@oh-my-pi/omp-stats 14.9.3 → 14.9.7

package/src/db.ts

@@ -4,6 +4,9 @@ import { type GeneratedProvider, getBundledModel, type Usage } from "@oh-my-pi/p
 import { getConfigRootDir, getStatsDbPath } from "@oh-my-pi/pi-utils";
 import type {
 	AggregatedStats,
+	BehaviorModelStats,
+	BehaviorOverallStats,
+	BehaviorTimeSeriesPoint,
 	CostTimeSeriesPoint,
 	FolderStats,
 	MessageStats,
@@ -11,6 +14,8 @@ import type {
 	ModelStats,
 	ModelTimeSeriesPoint,
 	TimeSeriesPoint,
+	UserMessageLink,
+	UserMessageStats,
 } from "./types";
 
 type ModelCost = { input: number; output: number; cacheRead: number; cacheWrite: number };
@@ -74,12 +79,42 @@ export async function initDb(): Promise<Database> {
 		CREATE INDEX IF NOT EXISTS idx_messages_model ON messages(model);
 		CREATE INDEX IF NOT EXISTS idx_messages_folder ON messages(folder);
 		CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_file);
+		CREATE INDEX IF NOT EXISTS idx_messages_timestamp_model_provider ON messages(timestamp, model, provider);
+		CREATE INDEX IF NOT EXISTS idx_messages_timestamp_folder ON messages(timestamp, folder);
+		CREATE INDEX IF NOT EXISTS idx_messages_stop_reason_timestamp ON messages(stop_reason, timestamp);
 
 		CREATE TABLE IF NOT EXISTS file_offsets (
 			session_file TEXT PRIMARY KEY,
 			offset INTEGER NOT NULL,
 			last_modified INTEGER NOT NULL
 		);
+
+		CREATE TABLE IF NOT EXISTS user_messages (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			session_file TEXT NOT NULL,
+			entry_id TEXT NOT NULL,
+			folder TEXT NOT NULL,
+			timestamp INTEGER NOT NULL,
+			model TEXT,
+			provider TEXT,
+			chars INTEGER NOT NULL,
+			words INTEGER NOT NULL,
+			yelling INTEGER NOT NULL,
+			profanity INTEGER NOT NULL,
+			anguish INTEGER NOT NULL,
+			negation INTEGER NOT NULL DEFAULT 0,
+			repetition INTEGER NOT NULL DEFAULT 0,
+			blame INTEGER NOT NULL DEFAULT 0,
+			UNIQUE(session_file, entry_id)
+		);
+
+		CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp ON user_messages(timestamp);
+		CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_model ON user_messages(timestamp, model, provider);
+
+		CREATE TABLE IF NOT EXISTS meta (
+			key TEXT PRIMARY KEY,
+			value TEXT NOT NULL
+		);
 	`);
 
 	const messageColumns = db.prepare("PRAGMA table_info(messages)").all() as { name: string }[];
@@ -87,6 +122,56 @@ export async function initDb(): Promise<Database> {
 		db.exec("ALTER TABLE messages ADD COLUMN premium_requests REAL NOT NULL DEFAULT 0");
 	}
 	db.exec("UPDATE messages SET premium_requests = 0 WHERE premium_requests IS NULL");
+	// Each behavior-metric bump invalidates previously-ingested rows. We detect
+	// the stale schema by column name and drop the table; `IF NOT EXISTS` above
+	// already produced the new schema, but we want a clean wipe + re-ingest.
+	// `backfillUserMessages` then clears `file_offsets` so the next sync
+	// re-parses every session under the current metric definitions.
+	//   v1 -> v2: yelling sentences replace `caps_words`.
+	//   v2 -> v3: `drama_runs` folded into a single `anguish` signal that
+	//             also captures elongated interjections, `dude`, and dot runs,
+	//             gated on a stripped prose-line budget.
+	//   v3 -> v4: added `negation`, `repetition`, `blame` frustration signals
+	//             plus profanity dictionary expansion + word-boundary fix.
+	//   v4 -> v5: column `yelling_sentences` renamed to `yelling` to match
+	//             the other single-word signal columns.
+	const userMessageColumns = db.prepare("PRAGMA table_info(user_messages)").all() as {
+		name: string;
+	}[];
+	const hasStaleColumn =
+		userMessageColumns.length > 0 &&
+		(userMessageColumns.some(column => column.name === "caps_words") ||
+			userMessageColumns.some(column => column.name === "drama_runs") ||
+			userMessageColumns.some(column => column.name === "yelling_sentences"));
+	const hasV4Columns = userMessageColumns.some(column => column.name === "negation");
+	const hasOldUserMessages = userMessageColumns.length > 0;
+	if (hasStaleColumn || (hasOldUserMessages && !hasV4Columns)) {
+		db.exec("DROP TABLE user_messages");
+		db.exec(`
+			CREATE TABLE user_messages (
+				id INTEGER PRIMARY KEY AUTOINCREMENT,
+				session_file TEXT NOT NULL,
+				entry_id TEXT NOT NULL,
+				folder TEXT NOT NULL,
+				timestamp INTEGER NOT NULL,
+				model TEXT,
+				provider TEXT,
+				chars INTEGER NOT NULL,
+				words INTEGER NOT NULL,
+				yelling INTEGER NOT NULL,
+				profanity INTEGER NOT NULL,
+				anguish INTEGER NOT NULL,
+				negation INTEGER NOT NULL DEFAULT 0,
+				repetition INTEGER NOT NULL DEFAULT 0,
+				blame INTEGER NOT NULL DEFAULT 0,
+				UNIQUE(session_file, entry_id)
+			);
+			CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp ON user_messages(timestamp);
+			CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_model ON user_messages(timestamp, model, provider);
+		`);
+	}
+	backfillUserMessages(db);
+	repairUserMessageLinks(db);
 	backfillMissingCatalogCosts(db);
 	return db;
 }
@@ -312,9 +397,10 @@ function buildAggregatedStats(rows: any[]): AggregatedStats {
 /**
  * Get overall aggregated stats.
  */
-export function getOverallStats(): AggregatedStats {
+export function getOverallStats(cutoff?: number): AggregatedStats {
 	if (!db) return buildAggregatedStats([]);
 
+	const hasCutoff = cutoff !== undefined && cutoff > 0;
 	const stmt = db.prepare(`
 		SELECT
 			COUNT(*) as total_requests,
@@ -331,18 +417,19 @@ export function getOverallStats(): AggregatedStats {
 			MIN(timestamp) as first_timestamp,
 			MAX(timestamp) as last_timestamp
 		FROM messages
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
 	`);
 
-	const rows = stmt.all();
-	return buildAggregatedStats(rows);
+	const rows = hasCutoff ? stmt.all(cutoff) : stmt.all();
+	return buildAggregatedStats(rows as any[]);
 }
-
 /**
  * Get stats grouped by model.
  */
-export function getStatsByModel(): ModelStats[] {
+export function getStatsByModel(cutoff?: number): ModelStats[] {
 	if (!db) return [];
 
+	const hasCutoff = cutoff !== undefined && cutoff > 0;
 	const stmt = db.prepare(`
 		SELECT
 			model,
@@ -361,11 +448,12 @@ export function getStatsByModel(): ModelStats[] {
 			MIN(timestamp) as first_timestamp,
 			MAX(timestamp) as last_timestamp
 		FROM messages
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
 		GROUP BY model, provider
 		ORDER BY total_requests DESC
 	`);
 
-	const rows = stmt.all() as any[];
+	const rows = (hasCutoff ? stmt.all(cutoff) : stmt.all()) as any[];
 	return rows.map(row => ({
 		model: row.model,
 		provider: row.provider,
@@ -376,9 +464,10 @@ export function getStatsByModel(): ModelStats[] {
 /**
  * Get stats grouped by folder.
  */
-export function getStatsByFolder(): FolderStats[] {
+export function getStatsByFolder(cutoff?: number): FolderStats[] {
 	if (!db) return [];
 
+	const hasCutoff = cutoff !== undefined && cutoff > 0;
 	const stmt = db.prepare(`
 		SELECT
 			folder,
@@ -396,11 +485,12 @@ export function getStatsByFolder(): FolderStats[] {
 			MIN(timestamp) as first_timestamp,
 			MAX(timestamp) as last_timestamp
 		FROM messages
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
 		GROUP BY folder
 		ORDER BY total_requests DESC
 	`);
 
-	const rows = stmt.all() as any[];
+	const rows = (hasCutoff ? stmt.all(cutoff) : stmt.all()) as any[];
 	return rows.map(row => ({
 		folder: row.folder,
 		...buildAggregatedStats([row]),
@@ -408,27 +498,30 @@ export function getStatsByFolder(): FolderStats[] {
 }
 
 /**
- * Get hourly time series data.
+ * Get time series data.
  */
-export function getTimeSeries(hours = 24): TimeSeriesPoint[] {
+export function getTimeSeries(hours = 24, cutoff?: number | null, bucketMs = 60 * 60 * 1000): TimeSeriesPoint[] {
 	if (!db) return [];
 
-	const cutoff = Date.now() - hours * 60 * 60 * 1000;
+	const hasCutoff = cutoff !== null;
+	const seriesCutoff = hasCutoff ? (cutoff ?? Date.now() - hours * 60 * 60 * 1000) : 0;
 
 	const stmt = db.prepare(`
 		SELECT
-			(timestamp / 3600000) * 3600000 as bucket,
+			(timestamp / ?) * ? as bucket,
 			COUNT(*) as requests,
 			SUM(CASE WHEN stop_reason = 'error' THEN 1 ELSE 0 END) as errors,
 			SUM(total_tokens) as tokens,
 			SUM(cost_total) as cost
 		FROM messages
-		WHERE timestamp >= ?
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
 		GROUP BY bucket
 		ORDER BY bucket ASC
 	`);
 
-	const rows = stmt.all(cutoff) as any[];
+	const rows = hasCutoff
+		? (stmt.all(bucketMs, bucketMs, seriesCutoff) as any[])
+		: (stmt.all(bucketMs, bucketMs) as any[]);
 	return rows.map(row => ({
 		timestamp: row.bucket,
 		requests: row.requests,
@@ -444,10 +537,11 @@ export function getTimeSeries(hours = 24): TimeSeriesPoint[] {
 /**
  * Get daily model usage time series data for the last N days.
  */
-export function getModelTimeSeries(days = 14): ModelTimeSeriesPoint[] {
+export function getModelTimeSeries(days = 14, cutoff?: number | null): ModelTimeSeriesPoint[] {
 	if (!db) return [];
 
-	const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+	const hasCutoff = cutoff !== null;
+	const seriesCutoff = hasCutoff ? (cutoff ?? Date.now() - days * 24 * 60 * 60 * 1000) : 0;
 
 	const stmt = db.prepare(`
 		SELECT
@@ -456,12 +550,12 @@ export function getModelTimeSeries(days = 14): ModelTimeSeriesPoint[] {
 			provider,
 			COUNT(*) as requests
 		FROM messages
-		WHERE timestamp >= ?
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
 		GROUP BY bucket, model, provider
 		ORDER BY bucket ASC
 	`);
 
-	const rows = stmt.all(cutoff) as any[];
+	const rows = hasCutoff ? (stmt.all(seriesCutoff) as any[]) : (stmt.all() as any[]);
 	return rows.map(row => ({
 		timestamp: row.bucket,
 		model: row.model,
@@ -473,10 +567,11 @@ export function getModelTimeSeries(days = 14): ModelTimeSeriesPoint[] {
 /**
  * Get daily model performance time series data for the last N days.
  */
-export function getModelPerformanceSeries(days = 14): ModelPerformancePoint[] {
+export function getModelPerformanceSeries(days = 14, cutoff?: number | null): ModelPerformancePoint[] {
 	if (!db) return [];
 
-	const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+	const hasCutoff = cutoff !== null;
+	const seriesCutoff = hasCutoff ? (cutoff ?? Date.now() - days * 24 * 60 * 60 * 1000) : 0;
 
 	const stmt = db.prepare(`
 		SELECT
@@ -487,12 +582,12 @@ export function getModelPerformanceSeries(days = 14): ModelPerformancePoint[] {
 			AVG(ttft) as avg_ttft,
 			AVG(CASE WHEN duration > 0 THEN output_tokens * 1000.0 / duration ELSE NULL END) as avg_tokens_per_second
 		FROM messages
-		WHERE timestamp >= ?
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
 		GROUP BY bucket, model, provider
 		ORDER BY bucket ASC
 	`);
 
-	const rows = stmt.all(cutoff) as any[];
+	const rows = hasCutoff ? (stmt.all(seriesCutoff) as any[]) : (stmt.all() as any[]);
 	return rows.map(row => ({
 		timestamp: row.bucket,
 		model: row.model,
@@ -586,10 +681,11 @@ export function getMessageById(id: number): MessageStats | null {
 /**
  * Get daily cost time series data for the last N days, broken down by model.
  */
-export function getCostTimeSeries(days = 90): CostTimeSeriesPoint[] {
+export function getCostTimeSeries(days = 90, cutoff?: number | null): CostTimeSeriesPoint[] {
 	if (!db) return [];
 
-	const cutoff = Date.now() - days * 24 * 60 * 60 * 1000;
+	const hasCutoff = cutoff !== null;
+	const seriesCutoff = hasCutoff ? (cutoff ?? Date.now() - days * 24 * 60 * 60 * 1000) : 0;
 
 	const stmt = db.prepare(`
 		SELECT
@@ -603,12 +699,12 @@ export function getCostTimeSeries(days = 90): CostTimeSeriesPoint[] {
 			SUM(cost_cache_write) as cost_cache_write,
 			COUNT(*) as requests
 		FROM messages
-		WHERE timestamp >= ?
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
 		GROUP BY bucket, model, provider
 		ORDER BY bucket ASC
 	`);
 
-	const rows = stmt.all(cutoff) as any[];
+	const rows = hasCutoff ? (stmt.all(seriesCutoff) as any[]) : (stmt.all() as any[]);
 	return rows.map(row => ({
 		timestamp: row.bucket,
 		model: row.model,
@@ -621,3 +717,301 @@ export function getCostTimeSeries(days = 90): CostTimeSeriesPoint[] {
 		requests: row.requests,
 	}));
 }
+
+/**
+ * Reset `file_offsets` (and any existing `user_messages` rows) so the next
+ * sync re-parses every session and re-derives behavioral metrics. Run once
+ * per metric-definition bump; the meta sentinel records the version.
+ *
+ * - v1: initial introduction of `user_messages`.
+ * - v2: yelling-sentence metric replaces caps-word counts; existing rows are
+ *   computed under the old definition and must be discarded.
+ * - v3: drama runs collapsed into `anguish` (drama + elongated interjections
+ *   + `dude` + dot runs), scored on a stripped prose body and gated on
+ *   line count. Existing rows used the narrower definition.
+ * - v4: added `negation` / `repetition` / `blame` signals and fixed a
+ *   latent word-boundary bug in the profanity / anguish regexes that had
+ *   left those metrics matching nothing in real prose.
+ * - v5: renamed `yelling_sentences` column to `yelling` to match the other
+ *   single-word signal columns (profanity, anguish, negation, ...).
+ *
+ * Existing `messages` rows are unaffected - `INSERT OR IGNORE` keeps them.
+ */
+function backfillUserMessages(database: Database): void {
+	const row = database.prepare("SELECT value FROM meta WHERE key = 'user_messages_v5'").get() as
+		| { value: string }
+		| undefined;
+	if (row) return;
+
+	database.exec("DELETE FROM user_messages");
+	database.exec("DELETE FROM file_offsets");
+	database
+		.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
+		.run("user_messages_v5", String(Date.now()));
+}
+
+/**
+ * One-shot wipe of `file_offsets` to force `parseSessionFile` to re-parse
+ * every session from byte zero. We don't touch `user_messages`; the parser
+ * now emits a `UserMessageLink` for every assistant->parent pair, and the
+ * guarded `updateUserMessageLinks` UPDATE fixes any row whose `model` was
+ * left NULL by the old in-pass-only linking logic. Idempotent: gated by a
+ * sentinel row in `meta`.
+ */
+function repairUserMessageLinks(database: Database): void {
+	const row = database.prepare("SELECT value FROM meta WHERE key = 'user_message_links_v1'").get() as
+		| { value: string }
+		| undefined;
+	if (row) return;
+
+	database.exec("DELETE FROM file_offsets");
+	database
+		.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)")
+		.run("user_message_links_v1", String(Date.now()));
+}
+
+/**
+ * Insert user-message stats. Idempotent via UNIQUE(session_file, entry_id).
+ */
+export function insertUserMessageStats(stats: UserMessageStats[]): number {
+	if (!db || stats.length === 0) return 0;
+
+	const stmt = db.prepare(`
+		INSERT OR IGNORE INTO user_messages (
+			session_file, entry_id, folder, timestamp, model, provider,
+			chars, words, yelling, profanity, anguish,
+			negation, repetition, blame
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+	`);
+
+	let inserted = 0;
+	const insert = db.transaction(() => {
+		for (const s of stats) {
+			const result = stmt.run(
+				s.sessionFile,
+				s.entryId,
+				s.folder,
+				s.timestamp,
+				s.model,
+				s.provider,
+				s.chars,
+				s.words,
+				s.yelling,
+				s.profanity,
+				s.anguish,
+				s.negation,
+				s.repetition,
+				s.blame,
+			);
+			if (result.changes > 0) inserted++;
+		}
+	});
+	insert();
+	return inserted;
+}
+
+/**
+ * Backfill the responding `model`/`provider` on user-message rows that were
+ * persisted before their assistant reply was parsed (a side effect of
+ * incremental `fromOffset` syncing: the `userByEntryId` map in
+ * `parseSessionFile` only spans a single pass). Each row is updated at most
+ * once because the `model IS NULL` guard short-circuits subsequent passes.
+ *
+ * Returns the number of rows actually updated.
+ */
+export function updateUserMessageLinks(links: UserMessageLink[]): number {
+	if (!db || links.length === 0) return 0;
+
+	const stmt = db.prepare(`
+		UPDATE user_messages
+		   SET model = ?, provider = ?
+		 WHERE session_file = ? AND entry_id = ? AND model IS NULL
+	`);
+
+	let updated = 0;
+	const apply = db.transaction(() => {
+		for (const link of links) {
+			const result = stmt.run(link.model, link.provider, link.sessionFile, link.entryId);
+			if (result.changes > 0) updated++;
+		}
+	});
+	apply();
+	return updated;
+}
+
+const UNKNOWN_MODEL = "unknown";
+
+interface BehaviorSeriesRow {
+	bucket: number;
+	model: string;
+	provider: string;
+	messages: number;
+	yelling: number | null;
+	profanity: number | null;
+	anguish: number | null;
+	negation: number | null;
+	repetition: number | null;
+	blame: number | null;
+	chars: number | null;
+}
+
+/**
+ * Daily behavioral time series, grouped by responding model+provider.
+ */
+export function getBehaviorTimeSeries(cutoff?: number | null): BehaviorTimeSeriesPoint[] {
+	if (!db) return [];
+	const hasCutoff = cutoff !== null && cutoff !== undefined && cutoff > 0;
+	const stmt = db.prepare(`
+		SELECT
+			(timestamp / 86400000) * 86400000 as bucket,
+			COALESCE(model, ?) as model,
+			COALESCE(provider, ?) as provider,
+			COUNT(*) as messages,
+			SUM(yelling) as yelling,
+			SUM(profanity) as profanity,
+			SUM(anguish) as anguish,
+			SUM(negation) as negation,
+			SUM(repetition) as repetition,
+			SUM(blame) as blame,
+			SUM(chars) as chars
+		FROM user_messages
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
+		GROUP BY bucket, model, provider
+		ORDER BY bucket ASC
+	`);
+	const rows = (
+		hasCutoff ? stmt.all(UNKNOWN_MODEL, UNKNOWN_MODEL, cutoff) : stmt.all(UNKNOWN_MODEL, UNKNOWN_MODEL)
+	) as BehaviorSeriesRow[];
+	return rows.map(row => ({
+		timestamp: row.bucket,
+		model: row.model,
+		provider: row.provider,
+		messages: row.messages,
+		yelling: row.yelling ?? 0,
+		profanity: row.profanity ?? 0,
+		anguish: row.anguish ?? 0,
+		negation: row.negation ?? 0,
+		repetition: row.repetition ?? 0,
+		blame: row.blame ?? 0,
+		chars: row.chars ?? 0,
+	}));
+}
+
+interface BehaviorOverallRow {
+	total_messages: number;
+	total_yelling: number | null;
+	total_profanity: number | null;
+	total_anguish: number | null;
+	total_negation: number | null;
+	total_repetition: number | null;
+	total_blame: number | null;
+	total_chars: number | null;
+	first_timestamp: number | null;
+	last_timestamp: number | null;
+}
+
+/**
+ * Overall behavioral totals across the cutoff window.
+ */
+export function getBehaviorOverall(cutoff?: number | null): BehaviorOverallStats {
+	const empty: BehaviorOverallStats = {
+		totalMessages: 0,
+		totalYelling: 0,
+		totalProfanity: 0,
+		totalAnguish: 0,
+		totalNegation: 0,
+		totalRepetition: 0,
+		totalBlame: 0,
+		totalChars: 0,
+		firstTimestamp: 0,
+		lastTimestamp: 0,
+	};
+	if (!db) return empty;
+	const hasCutoff = cutoff !== null && cutoff !== undefined && cutoff > 0;
+	const stmt = db.prepare(`
+		SELECT
+			COUNT(*) as total_messages,
+			SUM(yelling) as total_yelling,
+			SUM(profanity) as total_profanity,
+			SUM(anguish) as total_anguish,
+			SUM(negation) as total_negation,
+			SUM(repetition) as total_repetition,
+			SUM(blame) as total_blame,
+			SUM(chars) as total_chars,
+			MIN(timestamp) as first_timestamp,
+			MAX(timestamp) as last_timestamp
+		FROM user_messages
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
+	`);
+	const row = (hasCutoff ? stmt.get(cutoff) : stmt.get()) as BehaviorOverallRow | undefined;
+	if (!row?.total_messages) return empty;
+	return {
+		totalMessages: row.total_messages,
+		totalYelling: row.total_yelling ?? 0,
+		totalProfanity: row.total_profanity ?? 0,
+		totalAnguish: row.total_anguish ?? 0,
+		totalNegation: row.total_negation ?? 0,
+		totalRepetition: row.total_repetition ?? 0,
+		totalBlame: row.total_blame ?? 0,
+		totalChars: row.total_chars ?? 0,
+		firstTimestamp: row.first_timestamp ?? 0,
+		lastTimestamp: row.last_timestamp ?? 0,
+	};
+}
+
+interface BehaviorByModelRow {
+	model: string;
+	provider: string;
+	total_messages: number;
+	total_yelling: number | null;
+	total_profanity: number | null;
+	total_anguish: number | null;
+	total_negation: number | null;
+	total_repetition: number | null;
+	total_blame: number | null;
+	total_chars: number | null;
+	last_timestamp: number | null;
+}
+
+/**
+ * Per-model behavioral totals over the cutoff window. "Unknown" represents
+ * user messages that never received an assistant reply.
+ */
+export function getBehaviorByModel(cutoff?: number | null): BehaviorModelStats[] {
+	if (!db) return [];
+	const hasCutoff = cutoff !== null && cutoff !== undefined && cutoff > 0;
+	const stmt = db.prepare(`
+		SELECT
+			COALESCE(model, ?) as model,
+			COALESCE(provider, ?) as provider,
+			COUNT(*) as total_messages,
+			SUM(yelling) as total_yelling,
+			SUM(profanity) as total_profanity,
+			SUM(anguish) as total_anguish,
+			SUM(negation) as total_negation,
+			SUM(repetition) as total_repetition,
+			SUM(blame) as total_blame,
+			SUM(chars) as total_chars,
+			MAX(timestamp) as last_timestamp
+		FROM user_messages
+		${hasCutoff ? "WHERE timestamp >= ?" : ""}
+		GROUP BY model, provider
+		ORDER BY total_messages DESC
+	`);
+	const rows = (
+		hasCutoff ? stmt.all(UNKNOWN_MODEL, UNKNOWN_MODEL, cutoff) : stmt.all(UNKNOWN_MODEL, UNKNOWN_MODEL)
+	) as BehaviorByModelRow[];
+	return rows.map(row => ({
+		model: row.model,
+		provider: row.provider,
+		totalMessages: row.total_messages,
+		totalYelling: row.total_yelling ?? 0,
+		totalProfanity: row.total_profanity ?? 0,
+		totalAnguish: row.total_anguish ?? 0,
+		totalNegation: row.total_negation ?? 0,
+		totalRepetition: row.total_repetition ?? 0,
+		totalBlame: row.total_blame ?? 0,
+		totalChars: row.total_chars ?? 0,
+		lastTimestamp: row.last_timestamp ?? 0,
+	}));
+}

package/src/index.ts

@@ -6,7 +6,13 @@ import { getDashboardStats, getTotalMessageCount, syncAllSessions } from "./aggr
 import { closeDb } from "./db";
 import { startServer } from "./server";
 
-export { getDashboardStats, getTotalMessageCount, syncAllSessions } from "./aggregator";
+export {
+	getDashboardStats,
+	getTotalMessageCount,
+	type SyncOptions,
+	type SyncProgress,
+	syncAllSessions,
+} from "./aggregator";
 export { closeDb } from "./db";
 export { startServer } from "./server";
 export type {
@@ -112,8 +118,28 @@ Examples:
 
 	try {
 		// Sync first
-		console.log("Syncing session files...");
-		const { processed, files } = await syncAllSessions();
+		const tty = process.stderr.isTTY === true;
+		process.stderr.write("Syncing session files...\n");
+		let lastWidth = 0;
+		let lastRender = 0;
+		const { processed, files } = await syncAllSessions({
+			onProgress: event => {
+				if (!tty) return;
+				const now = Date.now();
+				if (event.current < event.total && now - lastRender < 33) return;
+				lastRender = now;
+				const marker = "/sessions/";
+				const idx = event.sessionFile.indexOf(marker);
+				const short = idx >= 0 ? event.sessionFile.slice(idx + marker.length) : event.sessionFile;
+				const pct = ((event.current / event.total) * 100).toFixed(0).padStart(3, " ");
+				const line = `[${event.current}/${event.total}] ${pct}%  ${short}`;
+				const columns = process.stderr.columns ?? 120;
+				const clipped = line.length > columns - 1 ? `${line.slice(0, columns - 2)}\u2026` : line;
+				process.stderr.write(`\r${clipped.padEnd(lastWidth)}`);
+				lastWidth = clipped.length;
+			},
+		});
+		if (tty && lastWidth > 0) process.stderr.write(`\r${" ".repeat(lastWidth)}\r`);
 		const total = await getTotalMessageCount();
 		console.log(`Synced ${processed} new entries from ${files} files (${total} total)\n`);