@oh-my-pi/omp-stats 18.3.4 → 18.4.0

package/src/live.ts

@@ -0,0 +1,259 @@
+/**
+ * Live ingest for a running dashboard: background session sync, rollup
+ * refresh, and a status stream the client subscribes to (`/api/events`).
+ *
+ * The dashboard never waits for ingest. {@link StatsLive.start} kicks a full
+ * sync in the background and watches the sessions directory; changed
+ * transcripts are re-synced (only those files) shortly after they are written.
+ * Every committed batch bumps {@link LiveStatus.version} (throttled), so open
+ * pages refetch and fill in while parsing is still running.
+ */
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { getSessionsDir, logger } from "@oh-my-pi/pi-utils";
+import { syncAllSessions } from "./aggregator";
+import { initDb } from "./db";
+import { getRollupStatus, refreshRollups } from "./rollup";
+import type { LiveStatus, LiveSyncStatus } from "./shared-types";
+
+/** Wait this long after the last transcript write before syncing it. */
+const WATCH_DEBOUNCE_MS = 800;
+/** Full resync cadence, catching anything the watcher missed (or everything, without one). */
+const FULL_SYNC_INTERVAL_MS = 5 * 60 * 1000;
+/** Minimum spacing of `version` bumps; each bump makes open pages refetch. */
+const VERSION_THROTTLE_MS = 1000;
+/** Delay before retrying a failed sync (typically a lock held by another omp process). */
+const SYNC_RETRY_MS = 10_000;
+/** Minimum spacing of progress-only status events. */
+const PROGRESS_THROTTLE_MS = 150;
+
+type Listener = (status: LiveStatus) => void;
+
+/** One per process; owned by the dashboard server (see `startServer`). */
+export class StatsLive {
+	#version = 1;
+	#sync: LiveSyncStatus = { phase: "idle", current: 0, total: 0, processed: 0, lastSyncedAt: null, error: null };
+	#indexingHours = 0;
+	#listeners = new Set<Listener>();
+
+	#started = false;
+	#watcher: fs.FSWatcher | null = null;
+	#fullSyncTimer: NodeJS.Timeout | null = null;
+	#debounceTimer: NodeJS.Timeout | null = null;
+	#versionTimer: NodeJS.Timeout | null = null;
+	#progressTimer: NodeJS.Timeout | null = null;
+	#retryTimer: NodeJS.Timeout | null = null;
+	#lastVersionAt = 0;
+
+	/** Queued work: specific files, or `"all"` for a full sync. */
+	#queued: Set<string> | "all" | null = null;
+	#syncing = false;
+	#refreshing = false;
+	#refreshAgain = false;
+
+	status(): LiveStatus {
+		return { version: this.#version, sync: { ...this.#sync }, indexingHours: this.#indexingHours };
+	}
+
+	/** Receive every status change; returns the unsubscribe function. */
+	subscribe(listener: Listener): () => void {
+		this.#listeners.add(listener);
+		return () => this.#listeners.delete(listener);
+	}
+
+	/** Begin background ingest: an immediate full sync, the transcript watcher, and periodic resyncs. Idempotent. */
+	start(): void {
+		if (this.#started) return;
+		this.#started = true;
+		this.#watch();
+		this.#fullSyncTimer = setInterval(() => this.requestSync(), FULL_SYNC_INTERVAL_MS);
+		this.#fullSyncTimer.unref?.();
+		this.requestSync();
+	}
+
+	stop(): void {
+		this.#started = false;
+		this.#watcher?.close();
+		this.#watcher = null;
+		clearInterval(this.#fullSyncTimer ?? undefined);
+		for (const timer of [this.#debounceTimer, this.#versionTimer, this.#progressTimer, this.#retryTimer]) {
+			clearTimeout(timer ?? undefined);
+		}
+		this.#fullSyncTimer = this.#debounceTimer = this.#versionTimer = this.#progressTimer = this.#retryTimer = null;
+		this.#queued = null;
+	}
+
+	/**
+	 * Queue a sync of `files`, or of every session when omitted. Requests made
+	 * while a sync runs coalesce into one follow-up run.
+	 */
+	requestSync(files?: readonly string[]): void {
+		if (!files) this.#queued = "all";
+		else if (this.#queued !== "all") {
+			this.#queued ??= new Set();
+			for (const file of files) this.#queued.add(file);
+		}
+		void this.#drain();
+	}
+
+	async #drain(): Promise<void> {
+		if (this.#syncing) return;
+		this.#syncing = true;
+		try {
+			while (this.#queued) {
+				const work = this.#queued;
+				this.#queued = null;
+				await this.#runSync(work === "all" ? undefined : [...work]);
+			}
+		} finally {
+			this.#syncing = false;
+		}
+	}
+
+	async #runSync(files: string[] | undefined): Promise<void> {
+		// Targeted syncs of a file or two stay quiet; only full syncs show progress.
+		const visible = files === undefined;
+		if (visible) {
+			this.#sync = { ...this.#sync, phase: "syncing", current: 0, total: 0, processed: 0, error: null };
+			this.#emit();
+		}
+		let committed = 0;
+		try {
+			await initDb();
+			const result = await syncAllSessions({
+				files,
+				onProgress: event => {
+					if (visible) {
+						this.#sync = {
+							...this.#sync,
+							current: event.current,
+							total: event.total,
+							processed: event.processed,
+						};
+						this.#emitProgress();
+					}
+					if (event.processed > committed) {
+						committed = event.processed;
+						this.#changed();
+					}
+				},
+			});
+			this.#sync = {
+				...this.#sync,
+				phase: "idle",
+				processed: visible ? result.processed : this.#sync.processed,
+				lastSyncedAt: Date.now(),
+				error: null,
+			};
+		} catch (error) {
+			logger.warn("Stats live sync failed", { error: String(error) });
+			this.#sync = { ...this.#sync, phase: "error", error: error instanceof Error ? error.message : String(error) };
+			// Usually lock contention with another omp process writing the same
+			// database; committed batches are kept, so a retry resumes where it stopped.
+			if (this.#started) {
+				clearTimeout(this.#retryTimer ?? undefined);
+				this.#retryTimer = setTimeout(() => {
+					this.#retryTimer = null;
+					this.requestSync(files);
+				}, SYNC_RETRY_MS);
+			}
+		}
+		// Another process may have ingested too; always let clients revalidate.
+		this.#changed();
+		this.#emit();
+	}
+
+	/** Stored data changed: refresh rollups and (throttled) bump the version. */
+	#changed(): void {
+		void this.#refresh();
+		if (this.#versionTimer) return;
+		const wait = Math.max(0, this.#lastVersionAt + VERSION_THROTTLE_MS - Date.now());
+		this.#versionTimer = setTimeout(() => {
+			this.#versionTimer = null;
+			this.#lastVersionAt = Date.now();
+			this.#version++;
+			this.#indexingHours = getRollupStatus().dirtyHours;
+			this.#emit();
+		}, wait);
+	}
+
+	async #refresh(): Promise<void> {
+		if (this.#refreshing) {
+			this.#refreshAgain = true;
+			return;
+		}
+		this.#refreshing = true;
+		try {
+			do {
+				this.#refreshAgain = false;
+				await refreshRollups({
+					onProgress: remaining => {
+						this.#indexingHours = remaining;
+						// Each rolled batch fills in more history for open pages.
+						if (remaining > 0) this.#changed();
+					},
+				});
+			} while (this.#refreshAgain);
+		} catch (error) {
+			logger.warn("Stats rollup refresh failed", { error: String(error) });
+		} finally {
+			this.#refreshing = false;
+		}
+		this.#indexingHours = getRollupStatus().dirtyHours;
+		this.#emit();
+	}
+
+	#watch(): void {
+		const root = getSessionsDir();
+		const pending = new Set<string>();
+		try {
+			this.#watcher = fs.watch(root, { recursive: true }, (_event, filename) => {
+				if (!filename?.endsWith(".jsonl")) return;
+				pending.add(path.join(root, filename));
+				clearTimeout(this.#debounceTimer ?? undefined);
+				this.#debounceTimer = setTimeout(() => {
+					this.#debounceTimer = null;
+					const files = [...pending];
+					pending.clear();
+					this.requestSync(files);
+				}, WATCH_DEBOUNCE_MS);
+			});
+			this.#watcher.on("error", error => {
+				logger.warn("Stats session watcher failed; relying on periodic sync", { error: String(error) });
+				this.#watcher?.close();
+				this.#watcher = null;
+			});
+		} catch (error) {
+			// No sessions dir yet, or recursive watch unsupported: periodic full syncs cover it.
+			logger.debug("Stats session watcher unavailable", { root, error: String(error) });
+		}
+	}
+
+	#emitProgress(): void {
+		if (this.#progressTimer) return;
+		this.#progressTimer = setTimeout(() => {
+			this.#progressTimer = null;
+			this.#emit();
+		}, PROGRESS_THROTTLE_MS);
+	}
+
+	#emit(): void {
+		const status = this.status();
+		for (const listener of this.#listeners) {
+			try {
+				listener(status);
+			} catch (error) {
+				logger.warn("Stats live listener failed", { error: String(error) });
+			}
+		}
+	}
+}
+
+let instance: StatsLive | null = null;
+
+/** The process-wide live hub. */
+export function statsLive(): StatsLive {
+	instance ??= new StatsLive();
+	return instance;
+}

package/src/parser.ts

@@ -25,7 +25,7 @@ import type {
 	UserMessageLink,
 	UserMessageStats,
 } from "./types";
-import { computeUserMessageMetrics } from "./user-metrics";
+import { computeUserMessageMetrics, judgeProse } from "./user-metrics";
 
 /** Basename of an advisor agent's transcript inside a session artifacts dir. */
 const ADVISOR_TRANSCRIPT_BASENAME = "__advisor.jsonl";
@@ -135,6 +135,7 @@ function extractUserStats(sessionFile: string, folder: string, entry: SessionMes
 	const text = extractUserText(msg.content);
 	if (!text.trim()) return null;
 	const metrics = computeUserMessageMetrics(text);
+	const prose = judgeProse(text);
 	const ts = Date.parse(entry.timestamp);
 	return {
 		sessionFile,
@@ -151,6 +152,8 @@ function extractUserStats(sessionFile: string, folder: string, entry: SessionMes
 		negation: metrics.negation,
 		repetition: metrics.repetition,
 		blame: metrics.blame,
+		prose,
+		proseHash: prose ? Bun.hash(prose).toString(16) : "",
 	};
 }
 
@@ -465,14 +468,11 @@ function visitSessionEntriesLenient(bytes: Uint8Array, visit: (entry: SessionEnt
 	return read;
 }
 
-function parseSessionEntriesLenient(bytes: Uint8Array): { entries: SessionEntry[]; read: number } {
-	const entries: SessionEntry[] = [];
-	const read = visitSessionEntriesLenient(bytes, entry => entries.push(entry));
-	return { entries, read };
-}
 /** Parse every well-formed entry in a transcript buffer (malformed lines skipped). */
 export function parseAllSessionEntries(bytes: Uint8Array): SessionEntry[] {
-	return parseSessionEntriesLenient(bytes).entries;
+	const entries: SessionEntry[] = [];
+	visitSessionEntriesLenient(bytes, entry => entries.push(entry));
+	return entries;
 }
 
 function scanLastServiceTier(bytes: Uint8Array): ServiceTierByFamily | undefined {
@@ -538,7 +538,53 @@ export async function parseSessionFile(
 	let info: nodeFs.Stats;
 	let checkpoint: string;
 	let read: number;
-	let entries: SessionEntry[];
+	const folder = extractFolderFromPath(sessionPath);
+	const agentType = classifyAgentType(sessionPath);
+	const stats: MessageStatsInput[] = [];
+	const userStats: UserMessageStats[] = [];
+	const userLinks: UserMessageLink[] = [];
+	const toolCalls: ToolCallStats[] = [];
+	const toolResults: ToolResultLink[] = [];
+
+	// Reduce each entry immediately so full replays do not retain tool-output/message bodies.
+	const visit = (entry: SessionEntry): void => {
+		if (isServiceTierChange(entry)) {
+			currentServiceTier = coerceServiceTierByFamily(entry.serviceTier);
+			return;
+		}
+		if (isUserMessage(entry)) {
+			const userMsg = extractUserStats(sessionPath, folder, entry);
+			if (userMsg) userStats.push(userMsg);
+			return;
+		}
+		if (isToolResultMessage(entry)) {
+			const link = extractToolResultLink(sessionPath, entry);
+			if (link) toolResults.push(link);
+			return;
+		}
+		if (isModelUsage(entry)) {
+			const modelUsageStats = extractModelUsageStats(sessionPath, folder, entry, agentType);
+			if (modelUsageStats) stats.push(modelUsageStats);
+			return;
+		}
+		if (isAssistantMessage(entry)) {
+			const msgStats = extractStats(sessionPath, folder, entry, currentServiceTier, agentType);
+			if (msgStats) stats.push(msgStats);
+			toolCalls.push(...extractToolCalls(sessionPath, folder, entry, agentType));
+			// Persist links even when the user entry was ingested in an earlier tail read.
+			const parentId = entry.parentId;
+			const msg = entry.message;
+			if (parentId && msg.role === "assistant" && msg.model && msg.provider) {
+				userLinks.push({
+					sessionFile: sessionPath,
+					entryId: parentId,
+					model: msg.model,
+					provider: msg.provider,
+				});
+			}
+		}
+	};
+
 	try {
 		const handle = await fs.open(sessionPath, "r");
 		try {
@@ -564,7 +610,7 @@ export async function parseSessionFile(
 			currentServiceTier = resume
 				? (state?.serviceTier ?? undefined)
 				: scanLastServiceTier(bytes.subarray(0, start));
-			({ entries, read } = parseSessionEntriesLenient(bytes.subarray(start - readStart)));
+			read = visitSessionEntriesLenient(bytes.subarray(start - readStart), visit);
 			const newOffset = start + read;
 			const checkpointStart = Math.max(0, newOffset - CHECKPOINT_BYTES);
 			const previous =
@@ -581,63 +627,6 @@ export async function parseSessionFile(
 		throw err;
 	}
 
-	const folder = extractFolderFromPath(sessionPath);
-	const agentType = classifyAgentType(sessionPath);
-	const stats: MessageStatsInput[] = [];
-	const userStats: UserMessageStats[] = [];
-	const userLinks: UserMessageLink[] = [];
-	const toolCalls: ToolCallStats[] = [];
-	const toolResults: ToolResultLink[] = [];
-	const userByEntryId = new Map<string, UserMessageStats>();
-	for (const entry of entries) {
-		if (isServiceTierChange(entry)) {
-			currentServiceTier = coerceServiceTierByFamily(entry.serviceTier);
-			continue;
-		}
-		if (isUserMessage(entry)) {
-			const userMsg = extractUserStats(sessionPath, folder, entry);
-			if (userMsg) {
-				userStats.push(userMsg);
-				userByEntryId.set(entry.id, userMsg);
-			}
-			continue;
-		}
-		if (isToolResultMessage(entry)) {
-			const link = extractToolResultLink(sessionPath, entry);
-			if (link) toolResults.push(link);
-			continue;
-		}
-		if (isModelUsage(entry)) {
-			const modelUsageStats = extractModelUsageStats(sessionPath, folder, entry, agentType);
-			if (modelUsageStats) stats.push(modelUsageStats);
-			continue;
-		}
-		if (isAssistantMessage(entry)) {
-			const msgStats = extractStats(sessionPath, folder, entry, currentServiceTier, agentType);
-			if (msgStats) stats.push(msgStats);
-			toolCalls.push(...extractToolCalls(sessionPath, folder, entry, agentType));
-			// Link assistant's responding model back to the user message it answered.
-			const parentId = (entry as SessionMessageEntry).parentId;
-			if (parentId) {
-				const msg = entry.message as AssistantMessage;
-				if (msg.model && msg.provider) {
-					// Emit unconditionally. The aggregator's UPDATE is guarded by
-					// `model IS NULL` so this is idempotent: a no-op for already
-					// linked rows, a fix-up for fresh inserts (which start NULL
-					// because the user row is recorded before its reply lands) and
-					// for cross-pass orphans whose parent was committed by an
-					// earlier incremental sync.
-					userLinks.push({
-						sessionFile: sessionPath,
-						entryId: parentId,
-						model: msg.model,
-						provider: msg.provider,
-					});
-				}
-			}
-		}
-	}
-
 	const newOffset = start + read;
 	return {
 		stats,