@linxiraos/pi-stats 1.1.15 → 1.1.16

package/src/aggregator.ts

@@ -1,7 +1,10 @@
 import * as fs from "node:fs";
-import * as fsp from "node:fs/promises";
-import { workerHostEntry } from "@linxiraos/pi-utils";
+import * as path from "node:path";
+import { getStatsDbPath, workerHostEntry } from "@linxiraos/pi-utils";
+import { withFileLock } from "@linxiraos/pi-utils/file-lock";
 import {
+	applySessionParseResult,
+	completeSessionSync,
 	getRecentErrors as dbGetRecentErrors,
 	getRecentRequests as dbGetRecentRequests,
 	getBehaviorByModel,
@@ -25,16 +28,18 @@ import {
 	getToolStatsByModel,
 	getToolTimeSeries,
 	initDb,
-	insertMessageStats,
-	insertToolCalls,
-	insertUserMessageStats,
 	markSessionBackfillsComplete,
+	prepareSessionSync,
 	pruneOrphanSessions,
-	setFileOffset,
-	updateToolResults,
-	updateUserMessageLinks,
 } from "./db";
-import { getSessionEntry, listAllSessionFiles, type ParseSessionResult, parseSessionFile } from "./parser";
+import {
+	getSessionEntry,
+	listAllSessionFiles,
+	matchesSessionFile,
+	type ParseSessionResult,
+	parseSessionFile,
+	type SessionParserState,
+} from "./parser";
 import type { SyncWorkerRequest, SyncWorkerResponse } from "./sync-worker";
 // Coding-agent binary/bundle workers route through the CLI entrypoint with a
 // hidden argv mode, so the compiled binary and npm bundle only need one
@@ -51,20 +56,23 @@ import type {
 } from "./types";
 import { computeUsageWindowStats, fetchUsageData } from "./usage-windows";
 
+const STATS_SYNC_LOCK_RETRY_MS = 25;
+const STATS_SYNC_LOCK_WAIT_MS = 60 * 60 * 1000;
+
 /**
- * Apply a freshly parsed result to the database. Runs entirely on the
- * main thread so the single SQLite handle owns every write.
+ * Serialize stats ingestion and archive reconciliation across processes.
+ * The lock covers file discovery, parsing, and the final SQLite write so a
+ * parse result for a session moved by GC can never commit after cleanup.
+ * The native lock is owned by an operating-system primitive, so an interrupted
+ * owner is released automatically and a live owner is never displaced.
  */
-function applyParseResult(sessionFile: string, lastModified: number, result: ParseSessionResult): number {
-	if (result.stats.length > 0) insertMessageStats(result.stats);
-	if (result.userStats.length > 0) insertUserMessageStats(result.userStats);
-	if (result.userLinks.length > 0) updateUserMessageLinks(result.userLinks);
-	if (result.toolCalls.length > 0) insertToolCalls(result.toolCalls);
-	if (result.toolResults.length > 0) updateToolResults(result.toolResults);
-	setFileOffset(sessionFile, result.newOffset, lastModified);
-	return result.stats.length + result.userStats.length;
+export async function withStatsSyncLock<T>(dbPath: string, fn: () => Promise<T>): Promise<T> {
+	await fs.promises.mkdir(path.dirname(dbPath), { recursive: true });
+	return await withFileLock(`${dbPath}.sync`, fn, {
+		retryDelayMs: STATS_SYNC_LOCK_RETRY_MS,
+		retries: Math.ceil(STATS_SYNC_LOCK_WAIT_MS / STATS_SYNC_LOCK_RETRY_MS),
+	});
 }
-
 /**
  * Progress event emitted after each session file is fully processed.
  * `current` is the number of files completed (skipped + parsed),
@@ -221,7 +229,23 @@ export async function smokeTestSyncWorker({ timeoutMs = 5_000 }: { timeoutMs?: n
  * bar walks at a steady rate).
  */
 export async function syncAllSessions(opts?: SyncOptions): Promise<{ processed: number; files: number }> {
+	return withStatsSyncLock(getStatsDbPath(), async () => {
+		let processed = 0;
+		let files = 0;
+		while (true) {
+			const result = await syncAllSessionsLocked(opts);
+			processed += result.processed;
+			files += result.files;
+			if (!result.reconcile) return { processed, files };
+		}
+	});
+}
+
+async function syncAllSessionsLocked(
+	opts?: SyncOptions,
+): Promise<{ processed: number; files: number; reconcile: boolean }> {
 	await initDb();
+	const replay = prepareSessionSync();
 
 	const files = await listAllSessionFiles();
 	// Drop rows for transcripts that no longer exist on disk (deleted temp/test
@@ -231,9 +255,11 @@ export async function syncAllSessions(opts?: SyncOptions): Promise<{ processed:
 	let filesProcessed = 0;
 	let completed = 0;
 	let cursor = 0;
+	let reconcile = false;
 	const finish = () => {
+		completeSessionSync(reconcile);
 		markSessionBackfillsComplete();
-		return { processed: totalProcessed, files: filesProcessed };
+		return { processed: totalProcessed, files: filesProcessed, reconcile };
 	};
 	if (files.length === 0) return finish();
 
@@ -249,7 +275,12 @@ export async function syncAllSessions(opts?: SyncOptions): Promise<{ processed:
 
 	const processFile = async (
 		sessionFile: string,
-		parse: (sessionFile: string, fromOffset: number) => Promise<ParseSessionResult>,
+		parse: (
+			sessionFile: string,
+			fromOffset: number,
+			state?: SessionParserState,
+			replay?: boolean,
+		) => Promise<ParseSessionResult>,
 	): Promise<void> => {
 		let fileStats: fs.Stats;
 		try {
@@ -260,14 +291,24 @@ export async function syncAllSessions(opts?: SyncOptions): Promise<{ processed:
 		}
 		const lastModified = fileStats.mtimeMs;
 		const stored = getFileOffset(sessionFile);
-		if (stored && stored.lastModified >= lastModified) {
+		if (
+			!replay &&
+			stored?.parserState &&
+			stored.lastModified === lastModified &&
+			stored.parserState.size === fileStats.size &&
+			matchesSessionFile(stored.parserState, fileStats)
+		) {
 			report(sessionFile);
 			return;
 		}
 
-		const fromOffset = stored?.offset ?? 0;
-		const result = await parse(sessionFile, fromOffset);
-		const inserted = applyParseResult(sessionFile, lastModified, result);
+		const unknownIdentity = stored !== null && !stored.parserState;
+		const fromOffset = unknownIdentity ? 0 : (stored?.offset ?? 0);
+		const result = await parse(sessionFile, fromOffset, stored?.parserState, replay);
+		if (unknownIdentity && result.parserState) result.reset = true;
+		const applied = applySessionParseResult(sessionFile, result, replay || !stored?.parserState);
+		const inserted = applied.processed;
+		if (applied.reconcile) reconcile = true;
 		if (inserted > 0) {
 			totalProcessed += inserted;
 			filesProcessed++;
@@ -293,7 +334,9 @@ export async function syncAllSessions(opts?: SyncOptions): Promise<{ processed:
 			const idx = cursor++;
 			if (idx >= files.length) return;
 			const sessionFile = files[idx];
-			await processFile(sessionFile, (file, fromOffset) => dispatch(handle, { sessionFile: file, fromOffset }));
+			await processFile(sessionFile, (file, fromOffset, parserState, replay) =>
+				dispatch(handle, { sessionFile: file, fromOffset, parserState, replay }),
+			);
 		}
 	}
 
@@ -553,79 +596,3 @@ export async function getProviderDashboardStats(range?: string | null): Promise<
 		windowInsights,
 	};
 }
-
-const STATS_LOCK_TIMEOUT_MS = 30_000;
-const STATS_LOCK_STALE_MS = 60_000;
-
-async function statIfPresent(path: string) {
-	try {
-		return await fsp.stat(path);
-	} catch (error) {
-		const code =
-			typeof error === "object" && error !== null && "code" in error ? (error as { code: string }).code : undefined;
-		if (code === "ENOENT") return null;
-		throw error;
-	}
-}
-
-function processExists(pid: number): boolean {
-	try {
-		process.kill(pid, 0);
-		return true;
-	} catch (error) {
-		const code =
-			typeof error === "object" && error !== null && "code" in error ? (error as { code: string }).code : undefined;
-		if (code === "ESRCH" || code === "EINVAL") return false;
-		return true;
-	}
-}
-
-async function isStatsLockStale(lockPath: string): Promise<boolean> {
-	const stat = await statIfPresent(lockPath);
-	if (!stat) return false;
-	const pid = Number.parseInt((await Bun.file(lockPath).text()).split(/\r?\n/, 1)[0] ?? "", 10);
-	if (Number.isSafeInteger(pid) && pid > 0) return !processExists(pid);
-	return Date.now() - stat.mtimeMs > STATS_LOCK_STALE_MS;
-}
-
-/**
- * Acquire an exclusive lock on the stats database and execute `fn`.
- * Used by GC to prevent stats row cleanup from racing with stats sync.
- */
-export async function withStatsSyncLock<T>(dbPath: string, fn: () => Promise<T>): Promise<T> {
-	const lockPath = `${dbPath}.lock`;
-	const deadline = Date.now() + STATS_LOCK_TIMEOUT_MS;
-
-	while (Date.now() < deadline) {
-		try {
-			const handle = await fsp.open(lockPath, "wx");
-			try {
-				await handle.writeFile(`${process.pid}\n${new Date().toISOString()}\n`);
-				return await fn();
-			} finally {
-				await handle.close();
-				await fsp.unlink(lockPath).catch(() => {});
-			}
-		} catch (error) {
-			const code =
-				typeof error === "object" && error !== null && "code" in error
-					? (error as { code: string }).code
-					: undefined;
-			if (code !== "EEXIST") throw error;
-		}
-		if (await isStatsLockStale(lockPath)) {
-			try {
-				await fsp.unlink(lockPath);
-			} catch (error) {
-				const code =
-					typeof error === "object" && error !== null && "code" in error
-						? (error as { code: string }).code
-						: undefined;
-				if (code !== "ENOENT") throw error;
-			}
-		}
-		await Bun.sleep(100);
-	}
-
-	throw new Error(`timed out waiting for stats lock: ${lockPath}`);
-}

package/src/client/styles.css

@@ -1511,10 +1511,35 @@
   border: 1px solid var(--border);
   border-radius: var(--radius-md);
   color: var(--text);
+  font-family: inherit;
   font-size: 12px;
   padding: 6px 10px;
 }
 
+/* Prevent iOS from zooming the viewport when a search input receives focus
+   (mirrors the .stats-select coarse-pointer treatment). */
+@media (pointer: coarse) {
+  .stats-trace-input {
+    font-size: 16px;
+  }
+}
+
+/* Native clear button is a dark glyph in WebKit; keep it visible in the dark
+   theme by inverting it (no-op in light). */
+.stats-trace-input::-webkit-search-cancel-button {
+  filter: invert(1);
+}
+
+@media (prefers-color-scheme: light) {
+  :root:not([data-theme]) .stats-trace-input::-webkit-search-cancel-button {
+    filter: none;
+  }
+}
+
+[data-theme="light"] .stats-trace-input::-webkit-search-cancel-button {
+  filter: none;
+}
+
 .stats-trace-input:focus-visible {
   outline: 2px solid var(--link);
   outline-offset: 1px;
@@ -1534,6 +1559,11 @@
   user-select: none;
 }
 
+/* Brand-tint the native checkbox instead of leaving the browser default. */
+.stats-trace-check input[type="checkbox"] {
+  accent-color: var(--accent);
+}
+
 .stats-trace-icon-btn {
   display: inline-flex;
   align-items: center;

package/src/db.ts

@@ -9,8 +9,7 @@ import {
 } from "@linxiraos/pi-catalog/models";
 import type { ModelCost } from "@linxiraos/pi-catalog/types";
 import { getConfigRootDir, getStatsDbPath } from "@linxiraos/pi-utils";
-
-import { classifyAgentType } from "./parser";
+import { classifyAgentType, type ParseSessionResult, type SessionParserState } from "./parser";
 import type {
 	AgentType,
 	AgentTypeStats,
@@ -145,7 +144,10 @@ const USER_MESSAGE_LINKS_REPAIR_KEY = "user_message_links_v1";
 const PRIORITY_PREMIUM_REQUESTS_BACKFILL_KEY = "premium_requests_priority_v1";
 const AGENT_TYPE_BACKFILL_KEY = "agent_type_v1";
 const FORK_DEDUPE_KEY = "fork_dedupe_v1";
-const TOOL_CALLS_BACKFILL_KEY = "tool_calls_v1";
+// v2: tool-name sanitization at ingest (see `sanitizeToolName` in parser.ts)
+// collapses provider-side garbage names; a full re-parse replaces the polluted
+// rows already stored in existing databases.
+const TOOL_CALLS_BACKFILL_KEY = "tool_calls_v2";
 // Older ingests dropped `Usage.orchestration` (never a stored column) when
 // pricing, so subscription models billed on orchestration tokens — multi-agent
 // Grok most notably — were priced from conversation buckets alone and could not
@@ -213,6 +215,7 @@ export async function initDb(): Promise<Database> {
 		);
 
 		CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp);
+		CREATE INDEX IF NOT EXISTS idx_messages_entry_timestamp ON messages(entry_id, timestamp);
 		CREATE INDEX IF NOT EXISTS idx_messages_model ON messages(model);
 		CREATE INDEX IF NOT EXISTS idx_messages_folder ON messages(folder);
 		CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_file);
@@ -223,7 +226,8 @@ export async function initDb(): Promise<Database> {
 		CREATE TABLE IF NOT EXISTS file_offsets (
 			session_file TEXT PRIMARY KEY,
 			offset INTEGER NOT NULL,
-			last_modified INTEGER NOT NULL
+			last_modified INTEGER NOT NULL,
+			parser_state TEXT
 		);
 
 		CREATE TABLE IF NOT EXISTS user_messages (
@@ -246,6 +250,7 @@ export async function initDb(): Promise<Database> {
 		);
 
 		CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp ON user_messages(timestamp);
+		CREATE INDEX IF NOT EXISTS idx_user_messages_entry_timestamp ON user_messages(entry_id, timestamp);
 		CREATE INDEX IF NOT EXISTS idx_user_messages_timestamp_model ON user_messages(timestamp, model, provider);
 
 		CREATE TABLE IF NOT EXISTS tool_calls (
@@ -267,6 +272,7 @@ export async function initDb(): Promise<Database> {
 		);
 
 		CREATE INDEX IF NOT EXISTS idx_tool_calls_timestamp ON tool_calls(timestamp);
+		CREATE INDEX IF NOT EXISTS idx_tool_calls_entry_timestamp ON tool_calls(entry_id, timestamp);
 		CREATE INDEX IF NOT EXISTS idx_tool_calls_tool_timestamp ON tool_calls(tool_name, timestamp);
 
 		CREATE TABLE IF NOT EXISTS meta (
@@ -275,6 +281,10 @@ export async function initDb(): Promise<Database> {
 		);
 	`);
 
+	const offsetColumns = db.prepare("PRAGMA table_info(file_offsets)").all() as { name: string }[];
+	if (!offsetColumns.some(column => column.name === "parser_state")) {
+		db.run("ALTER TABLE file_offsets ADD COLUMN parser_state TEXT");
+	}
 	const messageColumns = db.prepare("PRAGMA table_info(messages)").all() as { name: string }[];
 	if (!messageColumns.some(column => column.name === "premium_requests")) {
 		db.run("ALTER TABLE messages ADD COLUMN premium_requests REAL NOT NULL DEFAULT 0");
@@ -591,26 +601,104 @@ function backfillNoCacheInputCosts(database: Database): void {
 /**
  * Get the stored offset for a session file.
  */
-export function getFileOffset(sessionFile: string): { offset: number; lastModified: number } | null {
+export function getFileOffset(
+	sessionFile: string,
+): { offset: number; lastModified: number; parserState?: SessionParserState } | null {
 	if (!db) return null;
 
-	const stmt = db.prepare("SELECT offset, last_modified FROM file_offsets WHERE session_file = ?");
-	const row = stmt.get(sessionFile) as { offset: number; last_modified: number } | undefined;
-
-	return row ? { offset: row.offset, lastModified: row.last_modified } : null;
+	const stmt = db.prepare("SELECT offset, last_modified, parser_state FROM file_offsets WHERE session_file = ?");
+	const row = stmt.get(sessionFile) as
+		| { offset: number; last_modified: number; parser_state: string | null }
+		| undefined;
+	if (!row) return null;
+	let parserState: SessionParserState | undefined;
+	if (row.parser_state) {
+		try {
+			const state = JSON.parse(row.parser_state) as SessionParserState;
+			if (state?.version === 1 && state.offset === row.offset) parserState = state;
+		} catch {
+			/* A missing cursor is reconstructed from the transcript. */
+		}
+	}
+	return { offset: row.offset, lastModified: row.last_modified, parserState };
 }
 
 /**
  * Update the stored offset for a session file.
  */
-export function setFileOffset(sessionFile: string, offset: number, lastModified: number): void {
+export function setFileOffset(
+	sessionFile: string,
+	offset: number,
+	lastModified: number,
+	parserState?: SessionParserState,
+): void {
 	if (!db) return;
 
 	const stmt = db.prepare(`
-		INSERT OR REPLACE INTO file_offsets (session_file, offset, last_modified)
-		VALUES (?, ?, ?)
+		INSERT OR REPLACE INTO file_offsets (session_file, offset, last_modified, parser_state)
+		VALUES (?, ?, ?, ?)
 	`);
-	stmt.run(sessionFile, offset, lastModified);
+	stmt.run(sessionFile, offset, lastModified, parserState ? JSON.stringify(parserState) : null);
+}
+
+export function applySessionParseResult(
+	sessionFile: string,
+	result: ParseSessionResult,
+	rebuild = false,
+): { processed: number; reconcile: boolean } {
+	const parserState = result.parserState;
+	if (!db || !parserState) return { processed: 0, reconcile: false };
+	const database = db;
+	return database.transaction(() => {
+		let reconcile = result.reset ?? false;
+		if (result.reset || rebuild) {
+			const retainedMessages = new Set(result.stats.map(row => JSON.stringify([row.entryId, row.timestamp])));
+			const retainedUsers = new Set(result.userStats.map(row => JSON.stringify([row.entryId, row.timestamp])));
+			const retainedTools = new Set(
+				result.toolCalls.map(row => JSON.stringify([row.entryId, row.timestamp, row.toolCallId])),
+			);
+			const messages = database
+				.prepare("SELECT entry_id, timestamp FROM messages WHERE session_file = ?")
+				.all(sessionFile) as {
+				entry_id: string;
+				timestamp: number;
+			}[];
+			const users = database
+				.prepare("SELECT entry_id, timestamp FROM user_messages WHERE session_file = ?")
+				.all(sessionFile) as { entry_id: string; timestamp: number }[];
+			const tools = database
+				.prepare("SELECT entry_id, timestamp, tool_call_id FROM tool_calls WHERE session_file = ?")
+				.all(sessionFile) as { entry_id: string; timestamp: number; tool_call_id: string }[];
+			// A removed owner may have surviving fork copies skipped earlier in this pass.
+			reconcile ||=
+				messages.some(row => !retainedMessages.has(JSON.stringify([row.entry_id, row.timestamp]))) ||
+				users.some(row => !retainedUsers.has(JSON.stringify([row.entry_id, row.timestamp]))) ||
+				tools.some(row => !retainedTools.has(JSON.stringify([row.entry_id, row.timestamp, row.tool_call_id])));
+			database.prepare("DELETE FROM messages WHERE session_file = ?").run(sessionFile);
+			database.prepare("DELETE FROM user_messages WHERE session_file = ?").run(sessionFile);
+			database.prepare("DELETE FROM tool_calls WHERE session_file = ?").run(sessionFile);
+		}
+		if (reconcile) {
+			database
+				.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('session_reconciliation', 'pending')")
+				.run();
+		}
+		if (result.stats.length > 0) insertMessageStats(result.stats);
+		if (result.userStats.length > 0) insertUserMessageStats(result.userStats);
+		if (result.userLinks.length > 0) updateUserMessageLinks(result.userLinks);
+		if (result.toolCalls.length > 0) insertToolCalls(result.toolCalls);
+		if (result.toolResults.length > 0) updateToolResults(result.toolResults);
+		setFileOffset(sessionFile, result.newOffset, parserState.mtimeMs, parserState);
+		return { processed: result.stats.length + result.userStats.length, reconcile };
+	})();
+}
+
+export function prepareSessionSync(): boolean {
+	return Boolean(db?.prepare("SELECT 1 FROM meta WHERE key = 'session_reconciliation'").get());
+}
+
+export function completeSessionSync(reconcile: boolean): void {
+	if (!reconcile) db?.prepare("DELETE FROM meta WHERE key = 'session_reconciliation'").run();
 }
 
 /**