@gajae-code/stats 0.1.1

package/src/embedded-client.generated.txt

@@ -0,0 +1,7 @@
+/**
+ * Embedded stats dashboard bundle for compiled binaries.
+ *
+ * This file is generated by `bun --cwd=packages/stats scripts/generate-client-bundle.ts --generate` during
+ * binary builds. The checked-in value is intentionally empty.
+ */
+export const EMBEDDED_CLIENT_ARCHIVE_TAR_GZ_BASE64 = "";

package/src/index.ts

@@ -0,0 +1,182 @@
+#!/usr/bin/env bun
+
+import { parseArgs } from "node:util";
+import { formatDuration, formatNumber, formatPercent } from "@gajae-code/utils";
+import { getDashboardStats, getTotalMessageCount, syncAllSessions } from "./aggregator";
+import { closeDb } from "./db";
+import { startServer } from "./server";
+
+export {
+	getDashboardStats,
+	getTotalMessageCount,
+	type SyncOptions,
+	type SyncProgress,
+	smokeTestSyncWorker,
+	syncAllSessions,
+} from "./aggregator";
+export { closeDb } from "./db";
+export { startServer } from "./server";
+export type {
+	AggregatedStats,
+	DashboardStats,
+	FolderStats,
+	MessageStats,
+	ModelPerformancePoint,
+	ModelStats,
+	ModelTimeSeriesPoint,
+	TimeSeriesPoint,
+} from "./types";
+
+/**
+ * Format cost in dollars.
+ */
+function formatCost(n: number): string {
+	if (n < 0.01) return `$${n.toFixed(4)}`;
+	if (n < 1) return `$${n.toFixed(3)}`;
+	return `$${n.toFixed(2)}`;
+}
+
+function normalizePremiumRequests(n: number): number {
+	return Math.round((n + Number.EPSILON) * 100) / 100;
+}
+
+/**
+ * Print stats summary to console.
+ */
+async function printStats(): Promise<void> {
+	const stats = await getDashboardStats();
+	const { overall, byModel, byFolder } = stats;
+
+	console.log("\n=== AI Usage Statistics ===\n");
+
+	console.log("Overall:");
+	console.log(`  Requests: ${formatNumber(overall.totalRequests)} (${formatNumber(overall.failedRequests)} errors)`);
+	console.log(`  Error Rate: ${formatPercent(overall.errorRate)}`);
+	console.log(`  Total Tokens: ${formatNumber(overall.totalInputTokens + overall.totalOutputTokens)}`);
+	console.log(`  Input Tokens: ${formatNumber(overall.totalInputTokens)}`);
+	console.log(`  Output Tokens: ${formatNumber(overall.totalOutputTokens)}`);
+	console.log(`  Cache Rate: ${formatPercent(overall.cacheRate)}`);
+	console.log(`  Total Cost: ${formatCost(overall.totalCost)}`);
+	console.log(`  Premium Requests: ${formatNumber(normalizePremiumRequests(overall.totalPremiumRequests ?? 0))}`);
+	console.log(`  Avg Duration: ${overall.avgDuration !== null ? formatDuration(overall.avgDuration) : "-"}`);
+	console.log(`  Avg TTFT: ${overall.avgTtft !== null ? formatDuration(overall.avgTtft) : "-"}`);
+	if (overall.avgTokensPerSecond !== null) {
+		console.log(`  Avg Tokens/s: ${overall.avgTokensPerSecond.toFixed(1)}`);
+	}
+
+	if (byModel.length > 0) {
+		console.log("\nBy Model:");
+		for (const m of byModel.slice(0, 10)) {
+			console.log(
+				`  ${m.model}: ${formatNumber(m.totalRequests)} reqs, ${formatCost(m.totalCost)}, ${formatPercent(m.cacheRate)} cache`,
+			);
+		}
+	}
+
+	if (byFolder.length > 0) {
+		console.log("\nBy Folder:");
+		for (const f of byFolder.slice(0, 10)) {
+			console.log(`  ${f.folder}: ${formatNumber(f.totalRequests)} reqs, ${formatCost(f.totalCost)}`);
+		}
+	}
+
+	console.log("");
+}
+
+/**
+ * Main CLI entry point.
+ */
+async function main(): Promise<void> {
+	const { values } = parseArgs({
+		options: {
+			port: { type: "string", short: "p", default: "3847" },
+			json: { type: "boolean", short: "j", default: false },
+			sync: { type: "boolean", short: "s", default: false },
+			help: { type: "boolean", short: "h", default: false },
+		},
+		allowPositionals: true,
+	});
+
+	if (values.help) {
+		console.log(`
+gjc-stats - AI Usage Statistics Dashboard
+
+Usage:
+  gjc-stats [options]
+
+Options:
+  -p, --port <port>  Port for the dashboard server (default: 3847)
+  -j, --json         Output stats as JSON and exit
+  -s, --sync         Sync session files and show summary
+  -h, --help         Show this help message
+
+Examples:
+  gjc-stats              # Start dashboard server
+  gjc-stats --json       # Print stats as JSON
+  gjc-stats --port 8080  # Start on custom port
+  gjc-stats --sync       # Sync and show summary
+`);
+		return;
+	}
+
+	try {
+		// Sync first
+		const tty = process.stderr.isTTY === true;
+		process.stderr.write("Syncing session files...\n");
+		let lastWidth = 0;
+		let lastRender = 0;
+		const { processed, files } = await syncAllSessions({
+			onProgress: event => {
+				if (!tty) return;
+				const now = Date.now();
+				if (event.current < event.total && now - lastRender < 33) return;
+				lastRender = now;
+				const marker = "/sessions/";
+				const idx = event.sessionFile.indexOf(marker);
+				const short = idx >= 0 ? event.sessionFile.slice(idx + marker.length) : event.sessionFile;
+				const pct = ((event.current / event.total) * 100).toFixed(0).padStart(3, " ");
+				const line = `[${event.current}/${event.total}] ${pct}%  ${short}`;
+				const columns = process.stderr.columns ?? 120;
+				const clipped = line.length > columns - 1 ? `${line.slice(0, columns - 2)}\u2026` : line;
+				process.stderr.write(`\r${clipped.padEnd(lastWidth)}`);
+				lastWidth = clipped.length;
+			},
+		});
+		if (tty && lastWidth > 0) process.stderr.write(`\r${" ".repeat(lastWidth)}\r`);
+		const total = await getTotalMessageCount();
+		console.log(`Synced ${processed} new entries from ${files} files (${total} total)\n`);
+
+		if (values.json) {
+			const stats = await getDashboardStats();
+			console.log(JSON.stringify(stats, null, 2));
+			return;
+		}
+
+		if (values.sync) {
+			await printStats();
+			return;
+		}
+
+		// Start server
+		const port = parseInt(values.port || "3847", 10);
+		const { port: actualPort } = await startServer(port);
+		console.log(`Dashboard available at: http://localhost:${actualPort}`);
+		console.log("Press Ctrl+C to stop\n");
+
+		// Keep process running
+		process.on("SIGINT", () => {
+			console.log("\nShutting down...");
+			closeDb();
+			process.exit(0);
+		});
+	} catch (error) {
+		console.error("Error:", error);
+		closeDb();
+		process.exit(1);
+	}
+}
+
+// Run if executed directly
+if (import.meta.main) {
+	main();
+}

package/src/parser.ts

@@ -0,0 +1,334 @@
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import { type AssistantMessage, getPriorityPremiumRequests, type ServiceTier } from "@gajae-code/ai";
+import { getSessionsDir, isEnoent } from "@gajae-code/utils";
+import type {
+	MessageStats,
+	SessionEntry,
+	SessionMessageEntry,
+	SessionServiceTierChangeEntry,
+	UserMessageLink,
+	UserMessageStats,
+} from "./types";
+import { computeUserMessageMetrics } from "./user-metrics";
+
+/**
+ * Extract folder name from session filename.
+ * Session files are named like: --work--pi--/timestamp_uuid.jsonl
+ * The folder part uses -- as path separator.
+ */
+function extractFolderFromPath(sessionPath: string): string {
+	const sessionsDir = getSessionsDir();
+	const rel = path.relative(sessionsDir, sessionPath);
+	const projectDir = rel.split(path.sep)[0];
+	// Convert --work--pi-- to /work/pi
+	return projectDir.replace(/^--/, "/").replace(/--/g, "/");
+}
+
+/**
+ * Check if an entry is an assistant message.
+ */
+function isAssistantMessage(entry: SessionEntry): entry is SessionMessageEntry {
+	if (entry.type !== "message") return false;
+	const msgEntry = entry as SessionMessageEntry;
+	// Legacy sessions (pre-id tracking) recorded message entries without an `id`.
+	// They're not linkable and would violate the messages.entry_id NOT NULL
+	// constraint, so skip them at the parser boundary.
+	if (typeof msgEntry.id !== "string" || msgEntry.id.length === 0) return false;
+	return msgEntry.message?.role === "assistant";
+}
+
+/**
+ * Check if an entry is a user message (non-toolResult).
+ */
+function isUserMessage(entry: SessionEntry): entry is SessionMessageEntry {
+	if (entry.type !== "message") return false;
+	const msgEntry = entry as SessionMessageEntry;
+	if (typeof msgEntry.id !== "string" || msgEntry.id.length === 0) return false;
+	return msgEntry.message?.role === "user";
+}
+
+/**
+ * Check if an entry is a service-tier change.
+ */
+function isServiceTierChange(entry: SessionEntry): entry is SessionServiceTierChangeEntry {
+	return entry.type === "service_tier_change";
+}
+
+/**
+ * Extract plain text from a user message content payload.
+ */
+function extractUserText(content: unknown): string {
+	if (typeof content === "string") return content;
+	if (!Array.isArray(content)) return "";
+	const parts: string[] = [];
+	for (const block of content) {
+		if (block && typeof block === "object" && (block as { type?: unknown }).type === "text") {
+			const text = (block as { text?: unknown }).text;
+			if (typeof text === "string") parts.push(text);
+		}
+	}
+	return parts.join("");
+}
+
+/**
+ * Build user-message stats from an entry. Returns null for empty/synthetic content.
+ */
+function extractUserStats(sessionFile: string, folder: string, entry: SessionMessageEntry): UserMessageStats | null {
+	const msg = entry.message as { role: "user"; content?: unknown; synthetic?: boolean };
+	if (msg.role !== "user" || msg.synthetic) return null;
+	const text = extractUserText(msg.content);
+	if (!text.trim()) return null;
+	const metrics = computeUserMessageMetrics(text);
+	const ts = Date.parse(entry.timestamp);
+	return {
+		sessionFile,
+		entryId: entry.id,
+		folder,
+		timestamp: Number.isFinite(ts) ? ts : 0,
+		model: null,
+		provider: null,
+		chars: metrics.chars,
+		words: metrics.words,
+		yelling: metrics.yelling,
+		profanity: metrics.profanity,
+		anguish: metrics.anguish,
+		negation: metrics.negation,
+		repetition: metrics.repetition,
+		blame: metrics.blame,
+	};
+}
+
+/**
+ * Extract stats from an assistant message entry.
+ */
+function extractStats(
+	sessionFile: string,
+	folder: string,
+	entry: SessionMessageEntry,
+	currentServiceTier: ServiceTier | undefined,
+): MessageStats | null {
+	const msg = entry.message as AssistantMessage;
+	if (!msg || msg.role !== "assistant") return null;
+
+	// Backfill: when the session recorded `priority` as the active service tier
+	// at this point but the AI usage payload was captured before priority
+	// requests were folded into `premiumRequests`, derive the count here so the
+	// "Premium Reqs" stat aggregates priority traffic on re-sync. Trust any
+	// non-zero value already in `usage.premiumRequests` (Copilot multipliers or
+	// the new AI code path) and only synthesise when the field is missing/zero.
+	const recorded = msg.usage.premiumRequests ?? 0;
+	const derived = recorded > 0 ? recorded : getPriorityPremiumRequests(currentServiceTier, msg.provider);
+	const usage = derived === recorded ? msg.usage : { ...msg.usage, premiumRequests: derived };
+
+	return {
+		sessionFile,
+		entryId: entry.id,
+		folder,
+		model: msg.model,
+		provider: msg.provider,
+		api: msg.api,
+		timestamp: msg.timestamp,
+		duration: msg.duration ?? null,
+		ttft: msg.ttft ?? null,
+		stopReason: msg.stopReason,
+		errorMessage: msg.errorMessage ?? null,
+		usage,
+	};
+}
+
+const LF = 0x0a;
+
+function parseSessionEntriesLenient(bytes: Uint8Array): { entries: SessionEntry[]; read: number } {
+	const entries: SessionEntry[] = [];
+	let cursor = 0;
+
+	while (cursor < bytes.length) {
+		const { values, error, read, done } = Bun.JSONL.parseChunk(bytes, cursor, bytes.length);
+		if (values.length > 0) {
+			entries.push(...(values as SessionEntry[]));
+		}
+
+		if (error) {
+			const nextNewline = bytes.indexOf(LF, Math.max(read, cursor));
+			if (nextNewline === -1) break;
+			cursor = nextNewline + 1;
+			continue;
+		}
+
+		if (read <= cursor) break;
+		cursor = read;
+		if (done) break;
+	}
+
+	return { entries, read: cursor };
+}
+
+function scanLastServiceTier(bytes: Uint8Array): ServiceTier | undefined {
+	let cursor = 0;
+	let currentServiceTier: ServiceTier | undefined;
+
+	while (cursor < bytes.length) {
+		const { values, error, read, done } = Bun.JSONL.parseChunk(bytes, cursor, bytes.length);
+		for (const value of values as SessionEntry[]) {
+			if (isServiceTierChange(value)) currentServiceTier = value.serviceTier ?? undefined;
+		}
+
+		if (error) {
+			const nextNewline = bytes.indexOf(LF, Math.max(read, cursor));
+			if (nextNewline === -1) break;
+			cursor = nextNewline + 1;
+			continue;
+		}
+
+		if (read <= cursor) break;
+		cursor = read;
+		if (done) break;
+	}
+
+	return currentServiceTier;
+}
+/**
+ * Parse a session file and extract all assistant message stats.
+ * Uses incremental reading with offset tracking.
+ *
+ * Service-tier carry-over: `currentServiceTier` is a session-scoped piece of
+ * state derived from `service_tier_change` entries that affects whether
+ * subsequent OpenAI assistant replies count as premium requests. Incremental
+ * syncs that resume past the most-recent tier change would otherwise lose
+ * that state and silently record `premiumRequests = 0` for priority traffic
+ * (the coding-agent stopped folding the tier into `usage.premiumRequests`
+ * after 13f59162e — the parser is now the sole source of truth). When
+ * `fromOffset > 0` we therefore scan the bytes preceding `fromOffset`
+ * for the latest service-tier value before parsing the unprocessed tail.
+ * The scan only keeps the current tier and does not materialize prefix
+ * entries, preserving offset-based memory behavior for large sessions.
+ */
+export interface ParseSessionResult {
+	stats: MessageStats[];
+	userStats: UserMessageStats[];
+	userLinks: UserMessageLink[];
+	newOffset: number;
+}
+export async function parseSessionFile(sessionPath: string, fromOffset = 0): Promise<ParseSessionResult> {
+	let bytes: Uint8Array;
+	try {
+		bytes = await Bun.file(sessionPath).bytes();
+	} catch (err) {
+		if (isEnoent(err)) return { stats: [], userStats: [], userLinks: [], newOffset: fromOffset };
+		throw err;
+	}
+
+	const folder = extractFolderFromPath(sessionPath);
+	const stats: MessageStats[] = [];
+	const userStats: UserMessageStats[] = [];
+	const userLinks: UserMessageLink[] = [];
+	const userByEntryId = new Map<string, UserMessageStats>();
+	const start = Math.max(0, Math.min(fromOffset, bytes.length));
+	const unprocessed = bytes.subarray(start);
+	const { entries, read } = parseSessionEntriesLenient(unprocessed);
+	let currentServiceTier: ServiceTier | undefined;
+	if (start > 0) {
+		currentServiceTier = scanLastServiceTier(bytes.subarray(0, start));
+	}
+	for (const entry of entries) {
+		if (isServiceTierChange(entry)) {
+			currentServiceTier = entry.serviceTier ?? undefined;
+			continue;
+		}
+		if (isUserMessage(entry)) {
+			const userMsg = extractUserStats(sessionPath, folder, entry);
+			if (userMsg) {
+				userStats.push(userMsg);
+				userByEntryId.set(entry.id, userMsg);
+			}
+			continue;
+		}
+		if (isAssistantMessage(entry)) {
+			const msgStats = extractStats(sessionPath, folder, entry, currentServiceTier);
+			if (msgStats) stats.push(msgStats);
+			// Link assistant's responding model back to the user message it answered.
+			const parentId = (entry as SessionMessageEntry).parentId;
+			if (parentId) {
+				const msg = entry.message as AssistantMessage;
+				if (msg.model && msg.provider) {
+					// Emit unconditionally. The aggregator's UPDATE is guarded by
+					// `model IS NULL` so this is idempotent: a no-op for already
+					// linked rows, a fix-up for fresh inserts (which start NULL
+					// because the user row is recorded before its reply lands) and
+					// for cross-pass orphans whose parent was committed by an
+					// earlier incremental sync.
+					userLinks.push({
+						sessionFile: sessionPath,
+						entryId: parentId,
+						model: msg.model,
+						provider: msg.provider,
+					});
+				}
+			}
+		}
+	}
+
+	return { stats, userStats, userLinks, newOffset: start + read };
+}
+
+/**
+ * List all session directories (folders).
+ */
+export async function listSessionFolders(): Promise<string[]> {
+	try {
+		const sessionsDir = getSessionsDir();
+		const entries = await fs.readdir(sessionsDir, { withFileTypes: true });
+		return entries.filter(e => e.isDirectory()).map(e => path.join(sessionsDir, e.name));
+	} catch {
+		return [];
+	}
+}
+
+/**
+ * List all session files in a folder.
+ */
+export async function listSessionFiles(folderPath: string): Promise<string[]> {
+	try {
+		const entries = await fs.readdir(folderPath, { recursive: true, withFileTypes: true });
+		return entries.filter(e => e.isFile() && e.name.endsWith(".jsonl")).map(e => path.join(e.parentPath, e.name));
+	} catch {
+		return [];
+	}
+}
+
+/**
+ * List all session files across all folders.
+ */
+export async function listAllSessionFiles(): Promise<string[]> {
+	const folders = await listSessionFolders();
+	const allFiles: string[] = [];
+
+	for (const folder of folders) {
+		const files = await listSessionFiles(folder);
+		allFiles.push(...files);
+	}
+
+	return allFiles;
+}
+
+/**
+ * Find a specific entry in a session file.
+ */
+export async function getSessionEntry(sessionPath: string, entryId: string): Promise<SessionEntry | null> {
+	let bytes: Uint8Array;
+	try {
+		bytes = await Bun.file(sessionPath).bytes();
+	} catch (err) {
+		if (isEnoent(err)) return null;
+		throw err;
+	}
+
+	const { entries } = parseSessionEntriesLenient(bytes);
+	for (const entry of entries) {
+		if ("id" in entry && entry.id === entryId) {
+			return entry;
+		}
+	}
+	return null;
+}