@narumitw/pi-analytics 0.45.0

package/src/storage/queries.ts

@@ -0,0 +1,313 @@
+import type { Database } from "@tursodatabase/database";
+import type { ProviderErrorCategory } from "../types.js";
+
+export type TimeRangeId = "today" | "7d" | "30d" | "all";
+export interface TimeRange {
+	id?: TimeRangeId;
+	fromMs: number;
+	toMs: number;
+}
+
+export interface OverviewStats {
+	responseCycles: number;
+	llmCalls: number;
+	callsPerResponse: number;
+	p95CallsPerResponse: number;
+	toolCalls: number;
+	toolErrors: number;
+	skillActivations: number;
+	providerErrors: number;
+	recoveredErrors: number;
+}
+
+export interface ModelCount {
+	provider?: string;
+	model?: string;
+	count: number;
+}
+
+export interface SkillStats {
+	name: string;
+	count: number;
+	modelInitiated: number;
+	userInitiated: number;
+	lastOccurredAtMs: number;
+	models: ModelCount[];
+}
+
+export interface ToolStats {
+	name: string;
+	count: number;
+	errors: number;
+	averageDurationMs: number;
+	lastOccurredAtMs: number;
+	models: ModelCount[];
+}
+
+export interface ReliabilityStats {
+	http429: number;
+	http5xx: number;
+	recovered: number;
+	terminal: number;
+	categories: Record<ProviderErrorCategory, number>;
+}
+
+export interface ResponseStats {
+	count: number;
+	llmCalls: number;
+	average: number;
+	median: number;
+	p95: number;
+	maximum: number;
+	distribution: { one: number; twoToThree: number; fourToSix: number; sevenPlus: number };
+}
+
+export interface AnalyticsSnapshot {
+	overview: OverviewStats;
+	skills: SkillStats[];
+	tools: ToolStats[];
+	reliability: ReliabilityStats;
+	responses: ResponseStats;
+}
+
+const DAY_MS = 24 * 60 * 60 * 1_000;
+const ERROR_CATEGORIES: readonly ProviderErrorCategory[] = [
+	"dns",
+	"timeout",
+	"connection_refused",
+	"connection_reset",
+	"tls",
+	"network_other",
+	"provider_other",
+];
+
+export function resolveTimeRange(id: TimeRangeId, now = Date.now()): TimeRange {
+	let fromMs = 0;
+	if (id === "today") {
+		const date = new Date(now);
+		fromMs = new Date(date.getFullYear(), date.getMonth(), date.getDate()).getTime();
+	} else if (id === "7d") fromMs = now - 7 * DAY_MS;
+	else if (id === "30d") fromMs = now - 30 * DAY_MS;
+	return { id, fromMs, toMs: now + 1 };
+}
+
+export async function querySnapshot(
+	database: Database,
+	range: TimeRange,
+): Promise<AnalyticsSnapshot> {
+	const [runs, skillRows, toolRows, categoryRows, statusRows] = await Promise.all([
+		database.all(
+			`SELECT generation_count, tool_call_count, tool_error_count,
+			        skill_activation_count, provider_error_count, recovered_error_count
+			 FROM response_runs
+			 WHERE started_at_ms >= ? AND started_at_ms < ?
+			 ORDER BY generation_count`,
+			range.fromMs,
+			range.toMs,
+		),
+		database.all(
+			`SELECT s.skill_name, s.initiated_by, s.provider, s.model,
+			        COUNT(*) AS count, MAX(s.occurred_at_ms) AS last_at
+			 FROM skill_activations s
+			 JOIN response_runs r ON r.id = s.run_id
+			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
+			 GROUP BY s.skill_name, s.initiated_by, s.provider, s.model`,
+			range.fromMs,
+			range.toMs,
+		),
+		database.all(
+			`SELECT t.tool_name, t.provider, t.model, COUNT(*) AS count,
+			        SUM(t.is_error) AS errors, AVG(COALESCE(t.duration_ms, 0)) AS average_duration,
+			        MAX(t.started_at_ms) AS last_at
+			 FROM tool_calls t
+			 JOIN response_runs r ON r.id = t.run_id
+			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
+			 GROUP BY t.tool_name, t.provider, t.model`,
+			range.fromMs,
+			range.toMs,
+		),
+		database.all(
+			`SELECT e.category, COUNT(*) AS count, SUM(e.terminal) AS terminal
+			 FROM provider_errors e
+			 JOIN response_runs r ON r.id = e.run_id
+			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
+			 GROUP BY e.category`,
+			range.fromMs,
+			range.toMs,
+		),
+		database.all(
+			`SELECT p.status, COUNT(*) AS count
+			 FROM provider_responses p
+			 JOIN model_generations g ON g.id = p.generation_id
+			 JOIN response_runs r ON r.id = g.run_id
+			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
+			 GROUP BY p.status`,
+			range.fromMs,
+			range.toMs,
+		),
+	]);
+
+	const generationCounts = runs.map((row) => numberValue(row.generation_count));
+	const responseStats = responseStatistics(generationCounts);
+	const overview: OverviewStats = {
+		responseCycles: runs.length,
+		llmCalls: sum(generationCounts),
+		callsPerResponse: responseStats.average,
+		p95CallsPerResponse: responseStats.p95,
+		toolCalls: sum(runs.map((row) => numberValue(row.tool_call_count))),
+		toolErrors: sum(runs.map((row) => numberValue(row.tool_error_count))),
+		skillActivations: sum(runs.map((row) => numberValue(row.skill_activation_count))),
+		providerErrors: sum(runs.map((row) => numberValue(row.provider_error_count))),
+		recoveredErrors: sum(runs.map((row) => numberValue(row.recovered_error_count))),
+	};
+
+	const categories = Object.fromEntries(
+		ERROR_CATEGORIES.map((category) => [category, 0]),
+	) as Record<ProviderErrorCategory, number>;
+	let terminal = 0;
+	for (const row of categoryRows) {
+		const category = String(row.category) as ProviderErrorCategory;
+		if (category in categories) categories[category] = numberValue(row.count);
+		terminal += numberValue(row.terminal);
+	}
+	let http429 = 0;
+	let http5xx = 0;
+	for (const row of statusRows) {
+		const status = numberValue(row.status);
+		if (status === 429) http429 += numberValue(row.count);
+		if (status >= 500 && status < 600) http5xx += numberValue(row.count);
+	}
+
+	return {
+		overview,
+		skills: foldSkills(skillRows),
+		tools: foldTools(toolRows),
+		reliability: {
+			http429,
+			http5xx,
+			recovered: overview.recoveredErrors,
+			terminal,
+			categories,
+		},
+		responses: responseStats,
+	};
+}
+
+function foldSkills(rows: Array<Record<string, unknown>>): SkillStats[] {
+	const result = new Map<string, SkillStats>();
+	for (const row of rows) {
+		const name = String(row.skill_name);
+		const count = numberValue(row.count);
+		const item = result.get(name) ?? {
+			name,
+			count: 0,
+			modelInitiated: 0,
+			userInitiated: 0,
+			lastOccurredAtMs: 0,
+			models: [],
+		};
+		item.count += count;
+		if (row.initiated_by === "user") item.userInitiated += count;
+		else item.modelInitiated += count;
+		item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, numberValue(row.last_at));
+		mergeModelCount(item.models, {
+			provider: optionalString(row.provider),
+			model: optionalString(row.model),
+			count,
+		});
+		result.set(name, item);
+	}
+	return [...result.values()]
+		.map((item) => ({ ...item, models: sortModels(item.models) }))
+		.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name));
+}
+
+function foldTools(rows: Array<Record<string, unknown>>): ToolStats[] {
+	const result = new Map<string, ToolStats & { weightedDuration: number }>();
+	for (const row of rows) {
+		const name = String(row.tool_name);
+		const count = numberValue(row.count);
+		const average = numberValue(row.average_duration);
+		const item = result.get(name) ?? {
+			name,
+			count: 0,
+			errors: 0,
+			averageDurationMs: 0,
+			weightedDuration: 0,
+			lastOccurredAtMs: 0,
+			models: [],
+		};
+		item.count += count;
+		item.errors += numberValue(row.errors);
+		item.weightedDuration += average * count;
+		item.averageDurationMs = item.count > 0 ? item.weightedDuration / item.count : 0;
+		item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, numberValue(row.last_at));
+		mergeModelCount(item.models, {
+			provider: optionalString(row.provider),
+			model: optionalString(row.model),
+			count,
+		});
+		result.set(name, item);
+	}
+	return [...result.values()]
+		.map(({ weightedDuration: _, ...item }) => ({ ...item, models: sortModels(item.models) }))
+		.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name));
+}
+
+function responseStatistics(generationCounts: number[]): ResponseStats {
+	const sorted = [...generationCounts].sort((left, right) => left - right);
+	const count = sorted.length;
+	const llmCalls = sum(sorted);
+	const nearestRank = (percentile: number) =>
+		count === 0 ? 0 : (sorted[Math.max(0, Math.ceil(percentile * count) - 1)] ?? 0);
+	const median =
+		count === 0
+			? 0
+			: count % 2 === 1
+				? (sorted[Math.floor(count / 2)] ?? 0)
+				: ((sorted[count / 2 - 1] ?? 0) + (sorted[count / 2] ?? 0)) / 2;
+	return {
+		count,
+		llmCalls,
+		average: count > 0 ? llmCalls / count : 0,
+		median,
+		p95: nearestRank(0.95),
+		maximum: sorted.at(-1) ?? 0,
+		distribution: {
+			one: sorted.filter((value) => value === 1).length,
+			twoToThree: sorted.filter((value) => value >= 2 && value <= 3).length,
+			fourToSix: sorted.filter((value) => value >= 4 && value <= 6).length,
+			sevenPlus: sorted.filter((value) => value >= 7).length,
+		},
+	};
+}
+
+function mergeModelCount(models: ModelCount[], next: ModelCount): void {
+	const existing = models.find(
+		({ provider, model }) => provider === next.provider && model === next.model,
+	);
+	if (existing) existing.count += next.count;
+	else models.push(next);
+}
+
+function sortModels(models: ModelCount[]): ModelCount[] {
+	return models.sort(
+		(left, right) =>
+			right.count - left.count ||
+			`${left.provider ?? ""}/${left.model ?? ""}`.localeCompare(
+				`${right.provider ?? ""}/${right.model ?? ""}`,
+			),
+	);
+}
+
+function optionalString(value: unknown): string | undefined {
+	return typeof value === "string" ? value : undefined;
+}
+
+function numberValue(value: unknown): number {
+	return typeof value === "number" && Number.isFinite(value) ? value : Number(value) || 0;
+}
+
+function sum(values: readonly number[]): number {
+	return values.reduce((total, value) => total + value, 0);
+}

package/src/storage/store.ts

@@ -0,0 +1,249 @@
+import type { Database, Transaction } from "@tursodatabase/database";
+import type { SettledRun } from "../types.js";
+import type { OpenedAnalyticsDatabase } from "./database.js";
+import type { AnalyticsSnapshot, TimeRange } from "./queries.js";
+import { querySnapshot } from "./queries.js";
+
+const MAX_PENDING_RUNS = 100;
+const WRITE_ATTEMPTS = 6;
+
+export class AnalyticsStore {
+	private readonly pending: SettledRun[] = [];
+	private readonly activeQueries = new Set<Promise<unknown>>();
+	private readonly query: typeof querySnapshot;
+	private mutationTail: Promise<void> = Promise.resolve();
+	private closed = false;
+	private closePromise: Promise<void> | undefined;
+
+	constructor(
+		private readonly opened: OpenedAnalyticsDatabase,
+		dependencies: { querySnapshot?: typeof querySnapshot } = {},
+	) {
+		this.query = dependencies.querySnapshot ?? querySnapshot;
+	}
+
+	get path(): string {
+		return this.opened.path;
+	}
+
+	recordRun(run: SettledRun): Promise<void> {
+		if (this.closed) return Promise.reject(new Error("Analytics store is closed."));
+		if (this.pending.length >= MAX_PENDING_RUNS) this.pending.shift();
+		this.pending.push(run);
+		return this.enqueueMutation(() => this.flushPending());
+	}
+
+	getSnapshot(range: TimeRange): Promise<AnalyticsSnapshot> {
+		if (this.closed) return Promise.reject(new Error("Analytics store is closed."));
+		const query = (async () => {
+			await this.mutationTail;
+			return this.query(this.opened.connection, range);
+		})();
+		this.activeQueries.add(query);
+		void query.finally(() => this.activeQueries.delete(query)).catch(() => undefined);
+		return query;
+	}
+
+	clearAll(): Promise<number> {
+		if (this.closed) return Promise.reject(new Error("Analytics store is closed."));
+		let deleted = 0;
+		return this.enqueueMutation(async () => {
+			await withWriteRetry(async () => {
+				const transaction = this.opened.connection.transactionAsync(async (tx) => {
+					const row = (await tx.get("SELECT COUNT(*) AS count FROM response_runs")) as {
+						count?: number;
+					};
+					deleted = Number(row?.count ?? 0);
+					for (const table of [
+						"provider_responses",
+						"provider_errors",
+						"tool_calls",
+						"skill_activations",
+						"model_generations",
+						"response_runs",
+					]) {
+						await tx.exec(`DELETE FROM ${table}`);
+					}
+				});
+				await transaction.immediate();
+			});
+		}).then(() => deleted);
+	}
+
+	close(): Promise<void> {
+		if (this.closePromise) return this.closePromise;
+		this.closed = true;
+		this.closePromise = (async () => {
+			await this.mutationTail.catch(() => undefined);
+			await Promise.allSettled([...this.activeQueries]);
+			let pendingError: unknown;
+			try {
+				await this.flushPending();
+			} catch (error) {
+				pendingError = error;
+				this.pending.length = 0;
+			}
+			await this.opened.close();
+			if (pendingError !== undefined) {
+				throw new Error("Analytics pending writes could not be saved before close.", {
+					cause: pendingError,
+				});
+			}
+		})();
+		return this.closePromise;
+	}
+
+	private enqueueMutation(operation: () => Promise<void>): Promise<void> {
+		const result = this.mutationTail.then(operation);
+		this.mutationTail = result.catch(() => undefined);
+		return result;
+	}
+
+	private async flushPending(): Promise<void> {
+		while (this.pending.length > 0) {
+			const run = this.pending[0];
+			if (!run) return;
+			await withWriteRetry(() => writeRun(this.opened.connection, run));
+			this.pending.shift();
+		}
+	}
+}
+
+async function writeRun(database: Database, run: SettledRun): Promise<void> {
+	const transaction = database.transactionAsync(async (tx) => {
+		const existing = await tx.get("SELECT id FROM response_runs WHERE id = ?", run.id);
+		if (existing) return;
+		await insertRun(tx, run);
+		for (const generation of run.generations) {
+			await tx.run(
+				`INSERT INTO model_generations(
+					id, run_id, ordinal, provider, model, thinking_level, started_at_ms,
+					finished_at_ms, duration_ms, stop_reason, outcome
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+				generation.id,
+				run.id,
+				generation.ordinal,
+				generation.provider ?? null,
+				generation.model ?? null,
+				generation.thinkingLevel ?? null,
+				generation.startedAtMs,
+				generation.finishedAtMs ?? null,
+				generation.durationMs ?? null,
+				generation.stopReason ?? null,
+				generation.outcome,
+			);
+			for (const response of generation.responses) {
+				await tx.run(
+					`INSERT INTO provider_responses(generation_id, ordinal, occurred_at_ms, status)
+					 VALUES (?, ?, ?, ?)`,
+					generation.id,
+					response.ordinal,
+					response.occurredAtMs,
+					response.status,
+				);
+			}
+		}
+		for (const tool of run.tools) {
+			await tx.run(
+				`INSERT INTO tool_calls(
+					id, run_id, ordinal, tool_name, provider, model, started_at_ms,
+					finished_at_ms, duration_ms, is_error, completion_state
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+				tool.id,
+				run.id,
+				tool.ordinal,
+				tool.name,
+				tool.provider ?? null,
+				tool.model ?? null,
+				tool.startedAtMs,
+				tool.finishedAtMs ?? null,
+				tool.durationMs ?? null,
+				tool.isError ? 1 : 0,
+				tool.completionState,
+			);
+		}
+		for (const skill of run.skills) {
+			await tx.run(
+				`INSERT INTO skill_activations(
+					id, run_id, occurred_at_ms, skill_name, initiated_by, provider, model
+				) VALUES (?, ?, ?, ?, ?, ?, ?)`,
+				skill.id,
+				run.id,
+				skill.occurredAtMs,
+				skill.name,
+				skill.initiatedBy,
+				skill.provider ?? null,
+				skill.model ?? null,
+			);
+		}
+		for (const error of run.providerErrors) {
+			await tx.run(
+				`INSERT INTO provider_errors(
+					id, run_id, generation_id, occurred_at_ms, provider, model,
+					category, recovered, terminal
+				) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+				error.id,
+				run.id,
+				error.generationId ?? null,
+				error.occurredAtMs,
+				error.provider ?? null,
+				error.model ?? null,
+				error.category,
+				error.recovered ? 1 : 0,
+				error.terminal ? 1 : 0,
+			);
+		}
+	});
+	await transaction.immediate();
+}
+
+function insertRun(tx: Transaction, run: SettledRun): Promise<unknown> {
+	return tx.run(
+		`INSERT INTO response_runs(
+			id, started_at_ms, finished_at_ms, duration_ms, trigger_source,
+			initial_provider, initial_model, outcome, attempt_count, generation_count,
+			tool_call_count, tool_error_count, skill_activation_count,
+			provider_error_count, recovered_error_count
+		) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		run.id,
+		run.startedAtMs,
+		run.finishedAtMs,
+		run.durationMs,
+		run.triggerSource,
+		run.initialProvider ?? null,
+		run.initialModel ?? null,
+		run.outcome,
+		run.attemptCount,
+		run.generations.length,
+		run.tools.length,
+		run.toolErrorCount,
+		run.skills.length,
+		run.providerErrorCount,
+		run.recoveredErrorCount,
+	);
+}
+
+async function withWriteRetry(operation: () => Promise<void>): Promise<void> {
+	let lastError: unknown;
+	for (let attempt = 0; attempt < WRITE_ATTEMPTS; attempt += 1) {
+		try {
+			await operation();
+			return;
+		} catch (error) {
+			lastError = error;
+			if (!isConflict(error) || attempt + 1 >= WRITE_ATTEMPTS) throw error;
+			await new Promise((resolve) => setTimeout(resolve, 5 * (attempt + 1)));
+		}
+	}
+	throw lastError;
+}
+
+function isConflict(error: unknown): boolean {
+	const message =
+		error instanceof Error ? error.message.toLowerCase() : String(error).toLowerCase();
+	return (
+		message.includes("statement was interrupted") ||
+		message.includes("database is locked") ||
+		message.includes("database is busy")
+	);
+}

package/src/types.ts

@@ -0,0 +1,102 @@
+export type TriggerSource = "interactive" | "rpc" | "extension" | "unknown";
+export type RunOutcome =
+	| "success"
+	| "recovered_success"
+	| "error"
+	| "aborted"
+	| "length"
+	| "interrupted";
+export type GenerationOutcome =
+	| "pending"
+	| "stop"
+	| "tool_use"
+	| "error"
+	| "aborted"
+	| "length"
+	| "interrupted";
+export type ProviderErrorCategory =
+	| "dns"
+	| "timeout"
+	| "connection_refused"
+	| "connection_reset"
+	| "tls"
+	| "network_other"
+	| "provider_other";
+
+export interface ModelIdentity {
+	provider: string;
+	model: string;
+	thinkingLevel?: string;
+}
+
+export interface ProviderResponseRecord {
+	ordinal: number;
+	occurredAtMs: number;
+	status: number;
+}
+
+export interface GenerationRecord {
+	id: string;
+	ordinal: number;
+	provider?: string;
+	model?: string;
+	thinkingLevel?: string;
+	startedAtMs: number;
+	finishedAtMs?: number;
+	durationMs?: number;
+	stopReason?: string;
+	outcome: GenerationOutcome;
+	responses: ProviderResponseRecord[];
+}
+
+export interface ToolCallRecord {
+	id: string;
+	ordinal: number;
+	name: string;
+	provider?: string;
+	model?: string;
+	startedAtMs: number;
+	finishedAtMs?: number;
+	durationMs?: number;
+	isError: boolean;
+	completionState: "running" | "finished" | "interrupted";
+}
+
+export interface SkillActivationRecord {
+	id: string;
+	name: string;
+	initiatedBy: "user" | "model";
+	occurredAtMs: number;
+	provider?: string;
+	model?: string;
+}
+
+export interface ProviderErrorRecord {
+	id: string;
+	generationId?: string;
+	occurredAtMs: number;
+	provider?: string;
+	model?: string;
+	category: ProviderErrorCategory;
+	recovered: boolean;
+	terminal: boolean;
+}
+
+export interface SettledRun {
+	id: string;
+	startedAtMs: number;
+	finishedAtMs: number;
+	durationMs: number;
+	triggerSource: TriggerSource;
+	initialProvider?: string;
+	initialModel?: string;
+	outcome: RunOutcome;
+	attemptCount: number;
+	generations: GenerationRecord[];
+	tools: ToolCallRecord[];
+	skills: SkillActivationRecord[];
+	providerErrors: ProviderErrorRecord[];
+	toolErrorCount: number;
+	providerErrorCount: number;
+	recoveredErrorCount: number;
+}