@narumitw/pi-analytics 0.45.0 → 0.48.1

package/src/storage/format.ts

@@ -0,0 +1,281 @@
+import type {
+	GenerationOutcome,
+	GenerationRecord,
+	ProviderErrorCategory,
+	ProviderErrorRecord,
+	ProviderResponseRecord,
+	RunOutcome,
+	SettledRun,
+	SkillActivationRecord,
+	ToolCallRecord,
+} from "../types.js";
+
+export const MAX_STORED_RUN_BYTES = 1024 * 1024;
+const MAX_STRING_LENGTH = 4096;
+const MAX_NESTED_RECORDS = 20_000;
+const TRIGGER_SOURCES = ["interactive", "rpc", "extension", "unknown"] as const;
+const RUN_OUTCOMES = [
+	"success",
+	"recovered_success",
+	"error",
+	"aborted",
+	"length",
+	"interrupted",
+] as const;
+const GENERATION_OUTCOMES = [
+	"pending",
+	"stop",
+	"tool_use",
+	"error",
+	"aborted",
+	"length",
+	"interrupted",
+] as const;
+const ERROR_CATEGORIES = [
+	"dns",
+	"timeout",
+	"connection_refused",
+	"connection_reset",
+	"tls",
+	"network_other",
+	"provider_other",
+] as const;
+
+export class AnalyticsStorageFormatError extends Error {
+	constructor(message: string, options: { cause?: unknown } = {}) {
+		super(message, options);
+		this.name = "AnalyticsStorageFormatError";
+	}
+}
+
+export function encodeStoredRun(run: SettledRun): string {
+	const encoded = `${JSON.stringify({
+		formatVersion: 1,
+		run: parseRun(run, { remaining: MAX_NESTED_RECORDS }),
+	})}\n`;
+	if (Buffer.byteLength(encoded) > MAX_STORED_RUN_BYTES) {
+		throw new AnalyticsStorageFormatError("Analytics record is too large to store safely.");
+	}
+	return encoded;
+}
+
+export function decodeStoredRun(line: string): SettledRun {
+	if (Buffer.byteLength(line) > MAX_STORED_RUN_BYTES) {
+		throw new AnalyticsStorageFormatError("Analytics record is too large to read safely.");
+	}
+	let value: unknown;
+	try {
+		value = JSON.parse(line);
+	} catch (error) {
+		throw new AnalyticsStorageFormatError("Analytics record contains invalid JSON.", {
+			cause: error,
+		});
+	}
+	const envelope = asRecord(value, "analytics record");
+	if (envelope.formatVersion !== 1) {
+		throw new AnalyticsStorageFormatError("Analytics record uses an unsupported format version.");
+	}
+	return parseRun(envelope.run, { remaining: MAX_NESTED_RECORDS });
+}
+
+interface ParseBudget {
+	remaining: number;
+}
+
+function parseRun(value: unknown, budget: ParseBudget): SettledRun {
+	const run = asRecord(value, "run");
+	return {
+		id: requiredString(run.id, "run.id"),
+		startedAtMs: timestampValue(run.startedAtMs, "run.startedAtMs"),
+		finishedAtMs: timestampValue(run.finishedAtMs, "run.finishedAtMs"),
+		durationMs: durationValue(run.durationMs, "run.durationMs"),
+		triggerSource: enumValue(run.triggerSource, TRIGGER_SOURCES, "run.triggerSource"),
+		...optionalProperty(
+			"initialProvider",
+			optionalString(run.initialProvider, "run.initialProvider"),
+		),
+		...optionalProperty("initialModel", optionalString(run.initialModel, "run.initialModel")),
+		outcome: enumValue(run.outcome, RUN_OUTCOMES, "run.outcome") as RunOutcome,
+		attemptCount: boundedCount(run.attemptCount, "run.attemptCount"),
+		generations: boundedArray(run.generations, "run.generations", budget).map((item, index) =>
+			parseGeneration(item, index, budget),
+		),
+		tools: boundedArray(run.tools, "run.tools", budget).map(parseTool),
+		skills: boundedArray(run.skills, "run.skills", budget).map(parseSkill),
+		providerErrors: boundedArray(run.providerErrors, "run.providerErrors", budget).map(
+			parseProviderError,
+		),
+		toolErrorCount: boundedCount(run.toolErrorCount, "run.toolErrorCount"),
+		providerErrorCount: boundedCount(run.providerErrorCount, "run.providerErrorCount"),
+		recoveredErrorCount: boundedCount(run.recoveredErrorCount, "run.recoveredErrorCount"),
+	};
+}
+
+function parseGeneration(value: unknown, index: number, budget: ParseBudget): GenerationRecord {
+	const item = asRecord(value, `run.generations[${index}]`);
+	const prefix = `run.generations[${index}]`;
+	return {
+		id: requiredString(item.id, `${prefix}.id`),
+		ordinal: boundedCount(item.ordinal, `${prefix}.ordinal`),
+		...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
+		...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
+		...optionalProperty(
+			"thinkingLevel",
+			optionalString(item.thinkingLevel, `${prefix}.thinkingLevel`),
+		),
+		startedAtMs: timestampValue(item.startedAtMs, `${prefix}.startedAtMs`),
+		...optionalProperty(
+			"finishedAtMs",
+			optionalTimestamp(item.finishedAtMs, `${prefix}.finishedAtMs`),
+		),
+		...optionalProperty("durationMs", optionalDuration(item.durationMs, `${prefix}.durationMs`)),
+		...optionalProperty("stopReason", optionalString(item.stopReason, `${prefix}.stopReason`)),
+		outcome: enumValue(item.outcome, GENERATION_OUTCOMES, `${prefix}.outcome`) as GenerationOutcome,
+		responses: boundedArray(item.responses, `${prefix}.responses`, budget).map(
+			parseProviderResponse,
+		),
+	};
+}
+
+function parseProviderResponse(value: unknown, index: number): ProviderResponseRecord {
+	const item = asRecord(value, `provider response ${index}`);
+	return {
+		ordinal: boundedCount(item.ordinal, "providerResponse.ordinal"),
+		occurredAtMs: timestampValue(item.occurredAtMs, "providerResponse.occurredAtMs"),
+		status: boundedInteger(item.status, "providerResponse.status", 999),
+	};
+}
+
+function parseTool(value: unknown, index: number): ToolCallRecord {
+	const item = asRecord(value, `run.tools[${index}]`);
+	const prefix = `run.tools[${index}]`;
+	const ordinal = boundedCount(item.ordinal, `${prefix}.ordinal`);
+	return {
+		id: `tool-${ordinal}`,
+		ordinal,
+		name: requiredString(item.name, `${prefix}.name`),
+		...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
+		...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
+		startedAtMs: timestampValue(item.startedAtMs, `${prefix}.startedAtMs`),
+		...optionalProperty(
+			"finishedAtMs",
+			optionalTimestamp(item.finishedAtMs, `${prefix}.finishedAtMs`),
+		),
+		...optionalProperty("durationMs", optionalDuration(item.durationMs, `${prefix}.durationMs`)),
+		isError: booleanValue(item.isError, `${prefix}.isError`),
+		completionState: enumValue(
+			item.completionState,
+			["running", "finished", "interrupted"] as const,
+			`${prefix}.completionState`,
+		),
+	};
+}
+
+function parseSkill(value: unknown, index: number): SkillActivationRecord {
+	const item = asRecord(value, `run.skills[${index}]`);
+	const prefix = `run.skills[${index}]`;
+	return {
+		id: requiredString(item.id, `${prefix}.id`),
+		name: requiredString(item.name, `${prefix}.name`),
+		initiatedBy: enumValue(item.initiatedBy, ["user", "model"] as const, `${prefix}.initiatedBy`),
+		occurredAtMs: timestampValue(item.occurredAtMs, `${prefix}.occurredAtMs`),
+		...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
+		...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
+	};
+}
+
+function parseProviderError(value: unknown, index: number): ProviderErrorRecord {
+	const item = asRecord(value, `run.providerErrors[${index}]`);
+	const prefix = `run.providerErrors[${index}]`;
+	return {
+		id: requiredString(item.id, `${prefix}.id`),
+		...optionalProperty(
+			"generationId",
+			optionalString(item.generationId, `${prefix}.generationId`),
+		),
+		occurredAtMs: timestampValue(item.occurredAtMs, `${prefix}.occurredAtMs`),
+		...optionalProperty("provider", optionalString(item.provider, `${prefix}.provider`)),
+		...optionalProperty("model", optionalString(item.model, `${prefix}.model`)),
+		category: enumValue(
+			item.category,
+			ERROR_CATEGORIES,
+			`${prefix}.category`,
+		) as ProviderErrorCategory,
+		recovered: booleanValue(item.recovered, `${prefix}.recovered`),
+		terminal: booleanValue(item.terminal, `${prefix}.terminal`),
+	};
+}
+
+function asRecord(value: unknown, name: string): Record<string, unknown> {
+	if (typeof value !== "object" || value === null || Array.isArray(value)) invalid(name);
+	return value as Record<string, unknown>;
+}
+
+function boundedArray(value: unknown, name: string, budget: ParseBudget): unknown[] {
+	if (!Array.isArray(value)) invalid(name);
+	budget.remaining -= value.length;
+	if (budget.remaining < 0) {
+		throw new AnalyticsStorageFormatError("Analytics record is too large to process safely.");
+	}
+	return value;
+}
+
+function requiredString(value: unknown, name: string): string {
+	if (typeof value !== "string" || value.length === 0 || value.length > MAX_STRING_LENGTH) {
+		invalid(name);
+	}
+	return value;
+}
+
+function optionalString(value: unknown, name: string): string | undefined {
+	return value === undefined ? undefined : requiredString(value, name);
+}
+
+function boundedInteger(value: unknown, name: string, maximum: number): number {
+	if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0 || value > maximum) {
+		invalid(name);
+	}
+	return value;
+}
+
+function timestampValue(value: unknown, name: string): number {
+	return boundedInteger(value, name, Number.MAX_SAFE_INTEGER);
+}
+
+function optionalTimestamp(value: unknown, name: string): number | undefined {
+	return value === undefined ? undefined : timestampValue(value, name);
+}
+
+function durationValue(value: unknown, name: string): number {
+	return boundedInteger(value, name, Math.floor(Number.MAX_SAFE_INTEGER / MAX_NESTED_RECORDS));
+}
+
+function optionalDuration(value: unknown, name: string): number | undefined {
+	return value === undefined ? undefined : durationValue(value, name);
+}
+
+function boundedCount(value: unknown, name: string): number {
+	return boundedInteger(value, name, MAX_NESTED_RECORDS);
+}
+
+function booleanValue(value: unknown, name: string): boolean {
+	if (typeof value !== "boolean") invalid(name);
+	return value;
+}
+
+function enumValue<const T extends readonly string[]>(
+	value: unknown,
+	values: T,
+	name: string,
+): T[number] {
+	if (typeof value !== "string" || !values.includes(value)) invalid(name);
+	return value as T[number];
+}
+
+function optionalProperty<K extends string, T>(key: K, value: T | undefined): { [P in K]?: T } {
+	return value === undefined ? {} : ({ [key]: value } as { [P in K]?: T });
+}
+
+function invalid(name: string): never {
+	throw new AnalyticsStorageFormatError(`Analytics record has an invalid ${name}.`);
+}

package/src/storage/queries.ts

@@ -1,5 +1,4 @@
-import type { Database } from "@tursodatabase/database";
-import type { ProviderErrorCategory } from "../types.js";
+import type { ProviderErrorCategory, SettledRun } from "../types.js";
 
 export type TimeRangeId = "today" | "7d" | "30d" | "all";
 export interface TimeRange {
@@ -92,168 +91,120 @@ export function resolveTimeRange(id: TimeRangeId, now = Date.now()): TimeRange {
 }
 
 export async function querySnapshot(
-	database: Database,
+	runs: AsyncIterable<SettledRun> | Iterable<SettledRun>,
 	range: TimeRange,
+	signal?: AbortSignal,
 ): Promise<AnalyticsSnapshot> {
-	const [runs, skillRows, toolRows, categoryRows, statusRows] = await Promise.all([
-		database.all(
-			`SELECT generation_count, tool_call_count, tool_error_count,
-			        skill_activation_count, provider_error_count, recovered_error_count
-			 FROM response_runs
-			 WHERE started_at_ms >= ? AND started_at_ms < ?
-			 ORDER BY generation_count`,
-			range.fromMs,
-			range.toMs,
-		),
-		database.all(
-			`SELECT s.skill_name, s.initiated_by, s.provider, s.model,
-			        COUNT(*) AS count, MAX(s.occurred_at_ms) AS last_at
-			 FROM skill_activations s
-			 JOIN response_runs r ON r.id = s.run_id
-			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
-			 GROUP BY s.skill_name, s.initiated_by, s.provider, s.model`,
-			range.fromMs,
-			range.toMs,
-		),
-		database.all(
-			`SELECT t.tool_name, t.provider, t.model, COUNT(*) AS count,
-			        SUM(t.is_error) AS errors, AVG(COALESCE(t.duration_ms, 0)) AS average_duration,
-			        MAX(t.started_at_ms) AS last_at
-			 FROM tool_calls t
-			 JOIN response_runs r ON r.id = t.run_id
-			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
-			 GROUP BY t.tool_name, t.provider, t.model`,
-			range.fromMs,
-			range.toMs,
-		),
-		database.all(
-			`SELECT e.category, COUNT(*) AS count, SUM(e.terminal) AS terminal
-			 FROM provider_errors e
-			 JOIN response_runs r ON r.id = e.run_id
-			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
-			 GROUP BY e.category`,
-			range.fromMs,
-			range.toMs,
-		),
-		database.all(
-			`SELECT p.status, COUNT(*) AS count
-			 FROM provider_responses p
-			 JOIN model_generations g ON g.id = p.generation_id
-			 JOIN response_runs r ON r.id = g.run_id
-			 WHERE r.started_at_ms >= ? AND r.started_at_ms < ?
-			 GROUP BY p.status`,
-			range.fromMs,
-			range.toMs,
-		),
-	]);
-
-	const generationCounts = runs.map((row) => numberValue(row.generation_count));
-	const responseStats = responseStatistics(generationCounts);
-	const overview: OverviewStats = {
-		responseCycles: runs.length,
-		llmCalls: sum(generationCounts),
-		callsPerResponse: responseStats.average,
-		p95CallsPerResponse: responseStats.p95,
-		toolCalls: sum(runs.map((row) => numberValue(row.tool_call_count))),
-		toolErrors: sum(runs.map((row) => numberValue(row.tool_error_count))),
-		skillActivations: sum(runs.map((row) => numberValue(row.skill_activation_count))),
-		providerErrors: sum(runs.map((row) => numberValue(row.provider_error_count))),
-		recoveredErrors: sum(runs.map((row) => numberValue(row.recovered_error_count))),
-	};
-
+	const generationCounts: number[] = [];
+	const seenRunIds = new Set<string>();
+	const skills = new Map<string, SkillStats>();
+	const tools = new Map<string, ToolStats & { totalDurationMs: number }>();
 	const categories = Object.fromEntries(
 		ERROR_CATEGORIES.map((category) => [category, 0]),
 	) as Record<ProviderErrorCategory, number>;
-	let terminal = 0;
-	for (const row of categoryRows) {
-		const category = String(row.category) as ProviderErrorCategory;
-		if (category in categories) categories[category] = numberValue(row.count);
-		terminal += numberValue(row.terminal);
-	}
+	let toolErrors = 0;
+	let providerErrors = 0;
+	let recoveredErrors = 0;
 	let http429 = 0;
 	let http5xx = 0;
-	for (const row of statusRows) {
-		const status = numberValue(row.status);
-		if (status === 429) http429 += numberValue(row.count);
-		if (status >= 500 && status < 600) http5xx += numberValue(row.count);
+	let terminal = 0;
+
+	for await (const run of runs) {
+		throwIfAborted(signal);
+		if (seenRunIds.has(run.id)) continue;
+		seenRunIds.add(run.id);
+		if (run.startedAtMs < range.fromMs || run.startedAtMs >= range.toMs) continue;
+		generationCounts.push(run.generations.length);
+		toolErrors += run.toolErrorCount;
+		providerErrors += run.providerErrorCount;
+		recoveredErrors += run.recoveredErrorCount;
+
+		for (const skill of run.skills) {
+			const item = skills.get(skill.name) ?? {
+				name: skill.name,
+				count: 0,
+				modelInitiated: 0,
+				userInitiated: 0,
+				lastOccurredAtMs: 0,
+				models: [],
+			};
+			item.count += 1;
+			if (skill.initiatedBy === "user") item.userInitiated += 1;
+			else item.modelInitiated += 1;
+			item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, skill.occurredAtMs);
+			mergeModelCount(item.models, {
+				provider: skill.provider,
+				model: skill.model,
+				count: 1,
+			});
+			skills.set(skill.name, item);
+		}
+
+		for (const tool of run.tools) {
+			const item = tools.get(tool.name) ?? {
+				name: tool.name,
+				count: 0,
+				errors: 0,
+				averageDurationMs: 0,
+				totalDurationMs: 0,
+				lastOccurredAtMs: 0,
+				models: [],
+			};
+			item.count += 1;
+			item.errors += tool.isError ? 1 : 0;
+			item.totalDurationMs += tool.durationMs ?? 0;
+			item.averageDurationMs = item.totalDurationMs / item.count;
+			item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, tool.startedAtMs);
+			mergeModelCount(item.models, {
+				provider: tool.provider,
+				model: tool.model,
+				count: 1,
+			});
+			tools.set(tool.name, item);
+		}
+
+		for (const error of run.providerErrors) {
+			categories[error.category] += 1;
+			terminal += error.terminal ? 1 : 0;
+		}
+		for (const generation of run.generations) {
+			for (const response of generation.responses) {
+				if (response.status === 429) http429 += 1;
+				if (response.status >= 500 && response.status < 600) http5xx += 1;
+			}
+		}
 	}
 
+	const responses = responseStatistics(generationCounts);
 	return {
-		overview,
-		skills: foldSkills(skillRows),
-		tools: foldTools(toolRows),
+		overview: {
+			responseCycles: responses.count,
+			llmCalls: responses.llmCalls,
+			callsPerResponse: responses.average,
+			p95CallsPerResponse: responses.p95,
+			toolCalls: sum([...tools.values()].map(({ count }) => count)),
+			toolErrors,
+			skillActivations: sum([...skills.values()].map(({ count }) => count)),
+			providerErrors,
+			recoveredErrors,
+		},
+		skills: [...skills.values()]
+			.map((item) => ({ ...item, models: sortModels(item.models) }))
+			.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
+		tools: [...tools.values()]
+			.map(({ totalDurationMs: _, ...item }) => ({ ...item, models: sortModels(item.models) }))
+			.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name)),
 		reliability: {
 			http429,
 			http5xx,
-			recovered: overview.recoveredErrors,
+			recovered: recoveredErrors,
 			terminal,
 			categories,
 		},
-		responses: responseStats,
+		responses,
 	};
 }
 
-function foldSkills(rows: Array<Record<string, unknown>>): SkillStats[] {
-	const result = new Map<string, SkillStats>();
-	for (const row of rows) {
-		const name = String(row.skill_name);
-		const count = numberValue(row.count);
-		const item = result.get(name) ?? {
-			name,
-			count: 0,
-			modelInitiated: 0,
-			userInitiated: 0,
-			lastOccurredAtMs: 0,
-			models: [],
-		};
-		item.count += count;
-		if (row.initiated_by === "user") item.userInitiated += count;
-		else item.modelInitiated += count;
-		item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, numberValue(row.last_at));
-		mergeModelCount(item.models, {
-			provider: optionalString(row.provider),
-			model: optionalString(row.model),
-			count,
-		});
-		result.set(name, item);
-	}
-	return [...result.values()]
-		.map((item) => ({ ...item, models: sortModels(item.models) }))
-		.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name));
-}
-
-function foldTools(rows: Array<Record<string, unknown>>): ToolStats[] {
-	const result = new Map<string, ToolStats & { weightedDuration: number }>();
-	for (const row of rows) {
-		const name = String(row.tool_name);
-		const count = numberValue(row.count);
-		const average = numberValue(row.average_duration);
-		const item = result.get(name) ?? {
-			name,
-			count: 0,
-			errors: 0,
-			averageDurationMs: 0,
-			weightedDuration: 0,
-			lastOccurredAtMs: 0,
-			models: [],
-		};
-		item.count += count;
-		item.errors += numberValue(row.errors);
-		item.weightedDuration += average * count;
-		item.averageDurationMs = item.count > 0 ? item.weightedDuration / item.count : 0;
-		item.lastOccurredAtMs = Math.max(item.lastOccurredAtMs, numberValue(row.last_at));
-		mergeModelCount(item.models, {
-			provider: optionalString(row.provider),
-			model: optionalString(row.model),
-			count,
-		});
-		result.set(name, item);
-	}
-	return [...result.values()]
-		.map(({ weightedDuration: _, ...item }) => ({ ...item, models: sortModels(item.models) }))
-		.sort((left, right) => right.count - left.count || left.name.localeCompare(right.name));
-}
-
 function responseStatistics(generationCounts: number[]): ResponseStats {
 	const sorted = [...generationCounts].sort((left, right) => left - right);
 	const count = sorted.length;
@@ -300,12 +251,9 @@ function sortModels(models: ModelCount[]): ModelCount[] {
 	);
 }
 
-function optionalString(value: unknown): string | undefined {
-	return typeof value === "string" ? value : undefined;
-}
-
-function numberValue(value: unknown): number {
-	return typeof value === "number" && Number.isFinite(value) ? value : Number(value) || 0;
+function throwIfAborted(signal?: AbortSignal): void {
+	if (signal?.aborted)
+		throw signal.reason ?? new DOMException("Analytics query aborted", "AbortError");
 }
 
 function sum(values: readonly number[]): number {