@narumitw/pi-analytics 0.45.0

package/src/collector.ts

@@ -0,0 +1,297 @@
+import { randomUUID } from "node:crypto";
+import { classifyProviderError } from "./errors.js";
+import type {
+	GenerationOutcome,
+	GenerationRecord,
+	ModelIdentity,
+	ProviderErrorRecord,
+	RunOutcome,
+	SettledRun,
+	SkillActivationRecord,
+	ToolCallRecord,
+	TriggerSource,
+} from "./types.js";
+
+interface ActiveRun {
+	id: string;
+	startedAtMs: number;
+	triggerSource: TriggerSource;
+	initialModel?: ModelIdentity;
+	attemptCount: number;
+	generations: GenerationRecord[];
+	generationIds: Set<string>;
+	tools: ToolCallRecord[];
+	toolIds: Set<string>;
+	skills: SkillActivationRecord[];
+	skillIndexes: Map<string, number>;
+	providerErrors: ProviderErrorRecord[];
+}
+
+export class ResponseCollector {
+	private active: ActiveRun | undefined;
+
+	hasActiveRun(): boolean {
+		return this.active !== undefined;
+	}
+
+	begin(input: {
+		id: string;
+		now: number;
+		triggerSource: TriggerSource;
+		model?: ModelIdentity;
+	}): SettledRun | undefined {
+		const interrupted = this.active ? this.finalize(input.now, "interrupted") : undefined;
+		this.active = {
+			id: input.id,
+			startedAtMs: input.now,
+			triggerSource: input.triggerSource,
+			initialModel: input.model,
+			attemptCount: 0,
+			generations: [],
+			generationIds: new Set(),
+			tools: [],
+			toolIds: new Set(),
+			skills: [],
+			skillIndexes: new Map(),
+			providerErrors: [],
+		};
+		return interrupted;
+	}
+
+	beginAttempt(): void {
+		if (this.active) this.active.attemptCount += 1;
+	}
+
+	beginGeneration(input: { id: string; now: number; model?: ModelIdentity }): void {
+		const active = this.active;
+		if (!active || active.generationIds.has(input.id)) return;
+		active.generationIds.add(input.id);
+		active.generations.push({
+			id: input.id,
+			ordinal: active.generations.length,
+			provider: input.model?.provider,
+			model: input.model?.model,
+			thinkingLevel: input.model?.thinkingLevel,
+			startedAtMs: input.now,
+			outcome: "pending",
+			responses: [],
+		});
+	}
+
+	recordProviderResponse(input: { status: number; now: number }): void {
+		const generation = this.latestGeneration();
+		if (generation?.outcome !== "pending") return;
+		generation.responses.push({
+			ordinal: generation.responses.length,
+			occurredAtMs: input.now,
+			status: input.status,
+		});
+	}
+
+	finishGeneration(input: { now: number; stopReason: string; errorMessage?: string }): void {
+		const active = this.active;
+		const generation = this.latestGeneration();
+		if (!active || !generation || generation.outcome !== "pending") return;
+		generation.finishedAtMs = input.now;
+		generation.durationMs = elapsed(generation.startedAtMs, input.now);
+		generation.stopReason = input.stopReason;
+		generation.outcome = generationOutcome(input.stopReason);
+		if (generation.outcome === "error") {
+			active.providerErrors.push({
+				id: randomUUID(),
+				generationId: generation.id,
+				occurredAtMs: input.now,
+				provider: generation.provider,
+				model: generation.model,
+				category: classifyProviderError(input.errorMessage),
+				recovered: false,
+				terminal: true,
+			});
+		}
+	}
+
+	beginTool(input: { id: string; name: string; now: number; model?: ModelIdentity }): void {
+		const active = this.active;
+		if (!active || active.toolIds.has(input.id)) return;
+		active.toolIds.add(input.id);
+		active.tools.push({
+			id: input.id,
+			ordinal: active.tools.length,
+			name: input.name,
+			provider: input.model?.provider,
+			model: input.model?.model,
+			startedAtMs: input.now,
+			isError: false,
+			completionState: "running",
+		});
+	}
+
+	finishTool(input: { id: string; now: number; isError: boolean }): void {
+		const tool = this.active?.tools.find(({ id }) => id === input.id);
+		if (tool?.completionState !== "running") return;
+		tool.finishedAtMs = input.now;
+		tool.durationMs = elapsed(tool.startedAtMs, input.now);
+		tool.isError = input.isError;
+		tool.completionState = "finished";
+	}
+
+	activateSkill(input: {
+		name: string;
+		initiatedBy: "user" | "model";
+		now: number;
+		model?: ModelIdentity;
+	}): void {
+		const active = this.active;
+		if (!active) return;
+		const existingIndex = active.skillIndexes.get(input.name);
+		if (existingIndex !== undefined) {
+			const existing = active.skills[existingIndex];
+			if (existing && existing.initiatedBy === "model" && input.initiatedBy === "user") {
+				existing.initiatedBy = "user";
+				existing.occurredAtMs = input.now;
+				existing.provider = input.model?.provider;
+				existing.model = input.model?.model;
+			}
+			return;
+		}
+		active.skillIndexes.set(input.name, active.skills.length);
+		active.skills.push({
+			id: randomUUID(),
+			name: input.name,
+			initiatedBy: input.initiatedBy,
+			occurredAtMs: input.now,
+			provider: input.model?.provider,
+			model: input.model?.model,
+		});
+	}
+
+	settle(now: number): SettledRun | undefined {
+		return this.finalize(now);
+	}
+
+	interrupt(now: number): SettledRun | undefined {
+		return this.finalize(now, "interrupted");
+	}
+
+	private latestGeneration(): GenerationRecord | undefined {
+		return this.active?.generations.at(-1);
+	}
+
+	private finalize(now: number, forcedOutcome?: RunOutcome): SettledRun | undefined {
+		const active = this.active;
+		if (!active) return undefined;
+		this.active = undefined;
+
+		for (const generation of active.generations) {
+			if (generation.outcome !== "pending") continue;
+			generation.outcome = "interrupted";
+			generation.finishedAtMs = now;
+			generation.durationMs = elapsed(generation.startedAtMs, now);
+		}
+		for (const tool of active.tools) {
+			if (tool.completionState !== "running") continue;
+			tool.completionState = "interrupted";
+			tool.finishedAtMs = now;
+			tool.durationMs = elapsed(tool.startedAtMs, now);
+		}
+
+		const successfulGenerationIndexes = new Set(
+			active.generations
+				.map((generation, index) => ({ generation, index }))
+				.filter(({ generation }) => isSuccessfulGeneration(generation))
+				.map(({ index }) => index),
+		);
+		let recoveredHttpErrors = 0;
+		let httpErrors = 0;
+		for (const [generationIndex, generation] of active.generations.entries()) {
+			const hasLaterSuccess = [...successfulGenerationIndexes].some(
+				(index) => index > generationIndex,
+			);
+			for (const [responseIndex, response] of generation.responses.entries()) {
+				if (response.status < 400) continue;
+				httpErrors += 1;
+				const laterSuccessInGeneration = generation.responses
+					.slice(responseIndex + 1)
+					.some(({ status }) => status >= 200 && status < 400);
+				if (laterSuccessInGeneration || hasLaterSuccess) recoveredHttpErrors += 1;
+			}
+		}
+		for (const error of active.providerErrors) {
+			const generationIndex = active.generations.findIndex(({ id }) => id === error.generationId);
+			error.recovered = [...successfulGenerationIndexes].some((index) => index > generationIndex);
+			error.terminal = !error.recovered;
+		}
+
+		const recoveredGenerationErrors = active.providerErrors.filter(
+			({ recovered }) => recovered,
+		).length;
+		const providerErrorCount = httpErrors + active.providerErrors.length;
+		const recoveredErrorCount = recoveredHttpErrors + recoveredGenerationErrors;
+		const outcome = forcedOutcome ?? deriveOutcome(active.generations, providerErrorCount);
+
+		return {
+			id: active.id,
+			startedAtMs: active.startedAtMs,
+			finishedAtMs: now,
+			durationMs: elapsed(active.startedAtMs, now),
+			triggerSource: active.triggerSource,
+			initialProvider: active.initialModel?.provider,
+			initialModel: active.initialModel?.model,
+			outcome,
+			attemptCount: active.attemptCount,
+			generations: active.generations,
+			tools: active.tools,
+			skills: active.skills,
+			providerErrors: active.providerErrors,
+			toolErrorCount: active.tools.filter(({ isError }) => isError).length,
+			providerErrorCount,
+			recoveredErrorCount,
+		};
+	}
+}
+
+function elapsed(start: number, end: number): number {
+	return Math.max(0, end - start);
+}
+
+function generationOutcome(stopReason: string): GenerationOutcome {
+	switch (stopReason) {
+		case "stop":
+			return "stop";
+		case "toolUse":
+			return "tool_use";
+		case "error":
+			return "error";
+		case "aborted":
+			return "aborted";
+		case "length":
+			return "length";
+		default:
+			return "interrupted";
+	}
+}
+
+function isSuccessfulGeneration(generation: GenerationRecord): boolean {
+	return generation.outcome === "stop" || generation.outcome === "tool_use";
+}
+
+function deriveOutcome(
+	generations: readonly GenerationRecord[],
+	providerErrors: number,
+): RunOutcome {
+	const last = generations.at(-1);
+	if (!last) return providerErrors > 0 ? "error" : "success";
+	switch (last.outcome) {
+		case "stop":
+		case "tool_use":
+			return providerErrors > 0 ? "recovered_success" : "success";
+		case "error":
+			return "error";
+		case "aborted":
+			return "aborted";
+		case "length":
+			return "length";
+		default:
+			return "interrupted";
+	}
+}

package/src/errors.ts

@@ -0,0 +1,16 @@
+import type { ProviderErrorCategory } from "./types.js";
+
+export function classifyProviderError(message: string | undefined): ProviderErrorCategory {
+	const value = message?.toLowerCase() ?? "";
+	if (/\b(enotfound|eai_again|dns|getaddrinfo)\b/u.test(value)) return "dns";
+	if (/\b(etimedout|timeout|timed out)\b/u.test(value)) return "timeout";
+	if (/\b(econnrefused|connection refused)\b/u.test(value)) return "connection_refused";
+	if (/\b(econnreset|connection reset|socket hang up)\b/u.test(value)) {
+		return "connection_reset";
+	}
+	if (/\b(tls|ssl|certificate|cert_|handshake)\b/u.test(value)) return "tls";
+	if (/\b(fetch failed|network|socket|connection|transport)\b/u.test(value)) {
+		return "network_other";
+	}
+	return "provider_other";
+}

package/src/index.ts

@@ -0,0 +1 @@
+export { default } from "./analytics.js";

package/src/menu.ts

@@ -0,0 +1,372 @@
+import type { ExtensionCommandContext } from "@earendil-works/pi-coding-agent";
+import { defineMenu, runMenu, runTask } from "@narumitw/pi-tui-kit";
+import type {
+	AnalyticsSnapshot,
+	SkillStats,
+	TimeRange,
+	TimeRangeId,
+	ToolStats,
+} from "./storage/queries.js";
+import { resolveTimeRange } from "./storage/queries.js";
+
+export type AnalyticsLoadResult =
+	| { kind: "ready"; snapshot: AnalyticsSnapshot }
+	| { kind: "unavailable"; message: string };
+
+export interface AnalyticsMenuDataSource {
+	path: string;
+	load(range: TimeRange, signal: AbortSignal): Promise<AnalyticsLoadResult>;
+	clearAll(signal: AbortSignal): Promise<number>;
+}
+
+export interface AnalyticsMenuState {
+	rangeId: TimeRangeId;
+	range: TimeRange;
+	path: string;
+	result: AnalyticsLoadResult;
+}
+
+type Screen = "main" | "range" | "skills" | "tools" | "reliability" | "responses" | "privacy";
+type Action = "setRange" | "clearData";
+
+const RANGE_LABELS: Record<TimeRangeId, string> = {
+	today: "Today",
+	"7d": "Last 7 days",
+	"30d": "Last 30 days",
+	all: "All time",
+};
+
+export function createAnalyticsMenu(source: AnalyticsMenuDataSource, now: () => number = Date.now) {
+	let rangeId: TimeRangeId = "7d";
+	let cachedState: AnalyticsMenuState | undefined;
+	const loadState = async (signal: AbortSignal): Promise<AnalyticsMenuState> => {
+		if (cachedState?.rangeId === rangeId) return cachedState;
+		const range = resolveTimeRange(rangeId, now());
+		const loaded = { rangeId, range, path: source.path, result: await source.load(range, signal) };
+		if (!signal.aborted && rangeId === loaded.rangeId) cachedState = loaded;
+		return loaded;
+	};
+	const getState = ({ signal }: { signal: AbortSignal }): Promise<AnalyticsMenuState> =>
+		loadState(signal);
+	const menu = defineMenu<AnalyticsMenuState, Screen, Action>({
+		start: "main",
+		screens: {
+			main: ({ state }) => ({
+				kind: "actions",
+				title: `Analytics · ${RANGE_LABELS[state.rangeId]}`,
+				lines: overviewLines(state.result),
+				items: [
+					{ id: "range", label: "Change time range", to: "range" },
+					{ id: "skills", label: "Skills", to: "skills" },
+					{ id: "tools", label: "Tools", to: "tools" },
+					{ id: "reliability", label: "Provider reliability", to: "reliability" },
+					{ id: "responses", label: "Response cycles", to: "responses" },
+					{ id: "privacy", label: "Data & privacy", to: "privacy" },
+					{ id: "close", label: "Close", close: true },
+				],
+				hint: "close",
+			}),
+			range: ({ state }) => ({
+				kind: "choice",
+				title: "Analytics time range",
+				items: (Object.keys(RANGE_LABELS) as TimeRangeId[]).map((id) => ({
+					id,
+					label: RANGE_LABELS[id],
+				})),
+				action: "setRange",
+				currentItemId: state.rangeId,
+				initialItemId: state.rangeId,
+				hint: "back",
+			}),
+			skills: ({ state }) => skillsScreen(state.result),
+			tools: ({ state }) => toolsScreen(state.result),
+			reliability: ({ state }) => ({
+				kind: "detail",
+				title: `Provider reliability · ${RANGE_LABELS[state.rangeId]}`,
+				lines: reliabilityLines(state.result),
+				hint: "back",
+			}),
+			responses: ({ state }) => ({
+				kind: "detail",
+				title: `Response cycles · ${RANGE_LABELS[state.rangeId]}`,
+				lines: responseLines(state.result),
+				hint: "back",
+			}),
+			privacy: ({ state }) => ({
+				kind: "actions",
+				title: "Analytics data & privacy",
+				lines: privacyLines(state),
+				items: [
+					{
+						id: "clear",
+						label: "Clear analytics data…",
+						action: "clearData",
+						disabled: state.result.kind !== "ready",
+					},
+				],
+				hint: "back",
+			}),
+		},
+		actions: {
+			setRange: async ({ itemId, signal }) => {
+				if (!isRangeId(itemId)) return { kind: "rejected", error: new Error("Unknown range") };
+				rangeId = itemId;
+				cachedState = undefined;
+				await loadState(signal);
+				return signal.aborted ? { kind: "close" } : { kind: "to", screen: "main" };
+			},
+			clearData: async ({ ctx, state, signal }) => {
+				if (state.result.kind !== "ready") return { kind: "stay" };
+				const count = state.result.snapshot.overview.responseCycles;
+				const confirmed = await ctx.ui.confirm(
+					"Delete analytics data?",
+					`This will delete ${count} response cycles and their tool, skill, and reliability records from:\n\n${state.path}\n\nOther running Pi processes may add new records afterward.`,
+					{ signal },
+				);
+				if (!confirmed || signal.aborted) return { kind: "stay" };
+				const deleted = await source.clearAll(signal);
+				cachedState = undefined;
+				try {
+					ctx.ui.notify(`Deleted ${deleted} response cycles from local analytics.`, "info");
+				} catch {
+					// Session replacement can invalidate the UI after the database commit.
+				}
+				return signal.aborted ? { kind: "close" } : { kind: "to", screen: "main" };
+			},
+		},
+	});
+	return {
+		menu,
+		getState,
+		preload: (signal: AbortSignal) => loadState(signal),
+		get rangeId() {
+			return rangeId;
+		},
+	};
+}
+
+export async function showAnalyticsMenu(
+	ctx: ExtensionCommandContext,
+	source: AnalyticsMenuDataSource,
+	options: { signal: AbortSignal; isCurrent: () => boolean },
+): Promise<void> {
+	const controller = createAnalyticsMenu(source);
+	const loading = await runTask(ctx, {
+		label: "Loading local analytics…",
+		signal: options.signal,
+		isCurrent: options.isCurrent,
+		task: ({ signal }) => controller.preload(signal),
+		onError: () => undefined,
+	});
+	if (loading.kind !== "completed") {
+		if (loading.kind === "error") {
+			ctx.ui.notify(
+				"Analytics failed: The local analytics query could not be completed. Existing data was not changed.",
+				"error",
+			);
+		}
+		return;
+	}
+	await runMenu(ctx, controller.menu, {
+		getState: controller.getState,
+		signal: options.signal,
+		isCurrent: options.isCurrent,
+		onError: (_ctx, error) => {
+			ctx.ui.notify(`Analytics failed: ${safeErrorMessage(error)}`, "error");
+		},
+	});
+}
+
+function overviewLines(result: AnalyticsLoadResult): string[] {
+	if (result.kind === "unavailable") {
+		return [result.message, "", "No analytics are being collected."];
+	}
+	const stats = result.snapshot.overview;
+	if (stats.responseCycles === 0) {
+		return [
+			"No analytics yet.",
+			"Collection is active. Complete one Pi response cycle, then open /analytics again.",
+			"",
+			"Includes settled response cycles only.",
+		];
+	}
+	return [
+		metric("Response cycles", stats.responseCycles),
+		metric("LLM calls", stats.llmCalls),
+		metric(
+			"Calls per response",
+			`${formatDecimal(stats.callsPerResponse)} · P95 ${stats.p95CallsPerResponse}`,
+		),
+		metric("Tool calls", stats.toolCalls),
+		metric("Tool errors", stats.toolErrors),
+		metric("Skill activations", stats.skillActivations),
+		metric("Provider errors", stats.providerErrors),
+		metric("Recovered errors", stats.recoveredErrors),
+		"",
+		"Includes settled response cycles only.",
+	];
+}
+
+function skillsScreen(result: AnalyticsLoadResult) {
+	if (result.kind === "unavailable") {
+		return {
+			kind: "browse" as const,
+			title: "Skills",
+			lines: [result.message],
+			items: [],
+			hint: "back" as const,
+		};
+	}
+	return {
+		kind: "browse" as const,
+		title: "Skills",
+		lines:
+			result.snapshot.skills.length === 0
+				? ["No skill activations detected in this time range."]
+				: undefined,
+		items: result.snapshot.skills.map(skillItem),
+		viewportSize: "adaptive" as const,
+		hint: "back" as const,
+	};
+}
+
+function skillItem(skill: SkillStats) {
+	return {
+		id: skill.name,
+		label: skill.name,
+		statusText: `${skill.count} · ${skill.modelInitiated} model / ${skill.userInitiated} user`,
+		searchText: skill.models.map(modelLabel).join(" "),
+		details: [
+			metric("Activations", skill.count),
+			metric("Model initiated", skill.modelInitiated),
+			metric("User initiated", skill.userInitiated),
+			"",
+			"By model",
+			...skill.models.map((model) => `${modelLabel(model)}: ${model.count}`),
+			"",
+			`Last detected: ${formatTimestamp(skill.lastOccurredAtMs)}`,
+		],
+	};
+}
+
+function toolsScreen(result: AnalyticsLoadResult) {
+	if (result.kind === "unavailable") {
+		return {
+			kind: "browse" as const,
+			title: "Tools",
+			lines: [result.message],
+			items: [],
+			hint: "back" as const,
+		};
+	}
+	return {
+		kind: "browse" as const,
+		title: "Tools",
+		lines:
+			result.snapshot.tools.length === 0
+				? ["No tool calls detected in this time range."]
+				: undefined,
+		items: result.snapshot.tools.map(toolItem),
+		viewportSize: "adaptive" as const,
+		hint: "back" as const,
+	};
+}
+
+function toolItem(tool: ToolStats) {
+	return {
+		id: tool.name,
+		label: tool.name,
+		statusText: `${tool.count} · ${tool.errors} errors`,
+		searchText: tool.models.map(modelLabel).join(" "),
+		details: [
+			metric("Calls", tool.count),
+			metric("Errors", tool.errors),
+			`Average duration: ${formatDecimal(tool.averageDurationMs)} ms`,
+			"",
+			"By model",
+			...tool.models.map((model) => `${modelLabel(model)}: ${model.count}`),
+			"",
+			`Last detected: ${formatTimestamp(tool.lastOccurredAtMs)}`,
+		],
+	};
+}
+
+function reliabilityLines(result: AnalyticsLoadResult): string[] {
+	if (result.kind === "unavailable") return [result.message];
+	const value = result.snapshot.reliability;
+	return [
+		"Observed provider errors only; provider-internal failures may be invisible.",
+		"",
+		metric("HTTP 429", value.http429),
+		metric("HTTP 5xx", value.http5xx),
+		metric("DNS", value.categories.dns),
+		metric("Connection timeout", value.categories.timeout),
+		metric("Connection refused", value.categories.connection_refused),
+		metric("Connection reset", value.categories.connection_reset),
+		metric("TLS", value.categories.tls),
+		metric("Other network", value.categories.network_other),
+		metric("Other provider", value.categories.provider_other),
+		"",
+		metric("Recovered", value.recovered),
+		metric("Terminal failures", value.terminal),
+	];
+}
+
+function responseLines(result: AnalyticsLoadResult): string[] {
+	if (result.kind === "unavailable") return [result.message];
+	const value = result.snapshot.responses;
+	return [
+		metric("Cycles", value.count),
+		metric("LLM calls", value.llmCalls),
+		metric("Average", formatDecimal(value.average)),
+		metric("Median", formatDecimal(value.median)),
+		metric("P95", value.p95),
+		metric("Maximum", value.maximum),
+		"",
+		"Calls per response",
+		metric("1 call", value.distribution.one),
+		metric("2–3 calls", value.distribution.twoToThree),
+		metric("4–6 calls", value.distribution.fourToSix),
+		metric("7+ calls", value.distribution.sevenPlus),
+	];
+}
+
+function privacyLines(state: AnalyticsMenuState): string[] {
+	return [
+		"Local database:",
+		state.path,
+		"",
+		"Stored: timestamps, model/provider IDs, thinking level, tool and skill names, durations, counts, HTTP statuses, and classified errors.",
+		"Not stored: prompts, responses, thinking, tool arguments/results, raw errors, headers, cwd/file paths, session identity, or credentials.",
+		"",
+		"No Turso Cloud connection or other remote telemetry is used.",
+		"Turso Database is pre-1.0; analytics are non-critical derived metadata.",
+	];
+}
+
+function metric(label: string, value: string | number): string {
+	return `${label.padEnd(24)} ${value}`;
+}
+
+function formatDecimal(value: number): string {
+	return Number.isInteger(value)
+		? String(value)
+		: value.toFixed(2).replace(/0+$/u, "").replace(/\.$/u, "");
+}
+
+function formatTimestamp(value: number): string {
+	return new Date(value).toLocaleString();
+}
+
+function modelLabel(model: { provider?: string; model?: string }): string {
+	if (!model.provider && !model.model) return "unknown";
+	return `${model.provider ?? "unknown"}/${model.model ?? "unknown"}`;
+}
+
+function isRangeId(value: string): value is TimeRangeId {
+	return value === "today" || value === "7d" || value === "30d" || value === "all";
+}
+
+function safeErrorMessage(_error: unknown): string {
+	return "The local analytics query could not be completed. Try again; existing data was not changed.";
+}