gentle-pi 2.4.0 → 2.6.0

package/lib/runtime-metrics-pi-identity.ts

@@ -0,0 +1,113 @@
+import { findPackageJSON } from "node:module";
+
+// Catalog-name privacy classification ONLY. No identity issuance, route evidence,
+// registry reads, SDK hooks, auth resolution, network, or injected catalogs.
+export interface PiCatalogName {
+	readonly classification: "catalog_public" | "custom" | "unknown";
+	readonly modelId: string;
+}
+
+const UNKNOWN: PiCatalogName = Object.freeze({ classification: "unknown", modelId: "unknown" });
+const CUSTOM: PiCatalogName = Object.freeze({ classification: "custom", modelId: "custom" });
+// Initial provider coverage matches the accumulator. Other providers are custom;
+// additions require deliberate review. These are providers, not model-ID lists.
+const CATALOGS = ["anthropic", "openai", "openai-codex", "google", "google-vertex", "amazon-bedrock", "openrouter"] as const;
+const MAX_MODELS = 4096;
+const MAX_LOAD_ATTEMPTS = 3;
+
+function object(value: unknown): value is Record<string, unknown> {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function text(value: unknown, max: number): value is string {
+	return typeof value === "string" && value.length > 0 && value.length <= max;
+}
+
+function key(provider: string, modelId: string): string {
+	return JSON.stringify([provider, modelId]);
+}
+
+async function loadCatalog(): Promise<ReadonlyMap<string, PiCatalogName>> {
+	// Node >=22.19 supports findPackageJSON. Verify ESM lookup from this module
+	// selects Pi's own pi-ai package; fail rather than silently use another copy.
+	const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
+	const ownPackage = findPackageJSON("@earendil-works/pi-ai", import.meta.url);
+	if (!ownPackage || ownPackage !== findPackageJSON("@earendil-works/pi-ai", pi)) {
+		throw new Error("Pi catalog dependency mismatch");
+	}
+	const entries = new Map<string, PiCatalogName>();
+	let count = 0;
+	for (const provider of CATALOGS) {
+		// pi-ai 0.85.1 providers/* has an import-only export condition: use ESM,
+		// never require.resolve. Generated modules load packaged JSON, not registry
+		// overrides. Specifiers and export names come only from the fixed list.
+		const module = await import(`@earendil-works/pi-ai/providers/${provider}.models`);
+		const models: unknown = module[`${provider.replaceAll("-", "_").toUpperCase()}_MODELS`];
+		if (!object(models)) throw new Error("Invalid packaged model catalog");
+		for (const model of Object.values(models)) {
+			if (++count > MAX_MODELS) throw new RangeError("Packaged model catalog exceeds capacity");
+			if (!object(model) || model.provider !== provider || !text(model.id, 128)) {
+				throw new Error("Invalid packaged model name");
+			}
+			entries.set(key(provider, model.id), Object.freeze({ classification: "catalog_public", modelId: model.id }));
+		}
+	}
+	return entries;
+}
+
+export function createPiCatalogNameLookup(
+	load: () => Promise<ReadonlyMap<string, PiCatalogName>> = loadCatalog,
+	maxAttempts = MAX_LOAD_ATTEMPTS,
+): { lookup(input: unknown): Promise<PiCatalogName>; classify(input: unknown): PiCatalogName } {
+	let catalogPromise: Promise<ReadonlyMap<string, PiCatalogName>> | undefined;
+	let loadedCatalog: ReadonlyMap<string, PiCatalogName> | undefined;
+	let attempts = 0;
+	const classify = (input: unknown): PiCatalogName => {
+		if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
+		if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
+		return loadedCatalog ? loadedCatalog.get(key(input.provider, input.modelId)) ?? CUSTOM : UNKNOWN;
+	};
+	const lookup = async (input: unknown): Promise<PiCatalogName> => {
+		if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
+		if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
+		if (!loadedCatalog) {
+			if (!catalogPromise) {
+				if (attempts >= maxAttempts) throw new Error("Pi catalog load attempts exhausted");
+				attempts++;
+				catalogPromise = load().then(catalog => {
+					loadedCatalog = catalog;
+					return catalog;
+				}, error => {
+					catalogPromise = undefined;
+					throw error;
+				});
+			}
+			await catalogPromise;
+		}
+		return classify(input);
+	};
+	return { lookup, classify };
+}
+
+const catalogLookup = createPiCatalogNameLookup();
+
+/**
+ * Returns only a privacy-safe catalog name, NOT the model actually dispatched.
+ * Matching a public ID remains a public-name fact even on a custom endpoint;
+ * origin/API/endpoint strings are ignored, never treated as provenance evidence.
+ * Missing/malformed names are unknown; non-catalog names/aliases are custom.
+ * No modelVersion is invented. This labels a caller-observed selection only;
+ * actual dispatch/response identity requires separate future host evidence.
+ * One cached snapshot retains <=4096 names of <=128 code units; no input is kept.
+ * Missing/incompatible dependencies and catalog overflow reject, never skip.
+ */
+export async function lookupPiCatalogName(input: unknown): Promise<PiCatalogName> {
+	return catalogLookup.lookup(input);
+}
+
+/** Pure lookup after async catalog loading; unknown until initialization succeeds.
+ * Always recheck membership, never trust a caller's classification/public label.
+ */
+export function classifyPiCatalogName(input: unknown): PiCatalogName {
+	return catalogLookup.classify(input);
+}

package/lib/runtime-metrics-policy.ts

@@ -0,0 +1,51 @@
+import { gentleAiDevBinaryOverrideConfigured, resolveGentleAiBinary } from "./gentle-ai-binary.ts";
+import { createNodeExecFileAdapter, type ExecFileAdapter } from "./native-review-cli.ts";
+
+export interface RuntimeMetricsPolicyDeps { resolve?: () => string; exec?: ExecFileAdapter }
+
+export function runtimeMetricsEnvAllows(env: NodeJS.ProcessEnv): boolean {
+	// Unknown nonempty spellings veto too: never weaken a native environment veto.
+	const truthy = (value: string | undefined) => !["", "0", "false", "no", "off"].includes(value?.trim().toLowerCase() ?? "");
+	return !truthy(env.DO_NOT_TRACK) && !truthy(env.CI) && !truthy(env.GITHUB_ACTIONS)
+		&& env.GENTLE_AI_TELEMETRY?.trim() !== "0";
+}
+
+function publishedBinary(): string {
+	// This feature must not select an unpublished development binary or install one.
+	if (gentleAiDevBinaryOverrideConfigured()) throw new Error("Policy binary unavailable");
+	return resolveGentleAiBinary();
+}
+
+/** Only a validated boolean crosses this boundary; never raw stdout or errors.
+ * The shared adapter and outer timer bound asynchronous process waiting, NOT
+ * end-to-end time: the supported resolver synchronously reads/hashes the binary
+ * on every check. It exposes no supported validation cache. Keep verification
+ * intact rather than cache an unchecked path or invent stat-based trust.
+ * No retries, status/preview fallback, enrollment, installation or polling.
+ */
+export async function readRuntimeMetricsPolicy(cwd: string,
+	{ resolve = publishedBinary, exec = createNodeExecFileAdapter() }: RuntimeMetricsPolicyDeps = {}): Promise<boolean> {
+	const controller = new AbortController();
+	let timer: ReturnType<typeof setTimeout> | undefined;
+	try {
+		const deadline = new Promise<false>(done => {
+			timer = setTimeout(() => { controller.abort(); done(false); }, 1000);
+		});
+		const probe = async () => {
+			const result = await exec({ file: resolve(), arguments: ["telemetry", "policy", "--json"],
+				cwd, timeoutMs: 1000, maxBufferBytes: 2048, signal: controller.signal });
+			if (result.exitCode !== 0 || result.signal !== null || result.timedOut !== false
+				|| result.outputLimitExceeded !== false || result.stderr !== ""
+				|| typeof result.stdout !== "string" || Buffer.byteLength(result.stdout) > 2048) return false;
+			const value: unknown = JSON.parse(result.stdout);
+			if (!value || typeof value !== "object" || Array.isArray(value)) return false;
+			const row = value as Record<string, unknown>;
+			return Object.keys(row).sort().join(",") === "enabled,operation,reason,schema,source"
+				&& row.schema === "gentle-ai.telemetry-policy/v1" && row.operation === "policy"
+				&& typeof row.enabled === "boolean" && row.enabled && row.reason === "enabled"
+				&& ["DO_NOT_TRACK", "GENTLE_AI_TELEMETRY", "CI", "state", "default"].includes(row.source as string);
+		};
+		return await Promise.race([probe(), deadline]);
+	} catch { return false; }
+	finally { clearTimeout(timer); }
+}

package/lib/runtime-metrics.ts

@@ -0,0 +1,255 @@
+import { readFileSync } from "node:fs";
+import { classifyPiCatalogName } from "./runtime-metrics-pi-identity.ts";
+
+const runtimeSchema = JSON.parse(readFileSync(new URL("../contracts/telemetry/runtime-aggregate-v1.schema.json", import.meta.url), "utf8"));
+
+// Pure local accounting, not a telemetry transport or Pi event adapter.
+// Callers supply finalized assistant responses and authoritative classifications.
+// Never infer executor, usage availability, or measured timings from SDK defaults.
+const EXECUTORS = ["orchestrator", "worker", "reviewer", "unknown"] as const;
+const PROVIDERS = ["anthropic", "openai", "openai-codex", "google", "google-vertex", "amazon-bedrock", "openrouter", "custom", "unknown"] as const;
+// Stable families, not model IDs/versions: new models need no catalog update.
+// Callers map known native metadata to families; private aliases stay custom.
+const FAMILIES = ["claude", "gpt", "o-series", "gemini", "llama", "qwen", "deepseek", "kimi", "custom", "unknown"] as const;
+// Pi 0.85.1 docs/models.md, Thinking Level Map. These are selected Pi levels,
+// not inferred provider effort or a claim that each model supports every level.
+export const EFFORTS = ["off", "minimal", "low", "medium", "high", "xhigh", "max", "not_selected", "unsupported", "unavailable"] as const;
+const ERRORS = ["none", "aborted", "rate_limit", "authentication", "network", "provider", "unknown"] as const;
+const TOKEN_FIELDS = ["input", "output", "cacheRead", "cacheWrite", "reasoning", "totalTokens"] as const;
+declare const agentClassBrand: unique symbol;
+export type AgentClass = string & { readonly [agentClassBrand]: "AgentClass" };
+
+function agentClasses(): readonly AgentClass[] {
+	const values: unknown = runtimeSchema?.$defs?.row?.properties?.agent_class?.enum;
+	if (!Array.isArray(values) || !values.length || values.some(value => typeof value !== "string")) {
+		throw new Error("Invalid runtime telemetry agent_class schema");
+	}
+	return Object.freeze([...values]) as readonly AgentClass[];
+}
+
+// The mirrored transport contract is the runtime source of truth. Keeping this
+// data-driven lets packaged agent updates follow the closed enum without a
+// second name registry drifting in TypeScript.
+export const AGENT_CLASSES = agentClasses();
+export function parseAgentClass(value: unknown): AgentClass | undefined {
+	return typeof value === "string" && (AGENT_CLASSES as readonly string[]).includes(value) ? value as AgentClass : undefined;
+}
+function requiredAgentClass(value: string): AgentClass {
+	const parsed = parseAgentClass(value);
+	if (!parsed) throw new Error(`Runtime telemetry schema is missing required agent_class ${value}`);
+	return parsed;
+}
+export const UNKNOWN_AGENT_CLASS = requiredAgentClass("unknown");
+export const ORCHESTRATOR_AGENT_CLASS = requiredAgentClass("orchestrator");
+
+function registeredModels(): ReadonlySet<string> {
+	const rules: unknown = runtimeSchema?.$defs?.model?.oneOf;
+	if (!Array.isArray(rules) || !rules.length) throw new Error("Invalid runtime telemetry model schema");
+	const values = (rule: unknown, field: "provider" | "id"): string[] => {
+		if (!object(rule)) throw new Error("Invalid runtime telemetry model schema");
+		const properties = rule.properties;
+		if (!object(properties) || !object(properties[field])) throw new Error("Invalid runtime telemetry model schema");
+		const property = properties[field];
+		const candidates = typeof property.const === "string" ? [property.const] : property.enum;
+		if (!Array.isArray(candidates) || !candidates.length || candidates.some(value => typeof value !== "string")) {
+			throw new Error("Invalid runtime telemetry model schema");
+		}
+		return candidates as string[];
+	};
+	const models = new Set<string>();
+	for (const rule of rules) for (const provider of values(rule, "provider")) {
+		for (const id of values(rule, "id")) models.add(JSON.stringify([provider, id]));
+	}
+	return models;
+}
+
+const REGISTERED_MODELS = registeredModels();
+
+/** The mirrored transport registry is authoritative before the optional Pi
+ * catalog. Unknown registry pairs still pass through the existing privacy
+ * classifier, so private IDs can only become custom/unknown.
+ */
+export function classifyRuntimeModelId(provider: unknown, modelId: unknown,
+	classifyModel: typeof classifyPiCatalogName = classifyPiCatalogName): string {
+	if (typeof provider === "string" && typeof modelId === "string"
+		&& REGISTERED_MODELS.has(JSON.stringify([provider, modelId]))) return modelId;
+	return classifyModel({ provider, modelId }).modelId;
+}
+
+type Missing = { state: "unavailable" | "unsupported" };
+export type TokenMeasurement = Missing | { state: "reported"; value: number };
+export type DurationMeasurement = Missing | { state: "measured"; value: number };
+export interface FinalResponse {
+	kind: "final_assistant_response";
+	/** Local dedupe only: 1..128 UTF-16 code units; never exported. */
+	responseId: string;
+	/** Caller-observed selected SDK model ID, never dispatched/response identity.
+	 * Catalog must be loaded before accounting; only public catalog names survive.
+	 */
+	selectedModelId?: string;
+	/** Selection namespace, independent from observed response provider.
+	 * Omission preserves legacy same-provider callers; adapters must pass it explicitly.
+	 */
+	selectedProvider?: typeof PROVIDERS[number];
+	agentClass?: AgentClass;
+	/** Public catalog names from SDK response metadata, never endpoint proof. */
+	observedModelId?: string;
+	responseModelId?: string;
+	providerThinkingLevel?: typeof EFFORTS[number];
+	executor: typeof EXECUTORS[number];
+	provider: typeof PROVIDERS[number];
+	modelFamily: typeof FAMILIES[number];
+	effort: typeof EFFORTS[number];
+	error: typeof ERRORS[number];
+	/** Separate native counters; do not add cached tokens into input here. */
+	tokens: Record<"input" | "output" | "cacheRead" | "cacheWrite", TokenMeasurement>
+		& Partial<Record<"reasoning" | "totalTokens", TokenMeasurement>>;
+	/** Request start to response headers; not first token or full response. */
+	responseHeadersMs: DurationMeasurement;
+	/** Same request start to completed response; only explicitly measured values. */
+	fullResponseMs: DurationMeasurement;
+}
+
+interface TokenTotals { reported: number; unavailable: number; unsupported: number; sum: number }
+interface DurationTotals { measured: number; unavailable: number; unsupported: number; sum: number }
+export interface RuntimeMetricBucket {
+	hostAgent: "pi";
+	agentClass: AgentClass;
+	observedModelId: string;
+	responseModelId: string;
+	providerThinkingLevel: typeof EFFORTS[number];
+	selectedModelId: string;
+	selectedProvider: FinalResponse["provider"];
+	executor: FinalResponse["executor"];
+	provider: FinalResponse["provider"];
+	modelFamily: FinalResponse["modelFamily"];
+	effort: FinalResponse["effort"];
+	error: FinalResponse["error"];
+	responses: number;
+	tokens: Record<typeof TOKEN_FIELDS[number], TokenTotals>;
+	responseHeadersMs: DurationTotals;
+	fullResponseMs: DurationTotals;
+}
+
+function object(value: unknown): value is Record<string, unknown> {
+	return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function member<T extends string>(values: readonly T[], value: unknown): value is T {
+	return typeof value === "string" && values.includes(value as T);
+}
+
+function category<T extends string>(values: readonly T[], value: unknown, fallback: T): T {
+	return member(values, value) ? value : fallback;
+}
+
+function measurement(value: unknown, present: "reported" | "measured"): boolean {
+	if (!object(value)) return false;
+	if (value.state === "unavailable" || value.state === "unsupported") return !("value" in value);
+	if (value.state !== present || typeof value.value !== "number") return false;
+	const n = value.value;
+	// Hard ceilings keep every sum finite/exact for integer counters, even at capacity.
+	return Number.isFinite(n) && n >= 0 && (present === "reported"
+		? Number.isSafeInteger(n) && n <= 1_000_000_000
+		: n <= 86_400_000);
+}
+
+export function validRuntimeResponse(value: unknown): value is FinalResponse {
+	if (!object(value) || value.kind !== "final_assistant_response") return false;
+	if (typeof value.responseId !== "string" || value.responseId.length < 1 || value.responseId.length > 128) return false;
+	if (!member(EFFORTS, value.effort) || !member(ERRORS, value.error)) return false;
+	const tokens = value.tokens;
+	if (!object(tokens) || !TOKEN_FIELDS.every(key => measurement(tokens[key] === undefined && ["reasoning", "totalTokens"].includes(key)
+		? { state: "unavailable" } : tokens[key], "reported"))) return false;
+	if (!measurement(value.responseHeadersMs, "measured") || !measurement(value.fullResponseMs, "measured")) return false;
+	const headers = value.responseHeadersMs as DurationMeasurement;
+	const full = value.fullResponseMs as DurationMeasurement;
+	return headers.state !== "measured" || full.state !== "measured" || headers.value <= full.value;
+}
+
+function tokenTotals(): TokenTotals {
+	return { reported: 0, unavailable: 0, unsupported: 0, sum: 0 };
+}
+
+function durationTotals(): DurationTotals {
+	return { measured: 0, unavailable: 0, unsupported: 0, sum: 0 };
+}
+
+function addDuration(totals: DurationTotals, value: DurationMeasurement): void {
+	totals[value.state] += 1;
+	if (value.state === "measured") totals.sum += value.value;
+}
+
+/**
+ * At most 1024 accepted responses/IDs, 64 dimension buckets, and 128 code units
+ * per ID. No eviction: once capacity is reached, new records are rejected
+ * atomically (including existing buckets); accepted IDs remain deduplicated.
+ * Invalid/rejected IDs are not reserved. A new instance starts a new accounting
+ * window with NO cross-instance/lifetime dedupe guarantee. No reset/flush API:
+ * window ownership and delivery remain future work. Runtime consumption stays
+ * separate from deterministic SDD/RDD counts; no closure attribution or bridge.
+ * Snapshots contain only closed dimensions and bounded numeric aggregates.
+ */
+export class RuntimeMetrics {
+	#ids = new Set<string>();
+	#buckets = new Map<string, RuntimeMetricBucket>();
+	#maxResponses: number;
+	#maxBuckets: number;
+	#classifyModel: typeof classifyPiCatalogName;
+
+	constructor({ maxResponses = 1024, maxBuckets = 64, classifyModel = classifyPiCatalogName }:
+		{ maxResponses?: number; maxBuckets?: number; classifyModel?: typeof classifyPiCatalogName } = {}) {
+		if (!Number.isInteger(maxResponses) || maxResponses < 1 || maxResponses > 1024
+			|| !Number.isInteger(maxBuckets) || maxBuckets < 1 || maxBuckets > 64) {
+			throw new RangeError("Invalid runtime metrics capacity");
+		}
+		this.#maxResponses = maxResponses;
+		this.#maxBuckets = maxBuckets;
+		this.#classifyModel = classifyModel;
+	}
+
+	record(response: FinalResponse): "recorded" | "duplicate" | "invalid" | "capacity" {
+		if (!validRuntimeResponse(response)) return "invalid";
+		if (this.#ids.has(response.responseId)) return "duplicate";
+		const selectedProvider = response.selectedProvider ?? response.provider;
+		const dimensions = {
+			hostAgent: "pi" as const,
+			agentClass: category(AGENT_CLASSES, response.agentClass, UNKNOWN_AGENT_CLASS),
+			observedModelId: classifyRuntimeModelId(response.provider, response.observedModelId, this.#classifyModel),
+			responseModelId: classifyRuntimeModelId(response.provider, response.responseModelId, this.#classifyModel),
+			providerThinkingLevel: category(EFFORTS, response.providerThinkingLevel, "unavailable"),
+			selectedModelId: classifyRuntimeModelId(selectedProvider, response.selectedModelId, this.#classifyModel),
+			selectedProvider: category(PROVIDERS, selectedProvider, typeof selectedProvider === "string" && selectedProvider ? "custom" : "unknown"),
+			executor: category(EXECUTORS, response.executor, "unknown"),
+			provider: category(PROVIDERS, response.provider, typeof response.provider === "string" && response.provider ? "custom" : "unknown"),
+			modelFamily: category(FAMILIES, response.modelFamily, typeof response.modelFamily === "string" && response.modelFamily ? "custom" : "unknown"),
+			effort: response.effort,
+			error: response.error,
+		};
+		const key = JSON.stringify(dimensions);
+		let bucket = this.#buckets.get(key);
+		if (this.#ids.size >= this.#maxResponses || (!bucket && this.#buckets.size >= this.#maxBuckets)) return "capacity";
+		if (!bucket) {
+			bucket = {
+				...dimensions, responses: 0,
+				tokens: { input: tokenTotals(), output: tokenTotals(), cacheRead: tokenTotals(), cacheWrite: tokenTotals(), reasoning: tokenTotals(), totalTokens: tokenTotals() },
+				responseHeadersMs: durationTotals(), fullResponseMs: durationTotals(),
+			};
+			this.#buckets.set(key, bucket);
+		}
+		this.#ids.add(response.responseId);
+		bucket.responses += 1;
+		for (const field of TOKEN_FIELDS) {
+			const value = response.tokens[field] ?? { state: "unavailable" as const };
+			bucket.tokens[field][value.state] += 1;
+			if (value.state === "reported") bucket.tokens[field].sum += value.value;
+		}
+		addDuration(bucket.responseHeadersMs, response.responseHeadersMs);
+		addDuration(bucket.fullResponseMs, response.fullResponseMs);
+		return "recorded";
+	}
+
+	snapshot(): RuntimeMetricBucket[] {
+		return structuredClone([...this.#buckets.values()]);
+	}
+}