gentle-pi 2.6.0 → 2.6.1

package/tests/runtime-metrics-extension.test.ts

@@ -2,14 +2,12 @@ import assert from "node:assert/strict";
 import test from "node:test";
 import runtimeMetrics from "../extensions/runtime-metrics.ts";
 import { parseAgentClass, type RuntimeMetricBucket } from "../lib/runtime-metrics.ts";
-import { createPiCatalogNameLookup, type PiCatalogName } from "../lib/runtime-metrics-pi-identity.ts";
+import { encodeNativeRuntimeEvent } from "../lib/runtime-metrics-native.ts";
 import { CHILD_METRICS_EVENT, childEvent } from "../lib/runtime-metrics-children.ts";
 import { normalizeRpcEvent, TASK_EVENT } from "../lib/agents-protocol.ts";
 
 const tick = () => new Promise<void>(resolve => setImmediate(resolve));
-type CatalogLookup = NonNullable<NonNullable<Parameters<typeof runtimeMetrics>[2]>["lookup"]>;
-function harness(env: NodeJS.ProcessEnv = {}, lookup?: CatalogLookup,
-	classify?: (input: unknown) => PiCatalogName, mode: "tui" | "print" = "tui", shutdownWaitMs = 1500) {
+function harness(env: NodeJS.ProcessEnv = {}, mode: "tui" | "print" = "tui", shutdownWaitMs = 1500, model: unknown = { provider: "openai", id: "gpt-4o" }) {
 	const handlers = new Map<string, Function>();
 	const listeners = new Map<string, Function>();
 	let session = "first";
@@ -17,13 +15,13 @@ function harness(env: NodeJS.ProcessEnv = {}, lookup?: CatalogLookup,
 	let signal!: AbortSignal;
 	const sent: RuntimeMetricBucket[][] = [];
 	const launches: unknown[] = [];
-	const ctx: any = { cwd: "/fixture", mode, model: { provider: "openai", id: "gpt-4o" },
+	const ctx: any = { cwd: "/fixture", mode, model,
 		sessionManager: { getSessionId: () => session, getEntries: () => assert.fail("no reconstruction") } };
 	const pi: any = { on: (name: string, handler: Function) => handlers.set(name, handler),
 		getThinkingLevel: () => "high", registerCommand() {},
 		appendEntry: () => assert.fail("no metrics persistence"),
 		events: { on: (name: string, handler: Function) => { listeners.set(name, handler); return () => listeners.delete(name); } } };
-	runtimeMetrics(pi, env, { lookup, classify, now: () => 0, shutdownWaitMs, send: (rows, _cwd, deps) => {
+	runtimeMetrics(pi, env, { now: () => 0, shutdownWaitMs, send: (rows, _cwd, deps) => {
 		sent.push(structuredClone(rows)); launches.push(structuredClone(deps?.launches)); signal = deps!.signal!;
 		return new Promise(resolve => { finish = () => resolve("discarded"); });
 	} });
@@ -38,30 +36,41 @@ const final = (input = 7) => ({ role: "assistant", provider: "openai", model: "g
 	providerThinkingLevel: "low", stopReason: "stop", usage: { input, output: 3, cacheRead: 0 },
 	content: [{ text: "private response" }], errorMessage: "private error", path: "/private" });
 
-test("message classification retries a failed catalog load without awaiting it", async () => {
-	let calls = 0;
-	const publicName = Object.freeze({ classification: "catalog_public", modelId: "gpt-4o" }) satisfies PiCatalogName;
-	const catalog = new Map([[JSON.stringify(["openai", "gpt-4o"]), publicName]]);
-	const catalogLookup = createPiCatalogNameLookup(async () => {
-		calls++;
-		if (calls === 1) throw new Error("transient catalog failure");
-		return catalog;
-	});
-	const h = harness({}, catalogLookup.lookup, catalogLookup.classify);
-	h.emit("session_start");
-	await tick();
-	assert.equal(calls, 1);
-	assert.equal(h.emit("message_end", { message: final() }), undefined);
-	assert.equal(calls, 2);
+test("a generic open-weight selection reaches the wire by name; no catalog probe or gate", async () => {
+	const h = harness({}, "tui", 1500, { provider: "nan", id: "deepseek-v4-flash" });
+	await h.emit("session_start");
+	h.emit("turn_start"); h.emit("before_provider_request");
+	h.emit("message_end", { message: { role: "assistant", provider: "nan", model: "deepseek-v4-flash",
+		providerThinkingLevel: "low", stopReason: "stop", usage: { input: 3, output: 2 } } });
 	await tick();
 	assert.equal(h.sent.length, 1);
-	assert.equal(h.sent[0][0].observedModelId, "unknown", "the racing row remains fail-closed");
-	await h.finish();
-	h.emit("message_end", { message: final(8) });
+	assert.equal(h.sent[0][0].selectedProvider, "nan");
+	assert.equal(h.sent[0][0].selectedModelId, "deepseek-v4-flash");
+	const encoded = encodeNativeRuntimeEvent(h.sent[0]);
+	assert.ok(encoded);
+	const row = JSON.parse(encoded).rows[0];
+	assert.deepEqual(row.model, { provider: "nan", id: "deepseek-v4-flash" });
+	assert.equal(row.model_evidence, "selected");
+	await h.finish(); h.emit("session_shutdown");
+});
+
+test("an ambiguous turn with no captured selection still reports the observed model, as selected evidence", async () => {
+	const h = harness({}, "tui", 1500, { provider: "nan", id: "deepseek-v4-flash" });
+	await h.emit("session_start");
+	// No turn_start/before_provider_request: the turn is ambiguous, so no
+	// selection was captured for this response.
+	h.emit("message_end", { message: { role: "assistant", provider: "nan", model: "glm5.3",
+		providerThinkingLevel: "low", stopReason: "stop", usage: { input: 3, output: 2 } } });
 	await tick();
-	assert.equal(h.sent[1][0].observedModelId, "gpt-4o", "a later row uses the recovered catalog");
-	await h.finish();
-	h.emit("session_shutdown");
+	assert.equal(h.sent.length, 1);
+	assert.equal(h.sent[0][0].selectedModelId, "unknown");
+	assert.equal(h.sent[0][0].observedModelId, "glm5.3");
+	const encoded = encodeNativeRuntimeEvent(h.sent[0]);
+	assert.ok(encoded);
+	const row = JSON.parse(encoded).rows[0];
+	assert.deepEqual(row.model, { provider: "nan", id: "glm5.3" });
+	assert.equal(row.model_evidence, "selected");
+	await h.finish(); h.emit("session_shutdown");
 });
 
 test("final callback returns before blocked transport; busy and duplicate messages drop", async () => {
@@ -97,7 +106,7 @@ test("replacement before launch cancels stale work; shutdown does not await a bl
 });
 
 test("print shutdown joins an accepted orchestrator delivery before disposing", async () => {
-	const h = harness({}, undefined, undefined, "print", 50); await h.emit("session_start");
+	const h = harness({}, "print", 50); await h.emit("session_start");
 	h.emit("message_end", { message: final() }); await tick();
 	let stopped = false;
 	const shutdown = h.emit("session_shutdown").then(() => { stopped = true; });
@@ -109,7 +118,7 @@ test("print shutdown joins an accepted orchestrator delivery before disposing",
 });
 
 test("print shutdown joins an accepted child launch and response delivery", async () => {
-	const h = harness({}, undefined, undefined, "print", 50); await h.emit("session_start");
+	const h = harness({}, "print", 50); await h.emit("session_start");
 	const observation = normalizeRpcEvent({ type: "message_end", message: final() }, { observeResponses: true })
 		.find(event => event.type === TASK_EVENT.RESPONSE_OBSERVATION);
 	assert.ok(observation?.type === TASK_EVENT.RESPONSE_OBSERVATION);
@@ -127,7 +136,7 @@ test("print shutdown joins an accepted child launch and response delivery", asyn
 });
 
 test("print shutdown aborts a delivery after the bounded join deadline", async () => {
-	const h = harness({}, undefined, undefined, "print", 5); await h.emit("session_start");
+	const h = harness({}, "print", 5); await h.emit("session_start");
 	h.emit("message_end", { message: final() }); await tick();
 	await h.emit("session_shutdown");
 	assert.equal(h.signal().aborted, true);
@@ -135,7 +144,7 @@ test("print shutdown aborts a delivery after the bounded join deadline", async (
 });
 
 test("interactive shutdown never joins a blocked delivery", async () => {
-	const h = harness({}, undefined, undefined, "tui", 50); await h.emit("session_start");
+	const h = harness({}, "tui", 50); await h.emit("session_start");
 	h.emit("message_end", { message: final() }); await tick();
 	assert.equal(h.emit("session_shutdown"), undefined);
 	assert.equal(h.signal().aborted, true);

package/tests/runtime-metrics-model.test.ts

@@ -0,0 +1,76 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { normalizeRuntimeModel } from "../lib/runtime-metrics.ts";
+
+// Data-driven family-pattern normalizer (gentle-pi#968 / gentle-ai#4536). Rules:
+// - non-string or empty provider or id -> {provider: "unknown", id: "unknown"};
+// - id is trimmed, the LAST "/"-separated segment is kept, then lowercased; it is
+//   public only when it matches the schema id pattern within its maxLength,
+//   otherwise the id becomes "custom";
+// - provider is trimmed and lowercased; it is kept when it matches the schema
+//   provider pattern, otherwise it becomes "custom";
+// - when the id is not public the provider becomes "custom" too, except for the
+//   "opencode" provider, which yields "opencode"/"custom";
+// - the literal "unknown"/"unknown" and "custom"/"custom" pairs pass through.
+const cases: Array<[string, unknown, unknown, { provider: string; id: string }]> = [
+	["open-weight family on an arbitrary provider stays public", "nan", "deepseek-v4-flash", { provider: "nan", id: "deepseek-v4-flash" }],
+	["a short version suffix without a separator is still public", "nan", "glm5.3", { provider: "nan", id: "glm5.3" }],
+	["multiple suffix groups without a leading separator are still public", "nan", "glm5.3-flash", { provider: "nan", id: "glm5.3-flash" }],
+	["only the last '/'-separated id segment is kept", "nano-gpt", "TEE/glm-5.3", { provider: "nano-gpt", id: "glm-5.3" }],
+	["a nested provider path in the id is dropped, keeping the outer provider", "openrouter", "deepseek/deepseek-v4-flash", { provider: "openrouter", id: "deepseek-v4-flash" }],
+	["provider and id are lowercased", "OpenAI", "GPT-5.6-Sol", { provider: "openai", id: "gpt-5.6-sol" }],
+	["a long dotted/dashed anthropic id stays public", "anthropic", "claude-sonnet-5-20260101", { provider: "anthropic", id: "claude-sonnet-5-20260101" }],
+	["an id with no recognized family prefix collapses provider and id to custom", "acme", "acme-internal-finetune", { provider: "custom", id: "custom" }],
+	["a public id with a non-conforming provider is emitted as custom/<id>", "Acme Corp", "gpt-5.6", { provider: "custom", id: "gpt-5.6" }],
+	["an id longer than the schema maxLength collapses to custom/custom", "anthropic", `claude-${"x".repeat(64)}`, { provider: "custom", id: "custom" }],
+	["opencode keeps its provider identity even for a private id", "opencode", "private-thing", { provider: "opencode", id: "custom" }],
+	["empty provider and id fail closed to unknown/unknown", "", "", { provider: "unknown", id: "unknown" }],
+];
+
+for (const [name, provider, id, expected] of cases) {
+	test(`normalizeRuntimeModel: ${name}`, () => {
+		assert.deepEqual(normalizeRuntimeModel(provider, id), expected);
+	});
+}
+
+test("non-string provider or id fails closed to unknown/unknown", () => {
+	for (const [provider, id] of [[undefined, "gpt-5.6"], ["openai", undefined], [null, null], [42, "gpt-5.6"], ["openai", { id: "gpt-5.6" }]] as const) {
+		assert.deepEqual(normalizeRuntimeModel(provider, id), { provider: "unknown", id: "unknown" });
+	}
+});
+
+test("whitespace-only provider or id fails closed to unknown/unknown", () => {
+	assert.deepEqual(normalizeRuntimeModel("   ", "gpt-5.6"), { provider: "unknown", id: "unknown" });
+	assert.deepEqual(normalizeRuntimeModel("openai", "   "), { provider: "unknown", id: "unknown" });
+});
+
+test("literal unknown/unknown passes through without pattern matching", () => {
+	assert.deepEqual(normalizeRuntimeModel("unknown", "unknown"), { provider: "unknown", id: "unknown" });
+	assert.deepEqual(normalizeRuntimeModel("Unknown", "UNKNOWN"), { provider: "unknown", id: "unknown" });
+});
+
+test("literal custom/custom passes through", () => {
+	assert.deepEqual(normalizeRuntimeModel("custom", "custom"), { provider: "custom", id: "custom" });
+});
+
+test("surrounding whitespace is trimmed before matching", () => {
+	assert.deepEqual(normalizeRuntimeModel("  nan  ", "  deepseek-v4-flash  "), { provider: "nan", id: "deepseek-v4-flash" });
+});
+
+// The mirrored pattern is unambiguous (every suffix group starts with a
+// mandatory separator), so a long alphanumeric run followed by an invalid
+// character cannot trigger catastrophic backtracking in V8. The bound below
+// is generous on purpose: it catches an exponential regression, not jitter.
+test("a pathological 64-character id is rejected in linear time", () => {
+	const ids = [`gpt${"a".repeat(60)}!`, `glm${"5".repeat(60)}/`, `deepseek${"x".repeat(55)}-`];
+	const started = performance.now();
+	for (let round = 0; round < 50; round++) for (const id of ids) {
+		assert.deepEqual(normalizeRuntimeModel("nan", id), { provider: "custom", id: "custom" });
+	}
+	assert.ok(performance.now() - started < 500, "normalizer must not backtrack catastrophically");
+});
+
+test("attached run then separated groups stays public", () => {
+	assert.deepEqual(normalizeRuntimeModel("nan", "glm5.3-flash:thinking"), { provider: "nan", id: "glm5.3-flash:thinking" });
+	assert.deepEqual(normalizeRuntimeModel("nan", "gpt-1-2-3-4-5-6-7-8-9"), { provider: "custom", id: "custom" });
+});

package/tests/runtime-metrics-native.test.ts

@@ -75,18 +75,84 @@ test("encoder filters custom identities and never promotes SDK model to response
 	row.agentClass = "private-agent" as any; row.effort = "private-effort" as any;
 	row.error = "authentication";
 	const encoded = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
-	assert.deepEqual(encoded.rows[0].model, { provider: "custom", id: "custom" });
+	// "private-model" never normalizes to a public id, so it never claims
+	// response evidence; the row falls through to the still-public selection
+	// (source()'s default selectedModelId "gpt-5.4") instead of leaking it or
+	// collapsing a perfectly good selected identity to custom/custom.
+	assert.deepEqual(encoded.rows[0].model, { provider: "openai", id: "gpt-5.4" });
+	assert.equal(encoded.rows[0].model_evidence, "selected");
 	assert.equal(encoded.rows[0].agent_class, "unknown");
 	assert.equal(encoded.rows[0].selected_effort, "unavailable");
 	assert.equal(encoded.rows[0].error_category, "auth");
 	assert.ok(!JSON.stringify(encoded).includes("private"));
-	row.responseModelId = "unknown";
-	const selected = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
-	assert.equal(selected.rows[0].model_evidence, "selected");
-	assert.deepEqual(selected.rows[0].model, { provider: "openai", id: "gpt-5.4" });
 	row.selectedModelId = "unknown";
-	const unknown = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
-	assert.equal(unknown.rows[0].model_evidence, "unknown");
+	const observedFallback = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
+	// With no public selected model either, it falls through once more, but
+	// source()'s default observedModelId ("private-alias") is not public either.
+	assert.equal(observedFallback.rows[0].model_evidence, "unknown");
+	assert.deepEqual(observedFallback.rows[0].model, { provider: "unknown", id: "unknown" });
+});
+
+test("a public observed model is a fallback evidence tier, behind response and selected", () => {
+	const row = source();
+	row.responseModelId = "unknown"; row.selectedModelId = "unknown"; row.observedModelId = "glm5.3-flash"; row.provider = "nan";
+	const encoded = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
+	assert.deepEqual(encoded.rows[0].model, { provider: "nan", id: "glm5.3-flash" });
+	assert.equal(encoded.rows[0].model_evidence, "selected", "the transport has no separate observed evidence category");
+	// A public response model still wins over a public observed model.
+	const withResponse = source();
+	withResponse.selectedModelId = "unknown"; withResponse.observedModelId = "glm5.3-flash"; withResponse.provider = "nan";
+	const encodedResponse = JSON.parse(native.encodeNativeRuntimeEvent([withResponse])!);
+	assert.equal(encodedResponse.rows[0].model_evidence, "response");
+	// A public selected model still wins over a public observed model.
+	const withSelected = source();
+	withSelected.responseModelId = "unknown"; withSelected.observedModelId = "glm5.3-flash"; withSelected.provider = "nan";
+	const encodedSelected = JSON.parse(native.encodeNativeRuntimeEvent([withSelected])!);
+	assert.equal(encodedSelected.rows[0].model_evidence, "selected");
+	assert.deepEqual(encodedSelected.rows[0].model, { provider: withSelected.selectedProvider, id: withSelected.selectedModelId });
+	// A non-public observed model still falls through to unknown/unknown.
+	const noEvidence = source();
+	noEvidence.responseModelId = "unknown"; noEvidence.selectedModelId = "unknown"; noEvidence.observedModelId = "private-alias";
+	const encodedUnknown = JSON.parse(native.encodeNativeRuntimeEvent([noEvidence])!);
+	assert.deepEqual(encodedUnknown.rows[0].model, { provider: "unknown", id: "unknown" });
+	assert.equal(encodedUnknown.rows[0].model_evidence, "unknown");
+});
+
+test("every encoded model satisfies the schema's anyOf, by pattern or by sentinel", () => {
+	const idRule = schema.$defs.model.anyOf[0].properties.id as { pattern: string };
+	const providerRule = schema.$defs.model.anyOf[0].properties.provider as { pattern: string };
+	const idPattern = new RegExp(idRule.pattern);
+	const providerPattern = new RegExp(providerRule.pattern);
+	const sentinels = schema.$defs.model.anyOf.slice(1) as Array<{ properties: { provider: { const: string }; id: { const: string } } }>;
+	function matchesSchema(model: { provider: string; id: string }): boolean {
+		if (providerPattern.test(model.provider) && idPattern.test(model.id)) return true;
+		return sentinels.some(({ properties }) => properties.provider.const === model.provider && properties.id.const === model.id);
+	}
+	// Response/selected/observed rows only ever win evidence with an id that is
+	// already public, so their encoded model is either the public-pattern
+	// branch or the terminal unknown/unknown sentinel. The custom/custom and
+	// opencode/custom sentinels are reachable only through a launch row, which
+	// (already upstream-classified, always "selected" evidence) is paired
+	// through the same normalizer unconditionally.
+	const rows: RuntimeMetricBucket[] = [
+		source(),
+		{ ...source(), provider: "nan", selectedProvider: "nan", selectedModelId: "deepseek-v4-flash", responseModelId: "unknown" },
+		{ ...source(), responseModelId: "unknown", selectedModelId: "unknown", observedModelId: "unknown" },
+	];
+	const launches = [
+		{ evidence: "launch_configuration" as const, agentClass: parseAgentClass("worker")!, selectedProvider: "opencode", selectedModelId: "custom", selectedEffort: "high" as const, launches: 1 },
+		{ evidence: "launch_configuration" as const, agentClass: parseAgentClass("worker")!, selectedProvider: "custom", selectedModelId: "custom", selectedEffort: "high" as const, launches: 1 },
+		{ evidence: "launch_configuration" as const, agentClass: parseAgentClass("worker")!, selectedProvider: "unknown", selectedModelId: "unknown", selectedEffort: "high" as const, launches: 1 },
+	];
+	const payload = native.encodeNativeRuntimeEvent(rows, launches);
+	assert.ok(payload);
+	const decoded = JSON.parse(payload).rows as Array<{ model: { provider: string; id: string } }>;
+	assert.equal(decoded.length, rows.length + launches.length);
+	for (const row of decoded) assert.ok(matchesSchema(row.model), `model ${JSON.stringify(row.model)} must satisfy $defs.model.anyOf`);
+	assert.deepEqual(decoded.map(row => row.model), [
+		{ provider: "openai", id: "gpt-5.4" }, { provider: "nan", id: "deepseek-v4-flash" }, { provider: "unknown", id: "unknown" },
+		{ provider: "opencode", id: "custom" }, { provider: "custom", id: "custom" }, { provider: "unknown", id: "unknown" },
+	]);
 });
 
 function fixture() {

package/tests/runtime-metrics.test.ts

@@ -16,31 +16,32 @@ function response(responseId = "local-response"): FinalResponse {
 	};
 }
 
-test("selected provider is independent from response provider without relabeling", async () => {
-	const { lookupPiCatalogName } = await import("../lib/runtime-metrics-pi-identity.ts");
-	const { OPENAI_CODEX_MODELS } = await import("@earendil-works/pi-ai/providers/openai-codex.models");
-	const model = Object.values(OPENAI_CODEX_MODELS)[0];
-	await lookupPiCatalogName({ provider: model.provider, modelId: model.id });
+test("selected provider is independent from response provider without relabeling", () => {
 	const metrics = new RuntimeMetrics();
-	assert.equal(recordRaw(metrics, { ...response(), selectedProvider: model.provider,
-		selectedModelId: model.id, provider: "anthropic" }), "recorded");
+	assert.equal(recordRaw(metrics, { ...response(), selectedProvider: "openai-codex",
+		selectedModelId: "gpt-5.6-terra", provider: "anthropic" }), "recorded");
 	const [row] = metrics.snapshot();
-	assert.equal(row.selectedModelId, model.id);
-	assert.equal(row.selectedProvider, model.provider);
+	assert.equal(row.selectedModelId, "gpt-5.6-terra");
+	assert.equal(row.selectedProvider, "openai-codex");
 	assert.equal(row.provider, "anthropic");
-	assert.equal(recordRaw(metrics, { ...response("private"), selectedProvider: "private-provider",
-		selectedModelId: model.id }), "recorded");
-	assert.equal(metrics.snapshot()[1].selectedProvider, "custom");
+	// The provider dimension stays independent of a specific id's privacy outcome
+	// at record time; the pair is only coupled to a schema-conformant custom/custom
+	// model object later, at native encode time (see runtime-metrics-native.test.ts).
+	assert.equal(recordRaw(metrics, { ...response("private"), selectedProvider: "openai-codex",
+		selectedModelId: "private-internal-finetune" }), "recorded");
+	assert.equal(metrics.snapshot()[1].selectedProvider, "openai-codex");
 	assert.equal(metrics.snapshot()[1].selectedModelId, "custom");
 });
 
-test("mirrored registry identities survive an unavailable Pi catalog", () => {
-	const metrics = new RuntimeMetrics({ classifyModel: () => ({ classification: "unknown", modelId: "unknown" }) });
-	assert.equal(metrics.record({ ...response(), selectedProvider: "openai-codex", selectedModelId: "gpt-5.6-terra",
-		responseModelId: "gpt-5.6-sol" }), "recorded");
+test("open-weight models on arbitrary providers are reported by name, not squashed to a closed enum", () => {
+	const metrics = new RuntimeMetrics();
+	assert.equal(recordRaw(metrics, { ...response(), provider: "nan", selectedProvider: "nan",
+		selectedModelId: "deepseek-v4-flash", responseModelId: "glm5.3-flash" }), "recorded");
 	const [row] = metrics.snapshot();
-	assert.equal(row.selectedModelId, "gpt-5.6-terra");
-	assert.equal(row.responseModelId, "gpt-5.6-sol");
+	assert.equal(row.provider, "nan");
+	assert.equal(row.selectedProvider, "nan");
+	assert.equal(row.selectedModelId, "deepseek-v4-flash");
+	assert.equal(row.responseModelId, "glm5.3-flash");
 });
 
 // Deliberately bypass static types to exercise the runtime boundary.
@@ -96,7 +97,9 @@ test("effort selections and absence states remain distinct", () => {
 test("only closed dimensions survive; no prompt-based executor inference", () => {
 	const metrics = new RuntimeMetrics();
 	assert.equal(recordRaw(metrics, {
-		...response(), provider: "private-provider", modelFamily: "private-model-id",
+		// Free text with spaces/case never matches the schema provider pattern,
+		// unlike a genuine open-weight provider slug (see the dedicated test above).
+		...response(), provider: "Private Provider Free Text", modelFamily: "private-model-id",
 		executor: "SDD apply executor", systemPrompt: "reviewer", errorMessage: "private failure",
 	}), "recorded");
 	const [row] = metrics.snapshot();

package/lib/runtime-metrics-pi-identity.ts

@@ -1,113 +0,0 @@
-import { findPackageJSON } from "node:module";
-
-// Catalog-name privacy classification ONLY. No identity issuance, route evidence,
-// registry reads, SDK hooks, auth resolution, network, or injected catalogs.
-export interface PiCatalogName {
-	readonly classification: "catalog_public" | "custom" | "unknown";
-	readonly modelId: string;
-}
-
-const UNKNOWN: PiCatalogName = Object.freeze({ classification: "unknown", modelId: "unknown" });
-const CUSTOM: PiCatalogName = Object.freeze({ classification: "custom", modelId: "custom" });
-// Initial provider coverage matches the accumulator. Other providers are custom;
-// additions require deliberate review. These are providers, not model-ID lists.
-const CATALOGS = ["anthropic", "openai", "openai-codex", "google", "google-vertex", "amazon-bedrock", "openrouter"] as const;
-const MAX_MODELS = 4096;
-const MAX_LOAD_ATTEMPTS = 3;
-
-function object(value: unknown): value is Record<string, unknown> {
-	return value !== null && typeof value === "object" && !Array.isArray(value);
-}
-
-function text(value: unknown, max: number): value is string {
-	return typeof value === "string" && value.length > 0 && value.length <= max;
-}
-
-function key(provider: string, modelId: string): string {
-	return JSON.stringify([provider, modelId]);
-}
-
-async function loadCatalog(): Promise<ReadonlyMap<string, PiCatalogName>> {
-	// Node >=22.19 supports findPackageJSON. Verify ESM lookup from this module
-	// selects Pi's own pi-ai package; fail rather than silently use another copy.
-	const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
-	const ownPackage = findPackageJSON("@earendil-works/pi-ai", import.meta.url);
-	if (!ownPackage || ownPackage !== findPackageJSON("@earendil-works/pi-ai", pi)) {
-		throw new Error("Pi catalog dependency mismatch");
-	}
-	const entries = new Map<string, PiCatalogName>();
-	let count = 0;
-	for (const provider of CATALOGS) {
-		// pi-ai 0.85.1 providers/* has an import-only export condition: use ESM,
-		// never require.resolve. Generated modules load packaged JSON, not registry
-		// overrides. Specifiers and export names come only from the fixed list.
-		const module = await import(`@earendil-works/pi-ai/providers/${provider}.models`);
-		const models: unknown = module[`${provider.replaceAll("-", "_").toUpperCase()}_MODELS`];
-		if (!object(models)) throw new Error("Invalid packaged model catalog");
-		for (const model of Object.values(models)) {
-			if (++count > MAX_MODELS) throw new RangeError("Packaged model catalog exceeds capacity");
-			if (!object(model) || model.provider !== provider || !text(model.id, 128)) {
-				throw new Error("Invalid packaged model name");
-			}
-			entries.set(key(provider, model.id), Object.freeze({ classification: "catalog_public", modelId: model.id }));
-		}
-	}
-	return entries;
-}
-
-export function createPiCatalogNameLookup(
-	load: () => Promise<ReadonlyMap<string, PiCatalogName>> = loadCatalog,
-	maxAttempts = MAX_LOAD_ATTEMPTS,
-): { lookup(input: unknown): Promise<PiCatalogName>; classify(input: unknown): PiCatalogName } {
-	let catalogPromise: Promise<ReadonlyMap<string, PiCatalogName>> | undefined;
-	let loadedCatalog: ReadonlyMap<string, PiCatalogName> | undefined;
-	let attempts = 0;
-	const classify = (input: unknown): PiCatalogName => {
-		if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
-		if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
-		return loadedCatalog ? loadedCatalog.get(key(input.provider, input.modelId)) ?? CUSTOM : UNKNOWN;
-	};
-	const lookup = async (input: unknown): Promise<PiCatalogName> => {
-		if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
-		if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
-		if (!loadedCatalog) {
-			if (!catalogPromise) {
-				if (attempts >= maxAttempts) throw new Error("Pi catalog load attempts exhausted");
-				attempts++;
-				catalogPromise = load().then(catalog => {
-					loadedCatalog = catalog;
-					return catalog;
-				}, error => {
-					catalogPromise = undefined;
-					throw error;
-				});
-			}
-			await catalogPromise;
-		}
-		return classify(input);
-	};
-	return { lookup, classify };
-}
-
-const catalogLookup = createPiCatalogNameLookup();
-
-/**
- * Returns only a privacy-safe catalog name, NOT the model actually dispatched.
- * Matching a public ID remains a public-name fact even on a custom endpoint;
- * origin/API/endpoint strings are ignored, never treated as provenance evidence.
- * Missing/malformed names are unknown; non-catalog names/aliases are custom.
- * No modelVersion is invented. This labels a caller-observed selection only;
- * actual dispatch/response identity requires separate future host evidence.
- * One cached snapshot retains <=4096 names of <=128 code units; no input is kept.
- * Missing/incompatible dependencies and catalog overflow reject, never skip.
- */
-export async function lookupPiCatalogName(input: unknown): Promise<PiCatalogName> {
-	return catalogLookup.lookup(input);
-}
-
-/** Pure lookup after async catalog loading; unknown until initialization succeeds.
- * Always recheck membership, never trust a caller's classification/public label.
- */
-export function classifyPiCatalogName(input: unknown): PiCatalogName {
-	return catalogLookup.classify(input);
-}

package/tests/runtime-metrics-pi-identity.test.ts

@@ -1,113 +0,0 @@
-import assert from "node:assert/strict";
-import test from "node:test";
-import { findPackageJSON } from "node:module";
-import { OPENAI_CODEX_MODELS } from "@earendil-works/pi-ai/providers/openai-codex.models";
-import { classifyPiCatalogName, createPiCatalogNameLookup, lookupPiCatalogName, type PiCatalogName } from "../lib/runtime-metrics-pi-identity.ts";
-import { RuntimeMetrics, type FinalResponse } from "../lib/runtime-metrics.ts";
-
-const model = Object.values(OPENAI_CODEX_MODELS)[0];
-const name = { provider: model.provider, modelId: model.id };
-
-test("a failed catalog load is retried lazily and later classifications recover", async () => {
-	let attempts = 0;
-	const publicName = Object.freeze({ classification: "catalog_public", modelId: name.modelId }) satisfies PiCatalogName;
-	const catalog = new Map([[JSON.stringify([name.provider, name.modelId]), publicName]]);
-	const lookup = createPiCatalogNameLookup(async () => {
-		attempts++;
-		if (attempts === 1) throw new Error("transient catalog failure");
-		return catalog;
-	});
-	await assert.rejects(lookup.lookup(name), /transient catalog failure/);
-	assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
-	assert.strictEqual(await lookup.lookup(name), publicName);
-	assert.strictEqual(lookup.classify(name), publicName);
-	assert.equal(attempts, 2);
-});
-
-test("catalog loading attempts are capped per lookup instance", async () => {
-	let attempts = 0;
-	const lookup = createPiCatalogNameLookup(async () => {
-		attempts++;
-		throw new Error(`failure ${attempts}`);
-	});
-	for (let attempt = 0; attempt < 5; attempt++) await assert.rejects(lookup.lookup(name));
-	assert.equal(attempts, 3);
-	assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
-});
-
-test("uninitialized synchronous classification fails closed", () => {
-	assert.deepEqual(classifyPiCatalogName(name), { classification: "unknown", modelId: "unknown" });
-});
-
-test("missing catalog name remains unknown; no route evidence is invented", async () => {
-	for (const input of [undefined, null, {}, { provider: model.provider }, { modelId: model.id }]) {
-		assert.deepEqual(await lookupPiCatalogName(input), { classification: "unknown", modelId: "unknown" });
-	}
-});
-
-test("custom aliases never export supplied strings", async () => {
-	for (const input of [{ ...name, modelId: "private-alias" }, { ...name, provider: "private-provider" }]) {
-		const result = await lookupPiCatalogName(input);
-		assert.deepEqual(result, { classification: "custom", modelId: "custom" });
-		assert.ok(Object.isFrozen(result));
-	}
-});
-
-test("caller-created public identities cannot bypass the catalog boundary", async () => {
-	for (const input of [{ modelId: "private-model", classification: "catalog_public" },
-		{ ...name, modelId: "private-model", origin: "builtin" }]) {
-		assert.notEqual((await lookupPiCatalogName(input)).classification, "catalog_public");
-	}
-});
-
-test("malformed or oversized name metadata fails closed", async () => {
-	for (const patch of [{ modelId: "" }, { modelId: "x".repeat(129) }, { provider: 12 },
-		{ modelId: null }, { provider: "x".repeat(33) }]) {
-		assert.deepEqual(await lookupPiCatalogName({ ...name, ...patch }), { classification: "unknown", modelId: "unknown" });
-	}
-});
-
-test("actual installed ESM catalog supplies immutable public names, not route claims", async () => {
-	const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
-	assert.equal(findPackageJSON("@earendil-works/pi-ai", import.meta.url), findPackageJSON("@earendil-works/pi-ai", pi));
-	assert.ok(model);
-	const result = await lookupPiCatalogName(name);
-	assert.deepEqual(result, { classification: "catalog_public", modelId: model.id });
-	assert.ok(Object.isFrozen(result));
-	assert.strictEqual(await lookupPiCatalogName(name), result);
-	// Origin/endpoint assertions do not change privacy classification of a name.
-	// Neither a real catalog entry nor matching route strings establish dispatch.
-	for (const patch of [{ origin: "builtin" }, { origin: "unknown" }, { origin: "custom" },
-		{ baseUrl: "https://private.invalid" }, { api: "custom-api" }]) {
-		assert.strictEqual(await lookupPiCatalogName({ ...name, ...patch }), result);
-	}
-	assert.deepEqual(Object.keys(result), ["classification", "modelId"]);
-	assert.ok(!("modelVersion" in result));
-});
-
-test("catalog-public selections are counted without claiming actual route identity", async () => {
-	const catalogName = await lookupPiCatalogName(name);
-	const metrics = new RuntimeMetrics();
-	const missing = { state: "unavailable" } as const;
-	for (const [index, identity] of [catalogName, { ...name, origin: "builtin", api: model.api, baseUrl: model.baseUrl }].entries()) {
-		const record = {
-			kind: "final_assistant_response", responseId: String(index), identity, selectedModelId: model.id,
-			executor: "worker", provider: model.provider, modelFamily: "gpt", effort: "high", error: "none",
-			tokens: { input: missing, output: missing, cacheRead: missing, cacheWrite: missing },
-			responseHeadersMs: missing, fullResponseMs: missing,
-		};
-		assert.equal(metrics.record(record as FinalResponse), "recorded");
-	}
-	const [bucket] = metrics.snapshot();
-	assert.equal(bucket.selectedModelId, model.id);
-	assert.ok(!("modelId" in bucket));
-	assert.ok(!("route" in bucket));
-	assert.equal(bucket.responses, 2);
-	assert.equal(bucket.hostAgent, "pi");
-});
-
-test("selection classification rejects fabricated public labels and private IDs", async () => {
-	await lookupPiCatalogName(name);
-	assert.equal(classifyPiCatalogName({ ...name, modelId: "private-id", classification: "catalog_public" }).modelId, "custom");
-	assert.equal(classifyPiCatalogName({ ...name, provider: "anthropic" }).modelId, "custom");
-});