gentle-pi 2.5.0 → 2.6.0

package/tests/runtime-metrics-native.test.ts

@@ -0,0 +1,209 @@
+import assert from "node:assert/strict";
+import { EventEmitter } from "node:events";
+import { PassThrough } from "node:stream";
+import { test } from "node:test";
+import { readFileSync, realpathSync } from "node:fs";
+import { setGentleAiDevBinaryEnvironmentForTesting } from "../lib/gentle-ai-binary.ts";
+import * as native from "../lib/runtime-metrics-native.ts";
+import { parseAgentClass, type RuntimeMetricBucket } from "../lib/runtime-metrics.ts";
+import { createHash } from "node:crypto";
+import schema from "../contracts/telemetry/runtime-aggregate-v1.schema.json" with { type: "json" };
+import fixturePayloads from "./fixtures/runtime-metrics-native-batches.json" with { type: "json" };
+
+function source(): RuntimeMetricBucket {
+	const token = () => ({ reported: 1, unavailable: 0, unsupported: 0, sum: 7 });
+	const duration = () => ({ measured: 0, unavailable: 1, unsupported: 0, sum: 0 });
+	return { hostAgent: "pi", agentClass: parseAgentClass("worker")!, executor: "worker", provider: "openai", modelFamily: "gpt",
+		selectedProvider: "openai", selectedModelId: "gpt-5.4", observedModelId: "private-alias", responseModelId: "gpt-5.4",
+		effort: "high", providerThinkingLevel: "low", error: "none", responses: 1,
+		tokens: { input: token(), output: token(), cacheRead: token(), cacheWrite: token(), reasoning: token(), totalTokens: token() },
+		responseHeadersMs: duration(), fullResponseMs: duration() };
+}
+
+test("production encoder emits exact one-shot fixture with source occurrence coverage", () => {
+	const payload = native.encodeNativeRuntimeEvent([source()]);
+	assert.equal(typeof payload, "string", "production encoder must be enabled");
+	const value = JSON.parse(payload!);
+	assert.deepEqual(Object.keys(value).sort(), [...schema.required].sort());
+	assert.deepEqual(Object.keys(value.rows[0]).sort(), [...schema.$defs.row.required].sort());
+	assert.equal(value.rows[0].launches, null);
+	assert.equal(value.rows[0].responses, 1);
+	assert.equal(value.rows[0].selected_effort, "high");
+	assert.equal(value.rows[0].effective_effort, "low");
+	assert.equal(value.rows[0].model_evidence, "response");
+	assert.deepEqual(value.rows[0].model, { provider: "openai", id: "gpt-5.4" });
+	assert.ok(!payload!.includes("private"));
+	assert.ok(!payload!.includes("batch_id") && !payload!.includes("delivery_id"));
+	assert.deepEqual(fixturePayloads.batches, [payload]);
+	assert.equal(fixturePayloads.schema_sha256, createHash("sha256").update(readFileSync(new URL("../contracts/telemetry/runtime-aggregate-v1.schema.json", import.meta.url))).digest("hex"));
+});
+
+test("child launch occurrence retains selection evidence separately from response evidence", () => {
+	const payload = native.encodeNativeRuntimeEvent([source()], [{ evidence: "launch_configuration", agentClass: parseAgentClass("worker")!,
+		selectedProvider: "openai", selectedModelId: "gpt-5.4", selectedEffort: "high", launches: 1 }]);
+	const rows = JSON.parse(payload!).rows;
+	assert.equal(rows.length, 2);
+	assert.equal(rows[0].model_evidence, "response");
+	assert.equal(rows[1].model_evidence, "selected");
+	assert.equal(rows[1].launches, 1);
+	assert.equal(rows[1].responses, null);
+	assert.equal(rows[1].selected_effort, "high");
+	assert.equal(rows[1].effective_effort, "unavailable");
+	for (const field of ["input_tokens", "output_tokens", "cache_read_tokens", "cache_creation_tokens", "reasoning_tokens", "total_tokens"])
+		assert.deepEqual(rows[1][field], { reported: 0, unavailable: 0, unsupported: 0, sum: 0 });
+});
+
+test("encoder bounds and invalid coverage discard whole events without splitting", () => {
+	assert.equal(native.encodeNativeRuntimeEvent([]), undefined);
+	assert.equal(native.encodeNativeRuntimeEvent(Array(33).fill(source())), undefined);
+	const invalid = source(); invalid.tokens.input.reported = 0;
+	assert.equal(native.encodeNativeRuntimeEvent([invalid]), undefined);
+	invalid.tokens.input.sum = -1;
+	assert.equal(native.encodeNativeRuntimeEvent([invalid]), undefined);
+	const valid = source();
+	assert.equal(typeof native.encodeNativeRuntimeEvent([valid]), "string");
+	assert.equal(native.encodeNativeRuntimeEvent(Array(32).fill(valid)), undefined, "16 KiB bound applies even below row limit");
+	valid.fullResponseMs = { measured: 1, unavailable: 0, unsupported: 0, sum: 12.5 };
+	assert.deepEqual(JSON.parse(native.encodeNativeRuntimeEvent([valid])!).rows[0].duration,
+		{ kind: "request", measured_count: 1, sum_ms: 12.5 });
+	valid.tokens.input = { reported: 0, unavailable: 0, unsupported: 1, sum: 0 };
+	assert.deepEqual(JSON.parse(native.encodeNativeRuntimeEvent([valid])!).rows[0].input_tokens, valid.tokens.input);
+});
+
+test("encoder filters custom identities and never promotes SDK model to response proof", () => {
+	const row = source(); row.provider = "custom"; row.responseModelId = "private-model";
+	row.agentClass = "private-agent" as any; row.effort = "private-effort" as any;
+	row.error = "authentication";
+	const encoded = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
+	assert.deepEqual(encoded.rows[0].model, { provider: "custom", id: "custom" });
+	assert.equal(encoded.rows[0].agent_class, "unknown");
+	assert.equal(encoded.rows[0].selected_effort, "unavailable");
+	assert.equal(encoded.rows[0].error_category, "auth");
+	assert.ok(!JSON.stringify(encoded).includes("private"));
+	row.responseModelId = "unknown";
+	const selected = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
+	assert.equal(selected.rows[0].model_evidence, "selected");
+	assert.deepEqual(selected.rows[0].model, { provider: "openai", id: "gpt-5.4" });
+	row.selectedModelId = "unknown";
+	const unknown = JSON.parse(native.encodeNativeRuntimeEvent([row])!);
+	assert.equal(unknown.rows[0].model_evidence, "unknown");
+});
+
+function fixture() {
+	const child = Object.assign(new EventEmitter(), {
+		stdin: new PassThrough(), stdout: new PassThrough(), stderr: new PassThrough(),
+		kill: () => { killed++; return true; }, unref() {},
+	});
+	let killed = 0;
+	const calls: unknown[][] = [];
+	let input = "";
+	child.stdin.on("data", chunk => { input += chunk; });
+	const deps = {
+		env: {}, resolve: () => "fake-native",
+		encode: () => '{"host":"pi","rows":[]}',
+		spawn: (...args: unknown[]) => { calls.push(args); return child as any; },
+	};
+	return { deps, child, calls, input: () => input, killed: () => killed,
+		close: (decision = "stored") => {
+			child.stdout.write(JSON.stringify({ schema: native.NATIVE_SEND_ACK_SCHEMA, decision }));
+			child.emit("close", 0, null);
+		} };
+}
+
+test("one native stdin send; busy drops without another invocation or policy probe", async () => {
+	const f = fixture();
+	const first = native.sendNativeRuntimeEvent([], "/fixture", f.deps);
+	assert.equal(f.calls.length, 1);
+	assert.deepEqual(f.calls[0][1], ["telemetry", "runtime", "send", "--json"]);
+	assert.equal(f.input(), '{"host":"pi","rows":[]}');
+	assert.equal(await native.sendNativeRuntimeEvent([], "/fixture", f.deps), "discarded");
+	f.close();
+	assert.equal(await first, "stored");
+	assert.equal(f.calls.length, 1);
+});
+
+test("production encoder reaches the fake one-shot child without a test encoding override", async () => {
+	const f = fixture();
+	const { encode: _encode, ...deps } = f.deps;
+	const first = native.sendNativeRuntimeEvent([source()], "/fixture", deps);
+	assert.equal(f.calls.length, 1);
+	assert.equal(f.input(), fixturePayloads.batches[0]);
+	f.close(); assert.equal(await first, "stored");
+});
+
+test("validated dev override reaches the one-shot child through the production resolver", async () => {
+	const file = realpathSync(process.execPath);
+	setGentleAiDevBinaryEnvironmentForTesting({ env: { GENTLE_PI_GENTLE_AI_DEV_BINARY: file }, home: "/unused" });
+	try {
+		const f = fixture();
+		const { resolve: _resolve, encode: _encode, ...deps } = f.deps;
+		const pending = native.sendNativeRuntimeEvent([source()], "/fixture", deps);
+		f.close();
+		assert.equal(await pending, "stored");
+		assert.equal(f.calls.length, 1);
+		assert.equal(f.calls[0][0], file);
+		assert.deepEqual(f.calls[0][1], ["telemetry", "runtime", "send", "--json"]);
+		assert.equal(f.input(), fixturePayloads.batches[0]);
+	} finally {
+		setGentleAiDevBinaryEnvironmentForTesting(undefined);
+	}
+});
+
+test("invalid dev overrides fail closed without spawning or falling back", async () => {
+	for (const file of ["relative-binary", realpathSync(new URL(".", import.meta.url))]) {
+		setGentleAiDevBinaryEnvironmentForTesting({ env: { GENTLE_PI_GENTLE_AI_DEV_BINARY: file }, home: "/unused" });
+		try {
+			const f = fixture();
+			const { resolve: _resolve, ...deps } = f.deps;
+			assert.equal(await native.sendNativeRuntimeEvent([], "/fixture", deps), "discarded");
+			assert.equal(f.calls.length, 0);
+		} finally {
+			setGentleAiDevBinaryEnvironmentForTesting(undefined);
+		}
+	}
+});
+
+test("opt-out and pre-cancel cause zero encoding, resolution or launch", async () => {
+	const f = fixture();
+	f.deps.resolve = () => { throw new Error("must not resolve"); };
+	f.deps.encode = () => { throw new Error("must not encode"); };
+	assert.equal(await native.sendNativeRuntimeEvent([], "/fixture", { ...f.deps, env: { DO_NOT_TRACK: "1" } }), "disabled");
+	assert.equal(await native.sendNativeRuntimeEvent([], "/fixture", { ...f.deps, signal: AbortSignal.abort() }), "discarded");
+	assert.equal(f.calls.length, 0);
+});
+
+test("cancel kills once and keeps slot occupied until actual close", async () => {
+	const f = fixture();
+	const abort = new AbortController();
+	const first = native.sendNativeRuntimeEvent([], "/fixture", { ...f.deps, signal: abort.signal });
+	abort.abort(); abort.abort();
+	assert.equal(f.killed(), 1);
+	assert.equal(await native.sendNativeRuntimeEvent([], "/fixture", f.deps), "discarded");
+	f.close();
+	assert.equal(await first, "discarded");
+});
+
+for (const decision of ["discarded", "disabled", "stored", "duplicate", "sent", "private error"]) {
+	test(`native result ${decision} never triggers retry`, async () => {
+		const f = fixture();
+		const first = native.sendNativeRuntimeEvent([], "/fixture", f.deps);
+		f.close(decision);
+		assert.equal(await first, ["stored", "duplicate", "disabled"].includes(decision) ? decision : "discarded");
+		assert.equal(f.calls.length, 1);
+	});
+}
+
+test("metrics receiver, attempt gate and send transport have no filesystem persistence surface", () => {
+	for (const path of ["../extensions/runtime-metrics.ts", "../lib/runtime-metrics-delivery.ts", "../lib/runtime-metrics-native.ts"]) {
+		const source = readFileSync(new URL(path, import.meta.url), "utf8");
+		assert.doesNotMatch(source, /node:fs|appendEntry\s*\(|writeFile|mkdir|createWriteStream/);
+	}
+});
+
+test("spawn failure silently discards", async () => {
+	const f = fixture();
+	let calls = 0;
+	assert.equal(await native.sendNativeRuntimeEvent([], "/fixture", { ...f.deps,
+		spawn: () => { calls++; throw new Error("private error"); } }), "discarded");
+	assert.equal(calls, 1);
+});

package/tests/runtime-metrics-pi-identity.test.ts

@@ -0,0 +1,113 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { findPackageJSON } from "node:module";
+import { OPENAI_CODEX_MODELS } from "@earendil-works/pi-ai/providers/openai-codex.models";
+import { classifyPiCatalogName, createPiCatalogNameLookup, lookupPiCatalogName, type PiCatalogName } from "../lib/runtime-metrics-pi-identity.ts";
+import { RuntimeMetrics, type FinalResponse } from "../lib/runtime-metrics.ts";
+
+const model = Object.values(OPENAI_CODEX_MODELS)[0];
+const name = { provider: model.provider, modelId: model.id };
+
+test("a failed catalog load is retried lazily and later classifications recover", async () => {
+	let attempts = 0;
+	const publicName = Object.freeze({ classification: "catalog_public", modelId: name.modelId }) satisfies PiCatalogName;
+	const catalog = new Map([[JSON.stringify([name.provider, name.modelId]), publicName]]);
+	const lookup = createPiCatalogNameLookup(async () => {
+		attempts++;
+		if (attempts === 1) throw new Error("transient catalog failure");
+		return catalog;
+	});
+	await assert.rejects(lookup.lookup(name), /transient catalog failure/);
+	assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
+	assert.strictEqual(await lookup.lookup(name), publicName);
+	assert.strictEqual(lookup.classify(name), publicName);
+	assert.equal(attempts, 2);
+});
+
+test("catalog loading attempts are capped per lookup instance", async () => {
+	let attempts = 0;
+	const lookup = createPiCatalogNameLookup(async () => {
+		attempts++;
+		throw new Error(`failure ${attempts}`);
+	});
+	for (let attempt = 0; attempt < 5; attempt++) await assert.rejects(lookup.lookup(name));
+	assert.equal(attempts, 3);
+	assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
+});
+
+test("uninitialized synchronous classification fails closed", () => {
+	assert.deepEqual(classifyPiCatalogName(name), { classification: "unknown", modelId: "unknown" });
+});
+
+test("missing catalog name remains unknown; no route evidence is invented", async () => {
+	for (const input of [undefined, null, {}, { provider: model.provider }, { modelId: model.id }]) {
+		assert.deepEqual(await lookupPiCatalogName(input), { classification: "unknown", modelId: "unknown" });
+	}
+});
+
+test("custom aliases never export supplied strings", async () => {
+	for (const input of [{ ...name, modelId: "private-alias" }, { ...name, provider: "private-provider" }]) {
+		const result = await lookupPiCatalogName(input);
+		assert.deepEqual(result, { classification: "custom", modelId: "custom" });
+		assert.ok(Object.isFrozen(result));
+	}
+});
+
+test("caller-created public identities cannot bypass the catalog boundary", async () => {
+	for (const input of [{ modelId: "private-model", classification: "catalog_public" },
+		{ ...name, modelId: "private-model", origin: "builtin" }]) {
+		assert.notEqual((await lookupPiCatalogName(input)).classification, "catalog_public");
+	}
+});
+
+test("malformed or oversized name metadata fails closed", async () => {
+	for (const patch of [{ modelId: "" }, { modelId: "x".repeat(129) }, { provider: 12 },
+		{ modelId: null }, { provider: "x".repeat(33) }]) {
+		assert.deepEqual(await lookupPiCatalogName({ ...name, ...patch }), { classification: "unknown", modelId: "unknown" });
+	}
+});
+
+test("actual installed ESM catalog supplies immutable public names, not route claims", async () => {
+	const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
+	assert.equal(findPackageJSON("@earendil-works/pi-ai", import.meta.url), findPackageJSON("@earendil-works/pi-ai", pi));
+	assert.ok(model);
+	const result = await lookupPiCatalogName(name);
+	assert.deepEqual(result, { classification: "catalog_public", modelId: model.id });
+	assert.ok(Object.isFrozen(result));
+	assert.strictEqual(await lookupPiCatalogName(name), result);
+	// Origin/endpoint assertions do not change privacy classification of a name.
+	// Neither a real catalog entry nor matching route strings establish dispatch.
+	for (const patch of [{ origin: "builtin" }, { origin: "unknown" }, { origin: "custom" },
+		{ baseUrl: "https://private.invalid" }, { api: "custom-api" }]) {
+		assert.strictEqual(await lookupPiCatalogName({ ...name, ...patch }), result);
+	}
+	assert.deepEqual(Object.keys(result), ["classification", "modelId"]);
+	assert.ok(!("modelVersion" in result));
+});
+
+test("catalog-public selections are counted without claiming actual route identity", async () => {
+	const catalogName = await lookupPiCatalogName(name);
+	const metrics = new RuntimeMetrics();
+	const missing = { state: "unavailable" } as const;
+	for (const [index, identity] of [catalogName, { ...name, origin: "builtin", api: model.api, baseUrl: model.baseUrl }].entries()) {
+		const record = {
+			kind: "final_assistant_response", responseId: String(index), identity, selectedModelId: model.id,
+			executor: "worker", provider: model.provider, modelFamily: "gpt", effort: "high", error: "none",
+			tokens: { input: missing, output: missing, cacheRead: missing, cacheWrite: missing },
+			responseHeadersMs: missing, fullResponseMs: missing,
+		};
+		assert.equal(metrics.record(record as FinalResponse), "recorded");
+	}
+	const [bucket] = metrics.snapshot();
+	assert.equal(bucket.selectedModelId, model.id);
+	assert.ok(!("modelId" in bucket));
+	assert.ok(!("route" in bucket));
+	assert.equal(bucket.responses, 2);
+	assert.equal(bucket.hostAgent, "pi");
+});
+
+test("selection classification rejects fabricated public labels and private IDs", async () => {
+	await lookupPiCatalogName(name);
+	assert.equal(classifyPiCatalogName({ ...name, modelId: "private-id", classification: "catalog_public" }).modelId, "custom");
+	assert.equal(classifyPiCatalogName({ ...name, provider: "anthropic" }).modelId, "custom");
+});

package/tests/runtime-metrics-policy.test.ts

@@ -0,0 +1,62 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { readRuntimeMetricsPolicy, runtimeMetricsEnvAllows } from "../lib/runtime-metrics-policy.ts";
+
+const grant = { schema: "gentle-ai.telemetry-policy/v1", operation: "policy", enabled: true, source: "state", reason: "enabled" };
+const result = (value: unknown) => ({ stdout: JSON.stringify(value), stderr: "", exitCode: 0,
+	signal: null, timedOut: false, outputLimitExceeded: false });
+
+test("only strict enabled policy grants; uses bounded policy invocation exclusively", async () => {
+	let calls = 0;
+	assert.equal(await readRuntimeMetricsPolicy("/workspace", {
+		resolve: () => "/fixture/gentle-ai", exec: async request => {
+			calls++;
+			assert.deepEqual(request.arguments, ["telemetry", "policy", "--json"]);
+			assert.equal(request.file, "/fixture/gentle-ai");
+			assert.equal(request.cwd, "/workspace");
+			assert.equal(request.timeoutMs, 1000);
+			assert.equal(request.maxBufferBytes, 2048);
+			return result(grant);
+		},
+	}), true);
+	assert.equal(calls, 1);
+	for (const patch of [{ schema: "wrong" }, { operation: "status" }, { enabled: "true" },
+		{ enabled: false, reason: "disabled" }, { reason: "enrollment_pending" }, { reason: "state_unavailable" },
+		{ source: "private" }, { raw: "private" }, { reason: undefined }]) {
+		assert.equal(await readRuntimeMetricsPolicy("/workspace", {
+			resolve: () => "fixture", exec: async () => result({ ...grant, ...patch }),
+		}), false);
+	}
+});
+
+test("missing, old, timed-out and malformed binary responses fail closed without fallback", async () => {
+	for (const patch of [{ exitCode: 1 }, { timedOut: true }, { outputLimitExceeded: true },
+		{ signal: "SIGTERM" }, { stderr: "private error" }, { stdout: "unsupported command" },
+		{ stdout: "x".repeat(2049) }, { stdout: "null" }, { stdout: "[]" }]) {
+		let calls = 0;
+		assert.equal(await readRuntimeMetricsPolicy("/workspace", { resolve: () => "fixture",
+			exec: async () => { calls++; return { ...result(grant), ...patch } as any; } }), false);
+		assert.equal(calls, 1);
+	}
+	assert.equal(await readRuntimeMetricsPolicy("/workspace", {
+		resolve: () => { throw new Error("missing"); }, exec: async () => assert.fail("must not spawn"),
+	}), false);
+	assert.equal(await readRuntimeMetricsPolicy("/workspace", {
+		resolve: () => "fixture", exec: async () => { throw new Error("ENOENT"); },
+	}), false);
+});
+
+test("outer deadline releases callers even when an exec adapter never settles", async () => {
+	let signal: AbortSignal | undefined;
+	assert.equal(await readRuntimeMetricsPolicy("/workspace", { resolve: () => "fixture",
+		exec: request => { signal = request.signal; return new Promise(() => {}); } }), false);
+	assert.equal(signal?.aborted, true);
+});
+
+test("native-compatible environment veto accepts broad truthy spellings", () => {
+	for (const key of ["DO_NOT_TRACK", "CI", "GITHUB_ACTIONS"]) {
+		for (const value of ["1", "true", "TRUE", " yes ", "on", "t", "unknown"]) assert.equal(runtimeMetricsEnvAllows({ [key]: value }), false);
+	}
+	assert.equal(runtimeMetricsEnvAllows({ GENTLE_AI_TELEMETRY: "0" }), false);
+	assert.equal(runtimeMetricsEnvAllows({ DO_NOT_TRACK: "false", CI: "0" }), true);
+});

package/tests/runtime-metrics.test.ts

@@ -0,0 +1,184 @@
+import assert from "node:assert/strict";
+import test from "node:test";
+import { RuntimeMetrics, type FinalResponse } from "../lib/runtime-metrics.ts";
+
+function response(responseId = "local-response"): FinalResponse {
+	return {
+		kind: "final_assistant_response", responseId,
+		executor: "worker", provider: "openai-codex", modelFamily: "gpt",
+		effort: "high", error: "none",
+		tokens: {
+			input: { state: "reported", value: 0 }, output: { state: "reported", value: 12 },
+			cacheRead: { state: "unavailable" }, cacheWrite: { state: "unsupported" },
+		},
+		responseHeadersMs: { state: "measured", value: 25.5 },
+		fullResponseMs: { state: "measured", value: 100 },
+	};
+}
+
+test("selected provider is independent from response provider without relabeling", async () => {
+	const { lookupPiCatalogName } = await import("../lib/runtime-metrics-pi-identity.ts");
+	const { OPENAI_CODEX_MODELS } = await import("@earendil-works/pi-ai/providers/openai-codex.models");
+	const model = Object.values(OPENAI_CODEX_MODELS)[0];
+	await lookupPiCatalogName({ provider: model.provider, modelId: model.id });
+	const metrics = new RuntimeMetrics();
+	assert.equal(recordRaw(metrics, { ...response(), selectedProvider: model.provider,
+		selectedModelId: model.id, provider: "anthropic" }), "recorded");
+	const [row] = metrics.snapshot();
+	assert.equal(row.selectedModelId, model.id);
+	assert.equal(row.selectedProvider, model.provider);
+	assert.equal(row.provider, "anthropic");
+	assert.equal(recordRaw(metrics, { ...response("private"), selectedProvider: "private-provider",
+		selectedModelId: model.id }), "recorded");
+	assert.equal(metrics.snapshot()[1].selectedProvider, "custom");
+	assert.equal(metrics.snapshot()[1].selectedModelId, "custom");
+});
+
+test("mirrored registry identities survive an unavailable Pi catalog", () => {
+	const metrics = new RuntimeMetrics({ classifyModel: () => ({ classification: "unknown", modelId: "unknown" }) });
+	assert.equal(metrics.record({ ...response(), selectedProvider: "openai-codex", selectedModelId: "gpt-5.6-terra",
+		responseModelId: "gpt-5.6-sol" }), "recorded");
+	const [row] = metrics.snapshot();
+	assert.equal(row.selectedModelId, "gpt-5.6-terra");
+	assert.equal(row.responseModelId, "gpt-5.6-sol");
+});
+
+// Deliberately bypass static types to exercise the runtime boundary.
+function recordRaw(metrics: RuntimeMetrics, value: unknown) {
+	return metrics.record(value as FinalResponse);
+}
+
+test("optional native token fields preserve absence but reject explicit malformed values", () => {
+	for (const value of [null, -1, { state: "reported", value: NaN }, { state: "reported", value: 1_000_000_001 }]) {
+		assert.equal(recordRaw(new RuntimeMetrics(), { ...response(), tokens: { ...response().tokens, reasoning: value } }), "invalid");
+	}
+	const metrics = new RuntimeMetrics();
+	assert.equal(metrics.record(response()), "recorded");
+	assert.equal(metrics.snapshot()[0].tokens.reasoning.unavailable, 1);
+});
+
+test("accounts final responses with explicit zero, missing states and separate durations", () => {
+	const metrics = new RuntimeMetrics();
+	assert.equal(metrics.record(response()), "recorded");
+	assert.equal(metrics.record(response("second")), "recorded");
+	const [row] = metrics.snapshot();
+	assert.equal(row.responses, 2);
+	assert.deepEqual(row.tokens.input, { reported: 2, unavailable: 0, unsupported: 0, sum: 0 });
+	assert.equal(row.tokens.output.sum, 24);
+	assert.equal(row.tokens.cacheRead.unavailable, 2);
+	assert.equal(row.tokens.cacheWrite.unsupported, 2);
+	assert.equal(row.responseHeadersMs.sum, 51);
+	assert.equal(row.fullResponseMs.sum, 200);
+	assert.equal(row.executor, "worker");
+});
+
+test("deduplicates distinct local IDs without retaining them in snapshots", () => {
+	const metrics = new RuntimeMetrics();
+	assert.equal(metrics.record(response()), "recorded");
+	assert.equal(metrics.record({ ...response(), effort: "low" }), "duplicate");
+	assert.equal(metrics.record(response("second")), "recorded");
+	assert.equal(metrics.snapshot()[0].responses, 2);
+	assert.ok(!JSON.stringify(metrics.snapshot()).includes("local-response"));
+	const copy = metrics.snapshot();
+	copy[0].tokens.output.sum = 999;
+	assert.equal(metrics.snapshot()[0].tokens.output.sum, 24);
+});
+
+test("effort selections and absence states remain distinct", () => {
+	const metrics = new RuntimeMetrics();
+	const levels = ["off", "minimal", "low", "medium", "high", "xhigh", "max",
+		"not_selected", "unsupported", "unavailable"] as const;
+	for (const effort of levels) assert.equal(metrics.record({ ...response(effort), effort }), "recorded");
+	assert.deepEqual(metrics.snapshot().map(row => row.effort), levels);
+	assert.equal(recordRaw(metrics, { ...response("bad"), effort: "ultra" }), "invalid");
+});
+
+test("only closed dimensions survive; no prompt-based executor inference", () => {
+	const metrics = new RuntimeMetrics();
+	assert.equal(recordRaw(metrics, {
+		...response(), provider: "private-provider", modelFamily: "private-model-id",
+		executor: "SDD apply executor", systemPrompt: "reviewer", errorMessage: "private failure",
+	}), "recorded");
+	const [row] = metrics.snapshot();
+	assert.equal(row.provider, "custom");
+	assert.equal(row.modelFamily, "custom");
+	assert.equal(row.executor, "unknown");
+	assert.ok(!JSON.stringify(row).includes("private"));
+	assert.equal(recordRaw(metrics, { ...response("missing"), provider: undefined, modelFamily: null }), "recorded");
+	assert.equal(metrics.snapshot()[1].provider, "unknown");
+	assert.equal(metrics.snapshot()[1].modelFamily, "unknown");
+});
+
+test("rejects malformed numbers and ambiguous measurement states atomically", () => {
+	const metrics = new RuntimeMetrics();
+	for (const value of [-1, NaN, Infinity, "12", null, 1e20]) {
+		const base = response();
+		assert.equal(recordRaw(metrics, { ...base, tokens: { ...base.tokens, input: { state: "reported", value } } }), "invalid");
+		assert.equal(recordRaw(metrics, { ...base, fullResponseMs: { state: "measured", value } }), "invalid");
+	}
+	for (const measurement of [{ value: 0 }, { state: "reported" }, { state: "unsupported", value: 0 }]) {
+		const base = response();
+		assert.equal(recordRaw(metrics, { ...base, tokens: { ...base.tokens, input: measurement } }), "invalid");
+	}
+	assert.equal(recordRaw(metrics, { ...response(), tokens: { ...response().tokens, output: { state: "reported", value: 1.5 } } }), "invalid");
+	assert.equal(recordRaw(metrics, { ...response(), fullResponseMs: { state: "reported", value: 10 } }), "invalid");
+	assert.equal(recordRaw(metrics, { ...response(), fullResponseMs: { state: "measured", value: 10 } }), "invalid");
+	assert.deepEqual(metrics.snapshot(), []);
+	assert.equal(metrics.record(response()), "recorded", "invalid inputs do not reserve IDs");
+});
+
+test("rejects streaming, malformed records, identifiers and free-text errors", () => {
+	const metrics = new RuntimeMetrics();
+	for (const value of [null, {}, [], { ...response(), kind: "message_update" },
+		{ ...response(), responseId: "" }, { ...response(), responseId: "x".repeat(129) },
+		{ ...response(), error: "raw error text" }, { ...response(), tokens: null }]) {
+		assert.equal(recordRaw(metrics, value), "invalid");
+	}
+	for (const error of ["none", "aborted", "rate_limit", "authentication", "network", "provider", "unknown"] as const) {
+		assert.equal(metrics.record({ ...response(error), error, responseHeadersMs: { state: "unsupported" }, fullResponseMs: { state: "unavailable" } }), "recorded");
+	}
+	assert.equal(metrics.snapshot().length, 7);
+});
+
+test("host identity stays separate from role and rejects invented model dimensions", () => {
+	const metrics = new RuntimeMetrics();
+	assert.equal(recordRaw(metrics, { ...response("custom"), identity: { origin: "custom" } }), "recorded");
+	assert.equal(recordRaw(metrics, { ...response("forged"), identity: { hostAgent: "private-worker", modelId: "secret-model" } }), "recorded");
+	const rows = metrics.snapshot();
+	assert.deepEqual(rows.map(row => [row.hostAgent, row.executor, row.selectedModelId, row.effort]), [
+		["pi", "worker", "unknown", "high"],
+	]);
+	assert.ok(!JSON.stringify(rows).includes("secret-model"));
+});
+
+test("hard response and numeric ceilings keep aggregate totals bounded", () => {
+	const metrics = new RuntimeMetrics();
+	for (let i = 0; i < 1024; i++) {
+		const record = response(String(i));
+		record.tokens.input = { state: "reported", value: 1_000_000_000 };
+		record.responseHeadersMs = { state: "measured", value: 0 };
+		record.fullResponseMs = { state: "measured", value: 86_400_000 };
+		assert.equal(metrics.record(record), "recorded");
+	}
+	assert.equal(metrics.record(response("overflow")), "capacity");
+	const [row] = metrics.snapshot();
+	assert.equal(row.responses, 1024);
+	assert.equal(row.tokens.input.sum, 1_024_000_000_000);
+	assert.equal(row.fullResponseMs.sum, 88_473_600_000);
+	assert.equal(row.responseHeadersMs.measured, 1024);
+	assert.equal(row.responseHeadersMs.sum, 0);
+});
+
+test("capacity rejection keeps dedupe and accounting intact without eviction", () => {
+	const metrics = new RuntimeMetrics({ maxResponses: 2, maxBuckets: 1 });
+	assert.equal(metrics.record(response("one")), "recorded");
+	assert.equal(metrics.record({ ...response("two"), effort: "low" }), "capacity");
+	assert.equal(metrics.record(response("two")), "recorded");
+	assert.equal(metrics.record(response("three")), "capacity");
+	assert.equal(metrics.record(response("one")), "duplicate");
+	assert.equal(metrics.snapshot()[0].responses, 2);
+	for (const maxResponses of [0, -1, 1.5, Infinity, 1025]) {
+		assert.throws(() => new RuntimeMetrics({ maxResponses }), RangeError);
+	}
+	assert.throws(() => new RuntimeMetrics({ maxBuckets: 65 }), RangeError);
+});

package/tests/sdd-agent-tools.test.ts

@@ -67,7 +67,7 @@ const requiredToolsByAgent: Record<string, string[]> = {
 	"sdd-init.md": ["read", "grep", "find", "edit", "write", "bash", "mem_search", "mem_get_observation", "mem_save", "mem_update"],
 	"sdd-onboard.md": ["read", "grep", "find", "edit", "write", "bash", "mem_search", "mem_get_observation", "mem_save", "mem_update"],
 	"sdd-proposal.md": ["read", "grep", "find", "edit", "write", "mem_search", "mem_get_observation", "mem_save"],
-	"sdd-research.md": ["read", "grep", "find", "edit", "write", "mem_search", "mem_get_observation", "mem_save"],
+	"sdd-research.md": ["read", "grep", "find", "edit", "write", "mem_search", "mem_get_observation", "mem_save", "fetch_content", "web_search", "source_check", "get_search_content"],
 	"sdd-spec.md": ["read", "grep", "find", "edit", "write", "mem_search", "mem_get_observation", "mem_save"],
 	"sdd-status.md": ["read", "grep", "find", "bash", "mem_search", "mem_get_observation"],
 	"sdd-sync.md": ["read", "grep", "find", "edit", "write", "bash", "mem_search", "mem_get_observation", "mem_save", "mem_update"],
@@ -103,6 +103,15 @@ test("artifact-producing SDD agents can persist OpenSpec files while status rema
 	assert.ok(!statusTools.includes("write"), "sdd-status.md must remain read-only");
 });
 
+test("research instructions require executed evidence rather than blanket denial", () => {
+	const source = readFileSync(join(assetsAgentsDir, "sdd-research.md"), "utf8");
+	assert.doesNotMatch(source, /documentation=\[\]; open-web=\[\]/);
+	assert.match(source, /Actually call approved tools/);
+	assert.match(source, /claim maps to source IDs/);
+	assert.match(source, /proposal_ready: false/);
+	assert.ok(!readTools(join(assetsAgentsDir, "sdd-research.md")).includes("bash"));
+});
+
 test("project does not ship local SDD agent overrides", () => {
 	for (const relativeDir of [join(".pi", "agents"), join(".pi", "subagents")]) {
 		const dir = join(repoRoot, relativeDir);

package/tests/sdd-execution-routing-contract.test.ts

@@ -0,0 +1,28 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import test from "node:test";
+
+const guidancePaths = [
+	"assets/support/sdd-status-contract.md",
+	"assets/sdd-orchestrator-workflow.md",
+	"assets/agents/sdd-status.md",
+	"assets/agents/sdd-verify.md",
+	"assets/agents/sdd-archive.md",
+];
+
+for (const path of guidancePaths) {
+	test(`${path}: native v2 status remains read-only and authoritative`, () => {
+		const guidance = readFileSync(new URL(`../${path}`, import.meta.url), "utf8");
+		assert.match(guidance, /native.*(?:status|v2)|gentle-ai\.sdd-status/i);
+		assert.match(guidance, /read-only/i);
+		assert.doesNotMatch(guidance, /resolve-via-engram/i);
+		assert.doesNotMatch(guidance, /local SDD status engine|manual (?:fallback )?status|reconstruct(?:ing)? (?:native )?status/i);
+	});
+}
+
+test("workflow preserves explicit continuation and the manual sync resolver", () => {
+	const workflow = readFileSync(new URL("../assets/sdd-orchestrator-workflow.md", import.meta.url), "utf8");
+	assert.match(workflow, /only.*sdd-continue|sdd-continue.*only/i);
+	assert.match(workflow, /manual sdd-sync/i);
+	assert.doesNotMatch(workflow, /sdd-(?:apply|verify|archive).*local|local.*sdd-(?:apply|verify|archive)/i);
+});