gentle-pi 2.6.0 → 2.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +183 -943
  2. package/contracts/review-provider-contract-mirror/provider-contract.lock.json +9 -8
  3. package/contracts/review-provider-contract-mirror/v1.2.0/bundle/manifest.json +3 -3
  4. package/contracts/review-provider-contract-mirror/v1.2.0/bundle/orchestration/pi.md +7 -2
  5. package/contracts/review-provider-contract-mirror/v1.2.0/bundle/schemas/lens.schema.json +2 -2
  6. package/contracts/review-provider-contract-mirror/v1.2.0/bundle/schemas/targeted-validator.schema.json +1 -1
  7. package/contracts/review-provider-contract-mirror/v1.2.0/generated/provider-capabilities.baseline.json +1 -1
  8. package/contracts/review-provider-contract-mirror/v1.2.0/generated/provider-roles.baseline.json +2 -2
  9. package/contracts/telemetry/runtime-aggregate-v1.schema.json +8 -6
  10. package/docs/assets/brand/gentle-pi-banner.png +0 -0
  11. package/docs/assets/brand/gentle-pi-banner.svg +33 -0
  12. package/docs/assets/brand/terminal-divider.svg +17 -0
  13. package/docs/assets/diagrams/agent-orchestration.svg +19 -0
  14. package/docs/assets/diagrams/gentleman-workflow.svg +15 -0
  15. package/docs/assets/diagrams/native-review.svg +16 -0
  16. package/docs/assets/diagrams/sdd-cycle.svg +14 -0
  17. package/docs/assets/features/gentle-shell.png +0 -0
  18. package/docs/gentle-shell.md +151 -0
  19. package/docs/readme-reference.md +868 -0
  20. package/extensions/gentle-agents.ts +4 -5
  21. package/extensions/gentle-ai.ts +65 -22
  22. package/extensions/runtime-metrics.ts +10 -19
  23. package/lib/agents-history.ts +7 -1
  24. package/lib/native-review-cli.ts +14 -0
  25. package/lib/runtime-metrics-children.ts +3 -5
  26. package/lib/runtime-metrics-native.ts +17 -9
  27. package/lib/runtime-metrics.ts +95 -42
  28. package/package.json +1 -1
  29. package/runtime/native-review-cli.mjs +14 -0
  30. package/scripts/gentle-ai-installer.mjs +10 -10
  31. package/scripts/types-baseline.json +1 -5
  32. package/scripts/verify-package-files.mjs +3 -3
  33. package/tests/fixtures/runtime-metrics-native-batches.json +1 -1
  34. package/tests/gentle-agents.test.ts +22 -6
  35. package/tests/gentle-ai-binary.test.ts +1 -1
  36. package/tests/gentle-ai-installer.test.ts +47 -47
  37. package/tests/gentle-ai.test.ts +3 -2
  38. package/tests/native-review-capability-contract.test.ts +24 -1
  39. package/tests/package-manifest.test.ts +19 -18
  40. package/tests/review-authority-recovery-docs.test.ts +13 -13
  41. package/tests/review-controller-native-routing.test.ts +49 -0
  42. package/tests/review-ledger-contract.test.ts +7 -5
  43. package/tests/runtime-metrics-children.test.ts +4 -3
  44. package/tests/runtime-metrics-extension.test.ts +40 -31
  45. package/tests/runtime-metrics-model.test.ts +76 -0
  46. package/tests/runtime-metrics-native.test.ts +73 -7
  47. package/tests/runtime-metrics.test.ts +22 -19
  48. package/tests/sdd-managed-runtime-settlement.test.ts +37 -0
  49. package/tests/sdd-selection-transport.test.ts +57 -0
  50. package/tests/skill-collision-prefixes.test.ts +2 -2
  51. package/lib/runtime-metrics-pi-identity.ts +0 -113
  52. package/tests/runtime-metrics-pi-identity.test.ts +0 -113
@@ -16,31 +16,32 @@ function response(responseId = "local-response"): FinalResponse {
16
16
  };
17
17
  }
18
18
 
19
- test("selected provider is independent from response provider without relabeling", async () => {
20
- const { lookupPiCatalogName } = await import("../lib/runtime-metrics-pi-identity.ts");
21
- const { OPENAI_CODEX_MODELS } = await import("@earendil-works/pi-ai/providers/openai-codex.models");
22
- const model = Object.values(OPENAI_CODEX_MODELS)[0];
23
- await lookupPiCatalogName({ provider: model.provider, modelId: model.id });
19
+ test("selected provider is independent from response provider without relabeling", () => {
24
20
  const metrics = new RuntimeMetrics();
25
- assert.equal(recordRaw(metrics, { ...response(), selectedProvider: model.provider,
26
- selectedModelId: model.id, provider: "anthropic" }), "recorded");
21
+ assert.equal(recordRaw(metrics, { ...response(), selectedProvider: "openai-codex",
22
+ selectedModelId: "gpt-5.6-terra", provider: "anthropic" }), "recorded");
27
23
  const [row] = metrics.snapshot();
28
- assert.equal(row.selectedModelId, model.id);
29
- assert.equal(row.selectedProvider, model.provider);
24
+ assert.equal(row.selectedModelId, "gpt-5.6-terra");
25
+ assert.equal(row.selectedProvider, "openai-codex");
30
26
  assert.equal(row.provider, "anthropic");
31
- assert.equal(recordRaw(metrics, { ...response("private"), selectedProvider: "private-provider",
32
- selectedModelId: model.id }), "recorded");
33
- assert.equal(metrics.snapshot()[1].selectedProvider, "custom");
27
+ // The provider dimension stays independent of a specific id's privacy outcome
28
+ // at record time; the pair is only coupled to a schema-conformant custom/custom
29
+ // model object later, at native encode time (see runtime-metrics-native.test.ts).
30
+ assert.equal(recordRaw(metrics, { ...response("private"), selectedProvider: "openai-codex",
31
+ selectedModelId: "private-internal-finetune" }), "recorded");
32
+ assert.equal(metrics.snapshot()[1].selectedProvider, "openai-codex");
34
33
  assert.equal(metrics.snapshot()[1].selectedModelId, "custom");
35
34
  });
36
35
 
37
- test("mirrored registry identities survive an unavailable Pi catalog", () => {
38
- const metrics = new RuntimeMetrics({ classifyModel: () => ({ classification: "unknown", modelId: "unknown" }) });
39
- assert.equal(metrics.record({ ...response(), selectedProvider: "openai-codex", selectedModelId: "gpt-5.6-terra",
40
- responseModelId: "gpt-5.6-sol" }), "recorded");
36
+ test("open-weight models on arbitrary providers are reported by name, not squashed to a closed enum", () => {
37
+ const metrics = new RuntimeMetrics();
38
+ assert.equal(recordRaw(metrics, { ...response(), provider: "nan", selectedProvider: "nan",
39
+ selectedModelId: "deepseek-v4-flash", responseModelId: "glm5.3-flash" }), "recorded");
41
40
  const [row] = metrics.snapshot();
42
- assert.equal(row.selectedModelId, "gpt-5.6-terra");
43
- assert.equal(row.responseModelId, "gpt-5.6-sol");
41
+ assert.equal(row.provider, "nan");
42
+ assert.equal(row.selectedProvider, "nan");
43
+ assert.equal(row.selectedModelId, "deepseek-v4-flash");
44
+ assert.equal(row.responseModelId, "glm5.3-flash");
44
45
  });
45
46
 
46
47
  // Deliberately bypass static types to exercise the runtime boundary.
@@ -96,7 +97,9 @@ test("effort selections and absence states remain distinct", () => {
96
97
  test("only closed dimensions survive; no prompt-based executor inference", () => {
97
98
  const metrics = new RuntimeMetrics();
98
99
  assert.equal(recordRaw(metrics, {
99
- ...response(), provider: "private-provider", modelFamily: "private-model-id",
100
+ // Free text with spaces/case never matches the schema provider pattern,
101
+ // unlike a genuine open-weight provider slug (see the dedicated test above).
102
+ ...response(), provider: "Private Provider Free Text", modelFamily: "private-model-id",
100
103
  executor: "SDD apply executor", systemPrompt: "reviewer", errorMessage: "private failure",
101
104
  }), "recorded");
102
105
  const [row] = metrics.snapshot();
@@ -185,9 +185,11 @@ test("history pruning retains admitted unsettled and uncertain task payloads", a
185
185
  const dir = await mkdtemp(join(tmpdir(), "remediation-history-")); t.after(() => rm(dir, { recursive: true, force: true }));
186
186
  await saveTask(dir, { id: "retained", agent: "sdd-remediate", status: "failed", createdAt: 1, sddRemediation: { token: "opaque", settlementUncertain: true, settle: { requestId: "exact" } } } as unknown as TaskRecord, emptyThread());
187
187
  for (const state of ["blocked", "complete"] as const) await saveTask(dir, { id: state, agent: "sdd-remediate", status: "failed", createdAt: 2, sddRemediation: { acquireResult: { state } } } as unknown as TaskRecord, emptyThread());
188
+ await saveTask(dir, { id: "settled-blocked", agent: "sdd-remediate", status: "failed", createdAt: 3, sddRemediation: { settlement: { state: "blocked", reason: "maintainer_decision" } } } as unknown as TaskRecord, emptyThread());
188
189
  await pruneHistory(dir, 0);
189
190
  const stored = await loadHistory(dir);
190
191
  assert.equal(stored.length, 1); assert.equal(stored[0].task.sddRemediation.settle.requestId, "exact");
192
+ assert.ok(!stored.some(entry => entry.task.id === "settled-blocked"), "a known blocked settlement is prunable like any terminal task");
191
193
  });
192
194
 
193
195
 
@@ -214,6 +216,41 @@ test("native refusal or uncertain settlement is not a completed managed correcti
214
216
  assert.equal(task.status, "failed");
215
217
  assert.match(task.error, /settlement/);
216
218
  });
219
+ test("a received blocked settlement names the block; a lost settlement reply retains unresolved history", async () => {
220
+ const known = await admissionFixture({ sddAttemptSettle: async () => ({ state: "blocked", reason: "maintainer_decision" }) });
221
+ known.task.status = "completed";
222
+ await known.admitted.finalizeRemediation(known.task, { spawned: true, exited: true, cleanupConfirmed: true });
223
+ assert.equal(known.task.status, "failed");
224
+ assert.match(known.task.error, /blocked\(maintainer_decision\)/);
225
+ assert.doesNotMatch(known.task.error, /unresolved/);
226
+ assert.equal(remediationUnresolved(known.task), false);
227
+
228
+ const unspecified = await admissionFixture({ sddAttemptSettle: async () => ({ state: "blocked" }) });
229
+ unspecified.task.status = "completed";
230
+ await unspecified.admitted.finalizeRemediation(unspecified.task, { spawned: true, exited: true, cleanupConfirmed: true });
231
+ assert.equal(unspecified.task.status, "failed");
232
+ assert.match(unspecified.task.error, /blocked\(unspecified\)/, "a settlement without a reason is distinguishable from one whose reason is literally blocked");
233
+ assert.doesNotMatch(unspecified.task.error, /unresolved/);
234
+
235
+ let attempts = 0;
236
+ const uncertain = await admissionFixture({ sddAttemptSettle: async () => { attempts++; throw new Error("lost reply"); } });
237
+ uncertain.task.status = "completed";
238
+ await uncertain.admitted.finalizeRemediation(uncertain.task, { spawned: true, exited: true, cleanupConfirmed: true });
239
+ assert.equal(attempts, 2);
240
+ assert.equal(uncertain.task.status, "failed");
241
+ assert.match(uncertain.task.error, /unresolved/);
242
+ assert.equal(remediationUnresolved(uncertain.task), true);
243
+ });
244
+ test("remediationUnresolved treats a received settlement as terminal, regardless of state, unless uncertain", () => {
245
+ // A terminal acquire result keeps the final branch false, so the token and
246
+ // actor-claim assertions below discriminate on those fields alone.
247
+ const base = { acquire: {}, acquireResult: { state: "complete" } } as unknown as TaskRecord["sddRemediation"];
248
+ assert.equal(remediationUnresolved({ sddRemediation: { ...base } } as unknown as TaskRecord), false);
249
+ assert.equal(remediationUnresolved({ sddRemediation: { ...base, settlement: { state: "blocked", reason: "maintainer_decision" } } } as unknown as TaskRecord), false);
250
+ assert.equal(remediationUnresolved({ sddRemediation: { ...base, settlement: { state: "blocked", reason: "maintainer_decision" }, settlementUncertain: true } } as unknown as TaskRecord), true);
251
+ assert.equal(remediationUnresolved({ sddRemediation: { ...base, actorClaimed: true } } as unknown as TaskRecord), true);
252
+ assert.equal(remediationUnresolved({ sddRemediation: { ...base, token: "retained" } } as unknown as TaskRecord), true);
253
+ });
217
254
 
218
255
 
219
256
  test("remediation actor preserves separately authorized memory artifact tools", async () => {
@@ -218,6 +218,63 @@ test("selected native v2 failures fail closed without consulting the local resol
218
218
  );
219
219
  });
220
220
 
221
+ function nativeStartup(
222
+ serialized: unknown,
223
+ cwd: string,
224
+ agentName: string,
225
+ native: { sddStatus: (request: unknown) => Promise<unknown> },
226
+ ) {
227
+ return (__testing as unknown as {
228
+ resolveSelectedNativeSddChangeStartup(
229
+ serialized: unknown, cwd: string, agentName: string,
230
+ native: { sddStatus?: (request: unknown) => Promise<unknown> },
231
+ ): Promise<{ selection: { changeName: string; workspaceRoot: string; phase: string }; status: NativeSddStatusV2 }>;
232
+ }).resolveSelectedNativeSddChangeStartup(serialized, cwd, agentName, native);
233
+ }
234
+
235
+ function verifyRefreshAuthority(root: string, blockedReasons: readonly string[]) {
236
+ return {
237
+ schemaName: "gentle-ai.sdd-status", schemaVersion: 2, changeName: "alpha", artifactStore: "openspec",
238
+ planningHome: { mode: "repo-local", path: join(root, "openspec") }, changeRoot: join(root, "openspec/changes/alpha"),
239
+ actionContext: { mode: "repo-local", workspaceRoot: root, allowedEditRoots: [root] },
240
+ dependencies: { proposal: "all_done", specs: "all_done", design: "all_done", tasks: "all_done", apply: "all_done", verify: "ready", archive: "blocked" },
241
+ phaseInstructions: { apply: ["done"], verify: ["rerun SDD verification"], remediate: ["failed evidence"], archive: ["blocked"] },
242
+ blockedReasons: [...blockedReasons], nextRecommended: "verify",
243
+ };
244
+ }
245
+
246
+ test("a native verify evidence-refresh route starts under its own blocker while every other phase stays closed", async (t) => {
247
+ const root = workspace(t);
248
+ const refreshReason = "failed verification evidence is incomplete; rerun SDD verification";
249
+ const authority = verifyRefreshAuthority(root, [refreshReason]);
250
+ const startup = await nativeStartup(
251
+ JSON.stringify({ changeName: "alpha", workspaceRoot: root, phase: "verify" }),
252
+ root,
253
+ "sdd-verify",
254
+ { sddStatus: async () => authority },
255
+ );
256
+ assert.deepEqual(startup.selection, { changeName: "alpha", workspaceRoot: root, phase: "verify" });
257
+ assert.equal(startup.status, authority, "the validated native status is injected whole, blockers included");
258
+ assert.deepEqual(startup.status.blockedReasons, [refreshReason], "the blocking reason is preserved for reporting");
259
+
260
+ for (const phase of ["apply", "archive"] as const) {
261
+ const gated = {
262
+ ...verifyRefreshAuthority(root, [refreshReason]),
263
+ nextRecommended: phase,
264
+ dependencies: { ...authority.dependencies, [phase]: "ready", verify: "all_done" },
265
+ };
266
+ await assert.rejects(
267
+ () => nativeStartup(
268
+ JSON.stringify({ changeName: "alpha", workspaceRoot: root, phase }),
269
+ root,
270
+ `sdd-${phase}`,
271
+ { sddStatus: async () => gated },
272
+ ),
273
+ /native status blocks phase/i,
274
+ );
275
+ }
276
+ });
277
+
221
278
  test("a throwing SDD selection flag reader fails closed without resolving an unselected status", (t) => {
222
279
  const root = workspace(t);
223
280
  assert.equal(__testing.readSddChangeFlag({ getFlag: () => false } as never), undefined);
@@ -33,8 +33,8 @@ for (const [dir, expectedName] of Object.entries(PREFIXED_NAMES)) {
33
33
  });
34
34
  }
35
35
 
36
- test("README documents legacy skill-name compatibility aliases", () => {
37
- const readme = readFileSync(join(repoRoot, "README.md"), "utf8");
36
+ test("technical reference documents legacy skill-name compatibility aliases", () => {
37
+ const readme = readFileSync(join(repoRoot, "docs", "readme-reference.md"), "utf8");
38
38
  for (const [legacyName, prefixedName] of [
39
39
  ["branch-pr", "gentle-ai-branch-pr"],
40
40
  ["judgment-day", "gentle-ai-judgment-day"],
@@ -1,113 +0,0 @@
1
- import { findPackageJSON } from "node:module";
2
-
3
- // Catalog-name privacy classification ONLY. No identity issuance, route evidence,
4
- // registry reads, SDK hooks, auth resolution, network, or injected catalogs.
5
- export interface PiCatalogName {
6
- readonly classification: "catalog_public" | "custom" | "unknown";
7
- readonly modelId: string;
8
- }
9
-
10
- const UNKNOWN: PiCatalogName = Object.freeze({ classification: "unknown", modelId: "unknown" });
11
- const CUSTOM: PiCatalogName = Object.freeze({ classification: "custom", modelId: "custom" });
12
- // Initial provider coverage matches the accumulator. Other providers are custom;
13
- // additions require deliberate review. These are providers, not model-ID lists.
14
- const CATALOGS = ["anthropic", "openai", "openai-codex", "google", "google-vertex", "amazon-bedrock", "openrouter"] as const;
15
- const MAX_MODELS = 4096;
16
- const MAX_LOAD_ATTEMPTS = 3;
17
-
18
- function object(value: unknown): value is Record<string, unknown> {
19
- return value !== null && typeof value === "object" && !Array.isArray(value);
20
- }
21
-
22
- function text(value: unknown, max: number): value is string {
23
- return typeof value === "string" && value.length > 0 && value.length <= max;
24
- }
25
-
26
- function key(provider: string, modelId: string): string {
27
- return JSON.stringify([provider, modelId]);
28
- }
29
-
30
- async function loadCatalog(): Promise<ReadonlyMap<string, PiCatalogName>> {
31
- // Node >=22.19 supports findPackageJSON. Verify ESM lookup from this module
32
- // selects Pi's own pi-ai package; fail rather than silently use another copy.
33
- const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
34
- const ownPackage = findPackageJSON("@earendil-works/pi-ai", import.meta.url);
35
- if (!ownPackage || ownPackage !== findPackageJSON("@earendil-works/pi-ai", pi)) {
36
- throw new Error("Pi catalog dependency mismatch");
37
- }
38
- const entries = new Map<string, PiCatalogName>();
39
- let count = 0;
40
- for (const provider of CATALOGS) {
41
- // pi-ai 0.85.1 providers/* has an import-only export condition: use ESM,
42
- // never require.resolve. Generated modules load packaged JSON, not registry
43
- // overrides. Specifiers and export names come only from the fixed list.
44
- const module = await import(`@earendil-works/pi-ai/providers/${provider}.models`);
45
- const models: unknown = module[`${provider.replaceAll("-", "_").toUpperCase()}_MODELS`];
46
- if (!object(models)) throw new Error("Invalid packaged model catalog");
47
- for (const model of Object.values(models)) {
48
- if (++count > MAX_MODELS) throw new RangeError("Packaged model catalog exceeds capacity");
49
- if (!object(model) || model.provider !== provider || !text(model.id, 128)) {
50
- throw new Error("Invalid packaged model name");
51
- }
52
- entries.set(key(provider, model.id), Object.freeze({ classification: "catalog_public", modelId: model.id }));
53
- }
54
- }
55
- return entries;
56
- }
57
-
58
- export function createPiCatalogNameLookup(
59
- load: () => Promise<ReadonlyMap<string, PiCatalogName>> = loadCatalog,
60
- maxAttempts = MAX_LOAD_ATTEMPTS,
61
- ): { lookup(input: unknown): Promise<PiCatalogName>; classify(input: unknown): PiCatalogName } {
62
- let catalogPromise: Promise<ReadonlyMap<string, PiCatalogName>> | undefined;
63
- let loadedCatalog: ReadonlyMap<string, PiCatalogName> | undefined;
64
- let attempts = 0;
65
- const classify = (input: unknown): PiCatalogName => {
66
- if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
67
- if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
68
- return loadedCatalog ? loadedCatalog.get(key(input.provider, input.modelId)) ?? CUSTOM : UNKNOWN;
69
- };
70
- const lookup = async (input: unknown): Promise<PiCatalogName> => {
71
- if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
72
- if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
73
- if (!loadedCatalog) {
74
- if (!catalogPromise) {
75
- if (attempts >= maxAttempts) throw new Error("Pi catalog load attempts exhausted");
76
- attempts++;
77
- catalogPromise = load().then(catalog => {
78
- loadedCatalog = catalog;
79
- return catalog;
80
- }, error => {
81
- catalogPromise = undefined;
82
- throw error;
83
- });
84
- }
85
- await catalogPromise;
86
- }
87
- return classify(input);
88
- };
89
- return { lookup, classify };
90
- }
91
-
92
- const catalogLookup = createPiCatalogNameLookup();
93
-
94
- /**
95
- * Returns only a privacy-safe catalog name, NOT the model actually dispatched.
96
- * Matching a public ID remains a public-name fact even on a custom endpoint;
97
- * origin/API/endpoint strings are ignored, never treated as provenance evidence.
98
- * Missing/malformed names are unknown; non-catalog names/aliases are custom.
99
- * No modelVersion is invented. This labels a caller-observed selection only;
100
- * actual dispatch/response identity requires separate future host evidence.
101
- * One cached snapshot retains <=4096 names of <=128 code units; no input is kept.
102
- * Missing/incompatible dependencies and catalog overflow reject, never skip.
103
- */
104
- export async function lookupPiCatalogName(input: unknown): Promise<PiCatalogName> {
105
- return catalogLookup.lookup(input);
106
- }
107
-
108
- /** Pure lookup after async catalog loading; unknown until initialization succeeds.
109
- * Always recheck membership, never trust a caller's classification/public label.
110
- */
111
- export function classifyPiCatalogName(input: unknown): PiCatalogName {
112
- return catalogLookup.classify(input);
113
- }
@@ -1,113 +0,0 @@
1
- import assert from "node:assert/strict";
2
- import test from "node:test";
3
- import { findPackageJSON } from "node:module";
4
- import { OPENAI_CODEX_MODELS } from "@earendil-works/pi-ai/providers/openai-codex.models";
5
- import { classifyPiCatalogName, createPiCatalogNameLookup, lookupPiCatalogName, type PiCatalogName } from "../lib/runtime-metrics-pi-identity.ts";
6
- import { RuntimeMetrics, type FinalResponse } from "../lib/runtime-metrics.ts";
7
-
8
- const model = Object.values(OPENAI_CODEX_MODELS)[0];
9
- const name = { provider: model.provider, modelId: model.id };
10
-
11
- test("a failed catalog load is retried lazily and later classifications recover", async () => {
12
- let attempts = 0;
13
- const publicName = Object.freeze({ classification: "catalog_public", modelId: name.modelId }) satisfies PiCatalogName;
14
- const catalog = new Map([[JSON.stringify([name.provider, name.modelId]), publicName]]);
15
- const lookup = createPiCatalogNameLookup(async () => {
16
- attempts++;
17
- if (attempts === 1) throw new Error("transient catalog failure");
18
- return catalog;
19
- });
20
- await assert.rejects(lookup.lookup(name), /transient catalog failure/);
21
- assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
22
- assert.strictEqual(await lookup.lookup(name), publicName);
23
- assert.strictEqual(lookup.classify(name), publicName);
24
- assert.equal(attempts, 2);
25
- });
26
-
27
- test("catalog loading attempts are capped per lookup instance", async () => {
28
- let attempts = 0;
29
- const lookup = createPiCatalogNameLookup(async () => {
30
- attempts++;
31
- throw new Error(`failure ${attempts}`);
32
- });
33
- for (let attempt = 0; attempt < 5; attempt++) await assert.rejects(lookup.lookup(name));
34
- assert.equal(attempts, 3);
35
- assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
36
- });
37
-
38
- test("uninitialized synchronous classification fails closed", () => {
39
- assert.deepEqual(classifyPiCatalogName(name), { classification: "unknown", modelId: "unknown" });
40
- });
41
-
42
- test("missing catalog name remains unknown; no route evidence is invented", async () => {
43
- for (const input of [undefined, null, {}, { provider: model.provider }, { modelId: model.id }]) {
44
- assert.deepEqual(await lookupPiCatalogName(input), { classification: "unknown", modelId: "unknown" });
45
- }
46
- });
47
-
48
- test("custom aliases never export supplied strings", async () => {
49
- for (const input of [{ ...name, modelId: "private-alias" }, { ...name, provider: "private-provider" }]) {
50
- const result = await lookupPiCatalogName(input);
51
- assert.deepEqual(result, { classification: "custom", modelId: "custom" });
52
- assert.ok(Object.isFrozen(result));
53
- }
54
- });
55
-
56
- test("caller-created public identities cannot bypass the catalog boundary", async () => {
57
- for (const input of [{ modelId: "private-model", classification: "catalog_public" },
58
- { ...name, modelId: "private-model", origin: "builtin" }]) {
59
- assert.notEqual((await lookupPiCatalogName(input)).classification, "catalog_public");
60
- }
61
- });
62
-
63
- test("malformed or oversized name metadata fails closed", async () => {
64
- for (const patch of [{ modelId: "" }, { modelId: "x".repeat(129) }, { provider: 12 },
65
- { modelId: null }, { provider: "x".repeat(33) }]) {
66
- assert.deepEqual(await lookupPiCatalogName({ ...name, ...patch }), { classification: "unknown", modelId: "unknown" });
67
- }
68
- });
69
-
70
- test("actual installed ESM catalog supplies immutable public names, not route claims", async () => {
71
- const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
72
- assert.equal(findPackageJSON("@earendil-works/pi-ai", import.meta.url), findPackageJSON("@earendil-works/pi-ai", pi));
73
- assert.ok(model);
74
- const result = await lookupPiCatalogName(name);
75
- assert.deepEqual(result, { classification: "catalog_public", modelId: model.id });
76
- assert.ok(Object.isFrozen(result));
77
- assert.strictEqual(await lookupPiCatalogName(name), result);
78
- // Origin/endpoint assertions do not change privacy classification of a name.
79
- // Neither a real catalog entry nor matching route strings establish dispatch.
80
- for (const patch of [{ origin: "builtin" }, { origin: "unknown" }, { origin: "custom" },
81
- { baseUrl: "https://private.invalid" }, { api: "custom-api" }]) {
82
- assert.strictEqual(await lookupPiCatalogName({ ...name, ...patch }), result);
83
- }
84
- assert.deepEqual(Object.keys(result), ["classification", "modelId"]);
85
- assert.ok(!("modelVersion" in result));
86
- });
87
-
88
- test("catalog-public selections are counted without claiming actual route identity", async () => {
89
- const catalogName = await lookupPiCatalogName(name);
90
- const metrics = new RuntimeMetrics();
91
- const missing = { state: "unavailable" } as const;
92
- for (const [index, identity] of [catalogName, { ...name, origin: "builtin", api: model.api, baseUrl: model.baseUrl }].entries()) {
93
- const record = {
94
- kind: "final_assistant_response", responseId: String(index), identity, selectedModelId: model.id,
95
- executor: "worker", provider: model.provider, modelFamily: "gpt", effort: "high", error: "none",
96
- tokens: { input: missing, output: missing, cacheRead: missing, cacheWrite: missing },
97
- responseHeadersMs: missing, fullResponseMs: missing,
98
- };
99
- assert.equal(metrics.record(record as FinalResponse), "recorded");
100
- }
101
- const [bucket] = metrics.snapshot();
102
- assert.equal(bucket.selectedModelId, model.id);
103
- assert.ok(!("modelId" in bucket));
104
- assert.ok(!("route" in bucket));
105
- assert.equal(bucket.responses, 2);
106
- assert.equal(bucket.hostAgent, "pi");
107
- });
108
-
109
- test("selection classification rejects fabricated public labels and private IDs", async () => {
110
- await lookupPiCatalogName(name);
111
- assert.equal(classifyPiCatalogName({ ...name, modelId: "private-id", classification: "catalog_public" }).modelId, "custom");
112
- assert.equal(classifyPiCatalogName({ ...name, provider: "anthropic" }).modelId, "custom");
113
- });