gentle-pi 2.6.0 → 2.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +183 -943
- package/contracts/review-provider-contract-mirror/provider-contract.lock.json +9 -8
- package/contracts/review-provider-contract-mirror/v1.2.0/bundle/manifest.json +3 -3
- package/contracts/review-provider-contract-mirror/v1.2.0/bundle/orchestration/pi.md +7 -2
- package/contracts/review-provider-contract-mirror/v1.2.0/bundle/schemas/lens.schema.json +2 -2
- package/contracts/review-provider-contract-mirror/v1.2.0/bundle/schemas/targeted-validator.schema.json +1 -1
- package/contracts/review-provider-contract-mirror/v1.2.0/generated/provider-capabilities.baseline.json +1 -1
- package/contracts/review-provider-contract-mirror/v1.2.0/generated/provider-roles.baseline.json +2 -2
- package/contracts/telemetry/runtime-aggregate-v1.schema.json +8 -6
- package/docs/assets/brand/gentle-pi-banner.png +0 -0
- package/docs/assets/brand/gentle-pi-banner.svg +33 -0
- package/docs/assets/brand/terminal-divider.svg +17 -0
- package/docs/assets/diagrams/agent-orchestration.svg +19 -0
- package/docs/assets/diagrams/gentleman-workflow.svg +15 -0
- package/docs/assets/diagrams/native-review.svg +16 -0
- package/docs/assets/diagrams/sdd-cycle.svg +14 -0
- package/docs/assets/features/gentle-shell.png +0 -0
- package/docs/gentle-shell.md +151 -0
- package/docs/readme-reference.md +868 -0
- package/extensions/gentle-agents.ts +4 -5
- package/extensions/gentle-ai.ts +65 -22
- package/extensions/runtime-metrics.ts +10 -19
- package/lib/agents-history.ts +7 -1
- package/lib/native-review-cli.ts +14 -0
- package/lib/runtime-metrics-children.ts +3 -5
- package/lib/runtime-metrics-native.ts +17 -9
- package/lib/runtime-metrics.ts +95 -42
- package/package.json +1 -1
- package/runtime/native-review-cli.mjs +14 -0
- package/scripts/gentle-ai-installer.mjs +10 -10
- package/scripts/types-baseline.json +1 -5
- package/scripts/verify-package-files.mjs +3 -3
- package/tests/fixtures/runtime-metrics-native-batches.json +1 -1
- package/tests/gentle-agents.test.ts +22 -6
- package/tests/gentle-ai-binary.test.ts +1 -1
- package/tests/gentle-ai-installer.test.ts +47 -47
- package/tests/gentle-ai.test.ts +3 -2
- package/tests/native-review-capability-contract.test.ts +24 -1
- package/tests/package-manifest.test.ts +19 -18
- package/tests/review-authority-recovery-docs.test.ts +13 -13
- package/tests/review-controller-native-routing.test.ts +49 -0
- package/tests/review-ledger-contract.test.ts +7 -5
- package/tests/runtime-metrics-children.test.ts +4 -3
- package/tests/runtime-metrics-extension.test.ts +40 -31
- package/tests/runtime-metrics-model.test.ts +76 -0
- package/tests/runtime-metrics-native.test.ts +73 -7
- package/tests/runtime-metrics.test.ts +22 -19
- package/tests/sdd-managed-runtime-settlement.test.ts +37 -0
- package/tests/sdd-selection-transport.test.ts +57 -0
- package/tests/skill-collision-prefixes.test.ts +2 -2
- package/lib/runtime-metrics-pi-identity.ts +0 -113
- package/tests/runtime-metrics-pi-identity.test.ts +0 -113
|
@@ -16,31 +16,32 @@ function response(responseId = "local-response"): FinalResponse {
|
|
|
16
16
|
};
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
-
test("selected provider is independent from response provider without relabeling",
|
|
20
|
-
const { lookupPiCatalogName } = await import("../lib/runtime-metrics-pi-identity.ts");
|
|
21
|
-
const { OPENAI_CODEX_MODELS } = await import("@earendil-works/pi-ai/providers/openai-codex.models");
|
|
22
|
-
const model = Object.values(OPENAI_CODEX_MODELS)[0];
|
|
23
|
-
await lookupPiCatalogName({ provider: model.provider, modelId: model.id });
|
|
19
|
+
test("selected provider is independent from response provider without relabeling", () => {
|
|
24
20
|
const metrics = new RuntimeMetrics();
|
|
25
|
-
assert.equal(recordRaw(metrics, { ...response(), selectedProvider:
|
|
26
|
-
selectedModelId:
|
|
21
|
+
assert.equal(recordRaw(metrics, { ...response(), selectedProvider: "openai-codex",
|
|
22
|
+
selectedModelId: "gpt-5.6-terra", provider: "anthropic" }), "recorded");
|
|
27
23
|
const [row] = metrics.snapshot();
|
|
28
|
-
assert.equal(row.selectedModelId,
|
|
29
|
-
assert.equal(row.selectedProvider,
|
|
24
|
+
assert.equal(row.selectedModelId, "gpt-5.6-terra");
|
|
25
|
+
assert.equal(row.selectedProvider, "openai-codex");
|
|
30
26
|
assert.equal(row.provider, "anthropic");
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
27
|
+
// The provider dimension stays independent of a specific id's privacy outcome
|
|
28
|
+
// at record time; the pair is only coupled to a schema-conformant custom/custom
|
|
29
|
+
// model object later, at native encode time (see runtime-metrics-native.test.ts).
|
|
30
|
+
assert.equal(recordRaw(metrics, { ...response("private"), selectedProvider: "openai-codex",
|
|
31
|
+
selectedModelId: "private-internal-finetune" }), "recorded");
|
|
32
|
+
assert.equal(metrics.snapshot()[1].selectedProvider, "openai-codex");
|
|
34
33
|
assert.equal(metrics.snapshot()[1].selectedModelId, "custom");
|
|
35
34
|
});
|
|
36
35
|
|
|
37
|
-
test("
|
|
38
|
-
const metrics = new RuntimeMetrics(
|
|
39
|
-
assert.equal(metrics
|
|
40
|
-
responseModelId: "
|
|
36
|
+
test("open-weight models on arbitrary providers are reported by name, not squashed to a closed enum", () => {
|
|
37
|
+
const metrics = new RuntimeMetrics();
|
|
38
|
+
assert.equal(recordRaw(metrics, { ...response(), provider: "nan", selectedProvider: "nan",
|
|
39
|
+
selectedModelId: "deepseek-v4-flash", responseModelId: "glm5.3-flash" }), "recorded");
|
|
41
40
|
const [row] = metrics.snapshot();
|
|
42
|
-
assert.equal(row.
|
|
43
|
-
assert.equal(row.
|
|
41
|
+
assert.equal(row.provider, "nan");
|
|
42
|
+
assert.equal(row.selectedProvider, "nan");
|
|
43
|
+
assert.equal(row.selectedModelId, "deepseek-v4-flash");
|
|
44
|
+
assert.equal(row.responseModelId, "glm5.3-flash");
|
|
44
45
|
});
|
|
45
46
|
|
|
46
47
|
// Deliberately bypass static types to exercise the runtime boundary.
|
|
@@ -96,7 +97,9 @@ test("effort selections and absence states remain distinct", () => {
|
|
|
96
97
|
test("only closed dimensions survive; no prompt-based executor inference", () => {
|
|
97
98
|
const metrics = new RuntimeMetrics();
|
|
98
99
|
assert.equal(recordRaw(metrics, {
|
|
99
|
-
|
|
100
|
+
// Free text with spaces/case never matches the schema provider pattern,
|
|
101
|
+
// unlike a genuine open-weight provider slug (see the dedicated test above).
|
|
102
|
+
...response(), provider: "Private Provider Free Text", modelFamily: "private-model-id",
|
|
100
103
|
executor: "SDD apply executor", systemPrompt: "reviewer", errorMessage: "private failure",
|
|
101
104
|
}), "recorded");
|
|
102
105
|
const [row] = metrics.snapshot();
|
|
@@ -185,9 +185,11 @@ test("history pruning retains admitted unsettled and uncertain task payloads", a
|
|
|
185
185
|
const dir = await mkdtemp(join(tmpdir(), "remediation-history-")); t.after(() => rm(dir, { recursive: true, force: true }));
|
|
186
186
|
await saveTask(dir, { id: "retained", agent: "sdd-remediate", status: "failed", createdAt: 1, sddRemediation: { token: "opaque", settlementUncertain: true, settle: { requestId: "exact" } } } as unknown as TaskRecord, emptyThread());
|
|
187
187
|
for (const state of ["blocked", "complete"] as const) await saveTask(dir, { id: state, agent: "sdd-remediate", status: "failed", createdAt: 2, sddRemediation: { acquireResult: { state } } } as unknown as TaskRecord, emptyThread());
|
|
188
|
+
await saveTask(dir, { id: "settled-blocked", agent: "sdd-remediate", status: "failed", createdAt: 3, sddRemediation: { settlement: { state: "blocked", reason: "maintainer_decision" } } } as unknown as TaskRecord, emptyThread());
|
|
188
189
|
await pruneHistory(dir, 0);
|
|
189
190
|
const stored = await loadHistory(dir);
|
|
190
191
|
assert.equal(stored.length, 1); assert.equal(stored[0].task.sddRemediation.settle.requestId, "exact");
|
|
192
|
+
assert.ok(!stored.some(entry => entry.task.id === "settled-blocked"), "a known blocked settlement is prunable like any terminal task");
|
|
191
193
|
});
|
|
192
194
|
|
|
193
195
|
|
|
@@ -214,6 +216,41 @@ test("native refusal or uncertain settlement is not a completed managed correcti
|
|
|
214
216
|
assert.equal(task.status, "failed");
|
|
215
217
|
assert.match(task.error, /settlement/);
|
|
216
218
|
});
|
|
219
|
+
test("a received blocked settlement names the block; a lost settlement reply retains unresolved history", async () => {
|
|
220
|
+
const known = await admissionFixture({ sddAttemptSettle: async () => ({ state: "blocked", reason: "maintainer_decision" }) });
|
|
221
|
+
known.task.status = "completed";
|
|
222
|
+
await known.admitted.finalizeRemediation(known.task, { spawned: true, exited: true, cleanupConfirmed: true });
|
|
223
|
+
assert.equal(known.task.status, "failed");
|
|
224
|
+
assert.match(known.task.error, /blocked\(maintainer_decision\)/);
|
|
225
|
+
assert.doesNotMatch(known.task.error, /unresolved/);
|
|
226
|
+
assert.equal(remediationUnresolved(known.task), false);
|
|
227
|
+
|
|
228
|
+
const unspecified = await admissionFixture({ sddAttemptSettle: async () => ({ state: "blocked" }) });
|
|
229
|
+
unspecified.task.status = "completed";
|
|
230
|
+
await unspecified.admitted.finalizeRemediation(unspecified.task, { spawned: true, exited: true, cleanupConfirmed: true });
|
|
231
|
+
assert.equal(unspecified.task.status, "failed");
|
|
232
|
+
assert.match(unspecified.task.error, /blocked\(unspecified\)/, "a settlement without a reason is distinguishable from one whose reason is literally blocked");
|
|
233
|
+
assert.doesNotMatch(unspecified.task.error, /unresolved/);
|
|
234
|
+
|
|
235
|
+
let attempts = 0;
|
|
236
|
+
const uncertain = await admissionFixture({ sddAttemptSettle: async () => { attempts++; throw new Error("lost reply"); } });
|
|
237
|
+
uncertain.task.status = "completed";
|
|
238
|
+
await uncertain.admitted.finalizeRemediation(uncertain.task, { spawned: true, exited: true, cleanupConfirmed: true });
|
|
239
|
+
assert.equal(attempts, 2);
|
|
240
|
+
assert.equal(uncertain.task.status, "failed");
|
|
241
|
+
assert.match(uncertain.task.error, /unresolved/);
|
|
242
|
+
assert.equal(remediationUnresolved(uncertain.task), true);
|
|
243
|
+
});
|
|
244
|
+
test("remediationUnresolved treats a received settlement as terminal, regardless of state, unless uncertain", () => {
|
|
245
|
+
// A terminal acquire result keeps the final branch false, so the token and
|
|
246
|
+
// actor-claim assertions below discriminate on those fields alone.
|
|
247
|
+
const base = { acquire: {}, acquireResult: { state: "complete" } } as unknown as TaskRecord["sddRemediation"];
|
|
248
|
+
assert.equal(remediationUnresolved({ sddRemediation: { ...base } } as unknown as TaskRecord), false);
|
|
249
|
+
assert.equal(remediationUnresolved({ sddRemediation: { ...base, settlement: { state: "blocked", reason: "maintainer_decision" } } } as unknown as TaskRecord), false);
|
|
250
|
+
assert.equal(remediationUnresolved({ sddRemediation: { ...base, settlement: { state: "blocked", reason: "maintainer_decision" }, settlementUncertain: true } } as unknown as TaskRecord), true);
|
|
251
|
+
assert.equal(remediationUnresolved({ sddRemediation: { ...base, actorClaimed: true } } as unknown as TaskRecord), true);
|
|
252
|
+
assert.equal(remediationUnresolved({ sddRemediation: { ...base, token: "retained" } } as unknown as TaskRecord), true);
|
|
253
|
+
});
|
|
217
254
|
|
|
218
255
|
|
|
219
256
|
test("remediation actor preserves separately authorized memory artifact tools", async () => {
|
|
@@ -218,6 +218,63 @@ test("selected native v2 failures fail closed without consulting the local resol
|
|
|
218
218
|
);
|
|
219
219
|
});
|
|
220
220
|
|
|
221
|
+
function nativeStartup(
|
|
222
|
+
serialized: unknown,
|
|
223
|
+
cwd: string,
|
|
224
|
+
agentName: string,
|
|
225
|
+
native: { sddStatus: (request: unknown) => Promise<unknown> },
|
|
226
|
+
) {
|
|
227
|
+
return (__testing as unknown as {
|
|
228
|
+
resolveSelectedNativeSddChangeStartup(
|
|
229
|
+
serialized: unknown, cwd: string, agentName: string,
|
|
230
|
+
native: { sddStatus?: (request: unknown) => Promise<unknown> },
|
|
231
|
+
): Promise<{ selection: { changeName: string; workspaceRoot: string; phase: string }; status: NativeSddStatusV2 }>;
|
|
232
|
+
}).resolveSelectedNativeSddChangeStartup(serialized, cwd, agentName, native);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
function verifyRefreshAuthority(root: string, blockedReasons: readonly string[]) {
|
|
236
|
+
return {
|
|
237
|
+
schemaName: "gentle-ai.sdd-status", schemaVersion: 2, changeName: "alpha", artifactStore: "openspec",
|
|
238
|
+
planningHome: { mode: "repo-local", path: join(root, "openspec") }, changeRoot: join(root, "openspec/changes/alpha"),
|
|
239
|
+
actionContext: { mode: "repo-local", workspaceRoot: root, allowedEditRoots: [root] },
|
|
240
|
+
dependencies: { proposal: "all_done", specs: "all_done", design: "all_done", tasks: "all_done", apply: "all_done", verify: "ready", archive: "blocked" },
|
|
241
|
+
phaseInstructions: { apply: ["done"], verify: ["rerun SDD verification"], remediate: ["failed evidence"], archive: ["blocked"] },
|
|
242
|
+
blockedReasons: [...blockedReasons], nextRecommended: "verify",
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
test("a native verify evidence-refresh route starts under its own blocker while every other phase stays closed", async (t) => {
|
|
247
|
+
const root = workspace(t);
|
|
248
|
+
const refreshReason = "failed verification evidence is incomplete; rerun SDD verification";
|
|
249
|
+
const authority = verifyRefreshAuthority(root, [refreshReason]);
|
|
250
|
+
const startup = await nativeStartup(
|
|
251
|
+
JSON.stringify({ changeName: "alpha", workspaceRoot: root, phase: "verify" }),
|
|
252
|
+
root,
|
|
253
|
+
"sdd-verify",
|
|
254
|
+
{ sddStatus: async () => authority },
|
|
255
|
+
);
|
|
256
|
+
assert.deepEqual(startup.selection, { changeName: "alpha", workspaceRoot: root, phase: "verify" });
|
|
257
|
+
assert.equal(startup.status, authority, "the validated native status is injected whole, blockers included");
|
|
258
|
+
assert.deepEqual(startup.status.blockedReasons, [refreshReason], "the blocking reason is preserved for reporting");
|
|
259
|
+
|
|
260
|
+
for (const phase of ["apply", "archive"] as const) {
|
|
261
|
+
const gated = {
|
|
262
|
+
...verifyRefreshAuthority(root, [refreshReason]),
|
|
263
|
+
nextRecommended: phase,
|
|
264
|
+
dependencies: { ...authority.dependencies, [phase]: "ready", verify: "all_done" },
|
|
265
|
+
};
|
|
266
|
+
await assert.rejects(
|
|
267
|
+
() => nativeStartup(
|
|
268
|
+
JSON.stringify({ changeName: "alpha", workspaceRoot: root, phase }),
|
|
269
|
+
root,
|
|
270
|
+
`sdd-${phase}`,
|
|
271
|
+
{ sddStatus: async () => gated },
|
|
272
|
+
),
|
|
273
|
+
/native status blocks phase/i,
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
});
|
|
277
|
+
|
|
221
278
|
test("a throwing SDD selection flag reader fails closed without resolving an unselected status", (t) => {
|
|
222
279
|
const root = workspace(t);
|
|
223
280
|
assert.equal(__testing.readSddChangeFlag({ getFlag: () => false } as never), undefined);
|
|
@@ -33,8 +33,8 @@ for (const [dir, expectedName] of Object.entries(PREFIXED_NAMES)) {
|
|
|
33
33
|
});
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
test("
|
|
37
|
-
const readme = readFileSync(join(repoRoot, "
|
|
36
|
+
test("technical reference documents legacy skill-name compatibility aliases", () => {
|
|
37
|
+
const readme = readFileSync(join(repoRoot, "docs", "readme-reference.md"), "utf8");
|
|
38
38
|
for (const [legacyName, prefixedName] of [
|
|
39
39
|
["branch-pr", "gentle-ai-branch-pr"],
|
|
40
40
|
["judgment-day", "gentle-ai-judgment-day"],
|
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
import { findPackageJSON } from "node:module";
|
|
2
|
-
|
|
3
|
-
// Catalog-name privacy classification ONLY. No identity issuance, route evidence,
|
|
4
|
-
// registry reads, SDK hooks, auth resolution, network, or injected catalogs.
|
|
5
|
-
export interface PiCatalogName {
|
|
6
|
-
readonly classification: "catalog_public" | "custom" | "unknown";
|
|
7
|
-
readonly modelId: string;
|
|
8
|
-
}
|
|
9
|
-
|
|
10
|
-
const UNKNOWN: PiCatalogName = Object.freeze({ classification: "unknown", modelId: "unknown" });
|
|
11
|
-
const CUSTOM: PiCatalogName = Object.freeze({ classification: "custom", modelId: "custom" });
|
|
12
|
-
// Initial provider coverage matches the accumulator. Other providers are custom;
|
|
13
|
-
// additions require deliberate review. These are providers, not model-ID lists.
|
|
14
|
-
const CATALOGS = ["anthropic", "openai", "openai-codex", "google", "google-vertex", "amazon-bedrock", "openrouter"] as const;
|
|
15
|
-
const MAX_MODELS = 4096;
|
|
16
|
-
const MAX_LOAD_ATTEMPTS = 3;
|
|
17
|
-
|
|
18
|
-
function object(value: unknown): value is Record<string, unknown> {
|
|
19
|
-
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
function text(value: unknown, max: number): value is string {
|
|
23
|
-
return typeof value === "string" && value.length > 0 && value.length <= max;
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
function key(provider: string, modelId: string): string {
|
|
27
|
-
return JSON.stringify([provider, modelId]);
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
async function loadCatalog(): Promise<ReadonlyMap<string, PiCatalogName>> {
|
|
31
|
-
// Node >=22.19 supports findPackageJSON. Verify ESM lookup from this module
|
|
32
|
-
// selects Pi's own pi-ai package; fail rather than silently use another copy.
|
|
33
|
-
const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
|
|
34
|
-
const ownPackage = findPackageJSON("@earendil-works/pi-ai", import.meta.url);
|
|
35
|
-
if (!ownPackage || ownPackage !== findPackageJSON("@earendil-works/pi-ai", pi)) {
|
|
36
|
-
throw new Error("Pi catalog dependency mismatch");
|
|
37
|
-
}
|
|
38
|
-
const entries = new Map<string, PiCatalogName>();
|
|
39
|
-
let count = 0;
|
|
40
|
-
for (const provider of CATALOGS) {
|
|
41
|
-
// pi-ai 0.85.1 providers/* has an import-only export condition: use ESM,
|
|
42
|
-
// never require.resolve. Generated modules load packaged JSON, not registry
|
|
43
|
-
// overrides. Specifiers and export names come only from the fixed list.
|
|
44
|
-
const module = await import(`@earendil-works/pi-ai/providers/${provider}.models`);
|
|
45
|
-
const models: unknown = module[`${provider.replaceAll("-", "_").toUpperCase()}_MODELS`];
|
|
46
|
-
if (!object(models)) throw new Error("Invalid packaged model catalog");
|
|
47
|
-
for (const model of Object.values(models)) {
|
|
48
|
-
if (++count > MAX_MODELS) throw new RangeError("Packaged model catalog exceeds capacity");
|
|
49
|
-
if (!object(model) || model.provider !== provider || !text(model.id, 128)) {
|
|
50
|
-
throw new Error("Invalid packaged model name");
|
|
51
|
-
}
|
|
52
|
-
entries.set(key(provider, model.id), Object.freeze({ classification: "catalog_public", modelId: model.id }));
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
return entries;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
export function createPiCatalogNameLookup(
|
|
59
|
-
load: () => Promise<ReadonlyMap<string, PiCatalogName>> = loadCatalog,
|
|
60
|
-
maxAttempts = MAX_LOAD_ATTEMPTS,
|
|
61
|
-
): { lookup(input: unknown): Promise<PiCatalogName>; classify(input: unknown): PiCatalogName } {
|
|
62
|
-
let catalogPromise: Promise<ReadonlyMap<string, PiCatalogName>> | undefined;
|
|
63
|
-
let loadedCatalog: ReadonlyMap<string, PiCatalogName> | undefined;
|
|
64
|
-
let attempts = 0;
|
|
65
|
-
const classify = (input: unknown): PiCatalogName => {
|
|
66
|
-
if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
|
|
67
|
-
if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
|
|
68
|
-
return loadedCatalog ? loadedCatalog.get(key(input.provider, input.modelId)) ?? CUSTOM : UNKNOWN;
|
|
69
|
-
};
|
|
70
|
-
const lookup = async (input: unknown): Promise<PiCatalogName> => {
|
|
71
|
-
if (!object(input) || !text(input.provider, 32) || !text(input.modelId, 128)) return UNKNOWN;
|
|
72
|
-
if (!CATALOGS.includes(input.provider as typeof CATALOGS[number])) return CUSTOM;
|
|
73
|
-
if (!loadedCatalog) {
|
|
74
|
-
if (!catalogPromise) {
|
|
75
|
-
if (attempts >= maxAttempts) throw new Error("Pi catalog load attempts exhausted");
|
|
76
|
-
attempts++;
|
|
77
|
-
catalogPromise = load().then(catalog => {
|
|
78
|
-
loadedCatalog = catalog;
|
|
79
|
-
return catalog;
|
|
80
|
-
}, error => {
|
|
81
|
-
catalogPromise = undefined;
|
|
82
|
-
throw error;
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
await catalogPromise;
|
|
86
|
-
}
|
|
87
|
-
return classify(input);
|
|
88
|
-
};
|
|
89
|
-
return { lookup, classify };
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
const catalogLookup = createPiCatalogNameLookup();
|
|
93
|
-
|
|
94
|
-
/**
|
|
95
|
-
* Returns only a privacy-safe catalog name, NOT the model actually dispatched.
|
|
96
|
-
* Matching a public ID remains a public-name fact even on a custom endpoint;
|
|
97
|
-
* origin/API/endpoint strings are ignored, never treated as provenance evidence.
|
|
98
|
-
* Missing/malformed names are unknown; non-catalog names/aliases are custom.
|
|
99
|
-
* No modelVersion is invented. This labels a caller-observed selection only;
|
|
100
|
-
* actual dispatch/response identity requires separate future host evidence.
|
|
101
|
-
* One cached snapshot retains <=4096 names of <=128 code units; no input is kept.
|
|
102
|
-
* Missing/incompatible dependencies and catalog overflow reject, never skip.
|
|
103
|
-
*/
|
|
104
|
-
export async function lookupPiCatalogName(input: unknown): Promise<PiCatalogName> {
|
|
105
|
-
return catalogLookup.lookup(input);
|
|
106
|
-
}
|
|
107
|
-
|
|
108
|
-
/** Pure lookup after async catalog loading; unknown until initialization succeeds.
|
|
109
|
-
* Always recheck membership, never trust a caller's classification/public label.
|
|
110
|
-
*/
|
|
111
|
-
export function classifyPiCatalogName(input: unknown): PiCatalogName {
|
|
112
|
-
return catalogLookup.classify(input);
|
|
113
|
-
}
|
|
@@ -1,113 +0,0 @@
|
|
|
1
|
-
import assert from "node:assert/strict";
|
|
2
|
-
import test from "node:test";
|
|
3
|
-
import { findPackageJSON } from "node:module";
|
|
4
|
-
import { OPENAI_CODEX_MODELS } from "@earendil-works/pi-ai/providers/openai-codex.models";
|
|
5
|
-
import { classifyPiCatalogName, createPiCatalogNameLookup, lookupPiCatalogName, type PiCatalogName } from "../lib/runtime-metrics-pi-identity.ts";
|
|
6
|
-
import { RuntimeMetrics, type FinalResponse } from "../lib/runtime-metrics.ts";
|
|
7
|
-
|
|
8
|
-
const model = Object.values(OPENAI_CODEX_MODELS)[0];
|
|
9
|
-
const name = { provider: model.provider, modelId: model.id };
|
|
10
|
-
|
|
11
|
-
test("a failed catalog load is retried lazily and later classifications recover", async () => {
|
|
12
|
-
let attempts = 0;
|
|
13
|
-
const publicName = Object.freeze({ classification: "catalog_public", modelId: name.modelId }) satisfies PiCatalogName;
|
|
14
|
-
const catalog = new Map([[JSON.stringify([name.provider, name.modelId]), publicName]]);
|
|
15
|
-
const lookup = createPiCatalogNameLookup(async () => {
|
|
16
|
-
attempts++;
|
|
17
|
-
if (attempts === 1) throw new Error("transient catalog failure");
|
|
18
|
-
return catalog;
|
|
19
|
-
});
|
|
20
|
-
await assert.rejects(lookup.lookup(name), /transient catalog failure/);
|
|
21
|
-
assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
|
|
22
|
-
assert.strictEqual(await lookup.lookup(name), publicName);
|
|
23
|
-
assert.strictEqual(lookup.classify(name), publicName);
|
|
24
|
-
assert.equal(attempts, 2);
|
|
25
|
-
});
|
|
26
|
-
|
|
27
|
-
test("catalog loading attempts are capped per lookup instance", async () => {
|
|
28
|
-
let attempts = 0;
|
|
29
|
-
const lookup = createPiCatalogNameLookup(async () => {
|
|
30
|
-
attempts++;
|
|
31
|
-
throw new Error(`failure ${attempts}`);
|
|
32
|
-
});
|
|
33
|
-
for (let attempt = 0; attempt < 5; attempt++) await assert.rejects(lookup.lookup(name));
|
|
34
|
-
assert.equal(attempts, 3);
|
|
35
|
-
assert.deepEqual(lookup.classify(name), { classification: "unknown", modelId: "unknown" });
|
|
36
|
-
});
|
|
37
|
-
|
|
38
|
-
test("uninitialized synchronous classification fails closed", () => {
|
|
39
|
-
assert.deepEqual(classifyPiCatalogName(name), { classification: "unknown", modelId: "unknown" });
|
|
40
|
-
});
|
|
41
|
-
|
|
42
|
-
test("missing catalog name remains unknown; no route evidence is invented", async () => {
|
|
43
|
-
for (const input of [undefined, null, {}, { provider: model.provider }, { modelId: model.id }]) {
|
|
44
|
-
assert.deepEqual(await lookupPiCatalogName(input), { classification: "unknown", modelId: "unknown" });
|
|
45
|
-
}
|
|
46
|
-
});
|
|
47
|
-
|
|
48
|
-
test("custom aliases never export supplied strings", async () => {
|
|
49
|
-
for (const input of [{ ...name, modelId: "private-alias" }, { ...name, provider: "private-provider" }]) {
|
|
50
|
-
const result = await lookupPiCatalogName(input);
|
|
51
|
-
assert.deepEqual(result, { classification: "custom", modelId: "custom" });
|
|
52
|
-
assert.ok(Object.isFrozen(result));
|
|
53
|
-
}
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
test("caller-created public identities cannot bypass the catalog boundary", async () => {
|
|
57
|
-
for (const input of [{ modelId: "private-model", classification: "catalog_public" },
|
|
58
|
-
{ ...name, modelId: "private-model", origin: "builtin" }]) {
|
|
59
|
-
assert.notEqual((await lookupPiCatalogName(input)).classification, "catalog_public");
|
|
60
|
-
}
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
test("malformed or oversized name metadata fails closed", async () => {
|
|
64
|
-
for (const patch of [{ modelId: "" }, { modelId: "x".repeat(129) }, { provider: 12 },
|
|
65
|
-
{ modelId: null }, { provider: "x".repeat(33) }]) {
|
|
66
|
-
assert.deepEqual(await lookupPiCatalogName({ ...name, ...patch }), { classification: "unknown", modelId: "unknown" });
|
|
67
|
-
}
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
test("actual installed ESM catalog supplies immutable public names, not route claims", async () => {
|
|
71
|
-
const pi = import.meta.resolve("@earendil-works/pi-coding-agent");
|
|
72
|
-
assert.equal(findPackageJSON("@earendil-works/pi-ai", import.meta.url), findPackageJSON("@earendil-works/pi-ai", pi));
|
|
73
|
-
assert.ok(model);
|
|
74
|
-
const result = await lookupPiCatalogName(name);
|
|
75
|
-
assert.deepEqual(result, { classification: "catalog_public", modelId: model.id });
|
|
76
|
-
assert.ok(Object.isFrozen(result));
|
|
77
|
-
assert.strictEqual(await lookupPiCatalogName(name), result);
|
|
78
|
-
// Origin/endpoint assertions do not change privacy classification of a name.
|
|
79
|
-
// Neither a real catalog entry nor matching route strings establish dispatch.
|
|
80
|
-
for (const patch of [{ origin: "builtin" }, { origin: "unknown" }, { origin: "custom" },
|
|
81
|
-
{ baseUrl: "https://private.invalid" }, { api: "custom-api" }]) {
|
|
82
|
-
assert.strictEqual(await lookupPiCatalogName({ ...name, ...patch }), result);
|
|
83
|
-
}
|
|
84
|
-
assert.deepEqual(Object.keys(result), ["classification", "modelId"]);
|
|
85
|
-
assert.ok(!("modelVersion" in result));
|
|
86
|
-
});
|
|
87
|
-
|
|
88
|
-
test("catalog-public selections are counted without claiming actual route identity", async () => {
|
|
89
|
-
const catalogName = await lookupPiCatalogName(name);
|
|
90
|
-
const metrics = new RuntimeMetrics();
|
|
91
|
-
const missing = { state: "unavailable" } as const;
|
|
92
|
-
for (const [index, identity] of [catalogName, { ...name, origin: "builtin", api: model.api, baseUrl: model.baseUrl }].entries()) {
|
|
93
|
-
const record = {
|
|
94
|
-
kind: "final_assistant_response", responseId: String(index), identity, selectedModelId: model.id,
|
|
95
|
-
executor: "worker", provider: model.provider, modelFamily: "gpt", effort: "high", error: "none",
|
|
96
|
-
tokens: { input: missing, output: missing, cacheRead: missing, cacheWrite: missing },
|
|
97
|
-
responseHeadersMs: missing, fullResponseMs: missing,
|
|
98
|
-
};
|
|
99
|
-
assert.equal(metrics.record(record as FinalResponse), "recorded");
|
|
100
|
-
}
|
|
101
|
-
const [bucket] = metrics.snapshot();
|
|
102
|
-
assert.equal(bucket.selectedModelId, model.id);
|
|
103
|
-
assert.ok(!("modelId" in bucket));
|
|
104
|
-
assert.ok(!("route" in bucket));
|
|
105
|
-
assert.equal(bucket.responses, 2);
|
|
106
|
-
assert.equal(bucket.hostAgent, "pi");
|
|
107
|
-
});
|
|
108
|
-
|
|
109
|
-
test("selection classification rejects fabricated public labels and private IDs", async () => {
|
|
110
|
-
await lookupPiCatalogName(name);
|
|
111
|
-
assert.equal(classifyPiCatalogName({ ...name, modelId: "private-id", classification: "catalog_public" }).modelId, "custom");
|
|
112
|
-
assert.equal(classifyPiCatalogName({ ...name, provider: "anthropic" }).modelId, "custom");
|
|
113
|
-
});
|