@skill-harness/core 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/adapters/types.d.ts +30 -0
- package/dist/adjudication.d.ts +9 -1
- package/dist/adjudication.js +41 -33
- package/dist/arms.d.ts +51 -0
- package/dist/arms.js +255 -0
- package/dist/capture-trace-types.d.ts +2 -0
- package/dist/command-cost.d.ts +9 -0
- package/dist/command-cost.js +23 -0
- package/dist/comparison.d.ts +1 -0
- package/dist/comparison.js +9 -3
- package/dist/execution-trace.js +9 -1
- package/dist/grade.d.ts +2 -0
- package/dist/grade.js +2 -1
- package/dist/index.d.ts +14 -0
- package/dist/index.js +14 -0
- package/dist/journal.d.ts +2 -1
- package/dist/judge-agreement.d.ts +19 -0
- package/dist/judge-agreement.js +31 -0
- package/dist/judge-policy.js +5 -3
- package/dist/lift.js +1 -1
- package/dist/lint.js +1 -1
- package/dist/prompt-normalization.d.ts +6 -0
- package/dist/prompt-normalization.js +14 -0
- package/dist/provider-failure.d.ts +56 -0
- package/dist/provider-failure.js +98 -0
- package/dist/qualification-capture.d.ts +44 -0
- package/dist/qualification-capture.js +163 -0
- package/dist/qualification-config.d.ts +233 -0
- package/dist/qualification-config.js +694 -0
- package/dist/qualification-lock.d.ts +15 -0
- package/dist/qualification-lock.js +87 -0
- package/dist/qualification-oauth-directory.d.ts +79 -0
- package/dist/qualification-oauth-directory.js +444 -0
- package/dist/qualification-panel-store.d.ts +14 -0
- package/dist/qualification-panel-store.js +188 -0
- package/dist/qualification-panels.d.ts +99 -0
- package/dist/qualification-panels.js +91 -0
- package/dist/qualification-process.d.ts +13 -0
- package/dist/qualification-process.js +188 -0
- package/dist/qualification-runner.d.ts +203 -0
- package/dist/qualification-runner.js +2037 -0
- package/dist/qualification-store.d.ts +187 -0
- package/dist/qualification-store.js +833 -0
- package/dist/qualification-validation.d.ts +12 -0
- package/dist/qualification-validation.js +8 -0
- package/dist/regate.js +49 -10
- package/dist/regrade.d.ts +3 -1
- package/dist/regrade.js +62 -6
- package/dist/reps.d.ts +4 -2
- package/dist/reps.js +13 -4
- package/dist/rescore.js +9 -1
- package/dist/results.d.ts +99 -7
- package/dist/results.js +228 -10
- package/dist/run.d.ts +14 -0
- package/dist/run.js +260 -57
- package/dist/score.d.ts +4 -3
- package/dist/score.js +13 -4
- package/dist/screen.d.ts +31 -0
- package/dist/screen.js +105 -0
- package/dist/seeded.d.ts +12 -1
- package/dist/seeded.js +9 -2
- package/dist/sources.d.ts +7 -1
- package/dist/sources.js +13 -5
- package/dist/spec.d.ts +1 -1
- package/dist/spec.js +1 -1
- package/dist/stability.js +1 -1
- package/dist/trajectory-gates.d.ts +14 -2
- package/dist/trajectory-gates.js +26 -6
- package/dist/vote-panel.d.ts +33 -0
- package/dist/vote-panel.js +41 -0
- package/dist/workspace.js +34 -3
- package/package.json +3 -3
- package/schemas/qualification-cell-result-v1.schema.json +32 -0
- package/schemas/qualification-config-v1.schema.json +244 -0
- package/schemas/qualification-invocation-request-v1.schema.json +59 -0
- package/schemas/qualification-judge-panel-result-v1.schema.json +42 -0
- package/schemas/results-v2.schema.json +19 -21
- package/schemas/results-v3.schema.json +654 -0
- package/schemas/specification-v1.schema.json +1 -1
- package/schemas/trajectory-event-v1.schema.json +20 -1
|
@@ -0,0 +1,694 @@
|
|
|
1
|
+
import { execFileSync } from "node:child_process";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { closeSync, constants, fstatSync, lstatSync, openSync, readFileSync, realpathSync } from "node:fs";
|
|
4
|
+
import { isAbsolute, join } from "node:path";
|
|
5
|
+
import { QUALIFICATION_JUDGE_PANEL_POLICY } from "./qualification-panels.js";
|
|
6
|
+
export const QUALIFICATION_CONFIG_VERSION = "qualification-config-v1";
|
|
7
|
+
export const QUALIFICATION_RUNNER_VERSION = "qualification-runner-v1";
|
|
8
|
+
export const QUALIFICATION_REQUEST_VERSION = "qualification-invocation-request-v1";
|
|
9
|
+
export const QUALIFICATION_BOARD_VERSION = "qualification-board-v1";
|
|
10
|
+
export const QUALIFICATION_TERMINAL_RECEIPT_VERSION_V3 = "qualification-terminal-receipt-v3";
|
|
11
|
+
/** Historical configurations omit this field and retain the original closed policy. */
|
|
12
|
+
export const QUALIFICATION_OAUTH_DIRECTORY_POLICY_V1 = "qualification-oauth-directory-policy-v1";
|
|
13
|
+
/** New qualification configurations select this policy explicitly. */
|
|
14
|
+
export const QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2 = "qualification-oauth-directory-policy-v2";
|
|
15
|
+
export const PRINCIPAL_QUALIFICATION_PRODUCT_PIN = {
|
|
16
|
+
repository: "https://github.com/mojomanyana/principal-pi-skills",
|
|
17
|
+
commit: "a6596950d64a3a525f95329d5dbd3e38948be408",
|
|
18
|
+
tree: "960359d69deb6f216724b86e13eef67e2f6a6aa1",
|
|
19
|
+
package_sha256: "3677f76fb31dcaf6a28c9e8b9cf9d6358f998b78bd370f0a0f250d05e136fcc8",
|
|
20
|
+
package_bytes: 110168,
|
|
21
|
+
};
|
|
22
|
+
/** Immutable pi-daddy ledger-v3 producer merge head. */
|
|
23
|
+
export const PI_DADDY_QUALIFICATION_PRODUCER_PIN = {
|
|
24
|
+
repository: "https://github.com/mojomanyana/pi-daddy",
|
|
25
|
+
commit: "4a9524394ca995fd74ed9bbb836dc4e73cda3b8c",
|
|
26
|
+
tree: "7c006bff213142634f0f911ba9bd6add363ecaae",
|
|
27
|
+
version: "0.21.1",
|
|
28
|
+
ledger_version: 3,
|
|
29
|
+
ledger_schema_sha256: "64e3d875e74bc32fa43fb96892605548259cd16f6ed6678646d73cc56280c511",
|
|
30
|
+
};
|
|
31
|
+
export const QUALIFICATION_ACCOUNTING_POLICY = {
|
|
32
|
+
wave_a: { subject: 54, judge: 54 },
|
|
33
|
+
complete_program: { subject: 642, judge: 642 },
|
|
34
|
+
ceilings: { subject: 700, judge: 700 },
|
|
35
|
+
initial: { subject: 0, judge: 0 },
|
|
36
|
+
};
|
|
37
|
+
/** Prospective call ceilings for the universal two-plus-conditional-third panel. */
|
|
38
|
+
export const QUALIFICATION_PANEL_ACCOUNTING_POLICY = {
|
|
39
|
+
wave_a: { subject: 54, judge: 162 },
|
|
40
|
+
complete_program: { subject: 642, judge: 1926 },
|
|
41
|
+
ceilings: { subject: 700, judge: 2100 },
|
|
42
|
+
initial: { subject: 0, judge: 0 },
|
|
43
|
+
};
|
|
44
|
+
const SHA256_RE = /^[a-f0-9]{64}$/i;
|
|
45
|
+
const GIT_SHA_RE = /^[a-f0-9]{40}$/i;
|
|
46
|
+
const ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
|
|
47
|
+
const ENV_RE = /^[A-Z_][A-Z0-9_]*$/;
|
|
48
|
+
const PLACEHOLDER_RE = /\{([a-z_]+)\}/g;
|
|
49
|
+
const ALLOWED_ARGUMENT_PLACEHOLDERS = new Set(["input_path", "scenario_id", "invocation_id", "artifact_path"]);
|
|
50
|
+
/**
|
|
51
|
+
* Names that can alter provider credentials/routing or inject code into the child.
|
|
52
|
+
* Qualification children start from an allow-list regardless; this predicate also
|
|
53
|
+
* makes a dirty parent shell observable by NAME without ever retaining its value.
|
|
54
|
+
*/
|
|
55
|
+
export function isQualificationConflictingEnvironmentName(name) {
|
|
56
|
+
const upper = name.toUpperCase();
|
|
57
|
+
if (/^(?:LD_|DYLD_)/.test(upper))
|
|
58
|
+
return true;
|
|
59
|
+
if (upper.startsWith("PI_") && upper !== "PI_CODING_AGENT_DIR")
|
|
60
|
+
return true;
|
|
61
|
+
if (["NODE_OPTIONS", "BASH_ENV", "ENV", "CDPATH", "GIT_SSH_COMMAND", "SSH_ASKPASS"].includes(upper))
|
|
62
|
+
return true;
|
|
63
|
+
if (["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY"].includes(upper))
|
|
64
|
+
return true;
|
|
65
|
+
if (/^SKILL_(?:HARNESS|CHECK)_ALLOW_METERED_JUDGE$/.test(upper))
|
|
66
|
+
return true;
|
|
67
|
+
if (/(?:^|_)(?:API_KEY|ACCESS_KEY|SECRET_KEY|BEARER_TOKEN|AUTH_TOKEN|SESSION_TOKEN)$/.test(upper))
|
|
68
|
+
return true;
|
|
69
|
+
if (/^(?:HF_TOKEN|AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY|AWS_SESSION_TOKEN|AWS_BEARER_TOKEN_BEDROCK)$/.test(upper))
|
|
70
|
+
return true;
|
|
71
|
+
if (/(?:^|_)(?:BASE_URL|ENDPOINT_URL|RESOURCE_NAME|DEPLOYMENT_NAME_MAP|ORGANIZATION|ORG_ID|PROJECT|PROJECT_ID)$/.test(upper))
|
|
72
|
+
return true;
|
|
73
|
+
// Provider-prefixed ambient configuration is excluded as a class, not only
|
|
74
|
+
// for today's documented key names. That closes both credentials and routing
|
|
75
|
+
// knobs added by a future Pi/provider release without inheriting them first.
|
|
76
|
+
if (/^(?:OPENAI|AZURE_OPENAI|CODEX|CHATGPT|ANTHROPIC|ANT_LING|FIREWORKS|DEEPSEEK|OPENROUTER|XAI|GOOGLE|GEMINI|VERTEX|BEDROCK|AWS|CLOUDFLARE|TOGETHER|BASETEN|KIMI|MINIMAX|QWEN|XIAOMI|MISTRAL|GROQ|CEREBRAS|NVIDIA|ZAI|OPENCODE|RADIUS|VERCEL|AI_GATEWAY)_/.test(upper))
|
|
77
|
+
return true;
|
|
78
|
+
return false;
|
|
79
|
+
}
|
|
80
|
+
export function sanitizeQualificationEnvironment(parent, allowedNames, conflictPolicy) {
|
|
81
|
+
const unique = new Set();
|
|
82
|
+
for (const rawName of allowedNames) {
|
|
83
|
+
if (typeof rawName !== "string" || !ENV_RE.test(rawName))
|
|
84
|
+
throw new Error(`qualification environment name ${JSON.stringify(rawName)} is invalid`);
|
|
85
|
+
if (unique.has(rawName))
|
|
86
|
+
throw new Error(`qualification environment allowlist contains duplicate ${rawName}`);
|
|
87
|
+
if (isQualificationConflictingEnvironmentName(rawName))
|
|
88
|
+
throw new Error(`qualification environment allowlist must not allow ${rawName}`);
|
|
89
|
+
unique.add(rawName);
|
|
90
|
+
}
|
|
91
|
+
if (conflictPolicy !== "refuse" && conflictPolicy !== "remove-and-record") {
|
|
92
|
+
throw new Error(`qualification conflicting-parent-environment policy is unsupported: ${String(conflictPolicy)}`);
|
|
93
|
+
}
|
|
94
|
+
const conflicts = Object.keys(parent).filter(isQualificationConflictingEnvironmentName).sort();
|
|
95
|
+
if (conflictPolicy === "refuse" && conflicts.length > 0) {
|
|
96
|
+
throw new Error(`conflicting parent environment variables are present: ${conflicts.join(", ")}`);
|
|
97
|
+
}
|
|
98
|
+
const env = {};
|
|
99
|
+
for (const name of allowedNames) {
|
|
100
|
+
const value = parent[name];
|
|
101
|
+
if (value !== undefined)
|
|
102
|
+
env[name] = value;
|
|
103
|
+
}
|
|
104
|
+
return { env, removed_names: conflicts };
|
|
105
|
+
}
|
|
106
|
+
export function qualificationCanonicalJson(value) {
|
|
107
|
+
return stableStringify(value);
|
|
108
|
+
}
|
|
109
|
+
export function qualificationSha256(value) {
|
|
110
|
+
return createHash("sha256").update(value).digest("hex");
|
|
111
|
+
}
|
|
112
|
+
export function qualificationConfigDigest(config) {
|
|
113
|
+
return qualificationSha256(qualificationCanonicalJson(config));
|
|
114
|
+
}
|
|
115
|
+
export function qualificationOAuthDirectoryPolicy(config) {
|
|
116
|
+
return config.oauth_directory_policy ?? QUALIFICATION_OAUTH_DIRECTORY_POLICY_V1;
|
|
117
|
+
}
|
|
118
|
+
export function verifyQualificationPins(config) {
|
|
119
|
+
verifyQualificationExecutable(config.runner.executable);
|
|
120
|
+
for (const arm of config.arms) {
|
|
121
|
+
verifyQualificationExecutable(arm.executable);
|
|
122
|
+
for (const resource of arm.resources)
|
|
123
|
+
verifyQualificationResource(resource);
|
|
124
|
+
}
|
|
125
|
+
if (config.mode !== "production")
|
|
126
|
+
return;
|
|
127
|
+
if (process.platform !== "linux")
|
|
128
|
+
throw new Error("qualification production execution requires Linux process-occurrence and process-group identity support");
|
|
129
|
+
verifyRepository(config.product, "product");
|
|
130
|
+
verifyRepository(config.engine, "engine");
|
|
131
|
+
verifyRepository(config.producer, "producer");
|
|
132
|
+
verifyFileIdentity(config.product.package_path, config.product.package_sha256, config.product.package_bytes, "product package");
|
|
133
|
+
for (const name of ["core", "adapters", "cli", "meta"]) {
|
|
134
|
+
verifyFileIdentity(config.engine.package_paths[name], config.engine.package_sha256[name], undefined, `engine ${name} package`);
|
|
135
|
+
}
|
|
136
|
+
const producerPackagePath = join(config.producer.checkout_path, "packages", "pi-daddy", "package.json");
|
|
137
|
+
let producerPackage;
|
|
138
|
+
try {
|
|
139
|
+
producerPackage = JSON.parse(readFileSync(producerPackagePath, "utf8"));
|
|
140
|
+
}
|
|
141
|
+
catch {
|
|
142
|
+
throw new Error("qualification producer package.json is missing or invalid");
|
|
143
|
+
}
|
|
144
|
+
const producerManifest = object(producerPackage, "qualification producer package.json");
|
|
145
|
+
if (producerManifest.version !== config.producer.version)
|
|
146
|
+
throw new Error("qualification producer package version does not match its pin");
|
|
147
|
+
verifyFileIdentity(join(config.producer.checkout_path, "packages", "pi-daddy", "contracts", "ledger", "v3", "ledger-event.schema.json"), config.producer.ledger_schema_sha256, undefined, "producer ledger-v3 schema");
|
|
148
|
+
}
|
|
149
|
+
function verifyRepository(pin, label) {
|
|
150
|
+
let checkoutStat;
|
|
151
|
+
try {
|
|
152
|
+
checkoutStat = lstatSync(pin.checkout_path);
|
|
153
|
+
}
|
|
154
|
+
catch {
|
|
155
|
+
throw new Error(`qualification ${label} checkout is missing`);
|
|
156
|
+
}
|
|
157
|
+
if (checkoutStat.isSymbolicLink() || !checkoutStat.isDirectory())
|
|
158
|
+
throw new Error(`qualification ${label} checkout must be a regular non-symlink directory`);
|
|
159
|
+
let head, tree, dirty, remote;
|
|
160
|
+
try {
|
|
161
|
+
const gitOptions = { encoding: "utf8", timeout: 30_000, killSignal: "SIGKILL" };
|
|
162
|
+
head = execFileSync("git", ["-C", pin.checkout_path, "rev-parse", "HEAD"], gitOptions).trim();
|
|
163
|
+
tree = execFileSync("git", ["-C", pin.checkout_path, "rev-parse", "HEAD^{tree}"], gitOptions).trim();
|
|
164
|
+
dirty = execFileSync("git", ["-C", pin.checkout_path, "status", "--porcelain=v1", "--untracked-files=all"], gitOptions).trim();
|
|
165
|
+
remote = execFileSync("git", ["-C", pin.checkout_path, "remote", "get-url", "origin"], gitOptions).trim();
|
|
166
|
+
}
|
|
167
|
+
catch {
|
|
168
|
+
throw new Error(`qualification ${label} checkout is not a readable pinned Git repository`);
|
|
169
|
+
}
|
|
170
|
+
const finalStat = lstatSync(pin.checkout_path);
|
|
171
|
+
if (finalStat.isSymbolicLink() || finalStat.dev !== checkoutStat.dev || finalStat.ino !== checkoutStat.ino)
|
|
172
|
+
throw new Error(`qualification ${label} checkout identity changed while it was verified`);
|
|
173
|
+
if (head !== pin.commit || tree !== pin.tree)
|
|
174
|
+
throw new Error(`qualification ${label} checkout commit/tree does not match its pin`);
|
|
175
|
+
if (dirty)
|
|
176
|
+
throw new Error(`qualification ${label} checkout must be clean`);
|
|
177
|
+
if (repositorySlug(remote) !== repositorySlug(pin.repository))
|
|
178
|
+
throw new Error(`qualification ${label} checkout origin does not match its repository pin`);
|
|
179
|
+
}
|
|
180
|
+
function repositorySlug(value) {
|
|
181
|
+
const match = /github\.com(?::|\/)([^/]+\/[^/#]+?)(?:\.git)?$/i.exec(value.replace(/^git\+/, ""));
|
|
182
|
+
if (!match)
|
|
183
|
+
throw new Error(`qualification repository URL is not a pinned GitHub repository: ${value}`);
|
|
184
|
+
return match[1].replace(/\.git$/i, "").toLowerCase();
|
|
185
|
+
}
|
|
186
|
+
function readPinnedRegularFile(path, label) {
|
|
187
|
+
let fd;
|
|
188
|
+
try {
|
|
189
|
+
fd = openSync(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
|
|
190
|
+
const stat = fstatSync(fd);
|
|
191
|
+
const pathStat = lstatSync(path);
|
|
192
|
+
if (!stat.isFile() || pathStat.isSymbolicLink() || !pathStat.isFile() || stat.dev !== pathStat.dev || stat.ino !== pathStat.ino) {
|
|
193
|
+
throw new Error(`qualification ${label} must be a stable regular non-symlink file`);
|
|
194
|
+
}
|
|
195
|
+
const bytes = readFileSync(fd);
|
|
196
|
+
const resolved = realpathSync(path);
|
|
197
|
+
const finalStat = lstatSync(path);
|
|
198
|
+
if (finalStat.isSymbolicLink() || finalStat.dev !== stat.dev || finalStat.ino !== stat.ino)
|
|
199
|
+
throw new Error(`qualification ${label} path changed while it was verified`);
|
|
200
|
+
return { bytes, stat, realpath: resolved };
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
if (error instanceof Error && error.message.startsWith(`qualification ${label}`))
|
|
204
|
+
throw error;
|
|
205
|
+
throw new Error(`qualification ${label} is missing or unreadable: ${path}`);
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
if (fd !== undefined)
|
|
209
|
+
closeSync(fd);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
function verifyFileIdentity(path, expectedSha256, expectedBytes, label) {
|
|
213
|
+
const opened = readPinnedRegularFile(path, label);
|
|
214
|
+
if (expectedBytes !== undefined && opened.stat.size !== expectedBytes)
|
|
215
|
+
throw new Error(`qualification ${label} byte size does not match its pin`);
|
|
216
|
+
const actual = qualificationSha256(opened.bytes);
|
|
217
|
+
if (actual !== expectedSha256)
|
|
218
|
+
throw new Error(`qualification ${label} digest does not match its pin`);
|
|
219
|
+
}
|
|
220
|
+
export function verifyQualificationResource(pin) {
|
|
221
|
+
const path = absolutePath(pin.path, "qualification resource.path");
|
|
222
|
+
const expected = digest(pin.sha256, "qualification resource.sha256");
|
|
223
|
+
const opened = readPinnedRegularFile(path, "resource");
|
|
224
|
+
const actual = qualificationSha256(opened.bytes);
|
|
225
|
+
if (actual !== expected)
|
|
226
|
+
throw new Error(`qualification resource digest mismatch for ${path}: expected ${expected}, got ${actual}`);
|
|
227
|
+
return { realpath: opened.realpath, bytes: opened.stat.size, sha256: actual };
|
|
228
|
+
}
|
|
229
|
+
export function verifyQualificationExecutable(pin) {
|
|
230
|
+
const parsed = parseExecutablePin(pin, "executable");
|
|
231
|
+
const opened = readPinnedRegularFile(parsed.path, "executable");
|
|
232
|
+
if (process.platform !== "win32" && (opened.stat.mode & 0o111) === 0)
|
|
233
|
+
throw new Error(`qualification executable is not executable: ${parsed.path}`);
|
|
234
|
+
const digest = qualificationSha256(opened.bytes);
|
|
235
|
+
if (digest !== parsed.sha256)
|
|
236
|
+
throw new Error(`qualification executable digest mismatch for ${parsed.path}: expected ${parsed.sha256}, got ${digest}`);
|
|
237
|
+
return { realpath: opened.realpath, bytes: opened.stat.size, sha256: digest, device: opened.stat.dev, inode: opened.stat.ino, mtime_ms: opened.stat.mtimeMs };
|
|
238
|
+
}
|
|
239
|
+
export function parseQualificationConfig(value) {
|
|
240
|
+
const root = object(value, "qualification configuration");
|
|
241
|
+
exactKeys(root, ["schema_version", "mode", "product", "engine", "producer", "runner", "accounting", "arms"], "qualification configuration", ["oauth_directory_policy", "terminal_receipt_version", "judge_panel", "board"]);
|
|
242
|
+
if (root.schema_version !== QUALIFICATION_CONFIG_VERSION)
|
|
243
|
+
throw new Error(`qualification configuration schema_version must be ${QUALIFICATION_CONFIG_VERSION}`);
|
|
244
|
+
const oauthDirectoryPolicy = Object.hasOwn(root, "oauth_directory_policy")
|
|
245
|
+
? enumValue(root.oauth_directory_policy, [QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2], "qualification oauth_directory_policy")
|
|
246
|
+
: undefined;
|
|
247
|
+
const terminalReceiptVersion = Object.hasOwn(root, "terminal_receipt_version")
|
|
248
|
+
? enumValue(root.terminal_receipt_version, [QUALIFICATION_TERMINAL_RECEIPT_VERSION_V3], "qualification terminal_receipt_version")
|
|
249
|
+
: undefined;
|
|
250
|
+
const judgePanel = Object.hasOwn(root, "judge_panel")
|
|
251
|
+
? parseCanonicalJudgePanelPolicy(root.judge_panel)
|
|
252
|
+
: undefined;
|
|
253
|
+
const board = Object.hasOwn(root, "board") ? parseQualificationBoard(root.board) : undefined;
|
|
254
|
+
if (terminalReceiptVersion && oauthDirectoryPolicy !== QUALIFICATION_OAUTH_DIRECTORY_POLICY_V2)
|
|
255
|
+
throw new Error("qualification terminal receipt v3 requires oauth_directory_policy v2");
|
|
256
|
+
if (judgePanel && terminalReceiptVersion !== QUALIFICATION_TERMINAL_RECEIPT_VERSION_V3)
|
|
257
|
+
throw new Error("qualification judge_panel requires terminal_receipt_version v3 and oauth_directory_policy v2");
|
|
258
|
+
if (judgePanel && !board)
|
|
259
|
+
throw new Error("qualification judge_panel requires an externally approved read-only board");
|
|
260
|
+
if (!judgePanel && board)
|
|
261
|
+
throw new Error("qualification board requires judge_panel");
|
|
262
|
+
const mode = enumValue(root.mode, ["production", "test"], "qualification configuration mode");
|
|
263
|
+
const productObject = object(root.product, "qualification configuration product");
|
|
264
|
+
exactKeys(productObject, ["repository", "commit", "tree", "checkout_path", "package_path", "package_sha256", "package_bytes"], "qualification configuration product");
|
|
265
|
+
const product = {
|
|
266
|
+
...parseRepositoryPin(productObject, "product"),
|
|
267
|
+
checkout_path: absolutePath(productObject.checkout_path, "product.checkout_path"),
|
|
268
|
+
package_path: absolutePath(productObject.package_path, "product.package_path"),
|
|
269
|
+
package_sha256: digest(productObject.package_sha256, "product.package_sha256"),
|
|
270
|
+
package_bytes: positiveInteger(productObject.package_bytes, "product.package_bytes"),
|
|
271
|
+
};
|
|
272
|
+
const engineObject = object(root.engine, "qualification configuration engine");
|
|
273
|
+
exactKeys(engineObject, ["repository", "commit", "tree", "checkout_path", "package_paths", "package_sha256"], "qualification configuration engine");
|
|
274
|
+
const packageObject = object(engineObject.package_sha256, "engine.package_sha256");
|
|
275
|
+
exactKeys(packageObject, ["core", "adapters", "cli", "meta"], "engine.package_sha256");
|
|
276
|
+
const packagePaths = object(engineObject.package_paths, "engine.package_paths");
|
|
277
|
+
exactKeys(packagePaths, ["core", "adapters", "cli", "meta"], "engine.package_paths");
|
|
278
|
+
const engine = {
|
|
279
|
+
...parseRepositoryPin(engineObject, "engine"),
|
|
280
|
+
checkout_path: absolutePath(engineObject.checkout_path, "engine.checkout_path"),
|
|
281
|
+
package_paths: {
|
|
282
|
+
core: absolutePath(packagePaths.core, "engine.package_paths.core"),
|
|
283
|
+
adapters: absolutePath(packagePaths.adapters, "engine.package_paths.adapters"),
|
|
284
|
+
cli: absolutePath(packagePaths.cli, "engine.package_paths.cli"),
|
|
285
|
+
meta: absolutePath(packagePaths.meta, "engine.package_paths.meta"),
|
|
286
|
+
},
|
|
287
|
+
package_sha256: {
|
|
288
|
+
core: digest(packageObject.core, "engine.package_sha256.core"),
|
|
289
|
+
adapters: digest(packageObject.adapters, "engine.package_sha256.adapters"),
|
|
290
|
+
cli: digest(packageObject.cli, "engine.package_sha256.cli"),
|
|
291
|
+
meta: digest(packageObject.meta, "engine.package_sha256.meta"),
|
|
292
|
+
},
|
|
293
|
+
};
|
|
294
|
+
const producerObject = object(root.producer, "qualification configuration producer");
|
|
295
|
+
exactKeys(producerObject, ["repository", "commit", "tree", "checkout_path", "version", "ledger_version", "ledger_schema_sha256"], "qualification configuration producer");
|
|
296
|
+
if (producerObject.ledger_version !== 3)
|
|
297
|
+
throw new Error("qualification producer.ledger_version must be 3");
|
|
298
|
+
const producer = {
|
|
299
|
+
...parseRepositoryPin(producerObject, "producer"),
|
|
300
|
+
checkout_path: absolutePath(producerObject.checkout_path, "producer.checkout_path"),
|
|
301
|
+
version: nonEmpty(producerObject.version, "producer.version"),
|
|
302
|
+
ledger_version: 3,
|
|
303
|
+
ledger_schema_sha256: digest(producerObject.ledger_schema_sha256, "producer.ledger_schema_sha256"),
|
|
304
|
+
};
|
|
305
|
+
if (mode === "production") {
|
|
306
|
+
const productIdentity = {
|
|
307
|
+
repository: product.repository, commit: product.commit, tree: product.tree, package_sha256: product.package_sha256, package_bytes: product.package_bytes,
|
|
308
|
+
};
|
|
309
|
+
if (qualificationCanonicalJson(productIdentity) !== qualificationCanonicalJson(PRINCIPAL_QUALIFICATION_PRODUCT_PIN)) {
|
|
310
|
+
throw new Error("qualification production product pin does not match the authorized principal-pi-skills commit/tree/package identity");
|
|
311
|
+
}
|
|
312
|
+
const producerIdentity = {
|
|
313
|
+
repository: producer.repository, commit: producer.commit, tree: producer.tree, version: producer.version, ledger_version: producer.ledger_version,
|
|
314
|
+
ledger_schema_sha256: producer.ledger_schema_sha256,
|
|
315
|
+
};
|
|
316
|
+
if (qualificationCanonicalJson(producerIdentity) !== qualificationCanonicalJson(PI_DADDY_QUALIFICATION_PRODUCER_PIN)) {
|
|
317
|
+
throw new Error("qualification production producer pin does not match the authorized pi-daddy ledger-v3 identity");
|
|
318
|
+
}
|
|
319
|
+
if (repositorySlug(engine.repository) !== "mojomanyana/skill-harness")
|
|
320
|
+
throw new Error("qualification production engine repository must be mojomanyana/skill-harness");
|
|
321
|
+
}
|
|
322
|
+
const runnerObject = object(root.runner, "qualification configuration runner");
|
|
323
|
+
exactKeys(runnerObject, ["version", "executable", "conflicting_parent_environment"], "qualification configuration runner");
|
|
324
|
+
if (runnerObject.version !== QUALIFICATION_RUNNER_VERSION)
|
|
325
|
+
throw new Error(`qualification runner.version must be ${QUALIFICATION_RUNNER_VERSION}`);
|
|
326
|
+
const conflictPolicy = enumValue(runnerObject.conflicting_parent_environment, ["refuse", "remove-and-record"], "runner.conflicting_parent_environment");
|
|
327
|
+
const accounting = object(root.accounting, "qualification accounting");
|
|
328
|
+
const expectedAccounting = judgePanel
|
|
329
|
+
? QUALIFICATION_PANEL_ACCOUNTING_POLICY
|
|
330
|
+
: QUALIFICATION_ACCOUNTING_POLICY;
|
|
331
|
+
if (qualificationCanonicalJson(accounting) !== qualificationCanonicalJson(expectedAccounting)) {
|
|
332
|
+
throw new Error(judgePanel
|
|
333
|
+
? "qualification accounting must equal the canonical panel policy (Wave A 54/162, complete 642/1926, ceilings 700/2100, initial 0/0)"
|
|
334
|
+
: "qualification accounting must equal the canonical policy (Wave A 54/54, complete 642/642, ceilings 700/700, initial 0/0)");
|
|
335
|
+
}
|
|
336
|
+
if (!Array.isArray(root.arms) || root.arms.length === 0)
|
|
337
|
+
throw new Error("qualification configuration arms must be a non-empty array");
|
|
338
|
+
const arms = root.arms.map((entry, index) => parseArm(entry, mode, index));
|
|
339
|
+
const ids = new Set();
|
|
340
|
+
const tuples = new Set();
|
|
341
|
+
for (const arm of arms) {
|
|
342
|
+
if (ids.has(arm.id))
|
|
343
|
+
throw new Error(`duplicate arm id ${arm.id} in qualification configuration`);
|
|
344
|
+
ids.add(arm.id);
|
|
345
|
+
const tuple = `${arm.kind}\0${arm.provider}\0${arm.model}`;
|
|
346
|
+
if (tuples.has(tuple))
|
|
347
|
+
throw new Error(`duplicate arm ${arm.kind}:${arm.provider}:${arm.model} in qualification configuration`);
|
|
348
|
+
tuples.add(tuple);
|
|
349
|
+
}
|
|
350
|
+
if (!arms.some((arm) => arm.kind === "subject") || !arms.some((arm) => arm.kind === "judge")) {
|
|
351
|
+
throw new Error("qualification configuration requires at least one subject arm and one judge arm");
|
|
352
|
+
}
|
|
353
|
+
if (board) {
|
|
354
|
+
for (const cell of board.cells) {
|
|
355
|
+
if (!arms.some((arm) => arm.id === cell.subject_arm && arm.kind === "subject"))
|
|
356
|
+
throw new Error(`qualification board cell ${cell.id} names undeclared subject arm ${cell.subject_arm}`);
|
|
357
|
+
for (const judgeArm of cell.judge_arms)
|
|
358
|
+
if (!arms.some((arm) => arm.id === judgeArm && arm.kind === "judge"))
|
|
359
|
+
throw new Error(`qualification board cell ${cell.id} names undeclared judge arm ${judgeArm}`);
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
return {
|
|
363
|
+
schema_version: QUALIFICATION_CONFIG_VERSION,
|
|
364
|
+
...(oauthDirectoryPolicy ? { oauth_directory_policy: oauthDirectoryPolicy } : {}),
|
|
365
|
+
...(terminalReceiptVersion ? { terminal_receipt_version: terminalReceiptVersion } : {}),
|
|
366
|
+
...(judgePanel ? { judge_panel: judgePanel } : {}),
|
|
367
|
+
...(board ? { board } : {}),
|
|
368
|
+
mode,
|
|
369
|
+
product,
|
|
370
|
+
engine,
|
|
371
|
+
producer,
|
|
372
|
+
runner: {
|
|
373
|
+
version: QUALIFICATION_RUNNER_VERSION,
|
|
374
|
+
executable: parseExecutablePin(runnerObject.executable, "runner.executable"),
|
|
375
|
+
conflicting_parent_environment: conflictPolicy,
|
|
376
|
+
},
|
|
377
|
+
accounting: structuredClone(expectedAccounting),
|
|
378
|
+
arms,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
export function parseQualificationRequest(value, config) {
|
|
382
|
+
const root = object(value, "qualification invocation request");
|
|
383
|
+
exactKeys(root, ["schema_version", "measurement_identity_sha256", "invocation_id", "continuation_authority_sha256", "continuation_authority_expires_at", "scenario", "role", "counts_as_measurement", "arms", "selected_arm", "repetition"], "qualification invocation request", ["panel"]);
|
|
384
|
+
if (root.schema_version !== QUALIFICATION_REQUEST_VERSION)
|
|
385
|
+
throw new Error(`qualification request schema_version must be ${QUALIFICATION_REQUEST_VERSION}`);
|
|
386
|
+
const scenarioObject = object(root.scenario, "qualification request scenario");
|
|
387
|
+
exactKeys(scenarioObject, ["id", "version", "stimulus_sha256", "rubric_sha256", "input_path", "input_sha256", "working_directory"], "qualification request scenario");
|
|
388
|
+
const armsObject = object(root.arms, "qualification request arms");
|
|
389
|
+
exactKeys(armsObject, ["subject", "judge"], "qualification request arms");
|
|
390
|
+
const role = enumValue(root.role, ["holdout-author", "holdout-reviewer", "subject", "judge", "calibration", "canary"], "qualification role");
|
|
391
|
+
if (typeof root.counts_as_measurement !== "boolean")
|
|
392
|
+
throw new Error("qualification counts_as_measurement must be boolean");
|
|
393
|
+
const requiredMeasurement = role === "subject" || role === "judge";
|
|
394
|
+
if (root.counts_as_measurement !== requiredMeasurement) {
|
|
395
|
+
throw new Error(`qualification ${role} calls must be recorded as ${requiredMeasurement ? "measurement" : "non-measurement"}`);
|
|
396
|
+
}
|
|
397
|
+
const subjectArm = identifier(armsObject.subject, "qualification request subject arm");
|
|
398
|
+
const judgeArm = identifier(armsObject.judge, "qualification request judge arm");
|
|
399
|
+
const selectedArm = identifier(root.selected_arm, "qualification request selected arm");
|
|
400
|
+
const selected = config.arms.find((arm) => arm.id === selectedArm);
|
|
401
|
+
if (!selected)
|
|
402
|
+
throw new Error(`qualification request selected arm ${selectedArm} is not declared`);
|
|
403
|
+
if (!config.arms.some((arm) => arm.id === subjectArm && arm.kind === "subject"))
|
|
404
|
+
throw new Error(`qualification request subject arm ${subjectArm} is not a declared subject arm`);
|
|
405
|
+
if (!config.arms.some((arm) => arm.id === judgeArm && arm.kind === "judge"))
|
|
406
|
+
throw new Error(`qualification request judge arm ${judgeArm} is not a declared judge arm`);
|
|
407
|
+
if (role === "subject" && (selectedArm !== subjectArm || selected.kind !== "subject"))
|
|
408
|
+
throw new Error("qualification subject role must select the request's subject arm");
|
|
409
|
+
if (role === "judge" && (selectedArm !== judgeArm || selected.kind !== "judge"))
|
|
410
|
+
throw new Error("qualification judge role must select the request's judge arm");
|
|
411
|
+
if (![subjectArm, judgeArm].includes(selectedArm))
|
|
412
|
+
throw new Error("qualification selected arm must be the bound subject or judge arm");
|
|
413
|
+
if (config.board && role === "subject" && !config.board.cells.some((cell) => boardCellMatchesRequest(cell, root, scenarioObject, subjectArm) &&
|
|
414
|
+
cell.subject_input_sha256 === String(scenarioObject.input_sha256).toLowerCase() &&
|
|
415
|
+
cell.panels.some((panel) => panel.repetition === root.repetition))) {
|
|
416
|
+
throw new Error("qualification subject request identity/input is not declared by the approved board");
|
|
417
|
+
}
|
|
418
|
+
let panel;
|
|
419
|
+
if (Object.hasOwn(root, "panel")) {
|
|
420
|
+
if (!config.judge_panel)
|
|
421
|
+
throw new Error("qualification request panel metadata requires a judge_panel configuration");
|
|
422
|
+
if (role !== "judge")
|
|
423
|
+
throw new Error("qualification request panel metadata is valid only for judge role");
|
|
424
|
+
const panelObject = object(root.panel, "qualification request panel");
|
|
425
|
+
exactKeys(panelObject, ["id", "member_ordinal", "subject_invocation_id", "subject_artifact_sha256"], "qualification request panel");
|
|
426
|
+
const ordinal = boundedInteger(panelObject.member_ordinal, 1, 3, "qualification request panel.member_ordinal");
|
|
427
|
+
panel = {
|
|
428
|
+
id: identifier(panelObject.id, "qualification request panel.id"),
|
|
429
|
+
member_ordinal: ordinal,
|
|
430
|
+
subject_invocation_id: identifier(panelObject.subject_invocation_id, "qualification request panel.subject_invocation_id"),
|
|
431
|
+
subject_artifact_sha256: digest(panelObject.subject_artifact_sha256, "qualification request panel.subject_artifact_sha256"),
|
|
432
|
+
};
|
|
433
|
+
const cell = config.board?.cells.find((candidate) => candidate.panels.some((declared) => declared.id === panel.id && declared.repetition === root.repetition));
|
|
434
|
+
if (!cell || !boardCellMatchesRequest(cell, root, scenarioObject, subjectArm) || cell.judge_arms[ordinal - 1] !== selectedArm)
|
|
435
|
+
throw new Error("qualification request panel is not declared for this board identity/scenario/arm/repetition/judge ordinal");
|
|
436
|
+
}
|
|
437
|
+
else if (config.judge_panel && role === "judge") {
|
|
438
|
+
throw new Error("qualification judge request under judge_panel configuration requires panel metadata");
|
|
439
|
+
}
|
|
440
|
+
const inputPath = absolutePath(scenarioObject.input_path, "scenario.input_path");
|
|
441
|
+
const workingDirectory = absolutePath(scenarioObject.working_directory, "scenario.working_directory");
|
|
442
|
+
return {
|
|
443
|
+
schema_version: QUALIFICATION_REQUEST_VERSION,
|
|
444
|
+
measurement_identity_sha256: digest(root.measurement_identity_sha256, "measurement_identity_sha256"),
|
|
445
|
+
invocation_id: identifier(root.invocation_id, "invocation_id"),
|
|
446
|
+
continuation_authority_sha256: digest(root.continuation_authority_sha256, "continuation_authority_sha256"),
|
|
447
|
+
continuation_authority_expires_at: utcTimestamp(root.continuation_authority_expires_at, "continuation_authority_expires_at"),
|
|
448
|
+
scenario: {
|
|
449
|
+
id: identifier(scenarioObject.id, "scenario.id"),
|
|
450
|
+
version: identifier(scenarioObject.version, "scenario.version"),
|
|
451
|
+
stimulus_sha256: digest(scenarioObject.stimulus_sha256, "scenario.stimulus_sha256"),
|
|
452
|
+
rubric_sha256: digest(scenarioObject.rubric_sha256, "scenario.rubric_sha256"),
|
|
453
|
+
input_path: inputPath,
|
|
454
|
+
input_sha256: digest(scenarioObject.input_sha256, "scenario.input_sha256"),
|
|
455
|
+
working_directory: workingDirectory,
|
|
456
|
+
},
|
|
457
|
+
role,
|
|
458
|
+
counts_as_measurement: root.counts_as_measurement,
|
|
459
|
+
arms: { subject: subjectArm, judge: judgeArm },
|
|
460
|
+
selected_arm: selectedArm,
|
|
461
|
+
repetition: nonNegativeInteger(root.repetition, "repetition"),
|
|
462
|
+
...(panel ? { panel } : {}),
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
function parseArm(value, mode, index) {
|
|
466
|
+
const ctx = `qualification arm ${index}`;
|
|
467
|
+
const arm = object(value, ctx);
|
|
468
|
+
exactKeys(arm, ["id", "kind", "provider", "model", "authentication", "executable", "resources", "arguments", "allowed_environment_names", "timeout_ms", "output_limit_bytes", "artifact", "fallback", "metered_override"], ctx);
|
|
469
|
+
const kind = enumValue(arm.kind, ["subject", "judge"], `${ctx}.kind`);
|
|
470
|
+
const provider = nonEmpty(arm.provider, `${ctx}.provider`);
|
|
471
|
+
const model = nonEmpty(arm.model, `${ctx}.model`);
|
|
472
|
+
if (provider === "openai")
|
|
473
|
+
throw new Error(`${ctx} direct openai provider is prohibited`);
|
|
474
|
+
const authentication = enumValue(arm.authentication, ["chatgpt-oauth", "test-oauth"], `${ctx}.authentication`);
|
|
475
|
+
if (mode === "production" && (provider !== "openai-codex" || authentication !== "chatgpt-oauth")) {
|
|
476
|
+
throw new Error(`${ctx} production qualification requires exact provider openai-codex and ChatGPT OAuth`);
|
|
477
|
+
}
|
|
478
|
+
const allowedProductionModels = kind === "subject" ? new Set(["gpt-5.6-luna", "gpt-5.6-terra"]) : new Set(["gpt-5.6-sol"]);
|
|
479
|
+
if (mode === "production" && !allowedProductionModels.has(model)) {
|
|
480
|
+
throw new Error(`${ctx} production ${kind} model must be an exact Principal qualification model (${[...allowedProductionModels].join(", ")}); provider-qualified aliases and fallbacks are prohibited`);
|
|
481
|
+
}
|
|
482
|
+
if (mode === "test" && (provider !== "fake" || authentication !== "test-oauth")) {
|
|
483
|
+
throw new Error(`${ctx} test configuration permits only the inert fake provider and test-oauth metadata`);
|
|
484
|
+
}
|
|
485
|
+
if (arm.fallback !== false)
|
|
486
|
+
throw new Error(`${ctx}.fallback must be false`);
|
|
487
|
+
if (arm.metered_override !== false)
|
|
488
|
+
throw new Error(`${ctx}.metered_override must be false`);
|
|
489
|
+
if (!Array.isArray(arm.resources))
|
|
490
|
+
throw new Error(`${ctx}.resources must be an array`);
|
|
491
|
+
const resources = arm.resources.map((entry, resourceIndex) => {
|
|
492
|
+
const resource = object(entry, `${ctx}.resources[${resourceIndex}]`);
|
|
493
|
+
exactKeys(resource, ["kind", "path", "sha256"], `${ctx}.resources[${resourceIndex}]`);
|
|
494
|
+
return {
|
|
495
|
+
kind: enumValue(resource.kind, ["extension", "skill", "system-prompt"], `${ctx}.resources[${resourceIndex}].kind`),
|
|
496
|
+
path: absolutePath(resource.path, `${ctx}.resources[${resourceIndex}].path`),
|
|
497
|
+
sha256: digest(resource.sha256, `${ctx}.resources[${resourceIndex}].sha256`),
|
|
498
|
+
};
|
|
499
|
+
});
|
|
500
|
+
const resourceKeys = resources.map((resource) => `${resource.kind}\0${resource.path}`);
|
|
501
|
+
if (new Set(resourceKeys).size !== resourceKeys.length)
|
|
502
|
+
throw new Error(`${ctx}.resources contains a duplicate kind/path`);
|
|
503
|
+
if (!Array.isArray(arm.arguments) || arm.arguments.some((entry) => typeof entry !== "string"))
|
|
504
|
+
throw new Error(`${ctx}.arguments must be an array of strings`);
|
|
505
|
+
const argumentsList = arm.arguments;
|
|
506
|
+
if (argumentsList.some((entry) => entry.startsWith("-"))) {
|
|
507
|
+
throw new Error(`${ctx}.arguments must be positional inputs only; option-looking arguments are runner-owned`);
|
|
508
|
+
}
|
|
509
|
+
const placeholders = argumentsList.flatMap((entry) => [...entry.matchAll(PLACEHOLDER_RE)].map((match) => match[1]));
|
|
510
|
+
const unknownPlaceholder = placeholders.find((name) => !ALLOWED_ARGUMENT_PLACEHOLDERS.has(name));
|
|
511
|
+
if (unknownPlaceholder)
|
|
512
|
+
throw new Error(`${ctx}.arguments contains unknown placeholder {${unknownPlaceholder}}`);
|
|
513
|
+
if (!placeholders.includes("input_path"))
|
|
514
|
+
throw new Error(`${ctx}.arguments must contain {input_path}`);
|
|
515
|
+
if (!Array.isArray(arm.allowed_environment_names))
|
|
516
|
+
throw new Error(`${ctx}.allowed_environment_names must be an array`);
|
|
517
|
+
// Validation is intentionally shared with launch-time sanitation, including
|
|
518
|
+
// the sensitive-name and duplicate guards.
|
|
519
|
+
sanitizeQualificationEnvironment({}, arm.allowed_environment_names, "remove-and-record");
|
|
520
|
+
if (mode === "production" && !arm.allowed_environment_names.some((name) => name === "HOME" || name === "PI_CODING_AGENT_DIR")) {
|
|
521
|
+
throw new Error(`${ctx}.allowed_environment_names must include HOME or PI_CODING_AGENT_DIR for the dedicated OAuth credential store`);
|
|
522
|
+
}
|
|
523
|
+
const artifact = object(arm.artifact, `${ctx}.artifact`);
|
|
524
|
+
exactKeys(artifact, ["type", "relative_path_template"], `${ctx}.artifact`);
|
|
525
|
+
if (artifact.type !== "pi-jsonl")
|
|
526
|
+
throw new Error(`${ctx}.artifact.type must be pi-jsonl`);
|
|
527
|
+
if (artifact.relative_path_template !== "artifacts/{invocation_id}.jsonl") {
|
|
528
|
+
throw new Error(`${ctx}.artifact.relative_path_template must be artifacts/{invocation_id}.jsonl`);
|
|
529
|
+
}
|
|
530
|
+
return {
|
|
531
|
+
id: identifier(arm.id, `${ctx}.id`),
|
|
532
|
+
kind,
|
|
533
|
+
provider,
|
|
534
|
+
model,
|
|
535
|
+
authentication,
|
|
536
|
+
executable: parseExecutablePin(arm.executable, `${ctx}.executable`),
|
|
537
|
+
resources,
|
|
538
|
+
arguments: [...argumentsList],
|
|
539
|
+
allowed_environment_names: [...arm.allowed_environment_names],
|
|
540
|
+
timeout_ms: boundedInteger(arm.timeout_ms, 1, 24 * 60 * 60 * 1000, `${ctx}.timeout_ms`),
|
|
541
|
+
output_limit_bytes: boundedInteger(arm.output_limit_bytes, 1, 100 * 1024 * 1024, `${ctx}.output_limit_bytes`),
|
|
542
|
+
artifact: { type: "pi-jsonl", relative_path_template: "artifacts/{invocation_id}.jsonl" },
|
|
543
|
+
fallback: false,
|
|
544
|
+
metered_override: false,
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
function boardCellMatchesRequest(cell, request, scenario, subjectArm) {
|
|
548
|
+
return cell.measurement_identity_sha256 === String(request.measurement_identity_sha256).toLowerCase() &&
|
|
549
|
+
cell.scenario_id === scenario.id && cell.scenario_version === scenario.version &&
|
|
550
|
+
cell.stimulus_sha256 === String(scenario.stimulus_sha256).toLowerCase() &&
|
|
551
|
+
cell.rubric_sha256 === String(scenario.rubric_sha256).toLowerCase() && cell.subject_arm === subjectArm;
|
|
552
|
+
}
|
|
553
|
+
function parseQualificationBoard(value) {
|
|
554
|
+
const board = object(value, "qualification board");
|
|
555
|
+
exactKeys(board, ["schema_version", "read_only", "cells"], "qualification board");
|
|
556
|
+
if (board.schema_version !== QUALIFICATION_BOARD_VERSION || board.read_only !== true)
|
|
557
|
+
throw new Error("qualification board must be qualification-board-v1 with read_only true");
|
|
558
|
+
if (!Array.isArray(board.cells) || board.cells.length === 0)
|
|
559
|
+
throw new Error("qualification board cells must be a non-empty array");
|
|
560
|
+
const cells = board.cells.map((entry, index) => {
|
|
561
|
+
const cell = object(entry, `qualification board cell ${index}`);
|
|
562
|
+
exactKeys(cell, ["id", "scenario_id", "measurement_identity_sha256", "scenario_version", "stimulus_sha256", "rubric_sha256", "subject_input_sha256", "subject_arm", "judge_arms", "panels", "critical", "pass_threshold"], `qualification board cell ${index}`);
|
|
563
|
+
if (!Array.isArray(cell.panels) || cell.panels.length === 0)
|
|
564
|
+
throw new Error(`qualification board cell ${index}.panels must be non-empty`);
|
|
565
|
+
const panels = cell.panels.map((raw, panelIndex) => {
|
|
566
|
+
const panel = object(raw, `qualification board cell ${index}.panels[${panelIndex}]`);
|
|
567
|
+
exactKeys(panel, ["id", "repetition"], `qualification board cell ${index}.panels[${panelIndex}]`);
|
|
568
|
+
return { id: identifier(panel.id, `qualification board cell ${index} panel id`), repetition: nonNegativeInteger(panel.repetition, `qualification board cell ${index} panel repetition`) };
|
|
569
|
+
});
|
|
570
|
+
if (new Set(panels.map((panel) => panel.id)).size !== panels.length || new Set(panels.map((panel) => panel.repetition)).size !== panels.length)
|
|
571
|
+
throw new Error(`qualification board cell ${index} contains duplicate panel ids or repetitions`);
|
|
572
|
+
if (!Array.isArray(cell.judge_arms) || cell.judge_arms.length !== 3)
|
|
573
|
+
throw new Error(`qualification board cell ${index}.judge_arms must contain exactly three arm ids`);
|
|
574
|
+
const judgeArms = cell.judge_arms.map((arm, judgeIndex) => identifier(arm, `qualification board cell ${index}.judge_arms[${judgeIndex}]`));
|
|
575
|
+
if (typeof cell.critical !== "boolean")
|
|
576
|
+
throw new Error(`qualification board cell ${index}.critical must be boolean`);
|
|
577
|
+
const passThreshold = positiveInteger(cell.pass_threshold, `qualification board cell ${index}.pass_threshold`);
|
|
578
|
+
if (passThreshold > panels.length || (cell.critical && passThreshold !== panels.length))
|
|
579
|
+
throw new Error(`qualification board cell ${index}.pass_threshold contradicts its repetitions or Critical policy`);
|
|
580
|
+
return {
|
|
581
|
+
id: identifier(cell.id, `qualification board cell ${index}.id`),
|
|
582
|
+
scenario_id: identifier(cell.scenario_id, `qualification board cell ${index}.scenario_id`),
|
|
583
|
+
measurement_identity_sha256: digest(cell.measurement_identity_sha256, `qualification board cell ${index}.measurement_identity_sha256`),
|
|
584
|
+
scenario_version: identifier(cell.scenario_version, `qualification board cell ${index}.scenario_version`),
|
|
585
|
+
stimulus_sha256: digest(cell.stimulus_sha256, `qualification board cell ${index}.stimulus_sha256`),
|
|
586
|
+
rubric_sha256: digest(cell.rubric_sha256, `qualification board cell ${index}.rubric_sha256`),
|
|
587
|
+
subject_input_sha256: digest(cell.subject_input_sha256, `qualification board cell ${index}.subject_input_sha256`),
|
|
588
|
+
subject_arm: identifier(cell.subject_arm, `qualification board cell ${index}.subject_arm`), judge_arms: judgeArms,
|
|
589
|
+
panels, critical: cell.critical, pass_threshold: passThreshold,
|
|
590
|
+
};
|
|
591
|
+
});
|
|
592
|
+
if (new Set(cells.map((cell) => cell.id)).size !== cells.length)
|
|
593
|
+
throw new Error("qualification board contains duplicate cell ids");
|
|
594
|
+
const logicalSlots = cells.flatMap((cell) => cell.panels.map((panel) => `${cell.scenario_id}\0${cell.subject_arm}\0${panel.repetition}`));
|
|
595
|
+
if (new Set(logicalSlots).size !== logicalSlots.length)
|
|
596
|
+
throw new Error("qualification board duplicates a scenario/subject-arm/repetition slot");
|
|
597
|
+
const allPanelIds = cells.flatMap((cell) => cell.panels.map((panel) => panel.id));
|
|
598
|
+
if (new Set(allPanelIds).size !== allPanelIds.length)
|
|
599
|
+
throw new Error("qualification board contains duplicate panel ids across cells");
|
|
600
|
+
return { schema_version: QUALIFICATION_BOARD_VERSION, read_only: true, cells };
|
|
601
|
+
}
|
|
602
|
+
function parseCanonicalJudgePanelPolicy(value) {
|
|
603
|
+
if (qualificationCanonicalJson(value) !== qualificationCanonicalJson(QUALIFICATION_JUDGE_PANEL_POLICY)) {
|
|
604
|
+
throw new Error("qualification judge_panel must equal the canonical qualification-judge-panel-policy-v1 agreement, split, failure, and Wave A call-budget policy");
|
|
605
|
+
}
|
|
606
|
+
return structuredClone(QUALIFICATION_JUDGE_PANEL_POLICY);
|
|
607
|
+
}
|
|
608
|
+
function parseRepositoryPin(value, ctx) {
|
|
609
|
+
return {
|
|
610
|
+
repository: nonEmpty(value.repository, `${ctx}.repository`),
|
|
611
|
+
commit: gitSha(value.commit, `${ctx}.commit`),
|
|
612
|
+
tree: gitSha(value.tree, `${ctx}.tree`),
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
function parseExecutablePin(value, ctx) {
|
|
616
|
+
const objectValue = object(value, ctx);
|
|
617
|
+
exactKeys(objectValue, ["path", "sha256"], ctx);
|
|
618
|
+
return { path: absolutePath(objectValue.path, `${ctx}.path`), sha256: digest(objectValue.sha256, `${ctx}.sha256`) };
|
|
619
|
+
}
|
|
620
|
+
function object(value, ctx) {
|
|
621
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
622
|
+
throw new Error(`${ctx} must be an object`);
|
|
623
|
+
const prototype = Object.getPrototypeOf(value);
|
|
624
|
+
if (prototype !== Object.prototype && prototype !== null)
|
|
625
|
+
throw new Error(`${ctx} must be a plain object`);
|
|
626
|
+
return value;
|
|
627
|
+
}
|
|
628
|
+
function exactKeys(value, keys, ctx, optionalKeys = []) {
|
|
629
|
+
const allowed = new Set([...keys, ...optionalKeys]);
|
|
630
|
+
const unknown = Object.keys(value).find((key) => !allowed.has(key));
|
|
631
|
+
if (unknown)
|
|
632
|
+
throw new Error(`${ctx} contains unknown field ${unknown}`);
|
|
633
|
+
const missing = keys.find((key) => !Object.hasOwn(value, key));
|
|
634
|
+
if (missing)
|
|
635
|
+
throw new Error(`${ctx} is missing required field ${missing}`);
|
|
636
|
+
}
|
|
637
|
+
function nonEmpty(value, ctx) {
|
|
638
|
+
if (typeof value !== "string" || !value.trim())
|
|
639
|
+
throw new Error(`${ctx} must be a non-empty string`);
|
|
640
|
+
return value;
|
|
641
|
+
}
|
|
642
|
+
function identifier(value, ctx) {
|
|
643
|
+
const text = nonEmpty(value, ctx);
|
|
644
|
+
if (!ID_RE.test(text))
|
|
645
|
+
throw new Error(`${ctx} must be a bounded ASCII identifier`);
|
|
646
|
+
return text;
|
|
647
|
+
}
|
|
648
|
+
function digest(value, ctx) {
|
|
649
|
+
const text = nonEmpty(value, ctx);
|
|
650
|
+
if (!SHA256_RE.test(text))
|
|
651
|
+
throw new Error(`${ctx} must be a 64-character SHA-256 hex digest`);
|
|
652
|
+
return text.toLowerCase();
|
|
653
|
+
}
|
|
654
|
+
function gitSha(value, ctx) {
|
|
655
|
+
const text = nonEmpty(value, ctx);
|
|
656
|
+
if (!GIT_SHA_RE.test(text))
|
|
657
|
+
throw new Error(`${ctx} must be a pinned 40-character Git object id`);
|
|
658
|
+
return text.toLowerCase();
|
|
659
|
+
}
|
|
660
|
+
function absolutePath(value, ctx) {
|
|
661
|
+
const text = nonEmpty(value, ctx);
|
|
662
|
+
if (!isAbsolute(text))
|
|
663
|
+
throw new Error(`${ctx} must be an absolute path`);
|
|
664
|
+
return text;
|
|
665
|
+
}
|
|
666
|
+
function utcTimestamp(value, ctx) {
|
|
667
|
+
const text = nonEmpty(value, ctx);
|
|
668
|
+
if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?Z$/.test(text) || !Number.isFinite(Date.parse(text)))
|
|
669
|
+
throw new Error(`${ctx} must be an RFC 3339 UTC timestamp`);
|
|
670
|
+
return text;
|
|
671
|
+
}
|
|
672
|
+
function positiveInteger(value, ctx) { return boundedInteger(value, 1, Number.MAX_SAFE_INTEGER, ctx); }
|
|
673
|
+
function nonNegativeInteger(value, ctx) { return boundedInteger(value, 0, Number.MAX_SAFE_INTEGER, ctx); }
|
|
674
|
+
function boundedInteger(value, min, max, ctx) {
|
|
675
|
+
if (!Number.isSafeInteger(value) || Number(value) < min || Number(value) > max)
|
|
676
|
+
throw new Error(`${ctx} must be an integer in [${min}, ${max}]`);
|
|
677
|
+
return Number(value);
|
|
678
|
+
}
|
|
679
|
+
function enumValue(value, allowed, ctx) {
|
|
680
|
+
if (typeof value !== "string" || !allowed.includes(value))
|
|
681
|
+
throw new Error(`${ctx} must be one of ${allowed.join(", ")}`);
|
|
682
|
+
return value;
|
|
683
|
+
}
|
|
684
|
+
function stableStringify(value) {
|
|
685
|
+
if (value === null || typeof value !== "object")
|
|
686
|
+
return JSON.stringify(value) ?? "null";
|
|
687
|
+
if (Array.isArray(value))
|
|
688
|
+
return `[${value.map(stableStringify).join(",")}]`;
|
|
689
|
+
const entries = Object.entries(value)
|
|
690
|
+
.filter(([, entry]) => entry !== undefined)
|
|
691
|
+
.sort(([left], [right]) => left.localeCompare(right));
|
|
692
|
+
return `{${entries.map(([key, entry]) => `${JSON.stringify(key)}:${stableStringify(entry)}`).join(",")}}`;
|
|
693
|
+
}
|
|
694
|
+
//# sourceMappingURL=qualification-config.js.map
|