@tea-agent/loop-agent 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +59 -11
- package/README.md +1 -1
- package/dist/application/evaluation/alias.js +184 -0
- package/dist/application/evaluation/budget.js +192 -0
- package/dist/application/evaluation/campaign-hash.js +47 -0
- package/dist/application/evaluation/campaign-matrix.js +372 -0
- package/dist/application/evaluation/campaign-scorecard.js +135 -0
- package/dist/application/evaluation/campaign.js +370 -0
- package/dist/application/evaluation/candidate.js +23 -6
- package/dist/application/evaluation/corpus-hash.js +38 -0
- package/dist/application/evaluation/corpus.js +56 -0
- package/dist/application/evaluation/experiment.js +294 -0
- package/dist/application/evaluation/ignition.js +198 -0
- package/dist/application/evaluation/integrity-audit.js +162 -0
- package/dist/application/evaluation/outer-loop.js +132 -0
- package/dist/application/evaluation/pi-cell-executor.js +39 -0
- package/dist/application/evaluation/private-verifier.js +46 -0
- package/dist/application/evaluation/promotion-policy.js +151 -0
- package/dist/application/evaluation/proposer.js +98 -0
- package/dist/application/evaluation/types.js +522 -0
- package/dist/cli/command-definitions.js +19 -3
- package/dist/commands/eval.js +1176 -13
- package/dist/commands/init.js +4 -1
- package/dist/infrastructure/evaluation/alias-store.js +199 -0
- package/dist/infrastructure/evaluation/campaign-store.js +154 -0
- package/dist/infrastructure/evaluation/corpus-store.js +181 -0
- package/dist/infrastructure/evaluation/experiment-store.js +124 -0
- package/dist/infrastructure/evaluation/ignition-store.js +82 -0
- package/dist/infrastructure/evaluation/private-verifier-store.js +145 -0
- package/dist/infrastructure/evaluation/proposer-store.js +78 -0
- package/dist/worker/cli.js +6 -3
- package/dist/worker/delivery/final-verification.js +96 -8
- package/dist/worker/delivery/package.js +23 -4
- package/dist/worker/delivery/verification-bundle.js +521 -0
- package/dist/worker/feature/fullstack-validate.js +337 -0
- package/dist/worker/feature/profile-schema.js +44 -0
- package/dist/worker/feature/ready-plan-projection.js +1 -0
- package/dist/worker/feature/reducer.js +2 -0
- package/dist/worker/feature/review.js +106 -11
- package/dist/worker/materialize/harness-task-materializer.js +5 -0
- package/dist/worker/observability/read-model.js +7 -0
- package/dist/worker/observe/static/views/task.js +1 -0
- package/dist/worker/outcomes/adapters.js +144 -0
- package/dist/worker/outcomes/evidence-tokens.js +29 -0
- package/dist/worker/outcomes/gate.js +40 -0
- package/dist/worker/outcomes/projector.js +185 -0
- package/dist/worker/outcomes/registry.js +1 -0
- package/dist/worker/outcomes/store.js +131 -0
- package/dist/worker/outcomes/types.js +79 -0
- package/dist/worker/report/morning-report.js +4 -3
- package/dist/worker/run-task/run-task.js +85 -2
- package/dist/worker/runner/run-ready.js +32 -1
- package/dist/worker/task-graph/acceptance-schema.js +12 -0
- package/dist/worker/task-graph/ready-planner.js +131 -0
- package/dist/worker/task-graph/task-graph-schema.js +31 -0
- package/dist/worker/task-graph/validate.js +44 -4
- package/dist/worker/task-spec/schema.js +9 -0
- package/dist/worker/task-spec/validate.js +39 -0
- package/dist/worker/task-spec/workflow-routing.js +149 -0
- package/dist/workflows/dag/budget-enforcement.js +67 -0
- package/dist/workflows/dag/context-policy.js +137 -0
- package/dist/workflows/dag/knowledge-curator.js +3 -0
- package/dist/workflows/dag/node-execution.js +11 -4
- package/dist/workflows/dag/prompt.js +1 -1
- package/dist/workflows/dag/runner.js +43 -16
- package/dist/workflows/dag/skill-snapshot.js +11 -7
- package/dist/workflows/dag/types.js +18 -0
- package/docs/init-surface.manifest.json +3 -0
- package/docs/templates/evaluation/campaign-budget-v1.json +12 -0
- package/docs/templates/evaluation/campaign-dogfood-v0.json +24 -0
- package/docs/templates/evaluation/campaign-evidence-v1.json +44 -0
- package/docs/templates/evaluation/context-policy-baseline-v1.json +17 -0
- package/docs/templates/evaluation/context-policy-role-specialized-v1.json +28 -0
- package/docs/templates/evaluation/corpus-dogfood-v0.manifest.json +118 -0
- package/docs/templates/evaluation/matrix-dag-dry-run-v1.json +21 -0
- package/docs/templates/evaluation/matrix-fixture-v1.json +10 -0
- package/docs/templates/evaluation/private-verifier-dogfood-v0.json +16 -0
- package/docs/templates/product-line/AGENTS.md +1 -0
- package/docs/templates/product-line/README.md +17 -0
- package/docs/templates/product-line/acceptance.yaml +9 -0
- package/docs/templates/product-line/feature.yaml +11 -0
- package/docs/templates/product-line/task-graph.yaml +8 -0
- package/docs/templates/product-line/task.yaml +4 -0
- package/harness.json +1 -1
- package/package.json +1 -1
package/dist/commands/eval.js
CHANGED
|
@@ -3,8 +3,22 @@ import { writeTextAtomic } from "../infrastructure/harness/atomic-write.js";
|
|
|
3
3
|
import { readReplayScorecard } from "../infrastructure/evaluation/store.js";
|
|
4
4
|
import { formatReplayMarkdown, replayEvaluation, } from "../application/evaluation/replay.js";
|
|
5
5
|
import { formatCandidateMarkdown, listCandidates, registerCandidate, showCandidate, transitionCandidate, } from "../application/evaluation/candidate.js";
|
|
6
|
-
import {
|
|
7
|
-
|
|
6
|
+
import { formatAliasMarkdown, formatAliasMoveMarkdown, listAliases, promoteAlias, rollbackAlias, showAlias, } from "../application/evaluation/alias.js";
|
|
7
|
+
import { formatCorpusMarkdown, listCorpora, registerCorpus, showCorpus, validateCorpus, } from "../application/evaluation/corpus.js";
|
|
8
|
+
import { formatBudgetReportMarkdown, } from "../application/evaluation/budget.js";
|
|
9
|
+
import { createCampaign, formatCampaignMarkdown, listCampaigns, planCampaign, runCampaignPrivateChecks, showCampaign, validateCampaign, } from "../application/evaluation/campaign.js";
|
|
10
|
+
import { auditCampaignEvidence, buildCampaignScorecard, formatCampaignScorecardMarkdown, loadCampaignEvidence, requireCampaignPromotionGate, } from "../application/evaluation/campaign-scorecard.js";
|
|
11
|
+
import { acceptExperiment, createExperimentFromCurate, formatExperimentMarkdown, listExperiments, rejectExperiment, showExperiment, } from "../application/evaluation/experiment.js";
|
|
12
|
+
import { formatMatrixRunMarkdown, runCampaignMatrix, } from "../application/evaluation/campaign-matrix.js";
|
|
13
|
+
import { formatOuterLoopMarkdown, runOuterLoop, } from "../application/evaluation/outer-loop.js";
|
|
14
|
+
import { evaluateIgnition, formatIgnitionMarkdown, listIgnitions, recordIgnitionGeneration, showIgnition, } from "../application/evaluation/ignition.js";
|
|
15
|
+
import { createDagDryRunPiCellExecutor } from "../application/evaluation/pi-cell-executor.js";
|
|
16
|
+
import { formatProposerRunMarkdown, proposeFromCurate, } from "../application/evaluation/proposer.js";
|
|
17
|
+
import { formatPrivateVerifierMarkdown, listPrivateVerifiers, registerPrivateVerifier, showPrivateVerifier, validatePrivateVerifier, } from "../application/evaluation/private-verifier.js";
|
|
18
|
+
import { getContextPolicy, isContextPolicyId, listContextPolicies, } from "../workflows/dag/context-policy.js";
|
|
19
|
+
import { locateDagRun, readDagRunState } from "../workflows/dag/lifecycle.js";
|
|
20
|
+
import { lifecycleStateSchema, matrixModeSchema, } from "../application/evaluation/types.js";
|
|
21
|
+
const USAGE = "usage: eval <…|propose|outer-loop|ignition|campaign|experiment> ...; ignition <record|evaluate|show|list>; campaign matrix [--mode pi --dag-template <path>]; promote/ignition never auto-promote; ignition never claims RSI Level 1";
|
|
8
22
|
function parseFormatFlags(args) {
|
|
9
23
|
const json = args.includes("--json");
|
|
10
24
|
const markdown = args.includes("--markdown");
|
|
@@ -13,6 +27,15 @@ function parseFormatFlags(args) {
|
|
|
13
27
|
}
|
|
14
28
|
return { json: json || !markdown, markdown };
|
|
15
29
|
}
|
|
30
|
+
function parseDryRunFlags(args) {
|
|
31
|
+
const dryRun = args.includes("--dry-run");
|
|
32
|
+
const apply = args.includes("--apply");
|
|
33
|
+
if (dryRun && apply) {
|
|
34
|
+
throw new Error("eval accepts only one of --dry-run or --apply");
|
|
35
|
+
}
|
|
36
|
+
// Default dry-run: human gate; --apply commits alias move.
|
|
37
|
+
return !apply;
|
|
38
|
+
}
|
|
16
39
|
function flagValue(args, flag) {
|
|
17
40
|
const index = args.indexOf(flag);
|
|
18
41
|
if (index >= 0) {
|
|
@@ -25,6 +48,37 @@ function flagValue(args, flag) {
|
|
|
25
48
|
const prefix = `${flag}=`;
|
|
26
49
|
return args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length);
|
|
27
50
|
}
|
|
51
|
+
const VALUE_FLAGS = [
|
|
52
|
+
"--spec",
|
|
53
|
+
"--output",
|
|
54
|
+
"--replay-id",
|
|
55
|
+
"--manifest",
|
|
56
|
+
"--candidate-id",
|
|
57
|
+
"--corpus-id",
|
|
58
|
+
"--to",
|
|
59
|
+
"--to-candidate",
|
|
60
|
+
"--reason",
|
|
61
|
+
"--alias",
|
|
62
|
+
"--policy-id",
|
|
63
|
+
"--run-id",
|
|
64
|
+
"--private-verifier-id",
|
|
65
|
+
"--campaign-id",
|
|
66
|
+
"--evidence",
|
|
67
|
+
"--experiment-id",
|
|
68
|
+
"--mode",
|
|
69
|
+
"--fixture",
|
|
70
|
+
"--max-public-cells",
|
|
71
|
+
"--max",
|
|
72
|
+
"--proposer-run-id",
|
|
73
|
+
"--campaign-manifest",
|
|
74
|
+
"--matrix-mode",
|
|
75
|
+
"--max-proposals",
|
|
76
|
+
"--dag-template",
|
|
77
|
+
"--ignition-id",
|
|
78
|
+
"--generation",
|
|
79
|
+
"--parent-candidate",
|
|
80
|
+
"--proposer-run-id",
|
|
81
|
+
];
|
|
28
82
|
function assertKnownFlags(args, allowed) {
|
|
29
83
|
for (let i = 0; i < args.length; i += 1) {
|
|
30
84
|
const arg = args[i];
|
|
@@ -34,16 +88,7 @@ function assertKnownFlags(args, allowed) {
|
|
|
34
88
|
if (!allowed.includes(key)) {
|
|
35
89
|
throw new Error(`unknown eval argument: ${arg}`);
|
|
36
90
|
}
|
|
37
|
-
if (
|
|
38
|
-
"--spec",
|
|
39
|
-
"--output",
|
|
40
|
-
"--replay-id",
|
|
41
|
-
"--manifest",
|
|
42
|
-
"--candidate-id",
|
|
43
|
-
"--to",
|
|
44
|
-
"--reason",
|
|
45
|
-
].includes(key) &&
|
|
46
|
-
!arg.includes("=")) {
|
|
91
|
+
if (VALUE_FLAGS.includes(key) && !arg.includes("=")) {
|
|
47
92
|
i += 1;
|
|
48
93
|
}
|
|
49
94
|
}
|
|
@@ -116,6 +161,511 @@ function parseCandidateArgs(rest) {
|
|
|
116
161
|
}
|
|
117
162
|
throw new Error("usage: eval candidate <register|show|list|transition> ...");
|
|
118
163
|
}
|
|
164
|
+
function parseAliasArgs(rest) {
|
|
165
|
+
const [action, ...tail] = rest;
|
|
166
|
+
if (action === "show") {
|
|
167
|
+
assertKnownFlags(tail, ["--alias", "--json", "--markdown"]);
|
|
168
|
+
const alias = flagValue(tail, "--alias");
|
|
169
|
+
if (!alias)
|
|
170
|
+
throw new Error("eval alias show requires --alias <name>");
|
|
171
|
+
return {
|
|
172
|
+
command: "alias",
|
|
173
|
+
action: "show",
|
|
174
|
+
alias,
|
|
175
|
+
...parseFormatFlags(tail),
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
if (action === "list") {
|
|
179
|
+
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
180
|
+
return {
|
|
181
|
+
command: "alias",
|
|
182
|
+
action: "list",
|
|
183
|
+
...parseFormatFlags(tail),
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
throw new Error("usage: eval alias <show|list> ...");
|
|
187
|
+
}
|
|
188
|
+
function parseContextPolicyArgs(rest) {
|
|
189
|
+
const [action, ...tail] = rest;
|
|
190
|
+
if (action === "list") {
|
|
191
|
+
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
192
|
+
return {
|
|
193
|
+
command: "context-policy",
|
|
194
|
+
action: "list",
|
|
195
|
+
...parseFormatFlags(tail),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
if (action === "show") {
|
|
199
|
+
assertKnownFlags(tail, ["--policy-id", "--json", "--markdown"]);
|
|
200
|
+
const policyId = flagValue(tail, "--policy-id");
|
|
201
|
+
if (!policyId) {
|
|
202
|
+
throw new Error("eval context-policy show requires --policy-id <id>");
|
|
203
|
+
}
|
|
204
|
+
return {
|
|
205
|
+
command: "context-policy",
|
|
206
|
+
action: "show",
|
|
207
|
+
policyId,
|
|
208
|
+
...parseFormatFlags(tail),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
throw new Error("usage: eval context-policy <list|show> ...");
|
|
212
|
+
}
|
|
213
|
+
function parseBudgetArgs(rest) {
|
|
214
|
+
const [action, ...tail] = rest;
|
|
215
|
+
if (action === "show") {
|
|
216
|
+
assertKnownFlags(tail, ["--run-id", "--json", "--markdown"]);
|
|
217
|
+
const runId = flagValue(tail, "--run-id");
|
|
218
|
+
if (!runId) {
|
|
219
|
+
throw new Error("eval budget show requires --run-id <id>");
|
|
220
|
+
}
|
|
221
|
+
return {
|
|
222
|
+
command: "budget",
|
|
223
|
+
action: "show",
|
|
224
|
+
runId,
|
|
225
|
+
...parseFormatFlags(tail),
|
|
226
|
+
};
|
|
227
|
+
}
|
|
228
|
+
throw new Error("usage: eval budget show --run-id <id> [--json|--markdown]");
|
|
229
|
+
}
|
|
230
|
+
function parsePrivateVerifierArgs(rest) {
|
|
231
|
+
const [action, ...tail] = rest;
|
|
232
|
+
if (action === "register" || action === "validate") {
|
|
233
|
+
assertKnownFlags(tail, ["--manifest", "--json", "--markdown"]);
|
|
234
|
+
const manifestPath = flagValue(tail, "--manifest");
|
|
235
|
+
if (!manifestPath) {
|
|
236
|
+
throw new Error(`eval private-verifier ${action} requires --manifest <path>`);
|
|
237
|
+
}
|
|
238
|
+
return {
|
|
239
|
+
command: "private-verifier",
|
|
240
|
+
action,
|
|
241
|
+
manifestPath,
|
|
242
|
+
...parseFormatFlags(tail),
|
|
243
|
+
};
|
|
244
|
+
}
|
|
245
|
+
if (action === "show") {
|
|
246
|
+
assertKnownFlags(tail, ["--private-verifier-id", "--json", "--markdown"]);
|
|
247
|
+
const privateVerifierId = flagValue(tail, "--private-verifier-id");
|
|
248
|
+
if (!privateVerifierId) {
|
|
249
|
+
throw new Error("eval private-verifier show requires --private-verifier-id <id>");
|
|
250
|
+
}
|
|
251
|
+
return {
|
|
252
|
+
command: "private-verifier",
|
|
253
|
+
action: "show",
|
|
254
|
+
privateVerifierId,
|
|
255
|
+
...parseFormatFlags(tail),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
if (action === "list") {
|
|
259
|
+
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
260
|
+
return {
|
|
261
|
+
command: "private-verifier",
|
|
262
|
+
action: "list",
|
|
263
|
+
...parseFormatFlags(tail),
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
throw new Error("usage: eval private-verifier <register|show|list|validate> ...");
|
|
267
|
+
}
|
|
268
|
+
function parseCampaignArgs(rest) {
|
|
269
|
+
const [action, ...tail] = rest;
|
|
270
|
+
if (action === "create" || action === "validate") {
|
|
271
|
+
assertKnownFlags(tail, ["--manifest", "--json", "--markdown"]);
|
|
272
|
+
const manifestPath = flagValue(tail, "--manifest");
|
|
273
|
+
if (!manifestPath) {
|
|
274
|
+
throw new Error(`eval campaign ${action} requires --manifest <path>`);
|
|
275
|
+
}
|
|
276
|
+
return {
|
|
277
|
+
command: "campaign",
|
|
278
|
+
action,
|
|
279
|
+
manifestPath,
|
|
280
|
+
...parseFormatFlags(tail),
|
|
281
|
+
};
|
|
282
|
+
}
|
|
283
|
+
if (action === "show" || action === "plan" || action === "run") {
|
|
284
|
+
assertKnownFlags(tail, ["--campaign-id", "--json", "--markdown"]);
|
|
285
|
+
const campaignId = flagValue(tail, "--campaign-id");
|
|
286
|
+
if (!campaignId) {
|
|
287
|
+
throw new Error(`eval campaign ${action} requires --campaign-id <id>`);
|
|
288
|
+
}
|
|
289
|
+
return {
|
|
290
|
+
command: "campaign",
|
|
291
|
+
action,
|
|
292
|
+
campaignId,
|
|
293
|
+
...parseFormatFlags(tail),
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
if (action === "matrix") {
|
|
297
|
+
assertKnownFlags(tail, [
|
|
298
|
+
"--campaign-id",
|
|
299
|
+
"--mode",
|
|
300
|
+
"--fixture",
|
|
301
|
+
"--dag-template",
|
|
302
|
+
"--max-public-cells",
|
|
303
|
+
"--json",
|
|
304
|
+
"--markdown",
|
|
305
|
+
]);
|
|
306
|
+
const campaignId = flagValue(tail, "--campaign-id");
|
|
307
|
+
const modeRaw = flagValue(tail, "--mode") ?? "dry-run";
|
|
308
|
+
if (!campaignId) {
|
|
309
|
+
throw new Error("eval campaign matrix requires --campaign-id <id>");
|
|
310
|
+
}
|
|
311
|
+
const mode = matrixModeSchema.parse(modeRaw);
|
|
312
|
+
const maxRaw = flagValue(tail, "--max-public-cells");
|
|
313
|
+
return {
|
|
314
|
+
command: "campaign",
|
|
315
|
+
action: "matrix",
|
|
316
|
+
campaignId,
|
|
317
|
+
mode,
|
|
318
|
+
fixturePath: flagValue(tail, "--fixture"),
|
|
319
|
+
dagTemplatePath: flagValue(tail, "--dag-template"),
|
|
320
|
+
maxPublicCells: maxRaw ? Number(maxRaw) : undefined,
|
|
321
|
+
...parseFormatFlags(tail),
|
|
322
|
+
};
|
|
323
|
+
}
|
|
324
|
+
if (action === "scorecard" || action === "audit") {
|
|
325
|
+
assertKnownFlags(tail, [
|
|
326
|
+
"--campaign-id",
|
|
327
|
+
"--evidence",
|
|
328
|
+
"--acknowledge-review",
|
|
329
|
+
"--json",
|
|
330
|
+
"--markdown",
|
|
331
|
+
]);
|
|
332
|
+
const campaignId = flagValue(tail, "--campaign-id");
|
|
333
|
+
const evidencePath = flagValue(tail, "--evidence");
|
|
334
|
+
if (!campaignId) {
|
|
335
|
+
throw new Error(`eval campaign ${action} requires --campaign-id <id>`);
|
|
336
|
+
}
|
|
337
|
+
if (!evidencePath) {
|
|
338
|
+
throw new Error(`eval campaign ${action} requires --evidence <path>`);
|
|
339
|
+
}
|
|
340
|
+
return {
|
|
341
|
+
command: "campaign",
|
|
342
|
+
action,
|
|
343
|
+
campaignId,
|
|
344
|
+
evidencePath,
|
|
345
|
+
acknowledgeReview: tail.includes("--acknowledge-review"),
|
|
346
|
+
...parseFormatFlags(tail),
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
if (action === "list") {
|
|
350
|
+
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
351
|
+
return {
|
|
352
|
+
command: "campaign",
|
|
353
|
+
action: "list",
|
|
354
|
+
...parseFormatFlags(tail),
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
throw new Error("usage: eval campaign <create|show|list|validate|plan|run|matrix|scorecard|audit> ...");
|
|
358
|
+
}
|
|
359
|
+
function parseProposeArgs(rest) {
|
|
360
|
+
const [action, ...tail] = rest;
|
|
361
|
+
if (action !== "from-curate") {
|
|
362
|
+
throw new Error("usage: eval propose from-curate [--max N] [--dry-run|--apply]");
|
|
363
|
+
}
|
|
364
|
+
assertKnownFlags(tail, [
|
|
365
|
+
"--max",
|
|
366
|
+
"--proposer-run-id",
|
|
367
|
+
"--dry-run",
|
|
368
|
+
"--apply",
|
|
369
|
+
"--json",
|
|
370
|
+
"--markdown",
|
|
371
|
+
]);
|
|
372
|
+
const maxRaw = flagValue(tail, "--max");
|
|
373
|
+
return {
|
|
374
|
+
command: "propose",
|
|
375
|
+
action: "from-curate",
|
|
376
|
+
maxCandidates: maxRaw ? Number(maxRaw) : 1,
|
|
377
|
+
proposerRunId: flagValue(tail, "--proposer-run-id"),
|
|
378
|
+
dryRun: parseDryRunFlags(tail),
|
|
379
|
+
...parseFormatFlags(tail),
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
function parseOuterLoopArgs(rest) {
|
|
383
|
+
const [action, ...tail] = rest;
|
|
384
|
+
if (action !== "run") {
|
|
385
|
+
throw new Error("usage: eval outer-loop run --matrix-mode <dry-run|stub|pi-plan|pi> [--dag-template <path>] [--campaign-id|--campaign-manifest] [--propose-from-curate] ...");
|
|
386
|
+
}
|
|
387
|
+
assertKnownFlags(tail, [
|
|
388
|
+
"--campaign-id",
|
|
389
|
+
"--campaign-manifest",
|
|
390
|
+
"--propose-from-curate",
|
|
391
|
+
"--max-proposals",
|
|
392
|
+
"--matrix-mode",
|
|
393
|
+
"--fixture",
|
|
394
|
+
"--dag-template",
|
|
395
|
+
"--max-public-cells",
|
|
396
|
+
"--scorecard",
|
|
397
|
+
"--json",
|
|
398
|
+
"--markdown",
|
|
399
|
+
]);
|
|
400
|
+
const matrixModeRaw = flagValue(tail, "--matrix-mode");
|
|
401
|
+
if (!matrixModeRaw) {
|
|
402
|
+
throw new Error("eval outer-loop run requires --matrix-mode <mode>");
|
|
403
|
+
}
|
|
404
|
+
const matrixMode = matrixModeSchema.parse(matrixModeRaw);
|
|
405
|
+
const dagTemplatePath = flagValue(tail, "--dag-template");
|
|
406
|
+
if (matrixMode === "pi" && !dagTemplatePath) {
|
|
407
|
+
throw new Error("eval outer-loop refuses mode=pi without --dag-template (bounded dry-run Pi) or injected executor; use pi-plan or stub");
|
|
408
|
+
}
|
|
409
|
+
const maxRaw = flagValue(tail, "--max-public-cells");
|
|
410
|
+
const maxPropRaw = flagValue(tail, "--max-proposals");
|
|
411
|
+
return {
|
|
412
|
+
command: "outer-loop",
|
|
413
|
+
action: "run",
|
|
414
|
+
campaignId: flagValue(tail, "--campaign-id"),
|
|
415
|
+
campaignManifestPath: flagValue(tail, "--campaign-manifest"),
|
|
416
|
+
proposeFromCurate: tail.includes("--propose-from-curate"),
|
|
417
|
+
maxProposals: maxPropRaw ? Number(maxPropRaw) : 1,
|
|
418
|
+
matrixMode,
|
|
419
|
+
fixturePath: flagValue(tail, "--fixture"),
|
|
420
|
+
dagTemplatePath,
|
|
421
|
+
maxPublicCells: maxRaw ? Number(maxRaw) : undefined,
|
|
422
|
+
buildScorecard: tail.includes("--scorecard"),
|
|
423
|
+
...parseFormatFlags(tail),
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
function parseIgnitionArgs(rest) {
|
|
427
|
+
const [action, ...tail] = rest;
|
|
428
|
+
if (action === "list") {
|
|
429
|
+
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
430
|
+
return {
|
|
431
|
+
command: "ignition",
|
|
432
|
+
action: "list",
|
|
433
|
+
...parseFormatFlags(tail),
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
if (action === "evaluate" || action === "show") {
|
|
437
|
+
assertKnownFlags(tail, ["--ignition-id", "--json", "--markdown"]);
|
|
438
|
+
const ignitionId = flagValue(tail, "--ignition-id");
|
|
439
|
+
if (!ignitionId) {
|
|
440
|
+
throw new Error(`eval ignition ${action} requires --ignition-id <id>`);
|
|
441
|
+
}
|
|
442
|
+
return {
|
|
443
|
+
command: "ignition",
|
|
444
|
+
action,
|
|
445
|
+
ignitionId,
|
|
446
|
+
...parseFormatFlags(tail),
|
|
447
|
+
};
|
|
448
|
+
}
|
|
449
|
+
if (action === "record") {
|
|
450
|
+
assertKnownFlags(tail, [
|
|
451
|
+
"--ignition-id",
|
|
452
|
+
"--generation",
|
|
453
|
+
"--parent-candidate",
|
|
454
|
+
"--proposer-run-id",
|
|
455
|
+
"--campaign-id",
|
|
456
|
+
"--json",
|
|
457
|
+
"--markdown",
|
|
458
|
+
]);
|
|
459
|
+
const ignitionId = flagValue(tail, "--ignition-id");
|
|
460
|
+
const generationRaw = flagValue(tail, "--generation");
|
|
461
|
+
const proposerRunId = flagValue(tail, "--proposer-run-id");
|
|
462
|
+
if (!ignitionId) {
|
|
463
|
+
throw new Error("eval ignition record requires --ignition-id <id>");
|
|
464
|
+
}
|
|
465
|
+
if (generationRaw === undefined) {
|
|
466
|
+
throw new Error("eval ignition record requires --generation <n>");
|
|
467
|
+
}
|
|
468
|
+
if (!proposerRunId) {
|
|
469
|
+
throw new Error("eval ignition record requires --proposer-run-id <id>");
|
|
470
|
+
}
|
|
471
|
+
const parentRaw = flagValue(tail, "--parent-candidate");
|
|
472
|
+
return {
|
|
473
|
+
command: "ignition",
|
|
474
|
+
action: "record",
|
|
475
|
+
ignitionId,
|
|
476
|
+
generation: Number(generationRaw),
|
|
477
|
+
parentCandidateId: parentRaw === undefined || parentRaw === "null" ? null : parentRaw,
|
|
478
|
+
proposerRunId,
|
|
479
|
+
campaignId: flagValue(tail, "--campaign-id"),
|
|
480
|
+
...parseFormatFlags(tail),
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
throw new Error("usage: eval ignition <record|evaluate|show|list> ...");
|
|
484
|
+
}
|
|
485
|
+
function parseExperimentArgs(rest) {
|
|
486
|
+
const [action, ...tail] = rest;
|
|
487
|
+
if (action === "from-curate") {
|
|
488
|
+
assertKnownFlags(tail, [
|
|
489
|
+
"--experiment-id",
|
|
490
|
+
"--candidate-id",
|
|
491
|
+
"--json",
|
|
492
|
+
"--markdown",
|
|
493
|
+
]);
|
|
494
|
+
return {
|
|
495
|
+
command: "experiment",
|
|
496
|
+
action: "from-curate",
|
|
497
|
+
experimentId: flagValue(tail, "--experiment-id"),
|
|
498
|
+
candidateId: flagValue(tail, "--candidate-id"),
|
|
499
|
+
...parseFormatFlags(tail),
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
if (action === "show") {
|
|
503
|
+
assertKnownFlags(tail, ["--experiment-id", "--json", "--markdown"]);
|
|
504
|
+
const experimentId = flagValue(tail, "--experiment-id");
|
|
505
|
+
if (!experimentId) {
|
|
506
|
+
throw new Error("eval experiment show requires --experiment-id <id>");
|
|
507
|
+
}
|
|
508
|
+
return {
|
|
509
|
+
command: "experiment",
|
|
510
|
+
action: "show",
|
|
511
|
+
experimentId,
|
|
512
|
+
...parseFormatFlags(tail),
|
|
513
|
+
};
|
|
514
|
+
}
|
|
515
|
+
if (action === "list") {
|
|
516
|
+
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
517
|
+
return {
|
|
518
|
+
command: "experiment",
|
|
519
|
+
action: "list",
|
|
520
|
+
...parseFormatFlags(tail),
|
|
521
|
+
};
|
|
522
|
+
}
|
|
523
|
+
if (action === "accept") {
|
|
524
|
+
assertKnownFlags(tail, [
|
|
525
|
+
"--experiment-id",
|
|
526
|
+
"--campaign-id",
|
|
527
|
+
"--reason",
|
|
528
|
+
"--acknowledge-review",
|
|
529
|
+
"--dry-run",
|
|
530
|
+
"--apply",
|
|
531
|
+
"--json",
|
|
532
|
+
"--markdown",
|
|
533
|
+
]);
|
|
534
|
+
const experimentId = flagValue(tail, "--experiment-id");
|
|
535
|
+
const campaignId = flagValue(tail, "--campaign-id");
|
|
536
|
+
const reason = flagValue(tail, "--reason");
|
|
537
|
+
if (!experimentId) {
|
|
538
|
+
throw new Error("eval experiment accept requires --experiment-id <id>");
|
|
539
|
+
}
|
|
540
|
+
if (!campaignId) {
|
|
541
|
+
throw new Error("eval experiment accept requires --campaign-id <id>");
|
|
542
|
+
}
|
|
543
|
+
if (!reason) {
|
|
544
|
+
throw new Error("eval experiment accept requires --reason <text>");
|
|
545
|
+
}
|
|
546
|
+
return {
|
|
547
|
+
command: "experiment",
|
|
548
|
+
action: "accept",
|
|
549
|
+
experimentId,
|
|
550
|
+
campaignId,
|
|
551
|
+
reason,
|
|
552
|
+
acknowledgeReview: tail.includes("--acknowledge-review"),
|
|
553
|
+
dryRun: parseDryRunFlags(tail),
|
|
554
|
+
...parseFormatFlags(tail),
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
if (action === "reject") {
|
|
558
|
+
assertKnownFlags(tail, [
|
|
559
|
+
"--experiment-id",
|
|
560
|
+
"--reason",
|
|
561
|
+
"--dry-run",
|
|
562
|
+
"--apply",
|
|
563
|
+
"--json",
|
|
564
|
+
"--markdown",
|
|
565
|
+
]);
|
|
566
|
+
const experimentId = flagValue(tail, "--experiment-id");
|
|
567
|
+
const reason = flagValue(tail, "--reason");
|
|
568
|
+
if (!experimentId) {
|
|
569
|
+
throw new Error("eval experiment reject requires --experiment-id <id>");
|
|
570
|
+
}
|
|
571
|
+
if (!reason) {
|
|
572
|
+
throw new Error("eval experiment reject requires --reason <text>");
|
|
573
|
+
}
|
|
574
|
+
return {
|
|
575
|
+
command: "experiment",
|
|
576
|
+
action: "reject",
|
|
577
|
+
experimentId,
|
|
578
|
+
reason,
|
|
579
|
+
dryRun: parseDryRunFlags(tail),
|
|
580
|
+
...parseFormatFlags(tail),
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
throw new Error("usage: eval experiment <from-curate|show|list|accept|reject> ...");
|
|
584
|
+
}
|
|
585
|
+
function parseCorpusArgs(rest) {
|
|
586
|
+
const [action, ...tail] = rest;
|
|
587
|
+
if (action === "register" || action === "validate") {
|
|
588
|
+
assertKnownFlags(tail, ["--manifest", "--json", "--markdown"]);
|
|
589
|
+
const manifestPath = flagValue(tail, "--manifest");
|
|
590
|
+
if (!manifestPath) {
|
|
591
|
+
throw new Error(`eval corpus ${action} requires --manifest <path>`);
|
|
592
|
+
}
|
|
593
|
+
return {
|
|
594
|
+
command: "corpus",
|
|
595
|
+
action,
|
|
596
|
+
manifestPath,
|
|
597
|
+
...parseFormatFlags(tail),
|
|
598
|
+
};
|
|
599
|
+
}
|
|
600
|
+
if (action === "show") {
|
|
601
|
+
assertKnownFlags(tail, ["--corpus-id", "--json", "--markdown"]);
|
|
602
|
+
const corpusId = flagValue(tail, "--corpus-id");
|
|
603
|
+
if (!corpusId) {
|
|
604
|
+
throw new Error("eval corpus show requires --corpus-id <id>");
|
|
605
|
+
}
|
|
606
|
+
return {
|
|
607
|
+
command: "corpus",
|
|
608
|
+
action: "show",
|
|
609
|
+
corpusId,
|
|
610
|
+
...parseFormatFlags(tail),
|
|
611
|
+
};
|
|
612
|
+
}
|
|
613
|
+
if (action === "list") {
|
|
614
|
+
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
615
|
+
return {
|
|
616
|
+
command: "corpus",
|
|
617
|
+
action: "list",
|
|
618
|
+
...parseFormatFlags(tail),
|
|
619
|
+
};
|
|
620
|
+
}
|
|
621
|
+
throw new Error("usage: eval corpus <register|show|list|validate> ...");
|
|
622
|
+
}
|
|
623
|
+
function parseAliasMoveArgs(command, rest) {
|
|
624
|
+
const promoteOnlyFlags = command === "promote"
|
|
625
|
+
? ["--campaign-id", "--evidence", "--acknowledge-review"]
|
|
626
|
+
: [];
|
|
627
|
+
assertKnownFlags(rest, [
|
|
628
|
+
"--alias",
|
|
629
|
+
"--to-candidate",
|
|
630
|
+
"--reason",
|
|
631
|
+
"--dry-run",
|
|
632
|
+
"--apply",
|
|
633
|
+
"--json",
|
|
634
|
+
"--markdown",
|
|
635
|
+
...promoteOnlyFlags,
|
|
636
|
+
]);
|
|
637
|
+
const alias = flagValue(rest, "--alias");
|
|
638
|
+
const toCandidateId = flagValue(rest, "--to-candidate");
|
|
639
|
+
const reason = flagValue(rest, "--reason");
|
|
640
|
+
if (!alias)
|
|
641
|
+
throw new Error(`eval ${command} requires --alias <name>`);
|
|
642
|
+
if (!toCandidateId) {
|
|
643
|
+
throw new Error(`eval ${command} requires --to-candidate <id>`);
|
|
644
|
+
}
|
|
645
|
+
if (!reason)
|
|
646
|
+
throw new Error(`eval ${command} requires --reason <text>`);
|
|
647
|
+
if (command === "promote") {
|
|
648
|
+
return {
|
|
649
|
+
command,
|
|
650
|
+
alias,
|
|
651
|
+
toCandidateId,
|
|
652
|
+
reason,
|
|
653
|
+
campaignId: flagValue(rest, "--campaign-id"),
|
|
654
|
+
evidencePath: flagValue(rest, "--evidence"),
|
|
655
|
+
acknowledgeReview: rest.includes("--acknowledge-review"),
|
|
656
|
+
dryRun: parseDryRunFlags(rest),
|
|
657
|
+
...parseFormatFlags(rest),
|
|
658
|
+
};
|
|
659
|
+
}
|
|
660
|
+
return {
|
|
661
|
+
command,
|
|
662
|
+
alias,
|
|
663
|
+
toCandidateId,
|
|
664
|
+
reason,
|
|
665
|
+
dryRun: parseDryRunFlags(rest),
|
|
666
|
+
...parseFormatFlags(rest),
|
|
667
|
+
};
|
|
668
|
+
}
|
|
119
669
|
export function parseEvalArgs(args) {
|
|
120
670
|
const [command, ...rest] = args;
|
|
121
671
|
if (command === "replay") {
|
|
@@ -140,8 +690,48 @@ export function parseEvalArgs(args) {
|
|
|
140
690
|
if (command === "candidate") {
|
|
141
691
|
return parseCandidateArgs(rest);
|
|
142
692
|
}
|
|
693
|
+
if (command === "alias") {
|
|
694
|
+
return parseAliasArgs(rest);
|
|
695
|
+
}
|
|
696
|
+
if (command === "promote" || command === "rollback") {
|
|
697
|
+
return parseAliasMoveArgs(command, rest);
|
|
698
|
+
}
|
|
699
|
+
if (command === "corpus") {
|
|
700
|
+
return parseCorpusArgs(rest);
|
|
701
|
+
}
|
|
702
|
+
if (command === "context-policy") {
|
|
703
|
+
return parseContextPolicyArgs(rest);
|
|
704
|
+
}
|
|
705
|
+
if (command === "budget") {
|
|
706
|
+
return parseBudgetArgs(rest);
|
|
707
|
+
}
|
|
708
|
+
if (command === "private-verifier") {
|
|
709
|
+
return parsePrivateVerifierArgs(rest);
|
|
710
|
+
}
|
|
711
|
+
if (command === "campaign") {
|
|
712
|
+
return parseCampaignArgs(rest);
|
|
713
|
+
}
|
|
714
|
+
if (command === "experiment") {
|
|
715
|
+
return parseExperimentArgs(rest);
|
|
716
|
+
}
|
|
717
|
+
if (command === "propose") {
|
|
718
|
+
return parseProposeArgs(rest);
|
|
719
|
+
}
|
|
720
|
+
if (command === "outer-loop") {
|
|
721
|
+
return parseOuterLoopArgs(rest);
|
|
722
|
+
}
|
|
723
|
+
if (command === "ignition") {
|
|
724
|
+
return parseIgnitionArgs(rest);
|
|
725
|
+
}
|
|
143
726
|
throw new Error(USAGE);
|
|
144
727
|
}
|
|
728
|
+
function printIgnition(input) {
|
|
729
|
+
if (input.json) {
|
|
730
|
+
console.log(JSON.stringify(input.report, null, 2));
|
|
731
|
+
return;
|
|
732
|
+
}
|
|
733
|
+
process.stdout.write(formatIgnitionMarkdown(input.report));
|
|
734
|
+
}
|
|
145
735
|
function printScorecard(input) {
|
|
146
736
|
if (input.json) {
|
|
147
737
|
console.log(JSON.stringify(input.scorecard, null, 2));
|
|
@@ -156,6 +746,80 @@ function printCandidate(input) {
|
|
|
156
746
|
}
|
|
157
747
|
process.stdout.write(formatCandidateMarkdown(input.record));
|
|
158
748
|
}
|
|
749
|
+
function printAlias(input) {
|
|
750
|
+
if (input.json) {
|
|
751
|
+
console.log(JSON.stringify(input.alias, null, 2));
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
process.stdout.write(formatAliasMarkdown(input.alias));
|
|
755
|
+
}
|
|
756
|
+
function printAliasMove(input) {
|
|
757
|
+
if (input.json) {
|
|
758
|
+
console.log(JSON.stringify(input.result, null, 2));
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
process.stdout.write(formatAliasMoveMarkdown(input.result));
|
|
762
|
+
}
|
|
763
|
+
function printPrivateVerifier(input) {
|
|
764
|
+
if (input.json) {
|
|
765
|
+
console.log(JSON.stringify(input.manifest, null, 2));
|
|
766
|
+
return;
|
|
767
|
+
}
|
|
768
|
+
process.stdout.write(formatPrivateVerifierMarkdown(input.manifest));
|
|
769
|
+
}
|
|
770
|
+
function printCampaign(input) {
|
|
771
|
+
if (input.json) {
|
|
772
|
+
console.log(JSON.stringify({
|
|
773
|
+
manifest: input.manifest,
|
|
774
|
+
plan: input.plan,
|
|
775
|
+
}, null, 2));
|
|
776
|
+
return;
|
|
777
|
+
}
|
|
778
|
+
process.stdout.write(formatCampaignMarkdown({
|
|
779
|
+
manifest: input.manifest,
|
|
780
|
+
plan: input.plan,
|
|
781
|
+
}));
|
|
782
|
+
}
|
|
783
|
+
function printExperiment(input) {
|
|
784
|
+
if (input.json) {
|
|
785
|
+
console.log(JSON.stringify({
|
|
786
|
+
record: input.record,
|
|
787
|
+
candidate: input.candidate,
|
|
788
|
+
...input.extra,
|
|
789
|
+
}, null, 2));
|
|
790
|
+
return;
|
|
791
|
+
}
|
|
792
|
+
process.stdout.write(formatExperimentMarkdown({
|
|
793
|
+
record: input.record,
|
|
794
|
+
candidate: input.candidate,
|
|
795
|
+
}));
|
|
796
|
+
}
|
|
797
|
+
function printExperimentDisposition(input) {
|
|
798
|
+
if (input.json) {
|
|
799
|
+
console.log(JSON.stringify(input.result, null, 2));
|
|
800
|
+
return;
|
|
801
|
+
}
|
|
802
|
+
const prefix = input.result.dryRun ? "dry-run " : "";
|
|
803
|
+
process.stdout.write(`${prefix}experiment ${input.result.record.status} (changed=${input.result.changed})\n`);
|
|
804
|
+
process.stdout.write(formatExperimentMarkdown({
|
|
805
|
+
record: input.result.record,
|
|
806
|
+
candidate: input.result.candidate,
|
|
807
|
+
}));
|
|
808
|
+
}
|
|
809
|
+
function printCorpus(input) {
|
|
810
|
+
if (input.json) {
|
|
811
|
+
console.log(JSON.stringify({
|
|
812
|
+
summary: input.summary,
|
|
813
|
+
manifest: input.manifest,
|
|
814
|
+
...input.extra,
|
|
815
|
+
}, null, 2));
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
process.stdout.write(formatCorpusMarkdown({
|
|
819
|
+
manifest: input.manifest,
|
|
820
|
+
summary: input.summary,
|
|
821
|
+
}));
|
|
822
|
+
}
|
|
159
823
|
export async function runEval(repoRoot, args) {
|
|
160
824
|
const parsed = parseEvalArgs(args);
|
|
161
825
|
if (parsed.command === "replay") {
|
|
@@ -182,6 +846,505 @@ export async function runEval(repoRoot, args) {
|
|
|
182
846
|
printScorecard({ scorecard, markdown, json: parsed.json });
|
|
183
847
|
return;
|
|
184
848
|
}
|
|
849
|
+
if (parsed.command === "alias") {
|
|
850
|
+
if (parsed.action === "show") {
|
|
851
|
+
const alias = await showAlias({ repoRoot, alias: parsed.alias });
|
|
852
|
+
printAlias({ alias, json: parsed.json });
|
|
853
|
+
return;
|
|
854
|
+
}
|
|
855
|
+
const rows = await listAliases({ repoRoot });
|
|
856
|
+
if (parsed.json) {
|
|
857
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
const lines = [
|
|
861
|
+
"# Aliases",
|
|
862
|
+
"",
|
|
863
|
+
...rows.map((row) => `- \`${row.alias}\` → \`${row.candidateId}\` hash=\`${row.bundleHash}\` decision=\`${row.updatedByDecisionId}\``),
|
|
864
|
+
"",
|
|
865
|
+
];
|
|
866
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
867
|
+
return;
|
|
868
|
+
}
|
|
869
|
+
if (parsed.command === "promote" || parsed.command === "rollback") {
|
|
870
|
+
if (parsed.command === "promote" && parsed.campaignId) {
|
|
871
|
+
await requireCampaignPromotionGate({
|
|
872
|
+
repoRoot,
|
|
873
|
+
campaignId: parsed.campaignId,
|
|
874
|
+
toCandidateId: parsed.toCandidateId,
|
|
875
|
+
acknowledgeReview: parsed.acknowledgeReview,
|
|
876
|
+
evidencePath: parsed.evidencePath,
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
const mover = parsed.command === "promote" ? promoteAlias : rollbackAlias;
|
|
880
|
+
const result = await mover({
|
|
881
|
+
repoRoot,
|
|
882
|
+
alias: parsed.alias,
|
|
883
|
+
toCandidateId: parsed.toCandidateId,
|
|
884
|
+
reason: parsed.reason,
|
|
885
|
+
dryRun: parsed.dryRun,
|
|
886
|
+
});
|
|
887
|
+
printAliasMove({ result, json: parsed.json });
|
|
888
|
+
return;
|
|
889
|
+
}
|
|
890
|
+
if (parsed.command === "context-policy") {
|
|
891
|
+
if (parsed.action === "list") {
|
|
892
|
+
const rows = listContextPolicies().map((policy) => policy.toManifest());
|
|
893
|
+
if (parsed.json) {
|
|
894
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
const lines = [
|
|
898
|
+
"# Context Policies",
|
|
899
|
+
"",
|
|
900
|
+
...rows.map((row) => `- \`${row.policyId}\`: ${row.description}`),
|
|
901
|
+
"",
|
|
902
|
+
];
|
|
903
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
904
|
+
return;
|
|
905
|
+
}
|
|
906
|
+
if (!isContextPolicyId(parsed.policyId)) {
|
|
907
|
+
throw new Error(`unknown context policy id: ${parsed.policyId}`);
|
|
908
|
+
}
|
|
909
|
+
const manifest = getContextPolicy(parsed.policyId).toManifest();
|
|
910
|
+
if (parsed.json) {
|
|
911
|
+
console.log(JSON.stringify(manifest, null, 2));
|
|
912
|
+
return;
|
|
913
|
+
}
|
|
914
|
+
process.stdout.write([
|
|
915
|
+
`# Context Policy: ${manifest.policyId}`,
|
|
916
|
+
"",
|
|
917
|
+
manifest.description,
|
|
918
|
+
"",
|
|
919
|
+
"```json",
|
|
920
|
+
JSON.stringify(manifest.knobs, null, 2),
|
|
921
|
+
"```",
|
|
922
|
+
"",
|
|
923
|
+
].join("\n"));
|
|
924
|
+
return;
|
|
925
|
+
}
|
|
926
|
+
if (parsed.command === "budget") {
|
|
927
|
+
const located = await locateDagRun(repoRoot, parsed.runId);
|
|
928
|
+
if (!located) {
|
|
929
|
+
throw new Error(`dag run not found: ${parsed.runId}`);
|
|
930
|
+
}
|
|
931
|
+
const state = await readDagRunState(located.runDir);
|
|
932
|
+
const ledger = state.budgetLedger;
|
|
933
|
+
if (!ledger) {
|
|
934
|
+
throw new Error(`dag run ${parsed.runId} has no budget ledger (DagSpec.budget was not declared)`);
|
|
935
|
+
}
|
|
936
|
+
if (parsed.json) {
|
|
937
|
+
console.log(JSON.stringify(ledger, null, 2));
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
process.stdout.write(formatBudgetReportMarkdown(ledger));
|
|
941
|
+
return;
|
|
942
|
+
}
|
|
943
|
+
if (parsed.command === "private-verifier") {
|
|
944
|
+
if (parsed.action === "validate") {
|
|
945
|
+
const result = await validatePrivateVerifier({
|
|
946
|
+
repoRoot,
|
|
947
|
+
manifestPath: parsed.manifestPath,
|
|
948
|
+
});
|
|
949
|
+
printPrivateVerifier({ manifest: result.manifest, json: parsed.json });
|
|
950
|
+
return;
|
|
951
|
+
}
|
|
952
|
+
if (parsed.action === "register") {
|
|
953
|
+
const result = await registerPrivateVerifier({
|
|
954
|
+
repoRoot,
|
|
955
|
+
manifestPath: parsed.manifestPath,
|
|
956
|
+
});
|
|
957
|
+
printPrivateVerifier({ manifest: result.manifest, json: parsed.json });
|
|
958
|
+
return;
|
|
959
|
+
}
|
|
960
|
+
if (parsed.action === "list") {
|
|
961
|
+
const rows = await listPrivateVerifiers({ repoRoot });
|
|
962
|
+
if (parsed.json) {
|
|
963
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
964
|
+
return;
|
|
965
|
+
}
|
|
966
|
+
const lines = [
|
|
967
|
+
"# Private Verifiers",
|
|
968
|
+
"",
|
|
969
|
+
...rows.map((row) => `- \`${row.privateVerifierId}\`: checks=${row.checkCount} hash=\`${row.manifestHash}\``),
|
|
970
|
+
"",
|
|
971
|
+
];
|
|
972
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
973
|
+
return;
|
|
974
|
+
}
|
|
975
|
+
if (parsed.action !== "show") {
|
|
976
|
+
throw new Error("usage: eval private-verifier <register|show|list|validate> ...");
|
|
977
|
+
}
|
|
978
|
+
const result = await showPrivateVerifier({
|
|
979
|
+
repoRoot,
|
|
980
|
+
privateVerifierId: parsed.privateVerifierId,
|
|
981
|
+
});
|
|
982
|
+
printPrivateVerifier({ manifest: result.manifest, json: parsed.json });
|
|
983
|
+
return;
|
|
984
|
+
}
|
|
985
|
+
if (parsed.command === "propose") {
|
|
986
|
+
const result = await proposeFromCurate({
|
|
987
|
+
repoRoot,
|
|
988
|
+
maxCandidates: parsed.maxCandidates,
|
|
989
|
+
proposerRunId: parsed.proposerRunId,
|
|
990
|
+
dryRun: parsed.dryRun,
|
|
991
|
+
});
|
|
992
|
+
if (parsed.json) {
|
|
993
|
+
console.log(JSON.stringify(result, null, 2));
|
|
994
|
+
return;
|
|
995
|
+
}
|
|
996
|
+
process.stdout.write(formatProposerRunMarkdown(result));
|
|
997
|
+
return;
|
|
998
|
+
}
|
|
999
|
+
if (parsed.command === "outer-loop") {
|
|
1000
|
+
const piExecutor = parsed.matrixMode === "pi" && parsed.dagTemplatePath
|
|
1001
|
+
? createDagDryRunPiCellExecutor({
|
|
1002
|
+
dagTemplatePath: parsed.dagTemplatePath,
|
|
1003
|
+
})
|
|
1004
|
+
: undefined;
|
|
1005
|
+
const result = await runOuterLoop({
|
|
1006
|
+
repoRoot,
|
|
1007
|
+
campaignId: parsed.campaignId,
|
|
1008
|
+
campaignManifestPath: parsed.campaignManifestPath,
|
|
1009
|
+
proposeFromCurate: parsed.proposeFromCurate,
|
|
1010
|
+
maxProposals: parsed.maxProposals,
|
|
1011
|
+
matrixMode: parsed.matrixMode,
|
|
1012
|
+
fixturePath: parsed.fixturePath,
|
|
1013
|
+
maxPublicCells: parsed.maxPublicCells,
|
|
1014
|
+
buildScorecard: parsed.buildScorecard,
|
|
1015
|
+
piExecutor,
|
|
1016
|
+
});
|
|
1017
|
+
if (parsed.json) {
|
|
1018
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1019
|
+
return;
|
|
1020
|
+
}
|
|
1021
|
+
process.stdout.write(formatOuterLoopMarkdown(result));
|
|
1022
|
+
return;
|
|
1023
|
+
}
|
|
1024
|
+
if (parsed.command === "ignition") {
|
|
1025
|
+
if (parsed.action === "list") {
|
|
1026
|
+
const rows = await listIgnitions({ repoRoot });
|
|
1027
|
+
if (parsed.json) {
|
|
1028
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
1029
|
+
return;
|
|
1030
|
+
}
|
|
1031
|
+
const lines = [
|
|
1032
|
+
"# Ignition Research Gates",
|
|
1033
|
+
"",
|
|
1034
|
+
...rows.map((row) => `- \`${row.ignitionId}\`: generations=${row.generationCount} verdict=\`${row.verdict}\` rsiLevel1ClaimAllowed=\`${row.rsiLevel1ClaimAllowed}\``),
|
|
1035
|
+
"",
|
|
1036
|
+
];
|
|
1037
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1038
|
+
return;
|
|
1039
|
+
}
|
|
1040
|
+
if (parsed.action === "record") {
|
|
1041
|
+
const report = await recordIgnitionGeneration({
|
|
1042
|
+
repoRoot,
|
|
1043
|
+
ignitionId: parsed.ignitionId,
|
|
1044
|
+
generation: parsed.generation,
|
|
1045
|
+
parentCandidateId: parsed.parentCandidateId,
|
|
1046
|
+
proposerRunId: parsed.proposerRunId,
|
|
1047
|
+
campaignId: parsed.campaignId,
|
|
1048
|
+
});
|
|
1049
|
+
printIgnition({ report, json: parsed.json });
|
|
1050
|
+
return;
|
|
1051
|
+
}
|
|
1052
|
+
if (parsed.action === "evaluate") {
|
|
1053
|
+
const report = await evaluateIgnition({
|
|
1054
|
+
repoRoot,
|
|
1055
|
+
ignitionId: parsed.ignitionId,
|
|
1056
|
+
});
|
|
1057
|
+
printIgnition({ report, json: parsed.json });
|
|
1058
|
+
return;
|
|
1059
|
+
}
|
|
1060
|
+
if (parsed.action === "show") {
|
|
1061
|
+
const report = await showIgnition({
|
|
1062
|
+
repoRoot,
|
|
1063
|
+
ignitionId: parsed.ignitionId,
|
|
1064
|
+
});
|
|
1065
|
+
printIgnition({ report, json: parsed.json });
|
|
1066
|
+
return;
|
|
1067
|
+
}
|
|
1068
|
+
throw new Error("usage: eval ignition <record|evaluate|show|list> ...");
|
|
1069
|
+
}
|
|
1070
|
+
if (parsed.command === "experiment") {
|
|
1071
|
+
if (parsed.action === "from-curate") {
|
|
1072
|
+
const result = await createExperimentFromCurate({
|
|
1073
|
+
repoRoot,
|
|
1074
|
+
experimentId: parsed.experimentId,
|
|
1075
|
+
candidateId: parsed.candidateId,
|
|
1076
|
+
});
|
|
1077
|
+
printExperiment({
|
|
1078
|
+
record: result.record,
|
|
1079
|
+
candidate: result.candidate,
|
|
1080
|
+
json: parsed.json,
|
|
1081
|
+
extra: {
|
|
1082
|
+
idempotent: result.idempotent,
|
|
1083
|
+
curateMessage: result.curateMessage,
|
|
1084
|
+
},
|
|
1085
|
+
});
|
|
1086
|
+
return;
|
|
1087
|
+
}
|
|
1088
|
+
if (parsed.action === "list") {
|
|
1089
|
+
const rows = await listExperiments({ repoRoot });
|
|
1090
|
+
if (parsed.json) {
|
|
1091
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
const lines = [
|
|
1095
|
+
"# Experiments",
|
|
1096
|
+
"",
|
|
1097
|
+
...rows.map((row) => `- \`${row.experimentId}\`: status=${row.status} candidate=\`${row.candidateId}\` patterns=${row.patternCount}`),
|
|
1098
|
+
"",
|
|
1099
|
+
];
|
|
1100
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1101
|
+
return;
|
|
1102
|
+
}
|
|
1103
|
+
if (parsed.action === "show") {
|
|
1104
|
+
const result = await showExperiment({
|
|
1105
|
+
repoRoot,
|
|
1106
|
+
experimentId: parsed.experimentId,
|
|
1107
|
+
});
|
|
1108
|
+
printExperiment({
|
|
1109
|
+
record: result.record,
|
|
1110
|
+
candidate: result.candidate,
|
|
1111
|
+
json: parsed.json,
|
|
1112
|
+
});
|
|
1113
|
+
return;
|
|
1114
|
+
}
|
|
1115
|
+
if (parsed.action === "accept") {
|
|
1116
|
+
const result = await acceptExperiment({
|
|
1117
|
+
repoRoot,
|
|
1118
|
+
experimentId: parsed.experimentId,
|
|
1119
|
+
campaignId: parsed.campaignId,
|
|
1120
|
+
reason: parsed.reason,
|
|
1121
|
+
acknowledgeReview: parsed.acknowledgeReview,
|
|
1122
|
+
dryRun: parsed.dryRun,
|
|
1123
|
+
});
|
|
1124
|
+
printExperimentDisposition({ result, json: parsed.json });
|
|
1125
|
+
return;
|
|
1126
|
+
}
|
|
1127
|
+
if (parsed.action === "reject") {
|
|
1128
|
+
const result = await rejectExperiment({
|
|
1129
|
+
repoRoot,
|
|
1130
|
+
experimentId: parsed.experimentId,
|
|
1131
|
+
reason: parsed.reason,
|
|
1132
|
+
dryRun: parsed.dryRun,
|
|
1133
|
+
});
|
|
1134
|
+
printExperimentDisposition({ result, json: parsed.json });
|
|
1135
|
+
return;
|
|
1136
|
+
}
|
|
1137
|
+
throw new Error("usage: eval experiment <from-curate|show|list|accept|reject> ...");
|
|
1138
|
+
}
|
|
1139
|
+
if (parsed.command === "campaign") {
|
|
1140
|
+
if (parsed.action === "validate") {
|
|
1141
|
+
const result = await validateCampaign({
|
|
1142
|
+
repoRoot,
|
|
1143
|
+
manifestPath: parsed.manifestPath,
|
|
1144
|
+
});
|
|
1145
|
+
printCampaign({
|
|
1146
|
+
manifest: result.manifest,
|
|
1147
|
+
plan: result.plan,
|
|
1148
|
+
json: parsed.json,
|
|
1149
|
+
});
|
|
1150
|
+
return;
|
|
1151
|
+
}
|
|
1152
|
+
if (parsed.action === "create") {
|
|
1153
|
+
const result = await createCampaign({
|
|
1154
|
+
repoRoot,
|
|
1155
|
+
manifestPath: parsed.manifestPath,
|
|
1156
|
+
});
|
|
1157
|
+
printCampaign({
|
|
1158
|
+
manifest: result.manifest,
|
|
1159
|
+
plan: result.plan,
|
|
1160
|
+
json: parsed.json,
|
|
1161
|
+
});
|
|
1162
|
+
return;
|
|
1163
|
+
}
|
|
1164
|
+
if (parsed.action === "list") {
|
|
1165
|
+
const rows = await listCampaigns({ repoRoot });
|
|
1166
|
+
if (parsed.json) {
|
|
1167
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
1168
|
+
return;
|
|
1169
|
+
}
|
|
1170
|
+
const lines = [
|
|
1171
|
+
"# Campaigns",
|
|
1172
|
+
"",
|
|
1173
|
+
...rows.map((row) => `- \`${row.campaignId}\`: status=${row.status} corpus=\`${row.corpusId}\` cells=${row.cellCount ?? "-"}`),
|
|
1174
|
+
"",
|
|
1175
|
+
];
|
|
1176
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1177
|
+
return;
|
|
1178
|
+
}
|
|
1179
|
+
if (parsed.action === "plan") {
|
|
1180
|
+
const result = await planCampaign({
|
|
1181
|
+
repoRoot,
|
|
1182
|
+
campaignId: parsed.campaignId,
|
|
1183
|
+
});
|
|
1184
|
+
printCampaign({
|
|
1185
|
+
manifest: result.manifest,
|
|
1186
|
+
plan: result.plan,
|
|
1187
|
+
json: parsed.json,
|
|
1188
|
+
});
|
|
1189
|
+
return;
|
|
1190
|
+
}
|
|
1191
|
+
if (parsed.action === "run") {
|
|
1192
|
+
const result = await runCampaignPrivateChecks({
|
|
1193
|
+
repoRoot,
|
|
1194
|
+
campaignId: parsed.campaignId,
|
|
1195
|
+
});
|
|
1196
|
+
if (parsed.json) {
|
|
1197
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1198
|
+
return;
|
|
1199
|
+
}
|
|
1200
|
+
const failed = result.results.filter((row) => !row.ok);
|
|
1201
|
+
process.stdout.write(formatCampaignMarkdown({
|
|
1202
|
+
manifest: result.manifest,
|
|
1203
|
+
plan: result.plan,
|
|
1204
|
+
}));
|
|
1205
|
+
process.stdout.write(`\n## Private check results\n\n- total: ${result.results.length}\n- failed: ${failed.length}\n\n`);
|
|
1206
|
+
if (failed.length > 0) {
|
|
1207
|
+
throw new Error(`campaign private checks failed: ${failed.map((row) => `${row.cellId}/${row.checkId}`).join(", ")}`);
|
|
1208
|
+
}
|
|
1209
|
+
return;
|
|
1210
|
+
}
|
|
1211
|
+
if (parsed.action === "matrix") {
|
|
1212
|
+
if (parsed.mode === "pi" && !parsed.dagTemplatePath) {
|
|
1213
|
+
throw new Error("eval campaign matrix refuses mode=pi without --dag-template (bounded dry-run Pi) or injected executor; use --mode pi-plan or stub");
|
|
1214
|
+
}
|
|
1215
|
+
const piExecutor = parsed.mode === "pi" && parsed.dagTemplatePath
|
|
1216
|
+
? createDagDryRunPiCellExecutor({
|
|
1217
|
+
dagTemplatePath: parsed.dagTemplatePath,
|
|
1218
|
+
})
|
|
1219
|
+
: undefined;
|
|
1220
|
+
const result = await runCampaignMatrix({
|
|
1221
|
+
repoRoot,
|
|
1222
|
+
campaignId: parsed.campaignId,
|
|
1223
|
+
mode: parsed.mode,
|
|
1224
|
+
fixturePath: parsed.fixturePath,
|
|
1225
|
+
maxPublicCells: parsed.maxPublicCells,
|
|
1226
|
+
piExecutor,
|
|
1227
|
+
});
|
|
1228
|
+
if (parsed.json) {
|
|
1229
|
+
console.log(JSON.stringify(result.matrix, null, 2));
|
|
1230
|
+
return;
|
|
1231
|
+
}
|
|
1232
|
+
process.stdout.write(formatMatrixRunMarkdown(result.matrix));
|
|
1233
|
+
return;
|
|
1234
|
+
}
|
|
1235
|
+
if (parsed.action === "scorecard" || parsed.action === "audit") {
|
|
1236
|
+
const evidence = await loadCampaignEvidence({
|
|
1237
|
+
repoRoot,
|
|
1238
|
+
evidencePath: parsed.evidencePath,
|
|
1239
|
+
});
|
|
1240
|
+
if (parsed.action === "audit") {
|
|
1241
|
+
const result = await auditCampaignEvidence({
|
|
1242
|
+
repoRoot,
|
|
1243
|
+
campaignId: parsed.campaignId,
|
|
1244
|
+
evidence,
|
|
1245
|
+
});
|
|
1246
|
+
if (parsed.json) {
|
|
1247
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1248
|
+
return;
|
|
1249
|
+
}
|
|
1250
|
+
const scorecard = {
|
|
1251
|
+
schemaVersion: 1,
|
|
1252
|
+
campaignId: parsed.campaignId,
|
|
1253
|
+
planHash: "(audit-only)",
|
|
1254
|
+
createdAt: new Date().toISOString(),
|
|
1255
|
+
evidenceCellCount: evidence.cells.length,
|
|
1256
|
+
policy: result.policy,
|
|
1257
|
+
};
|
|
1258
|
+
process.stdout.write(formatCampaignScorecardMarkdown(scorecard));
|
|
1259
|
+
return;
|
|
1260
|
+
}
|
|
1261
|
+
const scorecard = await buildCampaignScorecard({
|
|
1262
|
+
repoRoot,
|
|
1263
|
+
campaignId: parsed.campaignId,
|
|
1264
|
+
evidence,
|
|
1265
|
+
acknowledgeReview: parsed.acknowledgeReview,
|
|
1266
|
+
persist: true,
|
|
1267
|
+
});
|
|
1268
|
+
if (parsed.json) {
|
|
1269
|
+
console.log(JSON.stringify(scorecard, null, 2));
|
|
1270
|
+
return;
|
|
1271
|
+
}
|
|
1272
|
+
process.stdout.write(formatCampaignScorecardMarkdown(scorecard));
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
1275
|
+
if (parsed.action !== "show") {
|
|
1276
|
+
throw new Error("usage: eval campaign <create|show|list|validate|plan|run|matrix|scorecard|audit> ...");
|
|
1277
|
+
}
|
|
1278
|
+
const result = await showCampaign({
|
|
1279
|
+
repoRoot,
|
|
1280
|
+
campaignId: parsed.campaignId,
|
|
1281
|
+
});
|
|
1282
|
+
printCampaign({
|
|
1283
|
+
manifest: result.manifest,
|
|
1284
|
+
plan: result.plan,
|
|
1285
|
+
json: parsed.json,
|
|
1286
|
+
});
|
|
1287
|
+
return;
|
|
1288
|
+
}
|
|
1289
|
+
if (parsed.command === "corpus") {
|
|
1290
|
+
if (parsed.action === "validate") {
|
|
1291
|
+
const result = await validateCorpus({
|
|
1292
|
+
repoRoot,
|
|
1293
|
+
manifestPath: parsed.manifestPath,
|
|
1294
|
+
});
|
|
1295
|
+
printCorpus({
|
|
1296
|
+
manifest: result.manifest,
|
|
1297
|
+
summary: result.summary,
|
|
1298
|
+
json: parsed.json,
|
|
1299
|
+
});
|
|
1300
|
+
return;
|
|
1301
|
+
}
|
|
1302
|
+
if (parsed.action === "register") {
|
|
1303
|
+
const result = await registerCorpus({
|
|
1304
|
+
repoRoot,
|
|
1305
|
+
manifestPath: parsed.manifestPath,
|
|
1306
|
+
});
|
|
1307
|
+
if (parsed.json) {
|
|
1308
|
+
printCorpus({
|
|
1309
|
+
manifest: result.manifest,
|
|
1310
|
+
summary: result.summary,
|
|
1311
|
+
json: true,
|
|
1312
|
+
extra: {
|
|
1313
|
+
idempotent: result.idempotent,
|
|
1314
|
+
manifestPath: result.manifestPath,
|
|
1315
|
+
},
|
|
1316
|
+
});
|
|
1317
|
+
return;
|
|
1318
|
+
}
|
|
1319
|
+
process.stdout.write(`${result.idempotent ? "idempotent " : ""}registered ${result.manifest.corpusId}\n${formatCorpusMarkdown(result)}`);
|
|
1320
|
+
return;
|
|
1321
|
+
}
|
|
1322
|
+
if (parsed.action === "show") {
|
|
1323
|
+
const result = await showCorpus({
|
|
1324
|
+
repoRoot,
|
|
1325
|
+
corpusId: parsed.corpusId,
|
|
1326
|
+
});
|
|
1327
|
+
printCorpus({
|
|
1328
|
+
manifest: result.manifest,
|
|
1329
|
+
summary: result.summary,
|
|
1330
|
+
json: parsed.json,
|
|
1331
|
+
});
|
|
1332
|
+
return;
|
|
1333
|
+
}
|
|
1334
|
+
const rows = await listCorpora({ repoRoot });
|
|
1335
|
+
if (parsed.json) {
|
|
1336
|
+
console.log(JSON.stringify(rows, null, 2));
|
|
1337
|
+
return;
|
|
1338
|
+
}
|
|
1339
|
+
const lines = [
|
|
1340
|
+
"# Corpora",
|
|
1341
|
+
"",
|
|
1342
|
+
...rows.map((row) => `- \`${row.corpusId}\` tasks=${row.taskCount} hash=\`${row.corpusHash}\` splits=public:${row.splits.public}/private:${row.splits.private}/held_out:${row.splits.held_out}`),
|
|
1343
|
+
"",
|
|
1344
|
+
];
|
|
1345
|
+
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1346
|
+
return;
|
|
1347
|
+
}
|
|
185
1348
|
// candidate subcommands
|
|
186
1349
|
if (parsed.action === "register") {
|
|
187
1350
|
const result = await registerCandidate({
|
|
@@ -217,7 +1380,7 @@ export async function runEval(repoRoot, args) {
|
|
|
217
1380
|
const lines = [
|
|
218
1381
|
"# Candidates",
|
|
219
1382
|
"",
|
|
220
|
-
...rows.map((row) => `- \`${row.candidateId}\` status=\`${row.status}\` hash=\`${row.bundleHash}\` promotionApplied
|
|
1383
|
+
...rows.map((row) => `- \`${row.candidateId}\` status=\`${row.status}\` hash=\`${row.bundleHash}\` promotionApplied=${row.promotionApplied}`),
|
|
221
1384
|
"",
|
|
222
1385
|
];
|
|
223
1386
|
process.stdout.write(`${lines.join("\n")}\n`);
|