@tea-agent/loop-agent 0.16.0 → 0.16.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +3 -55
- package/README.md +1 -1
- package/dist/application/evaluation/candidate.js +6 -23
- package/dist/application/evaluation/types.js +0 -522
- package/dist/cli/command-definitions.js +3 -19
- package/dist/commands/eval.js +13 -1176
- package/dist/commands/init.js +1 -4
- package/dist/executors/dag-pi-executor.js +44 -4
- package/dist/worker/delivery/verification-bundle.js +2 -13
- package/dist/worker/feature/review.js +2 -3
- package/dist/worker/outcomes/adapters.js +1 -4
- package/dist/worker/outcomes/gate.js +11 -10
- package/dist/worker/outcomes/projector.js +1 -10
- package/dist/worker/outcomes/types.js +0 -3
- package/dist/worker/run-task/run-task.js +4 -23
- package/dist/worker/task-graph/ready-planner.js +8 -14
- package/dist/worker/task-graph/task-graph-schema.js +3 -5
- package/dist/workflows/dag/init-hybrid.js +3 -2
- package/dist/workflows/dag/knowledge-curator.js +0 -3
- package/dist/workflows/dag/node-execution.js +4 -11
- package/dist/workflows/dag/prompt.js +1 -1
- package/dist/workflows/dag/runner.js +16 -43
- package/dist/workflows/dag/skill-snapshot.js +7 -11
- package/dist/workflows/dag/types.js +1 -18
- package/docs/init-surface.manifest.json +0 -3
- package/docs/templates/agent-dag.schema.json +5 -0
- package/docs/templates/product-line/AGENTS.md +0 -1
- package/docs/templates/product-line/README.md +0 -17
- package/docs/templates/product-line/acceptance.yaml +0 -9
- package/docs/templates/product-line/task-graph.yaml +0 -8
- package/docs/templates/product-line/task.yaml +0 -4
- package/package.json +1 -1
- package/skills/frontend-implementation/references/node-contracts.md +2 -2
- package/skills/loop-agent/references/hybrid-dag.md +1 -1
- package/dist/application/evaluation/alias.js +0 -184
- package/dist/application/evaluation/budget.js +0 -192
- package/dist/application/evaluation/campaign-hash.js +0 -47
- package/dist/application/evaluation/campaign-matrix.js +0 -372
- package/dist/application/evaluation/campaign-scorecard.js +0 -135
- package/dist/application/evaluation/campaign.js +0 -370
- package/dist/application/evaluation/corpus-hash.js +0 -38
- package/dist/application/evaluation/corpus.js +0 -56
- package/dist/application/evaluation/experiment.js +0 -294
- package/dist/application/evaluation/ignition.js +0 -198
- package/dist/application/evaluation/integrity-audit.js +0 -162
- package/dist/application/evaluation/outer-loop.js +0 -132
- package/dist/application/evaluation/pi-cell-executor.js +0 -39
- package/dist/application/evaluation/private-verifier.js +0 -46
- package/dist/application/evaluation/promotion-policy.js +0 -151
- package/dist/application/evaluation/proposer.js +0 -98
- package/dist/infrastructure/evaluation/alias-store.js +0 -199
- package/dist/infrastructure/evaluation/campaign-store.js +0 -154
- package/dist/infrastructure/evaluation/corpus-store.js +0 -181
- package/dist/infrastructure/evaluation/experiment-store.js +0 -124
- package/dist/infrastructure/evaluation/ignition-store.js +0 -82
- package/dist/infrastructure/evaluation/private-verifier-store.js +0 -145
- package/dist/infrastructure/evaluation/proposer-store.js +0 -78
- package/dist/worker/outcomes/evidence-tokens.js +0 -29
- package/dist/workflows/dag/budget-enforcement.js +0 -67
- package/dist/workflows/dag/context-policy.js +0 -137
- package/docs/templates/evaluation/campaign-budget-v1.json +0 -12
- package/docs/templates/evaluation/campaign-dogfood-v0.json +0 -24
- package/docs/templates/evaluation/campaign-evidence-v1.json +0 -44
- package/docs/templates/evaluation/context-policy-baseline-v1.json +0 -17
- package/docs/templates/evaluation/context-policy-role-specialized-v1.json +0 -28
- package/docs/templates/evaluation/corpus-dogfood-v0.manifest.json +0 -118
- package/docs/templates/evaluation/matrix-dag-dry-run-v1.json +0 -21
- package/docs/templates/evaluation/matrix-fixture-v1.json +0 -10
- package/docs/templates/evaluation/private-verifier-dogfood-v0.json +0 -16
- package/docs/templates/product-line/feature.yaml +0 -11
package/dist/commands/eval.js
CHANGED
|
@@ -3,22 +3,8 @@ import { writeTextAtomic } from "../infrastructure/harness/atomic-write.js";
|
|
|
3
3
|
import { readReplayScorecard } from "../infrastructure/evaluation/store.js";
|
|
4
4
|
import { formatReplayMarkdown, replayEvaluation, } from "../application/evaluation/replay.js";
|
|
5
5
|
import { formatCandidateMarkdown, listCandidates, registerCandidate, showCandidate, transitionCandidate, } from "../application/evaluation/candidate.js";
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
import { formatBudgetReportMarkdown, } from "../application/evaluation/budget.js";
|
|
9
|
-
import { createCampaign, formatCampaignMarkdown, listCampaigns, planCampaign, runCampaignPrivateChecks, showCampaign, validateCampaign, } from "../application/evaluation/campaign.js";
|
|
10
|
-
import { auditCampaignEvidence, buildCampaignScorecard, formatCampaignScorecardMarkdown, loadCampaignEvidence, requireCampaignPromotionGate, } from "../application/evaluation/campaign-scorecard.js";
|
|
11
|
-
import { acceptExperiment, createExperimentFromCurate, formatExperimentMarkdown, listExperiments, rejectExperiment, showExperiment, } from "../application/evaluation/experiment.js";
|
|
12
|
-
import { formatMatrixRunMarkdown, runCampaignMatrix, } from "../application/evaluation/campaign-matrix.js";
|
|
13
|
-
import { formatOuterLoopMarkdown, runOuterLoop, } from "../application/evaluation/outer-loop.js";
|
|
14
|
-
import { evaluateIgnition, formatIgnitionMarkdown, listIgnitions, recordIgnitionGeneration, showIgnition, } from "../application/evaluation/ignition.js";
|
|
15
|
-
import { createDagDryRunPiCellExecutor } from "../application/evaluation/pi-cell-executor.js";
|
|
16
|
-
import { formatProposerRunMarkdown, proposeFromCurate, } from "../application/evaluation/proposer.js";
|
|
17
|
-
import { formatPrivateVerifierMarkdown, listPrivateVerifiers, registerPrivateVerifier, showPrivateVerifier, validatePrivateVerifier, } from "../application/evaluation/private-verifier.js";
|
|
18
|
-
import { getContextPolicy, isContextPolicyId, listContextPolicies, } from "../workflows/dag/context-policy.js";
|
|
19
|
-
import { locateDagRun, readDagRunState } from "../workflows/dag/lifecycle.js";
|
|
20
|
-
import { lifecycleStateSchema, matrixModeSchema, } from "../application/evaluation/types.js";
|
|
21
|
-
const USAGE = "usage: eval <…|propose|outer-loop|ignition|campaign|experiment> ...; ignition <record|evaluate|show|list>; campaign matrix [--mode pi --dag-template <path>]; promote/ignition never auto-promote; ignition never claims RSI Level 1";
|
|
6
|
+
import { lifecycleStateSchema, } from "../application/evaluation/types.js";
|
|
7
|
+
const USAGE = "usage: eval <replay|report|candidate> ...; candidate <register|show|list|transition> [--json|--markdown]";
|
|
22
8
|
function parseFormatFlags(args) {
|
|
23
9
|
const json = args.includes("--json");
|
|
24
10
|
const markdown = args.includes("--markdown");
|
|
@@ -27,15 +13,6 @@ function parseFormatFlags(args) {
|
|
|
27
13
|
}
|
|
28
14
|
return { json: json || !markdown, markdown };
|
|
29
15
|
}
|
|
30
|
-
function parseDryRunFlags(args) {
|
|
31
|
-
const dryRun = args.includes("--dry-run");
|
|
32
|
-
const apply = args.includes("--apply");
|
|
33
|
-
if (dryRun && apply) {
|
|
34
|
-
throw new Error("eval accepts only one of --dry-run or --apply");
|
|
35
|
-
}
|
|
36
|
-
// Default dry-run: human gate; --apply commits alias move.
|
|
37
|
-
return !apply;
|
|
38
|
-
}
|
|
39
16
|
function flagValue(args, flag) {
|
|
40
17
|
const index = args.indexOf(flag);
|
|
41
18
|
if (index >= 0) {
|
|
@@ -48,37 +25,6 @@ function flagValue(args, flag) {
|
|
|
48
25
|
const prefix = `${flag}=`;
|
|
49
26
|
return args.find((arg) => arg.startsWith(prefix))?.slice(prefix.length);
|
|
50
27
|
}
|
|
51
|
-
const VALUE_FLAGS = [
|
|
52
|
-
"--spec",
|
|
53
|
-
"--output",
|
|
54
|
-
"--replay-id",
|
|
55
|
-
"--manifest",
|
|
56
|
-
"--candidate-id",
|
|
57
|
-
"--corpus-id",
|
|
58
|
-
"--to",
|
|
59
|
-
"--to-candidate",
|
|
60
|
-
"--reason",
|
|
61
|
-
"--alias",
|
|
62
|
-
"--policy-id",
|
|
63
|
-
"--run-id",
|
|
64
|
-
"--private-verifier-id",
|
|
65
|
-
"--campaign-id",
|
|
66
|
-
"--evidence",
|
|
67
|
-
"--experiment-id",
|
|
68
|
-
"--mode",
|
|
69
|
-
"--fixture",
|
|
70
|
-
"--max-public-cells",
|
|
71
|
-
"--max",
|
|
72
|
-
"--proposer-run-id",
|
|
73
|
-
"--campaign-manifest",
|
|
74
|
-
"--matrix-mode",
|
|
75
|
-
"--max-proposals",
|
|
76
|
-
"--dag-template",
|
|
77
|
-
"--ignition-id",
|
|
78
|
-
"--generation",
|
|
79
|
-
"--parent-candidate",
|
|
80
|
-
"--proposer-run-id",
|
|
81
|
-
];
|
|
82
28
|
function assertKnownFlags(args, allowed) {
|
|
83
29
|
for (let i = 0; i < args.length; i += 1) {
|
|
84
30
|
const arg = args[i];
|
|
@@ -88,7 +34,16 @@ function assertKnownFlags(args, allowed) {
|
|
|
88
34
|
if (!allowed.includes(key)) {
|
|
89
35
|
throw new Error(`unknown eval argument: ${arg}`);
|
|
90
36
|
}
|
|
91
|
-
if (
|
|
37
|
+
if ([
|
|
38
|
+
"--spec",
|
|
39
|
+
"--output",
|
|
40
|
+
"--replay-id",
|
|
41
|
+
"--manifest",
|
|
42
|
+
"--candidate-id",
|
|
43
|
+
"--to",
|
|
44
|
+
"--reason",
|
|
45
|
+
].includes(key) &&
|
|
46
|
+
!arg.includes("=")) {
|
|
92
47
|
i += 1;
|
|
93
48
|
}
|
|
94
49
|
}
|
|
@@ -161,511 +116,6 @@ function parseCandidateArgs(rest) {
|
|
|
161
116
|
}
|
|
162
117
|
throw new Error("usage: eval candidate <register|show|list|transition> ...");
|
|
163
118
|
}
|
|
164
|
-
function parseAliasArgs(rest) {
|
|
165
|
-
const [action, ...tail] = rest;
|
|
166
|
-
if (action === "show") {
|
|
167
|
-
assertKnownFlags(tail, ["--alias", "--json", "--markdown"]);
|
|
168
|
-
const alias = flagValue(tail, "--alias");
|
|
169
|
-
if (!alias)
|
|
170
|
-
throw new Error("eval alias show requires --alias <name>");
|
|
171
|
-
return {
|
|
172
|
-
command: "alias",
|
|
173
|
-
action: "show",
|
|
174
|
-
alias,
|
|
175
|
-
...parseFormatFlags(tail),
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
if (action === "list") {
|
|
179
|
-
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
180
|
-
return {
|
|
181
|
-
command: "alias",
|
|
182
|
-
action: "list",
|
|
183
|
-
...parseFormatFlags(tail),
|
|
184
|
-
};
|
|
185
|
-
}
|
|
186
|
-
throw new Error("usage: eval alias <show|list> ...");
|
|
187
|
-
}
|
|
188
|
-
function parseContextPolicyArgs(rest) {
|
|
189
|
-
const [action, ...tail] = rest;
|
|
190
|
-
if (action === "list") {
|
|
191
|
-
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
192
|
-
return {
|
|
193
|
-
command: "context-policy",
|
|
194
|
-
action: "list",
|
|
195
|
-
...parseFormatFlags(tail),
|
|
196
|
-
};
|
|
197
|
-
}
|
|
198
|
-
if (action === "show") {
|
|
199
|
-
assertKnownFlags(tail, ["--policy-id", "--json", "--markdown"]);
|
|
200
|
-
const policyId = flagValue(tail, "--policy-id");
|
|
201
|
-
if (!policyId) {
|
|
202
|
-
throw new Error("eval context-policy show requires --policy-id <id>");
|
|
203
|
-
}
|
|
204
|
-
return {
|
|
205
|
-
command: "context-policy",
|
|
206
|
-
action: "show",
|
|
207
|
-
policyId,
|
|
208
|
-
...parseFormatFlags(tail),
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
throw new Error("usage: eval context-policy <list|show> ...");
|
|
212
|
-
}
|
|
213
|
-
function parseBudgetArgs(rest) {
|
|
214
|
-
const [action, ...tail] = rest;
|
|
215
|
-
if (action === "show") {
|
|
216
|
-
assertKnownFlags(tail, ["--run-id", "--json", "--markdown"]);
|
|
217
|
-
const runId = flagValue(tail, "--run-id");
|
|
218
|
-
if (!runId) {
|
|
219
|
-
throw new Error("eval budget show requires --run-id <id>");
|
|
220
|
-
}
|
|
221
|
-
return {
|
|
222
|
-
command: "budget",
|
|
223
|
-
action: "show",
|
|
224
|
-
runId,
|
|
225
|
-
...parseFormatFlags(tail),
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
throw new Error("usage: eval budget show --run-id <id> [--json|--markdown]");
|
|
229
|
-
}
|
|
230
|
-
function parsePrivateVerifierArgs(rest) {
|
|
231
|
-
const [action, ...tail] = rest;
|
|
232
|
-
if (action === "register" || action === "validate") {
|
|
233
|
-
assertKnownFlags(tail, ["--manifest", "--json", "--markdown"]);
|
|
234
|
-
const manifestPath = flagValue(tail, "--manifest");
|
|
235
|
-
if (!manifestPath) {
|
|
236
|
-
throw new Error(`eval private-verifier ${action} requires --manifest <path>`);
|
|
237
|
-
}
|
|
238
|
-
return {
|
|
239
|
-
command: "private-verifier",
|
|
240
|
-
action,
|
|
241
|
-
manifestPath,
|
|
242
|
-
...parseFormatFlags(tail),
|
|
243
|
-
};
|
|
244
|
-
}
|
|
245
|
-
if (action === "show") {
|
|
246
|
-
assertKnownFlags(tail, ["--private-verifier-id", "--json", "--markdown"]);
|
|
247
|
-
const privateVerifierId = flagValue(tail, "--private-verifier-id");
|
|
248
|
-
if (!privateVerifierId) {
|
|
249
|
-
throw new Error("eval private-verifier show requires --private-verifier-id <id>");
|
|
250
|
-
}
|
|
251
|
-
return {
|
|
252
|
-
command: "private-verifier",
|
|
253
|
-
action: "show",
|
|
254
|
-
privateVerifierId,
|
|
255
|
-
...parseFormatFlags(tail),
|
|
256
|
-
};
|
|
257
|
-
}
|
|
258
|
-
if (action === "list") {
|
|
259
|
-
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
260
|
-
return {
|
|
261
|
-
command: "private-verifier",
|
|
262
|
-
action: "list",
|
|
263
|
-
...parseFormatFlags(tail),
|
|
264
|
-
};
|
|
265
|
-
}
|
|
266
|
-
throw new Error("usage: eval private-verifier <register|show|list|validate> ...");
|
|
267
|
-
}
|
|
268
|
-
function parseCampaignArgs(rest) {
|
|
269
|
-
const [action, ...tail] = rest;
|
|
270
|
-
if (action === "create" || action === "validate") {
|
|
271
|
-
assertKnownFlags(tail, ["--manifest", "--json", "--markdown"]);
|
|
272
|
-
const manifestPath = flagValue(tail, "--manifest");
|
|
273
|
-
if (!manifestPath) {
|
|
274
|
-
throw new Error(`eval campaign ${action} requires --manifest <path>`);
|
|
275
|
-
}
|
|
276
|
-
return {
|
|
277
|
-
command: "campaign",
|
|
278
|
-
action,
|
|
279
|
-
manifestPath,
|
|
280
|
-
...parseFormatFlags(tail),
|
|
281
|
-
};
|
|
282
|
-
}
|
|
283
|
-
if (action === "show" || action === "plan" || action === "run") {
|
|
284
|
-
assertKnownFlags(tail, ["--campaign-id", "--json", "--markdown"]);
|
|
285
|
-
const campaignId = flagValue(tail, "--campaign-id");
|
|
286
|
-
if (!campaignId) {
|
|
287
|
-
throw new Error(`eval campaign ${action} requires --campaign-id <id>`);
|
|
288
|
-
}
|
|
289
|
-
return {
|
|
290
|
-
command: "campaign",
|
|
291
|
-
action,
|
|
292
|
-
campaignId,
|
|
293
|
-
...parseFormatFlags(tail),
|
|
294
|
-
};
|
|
295
|
-
}
|
|
296
|
-
if (action === "matrix") {
|
|
297
|
-
assertKnownFlags(tail, [
|
|
298
|
-
"--campaign-id",
|
|
299
|
-
"--mode",
|
|
300
|
-
"--fixture",
|
|
301
|
-
"--dag-template",
|
|
302
|
-
"--max-public-cells",
|
|
303
|
-
"--json",
|
|
304
|
-
"--markdown",
|
|
305
|
-
]);
|
|
306
|
-
const campaignId = flagValue(tail, "--campaign-id");
|
|
307
|
-
const modeRaw = flagValue(tail, "--mode") ?? "dry-run";
|
|
308
|
-
if (!campaignId) {
|
|
309
|
-
throw new Error("eval campaign matrix requires --campaign-id <id>");
|
|
310
|
-
}
|
|
311
|
-
const mode = matrixModeSchema.parse(modeRaw);
|
|
312
|
-
const maxRaw = flagValue(tail, "--max-public-cells");
|
|
313
|
-
return {
|
|
314
|
-
command: "campaign",
|
|
315
|
-
action: "matrix",
|
|
316
|
-
campaignId,
|
|
317
|
-
mode,
|
|
318
|
-
fixturePath: flagValue(tail, "--fixture"),
|
|
319
|
-
dagTemplatePath: flagValue(tail, "--dag-template"),
|
|
320
|
-
maxPublicCells: maxRaw ? Number(maxRaw) : undefined,
|
|
321
|
-
...parseFormatFlags(tail),
|
|
322
|
-
};
|
|
323
|
-
}
|
|
324
|
-
if (action === "scorecard" || action === "audit") {
|
|
325
|
-
assertKnownFlags(tail, [
|
|
326
|
-
"--campaign-id",
|
|
327
|
-
"--evidence",
|
|
328
|
-
"--acknowledge-review",
|
|
329
|
-
"--json",
|
|
330
|
-
"--markdown",
|
|
331
|
-
]);
|
|
332
|
-
const campaignId = flagValue(tail, "--campaign-id");
|
|
333
|
-
const evidencePath = flagValue(tail, "--evidence");
|
|
334
|
-
if (!campaignId) {
|
|
335
|
-
throw new Error(`eval campaign ${action} requires --campaign-id <id>`);
|
|
336
|
-
}
|
|
337
|
-
if (!evidencePath) {
|
|
338
|
-
throw new Error(`eval campaign ${action} requires --evidence <path>`);
|
|
339
|
-
}
|
|
340
|
-
return {
|
|
341
|
-
command: "campaign",
|
|
342
|
-
action,
|
|
343
|
-
campaignId,
|
|
344
|
-
evidencePath,
|
|
345
|
-
acknowledgeReview: tail.includes("--acknowledge-review"),
|
|
346
|
-
...parseFormatFlags(tail),
|
|
347
|
-
};
|
|
348
|
-
}
|
|
349
|
-
if (action === "list") {
|
|
350
|
-
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
351
|
-
return {
|
|
352
|
-
command: "campaign",
|
|
353
|
-
action: "list",
|
|
354
|
-
...parseFormatFlags(tail),
|
|
355
|
-
};
|
|
356
|
-
}
|
|
357
|
-
throw new Error("usage: eval campaign <create|show|list|validate|plan|run|matrix|scorecard|audit> ...");
|
|
358
|
-
}
|
|
359
|
-
function parseProposeArgs(rest) {
|
|
360
|
-
const [action, ...tail] = rest;
|
|
361
|
-
if (action !== "from-curate") {
|
|
362
|
-
throw new Error("usage: eval propose from-curate [--max N] [--dry-run|--apply]");
|
|
363
|
-
}
|
|
364
|
-
assertKnownFlags(tail, [
|
|
365
|
-
"--max",
|
|
366
|
-
"--proposer-run-id",
|
|
367
|
-
"--dry-run",
|
|
368
|
-
"--apply",
|
|
369
|
-
"--json",
|
|
370
|
-
"--markdown",
|
|
371
|
-
]);
|
|
372
|
-
const maxRaw = flagValue(tail, "--max");
|
|
373
|
-
return {
|
|
374
|
-
command: "propose",
|
|
375
|
-
action: "from-curate",
|
|
376
|
-
maxCandidates: maxRaw ? Number(maxRaw) : 1,
|
|
377
|
-
proposerRunId: flagValue(tail, "--proposer-run-id"),
|
|
378
|
-
dryRun: parseDryRunFlags(tail),
|
|
379
|
-
...parseFormatFlags(tail),
|
|
380
|
-
};
|
|
381
|
-
}
|
|
382
|
-
function parseOuterLoopArgs(rest) {
|
|
383
|
-
const [action, ...tail] = rest;
|
|
384
|
-
if (action !== "run") {
|
|
385
|
-
throw new Error("usage: eval outer-loop run --matrix-mode <dry-run|stub|pi-plan|pi> [--dag-template <path>] [--campaign-id|--campaign-manifest] [--propose-from-curate] ...");
|
|
386
|
-
}
|
|
387
|
-
assertKnownFlags(tail, [
|
|
388
|
-
"--campaign-id",
|
|
389
|
-
"--campaign-manifest",
|
|
390
|
-
"--propose-from-curate",
|
|
391
|
-
"--max-proposals",
|
|
392
|
-
"--matrix-mode",
|
|
393
|
-
"--fixture",
|
|
394
|
-
"--dag-template",
|
|
395
|
-
"--max-public-cells",
|
|
396
|
-
"--scorecard",
|
|
397
|
-
"--json",
|
|
398
|
-
"--markdown",
|
|
399
|
-
]);
|
|
400
|
-
const matrixModeRaw = flagValue(tail, "--matrix-mode");
|
|
401
|
-
if (!matrixModeRaw) {
|
|
402
|
-
throw new Error("eval outer-loop run requires --matrix-mode <mode>");
|
|
403
|
-
}
|
|
404
|
-
const matrixMode = matrixModeSchema.parse(matrixModeRaw);
|
|
405
|
-
const dagTemplatePath = flagValue(tail, "--dag-template");
|
|
406
|
-
if (matrixMode === "pi" && !dagTemplatePath) {
|
|
407
|
-
throw new Error("eval outer-loop refuses mode=pi without --dag-template (bounded dry-run Pi) or injected executor; use pi-plan or stub");
|
|
408
|
-
}
|
|
409
|
-
const maxRaw = flagValue(tail, "--max-public-cells");
|
|
410
|
-
const maxPropRaw = flagValue(tail, "--max-proposals");
|
|
411
|
-
return {
|
|
412
|
-
command: "outer-loop",
|
|
413
|
-
action: "run",
|
|
414
|
-
campaignId: flagValue(tail, "--campaign-id"),
|
|
415
|
-
campaignManifestPath: flagValue(tail, "--campaign-manifest"),
|
|
416
|
-
proposeFromCurate: tail.includes("--propose-from-curate"),
|
|
417
|
-
maxProposals: maxPropRaw ? Number(maxPropRaw) : 1,
|
|
418
|
-
matrixMode,
|
|
419
|
-
fixturePath: flagValue(tail, "--fixture"),
|
|
420
|
-
dagTemplatePath,
|
|
421
|
-
maxPublicCells: maxRaw ? Number(maxRaw) : undefined,
|
|
422
|
-
buildScorecard: tail.includes("--scorecard"),
|
|
423
|
-
...parseFormatFlags(tail),
|
|
424
|
-
};
|
|
425
|
-
}
|
|
426
|
-
function parseIgnitionArgs(rest) {
|
|
427
|
-
const [action, ...tail] = rest;
|
|
428
|
-
if (action === "list") {
|
|
429
|
-
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
430
|
-
return {
|
|
431
|
-
command: "ignition",
|
|
432
|
-
action: "list",
|
|
433
|
-
...parseFormatFlags(tail),
|
|
434
|
-
};
|
|
435
|
-
}
|
|
436
|
-
if (action === "evaluate" || action === "show") {
|
|
437
|
-
assertKnownFlags(tail, ["--ignition-id", "--json", "--markdown"]);
|
|
438
|
-
const ignitionId = flagValue(tail, "--ignition-id");
|
|
439
|
-
if (!ignitionId) {
|
|
440
|
-
throw new Error(`eval ignition ${action} requires --ignition-id <id>`);
|
|
441
|
-
}
|
|
442
|
-
return {
|
|
443
|
-
command: "ignition",
|
|
444
|
-
action,
|
|
445
|
-
ignitionId,
|
|
446
|
-
...parseFormatFlags(tail),
|
|
447
|
-
};
|
|
448
|
-
}
|
|
449
|
-
if (action === "record") {
|
|
450
|
-
assertKnownFlags(tail, [
|
|
451
|
-
"--ignition-id",
|
|
452
|
-
"--generation",
|
|
453
|
-
"--parent-candidate",
|
|
454
|
-
"--proposer-run-id",
|
|
455
|
-
"--campaign-id",
|
|
456
|
-
"--json",
|
|
457
|
-
"--markdown",
|
|
458
|
-
]);
|
|
459
|
-
const ignitionId = flagValue(tail, "--ignition-id");
|
|
460
|
-
const generationRaw = flagValue(tail, "--generation");
|
|
461
|
-
const proposerRunId = flagValue(tail, "--proposer-run-id");
|
|
462
|
-
if (!ignitionId) {
|
|
463
|
-
throw new Error("eval ignition record requires --ignition-id <id>");
|
|
464
|
-
}
|
|
465
|
-
if (generationRaw === undefined) {
|
|
466
|
-
throw new Error("eval ignition record requires --generation <n>");
|
|
467
|
-
}
|
|
468
|
-
if (!proposerRunId) {
|
|
469
|
-
throw new Error("eval ignition record requires --proposer-run-id <id>");
|
|
470
|
-
}
|
|
471
|
-
const parentRaw = flagValue(tail, "--parent-candidate");
|
|
472
|
-
return {
|
|
473
|
-
command: "ignition",
|
|
474
|
-
action: "record",
|
|
475
|
-
ignitionId,
|
|
476
|
-
generation: Number(generationRaw),
|
|
477
|
-
parentCandidateId: parentRaw === undefined || parentRaw === "null" ? null : parentRaw,
|
|
478
|
-
proposerRunId,
|
|
479
|
-
campaignId: flagValue(tail, "--campaign-id"),
|
|
480
|
-
...parseFormatFlags(tail),
|
|
481
|
-
};
|
|
482
|
-
}
|
|
483
|
-
throw new Error("usage: eval ignition <record|evaluate|show|list> ...");
|
|
484
|
-
}
|
|
485
|
-
function parseExperimentArgs(rest) {
|
|
486
|
-
const [action, ...tail] = rest;
|
|
487
|
-
if (action === "from-curate") {
|
|
488
|
-
assertKnownFlags(tail, [
|
|
489
|
-
"--experiment-id",
|
|
490
|
-
"--candidate-id",
|
|
491
|
-
"--json",
|
|
492
|
-
"--markdown",
|
|
493
|
-
]);
|
|
494
|
-
return {
|
|
495
|
-
command: "experiment",
|
|
496
|
-
action: "from-curate",
|
|
497
|
-
experimentId: flagValue(tail, "--experiment-id"),
|
|
498
|
-
candidateId: flagValue(tail, "--candidate-id"),
|
|
499
|
-
...parseFormatFlags(tail),
|
|
500
|
-
};
|
|
501
|
-
}
|
|
502
|
-
if (action === "show") {
|
|
503
|
-
assertKnownFlags(tail, ["--experiment-id", "--json", "--markdown"]);
|
|
504
|
-
const experimentId = flagValue(tail, "--experiment-id");
|
|
505
|
-
if (!experimentId) {
|
|
506
|
-
throw new Error("eval experiment show requires --experiment-id <id>");
|
|
507
|
-
}
|
|
508
|
-
return {
|
|
509
|
-
command: "experiment",
|
|
510
|
-
action: "show",
|
|
511
|
-
experimentId,
|
|
512
|
-
...parseFormatFlags(tail),
|
|
513
|
-
};
|
|
514
|
-
}
|
|
515
|
-
if (action === "list") {
|
|
516
|
-
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
517
|
-
return {
|
|
518
|
-
command: "experiment",
|
|
519
|
-
action: "list",
|
|
520
|
-
...parseFormatFlags(tail),
|
|
521
|
-
};
|
|
522
|
-
}
|
|
523
|
-
if (action === "accept") {
|
|
524
|
-
assertKnownFlags(tail, [
|
|
525
|
-
"--experiment-id",
|
|
526
|
-
"--campaign-id",
|
|
527
|
-
"--reason",
|
|
528
|
-
"--acknowledge-review",
|
|
529
|
-
"--dry-run",
|
|
530
|
-
"--apply",
|
|
531
|
-
"--json",
|
|
532
|
-
"--markdown",
|
|
533
|
-
]);
|
|
534
|
-
const experimentId = flagValue(tail, "--experiment-id");
|
|
535
|
-
const campaignId = flagValue(tail, "--campaign-id");
|
|
536
|
-
const reason = flagValue(tail, "--reason");
|
|
537
|
-
if (!experimentId) {
|
|
538
|
-
throw new Error("eval experiment accept requires --experiment-id <id>");
|
|
539
|
-
}
|
|
540
|
-
if (!campaignId) {
|
|
541
|
-
throw new Error("eval experiment accept requires --campaign-id <id>");
|
|
542
|
-
}
|
|
543
|
-
if (!reason) {
|
|
544
|
-
throw new Error("eval experiment accept requires --reason <text>");
|
|
545
|
-
}
|
|
546
|
-
return {
|
|
547
|
-
command: "experiment",
|
|
548
|
-
action: "accept",
|
|
549
|
-
experimentId,
|
|
550
|
-
campaignId,
|
|
551
|
-
reason,
|
|
552
|
-
acknowledgeReview: tail.includes("--acknowledge-review"),
|
|
553
|
-
dryRun: parseDryRunFlags(tail),
|
|
554
|
-
...parseFormatFlags(tail),
|
|
555
|
-
};
|
|
556
|
-
}
|
|
557
|
-
if (action === "reject") {
|
|
558
|
-
assertKnownFlags(tail, [
|
|
559
|
-
"--experiment-id",
|
|
560
|
-
"--reason",
|
|
561
|
-
"--dry-run",
|
|
562
|
-
"--apply",
|
|
563
|
-
"--json",
|
|
564
|
-
"--markdown",
|
|
565
|
-
]);
|
|
566
|
-
const experimentId = flagValue(tail, "--experiment-id");
|
|
567
|
-
const reason = flagValue(tail, "--reason");
|
|
568
|
-
if (!experimentId) {
|
|
569
|
-
throw new Error("eval experiment reject requires --experiment-id <id>");
|
|
570
|
-
}
|
|
571
|
-
if (!reason) {
|
|
572
|
-
throw new Error("eval experiment reject requires --reason <text>");
|
|
573
|
-
}
|
|
574
|
-
return {
|
|
575
|
-
command: "experiment",
|
|
576
|
-
action: "reject",
|
|
577
|
-
experimentId,
|
|
578
|
-
reason,
|
|
579
|
-
dryRun: parseDryRunFlags(tail),
|
|
580
|
-
...parseFormatFlags(tail),
|
|
581
|
-
};
|
|
582
|
-
}
|
|
583
|
-
throw new Error("usage: eval experiment <from-curate|show|list|accept|reject> ...");
|
|
584
|
-
}
|
|
585
|
-
function parseCorpusArgs(rest) {
|
|
586
|
-
const [action, ...tail] = rest;
|
|
587
|
-
if (action === "register" || action === "validate") {
|
|
588
|
-
assertKnownFlags(tail, ["--manifest", "--json", "--markdown"]);
|
|
589
|
-
const manifestPath = flagValue(tail, "--manifest");
|
|
590
|
-
if (!manifestPath) {
|
|
591
|
-
throw new Error(`eval corpus ${action} requires --manifest <path>`);
|
|
592
|
-
}
|
|
593
|
-
return {
|
|
594
|
-
command: "corpus",
|
|
595
|
-
action,
|
|
596
|
-
manifestPath,
|
|
597
|
-
...parseFormatFlags(tail),
|
|
598
|
-
};
|
|
599
|
-
}
|
|
600
|
-
if (action === "show") {
|
|
601
|
-
assertKnownFlags(tail, ["--corpus-id", "--json", "--markdown"]);
|
|
602
|
-
const corpusId = flagValue(tail, "--corpus-id");
|
|
603
|
-
if (!corpusId) {
|
|
604
|
-
throw new Error("eval corpus show requires --corpus-id <id>");
|
|
605
|
-
}
|
|
606
|
-
return {
|
|
607
|
-
command: "corpus",
|
|
608
|
-
action: "show",
|
|
609
|
-
corpusId,
|
|
610
|
-
...parseFormatFlags(tail),
|
|
611
|
-
};
|
|
612
|
-
}
|
|
613
|
-
if (action === "list") {
|
|
614
|
-
assertKnownFlags(tail, ["--json", "--markdown"]);
|
|
615
|
-
return {
|
|
616
|
-
command: "corpus",
|
|
617
|
-
action: "list",
|
|
618
|
-
...parseFormatFlags(tail),
|
|
619
|
-
};
|
|
620
|
-
}
|
|
621
|
-
throw new Error("usage: eval corpus <register|show|list|validate> ...");
|
|
622
|
-
}
|
|
623
|
-
function parseAliasMoveArgs(command, rest) {
|
|
624
|
-
const promoteOnlyFlags = command === "promote"
|
|
625
|
-
? ["--campaign-id", "--evidence", "--acknowledge-review"]
|
|
626
|
-
: [];
|
|
627
|
-
assertKnownFlags(rest, [
|
|
628
|
-
"--alias",
|
|
629
|
-
"--to-candidate",
|
|
630
|
-
"--reason",
|
|
631
|
-
"--dry-run",
|
|
632
|
-
"--apply",
|
|
633
|
-
"--json",
|
|
634
|
-
"--markdown",
|
|
635
|
-
...promoteOnlyFlags,
|
|
636
|
-
]);
|
|
637
|
-
const alias = flagValue(rest, "--alias");
|
|
638
|
-
const toCandidateId = flagValue(rest, "--to-candidate");
|
|
639
|
-
const reason = flagValue(rest, "--reason");
|
|
640
|
-
if (!alias)
|
|
641
|
-
throw new Error(`eval ${command} requires --alias <name>`);
|
|
642
|
-
if (!toCandidateId) {
|
|
643
|
-
throw new Error(`eval ${command} requires --to-candidate <id>`);
|
|
644
|
-
}
|
|
645
|
-
if (!reason)
|
|
646
|
-
throw new Error(`eval ${command} requires --reason <text>`);
|
|
647
|
-
if (command === "promote") {
|
|
648
|
-
return {
|
|
649
|
-
command,
|
|
650
|
-
alias,
|
|
651
|
-
toCandidateId,
|
|
652
|
-
reason,
|
|
653
|
-
campaignId: flagValue(rest, "--campaign-id"),
|
|
654
|
-
evidencePath: flagValue(rest, "--evidence"),
|
|
655
|
-
acknowledgeReview: rest.includes("--acknowledge-review"),
|
|
656
|
-
dryRun: parseDryRunFlags(rest),
|
|
657
|
-
...parseFormatFlags(rest),
|
|
658
|
-
};
|
|
659
|
-
}
|
|
660
|
-
return {
|
|
661
|
-
command,
|
|
662
|
-
alias,
|
|
663
|
-
toCandidateId,
|
|
664
|
-
reason,
|
|
665
|
-
dryRun: parseDryRunFlags(rest),
|
|
666
|
-
...parseFormatFlags(rest),
|
|
667
|
-
};
|
|
668
|
-
}
|
|
669
119
|
export function parseEvalArgs(args) {
|
|
670
120
|
const [command, ...rest] = args;
|
|
671
121
|
if (command === "replay") {
|
|
@@ -690,48 +140,8 @@ export function parseEvalArgs(args) {
|
|
|
690
140
|
if (command === "candidate") {
|
|
691
141
|
return parseCandidateArgs(rest);
|
|
692
142
|
}
|
|
693
|
-
if (command === "alias") {
|
|
694
|
-
return parseAliasArgs(rest);
|
|
695
|
-
}
|
|
696
|
-
if (command === "promote" || command === "rollback") {
|
|
697
|
-
return parseAliasMoveArgs(command, rest);
|
|
698
|
-
}
|
|
699
|
-
if (command === "corpus") {
|
|
700
|
-
return parseCorpusArgs(rest);
|
|
701
|
-
}
|
|
702
|
-
if (command === "context-policy") {
|
|
703
|
-
return parseContextPolicyArgs(rest);
|
|
704
|
-
}
|
|
705
|
-
if (command === "budget") {
|
|
706
|
-
return parseBudgetArgs(rest);
|
|
707
|
-
}
|
|
708
|
-
if (command === "private-verifier") {
|
|
709
|
-
return parsePrivateVerifierArgs(rest);
|
|
710
|
-
}
|
|
711
|
-
if (command === "campaign") {
|
|
712
|
-
return parseCampaignArgs(rest);
|
|
713
|
-
}
|
|
714
|
-
if (command === "experiment") {
|
|
715
|
-
return parseExperimentArgs(rest);
|
|
716
|
-
}
|
|
717
|
-
if (command === "propose") {
|
|
718
|
-
return parseProposeArgs(rest);
|
|
719
|
-
}
|
|
720
|
-
if (command === "outer-loop") {
|
|
721
|
-
return parseOuterLoopArgs(rest);
|
|
722
|
-
}
|
|
723
|
-
if (command === "ignition") {
|
|
724
|
-
return parseIgnitionArgs(rest);
|
|
725
|
-
}
|
|
726
143
|
throw new Error(USAGE);
|
|
727
144
|
}
|
|
728
|
-
function printIgnition(input) {
|
|
729
|
-
if (input.json) {
|
|
730
|
-
console.log(JSON.stringify(input.report, null, 2));
|
|
731
|
-
return;
|
|
732
|
-
}
|
|
733
|
-
process.stdout.write(formatIgnitionMarkdown(input.report));
|
|
734
|
-
}
|
|
735
145
|
function printScorecard(input) {
|
|
736
146
|
if (input.json) {
|
|
737
147
|
console.log(JSON.stringify(input.scorecard, null, 2));
|
|
@@ -746,80 +156,6 @@ function printCandidate(input) {
|
|
|
746
156
|
}
|
|
747
157
|
process.stdout.write(formatCandidateMarkdown(input.record));
|
|
748
158
|
}
|
|
749
|
-
function printAlias(input) {
|
|
750
|
-
if (input.json) {
|
|
751
|
-
console.log(JSON.stringify(input.alias, null, 2));
|
|
752
|
-
return;
|
|
753
|
-
}
|
|
754
|
-
process.stdout.write(formatAliasMarkdown(input.alias));
|
|
755
|
-
}
|
|
756
|
-
function printAliasMove(input) {
|
|
757
|
-
if (input.json) {
|
|
758
|
-
console.log(JSON.stringify(input.result, null, 2));
|
|
759
|
-
return;
|
|
760
|
-
}
|
|
761
|
-
process.stdout.write(formatAliasMoveMarkdown(input.result));
|
|
762
|
-
}
|
|
763
|
-
function printPrivateVerifier(input) {
|
|
764
|
-
if (input.json) {
|
|
765
|
-
console.log(JSON.stringify(input.manifest, null, 2));
|
|
766
|
-
return;
|
|
767
|
-
}
|
|
768
|
-
process.stdout.write(formatPrivateVerifierMarkdown(input.manifest));
|
|
769
|
-
}
|
|
770
|
-
function printCampaign(input) {
|
|
771
|
-
if (input.json) {
|
|
772
|
-
console.log(JSON.stringify({
|
|
773
|
-
manifest: input.manifest,
|
|
774
|
-
plan: input.plan,
|
|
775
|
-
}, null, 2));
|
|
776
|
-
return;
|
|
777
|
-
}
|
|
778
|
-
process.stdout.write(formatCampaignMarkdown({
|
|
779
|
-
manifest: input.manifest,
|
|
780
|
-
plan: input.plan,
|
|
781
|
-
}));
|
|
782
|
-
}
|
|
783
|
-
function printExperiment(input) {
|
|
784
|
-
if (input.json) {
|
|
785
|
-
console.log(JSON.stringify({
|
|
786
|
-
record: input.record,
|
|
787
|
-
candidate: input.candidate,
|
|
788
|
-
...input.extra,
|
|
789
|
-
}, null, 2));
|
|
790
|
-
return;
|
|
791
|
-
}
|
|
792
|
-
process.stdout.write(formatExperimentMarkdown({
|
|
793
|
-
record: input.record,
|
|
794
|
-
candidate: input.candidate,
|
|
795
|
-
}));
|
|
796
|
-
}
|
|
797
|
-
function printExperimentDisposition(input) {
|
|
798
|
-
if (input.json) {
|
|
799
|
-
console.log(JSON.stringify(input.result, null, 2));
|
|
800
|
-
return;
|
|
801
|
-
}
|
|
802
|
-
const prefix = input.result.dryRun ? "dry-run " : "";
|
|
803
|
-
process.stdout.write(`${prefix}experiment ${input.result.record.status} (changed=${input.result.changed})\n`);
|
|
804
|
-
process.stdout.write(formatExperimentMarkdown({
|
|
805
|
-
record: input.result.record,
|
|
806
|
-
candidate: input.result.candidate,
|
|
807
|
-
}));
|
|
808
|
-
}
|
|
809
|
-
function printCorpus(input) {
|
|
810
|
-
if (input.json) {
|
|
811
|
-
console.log(JSON.stringify({
|
|
812
|
-
summary: input.summary,
|
|
813
|
-
manifest: input.manifest,
|
|
814
|
-
...input.extra,
|
|
815
|
-
}, null, 2));
|
|
816
|
-
return;
|
|
817
|
-
}
|
|
818
|
-
process.stdout.write(formatCorpusMarkdown({
|
|
819
|
-
manifest: input.manifest,
|
|
820
|
-
summary: input.summary,
|
|
821
|
-
}));
|
|
822
|
-
}
|
|
823
159
|
export async function runEval(repoRoot, args) {
|
|
824
160
|
const parsed = parseEvalArgs(args);
|
|
825
161
|
if (parsed.command === "replay") {
|
|
@@ -846,505 +182,6 @@ export async function runEval(repoRoot, args) {
|
|
|
846
182
|
printScorecard({ scorecard, markdown, json: parsed.json });
|
|
847
183
|
return;
|
|
848
184
|
}
|
|
849
|
-
if (parsed.command === "alias") {
|
|
850
|
-
if (parsed.action === "show") {
|
|
851
|
-
const alias = await showAlias({ repoRoot, alias: parsed.alias });
|
|
852
|
-
printAlias({ alias, json: parsed.json });
|
|
853
|
-
return;
|
|
854
|
-
}
|
|
855
|
-
const rows = await listAliases({ repoRoot });
|
|
856
|
-
if (parsed.json) {
|
|
857
|
-
console.log(JSON.stringify(rows, null, 2));
|
|
858
|
-
return;
|
|
859
|
-
}
|
|
860
|
-
const lines = [
|
|
861
|
-
"# Aliases",
|
|
862
|
-
"",
|
|
863
|
-
...rows.map((row) => `- \`${row.alias}\` → \`${row.candidateId}\` hash=\`${row.bundleHash}\` decision=\`${row.updatedByDecisionId}\``),
|
|
864
|
-
"",
|
|
865
|
-
];
|
|
866
|
-
process.stdout.write(`${lines.join("\n")}\n`);
|
|
867
|
-
return;
|
|
868
|
-
}
|
|
869
|
-
if (parsed.command === "promote" || parsed.command === "rollback") {
|
|
870
|
-
if (parsed.command === "promote" && parsed.campaignId) {
|
|
871
|
-
await requireCampaignPromotionGate({
|
|
872
|
-
repoRoot,
|
|
873
|
-
campaignId: parsed.campaignId,
|
|
874
|
-
toCandidateId: parsed.toCandidateId,
|
|
875
|
-
acknowledgeReview: parsed.acknowledgeReview,
|
|
876
|
-
evidencePath: parsed.evidencePath,
|
|
877
|
-
});
|
|
878
|
-
}
|
|
879
|
-
const mover = parsed.command === "promote" ? promoteAlias : rollbackAlias;
|
|
880
|
-
const result = await mover({
|
|
881
|
-
repoRoot,
|
|
882
|
-
alias: parsed.alias,
|
|
883
|
-
toCandidateId: parsed.toCandidateId,
|
|
884
|
-
reason: parsed.reason,
|
|
885
|
-
dryRun: parsed.dryRun,
|
|
886
|
-
});
|
|
887
|
-
printAliasMove({ result, json: parsed.json });
|
|
888
|
-
return;
|
|
889
|
-
}
|
|
890
|
-
if (parsed.command === "context-policy") {
|
|
891
|
-
if (parsed.action === "list") {
|
|
892
|
-
const rows = listContextPolicies().map((policy) => policy.toManifest());
|
|
893
|
-
if (parsed.json) {
|
|
894
|
-
console.log(JSON.stringify(rows, null, 2));
|
|
895
|
-
return;
|
|
896
|
-
}
|
|
897
|
-
const lines = [
|
|
898
|
-
"# Context Policies",
|
|
899
|
-
"",
|
|
900
|
-
...rows.map((row) => `- \`${row.policyId}\`: ${row.description}`),
|
|
901
|
-
"",
|
|
902
|
-
];
|
|
903
|
-
process.stdout.write(`${lines.join("\n")}\n`);
|
|
904
|
-
return;
|
|
905
|
-
}
|
|
906
|
-
if (!isContextPolicyId(parsed.policyId)) {
|
|
907
|
-
throw new Error(`unknown context policy id: ${parsed.policyId}`);
|
|
908
|
-
}
|
|
909
|
-
const manifest = getContextPolicy(parsed.policyId).toManifest();
|
|
910
|
-
if (parsed.json) {
|
|
911
|
-
console.log(JSON.stringify(manifest, null, 2));
|
|
912
|
-
return;
|
|
913
|
-
}
|
|
914
|
-
process.stdout.write([
|
|
915
|
-
`# Context Policy: ${manifest.policyId}`,
|
|
916
|
-
"",
|
|
917
|
-
manifest.description,
|
|
918
|
-
"",
|
|
919
|
-
"```json",
|
|
920
|
-
JSON.stringify(manifest.knobs, null, 2),
|
|
921
|
-
"```",
|
|
922
|
-
"",
|
|
923
|
-
].join("\n"));
|
|
924
|
-
return;
|
|
925
|
-
}
|
|
926
|
-
if (parsed.command === "budget") {
|
|
927
|
-
const located = await locateDagRun(repoRoot, parsed.runId);
|
|
928
|
-
if (!located) {
|
|
929
|
-
throw new Error(`dag run not found: ${parsed.runId}`);
|
|
930
|
-
}
|
|
931
|
-
const state = await readDagRunState(located.runDir);
|
|
932
|
-
const ledger = state.budgetLedger;
|
|
933
|
-
if (!ledger) {
|
|
934
|
-
throw new Error(`dag run ${parsed.runId} has no budget ledger (DagSpec.budget was not declared)`);
|
|
935
|
-
}
|
|
936
|
-
if (parsed.json) {
|
|
937
|
-
console.log(JSON.stringify(ledger, null, 2));
|
|
938
|
-
return;
|
|
939
|
-
}
|
|
940
|
-
process.stdout.write(formatBudgetReportMarkdown(ledger));
|
|
941
|
-
return;
|
|
942
|
-
}
|
|
943
|
-
if (parsed.command === "private-verifier") {
|
|
944
|
-
if (parsed.action === "validate") {
|
|
945
|
-
const result = await validatePrivateVerifier({
|
|
946
|
-
repoRoot,
|
|
947
|
-
manifestPath: parsed.manifestPath,
|
|
948
|
-
});
|
|
949
|
-
printPrivateVerifier({ manifest: result.manifest, json: parsed.json });
|
|
950
|
-
return;
|
|
951
|
-
}
|
|
952
|
-
if (parsed.action === "register") {
|
|
953
|
-
const result = await registerPrivateVerifier({
|
|
954
|
-
repoRoot,
|
|
955
|
-
manifestPath: parsed.manifestPath,
|
|
956
|
-
});
|
|
957
|
-
printPrivateVerifier({ manifest: result.manifest, json: parsed.json });
|
|
958
|
-
return;
|
|
959
|
-
}
|
|
960
|
-
if (parsed.action === "list") {
|
|
961
|
-
const rows = await listPrivateVerifiers({ repoRoot });
|
|
962
|
-
if (parsed.json) {
|
|
963
|
-
console.log(JSON.stringify(rows, null, 2));
|
|
964
|
-
return;
|
|
965
|
-
}
|
|
966
|
-
const lines = [
|
|
967
|
-
"# Private Verifiers",
|
|
968
|
-
"",
|
|
969
|
-
...rows.map((row) => `- \`${row.privateVerifierId}\`: checks=${row.checkCount} hash=\`${row.manifestHash}\``),
|
|
970
|
-
"",
|
|
971
|
-
];
|
|
972
|
-
process.stdout.write(`${lines.join("\n")}\n`);
|
|
973
|
-
return;
|
|
974
|
-
}
|
|
975
|
-
if (parsed.action !== "show") {
|
|
976
|
-
throw new Error("usage: eval private-verifier <register|show|list|validate> ...");
|
|
977
|
-
}
|
|
978
|
-
const result = await showPrivateVerifier({
|
|
979
|
-
repoRoot,
|
|
980
|
-
privateVerifierId: parsed.privateVerifierId,
|
|
981
|
-
});
|
|
982
|
-
printPrivateVerifier({ manifest: result.manifest, json: parsed.json });
|
|
983
|
-
return;
|
|
984
|
-
}
|
|
985
|
-
if (parsed.command === "propose") {
|
|
986
|
-
const result = await proposeFromCurate({
|
|
987
|
-
repoRoot,
|
|
988
|
-
maxCandidates: parsed.maxCandidates,
|
|
989
|
-
proposerRunId: parsed.proposerRunId,
|
|
990
|
-
dryRun: parsed.dryRun,
|
|
991
|
-
});
|
|
992
|
-
if (parsed.json) {
|
|
993
|
-
console.log(JSON.stringify(result, null, 2));
|
|
994
|
-
return;
|
|
995
|
-
}
|
|
996
|
-
process.stdout.write(formatProposerRunMarkdown(result));
|
|
997
|
-
return;
|
|
998
|
-
}
|
|
999
|
-
if (parsed.command === "outer-loop") {
|
|
1000
|
-
const piExecutor = parsed.matrixMode === "pi" && parsed.dagTemplatePath
|
|
1001
|
-
? createDagDryRunPiCellExecutor({
|
|
1002
|
-
dagTemplatePath: parsed.dagTemplatePath,
|
|
1003
|
-
})
|
|
1004
|
-
: undefined;
|
|
1005
|
-
const result = await runOuterLoop({
|
|
1006
|
-
repoRoot,
|
|
1007
|
-
campaignId: parsed.campaignId,
|
|
1008
|
-
campaignManifestPath: parsed.campaignManifestPath,
|
|
1009
|
-
proposeFromCurate: parsed.proposeFromCurate,
|
|
1010
|
-
maxProposals: parsed.maxProposals,
|
|
1011
|
-
matrixMode: parsed.matrixMode,
|
|
1012
|
-
fixturePath: parsed.fixturePath,
|
|
1013
|
-
maxPublicCells: parsed.maxPublicCells,
|
|
1014
|
-
buildScorecard: parsed.buildScorecard,
|
|
1015
|
-
piExecutor,
|
|
1016
|
-
});
|
|
1017
|
-
if (parsed.json) {
|
|
1018
|
-
console.log(JSON.stringify(result, null, 2));
|
|
1019
|
-
return;
|
|
1020
|
-
}
|
|
1021
|
-
process.stdout.write(formatOuterLoopMarkdown(result));
|
|
1022
|
-
return;
|
|
1023
|
-
}
|
|
1024
|
-
if (parsed.command === "ignition") {
|
|
1025
|
-
if (parsed.action === "list") {
|
|
1026
|
-
const rows = await listIgnitions({ repoRoot });
|
|
1027
|
-
if (parsed.json) {
|
|
1028
|
-
console.log(JSON.stringify(rows, null, 2));
|
|
1029
|
-
return;
|
|
1030
|
-
}
|
|
1031
|
-
const lines = [
|
|
1032
|
-
"# Ignition Research Gates",
|
|
1033
|
-
"",
|
|
1034
|
-
...rows.map((row) => `- \`${row.ignitionId}\`: generations=${row.generationCount} verdict=\`${row.verdict}\` rsiLevel1ClaimAllowed=\`${row.rsiLevel1ClaimAllowed}\``),
|
|
1035
|
-
"",
|
|
1036
|
-
];
|
|
1037
|
-
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1038
|
-
return;
|
|
1039
|
-
}
|
|
1040
|
-
if (parsed.action === "record") {
|
|
1041
|
-
const report = await recordIgnitionGeneration({
|
|
1042
|
-
repoRoot,
|
|
1043
|
-
ignitionId: parsed.ignitionId,
|
|
1044
|
-
generation: parsed.generation,
|
|
1045
|
-
parentCandidateId: parsed.parentCandidateId,
|
|
1046
|
-
proposerRunId: parsed.proposerRunId,
|
|
1047
|
-
campaignId: parsed.campaignId,
|
|
1048
|
-
});
|
|
1049
|
-
printIgnition({ report, json: parsed.json });
|
|
1050
|
-
return;
|
|
1051
|
-
}
|
|
1052
|
-
if (parsed.action === "evaluate") {
|
|
1053
|
-
const report = await evaluateIgnition({
|
|
1054
|
-
repoRoot,
|
|
1055
|
-
ignitionId: parsed.ignitionId,
|
|
1056
|
-
});
|
|
1057
|
-
printIgnition({ report, json: parsed.json });
|
|
1058
|
-
return;
|
|
1059
|
-
}
|
|
1060
|
-
if (parsed.action === "show") {
|
|
1061
|
-
const report = await showIgnition({
|
|
1062
|
-
repoRoot,
|
|
1063
|
-
ignitionId: parsed.ignitionId,
|
|
1064
|
-
});
|
|
1065
|
-
printIgnition({ report, json: parsed.json });
|
|
1066
|
-
return;
|
|
1067
|
-
}
|
|
1068
|
-
throw new Error("usage: eval ignition <record|evaluate|show|list> ...");
|
|
1069
|
-
}
|
|
1070
|
-
if (parsed.command === "experiment") {
|
|
1071
|
-
if (parsed.action === "from-curate") {
|
|
1072
|
-
const result = await createExperimentFromCurate({
|
|
1073
|
-
repoRoot,
|
|
1074
|
-
experimentId: parsed.experimentId,
|
|
1075
|
-
candidateId: parsed.candidateId,
|
|
1076
|
-
});
|
|
1077
|
-
printExperiment({
|
|
1078
|
-
record: result.record,
|
|
1079
|
-
candidate: result.candidate,
|
|
1080
|
-
json: parsed.json,
|
|
1081
|
-
extra: {
|
|
1082
|
-
idempotent: result.idempotent,
|
|
1083
|
-
curateMessage: result.curateMessage,
|
|
1084
|
-
},
|
|
1085
|
-
});
|
|
1086
|
-
return;
|
|
1087
|
-
}
|
|
1088
|
-
if (parsed.action === "list") {
|
|
1089
|
-
const rows = await listExperiments({ repoRoot });
|
|
1090
|
-
if (parsed.json) {
|
|
1091
|
-
console.log(JSON.stringify(rows, null, 2));
|
|
1092
|
-
return;
|
|
1093
|
-
}
|
|
1094
|
-
const lines = [
|
|
1095
|
-
"# Experiments",
|
|
1096
|
-
"",
|
|
1097
|
-
...rows.map((row) => `- \`${row.experimentId}\`: status=${row.status} candidate=\`${row.candidateId}\` patterns=${row.patternCount}`),
|
|
1098
|
-
"",
|
|
1099
|
-
];
|
|
1100
|
-
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1101
|
-
return;
|
|
1102
|
-
}
|
|
1103
|
-
if (parsed.action === "show") {
|
|
1104
|
-
const result = await showExperiment({
|
|
1105
|
-
repoRoot,
|
|
1106
|
-
experimentId: parsed.experimentId,
|
|
1107
|
-
});
|
|
1108
|
-
printExperiment({
|
|
1109
|
-
record: result.record,
|
|
1110
|
-
candidate: result.candidate,
|
|
1111
|
-
json: parsed.json,
|
|
1112
|
-
});
|
|
1113
|
-
return;
|
|
1114
|
-
}
|
|
1115
|
-
if (parsed.action === "accept") {
|
|
1116
|
-
const result = await acceptExperiment({
|
|
1117
|
-
repoRoot,
|
|
1118
|
-
experimentId: parsed.experimentId,
|
|
1119
|
-
campaignId: parsed.campaignId,
|
|
1120
|
-
reason: parsed.reason,
|
|
1121
|
-
acknowledgeReview: parsed.acknowledgeReview,
|
|
1122
|
-
dryRun: parsed.dryRun,
|
|
1123
|
-
});
|
|
1124
|
-
printExperimentDisposition({ result, json: parsed.json });
|
|
1125
|
-
return;
|
|
1126
|
-
}
|
|
1127
|
-
if (parsed.action === "reject") {
|
|
1128
|
-
const result = await rejectExperiment({
|
|
1129
|
-
repoRoot,
|
|
1130
|
-
experimentId: parsed.experimentId,
|
|
1131
|
-
reason: parsed.reason,
|
|
1132
|
-
dryRun: parsed.dryRun,
|
|
1133
|
-
});
|
|
1134
|
-
printExperimentDisposition({ result, json: parsed.json });
|
|
1135
|
-
return;
|
|
1136
|
-
}
|
|
1137
|
-
throw new Error("usage: eval experiment <from-curate|show|list|accept|reject> ...");
|
|
1138
|
-
}
|
|
1139
|
-
if (parsed.command === "campaign") {
|
|
1140
|
-
if (parsed.action === "validate") {
|
|
1141
|
-
const result = await validateCampaign({
|
|
1142
|
-
repoRoot,
|
|
1143
|
-
manifestPath: parsed.manifestPath,
|
|
1144
|
-
});
|
|
1145
|
-
printCampaign({
|
|
1146
|
-
manifest: result.manifest,
|
|
1147
|
-
plan: result.plan,
|
|
1148
|
-
json: parsed.json,
|
|
1149
|
-
});
|
|
1150
|
-
return;
|
|
1151
|
-
}
|
|
1152
|
-
if (parsed.action === "create") {
|
|
1153
|
-
const result = await createCampaign({
|
|
1154
|
-
repoRoot,
|
|
1155
|
-
manifestPath: parsed.manifestPath,
|
|
1156
|
-
});
|
|
1157
|
-
printCampaign({
|
|
1158
|
-
manifest: result.manifest,
|
|
1159
|
-
plan: result.plan,
|
|
1160
|
-
json: parsed.json,
|
|
1161
|
-
});
|
|
1162
|
-
return;
|
|
1163
|
-
}
|
|
1164
|
-
if (parsed.action === "list") {
|
|
1165
|
-
const rows = await listCampaigns({ repoRoot });
|
|
1166
|
-
if (parsed.json) {
|
|
1167
|
-
console.log(JSON.stringify(rows, null, 2));
|
|
1168
|
-
return;
|
|
1169
|
-
}
|
|
1170
|
-
const lines = [
|
|
1171
|
-
"# Campaigns",
|
|
1172
|
-
"",
|
|
1173
|
-
...rows.map((row) => `- \`${row.campaignId}\`: status=${row.status} corpus=\`${row.corpusId}\` cells=${row.cellCount ?? "-"}`),
|
|
1174
|
-
"",
|
|
1175
|
-
];
|
|
1176
|
-
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1177
|
-
return;
|
|
1178
|
-
}
|
|
1179
|
-
if (parsed.action === "plan") {
|
|
1180
|
-
const result = await planCampaign({
|
|
1181
|
-
repoRoot,
|
|
1182
|
-
campaignId: parsed.campaignId,
|
|
1183
|
-
});
|
|
1184
|
-
printCampaign({
|
|
1185
|
-
manifest: result.manifest,
|
|
1186
|
-
plan: result.plan,
|
|
1187
|
-
json: parsed.json,
|
|
1188
|
-
});
|
|
1189
|
-
return;
|
|
1190
|
-
}
|
|
1191
|
-
if (parsed.action === "run") {
|
|
1192
|
-
const result = await runCampaignPrivateChecks({
|
|
1193
|
-
repoRoot,
|
|
1194
|
-
campaignId: parsed.campaignId,
|
|
1195
|
-
});
|
|
1196
|
-
if (parsed.json) {
|
|
1197
|
-
console.log(JSON.stringify(result, null, 2));
|
|
1198
|
-
return;
|
|
1199
|
-
}
|
|
1200
|
-
const failed = result.results.filter((row) => !row.ok);
|
|
1201
|
-
process.stdout.write(formatCampaignMarkdown({
|
|
1202
|
-
manifest: result.manifest,
|
|
1203
|
-
plan: result.plan,
|
|
1204
|
-
}));
|
|
1205
|
-
process.stdout.write(`\n## Private check results\n\n- total: ${result.results.length}\n- failed: ${failed.length}\n\n`);
|
|
1206
|
-
if (failed.length > 0) {
|
|
1207
|
-
throw new Error(`campaign private checks failed: ${failed.map((row) => `${row.cellId}/${row.checkId}`).join(", ")}`);
|
|
1208
|
-
}
|
|
1209
|
-
return;
|
|
1210
|
-
}
|
|
1211
|
-
if (parsed.action === "matrix") {
|
|
1212
|
-
if (parsed.mode === "pi" && !parsed.dagTemplatePath) {
|
|
1213
|
-
throw new Error("eval campaign matrix refuses mode=pi without --dag-template (bounded dry-run Pi) or injected executor; use --mode pi-plan or stub");
|
|
1214
|
-
}
|
|
1215
|
-
const piExecutor = parsed.mode === "pi" && parsed.dagTemplatePath
|
|
1216
|
-
? createDagDryRunPiCellExecutor({
|
|
1217
|
-
dagTemplatePath: parsed.dagTemplatePath,
|
|
1218
|
-
})
|
|
1219
|
-
: undefined;
|
|
1220
|
-
const result = await runCampaignMatrix({
|
|
1221
|
-
repoRoot,
|
|
1222
|
-
campaignId: parsed.campaignId,
|
|
1223
|
-
mode: parsed.mode,
|
|
1224
|
-
fixturePath: parsed.fixturePath,
|
|
1225
|
-
maxPublicCells: parsed.maxPublicCells,
|
|
1226
|
-
piExecutor,
|
|
1227
|
-
});
|
|
1228
|
-
if (parsed.json) {
|
|
1229
|
-
console.log(JSON.stringify(result.matrix, null, 2));
|
|
1230
|
-
return;
|
|
1231
|
-
}
|
|
1232
|
-
process.stdout.write(formatMatrixRunMarkdown(result.matrix));
|
|
1233
|
-
return;
|
|
1234
|
-
}
|
|
1235
|
-
if (parsed.action === "scorecard" || parsed.action === "audit") {
|
|
1236
|
-
const evidence = await loadCampaignEvidence({
|
|
1237
|
-
repoRoot,
|
|
1238
|
-
evidencePath: parsed.evidencePath,
|
|
1239
|
-
});
|
|
1240
|
-
if (parsed.action === "audit") {
|
|
1241
|
-
const result = await auditCampaignEvidence({
|
|
1242
|
-
repoRoot,
|
|
1243
|
-
campaignId: parsed.campaignId,
|
|
1244
|
-
evidence,
|
|
1245
|
-
});
|
|
1246
|
-
if (parsed.json) {
|
|
1247
|
-
console.log(JSON.stringify(result, null, 2));
|
|
1248
|
-
return;
|
|
1249
|
-
}
|
|
1250
|
-
const scorecard = {
|
|
1251
|
-
schemaVersion: 1,
|
|
1252
|
-
campaignId: parsed.campaignId,
|
|
1253
|
-
planHash: "(audit-only)",
|
|
1254
|
-
createdAt: new Date().toISOString(),
|
|
1255
|
-
evidenceCellCount: evidence.cells.length,
|
|
1256
|
-
policy: result.policy,
|
|
1257
|
-
};
|
|
1258
|
-
process.stdout.write(formatCampaignScorecardMarkdown(scorecard));
|
|
1259
|
-
return;
|
|
1260
|
-
}
|
|
1261
|
-
const scorecard = await buildCampaignScorecard({
|
|
1262
|
-
repoRoot,
|
|
1263
|
-
campaignId: parsed.campaignId,
|
|
1264
|
-
evidence,
|
|
1265
|
-
acknowledgeReview: parsed.acknowledgeReview,
|
|
1266
|
-
persist: true,
|
|
1267
|
-
});
|
|
1268
|
-
if (parsed.json) {
|
|
1269
|
-
console.log(JSON.stringify(scorecard, null, 2));
|
|
1270
|
-
return;
|
|
1271
|
-
}
|
|
1272
|
-
process.stdout.write(formatCampaignScorecardMarkdown(scorecard));
|
|
1273
|
-
return;
|
|
1274
|
-
}
|
|
1275
|
-
if (parsed.action !== "show") {
|
|
1276
|
-
throw new Error("usage: eval campaign <create|show|list|validate|plan|run|matrix|scorecard|audit> ...");
|
|
1277
|
-
}
|
|
1278
|
-
const result = await showCampaign({
|
|
1279
|
-
repoRoot,
|
|
1280
|
-
campaignId: parsed.campaignId,
|
|
1281
|
-
});
|
|
1282
|
-
printCampaign({
|
|
1283
|
-
manifest: result.manifest,
|
|
1284
|
-
plan: result.plan,
|
|
1285
|
-
json: parsed.json,
|
|
1286
|
-
});
|
|
1287
|
-
return;
|
|
1288
|
-
}
|
|
1289
|
-
if (parsed.command === "corpus") {
|
|
1290
|
-
if (parsed.action === "validate") {
|
|
1291
|
-
const result = await validateCorpus({
|
|
1292
|
-
repoRoot,
|
|
1293
|
-
manifestPath: parsed.manifestPath,
|
|
1294
|
-
});
|
|
1295
|
-
printCorpus({
|
|
1296
|
-
manifest: result.manifest,
|
|
1297
|
-
summary: result.summary,
|
|
1298
|
-
json: parsed.json,
|
|
1299
|
-
});
|
|
1300
|
-
return;
|
|
1301
|
-
}
|
|
1302
|
-
if (parsed.action === "register") {
|
|
1303
|
-
const result = await registerCorpus({
|
|
1304
|
-
repoRoot,
|
|
1305
|
-
manifestPath: parsed.manifestPath,
|
|
1306
|
-
});
|
|
1307
|
-
if (parsed.json) {
|
|
1308
|
-
printCorpus({
|
|
1309
|
-
manifest: result.manifest,
|
|
1310
|
-
summary: result.summary,
|
|
1311
|
-
json: true,
|
|
1312
|
-
extra: {
|
|
1313
|
-
idempotent: result.idempotent,
|
|
1314
|
-
manifestPath: result.manifestPath,
|
|
1315
|
-
},
|
|
1316
|
-
});
|
|
1317
|
-
return;
|
|
1318
|
-
}
|
|
1319
|
-
process.stdout.write(`${result.idempotent ? "idempotent " : ""}registered ${result.manifest.corpusId}\n${formatCorpusMarkdown(result)}`);
|
|
1320
|
-
return;
|
|
1321
|
-
}
|
|
1322
|
-
if (parsed.action === "show") {
|
|
1323
|
-
const result = await showCorpus({
|
|
1324
|
-
repoRoot,
|
|
1325
|
-
corpusId: parsed.corpusId,
|
|
1326
|
-
});
|
|
1327
|
-
printCorpus({
|
|
1328
|
-
manifest: result.manifest,
|
|
1329
|
-
summary: result.summary,
|
|
1330
|
-
json: parsed.json,
|
|
1331
|
-
});
|
|
1332
|
-
return;
|
|
1333
|
-
}
|
|
1334
|
-
const rows = await listCorpora({ repoRoot });
|
|
1335
|
-
if (parsed.json) {
|
|
1336
|
-
console.log(JSON.stringify(rows, null, 2));
|
|
1337
|
-
return;
|
|
1338
|
-
}
|
|
1339
|
-
const lines = [
|
|
1340
|
-
"# Corpora",
|
|
1341
|
-
"",
|
|
1342
|
-
...rows.map((row) => `- \`${row.corpusId}\` tasks=${row.taskCount} hash=\`${row.corpusHash}\` splits=public:${row.splits.public}/private:${row.splits.private}/held_out:${row.splits.held_out}`),
|
|
1343
|
-
"",
|
|
1344
|
-
];
|
|
1345
|
-
process.stdout.write(`${lines.join("\n")}\n`);
|
|
1346
|
-
return;
|
|
1347
|
-
}
|
|
1348
185
|
// candidate subcommands
|
|
1349
186
|
if (parsed.action === "register") {
|
|
1350
187
|
const result = await registerCandidate({
|
|
@@ -1380,7 +217,7 @@ export async function runEval(repoRoot, args) {
|
|
|
1380
217
|
const lines = [
|
|
1381
218
|
"# Candidates",
|
|
1382
219
|
"",
|
|
1383
|
-
...rows.map((row) => `- \`${row.candidateId}\` status=\`${row.status}\` hash=\`${row.bundleHash}\` promotionApplied
|
|
220
|
+
...rows.map((row) => `- \`${row.candidateId}\` status=\`${row.status}\` hash=\`${row.bundleHash}\` promotionApplied=false`),
|
|
1384
221
|
"",
|
|
1385
222
|
];
|
|
1386
223
|
process.stdout.write(`${lines.join("\n")}\n`);
|