@tea-agent/loop-agent 0.16.0 → 0.16.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +7 -51
- package/README.md +1 -1
- package/dist/application/evaluation/candidate.js +6 -23
- package/dist/application/evaluation/types.js +0 -522
- package/dist/cli/command-definitions.js +3 -19
- package/dist/commands/eval.js +13 -1176
- package/dist/commands/init.js +1 -4
- package/dist/executors/dag-pi-executor.js +44 -4
- package/dist/worker/delivery/verification-bundle.js +2 -13
- package/dist/worker/feature/review.js +2 -3
- package/dist/worker/outcomes/adapters.js +1 -4
- package/dist/worker/outcomes/gate.js +11 -10
- package/dist/worker/outcomes/projector.js +1 -10
- package/dist/worker/outcomes/types.js +0 -3
- package/dist/worker/run-task/run-task.js +4 -23
- package/dist/worker/task-graph/ready-planner.js +8 -14
- package/dist/worker/task-graph/task-graph-schema.js +3 -5
- package/dist/workflows/dag/frontend-implementation-contract.js +77 -0
- package/dist/workflows/dag/init-hybrid.js +47 -3
- package/dist/workflows/dag/knowledge-curator.js +0 -3
- package/dist/workflows/dag/node-execution.js +4 -11
- package/dist/workflows/dag/prompt.js +1 -1
- package/dist/workflows/dag/runner.js +16 -43
- package/dist/workflows/dag/skill-snapshot.js +7 -11
- package/dist/workflows/dag/types.js +1 -18
- package/docs/init-surface.manifest.json +0 -3
- package/docs/templates/agent-dag.schema.json +5 -0
- package/docs/templates/product-line/AGENTS.md +0 -1
- package/docs/templates/product-line/README.md +0 -17
- package/docs/templates/product-line/acceptance.yaml +0 -9
- package/docs/templates/product-line/task-graph.yaml +0 -8
- package/docs/templates/product-line/task.yaml +0 -4
- package/package.json +1 -1
- package/skills/frontend-implementation/references/node-contracts.md +3 -3
- package/skills/loop-agent/references/hybrid-dag.md +1 -1
- package/dist/application/evaluation/alias.js +0 -184
- package/dist/application/evaluation/budget.js +0 -192
- package/dist/application/evaluation/campaign-hash.js +0 -47
- package/dist/application/evaluation/campaign-matrix.js +0 -372
- package/dist/application/evaluation/campaign-scorecard.js +0 -135
- package/dist/application/evaluation/campaign.js +0 -370
- package/dist/application/evaluation/corpus-hash.js +0 -38
- package/dist/application/evaluation/corpus.js +0 -56
- package/dist/application/evaluation/experiment.js +0 -294
- package/dist/application/evaluation/ignition.js +0 -198
- package/dist/application/evaluation/integrity-audit.js +0 -162
- package/dist/application/evaluation/outer-loop.js +0 -132
- package/dist/application/evaluation/pi-cell-executor.js +0 -39
- package/dist/application/evaluation/private-verifier.js +0 -46
- package/dist/application/evaluation/promotion-policy.js +0 -151
- package/dist/application/evaluation/proposer.js +0 -98
- package/dist/infrastructure/evaluation/alias-store.js +0 -199
- package/dist/infrastructure/evaluation/campaign-store.js +0 -154
- package/dist/infrastructure/evaluation/corpus-store.js +0 -181
- package/dist/infrastructure/evaluation/experiment-store.js +0 -124
- package/dist/infrastructure/evaluation/ignition-store.js +0 -82
- package/dist/infrastructure/evaluation/private-verifier-store.js +0 -145
- package/dist/infrastructure/evaluation/proposer-store.js +0 -78
- package/dist/worker/outcomes/evidence-tokens.js +0 -29
- package/dist/workflows/dag/budget-enforcement.js +0 -67
- package/dist/workflows/dag/context-policy.js +0 -137
- package/docs/templates/evaluation/campaign-budget-v1.json +0 -12
- package/docs/templates/evaluation/campaign-dogfood-v0.json +0 -24
- package/docs/templates/evaluation/campaign-evidence-v1.json +0 -44
- package/docs/templates/evaluation/context-policy-baseline-v1.json +0 -17
- package/docs/templates/evaluation/context-policy-role-specialized-v1.json +0 -28
- package/docs/templates/evaluation/corpus-dogfood-v0.manifest.json +0 -118
- package/docs/templates/evaluation/matrix-dag-dry-run-v1.json +0 -21
- package/docs/templates/evaluation/matrix-fixture-v1.json +0 -10
- package/docs/templates/evaluation/private-verifier-dogfood-v0.json +0 -16
- package/docs/templates/product-line/feature.yaml +0 -11
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
import { z } from "zod";
|
|
2
|
-
/**
|
|
3
|
-
* Campaign / DAG hard-budget contract (Eval Lab W3.4–W3.5 / M3).
|
|
4
|
-
*
|
|
5
|
-
* Token incompleteness: missing `tokensUsed` must never be treated as 0.
|
|
6
|
-
* When `maxTokens` is set, only known token samples accumulate; breach fires
|
|
7
|
-
* only when the known sum exceeds the limit. Calls + wall time + repair passes
|
|
8
|
-
* are the primary hard dimensions for v1.
|
|
9
|
-
*/
|
|
10
|
-
export const campaignBudgetModeSchema = z.enum(["hard", "record-only"]);
|
|
11
|
-
export const campaignBudgetLimitsSchema = z
|
|
12
|
-
.object({
|
|
13
|
-
maxTokens: z.number().int().positive().optional(),
|
|
14
|
-
maxWallTimeMs: z.number().int().positive().optional(),
|
|
15
|
-
maxExecutorCalls: z.number().int().positive().optional(),
|
|
16
|
-
maxRepairPasses: z.number().int().nonnegative().optional(),
|
|
17
|
-
maxConcurrency: z.number().int().positive().optional(),
|
|
18
|
-
maxContextChars: z.number().int().positive().optional(),
|
|
19
|
-
})
|
|
20
|
-
.strict()
|
|
21
|
-
.superRefine((limits, ctx) => {
|
|
22
|
-
const hasHardDim = limits.maxTokens !== undefined ||
|
|
23
|
-
limits.maxWallTimeMs !== undefined ||
|
|
24
|
-
limits.maxExecutorCalls !== undefined ||
|
|
25
|
-
limits.maxRepairPasses !== undefined ||
|
|
26
|
-
limits.maxContextChars !== undefined;
|
|
27
|
-
if (!hasHardDim) {
|
|
28
|
-
ctx.addIssue({
|
|
29
|
-
code: z.ZodIssueCode.custom,
|
|
30
|
-
message: "budget.limits requires at least one of maxTokens, maxWallTimeMs, maxExecutorCalls, maxRepairPasses, maxContextChars",
|
|
31
|
-
});
|
|
32
|
-
}
|
|
33
|
-
});
|
|
34
|
-
export const campaignBudgetSchema = z
|
|
35
|
-
.object({
|
|
36
|
-
schemaVersion: z.literal(1),
|
|
37
|
-
mode: campaignBudgetModeSchema.default("hard"),
|
|
38
|
-
limits: campaignBudgetLimitsSchema,
|
|
39
|
-
})
|
|
40
|
-
.strict();
|
|
41
|
-
export function createBudgetLedger(budget) {
|
|
42
|
-
return {
|
|
43
|
-
schemaVersion: 1,
|
|
44
|
-
mode: budget.mode,
|
|
45
|
-
limits: { ...budget.limits },
|
|
46
|
-
consumed: {
|
|
47
|
-
tokens: null,
|
|
48
|
-
wallTimeMs: 0,
|
|
49
|
-
executorCalls: 0,
|
|
50
|
-
repairPasses: 0,
|
|
51
|
-
peakContextChars: 0,
|
|
52
|
-
missingTokenNodeIds: [],
|
|
53
|
-
},
|
|
54
|
-
breaches: [],
|
|
55
|
-
status: budget.mode === "record-only" ? "record-only" : "ok",
|
|
56
|
-
};
|
|
57
|
-
}
|
|
58
|
-
export function resolveEffectiveMaxConcurrent(requested, budget) {
|
|
59
|
-
const limit = budget?.limits.maxConcurrency;
|
|
60
|
-
if (limit === undefined) {
|
|
61
|
-
return { maxConcurrent: Math.max(1, requested), clamped: false };
|
|
62
|
-
}
|
|
63
|
-
const maxConcurrent = Math.max(1, Math.min(requested, limit));
|
|
64
|
-
return { maxConcurrent, clamped: maxConcurrent < requested };
|
|
65
|
-
}
|
|
66
|
-
function pushBreach(ledger, breach) {
|
|
67
|
-
const full = {
|
|
68
|
-
...breach,
|
|
69
|
-
at: breach.at ?? new Date().toISOString(),
|
|
70
|
-
};
|
|
71
|
-
const duplicate = ledger.breaches.some((existing) => existing.dimension === full.dimension &&
|
|
72
|
-
existing.nodeId === full.nodeId &&
|
|
73
|
-
existing.limit === full.limit);
|
|
74
|
-
if (!duplicate) {
|
|
75
|
-
ledger.breaches.push(full);
|
|
76
|
-
}
|
|
77
|
-
if (ledger.mode === "hard") {
|
|
78
|
-
ledger.status = "breached";
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
function checkLimit(ledger, dimension, consumed, limit, opts) {
|
|
82
|
-
if (limit === undefined)
|
|
83
|
-
return undefined;
|
|
84
|
-
const exceeded = opts?.inclusive ? consumed >= limit : consumed > limit;
|
|
85
|
-
if (!exceeded)
|
|
86
|
-
return undefined;
|
|
87
|
-
const breach = {
|
|
88
|
-
dimension,
|
|
89
|
-
limit,
|
|
90
|
-
consumed,
|
|
91
|
-
at: new Date().toISOString(),
|
|
92
|
-
...(opts?.nodeId ? { nodeId: opts.nodeId } : {}),
|
|
93
|
-
};
|
|
94
|
-
pushBreach(ledger, breach);
|
|
95
|
-
return breach;
|
|
96
|
-
}
|
|
97
|
-
/**
|
|
98
|
-
* Pre-node check (no call increment yet).
|
|
99
|
-
* Discrete call budget uses inclusive compare so maxExecutorCalls=1 allows the
|
|
100
|
-
* first node then blocks starting a second.
|
|
101
|
-
*/
|
|
102
|
-
export function checkBudgetPreNode(ledger, sample) {
|
|
103
|
-
ledger.consumed.wallTimeMs = sample.wallTimeMs;
|
|
104
|
-
ledger.consumed.repairPasses = sample.repairPasses;
|
|
105
|
-
return (checkLimit(ledger, "executorCalls", ledger.consumed.executorCalls, ledger.limits.maxExecutorCalls, { inclusive: true }) ??
|
|
106
|
-
checkLimit(ledger, "wallTimeMs", sample.wallTimeMs, ledger.limits.maxWallTimeMs) ??
|
|
107
|
-
checkLimit(ledger, "repairPasses", sample.repairPasses, ledger.limits.maxRepairPasses));
|
|
108
|
-
}
|
|
109
|
-
/**
|
|
110
|
-
* Record one finished node execution attempt and evaluate hard limits.
|
|
111
|
-
* Increments executorCalls by 1. Does not invent tokens when missing.
|
|
112
|
-
*/
|
|
113
|
-
export function recordNodeBudgetSample(ledger, sample) {
|
|
114
|
-
ledger.consumed.executorCalls += 1;
|
|
115
|
-
ledger.consumed.wallTimeMs = sample.wallTimeMs;
|
|
116
|
-
ledger.consumed.repairPasses = sample.repairPasses;
|
|
117
|
-
if (sample.tokensUsed === undefined) {
|
|
118
|
-
if (!ledger.consumed.missingTokenNodeIds.includes(sample.nodeId)) {
|
|
119
|
-
ledger.consumed.missingTokenNodeIds.push(sample.nodeId);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
else {
|
|
123
|
-
ledger.consumed.tokens = (ledger.consumed.tokens ?? 0) + sample.tokensUsed;
|
|
124
|
-
}
|
|
125
|
-
if (sample.contextChars !== undefined) {
|
|
126
|
-
ledger.consumed.peakContextChars = Math.max(ledger.consumed.peakContextChars, sample.contextChars);
|
|
127
|
-
}
|
|
128
|
-
return (checkLimit(ledger, "executorCalls", ledger.consumed.executorCalls, ledger.limits.maxExecutorCalls, { nodeId: sample.nodeId, inclusive: true }) ??
|
|
129
|
-
checkLimit(ledger, "wallTimeMs", ledger.consumed.wallTimeMs, ledger.limits.maxWallTimeMs, { nodeId: sample.nodeId }) ??
|
|
130
|
-
checkLimit(ledger, "repairPasses", ledger.consumed.repairPasses, ledger.limits.maxRepairPasses, { nodeId: sample.nodeId }) ??
|
|
131
|
-
(ledger.consumed.tokens !== null
|
|
132
|
-
? checkLimit(ledger, "tokens", ledger.consumed.tokens, ledger.limits.maxTokens, { nodeId: sample.nodeId })
|
|
133
|
-
: undefined) ??
|
|
134
|
-
checkLimit(ledger, "contextChars", ledger.consumed.peakContextChars, ledger.limits.maxContextChars, { nodeId: sample.nodeId }));
|
|
135
|
-
}
|
|
136
|
-
export function isHardBudgetBreached(ledger) {
|
|
137
|
-
return Boolean(ledger && ledger.mode === "hard" && ledger.status === "breached");
|
|
138
|
-
}
|
|
139
|
-
/** Mark all PENDING nodes as SKIPPED due to budget breach (fail closed). */
|
|
140
|
-
export function skipPendingNodesForBudgetBreach(nodes, breach) {
|
|
141
|
-
const skipped = [];
|
|
142
|
-
const reason = `budget_breach:${breach.dimension}`;
|
|
143
|
-
for (const [id, node] of Object.entries(nodes)) {
|
|
144
|
-
if (node.status !== "PENDING")
|
|
145
|
-
continue;
|
|
146
|
-
node.status = "SKIPPED";
|
|
147
|
-
node.skippedReason = reason;
|
|
148
|
-
skipped.push(id);
|
|
149
|
-
}
|
|
150
|
-
return skipped;
|
|
151
|
-
}
|
|
152
|
-
export function formatBudgetReportMarkdown(ledger) {
|
|
153
|
-
const limits = ledger.limits;
|
|
154
|
-
const c = ledger.consumed;
|
|
155
|
-
const lines = [
|
|
156
|
-
`# Budget Ledger`,
|
|
157
|
-
``,
|
|
158
|
-
`- **mode**: ${ledger.mode}`,
|
|
159
|
-
`- **status**: ${ledger.status}`,
|
|
160
|
-
``,
|
|
161
|
-
`## Limits`,
|
|
162
|
-
``,
|
|
163
|
-
`| dimension | limit |`,
|
|
164
|
-
`|---|---|`,
|
|
165
|
-
`| maxTokens | ${limits.maxTokens ?? "n/a"} |`,
|
|
166
|
-
`| maxWallTimeMs | ${limits.maxWallTimeMs ?? "n/a"} |`,
|
|
167
|
-
`| maxExecutorCalls | ${limits.maxExecutorCalls ?? "n/a"} |`,
|
|
168
|
-
`| maxRepairPasses | ${limits.maxRepairPasses ?? "n/a"} |`,
|
|
169
|
-
`| maxConcurrency | ${limits.maxConcurrency ?? "n/a"} |`,
|
|
170
|
-
`| maxContextChars | ${limits.maxContextChars ?? "n/a"} |`,
|
|
171
|
-
``,
|
|
172
|
-
`## Consumed`,
|
|
173
|
-
``,
|
|
174
|
-
`| dimension | value |`,
|
|
175
|
-
`|---|---|`,
|
|
176
|
-
`| tokens | ${c.tokens ?? "n/a (missing)"} |`,
|
|
177
|
-
`| wallTimeMs | ${c.wallTimeMs} |`,
|
|
178
|
-
`| executorCalls | ${c.executorCalls} |`,
|
|
179
|
-
`| repairPasses | ${c.repairPasses} |`,
|
|
180
|
-
`| peakContextChars | ${c.peakContextChars} |`,
|
|
181
|
-
`| missingTokenNodes | ${c.missingTokenNodeIds.join(", ") || "none"} |`,
|
|
182
|
-
];
|
|
183
|
-
if (ledger.breaches.length > 0) {
|
|
184
|
-
lines.push(``, `## Breaches`, ``);
|
|
185
|
-
for (const breach of ledger.breaches) {
|
|
186
|
-
lines.push(`- **${breach.dimension}**: consumed ${breach.consumed} > limit ${breach.limit}` +
|
|
187
|
-
(breach.nodeId ? ` (node ${breach.nodeId})` : "") +
|
|
188
|
-
` at ${breach.at}`);
|
|
189
|
-
}
|
|
190
|
-
}
|
|
191
|
-
return `${lines.join("\n")}\n`;
|
|
192
|
-
}
|
|
@@ -1,47 +0,0 @@
|
|
|
1
|
-
import { formatContentSha, sha256Hex, } from "./candidate-hash.js";
|
|
2
|
-
/** Frozen placeholder for M4; M5 may replace with real scoring rules bytes. */
|
|
3
|
-
export const SCORING_RULES_V1 = "promotion-policy-v1";
|
|
4
|
-
export function computeScoringRulesHash(policyVersion = SCORING_RULES_V1) {
|
|
5
|
-
return formatContentSha(sha256Hex(policyVersion));
|
|
6
|
-
}
|
|
7
|
-
export function computeBudgetHash(budget) {
|
|
8
|
-
const canonical = {
|
|
9
|
-
schemaVersion: budget.schemaVersion,
|
|
10
|
-
mode: budget.mode,
|
|
11
|
-
limits: Object.fromEntries(Object.entries(budget.limits).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)),
|
|
12
|
-
};
|
|
13
|
-
return formatContentSha(sha256Hex(JSON.stringify(canonical)));
|
|
14
|
-
}
|
|
15
|
-
export function computePrivateVerifierManifestHash(manifest) {
|
|
16
|
-
const checks = [...manifest.checks]
|
|
17
|
-
.map((check) => ({
|
|
18
|
-
checkId: check.checkId,
|
|
19
|
-
kind: check.kind,
|
|
20
|
-
command: [...check.command],
|
|
21
|
-
cwd: check.cwd ?? ".",
|
|
22
|
-
timeoutMs: check.timeoutMs ?? 60_000,
|
|
23
|
-
expectedExitCode: check.expectedExitCode ?? 0,
|
|
24
|
-
}))
|
|
25
|
-
.sort((a, b) => a.checkId < b.checkId ? -1 : a.checkId > b.checkId ? 1 : 0);
|
|
26
|
-
const canonical = {
|
|
27
|
-
schemaVersion: 1,
|
|
28
|
-
privateVerifierId: manifest.privateVerifierId,
|
|
29
|
-
checks,
|
|
30
|
-
};
|
|
31
|
-
return formatContentSha(sha256Hex(JSON.stringify(canonical)));
|
|
32
|
-
}
|
|
33
|
-
export function computeCampaignPlanHash(campaignId, cells) {
|
|
34
|
-
const sorted = [...cells].sort((a, b) => a.cellId < b.cellId ? -1 : a.cellId > b.cellId ? 1 : 0);
|
|
35
|
-
const canonical = {
|
|
36
|
-
schemaVersion: 1,
|
|
37
|
-
campaignId,
|
|
38
|
-
cells: sorted,
|
|
39
|
-
};
|
|
40
|
-
return formatContentSha(sha256Hex(JSON.stringify(canonical)));
|
|
41
|
-
}
|
|
42
|
-
export function assertPrivateVerifierHash(manifest) {
|
|
43
|
-
const recomputed = computePrivateVerifierManifestHash(manifest);
|
|
44
|
-
if (recomputed !== manifest.manifestHash) {
|
|
45
|
-
throw new Error(`privateVerifier manifestHash mismatch for ${manifest.privateVerifierId}: expected ${recomputed}, got ${manifest.manifestHash}`);
|
|
46
|
-
}
|
|
47
|
-
}
|
|
@@ -1,372 +0,0 @@
|
|
|
1
|
-
import { mkdir, readFile } from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { campaignDir, readCampaignManifest, readCampaignPlan, writeCampaignScorecard, } from "../../infrastructure/evaluation/campaign-store.js";
|
|
4
|
-
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
5
|
-
import { planCampaign, runCampaignPrivateChecks, } from "./campaign.js";
|
|
6
|
-
import { evaluatePromotionPolicy } from "./promotion-policy.js";
|
|
7
|
-
import { matrixFixtureSchema, matrixRunSchema } from "./types.js";
|
|
8
|
-
function matrixJobsDir(repoRoot, campaignId) {
|
|
9
|
-
return path.join(campaignDir(repoRoot, campaignId), "matrix-jobs");
|
|
10
|
-
}
|
|
11
|
-
function matrixRunPath(repoRoot, campaignId) {
|
|
12
|
-
return path.join(campaignDir(repoRoot, campaignId), "matrix-run.json");
|
|
13
|
-
}
|
|
14
|
-
function evidenceDraftPath(repoRoot, campaignId) {
|
|
15
|
-
return path.join(campaignDir(repoRoot, campaignId), "evidence-draft.json");
|
|
16
|
-
}
|
|
17
|
-
export async function loadMatrixFixture(input) {
|
|
18
|
-
const absolute = path.resolve(input.repoRoot, input.fixturePath);
|
|
19
|
-
return matrixFixtureSchema.parse(JSON.parse(await readFile(absolute, "utf-8")));
|
|
20
|
-
}
|
|
21
|
-
function resolveStubResult(fixture, cell) {
|
|
22
|
-
for (const row of fixture.results) {
|
|
23
|
-
if (row.cellId && row.cellId === cell.cellId) {
|
|
24
|
-
return { verifyPassed: row.verifyPassed, runId: row.runId };
|
|
25
|
-
}
|
|
26
|
-
const match = (!row.candidateId || row.candidateId === cell.candidateId) &&
|
|
27
|
-
(!row.taskRef || row.taskRef === cell.taskRef) &&
|
|
28
|
-
(row.seed === undefined || row.seed === cell.seed) &&
|
|
29
|
-
(!row.split || row.split === cell.split);
|
|
30
|
-
if (match &&
|
|
31
|
-
(row.candidateId || row.taskRef || row.seed !== undefined || row.split)) {
|
|
32
|
-
return { verifyPassed: row.verifyPassed, runId: row.runId };
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
return { verifyPassed: fixture.defaultVerifyPassed };
|
|
36
|
-
}
|
|
37
|
-
export function buildMatrixJobSpec(input) {
|
|
38
|
-
return {
|
|
39
|
-
schemaVersion: 1,
|
|
40
|
-
cellId: input.cell.cellId,
|
|
41
|
-
evaluation: {
|
|
42
|
-
campaignId: input.campaignId,
|
|
43
|
-
candidateId: input.cell.candidateId,
|
|
44
|
-
bundleHash: input.cell.bundleHash,
|
|
45
|
-
seed: input.cell.seed,
|
|
46
|
-
split: input.cell.split,
|
|
47
|
-
taskRef: input.cell.taskRef,
|
|
48
|
-
},
|
|
49
|
-
executor: "pi",
|
|
50
|
-
autoPromote: false,
|
|
51
|
-
};
|
|
52
|
-
}
|
|
53
|
-
async function writeMatrixJob(input) {
|
|
54
|
-
const dir = matrixJobsDir(input.repoRoot, input.campaignId);
|
|
55
|
-
await mkdir(dir, { recursive: true });
|
|
56
|
-
const job = buildMatrixJobSpec({
|
|
57
|
-
cell: input.cell,
|
|
58
|
-
campaignId: input.campaignId,
|
|
59
|
-
});
|
|
60
|
-
const jobPath = path.join(dir, `${input.cell.cellId}.json`);
|
|
61
|
-
await writeJsonAtomic(jobPath, job, { repoRoot: input.repoRoot });
|
|
62
|
-
return {
|
|
63
|
-
jobPath: path.relative(input.repoRoot, jobPath).replace(/\\/g, "/"),
|
|
64
|
-
evaluation: job.evaluation,
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
/**
|
|
68
|
-
* Candidate matrix runner (M7 companion).
|
|
69
|
-
* Modes:
|
|
70
|
-
* - dry-run: plan public cells only
|
|
71
|
-
* - stub: resolve verifyPassed from fixture (tests / offline)
|
|
72
|
-
* - pi-plan: materialize per-cell Pi evaluation job specs (no spawn)
|
|
73
|
-
* - pi: require injected PiCellExecutor; never auto-promote
|
|
74
|
-
*/
|
|
75
|
-
export async function runCampaignMatrix(input) {
|
|
76
|
-
const planned = await planCampaign({
|
|
77
|
-
repoRoot: input.repoRoot,
|
|
78
|
-
campaignId: input.campaignId,
|
|
79
|
-
});
|
|
80
|
-
const publicCells = planned.plan.cells.filter((cell) => cell.executionRole === "candidate_run");
|
|
81
|
-
const limited = input.maxPublicCells !== undefined
|
|
82
|
-
? publicCells.slice(0, input.maxPublicCells)
|
|
83
|
-
: publicCells;
|
|
84
|
-
let fixture;
|
|
85
|
-
if (input.mode === "stub") {
|
|
86
|
-
if (!input.fixturePath) {
|
|
87
|
-
throw new Error("matrix mode=stub requires --fixture <path>");
|
|
88
|
-
}
|
|
89
|
-
fixture = await loadMatrixFixture({
|
|
90
|
-
repoRoot: input.repoRoot,
|
|
91
|
-
fixturePath: input.fixturePath,
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
if (input.mode === "pi" && !input.piExecutor) {
|
|
95
|
-
throw new Error("matrix mode=pi requires an injected PiCellExecutor (CLI refuses live Pi by default; use pi-plan or stub)");
|
|
96
|
-
}
|
|
97
|
-
const cellResults = [];
|
|
98
|
-
for (const cell of limited) {
|
|
99
|
-
if (input.mode === "dry-run") {
|
|
100
|
-
cellResults.push({
|
|
101
|
-
cellId: cell.cellId,
|
|
102
|
-
candidateId: cell.candidateId,
|
|
103
|
-
taskRef: cell.taskRef,
|
|
104
|
-
seed: cell.seed,
|
|
105
|
-
split: cell.split,
|
|
106
|
-
executionRole: cell.executionRole,
|
|
107
|
-
mode: "dry-run",
|
|
108
|
-
skipped: true,
|
|
109
|
-
detail: "dry-run: would execute candidate_run cell",
|
|
110
|
-
});
|
|
111
|
-
continue;
|
|
112
|
-
}
|
|
113
|
-
if (input.mode === "stub") {
|
|
114
|
-
const resolved = resolveStubResult(fixture, cell);
|
|
115
|
-
cellResults.push({
|
|
116
|
-
cellId: cell.cellId,
|
|
117
|
-
candidateId: cell.candidateId,
|
|
118
|
-
taskRef: cell.taskRef,
|
|
119
|
-
seed: cell.seed,
|
|
120
|
-
split: cell.split,
|
|
121
|
-
executionRole: cell.executionRole,
|
|
122
|
-
mode: "stub",
|
|
123
|
-
verifyPassed: resolved.verifyPassed,
|
|
124
|
-
runId: resolved.runId,
|
|
125
|
-
skipped: false,
|
|
126
|
-
});
|
|
127
|
-
continue;
|
|
128
|
-
}
|
|
129
|
-
const job = await writeMatrixJob({
|
|
130
|
-
repoRoot: input.repoRoot,
|
|
131
|
-
campaignId: input.campaignId,
|
|
132
|
-
cell,
|
|
133
|
-
});
|
|
134
|
-
if (input.mode === "pi-plan") {
|
|
135
|
-
cellResults.push({
|
|
136
|
-
cellId: cell.cellId,
|
|
137
|
-
candidateId: cell.candidateId,
|
|
138
|
-
taskRef: cell.taskRef,
|
|
139
|
-
seed: cell.seed,
|
|
140
|
-
split: cell.split,
|
|
141
|
-
executionRole: cell.executionRole,
|
|
142
|
-
mode: "pi-plan",
|
|
143
|
-
jobPath: job.jobPath,
|
|
144
|
-
skipped: true,
|
|
145
|
-
detail: "pi-plan: job materialized; Pi not spawned",
|
|
146
|
-
});
|
|
147
|
-
continue;
|
|
148
|
-
}
|
|
149
|
-
const executed = await input.piExecutor({
|
|
150
|
-
repoRoot: input.repoRoot,
|
|
151
|
-
cell,
|
|
152
|
-
jobPath: job.jobPath,
|
|
153
|
-
evaluation: job.evaluation,
|
|
154
|
-
});
|
|
155
|
-
cellResults.push({
|
|
156
|
-
cellId: cell.cellId,
|
|
157
|
-
candidateId: cell.candidateId,
|
|
158
|
-
taskRef: cell.taskRef,
|
|
159
|
-
seed: cell.seed,
|
|
160
|
-
split: cell.split,
|
|
161
|
-
executionRole: cell.executionRole,
|
|
162
|
-
mode: "pi",
|
|
163
|
-
verifyPassed: executed.verifyPassed,
|
|
164
|
-
runId: executed.runId,
|
|
165
|
-
jobPath: job.jobPath,
|
|
166
|
-
detail: executed.detail,
|
|
167
|
-
skipped: false,
|
|
168
|
-
});
|
|
169
|
-
}
|
|
170
|
-
// Mark held_out / remaining public as skipped for transparency
|
|
171
|
-
for (const cell of planned.plan.cells) {
|
|
172
|
-
if (cell.executionRole === "candidate_run") {
|
|
173
|
-
if (limited.some((row) => row.cellId === cell.cellId))
|
|
174
|
-
continue;
|
|
175
|
-
cellResults.push({
|
|
176
|
-
cellId: cell.cellId,
|
|
177
|
-
candidateId: cell.candidateId,
|
|
178
|
-
taskRef: cell.taskRef,
|
|
179
|
-
seed: cell.seed,
|
|
180
|
-
split: cell.split,
|
|
181
|
-
executionRole: cell.executionRole,
|
|
182
|
-
mode: input.mode,
|
|
183
|
-
skipped: true,
|
|
184
|
-
detail: "skipped by maxPublicCells",
|
|
185
|
-
});
|
|
186
|
-
continue;
|
|
187
|
-
}
|
|
188
|
-
if (cell.executionRole === "promotion_gate_only") {
|
|
189
|
-
cellResults.push({
|
|
190
|
-
cellId: cell.cellId,
|
|
191
|
-
candidateId: cell.candidateId,
|
|
192
|
-
taskRef: cell.taskRef,
|
|
193
|
-
seed: cell.seed,
|
|
194
|
-
split: cell.split,
|
|
195
|
-
executionRole: cell.executionRole,
|
|
196
|
-
mode: input.mode,
|
|
197
|
-
skipped: true,
|
|
198
|
-
detail: "held_out remains promotion_gate_only (not executed in matrix)",
|
|
199
|
-
});
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
let privateResults = [];
|
|
203
|
-
if (input.includePrivateChecks !== false && input.mode !== "dry-run") {
|
|
204
|
-
const privateRun = await runCampaignPrivateChecks({
|
|
205
|
-
repoRoot: input.repoRoot,
|
|
206
|
-
campaignId: input.campaignId,
|
|
207
|
-
});
|
|
208
|
-
privateResults = privateRun.results;
|
|
209
|
-
for (const cell of planned.plan.cells) {
|
|
210
|
-
if (cell.executionRole !== "private_verify")
|
|
211
|
-
continue;
|
|
212
|
-
const related = privateResults.filter((row) => row.cellId === cell.cellId);
|
|
213
|
-
const ok = related.length > 0 && related.every((row) => row.ok);
|
|
214
|
-
cellResults.push({
|
|
215
|
-
cellId: cell.cellId,
|
|
216
|
-
candidateId: cell.candidateId,
|
|
217
|
-
taskRef: cell.taskRef,
|
|
218
|
-
seed: cell.seed,
|
|
219
|
-
split: cell.split,
|
|
220
|
-
executionRole: "private_verify",
|
|
221
|
-
mode: input.mode,
|
|
222
|
-
verifyPassed: ok,
|
|
223
|
-
skipped: false,
|
|
224
|
-
detail: `private checks=${related.length}`,
|
|
225
|
-
});
|
|
226
|
-
}
|
|
227
|
-
}
|
|
228
|
-
const now = input.now ?? new Date().toISOString();
|
|
229
|
-
let evidence;
|
|
230
|
-
let evidencePath;
|
|
231
|
-
if (input.writeEvidenceDraft !== false && input.mode !== "dry-run") {
|
|
232
|
-
evidence = buildEvidenceDraft({
|
|
233
|
-
campaignId: input.campaignId,
|
|
234
|
-
manifest: planned.manifest,
|
|
235
|
-
cellResults,
|
|
236
|
-
});
|
|
237
|
-
evidencePath = path
|
|
238
|
-
.relative(input.repoRoot, evidenceDraftPath(input.repoRoot, input.campaignId))
|
|
239
|
-
.replace(/\\/g, "/");
|
|
240
|
-
await writeJsonAtomic(evidenceDraftPath(input.repoRoot, input.campaignId), evidence, { repoRoot: input.repoRoot });
|
|
241
|
-
}
|
|
242
|
-
const matrix = matrixRunSchema.parse({
|
|
243
|
-
schemaVersion: 1,
|
|
244
|
-
campaignId: input.campaignId,
|
|
245
|
-
planHash: planned.plan.planHash,
|
|
246
|
-
mode: input.mode,
|
|
247
|
-
createdAt: now,
|
|
248
|
-
cells: cellResults,
|
|
249
|
-
privateCheckCount: privateResults.length,
|
|
250
|
-
privateCheckFailed: privateResults.filter((row) => !row.ok).length,
|
|
251
|
-
evidencePath,
|
|
252
|
-
autoPromote: false,
|
|
253
|
-
});
|
|
254
|
-
await writeJsonAtomic(matrixRunPath(input.repoRoot, input.campaignId), matrix, {
|
|
255
|
-
repoRoot: input.repoRoot,
|
|
256
|
-
});
|
|
257
|
-
return { matrix, privateResults, evidence };
|
|
258
|
-
}
|
|
259
|
-
function buildEvidenceDraft(input) {
|
|
260
|
-
const challengerCandidateId = input.manifest.challengerCandidateIds[0];
|
|
261
|
-
if (!challengerCandidateId) {
|
|
262
|
-
throw new Error("campaign has no challenger for evidence draft");
|
|
263
|
-
}
|
|
264
|
-
const cells = [];
|
|
265
|
-
for (const row of input.cellResults) {
|
|
266
|
-
if (row.verifyPassed === undefined)
|
|
267
|
-
continue;
|
|
268
|
-
if (row.executionRole !== "candidate_run" &&
|
|
269
|
-
row.executionRole !== "private_verify" &&
|
|
270
|
-
row.executionRole !== "promotion_gate_only") {
|
|
271
|
-
continue;
|
|
272
|
-
}
|
|
273
|
-
// Only include executed public/private pairs; held_out needs external evidence.
|
|
274
|
-
if (row.executionRole === "promotion_gate_only")
|
|
275
|
-
continue;
|
|
276
|
-
cells.push({
|
|
277
|
-
cellId: row.cellId,
|
|
278
|
-
candidateId: row.candidateId,
|
|
279
|
-
taskRef: row.taskRef,
|
|
280
|
-
seed: row.seed,
|
|
281
|
-
split: row.split,
|
|
282
|
-
verifyPassed: row.verifyPassed,
|
|
283
|
-
runId: row.runId,
|
|
284
|
-
});
|
|
285
|
-
}
|
|
286
|
-
if (cells.length === 0) {
|
|
287
|
-
// Minimal placeholder so draft is schema-valid for inspection; scorecard needs more.
|
|
288
|
-
cells.push({
|
|
289
|
-
candidateId: input.manifest.incumbentCandidateId,
|
|
290
|
-
taskRef: "matrix/placeholder",
|
|
291
|
-
seed: 0,
|
|
292
|
-
split: "public",
|
|
293
|
-
verifyPassed: false,
|
|
294
|
-
});
|
|
295
|
-
cells.push({
|
|
296
|
-
candidateId: challengerCandidateId,
|
|
297
|
-
taskRef: "matrix/placeholder",
|
|
298
|
-
seed: 0,
|
|
299
|
-
split: "public",
|
|
300
|
-
verifyPassed: false,
|
|
301
|
-
});
|
|
302
|
-
}
|
|
303
|
-
return {
|
|
304
|
-
schemaVersion: 1,
|
|
305
|
-
campaignId: input.campaignId,
|
|
306
|
-
challengerCandidateId,
|
|
307
|
-
cells,
|
|
308
|
-
diffs: [],
|
|
309
|
-
verifyCommands: { incumbent: [], challenger: [] },
|
|
310
|
-
budgetStatus: { incumbent: "missing", challenger: "missing" },
|
|
311
|
-
};
|
|
312
|
-
}
|
|
313
|
-
export async function readMatrixRun(repoRoot, campaignId) {
|
|
314
|
-
try {
|
|
315
|
-
const raw = await readFile(matrixRunPath(repoRoot, campaignId), "utf-8");
|
|
316
|
-
return matrixRunSchema.parse(JSON.parse(raw));
|
|
317
|
-
}
|
|
318
|
-
catch (error) {
|
|
319
|
-
const code = error.code;
|
|
320
|
-
if (code === "ENOENT")
|
|
321
|
-
return undefined;
|
|
322
|
-
throw error;
|
|
323
|
-
}
|
|
324
|
-
}
|
|
325
|
-
export function formatMatrixRunMarkdown(matrix) {
|
|
326
|
-
const executed = matrix.cells.filter((cell) => !cell.skipped);
|
|
327
|
-
const lines = [
|
|
328
|
-
`# Campaign Matrix: ${matrix.campaignId}`,
|
|
329
|
-
"",
|
|
330
|
-
`- mode: \`${matrix.mode}\``,
|
|
331
|
-
`- planHash: \`${matrix.planHash}\``,
|
|
332
|
-
`- cells: ${matrix.cells.length} (executed/recorded=${executed.length})`,
|
|
333
|
-
`- privateChecks: ${matrix.privateCheckCount} (failed=${matrix.privateCheckFailed})`,
|
|
334
|
-
`- evidencePath: \`${matrix.evidencePath ?? "(none)"}\``,
|
|
335
|
-
`- autoPromote: \`${matrix.autoPromote}\``,
|
|
336
|
-
"",
|
|
337
|
-
"## Cell results",
|
|
338
|
-
"",
|
|
339
|
-
];
|
|
340
|
-
for (const cell of matrix.cells.slice(0, 40)) {
|
|
341
|
-
lines.push(`- \`${cell.cellId}\` role=${cell.executionRole} skipped=${cell.skipped} passed=${cell.verifyPassed ?? "-"} ${cell.detail ?? ""}`);
|
|
342
|
-
}
|
|
343
|
-
if (matrix.cells.length > 40) {
|
|
344
|
-
lines.push(`- … ${matrix.cells.length - 40} more`);
|
|
345
|
-
}
|
|
346
|
-
lines.push("", "> Matrix never auto-promotes. Use `eval campaign scorecard` + human-gated `eval promote` / `eval experiment accept`.", "");
|
|
347
|
-
return `${lines.join("\n")}\n`;
|
|
348
|
-
}
|
|
349
|
-
/** Optional helper: build scorecard from matrix evidence draft when complete enough. */
|
|
350
|
-
export async function buildScorecardFromMatrixEvidence(input) {
|
|
351
|
-
const manifest = await readCampaignManifest(input.repoRoot, input.campaignId);
|
|
352
|
-
const plan = await readCampaignPlan(input.repoRoot, input.campaignId);
|
|
353
|
-
if (!plan)
|
|
354
|
-
throw new Error(`campaign ${input.campaignId} has no plan`);
|
|
355
|
-
const policy = evaluatePromotionPolicy({
|
|
356
|
-
campaignId: input.campaignId,
|
|
357
|
-
incumbentCandidateId: manifest.incumbentCandidateId,
|
|
358
|
-
evidence: input.evidence,
|
|
359
|
-
});
|
|
360
|
-
await writeCampaignScorecard({
|
|
361
|
-
repoRoot: input.repoRoot,
|
|
362
|
-
scorecard: {
|
|
363
|
-
schemaVersion: 1,
|
|
364
|
-
campaignId: input.campaignId,
|
|
365
|
-
planHash: plan.planHash,
|
|
366
|
-
createdAt: new Date().toISOString(),
|
|
367
|
-
evidenceCellCount: input.evidence.cells.length,
|
|
368
|
-
policy,
|
|
369
|
-
},
|
|
370
|
-
evidence: input.evidence,
|
|
371
|
-
});
|
|
372
|
-
}
|