@tea-agent/loop-agent 0.16.1-beta.1 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -8
- package/CHANGELOG.md +59 -16
- package/README.md +76 -299
- package/dist/application/evaluation/alias.js +184 -0
- package/dist/application/evaluation/budget.js +192 -0
- package/dist/application/evaluation/campaign-hash.js +47 -0
- package/dist/application/evaluation/campaign-matrix.js +372 -0
- package/dist/application/evaluation/campaign-scorecard.js +135 -0
- package/dist/application/evaluation/campaign.js +370 -0
- package/dist/application/evaluation/candidate.js +23 -6
- package/dist/application/evaluation/corpus-hash.js +38 -0
- package/dist/application/evaluation/corpus.js +56 -0
- package/dist/application/evaluation/experiment.js +294 -0
- package/dist/application/evaluation/ignition.js +198 -0
- package/dist/application/evaluation/integrity-audit.js +162 -0
- package/dist/application/evaluation/outer-loop.js +132 -0
- package/dist/application/evaluation/pi-cell-executor.js +39 -0
- package/dist/application/evaluation/private-verifier.js +46 -0
- package/dist/application/evaluation/promotion-policy.js +151 -0
- package/dist/application/evaluation/proposer.js +98 -0
- package/dist/application/evaluation/types.js +522 -0
- package/dist/cli/command-definitions.js +19 -3
- package/dist/commands/dag-reconcile-run.js +3 -116
- package/dist/commands/eval.js +1176 -13
- package/dist/commands/init.js +7 -1
- package/dist/executors/dag-pi-executor.js +4 -44
- package/dist/executors/shell-executor.js +1 -1
- package/dist/infrastructure/evaluation/alias-store.js +199 -0
- package/dist/infrastructure/evaluation/campaign-store.js +154 -0
- package/dist/infrastructure/evaluation/corpus-store.js +181 -0
- package/dist/infrastructure/evaluation/experiment-store.js +124 -0
- package/dist/infrastructure/evaluation/ignition-store.js +82 -0
- package/dist/infrastructure/evaluation/private-verifier-store.js +145 -0
- package/dist/infrastructure/evaluation/proposer-store.js +78 -0
- package/dist/records/promotion.js +3 -1
- package/dist/worker/cli.js +83 -0
- package/dist/worker/delivery/git-transaction.js +75 -0
- package/dist/worker/delivery/verification-bundle.js +13 -2
- package/dist/worker/feature/review.js +3 -2
- package/dist/worker/observe/static/dag-helpers.js +0 -62
- package/dist/worker/observe/static/styles.css +18 -55
- package/dist/worker/observe/static/views/dag.js +13 -5
- package/dist/worker/outcomes/adapters.js +4 -1
- package/dist/worker/outcomes/declared-artifacts.js +103 -0
- package/dist/worker/outcomes/evidence-tokens.js +29 -0
- package/dist/worker/outcomes/gate.js +10 -11
- package/dist/worker/outcomes/projector.js +30 -4
- package/dist/worker/outcomes/types.js +3 -0
- package/dist/worker/pool/reconcile.js +285 -0
- package/dist/worker/run-task/run-task.js +81 -4
- package/dist/worker/runner/run-ready.js +25 -2
- package/dist/worker/task-graph/ready-planner.js +14 -8
- package/dist/worker/task-graph/task-graph-schema.js +5 -3
- package/dist/workflows/dag/budget-enforcement.js +67 -0
- package/dist/workflows/dag/context-policy.js +137 -0
- package/dist/workflows/dag/failure-routing.js +7 -0
- package/dist/workflows/dag/frontend-implementation-contract.js +0 -77
- package/dist/workflows/dag/init-hybrid.js +33 -53
- package/dist/workflows/dag/knowledge-curator.js +3 -0
- package/dist/workflows/dag/node-execution.js +11 -4
- package/dist/workflows/dag/prompt.js +1 -1
- package/dist/workflows/dag/reconcile-run.js +121 -0
- package/dist/workflows/dag/report.js +12 -0
- package/dist/workflows/dag/runner.js +43 -16
- package/dist/workflows/dag/skill-snapshot.js +11 -7
- package/dist/workflows/dag/types.js +18 -1
- package/dist/workflows/dag/validate.js +15 -1
- package/docs/README.md +3 -1
- package/docs/architecture/runtime-boundaries.md +3 -2
- package/docs/init-surface.manifest.json +4 -0
- package/docs/local-development-environment.md +52 -0
- package/docs/templates/agent-dag.schema.json +0 -5
- package/docs/templates/agent-dag.supervised-implementation.json +23 -4
- package/docs/templates/branch-merge-report.md +14 -0
- package/docs/templates/evaluation/campaign-budget-v1.json +12 -0
- package/docs/templates/evaluation/campaign-dogfood-v0.json +24 -0
- package/docs/templates/evaluation/campaign-evidence-v1.json +44 -0
- package/docs/templates/evaluation/context-policy-baseline-v1.json +17 -0
- package/docs/templates/evaluation/context-policy-role-specialized-v1.json +28 -0
- package/docs/templates/evaluation/corpus-dogfood-v0.manifest.json +118 -0
- package/docs/templates/evaluation/matrix-dag-dry-run-v1.json +21 -0
- package/docs/templates/evaluation/matrix-fixture-v1.json +10 -0
- package/docs/templates/evaluation/private-verifier-dogfood-v0.json +16 -0
- package/docs/templates/product-line/AGENTS.md +1 -0
- package/docs/templates/product-line/README.md +17 -0
- package/docs/templates/product-line/acceptance.yaml +9 -0
- package/docs/templates/product-line/feature.yaml +11 -0
- package/docs/templates/product-line/task-graph.yaml +8 -0
- package/docs/templates/product-line/task.yaml +4 -0
- package/package.json +2 -1
- package/skills/frontend-implementation/references/node-contracts.md +3 -3
- package/skills/loop-agent/references/command-reference.md +5 -0
- package/skills/loop-agent/references/hybrid-dag.md +3 -3
package/dist/commands/init.js
CHANGED
|
@@ -859,6 +859,12 @@ export function buildManagedGitignoreBlock() {
|
|
|
859
859
|
".harness/*.log",
|
|
860
860
|
".harness/init-surface.json",
|
|
861
861
|
"",
|
|
862
|
+
"# Eval Lab runtime (candidates, campaigns, scorecards, dogfood scratch)",
|
|
863
|
+
".harness/evaluation/",
|
|
864
|
+
"",
|
|
865
|
+
"# fullstack / worker dogfood evidence (local campaign scratch)",
|
|
866
|
+
".harness/dogfood-evidence/",
|
|
867
|
+
"",
|
|
862
868
|
"# worker task pool",
|
|
863
869
|
".harness/task-pool/*",
|
|
864
870
|
".task-pool/",
|
|
@@ -2089,7 +2095,7 @@ export function buildInitInstructions(input) {
|
|
|
2089
2095
|
"- Enrich the root `README.md`: keep the deterministic project title and the loop-agent managed block intact, and fill the human-authored sections (项目概览, 技术栈与目录结构, 开发与验证) from the target project's actual files. The root README must serve both as a human-first project entry and as an agent work entry; replace the initialization-model supplement comments when the project files provide the information.",
|
|
2090
2096
|
`- Populate \`${governanceRoot}/verification-matrix.md\` with the target project's actual quick, standard, and full verification commands derived from its real language and toolchain, keeping the governance rows intact.`,
|
|
2091
2097
|
"- Project repo-local skills live in `.agents/skills/`. Do not create a root `skills/` directory in the target project; the package's bundled `skills/` remains the built-in fallback.",
|
|
2092
|
-
"- Merge a loop-agent managed block into `.gitignore` that ignores harness runtime facts (tasks, dag-runs, runs, live, cache, init-surface.json, .harness/task-pool, legacy .task-pool residue) while keeping prompts and directory placeholders shareable.",
|
|
2098
|
+
"- Merge a loop-agent managed block into `.gitignore` that ignores harness runtime facts (tasks, dag-runs, runs, evaluation, dogfood-evidence, live, cache, init-surface.json, .harness/task-pool, legacy .task-pool residue) while keeping prompts and directory placeholders shareable.",
|
|
2093
2099
|
"- Do not copy examples by default; examples stay bundled in the tool and are available through `loop-agent examples`.",
|
|
2094
2100
|
"- Add or update a loop-agent managed block in AGENTS.md.",
|
|
2095
2101
|
"- The generated AGENTS.md must include documentation convergence and structured DAG write-boundary rules so target projects keep the same working discipline as this repository.",
|
|
@@ -272,7 +272,7 @@ export async function executeDagPiNode(input, meta, piStepFn = executePiStep) {
|
|
|
272
272
|
persona,
|
|
273
273
|
step,
|
|
274
274
|
});
|
|
275
|
-
const mapped = mapPiResultToDagNodeResult(result
|
|
275
|
+
const mapped = mapPiResultToDagNodeResult(result);
|
|
276
276
|
if (!isWriteTask) {
|
|
277
277
|
return mapped;
|
|
278
278
|
}
|
|
@@ -313,57 +313,17 @@ export async function executeDagPiNode(input, meta, piStepFn = executePiStep) {
|
|
|
313
313
|
durationMs: mapped.durationMs || Date.now() - started,
|
|
314
314
|
};
|
|
315
315
|
}
|
|
316
|
-
export function mapPiResultToDagNodeResult(result
|
|
317
|
-
const assistantText = canonicalizeProtocolFirstLine(result.assistantText, firstProtocolLine);
|
|
316
|
+
export function mapPiResultToDagNodeResult(result) {
|
|
318
317
|
return {
|
|
319
318
|
ok: result.ok,
|
|
320
|
-
stdout: assistantText || result.stdout,
|
|
319
|
+
stdout: result.assistantText || result.stdout,
|
|
321
320
|
stderr: result.stderr,
|
|
322
321
|
failureCategory: result.failureCategory,
|
|
323
322
|
durationMs: result.durationMs,
|
|
324
|
-
assistantText,
|
|
323
|
+
assistantText: result.assistantText,
|
|
325
324
|
backend: result.backend,
|
|
326
325
|
sdkAttempted: result.sdkAttempted,
|
|
327
326
|
tokensUsed: result.tokensUsed,
|
|
328
327
|
parsedEvents: result.parsedEvents,
|
|
329
328
|
};
|
|
330
329
|
}
|
|
331
|
-
function canonicalizeProtocolFirstLine(assistantText, firstProtocolLine) {
|
|
332
|
-
if (!assistantText || !firstProtocolLine)
|
|
333
|
-
return assistantText;
|
|
334
|
-
const lines = assistantText.split(/\r?\n/);
|
|
335
|
-
let protocolIndex = -1;
|
|
336
|
-
let protocolLine = "";
|
|
337
|
-
for (const [index, line] of lines.entries()) {
|
|
338
|
-
const normalized = normalizeProtocolLine(line);
|
|
339
|
-
if (normalized.startsWith(firstProtocolLine)) {
|
|
340
|
-
protocolIndex = index;
|
|
341
|
-
protocolLine = normalized;
|
|
342
|
-
break;
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
if (protocolIndex < 0)
|
|
346
|
-
return assistantText;
|
|
347
|
-
if (protocolIndex === 0) {
|
|
348
|
-
return [protocolLine, ...lines.slice(1)].join("\n");
|
|
349
|
-
}
|
|
350
|
-
const before = lines.slice(0, protocolIndex);
|
|
351
|
-
const after = lines.slice(protocolIndex + 1);
|
|
352
|
-
while (before.at(-1)?.trim() === "" &&
|
|
353
|
-
after.at(0)?.trim() === "") {
|
|
354
|
-
after.shift();
|
|
355
|
-
}
|
|
356
|
-
const bodyLines = [...before, ...after];
|
|
357
|
-
while (bodyLines.at(0)?.trim() === "")
|
|
358
|
-
bodyLines.shift();
|
|
359
|
-
while (bodyLines.at(-1)?.trim() === "")
|
|
360
|
-
bodyLines.pop();
|
|
361
|
-
return bodyLines.length > 0
|
|
362
|
-
? `${protocolLine}\n\n${bodyLines.join("\n")}`
|
|
363
|
-
: protocolLine;
|
|
364
|
-
}
|
|
365
|
-
function normalizeProtocolLine(line) {
|
|
366
|
-
const trimmed = line.trim();
|
|
367
|
-
const emphasized = trimmed.match(/^(\*{1,3})\s*(.*?)\s*\1$/);
|
|
368
|
-
return (emphasized?.[2] ?? trimmed).trim();
|
|
369
|
-
}
|
|
@@ -98,7 +98,7 @@ export async function executeShellCommand(input) {
|
|
|
98
98
|
HARNESS_DAG_RUN_ID: input.dagRunMeta.runId,
|
|
99
99
|
}
|
|
100
100
|
: undefined;
|
|
101
|
-
const child = spawn(resolveBashExecutable(), ["-
|
|
101
|
+
const child = spawn(resolveBashExecutable(), ["-c", input.command], {
|
|
102
102
|
cwd: input.cwd,
|
|
103
103
|
env: buildShellProcessEnv(input.envAllowlist, injectedEnv),
|
|
104
104
|
stdio: ["ignore", "pipe", "pipe"],
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { access, mkdir, readdir, readFile, rm, writeFile, } from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { aliasHistoryEventSchema, aliasNameSchema, incumbentAliasSchema, promotionDecisionSchema, } from "../../application/evaluation/types.js";
|
|
5
|
+
import { appendJsonlLineAtomic, writeJsonAtomic, writeTextAtomic, } from "../harness/atomic-write.js";
|
|
6
|
+
import { EVALUATION_ROOT } from "./store.js";
|
|
7
|
+
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
8
|
+
const ALIAS_LOCK_DIR = ".lock";
|
|
9
|
+
const ALIAS_LOCK_RETRIES = 40;
|
|
10
|
+
const ALIAS_LOCK_DELAY_MS = 25;
|
|
11
|
+
const CURRENT_INTEGRITY_FILE = "current.sha256";
|
|
12
|
+
export function assertAliasName(value) {
|
|
13
|
+
aliasNameSchema.parse(value);
|
|
14
|
+
if (!SAFE_ID.test(value)) {
|
|
15
|
+
throw new Error(`alias must contain only letters, numbers, dot, underscore, or hyphen: ${value}`);
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export function aliasDir(repoRoot, alias) {
|
|
19
|
+
assertAliasName(alias);
|
|
20
|
+
return path.join(repoRoot, EVALUATION_ROOT, "aliases", alias);
|
|
21
|
+
}
|
|
22
|
+
export function aliasCurrentPath(repoRoot, alias) {
|
|
23
|
+
return path.join(aliasDir(repoRoot, alias), "current.json");
|
|
24
|
+
}
|
|
25
|
+
export function aliasHistoryPath(repoRoot, alias) {
|
|
26
|
+
return path.join(aliasDir(repoRoot, alias), "history.jsonl");
|
|
27
|
+
}
|
|
28
|
+
export function decisionDir(repoRoot) {
|
|
29
|
+
return path.join(repoRoot, EVALUATION_ROOT, "decisions");
|
|
30
|
+
}
|
|
31
|
+
export function decisionPath(repoRoot, decisionId) {
|
|
32
|
+
if (!SAFE_ID.test(decisionId)) {
|
|
33
|
+
throw new Error(`decision-id must contain only letters, numbers, dot, underscore, or hyphen: ${decisionId}`);
|
|
34
|
+
}
|
|
35
|
+
return path.join(decisionDir(repoRoot), `${decisionId}.json`);
|
|
36
|
+
}
|
|
37
|
+
function sha256Hex(content) {
|
|
38
|
+
return createHash("sha256").update(content, "utf8").digest("hex");
|
|
39
|
+
}
|
|
40
|
+
function serializeAlias(alias) {
|
|
41
|
+
return `${JSON.stringify(alias, null, 2)}\n`;
|
|
42
|
+
}
|
|
43
|
+
async function pathExists(filePath) {
|
|
44
|
+
try {
|
|
45
|
+
await access(filePath);
|
|
46
|
+
return true;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
async function acquireAliasLock(repoRoot, alias) {
|
|
53
|
+
const lockPath = path.join(aliasDir(repoRoot, alias), ALIAS_LOCK_DIR);
|
|
54
|
+
await mkdir(aliasDir(repoRoot, alias), { recursive: true });
|
|
55
|
+
for (let attempt = 0; attempt < ALIAS_LOCK_RETRIES; attempt += 1) {
|
|
56
|
+
try {
|
|
57
|
+
await mkdir(lockPath);
|
|
58
|
+
return async () => {
|
|
59
|
+
await rm(lockPath, { recursive: true, force: true });
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
if (error.code !== "EEXIST")
|
|
64
|
+
throw error;
|
|
65
|
+
await new Promise((resolve) => setTimeout(resolve, ALIAS_LOCK_DELAY_MS));
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
throw new Error(`timed out acquiring alias lock for ${alias}`);
|
|
69
|
+
}
|
|
70
|
+
async function assertCurrentIntegrity(repoRoot, alias, raw) {
|
|
71
|
+
const integrityPath = path.join(aliasDir(repoRoot, alias), CURRENT_INTEGRITY_FILE);
|
|
72
|
+
if (!(await pathExists(integrityPath))) {
|
|
73
|
+
throw new Error(`missing alias integrity file for ${alias}`);
|
|
74
|
+
}
|
|
75
|
+
const expected = (await readFile(integrityPath, "utf-8")).trim();
|
|
76
|
+
const actual = sha256Hex(raw);
|
|
77
|
+
if (expected !== actual) {
|
|
78
|
+
throw new Error(`alias integrity mismatch for ${alias}: expected ${expected}, got ${actual}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
export async function readIncumbentAlias(repoRoot, alias) {
|
|
82
|
+
assertAliasName(alias);
|
|
83
|
+
const currentPath = aliasCurrentPath(repoRoot, alias);
|
|
84
|
+
if (!(await pathExists(currentPath)))
|
|
85
|
+
return null;
|
|
86
|
+
const raw = await readFile(currentPath, "utf-8");
|
|
87
|
+
await assertCurrentIntegrity(repoRoot, alias, raw);
|
|
88
|
+
const parsed = incumbentAliasSchema.parse(JSON.parse(raw));
|
|
89
|
+
if (parsed.alias !== alias) {
|
|
90
|
+
throw new Error(`alias identity mismatch: directory=${alias}, current=${parsed.alias}`);
|
|
91
|
+
}
|
|
92
|
+
return parsed;
|
|
93
|
+
}
|
|
94
|
+
export async function listAliasNames(repoRoot) {
|
|
95
|
+
const root = path.join(repoRoot, EVALUATION_ROOT, "aliases");
|
|
96
|
+
try {
|
|
97
|
+
const entries = await readdir(root, { withFileTypes: true });
|
|
98
|
+
return entries
|
|
99
|
+
.filter((entry) => entry.isDirectory() && SAFE_ID.test(entry.name))
|
|
100
|
+
.map((entry) => entry.name)
|
|
101
|
+
.sort();
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
const code = error.code;
|
|
105
|
+
if (code === "ENOENT")
|
|
106
|
+
return [];
|
|
107
|
+
throw error;
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
export async function listIncumbentCandidateIds(repoRoot) {
|
|
111
|
+
const names = await listAliasNames(repoRoot);
|
|
112
|
+
const ids = new Set();
|
|
113
|
+
for (const name of names) {
|
|
114
|
+
const current = await readIncumbentAlias(repoRoot, name);
|
|
115
|
+
if (current)
|
|
116
|
+
ids.add(current.candidateId);
|
|
117
|
+
}
|
|
118
|
+
return ids;
|
|
119
|
+
}
|
|
120
|
+
export async function readPromotionDecision(repoRoot, decisionId) {
|
|
121
|
+
const raw = await readFile(decisionPath(repoRoot, decisionId), "utf-8");
|
|
122
|
+
return promotionDecisionSchema.parse(JSON.parse(raw));
|
|
123
|
+
}
|
|
124
|
+
async function writeAliasCurrentAtomic(input) {
|
|
125
|
+
const dir = aliasDir(input.repoRoot, input.alias.alias);
|
|
126
|
+
await mkdir(dir, { recursive: true });
|
|
127
|
+
const serialized = serializeAlias(input.alias);
|
|
128
|
+
await writeTextAtomic(aliasCurrentPath(input.repoRoot, input.alias.alias), serialized, {
|
|
129
|
+
repoRoot: input.repoRoot,
|
|
130
|
+
});
|
|
131
|
+
await writeTextAtomic(path.join(dir, CURRENT_INTEGRITY_FILE), `${sha256Hex(serialized)}\n`, { repoRoot: input.repoRoot });
|
|
132
|
+
}
|
|
133
|
+
export async function commitAliasMove(input) {
|
|
134
|
+
const { repoRoot, decision, nextAlias, historyEvent } = input;
|
|
135
|
+
assertAliasName(decision.alias);
|
|
136
|
+
promotionDecisionSchema.parse(decision);
|
|
137
|
+
incumbentAliasSchema.parse(nextAlias);
|
|
138
|
+
aliasHistoryEventSchema.parse(historyEvent);
|
|
139
|
+
if (decision.decisionId !== nextAlias.updatedByDecisionId) {
|
|
140
|
+
throw new Error("decisionId must match alias.updatedByDecisionId");
|
|
141
|
+
}
|
|
142
|
+
if (decision.decisionId !== historyEvent.decisionId) {
|
|
143
|
+
throw new Error("decisionId must match history event");
|
|
144
|
+
}
|
|
145
|
+
const releaseLock = await acquireAliasLock(repoRoot, decision.alias);
|
|
146
|
+
const previousRaw = await readFile(aliasCurrentPath(repoRoot, decision.alias), "utf-8").catch(() => null);
|
|
147
|
+
const previousIntegrity = await readFile(path.join(aliasDir(repoRoot, decision.alias), CURRENT_INTEGRITY_FILE), "utf-8").catch(() => null);
|
|
148
|
+
const decisionFile = decisionPath(repoRoot, decision.decisionId);
|
|
149
|
+
let decisionWritten = false;
|
|
150
|
+
let historyWritten = false;
|
|
151
|
+
try {
|
|
152
|
+
if (await pathExists(decisionFile)) {
|
|
153
|
+
throw new Error(`decision already exists: ${decision.decisionId}`);
|
|
154
|
+
}
|
|
155
|
+
await mkdir(decisionDir(repoRoot), { recursive: true });
|
|
156
|
+
await writeJsonAtomic(decisionFile, decision, { repoRoot });
|
|
157
|
+
decisionWritten = true;
|
|
158
|
+
await writeAliasCurrentAtomic({ repoRoot, alias: nextAlias });
|
|
159
|
+
await appendJsonlLineAtomic(aliasHistoryPath(repoRoot, decision.alias), historyEvent, { repoRoot });
|
|
160
|
+
historyWritten = true;
|
|
161
|
+
}
|
|
162
|
+
catch (error) {
|
|
163
|
+
// Best-effort rollback of alias pointer + decision; history append is last.
|
|
164
|
+
if (previousRaw !== null) {
|
|
165
|
+
await writeTextAtomic(aliasCurrentPath(repoRoot, decision.alias), previousRaw, { repoRoot }).catch(() => undefined);
|
|
166
|
+
if (previousIntegrity !== null) {
|
|
167
|
+
await writeTextAtomic(path.join(aliasDir(repoRoot, decision.alias), CURRENT_INTEGRITY_FILE), previousIntegrity, { repoRoot }).catch(() => undefined);
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
else if (!historyWritten) {
|
|
171
|
+
await rm(aliasCurrentPath(repoRoot, decision.alias), {
|
|
172
|
+
force: true,
|
|
173
|
+
}).catch(() => undefined);
|
|
174
|
+
await rm(path.join(aliasDir(repoRoot, decision.alias), CURRENT_INTEGRITY_FILE), { force: true }).catch(() => undefined);
|
|
175
|
+
}
|
|
176
|
+
if (decisionWritten) {
|
|
177
|
+
await rm(decisionFile, { force: true }).catch(() => undefined);
|
|
178
|
+
}
|
|
179
|
+
throw error;
|
|
180
|
+
}
|
|
181
|
+
finally {
|
|
182
|
+
await releaseLock();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
export function makeDecisionId(input) {
|
|
186
|
+
const stamp = input.createdAt.replace(/[^A-Za-z0-9._-]/g, "-");
|
|
187
|
+
const raw = `dec-${input.alias}-${input.action}-${input.toCandidateId}-${stamp}`;
|
|
188
|
+
return raw.slice(0, 180);
|
|
189
|
+
}
|
|
190
|
+
/** Test helper: force-corrupt integrity without going through commit. */
|
|
191
|
+
export async function writeAliasCurrentForTest(input) {
|
|
192
|
+
await writeAliasCurrentAtomic({
|
|
193
|
+
repoRoot: input.repoRoot,
|
|
194
|
+
alias: input.alias,
|
|
195
|
+
});
|
|
196
|
+
if (input.corruptIntegrity) {
|
|
197
|
+
await writeFile(path.join(aliasDir(input.repoRoot, input.alias.alias), CURRENT_INTEGRITY_FILE), `${"0".repeat(64)}\n`, "utf-8");
|
|
198
|
+
}
|
|
199
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import { access, mkdir, mkdtemp, readdir, readFile, rename, rm } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { campaignEvidenceSchema, campaignIdSchema, campaignManifestSchema, campaignPlanSchema, } from "../../application/evaluation/types.js";
|
|
4
|
+
import { sha256Hex } from "../../application/evaluation/candidate-hash.js";
|
|
5
|
+
import { writeJsonAtomic, writeTextAtomic } from "../harness/atomic-write.js";
|
|
6
|
+
import { EVALUATION_ROOT } from "./store.js";
|
|
7
|
+
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
8
|
+
const MANIFEST_INTEGRITY_FILE = "manifest.sha256";
|
|
9
|
+
export function assertCampaignId(value) {
|
|
10
|
+
campaignIdSchema.parse(value);
|
|
11
|
+
}
|
|
12
|
+
export function campaignDir(repoRoot, campaignId) {
|
|
13
|
+
assertCampaignId(campaignId);
|
|
14
|
+
return path.join(repoRoot, EVALUATION_ROOT, "campaigns", campaignId);
|
|
15
|
+
}
|
|
16
|
+
export function campaignManifestPath(repoRoot, campaignId) {
|
|
17
|
+
return path.join(campaignDir(repoRoot, campaignId), "manifest.json");
|
|
18
|
+
}
|
|
19
|
+
export function campaignPlanPath(repoRoot, campaignId) {
|
|
20
|
+
return path.join(campaignDir(repoRoot, campaignId), "plan.json");
|
|
21
|
+
}
|
|
22
|
+
export function campaignScorecardPath(repoRoot, campaignId) {
|
|
23
|
+
return path.join(campaignDir(repoRoot, campaignId), "scorecard.json");
|
|
24
|
+
}
|
|
25
|
+
export function campaignEvidencePath(repoRoot, campaignId) {
|
|
26
|
+
return path.join(campaignDir(repoRoot, campaignId), "evidence.json");
|
|
27
|
+
}
|
|
28
|
+
async function pathExists(filePath) {
|
|
29
|
+
try {
|
|
30
|
+
await access(filePath);
|
|
31
|
+
return true;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
return false;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
function serializeManifest(manifest) {
|
|
38
|
+
return `${JSON.stringify(manifest, null, 2)}\n`;
|
|
39
|
+
}
|
|
40
|
+
function manifestsEqual(a, b) {
|
|
41
|
+
return JSON.stringify(a) === JSON.stringify(b);
|
|
42
|
+
}
|
|
43
|
+
export async function readCampaignManifest(repoRoot, campaignId) {
|
|
44
|
+
assertCampaignId(campaignId);
|
|
45
|
+
const manifestPath = campaignManifestPath(repoRoot, campaignId);
|
|
46
|
+
const rawText = await readFile(manifestPath, "utf-8");
|
|
47
|
+
const expected = (await readFile(path.join(campaignDir(repoRoot, campaignId), MANIFEST_INTEGRITY_FILE), "utf-8")).trim();
|
|
48
|
+
const actual = sha256Hex(rawText);
|
|
49
|
+
if (expected !== actual) {
|
|
50
|
+
throw new Error(`campaign manifest integrity mismatch for ${campaignId}`);
|
|
51
|
+
}
|
|
52
|
+
const stored = campaignManifestSchema.parse(JSON.parse(rawText));
|
|
53
|
+
if (stored.campaignId !== campaignId) {
|
|
54
|
+
throw new Error(`campaign identity mismatch: directory=${campaignId}, manifest=${stored.campaignId}`);
|
|
55
|
+
}
|
|
56
|
+
return stored;
|
|
57
|
+
}
|
|
58
|
+
export async function readCampaignPlan(repoRoot, campaignId) {
|
|
59
|
+
const planPath = campaignPlanPath(repoRoot, campaignId);
|
|
60
|
+
if (!(await pathExists(planPath)))
|
|
61
|
+
return undefined;
|
|
62
|
+
return campaignPlanSchema.parse(JSON.parse(await readFile(planPath, "utf-8")));
|
|
63
|
+
}
|
|
64
|
+
export async function listCampaignIds(repoRoot) {
|
|
65
|
+
const root = path.join(repoRoot, EVALUATION_ROOT, "campaigns");
|
|
66
|
+
try {
|
|
67
|
+
const entries = await readdir(root, { withFileTypes: true });
|
|
68
|
+
return entries
|
|
69
|
+
.filter((entry) => entry.isDirectory() && SAFE_ID.test(entry.name))
|
|
70
|
+
.map((entry) => entry.name)
|
|
71
|
+
.sort();
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
const code = error.code;
|
|
75
|
+
if (code === "ENOENT")
|
|
76
|
+
return [];
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
export async function registerCampaignManifest(input) {
|
|
81
|
+
const { repoRoot, manifest } = input;
|
|
82
|
+
assertCampaignId(manifest.campaignId);
|
|
83
|
+
const manifestPath = campaignManifestPath(repoRoot, manifest.campaignId);
|
|
84
|
+
if (await pathExists(manifestPath)) {
|
|
85
|
+
const existing = await readCampaignManifest(repoRoot, manifest.campaignId);
|
|
86
|
+
if (!manifestsEqual(existing, manifest)) {
|
|
87
|
+
throw new Error(`campaign already exists with different content: ${manifest.campaignId}`);
|
|
88
|
+
}
|
|
89
|
+
return { manifest: existing, idempotent: true, manifestPath };
|
|
90
|
+
}
|
|
91
|
+
const root = path.dirname(campaignDir(repoRoot, manifest.campaignId));
|
|
92
|
+
await mkdir(root, { recursive: true });
|
|
93
|
+
const stagingDir = await mkdtemp(path.join(root, `.${manifest.campaignId}.register-`));
|
|
94
|
+
try {
|
|
95
|
+
await writeJsonAtomic(path.join(stagingDir, "manifest.json"), manifest, {
|
|
96
|
+
repoRoot,
|
|
97
|
+
});
|
|
98
|
+
await writeTextAtomic(path.join(stagingDir, MANIFEST_INTEGRITY_FILE), `${sha256Hex(serializeManifest(manifest))}\n`, { repoRoot });
|
|
99
|
+
try {
|
|
100
|
+
await rename(stagingDir, campaignDir(repoRoot, manifest.campaignId));
|
|
101
|
+
}
|
|
102
|
+
catch (error) {
|
|
103
|
+
const code = error.code;
|
|
104
|
+
const isCreateRace = code === "EEXIST" ||
|
|
105
|
+
code === "ENOTEMPTY" ||
|
|
106
|
+
(code === "EPERM" &&
|
|
107
|
+
(await pathExists(campaignDir(repoRoot, manifest.campaignId))));
|
|
108
|
+
if (!isCreateRace)
|
|
109
|
+
throw error;
|
|
110
|
+
await rm(stagingDir, { recursive: true, force: true });
|
|
111
|
+
const existing = await readCampaignManifest(repoRoot, manifest.campaignId);
|
|
112
|
+
if (!manifestsEqual(existing, manifest)) {
|
|
113
|
+
throw new Error(`campaign already exists with different content: ${manifest.campaignId}`);
|
|
114
|
+
}
|
|
115
|
+
return { manifest: existing, idempotent: true, manifestPath };
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
catch (error) {
|
|
119
|
+
await rm(stagingDir, { recursive: true, force: true });
|
|
120
|
+
throw error;
|
|
121
|
+
}
|
|
122
|
+
return { manifest, idempotent: false, manifestPath };
|
|
123
|
+
}
|
|
124
|
+
export async function writeCampaignPlan(input) {
|
|
125
|
+
const planPath = campaignPlanPath(input.repoRoot, input.plan.campaignId);
|
|
126
|
+
await writeJsonAtomic(planPath, input.plan, { repoRoot: input.repoRoot });
|
|
127
|
+
return planPath;
|
|
128
|
+
}
|
|
129
|
+
export async function writeCampaignScorecard(input) {
|
|
130
|
+
const scorecardPath = campaignScorecardPath(input.repoRoot, input.scorecard.campaignId);
|
|
131
|
+
await writeJsonAtomic(scorecardPath, input.scorecard, {
|
|
132
|
+
repoRoot: input.repoRoot,
|
|
133
|
+
});
|
|
134
|
+
if (input.evidence) {
|
|
135
|
+
await writeJsonAtomic(campaignEvidencePath(input.repoRoot, input.scorecard.campaignId), input.evidence, { repoRoot: input.repoRoot });
|
|
136
|
+
}
|
|
137
|
+
return scorecardPath;
|
|
138
|
+
}
|
|
139
|
+
export async function readCampaignScorecard(repoRoot, campaignId) {
|
|
140
|
+
const scorecardPath = campaignScorecardPath(repoRoot, campaignId);
|
|
141
|
+
if (!(await pathExists(scorecardPath)))
|
|
142
|
+
return undefined;
|
|
143
|
+
const raw = JSON.parse(await readFile(scorecardPath, "utf-8"));
|
|
144
|
+
if (raw.schemaVersion !== 1 || raw.campaignId !== campaignId) {
|
|
145
|
+
throw new Error(`invalid campaign scorecard for ${campaignId}`);
|
|
146
|
+
}
|
|
147
|
+
return raw;
|
|
148
|
+
}
|
|
149
|
+
export async function readCampaignEvidence(repoRoot, campaignId) {
|
|
150
|
+
const evidencePath = campaignEvidencePath(repoRoot, campaignId);
|
|
151
|
+
if (!(await pathExists(evidencePath)))
|
|
152
|
+
return undefined;
|
|
153
|
+
return campaignEvidenceSchema.parse(JSON.parse(await readFile(evidencePath, "utf-8")));
|
|
154
|
+
}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { access, mkdir, readdir, readFile, rename, rm } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { mkdtemp } from "node:fs/promises";
|
|
4
|
+
import { computeCorpusHash, } from "../../application/evaluation/corpus-hash.js";
|
|
5
|
+
import { formatContentSha, normalizeContentSha, sha256Hex, } from "../../application/evaluation/candidate-hash.js";
|
|
6
|
+
import { corpusIdSchema, corpusManifestInputSchema, corpusManifestSchema, } from "../../application/evaluation/types.js";
|
|
7
|
+
import { writeJsonAtomic, writeTextAtomic } from "../harness/atomic-write.js";
|
|
8
|
+
import { EVALUATION_ROOT } from "./store.js";
|
|
9
|
+
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
10
|
+
const MANIFEST_INTEGRITY_FILE = "manifest.sha256";
|
|
11
|
+
export function assertCorpusId(value) {
|
|
12
|
+
corpusIdSchema.parse(value);
|
|
13
|
+
}
|
|
14
|
+
export function corpusDir(repoRoot, corpusId) {
|
|
15
|
+
assertCorpusId(corpusId);
|
|
16
|
+
return path.join(repoRoot, EVALUATION_ROOT, "corpora", corpusId);
|
|
17
|
+
}
|
|
18
|
+
export function corpusManifestPath(repoRoot, corpusId) {
|
|
19
|
+
return path.join(corpusDir(repoRoot, corpusId), "manifest.json");
|
|
20
|
+
}
|
|
21
|
+
async function pathExists(filePath) {
|
|
22
|
+
try {
|
|
23
|
+
await access(filePath);
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
function serializeManifest(manifest) {
|
|
31
|
+
return `${JSON.stringify(manifest, null, 2)}\n`;
|
|
32
|
+
}
|
|
33
|
+
function normalizeTasksForStorage(tasks) {
|
|
34
|
+
return [...tasks]
|
|
35
|
+
.map((task) => ({
|
|
36
|
+
...task,
|
|
37
|
+
taskRef: task.taskRef.replace(/\\/g, "/"),
|
|
38
|
+
seeds: [...task.seeds].sort((a, b) => a - b),
|
|
39
|
+
}))
|
|
40
|
+
.sort((a, b) => a.taskRef < b.taskRef ? -1 : a.taskRef > b.taskRef ? 1 : 0);
|
|
41
|
+
}
|
|
42
|
+
export async function materializeCorpusManifest(raw) {
|
|
43
|
+
const input = corpusManifestInputSchema.parse(raw);
|
|
44
|
+
const tasks = normalizeTasksForStorage(input.tasks);
|
|
45
|
+
const withoutHash = {
|
|
46
|
+
schemaVersion: 1,
|
|
47
|
+
corpusId: input.corpusId,
|
|
48
|
+
createdAt: input.createdAt,
|
|
49
|
+
...(input.description !== undefined
|
|
50
|
+
? { description: input.description }
|
|
51
|
+
: {}),
|
|
52
|
+
tasks,
|
|
53
|
+
};
|
|
54
|
+
const corpusHash = computeCorpusHash(withoutHash);
|
|
55
|
+
if (input.corpusHash) {
|
|
56
|
+
const provided = formatContentSha(normalizeContentSha(input.corpusHash));
|
|
57
|
+
if (provided !== corpusHash) {
|
|
58
|
+
throw new Error(`corpusHash mismatch: expected ${corpusHash}, got ${provided}`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return corpusManifestSchema.parse({
|
|
62
|
+
...withoutHash,
|
|
63
|
+
corpusHash,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async function assertManifestIntegrity(repoRoot, corpusId, rawManifest) {
|
|
67
|
+
const integrityPath = path.join(corpusDir(repoRoot, corpusId), MANIFEST_INTEGRITY_FILE);
|
|
68
|
+
const expected = (await readFile(integrityPath, "utf-8")).trim();
|
|
69
|
+
const actual = sha256Hex(rawManifest);
|
|
70
|
+
if (expected !== actual) {
|
|
71
|
+
throw new Error(`corpus manifest integrity mismatch for ${corpusId}: expected ${expected}, got ${actual}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
function manifestsEqual(a, b) {
|
|
75
|
+
return (a.corpusId === b.corpusId &&
|
|
76
|
+
a.corpusHash === b.corpusHash &&
|
|
77
|
+
a.createdAt === b.createdAt &&
|
|
78
|
+
(a.description ?? "") === (b.description ?? "") &&
|
|
79
|
+
JSON.stringify(a.tasks) === JSON.stringify(b.tasks));
|
|
80
|
+
}
|
|
81
|
+
export async function loadCorpusManifestFromPath(repoRoot, manifestPath) {
|
|
82
|
+
const resolved = path.isAbsolute(manifestPath)
|
|
83
|
+
? manifestPath
|
|
84
|
+
: path.resolve(repoRoot, manifestPath);
|
|
85
|
+
const raw = JSON.parse(await readFile(resolved, "utf-8"));
|
|
86
|
+
return corpusManifestInputSchema.parse(raw);
|
|
87
|
+
}
|
|
88
|
+
export async function readCorpusManifest(repoRoot, corpusId) {
|
|
89
|
+
assertCorpusId(corpusId);
|
|
90
|
+
const manifestPath = corpusManifestPath(repoRoot, corpusId);
|
|
91
|
+
const rawText = await readFile(manifestPath, "utf-8");
|
|
92
|
+
await assertManifestIntegrity(repoRoot, corpusId, rawText);
|
|
93
|
+
const stored = corpusManifestSchema.parse(JSON.parse(rawText));
|
|
94
|
+
if (stored.corpusId !== corpusId) {
|
|
95
|
+
throw new Error(`corpus identity mismatch: directory=${corpusId}, manifest=${stored.corpusId}`);
|
|
96
|
+
}
|
|
97
|
+
const recomputed = computeCorpusHash(stored);
|
|
98
|
+
if (recomputed !== stored.corpusHash) {
|
|
99
|
+
throw new Error(`corpusHash mismatch for ${corpusId}: expected ${recomputed}, got ${stored.corpusHash}`);
|
|
100
|
+
}
|
|
101
|
+
return stored;
|
|
102
|
+
}
|
|
103
|
+
export async function listCorpusIds(repoRoot) {
|
|
104
|
+
const root = path.join(repoRoot, EVALUATION_ROOT, "corpora");
|
|
105
|
+
try {
|
|
106
|
+
const entries = await readdir(root, { withFileTypes: true });
|
|
107
|
+
return entries
|
|
108
|
+
.filter((entry) => entry.isDirectory() && SAFE_ID.test(entry.name))
|
|
109
|
+
.map((entry) => entry.name)
|
|
110
|
+
.sort();
|
|
111
|
+
}
|
|
112
|
+
catch (error) {
|
|
113
|
+
const code = error.code;
|
|
114
|
+
if (code === "ENOENT")
|
|
115
|
+
return [];
|
|
116
|
+
throw error;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
export async function registerCorpusManifest(input) {
|
|
120
|
+
const { repoRoot, manifest } = input;
|
|
121
|
+
assertCorpusId(manifest.corpusId);
|
|
122
|
+
const manifestPath = corpusManifestPath(repoRoot, manifest.corpusId);
|
|
123
|
+
if (await pathExists(manifestPath)) {
|
|
124
|
+
const existing = await readCorpusManifest(repoRoot, manifest.corpusId);
|
|
125
|
+
if (!manifestsEqual(existing, manifest)) {
|
|
126
|
+
throw new Error(`corpus already exists with different content: ${manifest.corpusId}`);
|
|
127
|
+
}
|
|
128
|
+
return { manifest: existing, idempotent: true, manifestPath };
|
|
129
|
+
}
|
|
130
|
+
const corporaRoot = path.dirname(corpusDir(repoRoot, manifest.corpusId));
|
|
131
|
+
await mkdir(corporaRoot, { recursive: true });
|
|
132
|
+
const stagingDir = await mkdtemp(path.join(corporaRoot, `.${manifest.corpusId}.register-`));
|
|
133
|
+
try {
|
|
134
|
+
await writeJsonAtomic(path.join(stagingDir, "manifest.json"), manifest, {
|
|
135
|
+
repoRoot,
|
|
136
|
+
});
|
|
137
|
+
await writeTextAtomic(path.join(stagingDir, MANIFEST_INTEGRITY_FILE), `${sha256Hex(serializeManifest(manifest))}\n`, { repoRoot });
|
|
138
|
+
try {
|
|
139
|
+
await rename(stagingDir, corpusDir(repoRoot, manifest.corpusId));
|
|
140
|
+
}
|
|
141
|
+
catch (error) {
|
|
142
|
+
const code = error.code;
|
|
143
|
+
const isCreateRace = code === "EEXIST" ||
|
|
144
|
+
code === "ENOTEMPTY" ||
|
|
145
|
+
(code === "EPERM" &&
|
|
146
|
+
(await pathExists(corpusDir(repoRoot, manifest.corpusId))));
|
|
147
|
+
if (!isCreateRace)
|
|
148
|
+
throw error;
|
|
149
|
+
await rm(stagingDir, { recursive: true, force: true });
|
|
150
|
+
const existing = await readCorpusManifest(repoRoot, manifest.corpusId);
|
|
151
|
+
if (!manifestsEqual(existing, manifest)) {
|
|
152
|
+
throw new Error(`corpus already exists with different content: ${manifest.corpusId}`);
|
|
153
|
+
}
|
|
154
|
+
return { manifest: existing, idempotent: true, manifestPath };
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
catch (error) {
|
|
158
|
+
await rm(stagingDir, { recursive: true, force: true });
|
|
159
|
+
throw error;
|
|
160
|
+
}
|
|
161
|
+
return { manifest, idempotent: false, manifestPath };
|
|
162
|
+
}
|
|
163
|
+
export function summarizeCorpus(manifest) {
|
|
164
|
+
const splits = {
|
|
165
|
+
public: 0,
|
|
166
|
+
private: 0,
|
|
167
|
+
held_out: 0,
|
|
168
|
+
};
|
|
169
|
+
const classes = new Set();
|
|
170
|
+
for (const task of manifest.tasks) {
|
|
171
|
+
splits[task.split] += 1;
|
|
172
|
+
classes.add(task.class);
|
|
173
|
+
}
|
|
174
|
+
return {
|
|
175
|
+
corpusId: manifest.corpusId,
|
|
176
|
+
corpusHash: manifest.corpusHash,
|
|
177
|
+
taskCount: manifest.tasks.length,
|
|
178
|
+
splits,
|
|
179
|
+
classes: [...classes].sort(),
|
|
180
|
+
};
|
|
181
|
+
}
|