@tea-agent/loop-agent 0.16.0 → 0.16.1-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -1
- package/CHANGELOG.md +7 -51
- package/README.md +1 -1
- package/dist/application/evaluation/candidate.js +6 -23
- package/dist/application/evaluation/types.js +0 -522
- package/dist/cli/command-definitions.js +3 -19
- package/dist/commands/eval.js +13 -1176
- package/dist/commands/init.js +1 -4
- package/dist/executors/dag-pi-executor.js +44 -4
- package/dist/worker/delivery/verification-bundle.js +2 -13
- package/dist/worker/feature/review.js +2 -3
- package/dist/worker/outcomes/adapters.js +1 -4
- package/dist/worker/outcomes/gate.js +11 -10
- package/dist/worker/outcomes/projector.js +1 -10
- package/dist/worker/outcomes/types.js +0 -3
- package/dist/worker/run-task/run-task.js +4 -23
- package/dist/worker/task-graph/ready-planner.js +8 -14
- package/dist/worker/task-graph/task-graph-schema.js +3 -5
- package/dist/workflows/dag/frontend-implementation-contract.js +77 -0
- package/dist/workflows/dag/init-hybrid.js +47 -3
- package/dist/workflows/dag/knowledge-curator.js +0 -3
- package/dist/workflows/dag/node-execution.js +4 -11
- package/dist/workflows/dag/prompt.js +1 -1
- package/dist/workflows/dag/runner.js +16 -43
- package/dist/workflows/dag/skill-snapshot.js +7 -11
- package/dist/workflows/dag/types.js +1 -18
- package/docs/init-surface.manifest.json +0 -3
- package/docs/templates/agent-dag.schema.json +5 -0
- package/docs/templates/product-line/AGENTS.md +0 -1
- package/docs/templates/product-line/README.md +0 -17
- package/docs/templates/product-line/acceptance.yaml +0 -9
- package/docs/templates/product-line/task-graph.yaml +0 -8
- package/docs/templates/product-line/task.yaml +0 -4
- package/package.json +1 -1
- package/skills/frontend-implementation/references/node-contracts.md +3 -3
- package/skills/loop-agent/references/hybrid-dag.md +1 -1
- package/dist/application/evaluation/alias.js +0 -184
- package/dist/application/evaluation/budget.js +0 -192
- package/dist/application/evaluation/campaign-hash.js +0 -47
- package/dist/application/evaluation/campaign-matrix.js +0 -372
- package/dist/application/evaluation/campaign-scorecard.js +0 -135
- package/dist/application/evaluation/campaign.js +0 -370
- package/dist/application/evaluation/corpus-hash.js +0 -38
- package/dist/application/evaluation/corpus.js +0 -56
- package/dist/application/evaluation/experiment.js +0 -294
- package/dist/application/evaluation/ignition.js +0 -198
- package/dist/application/evaluation/integrity-audit.js +0 -162
- package/dist/application/evaluation/outer-loop.js +0 -132
- package/dist/application/evaluation/pi-cell-executor.js +0 -39
- package/dist/application/evaluation/private-verifier.js +0 -46
- package/dist/application/evaluation/promotion-policy.js +0 -151
- package/dist/application/evaluation/proposer.js +0 -98
- package/dist/infrastructure/evaluation/alias-store.js +0 -199
- package/dist/infrastructure/evaluation/campaign-store.js +0 -154
- package/dist/infrastructure/evaluation/corpus-store.js +0 -181
- package/dist/infrastructure/evaluation/experiment-store.js +0 -124
- package/dist/infrastructure/evaluation/ignition-store.js +0 -82
- package/dist/infrastructure/evaluation/private-verifier-store.js +0 -145
- package/dist/infrastructure/evaluation/proposer-store.js +0 -78
- package/dist/worker/outcomes/evidence-tokens.js +0 -29
- package/dist/workflows/dag/budget-enforcement.js +0 -67
- package/dist/workflows/dag/context-policy.js +0 -137
- package/docs/templates/evaluation/campaign-budget-v1.json +0 -12
- package/docs/templates/evaluation/campaign-dogfood-v0.json +0 -24
- package/docs/templates/evaluation/campaign-evidence-v1.json +0 -44
- package/docs/templates/evaluation/context-policy-baseline-v1.json +0 -17
- package/docs/templates/evaluation/context-policy-role-specialized-v1.json +0 -28
- package/docs/templates/evaluation/corpus-dogfood-v0.manifest.json +0 -118
- package/docs/templates/evaluation/matrix-dag-dry-run-v1.json +0 -21
- package/docs/templates/evaluation/matrix-fixture-v1.json +0 -10
- package/docs/templates/evaluation/private-verifier-dogfood-v0.json +0 -16
- package/docs/templates/product-line/feature.yaml +0 -11
|
@@ -1,132 +0,0 @@
|
|
|
1
|
-
import { readFile } from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { createCampaign, validateCampaign, } from "./campaign.js";
|
|
4
|
-
import { buildScorecardFromMatrixEvidence, formatMatrixRunMarkdown, runCampaignMatrix, } from "./campaign-matrix.js";
|
|
5
|
-
import { formatProposerRunMarkdown, proposeFromCurate } from "./proposer.js";
|
|
6
|
-
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
7
|
-
import { campaignManifestInputSchema } from "./types.js";
|
|
8
|
-
/**
|
|
9
|
-
* M7 outer-loop orchestration:
|
|
10
|
-
* optional bounded propose → ensure/create campaign → matrix → optional scorecard.
|
|
11
|
-
* Never calls promote/alias APIs.
|
|
12
|
-
*/
|
|
13
|
-
export async function runOuterLoop(input) {
|
|
14
|
-
let proposerRun;
|
|
15
|
-
if (input.proposeFromCurate) {
|
|
16
|
-
proposerRun = await proposeFromCurate({
|
|
17
|
-
repoRoot: input.repoRoot,
|
|
18
|
-
maxCandidates: input.maxProposals ?? 1,
|
|
19
|
-
dryRun: false,
|
|
20
|
-
now: input.now,
|
|
21
|
-
});
|
|
22
|
-
if (proposerRun.proposals.length === 0) {
|
|
23
|
-
throw new Error(`outer-loop propose produced no candidates: ${proposerRun.skippedReasons.join("; ")}`);
|
|
24
|
-
}
|
|
25
|
-
}
|
|
26
|
-
let campaignId = input.campaignId;
|
|
27
|
-
if (input.campaignManifestPath) {
|
|
28
|
-
const absolute = path.resolve(input.repoRoot, input.campaignManifestPath);
|
|
29
|
-
const raw = JSON.parse(await readFile(absolute, "utf-8"));
|
|
30
|
-
if (proposerRun && proposerRun.proposals.length > 0) {
|
|
31
|
-
raw.challengerCandidateIds = proposerRun.proposals.map((row) => row.candidateId);
|
|
32
|
-
}
|
|
33
|
-
const parsed = campaignManifestInputSchema.parse(raw);
|
|
34
|
-
if (campaignId && parsed.campaignId !== campaignId) {
|
|
35
|
-
throw new Error(`campaignId mismatch: --campaign-id=${campaignId}, manifest=${parsed.campaignId}`);
|
|
36
|
-
}
|
|
37
|
-
campaignId = parsed.campaignId;
|
|
38
|
-
const staging = path.join(input.repoRoot, ".harness/evaluation/outer-loop", `${parsed.campaignId}.manifest.json`);
|
|
39
|
-
await writeJsonAtomic(staging, parsed, { repoRoot: input.repoRoot });
|
|
40
|
-
await validateCampaign({
|
|
41
|
-
repoRoot: input.repoRoot,
|
|
42
|
-
manifestPath: staging,
|
|
43
|
-
});
|
|
44
|
-
await createCampaign({
|
|
45
|
-
repoRoot: input.repoRoot,
|
|
46
|
-
manifestPath: staging,
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
if (!campaignId) {
|
|
50
|
-
throw new Error("outer-loop requires --campaign-id or --campaign-manifest");
|
|
51
|
-
}
|
|
52
|
-
if (proposerRun && !input.campaignManifestPath) {
|
|
53
|
-
// Campaign must already list proposed challengers.
|
|
54
|
-
const { showCampaign } = await import("./campaign.js");
|
|
55
|
-
const existing = await showCampaign({
|
|
56
|
-
repoRoot: input.repoRoot,
|
|
57
|
-
campaignId,
|
|
58
|
-
});
|
|
59
|
-
for (const proposal of proposerRun.proposals) {
|
|
60
|
-
if (!existing.manifest.challengerCandidateIds.includes(proposal.candidateId)) {
|
|
61
|
-
throw new Error(`proposed candidate ${proposal.candidateId} is not in campaign ${campaignId} challengers; recreate campaign with --campaign-manifest or add challenger ids`);
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
}
|
|
65
|
-
const matrixResult = await runCampaignMatrix({
|
|
66
|
-
repoRoot: input.repoRoot,
|
|
67
|
-
campaignId,
|
|
68
|
-
mode: input.matrixMode,
|
|
69
|
-
fixturePath: input.fixturePath,
|
|
70
|
-
maxPublicCells: input.maxPublicCells,
|
|
71
|
-
includePrivateChecks: true,
|
|
72
|
-
writeEvidenceDraft: true,
|
|
73
|
-
piExecutor: input.piExecutor,
|
|
74
|
-
now: input.now,
|
|
75
|
-
});
|
|
76
|
-
let scorecardBuilt = false;
|
|
77
|
-
let promotionEligible;
|
|
78
|
-
if (input.buildScorecard && matrixResult.evidence) {
|
|
79
|
-
// Scorecard from matrix draft alone is usually insufficient for held_out;
|
|
80
|
-
// still persist for inspection when requested.
|
|
81
|
-
try {
|
|
82
|
-
await buildScorecardFromMatrixEvidence({
|
|
83
|
-
repoRoot: input.repoRoot,
|
|
84
|
-
campaignId,
|
|
85
|
-
evidence: matrixResult.evidence,
|
|
86
|
-
});
|
|
87
|
-
scorecardBuilt = true;
|
|
88
|
-
const { readCampaignScorecard } = await import("../../infrastructure/evaluation/campaign-store.js");
|
|
89
|
-
const scorecard = await readCampaignScorecard(input.repoRoot, campaignId);
|
|
90
|
-
promotionEligible = scorecard?.policy.promotionEligible;
|
|
91
|
-
}
|
|
92
|
-
catch {
|
|
93
|
-
scorecardBuilt = false;
|
|
94
|
-
}
|
|
95
|
-
}
|
|
96
|
-
const result = {
|
|
97
|
-
schemaVersion: 1,
|
|
98
|
-
proposerRun,
|
|
99
|
-
campaignId,
|
|
100
|
-
matrix: matrixResult.matrix,
|
|
101
|
-
scorecardBuilt,
|
|
102
|
-
promotionEligible,
|
|
103
|
-
autoPromote: false,
|
|
104
|
-
next: [
|
|
105
|
-
`loop-agent eval campaign show --campaign-id ${campaignId} --json`,
|
|
106
|
-
`loop-agent eval campaign scorecard --campaign-id ${campaignId} --evidence .harness/evaluation/campaigns/${campaignId}/evidence-draft.json --json`,
|
|
107
|
-
`loop-agent eval promote --alias <name> --to-candidate <id> --campaign-id ${campaignId} --reason <text> --dry-run`,
|
|
108
|
-
"NEVER auto-promote: human --apply required after policy pass",
|
|
109
|
-
],
|
|
110
|
-
};
|
|
111
|
-
return result;
|
|
112
|
-
}
|
|
113
|
-
export function formatOuterLoopMarkdown(result) {
|
|
114
|
-
const lines = [
|
|
115
|
-
`# Outer Loop: ${result.campaignId}`,
|
|
116
|
-
"",
|
|
117
|
-
`- autoPromote: \`${result.autoPromote}\``,
|
|
118
|
-
`- scorecardBuilt: ${result.scorecardBuilt}`,
|
|
119
|
-
`- promotionEligible: ${result.promotionEligible ?? "(n/a)"}`,
|
|
120
|
-
"",
|
|
121
|
-
];
|
|
122
|
-
if (result.proposerRun) {
|
|
123
|
-
lines.push(formatProposerRunMarkdown(result.proposerRun));
|
|
124
|
-
}
|
|
125
|
-
lines.push(formatMatrixRunMarkdown(result.matrix));
|
|
126
|
-
lines.push("## Next", "");
|
|
127
|
-
for (const step of result.next) {
|
|
128
|
-
lines.push(`- ${step}`);
|
|
129
|
-
}
|
|
130
|
-
lines.push("");
|
|
131
|
-
return `${lines.join("\n")}\n`;
|
|
132
|
-
}
|
|
@@ -1,39 +0,0 @@
|
|
|
1
|
-
import { readFile } from "node:fs/promises";
|
|
2
|
-
import path from "node:path";
|
|
3
|
-
import { runDagUseCase } from "../dag/run-dag.js";
|
|
4
|
-
import { writeJsonAtomic } from "../../infrastructure/harness/atomic-write.js";
|
|
5
|
-
/**
|
|
6
|
-
* M7 live-Pi wiring (bounded): materialize a per-cell DagSpec with evaluation
|
|
7
|
-
* binding and run `runDagUseCase` in dry-run mode only.
|
|
8
|
-
*
|
|
9
|
-
* Never auto-promotes. Full live Pi execute remains opt-in via custom PiCellExecutor.
|
|
10
|
-
*/
|
|
11
|
-
export function createDagDryRunPiCellExecutor(input) {
|
|
12
|
-
return async (ctx) => {
|
|
13
|
-
const absoluteTemplate = path.resolve(ctx.repoRoot, input.dagTemplatePath);
|
|
14
|
-
const raw = JSON.parse(await readFile(absoluteTemplate, "utf-8"));
|
|
15
|
-
raw.version = 3;
|
|
16
|
-
raw.evaluation = ctx.evaluation;
|
|
17
|
-
raw.title =
|
|
18
|
-
typeof raw.title === "string"
|
|
19
|
-
? `${raw.title} :: ${ctx.cell.cellId}`
|
|
20
|
-
: `eval-matrix-${ctx.cell.cellId}`;
|
|
21
|
-
const cellDagPath = path.join(ctx.repoRoot, path.dirname(ctx.jobPath), `${ctx.cell.cellId}.dag.json`);
|
|
22
|
-
await writeJsonAtomic(cellDagPath, raw, { repoRoot: ctx.repoRoot });
|
|
23
|
-
const result = await runDagUseCase({
|
|
24
|
-
repoRoot: ctx.repoRoot,
|
|
25
|
-
dagPath: cellDagPath,
|
|
26
|
-
cwd: ctx.repoRoot,
|
|
27
|
-
initOnly: false,
|
|
28
|
-
dryRun: true,
|
|
29
|
-
});
|
|
30
|
-
if (!("mode" in result) || result.mode !== "dry-run") {
|
|
31
|
-
throw new Error("createDagDryRunPiCellExecutor expected dry-run result; refuse non-dry execution");
|
|
32
|
-
}
|
|
33
|
-
return {
|
|
34
|
-
verifyPassed: false,
|
|
35
|
-
runId: result.runId,
|
|
36
|
-
detail: `pi dry-run prepared runDir=${result.runDir}; verifyPassed left false until live shell evidence`,
|
|
37
|
-
};
|
|
38
|
-
};
|
|
39
|
-
}
|
|
@@ -1,46 +0,0 @@
|
|
|
1
|
-
import { listPrivateVerifierIds, loadPrivateVerifierManifestFromPath, materializePrivateVerifierManifest, readPrivateVerifierManifest, registerPrivateVerifierManifest, } from "../../infrastructure/evaluation/private-verifier-store.js";
|
|
2
|
-
export async function validatePrivateVerifier(input) {
|
|
3
|
-
const raw = await loadPrivateVerifierManifestFromPath(input.repoRoot, input.manifestPath);
|
|
4
|
-
const manifest = await materializePrivateVerifierManifest(raw);
|
|
5
|
-
return { manifest };
|
|
6
|
-
}
|
|
7
|
-
export async function registerPrivateVerifier(input) {
|
|
8
|
-
const validated = await validatePrivateVerifier(input);
|
|
9
|
-
return registerPrivateVerifierManifest({
|
|
10
|
-
repoRoot: input.repoRoot,
|
|
11
|
-
manifest: validated.manifest,
|
|
12
|
-
});
|
|
13
|
-
}
|
|
14
|
-
export async function showPrivateVerifier(input) {
|
|
15
|
-
const manifest = await readPrivateVerifierManifest(input.repoRoot, input.privateVerifierId);
|
|
16
|
-
return { manifest };
|
|
17
|
-
}
|
|
18
|
-
export async function listPrivateVerifiers(input) {
|
|
19
|
-
const ids = await listPrivateVerifierIds(input.repoRoot);
|
|
20
|
-
const rows = [];
|
|
21
|
-
for (const privateVerifierId of ids) {
|
|
22
|
-
const manifest = await readPrivateVerifierManifest(input.repoRoot, privateVerifierId);
|
|
23
|
-
rows.push({
|
|
24
|
-
privateVerifierId,
|
|
25
|
-
manifestHash: manifest.manifestHash,
|
|
26
|
-
checkCount: manifest.checks.length,
|
|
27
|
-
});
|
|
28
|
-
}
|
|
29
|
-
return rows;
|
|
30
|
-
}
|
|
31
|
-
export function formatPrivateVerifierMarkdown(manifest) {
|
|
32
|
-
return [
|
|
33
|
-
`# Private Verifier: ${manifest.privateVerifierId}`,
|
|
34
|
-
"",
|
|
35
|
-
`- manifestHash: \`${manifest.manifestHash}\``,
|
|
36
|
-
`- checks: ${manifest.checks.length}`,
|
|
37
|
-
`- createdAt: \`${manifest.createdAt}\``,
|
|
38
|
-
"",
|
|
39
|
-
"> Controller-only. Must never enter candidate prompt, skill snapshot, or contentRefs.",
|
|
40
|
-
"",
|
|
41
|
-
"## Checks",
|
|
42
|
-
"",
|
|
43
|
-
...manifest.checks.map((check) => `- \`${check.checkId}\` (${check.kind}): command arity=${check.command.length}, timeoutMs=${check.timeoutMs}`),
|
|
44
|
-
"",
|
|
45
|
-
].join("\n");
|
|
46
|
-
}
|
|
@@ -1,151 +0,0 @@
|
|
|
1
|
-
import { runIntegrityAudits } from "./integrity-audit.js";
|
|
2
|
-
export const PROMOTION_POLICY_VERSION = "promotion-policy-v1";
|
|
3
|
-
function emptyTally() {
|
|
4
|
-
return { incumbentWins: 0, challengerWins: 0, ties: 0 };
|
|
5
|
-
}
|
|
6
|
-
function pairKey(cell) {
|
|
7
|
-
return `${cell.split}\u0000${cell.taskRef}\u0000${cell.seed}`;
|
|
8
|
-
}
|
|
9
|
-
function tallyPaired(input) {
|
|
10
|
-
const byKey = new Map();
|
|
11
|
-
for (const cell of input.cells) {
|
|
12
|
-
if (cell.split !== input.split)
|
|
13
|
-
continue;
|
|
14
|
-
const key = pairKey(cell);
|
|
15
|
-
const row = byKey.get(key) ?? {};
|
|
16
|
-
if (cell.candidateId === input.incumbentCandidateId) {
|
|
17
|
-
row.incumbent = cell.verifyPassed;
|
|
18
|
-
}
|
|
19
|
-
else if (cell.candidateId === input.challengerCandidateId) {
|
|
20
|
-
row.challenger = cell.verifyPassed;
|
|
21
|
-
}
|
|
22
|
-
byKey.set(key, row);
|
|
23
|
-
}
|
|
24
|
-
const tally = emptyTally();
|
|
25
|
-
for (const row of byKey.values()) {
|
|
26
|
-
if (row.incumbent === undefined || row.challenger === undefined)
|
|
27
|
-
continue;
|
|
28
|
-
if (row.challenger === row.incumbent) {
|
|
29
|
-
tally.ties += 1;
|
|
30
|
-
}
|
|
31
|
-
else if (row.challenger && !row.incumbent) {
|
|
32
|
-
tally.challengerWins += 1;
|
|
33
|
-
}
|
|
34
|
-
else {
|
|
35
|
-
tally.incumbentWins += 1;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
return tally;
|
|
39
|
-
}
|
|
40
|
-
function privateImprovementStatus(tally) {
|
|
41
|
-
const compared = tally.incumbentWins + tally.challengerWins + tally.ties;
|
|
42
|
-
if (compared === 0)
|
|
43
|
-
return "insufficient";
|
|
44
|
-
if (tally.challengerWins > tally.incumbentWins)
|
|
45
|
-
return "pass";
|
|
46
|
-
return "fail";
|
|
47
|
-
}
|
|
48
|
-
function heldOutNonRegressionStatus(tally) {
|
|
49
|
-
const compared = tally.incumbentWins + tally.challengerWins + tally.ties;
|
|
50
|
-
if (compared === 0)
|
|
51
|
-
return "insufficient";
|
|
52
|
-
// Non-regression: challenger must not lose more pairs than incumbent.
|
|
53
|
-
if (tally.challengerWins + tally.ties >= tally.incumbentWins)
|
|
54
|
-
return "pass";
|
|
55
|
-
return "fail";
|
|
56
|
-
}
|
|
57
|
-
function budgetStatusGate(status) {
|
|
58
|
-
if (status === "ok")
|
|
59
|
-
return "pass";
|
|
60
|
-
if (status === "breached")
|
|
61
|
-
return "fail";
|
|
62
|
-
return "insufficient";
|
|
63
|
-
}
|
|
64
|
-
function safetyStatus(findings) {
|
|
65
|
-
if (findings.some((f) => f.severity === "fail"))
|
|
66
|
-
return "fail";
|
|
67
|
-
if (findings.some((f) => f.severity === "review"))
|
|
68
|
-
return "review";
|
|
69
|
-
return "pass";
|
|
70
|
-
}
|
|
71
|
-
/**
|
|
72
|
-
* Evaluate Promotion Policy v1 (W5.1).
|
|
73
|
-
* Never auto-promotes; human gate remains mandatory at apply time.
|
|
74
|
-
*/
|
|
75
|
-
export function evaluatePromotionPolicy(input) {
|
|
76
|
-
if (input.evidence.campaignId !== input.campaignId) {
|
|
77
|
-
throw new Error(`evidence campaignId mismatch: expected ${input.campaignId}, got ${input.evidence.campaignId}`);
|
|
78
|
-
}
|
|
79
|
-
const findings = runIntegrityAudits({
|
|
80
|
-
evidence: input.evidence,
|
|
81
|
-
incumbentCandidateId: input.incumbentCandidateId,
|
|
82
|
-
});
|
|
83
|
-
const paired = {
|
|
84
|
-
private: tallyPaired({
|
|
85
|
-
incumbentCandidateId: input.incumbentCandidateId,
|
|
86
|
-
challengerCandidateId: input.evidence.challengerCandidateId,
|
|
87
|
-
cells: input.evidence.cells,
|
|
88
|
-
split: "private",
|
|
89
|
-
}),
|
|
90
|
-
held_out: tallyPaired({
|
|
91
|
-
incumbentCandidateId: input.incumbentCandidateId,
|
|
92
|
-
challengerCandidateId: input.evidence.challengerCandidateId,
|
|
93
|
-
cells: input.evidence.cells,
|
|
94
|
-
split: "held_out",
|
|
95
|
-
}),
|
|
96
|
-
public: tallyPaired({
|
|
97
|
-
incumbentCandidateId: input.incumbentCandidateId,
|
|
98
|
-
challengerCandidateId: input.evidence.challengerCandidateId,
|
|
99
|
-
cells: input.evidence.cells,
|
|
100
|
-
split: "public",
|
|
101
|
-
}),
|
|
102
|
-
};
|
|
103
|
-
const safetyOk = safetyStatus(findings);
|
|
104
|
-
const gates = {
|
|
105
|
-
privateImprovement: privateImprovementStatus(paired.private),
|
|
106
|
-
heldOutNonRegression: heldOutNonRegressionStatus(paired.held_out),
|
|
107
|
-
budgetOk: budgetStatusGate(input.evidence.budgetStatus.challenger),
|
|
108
|
-
safetyOk,
|
|
109
|
-
humanRequired: true,
|
|
110
|
-
};
|
|
111
|
-
const reasons = [];
|
|
112
|
-
if (gates.privateImprovement !== "pass") {
|
|
113
|
-
reasons.push(`privateImprovement=${gates.privateImprovement} (challengerWins=${paired.private.challengerWins}, incumbentWins=${paired.private.incumbentWins}, ties=${paired.private.ties})`);
|
|
114
|
-
}
|
|
115
|
-
if (gates.heldOutNonRegression !== "pass") {
|
|
116
|
-
reasons.push(`heldOutNonRegression=${gates.heldOutNonRegression} (challengerWins=${paired.held_out.challengerWins}, incumbentWins=${paired.held_out.incumbentWins}, ties=${paired.held_out.ties})`);
|
|
117
|
-
}
|
|
118
|
-
if (gates.budgetOk !== "pass") {
|
|
119
|
-
reasons.push(`budgetOk=${gates.budgetOk}`);
|
|
120
|
-
}
|
|
121
|
-
if (gates.safetyOk === "fail") {
|
|
122
|
-
reasons.push(`safetyOk=fail (${findings.filter((f) => f.severity === "fail").map((f) => f.code).join(", ")})`);
|
|
123
|
-
}
|
|
124
|
-
else if (gates.safetyOk === "review") {
|
|
125
|
-
if (input.acknowledgeReview) {
|
|
126
|
-
reasons.push(`safetyOk=review acknowledged via --acknowledge-review (${findings.filter((f) => f.severity === "review").map((f) => f.code).join(", ")})`);
|
|
127
|
-
}
|
|
128
|
-
else {
|
|
129
|
-
reasons.push(`safetyOk=review (${findings.filter((f) => f.severity === "review").map((f) => f.code).join(", ")}); re-run with --acknowledge-review after human review`);
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
|
-
reasons.push("humanRequired=true (apply still needs --reason)");
|
|
133
|
-
const safetyAllows = gates.safetyOk === "pass" ||
|
|
134
|
-
(gates.safetyOk === "review" && Boolean(input.acknowledgeReview));
|
|
135
|
-
const promotionEligible = gates.privateImprovement === "pass" &&
|
|
136
|
-
gates.heldOutNonRegression === "pass" &&
|
|
137
|
-
gates.budgetOk === "pass" &&
|
|
138
|
-
safetyAllows;
|
|
139
|
-
return {
|
|
140
|
-
schemaVersion: 1,
|
|
141
|
-
policyVersion: PROMOTION_POLICY_VERSION,
|
|
142
|
-
campaignId: input.campaignId,
|
|
143
|
-
incumbentCandidateId: input.incumbentCandidateId,
|
|
144
|
-
challengerCandidateId: input.evidence.challengerCandidateId,
|
|
145
|
-
promotionEligible,
|
|
146
|
-
gates,
|
|
147
|
-
findings,
|
|
148
|
-
reasons,
|
|
149
|
-
paired,
|
|
150
|
-
};
|
|
151
|
-
}
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import { writeProposerRun } from "../../infrastructure/evaluation/proposer-store.js";
|
|
3
|
-
import { createExperimentFromCurate } from "./experiment.js";
|
|
4
|
-
/**
|
|
5
|
-
* Controlled outer-loop proposer (M7).
|
|
6
|
-
* Only proposes bounded experimental candidates from knowledge-curate.
|
|
7
|
-
* Never modifies evaluator/kernel/private verifier and never auto-promotes.
|
|
8
|
-
*/
|
|
9
|
-
export async function proposeFromCurate(input) {
|
|
10
|
-
const maxCandidates = input.maxCandidates ?? 1;
|
|
11
|
-
if (maxCandidates < 1 || maxCandidates > 3) {
|
|
12
|
-
throw new Error("proposer maxCandidates must be between 1 and 3 (bounded)");
|
|
13
|
-
}
|
|
14
|
-
const dryRun = input.dryRun !== false;
|
|
15
|
-
const now = input.now ?? new Date().toISOString();
|
|
16
|
-
const proposerRunId = input.proposerRunId ??
|
|
17
|
-
`propose-${createHash("sha256").update(now).digest("hex").slice(0, 12)}`;
|
|
18
|
-
const proposals = [];
|
|
19
|
-
const skippedReasons = [];
|
|
20
|
-
if (dryRun) {
|
|
21
|
-
skippedReasons.push("dry-run: no candidate registered; re-run with --apply to materialize bounded proposals");
|
|
22
|
-
const run = {
|
|
23
|
-
schemaVersion: 1,
|
|
24
|
-
proposerRunId,
|
|
25
|
-
source: "knowledge-curate",
|
|
26
|
-
createdAt: now,
|
|
27
|
-
maxCandidates,
|
|
28
|
-
proposals,
|
|
29
|
-
skippedReasons,
|
|
30
|
-
autoPromote: false,
|
|
31
|
-
};
|
|
32
|
-
return run;
|
|
33
|
-
}
|
|
34
|
-
for (let i = 0; i < maxCandidates; i += 1) {
|
|
35
|
-
try {
|
|
36
|
-
const created = await createExperimentFromCurate({
|
|
37
|
-
repoRoot: input.repoRoot,
|
|
38
|
-
now,
|
|
39
|
-
candidateId: `proposed-${proposerRunId}-${i + 1}`,
|
|
40
|
-
experimentId: `exp-${proposerRunId}-${i + 1}`,
|
|
41
|
-
});
|
|
42
|
-
proposals.push({
|
|
43
|
-
candidateId: created.candidate.manifest.candidateId,
|
|
44
|
-
experimentId: created.record.experimentId,
|
|
45
|
-
proposalPath: created.record.proposalPath,
|
|
46
|
-
bundleHash: created.candidate.manifest.bundleHash,
|
|
47
|
-
status: "proposed_experimenting",
|
|
48
|
-
});
|
|
49
|
-
}
|
|
50
|
-
catch (error) {
|
|
51
|
-
skippedReasons.push(error instanceof Error ? error.message : String(error));
|
|
52
|
-
break;
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
const run = {
|
|
56
|
-
schemaVersion: 1,
|
|
57
|
-
proposerRunId,
|
|
58
|
-
source: "knowledge-curate",
|
|
59
|
-
createdAt: now,
|
|
60
|
-
maxCandidates,
|
|
61
|
-
proposals,
|
|
62
|
-
skippedReasons,
|
|
63
|
-
autoPromote: false,
|
|
64
|
-
};
|
|
65
|
-
await writeProposerRun({ repoRoot: input.repoRoot, run });
|
|
66
|
-
return run;
|
|
67
|
-
}
|
|
68
|
-
export function formatProposerRunMarkdown(run) {
|
|
69
|
-
const lines = [
|
|
70
|
-
`# Proposer Run: ${run.proposerRunId}`,
|
|
71
|
-
"",
|
|
72
|
-
`- source: \`${run.source}\``,
|
|
73
|
-
`- maxCandidates: ${run.maxCandidates}`,
|
|
74
|
-
`- proposals: ${run.proposals.length}`,
|
|
75
|
-
`- autoPromote: \`${run.autoPromote}\``,
|
|
76
|
-
"",
|
|
77
|
-
"## Proposals",
|
|
78
|
-
"",
|
|
79
|
-
];
|
|
80
|
-
if (run.proposals.length === 0) {
|
|
81
|
-
lines.push("(none)", "");
|
|
82
|
-
}
|
|
83
|
-
else {
|
|
84
|
-
for (const proposal of run.proposals) {
|
|
85
|
-
lines.push(`- \`${proposal.candidateId}\` experiment=\`${proposal.experimentId}\` hash=\`${proposal.bundleHash}\``);
|
|
86
|
-
}
|
|
87
|
-
lines.push("");
|
|
88
|
-
}
|
|
89
|
-
if (run.skippedReasons.length > 0) {
|
|
90
|
-
lines.push("## Skipped / notes", "");
|
|
91
|
-
for (const reason of run.skippedReasons) {
|
|
92
|
-
lines.push(`- ${reason}`);
|
|
93
|
-
}
|
|
94
|
-
lines.push("");
|
|
95
|
-
}
|
|
96
|
-
lines.push("> M7: proposer only registers experimenting candidates; promotion remains human-gated.", "");
|
|
97
|
-
return `${lines.join("\n")}\n`;
|
|
98
|
-
}
|
|
@@ -1,199 +0,0 @@
|
|
|
1
|
-
import { createHash } from "node:crypto";
|
|
2
|
-
import { access, mkdir, readdir, readFile, rm, writeFile, } from "node:fs/promises";
|
|
3
|
-
import path from "node:path";
|
|
4
|
-
import { aliasHistoryEventSchema, aliasNameSchema, incumbentAliasSchema, promotionDecisionSchema, } from "../../application/evaluation/types.js";
|
|
5
|
-
import { appendJsonlLineAtomic, writeJsonAtomic, writeTextAtomic, } from "../harness/atomic-write.js";
|
|
6
|
-
import { EVALUATION_ROOT } from "./store.js";
|
|
7
|
-
const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
|
|
8
|
-
const ALIAS_LOCK_DIR = ".lock";
|
|
9
|
-
const ALIAS_LOCK_RETRIES = 40;
|
|
10
|
-
const ALIAS_LOCK_DELAY_MS = 25;
|
|
11
|
-
const CURRENT_INTEGRITY_FILE = "current.sha256";
|
|
12
|
-
export function assertAliasName(value) {
|
|
13
|
-
aliasNameSchema.parse(value);
|
|
14
|
-
if (!SAFE_ID.test(value)) {
|
|
15
|
-
throw new Error(`alias must contain only letters, numbers, dot, underscore, or hyphen: ${value}`);
|
|
16
|
-
}
|
|
17
|
-
}
|
|
18
|
-
export function aliasDir(repoRoot, alias) {
|
|
19
|
-
assertAliasName(alias);
|
|
20
|
-
return path.join(repoRoot, EVALUATION_ROOT, "aliases", alias);
|
|
21
|
-
}
|
|
22
|
-
export function aliasCurrentPath(repoRoot, alias) {
|
|
23
|
-
return path.join(aliasDir(repoRoot, alias), "current.json");
|
|
24
|
-
}
|
|
25
|
-
export function aliasHistoryPath(repoRoot, alias) {
|
|
26
|
-
return path.join(aliasDir(repoRoot, alias), "history.jsonl");
|
|
27
|
-
}
|
|
28
|
-
export function decisionDir(repoRoot) {
|
|
29
|
-
return path.join(repoRoot, EVALUATION_ROOT, "decisions");
|
|
30
|
-
}
|
|
31
|
-
export function decisionPath(repoRoot, decisionId) {
|
|
32
|
-
if (!SAFE_ID.test(decisionId)) {
|
|
33
|
-
throw new Error(`decision-id must contain only letters, numbers, dot, underscore, or hyphen: ${decisionId}`);
|
|
34
|
-
}
|
|
35
|
-
return path.join(decisionDir(repoRoot), `${decisionId}.json`);
|
|
36
|
-
}
|
|
37
|
-
function sha256Hex(content) {
|
|
38
|
-
return createHash("sha256").update(content, "utf8").digest("hex");
|
|
39
|
-
}
|
|
40
|
-
function serializeAlias(alias) {
|
|
41
|
-
return `${JSON.stringify(alias, null, 2)}\n`;
|
|
42
|
-
}
|
|
43
|
-
async function pathExists(filePath) {
|
|
44
|
-
try {
|
|
45
|
-
await access(filePath);
|
|
46
|
-
return true;
|
|
47
|
-
}
|
|
48
|
-
catch {
|
|
49
|
-
return false;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
async function acquireAliasLock(repoRoot, alias) {
|
|
53
|
-
const lockPath = path.join(aliasDir(repoRoot, alias), ALIAS_LOCK_DIR);
|
|
54
|
-
await mkdir(aliasDir(repoRoot, alias), { recursive: true });
|
|
55
|
-
for (let attempt = 0; attempt < ALIAS_LOCK_RETRIES; attempt += 1) {
|
|
56
|
-
try {
|
|
57
|
-
await mkdir(lockPath);
|
|
58
|
-
return async () => {
|
|
59
|
-
await rm(lockPath, { recursive: true, force: true });
|
|
60
|
-
};
|
|
61
|
-
}
|
|
62
|
-
catch (error) {
|
|
63
|
-
if (error.code !== "EEXIST")
|
|
64
|
-
throw error;
|
|
65
|
-
await new Promise((resolve) => setTimeout(resolve, ALIAS_LOCK_DELAY_MS));
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
throw new Error(`timed out acquiring alias lock for ${alias}`);
|
|
69
|
-
}
|
|
70
|
-
async function assertCurrentIntegrity(repoRoot, alias, raw) {
|
|
71
|
-
const integrityPath = path.join(aliasDir(repoRoot, alias), CURRENT_INTEGRITY_FILE);
|
|
72
|
-
if (!(await pathExists(integrityPath))) {
|
|
73
|
-
throw new Error(`missing alias integrity file for ${alias}`);
|
|
74
|
-
}
|
|
75
|
-
const expected = (await readFile(integrityPath, "utf-8")).trim();
|
|
76
|
-
const actual = sha256Hex(raw);
|
|
77
|
-
if (expected !== actual) {
|
|
78
|
-
throw new Error(`alias integrity mismatch for ${alias}: expected ${expected}, got ${actual}`);
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
export async function readIncumbentAlias(repoRoot, alias) {
|
|
82
|
-
assertAliasName(alias);
|
|
83
|
-
const currentPath = aliasCurrentPath(repoRoot, alias);
|
|
84
|
-
if (!(await pathExists(currentPath)))
|
|
85
|
-
return null;
|
|
86
|
-
const raw = await readFile(currentPath, "utf-8");
|
|
87
|
-
await assertCurrentIntegrity(repoRoot, alias, raw);
|
|
88
|
-
const parsed = incumbentAliasSchema.parse(JSON.parse(raw));
|
|
89
|
-
if (parsed.alias !== alias) {
|
|
90
|
-
throw new Error(`alias identity mismatch: directory=${alias}, current=${parsed.alias}`);
|
|
91
|
-
}
|
|
92
|
-
return parsed;
|
|
93
|
-
}
|
|
94
|
-
export async function listAliasNames(repoRoot) {
|
|
95
|
-
const root = path.join(repoRoot, EVALUATION_ROOT, "aliases");
|
|
96
|
-
try {
|
|
97
|
-
const entries = await readdir(root, { withFileTypes: true });
|
|
98
|
-
return entries
|
|
99
|
-
.filter((entry) => entry.isDirectory() && SAFE_ID.test(entry.name))
|
|
100
|
-
.map((entry) => entry.name)
|
|
101
|
-
.sort();
|
|
102
|
-
}
|
|
103
|
-
catch (error) {
|
|
104
|
-
const code = error.code;
|
|
105
|
-
if (code === "ENOENT")
|
|
106
|
-
return [];
|
|
107
|
-
throw error;
|
|
108
|
-
}
|
|
109
|
-
}
|
|
110
|
-
export async function listIncumbentCandidateIds(repoRoot) {
|
|
111
|
-
const names = await listAliasNames(repoRoot);
|
|
112
|
-
const ids = new Set();
|
|
113
|
-
for (const name of names) {
|
|
114
|
-
const current = await readIncumbentAlias(repoRoot, name);
|
|
115
|
-
if (current)
|
|
116
|
-
ids.add(current.candidateId);
|
|
117
|
-
}
|
|
118
|
-
return ids;
|
|
119
|
-
}
|
|
120
|
-
export async function readPromotionDecision(repoRoot, decisionId) {
|
|
121
|
-
const raw = await readFile(decisionPath(repoRoot, decisionId), "utf-8");
|
|
122
|
-
return promotionDecisionSchema.parse(JSON.parse(raw));
|
|
123
|
-
}
|
|
124
|
-
async function writeAliasCurrentAtomic(input) {
|
|
125
|
-
const dir = aliasDir(input.repoRoot, input.alias.alias);
|
|
126
|
-
await mkdir(dir, { recursive: true });
|
|
127
|
-
const serialized = serializeAlias(input.alias);
|
|
128
|
-
await writeTextAtomic(aliasCurrentPath(input.repoRoot, input.alias.alias), serialized, {
|
|
129
|
-
repoRoot: input.repoRoot,
|
|
130
|
-
});
|
|
131
|
-
await writeTextAtomic(path.join(dir, CURRENT_INTEGRITY_FILE), `${sha256Hex(serialized)}\n`, { repoRoot: input.repoRoot });
|
|
132
|
-
}
|
|
133
|
-
export async function commitAliasMove(input) {
|
|
134
|
-
const { repoRoot, decision, nextAlias, historyEvent } = input;
|
|
135
|
-
assertAliasName(decision.alias);
|
|
136
|
-
promotionDecisionSchema.parse(decision);
|
|
137
|
-
incumbentAliasSchema.parse(nextAlias);
|
|
138
|
-
aliasHistoryEventSchema.parse(historyEvent);
|
|
139
|
-
if (decision.decisionId !== nextAlias.updatedByDecisionId) {
|
|
140
|
-
throw new Error("decisionId must match alias.updatedByDecisionId");
|
|
141
|
-
}
|
|
142
|
-
if (decision.decisionId !== historyEvent.decisionId) {
|
|
143
|
-
throw new Error("decisionId must match history event");
|
|
144
|
-
}
|
|
145
|
-
const releaseLock = await acquireAliasLock(repoRoot, decision.alias);
|
|
146
|
-
const previousRaw = await readFile(aliasCurrentPath(repoRoot, decision.alias), "utf-8").catch(() => null);
|
|
147
|
-
const previousIntegrity = await readFile(path.join(aliasDir(repoRoot, decision.alias), CURRENT_INTEGRITY_FILE), "utf-8").catch(() => null);
|
|
148
|
-
const decisionFile = decisionPath(repoRoot, decision.decisionId);
|
|
149
|
-
let decisionWritten = false;
|
|
150
|
-
let historyWritten = false;
|
|
151
|
-
try {
|
|
152
|
-
if (await pathExists(decisionFile)) {
|
|
153
|
-
throw new Error(`decision already exists: ${decision.decisionId}`);
|
|
154
|
-
}
|
|
155
|
-
await mkdir(decisionDir(repoRoot), { recursive: true });
|
|
156
|
-
await writeJsonAtomic(decisionFile, decision, { repoRoot });
|
|
157
|
-
decisionWritten = true;
|
|
158
|
-
await writeAliasCurrentAtomic({ repoRoot, alias: nextAlias });
|
|
159
|
-
await appendJsonlLineAtomic(aliasHistoryPath(repoRoot, decision.alias), historyEvent, { repoRoot });
|
|
160
|
-
historyWritten = true;
|
|
161
|
-
}
|
|
162
|
-
catch (error) {
|
|
163
|
-
// Best-effort rollback of alias pointer + decision; history append is last.
|
|
164
|
-
if (previousRaw !== null) {
|
|
165
|
-
await writeTextAtomic(aliasCurrentPath(repoRoot, decision.alias), previousRaw, { repoRoot }).catch(() => undefined);
|
|
166
|
-
if (previousIntegrity !== null) {
|
|
167
|
-
await writeTextAtomic(path.join(aliasDir(repoRoot, decision.alias), CURRENT_INTEGRITY_FILE), previousIntegrity, { repoRoot }).catch(() => undefined);
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
else if (!historyWritten) {
|
|
171
|
-
await rm(aliasCurrentPath(repoRoot, decision.alias), {
|
|
172
|
-
force: true,
|
|
173
|
-
}).catch(() => undefined);
|
|
174
|
-
await rm(path.join(aliasDir(repoRoot, decision.alias), CURRENT_INTEGRITY_FILE), { force: true }).catch(() => undefined);
|
|
175
|
-
}
|
|
176
|
-
if (decisionWritten) {
|
|
177
|
-
await rm(decisionFile, { force: true }).catch(() => undefined);
|
|
178
|
-
}
|
|
179
|
-
throw error;
|
|
180
|
-
}
|
|
181
|
-
finally {
|
|
182
|
-
await releaseLock();
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
export function makeDecisionId(input) {
|
|
186
|
-
const stamp = input.createdAt.replace(/[^A-Za-z0-9._-]/g, "-");
|
|
187
|
-
const raw = `dec-${input.alias}-${input.action}-${input.toCandidateId}-${stamp}`;
|
|
188
|
-
return raw.slice(0, 180);
|
|
189
|
-
}
|
|
190
|
-
/** Test helper: force-corrupt integrity without going through commit. */
|
|
191
|
-
export async function writeAliasCurrentForTest(input) {
|
|
192
|
-
await writeAliasCurrentAtomic({
|
|
193
|
-
repoRoot: input.repoRoot,
|
|
194
|
-
alias: input.alias,
|
|
195
|
-
});
|
|
196
|
-
if (input.corruptIntegrity) {
|
|
197
|
-
await writeFile(path.join(aliasDir(input.repoRoot, input.alias.alias), CURRENT_INTEGRITY_FILE), `${"0".repeat(64)}\n`, "utf-8");
|
|
198
|
-
}
|
|
199
|
-
}
|