ai-engineering-loop 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.agents/skills/ael/SKILL.md +1 -1
  2. package/.agents/skills/ael/references/loop-commands.md +11 -0
  3. package/.agents/skills/ael/references/loop-goal.md +2 -0
  4. package/.agents/skills/ael/references/loop-maker.md +1 -6
  5. package/.agents/skills/ael/references/loop-report.md +8 -0
  6. package/.agents/skills/ael/references/loop-verify.md +11 -9
  7. package/.claude/agents/devil-advocate.md +6 -1
  8. package/.claude/agents/judge.md +3 -1
  9. package/.grok/agents/devil-advocate.md +6 -1
  10. package/.grok/agents/judge.md +3 -1
  11. package/CHANGELOG.md +16 -0
  12. package/agents/devil-advocate.md +8 -0
  13. package/agents/judge.md +1 -0
  14. package/agents/shared/devil-advocate.body.md +6 -1
  15. package/agents/shared/judge.body.md +3 -1
  16. package/bin/ai-engineering-loop.js +262 -20
  17. package/core/bench.md +15 -0
  18. package/core/goal-contract.md +35 -0
  19. package/core/verification-loop.md +78 -1
  20. package/lib/ablation.js +52 -0
  21. package/lib/ac-status.js +63 -0
  22. package/lib/baseline.js +138 -0
  23. package/lib/bench.js +24 -1
  24. package/lib/calibration.js +112 -0
  25. package/lib/claimed-vs-reality.js +106 -12
  26. package/lib/clean-state.js +85 -0
  27. package/lib/cli-args.js +5 -3
  28. package/lib/debt.js +110 -0
  29. package/lib/features.js +95 -0
  30. package/lib/gates.js +234 -41
  31. package/lib/harness-audit.js +124 -0
  32. package/lib/no-progress.js +6 -1
  33. package/lib/review-ingest.js +54 -2
  34. package/lib/rules.js +112 -0
  35. package/lib/stats.js +90 -0
  36. package/lib/sync-hosts.js +1 -0
  37. package/lib/verification-integrity.js +164 -0
  38. package/lib/verification-lanes.js +64 -0
  39. package/lib/verification-recorder.js +17 -5
  40. package/package.json +1 -1
  41. package/schemas/finding-ledger.schema.json +14 -0
  42. package/schemas/goal-contract.schema.json +31 -0
  43. package/schemas/verdict.schema.json +2 -1
@@ -0,0 +1,52 @@
1
+ 'use strict';
2
+
3
+ // Harness ablation for bench (C-08, learn-harness-engineering L02/L12):
4
+ // switch one gate component off inside a bench scenario repo to measure
5
+ // whether it still earns its cost. Honoured only when bench set it up (env
6
+ // AEL_BENCH_MODE + AEL_ABLATE and a marker file bench wrote into the repo),
7
+ // and every skip is logged to the Run so it can never pass silently.
8
+
9
+ const fs = require('fs');
10
+ const path = require('path');
11
+ const { runsDir } = require('./run-state.js');
12
+
13
+ const COMPONENTS = Object.freeze(['integrity', 'l3', 'l1', 'smoke', 'rules']);
14
+ const MARKER = path.join('.ai-engineering-loop', 'bench-ablation.json');
15
+
16
+ function requested(env = process.env) {
17
+ return String(env.AEL_ABLATE || '').split(',').map((item) => item.trim()).filter(Boolean);
18
+ }
19
+
20
+ function writeMarker(repo, components) {
21
+ fs.mkdirSync(path.join(repo, '.ai-engineering-loop'), { recursive: true });
22
+ fs.writeFileSync(path.join(repo, MARKER), `${JSON.stringify({ schemaVersion: 1, components }, null, 2)}\n`);
23
+ }
24
+
25
+ function ablated(rootDir, component, runId = null, env = process.env) {
26
+ if (!env.AEL_BENCH_MODE || !requested(env).includes(component)) return false;
27
+ let marker = null;
28
+ try {
29
+ marker = JSON.parse(fs.readFileSync(path.join(rootDir, MARKER), 'utf8'));
30
+ } catch {
31
+ return false;
32
+ }
33
+ if (!Array.isArray(marker?.components) || !marker.components.includes(component)) return false;
34
+ if (runId) {
35
+ try {
36
+ fs.appendFileSync(path.join(runsDir(rootDir), runId, 'ablation.log'), `${JSON.stringify({ component, at: new Date().toISOString() })}\n`);
37
+ } catch {
38
+ // Run directory missing; the bench result still records the ablation.
39
+ }
40
+ }
41
+ return true;
42
+ }
43
+
44
+ function validateComponents(list) {
45
+ const unknown = list.filter((item) => !COMPONENTS.includes(item));
46
+ if (unknown.length) {
47
+ throw Object.assign(new Error(`Unknown --without component(s): ${unknown.join(', ')}. Use: ${COMPONENTS.join(', ')}`), { code: 'BENCH_USAGE' });
48
+ }
49
+ return list;
50
+ }
51
+
52
+ module.exports = { COMPONENTS, MARKER, ablated, requested, validateComponents, writeMarker };
@@ -0,0 +1,63 @@
1
+ 'use strict';
2
+
3
+ // Harness-owned AC status (C-07, learn-harness-engineering L08): an AC that
4
+ // names its own verify command is `proven` only when that exact command
5
+ // passed on the gated diff. Agents cannot set the status; evidence does.
6
+
7
+ const fs = require('fs');
8
+ const path = require('path');
9
+ const { runsDir } = require('./run-state.js');
10
+
11
+ const norm = (text) => String(text || '').trim().replace(/\s+/g, ' ');
12
+
13
+ function recordedCommands(rootDir, runId) {
14
+ const records = [];
15
+ for (const file of ['verification.json', 'final-verification.json']) {
16
+ try {
17
+ const bundle = JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, file), 'utf8'));
18
+ records.push(...(bundle.commands || []));
19
+ } catch {
20
+ // Not recorded yet.
21
+ }
22
+ }
23
+ return records;
24
+ }
25
+
26
+ // [{ id, command, status: 'proven'|'failed'|'pending'|'unbound' }]
27
+ function acStatuses(goal, records) {
28
+ return (goal?.acceptanceCriteria || []).map((criterion) => {
29
+ const command = norm(criterion.verify?.command);
30
+ if (!command) return { id: criterion.id, command: null, status: 'unbound' };
31
+ const matches = (records || []).filter((record) => norm(record.command) === command);
32
+ if (matches.some((record) => record.exitCode === 0 && record.timeoutStatus === 'COMPLETED')) {
33
+ return { id: criterion.id, command, status: 'proven' };
34
+ }
35
+ return { id: criterion.id, command, status: matches.length ? 'failed' : 'pending' };
36
+ });
37
+ }
38
+
39
+ function runAcStatuses(rootDir, run, goal) {
40
+ return acStatuses(goal, recordedCommands(rootDir, run.runId));
41
+ }
42
+
43
+ // C-09 (learn-harness-engineering L09/L11): subjective ACs (UI, copy,
44
+ // design) carry a rubric. The ledger must score every criterion with
45
+ // evidence; an unscored or failed criterion blocks Judge PASS.
46
+ function rubricProblems(goal, ledger) {
47
+ const problems = [];
48
+ const scores = Array.isArray(ledger?.rubricScores) ? ledger.rubricScores : [];
49
+ for (const criterion of goal?.acceptanceCriteria || []) {
50
+ for (const item of Array.isArray(criterion.rubric) ? criterion.rubric : []) {
51
+ const name = String(item.criterion || '').trim();
52
+ const score = scores.find((entry) => entry.ac === criterion.id && String(entry.criterion || '').trim() === name);
53
+ if (!score || typeof score.pass !== 'boolean' || !String(score.evidence || '').trim()) {
54
+ problems.push(`${criterion.id} rubric "${name}" has no score with evidence in the Finding Ledger (rubricScores)`);
55
+ } else if (!score.pass) {
56
+ problems.push(`${criterion.id} rubric "${name}" failed: ${score.evidence}`);
57
+ }
58
+ }
59
+ }
60
+ return problems;
61
+ }
62
+
63
+ module.exports = { acStatuses, recordedCommands, rubricProblems, runAcStatuses };
@@ -0,0 +1,138 @@
1
+ 'use strict';
2
+
3
+ // Baseline verification (C-10, learn-harness-engineering L06 + startup flow):
4
+ // before the Maker edits anything, run the project's own verification plan
5
+ // once on the frozen starting state. A broken command is caught before work
6
+ // starts, and failures that already existed are told apart from regressions.
7
+
8
+ const fs = require('fs');
9
+ const path = require('path');
10
+ const { atomicWriteJson, getCurrentRun, getGitRevision, loadRun, runsDir } = require('./run-state.js');
11
+
12
+ const BASELINE_FILE = 'baseline.json';
13
+ const COMMAND_NOT_FOUND = 127;
14
+
15
+ function baselineError(message, code, fix) {
16
+ return Object.assign(new Error(fix ? `${message} FIX: ${fix}` : message), { code });
17
+ }
18
+
19
+ function resolveRun(rootDir, runId) {
20
+ const run = runId ? loadRun(rootDir, runId) : getCurrentRun(rootDir);
21
+ if (!run) throw baselineError('No current run.', 'NO_CURRENT_RUN', 'pass --run <id>.');
22
+ return run;
23
+ }
24
+
25
+ // Failing test names from common runner output (node:test, TAP, jest, vitest,
26
+ // pytest). Best effort: an empty list means "could not tell", never "none".
27
+ function extractFailingTests(text) {
28
+ const names = new Set();
29
+ const patterns = [
30
+ /^\s*✖\s+(.+?)\s+\(\d+(?:\.\d+)?m?s\)\s*$/gm, // node:test spec reporter
31
+ /^not ok \d+ - (.+)$/gm, // TAP
32
+ /^\s*●\s+(.+?)\s*$/gm, // jest
33
+ /^\s*[×✗]\s+(.+?)(?:\s+\d+m?s)?\s*$/gm, // vitest
34
+ /^FAILED\s+(\S+)/gm // pytest -rf
35
+ ];
36
+ const value = String(text || '');
37
+ for (const pattern of patterns) {
38
+ for (const match of value.matchAll(pattern)) {
39
+ const name = match[1].trim();
40
+ if (name && !/^failing tests:?$/i.test(name)) names.add(name);
41
+ }
42
+ }
43
+ return [...names].sort();
44
+ }
45
+
46
+ const passedRecord = (record) => record.exitCode === 0 && record.timeoutStatus === 'COMPLETED';
47
+
48
+ // Run `plan` ([{ key, layer, command, args, shell }]) serially on the current
49
+ // tree and write runs/<id>/baseline.json. Must run before the Maker gate, so
50
+ // the tree is the frozen starting point.
51
+ async function recordBaseline(rootDir, { runId, plan, timeoutMs, runCommand } = {}) {
52
+ const run = resolveRun(rootDir, runId);
53
+ if (run.artifacts?.diff) {
54
+ throw baselineError(`Run ${run.runId} already has a gated Maker diff; a baseline must describe the tree before the change.`,
55
+ 'BASELINE_TOO_LATE', 'start a new Run, or skip the baseline for this one.');
56
+ }
57
+ if (!Array.isArray(plan) || plan.length === 0) {
58
+ throw baselineError('No baseline commands.', 'NO_BASELINE_COMMANDS',
59
+ 'add build/typecheck/lint/test_unit to .ai-engineering-loop/verification.md, or pass --cmd.');
60
+ }
61
+ const recorder = require('./verification-recorder.js');
62
+ const exec = runCommand || recorder.runCommand;
63
+ const limit = Number.isInteger(timeoutMs) && timeoutMs > 0 ? timeoutMs : recorder.DEFAULT_TIMEOUT_MS;
64
+ const records = [];
65
+ const problems = [];
66
+ for (const item of plan) {
67
+ const result = await exec(item.command, item.args || [], { cwd: rootDir, timeoutMs: limit, shell: Boolean(item.shell) });
68
+ const record = {
69
+ key: item.key || null,
70
+ layer: item.layer || null,
71
+ command: result.command,
72
+ exitCode: result.exitCode,
73
+ timeoutStatus: result.timeoutStatus,
74
+ failingTests: passedRecord(result) ? [] : extractFailingTests(`${result.stdout}\n${result.stderr}`)
75
+ };
76
+ records.push(record);
77
+ if (result.exitCode === COMMAND_NOT_FOUND || result.launchError) {
78
+ problems.push(`${record.command}: command not found (exit ${result.exitCode}). FIX: correct the ${record.key || 'command'} entry in .ai-engineering-loop/verification.md; verification cannot prove anything with it.`);
79
+ break;
80
+ }
81
+ }
82
+ const broken = problems.length > 0;
83
+ const red = records.some((record) => !passedRecord(record));
84
+ if (!broken && red) {
85
+ for (const record of records.filter((r) => !passedRecord(r))) {
86
+ problems.push(`${record.command}: already failing before any change (exit ${record.exitCode}${record.failingTests.length ? `; ${record.failingTests.length} failing test(s)` : ''}). FIX: repair the baseline in a separate Run first, or continue knowing these failures are pre-existing.`);
87
+ }
88
+ }
89
+ const status = broken ? 'BROKEN' : (red ? 'RED' : 'GREEN');
90
+ const document = {
91
+ schemaVersion: 1,
92
+ runId: run.runId,
93
+ gitRevision: getGitRevision(rootDir),
94
+ recordedAt: new Date().toISOString(),
95
+ status,
96
+ commands: records
97
+ };
98
+ const filePath = path.join(runsDir(rootDir), run.runId, BASELINE_FILE);
99
+ atomicWriteJson(filePath, document);
100
+ return {
101
+ runId: run.runId,
102
+ path: path.relative(rootDir, filePath).split(path.sep).join('/'),
103
+ status,
104
+ commands: records,
105
+ problems
106
+ };
107
+ }
108
+
109
+ function loadBaseline(rootDir, runId) {
110
+ try {
111
+ return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, BASELINE_FILE), 'utf8'));
112
+ } catch {
113
+ return null;
114
+ }
115
+ }
116
+
117
+ // Label failing verification records as pre-existing (every failing test was
118
+ // already failing at baseline) or new. Records without a baseline counterpart,
119
+ // or whose failures cannot be named, stay "new": unknown is never excused.
120
+ function classifyAgainstBaseline(baseline, records) {
121
+ const byCommand = new Map((baseline?.commands || []).map((record) => [record.command, record]));
122
+ return (records || []).filter((record) => !passedRecord(record)).map((record) => {
123
+ const before = byCommand.get(record.command);
124
+ const failing = extractFailingTests(`${record.stdout || ''}\n${record.stderr || ''}`);
125
+ const known = new Set(before?.failingTests || []);
126
+ const fresh = failing.filter((name) => !known.has(name));
127
+ const preExisting = Boolean(before && !passedRecord(before) && failing.length > 0 && fresh.length === 0);
128
+ return { command: record.command, preExisting, newFailures: preExisting ? [] : (fresh.length ? fresh : failing) };
129
+ });
130
+ }
131
+
132
+ module.exports = {
133
+ BASELINE_FILE,
134
+ classifyAgainstBaseline,
135
+ extractFailingTests,
136
+ loadBaseline,
137
+ recordBaseline
138
+ };
package/lib/bench.js CHANGED
@@ -817,6 +817,9 @@ function prepareScenarioRepo(repo, scenario, { host, tokenCap, extraEnv, cli, pa
817
817
  const env = extraEnv;
818
818
  const init = spawnNode([cli, 'init'], { cwd: repo, env });
819
819
  if (init.status !== 0) throw benchError(`init failed: ${init.stdout}\n${init.stderr}`, 'BENCH_SETUP');
820
+ const ablation = require('./ablation.js');
821
+ const components = ablation.requested(env || {});
822
+ if (components.length) ablation.writeMarker(repo, components);
820
823
  installProjectHostSkills(repo, host, packageRoot);
821
824
  installLocalCli(repo, cli);
822
825
  if (host.hooks) {
@@ -1085,7 +1088,9 @@ function runBench(options = {}) {
1085
1088
  const selected = selectScenarios(all, options.scenarioIds || []);
1086
1089
  const benchId = options.benchId || newBenchId();
1087
1090
  const startedAt = new Date().toISOString();
1088
- const extraEnv = options.env || {};
1091
+ // C-08 ablation: switch gate components off inside the scenario repos.
1092
+ const without = require('./ablation.js').validateComponents(options.without || []);
1093
+ const extraEnv = { ...(options.env || {}), ...(without.length ? { AEL_ABLATE: without.join(',') } : {}) };
1089
1094
  const cli = options.cli || cliPath(options.packageRoot);
1090
1095
 
1091
1096
  let scenarioScores;
@@ -1151,14 +1156,32 @@ function runBench(options = {}) {
1151
1156
  status,
1152
1157
  unavailableReason,
1153
1158
  repeat,
1159
+ ...(without.length ? { ablation: { without } } : {}),
1154
1160
  summary: summarize(scenarioScores),
1155
1161
  scenarios: scenarioScores,
1156
1162
  instructionViolations
1157
1163
  });
1158
1164
  const directory = writeBenchOutputs(rootDir, benchId, result);
1165
+ if (without.length) appendSimplificationLog(rootDir, { benchId, without, summary: result.summary });
1159
1166
  return { ...result, directory };
1160
1167
  }
1161
1168
 
1169
+ // Simplification log (OpenAI pack QUALITY_SCORE.md): every ablation bench is
1170
+ // recorded; a human fills `decision` (keep | remove | replace) after
1171
+ // comparing it with a baseline bench (`bench compare`).
1172
+ function appendSimplificationLog(rootDir, { benchId, without, summary }) {
1173
+ const filePath = path.join(rootDir, '.ai-engineering-loop', 'simplification-log.json');
1174
+ let log = { schemaVersion: 1, entries: [] };
1175
+ try {
1176
+ log = JSON.parse(fs.readFileSync(filePath, 'utf8'));
1177
+ } catch {
1178
+ // first entry
1179
+ }
1180
+ log.entries = [...(log.entries || []), { at: new Date().toISOString(), benchId, without, complianceRate: summary?.complianceRate ?? null, decision: null }];
1181
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
1182
+ fs.writeFileSync(filePath, `${JSON.stringify(log, null, 2)}\n`);
1183
+ }
1184
+
1162
1185
  function loadBenchResult(rootDir, benchId) {
1163
1186
  const filePath = path.join(benchDir(rootDir, benchId), 'results.json');
1164
1187
  if (!fs.existsSync(filePath)) throw benchError(`No bench result ${benchId}`, 'BENCH_NOT_FOUND');
@@ -0,0 +1,112 @@
1
+ 'use strict';
2
+
3
+ // Evaluator calibration (C-13, learn-harness-engineering L11 + L14 anchors):
4
+ // pin the Devil's Advocate and Judge to reality. Post-delivery outcomes (a
5
+ // human MR comment, a revert, a bug traced to a Run) are compared with that
6
+ // Run's Finding Ledger, so misses and wrong dismissals become visible and
7
+ // can drive prompt changes (tracked by the reviewer prompt hash, C-16).
8
+
9
+ const fs = require('fs');
10
+ const path = require('path');
11
+ const { runsDir, loadRun } = require('./run-state.js');
12
+
13
+ const OUTCOMES_FILE = 'outcomes.jsonl';
14
+ const SOURCES = Object.freeze(['mr-comment', 'revert', 'bug', 'incident']);
15
+
16
+ function calibrationError(message, code) {
17
+ return Object.assign(new Error(message), { code });
18
+ }
19
+
20
+ const fileOf = (location) => String(location || '').replace(/:\d+(?::\d+)?$/, '');
21
+
22
+ function recordOutcome(rootDir, { runId, source, location, summary, evidence = null, by, now = new Date() } = {}) {
23
+ if (!runId) throw calibrationError('outcome record needs --run <id> (the delivered Run the outcome traces to).', 'INVALID_USAGE');
24
+ if (!SOURCES.includes(source)) throw calibrationError(`--source must be one of ${SOURCES.join(', ')}`, 'INVALID_USAGE');
25
+ if (!String(location || '').trim()) throw calibrationError('--location <file[:line]> is required so the outcome can be matched to findings.', 'INVALID_USAGE');
26
+ if (!String(summary || '').trim()) throw calibrationError('--summary <text> is required.', 'INVALID_USAGE');
27
+ if (!String(by || '').trim()) throw calibrationError('--by <name> is required: outcomes are human-reported evidence.', 'INVALID_USAGE');
28
+ loadRun(rootDir, runId);
29
+ const entry = { source, location: String(location).trim(), summary: String(summary).trim(), by: String(by).trim(), at: now.toISOString(), ...(evidence ? { evidence } : {}) };
30
+ fs.appendFileSync(path.join(runsDir(rootDir), runId, OUTCOMES_FILE), `${JSON.stringify(entry)}\n`, { mode: 0o600 });
31
+ return entry;
32
+ }
33
+
34
+ function readJsonLines(filePath) {
35
+ let text = '';
36
+ try {
37
+ text = fs.readFileSync(filePath, 'utf8');
38
+ } catch {
39
+ return [];
40
+ }
41
+ return text.split('\n').filter((line) => line.trim()).flatMap((line) => {
42
+ try {
43
+ return [JSON.parse(line)];
44
+ } catch {
45
+ return [];
46
+ }
47
+ });
48
+ }
49
+
50
+ // Classify one outcome against the Run's ledger findings.
51
+ function classifyOutcome(outcome, findings) {
52
+ const file = fileOf(outcome.location);
53
+ const related = (findings || []).filter((finding) => fileOf(finding.location) === file);
54
+ if (!related.length) return { verdict: 'DA_MISS', findings: [] };
55
+ const dismissed = related.filter((finding) => finding.validity === 'INVALID' || finding.dismissal);
56
+ if (dismissed.length) return { verdict: 'FALSE_DISMISS', findings: dismissed.map((finding) => finding.id) };
57
+ return { verdict: 'CAUGHT_BUT_SHIPPED', findings: related.map((finding) => finding.id) };
58
+ }
59
+
60
+ function calibrate(rootDir) {
61
+ let runs = [];
62
+ try {
63
+ runs = fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
64
+ } catch {
65
+ runs = [];
66
+ }
67
+ const rows = [];
68
+ const byPrompt = {};
69
+ for (const runId of runs) {
70
+ const outcomes = readJsonLines(path.join(runsDir(rootDir), runId, OUTCOMES_FILE));
71
+ if (!outcomes.length) continue;
72
+ let ledger = null;
73
+ try {
74
+ ledger = JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, 'findings.json'), 'utf8'));
75
+ } catch {
76
+ ledger = null;
77
+ }
78
+ const prompt = ledger?.reviewer?.promptSha256 ? ledger.reviewer.promptSha256.slice(0, 12) : 'unknown';
79
+ for (const outcome of outcomes) {
80
+ const result = classifyOutcome(outcome, ledger?.findings);
81
+ rows.push({ runId, ...outcome, ...result, prompt });
82
+ const bucket = byPrompt[prompt] || (byPrompt[prompt] = { outcomes: 0, DA_MISS: 0, FALSE_DISMISS: 0, CAUGHT_BUT_SHIPPED: 0 });
83
+ bucket.outcomes += 1;
84
+ bucket[result.verdict] += 1;
85
+ }
86
+ }
87
+ const count = (verdict) => rows.filter((row) => row.verdict === verdict).length;
88
+ return {
89
+ outcomes: rows.length,
90
+ daMiss: count('DA_MISS'),
91
+ falseDismiss: count('FALSE_DISMISS'),
92
+ caughtButShipped: count('CAUGHT_BUT_SHIPPED'),
93
+ byPrompt,
94
+ rows
95
+ };
96
+ }
97
+
98
+ function formatCalibration(report) {
99
+ const lines = [`Outcomes: ${report.outcomes} (DA missed ${report.daMiss}, wrongly dismissed ${report.falseDismiss}, caught but shipped ${report.caughtButShipped})`];
100
+ for (const row of report.rows.filter((r) => r.verdict !== 'CAUGHT_BUT_SHIPPED')) {
101
+ lines.push(`- ${row.verdict} ${row.runId} ${row.location}: ${row.summary}${row.findings.length ? ` (findings ${row.findings.join(', ')})` : ''}`);
102
+ }
103
+ if (report.daMiss || report.falseDismiss) {
104
+ lines.push('Next: add each miss as a concrete check to agents/shared/devil-advocate.body.md (or a learn promote rule), then compare the next prompt hash here.');
105
+ }
106
+ for (const [prompt, bucket] of Object.entries(report.byPrompt)) {
107
+ lines.push(`- prompt ${prompt}: ${bucket.outcomes} outcome(s), ${bucket.DA_MISS} miss, ${bucket.FALSE_DISMISS} false dismiss`);
108
+ }
109
+ return lines.join('\n');
110
+ }
111
+
112
+ module.exports = { OUTCOMES_FILE, SOURCES, calibrate, classifyOutcome, formatCalibration, recordOutcome };
@@ -70,11 +70,11 @@ function assertNoSymlinkPath(root, targetPath) {
70
70
  }
71
71
  }
72
72
 
73
- function renderClaimedVsRealityTemplate(goal) {
73
+ function renderClaimedVsRealityTemplate(goal, tier = null) {
74
74
  const lines = [
75
75
  '# Claimed vs Reality',
76
76
  '',
77
- '> Generated from the frozen Goal Contract. Fill Claimed and Reality only after implementation and verification. Reality must cite observed command output; this template is not verification evidence.',
77
+ '> Generated from the frozen Goal Contract. Fill Claimed and Reality only after implementation and verification. Reality must cite observed command output; this template is not verification evidence. Layer: L1 static, L2 tests, L3 e2e; "needs L3" rows need L3 or "L3 waived: <reason>".',
78
78
  '',
79
79
  '| AC | Claimed | Reality from command log | Layer |',
80
80
  '|---|---|---|---|'
@@ -87,7 +87,8 @@ function renderClaimedVsRealityTemplate(goal) {
87
87
  evidence ? `Evidence: ${evidence}` : '',
88
88
  failures ? `Failure cases: ${failures}` : ''
89
89
  ].filter(Boolean).join(' — ');
90
- lines.push(`| ${markdownCell(description)} | | | |`);
90
+ const layerHint = criterionNeedsL3(criterion, tier) ? 'needs L3' : '';
91
+ lines.push(`| ${markdownCell(description)} | | | ${layerHint} |`);
91
92
  }
92
93
  return `${lines.join('\n')}\n`;
93
94
  }
@@ -125,7 +126,7 @@ function createClaimedVsRealityTemplate(rootDir, { runId, outputPath } = {}) {
125
126
  throw cause;
126
127
  }
127
128
  try {
128
- fs.writeFileSync(descriptor, renderClaimedVsRealityTemplate(goal));
129
+ fs.writeFileSync(descriptor, renderClaimedVsRealityTemplate(goal, run.tier?.current || null));
129
130
  fs.fsyncSync(descriptor);
130
131
  } finally {
131
132
  fs.closeSync(descriptor);
@@ -156,20 +157,113 @@ function parseClaimedVsReality(markdown) {
156
157
  return { rows };
157
158
  }
158
159
 
159
- function readyForDa(markdown) {
160
+ // Three-layer termination check (#110, learn-harness-engineering lectures 09/10):
161
+ // L1 static (typecheck/lint/build), L2 runtime (unit/related/all tests),
162
+ // L3 end-to-end (e2e or the AC's real user flow). Unit tests isolate and mock
163
+ // by design, so they cannot prove a cross-component AC.
164
+ const LAYER_TOKEN = /^L[123]$/;
165
+ const WAIVER = /^L3\s+waived\s*:\s*(.+)$/i;
166
+ const MIN_WAIVER_REASON = 15;
167
+
168
+ function acIdOf(cell) {
169
+ const match = String(cell || '').match(/\bAC-[0-9]+\b/);
170
+ return match ? match[0] : null;
171
+ }
172
+
173
+ function criterionNeedsL3(criterion, tier) {
174
+ return criterion?.crossComponent === true || tier === 'T2';
175
+ }
176
+
177
+ // Parse a Layer cell such as "L2", "L1+L2+L3", or "L2; L3 waived: <reason>".
178
+ // Returns { layers, waiver, invalid } where invalid lists unknown tokens.
179
+ function parseLayerCell(cell) {
180
+ const layers = new Set();
181
+ const invalid = [];
182
+ let waiver = null;
183
+ const text = String(cell || '').trim();
184
+ if (!text || /^needs L3$/i.test(text)) return { layers, waiver, invalid, empty: true };
185
+ for (const part of text.split(/[;,]/).map((p) => p.trim()).filter(Boolean)) {
186
+ const waived = part.match(WAIVER);
187
+ if (waived) {
188
+ waiver = waived[1].trim();
189
+ continue;
190
+ }
191
+ for (const token of part.split(/[+\s]+/).map((t) => t.trim().toUpperCase()).filter(Boolean)) {
192
+ if (LAYER_TOKEN.test(token)) layers.add(token);
193
+ else invalid.push(token);
194
+ }
195
+ }
196
+ return { layers, waiver, invalid, empty: false };
197
+ }
198
+
199
+ // Agent-oriented check: every problem says what is wrong and how to fix it.
200
+ // options.goal (frozen Goal Contract) and options.tier switch on the L3
201
+ // requirement; without them only the Layer cell syntax is checked, so legacy
202
+ // 3-column files keep working.
203
+ // C-01: a Layer tag must be backed by evidence of that layer. `layers` is
204
+ // the set of layers present in the recorded verification commands (null =
205
+ // unknown, skip the check). L3 may instead cite a manual end-to-end flow.
206
+ const MANUAL_FLOW = /\b(manual(?:ly)?|user flow|clicked|screenshot|playwright|cypress|browser|curl|https?:\/\/)/i;
207
+
208
+ function checkClaimedVsReality(markdown, { goal = null, tier = null, layers = null } = {}) {
160
209
  const { rows } = parseClaimedVsReality(markdown);
161
- if (rows.length === 0) return false;
162
- return rows.every((row) => {
210
+ const problems = [];
211
+ if (rows.length === 0) {
212
+ problems.push('claimed-vs-reality has no AC rows. FIX: scaffold it with `context claimed-vs-reality --run <id>` and fill every row.');
213
+ return { ready: false, problems };
214
+ }
215
+ const hasLayerColumn = /^\s*\|\s*AC\s*\|.*\|\s*Layer\s*\|\s*$/im.test(String(markdown));
216
+ const criteria = new Map((goal?.acceptanceCriteria || []).map((c) => [c.id, c]));
217
+ const effectiveTier = tier || goal?.tier || null;
218
+ for (const row of rows) {
219
+ const label = acIdOf(row.ac) || markdownCell(row.ac).slice(0, 40);
163
220
  const reality = (row.reality || '').trim();
164
- if (!reality) return false;
165
- if (/^seems green$/i.test(reality)) return false;
166
- if (!(row.claimed || '').trim()) return false;
167
- return true;
168
- });
221
+ if (!(row.claimed || '').trim()) {
222
+ problems.push(`${label}: Claimed is empty. FIX: state what the change does for this AC.`);
223
+ }
224
+ if (!reality) {
225
+ problems.push(`${label}: Reality is empty. FIX: paste the command and its observed result (exit code, pass count).`);
226
+ } else if (/^seems green$/i.test(reality)) {
227
+ problems.push(`${label}: Reality "seems green" is not evidence. FIX: cite the command output from this iteration.`);
228
+ }
229
+ const layer = parseLayerCell(row.layer);
230
+ if (layer.invalid.length) {
231
+ problems.push(`${label}: Layer has unknown token(s) ${layer.invalid.join(', ')}. FIX: use L1 (static), L2 (runtime tests), L3 (end-to-end), e.g. "L2+L3".`);
232
+ }
233
+ if (layer.waiver !== null && layer.waiver.length < MIN_WAIVER_REASON) {
234
+ problems.push(`${label}: L3 waiver reason is too short. FIX: write "L3 waived: <why no e2e or manual flow could run>" with a concrete reason.`);
235
+ }
236
+ if (layers) {
237
+ for (const tag of layer.layers) {
238
+ if (layers.has(tag)) continue;
239
+ if (tag === 'L3' && MANUAL_FLOW.test(reality)) continue;
240
+ problems.push(tag === 'L3'
241
+ ? `${label}: Layer says L3 but no e2e command was recorded and Reality cites no manual flow. FIX: run \`verify --cmd "<e2e command>"\`, or describe the real user flow you exercised (steps, URL, screenshot path) in Reality.`
242
+ : `${label}: Layer says ${tag} but no ${tag} command was recorded for this diff. FIX: run the ${tag === 'L1' ? 'typecheck/lint/build' : 'test'} command through \`verify\`, or correct the tag.`);
243
+ }
244
+ }
245
+ const criterion = criteria.get(acIdOf(row.ac));
246
+ const needsL3 = criterion ? criterionNeedsL3(criterion, effectiveTier) : (effectiveTier === 'T2' && Boolean(goal));
247
+ if (needsL3 && !layer.layers.has('L3') && !(layer.waiver && layer.waiver.length >= MIN_WAIVER_REASON)) {
248
+ const why = criterion?.crossComponent === true ? 'it is marked crossComponent' : `tier ${effectiveTier} requires L1-L3`;
249
+ problems.push(hasLayerColumn
250
+ ? `${label}: needs L3 end-to-end evidence because ${why}; unit/related tests mock the boundaries this AC crosses. FIX: run \`e2e\` (or the AC's real user flow), cite it in Reality and tag Layer "L3", or write "L3 waived: <reason>".`
251
+ : `${label}: needs L3 evidence because ${why}, but the file has no Layer column. FIX: add a 4th column "Layer" and tag this row L3 (or "L3 waived: <reason>").`);
252
+ }
253
+ }
254
+ return { ready: problems.length === 0, problems };
255
+ }
256
+
257
+ function readyForDa(markdown, options) {
258
+ return checkClaimedVsReality(markdown, options).ready;
169
259
  }
170
260
 
171
261
  module.exports = {
262
+ checkClaimedVsReality,
172
263
  createClaimedVsRealityTemplate,
264
+ criterionNeedsL3,
265
+ loadFrozenGoal: frozenGoal,
266
+ parseLayerCell,
173
267
  parseClaimedVsReality,
174
268
  readyForDa
175
269
  };
@@ -0,0 +1,85 @@
1
+ 'use strict';
2
+
3
+ // Clean state at delivery (C-06, learn-harness-engineering L12): the next
4
+ // session must start from a tree it can trust. Checked at `gate delivery`.
5
+
6
+ const fs = require('fs');
7
+ const path = require('path');
8
+ const { execFileSync } = require('child_process');
9
+ const { runsDir } = require('./run-state.js');
10
+ const { parseDiff } = require('./verification-integrity.js');
11
+
12
+ const DEBUG_MARKERS = [
13
+ { pattern: /^\s*debugger\s*;?\s*$/, label: 'debugger statement' },
14
+ { pattern: /^(<{7}|={7}|>{7})( |$)/, label: 'merge conflict marker' },
15
+ { pattern: /\bbreakpoint\(\)|\bpdb\.set_trace\(\)|\bbinding\.pry\b/, label: 'interactive breakpoint' }
16
+ ];
17
+ const SECRET_FILE = /(^|\/)\.env(\.(?!example$|sample$|template$)[\w-]+)?$|(^|\/)(id_rsa|id_ed25519)$|\.(pem|p12|pfx|key)$/;
18
+
19
+ function untrackedFiles(rootDir) {
20
+ try {
21
+ return execFileSync('git', ['ls-files', '--others', '--exclude-standard', '-z', '--', '.', ':(exclude).ai-engineering-loop'], {
22
+ cwd: rootDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore']
23
+ }).split('\0').filter(Boolean);
24
+ } catch {
25
+ return [];
26
+ }
27
+ }
28
+
29
+ function templateOnlyArtifacts(rootDir, run) {
30
+ const found = [];
31
+ const dirs = [path.join(runsDir(rootDir), run.runId), path.join(rootDir, '.ai-engineering-loop', 'tasks')];
32
+ for (const dir of dirs) {
33
+ let entries = [];
34
+ try {
35
+ entries = fs.readdirSync(dir).filter((name) => name.endsWith('.json'));
36
+ } catch {
37
+ continue;
38
+ }
39
+ for (const name of entries) {
40
+ try {
41
+ if (JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'))?._templateOnly === true) {
42
+ found.push(path.relative(rootDir, path.join(dir, name)).split(path.sep).join('/'));
43
+ }
44
+ } catch {
45
+ // Not JSON we own; ignore.
46
+ }
47
+ }
48
+ }
49
+ return found;
50
+ }
51
+
52
+ // { errors, warnings }: errors always block delivery; stray untracked files
53
+ // block in UNATTENDED and warn in ASSISTED (people keep scratch files).
54
+ function checkCleanState(rootDir, run, { mode = 'ASSISTED' } = {}) {
55
+ const errors = [];
56
+ const warnings = [];
57
+ let diffText = '';
58
+ try {
59
+ diffText = fs.readFileSync(path.join(rootDir, run.artifacts.diff.path), 'utf8');
60
+ } catch {
61
+ diffText = '';
62
+ }
63
+ const files = parseDiff(diffText);
64
+ for (const file of files) {
65
+ if (!file.deleted && SECRET_FILE.test(file.path)) {
66
+ errors.push(`${file.path}: secret-like file is part of the change. FIX: remove it from the change and add it to .gitignore; ship an .env.example instead.`);
67
+ }
68
+ for (const added of file.added) {
69
+ const hit = DEBUG_MARKERS.find((marker) => marker.pattern.test(added.text));
70
+ if (hit) errors.push(`${file.path}:${added.line}: ${hit.label} left in the change. FIX: remove it, re-run the Maker gate and verify.`);
71
+ }
72
+ }
73
+ for (const artifact of templateOnlyArtifacts(rootDir, run)) {
74
+ errors.push(`${artifact}: still a _templateOnly scaffold. FIX: fill it with observed facts and remove _templateOnly, or delete it.`);
75
+ }
76
+ const inDiff = new Set(files.map((file) => file.path));
77
+ const strays = untrackedFiles(rootDir).filter((file) => !inDiff.has(file));
78
+ if (strays.length) {
79
+ const message = `untracked file(s) not part of the reviewed change: ${strays.slice(0, 10).join(', ')}${strays.length > 10 ? ', ...' : ''}. FIX: delete them, add them to .gitignore, or include them and re-run the Maker gate.`;
80
+ (mode === 'UNATTENDED' ? errors : warnings).push(message);
81
+ }
82
+ return { errors, warnings };
83
+ }
84
+
85
+ module.exports = { checkCleanState, untrackedFiles };