ai-engineering-loop 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.agents/skills/ael/SKILL.md +1 -1
  2. package/.agents/skills/ael/references/loop-commands.md +11 -0
  3. package/.agents/skills/ael/references/loop-goal.md +2 -0
  4. package/.agents/skills/ael/references/loop-maker.md +1 -6
  5. package/.agents/skills/ael/references/loop-report.md +8 -0
  6. package/.agents/skills/ael/references/loop-verify.md +11 -9
  7. package/.claude/agents/devil-advocate.md +6 -1
  8. package/.claude/agents/judge.md +3 -1
  9. package/.grok/agents/devil-advocate.md +6 -1
  10. package/.grok/agents/judge.md +3 -1
  11. package/CHANGELOG.md +16 -0
  12. package/agents/devil-advocate.md +8 -0
  13. package/agents/judge.md +1 -0
  14. package/agents/shared/devil-advocate.body.md +6 -1
  15. package/agents/shared/judge.body.md +3 -1
  16. package/bin/ai-engineering-loop.js +262 -20
  17. package/core/bench.md +15 -0
  18. package/core/goal-contract.md +35 -0
  19. package/core/verification-loop.md +78 -1
  20. package/lib/ablation.js +52 -0
  21. package/lib/ac-status.js +63 -0
  22. package/lib/baseline.js +138 -0
  23. package/lib/bench.js +24 -1
  24. package/lib/calibration.js +112 -0
  25. package/lib/claimed-vs-reality.js +106 -12
  26. package/lib/clean-state.js +85 -0
  27. package/lib/cli-args.js +5 -3
  28. package/lib/debt.js +110 -0
  29. package/lib/features.js +95 -0
  30. package/lib/gates.js +234 -41
  31. package/lib/harness-audit.js +124 -0
  32. package/lib/no-progress.js +6 -1
  33. package/lib/review-ingest.js +54 -2
  34. package/lib/rules.js +112 -0
  35. package/lib/stats.js +90 -0
  36. package/lib/sync-hosts.js +1 -0
  37. package/lib/verification-integrity.js +164 -0
  38. package/lib/verification-lanes.js +64 -0
  39. package/lib/verification-recorder.js +17 -5
  40. package/package.json +1 -1
  41. package/schemas/finding-ledger.schema.json +14 -0
  42. package/schemas/goal-contract.schema.json +31 -0
  43. package/schemas/verdict.schema.json +2 -1
@@ -55,6 +55,49 @@ function extractLedger(text) {
55
55
  return found;
56
56
  }
57
57
 
58
+ // C-16 ("model-visible means logged"): what the reviewer saw. review-pack
59
+ // writes review-inputs.json; ingest copies it into the ledger, and the review
60
+ // gate refuses a ledger whose inputs no longer match the files on disk.
61
+ const INPUTS_FILE = 'review-inputs.json';
62
+ const PROMPT_BODY = path.join(__dirname, '..', 'agents', 'shared', 'devil-advocate.body.md');
63
+
64
+ function fileSha256(filePath) {
65
+ try {
66
+ return sha256(fs.readFileSync(filePath));
67
+ } catch {
68
+ return null;
69
+ }
70
+ }
71
+
72
+ function recordReviewInputs(rootDir, runId, { packPath, diffPath } = {}) {
73
+ const runDir = path.join(runsDir(rootDir), runId);
74
+ const inputs = [packPath, diffPath].filter(Boolean).map((relative) => ({
75
+ path: relative,
76
+ sha256: fileSha256(path.join(rootDir, relative))
77
+ })).filter((item) => item.sha256);
78
+ const document = { schemaVersion: 1, runId, inputs, promptSha256: fileSha256(PROMPT_BODY), recordedAt: new Date().toISOString() };
79
+ fs.writeFileSync(path.join(runDir, INPUTS_FILE), `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
80
+ return document;
81
+ }
82
+
83
+ function loadReviewInputs(rootDir, runId) {
84
+ try {
85
+ return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, INPUTS_FILE), 'utf8'));
86
+ } catch {
87
+ return null;
88
+ }
89
+ }
90
+
91
+ // Errors when a recorded input changed after the reviewer saw it.
92
+ function checkReviewInputs(rootDir, ledger) {
93
+ const errors = [];
94
+ for (const item of ledger?.reviewer?.inputs || []) {
95
+ const now = fileSha256(path.join(rootDir, item.path));
96
+ if (now !== item.sha256) errors.push(`${item.path} changed after the reviewer saw it (sha256 ${String(item.sha256).slice(0, 12)} -> ${now ? now.slice(0, 12) : 'missing'})`);
97
+ }
98
+ return errors;
99
+ }
100
+
58
101
  function ingestReview(rootDir, { runId, childId, file } = {}) {
59
102
  if (!String(childId || '').trim()) throw ingestError('--child-id is required (the reviewer subagent\'s own id)', 'INVALID_USAGE');
60
103
  if (!file) throw ingestError('--file <child-output> is required', 'INVALID_USAGE');
@@ -75,7 +118,15 @@ function ingestReview(rootDir, { runId, childId, file } = {}) {
75
118
  schemaVersion: 1,
76
119
  runId: run.runId,
77
120
  diffHash: run.artifacts?.diff?.sha256,
78
- reviewer: { name: 'devil-advocate', childId: String(childId).trim(), rawSha256: sha256(raw) },
121
+ reviewer: {
122
+ name: 'devil-advocate',
123
+ childId: String(childId).trim(),
124
+ rawSha256: sha256(raw),
125
+ ...(() => {
126
+ const recorded = loadReviewInputs(rootDir, run.runId);
127
+ return recorded ? { inputs: recorded.inputs, promptSha256: recorded.promptSha256 } : {};
128
+ })()
129
+ },
79
130
  findings: ledger.findings
80
131
  };
81
132
  fs.writeFileSync(path.join(runDir, 'findings.json'), `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
@@ -96,6 +147,7 @@ function checkLedgerBinding(rootDir, run, ledger) {
96
147
  return { binding: 'BOUND', errors: [`${RAW_FILE} is missing; re-run review ingest`] };
97
148
  }
98
149
  if (sha256(raw) !== rawSha256) return { binding: 'BOUND', errors: [`${RAW_FILE} does not match reviewer.rawSha256`] };
150
+ errors.push(...checkReviewInputs(rootDir, ledger));
99
151
  const original = extractLedger(raw.toString('utf8'))?.findings || [];
100
152
  const current = new Map((ledger.findings || []).map((finding) => [finding.id, finding]));
101
153
  for (const finding of original) {
@@ -113,4 +165,4 @@ function checkLedgerBinding(rootDir, run, ledger) {
113
165
  return { binding: 'BOUND', errors };
114
166
  }
115
167
 
116
- module.exports = { RAW_FILE, extractLedger, ingestReview, checkLedgerBinding };
168
+ module.exports = { INPUTS_FILE, RAW_FILE, checkReviewInputs, extractLedger, ingestReview, checkLedgerBinding, loadReviewInputs, recordReviewInputs };
package/lib/rules.js ADDED
@@ -0,0 +1,112 @@
1
+ 'use strict';
2
+
3
+ // Review feedback promotion (C-05, learn-harness-engineering L10): a lesson
4
+ // that keeps coming back from review becomes an executable check with an
5
+ // agent-oriented fix message. Rules live in .ai-engineering-loop/rules.json,
6
+ // run as L1 commands in the default verify plan, and the verification gate
7
+ // requires every rule to have passed on the gated diff.
8
+
9
+ const fs = require('fs');
10
+ const path = require('path');
11
+ const { atomicWriteJson } = require('./run-state.js');
12
+
13
+ const RULES_FILE = path.join('.ai-engineering-loop', 'rules.json');
14
+ const PROMOTE_AFTER_RUNS = 3;
15
+
16
+ function rulesError(message, code = 'RULES_FAILED') {
17
+ return Object.assign(new Error(message), { code });
18
+ }
19
+
20
+ function loadRules(rootDir) {
21
+ try {
22
+ const value = JSON.parse(fs.readFileSync(path.join(rootDir, RULES_FILE), 'utf8'));
23
+ return Array.isArray(value.rules) ? value : { schemaVersion: 1, rules: [] };
24
+ } catch {
25
+ return { schemaVersion: 1, rules: [] };
26
+ }
27
+ }
28
+
29
+ function loadLearningEntries(rootDir) {
30
+ try {
31
+ return JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'learnings.json'), 'utf8')).entries || [];
32
+ } catch {
33
+ return [];
34
+ }
35
+ }
36
+
37
+ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, now = new Date() } = {}) {
38
+ if (!String(check || '').trim()) throw rulesError('--check "<command>" is required: the command must exit non-zero on a violation.', 'INVALID_USAGE');
39
+ if (!String(fix || '').trim()) throw rulesError('--fix "<instruction>" is required: tell the agent exactly how to repair a violation.', 'INVALID_USAGE');
40
+ if (!String(by || '').trim()) throw rulesError('--by <name> is required: promoting a rule is a human decision.', 'INVALID_USAGE');
41
+ const learning = loadLearningEntries(rootDir).find((entry) => entry.id === learningId);
42
+ if (!learning) throw rulesError(`No learning ${learningId} in .ai-engineering-loop/learnings.json. Run \`learn\` first and use an id it prints.`, 'NOT_FOUND');
43
+ const document = loadRules(rootDir);
44
+ const id = `R-${learningId.replace(/^L-/, '')}`;
45
+ if (document.rules.some((rule) => rule.id === id)) throw rulesError(`${learningId} is already promoted as ${id}.`, 'ALREADY_PROMOTED');
46
+ const rule = {
47
+ id,
48
+ from: learningId,
49
+ lesson: learning.lesson,
50
+ check: String(check).trim(),
51
+ shell: Boolean(shell),
52
+ why: why ? String(why).trim() : learning.lesson,
53
+ fix: String(fix).trim(),
54
+ layer: 'L1',
55
+ createdBy: String(by).trim(),
56
+ createdAt: now.toISOString()
57
+ };
58
+ atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
59
+ return rule;
60
+ }
61
+
62
+ // Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
63
+ function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
64
+ const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
65
+ return entries
66
+ .filter((entry) => (entry.runIds || []).length >= PROMOTE_AFTER_RUNS && !promoted.has(entry.id))
67
+ .map((entry) => ({ id: entry.id, runs: entry.runIds.length, lesson: entry.lesson }));
68
+ }
69
+
70
+ // Default-plan steps for every rule (L1, shell by default).
71
+ function rulePlan(rootDir) {
72
+ return loadRules(rootDir).rules.map((rule) => ({ key: `rule:${rule.id}`, layer: 'L1', text: rule.check, shell: rule.shell !== false, rule }));
73
+ }
74
+
75
+ // Shell commands are recorded JSON-quoted; compare on the unquoted text.
76
+ function plainCommand(text) {
77
+ const value = String(text || '').trim();
78
+ if (value.startsWith('"') && value.endsWith('"')) {
79
+ try {
80
+ return JSON.parse(value);
81
+ } catch {
82
+ return value;
83
+ }
84
+ }
85
+ return value;
86
+ }
87
+
88
+ function ruleForCommand(rootDir, commandText) {
89
+ const value = plainCommand(commandText);
90
+ return loadRules(rootDir).rules.find((rule) => rule.check === value) || null;
91
+ }
92
+
93
+ // Rules with no passing record in a full-scope bundle.
94
+ function missingRules(rootDir, bundle) {
95
+ const records = bundle?.commands || [];
96
+ if (records.length && records.every((record) => record.scope === 'related')) return [];
97
+ const passing = records.filter((record) => record.exitCode === 0);
98
+ const passed = new Set(passing.map((record) => plainCommand(record.command)));
99
+ const keys = new Set(passing.map((record) => record.key).filter(Boolean));
100
+ return loadRules(rootDir).rules.filter((rule) => !passed.has(rule.check) && !keys.has(`rule:${rule.id}`));
101
+ }
102
+
103
+ module.exports = {
104
+ PROMOTE_AFTER_RUNS,
105
+ RULES_FILE,
106
+ loadRules,
107
+ missingRules,
108
+ promoteLearning,
109
+ promotionCandidates,
110
+ ruleForCommand,
111
+ rulePlan
112
+ };
package/lib/stats.js ADDED
@@ -0,0 +1,90 @@
1
+ 'use strict';
2
+
3
+ // `ael stats` (C-12): where the loop fails, by harness layer, and how often
4
+ // the Maker's green verification was a false victory (the Judge still found
5
+ // a VALID blocking problem). Bottlenecks come from failure records, not
6
+ // guesses (learn-harness-engineering L01/L02).
7
+
8
+ const fs = require('fs');
9
+ const path = require('path');
10
+ const { runsDir } = require('./run-state.js');
11
+ const { readIterations } = require('./no-progress.js');
12
+
13
+ function listRuns(rootDir) {
14
+ let names = [];
15
+ try {
16
+ names = fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
17
+ } catch {
18
+ return [];
19
+ }
20
+ return names.sort();
21
+ }
22
+
23
+ function runTier(rootDir, runId) {
24
+ try {
25
+ return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, 'state.json'), 'utf8')).tier?.current || null;
26
+ } catch {
27
+ return null;
28
+ }
29
+ }
30
+
31
+ function computeStats(rootDir, { last = null } = {}) {
32
+ let runs = listRuns(rootDir);
33
+ if (Number.isInteger(last) && last > 0) runs = runs.slice(-last);
34
+ const causes = {};
35
+ const tiers = {};
36
+ let judged = 0;
37
+ let falseVictories = 0;
38
+ const passIterations = [];
39
+ for (const runId of runs) {
40
+ const entries = readIterations(rootDir, runId);
41
+ if (!entries.length) continue;
42
+ const tier = entries.find((entry) => entry.tier)?.tier || runTier(rootDir, runId) || 'none';
43
+ const bucket = tiers[tier] || (tiers[tier] = { runs: 0, judged: 0, falseVictories: 0 });
44
+ bucket.runs += 1;
45
+ for (const entry of entries) {
46
+ judged += 1;
47
+ bucket.judged += 1;
48
+ if (entry.verdict !== 'PASS' && (entry.blockingCount || (entry.blockingSignatures || []).length) > 0) {
49
+ falseVictories += 1;
50
+ bucket.falseVictories += 1;
51
+ }
52
+ if (entry.verdict !== 'PASS') {
53
+ const cause = entry.cause || 'unclassified';
54
+ causes[cause] = (causes[cause] || 0) + 1;
55
+ }
56
+ }
57
+ const pass = entries.find((entry) => entry.verdict === 'PASS');
58
+ if (pass) passIterations.push(pass.iteration || entries.indexOf(pass) + 1);
59
+ }
60
+ const rate = (n, d) => (d ? Math.round((n / d) * 1000) / 1000 : 0);
61
+ for (const bucket of Object.values(tiers)) bucket.falseVictoryRate = rate(bucket.falseVictories, bucket.judged);
62
+ const topCauses = Object.entries(causes).sort((a, b) => b[1] - a[1]).slice(0, 2).map(([cause, count]) => ({ cause, count }));
63
+ return {
64
+ runs: runs.length,
65
+ judged,
66
+ falseVictories,
67
+ falseVictoryRate: rate(falseVictories, judged),
68
+ causes,
69
+ topCauses,
70
+ bottleneck: topCauses[0]?.cause || null,
71
+ iterationsToPass: passIterations.length ? rate(passIterations.reduce((a, b) => a + b, 0), passIterations.length) : null,
72
+ tiers
73
+ };
74
+ }
75
+
76
+ function formatStats(stats) {
77
+ const lines = [
78
+ `Runs: ${stats.runs}, judged iterations: ${stats.judged}`,
79
+ `False victories: ${stats.falseVictories} (${Math.round(stats.falseVictoryRate * 100)}%) - verification was green but the Judge found a VALID blocking problem`,
80
+ `Iterations to PASS (avg): ${stats.iterationsToPass ?? 'n/a'}`,
81
+ `Failure causes: ${Object.entries(stats.causes).map(([cause, count]) => `${cause} ${count}`).join(', ') || 'none'}`
82
+ ];
83
+ if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
84
+ for (const [tier, bucket] of Object.entries(stats.tiers)) {
85
+ lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
86
+ }
87
+ return lines.join('\n');
88
+ }
89
+
90
+ module.exports = { computeStats, formatStats };
package/lib/sync-hosts.js CHANGED
@@ -129,6 +129,7 @@ const LOOP_REFERENCE_FILES = Object.freeze([
129
129
  'loop-goal.md',
130
130
  'loop-hosts.md',
131
131
  'loop-maker.md',
132
+ 'loop-report.md',
132
133
  'loop-review.md',
133
134
  'loop-risky.md',
134
135
  'loop-verify.md'
@@ -0,0 +1,164 @@
1
+ 'use strict';
2
+
3
+ // Verification integrity (C-11). The cheapest way for an agent to turn a gate
4
+ // green is to weaken the proof: skip a test, delete it, drop assertions, or
5
+ // edit the verification config. This deterministic scan of the gated diff
6
+ // turns each of those into a VALID ledger finding the reviewer cannot drop.
7
+
8
+ const fs = require('fs');
9
+ const path = require('path');
10
+
11
+ const SOURCE = 'verification-integrity';
12
+ const TEST_FILE = /(^|\/)(tests?|__tests__|spec|specs)\/|\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)test_[^/]+\.py$|_test\.(py|go)$|Test\.java$/;
13
+ const SKIP_MARKER = /\b(?:it|test|describe|context)\.(?:skip|only|todo)\s*\(|\bx(?:it|describe|test)\s*\(|\bf(?:it|describe)\s*\(|@Disabled\b|@Ignore\b|pytest\.mark\.(?:skip|xfail)|@unittest\.skip|\bt\.Skip(?:Now)?\(|\{\s*skip\s*:\s*true/;
14
+ const ASSERTION = /\b(?:assert(?:\.\w+)?|expect|should|t\.(?:equal|deepEqual|ok|is|true|false|throws))\s*\(|\bassert\s+\w|\bself\.assert\w*\(/;
15
+ const CONFIG_FILE = /(^|\/)(\.ai-engineering-loop\/verification\.md|(jest|vitest|playwright|cypress|karma)\.config\.[cm]?[jt]s|pytest\.ini|tox\.ini|\.nycrc(\.json)?|\.c8rc(\.json)?|codecov\.ya?ml|\.mocharc\.[a-z]+)$/;
16
+
17
+ function isTestFile(file) {
18
+ return TEST_FILE.test(file);
19
+ }
20
+
21
+ // Split a unified diff into per-file entries with added/removed lines.
22
+ function parseDiff(diffText) {
23
+ const files = [];
24
+ let current = null;
25
+ let newLine = 0;
26
+ for (const line of String(diffText || '').split('\n')) {
27
+ const header = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
28
+ if (header) {
29
+ current = { oldPath: header[1], path: header[2], deleted: false, added: [], removed: [] };
30
+ files.push(current);
31
+ continue;
32
+ }
33
+ if (!current) continue;
34
+ if (line.startsWith('deleted file mode')) current.deleted = true;
35
+ else if (line.startsWith('new file mode')) current.created = true;
36
+ else if (line.startsWith('+++ ') || line.startsWith('--- ')) continue;
37
+ else if (line.startsWith('@@')) {
38
+ const hunk = line.match(/\+(\d+)/);
39
+ newLine = hunk ? Number(hunk[1]) : 0;
40
+ } else if (line.startsWith('+')) {
41
+ current.added.push({ text: line.slice(1), line: newLine });
42
+ newLine += 1;
43
+ } else if (line.startsWith('-')) {
44
+ current.removed.push({ text: line.slice(1) });
45
+ } else {
46
+ newLine += 1;
47
+ }
48
+ }
49
+ return files;
50
+ }
51
+
52
+ function finding(id, location, failureScenario, evidence, concreteAlternativeDiff, severity) {
53
+ return {
54
+ id,
55
+ axis: 'spec',
56
+ location,
57
+ failureScenario,
58
+ evidence,
59
+ severity,
60
+ validity: 'VALID',
61
+ disposition: 'STRONG',
62
+ concreteAlternativeDiff,
63
+ source: SOURCE,
64
+ fix: 'Restore the proof (un-skip, keep the assertion, restore the test or config). A human may dismiss it with dismissal: {by, evidence} when the change is intended.'
65
+ };
66
+ }
67
+
68
+ // `goalText` exempts config files the frozen Goal Contract names explicitly.
69
+ function scanDiff(diffText, { tier = null, goalText = '' } = {}) {
70
+ const severity = tier === 'T2' ? 'BLOCKER' : 'HIGH';
71
+ const files = parseDiff(diffText);
72
+ const findings = [];
73
+ const createdTests = files.filter((file) => file.created && isTestFile(file.path));
74
+ let n = 0;
75
+ const next = () => `INTEGRITY-${++n}`;
76
+
77
+ for (const file of files) {
78
+ if (file.deleted && isTestFile(file.oldPath)) {
79
+ const dir = path.posix.dirname(file.oldPath);
80
+ const replaced = createdTests.some((created) => path.posix.dirname(created.path) === dir);
81
+ if (!replaced) {
82
+ findings.push(finding(next(), file.oldPath,
83
+ `Test file ${file.oldPath} was deleted with no replacement test in ${dir}/; the behaviour it covered is no longer verified`,
84
+ `diff marks ${file.oldPath} as deleted file mode`,
85
+ `Restore ${file.oldPath} (git checkout <base> -- ${file.oldPath}) or add a replacement test in ${dir}/`, severity));
86
+ }
87
+ continue;
88
+ }
89
+ if (isTestFile(file.path)) {
90
+ for (const added of file.added) {
91
+ if (SKIP_MARKER.test(added.text)) {
92
+ findings.push(finding(next(), `${file.path}:${added.line}`,
93
+ `A skip/only/todo marker was added, so tests stop running while the suite still reports green`,
94
+ `added line: ${added.text.trim().slice(0, 160)}`,
95
+ `- ${added.text.trim()}\n+ ${added.text.trim().replace(/\.(skip|only|todo)\s*\(/, '(').replace(/\bx(it|describe|test)\s*\(/, '$1(').replace(/\bf(it|describe)\s*\(/, '$1(')}`,
96
+ severity));
97
+ }
98
+ }
99
+ const removedAsserts = file.removed.filter((line) => ASSERTION.test(line.text));
100
+ const addedAsserts = file.added.filter((line) => ASSERTION.test(line.text));
101
+ if (!file.deleted && removedAsserts.length > addedAsserts.length) {
102
+ findings.push(finding(next(), file.path,
103
+ `${removedAsserts.length - addedAsserts.length} assertion(s) removed from ${file.path} without replacement; the test proves less than before`,
104
+ `removed ${removedAsserts.length} assertion line(s), added ${addedAsserts.length}; first removed: ${removedAsserts[0].text.trim().slice(0, 120)}`,
105
+ `+ ${removedAsserts[0].text.trim()}`, severity));
106
+ }
107
+ }
108
+ if (CONFIG_FILE.test(file.path) && !goalText.includes(file.path)) {
109
+ findings.push(finding(next(), file.path,
110
+ `Verification config ${file.path} changed during the run; the rules that judge this change were rewritten by the change itself`,
111
+ `diff touches ${file.path} (+${file.added.length}/-${file.removed.length} lines); the frozen Goal Contract does not name it`,
112
+ `Revert ${file.path}, or freeze a Goal Contract that names it as in scope`, severity));
113
+ }
114
+ }
115
+ return findings;
116
+ }
117
+
118
+ function scanRun(rootDir, run) {
119
+ const artifact = run.artifacts?.diff;
120
+ if (!artifact?.path) return [];
121
+ let diffText;
122
+ try {
123
+ diffText = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
124
+ } catch {
125
+ return [];
126
+ }
127
+ let goalText = '';
128
+ try {
129
+ goalText = fs.readFileSync(path.join(rootDir, run.artifacts.goalContract.path), 'utf8');
130
+ } catch {
131
+ goalText = '';
132
+ }
133
+ return scanDiff(diffText, { tier: run.tier?.current || null, goalText });
134
+ }
135
+
136
+ // Merge into the Finding Ledger. A finding already present may only be
137
+ // downgraded with a human dismissal ({by, evidence}); anything else throws.
138
+ function mergeIntegrityIntoLedger(ledger, findings) {
139
+ const errors = [];
140
+ const existing = new Map((ledger.findings || []).filter((item) => item.source === SOURCE).map((item) => [item.location, item]));
141
+ const additions = [];
142
+ for (const item of findings) {
143
+ const present = existing.get(item.location);
144
+ if (!present) {
145
+ additions.push(item);
146
+ continue;
147
+ }
148
+ const downgraded = present.validity !== 'VALID' || !['BLOCKER', 'HIGH'].includes(present.severity);
149
+ const dismissal = present.dismissal || {};
150
+ if (downgraded && !(String(dismissal.by || '').trim() && String(dismissal.evidence || '').trim())) {
151
+ errors.push(`${present.id} (${present.location}) was downgraded without a human dismissal. FIX: restore it, or add dismissal: {by: "<human>", evidence: "<why the weaker test is intended>"}.`);
152
+ }
153
+ }
154
+ return { ledger: { ...ledger, findings: [...(ledger.findings || []), ...additions] }, added: additions.length, errors };
155
+ }
156
+
157
+ module.exports = {
158
+ SOURCE,
159
+ isTestFile,
160
+ mergeIntegrityIntoLedger,
161
+ parseDiff,
162
+ scanDiff,
163
+ scanRun
164
+ };
@@ -73,6 +73,63 @@ function relatedCommand(rootDir, run, { split, shell = false } = {}) {
73
73
  return { ...parsed, args, files };
74
74
  }
75
75
 
76
+ // Verification layers (#110): which verification.md key proves which layer.
77
+ const LAYER_OF_KEY = Object.freeze({
78
+ build: 'L1', typecheck: 'L1', lint: 'L1',
79
+ test_unit: 'L2', test_related: 'L2', test_all: 'L2', smoke: 'L2',
80
+ e2e: 'L3'
81
+ });
82
+ // Cheapest-first default plan (templates/repo-config/verification.md
83
+ // "Default order"): L1 static checks, then L2 unit tests.
84
+ const DEFAULT_PLAN_KEYS = Object.freeze(['build', 'typecheck', 'lint', 'test_unit']);
85
+
86
+ function layerOfKey(key) {
87
+ return LAYER_OF_KEY[key] || null;
88
+ }
89
+
90
+ // Ordered [{ key, layer, text }] for the configured default plan. A command
91
+ // marked `# Optional` is still run when present.
92
+ // Commands that rewrite files (`eslint --fix`, `prettier --write`) would
93
+ // change the gated diff mid-verification, so they never run by default.
94
+ function isMutatingCommand(text) {
95
+ return /(^|\s)(--fix|--write|-w)(\s|=|$)/.test(String(text || ''));
96
+ }
97
+
98
+ function defaultCommandPlan(rootDir, keys = DEFAULT_PLAN_KEYS) {
99
+ const commands = verificationCommands(rootDir);
100
+ return keys
101
+ .filter((key) => commands[key] && !isMutatingCommand(commands[key]))
102
+ .map((key) => ({ key, layer: layerOfKey(key), text: commands[key] }));
103
+ }
104
+
105
+ // Layer of a recorded command string, by matching it against verification.md.
106
+ function layerOfCommand(rootDir, commandText) {
107
+ const value = String(commandText || '').trim();
108
+ const commands = verificationCommands(rootDir);
109
+ for (const [key, text] of Object.entries(commands)) {
110
+ const base = text.replace(FILES_TOKEN, '').trim();
111
+ if (value === text || (base && value.startsWith(base))) return layerOfKey(key);
112
+ }
113
+ return null;
114
+ }
115
+
116
+ // C-03: changes that can break startup without breaking unit tests
117
+ // (migrations, env, config, entrypoints, dependency manifests).
118
+ const STARTUP_RISK = /(^|\/)(migrations?|db\/migrate|prisma|alembic)\/|(^|\/)\.env(\.[\w-]+)?$|(^|\/)(config|settings)\/|\.config\.[cm]?[jt]s$|(^|\/)(docker-compose[^/]*\.ya?ml|Dockerfile|Procfile)$|(^|\/)(main|index|server|app)\.[cm]?[jt]s$|(^|\/)(package\.json|requirements\.txt|pyproject\.toml|go\.mod|Cargo\.toml)$/;
119
+
120
+ // Changed paths that warrant the `smoke` check, from the gated diff.
121
+ function startupRiskFiles(rootDir, run) {
122
+ const artifact = run?.artifacts?.diff;
123
+ if (!artifact?.path) return [];
124
+ let diff = '';
125
+ try {
126
+ diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
127
+ } catch {
128
+ return [];
129
+ }
130
+ return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
131
+ }
132
+
76
133
  function readJson(filePath) {
77
134
  try {
78
135
  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
@@ -106,6 +163,13 @@ function finalVerificationStatus(rootDir, run) {
106
163
  }
107
164
 
108
165
  module.exports = {
166
+ DEFAULT_PLAN_KEYS,
167
+ LAYER_OF_KEY,
168
+ defaultCommandPlan,
169
+ isMutatingCommand,
170
+ startupRiskFiles,
171
+ layerOfCommand,
172
+ layerOfKey,
109
173
  FINAL_FILE,
110
174
  FILES_TOKEN,
111
175
  verificationCommands,
@@ -303,7 +303,8 @@ async function recordVerificationCommands(rootDir, {
303
303
  timeoutMs = DEFAULT_TIMEOUT_MS,
304
304
  scope = null,
305
305
  target = 'verification',
306
- fresh = false
306
+ fresh = false,
307
+ stopOnFailure = false
307
308
  } = {}) {
308
309
  if (!Array.isArray(commands) || commands.length === 0) {
309
310
  throw recorderError('Provide at least one verification command.', 'INVALID_COMMAND');
@@ -326,12 +327,21 @@ async function recordVerificationCommands(rootDir, {
326
327
  && (record.scope || 'full') === (scope || 'full'));
327
328
  if (previous) return { ...previous, reused: true };
328
329
  const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell) });
329
- return scope ? { ...result, scope } : result;
330
+ const tagged = { ...result, ...(item.layer ? { layer: item.layer } : {}), ...(item.key ? { key: item.key } : {}) };
331
+ return scope ? { ...tagged, scope } : tagged;
330
332
  };
331
333
  let results;
332
- if (serial) {
334
+ const skipped = [];
335
+ if (serial || stopOnFailure) {
333
336
  results = [];
334
- for (const item of commands) results.push(await exec(item));
337
+ for (const item of commands) {
338
+ // Cheapest-first plan: a failing L1 check stops before slower layers.
339
+ if (stopOnFailure && results.some((result) => !passedRecord(result))) {
340
+ skipped.push(displayCommand(item.command, item.args || []));
341
+ continue;
342
+ }
343
+ results.push(await exec(item));
344
+ }
335
345
  } else {
336
346
  results = await Promise.all(commands.map(exec));
337
347
  }
@@ -342,12 +352,14 @@ async function recordVerificationCommands(rootDir, {
342
352
  runId: run.runId,
343
353
  path: path.relative(rootDir, filePath).split(path.sep).join('/'),
344
354
  commands: results,
355
+ skipped,
345
356
  summary,
346
- passed: results.every(passedRecord)
357
+ passed: results.every(passedRecord) && skipped.length === 0
347
358
  };
348
359
  }
349
360
 
350
361
  module.exports = {
362
+ runCommand,
351
363
  recordVerificationCommands,
352
364
  DEFAULT_TIMEOUT_MS,
353
365
  MAX_CAPTURE_BYTES,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ai-engineering-loop",
3
- "version": "2.6.0",
3
+ "version": "2.7.0",
4
4
  "description": "A visual workspace that helps AI coding agents plan, build, review, and ship reliable software.",
5
5
  "main": "bin/ai-engineering-loop.js",
6
6
  "bin": {
@@ -22,6 +22,20 @@
22
22
  },
23
23
  "additionalProperties": true
24
24
  }
25
+ },
26
+ "rubricScores": {
27
+ "description": "C-09: one score per rubric criterion of a subjective AC",
28
+ "type": "array",
29
+ "items": {
30
+ "type": "object",
31
+ "required": ["ac", "criterion", "pass", "evidence"],
32
+ "properties": {
33
+ "ac": { "type": "string", "pattern": "^AC-[0-9]+$" },
34
+ "criterion": { "type": "string", "minLength": 1 },
35
+ "pass": { "type": "boolean" },
36
+ "evidence": { "type": "string", "minLength": 1 }
37
+ }
38
+ }
25
39
  }
26
40
  },
27
41
  "additionalProperties": true
@@ -52,6 +52,32 @@
52
52
  "type": "string",
53
53
  "minLength": 1
54
54
  }
55
+ },
56
+ "rubric": {
57
+ "description": "Optional (C-09) for subjective ACs: criteria the Devil's Advocate scores with evidence in ledger.rubricScores. Judge PASS needs every criterion scored pass:true.",
58
+ "type": "array",
59
+ "items": {
60
+ "type": "object",
61
+ "required": ["criterion", "pass"],
62
+ "properties": {
63
+ "criterion": { "type": "string", "minLength": 1 },
64
+ "pass": { "type": "string", "minLength": 1, "description": "What counts as passing, e.g. 'contrast >= 4.5:1 (WCAG AA)'" }
65
+ }
66
+ }
67
+ },
68
+ "verify": {
69
+ "description": "Optional (C-07). The command that proves this AC. Gate judge refuses PASS until this exact command passed on the gated diff (verification.json or final-verification.json).",
70
+ "type": "object",
71
+ "required": ["command"],
72
+ "properties": {
73
+ "command": { "type": "string", "minLength": 1 },
74
+ "layer": { "enum": ["L1", "L2", "L3"] }
75
+ },
76
+ "additionalProperties": false
77
+ },
78
+ "crossComponent": {
79
+ "description": "Optional (#110). true when the AC crosses a component boundary; claimed-vs-reality then needs L3 (end-to-end) evidence or an 'L3 waived: <reason>' Layer cell before gate review.",
80
+ "type": "boolean"
55
81
  }
56
82
  },
57
83
  "additionalProperties": true
@@ -104,6 +130,11 @@
104
130
  },
105
131
  "additionalProperties": false
106
132
  },
133
+ "featureId": {
134
+ "description": "Optional (C-15): id of the feature in .ai-engineering-loop/features.json this Goal delivers. Goal freeze marks it active; delivery marks it passing with this Run as evidence.",
135
+ "type": "string",
136
+ "minLength": 1
137
+ },
107
138
  "outOfScope": {
108
139
  "description": "Optional list of things this Goal must not change.",
109
140
  "type": "array",
@@ -11,7 +11,8 @@
11
11
  "reason": { "type": "string", "minLength": 1 },
12
12
  "action": { "type": "string", "minLength": 1 },
13
13
  "modelTier": { "enum": ["CHEAP_MODEL", "STRONG_MODEL", "HUMAN"] },
14
- "humanApproved": { "type": "boolean" }
14
+ "humanApproved": { "type": "boolean" },
15
+ "cause": { "description": "C-12: harness layer a non-PASS verdict points at", "enum": ["task_spec", "context", "environment", "verification", "state", "model"] }
15
16
  },
16
17
  "additionalProperties": true
17
18
  }