ai-engineering-loop 2.6.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/.agents/skills/ael/SKILL.md +1 -1
  2. package/.agents/skills/ael/references/loop-commands.md +13 -1
  3. package/.agents/skills/ael/references/loop-deliver.md +1 -1
  4. package/.agents/skills/ael/references/loop-goal.md +2 -0
  5. package/.agents/skills/ael/references/loop-maker.md +1 -6
  6. package/.agents/skills/ael/references/loop-report.md +8 -0
  7. package/.agents/skills/ael/references/loop-verify.md +11 -9
  8. package/.claude/agents/devil-advocate.md +7 -2
  9. package/.claude/agents/judge.md +3 -1
  10. package/.grok/agents/devil-advocate.md +7 -2
  11. package/.grok/agents/judge.md +3 -1
  12. package/CHANGELOG.md +31 -0
  13. package/agents/devil-advocate.md +8 -0
  14. package/agents/judge.md +1 -0
  15. package/agents/shared/devil-advocate.body.md +7 -2
  16. package/agents/shared/judge.body.md +3 -1
  17. package/bin/ai-engineering-loop.js +326 -21
  18. package/core/bench.md +15 -0
  19. package/core/formal-statechart.md +49 -0
  20. package/core/goal-contract.md +53 -0
  21. package/core/verification-loop.md +106 -1
  22. package/lib/ablation.js +52 -0
  23. package/lib/ac-status.js +95 -0
  24. package/lib/baseline.js +138 -0
  25. package/lib/bench.js +24 -1
  26. package/lib/budget.js +9 -1
  27. package/lib/calibration.js +129 -0
  28. package/lib/claimed-vs-reality.js +106 -12
  29. package/lib/clean-state.js +85 -0
  30. package/lib/cli-args.js +5 -3
  31. package/lib/comment-findings.js +111 -0
  32. package/lib/debt.js +110 -0
  33. package/lib/feature-map.js +57 -0
  34. package/lib/features.js +95 -0
  35. package/lib/flow.js +86 -0
  36. package/lib/gates.js +257 -41
  37. package/lib/harness-audit.js +141 -0
  38. package/lib/learn.js +24 -1
  39. package/lib/no-progress.js +6 -1
  40. package/lib/receipt.js +33 -2
  41. package/lib/reverts.js +77 -0
  42. package/lib/review-ingest.js +54 -2
  43. package/lib/rule-check.js +90 -0
  44. package/lib/rules.js +205 -0
  45. package/lib/stats.js +110 -0
  46. package/lib/sync-hosts.js +1 -0
  47. package/lib/verification-integrity.js +164 -0
  48. package/lib/verification-lanes.js +96 -0
  49. package/lib/verification-recorder.js +20 -8
  50. package/package.json +1 -1
  51. package/schemas/finding-ledger.schema.json +14 -0
  52. package/schemas/goal-contract.schema.json +50 -0
  53. package/schemas/verdict.schema.json +2 -1
package/lib/rules.js ADDED
@@ -0,0 +1,205 @@
1
+ 'use strict';
2
+
3
+ // Review feedback promotion (C-05, learn-harness-engineering L10): a lesson
4
+ // that keeps coming back from review becomes an executable check with an
5
+ // agent-oriented fix message. Rules live in .ai-engineering-loop/rules.json,
6
+ // run as L1 commands in the default verify plan, and the verification gate
7
+ // requires every rule to have passed on the gated diff.
8
+
9
+ const fs = require('fs');
10
+ const path = require('path');
11
+ const { atomicWriteJson } = require('./run-state.js');
12
+
13
+ const RULES_FILE = path.join('.ai-engineering-loop', 'rules.json');
14
+ const PROMOTE_AFTER_RUNS = 3;
15
+
16
+ function rulesError(message, code = 'RULES_FAILED') {
17
+ return Object.assign(new Error(message), { code });
18
+ }
19
+
20
+ function loadRules(rootDir) {
21
+ try {
22
+ const value = JSON.parse(fs.readFileSync(path.join(rootDir, RULES_FILE), 'utf8'));
23
+ return Array.isArray(value.rules) ? value : { schemaVersion: 1, rules: [] };
24
+ } catch {
25
+ return { schemaVersion: 1, rules: [] };
26
+ }
27
+ }
28
+
29
+ function loadLearningEntries(rootDir) {
30
+ try {
31
+ return JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'learnings.json'), 'utf8')).entries || [];
32
+ } catch {
33
+ return [];
34
+ }
35
+ }
36
+
37
+ const SCOPES = Object.freeze(['repo', 'diff']);
38
+
39
+ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, now = new Date() } = {}) {
40
+ if (!SCOPES.includes(scope)) throw rulesError(`--scope must be repo or diff (got ${scope}).`, 'INVALID_USAGE');
41
+ // C-20 built-in kinds: no hand-written check, always judged on added lines.
42
+ let builtin = null;
43
+ if (forbidImport && forbidApi) throw rulesError('Use one of --forbid-import or --forbid-api per rule.', 'INVALID_USAGE');
44
+ if (forbidImport) {
45
+ require('./rule-check.js').parseImportSpec(forbidImport);
46
+ builtin = { kind: 'forbid-import', spec: String(forbidImport).trim() };
47
+ } else if (forbidApi) {
48
+ try {
49
+ new RegExp(forbidApi);
50
+ } catch (cause) {
51
+ throw rulesError(`--forbid-api is not a valid regex: ${cause.message}`, 'INVALID_USAGE');
52
+ }
53
+ builtin = { kind: 'forbid-api', pattern: String(forbidApi), ...(except ? { except: String(except).trim() } : {}) };
54
+ }
55
+ if (builtin) {
56
+ check = `ael rules check R-${learningId.replace(/^L-/, '')}`;
57
+ scope = 'diff';
58
+ shell = true;
59
+ }
60
+ if (!String(check || '').trim()) throw rulesError('--check "<command>" (or --forbid-import / --forbid-api) is required: the check must exit non-zero on a violation.', 'INVALID_USAGE');
61
+ if (!String(fix || '').trim()) throw rulesError('--fix "<instruction>" is required: tell the agent exactly how to repair a violation.', 'INVALID_USAGE');
62
+ if (!String(by || '').trim()) throw rulesError('--by <name> is required: promoting a rule is a human decision.', 'INVALID_USAGE');
63
+ const learning = loadLearningEntries(rootDir).find((entry) => entry.id === learningId);
64
+ if (!learning) throw rulesError(`No learning ${learningId} in .ai-engineering-loop/learnings.json. Run \`learn\` first and use an id it prints.`, 'NOT_FOUND');
65
+ const document = loadRules(rootDir);
66
+ const id = `R-${learningId.replace(/^L-/, '')}`;
67
+ if (document.rules.some((rule) => rule.id === id)) throw rulesError(`${learningId} is already promoted as ${id}.`, 'ALREADY_PROMOTED');
68
+ const rule = {
69
+ id,
70
+ from: learningId,
71
+ lesson: learning.lesson,
72
+ check: String(check).trim(),
73
+ shell: Boolean(shell),
74
+ why: why ? String(why).trim() : learning.lesson,
75
+ fix: String(fix).trim(),
76
+ layer: 'L1',
77
+ scope,
78
+ ...(builtin || {}),
79
+ createdBy: String(by).trim(),
80
+ createdAt: now.toISOString()
81
+ };
82
+ atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
83
+ return rule;
84
+ }
85
+
86
+ // Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
87
+ function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
88
+ const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
89
+ return entries
90
+ .filter((entry) => (entry.runIds || []).length >= PROMOTE_AFTER_RUNS && !promoted.has(entry.id))
91
+ .map((entry) => ({ id: entry.id, runs: entry.runIds.length, lesson: entry.lesson }));
92
+ }
93
+
94
+ // C-17 ratchet: a `diff` rule judges only what this change adds. Its check
95
+ // receives AEL_CHANGED_FILES (one path per line) and AEL_ADDED_LINES_FILE
96
+ // (`path:line:text` per added line, the format of `grep -n`), so
97
+ // `! grep -E "<pattern>" "$AEL_ADDED_LINES_FILE"` flags new occurrences only.
98
+ function writeAddedLines(rootDir, run) {
99
+ const { parseDiff } = require('./verification-integrity.js');
100
+ let diffText = '';
101
+ try {
102
+ diffText = fs.readFileSync(path.join(rootDir, run.artifacts.diff.path), 'utf8');
103
+ } catch {
104
+ diffText = '';
105
+ }
106
+ const files = parseDiff(diffText).filter((file) => !file.deleted);
107
+ const lines = files.flatMap((file) => file.added.map((added) => `${file.path}:${added.line}:${added.text}`));
108
+ const dir = path.join(rootDir, '.ai-engineering-loop', 'runs', run.runId);
109
+ fs.mkdirSync(dir, { recursive: true });
110
+ const addedPath = path.join(dir, 'rule-added-lines.txt');
111
+ fs.writeFileSync(addedPath, lines.length ? `${lines.join('\n')}\n` : '');
112
+ return { AEL_CHANGED_FILES: files.map((file) => file.path).join('\n'), AEL_ADDED_LINES_FILE: addedPath };
113
+ }
114
+
115
+ // Built-in kinds run through this CLI (`rules check <id>`); others run their
116
+ // stored check.
117
+ function commandForRule(rule) {
118
+ return rule.kind ? require('./rule-check.js').builtinCommand(rule) : rule.check;
119
+ }
120
+
121
+ // Default-plan steps for every rule (L1, shell by default). `run` enables
122
+ // the diff env for `scope: "diff"` rules.
123
+ function rulePlan(rootDir, run = null) {
124
+ const rules = loadRules(rootDir).rules;
125
+ const diffEnv = run && rules.some((rule) => rule.scope === 'diff') ? writeAddedLines(rootDir, run) : null;
126
+ return rules.map((rule) => ({
127
+ key: `rule:${rule.id}`,
128
+ layer: 'L1',
129
+ text: commandForRule(rule),
130
+ shell: rule.kind ? true : rule.shell !== false,
131
+ rule,
132
+ ...(rule.scope === 'diff' ? { env: diffEnv || { AEL_CHANGED_FILES: '', AEL_ADDED_LINES_FILE: '' } } : {})
133
+ }));
134
+ }
135
+
136
+ // Legacy backlog of a `diff` rule: run its check against every tracked line
137
+ // (as if all were added). Exit 0 = clean, ready to switch to `repo`.
138
+ function legacyBacklog(rootDir, rule, { maxBytes = 32 * 1024 * 1024 } = {}) {
139
+ const { execFileSync, spawnSync } = require('child_process');
140
+ let dump = '';
141
+ try {
142
+ dump = execFileSync('git', ['grep', '-n', '-I', '-e', '', '--', '.', ':(exclude).ai-engineering-loop'], { cwd: rootDir, encoding: 'utf8', maxBuffer: maxBytes });
143
+ } catch (cause) {
144
+ if (cause.status !== 1) return { rule: rule.id, error: `git grep failed: ${cause.message.split('\n')[0]}` };
145
+ }
146
+ const os = require('os');
147
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-backlog-'));
148
+ const addedPath = path.join(tmp, 'all-lines.txt');
149
+ fs.writeFileSync(addedPath, dump);
150
+ const files = [...new Set(dump.split('\n').filter(Boolean).map((line) => line.split(':')[0]))];
151
+ const result = rule.kind || rule.shell !== false
152
+ ? spawnSync(commandForRule(rule), { cwd: rootDir, shell: true, encoding: 'utf8', env: { ...process.env, AEL_CHANGED_FILES: files.join('\n'), AEL_ADDED_LINES_FILE: addedPath }, maxBuffer: maxBytes })
153
+ : spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: rootDir, encoding: 'utf8', env: { ...process.env, AEL_CHANGED_FILES: files.join('\n'), AEL_ADDED_LINES_FILE: addedPath } });
154
+ fs.rmSync(tmp, { recursive: true, force: true });
155
+ const output = `${result.stdout || ''}${result.stderr || ''}`.split('\n').filter((line) => line.trim());
156
+ return { rule: rule.id, clean: result.status === 0, outputLines: output.length, sample: output.slice(0, 3) };
157
+ }
158
+
159
+ // Shell commands are recorded JSON-quoted; compare on the unquoted text.
160
+ function plainCommand(text) {
161
+ const value = String(text || '').trim();
162
+ if (value.startsWith('"') && value.endsWith('"')) {
163
+ try {
164
+ return JSON.parse(value);
165
+ } catch {
166
+ return value;
167
+ }
168
+ }
169
+ return value;
170
+ }
171
+
172
+ function ruleForCommand(rootDir, commandText) {
173
+ const value = plainCommand(commandText);
174
+ return loadRules(rootDir).rules.find((rule) => rule.check === value || commandForRule(rule) === value) || null;
175
+ }
176
+
177
+ function findRule(rootDir, id) {
178
+ return loadRules(rootDir).rules.find((rule) => rule.id === id) || null;
179
+ }
180
+
181
+ // Rules with no passing record in a full-scope bundle.
182
+ function missingRules(rootDir, bundle) {
183
+ const records = bundle?.commands || [];
184
+ if (records.length && records.every((record) => record.scope === 'related')) return [];
185
+ const passing = records.filter((record) => record.exitCode === 0);
186
+ const passed = new Set(passing.map((record) => plainCommand(record.command)));
187
+ const keys = new Set(passing.map((record) => record.key).filter(Boolean));
188
+ return loadRules(rootDir).rules.filter((rule) => !passed.has(rule.check) && !keys.has(`rule:${rule.id}`));
189
+ }
190
+
191
+ module.exports = {
192
+ SCOPES,
193
+ commandForRule,
194
+ findRule,
195
+ legacyBacklog,
196
+ writeAddedLines,
197
+ PROMOTE_AFTER_RUNS,
198
+ RULES_FILE,
199
+ loadRules,
200
+ missingRules,
201
+ promoteLearning,
202
+ promotionCandidates,
203
+ ruleForCommand,
204
+ rulePlan
205
+ };
package/lib/stats.js ADDED
@@ -0,0 +1,110 @@
1
+ 'use strict';
2
+
3
+ // `ael stats` (C-12): where the loop fails, by harness layer, and how often
4
+ // the Maker's green verification was a false victory (the Judge still found
5
+ // a VALID blocking problem). Bottlenecks come from failure records, not
6
+ // guesses (learn-harness-engineering L01/L02).
7
+
8
+ const fs = require('fs');
9
+ const path = require('path');
10
+ const { runsDir } = require('./run-state.js');
11
+ const { readIterations } = require('./no-progress.js');
12
+
13
+ function listRuns(rootDir) {
14
+ let names = [];
15
+ try {
16
+ names = fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
17
+ } catch {
18
+ return [];
19
+ }
20
+ return names.sort();
21
+ }
22
+
23
+ function runTier(rootDir, runId) {
24
+ try {
25
+ return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, 'state.json'), 'utf8')).tier?.current || null;
26
+ } catch {
27
+ return null;
28
+ }
29
+ }
30
+
31
+ function computeStats(rootDir, { last = null } = {}) {
32
+ let runs = listRuns(rootDir);
33
+ if (Number.isInteger(last) && last > 0) runs = runs.slice(-last);
34
+ const causes = {};
35
+ const tiers = {};
36
+ let judged = 0;
37
+ let falseVictories = 0;
38
+ const passIterations = [];
39
+ for (const runId of runs) {
40
+ const entries = readIterations(rootDir, runId);
41
+ if (!entries.length) continue;
42
+ const tier = entries.find((entry) => entry.tier)?.tier || runTier(rootDir, runId) || 'none';
43
+ const bucket = tiers[tier] || (tiers[tier] = { runs: 0, judged: 0, falseVictories: 0 });
44
+ bucket.runs += 1;
45
+ for (const entry of entries) {
46
+ judged += 1;
47
+ bucket.judged += 1;
48
+ if (entry.verdict !== 'PASS' && (entry.blockingCount || (entry.blockingSignatures || []).length) > 0) {
49
+ falseVictories += 1;
50
+ bucket.falseVictories += 1;
51
+ }
52
+ if (entry.verdict !== 'PASS') {
53
+ const cause = entry.cause || 'unclassified';
54
+ causes[cause] = (causes[cause] || 0) + 1;
55
+ }
56
+ }
57
+ const pass = entries.find((entry) => entry.verdict === 'PASS');
58
+ if (pass) passIterations.push(pass.iteration || entries.indexOf(pass) + 1);
59
+ }
60
+ const rate = (n, d) => (d ? Math.round((n / d) * 1000) / 1000 : 0);
61
+ for (const bucket of Object.values(tiers)) bucket.falseVictoryRate = rate(bucket.falseVictories, bucket.judged);
62
+ const topCauses = Object.entries(causes).sort((a, b) => b[1] - a[1]).slice(0, 2).map(([cause, count]) => ({ cause, count }));
63
+ // C-24: revert rate from git, token share per role from the usage ledger.
64
+ const reverts = require('./reverts.js').detectReverts(rootDir);
65
+ const tokensByRole = {};
66
+ try {
67
+ for (const entry of require('./budget.js').readAllUsage(rootDir)) {
68
+ const role = entry.role || 'unlabeled';
69
+ tokensByRole[role] = (tokensByRole[role] || 0) + entry.totalTokens;
70
+ }
71
+ } catch {
72
+ // Unreadable ledger: report no token split rather than fail stats.
73
+ }
74
+ const labeled = Object.entries(tokensByRole).filter(([role]) => role !== 'unlabeled').reduce((sum, [, n]) => sum + n, 0);
75
+ const checking = ['reviewer', 'judge', 'verification'].reduce((sum, role) => sum + (tokensByRole[role] || 0), 0);
76
+ return {
77
+ revertRate: reverts.rate,
78
+ reverted: reverts.reverted,
79
+ delivered: reverts.delivered,
80
+ tokensByRole,
81
+ verificationTokenShare: labeled ? rate(checking, labeled) : null,
82
+ runs: runs.length,
83
+ judged,
84
+ falseVictories,
85
+ falseVictoryRate: rate(falseVictories, judged),
86
+ causes,
87
+ topCauses,
88
+ bottleneck: topCauses[0]?.cause || null,
89
+ iterationsToPass: passIterations.length ? rate(passIterations.reduce((a, b) => a + b, 0), passIterations.length) : null,
90
+ tiers
91
+ };
92
+ }
93
+
94
+ function formatStats(stats) {
95
+ const lines = [
96
+ `Runs: ${stats.runs}, judged iterations: ${stats.judged}`,
97
+ `False victories: ${stats.falseVictories} (${Math.round(stats.falseVictoryRate * 100)}%) - verification was green but the Judge found a VALID blocking problem`,
98
+ `Iterations to PASS (avg): ${stats.iterationsToPass ?? 'n/a'}`,
99
+ `Failure causes: ${Object.entries(stats.causes).map(([cause, count]) => `${cause} ${count}`).join(', ') || 'none'}`
100
+ ];
101
+ lines.push(`Revert rate: ${stats.reverted.length}/${stats.delivered} delivered Run(s) (${Math.round(stats.revertRate * 1000) / 10}%)${stats.reverted.length ? `: ${stats.reverted.map((item) => item.runId).join(', ')}` : ''}`);
102
+ if (stats.verificationTokenShare != null) lines.push(`Verification token share: ${Math.round(stats.verificationTokenShare * 100)}% (reviewer + judge + verification of role-labelled usage)`);
103
+ if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
104
+ for (const [tier, bucket] of Object.entries(stats.tiers)) {
105
+ lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
106
+ }
107
+ return lines.join('\n');
108
+ }
109
+
110
+ module.exports = { computeStats, formatStats };
package/lib/sync-hosts.js CHANGED
@@ -129,6 +129,7 @@ const LOOP_REFERENCE_FILES = Object.freeze([
129
129
  'loop-goal.md',
130
130
  'loop-hosts.md',
131
131
  'loop-maker.md',
132
+ 'loop-report.md',
132
133
  'loop-review.md',
133
134
  'loop-risky.md',
134
135
  'loop-verify.md'
@@ -0,0 +1,164 @@
1
+ 'use strict';
2
+
3
+ // Verification integrity (C-11). The cheapest way for an agent to turn a gate
4
+ // green is to weaken the proof: skip a test, delete it, drop assertions, or
5
+ // edit the verification config. This deterministic scan of the gated diff
6
+ // turns each of those into a VALID ledger finding the reviewer cannot drop.
7
+
8
+ const fs = require('fs');
9
+ const path = require('path');
10
+
11
+ const SOURCE = 'verification-integrity';
12
+ const TEST_FILE = /(^|\/)(tests?|__tests__|spec|specs)\/|\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)test_[^/]+\.py$|_test\.(py|go)$|Test\.java$/;
13
+ const SKIP_MARKER = /\b(?:it|test|describe|context)\.(?:skip|only|todo)\s*\(|\bx(?:it|describe|test)\s*\(|\bf(?:it|describe)\s*\(|@Disabled\b|@Ignore\b|pytest\.mark\.(?:skip|xfail)|@unittest\.skip|\bt\.Skip(?:Now)?\(|\{\s*skip\s*:\s*true/;
14
+ const ASSERTION = /\b(?:assert(?:\.\w+)?|expect|should|t\.(?:equal|deepEqual|ok|is|true|false|throws))\s*\(|\bassert\s+\w|\bself\.assert\w*\(/;
15
+ const CONFIG_FILE = /(^|\/)(\.ai-engineering-loop\/verification\.md|(jest|vitest|playwright|cypress|karma)\.config\.[cm]?[jt]s|pytest\.ini|tox\.ini|\.nycrc(\.json)?|\.c8rc(\.json)?|codecov\.ya?ml|\.mocharc\.[a-z]+)$/;
16
+
17
+ function isTestFile(file) {
18
+ return TEST_FILE.test(file);
19
+ }
20
+
21
+ // Split a unified diff into per-file entries with added/removed lines.
22
+ function parseDiff(diffText) {
23
+ const files = [];
24
+ let current = null;
25
+ let newLine = 0;
26
+ for (const line of String(diffText || '').split('\n')) {
27
+ const header = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
28
+ if (header) {
29
+ current = { oldPath: header[1], path: header[2], deleted: false, added: [], removed: [] };
30
+ files.push(current);
31
+ continue;
32
+ }
33
+ if (!current) continue;
34
+ if (line.startsWith('deleted file mode')) current.deleted = true;
35
+ else if (line.startsWith('new file mode')) current.created = true;
36
+ else if (line.startsWith('+++ ') || line.startsWith('--- ')) continue;
37
+ else if (line.startsWith('@@')) {
38
+ const hunk = line.match(/\+(\d+)/);
39
+ newLine = hunk ? Number(hunk[1]) : 0;
40
+ } else if (line.startsWith('+')) {
41
+ current.added.push({ text: line.slice(1), line: newLine });
42
+ newLine += 1;
43
+ } else if (line.startsWith('-')) {
44
+ current.removed.push({ text: line.slice(1) });
45
+ } else {
46
+ newLine += 1;
47
+ }
48
+ }
49
+ return files;
50
+ }
51
+
52
+ function finding(id, location, failureScenario, evidence, concreteAlternativeDiff, severity) {
53
+ return {
54
+ id,
55
+ axis: 'spec',
56
+ location,
57
+ failureScenario,
58
+ evidence,
59
+ severity,
60
+ validity: 'VALID',
61
+ disposition: 'STRONG',
62
+ concreteAlternativeDiff,
63
+ source: SOURCE,
64
+ fix: 'Restore the proof (un-skip, keep the assertion, restore the test or config). A human may dismiss it with dismissal: {by, evidence} when the change is intended.'
65
+ };
66
+ }
67
+
68
+ // `goalText` exempts config files the frozen Goal Contract names explicitly.
69
+ function scanDiff(diffText, { tier = null, goalText = '' } = {}) {
70
+ const severity = tier === 'T2' ? 'BLOCKER' : 'HIGH';
71
+ const files = parseDiff(diffText);
72
+ const findings = [];
73
+ const createdTests = files.filter((file) => file.created && isTestFile(file.path));
74
+ let n = 0;
75
+ const next = () => `INTEGRITY-${++n}`;
76
+
77
+ for (const file of files) {
78
+ if (file.deleted && isTestFile(file.oldPath)) {
79
+ const dir = path.posix.dirname(file.oldPath);
80
+ const replaced = createdTests.some((created) => path.posix.dirname(created.path) === dir);
81
+ if (!replaced) {
82
+ findings.push(finding(next(), file.oldPath,
83
+ `Test file ${file.oldPath} was deleted with no replacement test in ${dir}/; the behaviour it covered is no longer verified`,
84
+ `diff marks ${file.oldPath} as deleted file mode`,
85
+ `Restore ${file.oldPath} (git checkout <base> -- ${file.oldPath}) or add a replacement test in ${dir}/`, severity));
86
+ }
87
+ continue;
88
+ }
89
+ if (isTestFile(file.path)) {
90
+ for (const added of file.added) {
91
+ if (SKIP_MARKER.test(added.text)) {
92
+ findings.push(finding(next(), `${file.path}:${added.line}`,
93
+ `A skip/only/todo marker was added, so tests stop running while the suite still reports green`,
94
+ `added line: ${added.text.trim().slice(0, 160)}`,
95
+ `- ${added.text.trim()}\n+ ${added.text.trim().replace(/\.(skip|only|todo)\s*\(/, '(').replace(/\bx(it|describe|test)\s*\(/, '$1(').replace(/\bf(it|describe)\s*\(/, '$1(')}`,
96
+ severity));
97
+ }
98
+ }
99
+ const removedAsserts = file.removed.filter((line) => ASSERTION.test(line.text));
100
+ const addedAsserts = file.added.filter((line) => ASSERTION.test(line.text));
101
+ if (!file.deleted && removedAsserts.length > addedAsserts.length) {
102
+ findings.push(finding(next(), file.path,
103
+ `${removedAsserts.length - addedAsserts.length} assertion(s) removed from ${file.path} without replacement; the test proves less than before`,
104
+ `removed ${removedAsserts.length} assertion line(s), added ${addedAsserts.length}; first removed: ${removedAsserts[0].text.trim().slice(0, 120)}`,
105
+ `+ ${removedAsserts[0].text.trim()}`, severity));
106
+ }
107
+ }
108
+ if (CONFIG_FILE.test(file.path) && !goalText.includes(file.path)) {
109
+ findings.push(finding(next(), file.path,
110
+ `Verification config ${file.path} changed during the run; the rules that judge this change were rewritten by the change itself`,
111
+ `diff touches ${file.path} (+${file.added.length}/-${file.removed.length} lines); the frozen Goal Contract does not name it`,
112
+ `Revert ${file.path}, or freeze a Goal Contract that names it as in scope`, severity));
113
+ }
114
+ }
115
+ return findings;
116
+ }
117
+
118
+ function scanRun(rootDir, run) {
119
+ const artifact = run.artifacts?.diff;
120
+ if (!artifact?.path) return [];
121
+ let diffText;
122
+ try {
123
+ diffText = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
124
+ } catch {
125
+ return [];
126
+ }
127
+ let goalText = '';
128
+ try {
129
+ goalText = fs.readFileSync(path.join(rootDir, run.artifacts.goalContract.path), 'utf8');
130
+ } catch {
131
+ goalText = '';
132
+ }
133
+ return scanDiff(diffText, { tier: run.tier?.current || null, goalText });
134
+ }
135
+
136
+ // Merge into the Finding Ledger. A finding already present may only be
137
+ // downgraded with a human dismissal ({by, evidence}); anything else throws.
138
+ function mergeIntegrityIntoLedger(ledger, findings) {
139
+ const errors = [];
140
+ const existing = new Map((ledger.findings || []).filter((item) => item.source === SOURCE).map((item) => [item.location, item]));
141
+ const additions = [];
142
+ for (const item of findings) {
143
+ const present = existing.get(item.location);
144
+ if (!present) {
145
+ additions.push(item);
146
+ continue;
147
+ }
148
+ const downgraded = present.validity !== 'VALID' || !['BLOCKER', 'HIGH'].includes(present.severity);
149
+ const dismissal = present.dismissal || {};
150
+ if (downgraded && !(String(dismissal.by || '').trim() && String(dismissal.evidence || '').trim())) {
151
+ errors.push(`${present.id} (${present.location}) was downgraded without a human dismissal. FIX: restore it, or add dismissal: {by: "<human>", evidence: "<why the weaker test is intended>"}.`);
152
+ }
153
+ }
154
+ return { ledger: { ...ledger, findings: [...(ledger.findings || []), ...additions] }, added: additions.length, errors };
155
+ }
156
+
157
+ module.exports = {
158
+ SOURCE,
159
+ isTestFile,
160
+ mergeIntegrityIntoLedger,
161
+ parseDiff,
162
+ scanDiff,
163
+ scanRun
164
+ };
@@ -73,6 +73,93 @@ function relatedCommand(rootDir, run, { split, shell = false } = {}) {
73
73
  return { ...parsed, args, files };
74
74
  }
75
75
 
76
+ // Verification catalog (#110, C-23): every kind of proof a project may
77
+ // configure in verification.md, the layer it proves, and whether the default
78
+ // plan runs it. AEL does not ship these tools; it recognises them, orders
79
+ // them, binds their evidence, and lets an AC require one (`verify.key`).
80
+ // Layers: L1 static, L2 runtime, L3 end-to-end, LF formal (model checking,
81
+ // proofs; slow, so never in the default plan).
82
+ const VERIFICATION_CATALOG = Object.freeze({
83
+ build: { layer: 'L1', plan: true, what: 'compiles or bundles' },
84
+ typecheck: { layer: 'L1', plan: true, what: 'static types' },
85
+ lint: { layer: 'L1', plan: true, what: 'lint and static analysis' },
86
+ security: { layer: 'L1', plan: false, what: 'dependency or static security scan' },
87
+ test_unit: { layer: 'L2', plan: true, what: 'unit tests' },
88
+ test_related: { layer: 'L2', plan: false, what: 'tests related to changed files' },
89
+ test_all: { layer: 'L2', plan: false, what: 'full regression suite' },
90
+ property: { layer: 'L2', plan: true, what: 'property-based tests' },
91
+ fuzz: { layer: 'L2', plan: false, what: 'fuzz tests' },
92
+ mutation: { layer: 'L2', plan: false, what: 'mutation tests (do the tests catch broken code?)' },
93
+ coverage: { layer: 'L2', plan: false, what: 'coverage threshold' },
94
+ benchmark: { layer: 'L2', plan: false, what: 'benchmark or performance tests' },
95
+ smoke: { layer: 'L2', plan: false, what: 'start, probe readiness, stop' },
96
+ contract: { layer: 'L3', plan: false, what: 'API or consumer contract tests' },
97
+ e2e: { layer: 'L3', plan: false, what: 'end-to-end user flows' },
98
+ pentest: { layer: 'L3', plan: false, what: 'penetration or DAST scan' },
99
+ formal: { layer: 'LF', plan: false, what: 'formal verification (TLA+, Lean, model checking)' }
100
+ });
101
+ const LAYER_OF_KEY = Object.freeze(Object.fromEntries(Object.entries(VERIFICATION_CATALOG).map(([key, entry]) => [key, entry.layer])));
102
+ // Cheapest-first default plan (templates/repo-config/verification.md
103
+ // "Default order"): L1 static checks, then L2 unit and property tests.
104
+ const DEFAULT_PLAN_KEYS = Object.freeze(Object.keys(VERIFICATION_CATALOG).filter((key) => VERIFICATION_CATALOG[key].plan));
105
+
106
+ function layerOfKey(key) {
107
+ return LAYER_OF_KEY[key] || null;
108
+ }
109
+
110
+ // verification.md key of a recorded command string (null when unmapped).
111
+ function keyOfCommand(rootDir, commandText) {
112
+ const value = String(commandText || '').trim();
113
+ for (const [key, text] of Object.entries(verificationCommands(rootDir))) {
114
+ const base = text.replace(FILES_TOKEN, '').trim();
115
+ if (value === text || (base && value.startsWith(base))) return key;
116
+ }
117
+ return null;
118
+ }
119
+
120
+ // Ordered [{ key, layer, text }] for the configured default plan. A command
121
+ // marked `# Optional` is still run when present.
122
+ // Commands that rewrite files (`eslint --fix`, `prettier --write`) would
123
+ // change the gated diff mid-verification, so they never run by default.
124
+ function isMutatingCommand(text) {
125
+ return /(^|\s)(--fix|--write|-w)(\s|=|$)/.test(String(text || ''));
126
+ }
127
+
128
+ function defaultCommandPlan(rootDir, keys = DEFAULT_PLAN_KEYS) {
129
+ const commands = verificationCommands(rootDir);
130
+ return keys
131
+ .filter((key) => commands[key] && !isMutatingCommand(commands[key]))
132
+ .map((key) => ({ key, layer: layerOfKey(key), text: commands[key] }));
133
+ }
134
+
135
+ // Layer of a recorded command string, by matching it against verification.md.
136
+ function layerOfCommand(rootDir, commandText) {
137
+ const value = String(commandText || '').trim();
138
+ const commands = verificationCommands(rootDir);
139
+ for (const [key, text] of Object.entries(commands)) {
140
+ const base = text.replace(FILES_TOKEN, '').trim();
141
+ if (value === text || (base && value.startsWith(base))) return layerOfKey(key);
142
+ }
143
+ return null;
144
+ }
145
+
146
+ // C-03: changes that can break startup without breaking unit tests
147
+ // (migrations, env, config, entrypoints, dependency manifests).
148
+ const STARTUP_RISK = /(^|\/)(migrations?|db\/migrate|prisma|alembic)\/|(^|\/)\.env(\.[\w-]+)?$|(^|\/)(config|settings)\/|\.config\.[cm]?[jt]s$|(^|\/)(docker-compose[^/]*\.ya?ml|Dockerfile|Procfile)$|(^|\/)(main|index|server|app)\.[cm]?[jt]s$|(^|\/)(package\.json|requirements\.txt|pyproject\.toml|go\.mod|Cargo\.toml)$/;
149
+
150
+ // Changed paths that warrant the `smoke` check, from the gated diff.
151
+ function startupRiskFiles(rootDir, run) {
152
+ const artifact = run?.artifacts?.diff;
153
+ if (!artifact?.path) return [];
154
+ let diff = '';
155
+ try {
156
+ diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
157
+ } catch {
158
+ return [];
159
+ }
160
+ return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
161
+ }
162
+
76
163
  function readJson(filePath) {
77
164
  try {
78
165
  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
@@ -106,6 +193,15 @@ function finalVerificationStatus(rootDir, run) {
106
193
  }
107
194
 
108
195
  module.exports = {
196
+ VERIFICATION_CATALOG,
197
+ keyOfCommand,
198
+ DEFAULT_PLAN_KEYS,
199
+ LAYER_OF_KEY,
200
+ defaultCommandPlan,
201
+ isMutatingCommand,
202
+ startupRiskFiles,
203
+ layerOfCommand,
204
+ layerOfKey,
109
205
  FINAL_FILE,
110
206
  FILES_TOKEN,
111
207
  verificationCommands,
@@ -134,7 +134,7 @@ function displayCommand(command, args) {
134
134
  return redactHomePaths(redactSecrets(rendered).text);
135
135
  }
136
136
 
137
- function runCommand(command, args, { cwd, timeoutMs, shell }) {
137
+ function runCommand(command, args, { cwd, timeoutMs, shell, env: extraEnv = null }) {
138
138
  return new Promise((resolve) => {
139
139
  const started = new Date();
140
140
  let timedOut = false;
@@ -147,7 +147,7 @@ function runCommand(command, args, { cwd, timeoutMs, shell }) {
147
147
  const detached = process.platform !== 'win32';
148
148
  const child = spawn(target.command, target.args, {
149
149
  cwd,
150
- env: process.env,
150
+ env: extraEnv ? { ...process.env, ...extraEnv } : process.env,
151
151
  shell: Boolean(shell),
152
152
  detached,
153
153
  windowsHide: true,
@@ -303,7 +303,8 @@ async function recordVerificationCommands(rootDir, {
303
303
  timeoutMs = DEFAULT_TIMEOUT_MS,
304
304
  scope = null,
305
305
  target = 'verification',
306
- fresh = false
306
+ fresh = false,
307
+ stopOnFailure = false
307
308
  } = {}) {
308
309
  if (!Array.isArray(commands) || commands.length === 0) {
309
310
  throw recorderError('Provide at least one verification command.', 'INVALID_COMMAND');
@@ -325,13 +326,22 @@ async function recordVerificationCommands(rootDir, {
325
326
  const previous = fresh ? null : bundle.commands.find((record) => record.command === display && passedRecord(record)
326
327
  && (record.scope || 'full') === (scope || 'full'));
327
328
  if (previous) return { ...previous, reused: true };
328
- const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell) });
329
- return scope ? { ...result, scope } : result;
329
+ const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell), env: item.env || null });
330
+ const tagged = { ...result, ...(item.layer ? { layer: item.layer } : {}), ...(item.key ? { key: item.key } : {}) };
331
+ return scope ? { ...tagged, scope } : tagged;
330
332
  };
331
333
  let results;
332
- if (serial) {
334
+ const skipped = [];
335
+ if (serial || stopOnFailure) {
333
336
  results = [];
334
- for (const item of commands) results.push(await exec(item));
337
+ for (const item of commands) {
338
+ // Cheapest-first plan: a failing L1 check stops before slower layers.
339
+ if (stopOnFailure && results.some((result) => !passedRecord(result))) {
340
+ skipped.push(displayCommand(item.command, item.args || []));
341
+ continue;
342
+ }
343
+ results.push(await exec(item));
344
+ }
335
345
  } else {
336
346
  results = await Promise.all(commands.map(exec));
337
347
  }
@@ -342,12 +352,14 @@ async function recordVerificationCommands(rootDir, {
342
352
  runId: run.runId,
343
353
  path: path.relative(rootDir, filePath).split(path.sep).join('/'),
344
354
  commands: results,
355
+ skipped,
345
356
  summary,
346
- passed: results.every(passedRecord)
357
+ passed: results.every(passedRecord) && skipped.length === 0
347
358
  };
348
359
  }
349
360
 
350
361
  module.exports = {
362
+ runCommand,
351
363
  recordVerificationCommands,
352
364
  DEFAULT_TIMEOUT_MS,
353
365
  MAX_CAPTURE_BYTES,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ai-engineering-loop",
3
- "version": "2.6.0",
3
+ "version": "2.8.0",
4
4
  "description": "A visual workspace that helps AI coding agents plan, build, review, and ship reliable software.",
5
5
  "main": "bin/ai-engineering-loop.js",
6
6
  "bin": {