ai-engineering-loop 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +13 -1
- package/.agents/skills/ael/references/loop-deliver.md +1 -1
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +11 -9
- package/.claude/agents/devil-advocate.md +7 -2
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +7 -2
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +31 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +7 -2
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +326 -21
- package/core/bench.md +15 -0
- package/core/formal-statechart.md +49 -0
- package/core/goal-contract.md +53 -0
- package/core/verification-loop.md +106 -1
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +95 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/budget.js +9 -1
- package/lib/calibration.js +129 -0
- package/lib/claimed-vs-reality.js +106 -12
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/comment-findings.js +111 -0
- package/lib/debt.js +110 -0
- package/lib/feature-map.js +57 -0
- package/lib/features.js +95 -0
- package/lib/flow.js +86 -0
- package/lib/gates.js +257 -41
- package/lib/harness-audit.js +141 -0
- package/lib/learn.js +24 -1
- package/lib/no-progress.js +6 -1
- package/lib/receipt.js +33 -2
- package/lib/reverts.js +77 -0
- package/lib/review-ingest.js +54 -2
- package/lib/rule-check.js +90 -0
- package/lib/rules.js +205 -0
- package/lib/stats.js +110 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +96 -0
- package/lib/verification-recorder.js +20 -8
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +50 -0
- package/schemas/verdict.schema.json +2 -1
package/lib/rules.js
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Review feedback promotion (C-05, learn-harness-engineering L10): a lesson
|
|
4
|
+
// that keeps coming back from review becomes an executable check with an
|
|
5
|
+
// agent-oriented fix message. Rules live in .ai-engineering-loop/rules.json,
|
|
6
|
+
// run as L1 commands in the default verify plan, and the verification gate
|
|
7
|
+
// requires every rule to have passed on the gated diff.
|
|
8
|
+
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const { atomicWriteJson } = require('./run-state.js');
|
|
12
|
+
|
|
13
|
+
const RULES_FILE = path.join('.ai-engineering-loop', 'rules.json');
|
|
14
|
+
const PROMOTE_AFTER_RUNS = 3;
|
|
15
|
+
|
|
16
|
+
function rulesError(message, code = 'RULES_FAILED') {
|
|
17
|
+
return Object.assign(new Error(message), { code });
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function loadRules(rootDir) {
|
|
21
|
+
try {
|
|
22
|
+
const value = JSON.parse(fs.readFileSync(path.join(rootDir, RULES_FILE), 'utf8'));
|
|
23
|
+
return Array.isArray(value.rules) ? value : { schemaVersion: 1, rules: [] };
|
|
24
|
+
} catch {
|
|
25
|
+
return { schemaVersion: 1, rules: [] };
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function loadLearningEntries(rootDir) {
|
|
30
|
+
try {
|
|
31
|
+
return JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'learnings.json'), 'utf8')).entries || [];
|
|
32
|
+
} catch {
|
|
33
|
+
return [];
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const SCOPES = Object.freeze(['repo', 'diff']);
|
|
38
|
+
|
|
39
|
+
function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, now = new Date() } = {}) {
|
|
40
|
+
if (!SCOPES.includes(scope)) throw rulesError(`--scope must be repo or diff (got ${scope}).`, 'INVALID_USAGE');
|
|
41
|
+
// C-20 built-in kinds: no hand-written check, always judged on added lines.
|
|
42
|
+
let builtin = null;
|
|
43
|
+
if (forbidImport && forbidApi) throw rulesError('Use one of --forbid-import or --forbid-api per rule.', 'INVALID_USAGE');
|
|
44
|
+
if (forbidImport) {
|
|
45
|
+
require('./rule-check.js').parseImportSpec(forbidImport);
|
|
46
|
+
builtin = { kind: 'forbid-import', spec: String(forbidImport).trim() };
|
|
47
|
+
} else if (forbidApi) {
|
|
48
|
+
try {
|
|
49
|
+
new RegExp(forbidApi);
|
|
50
|
+
} catch (cause) {
|
|
51
|
+
throw rulesError(`--forbid-api is not a valid regex: ${cause.message}`, 'INVALID_USAGE');
|
|
52
|
+
}
|
|
53
|
+
builtin = { kind: 'forbid-api', pattern: String(forbidApi), ...(except ? { except: String(except).trim() } : {}) };
|
|
54
|
+
}
|
|
55
|
+
if (builtin) {
|
|
56
|
+
check = `ael rules check R-${learningId.replace(/^L-/, '')}`;
|
|
57
|
+
scope = 'diff';
|
|
58
|
+
shell = true;
|
|
59
|
+
}
|
|
60
|
+
if (!String(check || '').trim()) throw rulesError('--check "<command>" (or --forbid-import / --forbid-api) is required: the check must exit non-zero on a violation.', 'INVALID_USAGE');
|
|
61
|
+
if (!String(fix || '').trim()) throw rulesError('--fix "<instruction>" is required: tell the agent exactly how to repair a violation.', 'INVALID_USAGE');
|
|
62
|
+
if (!String(by || '').trim()) throw rulesError('--by <name> is required: promoting a rule is a human decision.', 'INVALID_USAGE');
|
|
63
|
+
const learning = loadLearningEntries(rootDir).find((entry) => entry.id === learningId);
|
|
64
|
+
if (!learning) throw rulesError(`No learning ${learningId} in .ai-engineering-loop/learnings.json. Run \`learn\` first and use an id it prints.`, 'NOT_FOUND');
|
|
65
|
+
const document = loadRules(rootDir);
|
|
66
|
+
const id = `R-${learningId.replace(/^L-/, '')}`;
|
|
67
|
+
if (document.rules.some((rule) => rule.id === id)) throw rulesError(`${learningId} is already promoted as ${id}.`, 'ALREADY_PROMOTED');
|
|
68
|
+
const rule = {
|
|
69
|
+
id,
|
|
70
|
+
from: learningId,
|
|
71
|
+
lesson: learning.lesson,
|
|
72
|
+
check: String(check).trim(),
|
|
73
|
+
shell: Boolean(shell),
|
|
74
|
+
why: why ? String(why).trim() : learning.lesson,
|
|
75
|
+
fix: String(fix).trim(),
|
|
76
|
+
layer: 'L1',
|
|
77
|
+
scope,
|
|
78
|
+
...(builtin || {}),
|
|
79
|
+
createdBy: String(by).trim(),
|
|
80
|
+
createdAt: now.toISOString()
|
|
81
|
+
};
|
|
82
|
+
atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
|
|
83
|
+
return rule;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
|
|
87
|
+
function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
|
|
88
|
+
const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
|
|
89
|
+
return entries
|
|
90
|
+
.filter((entry) => (entry.runIds || []).length >= PROMOTE_AFTER_RUNS && !promoted.has(entry.id))
|
|
91
|
+
.map((entry) => ({ id: entry.id, runs: entry.runIds.length, lesson: entry.lesson }));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// C-17 ratchet: a `diff` rule judges only what this change adds. Its check
|
|
95
|
+
// receives AEL_CHANGED_FILES (one path per line) and AEL_ADDED_LINES_FILE
|
|
96
|
+
// (`path:line:text` per added line, the format of `grep -n`), so
|
|
97
|
+
// `! grep -E "<pattern>" "$AEL_ADDED_LINES_FILE"` flags new occurrences only.
|
|
98
|
+
function writeAddedLines(rootDir, run) {
|
|
99
|
+
const { parseDiff } = require('./verification-integrity.js');
|
|
100
|
+
let diffText = '';
|
|
101
|
+
try {
|
|
102
|
+
diffText = fs.readFileSync(path.join(rootDir, run.artifacts.diff.path), 'utf8');
|
|
103
|
+
} catch {
|
|
104
|
+
diffText = '';
|
|
105
|
+
}
|
|
106
|
+
const files = parseDiff(diffText).filter((file) => !file.deleted);
|
|
107
|
+
const lines = files.flatMap((file) => file.added.map((added) => `${file.path}:${added.line}:${added.text}`));
|
|
108
|
+
const dir = path.join(rootDir, '.ai-engineering-loop', 'runs', run.runId);
|
|
109
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
110
|
+
const addedPath = path.join(dir, 'rule-added-lines.txt');
|
|
111
|
+
fs.writeFileSync(addedPath, lines.length ? `${lines.join('\n')}\n` : '');
|
|
112
|
+
return { AEL_CHANGED_FILES: files.map((file) => file.path).join('\n'), AEL_ADDED_LINES_FILE: addedPath };
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
// Built-in kinds run through this CLI (`rules check <id>`); others run their
|
|
116
|
+
// stored check.
|
|
117
|
+
function commandForRule(rule) {
|
|
118
|
+
return rule.kind ? require('./rule-check.js').builtinCommand(rule) : rule.check;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
// Default-plan steps for every rule (L1, shell by default). `run` enables
|
|
122
|
+
// the diff env for `scope: "diff"` rules.
|
|
123
|
+
function rulePlan(rootDir, run = null) {
|
|
124
|
+
const rules = loadRules(rootDir).rules;
|
|
125
|
+
const diffEnv = run && rules.some((rule) => rule.scope === 'diff') ? writeAddedLines(rootDir, run) : null;
|
|
126
|
+
return rules.map((rule) => ({
|
|
127
|
+
key: `rule:${rule.id}`,
|
|
128
|
+
layer: 'L1',
|
|
129
|
+
text: commandForRule(rule),
|
|
130
|
+
shell: rule.kind ? true : rule.shell !== false,
|
|
131
|
+
rule,
|
|
132
|
+
...(rule.scope === 'diff' ? { env: diffEnv || { AEL_CHANGED_FILES: '', AEL_ADDED_LINES_FILE: '' } } : {})
|
|
133
|
+
}));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Legacy backlog of a `diff` rule: run its check against every tracked line
|
|
137
|
+
// (as if all were added). Exit 0 = clean, ready to switch to `repo`.
|
|
138
|
+
function legacyBacklog(rootDir, rule, { maxBytes = 32 * 1024 * 1024 } = {}) {
|
|
139
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
140
|
+
let dump = '';
|
|
141
|
+
try {
|
|
142
|
+
dump = execFileSync('git', ['grep', '-n', '-I', '-e', '', '--', '.', ':(exclude).ai-engineering-loop'], { cwd: rootDir, encoding: 'utf8', maxBuffer: maxBytes });
|
|
143
|
+
} catch (cause) {
|
|
144
|
+
if (cause.status !== 1) return { rule: rule.id, error: `git grep failed: ${cause.message.split('\n')[0]}` };
|
|
145
|
+
}
|
|
146
|
+
const os = require('os');
|
|
147
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-backlog-'));
|
|
148
|
+
const addedPath = path.join(tmp, 'all-lines.txt');
|
|
149
|
+
fs.writeFileSync(addedPath, dump);
|
|
150
|
+
const files = [...new Set(dump.split('\n').filter(Boolean).map((line) => line.split(':')[0]))];
|
|
151
|
+
const result = rule.kind || rule.shell !== false
|
|
152
|
+
? spawnSync(commandForRule(rule), { cwd: rootDir, shell: true, encoding: 'utf8', env: { ...process.env, AEL_CHANGED_FILES: files.join('\n'), AEL_ADDED_LINES_FILE: addedPath }, maxBuffer: maxBytes })
|
|
153
|
+
: spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: rootDir, encoding: 'utf8', env: { ...process.env, AEL_CHANGED_FILES: files.join('\n'), AEL_ADDED_LINES_FILE: addedPath } });
|
|
154
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
155
|
+
const output = `${result.stdout || ''}${result.stderr || ''}`.split('\n').filter((line) => line.trim());
|
|
156
|
+
return { rule: rule.id, clean: result.status === 0, outputLines: output.length, sample: output.slice(0, 3) };
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Shell commands are recorded JSON-quoted; compare on the unquoted text.
|
|
160
|
+
function plainCommand(text) {
|
|
161
|
+
const value = String(text || '').trim();
|
|
162
|
+
if (value.startsWith('"') && value.endsWith('"')) {
|
|
163
|
+
try {
|
|
164
|
+
return JSON.parse(value);
|
|
165
|
+
} catch {
|
|
166
|
+
return value;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
return value;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function ruleForCommand(rootDir, commandText) {
|
|
173
|
+
const value = plainCommand(commandText);
|
|
174
|
+
return loadRules(rootDir).rules.find((rule) => rule.check === value || commandForRule(rule) === value) || null;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
function findRule(rootDir, id) {
|
|
178
|
+
return loadRules(rootDir).rules.find((rule) => rule.id === id) || null;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Rules with no passing record in a full-scope bundle.
|
|
182
|
+
function missingRules(rootDir, bundle) {
|
|
183
|
+
const records = bundle?.commands || [];
|
|
184
|
+
if (records.length && records.every((record) => record.scope === 'related')) return [];
|
|
185
|
+
const passing = records.filter((record) => record.exitCode === 0);
|
|
186
|
+
const passed = new Set(passing.map((record) => plainCommand(record.command)));
|
|
187
|
+
const keys = new Set(passing.map((record) => record.key).filter(Boolean));
|
|
188
|
+
return loadRules(rootDir).rules.filter((rule) => !passed.has(rule.check) && !keys.has(`rule:${rule.id}`));
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
module.exports = {
|
|
192
|
+
SCOPES,
|
|
193
|
+
commandForRule,
|
|
194
|
+
findRule,
|
|
195
|
+
legacyBacklog,
|
|
196
|
+
writeAddedLines,
|
|
197
|
+
PROMOTE_AFTER_RUNS,
|
|
198
|
+
RULES_FILE,
|
|
199
|
+
loadRules,
|
|
200
|
+
missingRules,
|
|
201
|
+
promoteLearning,
|
|
202
|
+
promotionCandidates,
|
|
203
|
+
ruleForCommand,
|
|
204
|
+
rulePlan
|
|
205
|
+
};
|
package/lib/stats.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// `ael stats` (C-12): where the loop fails, by harness layer, and how often
|
|
4
|
+
// the Maker's green verification was a false victory (the Judge still found
|
|
5
|
+
// a VALID blocking problem). Bottlenecks come from failure records, not
|
|
6
|
+
// guesses (learn-harness-engineering L01/L02).
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
const { runsDir } = require('./run-state.js');
|
|
11
|
+
const { readIterations } = require('./no-progress.js');
|
|
12
|
+
|
|
13
|
+
function listRuns(rootDir) {
|
|
14
|
+
let names = [];
|
|
15
|
+
try {
|
|
16
|
+
names = fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
|
|
17
|
+
} catch {
|
|
18
|
+
return [];
|
|
19
|
+
}
|
|
20
|
+
return names.sort();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function runTier(rootDir, runId) {
|
|
24
|
+
try {
|
|
25
|
+
return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, 'state.json'), 'utf8')).tier?.current || null;
|
|
26
|
+
} catch {
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function computeStats(rootDir, { last = null } = {}) {
|
|
32
|
+
let runs = listRuns(rootDir);
|
|
33
|
+
if (Number.isInteger(last) && last > 0) runs = runs.slice(-last);
|
|
34
|
+
const causes = {};
|
|
35
|
+
const tiers = {};
|
|
36
|
+
let judged = 0;
|
|
37
|
+
let falseVictories = 0;
|
|
38
|
+
const passIterations = [];
|
|
39
|
+
for (const runId of runs) {
|
|
40
|
+
const entries = readIterations(rootDir, runId);
|
|
41
|
+
if (!entries.length) continue;
|
|
42
|
+
const tier = entries.find((entry) => entry.tier)?.tier || runTier(rootDir, runId) || 'none';
|
|
43
|
+
const bucket = tiers[tier] || (tiers[tier] = { runs: 0, judged: 0, falseVictories: 0 });
|
|
44
|
+
bucket.runs += 1;
|
|
45
|
+
for (const entry of entries) {
|
|
46
|
+
judged += 1;
|
|
47
|
+
bucket.judged += 1;
|
|
48
|
+
if (entry.verdict !== 'PASS' && (entry.blockingCount || (entry.blockingSignatures || []).length) > 0) {
|
|
49
|
+
falseVictories += 1;
|
|
50
|
+
bucket.falseVictories += 1;
|
|
51
|
+
}
|
|
52
|
+
if (entry.verdict !== 'PASS') {
|
|
53
|
+
const cause = entry.cause || 'unclassified';
|
|
54
|
+
causes[cause] = (causes[cause] || 0) + 1;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const pass = entries.find((entry) => entry.verdict === 'PASS');
|
|
58
|
+
if (pass) passIterations.push(pass.iteration || entries.indexOf(pass) + 1);
|
|
59
|
+
}
|
|
60
|
+
const rate = (n, d) => (d ? Math.round((n / d) * 1000) / 1000 : 0);
|
|
61
|
+
for (const bucket of Object.values(tiers)) bucket.falseVictoryRate = rate(bucket.falseVictories, bucket.judged);
|
|
62
|
+
const topCauses = Object.entries(causes).sort((a, b) => b[1] - a[1]).slice(0, 2).map(([cause, count]) => ({ cause, count }));
|
|
63
|
+
// C-24: revert rate from git, token share per role from the usage ledger.
|
|
64
|
+
const reverts = require('./reverts.js').detectReverts(rootDir);
|
|
65
|
+
const tokensByRole = {};
|
|
66
|
+
try {
|
|
67
|
+
for (const entry of require('./budget.js').readAllUsage(rootDir)) {
|
|
68
|
+
const role = entry.role || 'unlabeled';
|
|
69
|
+
tokensByRole[role] = (tokensByRole[role] || 0) + entry.totalTokens;
|
|
70
|
+
}
|
|
71
|
+
} catch {
|
|
72
|
+
// Unreadable ledger: report no token split rather than fail stats.
|
|
73
|
+
}
|
|
74
|
+
const labeled = Object.entries(tokensByRole).filter(([role]) => role !== 'unlabeled').reduce((sum, [, n]) => sum + n, 0);
|
|
75
|
+
const checking = ['reviewer', 'judge', 'verification'].reduce((sum, role) => sum + (tokensByRole[role] || 0), 0);
|
|
76
|
+
return {
|
|
77
|
+
revertRate: reverts.rate,
|
|
78
|
+
reverted: reverts.reverted,
|
|
79
|
+
delivered: reverts.delivered,
|
|
80
|
+
tokensByRole,
|
|
81
|
+
verificationTokenShare: labeled ? rate(checking, labeled) : null,
|
|
82
|
+
runs: runs.length,
|
|
83
|
+
judged,
|
|
84
|
+
falseVictories,
|
|
85
|
+
falseVictoryRate: rate(falseVictories, judged),
|
|
86
|
+
causes,
|
|
87
|
+
topCauses,
|
|
88
|
+
bottleneck: topCauses[0]?.cause || null,
|
|
89
|
+
iterationsToPass: passIterations.length ? rate(passIterations.reduce((a, b) => a + b, 0), passIterations.length) : null,
|
|
90
|
+
tiers
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function formatStats(stats) {
|
|
95
|
+
const lines = [
|
|
96
|
+
`Runs: ${stats.runs}, judged iterations: ${stats.judged}`,
|
|
97
|
+
`False victories: ${stats.falseVictories} (${Math.round(stats.falseVictoryRate * 100)}%) - verification was green but the Judge found a VALID blocking problem`,
|
|
98
|
+
`Iterations to PASS (avg): ${stats.iterationsToPass ?? 'n/a'}`,
|
|
99
|
+
`Failure causes: ${Object.entries(stats.causes).map(([cause, count]) => `${cause} ${count}`).join(', ') || 'none'}`
|
|
100
|
+
];
|
|
101
|
+
lines.push(`Revert rate: ${stats.reverted.length}/${stats.delivered} delivered Run(s) (${Math.round(stats.revertRate * 1000) / 10}%)${stats.reverted.length ? `: ${stats.reverted.map((item) => item.runId).join(', ')}` : ''}`);
|
|
102
|
+
if (stats.verificationTokenShare != null) lines.push(`Verification token share: ${Math.round(stats.verificationTokenShare * 100)}% (reviewer + judge + verification of role-labelled usage)`);
|
|
103
|
+
if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
|
|
104
|
+
for (const [tier, bucket] of Object.entries(stats.tiers)) {
|
|
105
|
+
lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
|
|
106
|
+
}
|
|
107
|
+
return lines.join('\n');
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
module.exports = { computeStats, formatStats };
|
package/lib/sync-hosts.js
CHANGED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Verification integrity (C-11). The cheapest way for an agent to turn a gate
|
|
4
|
+
// green is to weaken the proof: skip a test, delete it, drop assertions, or
|
|
5
|
+
// edit the verification config. This deterministic scan of the gated diff
|
|
6
|
+
// turns each of those into a VALID ledger finding the reviewer cannot drop.
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
|
|
11
|
+
const SOURCE = 'verification-integrity';
|
|
12
|
+
const TEST_FILE = /(^|\/)(tests?|__tests__|spec|specs)\/|\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)test_[^/]+\.py$|_test\.(py|go)$|Test\.java$/;
|
|
13
|
+
const SKIP_MARKER = /\b(?:it|test|describe|context)\.(?:skip|only|todo)\s*\(|\bx(?:it|describe|test)\s*\(|\bf(?:it|describe)\s*\(|@Disabled\b|@Ignore\b|pytest\.mark\.(?:skip|xfail)|@unittest\.skip|\bt\.Skip(?:Now)?\(|\{\s*skip\s*:\s*true/;
|
|
14
|
+
const ASSERTION = /\b(?:assert(?:\.\w+)?|expect|should|t\.(?:equal|deepEqual|ok|is|true|false|throws))\s*\(|\bassert\s+\w|\bself\.assert\w*\(/;
|
|
15
|
+
const CONFIG_FILE = /(^|\/)(\.ai-engineering-loop\/verification\.md|(jest|vitest|playwright|cypress|karma)\.config\.[cm]?[jt]s|pytest\.ini|tox\.ini|\.nycrc(\.json)?|\.c8rc(\.json)?|codecov\.ya?ml|\.mocharc\.[a-z]+)$/;
|
|
16
|
+
|
|
17
|
+
function isTestFile(file) {
|
|
18
|
+
return TEST_FILE.test(file);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Split a unified diff into per-file entries with added/removed lines.
|
|
22
|
+
function parseDiff(diffText) {
|
|
23
|
+
const files = [];
|
|
24
|
+
let current = null;
|
|
25
|
+
let newLine = 0;
|
|
26
|
+
for (const line of String(diffText || '').split('\n')) {
|
|
27
|
+
const header = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
|
|
28
|
+
if (header) {
|
|
29
|
+
current = { oldPath: header[1], path: header[2], deleted: false, added: [], removed: [] };
|
|
30
|
+
files.push(current);
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (!current) continue;
|
|
34
|
+
if (line.startsWith('deleted file mode')) current.deleted = true;
|
|
35
|
+
else if (line.startsWith('new file mode')) current.created = true;
|
|
36
|
+
else if (line.startsWith('+++ ') || line.startsWith('--- ')) continue;
|
|
37
|
+
else if (line.startsWith('@@')) {
|
|
38
|
+
const hunk = line.match(/\+(\d+)/);
|
|
39
|
+
newLine = hunk ? Number(hunk[1]) : 0;
|
|
40
|
+
} else if (line.startsWith('+')) {
|
|
41
|
+
current.added.push({ text: line.slice(1), line: newLine });
|
|
42
|
+
newLine += 1;
|
|
43
|
+
} else if (line.startsWith('-')) {
|
|
44
|
+
current.removed.push({ text: line.slice(1) });
|
|
45
|
+
} else {
|
|
46
|
+
newLine += 1;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return files;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function finding(id, location, failureScenario, evidence, concreteAlternativeDiff, severity) {
|
|
53
|
+
return {
|
|
54
|
+
id,
|
|
55
|
+
axis: 'spec',
|
|
56
|
+
location,
|
|
57
|
+
failureScenario,
|
|
58
|
+
evidence,
|
|
59
|
+
severity,
|
|
60
|
+
validity: 'VALID',
|
|
61
|
+
disposition: 'STRONG',
|
|
62
|
+
concreteAlternativeDiff,
|
|
63
|
+
source: SOURCE,
|
|
64
|
+
fix: 'Restore the proof (un-skip, keep the assertion, restore the test or config). A human may dismiss it with dismissal: {by, evidence} when the change is intended.'
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// `goalText` exempts config files the frozen Goal Contract names explicitly.
|
|
69
|
+
function scanDiff(diffText, { tier = null, goalText = '' } = {}) {
|
|
70
|
+
const severity = tier === 'T2' ? 'BLOCKER' : 'HIGH';
|
|
71
|
+
const files = parseDiff(diffText);
|
|
72
|
+
const findings = [];
|
|
73
|
+
const createdTests = files.filter((file) => file.created && isTestFile(file.path));
|
|
74
|
+
let n = 0;
|
|
75
|
+
const next = () => `INTEGRITY-${++n}`;
|
|
76
|
+
|
|
77
|
+
for (const file of files) {
|
|
78
|
+
if (file.deleted && isTestFile(file.oldPath)) {
|
|
79
|
+
const dir = path.posix.dirname(file.oldPath);
|
|
80
|
+
const replaced = createdTests.some((created) => path.posix.dirname(created.path) === dir);
|
|
81
|
+
if (!replaced) {
|
|
82
|
+
findings.push(finding(next(), file.oldPath,
|
|
83
|
+
`Test file ${file.oldPath} was deleted with no replacement test in ${dir}/; the behaviour it covered is no longer verified`,
|
|
84
|
+
`diff marks ${file.oldPath} as deleted file mode`,
|
|
85
|
+
`Restore ${file.oldPath} (git checkout <base> -- ${file.oldPath}) or add a replacement test in ${dir}/`, severity));
|
|
86
|
+
}
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (isTestFile(file.path)) {
|
|
90
|
+
for (const added of file.added) {
|
|
91
|
+
if (SKIP_MARKER.test(added.text)) {
|
|
92
|
+
findings.push(finding(next(), `${file.path}:${added.line}`,
|
|
93
|
+
`A skip/only/todo marker was added, so tests stop running while the suite still reports green`,
|
|
94
|
+
`added line: ${added.text.trim().slice(0, 160)}`,
|
|
95
|
+
`- ${added.text.trim()}\n+ ${added.text.trim().replace(/\.(skip|only|todo)\s*\(/, '(').replace(/\bx(it|describe|test)\s*\(/, '$1(').replace(/\bf(it|describe)\s*\(/, '$1(')}`,
|
|
96
|
+
severity));
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const removedAsserts = file.removed.filter((line) => ASSERTION.test(line.text));
|
|
100
|
+
const addedAsserts = file.added.filter((line) => ASSERTION.test(line.text));
|
|
101
|
+
if (!file.deleted && removedAsserts.length > addedAsserts.length) {
|
|
102
|
+
findings.push(finding(next(), file.path,
|
|
103
|
+
`${removedAsserts.length - addedAsserts.length} assertion(s) removed from ${file.path} without replacement; the test proves less than before`,
|
|
104
|
+
`removed ${removedAsserts.length} assertion line(s), added ${addedAsserts.length}; first removed: ${removedAsserts[0].text.trim().slice(0, 120)}`,
|
|
105
|
+
`+ ${removedAsserts[0].text.trim()}`, severity));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (CONFIG_FILE.test(file.path) && !goalText.includes(file.path)) {
|
|
109
|
+
findings.push(finding(next(), file.path,
|
|
110
|
+
`Verification config ${file.path} changed during the run; the rules that judge this change were rewritten by the change itself`,
|
|
111
|
+
`diff touches ${file.path} (+${file.added.length}/-${file.removed.length} lines); the frozen Goal Contract does not name it`,
|
|
112
|
+
`Revert ${file.path}, or freeze a Goal Contract that names it as in scope`, severity));
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return findings;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function scanRun(rootDir, run) {
|
|
119
|
+
const artifact = run.artifacts?.diff;
|
|
120
|
+
if (!artifact?.path) return [];
|
|
121
|
+
let diffText;
|
|
122
|
+
try {
|
|
123
|
+
diffText = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
|
|
124
|
+
} catch {
|
|
125
|
+
return [];
|
|
126
|
+
}
|
|
127
|
+
let goalText = '';
|
|
128
|
+
try {
|
|
129
|
+
goalText = fs.readFileSync(path.join(rootDir, run.artifacts.goalContract.path), 'utf8');
|
|
130
|
+
} catch {
|
|
131
|
+
goalText = '';
|
|
132
|
+
}
|
|
133
|
+
return scanDiff(diffText, { tier: run.tier?.current || null, goalText });
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Merge into the Finding Ledger. A finding already present may only be
|
|
137
|
+
// downgraded with a human dismissal ({by, evidence}); anything else throws.
|
|
138
|
+
function mergeIntegrityIntoLedger(ledger, findings) {
|
|
139
|
+
const errors = [];
|
|
140
|
+
const existing = new Map((ledger.findings || []).filter((item) => item.source === SOURCE).map((item) => [item.location, item]));
|
|
141
|
+
const additions = [];
|
|
142
|
+
for (const item of findings) {
|
|
143
|
+
const present = existing.get(item.location);
|
|
144
|
+
if (!present) {
|
|
145
|
+
additions.push(item);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const downgraded = present.validity !== 'VALID' || !['BLOCKER', 'HIGH'].includes(present.severity);
|
|
149
|
+
const dismissal = present.dismissal || {};
|
|
150
|
+
if (downgraded && !(String(dismissal.by || '').trim() && String(dismissal.evidence || '').trim())) {
|
|
151
|
+
errors.push(`${present.id} (${present.location}) was downgraded without a human dismissal. FIX: restore it, or add dismissal: {by: "<human>", evidence: "<why the weaker test is intended>"}.`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return { ledger: { ...ledger, findings: [...(ledger.findings || []), ...additions] }, added: additions.length, errors };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
module.exports = {
|
|
158
|
+
SOURCE,
|
|
159
|
+
isTestFile,
|
|
160
|
+
mergeIntegrityIntoLedger,
|
|
161
|
+
parseDiff,
|
|
162
|
+
scanDiff,
|
|
163
|
+
scanRun
|
|
164
|
+
};
|
|
@@ -73,6 +73,93 @@ function relatedCommand(rootDir, run, { split, shell = false } = {}) {
|
|
|
73
73
|
return { ...parsed, args, files };
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
// Verification catalog (#110, C-23): every kind of proof a project may
|
|
77
|
+
// configure in verification.md, the layer it proves, and whether the default
|
|
78
|
+
// plan runs it. AEL does not ship these tools; it recognises them, orders
|
|
79
|
+
// them, binds their evidence, and lets an AC require one (`verify.key`).
|
|
80
|
+
// Layers: L1 static, L2 runtime, L3 end-to-end, LF formal (model checking,
|
|
81
|
+
// proofs; slow, so never in the default plan).
|
|
82
|
+
const VERIFICATION_CATALOG = Object.freeze({
|
|
83
|
+
build: { layer: 'L1', plan: true, what: 'compiles or bundles' },
|
|
84
|
+
typecheck: { layer: 'L1', plan: true, what: 'static types' },
|
|
85
|
+
lint: { layer: 'L1', plan: true, what: 'lint and static analysis' },
|
|
86
|
+
security: { layer: 'L1', plan: false, what: 'dependency or static security scan' },
|
|
87
|
+
test_unit: { layer: 'L2', plan: true, what: 'unit tests' },
|
|
88
|
+
test_related: { layer: 'L2', plan: false, what: 'tests related to changed files' },
|
|
89
|
+
test_all: { layer: 'L2', plan: false, what: 'full regression suite' },
|
|
90
|
+
property: { layer: 'L2', plan: true, what: 'property-based tests' },
|
|
91
|
+
fuzz: { layer: 'L2', plan: false, what: 'fuzz tests' },
|
|
92
|
+
mutation: { layer: 'L2', plan: false, what: 'mutation tests (do the tests catch broken code?)' },
|
|
93
|
+
coverage: { layer: 'L2', plan: false, what: 'coverage threshold' },
|
|
94
|
+
benchmark: { layer: 'L2', plan: false, what: 'benchmark or performance tests' },
|
|
95
|
+
smoke: { layer: 'L2', plan: false, what: 'start, probe readiness, stop' },
|
|
96
|
+
contract: { layer: 'L3', plan: false, what: 'API or consumer contract tests' },
|
|
97
|
+
e2e: { layer: 'L3', plan: false, what: 'end-to-end user flows' },
|
|
98
|
+
pentest: { layer: 'L3', plan: false, what: 'penetration or DAST scan' },
|
|
99
|
+
formal: { layer: 'LF', plan: false, what: 'formal verification (TLA+, Lean, model checking)' }
|
|
100
|
+
});
|
|
101
|
+
const LAYER_OF_KEY = Object.freeze(Object.fromEntries(Object.entries(VERIFICATION_CATALOG).map(([key, entry]) => [key, entry.layer])));
|
|
102
|
+
// Cheapest-first default plan (templates/repo-config/verification.md
|
|
103
|
+
// "Default order"): L1 static checks, then L2 unit and property tests.
|
|
104
|
+
const DEFAULT_PLAN_KEYS = Object.freeze(Object.keys(VERIFICATION_CATALOG).filter((key) => VERIFICATION_CATALOG[key].plan));
|
|
105
|
+
|
|
106
|
+
function layerOfKey(key) {
|
|
107
|
+
return LAYER_OF_KEY[key] || null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// verification.md key of a recorded command string (null when unmapped).
|
|
111
|
+
function keyOfCommand(rootDir, commandText) {
|
|
112
|
+
const value = String(commandText || '').trim();
|
|
113
|
+
for (const [key, text] of Object.entries(verificationCommands(rootDir))) {
|
|
114
|
+
const base = text.replace(FILES_TOKEN, '').trim();
|
|
115
|
+
if (value === text || (base && value.startsWith(base))) return key;
|
|
116
|
+
}
|
|
117
|
+
return null;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// Ordered [{ key, layer, text }] for the configured default plan. A command
|
|
121
|
+
// marked `# Optional` is still run when present.
|
|
122
|
+
// Commands that rewrite files (`eslint --fix`, `prettier --write`) would
|
|
123
|
+
// change the gated diff mid-verification, so they never run by default.
|
|
124
|
+
function isMutatingCommand(text) {
|
|
125
|
+
return /(^|\s)(--fix|--write|-w)(\s|=|$)/.test(String(text || ''));
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function defaultCommandPlan(rootDir, keys = DEFAULT_PLAN_KEYS) {
|
|
129
|
+
const commands = verificationCommands(rootDir);
|
|
130
|
+
return keys
|
|
131
|
+
.filter((key) => commands[key] && !isMutatingCommand(commands[key]))
|
|
132
|
+
.map((key) => ({ key, layer: layerOfKey(key), text: commands[key] }));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
// Layer of a recorded command string, by matching it against verification.md.
|
|
136
|
+
function layerOfCommand(rootDir, commandText) {
|
|
137
|
+
const value = String(commandText || '').trim();
|
|
138
|
+
const commands = verificationCommands(rootDir);
|
|
139
|
+
for (const [key, text] of Object.entries(commands)) {
|
|
140
|
+
const base = text.replace(FILES_TOKEN, '').trim();
|
|
141
|
+
if (value === text || (base && value.startsWith(base))) return layerOfKey(key);
|
|
142
|
+
}
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// C-03: changes that can break startup without breaking unit tests
|
|
147
|
+
// (migrations, env, config, entrypoints, dependency manifests).
|
|
148
|
+
const STARTUP_RISK = /(^|\/)(migrations?|db\/migrate|prisma|alembic)\/|(^|\/)\.env(\.[\w-]+)?$|(^|\/)(config|settings)\/|\.config\.[cm]?[jt]s$|(^|\/)(docker-compose[^/]*\.ya?ml|Dockerfile|Procfile)$|(^|\/)(main|index|server|app)\.[cm]?[jt]s$|(^|\/)(package\.json|requirements\.txt|pyproject\.toml|go\.mod|Cargo\.toml)$/;
|
|
149
|
+
|
|
150
|
+
// Changed paths that warrant the `smoke` check, from the gated diff.
|
|
151
|
+
function startupRiskFiles(rootDir, run) {
|
|
152
|
+
const artifact = run?.artifacts?.diff;
|
|
153
|
+
if (!artifact?.path) return [];
|
|
154
|
+
let diff = '';
|
|
155
|
+
try {
|
|
156
|
+
diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
|
|
157
|
+
} catch {
|
|
158
|
+
return [];
|
|
159
|
+
}
|
|
160
|
+
return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
|
|
161
|
+
}
|
|
162
|
+
|
|
76
163
|
function readJson(filePath) {
|
|
77
164
|
try {
|
|
78
165
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
@@ -106,6 +193,15 @@ function finalVerificationStatus(rootDir, run) {
|
|
|
106
193
|
}
|
|
107
194
|
|
|
108
195
|
module.exports = {
|
|
196
|
+
VERIFICATION_CATALOG,
|
|
197
|
+
keyOfCommand,
|
|
198
|
+
DEFAULT_PLAN_KEYS,
|
|
199
|
+
LAYER_OF_KEY,
|
|
200
|
+
defaultCommandPlan,
|
|
201
|
+
isMutatingCommand,
|
|
202
|
+
startupRiskFiles,
|
|
203
|
+
layerOfCommand,
|
|
204
|
+
layerOfKey,
|
|
109
205
|
FINAL_FILE,
|
|
110
206
|
FILES_TOKEN,
|
|
111
207
|
verificationCommands,
|
|
@@ -134,7 +134,7 @@ function displayCommand(command, args) {
|
|
|
134
134
|
return redactHomePaths(redactSecrets(rendered).text);
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
-
function runCommand(command, args, { cwd, timeoutMs, shell }) {
|
|
137
|
+
function runCommand(command, args, { cwd, timeoutMs, shell, env: extraEnv = null }) {
|
|
138
138
|
return new Promise((resolve) => {
|
|
139
139
|
const started = new Date();
|
|
140
140
|
let timedOut = false;
|
|
@@ -147,7 +147,7 @@ function runCommand(command, args, { cwd, timeoutMs, shell }) {
|
|
|
147
147
|
const detached = process.platform !== 'win32';
|
|
148
148
|
const child = spawn(target.command, target.args, {
|
|
149
149
|
cwd,
|
|
150
|
-
env: process.env,
|
|
150
|
+
env: extraEnv ? { ...process.env, ...extraEnv } : process.env,
|
|
151
151
|
shell: Boolean(shell),
|
|
152
152
|
detached,
|
|
153
153
|
windowsHide: true,
|
|
@@ -303,7 +303,8 @@ async function recordVerificationCommands(rootDir, {
|
|
|
303
303
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
304
304
|
scope = null,
|
|
305
305
|
target = 'verification',
|
|
306
|
-
fresh = false
|
|
306
|
+
fresh = false,
|
|
307
|
+
stopOnFailure = false
|
|
307
308
|
} = {}) {
|
|
308
309
|
if (!Array.isArray(commands) || commands.length === 0) {
|
|
309
310
|
throw recorderError('Provide at least one verification command.', 'INVALID_COMMAND');
|
|
@@ -325,13 +326,22 @@ async function recordVerificationCommands(rootDir, {
|
|
|
325
326
|
const previous = fresh ? null : bundle.commands.find((record) => record.command === display && passedRecord(record)
|
|
326
327
|
&& (record.scope || 'full') === (scope || 'full'));
|
|
327
328
|
if (previous) return { ...previous, reused: true };
|
|
328
|
-
const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell) });
|
|
329
|
-
|
|
329
|
+
const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell), env: item.env || null });
|
|
330
|
+
const tagged = { ...result, ...(item.layer ? { layer: item.layer } : {}), ...(item.key ? { key: item.key } : {}) };
|
|
331
|
+
return scope ? { ...tagged, scope } : tagged;
|
|
330
332
|
};
|
|
331
333
|
let results;
|
|
332
|
-
|
|
334
|
+
const skipped = [];
|
|
335
|
+
if (serial || stopOnFailure) {
|
|
333
336
|
results = [];
|
|
334
|
-
for (const item of commands)
|
|
337
|
+
for (const item of commands) {
|
|
338
|
+
// Cheapest-first plan: a failing L1 check stops before slower layers.
|
|
339
|
+
if (stopOnFailure && results.some((result) => !passedRecord(result))) {
|
|
340
|
+
skipped.push(displayCommand(item.command, item.args || []));
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
results.push(await exec(item));
|
|
344
|
+
}
|
|
335
345
|
} else {
|
|
336
346
|
results = await Promise.all(commands.map(exec));
|
|
337
347
|
}
|
|
@@ -342,12 +352,14 @@ async function recordVerificationCommands(rootDir, {
|
|
|
342
352
|
runId: run.runId,
|
|
343
353
|
path: path.relative(rootDir, filePath).split(path.sep).join('/'),
|
|
344
354
|
commands: results,
|
|
355
|
+
skipped,
|
|
345
356
|
summary,
|
|
346
|
-
passed: results.every(passedRecord)
|
|
357
|
+
passed: results.every(passedRecord) && skipped.length === 0
|
|
347
358
|
};
|
|
348
359
|
}
|
|
349
360
|
|
|
350
361
|
module.exports = {
|
|
362
|
+
runCommand,
|
|
351
363
|
recordVerificationCommands,
|
|
352
364
|
DEFAULT_TIMEOUT_MS,
|
|
353
365
|
MAX_CAPTURE_BYTES,
|
package/package.json
CHANGED