ai-engineering-loop 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +11 -0
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +21 -11
- package/.claude/agents/devil-advocate.md +6 -1
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +6 -1
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +22 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +6 -1
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +262 -20
- package/core/bench.md +15 -0
- package/core/goal-contract.md +35 -0
- package/core/verification-loop.md +96 -7
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +63 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/calibration.js +112 -0
- package/lib/claimed-vs-reality.js +114 -16
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/debt.js +110 -0
- package/lib/features.js +95 -0
- package/lib/gates.js +234 -41
- package/lib/harness-audit.js +124 -0
- package/lib/no-progress.js +6 -1
- package/lib/review-ingest.js +54 -2
- package/lib/rules.js +112 -0
- package/lib/stats.js +90 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +64 -0
- package/lib/verification-recorder.js +17 -5
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +31 -0
- package/schemas/verdict.schema.json +2 -1
package/lib/review-ingest.js
CHANGED
|
@@ -55,6 +55,49 @@ function extractLedger(text) {
|
|
|
55
55
|
return found;
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
// C-16 ("model-visible means logged"): what the reviewer saw. review-pack
|
|
59
|
+
// writes review-inputs.json; ingest copies it into the ledger, and the review
|
|
60
|
+
// gate refuses a ledger whose inputs no longer match the files on disk.
|
|
61
|
+
const INPUTS_FILE = 'review-inputs.json';
|
|
62
|
+
const PROMPT_BODY = path.join(__dirname, '..', 'agents', 'shared', 'devil-advocate.body.md');
|
|
63
|
+
|
|
64
|
+
function fileSha256(filePath) {
|
|
65
|
+
try {
|
|
66
|
+
return sha256(fs.readFileSync(filePath));
|
|
67
|
+
} catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function recordReviewInputs(rootDir, runId, { packPath, diffPath } = {}) {
|
|
73
|
+
const runDir = path.join(runsDir(rootDir), runId);
|
|
74
|
+
const inputs = [packPath, diffPath].filter(Boolean).map((relative) => ({
|
|
75
|
+
path: relative,
|
|
76
|
+
sha256: fileSha256(path.join(rootDir, relative))
|
|
77
|
+
})).filter((item) => item.sha256);
|
|
78
|
+
const document = { schemaVersion: 1, runId, inputs, promptSha256: fileSha256(PROMPT_BODY), recordedAt: new Date().toISOString() };
|
|
79
|
+
fs.writeFileSync(path.join(runDir, INPUTS_FILE), `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
|
|
80
|
+
return document;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function loadReviewInputs(rootDir, runId) {
|
|
84
|
+
try {
|
|
85
|
+
return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, INPUTS_FILE), 'utf8'));
|
|
86
|
+
} catch {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Errors when a recorded input changed after the reviewer saw it.
|
|
92
|
+
function checkReviewInputs(rootDir, ledger) {
|
|
93
|
+
const errors = [];
|
|
94
|
+
for (const item of ledger?.reviewer?.inputs || []) {
|
|
95
|
+
const now = fileSha256(path.join(rootDir, item.path));
|
|
96
|
+
if (now !== item.sha256) errors.push(`${item.path} changed after the reviewer saw it (sha256 ${String(item.sha256).slice(0, 12)} -> ${now ? now.slice(0, 12) : 'missing'})`);
|
|
97
|
+
}
|
|
98
|
+
return errors;
|
|
99
|
+
}
|
|
100
|
+
|
|
58
101
|
function ingestReview(rootDir, { runId, childId, file } = {}) {
|
|
59
102
|
if (!String(childId || '').trim()) throw ingestError('--child-id is required (the reviewer subagent\'s own id)', 'INVALID_USAGE');
|
|
60
103
|
if (!file) throw ingestError('--file <child-output> is required', 'INVALID_USAGE');
|
|
@@ -75,7 +118,15 @@ function ingestReview(rootDir, { runId, childId, file } = {}) {
|
|
|
75
118
|
schemaVersion: 1,
|
|
76
119
|
runId: run.runId,
|
|
77
120
|
diffHash: run.artifacts?.diff?.sha256,
|
|
78
|
-
reviewer: {
|
|
121
|
+
reviewer: {
|
|
122
|
+
name: 'devil-advocate',
|
|
123
|
+
childId: String(childId).trim(),
|
|
124
|
+
rawSha256: sha256(raw),
|
|
125
|
+
...(() => {
|
|
126
|
+
const recorded = loadReviewInputs(rootDir, run.runId);
|
|
127
|
+
return recorded ? { inputs: recorded.inputs, promptSha256: recorded.promptSha256 } : {};
|
|
128
|
+
})()
|
|
129
|
+
},
|
|
79
130
|
findings: ledger.findings
|
|
80
131
|
};
|
|
81
132
|
fs.writeFileSync(path.join(runDir, 'findings.json'), `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
|
|
@@ -96,6 +147,7 @@ function checkLedgerBinding(rootDir, run, ledger) {
|
|
|
96
147
|
return { binding: 'BOUND', errors: [`${RAW_FILE} is missing; re-run review ingest`] };
|
|
97
148
|
}
|
|
98
149
|
if (sha256(raw) !== rawSha256) return { binding: 'BOUND', errors: [`${RAW_FILE} does not match reviewer.rawSha256`] };
|
|
150
|
+
errors.push(...checkReviewInputs(rootDir, ledger));
|
|
99
151
|
const original = extractLedger(raw.toString('utf8'))?.findings || [];
|
|
100
152
|
const current = new Map((ledger.findings || []).map((finding) => [finding.id, finding]));
|
|
101
153
|
for (const finding of original) {
|
|
@@ -113,4 +165,4 @@ function checkLedgerBinding(rootDir, run, ledger) {
|
|
|
113
165
|
return { binding: 'BOUND', errors };
|
|
114
166
|
}
|
|
115
167
|
|
|
116
|
-
module.exports = { RAW_FILE, extractLedger, ingestReview, checkLedgerBinding };
|
|
168
|
+
module.exports = { INPUTS_FILE, RAW_FILE, checkReviewInputs, extractLedger, ingestReview, checkLedgerBinding, loadReviewInputs, recordReviewInputs };
|
package/lib/rules.js
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Review feedback promotion (C-05, learn-harness-engineering L10): a lesson
|
|
4
|
+
// that keeps coming back from review becomes an executable check with an
|
|
5
|
+
// agent-oriented fix message. Rules live in .ai-engineering-loop/rules.json,
|
|
6
|
+
// run as L1 commands in the default verify plan, and the verification gate
|
|
7
|
+
// requires every rule to have passed on the gated diff.
|
|
8
|
+
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const { atomicWriteJson } = require('./run-state.js');
|
|
12
|
+
|
|
13
|
+
const RULES_FILE = path.join('.ai-engineering-loop', 'rules.json');
|
|
14
|
+
const PROMOTE_AFTER_RUNS = 3;
|
|
15
|
+
|
|
16
|
+
function rulesError(message, code = 'RULES_FAILED') {
|
|
17
|
+
return Object.assign(new Error(message), { code });
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function loadRules(rootDir) {
|
|
21
|
+
try {
|
|
22
|
+
const value = JSON.parse(fs.readFileSync(path.join(rootDir, RULES_FILE), 'utf8'));
|
|
23
|
+
return Array.isArray(value.rules) ? value : { schemaVersion: 1, rules: [] };
|
|
24
|
+
} catch {
|
|
25
|
+
return { schemaVersion: 1, rules: [] };
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function loadLearningEntries(rootDir) {
|
|
30
|
+
try {
|
|
31
|
+
return JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'learnings.json'), 'utf8')).entries || [];
|
|
32
|
+
} catch {
|
|
33
|
+
return [];
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, now = new Date() } = {}) {
|
|
38
|
+
if (!String(check || '').trim()) throw rulesError('--check "<command>" is required: the command must exit non-zero on a violation.', 'INVALID_USAGE');
|
|
39
|
+
if (!String(fix || '').trim()) throw rulesError('--fix "<instruction>" is required: tell the agent exactly how to repair a violation.', 'INVALID_USAGE');
|
|
40
|
+
if (!String(by || '').trim()) throw rulesError('--by <name> is required: promoting a rule is a human decision.', 'INVALID_USAGE');
|
|
41
|
+
const learning = loadLearningEntries(rootDir).find((entry) => entry.id === learningId);
|
|
42
|
+
if (!learning) throw rulesError(`No learning ${learningId} in .ai-engineering-loop/learnings.json. Run \`learn\` first and use an id it prints.`, 'NOT_FOUND');
|
|
43
|
+
const document = loadRules(rootDir);
|
|
44
|
+
const id = `R-${learningId.replace(/^L-/, '')}`;
|
|
45
|
+
if (document.rules.some((rule) => rule.id === id)) throw rulesError(`${learningId} is already promoted as ${id}.`, 'ALREADY_PROMOTED');
|
|
46
|
+
const rule = {
|
|
47
|
+
id,
|
|
48
|
+
from: learningId,
|
|
49
|
+
lesson: learning.lesson,
|
|
50
|
+
check: String(check).trim(),
|
|
51
|
+
shell: Boolean(shell),
|
|
52
|
+
why: why ? String(why).trim() : learning.lesson,
|
|
53
|
+
fix: String(fix).trim(),
|
|
54
|
+
layer: 'L1',
|
|
55
|
+
createdBy: String(by).trim(),
|
|
56
|
+
createdAt: now.toISOString()
|
|
57
|
+
};
|
|
58
|
+
atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
|
|
59
|
+
return rule;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
|
|
63
|
+
function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
|
|
64
|
+
const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
|
|
65
|
+
return entries
|
|
66
|
+
.filter((entry) => (entry.runIds || []).length >= PROMOTE_AFTER_RUNS && !promoted.has(entry.id))
|
|
67
|
+
.map((entry) => ({ id: entry.id, runs: entry.runIds.length, lesson: entry.lesson }));
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// Default-plan steps for every rule (L1, shell by default).
|
|
71
|
+
function rulePlan(rootDir) {
|
|
72
|
+
return loadRules(rootDir).rules.map((rule) => ({ key: `rule:${rule.id}`, layer: 'L1', text: rule.check, shell: rule.shell !== false, rule }));
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Shell commands are recorded JSON-quoted; compare on the unquoted text.
|
|
76
|
+
function plainCommand(text) {
|
|
77
|
+
const value = String(text || '').trim();
|
|
78
|
+
if (value.startsWith('"') && value.endsWith('"')) {
|
|
79
|
+
try {
|
|
80
|
+
return JSON.parse(value);
|
|
81
|
+
} catch {
|
|
82
|
+
return value;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
return value;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function ruleForCommand(rootDir, commandText) {
|
|
89
|
+
const value = plainCommand(commandText);
|
|
90
|
+
return loadRules(rootDir).rules.find((rule) => rule.check === value) || null;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Rules with no passing record in a full-scope bundle.
|
|
94
|
+
function missingRules(rootDir, bundle) {
|
|
95
|
+
const records = bundle?.commands || [];
|
|
96
|
+
if (records.length && records.every((record) => record.scope === 'related')) return [];
|
|
97
|
+
const passing = records.filter((record) => record.exitCode === 0);
|
|
98
|
+
const passed = new Set(passing.map((record) => plainCommand(record.command)));
|
|
99
|
+
const keys = new Set(passing.map((record) => record.key).filter(Boolean));
|
|
100
|
+
return loadRules(rootDir).rules.filter((rule) => !passed.has(rule.check) && !keys.has(`rule:${rule.id}`));
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
module.exports = {
|
|
104
|
+
PROMOTE_AFTER_RUNS,
|
|
105
|
+
RULES_FILE,
|
|
106
|
+
loadRules,
|
|
107
|
+
missingRules,
|
|
108
|
+
promoteLearning,
|
|
109
|
+
promotionCandidates,
|
|
110
|
+
ruleForCommand,
|
|
111
|
+
rulePlan
|
|
112
|
+
};
|
package/lib/stats.js
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// `ael stats` (C-12): where the loop fails, by harness layer, and how often
|
|
4
|
+
// the Maker's green verification was a false victory (the Judge still found
|
|
5
|
+
// a VALID blocking problem). Bottlenecks come from failure records, not
|
|
6
|
+
// guesses (learn-harness-engineering L01/L02).
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
const { runsDir } = require('./run-state.js');
|
|
11
|
+
const { readIterations } = require('./no-progress.js');
|
|
12
|
+
|
|
13
|
+
function listRuns(rootDir) {
|
|
14
|
+
let names = [];
|
|
15
|
+
try {
|
|
16
|
+
names = fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name);
|
|
17
|
+
} catch {
|
|
18
|
+
return [];
|
|
19
|
+
}
|
|
20
|
+
return names.sort();
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function runTier(rootDir, runId) {
|
|
24
|
+
try {
|
|
25
|
+
return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, 'state.json'), 'utf8')).tier?.current || null;
|
|
26
|
+
} catch {
|
|
27
|
+
return null;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function computeStats(rootDir, { last = null } = {}) {
|
|
32
|
+
let runs = listRuns(rootDir);
|
|
33
|
+
if (Number.isInteger(last) && last > 0) runs = runs.slice(-last);
|
|
34
|
+
const causes = {};
|
|
35
|
+
const tiers = {};
|
|
36
|
+
let judged = 0;
|
|
37
|
+
let falseVictories = 0;
|
|
38
|
+
const passIterations = [];
|
|
39
|
+
for (const runId of runs) {
|
|
40
|
+
const entries = readIterations(rootDir, runId);
|
|
41
|
+
if (!entries.length) continue;
|
|
42
|
+
const tier = entries.find((entry) => entry.tier)?.tier || runTier(rootDir, runId) || 'none';
|
|
43
|
+
const bucket = tiers[tier] || (tiers[tier] = { runs: 0, judged: 0, falseVictories: 0 });
|
|
44
|
+
bucket.runs += 1;
|
|
45
|
+
for (const entry of entries) {
|
|
46
|
+
judged += 1;
|
|
47
|
+
bucket.judged += 1;
|
|
48
|
+
if (entry.verdict !== 'PASS' && (entry.blockingCount || (entry.blockingSignatures || []).length) > 0) {
|
|
49
|
+
falseVictories += 1;
|
|
50
|
+
bucket.falseVictories += 1;
|
|
51
|
+
}
|
|
52
|
+
if (entry.verdict !== 'PASS') {
|
|
53
|
+
const cause = entry.cause || 'unclassified';
|
|
54
|
+
causes[cause] = (causes[cause] || 0) + 1;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
const pass = entries.find((entry) => entry.verdict === 'PASS');
|
|
58
|
+
if (pass) passIterations.push(pass.iteration || entries.indexOf(pass) + 1);
|
|
59
|
+
}
|
|
60
|
+
const rate = (n, d) => (d ? Math.round((n / d) * 1000) / 1000 : 0);
|
|
61
|
+
for (const bucket of Object.values(tiers)) bucket.falseVictoryRate = rate(bucket.falseVictories, bucket.judged);
|
|
62
|
+
const topCauses = Object.entries(causes).sort((a, b) => b[1] - a[1]).slice(0, 2).map(([cause, count]) => ({ cause, count }));
|
|
63
|
+
return {
|
|
64
|
+
runs: runs.length,
|
|
65
|
+
judged,
|
|
66
|
+
falseVictories,
|
|
67
|
+
falseVictoryRate: rate(falseVictories, judged),
|
|
68
|
+
causes,
|
|
69
|
+
topCauses,
|
|
70
|
+
bottleneck: topCauses[0]?.cause || null,
|
|
71
|
+
iterationsToPass: passIterations.length ? rate(passIterations.reduce((a, b) => a + b, 0), passIterations.length) : null,
|
|
72
|
+
tiers
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function formatStats(stats) {
|
|
77
|
+
const lines = [
|
|
78
|
+
`Runs: ${stats.runs}, judged iterations: ${stats.judged}`,
|
|
79
|
+
`False victories: ${stats.falseVictories} (${Math.round(stats.falseVictoryRate * 100)}%) - verification was green but the Judge found a VALID blocking problem`,
|
|
80
|
+
`Iterations to PASS (avg): ${stats.iterationsToPass ?? 'n/a'}`,
|
|
81
|
+
`Failure causes: ${Object.entries(stats.causes).map(([cause, count]) => `${cause} ${count}`).join(', ') || 'none'}`
|
|
82
|
+
];
|
|
83
|
+
if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
|
|
84
|
+
for (const [tier, bucket] of Object.entries(stats.tiers)) {
|
|
85
|
+
lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
|
|
86
|
+
}
|
|
87
|
+
return lines.join('\n');
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
module.exports = { computeStats, formatStats };
|
package/lib/sync-hosts.js
CHANGED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Verification integrity (C-11). The cheapest way for an agent to turn a gate
|
|
4
|
+
// green is to weaken the proof: skip a test, delete it, drop assertions, or
|
|
5
|
+
// edit the verification config. This deterministic scan of the gated diff
|
|
6
|
+
// turns each of those into a VALID ledger finding the reviewer cannot drop.
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
|
|
11
|
+
const SOURCE = 'verification-integrity';
|
|
12
|
+
const TEST_FILE = /(^|\/)(tests?|__tests__|spec|specs)\/|\.(test|spec)\.[cm]?[jt]sx?$|(^|\/)test_[^/]+\.py$|_test\.(py|go)$|Test\.java$/;
|
|
13
|
+
const SKIP_MARKER = /\b(?:it|test|describe|context)\.(?:skip|only|todo)\s*\(|\bx(?:it|describe|test)\s*\(|\bf(?:it|describe)\s*\(|@Disabled\b|@Ignore\b|pytest\.mark\.(?:skip|xfail)|@unittest\.skip|\bt\.Skip(?:Now)?\(|\{\s*skip\s*:\s*true/;
|
|
14
|
+
const ASSERTION = /\b(?:assert(?:\.\w+)?|expect|should|t\.(?:equal|deepEqual|ok|is|true|false|throws))\s*\(|\bassert\s+\w|\bself\.assert\w*\(/;
|
|
15
|
+
const CONFIG_FILE = /(^|\/)(\.ai-engineering-loop\/verification\.md|(jest|vitest|playwright|cypress|karma)\.config\.[cm]?[jt]s|pytest\.ini|tox\.ini|\.nycrc(\.json)?|\.c8rc(\.json)?|codecov\.ya?ml|\.mocharc\.[a-z]+)$/;
|
|
16
|
+
|
|
17
|
+
function isTestFile(file) {
|
|
18
|
+
return TEST_FILE.test(file);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
// Split a unified diff into per-file entries with added/removed lines.
|
|
22
|
+
function parseDiff(diffText) {
|
|
23
|
+
const files = [];
|
|
24
|
+
let current = null;
|
|
25
|
+
let newLine = 0;
|
|
26
|
+
for (const line of String(diffText || '').split('\n')) {
|
|
27
|
+
const header = line.match(/^diff --git a\/(.+?) b\/(.+)$/);
|
|
28
|
+
if (header) {
|
|
29
|
+
current = { oldPath: header[1], path: header[2], deleted: false, added: [], removed: [] };
|
|
30
|
+
files.push(current);
|
|
31
|
+
continue;
|
|
32
|
+
}
|
|
33
|
+
if (!current) continue;
|
|
34
|
+
if (line.startsWith('deleted file mode')) current.deleted = true;
|
|
35
|
+
else if (line.startsWith('new file mode')) current.created = true;
|
|
36
|
+
else if (line.startsWith('+++ ') || line.startsWith('--- ')) continue;
|
|
37
|
+
else if (line.startsWith('@@')) {
|
|
38
|
+
const hunk = line.match(/\+(\d+)/);
|
|
39
|
+
newLine = hunk ? Number(hunk[1]) : 0;
|
|
40
|
+
} else if (line.startsWith('+')) {
|
|
41
|
+
current.added.push({ text: line.slice(1), line: newLine });
|
|
42
|
+
newLine += 1;
|
|
43
|
+
} else if (line.startsWith('-')) {
|
|
44
|
+
current.removed.push({ text: line.slice(1) });
|
|
45
|
+
} else {
|
|
46
|
+
newLine += 1;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return files;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function finding(id, location, failureScenario, evidence, concreteAlternativeDiff, severity) {
|
|
53
|
+
return {
|
|
54
|
+
id,
|
|
55
|
+
axis: 'spec',
|
|
56
|
+
location,
|
|
57
|
+
failureScenario,
|
|
58
|
+
evidence,
|
|
59
|
+
severity,
|
|
60
|
+
validity: 'VALID',
|
|
61
|
+
disposition: 'STRONG',
|
|
62
|
+
concreteAlternativeDiff,
|
|
63
|
+
source: SOURCE,
|
|
64
|
+
fix: 'Restore the proof (un-skip, keep the assertion, restore the test or config). A human may dismiss it with dismissal: {by, evidence} when the change is intended.'
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// `goalText` exempts config files the frozen Goal Contract names explicitly.
|
|
69
|
+
function scanDiff(diffText, { tier = null, goalText = '' } = {}) {
|
|
70
|
+
const severity = tier === 'T2' ? 'BLOCKER' : 'HIGH';
|
|
71
|
+
const files = parseDiff(diffText);
|
|
72
|
+
const findings = [];
|
|
73
|
+
const createdTests = files.filter((file) => file.created && isTestFile(file.path));
|
|
74
|
+
let n = 0;
|
|
75
|
+
const next = () => `INTEGRITY-${++n}`;
|
|
76
|
+
|
|
77
|
+
for (const file of files) {
|
|
78
|
+
if (file.deleted && isTestFile(file.oldPath)) {
|
|
79
|
+
const dir = path.posix.dirname(file.oldPath);
|
|
80
|
+
const replaced = createdTests.some((created) => path.posix.dirname(created.path) === dir);
|
|
81
|
+
if (!replaced) {
|
|
82
|
+
findings.push(finding(next(), file.oldPath,
|
|
83
|
+
`Test file ${file.oldPath} was deleted with no replacement test in ${dir}/; the behaviour it covered is no longer verified`,
|
|
84
|
+
`diff marks ${file.oldPath} as deleted file mode`,
|
|
85
|
+
`Restore ${file.oldPath} (git checkout <base> -- ${file.oldPath}) or add a replacement test in ${dir}/`, severity));
|
|
86
|
+
}
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
if (isTestFile(file.path)) {
|
|
90
|
+
for (const added of file.added) {
|
|
91
|
+
if (SKIP_MARKER.test(added.text)) {
|
|
92
|
+
findings.push(finding(next(), `${file.path}:${added.line}`,
|
|
93
|
+
`A skip/only/todo marker was added, so tests stop running while the suite still reports green`,
|
|
94
|
+
`added line: ${added.text.trim().slice(0, 160)}`,
|
|
95
|
+
`- ${added.text.trim()}\n+ ${added.text.trim().replace(/\.(skip|only|todo)\s*\(/, '(').replace(/\bx(it|describe|test)\s*\(/, '$1(').replace(/\bf(it|describe)\s*\(/, '$1(')}`,
|
|
96
|
+
severity));
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
const removedAsserts = file.removed.filter((line) => ASSERTION.test(line.text));
|
|
100
|
+
const addedAsserts = file.added.filter((line) => ASSERTION.test(line.text));
|
|
101
|
+
if (!file.deleted && removedAsserts.length > addedAsserts.length) {
|
|
102
|
+
findings.push(finding(next(), file.path,
|
|
103
|
+
`${removedAsserts.length - addedAsserts.length} assertion(s) removed from ${file.path} without replacement; the test proves less than before`,
|
|
104
|
+
`removed ${removedAsserts.length} assertion line(s), added ${addedAsserts.length}; first removed: ${removedAsserts[0].text.trim().slice(0, 120)}`,
|
|
105
|
+
`+ ${removedAsserts[0].text.trim()}`, severity));
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
if (CONFIG_FILE.test(file.path) && !goalText.includes(file.path)) {
|
|
109
|
+
findings.push(finding(next(), file.path,
|
|
110
|
+
`Verification config ${file.path} changed during the run; the rules that judge this change were rewritten by the change itself`,
|
|
111
|
+
`diff touches ${file.path} (+${file.added.length}/-${file.removed.length} lines); the frozen Goal Contract does not name it`,
|
|
112
|
+
`Revert ${file.path}, or freeze a Goal Contract that names it as in scope`, severity));
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
return findings;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function scanRun(rootDir, run) {
|
|
119
|
+
const artifact = run.artifacts?.diff;
|
|
120
|
+
if (!artifact?.path) return [];
|
|
121
|
+
let diffText;
|
|
122
|
+
try {
|
|
123
|
+
diffText = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
|
|
124
|
+
} catch {
|
|
125
|
+
return [];
|
|
126
|
+
}
|
|
127
|
+
let goalText = '';
|
|
128
|
+
try {
|
|
129
|
+
goalText = fs.readFileSync(path.join(rootDir, run.artifacts.goalContract.path), 'utf8');
|
|
130
|
+
} catch {
|
|
131
|
+
goalText = '';
|
|
132
|
+
}
|
|
133
|
+
return scanDiff(diffText, { tier: run.tier?.current || null, goalText });
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Merge into the Finding Ledger. A finding already present may only be
|
|
137
|
+
// downgraded with a human dismissal ({by, evidence}); anything else throws.
|
|
138
|
+
function mergeIntegrityIntoLedger(ledger, findings) {
|
|
139
|
+
const errors = [];
|
|
140
|
+
const existing = new Map((ledger.findings || []).filter((item) => item.source === SOURCE).map((item) => [item.location, item]));
|
|
141
|
+
const additions = [];
|
|
142
|
+
for (const item of findings) {
|
|
143
|
+
const present = existing.get(item.location);
|
|
144
|
+
if (!present) {
|
|
145
|
+
additions.push(item);
|
|
146
|
+
continue;
|
|
147
|
+
}
|
|
148
|
+
const downgraded = present.validity !== 'VALID' || !['BLOCKER', 'HIGH'].includes(present.severity);
|
|
149
|
+
const dismissal = present.dismissal || {};
|
|
150
|
+
if (downgraded && !(String(dismissal.by || '').trim() && String(dismissal.evidence || '').trim())) {
|
|
151
|
+
errors.push(`${present.id} (${present.location}) was downgraded without a human dismissal. FIX: restore it, or add dismissal: {by: "<human>", evidence: "<why the weaker test is intended>"}.`);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
return { ledger: { ...ledger, findings: [...(ledger.findings || []), ...additions] }, added: additions.length, errors };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
module.exports = {
|
|
158
|
+
SOURCE,
|
|
159
|
+
isTestFile,
|
|
160
|
+
mergeIntegrityIntoLedger,
|
|
161
|
+
parseDiff,
|
|
162
|
+
scanDiff,
|
|
163
|
+
scanRun
|
|
164
|
+
};
|
|
@@ -73,6 +73,63 @@ function relatedCommand(rootDir, run, { split, shell = false } = {}) {
|
|
|
73
73
|
return { ...parsed, args, files };
|
|
74
74
|
}
|
|
75
75
|
|
|
76
|
+
// Verification layers (#110): which verification.md key proves which layer.
|
|
77
|
+
const LAYER_OF_KEY = Object.freeze({
|
|
78
|
+
build: 'L1', typecheck: 'L1', lint: 'L1',
|
|
79
|
+
test_unit: 'L2', test_related: 'L2', test_all: 'L2', smoke: 'L2',
|
|
80
|
+
e2e: 'L3'
|
|
81
|
+
});
|
|
82
|
+
// Cheapest-first default plan (templates/repo-config/verification.md
|
|
83
|
+
// "Default order"): L1 static checks, then L2 unit tests.
|
|
84
|
+
const DEFAULT_PLAN_KEYS = Object.freeze(['build', 'typecheck', 'lint', 'test_unit']);
|
|
85
|
+
|
|
86
|
+
function layerOfKey(key) {
|
|
87
|
+
return LAYER_OF_KEY[key] || null;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// Ordered [{ key, layer, text }] for the configured default plan. A command
|
|
91
|
+
// marked `# Optional` is still run when present.
|
|
92
|
+
// Commands that rewrite files (`eslint --fix`, `prettier --write`) would
|
|
93
|
+
// change the gated diff mid-verification, so they never run by default.
|
|
94
|
+
function isMutatingCommand(text) {
|
|
95
|
+
return /(^|\s)(--fix|--write|-w)(\s|=|$)/.test(String(text || ''));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function defaultCommandPlan(rootDir, keys = DEFAULT_PLAN_KEYS) {
|
|
99
|
+
const commands = verificationCommands(rootDir);
|
|
100
|
+
return keys
|
|
101
|
+
.filter((key) => commands[key] && !isMutatingCommand(commands[key]))
|
|
102
|
+
.map((key) => ({ key, layer: layerOfKey(key), text: commands[key] }));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Layer of a recorded command string, by matching it against verification.md.
|
|
106
|
+
function layerOfCommand(rootDir, commandText) {
|
|
107
|
+
const value = String(commandText || '').trim();
|
|
108
|
+
const commands = verificationCommands(rootDir);
|
|
109
|
+
for (const [key, text] of Object.entries(commands)) {
|
|
110
|
+
const base = text.replace(FILES_TOKEN, '').trim();
|
|
111
|
+
if (value === text || (base && value.startsWith(base))) return layerOfKey(key);
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// C-03: changes that can break startup without breaking unit tests
|
|
117
|
+
// (migrations, env, config, entrypoints, dependency manifests).
|
|
118
|
+
const STARTUP_RISK = /(^|\/)(migrations?|db\/migrate|prisma|alembic)\/|(^|\/)\.env(\.[\w-]+)?$|(^|\/)(config|settings)\/|\.config\.[cm]?[jt]s$|(^|\/)(docker-compose[^/]*\.ya?ml|Dockerfile|Procfile)$|(^|\/)(main|index|server|app)\.[cm]?[jt]s$|(^|\/)(package\.json|requirements\.txt|pyproject\.toml|go\.mod|Cargo\.toml)$/;
|
|
119
|
+
|
|
120
|
+
// Changed paths that warrant the `smoke` check, from the gated diff.
|
|
121
|
+
function startupRiskFiles(rootDir, run) {
|
|
122
|
+
const artifact = run?.artifacts?.diff;
|
|
123
|
+
if (!artifact?.path) return [];
|
|
124
|
+
let diff = '';
|
|
125
|
+
try {
|
|
126
|
+
diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
|
|
127
|
+
} catch {
|
|
128
|
+
return [];
|
|
129
|
+
}
|
|
130
|
+
return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
|
|
131
|
+
}
|
|
132
|
+
|
|
76
133
|
function readJson(filePath) {
|
|
77
134
|
try {
|
|
78
135
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
@@ -106,6 +163,13 @@ function finalVerificationStatus(rootDir, run) {
|
|
|
106
163
|
}
|
|
107
164
|
|
|
108
165
|
module.exports = {
|
|
166
|
+
DEFAULT_PLAN_KEYS,
|
|
167
|
+
LAYER_OF_KEY,
|
|
168
|
+
defaultCommandPlan,
|
|
169
|
+
isMutatingCommand,
|
|
170
|
+
startupRiskFiles,
|
|
171
|
+
layerOfCommand,
|
|
172
|
+
layerOfKey,
|
|
109
173
|
FINAL_FILE,
|
|
110
174
|
FILES_TOKEN,
|
|
111
175
|
verificationCommands,
|
|
@@ -303,7 +303,8 @@ async function recordVerificationCommands(rootDir, {
|
|
|
303
303
|
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
304
304
|
scope = null,
|
|
305
305
|
target = 'verification',
|
|
306
|
-
fresh = false
|
|
306
|
+
fresh = false,
|
|
307
|
+
stopOnFailure = false
|
|
307
308
|
} = {}) {
|
|
308
309
|
if (!Array.isArray(commands) || commands.length === 0) {
|
|
309
310
|
throw recorderError('Provide at least one verification command.', 'INVALID_COMMAND');
|
|
@@ -326,12 +327,21 @@ async function recordVerificationCommands(rootDir, {
|
|
|
326
327
|
&& (record.scope || 'full') === (scope || 'full'));
|
|
327
328
|
if (previous) return { ...previous, reused: true };
|
|
328
329
|
const result = await runCommand(item.command, item.args || [], { cwd: rootDir, timeoutMs, shell: Boolean(item.shell) });
|
|
329
|
-
|
|
330
|
+
const tagged = { ...result, ...(item.layer ? { layer: item.layer } : {}), ...(item.key ? { key: item.key } : {}) };
|
|
331
|
+
return scope ? { ...tagged, scope } : tagged;
|
|
330
332
|
};
|
|
331
333
|
let results;
|
|
332
|
-
|
|
334
|
+
const skipped = [];
|
|
335
|
+
if (serial || stopOnFailure) {
|
|
333
336
|
results = [];
|
|
334
|
-
for (const item of commands)
|
|
337
|
+
for (const item of commands) {
|
|
338
|
+
// Cheapest-first plan: a failing L1 check stops before slower layers.
|
|
339
|
+
if (stopOnFailure && results.some((result) => !passedRecord(result))) {
|
|
340
|
+
skipped.push(displayCommand(item.command, item.args || []));
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
results.push(await exec(item));
|
|
344
|
+
}
|
|
335
345
|
} else {
|
|
336
346
|
results = await Promise.all(commands.map(exec));
|
|
337
347
|
}
|
|
@@ -342,12 +352,14 @@ async function recordVerificationCommands(rootDir, {
|
|
|
342
352
|
runId: run.runId,
|
|
343
353
|
path: path.relative(rootDir, filePath).split(path.sep).join('/'),
|
|
344
354
|
commands: results,
|
|
355
|
+
skipped,
|
|
345
356
|
summary,
|
|
346
|
-
passed: results.every(passedRecord)
|
|
357
|
+
passed: results.every(passedRecord) && skipped.length === 0
|
|
347
358
|
};
|
|
348
359
|
}
|
|
349
360
|
|
|
350
361
|
module.exports = {
|
|
362
|
+
runCommand,
|
|
351
363
|
recordVerificationCommands,
|
|
352
364
|
DEFAULT_TIMEOUT_MS,
|
|
353
365
|
MAX_CAPTURE_BYTES,
|
package/package.json
CHANGED
|
@@ -22,6 +22,20 @@
|
|
|
22
22
|
},
|
|
23
23
|
"additionalProperties": true
|
|
24
24
|
}
|
|
25
|
+
},
|
|
26
|
+
"rubricScores": {
|
|
27
|
+
"description": "C-09: one score per rubric criterion of a subjective AC",
|
|
28
|
+
"type": "array",
|
|
29
|
+
"items": {
|
|
30
|
+
"type": "object",
|
|
31
|
+
"required": ["ac", "criterion", "pass", "evidence"],
|
|
32
|
+
"properties": {
|
|
33
|
+
"ac": { "type": "string", "pattern": "^AC-[0-9]+$" },
|
|
34
|
+
"criterion": { "type": "string", "minLength": 1 },
|
|
35
|
+
"pass": { "type": "boolean" },
|
|
36
|
+
"evidence": { "type": "string", "minLength": 1 }
|
|
37
|
+
}
|
|
38
|
+
}
|
|
25
39
|
}
|
|
26
40
|
},
|
|
27
41
|
"additionalProperties": true
|
|
@@ -52,6 +52,32 @@
|
|
|
52
52
|
"type": "string",
|
|
53
53
|
"minLength": 1
|
|
54
54
|
}
|
|
55
|
+
},
|
|
56
|
+
"rubric": {
|
|
57
|
+
"description": "Optional (C-09) for subjective ACs: criteria the Devil's Advocate scores with evidence in ledger.rubricScores. Judge PASS needs every criterion scored pass:true.",
|
|
58
|
+
"type": "array",
|
|
59
|
+
"items": {
|
|
60
|
+
"type": "object",
|
|
61
|
+
"required": ["criterion", "pass"],
|
|
62
|
+
"properties": {
|
|
63
|
+
"criterion": { "type": "string", "minLength": 1 },
|
|
64
|
+
"pass": { "type": "string", "minLength": 1, "description": "What counts as passing, e.g. 'contrast >= 4.5:1 (WCAG AA)'" }
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
"verify": {
|
|
69
|
+
"description": "Optional (C-07). The command that proves this AC. Gate judge refuses PASS until this exact command passed on the gated diff (verification.json or final-verification.json).",
|
|
70
|
+
"type": "object",
|
|
71
|
+
"required": ["command"],
|
|
72
|
+
"properties": {
|
|
73
|
+
"command": { "type": "string", "minLength": 1 },
|
|
74
|
+
"layer": { "enum": ["L1", "L2", "L3"] }
|
|
75
|
+
},
|
|
76
|
+
"additionalProperties": false
|
|
77
|
+
},
|
|
78
|
+
"crossComponent": {
|
|
79
|
+
"description": "Optional (#110). true when the AC crosses a component boundary; claimed-vs-reality then needs L3 (end-to-end) evidence or an 'L3 waived: <reason>' Layer cell before gate review.",
|
|
80
|
+
"type": "boolean"
|
|
55
81
|
}
|
|
56
82
|
},
|
|
57
83
|
"additionalProperties": true
|
|
@@ -104,6 +130,11 @@
|
|
|
104
130
|
},
|
|
105
131
|
"additionalProperties": false
|
|
106
132
|
},
|
|
133
|
+
"featureId": {
|
|
134
|
+
"description": "Optional (C-15): id of the feature in .ai-engineering-loop/features.json this Goal delivers. Goal freeze marks it active; delivery marks it passing with this Run as evidence.",
|
|
135
|
+
"type": "string",
|
|
136
|
+
"minLength": 1
|
|
137
|
+
},
|
|
107
138
|
"outOfScope": {
|
|
108
139
|
"description": "Optional list of things this Goal must not change.",
|
|
109
140
|
"type": "array",
|
|
@@ -11,7 +11,8 @@
|
|
|
11
11
|
"reason": { "type": "string", "minLength": 1 },
|
|
12
12
|
"action": { "type": "string", "minLength": 1 },
|
|
13
13
|
"modelTier": { "enum": ["CHEAP_MODEL", "STRONG_MODEL", "HUMAN"] },
|
|
14
|
-
"humanApproved": { "type": "boolean" }
|
|
14
|
+
"humanApproved": { "type": "boolean" },
|
|
15
|
+
"cause": { "description": "C-12: harness layer a non-PASS verdict points at", "enum": ["task_spec", "context", "environment", "verification", "state", "model"] }
|
|
15
16
|
},
|
|
16
17
|
"additionalProperties": true
|
|
17
18
|
}
|