ai-engineering-loop 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +11 -0
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +21 -11
- package/.claude/agents/devil-advocate.md +6 -1
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +6 -1
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +22 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +6 -1
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +262 -20
- package/core/bench.md +15 -0
- package/core/goal-contract.md +35 -0
- package/core/verification-loop.md +96 -7
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +63 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/calibration.js +112 -0
- package/lib/claimed-vs-reality.js +114 -16
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/debt.js +110 -0
- package/lib/features.js +95 -0
- package/lib/gates.js +234 -41
- package/lib/harness-audit.js +124 -0
- package/lib/no-progress.js +6 -1
- package/lib/review-ingest.js +54 -2
- package/lib/rules.js +112 -0
- package/lib/stats.js +90 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +64 -0
- package/lib/verification-recorder.js +17 -5
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +31 -0
- package/schemas/verdict.schema.json +2 -1
|
@@ -70,14 +70,14 @@ function assertNoSymlinkPath(root, targetPath) {
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
function renderClaimedVsRealityTemplate(goal) {
|
|
73
|
+
function renderClaimedVsRealityTemplate(goal, tier = null) {
|
|
74
74
|
const lines = [
|
|
75
75
|
'# Claimed vs Reality',
|
|
76
76
|
'',
|
|
77
|
-
'> Generated from the frozen Goal Contract. Fill Claimed and Reality only after implementation and verification. Reality must cite observed command output; this template is not verification evidence.',
|
|
77
|
+
'> Generated from the frozen Goal Contract. Fill Claimed and Reality only after implementation and verification. Reality must cite observed command output; this template is not verification evidence. Layer: L1 static, L2 tests, L3 e2e; "needs L3" rows need L3 or "L3 waived: <reason>".',
|
|
78
78
|
'',
|
|
79
|
-
'| AC | Claimed | Reality from command log |',
|
|
80
|
-
'
|
|
79
|
+
'| AC | Claimed | Reality from command log | Layer |',
|
|
80
|
+
'|---|---|---|---|'
|
|
81
81
|
];
|
|
82
82
|
for (const criterion of goal.acceptanceCriteria) {
|
|
83
83
|
const evidence = markdownCell(criterion.evidenceRequired);
|
|
@@ -87,7 +87,8 @@ function renderClaimedVsRealityTemplate(goal) {
|
|
|
87
87
|
evidence ? `Evidence: ${evidence}` : '',
|
|
88
88
|
failures ? `Failure cases: ${failures}` : ''
|
|
89
89
|
].filter(Boolean).join(' — ');
|
|
90
|
-
|
|
90
|
+
const layerHint = criterionNeedsL3(criterion, tier) ? 'needs L3' : '';
|
|
91
|
+
lines.push(`| ${markdownCell(description)} | | | ${layerHint} |`);
|
|
91
92
|
}
|
|
92
93
|
return `${lines.join('\n')}\n`;
|
|
93
94
|
}
|
|
@@ -125,7 +126,7 @@ function createClaimedVsRealityTemplate(rootDir, { runId, outputPath } = {}) {
|
|
|
125
126
|
throw cause;
|
|
126
127
|
}
|
|
127
128
|
try {
|
|
128
|
-
fs.writeFileSync(descriptor, renderClaimedVsRealityTemplate(goal));
|
|
129
|
+
fs.writeFileSync(descriptor, renderClaimedVsRealityTemplate(goal, run.tier?.current || null));
|
|
129
130
|
fs.fsyncSync(descriptor);
|
|
130
131
|
} finally {
|
|
131
132
|
fs.closeSync(descriptor);
|
|
@@ -143,29 +144,126 @@ function parseClaimedVsReality(markdown) {
|
|
|
143
144
|
if (/^\|\s*-+\s*\|/.test(trimmed) || /^\|\s*AC\s*\|/i.test(trimmed)) continue;
|
|
144
145
|
const cells = trimmed.split('|').map((c) => c.trim()).filter((_, i, arr) => i > 0 && i < arr.length - 1);
|
|
145
146
|
if (cells.length < 3) continue;
|
|
146
|
-
|
|
147
|
+
const row = {
|
|
147
148
|
ac: cells[0],
|
|
148
149
|
claimed: cells[1],
|
|
149
150
|
reality: cells[2]
|
|
150
|
-
}
|
|
151
|
+
};
|
|
152
|
+
if (cells.length >= 4 && cells[3]) {
|
|
153
|
+
row.layer = cells[3];
|
|
154
|
+
}
|
|
155
|
+
rows.push(row);
|
|
151
156
|
}
|
|
152
157
|
return { rows };
|
|
153
158
|
}
|
|
154
159
|
|
|
155
|
-
|
|
160
|
+
// Three-layer termination check (#110, learn-harness-engineering lectures 09/10):
|
|
161
|
+
// L1 static (typecheck/lint/build), L2 runtime (unit/related/all tests),
|
|
162
|
+
// L3 end-to-end (e2e or the AC's real user flow). Unit tests isolate and mock
|
|
163
|
+
// by design, so they cannot prove a cross-component AC.
|
|
164
|
+
const LAYER_TOKEN = /^L[123]$/;
|
|
165
|
+
const WAIVER = /^L3\s+waived\s*:\s*(.+)$/i;
|
|
166
|
+
const MIN_WAIVER_REASON = 15;
|
|
167
|
+
|
|
168
|
+
function acIdOf(cell) {
|
|
169
|
+
const match = String(cell || '').match(/\bAC-[0-9]+\b/);
|
|
170
|
+
return match ? match[0] : null;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
function criterionNeedsL3(criterion, tier) {
|
|
174
|
+
return criterion?.crossComponent === true || tier === 'T2';
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Parse a Layer cell such as "L2", "L1+L2+L3", or "L2; L3 waived: <reason>".
|
|
178
|
+
// Returns { layers, waiver, invalid } where invalid lists unknown tokens.
|
|
179
|
+
function parseLayerCell(cell) {
|
|
180
|
+
const layers = new Set();
|
|
181
|
+
const invalid = [];
|
|
182
|
+
let waiver = null;
|
|
183
|
+
const text = String(cell || '').trim();
|
|
184
|
+
if (!text || /^needs L3$/i.test(text)) return { layers, waiver, invalid, empty: true };
|
|
185
|
+
for (const part of text.split(/[;,]/).map((p) => p.trim()).filter(Boolean)) {
|
|
186
|
+
const waived = part.match(WAIVER);
|
|
187
|
+
if (waived) {
|
|
188
|
+
waiver = waived[1].trim();
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
for (const token of part.split(/[+\s]+/).map((t) => t.trim().toUpperCase()).filter(Boolean)) {
|
|
192
|
+
if (LAYER_TOKEN.test(token)) layers.add(token);
|
|
193
|
+
else invalid.push(token);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return { layers, waiver, invalid, empty: false };
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// Agent-oriented check: every problem says what is wrong and how to fix it.
|
|
200
|
+
// options.goal (frozen Goal Contract) and options.tier switch on the L3
|
|
201
|
+
// requirement; without them only the Layer cell syntax is checked, so legacy
|
|
202
|
+
// 3-column files keep working.
|
|
203
|
+
// C-01: a Layer tag must be backed by evidence of that layer. `layers` is
|
|
204
|
+
// the set of layers present in the recorded verification commands (null =
|
|
205
|
+
// unknown, skip the check). L3 may instead cite a manual end-to-end flow.
|
|
206
|
+
const MANUAL_FLOW = /\b(manual(?:ly)?|user flow|clicked|screenshot|playwright|cypress|browser|curl|https?:\/\/)/i;
|
|
207
|
+
|
|
208
|
+
function checkClaimedVsReality(markdown, { goal = null, tier = null, layers = null } = {}) {
|
|
156
209
|
const { rows } = parseClaimedVsReality(markdown);
|
|
157
|
-
|
|
158
|
-
|
|
210
|
+
const problems = [];
|
|
211
|
+
if (rows.length === 0) {
|
|
212
|
+
problems.push('claimed-vs-reality has no AC rows. FIX: scaffold it with `context claimed-vs-reality --run <id>` and fill every row.');
|
|
213
|
+
return { ready: false, problems };
|
|
214
|
+
}
|
|
215
|
+
const hasLayerColumn = /^\s*\|\s*AC\s*\|.*\|\s*Layer\s*\|\s*$/im.test(String(markdown));
|
|
216
|
+
const criteria = new Map((goal?.acceptanceCriteria || []).map((c) => [c.id, c]));
|
|
217
|
+
const effectiveTier = tier || goal?.tier || null;
|
|
218
|
+
for (const row of rows) {
|
|
219
|
+
const label = acIdOf(row.ac) || markdownCell(row.ac).slice(0, 40);
|
|
159
220
|
const reality = (row.reality || '').trim();
|
|
160
|
-
if (!
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
221
|
+
if (!(row.claimed || '').trim()) {
|
|
222
|
+
problems.push(`${label}: Claimed is empty. FIX: state what the change does for this AC.`);
|
|
223
|
+
}
|
|
224
|
+
if (!reality) {
|
|
225
|
+
problems.push(`${label}: Reality is empty. FIX: paste the command and its observed result (exit code, pass count).`);
|
|
226
|
+
} else if (/^seems green$/i.test(reality)) {
|
|
227
|
+
problems.push(`${label}: Reality "seems green" is not evidence. FIX: cite the command output from this iteration.`);
|
|
228
|
+
}
|
|
229
|
+
const layer = parseLayerCell(row.layer);
|
|
230
|
+
if (layer.invalid.length) {
|
|
231
|
+
problems.push(`${label}: Layer has unknown token(s) ${layer.invalid.join(', ')}. FIX: use L1 (static), L2 (runtime tests), L3 (end-to-end), e.g. "L2+L3".`);
|
|
232
|
+
}
|
|
233
|
+
if (layer.waiver !== null && layer.waiver.length < MIN_WAIVER_REASON) {
|
|
234
|
+
problems.push(`${label}: L3 waiver reason is too short. FIX: write "L3 waived: <why no e2e or manual flow could run>" with a concrete reason.`);
|
|
235
|
+
}
|
|
236
|
+
if (layers) {
|
|
237
|
+
for (const tag of layer.layers) {
|
|
238
|
+
if (layers.has(tag)) continue;
|
|
239
|
+
if (tag === 'L3' && MANUAL_FLOW.test(reality)) continue;
|
|
240
|
+
problems.push(tag === 'L3'
|
|
241
|
+
? `${label}: Layer says L3 but no e2e command was recorded and Reality cites no manual flow. FIX: run \`verify --cmd "<e2e command>"\`, or describe the real user flow you exercised (steps, URL, screenshot path) in Reality.`
|
|
242
|
+
: `${label}: Layer says ${tag} but no ${tag} command was recorded for this diff. FIX: run the ${tag === 'L1' ? 'typecheck/lint/build' : 'test'} command through \`verify\`, or correct the tag.`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
const criterion = criteria.get(acIdOf(row.ac));
|
|
246
|
+
const needsL3 = criterion ? criterionNeedsL3(criterion, effectiveTier) : (effectiveTier === 'T2' && Boolean(goal));
|
|
247
|
+
if (needsL3 && !layer.layers.has('L3') && !(layer.waiver && layer.waiver.length >= MIN_WAIVER_REASON)) {
|
|
248
|
+
const why = criterion?.crossComponent === true ? 'it is marked crossComponent' : `tier ${effectiveTier} requires L1-L3`;
|
|
249
|
+
problems.push(hasLayerColumn
|
|
250
|
+
? `${label}: needs L3 end-to-end evidence because ${why}; unit/related tests mock the boundaries this AC crosses. FIX: run \`e2e\` (or the AC's real user flow), cite it in Reality and tag Layer "L3", or write "L3 waived: <reason>".`
|
|
251
|
+
: `${label}: needs L3 evidence because ${why}, but the file has no Layer column. FIX: add a 4th column "Layer" and tag this row L3 (or "L3 waived: <reason>").`);
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
return { ready: problems.length === 0, problems };
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
function readyForDa(markdown, options) {
|
|
258
|
+
return checkClaimedVsReality(markdown, options).ready;
|
|
165
259
|
}
|
|
166
260
|
|
|
167
261
|
module.exports = {
|
|
262
|
+
checkClaimedVsReality,
|
|
168
263
|
createClaimedVsRealityTemplate,
|
|
264
|
+
criterionNeedsL3,
|
|
265
|
+
loadFrozenGoal: frozenGoal,
|
|
266
|
+
parseLayerCell,
|
|
169
267
|
parseClaimedVsReality,
|
|
170
268
|
readyForDa
|
|
171
269
|
};
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Clean state at delivery (C-06, learn-harness-engineering L12): the next
|
|
4
|
+
// session must start from a tree it can trust. Checked at `gate delivery`.
|
|
5
|
+
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const { execFileSync } = require('child_process');
|
|
9
|
+
const { runsDir } = require('./run-state.js');
|
|
10
|
+
const { parseDiff } = require('./verification-integrity.js');
|
|
11
|
+
|
|
12
|
+
const DEBUG_MARKERS = [
|
|
13
|
+
{ pattern: /^\s*debugger\s*;?\s*$/, label: 'debugger statement' },
|
|
14
|
+
{ pattern: /^(<{7}|={7}|>{7})( |$)/, label: 'merge conflict marker' },
|
|
15
|
+
{ pattern: /\bbreakpoint\(\)|\bpdb\.set_trace\(\)|\bbinding\.pry\b/, label: 'interactive breakpoint' }
|
|
16
|
+
];
|
|
17
|
+
const SECRET_FILE = /(^|\/)\.env(\.(?!example$|sample$|template$)[\w-]+)?$|(^|\/)(id_rsa|id_ed25519)$|\.(pem|p12|pfx|key)$/;
|
|
18
|
+
|
|
19
|
+
function untrackedFiles(rootDir) {
|
|
20
|
+
try {
|
|
21
|
+
return execFileSync('git', ['ls-files', '--others', '--exclude-standard', '-z', '--', '.', ':(exclude).ai-engineering-loop'], {
|
|
22
|
+
cwd: rootDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore']
|
|
23
|
+
}).split('\0').filter(Boolean);
|
|
24
|
+
} catch {
|
|
25
|
+
return [];
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function templateOnlyArtifacts(rootDir, run) {
|
|
30
|
+
const found = [];
|
|
31
|
+
const dirs = [path.join(runsDir(rootDir), run.runId), path.join(rootDir, '.ai-engineering-loop', 'tasks')];
|
|
32
|
+
for (const dir of dirs) {
|
|
33
|
+
let entries = [];
|
|
34
|
+
try {
|
|
35
|
+
entries = fs.readdirSync(dir).filter((name) => name.endsWith('.json'));
|
|
36
|
+
} catch {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
for (const name of entries) {
|
|
40
|
+
try {
|
|
41
|
+
if (JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'))?._templateOnly === true) {
|
|
42
|
+
found.push(path.relative(rootDir, path.join(dir, name)).split(path.sep).join('/'));
|
|
43
|
+
}
|
|
44
|
+
} catch {
|
|
45
|
+
// Not JSON we own; ignore.
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return found;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// { errors, warnings }: errors always block delivery; stray untracked files
|
|
53
|
+
// block in UNATTENDED and warn in ASSISTED (people keep scratch files).
|
|
54
|
+
function checkCleanState(rootDir, run, { mode = 'ASSISTED' } = {}) {
|
|
55
|
+
const errors = [];
|
|
56
|
+
const warnings = [];
|
|
57
|
+
let diffText = '';
|
|
58
|
+
try {
|
|
59
|
+
diffText = fs.readFileSync(path.join(rootDir, run.artifacts.diff.path), 'utf8');
|
|
60
|
+
} catch {
|
|
61
|
+
diffText = '';
|
|
62
|
+
}
|
|
63
|
+
const files = parseDiff(diffText);
|
|
64
|
+
for (const file of files) {
|
|
65
|
+
if (!file.deleted && SECRET_FILE.test(file.path)) {
|
|
66
|
+
errors.push(`${file.path}: secret-like file is part of the change. FIX: remove it from the change and add it to .gitignore; ship an .env.example instead.`);
|
|
67
|
+
}
|
|
68
|
+
for (const added of file.added) {
|
|
69
|
+
const hit = DEBUG_MARKERS.find((marker) => marker.pattern.test(added.text));
|
|
70
|
+
if (hit) errors.push(`${file.path}:${added.line}: ${hit.label} left in the change. FIX: remove it, re-run the Maker gate and verify.`);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
for (const artifact of templateOnlyArtifacts(rootDir, run)) {
|
|
74
|
+
errors.push(`${artifact}: still a _templateOnly scaffold. FIX: fill it with observed facts and remove _templateOnly, or delete it.`);
|
|
75
|
+
}
|
|
76
|
+
const inDiff = new Set(files.map((file) => file.path));
|
|
77
|
+
const strays = untrackedFiles(rootDir).filter((file) => !inDiff.has(file));
|
|
78
|
+
if (strays.length) {
|
|
79
|
+
const message = `untracked file(s) not part of the reviewed change: ${strays.slice(0, 10).join(', ')}${strays.length > 10 ? ', ...' : ''}. FIX: delete them, add them to .gitignore, or include them and re-run the Maker gate.`;
|
|
80
|
+
(mode === 'UNATTENDED' ? errors : warnings).push(message);
|
|
81
|
+
}
|
|
82
|
+
return { errors, warnings };
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
module.exports = { checkCleanState, untrackedFiles };
|
package/lib/cli-args.js
CHANGED
|
@@ -21,14 +21,16 @@ const WRITER_COMMANDS = Object.freeze({
|
|
|
21
21
|
'run:abandon': { flags: ['--abandon', '--yes', '--json'], valued: ['--run', '--reason', '--by'], positionals: [] },
|
|
22
22
|
bench: {
|
|
23
23
|
flags: ['--json', '--sandbox-home'],
|
|
24
|
-
valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg'],
|
|
24
|
+
valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg', '--without'],
|
|
25
25
|
positionals: []
|
|
26
26
|
},
|
|
27
27
|
migrate: { flags: ['--json', '--dry-run'], valued: ['--from'], positionals: [] },
|
|
28
|
-
|
|
28
|
+
debt: { flags: ['--json'], valued: ['--by', '--note', '--file'], positionals: null },
|
|
29
|
+
outcome: { flags: ['--json'], valued: ['--run', '--source', '--location', '--summary', '--evidence', '--by'], positionals: ['record'] },
|
|
30
|
+
verify: { flags: ['--json', '--serial', '--shell', '--related', '--final', '--fresh', '--baseline'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
|
|
29
31
|
quick: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms', '--mode'], positionals: null },
|
|
30
32
|
review: { flags: ['--json'], valued: ['--run', '--child-id', '--file'], positionals: ['ingest'] },
|
|
31
|
-
learn: { flags: ['--json', '--dry-run', '--prune'], valued: ['--max-age-runs'], positionals:
|
|
33
|
+
learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by'], positionals: null },
|
|
32
34
|
guard: { flags: ['--json'], valued: ['--run', '--tool', '--input', '--path', '--command', '--hook'], positionals: ['check'] },
|
|
33
35
|
hooks: { flags: ['--json'], valued: ['--host'], positionals: ['install', 'uninstall', 'status'] },
|
|
34
36
|
impact: { flags: ['--json', '--force'], valued: ['--run', '--by', '--reason', '--allow', '--remove', '--limit'], positionals: ['map', 'show', 'confirm', 'skip', 'drift'] },
|
package/lib/debt.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tech-debt ledger (C-14, OpenAI pack tech-debt-tracker, L12 "clean up later
|
|
4
|
+
// means never"). Trade-offs the Judge accepted and L3 waivers outlive the Run
|
|
5
|
+
// in .ai-engineering-loop/debt.json and come back when a later change
|
|
6
|
+
// touches the same file.
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
const { atomicWriteJson, runsDir } = require('./run-state.js');
|
|
11
|
+
const { parseClaimedVsReality, parseLayerCell } = require('./claimed-vs-reality.js');
|
|
12
|
+
|
|
13
|
+
const DEBT_FILE = path.join('.ai-engineering-loop', 'debt.json');
|
|
14
|
+
|
|
15
|
+
function debtPath(rootDir) {
|
|
16
|
+
return path.join(rootDir, DEBT_FILE);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function loadDebt(rootDir) {
|
|
20
|
+
try {
|
|
21
|
+
const value = JSON.parse(fs.readFileSync(debtPath(rootDir), 'utf8'));
|
|
22
|
+
return Array.isArray(value.items) ? value : { schemaVersion: 1, items: [] };
|
|
23
|
+
} catch {
|
|
24
|
+
return { schemaVersion: 1, items: [] };
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const fileOf = (location) => String(location || '').replace(/:\d+(?::\d+)?$/, '');
|
|
29
|
+
|
|
30
|
+
// Debt items for one delivered Run: VALID MEDIUM/LOW findings the Judge
|
|
31
|
+
// accepted as trade-offs, and claimed-vs-reality rows with an L3 waiver.
|
|
32
|
+
function collectRunDebt(rootDir, runId, { now = new Date() } = {}) {
|
|
33
|
+
const items = [];
|
|
34
|
+
const dir = path.join(runsDir(rootDir), runId);
|
|
35
|
+
let ledger = null;
|
|
36
|
+
try {
|
|
37
|
+
ledger = JSON.parse(fs.readFileSync(path.join(dir, 'findings.json'), 'utf8'));
|
|
38
|
+
} catch {
|
|
39
|
+
ledger = null;
|
|
40
|
+
}
|
|
41
|
+
for (const finding of ledger?.findings || []) {
|
|
42
|
+
if (finding.validity !== 'VALID' || !['MEDIUM', 'LOW'].includes(finding.severity)) continue;
|
|
43
|
+
items.push({
|
|
44
|
+
id: `${runId}:${finding.id}`,
|
|
45
|
+
runId,
|
|
46
|
+
kind: 'accepted-finding',
|
|
47
|
+
file: fileOf(finding.location),
|
|
48
|
+
location: finding.location,
|
|
49
|
+
debt: finding.failureScenario,
|
|
50
|
+
whyDeferred: `Judge accepted ${finding.severity} ${finding.axis || 'spec'} finding as a trade-off`,
|
|
51
|
+
risk: finding.severity,
|
|
52
|
+
nextTrigger: `next change touching ${fileOf(finding.location)}`,
|
|
53
|
+
status: 'open',
|
|
54
|
+
createdAt: now.toISOString()
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
let markdown = '';
|
|
58
|
+
try {
|
|
59
|
+
markdown = fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'tasks', 'claimed-vs-reality.md'), 'utf8');
|
|
60
|
+
} catch {
|
|
61
|
+
markdown = '';
|
|
62
|
+
}
|
|
63
|
+
for (const row of parseClaimedVsReality(markdown).rows) {
|
|
64
|
+
const waiver = parseLayerCell(row.layer).waiver;
|
|
65
|
+
if (!waiver) continue;
|
|
66
|
+
const ac = (String(row.ac).match(/\bAC-\d+\b/) || [row.ac.slice(0, 20)])[0];
|
|
67
|
+
items.push({
|
|
68
|
+
id: `${runId}:${ac}:L3`,
|
|
69
|
+
runId,
|
|
70
|
+
kind: 'l3-waiver',
|
|
71
|
+
file: null,
|
|
72
|
+
location: ac,
|
|
73
|
+
debt: `${ac} shipped without end-to-end (L3) evidence`,
|
|
74
|
+
whyDeferred: waiver,
|
|
75
|
+
risk: 'MEDIUM',
|
|
76
|
+
nextTrigger: 'when an e2e command is added to verification.md',
|
|
77
|
+
status: 'open',
|
|
78
|
+
createdAt: now.toISOString()
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
return items;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function recordRunDebt(rootDir, runId, options) {
|
|
85
|
+
const ledger = loadDebt(rootDir);
|
|
86
|
+
const known = new Set(ledger.items.map((item) => item.id));
|
|
87
|
+
const added = collectRunDebt(rootDir, runId, options).filter((item) => !known.has(item.id));
|
|
88
|
+
if (!added.length) return { added: 0, path: DEBT_FILE.split(path.sep).join('/') };
|
|
89
|
+
atomicWriteJson(debtPath(rootDir), { schemaVersion: 1, items: [...ledger.items, ...added] });
|
|
90
|
+
return { added: added.length, path: DEBT_FILE.split(path.sep).join('/') };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Open debt on any of `files` (plus file-less waivers when `includeGlobal`).
|
|
94
|
+
function openDebtFor(rootDir, files = [], { includeGlobal = false } = {}) {
|
|
95
|
+
const wanted = new Set(files);
|
|
96
|
+
return loadDebt(rootDir).items.filter((item) => item.status === 'open'
|
|
97
|
+
&& ((item.file && wanted.has(item.file)) || (includeGlobal && !item.file)));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function closeDebt(rootDir, id, { by, note } = {}) {
|
|
101
|
+
if (!String(by || '').trim()) throw Object.assign(new Error('debt close needs --by <name>'), { code: 'INVALID_USAGE' });
|
|
102
|
+
const ledger = loadDebt(rootDir);
|
|
103
|
+
const item = ledger.items.find((entry) => entry.id === id);
|
|
104
|
+
if (!item) throw Object.assign(new Error(`No debt item ${id}`), { code: 'NOT_FOUND' });
|
|
105
|
+
Object.assign(item, { status: 'closed', closedBy: by, closedAt: new Date().toISOString(), ...(note ? { note } : {}) });
|
|
106
|
+
atomicWriteJson(debtPath(rootDir), ledger);
|
|
107
|
+
return item;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
module.exports = { DEBT_FILE, closeDebt, collectRunDebt, loadDebt, openDebtFor, recordRunDebt };
|
package/lib/features.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Project feature ledger (C-15, learn-harness-engineering L07/L08): the
|
|
4
|
+
// cross-Run list of what the project must do. Each feature is a triple
|
|
5
|
+
// (behavior, verify, status). Status moves only through Run transitions:
|
|
6
|
+
// a frozen Goal with `featureId` makes it `active`, delivery makes it
|
|
7
|
+
// `passing` with the Run as evidence. `next` picks the next feature and
|
|
8
|
+
// reports VCR = verified / activated.
|
|
9
|
+
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { atomicWriteJson, loadRun } = require('./run-state.js');
|
|
13
|
+
|
|
14
|
+
const FEATURES_FILE = path.join('.ai-engineering-loop', 'features.json');
|
|
15
|
+
const STATUSES = Object.freeze(['not_started', 'active', 'blocked', 'passing']);
|
|
16
|
+
|
|
17
|
+
function featuresPath(rootDir) {
|
|
18
|
+
return path.join(rootDir, FEATURES_FILE);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function loadFeatures(rootDir) {
|
|
22
|
+
try {
|
|
23
|
+
const value = JSON.parse(fs.readFileSync(featuresPath(rootDir), 'utf8'));
|
|
24
|
+
return Array.isArray(value.features) ? value : null;
|
|
25
|
+
} catch {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function save(rootDir, document) {
|
|
31
|
+
atomicWriteJson(featuresPath(rootDir), document);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function transition(rootDir, featureId, to, { runId, now = new Date() } = {}) {
|
|
35
|
+
const document = loadFeatures(rootDir);
|
|
36
|
+
const feature = document?.features.find((item) => item.id === featureId);
|
|
37
|
+
if (!feature) return null;
|
|
38
|
+
if (feature.status === 'passing' && to !== 'passing') return feature; // passing is final
|
|
39
|
+
feature.status = to;
|
|
40
|
+
feature.history = [...(feature.history || []), { to, runId, at: now.toISOString() }];
|
|
41
|
+
if (to === 'passing') feature.evidence = { runId, deliveredAt: now.toISOString() };
|
|
42
|
+
save(rootDir, document);
|
|
43
|
+
return feature;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Called by the Goal gate.
|
|
47
|
+
function onGoalFrozen(rootDir, run, goal) {
|
|
48
|
+
if (!goal?.featureId) return null;
|
|
49
|
+
const feature = loadFeatures(rootDir)?.features.find((item) => item.id === goal.featureId);
|
|
50
|
+
if (!feature || feature.status === 'passing') return feature || null;
|
|
51
|
+
return transition(rootDir, goal.featureId, 'active', { runId: run.runId });
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Called by the delivery gate.
|
|
55
|
+
function onDelivered(rootDir, run, goal) {
|
|
56
|
+
if (!goal?.featureId) return null;
|
|
57
|
+
return transition(rootDir, goal.featureId, 'passing', { runId: run.runId });
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function vcr(document) {
|
|
61
|
+
const features = document?.features || [];
|
|
62
|
+
const activated = features.filter((item) => item.status !== 'not_started').length;
|
|
63
|
+
const verified = features.filter((item) => item.status === 'passing').length;
|
|
64
|
+
return { verified, activated, value: activated ? Math.round((verified / activated) * 1000) / 1000 : 1 };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function nextFeature(document) {
|
|
68
|
+
const features = document?.features || [];
|
|
69
|
+
const passing = new Set(features.filter((item) => item.status === 'passing').map((item) => item.id));
|
|
70
|
+
return features
|
|
71
|
+
.filter((item) => item.status === 'not_started' && (item.dependsOn || []).every((id) => passing.has(id)))
|
|
72
|
+
.sort((a, b) => (a.priority ?? 999) - (b.priority ?? 999))[0] || null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// A `passing` feature must point at a delivered Run whose Goal names it.
|
|
76
|
+
function auditFeatures(rootDir, document = loadFeatures(rootDir)) {
|
|
77
|
+
const problems = [];
|
|
78
|
+
for (const feature of document?.features || []) {
|
|
79
|
+
if (!STATUSES.includes(feature.status)) problems.push(`${feature.id}: unknown status ${feature.status}`);
|
|
80
|
+
if (feature.status !== 'passing') continue;
|
|
81
|
+
const runId = feature.evidence?.runId;
|
|
82
|
+
let run = null;
|
|
83
|
+
try {
|
|
84
|
+
run = runId ? loadRun(rootDir, runId) : null;
|
|
85
|
+
} catch {
|
|
86
|
+
run = null;
|
|
87
|
+
}
|
|
88
|
+
if (!run || run.state !== 'DELIVERED') {
|
|
89
|
+
problems.push(`${feature.id}: marked passing without a delivered Run as evidence. FIX: set it back to not_started and deliver it through a Run whose Goal has featureId "${feature.id}".`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return problems;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
module.exports = { FEATURES_FILE, STATUSES, auditFeatures, loadFeatures, nextFeature, onDelivered, onGoalFrozen, vcr };
|