ai-engineering-loop 2.6.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +11 -0
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +11 -9
- package/.claude/agents/devil-advocate.md +6 -1
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +6 -1
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +16 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +6 -1
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +262 -20
- package/core/bench.md +15 -0
- package/core/goal-contract.md +35 -0
- package/core/verification-loop.md +78 -1
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +63 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/calibration.js +112 -0
- package/lib/claimed-vs-reality.js +106 -12
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/debt.js +110 -0
- package/lib/features.js +95 -0
- package/lib/gates.js +234 -41
- package/lib/harness-audit.js +124 -0
- package/lib/no-progress.js +6 -1
- package/lib/review-ingest.js +54 -2
- package/lib/rules.js +112 -0
- package/lib/stats.js +90 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +64 -0
- package/lib/verification-recorder.js +17 -5
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +31 -0
- package/schemas/verdict.schema.json +2 -1
package/lib/cli-args.js
CHANGED
|
@@ -21,14 +21,16 @@ const WRITER_COMMANDS = Object.freeze({
|
|
|
21
21
|
'run:abandon': { flags: ['--abandon', '--yes', '--json'], valued: ['--run', '--reason', '--by'], positionals: [] },
|
|
22
22
|
bench: {
|
|
23
23
|
flags: ['--json', '--sandbox-home'],
|
|
24
|
-
valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg'],
|
|
24
|
+
valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg', '--without'],
|
|
25
25
|
positionals: []
|
|
26
26
|
},
|
|
27
27
|
migrate: { flags: ['--json', '--dry-run'], valued: ['--from'], positionals: [] },
|
|
28
|
-
|
|
28
|
+
debt: { flags: ['--json'], valued: ['--by', '--note', '--file'], positionals: null },
|
|
29
|
+
outcome: { flags: ['--json'], valued: ['--run', '--source', '--location', '--summary', '--evidence', '--by'], positionals: ['record'] },
|
|
30
|
+
verify: { flags: ['--json', '--serial', '--shell', '--related', '--final', '--fresh', '--baseline'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
|
|
29
31
|
quick: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms', '--mode'], positionals: null },
|
|
30
32
|
review: { flags: ['--json'], valued: ['--run', '--child-id', '--file'], positionals: ['ingest'] },
|
|
31
|
-
learn: { flags: ['--json', '--dry-run', '--prune'], valued: ['--max-age-runs'], positionals:
|
|
33
|
+
learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by'], positionals: null },
|
|
32
34
|
guard: { flags: ['--json'], valued: ['--run', '--tool', '--input', '--path', '--command', '--hook'], positionals: ['check'] },
|
|
33
35
|
hooks: { flags: ['--json'], valued: ['--host'], positionals: ['install', 'uninstall', 'status'] },
|
|
34
36
|
impact: { flags: ['--json', '--force'], valued: ['--run', '--by', '--reason', '--allow', '--remove', '--limit'], positionals: ['map', 'show', 'confirm', 'skip', 'drift'] },
|
package/lib/debt.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Tech-debt ledger (C-14, OpenAI pack tech-debt-tracker, L12 "clean up later
|
|
4
|
+
// means never"). Trade-offs the Judge accepted and L3 waivers outlive the Run
|
|
5
|
+
// in .ai-engineering-loop/debt.json and come back when a later change
|
|
6
|
+
// touches the same file.
|
|
7
|
+
|
|
8
|
+
const fs = require('fs');
|
|
9
|
+
const path = require('path');
|
|
10
|
+
const { atomicWriteJson, runsDir } = require('./run-state.js');
|
|
11
|
+
const { parseClaimedVsReality, parseLayerCell } = require('./claimed-vs-reality.js');
|
|
12
|
+
|
|
13
|
+
const DEBT_FILE = path.join('.ai-engineering-loop', 'debt.json');
|
|
14
|
+
|
|
15
|
+
function debtPath(rootDir) {
|
|
16
|
+
return path.join(rootDir, DEBT_FILE);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function loadDebt(rootDir) {
|
|
20
|
+
try {
|
|
21
|
+
const value = JSON.parse(fs.readFileSync(debtPath(rootDir), 'utf8'));
|
|
22
|
+
return Array.isArray(value.items) ? value : { schemaVersion: 1, items: [] };
|
|
23
|
+
} catch {
|
|
24
|
+
return { schemaVersion: 1, items: [] };
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const fileOf = (location) => String(location || '').replace(/:\d+(?::\d+)?$/, '');
|
|
29
|
+
|
|
30
|
+
// Debt items for one delivered Run: VALID MEDIUM/LOW findings the Judge
|
|
31
|
+
// accepted as trade-offs, and claimed-vs-reality rows with an L3 waiver.
|
|
32
|
+
function collectRunDebt(rootDir, runId, { now = new Date() } = {}) {
|
|
33
|
+
const items = [];
|
|
34
|
+
const dir = path.join(runsDir(rootDir), runId);
|
|
35
|
+
let ledger = null;
|
|
36
|
+
try {
|
|
37
|
+
ledger = JSON.parse(fs.readFileSync(path.join(dir, 'findings.json'), 'utf8'));
|
|
38
|
+
} catch {
|
|
39
|
+
ledger = null;
|
|
40
|
+
}
|
|
41
|
+
for (const finding of ledger?.findings || []) {
|
|
42
|
+
if (finding.validity !== 'VALID' || !['MEDIUM', 'LOW'].includes(finding.severity)) continue;
|
|
43
|
+
items.push({
|
|
44
|
+
id: `${runId}:${finding.id}`,
|
|
45
|
+
runId,
|
|
46
|
+
kind: 'accepted-finding',
|
|
47
|
+
file: fileOf(finding.location),
|
|
48
|
+
location: finding.location,
|
|
49
|
+
debt: finding.failureScenario,
|
|
50
|
+
whyDeferred: `Judge accepted ${finding.severity} ${finding.axis || 'spec'} finding as a trade-off`,
|
|
51
|
+
risk: finding.severity,
|
|
52
|
+
nextTrigger: `next change touching ${fileOf(finding.location)}`,
|
|
53
|
+
status: 'open',
|
|
54
|
+
createdAt: now.toISOString()
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
let markdown = '';
|
|
58
|
+
try {
|
|
59
|
+
markdown = fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'tasks', 'claimed-vs-reality.md'), 'utf8');
|
|
60
|
+
} catch {
|
|
61
|
+
markdown = '';
|
|
62
|
+
}
|
|
63
|
+
for (const row of parseClaimedVsReality(markdown).rows) {
|
|
64
|
+
const waiver = parseLayerCell(row.layer).waiver;
|
|
65
|
+
if (!waiver) continue;
|
|
66
|
+
const ac = (String(row.ac).match(/\bAC-\d+\b/) || [row.ac.slice(0, 20)])[0];
|
|
67
|
+
items.push({
|
|
68
|
+
id: `${runId}:${ac}:L3`,
|
|
69
|
+
runId,
|
|
70
|
+
kind: 'l3-waiver',
|
|
71
|
+
file: null,
|
|
72
|
+
location: ac,
|
|
73
|
+
debt: `${ac} shipped without end-to-end (L3) evidence`,
|
|
74
|
+
whyDeferred: waiver,
|
|
75
|
+
risk: 'MEDIUM',
|
|
76
|
+
nextTrigger: 'when an e2e command is added to verification.md',
|
|
77
|
+
status: 'open',
|
|
78
|
+
createdAt: now.toISOString()
|
|
79
|
+
});
|
|
80
|
+
}
|
|
81
|
+
return items;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function recordRunDebt(rootDir, runId, options) {
|
|
85
|
+
const ledger = loadDebt(rootDir);
|
|
86
|
+
const known = new Set(ledger.items.map((item) => item.id));
|
|
87
|
+
const added = collectRunDebt(rootDir, runId, options).filter((item) => !known.has(item.id));
|
|
88
|
+
if (!added.length) return { added: 0, path: DEBT_FILE.split(path.sep).join('/') };
|
|
89
|
+
atomicWriteJson(debtPath(rootDir), { schemaVersion: 1, items: [...ledger.items, ...added] });
|
|
90
|
+
return { added: added.length, path: DEBT_FILE.split(path.sep).join('/') };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Open debt on any of `files` (plus file-less waivers when `includeGlobal`).
|
|
94
|
+
function openDebtFor(rootDir, files = [], { includeGlobal = false } = {}) {
|
|
95
|
+
const wanted = new Set(files);
|
|
96
|
+
return loadDebt(rootDir).items.filter((item) => item.status === 'open'
|
|
97
|
+
&& ((item.file && wanted.has(item.file)) || (includeGlobal && !item.file)));
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function closeDebt(rootDir, id, { by, note } = {}) {
|
|
101
|
+
if (!String(by || '').trim()) throw Object.assign(new Error('debt close needs --by <name>'), { code: 'INVALID_USAGE' });
|
|
102
|
+
const ledger = loadDebt(rootDir);
|
|
103
|
+
const item = ledger.items.find((entry) => entry.id === id);
|
|
104
|
+
if (!item) throw Object.assign(new Error(`No debt item ${id}`), { code: 'NOT_FOUND' });
|
|
105
|
+
Object.assign(item, { status: 'closed', closedBy: by, closedAt: new Date().toISOString(), ...(note ? { note } : {}) });
|
|
106
|
+
atomicWriteJson(debtPath(rootDir), ledger);
|
|
107
|
+
return item;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
module.exports = { DEBT_FILE, closeDebt, collectRunDebt, loadDebt, openDebtFor, recordRunDebt };
|
package/lib/features.js
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Project feature ledger (C-15, learn-harness-engineering L07/L08): the
|
|
4
|
+
// cross-Run list of what the project must do. Each feature is a triple
|
|
5
|
+
// (behavior, verify, status). Status moves only through Run transitions:
|
|
6
|
+
// a frozen Goal with `featureId` makes it `active`, delivery makes it
|
|
7
|
+
// `passing` with the Run as evidence. `next` picks the next feature and
|
|
8
|
+
// reports VCR = verified / activated.
|
|
9
|
+
|
|
10
|
+
const fs = require('fs');
|
|
11
|
+
const path = require('path');
|
|
12
|
+
const { atomicWriteJson, loadRun } = require('./run-state.js');
|
|
13
|
+
|
|
14
|
+
const FEATURES_FILE = path.join('.ai-engineering-loop', 'features.json');
|
|
15
|
+
const STATUSES = Object.freeze(['not_started', 'active', 'blocked', 'passing']);
|
|
16
|
+
|
|
17
|
+
function featuresPath(rootDir) {
|
|
18
|
+
return path.join(rootDir, FEATURES_FILE);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function loadFeatures(rootDir) {
|
|
22
|
+
try {
|
|
23
|
+
const value = JSON.parse(fs.readFileSync(featuresPath(rootDir), 'utf8'));
|
|
24
|
+
return Array.isArray(value.features) ? value : null;
|
|
25
|
+
} catch {
|
|
26
|
+
return null;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function save(rootDir, document) {
|
|
31
|
+
atomicWriteJson(featuresPath(rootDir), document);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function transition(rootDir, featureId, to, { runId, now = new Date() } = {}) {
|
|
35
|
+
const document = loadFeatures(rootDir);
|
|
36
|
+
const feature = document?.features.find((item) => item.id === featureId);
|
|
37
|
+
if (!feature) return null;
|
|
38
|
+
if (feature.status === 'passing' && to !== 'passing') return feature; // passing is final
|
|
39
|
+
feature.status = to;
|
|
40
|
+
feature.history = [...(feature.history || []), { to, runId, at: now.toISOString() }];
|
|
41
|
+
if (to === 'passing') feature.evidence = { runId, deliveredAt: now.toISOString() };
|
|
42
|
+
save(rootDir, document);
|
|
43
|
+
return feature;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// Called by the Goal gate.
|
|
47
|
+
function onGoalFrozen(rootDir, run, goal) {
|
|
48
|
+
if (!goal?.featureId) return null;
|
|
49
|
+
const feature = loadFeatures(rootDir)?.features.find((item) => item.id === goal.featureId);
|
|
50
|
+
if (!feature || feature.status === 'passing') return feature || null;
|
|
51
|
+
return transition(rootDir, goal.featureId, 'active', { runId: run.runId });
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Called by the delivery gate.
|
|
55
|
+
function onDelivered(rootDir, run, goal) {
|
|
56
|
+
if (!goal?.featureId) return null;
|
|
57
|
+
return transition(rootDir, goal.featureId, 'passing', { runId: run.runId });
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function vcr(document) {
|
|
61
|
+
const features = document?.features || [];
|
|
62
|
+
const activated = features.filter((item) => item.status !== 'not_started').length;
|
|
63
|
+
const verified = features.filter((item) => item.status === 'passing').length;
|
|
64
|
+
return { verified, activated, value: activated ? Math.round((verified / activated) * 1000) / 1000 : 1 };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function nextFeature(document) {
|
|
68
|
+
const features = document?.features || [];
|
|
69
|
+
const passing = new Set(features.filter((item) => item.status === 'passing').map((item) => item.id));
|
|
70
|
+
return features
|
|
71
|
+
.filter((item) => item.status === 'not_started' && (item.dependsOn || []).every((id) => passing.has(id)))
|
|
72
|
+
.sort((a, b) => (a.priority ?? 999) - (b.priority ?? 999))[0] || null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// A `passing` feature must point at a delivered Run whose Goal names it.
|
|
76
|
+
function auditFeatures(rootDir, document = loadFeatures(rootDir)) {
|
|
77
|
+
const problems = [];
|
|
78
|
+
for (const feature of document?.features || []) {
|
|
79
|
+
if (!STATUSES.includes(feature.status)) problems.push(`${feature.id}: unknown status ${feature.status}`);
|
|
80
|
+
if (feature.status !== 'passing') continue;
|
|
81
|
+
const runId = feature.evidence?.runId;
|
|
82
|
+
let run = null;
|
|
83
|
+
try {
|
|
84
|
+
run = runId ? loadRun(rootDir, runId) : null;
|
|
85
|
+
} catch {
|
|
86
|
+
run = null;
|
|
87
|
+
}
|
|
88
|
+
if (!run || run.state !== 'DELIVERED') {
|
|
89
|
+
problems.push(`${feature.id}: marked passing without a delivered Run as evidence. FIX: set it back to not_started and deliver it through a Run whose Goal has featureId "${feature.id}".`);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
return problems;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
module.exports = { FEATURES_FILE, STATUSES, auditFeatures, loadFeatures, nextFeature, onDelivered, onGoalFrozen, vcr };
|