ai-engineering-loop 2.10.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +1 -0
- package/.agents/skills/ael/references/loop-risky.md +1 -0
- package/.claude/agents/devil-advocate.md +2 -1
- package/.claude/agents/judge.md +2 -0
- package/.grok/agents/devil-advocate.md +2 -1
- package/.grok/agents/judge.md +2 -0
- package/CHANGELOG.md +17 -0
- package/README.md +80 -75
- package/agents/shared/devil-advocate.body.md +2 -1
- package/agents/shared/judge.body.md +2 -0
- package/bin/ai-engineering-loop.js +50 -0
- package/core/controlled-workflow-runtime.md +1 -0
- package/core/goal-contract.md +4 -0
- package/core/iteration-policy.md +4 -0
- package/core/verification-loop.md +25 -0
- package/docs/images/ael-hero.svg +19 -0
- package/docs/images/readme-banner.png +0 -0
- package/docs/images/social-preview.png +0 -0
- package/docs/images/studio-hero-orange.svg +136 -0
- package/lib/ablation.js +1 -1
- package/lib/cli-args.js +3 -1
- package/lib/finding-proofs.js +72 -0
- package/lib/gates.js +36 -0
- package/lib/harness-audit.js +20 -0
- package/lib/interventions.js +56 -0
- package/lib/measurements.js +51 -0
- package/lib/orchestration.js +1 -0
- package/lib/recap-html.js +2 -2
- package/lib/recap.js +2 -0
- package/lib/rules.js +51 -1
- package/lib/run-state.js +2 -1
- package/lib/safety-fact.js +64 -0
- package/lib/stats.js +9 -0
- package/lib/verification-lanes.js +23 -0
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +4 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="1600" height="900" viewBox="0 0 1600 900" role="img" aria-labelledby="title desc">
|
|
2
|
+
<title id="title">AI Engineering Loop Workflow Studio</title>
|
|
3
|
+
<desc id="desc">A dark Workflow Studio live-run view: a stage rail from goal to deliver with review in progress, a pending question banner, compliance chips, a short timeline, and a Start with AI Agent button.</desc>
|
|
4
|
+
<defs>
|
|
5
|
+
<linearGradient id="bg" x1="0" y1="0" x2="1" y2="1">
|
|
6
|
+
<stop stop-color="#111512"/>
|
|
7
|
+
<stop offset="1" stop-color="#080a09"/>
|
|
8
|
+
</linearGradient>
|
|
9
|
+
<linearGradient id="glow" x1="0" y1="0" x2="1" y2="1">
|
|
10
|
+
<stop stop-color="#fc6404" stop-opacity=".28"/>
|
|
11
|
+
<stop offset="1" stop-color="#fc6404" stop-opacity="0"/>
|
|
12
|
+
</linearGradient>
|
|
13
|
+
<filter id="shadow" x="-30%" y="-30%" width="160%" height="160%">
|
|
14
|
+
<feDropShadow dx="0" dy="18" stdDeviation="24" flood-color="#000" flood-opacity=".45"/>
|
|
15
|
+
</filter>
|
|
16
|
+
<filter id="soft">
|
|
17
|
+
<feGaussianBlur stdDeviation="55"/>
|
|
18
|
+
</filter>
|
|
19
|
+
<pattern id="grid" width="32" height="32" patternUnits="userSpaceOnUse">
|
|
20
|
+
<path d="M32 0H0V32" fill="none" stroke="#fff" stroke-opacity=".035"/>
|
|
21
|
+
</pattern>
|
|
22
|
+
<marker id="arrow" markerWidth="10" markerHeight="10" refX="8" refY="5" orient="auto">
|
|
23
|
+
<path d="M0 0L10 5L0 10Z" fill="#5b625d"/>
|
|
24
|
+
</marker>
|
|
25
|
+
<style>
|
|
26
|
+
.ui{font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}
|
|
27
|
+
.mono{font-family:"SFMono-Regular",Consolas,monospace}
|
|
28
|
+
.eyebrow{font-size:14px;font-weight:700;letter-spacing:2.4px;fill:#7f8881}
|
|
29
|
+
.node-title{font-size:22px;font-weight:700;fill:#f4f6f4}
|
|
30
|
+
.node-copy{font-size:14px;fill:#888f8a}
|
|
31
|
+
.tiny{font-size:12px;fill:#707873}
|
|
32
|
+
.flow{fill:none;stroke:#5b625d;stroke-width:3;marker-end:url(#arrow)}
|
|
33
|
+
</style>
|
|
34
|
+
</defs>
|
|
35
|
+
|
|
36
|
+
<rect width="1600" height="900" fill="url(#bg)"/>
|
|
37
|
+
<circle cx="1240" cy="70" r="330" fill="#fc6404" opacity=".12" filter="url(#soft)"/>
|
|
38
|
+
<circle cx="280" cy="880" r="260" fill="#fc6404" opacity=".08" filter="url(#soft)"/>
|
|
39
|
+
|
|
40
|
+
<g class="ui">
|
|
41
|
+
<text x="110" y="112" font-size="16" font-weight="800" letter-spacing="2" fill="#fcaa72">AI ENGINEERING LOOP</text>
|
|
42
|
+
<text x="110" y="188" font-size="64" font-weight="750" letter-spacing="-2.8" fill="#f7f9f7">From idea to trusted code.</text>
|
|
43
|
+
<text x="112" y="232" font-size="21" fill="#9da49f">Your AI handles the workflow. You keep the clarity and control.</text>
|
|
44
|
+
</g>
|
|
45
|
+
|
|
46
|
+
<g filter="url(#shadow)">
|
|
47
|
+
<rect x="105" y="290" width="1390" height="510" rx="24" fill="#151816" stroke="#303632"/>
|
|
48
|
+
<rect x="105" y="290" width="1390" height="64" rx="24" fill="#1b1f1c"/>
|
|
49
|
+
<path d="M105 330V354H1495V330" fill="#1b1f1c"/>
|
|
50
|
+
<circle cx="140" cy="322" r="6" fill="#4d544f"/>
|
|
51
|
+
<circle cx="160" cy="322" r="6" fill="#4d544f"/>
|
|
52
|
+
<circle cx="180" cy="322" r="6" fill="#fc6404"/>
|
|
53
|
+
<rect x="680" y="307" width="240" height="30" rx="15" fill="#111411"/>
|
|
54
|
+
<circle cx="704" cy="322" r="6" fill="#fc6404"/>
|
|
55
|
+
<text class="ui tiny" x="720" y="326">Workflow Studio</text>
|
|
56
|
+
|
|
57
|
+
<rect x="105" y="354" width="72" height="446" fill="#101311"/>
|
|
58
|
+
<rect x="123" y="381" width="36" height="36" rx="10" fill="#fc6404"/>
|
|
59
|
+
<path d="M129 400h6l3-8 5 16 3-8h6" fill="none" stroke="#fff" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"/>
|
|
60
|
+
<circle cx="141" cy="449" r="15" fill="#252a26"/>
|
|
61
|
+
<path d="M134 444h14M134 449h14M134 454h9" stroke="#818982" stroke-width="2" stroke-linecap="round"/>
|
|
62
|
+
<circle cx="141" cy="500" r="15" fill="#252a26"/>
|
|
63
|
+
<circle cx="135" cy="505" r="3" fill="none" stroke="#818982" stroke-width="2"/>
|
|
64
|
+
<circle cx="147" cy="495" r="3" fill="none" stroke="#818982" stroke-width="2"/>
|
|
65
|
+
<path d="M137.5 502.5l7-5" stroke="#818982" stroke-width="2"/>
|
|
66
|
+
|
|
67
|
+
<rect x="177" y="354" width="1318" height="446" fill="#111411"/>
|
|
68
|
+
<rect x="177" y="354" width="1318" height="446" fill="url(#grid)"/>
|
|
69
|
+
<ellipse cx="1220" cy="410" rx="300" ry="210" fill="url(#glow)"/>
|
|
70
|
+
|
|
71
|
+
<text class="ui eyebrow" x="222" y="397">LIVE RUN / PRODUCT LAUNCH</text>
|
|
72
|
+
<text class="ui tiny" x="222" y="420">T1 · ASSISTED · iteration 1 / 3 · budget 38%</text>
|
|
73
|
+
<rect x="1180" y="375" width="262" height="42" rx="9" fill="#1d221e" stroke="#343a35"/>
|
|
74
|
+
<circle cx="1206" cy="396" r="6" fill="#fc6404"/>
|
|
75
|
+
<circle cx="1206" cy="396" r="11" fill="none" stroke="#fc6404" stroke-opacity=".35" stroke-width="2"/>
|
|
76
|
+
<text class="ui tiny" x="1224" y="400">live: watch · heartbeat 3s ago</text>
|
|
77
|
+
|
|
78
|
+
<rect x="222" y="444" width="1220" height="50" rx="12" fill="#1a2418" stroke="#2f5a22"/>
|
|
79
|
+
<rect x="222" y="444" width="6" height="50" rx="3" fill="#fcaa72"/>
|
|
80
|
+
<text class="ui" x="248" y="474" font-size="13" font-weight="800" letter-spacing="1.4" fill="#fcaa72">QUESTION</text>
|
|
81
|
+
<text class="ui" x="340" y="474" font-size="15" fill="#e4e8e5">Should an empty cart show the “Continue shopping” link?</text>
|
|
82
|
+
<rect x="1316" y="456" width="110" height="26" rx="7" fill="#1d221e" stroke="#3b6b2c"/>
|
|
83
|
+
<text class="ui" x="1371" y="474" text-anchor="middle" font-size="12" font-weight="750" fill="#c9f0bd">Answer</text>
|
|
84
|
+
|
|
85
|
+
<line x1="300" y1="560" x2="1365" y2="560" stroke="#343a35" stroke-width="4" stroke-linecap="round"/>
|
|
86
|
+
<line x1="300" y1="560" x2="939" y2="560" stroke="#fc6404" stroke-width="4" stroke-linecap="round"/>
|
|
87
|
+
<circle cx="300" cy="560" r="18" fill="#fc6404"/>
|
|
88
|
+
<path d="M293 560l5 5 10-11" fill="none" stroke="#fff" stroke-width="3.5" stroke-linecap="round" stroke-linejoin="round"/>
|
|
89
|
+
<text class="ui" x="300" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#c7ccc8">GOAL</text>
|
|
90
|
+
<circle cx="513" cy="560" r="18" fill="#fc6404"/>
|
|
91
|
+
<path d="M506 560l5 5 10-11" fill="none" stroke="#fff" stroke-width="3.5" stroke-linecap="round" stroke-linejoin="round"/>
|
|
92
|
+
<text class="ui" x="513" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#c7ccc8">MAKER</text>
|
|
93
|
+
<circle cx="726" cy="560" r="18" fill="#fc6404"/>
|
|
94
|
+
<path d="M719 560l5 5 10-11" fill="none" stroke="#fff" stroke-width="3.5" stroke-linecap="round" stroke-linejoin="round"/>
|
|
95
|
+
<text class="ui" x="726" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#c7ccc8">VERIFY</text>
|
|
96
|
+
<circle cx="939" cy="560" r="28" fill="#fc6404" opacity=".18"/>
|
|
97
|
+
<circle cx="939" cy="560" r="18" fill="#1c211d" stroke="#fc6404" stroke-width="4"/>
|
|
98
|
+
<circle cx="939" cy="560" r="6" fill="#fc6404"/>
|
|
99
|
+
<text class="ui" x="939" y="608" text-anchor="middle" font-size="13" font-weight="800" letter-spacing="1.6" fill="#f4f6f4">REVIEW</text>
|
|
100
|
+
<circle cx="1152" cy="560" r="18" fill="#1c211d" stroke="#3b433d" stroke-width="3"/>
|
|
101
|
+
<text class="ui" x="1152" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#707873">JUDGE</text>
|
|
102
|
+
<circle cx="1365" cy="560" r="18" fill="#1c211d" stroke="#3b433d" stroke-width="3"/>
|
|
103
|
+
<text class="ui" x="1365" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#707873">DELIVER</text>
|
|
104
|
+
<text class="ui tiny" x="939" y="628" text-anchor="middle" fill="#fcaa72">Devil's Advocate running</text>
|
|
105
|
+
|
|
106
|
+
<rect x="222" y="652" width="600" height="62" rx="12" fill="#191d1a" stroke="#2b312c"/>
|
|
107
|
+
<text class="ui eyebrow" x="246" y="688" font-size="12">COMPLIANCE</text>
|
|
108
|
+
<rect x="384" y="667" width="134" height="32" rx="16" fill="#1c2a18" stroke="#2f5a22"/>
|
|
109
|
+
<circle cx="402" cy="683" r="5" fill="#fc6404"/>
|
|
110
|
+
<text class="ui" x="414" y="688" font-size="13" fill="#c9f0bd">Guard 0 blocks</text>
|
|
111
|
+
<rect x="526" y="667" width="120" height="32" rx="16" fill="#1c2a18" stroke="#2f5a22"/>
|
|
112
|
+
<circle cx="543" cy="683" r="5" fill="#fc6404"/>
|
|
113
|
+
<text class="ui" x="555" y="688" font-size="13" fill="#c9f0bd">Scope in map</text>
|
|
114
|
+
<rect x="654" y="667" width="148" height="32" rx="16" fill="#1d221e" stroke="#343a35"/>
|
|
115
|
+
<circle cx="671" cy="683" r="5" fill="#707873"/>
|
|
116
|
+
<text class="ui" x="683" y="688" font-size="13" fill="#9da49f">Receipt pending</text>
|
|
117
|
+
|
|
118
|
+
<rect x="842" y="652" width="600" height="62" rx="12" fill="#191d1a" stroke="#2b312c"/>
|
|
119
|
+
<text class="ui eyebrow" x="866" y="688" font-size="12">TIMELINE</text>
|
|
120
|
+
<text class="mono tiny" x="966" y="677">10:42</text>
|
|
121
|
+
<text class="ui" x="1012" y="677" font-size="13" fill="#c7ccc8">verify · npm test · 511 passed</text>
|
|
122
|
+
<text class="mono tiny" x="966" y="700">10:44</text>
|
|
123
|
+
<text class="ui" x="1012" y="700" font-size="13" fill="#c7ccc8">review · 2 findings, 0 blocking</text>
|
|
124
|
+
|
|
125
|
+
<rect x="224" y="734" width="1218" height="44" rx="12" fill="#191d1a" stroke="#2b312c"/>
|
|
126
|
+
<text class="mono tiny" x="248" y="761">Task</text>
|
|
127
|
+
<text class="ui" x="300" y="761" font-size="14" fill="#c7ccc8">Ship a reliable feature without babysitting every command</text>
|
|
128
|
+
<rect x="1250" y="743" width="174" height="26" rx="7" fill="#fc6404"/>
|
|
129
|
+
<text class="ui" x="1337" y="761" text-anchor="middle" font-size="12" font-weight="750" fill="#fff">Start with AI Agent</text>
|
|
130
|
+
</g>
|
|
131
|
+
|
|
132
|
+
<g class="ui">
|
|
133
|
+
<text x="112" y="852" font-size="15" fill="#6f7771">Live runs · compliance you can see · evidence you can trust</text>
|
|
134
|
+
<text x="1488" y="852" text-anchor="end" font-size="15" font-weight="700" fill="#fc6404">OPEN SOURCE</text>
|
|
135
|
+
</g>
|
|
136
|
+
</svg>
|
package/lib/ablation.js
CHANGED
|
@@ -10,7 +10,7 @@ const fs = require('fs');
|
|
|
10
10
|
const path = require('path');
|
|
11
11
|
const { runsDir } = require('./run-state.js');
|
|
12
12
|
|
|
13
|
-
const COMPONENTS = Object.freeze(['integrity', 'comments', 'l3', 'l1', 'smoke', 'rules']);
|
|
13
|
+
const COMPONENTS = Object.freeze(['integrity', 'comments', 'l3', 'l1', 'smoke', 'rules', 'proofs']);
|
|
14
14
|
const MARKER = path.join('.ai-engineering-loop', 'bench-ablation.json');
|
|
15
15
|
|
|
16
16
|
function requested(env = process.env) {
|
package/lib/cli-args.js
CHANGED
|
@@ -24,6 +24,8 @@ const WRITER_COMMANDS = Object.freeze({
|
|
|
24
24
|
valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg', '--without'],
|
|
25
25
|
positionals: []
|
|
26
26
|
},
|
|
27
|
+
safety: { flags: ['--json', '--live'], valued: ['--run', '--fact', '--cmd', '--level'], positionals: ['record'] },
|
|
28
|
+
measurement: { flags: ['--json'], valued: ['--run', '--ac', '--runs', '--spread', '--effect', '--limiter'], positionals: ['record'] },
|
|
27
29
|
recap: { flags: ['--json', '--open'], valued: ['--days', '--for', '--projects', '--out', '--now', '--lang'], positionals: [] },
|
|
28
30
|
migrate: { flags: ['--json', '--dry-run'], valued: ['--from'], positionals: [] },
|
|
29
31
|
debt: { flags: ['--json'], valued: ['--by', '--note', '--file'], positionals: null },
|
|
@@ -31,7 +33,7 @@ const WRITER_COMMANDS = Object.freeze({
|
|
|
31
33
|
verify: { flags: ['--json', '--serial', '--shell', '--related', '--final', '--fresh', '--baseline'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
|
|
32
34
|
quick: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms', '--mode'], positionals: null },
|
|
33
35
|
review: { flags: ['--json'], valued: ['--run', '--child-id', '--file'], positionals: ['ingest'] },
|
|
34
|
-
learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by', '--scope', '--forbid-import', '--forbid-api', '--except'], positionals: null },
|
|
36
|
+
learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by', '--scope', '--forbid-import', '--forbid-api', '--except', '--prove'], positionals: null },
|
|
35
37
|
guard: { flags: ['--json'], valued: ['--run', '--tool', '--input', '--path', '--command', '--hook'], positionals: ['check'] },
|
|
36
38
|
hooks: { flags: ['--json'], valued: ['--host'], positionals: ['install', 'uninstall', 'status'] },
|
|
37
39
|
impact: { flags: ['--json', '--force'], valued: ['--run', '--by', '--reason', '--allow', '--remove', '--limit'], positionals: ['map', 'show', 'confirm', 'skip', 'drift'] },
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { spawnSync } = require('child_process');
|
|
6
|
+
const { destructiveFindings } = require('./guard.js');
|
|
7
|
+
const { redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
8
|
+
|
|
9
|
+
const PROOF_TIMEOUT_MS = 120000;
|
|
10
|
+
const TAIL = 2000;
|
|
11
|
+
|
|
12
|
+
const tail = (text) => redactHomePaths(redactSecrets(String(text || '')).text).slice(-TAIL);
|
|
13
|
+
|
|
14
|
+
function runProof(rootDir, proof, timeoutMs) {
|
|
15
|
+
const command = String(proof.command || '').trim();
|
|
16
|
+
if (!command) return { status: 'refused', reason: 'proof.command is empty' };
|
|
17
|
+
const destructive = destructiveFindings(command);
|
|
18
|
+
if (destructive.length) return { status: 'refused', reason: destructive.map((item) => item.reason).join('; ') };
|
|
19
|
+
const started = Date.now();
|
|
20
|
+
const result = proof.shell === false
|
|
21
|
+
? spawnSync(command.split(/\s+/)[0], command.split(/\s+/).slice(1), { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs })
|
|
22
|
+
: spawnSync(command, { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs, shell: true });
|
|
23
|
+
const durationMs = Date.now() - started;
|
|
24
|
+
if (result.error?.code === 'ETIMEDOUT' || (result.signal && durationMs >= timeoutMs - 50)) {
|
|
25
|
+
return { status: 'timed_out', durationMs };
|
|
26
|
+
}
|
|
27
|
+
if (result.error) return { status: 'refused', reason: result.error.message, durationMs };
|
|
28
|
+
return {
|
|
29
|
+
status: result.status === 0 ? 'not_reproduced' : 'reproduced',
|
|
30
|
+
exitCode: result.status,
|
|
31
|
+
durationMs,
|
|
32
|
+
stdoutTail: tail(result.stdout),
|
|
33
|
+
stderrTail: tail(result.stderr)
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function runFindingProofs(rootDir, run, { timeoutMs = PROOF_TIMEOUT_MS } = {}) {
|
|
38
|
+
const runDir = path.join(rootDir, '.ai-engineering-loop', 'runs', run.runId);
|
|
39
|
+
const ledgerPath = path.join(runDir, 'findings.json');
|
|
40
|
+
const counts = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
|
|
41
|
+
let ledger;
|
|
42
|
+
try {
|
|
43
|
+
ledger = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'));
|
|
44
|
+
} catch {
|
|
45
|
+
return counts;
|
|
46
|
+
}
|
|
47
|
+
if (!Array.isArray(ledger.findings)) return counts;
|
|
48
|
+
const records = [];
|
|
49
|
+
let changed = false;
|
|
50
|
+
ledger.findings = ledger.findings.map((finding) => {
|
|
51
|
+
const next = { ...finding };
|
|
52
|
+
if ('proofStatus' in next) {
|
|
53
|
+
delete next.proofStatus;
|
|
54
|
+
changed = true;
|
|
55
|
+
}
|
|
56
|
+
const runnable = next.validity === 'VALID' && ['BLOCKER', 'HIGH'].includes(next.severity) && next.proof && typeof next.proof === 'object';
|
|
57
|
+
if (!runnable) return next;
|
|
58
|
+
const outcome = runProof(rootDir, next.proof, timeoutMs);
|
|
59
|
+
next.proofStatus = outcome.status;
|
|
60
|
+
counts[outcome.status] += 1;
|
|
61
|
+
records.push({ id: next.id, command: tail(next.proof.command), ...outcome });
|
|
62
|
+
changed = true;
|
|
63
|
+
return next;
|
|
64
|
+
});
|
|
65
|
+
if (changed) fs.writeFileSync(ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, { mode: 0o600 });
|
|
66
|
+
if (records.length) {
|
|
67
|
+
fs.writeFileSync(path.join(runDir, 'proofs.json'), `${JSON.stringify({ schemaVersion: 1, runId: run.runId, diffHash: ledger.diffHash, proofs: records }, null, 2)}\n`, { mode: 0o600 });
|
|
68
|
+
}
|
|
69
|
+
return counts;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
module.exports = { PROOF_TIMEOUT_MS, runFindingProofs };
|
package/lib/gates.js
CHANGED
|
@@ -545,6 +545,19 @@ function mergeCommentJustifications(rootDir, run) {
|
|
|
545
545
|
return merged.added;
|
|
546
546
|
}
|
|
547
547
|
|
|
548
|
+
const ITERATION_EVIDENCE = ['verification.json', 'verification-summary.json', 'final-verification.json'];
|
|
549
|
+
|
|
550
|
+
function archiveIterationEvidence(rootDir, run) {
|
|
551
|
+
const runDir = path.dirname(artifactPath(rootDir, run.runId, 'verification.json'));
|
|
552
|
+
const target = path.join(runDir, 'iterations', String(run.iteration || 1));
|
|
553
|
+
for (const name of ITERATION_EVIDENCE) {
|
|
554
|
+
const from = path.join(runDir, name);
|
|
555
|
+
if (!fs.existsSync(from)) continue;
|
|
556
|
+
fs.mkdirSync(target, { recursive: true, mode: 0o700 });
|
|
557
|
+
fs.renameSync(from, path.join(target, name));
|
|
558
|
+
}
|
|
559
|
+
}
|
|
560
|
+
|
|
548
561
|
function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
549
562
|
if (!Object.hasOwn(GATE_FILES, gate)) {
|
|
550
563
|
throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`, [], { why: 'Only the listed stages exist.', fix: `Use one of: ${Object.keys(GATE_FILES).join(', ')}.` });
|
|
@@ -673,6 +686,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
673
686
|
mergeImpactDrift(rootDir, run);
|
|
674
687
|
mergeVerificationIntegrity(rootDir, run);
|
|
675
688
|
mergeCommentJustifications(rootDir, run);
|
|
689
|
+
if (!require('./ablation.js').ablated(rootDir, 'proofs', run.runId)) require('./finding-proofs.js').runFindingProofs(rootDir, run);
|
|
676
690
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
|
|
677
691
|
requireRunMetadata(value, run, 'Finding Ledger');
|
|
678
692
|
if (value.diffHash !== run.artifacts.diff?.sha256) {
|
|
@@ -720,6 +734,14 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
720
734
|
}
|
|
721
735
|
const blocking = expected.blockingFindings || [];
|
|
722
736
|
const cause = value.cause || inferCause(expected.verdict, blocking);
|
|
737
|
+
if (expected.verdict === 'ITERATE' && value.cause) {
|
|
738
|
+
const previous = require('./no-progress.js').readIterations(rootDir, run.runId).filter((entry) => entry.verdict === 'ITERATE').pop();
|
|
739
|
+
const premise = value.premise || {};
|
|
740
|
+
if (previous && !previous.causeInferred && previous.cause === cause && !(String(premise.statement || '').trim() && String(premise.test || '').trim())) {
|
|
741
|
+
throw gateError(`Judge ITERATE repeats cause "${cause}": state and test the shared premise first (PREMISE_REQUIRED)`, [`iteration ${previous.iteration || run.iteration - 1} also failed on ${cause}`],
|
|
742
|
+
{ why: 'When two fixes that share one premise fail the same gate, the premise is the likely bug; a third fix on the same premise usually fails too.', fix: 'Add premise: { statement: "<the one sentence every failed fix assumed>", test: "<the observation that would differ if it were false, and what it showed>" } to verdict.json, then retry `gate judge`.' });
|
|
743
|
+
}
|
|
744
|
+
}
|
|
723
745
|
recordIteration(rootDir, run, {
|
|
724
746
|
verdict: expected.verdict,
|
|
725
747
|
diffHash: run.artifacts?.diff?.sha256,
|
|
@@ -734,6 +756,18 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
734
756
|
throw gateError('Judge PASS refused: rubric criteria are not scored or failed (AC_RUBRIC)', rubric,
|
|
735
757
|
{ why: 'Subjective ACs are judged generously by default; a rubric scored with evidence per criterion makes the judgment reproducible.', fix: 'Have the Devil\'s Advocate add rubricScores [{ac, criterion, pass, evidence}] for every criterion (screenshot path, DOM snapshot, measured value), re-ingest, then retry; or issue ITERATE for failed criteria.' });
|
|
736
758
|
}
|
|
759
|
+
if (tierOf(run) === 'T2') {
|
|
760
|
+
const safety = require('./safety-fact.js').safetyFactProblem(rootDir, run);
|
|
761
|
+
if (safety) {
|
|
762
|
+
throw gateError('Judge PASS refused: T2 needs a proven safety fact (SAFETY_FACT_UNPROVEN)', [safety],
|
|
763
|
+
{ why: 'A high-risk change is safe because of one fact; a writeup that sounds right proves nothing until real code runs.', fix: `Run \`safety record --run ${run.runId} --fact "<the one fact>" --cmd "<script or test that passes only if the fact holds>"\`, then retry \`gate judge\`.` });
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
const measured = require('./measurements.js').measurementProblems(rootDir, run, loadFrozenGoal(rootDir, run));
|
|
767
|
+
if (measured.length) {
|
|
768
|
+
throw gateError('Judge PASS refused: a performance number is not explained (MEASUREMENT_UNEXPLAINED)', measured,
|
|
769
|
+
{ why: 'A single run or a gap smaller than the noise says nothing; the limiter tells you whether the number measured the change at all.', fix: `Measure at least 3 runs, find what bounds the result, then \`measurement record --run ${run.runId} --ac <AC-n> --runs <n> --spread <pct> --effect <pct> --limiter "<what bounds it>"\` and retry.` });
|
|
770
|
+
}
|
|
737
771
|
const unproven = unprovenCriteria(rootDir, run);
|
|
738
772
|
if (unproven.length) {
|
|
739
773
|
throw gateError('Judge PASS refused: acceptance criteria are not proven (AC_NOT_PROVEN)',
|
|
@@ -749,9 +783,11 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
749
783
|
reason: expected.reason
|
|
750
784
|
});
|
|
751
785
|
}
|
|
786
|
+
archiveIterationEvidence(rootDir, run);
|
|
752
787
|
return transitionRun(rootDir, run.runId, RUN_STATES.GOAL_FROZEN, {
|
|
753
788
|
gate,
|
|
754
789
|
artifacts,
|
|
790
|
+
dropArtifacts: ['verification'],
|
|
755
791
|
reason: expected.reason,
|
|
756
792
|
incrementIteration: true
|
|
757
793
|
});
|
package/lib/harness-audit.js
CHANGED
|
@@ -92,6 +92,7 @@ function auditRules(rootDir, now = new Date()) {
|
|
|
92
92
|
else if (backlog.clean) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff) has no legacy violations left. Switch it to scope repo so the whole codebase stays clean.` });
|
|
93
93
|
else items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff): legacy violations remain (${backlog.outputLines} output line(s), e.g. ${backlog.sample[0] || 'n/a'}). Clean them up over time; new ones are already blocked.` });
|
|
94
94
|
}
|
|
95
|
+
if (!rule.provenBy) items.push({ id: 'rule-unproven', level: 'info', detail: `${rule.id} was never shown to catch a real mistake. FIX: re-promote it with \`learn promote ${rule.from || '<learning>'} ... --prove <commit that introduced the mistake>\`.` });
|
|
95
96
|
const age = (now - new Date(rule.createdAt || 0)) / 86400000;
|
|
96
97
|
if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
|
|
97
98
|
}
|
|
@@ -127,10 +128,29 @@ function auditFeatureMap(rootDir) {
|
|
|
127
128
|
return [{ id: 'feature-map', level: 'warn', detail: 'e2e/smoke is configured but .ai-engineering-loop/feature-map.md is missing or empty. FIX: list user-facing features (Feature | Where | Paths | How to reach | Driver) so agents and reviewers share one vocabulary for runtime checks.' }];
|
|
128
129
|
}
|
|
129
130
|
|
|
131
|
+
function auditFeatureMapDrift(rootDir) {
|
|
132
|
+
const entries = require('./feature-map.js').loadFeatureMap(rootDir);
|
|
133
|
+
if (!entries.length) return [];
|
|
134
|
+
let files = [];
|
|
135
|
+
try {
|
|
136
|
+
files = require('child_process').execFileSync('git', ['ls-files'], { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).split('\n').filter(Boolean);
|
|
137
|
+
} catch {
|
|
138
|
+
return [];
|
|
139
|
+
}
|
|
140
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
141
|
+
const items = [];
|
|
142
|
+
for (const entry of entries) {
|
|
143
|
+
const dead = entry.paths.filter((glob) => !files.some((file) => matchesGlob(file, glob)));
|
|
144
|
+
if (dead.length) items.push({ id: 'feature-map-drift', level: 'warn', detail: `Feature "${entry.id}" lists paths that match no tracked file: ${dead.join(', ')}. FIX: update the Paths column in .ai-engineering-loop/feature-map.md, or remove the feature if it is gone.` });
|
|
145
|
+
}
|
|
146
|
+
return items;
|
|
147
|
+
}
|
|
148
|
+
|
|
130
149
|
function auditHarness(rootDir, { budget = true } = {}) {
|
|
131
150
|
const items = [
|
|
132
151
|
...auditEntryFile(rootDir),
|
|
133
152
|
...auditFeatureMap(rootDir),
|
|
153
|
+
...auditFeatureMapDrift(rootDir),
|
|
134
154
|
...auditLearnings(rootDir),
|
|
135
155
|
...auditRules(rootDir),
|
|
136
156
|
...(budget ? auditBudget(rootDir) : [])
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
const lines = (file) => {
|
|
7
|
+
try {
|
|
8
|
+
return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
|
|
9
|
+
try {
|
|
10
|
+
return [JSON.parse(line)];
|
|
11
|
+
} catch {
|
|
12
|
+
return [];
|
|
13
|
+
}
|
|
14
|
+
});
|
|
15
|
+
} catch {
|
|
16
|
+
return [];
|
|
17
|
+
}
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const readJson = (file) => {
|
|
21
|
+
try {
|
|
22
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
23
|
+
} catch {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
function runInterventions(runDir) {
|
|
29
|
+
const answers = lines(path.join(runDir, 'interactions.jsonl')).filter((entry) => entry.type === 'ANSWER').length;
|
|
30
|
+
const state = readJson(path.join(runDir, 'state.json'));
|
|
31
|
+
const escalated = state?.state === 'ESCALATED' || (state?.history || []).some((entry) => entry.to === 'ESCALATED') ? 1 : 0;
|
|
32
|
+
const outcomes = lines(path.join(runDir, 'outcomes.jsonl')).length;
|
|
33
|
+
const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
|
|
34
|
+
for (const proof of readJson(path.join(runDir, 'proofs.json'))?.proofs || []) {
|
|
35
|
+
if (Object.hasOwn(proofs, proof.status)) proofs[proof.status] += 1;
|
|
36
|
+
}
|
|
37
|
+
return { answers, escalated, outcomes, total: answers + escalated + outcomes, proofs };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function summarizeInterventions(perRun) {
|
|
41
|
+
const sum = (key) => perRun.reduce((n, item) => n + item[key], 0);
|
|
42
|
+
const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
|
|
43
|
+
for (const item of perRun) for (const key of Object.keys(proofs)) proofs[key] += item.proofs[key];
|
|
44
|
+
const ran = proofs.reproduced + proofs.not_reproduced;
|
|
45
|
+
return {
|
|
46
|
+
runs: perRun.length,
|
|
47
|
+
answers: sum('answers'),
|
|
48
|
+
escalations: sum('escalated'),
|
|
49
|
+
outcomes: sum('outcomes'),
|
|
50
|
+
perRun: perRun.length ? Math.round((sum('total') / perRun.length) * 100) / 100 : null,
|
|
51
|
+
proofs,
|
|
52
|
+
reproducedRate: ran ? Math.round((proofs.reproduced / ran) * 1000) / 1000 : null
|
|
53
|
+
};
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
module.exports = { runInterventions, summarizeInterventions };
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
const FILE = 'measurements.json';
|
|
7
|
+
const MIN_RUNS = 3;
|
|
8
|
+
|
|
9
|
+
const fail = (message) => Object.assign(new Error(message), { code: 'INVALID_USAGE' });
|
|
10
|
+
const filePath = (rootDir, runId) => path.join(rootDir, '.ai-engineering-loop', 'runs', runId, FILE);
|
|
11
|
+
const percent = (value, flag) => {
|
|
12
|
+
const number = Number(String(value ?? '').replace(/%$/, ''));
|
|
13
|
+
if (!Number.isFinite(number) || number < 0) throw fail(`${flag} must be a non-negative percentage, such as 4 or 4%`);
|
|
14
|
+
return number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
function loadMeasurements(rootDir, runId) {
|
|
18
|
+
try {
|
|
19
|
+
return JSON.parse(fs.readFileSync(filePath(rootDir, runId), 'utf8')).measurements || [];
|
|
20
|
+
} catch {
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function recordMeasurement(rootDir, runId, { ac, runs, spread, effect, limiter, now = new Date() }) {
|
|
26
|
+
if (!/^AC-\d+$/.test(String(ac || ''))) throw fail('--ac must be an AC id such as AC-2');
|
|
27
|
+
const count = Number(runs);
|
|
28
|
+
if (!Number.isInteger(count) || count < 1) throw fail('--runs must be a whole number of measured runs');
|
|
29
|
+
const reason = String(limiter || '').trim();
|
|
30
|
+
if (!reason) throw fail('--limiter is required: the resource or code path that bounds the number, from a profile or counters');
|
|
31
|
+
const entry = { ac, runs: count, spread: percent(spread, '--spread'), effect: percent(effect, '--effect'), limiter: reason, recordedAt: now.toISOString() };
|
|
32
|
+
const measurements = loadMeasurements(rootDir, runId).filter((item) => item.ac !== ac).concat(entry);
|
|
33
|
+
fs.writeFileSync(filePath(rootDir, runId), `${JSON.stringify({ schemaVersion: 1, runId, measurements }, null, 2)}\n`, { mode: 0o600 });
|
|
34
|
+
return entry;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function measurementProblems(rootDir, run, goal) {
|
|
38
|
+
const needed = (goal?.acceptanceCriteria || []).filter((ac) => ac.measure === true);
|
|
39
|
+
if (!needed.length) return [];
|
|
40
|
+
const recorded = new Map(loadMeasurements(rootDir, run.runId).map((item) => [item.ac, item]));
|
|
41
|
+
const problems = [];
|
|
42
|
+
for (const ac of needed) {
|
|
43
|
+
const m = recorded.get(ac.id);
|
|
44
|
+
if (!m) problems.push(`${ac.id}: no measurement recorded`);
|
|
45
|
+
else if (m.runs < MIN_RUNS) problems.push(`${ac.id}: ${m.runs} run(s); at least ${MIN_RUNS} are needed to see the noise`);
|
|
46
|
+
else if (m.spread >= m.effect) problems.push(`${ac.id}: run-to-run spread ${m.spread}% is not smaller than the effect ${m.effect}%`);
|
|
47
|
+
}
|
|
48
|
+
return problems;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
module.exports = { MIN_RUNS, loadMeasurements, measurementProblems, recordMeasurement };
|
package/lib/orchestration.js
CHANGED
|
@@ -151,6 +151,7 @@ function validateFindingLedger(ledger) {
|
|
|
151
151
|
function isBlockingFinding(finding) {
|
|
152
152
|
if (!finding || finding.validity !== 'VALID') return false;
|
|
153
153
|
if (finding.severity !== 'BLOCKER' && finding.severity !== 'HIGH') return false;
|
|
154
|
+
if (finding.proofStatus === 'not_reproduced') return false;
|
|
154
155
|
const axis = finding.axis || 'spec';
|
|
155
156
|
if (axis === 'standards' && finding.hardConvention !== true) return false;
|
|
156
157
|
return true;
|
package/lib/recap-html.js
CHANGED
|
@@ -195,11 +195,11 @@ function renderRecap(data, { audience = 'me', now = new Date(), lang = null } =
|
|
|
195
195
|
const contentLang = lang ? ` lang="${esc(lang)}"` : '';
|
|
196
196
|
const technical = runs.filter((run) => run.delivered && run.clientTextSource === 'objective');
|
|
197
197
|
const empty = `<p class="empty">No runs in the last ${esc(data.window.days)} days. Try a longer window: <code>ael recap --days ${esc(data.window.days * 4)}</code></p>`;
|
|
198
|
-
const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}</div>
|
|
198
|
+
const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Times you had to step in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>
|
|
199
199
|
<div class="panel"><h2>Activity per day</h2>${heatmap(data)}</div>
|
|
200
200
|
<div class="panel"><h2>Where the time went</h2>${stageBars(runs)}</div>
|
|
201
201
|
<div class="panel"><h2>Why runs had to iterate</h2>${causes(metrics)}</div>` : `<div class="panel">${empty}</div>`;
|
|
202
|
-
const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
|
|
202
|
+
const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}${metric('Times a human stepped in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
|
|
203
203
|
const tab = (view, text) => `<button type="button" aria-controls="${view}" data-view="${view}" aria-pressed="${view === audience}">${text}</button>`;
|
|
204
204
|
const section = (view, html) => `<section class="view${view === audience ? ' on' : ''}" id="${view}" aria-label="${esc({ me: 'For me', lead: 'For the lead', client: 'For the client' }[view])}">${html}</section>`;
|
|
205
205
|
const range = `${fmtDate(data.window.from)} – ${fmtDate(new Date(data.window.to.getTime() - 1))}`;
|
package/lib/recap.js
CHANGED
|
@@ -110,6 +110,7 @@ function loadRun(projectPath, runId) {
|
|
|
110
110
|
outOfMap: Array.isArray(receipt.impact?.outOfMap) ? receipt.impact.outOfMap : [],
|
|
111
111
|
openFindings: Array.isArray(summary.openFindings) ? summary.openFindings.length : 0,
|
|
112
112
|
stages: stageDurations(history),
|
|
113
|
+
interventions: require('./interventions.js').runInterventions(dir),
|
|
113
114
|
files
|
|
114
115
|
};
|
|
115
116
|
}
|
|
@@ -146,6 +147,7 @@ function metricsOf(runs) {
|
|
|
146
147
|
firstPassRate: judged.length ? Math.round((judged.filter((run) => run.iterations[0].verdict === 'PASS').length / judged.length) * 100) : null,
|
|
147
148
|
medianCycleMs: median(runs.filter((run) => run.delivered).map(cycle).filter(Boolean)),
|
|
148
149
|
t2: runs.filter((run) => run.tier === 'T2').length,
|
|
150
|
+
interventionsPerRun: require('./interventions.js').summarizeInterventions(runs.map((run) => run.interventions)).perRun,
|
|
149
151
|
causes
|
|
150
152
|
};
|
|
151
153
|
}
|
package/lib/rules.js
CHANGED
|
@@ -36,7 +36,7 @@ function loadLearningEntries(rootDir) {
|
|
|
36
36
|
|
|
37
37
|
const SCOPES = Object.freeze(['repo', 'diff']);
|
|
38
38
|
|
|
39
|
-
function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, now = new Date() } = {}) {
|
|
39
|
+
function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, prove = null, now = new Date() } = {}) {
|
|
40
40
|
if (!SCOPES.includes(scope)) throw rulesError(`--scope must be repo or diff (got ${scope}).`, 'INVALID_USAGE');
|
|
41
41
|
// C-20 built-in kinds: no hand-written check, always judged on added lines.
|
|
42
42
|
let builtin = null;
|
|
@@ -79,10 +79,59 @@ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shel
|
|
|
79
79
|
createdBy: String(by).trim(),
|
|
80
80
|
createdAt: now.toISOString()
|
|
81
81
|
};
|
|
82
|
+
if (prove) rule.provenBy = proveRule(rootDir, rule, prove, { now });
|
|
82
83
|
atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
|
|
83
84
|
return rule;
|
|
84
85
|
}
|
|
85
86
|
|
|
87
|
+
function proveRule(rootDir, rule, commit, { now = new Date() } = {}) {
|
|
88
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
89
|
+
const os = require('os');
|
|
90
|
+
const git = (...args) => execFileSync('git', args, { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
|
|
91
|
+
let sha;
|
|
92
|
+
try {
|
|
93
|
+
sha = git('rev-parse', '--verify', `${commit}^{commit}`).trim();
|
|
94
|
+
} catch {
|
|
95
|
+
throw rulesError(`--prove ${commit} is not a commit in this repository.`, 'INVALID_USAGE');
|
|
96
|
+
}
|
|
97
|
+
let failed;
|
|
98
|
+
if (rule.scope === 'diff') {
|
|
99
|
+
const { parseDiff } = require('./verification-integrity.js');
|
|
100
|
+
const files = parseDiff(git('show', '--format=', '--no-color', sha, '--', '.', ':(exclude).ai-engineering-loop')).filter((file) => !file.deleted);
|
|
101
|
+
const lines = files.flatMap((file) => file.added.map((added) => ({ file: file.path, line: added.line, text: added.text })));
|
|
102
|
+
if (rule.kind) {
|
|
103
|
+
failed = require('./rule-check.js').checkRule(rule, lines).length > 0;
|
|
104
|
+
} else {
|
|
105
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-'));
|
|
106
|
+
const addedPath = path.join(tmp, 'added-lines.txt');
|
|
107
|
+
fs.writeFileSync(addedPath, lines.map((l) => `${l.file}:${l.line}:${l.text}`).join('\n'));
|
|
108
|
+
const env = { ...process.env, AEL_CHANGED_FILES: files.map((file) => file.path).join('\n'), AEL_ADDED_LINES_FILE: addedPath };
|
|
109
|
+
const result = rule.shell !== false
|
|
110
|
+
? spawnSync(rule.check, { cwd: rootDir, shell: true, env, encoding: 'utf8' })
|
|
111
|
+
: spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: rootDir, env, encoding: 'utf8' });
|
|
112
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
113
|
+
failed = result.status !== 0;
|
|
114
|
+
}
|
|
115
|
+
} else {
|
|
116
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-wt-'));
|
|
117
|
+
const tree = path.join(tmp, 'tree');
|
|
118
|
+
git('worktree', 'add', '--detach', '--quiet', tree, sha);
|
|
119
|
+
try {
|
|
120
|
+
const result = rule.shell !== false
|
|
121
|
+
? spawnSync(rule.check, { cwd: tree, shell: true, encoding: 'utf8' })
|
|
122
|
+
: spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: tree, encoding: 'utf8' });
|
|
123
|
+
failed = result.status !== 0;
|
|
124
|
+
} finally {
|
|
125
|
+
git('worktree', 'remove', '--force', tree);
|
|
126
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
if (!failed) {
|
|
130
|
+
throw rulesError(`The check did not fail on ${sha.slice(0, 12)}, so it would not have caught that mistake. Pick the commit that introduced the mistake, or fix the check.`, 'RULE_NOT_PROVEN');
|
|
131
|
+
}
|
|
132
|
+
return { commit: sha, at: now.toISOString() };
|
|
133
|
+
}
|
|
134
|
+
|
|
86
135
|
// Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
|
|
87
136
|
function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
|
|
88
137
|
const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
|
|
@@ -189,6 +238,7 @@ function missingRules(rootDir, bundle) {
|
|
|
189
238
|
}
|
|
190
239
|
|
|
191
240
|
module.exports = {
|
|
241
|
+
proveRule,
|
|
192
242
|
SCOPES,
|
|
193
243
|
commandForRule,
|
|
194
244
|
findRule,
|
package/lib/run-state.js
CHANGED
|
@@ -396,6 +396,7 @@ function transitionRunLocked(
|
|
|
396
396
|
incrementIteration = false,
|
|
397
397
|
statePatch = {},
|
|
398
398
|
historyMetadata = {},
|
|
399
|
+
dropArtifacts = [],
|
|
399
400
|
now = new Date()
|
|
400
401
|
} = {}
|
|
401
402
|
) {
|
|
@@ -419,7 +420,7 @@ function transitionRunLocked(
|
|
|
419
420
|
state: to,
|
|
420
421
|
iteration: incrementIteration ? run.iteration + 1 : run.iteration,
|
|
421
422
|
updatedAt: timestamp,
|
|
422
|
-
artifacts: { ...run.artifacts, ...artifacts },
|
|
423
|
+
artifacts: Object.fromEntries(Object.entries({ ...run.artifacts, ...artifacts }).filter(([key]) => !dropArtifacts.includes(key))),
|
|
423
424
|
history: [
|
|
424
425
|
...run.history,
|
|
425
426
|
{
|