ai-engineering-loop 2.10.0 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +1 -0
- package/.agents/skills/ael/references/loop-risky.md +1 -0
- package/.claude/agents/devil-advocate.md +2 -1
- package/.claude/agents/judge.md +2 -0
- package/.grok/agents/devil-advocate.md +2 -1
- package/.grok/agents/judge.md +2 -0
- package/CHANGELOG.md +17 -0
- package/README.md +80 -75
- package/agents/shared/devil-advocate.body.md +2 -1
- package/agents/shared/judge.body.md +2 -0
- package/bin/ai-engineering-loop.js +50 -0
- package/core/controlled-workflow-runtime.md +1 -0
- package/core/goal-contract.md +4 -0
- package/core/iteration-policy.md +4 -0
- package/core/verification-loop.md +25 -0
- package/docs/images/ael-hero.svg +19 -0
- package/docs/images/readme-banner.png +0 -0
- package/docs/images/social-preview.png +0 -0
- package/docs/images/studio-hero-orange.svg +136 -0
- package/lib/ablation.js +1 -1
- package/lib/cli-args.js +3 -1
- package/lib/finding-proofs.js +72 -0
- package/lib/gates.js +36 -0
- package/lib/harness-audit.js +20 -0
- package/lib/interventions.js +56 -0
- package/lib/measurements.js +51 -0
- package/lib/orchestration.js +1 -0
- package/lib/recap-html.js +2 -2
- package/lib/recap.js +2 -0
- package/lib/rules.js +51 -1
- package/lib/run-state.js +2 -1
- package/lib/safety-fact.js +64 -0
- package/lib/stats.js +9 -0
- package/lib/verification-lanes.js +23 -0
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +4 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { spawnSync } = require('child_process');
|
|
6
|
+
const { destructiveFindings } = require('./guard.js');
|
|
7
|
+
const { redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
8
|
+
|
|
9
|
+
const FILE = 'safety-fact.json';
|
|
10
|
+
const LEVELS = {
|
|
11
|
+
1: 'stated',
|
|
12
|
+
2: 'pointed at the line',
|
|
13
|
+
3: 'walked the failure path',
|
|
14
|
+
4: 'ran real code',
|
|
15
|
+
5: 'reproduced in the running app'
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const fail = (message, code = 'INVALID_USAGE') => Object.assign(new Error(message), { code });
|
|
19
|
+
const factPath = (rootDir, runId) => path.join(rootDir, '.ai-engineering-loop', 'runs', runId, FILE);
|
|
20
|
+
|
|
21
|
+
function recordSafetyFact(rootDir, runId, { fact, command = null, live = false, level = null, timeoutMs = 120000, now = new Date() }) {
|
|
22
|
+
const text = String(fact || '').trim();
|
|
23
|
+
if (!text) throw fail('--fact is required: the one fact this change is safe because of');
|
|
24
|
+
let record;
|
|
25
|
+
if (command) {
|
|
26
|
+
if (destructiveFindings(command).length) throw fail('The proof command looks destructive; prove the fact with a read-only script or test');
|
|
27
|
+
const result = spawnSync(command, { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs, shell: true });
|
|
28
|
+
if (result.status !== 0) {
|
|
29
|
+
throw fail(`The proof command did not pass (exit ${result.status ?? 'none'}), so the fact is not proven. Fix the change or the fact, then record again.`, 'SAFETY_FACT_NOT_PROVEN');
|
|
30
|
+
}
|
|
31
|
+
record = {
|
|
32
|
+
fact: text,
|
|
33
|
+
level: live ? 5 : 4,
|
|
34
|
+
command: redactHomePaths(redactSecrets(command).text),
|
|
35
|
+
exitCode: 0,
|
|
36
|
+
outputTail: redactHomePaths(redactSecrets(String(result.stdout || '')).text).slice(-1000)
|
|
37
|
+
};
|
|
38
|
+
} else {
|
|
39
|
+
const claimed = level == null ? 1 : Number(level);
|
|
40
|
+
if (![1, 2, 3].includes(claimed)) throw fail('--level without --cmd must be 1, 2 or 3; levels 4 and 5 need a proof command that passes');
|
|
41
|
+
if (live) throw fail('--live needs --cmd: level 5 is a passing proof run against the running app');
|
|
42
|
+
record = { fact: text, level: claimed };
|
|
43
|
+
}
|
|
44
|
+
const value = { schemaVersion: 1, runId, ...record, levelName: LEVELS[record.level], recordedAt: now.toISOString() };
|
|
45
|
+
fs.writeFileSync(factPath(rootDir, runId), `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
|
|
46
|
+
return value;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function loadSafetyFact(rootDir, runId) {
|
|
50
|
+
try {
|
|
51
|
+
return JSON.parse(fs.readFileSync(factPath(rootDir, runId), 'utf8'));
|
|
52
|
+
} catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function safetyFactProblem(rootDir, run, { minimum = 4 } = {}) {
|
|
58
|
+
const value = loadSafetyFact(rootDir, run.runId);
|
|
59
|
+
if (!value) return 'no safety fact recorded';
|
|
60
|
+
if (value.level < minimum) return `safety fact is level ${value.level} (${LEVELS[value.level]}); level ${minimum} needs a proof command that passes`;
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
module.exports = { LEVELS, loadSafetyFact, recordSafetyFact, safetyFactProblem };
|
package/lib/stats.js
CHANGED
|
@@ -73,7 +73,10 @@ function computeStats(rootDir, { last = null } = {}) {
|
|
|
73
73
|
}
|
|
74
74
|
const labeled = Object.entries(tokensByRole).filter(([role]) => role !== 'unlabeled').reduce((sum, [, n]) => sum + n, 0);
|
|
75
75
|
const checking = ['reviewer', 'judge', 'verification'].reduce((sum, role) => sum + (tokensByRole[role] || 0), 0);
|
|
76
|
+
const { runInterventions, summarizeInterventions } = require('./interventions.js');
|
|
77
|
+
const interventions = summarizeInterventions(runs.map((runId) => runInterventions(path.join(runsDir(rootDir), runId))));
|
|
76
78
|
return {
|
|
79
|
+
interventions,
|
|
77
80
|
revertRate: reverts.rate,
|
|
78
81
|
reverted: reverts.reverted,
|
|
79
82
|
delivered: reverts.delivered,
|
|
@@ -100,6 +103,12 @@ function formatStats(stats) {
|
|
|
100
103
|
];
|
|
101
104
|
lines.push(`Revert rate: ${stats.reverted.length}/${stats.delivered} delivered Run(s) (${Math.round(stats.revertRate * 1000) / 10}%)${stats.reverted.length ? `: ${stats.reverted.map((item) => item.runId).join(', ')}` : ''}`);
|
|
102
105
|
if (stats.verificationTokenShare != null) lines.push(`Verification token share: ${Math.round(stats.verificationTokenShare * 100)}% (reviewer + judge + verification of role-labelled usage)`);
|
|
106
|
+
if (stats.interventions?.runs) {
|
|
107
|
+
const i = stats.interventions;
|
|
108
|
+
lines.push(`Human interventions per Run: ${i.perRun} (${i.answers} answered questions, ${i.escalations} escalations, ${i.outcomes} problems found after delivery)`);
|
|
109
|
+
const ran = i.proofs.reproduced + i.proofs.not_reproduced;
|
|
110
|
+
if (ran) lines.push(`Finding proofs: ${i.proofs.reproduced}/${ran} reproduced (${Math.round(i.reproducedRate * 100)}%); ${i.proofs.not_reproduced} blocking finding(s) did not reproduce and stopped blocking`);
|
|
111
|
+
}
|
|
103
112
|
if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
|
|
104
113
|
for (const [tier, bucket] of Object.entries(stats.tiers)) {
|
|
105
114
|
lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
|
|
@@ -160,6 +160,28 @@ function startupRiskFiles(rootDir, run) {
|
|
|
160
160
|
return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
+
function formalCoveredFiles(rootDir, run) {
|
|
164
|
+
let text = '';
|
|
165
|
+
try {
|
|
166
|
+
text = fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'verification.md'), 'utf8');
|
|
167
|
+
} catch {
|
|
168
|
+
return [];
|
|
169
|
+
}
|
|
170
|
+
const match = text.match(/\*\*formal_covers\*\*:\s*(.+)/i);
|
|
171
|
+
if (!match) return [];
|
|
172
|
+
const globs = match[1].replace(/`/g, '').split(',').map((glob) => glob.trim()).filter(Boolean);
|
|
173
|
+
const artifact = run?.artifacts?.diff;
|
|
174
|
+
if (!artifact?.path || !globs.length) return [];
|
|
175
|
+
let diff = '';
|
|
176
|
+
try {
|
|
177
|
+
diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
|
|
178
|
+
} catch {
|
|
179
|
+
return [];
|
|
180
|
+
}
|
|
181
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
182
|
+
return [...new Set(extractChangedPaths(diff))].filter((file) => globs.some((glob) => matchesGlob(file, glob)));
|
|
183
|
+
}
|
|
184
|
+
|
|
163
185
|
function readJson(filePath) {
|
|
164
186
|
try {
|
|
165
187
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
@@ -193,6 +215,7 @@ function finalVerificationStatus(rootDir, run) {
|
|
|
193
215
|
}
|
|
194
216
|
|
|
195
217
|
module.exports = {
|
|
218
|
+
formalCoveredFiles,
|
|
196
219
|
VERIFICATION_CATALOG,
|
|
197
220
|
keyOfCommand,
|
|
198
221
|
DEFAULT_PLAN_KEYS,
|
package/package.json
CHANGED
|
@@ -53,6 +53,10 @@
|
|
|
53
53
|
"minLength": 1
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
|
+
"measure": {
|
|
57
|
+
"description": "Optional (C-33) for performance ACs: Judge PASS needs `measurement record` with at least 3 runs, a limiter, and spread smaller than the effect.",
|
|
58
|
+
"type": "boolean"
|
|
59
|
+
},
|
|
56
60
|
"rubric": {
|
|
57
61
|
"description": "Optional (C-09) for subjective ACs: criteria the Devil's Advocate scores with evidence in ledger.rubricScores. Judge PASS needs every criterion scored pass:true.",
|
|
58
62
|
"type": "array",
|