ai-engineering-loop 2.10.1 → 2.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +1 -0
- package/.agents/skills/ael/references/loop-risky.md +1 -0
- package/.claude/agents/devil-advocate.md +2 -1
- package/.claude/agents/judge.md +2 -0
- package/.grok/agents/devil-advocate.md +2 -1
- package/.grok/agents/judge.md +2 -0
- package/CHANGELOG.md +13 -0
- package/README.md +80 -75
- package/agents/shared/devil-advocate.body.md +2 -1
- package/agents/shared/judge.body.md +2 -0
- package/bin/ai-engineering-loop.js +50 -0
- package/core/goal-contract.md +4 -0
- package/core/iteration-policy.md +4 -0
- package/core/verification-loop.md +25 -0
- package/docs/images/ael-hero.svg +19 -0
- package/docs/images/readme-banner.png +0 -0
- package/docs/images/social-preview.png +0 -0
- package/docs/images/studio-hero-orange.svg +136 -0
- package/lib/ablation.js +1 -1
- package/lib/cli-args.js +3 -1
- package/lib/finding-proofs.js +72 -0
- package/lib/gates.js +21 -0
- package/lib/harness-audit.js +20 -0
- package/lib/interventions.js +56 -0
- package/lib/measurements.js +51 -0
- package/lib/orchestration.js +1 -0
- package/lib/recap-html.js +2 -2
- package/lib/recap.js +2 -0
- package/lib/rules.js +51 -1
- package/lib/safety-fact.js +64 -0
- package/lib/stats.js +9 -0
- package/lib/verification-lanes.js +23 -0
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +4 -0
package/lib/stats.js
CHANGED
|
@@ -73,7 +73,10 @@ function computeStats(rootDir, { last = null } = {}) {
|
|
|
73
73
|
}
|
|
74
74
|
const labeled = Object.entries(tokensByRole).filter(([role]) => role !== 'unlabeled').reduce((sum, [, n]) => sum + n, 0);
|
|
75
75
|
const checking = ['reviewer', 'judge', 'verification'].reduce((sum, role) => sum + (tokensByRole[role] || 0), 0);
|
|
76
|
+
const { runInterventions, summarizeInterventions } = require('./interventions.js');
|
|
77
|
+
const interventions = summarizeInterventions(runs.map((runId) => runInterventions(path.join(runsDir(rootDir), runId))));
|
|
76
78
|
return {
|
|
79
|
+
interventions,
|
|
77
80
|
revertRate: reverts.rate,
|
|
78
81
|
reverted: reverts.reverted,
|
|
79
82
|
delivered: reverts.delivered,
|
|
@@ -100,6 +103,12 @@ function formatStats(stats) {
|
|
|
100
103
|
];
|
|
101
104
|
lines.push(`Revert rate: ${stats.reverted.length}/${stats.delivered} delivered Run(s) (${Math.round(stats.revertRate * 1000) / 10}%)${stats.reverted.length ? `: ${stats.reverted.map((item) => item.runId).join(', ')}` : ''}`);
|
|
102
105
|
if (stats.verificationTokenShare != null) lines.push(`Verification token share: ${Math.round(stats.verificationTokenShare * 100)}% (reviewer + judge + verification of role-labelled usage)`);
|
|
106
|
+
if (stats.interventions?.runs) {
|
|
107
|
+
const i = stats.interventions;
|
|
108
|
+
lines.push(`Human interventions per Run: ${i.perRun} (${i.answers} answered questions, ${i.escalations} escalations, ${i.outcomes} problems found after delivery)`);
|
|
109
|
+
const ran = i.proofs.reproduced + i.proofs.not_reproduced;
|
|
110
|
+
if (ran) lines.push(`Finding proofs: ${i.proofs.reproduced}/${ran} reproduced (${Math.round(i.reproducedRate * 100)}%); ${i.proofs.not_reproduced} blocking finding(s) did not reproduce and stopped blocking`);
|
|
111
|
+
}
|
|
103
112
|
if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
|
|
104
113
|
for (const [tier, bucket] of Object.entries(stats.tiers)) {
|
|
105
114
|
lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
|
|
@@ -160,6 +160,28 @@ function startupRiskFiles(rootDir, run) {
|
|
|
160
160
|
return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
+
function formalCoveredFiles(rootDir, run) {
|
|
164
|
+
let text = '';
|
|
165
|
+
try {
|
|
166
|
+
text = fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'verification.md'), 'utf8');
|
|
167
|
+
} catch {
|
|
168
|
+
return [];
|
|
169
|
+
}
|
|
170
|
+
const match = text.match(/\*\*formal_covers\*\*:\s*(.+)/i);
|
|
171
|
+
if (!match) return [];
|
|
172
|
+
const globs = match[1].replace(/`/g, '').split(',').map((glob) => glob.trim()).filter(Boolean);
|
|
173
|
+
const artifact = run?.artifacts?.diff;
|
|
174
|
+
if (!artifact?.path || !globs.length) return [];
|
|
175
|
+
let diff = '';
|
|
176
|
+
try {
|
|
177
|
+
diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
|
|
178
|
+
} catch {
|
|
179
|
+
return [];
|
|
180
|
+
}
|
|
181
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
182
|
+
return [...new Set(extractChangedPaths(diff))].filter((file) => globs.some((glob) => matchesGlob(file, glob)));
|
|
183
|
+
}
|
|
184
|
+
|
|
163
185
|
function readJson(filePath) {
|
|
164
186
|
try {
|
|
165
187
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
@@ -193,6 +215,7 @@ function finalVerificationStatus(rootDir, run) {
|
|
|
193
215
|
}
|
|
194
216
|
|
|
195
217
|
module.exports = {
|
|
218
|
+
formalCoveredFiles,
|
|
196
219
|
VERIFICATION_CATALOG,
|
|
197
220
|
keyOfCommand,
|
|
198
221
|
DEFAULT_PLAN_KEYS,
|
package/package.json
CHANGED
|
@@ -53,6 +53,10 @@
|
|
|
53
53
|
"minLength": 1
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
|
+
"measure": {
|
|
57
|
+
"description": "Optional (C-33) for performance ACs: Judge PASS needs `measurement record` with at least 3 runs, a limiter, and spread smaller than the effect.",
|
|
58
|
+
"type": "boolean"
|
|
59
|
+
},
|
|
56
60
|
"rubric": {
|
|
57
61
|
"description": "Optional (C-09) for subjective ACs: criteria the Devil's Advocate scores with evidence in ledger.rubricScores. Judge PASS needs every criterion scored pass:true.",
|
|
58
62
|
"type": "array",
|