ai-engineering-loop 2.10.1 → 2.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +2 -0
- package/.agents/skills/ael/references/loop-deliver.md +1 -0
- package/.agents/skills/ael/references/loop-goal.md +1 -0
- package/.agents/skills/ael/references/loop-risky.md +1 -0
- package/.claude/agents/devil-advocate.md +2 -1
- package/.claude/agents/judge.md +2 -0
- package/.grok/agents/devil-advocate.md +2 -1
- package/.grok/agents/judge.md +2 -0
- package/CHANGELOG.md +24 -0
- package/README.md +80 -75
- package/agents/shared/devil-advocate.body.md +2 -1
- package/agents/shared/judge.body.md +2 -0
- package/bin/ai-engineering-loop.js +153 -5
- package/core/goal-contract.md +4 -0
- package/core/iteration-policy.md +4 -0
- package/core/run-isolation.md +55 -0
- package/core/verification-loop.md +25 -0
- package/docs/images/ael-hero.svg +19 -0
- package/docs/images/readme-banner.png +0 -0
- package/docs/images/social-preview.png +0 -0
- package/docs/images/studio-hero-orange.svg +136 -0
- package/lib/ablation.js +1 -1
- package/lib/cli-args.js +5 -2
- package/lib/finding-proofs.js +72 -0
- package/lib/gates.js +21 -0
- package/lib/harness-audit.js +20 -0
- package/lib/interventions.js +56 -0
- package/lib/measurements.js +51 -0
- package/lib/orchestration.js +1 -0
- package/lib/recap-html.js +2 -2
- package/lib/recap.js +2 -0
- package/lib/rules.js +51 -1
- package/lib/run-isolation.js +293 -0
- package/lib/run-state.js +2 -1
- package/lib/safety-fact.js +64 -0
- package/lib/stats.js +9 -0
- package/lib/verification-lanes.js +23 -0
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +4 -0
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { execFileSync, spawnSync } = require('child_process');
|
|
6
|
+
|
|
7
|
+
const LIVE_MINUTES = 15;
|
|
8
|
+
const IDLE_DAYS = 7;
|
|
9
|
+
const REGISTRY = path.join('.ai-engineering-loop', 'worktrees.json');
|
|
10
|
+
const TERMINAL = new Set(['REPORTED', 'DELIVERED', 'ESCALATED', 'CANCELLED']);
|
|
11
|
+
const CONTEXT_SKIP = new Set(['runs', 'recaps', 'tasks', 'current.json', 'worktrees.json', 'bench-ablation.json']);
|
|
12
|
+
|
|
13
|
+
const fail = (message, code = 'INVALID_USAGE', extra = {}) => Object.assign(new Error(message), { code, ...extra });
|
|
14
|
+
|
|
15
|
+
function git(cwd, args, { allowFail = false } = {}) {
|
|
16
|
+
try {
|
|
17
|
+
return execFileSync('git', args, { cwd, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], maxBuffer: 64 * 1024 * 1024 }).trim();
|
|
18
|
+
} catch (error) {
|
|
19
|
+
if (allowFail) return null;
|
|
20
|
+
throw error;
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function readJson(file) {
|
|
25
|
+
try {
|
|
26
|
+
return JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
27
|
+
} catch {
|
|
28
|
+
return null;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function lastLifecycleAt(rootDir, runId) {
|
|
33
|
+
try {
|
|
34
|
+
const lines = fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'runs', runId, 'lifecycle.jsonl'), 'utf8').trim().split('\n');
|
|
35
|
+
return Date.parse(JSON.parse(lines[lines.length - 1]).at) || 0;
|
|
36
|
+
} catch {
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function runActivity(rootDir, run, { now = new Date() } = {}) {
|
|
42
|
+
const last = Math.max(Date.parse(run.updatedAt || run.createdAt || 0) || 0, lastLifecycleAt(rootDir, run.runId));
|
|
43
|
+
const idleMinutes = last ? Math.max(0, Math.floor((now.getTime() - last) / 60000)) : null;
|
|
44
|
+
return { lastActivity: last ? new Date(last).toISOString() : null, idleMinutes, live: idleMinutes != null && idleMinutes < LIVE_MINUTES };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const normalizeTask = (text) => String(text || '').trim().replace(/\s+/g, ' ').toLowerCase();
|
|
48
|
+
|
|
49
|
+
function changedFileCount(rootDir) {
|
|
50
|
+
const out = git(rootDir, ['status', '--porcelain', '--', '.', ':(exclude).ai-engineering-loop'], { allowFail: true });
|
|
51
|
+
return out ? out.split('\n').filter(Boolean).length : 0;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function runConflict(rootDir, current, task, { now = new Date() } = {}) {
|
|
55
|
+
if (!current || TERMINAL.has(current.state)) return null;
|
|
56
|
+
const wanted = normalizeTask(task);
|
|
57
|
+
if (!wanted || wanted === normalizeTask(current.task)) return null;
|
|
58
|
+
const activity = runActivity(rootDir, current, { now });
|
|
59
|
+
const quoted = JSON.stringify(String(task).trim());
|
|
60
|
+
return {
|
|
61
|
+
runId: current.runId,
|
|
62
|
+
state: current.state,
|
|
63
|
+
task: current.task || null,
|
|
64
|
+
...activity,
|
|
65
|
+
changedFiles: changedFileCount(rootDir),
|
|
66
|
+
recommended: activity.live || changedFileCount(rootDir) ? 'worktree' : 'abandon',
|
|
67
|
+
options: [
|
|
68
|
+
{ id: 'worktree', label: 'New worktree for this task', command: `ai-engineering-loop run --worktree --json ${quoted}` },
|
|
69
|
+
{ id: 'resume', label: `Resume ${current.runId} instead`, command: 'ai-engineering-loop run --json' },
|
|
70
|
+
{ id: 'abandon', label: `Abandon ${current.runId}, then start this task`, command: `ai-engineering-loop run --abandon --run ${current.runId} --reason "<why>" --by <name>${activity.live ? ' --force-live' : ''}` }
|
|
71
|
+
]
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function mainCheckout(cwd) {
|
|
76
|
+
const common = git(cwd, ['rev-parse', '--git-common-dir'], { allowFail: true });
|
|
77
|
+
if (!common) return null;
|
|
78
|
+
return path.dirname(path.resolve(cwd, common));
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const realpath = (p) => {
|
|
82
|
+
try {
|
|
83
|
+
return fs.realpathSync(p);
|
|
84
|
+
} catch {
|
|
85
|
+
return path.resolve(p);
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
|
|
89
|
+
function loadRegistry(mainRoot) {
|
|
90
|
+
return readJson(path.join(mainRoot, REGISTRY)) || { schemaVersion: 1, worktrees: [] };
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function saveRegistry(mainRoot, registry) {
|
|
94
|
+
fs.mkdirSync(path.join(mainRoot, '.ai-engineering-loop'), { recursive: true });
|
|
95
|
+
fs.writeFileSync(path.join(mainRoot, REGISTRY), `${JSON.stringify(registry, null, 2)}\n`, { mode: 0o600 });
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function slugOf(task) {
|
|
99
|
+
return normalizeTask(task).replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '').slice(0, 32) || 'task';
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function copyContext(fromRoot, toRoot) {
|
|
103
|
+
const from = path.join(fromRoot, '.ai-engineering-loop');
|
|
104
|
+
const to = path.join(toRoot, '.ai-engineering-loop');
|
|
105
|
+
if (!fs.existsSync(from)) return;
|
|
106
|
+
fs.mkdirSync(to, { recursive: true });
|
|
107
|
+
for (const name of fs.readdirSync(from)) {
|
|
108
|
+
if (CONTEXT_SKIP.has(name)) continue;
|
|
109
|
+
const target = path.join(to, name);
|
|
110
|
+
if (!fs.existsSync(target)) fs.cpSync(path.join(from, name), target, { recursive: true });
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function createWorktree(rootDir, { task, now = new Date() }) {
|
|
115
|
+
if (!String(task || '').trim()) throw fail('run --worktree needs a task: run --worktree "<task>"');
|
|
116
|
+
const mainRoot = mainCheckout(rootDir);
|
|
117
|
+
if (!mainRoot) throw fail('run --worktree needs a git repository');
|
|
118
|
+
const id = now.toISOString().replace(/\D/g, '').slice(2, 12);
|
|
119
|
+
const slug = `${slugOf(task)}-${id}`;
|
|
120
|
+
const branch = `ael/${slug}`;
|
|
121
|
+
const worktreePath = path.join(path.dirname(mainRoot), `${path.basename(mainRoot)}.ael-worktrees`, slug);
|
|
122
|
+
const base = git(mainRoot, ['rev-parse', 'HEAD']);
|
|
123
|
+
fs.mkdirSync(path.dirname(worktreePath), { recursive: true });
|
|
124
|
+
git(mainRoot, ['worktree', 'add', '--quiet', '-b', branch, worktreePath, base]);
|
|
125
|
+
copyContext(mainRoot, worktreePath);
|
|
126
|
+
return { mainRoot, path: worktreePath, branch, base };
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function registerWorktree(mainRoot, entry) {
|
|
130
|
+
const registry = loadRegistry(mainRoot);
|
|
131
|
+
registry.worktrees = registry.worktrees.filter((item) => realpath(item.path) !== realpath(entry.path)).concat(entry);
|
|
132
|
+
saveRegistry(mainRoot, registry);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function isDirty(worktreePath) {
|
|
136
|
+
const out = git(worktreePath, ['status', '--porcelain', '--', '.', ':(exclude).ai-engineering-loop'], { allowFail: true });
|
|
137
|
+
return out == null ? true : out.split('\n').filter(Boolean).length > 0;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function copyEvidence(worktreePath, mainRoot, runId) {
|
|
141
|
+
if (!runId) return false;
|
|
142
|
+
const from = path.join(worktreePath, '.ai-engineering-loop', 'runs', runId);
|
|
143
|
+
if (!fs.existsSync(from)) return false;
|
|
144
|
+
const to = path.join(mainRoot, '.ai-engineering-loop', 'runs', runId);
|
|
145
|
+
fs.mkdirSync(path.dirname(to), { recursive: true });
|
|
146
|
+
fs.cpSync(from, to, { recursive: true, force: true });
|
|
147
|
+
return true;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function removeWorktree(mainRoot, worktreePath) {
|
|
151
|
+
git(mainRoot, ['worktree', 'remove', '--force', worktreePath]);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function cleanupAfterDeliver(cwd, runId, { now = new Date() } = {}) {
|
|
155
|
+
const mainRoot = mainCheckout(cwd);
|
|
156
|
+
if (!mainRoot || realpath(mainRoot) === realpath(cwd)) return null;
|
|
157
|
+
const registry = loadRegistry(mainRoot);
|
|
158
|
+
const entry = registry.worktrees.find((item) => realpath(item.path) === realpath(cwd));
|
|
159
|
+
if (!entry) return null;
|
|
160
|
+
const evidenceCopied = copyEvidence(cwd, mainRoot, runId || entry.runId);
|
|
161
|
+
const result = { worktree: entry.path, branch: entry.branch, mainCheckout: mainRoot, evidenceCopied, removed: false };
|
|
162
|
+
if (isDirty(cwd)) {
|
|
163
|
+
result.keptReason = 'the worktree has uncommitted or untracked changes; commit or discard them, then run `worktree prune`';
|
|
164
|
+
entry.state = 'delivered';
|
|
165
|
+
} else {
|
|
166
|
+
removeWorktree(mainRoot, cwd);
|
|
167
|
+
result.removed = true;
|
|
168
|
+
entry.state = 'removed';
|
|
169
|
+
entry.removedAt = now.toISOString();
|
|
170
|
+
}
|
|
171
|
+
entry.deliveredAt = now.toISOString();
|
|
172
|
+
saveRegistry(mainRoot, registry);
|
|
173
|
+
return result;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function defaultBranch(mainRoot) {
|
|
177
|
+
const head = git(mainRoot, ['symbolic-ref', '--quiet', 'refs/remotes/origin/HEAD'], { allowFail: true });
|
|
178
|
+
if (head) return head.replace('refs/remotes/', '');
|
|
179
|
+
for (const name of ['main', 'master']) {
|
|
180
|
+
if (git(mainRoot, ['rev-parse', '--verify', '--quiet', name], { allowFail: true })) return name;
|
|
181
|
+
}
|
|
182
|
+
return null;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
function isMerged(mainRoot, branch) {
|
|
186
|
+
if (!git(mainRoot, ['rev-parse', '--verify', '--quiet', branch], { allowFail: true })) return { merged: false, gone: true };
|
|
187
|
+
const target = defaultBranch(mainRoot);
|
|
188
|
+
if (target && spawnSync('git', ['merge-base', '--is-ancestor', branch, target], { cwd: mainRoot }).status === 0) return { merged: true };
|
|
189
|
+
const gh = spawnSync('gh', ['pr', 'list', '--head', branch.replace(/^origin\//, ''), '--state', 'merged', '--json', 'number'], { cwd: mainRoot, encoding: 'utf8', timeout: 15000 });
|
|
190
|
+
if (gh.status === 0) {
|
|
191
|
+
try {
|
|
192
|
+
if (JSON.parse(gh.stdout || '[]').length) return { merged: true };
|
|
193
|
+
} catch {
|
|
194
|
+
return { merged: false };
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
return { merged: false };
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
function runStateOf(entry, mainRoot) {
|
|
201
|
+
for (const root of [entry.path, mainRoot]) {
|
|
202
|
+
const state = readJson(path.join(root, '.ai-engineering-loop', 'runs', entry.runId || '-', 'state.json'));
|
|
203
|
+
if (state) return state;
|
|
204
|
+
}
|
|
205
|
+
return null;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
function planPrune(rootDir, { now = new Date(), checkMerged = true } = {}) {
|
|
209
|
+
const mainRoot = mainCheckout(rootDir) || rootDir;
|
|
210
|
+
const registry = loadRegistry(mainRoot);
|
|
211
|
+
const plan = [];
|
|
212
|
+
for (const entry of registry.worktrees) {
|
|
213
|
+
if (entry.branchDeleted) continue;
|
|
214
|
+
const exists = fs.existsSync(entry.path);
|
|
215
|
+
const run = runStateOf(entry, mainRoot);
|
|
216
|
+
const state = run?.state || null;
|
|
217
|
+
const row = { path: entry.path, branch: entry.branch, runId: entry.runId || null, runState: state, actions: [], notes: [] };
|
|
218
|
+
if (!exists && entry.state !== 'removed') {
|
|
219
|
+
row.actions.push('forget-missing-folder');
|
|
220
|
+
} else if (exists) {
|
|
221
|
+
if (state === 'DELIVERED' || state === 'CANCELLED' || state === 'ESCALATED') {
|
|
222
|
+
if (isDirty(entry.path)) row.notes.push('kept: uncommitted or untracked changes');
|
|
223
|
+
else row.actions.push('remove-worktree');
|
|
224
|
+
} else if (run) {
|
|
225
|
+
const activity = runActivity(entry.path, run, { now });
|
|
226
|
+
if (activity.idleMinutes != null && activity.idleMinutes >= IDLE_DAYS * 1440) row.notes.push(`run ${run.runId} idle ${Math.floor(activity.idleMinutes / 1440)} days: abandon it there if it is dead`);
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
if (checkMerged && entry.branch && (state === 'DELIVERED' || entry.state === 'removed' || entry.state === 'delivered')) {
|
|
230
|
+
const merged = isMerged(mainRoot, entry.branch);
|
|
231
|
+
if (merged.merged) {
|
|
232
|
+
if (exists && !row.actions.includes('remove-worktree')) row.notes.push('branch merged but worktree kept until it is clean');
|
|
233
|
+
else row.actions.push('delete-branch');
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
if (row.actions.length || row.notes.length) plan.push(row);
|
|
237
|
+
}
|
|
238
|
+
return { mainRoot, plan };
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
function applyPrune(rootDir, { now = new Date() } = {}) {
|
|
242
|
+
const { mainRoot, plan } = planPrune(rootDir, { now });
|
|
243
|
+
const registry = loadRegistry(mainRoot);
|
|
244
|
+
const done = [];
|
|
245
|
+
for (const row of plan) {
|
|
246
|
+
const entry = registry.worktrees.find((item) => item.path === row.path);
|
|
247
|
+
for (const action of row.actions) {
|
|
248
|
+
if (action === 'remove-worktree') {
|
|
249
|
+
if (isDirty(row.path)) continue;
|
|
250
|
+
copyEvidence(row.path, mainRoot, row.runId);
|
|
251
|
+
removeWorktree(mainRoot, row.path);
|
|
252
|
+
entry.state = 'removed';
|
|
253
|
+
entry.removedAt = now.toISOString();
|
|
254
|
+
} else if (action === 'forget-missing-folder') {
|
|
255
|
+
git(mainRoot, ['worktree', 'prune'], { allowFail: true });
|
|
256
|
+
entry.state = 'removed';
|
|
257
|
+
entry.removedAt = now.toISOString();
|
|
258
|
+
} else if (action === 'delete-branch') {
|
|
259
|
+
if (fs.existsSync(row.path)) continue;
|
|
260
|
+
git(mainRoot, ['branch', '-D', row.branch], { allowFail: true });
|
|
261
|
+
entry.branchDeleted = true;
|
|
262
|
+
}
|
|
263
|
+
done.push({ path: row.path, action });
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
saveRegistry(mainRoot, registry);
|
|
267
|
+
return { mainRoot, done, plan };
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
function pruneHint(rootDir) {
|
|
271
|
+
try {
|
|
272
|
+
const { plan } = planPrune(rootDir, { checkMerged: false });
|
|
273
|
+
const ready = plan.filter((row) => row.actions.length).length;
|
|
274
|
+
return ready ? `${ready} AEL worktree(s) can be cleaned up: ai-engineering-loop worktree prune` : null;
|
|
275
|
+
} catch {
|
|
276
|
+
return null;
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
module.exports = {
|
|
281
|
+
IDLE_DAYS,
|
|
282
|
+
LIVE_MINUTES,
|
|
283
|
+
applyPrune,
|
|
284
|
+
cleanupAfterDeliver,
|
|
285
|
+
createWorktree,
|
|
286
|
+
loadRegistry,
|
|
287
|
+
mainCheckout,
|
|
288
|
+
planPrune,
|
|
289
|
+
pruneHint,
|
|
290
|
+
registerWorktree,
|
|
291
|
+
runActivity,
|
|
292
|
+
runConflict
|
|
293
|
+
};
|
package/lib/run-state.js
CHANGED
|
@@ -303,7 +303,8 @@ function createOrResumeRun(
|
|
|
303
303
|
const current = getCurrentRun(rootDir);
|
|
304
304
|
const requestedTask = String(task || '').trim();
|
|
305
305
|
if (current && !TERMINAL_STATES.has(current.state) && !forceNew) {
|
|
306
|
-
|
|
306
|
+
const same = (a, b) => String(a).trim().replace(/\s+/g, ' ').toLowerCase() === String(b).trim().replace(/\s+/g, ' ').toLowerCase();
|
|
307
|
+
if (requestedTask && current.task && !same(requestedTask, current.task)) {
|
|
307
308
|
const err = new Error(
|
|
308
309
|
`Run ${current.runId} is still ${current.state}. Resume it or pass --new to start another task.`
|
|
309
310
|
);
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { spawnSync } = require('child_process');
|
|
6
|
+
const { destructiveFindings } = require('./guard.js');
|
|
7
|
+
const { redactHomePaths, redactSecrets } = require('./safe-context.js');
|
|
8
|
+
|
|
9
|
+
const FILE = 'safety-fact.json';
|
|
10
|
+
const LEVELS = {
|
|
11
|
+
1: 'stated',
|
|
12
|
+
2: 'pointed at the line',
|
|
13
|
+
3: 'walked the failure path',
|
|
14
|
+
4: 'ran real code',
|
|
15
|
+
5: 'reproduced in the running app'
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const fail = (message, code = 'INVALID_USAGE') => Object.assign(new Error(message), { code });
|
|
19
|
+
const factPath = (rootDir, runId) => path.join(rootDir, '.ai-engineering-loop', 'runs', runId, FILE);
|
|
20
|
+
|
|
21
|
+
function recordSafetyFact(rootDir, runId, { fact, command = null, live = false, level = null, timeoutMs = 120000, now = new Date() }) {
|
|
22
|
+
const text = String(fact || '').trim();
|
|
23
|
+
if (!text) throw fail('--fact is required: the one fact this change is safe because of');
|
|
24
|
+
let record;
|
|
25
|
+
if (command) {
|
|
26
|
+
if (destructiveFindings(command).length) throw fail('The proof command looks destructive; prove the fact with a read-only script or test');
|
|
27
|
+
const result = spawnSync(command, { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs, shell: true });
|
|
28
|
+
if (result.status !== 0) {
|
|
29
|
+
throw fail(`The proof command did not pass (exit ${result.status ?? 'none'}), so the fact is not proven. Fix the change or the fact, then record again.`, 'SAFETY_FACT_NOT_PROVEN');
|
|
30
|
+
}
|
|
31
|
+
record = {
|
|
32
|
+
fact: text,
|
|
33
|
+
level: live ? 5 : 4,
|
|
34
|
+
command: redactHomePaths(redactSecrets(command).text),
|
|
35
|
+
exitCode: 0,
|
|
36
|
+
outputTail: redactHomePaths(redactSecrets(String(result.stdout || '')).text).slice(-1000)
|
|
37
|
+
};
|
|
38
|
+
} else {
|
|
39
|
+
const claimed = level == null ? 1 : Number(level);
|
|
40
|
+
if (![1, 2, 3].includes(claimed)) throw fail('--level without --cmd must be 1, 2 or 3; levels 4 and 5 need a proof command that passes');
|
|
41
|
+
if (live) throw fail('--live needs --cmd: level 5 is a passing proof run against the running app');
|
|
42
|
+
record = { fact: text, level: claimed };
|
|
43
|
+
}
|
|
44
|
+
const value = { schemaVersion: 1, runId, ...record, levelName: LEVELS[record.level], recordedAt: now.toISOString() };
|
|
45
|
+
fs.writeFileSync(factPath(rootDir, runId), `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
|
|
46
|
+
return value;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function loadSafetyFact(rootDir, runId) {
|
|
50
|
+
try {
|
|
51
|
+
return JSON.parse(fs.readFileSync(factPath(rootDir, runId), 'utf8'));
|
|
52
|
+
} catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function safetyFactProblem(rootDir, run, { minimum = 4 } = {}) {
|
|
58
|
+
const value = loadSafetyFact(rootDir, run.runId);
|
|
59
|
+
if (!value) return 'no safety fact recorded';
|
|
60
|
+
if (value.level < minimum) return `safety fact is level ${value.level} (${LEVELS[value.level]}); level ${minimum} needs a proof command that passes`;
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
module.exports = { LEVELS, loadSafetyFact, recordSafetyFact, safetyFactProblem };
|
package/lib/stats.js
CHANGED
|
@@ -73,7 +73,10 @@ function computeStats(rootDir, { last = null } = {}) {
|
|
|
73
73
|
}
|
|
74
74
|
const labeled = Object.entries(tokensByRole).filter(([role]) => role !== 'unlabeled').reduce((sum, [, n]) => sum + n, 0);
|
|
75
75
|
const checking = ['reviewer', 'judge', 'verification'].reduce((sum, role) => sum + (tokensByRole[role] || 0), 0);
|
|
76
|
+
const { runInterventions, summarizeInterventions } = require('./interventions.js');
|
|
77
|
+
const interventions = summarizeInterventions(runs.map((runId) => runInterventions(path.join(runsDir(rootDir), runId))));
|
|
76
78
|
return {
|
|
79
|
+
interventions,
|
|
77
80
|
revertRate: reverts.rate,
|
|
78
81
|
reverted: reverts.reverted,
|
|
79
82
|
delivered: reverts.delivered,
|
|
@@ -100,6 +103,12 @@ function formatStats(stats) {
|
|
|
100
103
|
];
|
|
101
104
|
lines.push(`Revert rate: ${stats.reverted.length}/${stats.delivered} delivered Run(s) (${Math.round(stats.revertRate * 1000) / 10}%)${stats.reverted.length ? `: ${stats.reverted.map((item) => item.runId).join(', ')}` : ''}`);
|
|
102
105
|
if (stats.verificationTokenShare != null) lines.push(`Verification token share: ${Math.round(stats.verificationTokenShare * 100)}% (reviewer + judge + verification of role-labelled usage)`);
|
|
106
|
+
if (stats.interventions?.runs) {
|
|
107
|
+
const i = stats.interventions;
|
|
108
|
+
lines.push(`Human interventions per Run: ${i.perRun} (${i.answers} answered questions, ${i.escalations} escalations, ${i.outcomes} problems found after delivery)`);
|
|
109
|
+
const ran = i.proofs.reproduced + i.proofs.not_reproduced;
|
|
110
|
+
if (ran) lines.push(`Finding proofs: ${i.proofs.reproduced}/${ran} reproduced (${Math.round(i.reproducedRate * 100)}%); ${i.proofs.not_reproduced} blocking finding(s) did not reproduce and stopped blocking`);
|
|
111
|
+
}
|
|
103
112
|
if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
|
|
104
113
|
for (const [tier, bucket] of Object.entries(stats.tiers)) {
|
|
105
114
|
lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
|
|
@@ -160,6 +160,28 @@ function startupRiskFiles(rootDir, run) {
|
|
|
160
160
|
return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
+
function formalCoveredFiles(rootDir, run) {
|
|
164
|
+
let text = '';
|
|
165
|
+
try {
|
|
166
|
+
text = fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'verification.md'), 'utf8');
|
|
167
|
+
} catch {
|
|
168
|
+
return [];
|
|
169
|
+
}
|
|
170
|
+
const match = text.match(/\*\*formal_covers\*\*:\s*(.+)/i);
|
|
171
|
+
if (!match) return [];
|
|
172
|
+
const globs = match[1].replace(/`/g, '').split(',').map((glob) => glob.trim()).filter(Boolean);
|
|
173
|
+
const artifact = run?.artifacts?.diff;
|
|
174
|
+
if (!artifact?.path || !globs.length) return [];
|
|
175
|
+
let diff = '';
|
|
176
|
+
try {
|
|
177
|
+
diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
|
|
178
|
+
} catch {
|
|
179
|
+
return [];
|
|
180
|
+
}
|
|
181
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
182
|
+
return [...new Set(extractChangedPaths(diff))].filter((file) => globs.some((glob) => matchesGlob(file, glob)));
|
|
183
|
+
}
|
|
184
|
+
|
|
163
185
|
function readJson(filePath) {
|
|
164
186
|
try {
|
|
165
187
|
return JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
@@ -193,6 +215,7 @@ function finalVerificationStatus(rootDir, run) {
|
|
|
193
215
|
}
|
|
194
216
|
|
|
195
217
|
module.exports = {
|
|
218
|
+
formalCoveredFiles,
|
|
196
219
|
VERIFICATION_CATALOG,
|
|
197
220
|
keyOfCommand,
|
|
198
221
|
DEFAULT_PLAN_KEYS,
|
package/package.json
CHANGED
|
@@ -53,6 +53,10 @@
|
|
|
53
53
|
"minLength": 1
|
|
54
54
|
}
|
|
55
55
|
},
|
|
56
|
+
"measure": {
|
|
57
|
+
"description": "Optional (C-33) for performance ACs: Judge PASS needs `measurement record` with at least 3 runs, a limiter, and spread smaller than the effect.",
|
|
58
|
+
"type": "boolean"
|
|
59
|
+
},
|
|
56
60
|
"rubric": {
|
|
57
61
|
"description": "Optional (C-09) for subjective ACs: criteria the Devil's Advocate scores with evidence in ledger.rubricScores. Judge PASS needs every criterion scored pass:true.",
|
|
58
62
|
"type": "array",
|