ai-engineering-loop 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +13 -1
- package/.agents/skills/ael/references/loop-deliver.md +1 -1
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +11 -9
- package/.claude/agents/devil-advocate.md +7 -2
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +7 -2
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +31 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +7 -2
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +326 -21
- package/core/bench.md +15 -0
- package/core/formal-statechart.md +49 -0
- package/core/goal-contract.md +53 -0
- package/core/verification-loop.md +106 -1
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +95 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/budget.js +9 -1
- package/lib/calibration.js +129 -0
- package/lib/claimed-vs-reality.js +106 -12
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/comment-findings.js +111 -0
- package/lib/debt.js +110 -0
- package/lib/feature-map.js +57 -0
- package/lib/features.js +95 -0
- package/lib/flow.js +86 -0
- package/lib/gates.js +257 -41
- package/lib/harness-audit.js +141 -0
- package/lib/learn.js +24 -1
- package/lib/no-progress.js +6 -1
- package/lib/receipt.js +33 -2
- package/lib/reverts.js +77 -0
- package/lib/review-ingest.js +54 -2
- package/lib/rule-check.js +90 -0
- package/lib/rules.js +205 -0
- package/lib/stats.js +110 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +96 -0
- package/lib/verification-recorder.js +20 -8
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +50 -0
- package/schemas/verdict.schema.json +2 -1
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// `doctor --harness` (C-08, learn-harness-engineering L03/L04/L12): harness
|
|
4
|
+
// debt in the project, reported as candidates to fix or delete. Harness rots
|
|
5
|
+
// like code: entry files bloat, learnings go stale, rules outlive their need.
|
|
6
|
+
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
const { runsDir } = require('./run-state.js');
|
|
10
|
+
|
|
11
|
+
const ENTRY_MIN = 50;
|
|
12
|
+
const ENTRY_MAX = 200;
|
|
13
|
+
const STALE_LEARNING_RUNS = 10;
|
|
14
|
+
const RULE_REVIEW_DAYS = 90;
|
|
15
|
+
const BUDGET_HEADROOM = 0.02;
|
|
16
|
+
|
|
17
|
+
// Only structured lines count (headings, lists, tables, code, bold leads), so
|
|
18
|
+
// sprinkling keywords into prose cannot game the score (Goodhart, L14).
|
|
19
|
+
// Adapted from harness-creator's structuredText (MIT).
|
|
20
|
+
function structuredText(markdown) {
|
|
21
|
+
const kept = [];
|
|
22
|
+
let inFence = false;
|
|
23
|
+
for (const raw of String(markdown || '').split(/\r?\n/)) {
|
|
24
|
+
const line = raw.trim();
|
|
25
|
+
if (/^(```|~~~)/.test(line)) {
|
|
26
|
+
inFence = !inFence;
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
if (inFence || /^#{1,6}\s/.test(line) || /^([-*+]|\d+\.)\s/.test(line) || line.startsWith('|') || /^\*\*[^*]+\*\*/.test(line)) {
|
|
30
|
+
if (line) kept.push(line);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return kept.join('\n').toLowerCase();
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function auditEntryFile(rootDir) {
|
|
37
|
+
const name = ['AGENTS.md', 'CLAUDE.md'].find((file) => fs.existsSync(path.join(rootDir, file)));
|
|
38
|
+
if (!name) return [{ id: 'entry-file', level: 'warn', detail: 'No AGENTS.md or CLAUDE.md. FIX: add a 50-200 line router: overview, run/verify commands, hard rules, links to docs.' }];
|
|
39
|
+
const text = fs.readFileSync(path.join(rootDir, name), 'utf8');
|
|
40
|
+
const lines = text.split(/\r?\n/).length;
|
|
41
|
+
const items = [];
|
|
42
|
+
if (lines > ENTRY_MAX) items.push({ id: 'entry-file-size', level: 'warn', detail: `${name} has ${lines} lines (> ${ENTRY_MAX}). FIX: keep it a router; move topic detail to docs/ and link it.` });
|
|
43
|
+
if (lines < ENTRY_MIN) items.push({ id: 'entry-file-size', level: 'info', detail: `${name} has ${lines} lines; fine if it still answers what/run/verify.` });
|
|
44
|
+
const structured = structuredText(text);
|
|
45
|
+
const needs = [
|
|
46
|
+
['verification', /verif|test|check/],
|
|
47
|
+
['definition of done', /definition of done|done when|done only when/],
|
|
48
|
+
['startup', /startup|start here|quick start|before writing|setup/]
|
|
49
|
+
];
|
|
50
|
+
for (const [label, pattern] of needs) {
|
|
51
|
+
if (!pattern.test(structured)) items.push({ id: 'entry-file-structure', level: 'warn', detail: `${name} has no structured ${label} section (heading or list). FIX: add one; prose mentions do not count.` });
|
|
52
|
+
}
|
|
53
|
+
return items;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function listRunIds(rootDir) {
|
|
57
|
+
try {
|
|
58
|
+
return fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
59
|
+
} catch {
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function auditLearnings(rootDir) {
|
|
65
|
+
let entries = [];
|
|
66
|
+
try {
|
|
67
|
+
entries = JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'learnings.json'), 'utf8')).entries || [];
|
|
68
|
+
} catch {
|
|
69
|
+
return [];
|
|
70
|
+
}
|
|
71
|
+
const runs = listRunIds(rootDir);
|
|
72
|
+
const recent = new Set(runs.slice(-STALE_LEARNING_RUNS));
|
|
73
|
+
if (runs.length <= STALE_LEARNING_RUNS) return [];
|
|
74
|
+
return entries
|
|
75
|
+
.filter((entry) => !(entry.runIds || []).some((runId) => recent.has(runId)))
|
|
76
|
+
.map((entry) => ({ id: 'stale-learning', level: 'warn', detail: `${entry.id} not seen in the last ${STALE_LEARNING_RUNS} Runs: ${String(entry.lesson || '').slice(0, 80)}. FIX: \`learn --prune\` or delete it if the problem is gone.` }));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function auditRules(rootDir, now = new Date()) {
|
|
80
|
+
let rules = [];
|
|
81
|
+
try {
|
|
82
|
+
rules = JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'rules.json'), 'utf8')).rules || [];
|
|
83
|
+
} catch {
|
|
84
|
+
return [];
|
|
85
|
+
}
|
|
86
|
+
const items = [];
|
|
87
|
+
for (const rule of rules) {
|
|
88
|
+
if (!rule.from || !rule.why || !rule.fix) items.push({ id: 'rule-provenance', level: 'warn', detail: `${rule.id} lacks a source (from), why or fix. FIX: record where it came from and how to repair a violation.` });
|
|
89
|
+
if (rule.scope === 'diff') {
|
|
90
|
+
const backlog = require('./rules.js').legacyBacklog(rootDir, rule);
|
|
91
|
+
if (backlog.error) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id}: ${backlog.error}` });
|
|
92
|
+
else if (backlog.clean) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff) has no legacy violations left. Switch it to scope repo so the whole codebase stays clean.` });
|
|
93
|
+
else items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff): legacy violations remain (${backlog.outputLines} output line(s), e.g. ${backlog.sample[0] || 'n/a'}). Clean them up over time; new ones are already blocked.` });
|
|
94
|
+
}
|
|
95
|
+
const age = (now - new Date(rule.createdAt || 0)) / 86400000;
|
|
96
|
+
if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
|
|
97
|
+
}
|
|
98
|
+
return items;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// Inside the AEL package repo: T1 skill-context headroom from scripts/measure.js.
|
|
102
|
+
function auditBudget(rootDir) {
|
|
103
|
+
const script = path.join(rootDir, 'scripts', 'measure.js');
|
|
104
|
+
if (!fs.existsSync(script)) return [];
|
|
105
|
+
try {
|
|
106
|
+
const { measureAll, BASELINE } = require(script);
|
|
107
|
+
const t1 = measureAll().results.find((row) => row.tier === 'T1');
|
|
108
|
+
const limit = BASELINE.parentContextChars / 2;
|
|
109
|
+
const headroom = (limit - t1.parentContextChars) / limit;
|
|
110
|
+
return [{
|
|
111
|
+
id: 'skill-budget',
|
|
112
|
+
level: headroom < BUDGET_HEADROOM ? 'warn' : 'info',
|
|
113
|
+
detail: `T1 skill context ${t1.parentContextChars}/${limit} chars (${(headroom * 100).toFixed(2)}% headroom).${headroom < BUDGET_HEADROOM ? ' FIX: trim skill-src references before adding text.' : ''}`
|
|
114
|
+
}];
|
|
115
|
+
} catch (cause) {
|
|
116
|
+
return [{ id: 'skill-budget', level: 'warn', detail: `measure failed: ${cause.message}` }];
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
// C-22: runtime verification without a feature map makes agents guess what
|
|
121
|
+
// to exercise.
|
|
122
|
+
function auditFeatureMap(rootDir) {
|
|
123
|
+
const lanes = require('./verification-lanes.js');
|
|
124
|
+
const commands = lanes.verificationCommands(rootDir);
|
|
125
|
+
if (!commands.e2e && !commands.smoke) return [];
|
|
126
|
+
if (require('./feature-map.js').loadFeatureMap(rootDir).length) return [];
|
|
127
|
+
return [{ id: 'feature-map', level: 'warn', detail: 'e2e/smoke is configured but .ai-engineering-loop/feature-map.md is missing or empty. FIX: list user-facing features (Feature | Where | Paths | How to reach | Driver) so agents and reviewers share one vocabulary for runtime checks.' }];
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function auditHarness(rootDir, { budget = true } = {}) {
|
|
131
|
+
const items = [
|
|
132
|
+
...auditEntryFile(rootDir),
|
|
133
|
+
...auditFeatureMap(rootDir),
|
|
134
|
+
...auditLearnings(rootDir),
|
|
135
|
+
...auditRules(rootDir),
|
|
136
|
+
...(budget ? auditBudget(rootDir) : [])
|
|
137
|
+
];
|
|
138
|
+
return { ok: !items.some((item) => item.level === 'warn'), items };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
module.exports = { auditHarness, structuredText };
|
package/lib/learn.js
CHANGED
|
@@ -255,6 +255,25 @@ function relevance(entry, files) {
|
|
|
255
255
|
return score;
|
|
256
256
|
}
|
|
257
257
|
|
|
258
|
+
// C-19 ("All Code Is Slop (Except Mine)", 06:46): agents over-rotate on a
|
|
259
|
+
// single correction and treat it as a hard rule. Confidence follows how many
|
|
260
|
+
// Runs produced the lesson; only rule-candidates read as firm guidance.
|
|
261
|
+
const PROMOTE_AFTER_RUNS = 3;
|
|
262
|
+
|
|
263
|
+
function confidenceOf(entry) {
|
|
264
|
+
const runs = (entry.runIds || []).length;
|
|
265
|
+
if (runs >= PROMOTE_AFTER_RUNS) return 'rule-candidate';
|
|
266
|
+
if (runs === 2) return 'recurring';
|
|
267
|
+
return 'tentative';
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
// Wording that matches the confidence: a one-off lesson is a hint, never a rule.
|
|
271
|
+
function guidanceFor(confidence, lesson) {
|
|
272
|
+
if (confidence === 'tentative') return `Consider (seen in 1 Run, not a rule): ${lesson}`;
|
|
273
|
+
if (confidence === 'recurring') return `Recurring (2 Runs), weigh it: ${lesson}`;
|
|
274
|
+
return `Repeated in ${PROMOTE_AFTER_RUNS}+ Runs (promote it to a rule): ${lesson}`;
|
|
275
|
+
}
|
|
276
|
+
|
|
258
277
|
// Top-K learnings for a context pack, greedily kept inside `budgetTokens`.
|
|
259
278
|
// Entries relevant to the pack's files rank first; project-wide recurring
|
|
260
279
|
// lessons (2+ runs) fill the rest.
|
|
@@ -269,11 +288,13 @@ function selectLearnings(rootDir, { files = [], budgetTokens = 0, limit = MAX_CO
|
|
|
269
288
|
let tokens = 0;
|
|
270
289
|
for (const { entry } of ranked) {
|
|
271
290
|
if (selected.length >= limit) break;
|
|
291
|
+
const confidence = confidenceOf(entry);
|
|
272
292
|
const item = {
|
|
273
293
|
id: entry.id,
|
|
274
294
|
axis: entry.axis,
|
|
275
295
|
severity: entry.severity,
|
|
276
|
-
|
|
296
|
+
confidence,
|
|
297
|
+
lesson: guidanceFor(confidence, safeText(entry.lesson)),
|
|
277
298
|
locations: entry.locations.slice(0, 5),
|
|
278
299
|
occurrences: entry.occurrences
|
|
279
300
|
};
|
|
@@ -291,6 +312,8 @@ module.exports = {
|
|
|
291
312
|
LEARNINGS_BUDGET_SHARE,
|
|
292
313
|
DEFAULT_MAX_AGE_RUNS,
|
|
293
314
|
isLesson,
|
|
315
|
+
confidenceOf,
|
|
316
|
+
guidanceFor,
|
|
294
317
|
learn,
|
|
295
318
|
loadLearnings,
|
|
296
319
|
selectLearnings,
|
package/lib/no-progress.js
CHANGED
|
@@ -87,12 +87,17 @@ function detectNoProgress(rootDir, run, { diffHash, blockingFindings = [] }) {
|
|
|
87
87
|
return null;
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
function recordIteration(rootDir, run, { verdict, diffHash, blockingFindings = [], now = new Date() }) {
|
|
90
|
+
function recordIteration(rootDir, run, { verdict, diffHash, blockingFindings = [], cause = null, causeInferred = false, now = new Date() }) {
|
|
91
91
|
const entry = {
|
|
92
92
|
iteration: run.iteration,
|
|
93
93
|
verdict,
|
|
94
94
|
diffHash: diffHash || null,
|
|
95
95
|
blockingSignatures: blockingFindings.map(findingSignature),
|
|
96
|
+
// C-12: verification was green when the Judge ran, so blocking findings
|
|
97
|
+
// here mean the Maker's "done" was premature (a false victory).
|
|
98
|
+
blockingCount: blockingFindings.length,
|
|
99
|
+
...(cause ? { cause, causeInferred: Boolean(causeInferred) } : {}),
|
|
100
|
+
tier: run.tier?.current || null,
|
|
96
101
|
at: now.toISOString()
|
|
97
102
|
};
|
|
98
103
|
const filePath = iterationsPath(rootDir, run.runId);
|
package/lib/receipt.js
CHANGED
|
@@ -134,10 +134,27 @@ function coverageSummary(coverage) {
|
|
|
134
134
|
return numbers.length ? Object.fromEntries(numbers.sort(([a], [b]) => a.localeCompare(b))) : null;
|
|
135
135
|
}
|
|
136
136
|
|
|
137
|
+
// C-21: ACs whose proof is visual or crosses components need an artifact
|
|
138
|
+
// (video, screenshot, diagram) a reviewer can look at instead of the diff.
|
|
139
|
+
function needsArtifact(criterion) {
|
|
140
|
+
return criterion.crossComponent === true || (Array.isArray(criterion.rubric) && criterion.rubric.length > 0);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function artifactKind(url) {
|
|
144
|
+
const value = String(url).toLowerCase();
|
|
145
|
+
if (/\.(mp4|webm|mov|gif)(\?|$)|loom\.com|youtu/.test(value)) return 'video';
|
|
146
|
+
if (/\.(png|jpe?g|webp)(\?|$)/.test(value)) return 'screenshot';
|
|
147
|
+
if (/\.(svg|drawio)(\?|$)|mermaid|excalidraw|figma\.com/.test(value)) return 'diagram';
|
|
148
|
+
return 'link';
|
|
149
|
+
}
|
|
150
|
+
|
|
137
151
|
function criterionEntry(criterion, { commands, claims, links }) {
|
|
138
152
|
const claim = claims.find((row) => row.ac === criterion.id);
|
|
139
153
|
const evidence = [];
|
|
140
154
|
const missing = [];
|
|
155
|
+
if (needsArtifact(criterion) && !(links[criterion.id] || []).length) {
|
|
156
|
+
missing.push('no verification artifact (video, screenshot or diagram); pass --evidence ' + criterion.id + '=<https-url>');
|
|
157
|
+
}
|
|
141
158
|
if (claim && String(claim.reality || '').trim()) {
|
|
142
159
|
evidence.push({ type: 'claim', claimed: safeText(claim.claimed), reality: safeText(claim.reality) });
|
|
143
160
|
} else {
|
|
@@ -146,7 +163,7 @@ function criterionEntry(criterion, { commands, claims, links }) {
|
|
|
146
163
|
if (!commands.length) missing.push('no recorded verification command');
|
|
147
164
|
else if (commands.some((item) => !item.passed)) missing.push('a verification command did not pass');
|
|
148
165
|
evidence.push(...commands);
|
|
149
|
-
for (const url of links[criterion.id] || []) evidence.push({ type: 'link', url });
|
|
166
|
+
for (const url of links[criterion.id] || []) evidence.push({ type: 'link', kind: artifactKind(url), url });
|
|
150
167
|
return {
|
|
151
168
|
id: criterion.id,
|
|
152
169
|
statement: safeText(criterion.statement),
|
|
@@ -199,7 +216,8 @@ function buildReceipt(rootDir, { runId, host, model, head, evidenceLinks = [], a
|
|
|
199
216
|
sha256: hashBytes(fs.readFileSync(goalFile)),
|
|
200
217
|
contentHash: sha256(goal),
|
|
201
218
|
version: run.goal?.version || 1,
|
|
202
|
-
objective: safeText(goal.objective)
|
|
219
|
+
objective: safeText(goal.objective),
|
|
220
|
+
...(Array.isArray(goal.flow) && goal.flow.length ? { flow: goal.flow.map((step) => ({ from: safeText(step.from), event: safeText(step.event), to: safeText(step.to) })) } : {})
|
|
203
221
|
},
|
|
204
222
|
acceptanceCriteria: (goal.acceptanceCriteria || []).map((criterion) => criterionEntry(criterion, { commands, claims, links })),
|
|
205
223
|
verification: {
|
|
@@ -259,6 +277,17 @@ function renderReceiptMarkdown(receipt) {
|
|
|
259
277
|
`Run \`${receipt.runId}\` · Goal v${receipt.goal.version} \`${short(receipt.goal.contentHash)}\` · base \`${short(receipt.base.sha, 7)}\` → head \`${short(receipt.head.sha, 7)}\` · digest \`${short(receipt.digest)}\``,
|
|
260
278
|
''
|
|
261
279
|
];
|
|
280
|
+
// C-21: review artifacts first (talk 09:26: review shrinks to looking at
|
|
281
|
+
// videos, screenshots and diagrams), then the intended flow, then the rest.
|
|
282
|
+
const artifacts = receipt.acceptanceCriteria.flatMap((criterion) => criterion.evidence
|
|
283
|
+
.filter((item) => item.type === 'link')
|
|
284
|
+
.map((item) => `- **${criterion.id}** ${item.kind || 'link'}: [${item.kind || 'evidence'}](${item.url})`));
|
|
285
|
+
if (artifacts.length) lines.push('### Verification artifacts', '', ...artifacts, '');
|
|
286
|
+
if (receipt.goal.flow) {
|
|
287
|
+
const flowLib = require('./flow.js');
|
|
288
|
+
lines.push('### Intended flow', '', '```mermaid', flowLib.renderMermaid(receipt.goal.flow), '```', '', flowLib.renderFlow(receipt.goal.flow), '');
|
|
289
|
+
}
|
|
290
|
+
lines.push('### Acceptance criteria', '');
|
|
262
291
|
for (const criterion of receipt.acceptanceCriteria) {
|
|
263
292
|
const parts = [];
|
|
264
293
|
for (const item of criterion.evidence) {
|
|
@@ -358,6 +387,8 @@ function verifyReceipt(rootDir, { runId, receiptPath, head } = {}) {
|
|
|
358
387
|
}
|
|
359
388
|
|
|
360
389
|
module.exports = {
|
|
390
|
+
criterionEntry,
|
|
391
|
+
needsArtifact,
|
|
361
392
|
RECEIPT_FILE,
|
|
362
393
|
RECEIPT_MARKDOWN_FILE,
|
|
363
394
|
buildReceipt,
|
package/lib/reverts.js
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Revert detection (C-24, "All Code Is Slop (Except Mine)" 13:26-14:46):
|
|
4
|
+
// the revert rate of delivered work is a quality anchor that needs no human
|
|
5
|
+
// input. A git `Revert "<subject>"` commit is matched to a DELIVERED Run when
|
|
6
|
+
// it reverts that Run's receipt head (by sha) or a commit with the same
|
|
7
|
+
// subject (squash merges change the sha but keep the subject).
|
|
8
|
+
|
|
9
|
+
const fs = require('fs');
|
|
10
|
+
const path = require('path');
|
|
11
|
+
const { execFileSync } = require('child_process');
|
|
12
|
+
const { runsDir } = require('./run-state.js');
|
|
13
|
+
|
|
14
|
+
function git(rootDir, args) {
|
|
15
|
+
try {
|
|
16
|
+
return execFileSync('git', args, { cwd: rootDir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 16 * 1024 * 1024 });
|
|
17
|
+
} catch {
|
|
18
|
+
return '';
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function deliveredRuns(rootDir) {
|
|
23
|
+
let names = [];
|
|
24
|
+
try {
|
|
25
|
+
names = fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
26
|
+
} catch {
|
|
27
|
+
return [];
|
|
28
|
+
}
|
|
29
|
+
const runs = [];
|
|
30
|
+
for (const runId of names) {
|
|
31
|
+
const dir = path.join(runsDir(rootDir), runId);
|
|
32
|
+
let state = null;
|
|
33
|
+
let receipt = null;
|
|
34
|
+
try {
|
|
35
|
+
state = JSON.parse(fs.readFileSync(path.join(dir, 'state.json'), 'utf8'));
|
|
36
|
+
} catch {
|
|
37
|
+
continue;
|
|
38
|
+
}
|
|
39
|
+
if (state.state !== 'DELIVERED') continue;
|
|
40
|
+
try {
|
|
41
|
+
receipt = JSON.parse(fs.readFileSync(path.join(dir, 'receipt.json'), 'utf8'));
|
|
42
|
+
} catch {
|
|
43
|
+
receipt = null;
|
|
44
|
+
}
|
|
45
|
+
const head = receipt?.head?.sha || null;
|
|
46
|
+
const subject = head ? git(rootDir, ['log', '-1', '--format=%s', head]).trim() : '';
|
|
47
|
+
runs.push({ runId, head, subject });
|
|
48
|
+
}
|
|
49
|
+
return runs;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function revertCommits(rootDir) {
|
|
53
|
+
const out = git(rootDir, ['log', '--all', '-E', '--grep=^Revert "', '--format=%H%x1f%s%x1f%b%x1e']);
|
|
54
|
+
return out.split('\x1e').map((chunk) => chunk.trim()).filter(Boolean).map((chunk) => {
|
|
55
|
+
const [sha, subject, body] = chunk.split('\x1f');
|
|
56
|
+
const reverted = (body || '').match(/This reverts commit ([0-9a-f]{7,40})/);
|
|
57
|
+
const original = (subject || '').match(/^Revert "(.+)"$/);
|
|
58
|
+
return { sha, revertedSha: reverted ? reverted[1] : null, originalSubject: original ? original[1] : null };
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function detectReverts(rootDir) {
|
|
63
|
+
const runs = deliveredRuns(rootDir);
|
|
64
|
+
const reverts = revertCommits(rootDir);
|
|
65
|
+
const reverted = [];
|
|
66
|
+
for (const run of runs) {
|
|
67
|
+
const hit = reverts.find((revert) => (run.head && revert.revertedSha && (run.head.startsWith(revert.revertedSha) || revert.revertedSha.startsWith(run.head)))
|
|
68
|
+
|| (run.subject && revert.originalSubject === run.subject));
|
|
69
|
+
if (hit) {
|
|
70
|
+
const files = run.head ? git(rootDir, ['show', '--name-only', '--format=', run.head]).split('\n').filter(Boolean) : [];
|
|
71
|
+
reverted.push({ runId: run.runId, revertSha: hit.sha, location: files[0] || run.runId });
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return { delivered: runs.length, reverted, rate: runs.length ? Math.round((reverted.length / runs.length) * 1000) / 1000 : 0 };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { detectReverts };
|
package/lib/review-ingest.js
CHANGED
|
@@ -55,6 +55,49 @@ function extractLedger(text) {
|
|
|
55
55
|
return found;
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
// C-16 ("model-visible means logged"): what the reviewer saw. review-pack
|
|
59
|
+
// writes review-inputs.json; ingest copies it into the ledger, and the review
|
|
60
|
+
// gate refuses a ledger whose inputs no longer match the files on disk.
|
|
61
|
+
const INPUTS_FILE = 'review-inputs.json';
|
|
62
|
+
const PROMPT_BODY = path.join(__dirname, '..', 'agents', 'shared', 'devil-advocate.body.md');
|
|
63
|
+
|
|
64
|
+
function fileSha256(filePath) {
|
|
65
|
+
try {
|
|
66
|
+
return sha256(fs.readFileSync(filePath));
|
|
67
|
+
} catch {
|
|
68
|
+
return null;
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function recordReviewInputs(rootDir, runId, { packPath, diffPath } = {}) {
|
|
73
|
+
const runDir = path.join(runsDir(rootDir), runId);
|
|
74
|
+
const inputs = [packPath, diffPath].filter(Boolean).map((relative) => ({
|
|
75
|
+
path: relative,
|
|
76
|
+
sha256: fileSha256(path.join(rootDir, relative))
|
|
77
|
+
})).filter((item) => item.sha256);
|
|
78
|
+
const document = { schemaVersion: 1, runId, inputs, promptSha256: fileSha256(PROMPT_BODY), recordedAt: new Date().toISOString() };
|
|
79
|
+
fs.writeFileSync(path.join(runDir, INPUTS_FILE), `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
|
|
80
|
+
return document;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function loadReviewInputs(rootDir, runId) {
|
|
84
|
+
try {
|
|
85
|
+
return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, INPUTS_FILE), 'utf8'));
|
|
86
|
+
} catch {
|
|
87
|
+
return null;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Errors when a recorded input changed after the reviewer saw it.
|
|
92
|
+
function checkReviewInputs(rootDir, ledger) {
|
|
93
|
+
const errors = [];
|
|
94
|
+
for (const item of ledger?.reviewer?.inputs || []) {
|
|
95
|
+
const now = fileSha256(path.join(rootDir, item.path));
|
|
96
|
+
if (now !== item.sha256) errors.push(`${item.path} changed after the reviewer saw it (sha256 ${String(item.sha256).slice(0, 12)} -> ${now ? now.slice(0, 12) : 'missing'})`);
|
|
97
|
+
}
|
|
98
|
+
return errors;
|
|
99
|
+
}
|
|
100
|
+
|
|
58
101
|
function ingestReview(rootDir, { runId, childId, file } = {}) {
|
|
59
102
|
if (!String(childId || '').trim()) throw ingestError('--child-id is required (the reviewer subagent\'s own id)', 'INVALID_USAGE');
|
|
60
103
|
if (!file) throw ingestError('--file <child-output> is required', 'INVALID_USAGE');
|
|
@@ -75,7 +118,15 @@ function ingestReview(rootDir, { runId, childId, file } = {}) {
|
|
|
75
118
|
schemaVersion: 1,
|
|
76
119
|
runId: run.runId,
|
|
77
120
|
diffHash: run.artifacts?.diff?.sha256,
|
|
78
|
-
reviewer: {
|
|
121
|
+
reviewer: {
|
|
122
|
+
name: 'devil-advocate',
|
|
123
|
+
childId: String(childId).trim(),
|
|
124
|
+
rawSha256: sha256(raw),
|
|
125
|
+
...(() => {
|
|
126
|
+
const recorded = loadReviewInputs(rootDir, run.runId);
|
|
127
|
+
return recorded ? { inputs: recorded.inputs, promptSha256: recorded.promptSha256 } : {};
|
|
128
|
+
})()
|
|
129
|
+
},
|
|
79
130
|
findings: ledger.findings
|
|
80
131
|
};
|
|
81
132
|
fs.writeFileSync(path.join(runDir, 'findings.json'), `${JSON.stringify(document, null, 2)}\n`, { mode: 0o600 });
|
|
@@ -96,6 +147,7 @@ function checkLedgerBinding(rootDir, run, ledger) {
|
|
|
96
147
|
return { binding: 'BOUND', errors: [`${RAW_FILE} is missing; re-run review ingest`] };
|
|
97
148
|
}
|
|
98
149
|
if (sha256(raw) !== rawSha256) return { binding: 'BOUND', errors: [`${RAW_FILE} does not match reviewer.rawSha256`] };
|
|
150
|
+
errors.push(...checkReviewInputs(rootDir, ledger));
|
|
99
151
|
const original = extractLedger(raw.toString('utf8'))?.findings || [];
|
|
100
152
|
const current = new Map((ledger.findings || []).map((finding) => [finding.id, finding]));
|
|
101
153
|
for (const finding of original) {
|
|
@@ -113,4 +165,4 @@ function checkLedgerBinding(rootDir, run, ledger) {
|
|
|
113
165
|
return { binding: 'BOUND', errors };
|
|
114
166
|
}
|
|
115
167
|
|
|
116
|
-
module.exports = { RAW_FILE, extractLedger, ingestReview, checkLedgerBinding };
|
|
168
|
+
module.exports = { INPUTS_FILE, RAW_FILE, checkReviewInputs, extractLedger, ingestReview, checkLedgerBinding, loadReviewInputs, recordReviewInputs };
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Built-in rule kinds (C-20, "All Code Is Slop (Except Mine)" 05:24-06:46):
|
|
4
|
+
// the two corrections teams make most often, without hand-written grep.
|
|
5
|
+
// - forbid-import "<fromGlob> -> <toGlob>": files under fromGlob may not
|
|
6
|
+
// import modules under toGlob (relative paths are resolved; bare names
|
|
7
|
+
// such as `fs` or `react` are matched as written).
|
|
8
|
+
// - forbid-api "<regex>" except "<glob>": the API may only appear inside
|
|
9
|
+
// the excepted paths (for example a framework folder).
|
|
10
|
+
// Both judge only lines the change adds (the C-17 ratchet env).
|
|
11
|
+
|
|
12
|
+
const fs = require('fs');
|
|
13
|
+
const path = require('path');
|
|
14
|
+
const { matchesGlob } = require('./impact-map.js');
|
|
15
|
+
|
|
16
|
+
const IMPORT_SPECIFIER = /(?:\bfrom\s*|\bimport\s*\(\s*|\brequire\s*\(\s*|^\s*import\s+)['"]([^'"]+)['"]/g;
|
|
17
|
+
const EXTENSIONS = ['', '.js', '.ts', '.tsx', '.jsx', '.mjs', '.cjs', '/index.js', '/index.ts'];
|
|
18
|
+
|
|
19
|
+
function parseImportSpec(spec) {
|
|
20
|
+
const parts = String(spec || '').split('->').map((part) => part.trim());
|
|
21
|
+
if (parts.length !== 2 || !parts[0] || !parts[1]) {
|
|
22
|
+
throw Object.assign(new Error('--forbid-import needs "<fromGlob> -> <toGlob>", e.g. "src/renderer/** -> src/main/**"'), { code: 'INVALID_USAGE' });
|
|
23
|
+
}
|
|
24
|
+
return { from: parts[0], to: parts[1] };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function readAddedLines(env = process.env) {
|
|
28
|
+
const file = env.AEL_ADDED_LINES_FILE;
|
|
29
|
+
if (!file) return [];
|
|
30
|
+
let text = '';
|
|
31
|
+
try {
|
|
32
|
+
text = fs.readFileSync(file, 'utf8');
|
|
33
|
+
} catch {
|
|
34
|
+
return [];
|
|
35
|
+
}
|
|
36
|
+
return text.split('\n').filter(Boolean).flatMap((raw) => {
|
|
37
|
+
const match = raw.match(/^(.+?):(\d+):(.*)$/);
|
|
38
|
+
return match ? [{ file: match[1].replace(/\\/g, '/'), line: Number(match[2]), text: match[3] }] : [];
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function importTargets(file, text) {
|
|
43
|
+
const targets = [];
|
|
44
|
+
for (const match of text.matchAll(IMPORT_SPECIFIER)) {
|
|
45
|
+
const spec = match[1];
|
|
46
|
+
if (spec.startsWith('.')) {
|
|
47
|
+
const resolved = path.posix.normalize(path.posix.join(path.posix.dirname(file), spec));
|
|
48
|
+
targets.push({ spec, candidates: EXTENSIONS.map((ext) => `${resolved}${ext}`) });
|
|
49
|
+
} else {
|
|
50
|
+
targets.push({ spec, candidates: [spec] });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
return targets;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// [{ file, line, message }]
|
|
57
|
+
function checkRule(rule, lines) {
|
|
58
|
+
const violations = [];
|
|
59
|
+
if (rule.kind === 'forbid-import') {
|
|
60
|
+
const { from, to } = parseImportSpec(rule.spec);
|
|
61
|
+
for (const added of lines) {
|
|
62
|
+
if (!matchesGlob(added.file, from)) continue;
|
|
63
|
+
for (const target of importTargets(added.file, added.text)) {
|
|
64
|
+
if (target.candidates.some((candidate) => matchesGlob(candidate, to) || candidate === to)) {
|
|
65
|
+
violations.push({ file: added.file, line: added.line, message: `imports "${target.spec}" (${to}), forbidden from ${from}` });
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
} else if (rule.kind === 'forbid-api') {
|
|
70
|
+
const pattern = new RegExp(rule.pattern);
|
|
71
|
+
for (const added of lines) {
|
|
72
|
+
if (rule.except && matchesGlob(added.file, rule.except)) continue;
|
|
73
|
+
if (pattern.test(added.text)) {
|
|
74
|
+
violations.push({ file: added.file, line: added.line, message: `uses /${rule.pattern}/ outside ${rule.except || 'the allowed paths'}` });
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
} else {
|
|
78
|
+
throw Object.assign(new Error(`Rule ${rule.id} has no built-in kind`), { code: 'INVALID_RULE' });
|
|
79
|
+
}
|
|
80
|
+
return violations;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Shell text that runs `rules check <id>` with this very CLI, so a rules.json
|
|
84
|
+
// shared across machines never stores a machine-specific path.
|
|
85
|
+
function builtinCommand(rule) {
|
|
86
|
+
const bin = path.join(__dirname, '..', 'bin', 'ai-engineering-loop.js');
|
|
87
|
+
return `"${process.execPath}" "${bin}" rules check ${rule.id}`;
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
module.exports = { builtinCommand, checkRule, importTargets, parseImportSpec, readAddedLines };
|