ai-engineering-loop 2.10.1 → 2.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/.agents/skills/ael/references/loop-commands.md +2 -0
  2. package/.agents/skills/ael/references/loop-deliver.md +1 -0
  3. package/.agents/skills/ael/references/loop-goal.md +1 -0
  4. package/.agents/skills/ael/references/loop-risky.md +1 -0
  5. package/.claude/agents/devil-advocate.md +2 -1
  6. package/.claude/agents/judge.md +2 -0
  7. package/.grok/agents/devil-advocate.md +2 -1
  8. package/.grok/agents/judge.md +2 -0
  9. package/CHANGELOG.md +24 -0
  10. package/README.md +80 -75
  11. package/agents/shared/devil-advocate.body.md +2 -1
  12. package/agents/shared/judge.body.md +2 -0
  13. package/bin/ai-engineering-loop.js +153 -5
  14. package/core/goal-contract.md +4 -0
  15. package/core/iteration-policy.md +4 -0
  16. package/core/run-isolation.md +55 -0
  17. package/core/verification-loop.md +25 -0
  18. package/docs/images/ael-hero.svg +19 -0
  19. package/docs/images/readme-banner.png +0 -0
  20. package/docs/images/social-preview.png +0 -0
  21. package/docs/images/studio-hero-orange.svg +136 -0
  22. package/lib/ablation.js +1 -1
  23. package/lib/cli-args.js +5 -2
  24. package/lib/finding-proofs.js +72 -0
  25. package/lib/gates.js +21 -0
  26. package/lib/harness-audit.js +20 -0
  27. package/lib/interventions.js +56 -0
  28. package/lib/measurements.js +51 -0
  29. package/lib/orchestration.js +1 -0
  30. package/lib/recap-html.js +2 -2
  31. package/lib/recap.js +2 -0
  32. package/lib/rules.js +51 -1
  33. package/lib/run-isolation.js +293 -0
  34. package/lib/run-state.js +2 -1
  35. package/lib/safety-fact.js +64 -0
  36. package/lib/stats.js +9 -0
  37. package/lib/verification-lanes.js +23 -0
  38. package/package.json +1 -1
  39. package/schemas/goal-contract.schema.json +4 -0
@@ -0,0 +1,72 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+ const { spawnSync } = require('child_process');
6
+ const { destructiveFindings } = require('./guard.js');
7
+ const { redactHomePaths, redactSecrets } = require('./safe-context.js');
8
+
9
+ const PROOF_TIMEOUT_MS = 120000;
10
+ const TAIL = 2000;
11
+
12
+ const tail = (text) => redactHomePaths(redactSecrets(String(text || '')).text).slice(-TAIL);
13
+
14
+ function runProof(rootDir, proof, timeoutMs) {
15
+ const command = String(proof.command || '').trim();
16
+ if (!command) return { status: 'refused', reason: 'proof.command is empty' };
17
+ const destructive = destructiveFindings(command);
18
+ if (destructive.length) return { status: 'refused', reason: destructive.map((item) => item.reason).join('; ') };
19
+ const started = Date.now();
20
+ const result = proof.shell === false
21
+ ? spawnSync(command.split(/\s+/)[0], command.split(/\s+/).slice(1), { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs })
22
+ : spawnSync(command, { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs, shell: true });
23
+ const durationMs = Date.now() - started;
24
+ if (result.error?.code === 'ETIMEDOUT' || (result.signal && durationMs >= timeoutMs - 50)) {
25
+ return { status: 'timed_out', durationMs };
26
+ }
27
+ if (result.error) return { status: 'refused', reason: result.error.message, durationMs };
28
+ return {
29
+ status: result.status === 0 ? 'not_reproduced' : 'reproduced',
30
+ exitCode: result.status,
31
+ durationMs,
32
+ stdoutTail: tail(result.stdout),
33
+ stderrTail: tail(result.stderr)
34
+ };
35
+ }
36
+
37
+ function runFindingProofs(rootDir, run, { timeoutMs = PROOF_TIMEOUT_MS } = {}) {
38
+ const runDir = path.join(rootDir, '.ai-engineering-loop', 'runs', run.runId);
39
+ const ledgerPath = path.join(runDir, 'findings.json');
40
+ const counts = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
41
+ let ledger;
42
+ try {
43
+ ledger = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'));
44
+ } catch {
45
+ return counts;
46
+ }
47
+ if (!Array.isArray(ledger.findings)) return counts;
48
+ const records = [];
49
+ let changed = false;
50
+ ledger.findings = ledger.findings.map((finding) => {
51
+ const next = { ...finding };
52
+ if ('proofStatus' in next) {
53
+ delete next.proofStatus;
54
+ changed = true;
55
+ }
56
+ const runnable = next.validity === 'VALID' && ['BLOCKER', 'HIGH'].includes(next.severity) && next.proof && typeof next.proof === 'object';
57
+ if (!runnable) return next;
58
+ const outcome = runProof(rootDir, next.proof, timeoutMs);
59
+ next.proofStatus = outcome.status;
60
+ counts[outcome.status] += 1;
61
+ records.push({ id: next.id, command: tail(next.proof.command), ...outcome });
62
+ changed = true;
63
+ return next;
64
+ });
65
+ if (changed) fs.writeFileSync(ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, { mode: 0o600 });
66
+ if (records.length) {
67
+ fs.writeFileSync(path.join(runDir, 'proofs.json'), `${JSON.stringify({ schemaVersion: 1, runId: run.runId, diffHash: ledger.diffHash, proofs: records }, null, 2)}\n`, { mode: 0o600 });
68
+ }
69
+ return counts;
70
+ }
71
+
72
+ module.exports = { PROOF_TIMEOUT_MS, runFindingProofs };
package/lib/gates.js CHANGED
@@ -686,6 +686,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
686
686
  mergeImpactDrift(rootDir, run);
687
687
  mergeVerificationIntegrity(rootDir, run);
688
688
  mergeCommentJustifications(rootDir, run);
689
+ if (!require('./ablation.js').ablated(rootDir, 'proofs', run.runId)) require('./finding-proofs.js').runFindingProofs(rootDir, run);
689
690
  const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
690
691
  requireRunMetadata(value, run, 'Finding Ledger');
691
692
  if (value.diffHash !== run.artifacts.diff?.sha256) {
@@ -733,6 +734,14 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
733
734
  }
734
735
  const blocking = expected.blockingFindings || [];
735
736
  const cause = value.cause || inferCause(expected.verdict, blocking);
737
+ if (expected.verdict === 'ITERATE' && value.cause) {
738
+ const previous = require('./no-progress.js').readIterations(rootDir, run.runId).filter((entry) => entry.verdict === 'ITERATE').pop();
739
+ const premise = value.premise || {};
740
+ if (previous && !previous.causeInferred && previous.cause === cause && !(String(premise.statement || '').trim() && String(premise.test || '').trim())) {
741
+ throw gateError(`Judge ITERATE repeats cause "${cause}": state and test the shared premise first (PREMISE_REQUIRED)`, [`iteration ${previous.iteration || run.iteration - 1} also failed on ${cause}`],
742
+ { why: 'When two fixes that share one premise fail the same gate, the premise is the likely bug; a third fix on the same premise usually fails too.', fix: 'Add premise: { statement: "<the one sentence every failed fix assumed>", test: "<the observation that would differ if it were false, and what it showed>" } to verdict.json, then retry `gate judge`.' });
743
+ }
744
+ }
736
745
  recordIteration(rootDir, run, {
737
746
  verdict: expected.verdict,
738
747
  diffHash: run.artifacts?.diff?.sha256,
@@ -747,6 +756,18 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
747
756
  throw gateError('Judge PASS refused: rubric criteria are not scored or failed (AC_RUBRIC)', rubric,
748
757
  { why: 'Subjective ACs are judged generously by default; a rubric scored with evidence per criterion makes the judgment reproducible.', fix: 'Have the Devil\'s Advocate add rubricScores [{ac, criterion, pass, evidence}] for every criterion (screenshot path, DOM snapshot, measured value), re-ingest, then retry; or issue ITERATE for failed criteria.' });
749
758
  }
759
+ if (tierOf(run) === 'T2') {
760
+ const safety = require('./safety-fact.js').safetyFactProblem(rootDir, run);
761
+ if (safety) {
762
+ throw gateError('Judge PASS refused: T2 needs a proven safety fact (SAFETY_FACT_UNPROVEN)', [safety],
763
+ { why: 'A high-risk change is safe because of one fact; a writeup that sounds right proves nothing until real code runs.', fix: `Run \`safety record --run ${run.runId} --fact "<the one fact>" --cmd "<script or test that passes only if the fact holds>"\`, then retry \`gate judge\`.` });
764
+ }
765
+ }
766
+ const measured = require('./measurements.js').measurementProblems(rootDir, run, loadFrozenGoal(rootDir, run));
767
+ if (measured.length) {
768
+ throw gateError('Judge PASS refused: a performance number is not explained (MEASUREMENT_UNEXPLAINED)', measured,
769
+ { why: 'A single run or a gap smaller than the noise says nothing; the limiter tells you whether the number measured the change at all.', fix: `Measure at least 3 runs, find what bounds the result, then \`measurement record --run ${run.runId} --ac <AC-n> --runs <n> --spread <pct> --effect <pct> --limiter "<what bounds it>"\` and retry.` });
770
+ }
750
771
  const unproven = unprovenCriteria(rootDir, run);
751
772
  if (unproven.length) {
752
773
  throw gateError('Judge PASS refused: acceptance criteria are not proven (AC_NOT_PROVEN)',
@@ -92,6 +92,7 @@ function auditRules(rootDir, now = new Date()) {
92
92
  else if (backlog.clean) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff) has no legacy violations left. Switch it to scope repo so the whole codebase stays clean.` });
93
93
  else items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff): legacy violations remain (${backlog.outputLines} output line(s), e.g. ${backlog.sample[0] || 'n/a'}). Clean them up over time; new ones are already blocked.` });
94
94
  }
95
+ if (!rule.provenBy) items.push({ id: 'rule-unproven', level: 'info', detail: `${rule.id} was never shown to catch a real mistake. FIX: re-promote it with \`learn promote ${rule.from || '<learning>'} ... --prove <commit that introduced the mistake>\`.` });
95
96
  const age = (now - new Date(rule.createdAt || 0)) / 86400000;
96
97
  if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
97
98
  }
@@ -127,10 +128,29 @@ function auditFeatureMap(rootDir) {
127
128
  return [{ id: 'feature-map', level: 'warn', detail: 'e2e/smoke is configured but .ai-engineering-loop/feature-map.md is missing or empty. FIX: list user-facing features (Feature | Where | Paths | How to reach | Driver) so agents and reviewers share one vocabulary for runtime checks.' }];
128
129
  }
129
130
 
131
+ function auditFeatureMapDrift(rootDir) {
132
+ const entries = require('./feature-map.js').loadFeatureMap(rootDir);
133
+ if (!entries.length) return [];
134
+ let files = [];
135
+ try {
136
+ files = require('child_process').execFileSync('git', ['ls-files'], { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).split('\n').filter(Boolean);
137
+ } catch {
138
+ return [];
139
+ }
140
+ const { matchesGlob } = require('./impact-map.js');
141
+ const items = [];
142
+ for (const entry of entries) {
143
+ const dead = entry.paths.filter((glob) => !files.some((file) => matchesGlob(file, glob)));
144
+ if (dead.length) items.push({ id: 'feature-map-drift', level: 'warn', detail: `Feature "${entry.id}" lists paths that match no tracked file: ${dead.join(', ')}. FIX: update the Paths column in .ai-engineering-loop/feature-map.md, or remove the feature if it is gone.` });
145
+ }
146
+ return items;
147
+ }
148
+
130
149
  function auditHarness(rootDir, { budget = true } = {}) {
131
150
  const items = [
132
151
  ...auditEntryFile(rootDir),
133
152
  ...auditFeatureMap(rootDir),
153
+ ...auditFeatureMapDrift(rootDir),
134
154
  ...auditLearnings(rootDir),
135
155
  ...auditRules(rootDir),
136
156
  ...(budget ? auditBudget(rootDir) : [])
@@ -0,0 +1,56 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+
6
+ const lines = (file) => {
7
+ try {
8
+ return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
9
+ try {
10
+ return [JSON.parse(line)];
11
+ } catch {
12
+ return [];
13
+ }
14
+ });
15
+ } catch {
16
+ return [];
17
+ }
18
+ };
19
+
20
+ const readJson = (file) => {
21
+ try {
22
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
23
+ } catch {
24
+ return null;
25
+ }
26
+ };
27
+
28
+ function runInterventions(runDir) {
29
+ const answers = lines(path.join(runDir, 'interactions.jsonl')).filter((entry) => entry.type === 'ANSWER').length;
30
+ const state = readJson(path.join(runDir, 'state.json'));
31
+ const escalated = state?.state === 'ESCALATED' || (state?.history || []).some((entry) => entry.to === 'ESCALATED') ? 1 : 0;
32
+ const outcomes = lines(path.join(runDir, 'outcomes.jsonl')).length;
33
+ const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
34
+ for (const proof of readJson(path.join(runDir, 'proofs.json'))?.proofs || []) {
35
+ if (Object.hasOwn(proofs, proof.status)) proofs[proof.status] += 1;
36
+ }
37
+ return { answers, escalated, outcomes, total: answers + escalated + outcomes, proofs };
38
+ }
39
+
40
+ function summarizeInterventions(perRun) {
41
+ const sum = (key) => perRun.reduce((n, item) => n + item[key], 0);
42
+ const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
43
+ for (const item of perRun) for (const key of Object.keys(proofs)) proofs[key] += item.proofs[key];
44
+ const ran = proofs.reproduced + proofs.not_reproduced;
45
+ return {
46
+ runs: perRun.length,
47
+ answers: sum('answers'),
48
+ escalations: sum('escalated'),
49
+ outcomes: sum('outcomes'),
50
+ perRun: perRun.length ? Math.round((sum('total') / perRun.length) * 100) / 100 : null,
51
+ proofs,
52
+ reproducedRate: ran ? Math.round((proofs.reproduced / ran) * 1000) / 1000 : null
53
+ };
54
+ }
55
+
56
+ module.exports = { runInterventions, summarizeInterventions };
@@ -0,0 +1,51 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+
6
+ const FILE = 'measurements.json';
7
+ const MIN_RUNS = 3;
8
+
9
+ const fail = (message) => Object.assign(new Error(message), { code: 'INVALID_USAGE' });
10
+ const filePath = (rootDir, runId) => path.join(rootDir, '.ai-engineering-loop', 'runs', runId, FILE);
11
+ const percent = (value, flag) => {
12
+ const number = Number(String(value ?? '').replace(/%$/, ''));
13
+ if (!Number.isFinite(number) || number < 0) throw fail(`${flag} must be a non-negative percentage, such as 4 or 4%`);
14
+ return number;
15
+ };
16
+
17
+ function loadMeasurements(rootDir, runId) {
18
+ try {
19
+ return JSON.parse(fs.readFileSync(filePath(rootDir, runId), 'utf8')).measurements || [];
20
+ } catch {
21
+ return [];
22
+ }
23
+ }
24
+
25
+ function recordMeasurement(rootDir, runId, { ac, runs, spread, effect, limiter, now = new Date() }) {
26
+ if (!/^AC-\d+$/.test(String(ac || ''))) throw fail('--ac must be an AC id such as AC-2');
27
+ const count = Number(runs);
28
+ if (!Number.isInteger(count) || count < 1) throw fail('--runs must be a whole number of measured runs');
29
+ const reason = String(limiter || '').trim();
30
+ if (!reason) throw fail('--limiter is required: the resource or code path that bounds the number, from a profile or counters');
31
+ const entry = { ac, runs: count, spread: percent(spread, '--spread'), effect: percent(effect, '--effect'), limiter: reason, recordedAt: now.toISOString() };
32
+ const measurements = loadMeasurements(rootDir, runId).filter((item) => item.ac !== ac).concat(entry);
33
+ fs.writeFileSync(filePath(rootDir, runId), `${JSON.stringify({ schemaVersion: 1, runId, measurements }, null, 2)}\n`, { mode: 0o600 });
34
+ return entry;
35
+ }
36
+
37
+ function measurementProblems(rootDir, run, goal) {
38
+ const needed = (goal?.acceptanceCriteria || []).filter((ac) => ac.measure === true);
39
+ if (!needed.length) return [];
40
+ const recorded = new Map(loadMeasurements(rootDir, run.runId).map((item) => [item.ac, item]));
41
+ const problems = [];
42
+ for (const ac of needed) {
43
+ const m = recorded.get(ac.id);
44
+ if (!m) problems.push(`${ac.id}: no measurement recorded`);
45
+ else if (m.runs < MIN_RUNS) problems.push(`${ac.id}: ${m.runs} run(s); at least ${MIN_RUNS} are needed to see the noise`);
46
+ else if (m.spread >= m.effect) problems.push(`${ac.id}: run-to-run spread ${m.spread}% is not smaller than the effect ${m.effect}%`);
47
+ }
48
+ return problems;
49
+ }
50
+
51
+ module.exports = { MIN_RUNS, loadMeasurements, measurementProblems, recordMeasurement };
@@ -151,6 +151,7 @@ function validateFindingLedger(ledger) {
151
151
  function isBlockingFinding(finding) {
152
152
  if (!finding || finding.validity !== 'VALID') return false;
153
153
  if (finding.severity !== 'BLOCKER' && finding.severity !== 'HIGH') return false;
154
+ if (finding.proofStatus === 'not_reproduced') return false;
154
155
  const axis = finding.axis || 'spec';
155
156
  if (axis === 'standards' && finding.hardConvention !== true) return false;
156
157
  return true;
package/lib/recap-html.js CHANGED
@@ -195,11 +195,11 @@ function renderRecap(data, { audience = 'me', now = new Date(), lang = null } =
195
195
  const contentLang = lang ? ` lang="${esc(lang)}"` : '';
196
196
  const technical = runs.filter((run) => run.delivered && run.clientTextSource === 'objective');
197
197
  const empty = `<p class="empty">No runs in the last ${esc(data.window.days)} days. Try a longer window: <code>ael recap --days ${esc(data.window.days * 4)}</code></p>`;
198
- const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}</div>
198
+ const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Times you had to step in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>
199
199
  <div class="panel"><h2>Activity per day</h2>${heatmap(data)}</div>
200
200
  <div class="panel"><h2>Where the time went</h2>${stageBars(runs)}</div>
201
201
  <div class="panel"><h2>Why runs had to iterate</h2>${causes(metrics)}</div>` : `<div class="panel">${empty}</div>`;
202
- const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
202
+ const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}${metric('Times a human stepped in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
203
203
  const tab = (view, text) => `<button type="button" aria-controls="${view}" data-view="${view}" aria-pressed="${view === audience}">${text}</button>`;
204
204
  const section = (view, html) => `<section class="view${view === audience ? ' on' : ''}" id="${view}" aria-label="${esc({ me: 'For me', lead: 'For the lead', client: 'For the client' }[view])}">${html}</section>`;
205
205
  const range = `${fmtDate(data.window.from)} – ${fmtDate(new Date(data.window.to.getTime() - 1))}`;
package/lib/recap.js CHANGED
@@ -110,6 +110,7 @@ function loadRun(projectPath, runId) {
110
110
  outOfMap: Array.isArray(receipt.impact?.outOfMap) ? receipt.impact.outOfMap : [],
111
111
  openFindings: Array.isArray(summary.openFindings) ? summary.openFindings.length : 0,
112
112
  stages: stageDurations(history),
113
+ interventions: require('./interventions.js').runInterventions(dir),
113
114
  files
114
115
  };
115
116
  }
@@ -146,6 +147,7 @@ function metricsOf(runs) {
146
147
  firstPassRate: judged.length ? Math.round((judged.filter((run) => run.iterations[0].verdict === 'PASS').length / judged.length) * 100) : null,
147
148
  medianCycleMs: median(runs.filter((run) => run.delivered).map(cycle).filter(Boolean)),
148
149
  t2: runs.filter((run) => run.tier === 'T2').length,
150
+ interventionsPerRun: require('./interventions.js').summarizeInterventions(runs.map((run) => run.interventions)).perRun,
149
151
  causes
150
152
  };
151
153
  }
package/lib/rules.js CHANGED
@@ -36,7 +36,7 @@ function loadLearningEntries(rootDir) {
36
36
 
37
37
  const SCOPES = Object.freeze(['repo', 'diff']);
38
38
 
39
- function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, now = new Date() } = {}) {
39
+ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, prove = null, now = new Date() } = {}) {
40
40
  if (!SCOPES.includes(scope)) throw rulesError(`--scope must be repo or diff (got ${scope}).`, 'INVALID_USAGE');
41
41
  // C-20 built-in kinds: no hand-written check, always judged on added lines.
42
42
  let builtin = null;
@@ -79,10 +79,59 @@ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shel
79
79
  createdBy: String(by).trim(),
80
80
  createdAt: now.toISOString()
81
81
  };
82
+ if (prove) rule.provenBy = proveRule(rootDir, rule, prove, { now });
82
83
  atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
83
84
  return rule;
84
85
  }
85
86
 
87
+ function proveRule(rootDir, rule, commit, { now = new Date() } = {}) {
88
+ const { execFileSync, spawnSync } = require('child_process');
89
+ const os = require('os');
90
+ const git = (...args) => execFileSync('git', args, { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
91
+ let sha;
92
+ try {
93
+ sha = git('rev-parse', '--verify', `${commit}^{commit}`).trim();
94
+ } catch {
95
+ throw rulesError(`--prove ${commit} is not a commit in this repository.`, 'INVALID_USAGE');
96
+ }
97
+ let failed;
98
+ if (rule.scope === 'diff') {
99
+ const { parseDiff } = require('./verification-integrity.js');
100
+ const files = parseDiff(git('show', '--format=', '--no-color', sha, '--', '.', ':(exclude).ai-engineering-loop')).filter((file) => !file.deleted);
101
+ const lines = files.flatMap((file) => file.added.map((added) => ({ file: file.path, line: added.line, text: added.text })));
102
+ if (rule.kind) {
103
+ failed = require('./rule-check.js').checkRule(rule, lines).length > 0;
104
+ } else {
105
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-'));
106
+ const addedPath = path.join(tmp, 'added-lines.txt');
107
+ fs.writeFileSync(addedPath, lines.map((l) => `${l.file}:${l.line}:${l.text}`).join('\n'));
108
+ const env = { ...process.env, AEL_CHANGED_FILES: files.map((file) => file.path).join('\n'), AEL_ADDED_LINES_FILE: addedPath };
109
+ const result = rule.shell !== false
110
+ ? spawnSync(rule.check, { cwd: rootDir, shell: true, env, encoding: 'utf8' })
111
+ : spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: rootDir, env, encoding: 'utf8' });
112
+ fs.rmSync(tmp, { recursive: true, force: true });
113
+ failed = result.status !== 0;
114
+ }
115
+ } else {
116
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-wt-'));
117
+ const tree = path.join(tmp, 'tree');
118
+ git('worktree', 'add', '--detach', '--quiet', tree, sha);
119
+ try {
120
+ const result = rule.shell !== false
121
+ ? spawnSync(rule.check, { cwd: tree, shell: true, encoding: 'utf8' })
122
+ : spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: tree, encoding: 'utf8' });
123
+ failed = result.status !== 0;
124
+ } finally {
125
+ git('worktree', 'remove', '--force', tree);
126
+ fs.rmSync(tmp, { recursive: true, force: true });
127
+ }
128
+ }
129
+ if (!failed) {
130
+ throw rulesError(`The check did not fail on ${sha.slice(0, 12)}, so it would not have caught that mistake. Pick the commit that introduced the mistake, or fix the check.`, 'RULE_NOT_PROVEN');
131
+ }
132
+ return { commit: sha, at: now.toISOString() };
133
+ }
134
+
86
135
  // Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
87
136
  function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
88
137
  const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
@@ -189,6 +238,7 @@ function missingRules(rootDir, bundle) {
189
238
  }
190
239
 
191
240
  module.exports = {
241
+ proveRule,
192
242
  SCOPES,
193
243
  commandForRule,
194
244
  findRule,