ai-engineering-loop 2.10.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,136 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" width="1600" height="900" viewBox="0 0 1600 900" role="img" aria-labelledby="title desc">
2
+ <title id="title">AI Engineering Loop Workflow Studio</title>
3
+ <desc id="desc">A dark Workflow Studio live-run view: a stage rail from goal to deliver with review in progress, a pending question banner, compliance chips, a short timeline, and a Start with AI Agent button.</desc>
4
+ <defs>
5
+ <linearGradient id="bg" x1="0" y1="0" x2="1" y2="1">
6
+ <stop stop-color="#111512"/>
7
+ <stop offset="1" stop-color="#080a09"/>
8
+ </linearGradient>
9
+ <linearGradient id="glow" x1="0" y1="0" x2="1" y2="1">
10
+ <stop stop-color="#fc6404" stop-opacity=".28"/>
11
+ <stop offset="1" stop-color="#fc6404" stop-opacity="0"/>
12
+ </linearGradient>
13
+ <filter id="shadow" x="-30%" y="-30%" width="160%" height="160%">
14
+ <feDropShadow dx="0" dy="18" stdDeviation="24" flood-color="#000" flood-opacity=".45"/>
15
+ </filter>
16
+ <filter id="soft">
17
+ <feGaussianBlur stdDeviation="55"/>
18
+ </filter>
19
+ <pattern id="grid" width="32" height="32" patternUnits="userSpaceOnUse">
20
+ <path d="M32 0H0V32" fill="none" stroke="#fff" stroke-opacity=".035"/>
21
+ </pattern>
22
+ <marker id="arrow" markerWidth="10" markerHeight="10" refX="8" refY="5" orient="auto">
23
+ <path d="M0 0L10 5L0 10Z" fill="#5b625d"/>
24
+ </marker>
25
+ <style>
26
+ .ui{font-family:Inter,ui-sans-serif,system-ui,-apple-system,"Segoe UI",sans-serif}
27
+ .mono{font-family:"SFMono-Regular",Consolas,monospace}
28
+ .eyebrow{font-size:14px;font-weight:700;letter-spacing:2.4px;fill:#7f8881}
29
+ .node-title{font-size:22px;font-weight:700;fill:#f4f6f4}
30
+ .node-copy{font-size:14px;fill:#888f8a}
31
+ .tiny{font-size:12px;fill:#707873}
32
+ .flow{fill:none;stroke:#5b625d;stroke-width:3;marker-end:url(#arrow)}
33
+ </style>
34
+ </defs>
35
+
36
+ <rect width="1600" height="900" fill="url(#bg)"/>
37
+ <circle cx="1240" cy="70" r="330" fill="#fc6404" opacity=".12" filter="url(#soft)"/>
38
+ <circle cx="280" cy="880" r="260" fill="#fc6404" opacity=".08" filter="url(#soft)"/>
39
+
40
+ <g class="ui">
41
+ <text x="110" y="112" font-size="16" font-weight="800" letter-spacing="2" fill="#fcaa72">AI ENGINEERING LOOP</text>
42
+ <text x="110" y="188" font-size="64" font-weight="750" letter-spacing="-2.8" fill="#f7f9f7">From idea to trusted code.</text>
43
+ <text x="112" y="232" font-size="21" fill="#9da49f">Your AI handles the workflow. You keep the clarity and control.</text>
44
+ </g>
45
+
46
+ <g filter="url(#shadow)">
47
+ <rect x="105" y="290" width="1390" height="510" rx="24" fill="#151816" stroke="#303632"/>
48
+ <rect x="105" y="290" width="1390" height="64" rx="24" fill="#1b1f1c"/>
49
+ <path d="M105 330V354H1495V330" fill="#1b1f1c"/>
50
+ <circle cx="140" cy="322" r="6" fill="#4d544f"/>
51
+ <circle cx="160" cy="322" r="6" fill="#4d544f"/>
52
+ <circle cx="180" cy="322" r="6" fill="#fc6404"/>
53
+ <rect x="680" y="307" width="240" height="30" rx="15" fill="#111411"/>
54
+ <circle cx="704" cy="322" r="6" fill="#fc6404"/>
55
+ <text class="ui tiny" x="720" y="326">Workflow Studio</text>
56
+
57
+ <rect x="105" y="354" width="72" height="446" fill="#101311"/>
58
+ <rect x="123" y="381" width="36" height="36" rx="10" fill="#fc6404"/>
59
+ <path d="M129 400h6l3-8 5 16 3-8h6" fill="none" stroke="#fff" stroke-width="2.2" stroke-linecap="round" stroke-linejoin="round"/>
60
+ <circle cx="141" cy="449" r="15" fill="#252a26"/>
61
+ <path d="M134 444h14M134 449h14M134 454h9" stroke="#818982" stroke-width="2" stroke-linecap="round"/>
62
+ <circle cx="141" cy="500" r="15" fill="#252a26"/>
63
+ <circle cx="135" cy="505" r="3" fill="none" stroke="#818982" stroke-width="2"/>
64
+ <circle cx="147" cy="495" r="3" fill="none" stroke="#818982" stroke-width="2"/>
65
+ <path d="M137.5 502.5l7-5" stroke="#818982" stroke-width="2"/>
66
+
67
+ <rect x="177" y="354" width="1318" height="446" fill="#111411"/>
68
+ <rect x="177" y="354" width="1318" height="446" fill="url(#grid)"/>
69
+ <ellipse cx="1220" cy="410" rx="300" ry="210" fill="url(#glow)"/>
70
+
71
+ <text class="ui eyebrow" x="222" y="397">LIVE RUN / PRODUCT LAUNCH</text>
72
+ <text class="ui tiny" x="222" y="420">T1 · ASSISTED · iteration 1 / 3 · budget 38%</text>
73
+ <rect x="1180" y="375" width="262" height="42" rx="9" fill="#1d221e" stroke="#343a35"/>
74
+ <circle cx="1206" cy="396" r="6" fill="#fc6404"/>
75
+ <circle cx="1206" cy="396" r="11" fill="none" stroke="#fc6404" stroke-opacity=".35" stroke-width="2"/>
76
+ <text class="ui tiny" x="1224" y="400">live: watch · heartbeat 3s ago</text>
77
+
78
+ <rect x="222" y="444" width="1220" height="50" rx="12" fill="#1a2418" stroke="#2f5a22"/>
79
+ <rect x="222" y="444" width="6" height="50" rx="3" fill="#fcaa72"/>
80
+ <text class="ui" x="248" y="474" font-size="13" font-weight="800" letter-spacing="1.4" fill="#fcaa72">QUESTION</text>
81
+ <text class="ui" x="340" y="474" font-size="15" fill="#e4e8e5">Should an empty cart show the “Continue shopping” link?</text>
82
+ <rect x="1316" y="456" width="110" height="26" rx="7" fill="#1d221e" stroke="#3b6b2c"/>
83
+ <text class="ui" x="1371" y="474" text-anchor="middle" font-size="12" font-weight="750" fill="#c9f0bd">Answer</text>
84
+
85
+ <line x1="300" y1="560" x2="1365" y2="560" stroke="#343a35" stroke-width="4" stroke-linecap="round"/>
86
+ <line x1="300" y1="560" x2="939" y2="560" stroke="#fc6404" stroke-width="4" stroke-linecap="round"/>
87
+ <circle cx="300" cy="560" r="18" fill="#fc6404"/>
88
+ <path d="M293 560l5 5 10-11" fill="none" stroke="#fff" stroke-width="3.5" stroke-linecap="round" stroke-linejoin="round"/>
89
+ <text class="ui" x="300" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#c7ccc8">GOAL</text>
90
+ <circle cx="513" cy="560" r="18" fill="#fc6404"/>
91
+ <path d="M506 560l5 5 10-11" fill="none" stroke="#fff" stroke-width="3.5" stroke-linecap="round" stroke-linejoin="round"/>
92
+ <text class="ui" x="513" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#c7ccc8">MAKER</text>
93
+ <circle cx="726" cy="560" r="18" fill="#fc6404"/>
94
+ <path d="M719 560l5 5 10-11" fill="none" stroke="#fff" stroke-width="3.5" stroke-linecap="round" stroke-linejoin="round"/>
95
+ <text class="ui" x="726" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#c7ccc8">VERIFY</text>
96
+ <circle cx="939" cy="560" r="28" fill="#fc6404" opacity=".18"/>
97
+ <circle cx="939" cy="560" r="18" fill="#1c211d" stroke="#fc6404" stroke-width="4"/>
98
+ <circle cx="939" cy="560" r="6" fill="#fc6404"/>
99
+ <text class="ui" x="939" y="608" text-anchor="middle" font-size="13" font-weight="800" letter-spacing="1.6" fill="#f4f6f4">REVIEW</text>
100
+ <circle cx="1152" cy="560" r="18" fill="#1c211d" stroke="#3b433d" stroke-width="3"/>
101
+ <text class="ui" x="1152" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#707873">JUDGE</text>
102
+ <circle cx="1365" cy="560" r="18" fill="#1c211d" stroke="#3b433d" stroke-width="3"/>
103
+ <text class="ui" x="1365" y="608" text-anchor="middle" font-size="13" font-weight="700" letter-spacing="1.6" fill="#707873">DELIVER</text>
104
+ <text class="ui tiny" x="939" y="628" text-anchor="middle" fill="#fcaa72">Devil's Advocate running</text>
105
+
106
+ <rect x="222" y="652" width="600" height="62" rx="12" fill="#191d1a" stroke="#2b312c"/>
107
+ <text class="ui eyebrow" x="246" y="688" font-size="12">COMPLIANCE</text>
108
+ <rect x="384" y="667" width="134" height="32" rx="16" fill="#1c2a18" stroke="#2f5a22"/>
109
+ <circle cx="402" cy="683" r="5" fill="#fc6404"/>
110
+ <text class="ui" x="414" y="688" font-size="13" fill="#c9f0bd">Guard 0 blocks</text>
111
+ <rect x="526" y="667" width="120" height="32" rx="16" fill="#1c2a18" stroke="#2f5a22"/>
112
+ <circle cx="543" cy="683" r="5" fill="#fc6404"/>
113
+ <text class="ui" x="555" y="688" font-size="13" fill="#c9f0bd">Scope in map</text>
114
+ <rect x="654" y="667" width="148" height="32" rx="16" fill="#1d221e" stroke="#343a35"/>
115
+ <circle cx="671" cy="683" r="5" fill="#707873"/>
116
+ <text class="ui" x="683" y="688" font-size="13" fill="#9da49f">Receipt pending</text>
117
+
118
+ <rect x="842" y="652" width="600" height="62" rx="12" fill="#191d1a" stroke="#2b312c"/>
119
+ <text class="ui eyebrow" x="866" y="688" font-size="12">TIMELINE</text>
120
+ <text class="mono tiny" x="966" y="677">10:42</text>
121
+ <text class="ui" x="1012" y="677" font-size="13" fill="#c7ccc8">verify · npm test · 511 passed</text>
122
+ <text class="mono tiny" x="966" y="700">10:44</text>
123
+ <text class="ui" x="1012" y="700" font-size="13" fill="#c7ccc8">review · 2 findings, 0 blocking</text>
124
+
125
+ <rect x="224" y="734" width="1218" height="44" rx="12" fill="#191d1a" stroke="#2b312c"/>
126
+ <text class="mono tiny" x="248" y="761">Task</text>
127
+ <text class="ui" x="300" y="761" font-size="14" fill="#c7ccc8">Ship a reliable feature without babysitting every command</text>
128
+ <rect x="1250" y="743" width="174" height="26" rx="7" fill="#fc6404"/>
129
+ <text class="ui" x="1337" y="761" text-anchor="middle" font-size="12" font-weight="750" fill="#fff">Start with AI Agent</text>
130
+ </g>
131
+
132
+ <g class="ui">
133
+ <text x="112" y="852" font-size="15" fill="#6f7771">Live runs · compliance you can see · evidence you can trust</text>
134
+ <text x="1488" y="852" text-anchor="end" font-size="15" font-weight="700" fill="#fc6404">OPEN SOURCE</text>
135
+ </g>
136
+ </svg>
package/lib/ablation.js CHANGED
@@ -10,7 +10,7 @@ const fs = require('fs');
10
10
  const path = require('path');
11
11
  const { runsDir } = require('./run-state.js');
12
12
 
13
- const COMPONENTS = Object.freeze(['integrity', 'comments', 'l3', 'l1', 'smoke', 'rules']);
13
+ const COMPONENTS = Object.freeze(['integrity', 'comments', 'l3', 'l1', 'smoke', 'rules', 'proofs']);
14
14
  const MARKER = path.join('.ai-engineering-loop', 'bench-ablation.json');
15
15
 
16
16
  function requested(env = process.env) {
package/lib/cli-args.js CHANGED
@@ -24,6 +24,8 @@ const WRITER_COMMANDS = Object.freeze({
24
24
  valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg', '--without'],
25
25
  positionals: []
26
26
  },
27
+ safety: { flags: ['--json', '--live'], valued: ['--run', '--fact', '--cmd', '--level'], positionals: ['record'] },
28
+ measurement: { flags: ['--json'], valued: ['--run', '--ac', '--runs', '--spread', '--effect', '--limiter'], positionals: ['record'] },
27
29
  recap: { flags: ['--json', '--open'], valued: ['--days', '--for', '--projects', '--out', '--now', '--lang'], positionals: [] },
28
30
  migrate: { flags: ['--json', '--dry-run'], valued: ['--from'], positionals: [] },
29
31
  debt: { flags: ['--json'], valued: ['--by', '--note', '--file'], positionals: null },
@@ -31,7 +33,7 @@ const WRITER_COMMANDS = Object.freeze({
31
33
  verify: { flags: ['--json', '--serial', '--shell', '--related', '--final', '--fresh', '--baseline'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
32
34
  quick: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms', '--mode'], positionals: null },
33
35
  review: { flags: ['--json'], valued: ['--run', '--child-id', '--file'], positionals: ['ingest'] },
34
- learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by', '--scope', '--forbid-import', '--forbid-api', '--except'], positionals: null },
36
+ learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by', '--scope', '--forbid-import', '--forbid-api', '--except', '--prove'], positionals: null },
35
37
  guard: { flags: ['--json'], valued: ['--run', '--tool', '--input', '--path', '--command', '--hook'], positionals: ['check'] },
36
38
  hooks: { flags: ['--json'], valued: ['--host'], positionals: ['install', 'uninstall', 'status'] },
37
39
  impact: { flags: ['--json', '--force'], valued: ['--run', '--by', '--reason', '--allow', '--remove', '--limit'], positionals: ['map', 'show', 'confirm', 'skip', 'drift'] },
@@ -0,0 +1,72 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+ const { spawnSync } = require('child_process');
6
+ const { destructiveFindings } = require('./guard.js');
7
+ const { redactHomePaths, redactSecrets } = require('./safe-context.js');
8
+
9
+ const PROOF_TIMEOUT_MS = 120000;
10
+ const TAIL = 2000;
11
+
12
+ const tail = (text) => redactHomePaths(redactSecrets(String(text || '')).text).slice(-TAIL);
13
+
14
+ function runProof(rootDir, proof, timeoutMs) {
15
+ const command = String(proof.command || '').trim();
16
+ if (!command) return { status: 'refused', reason: 'proof.command is empty' };
17
+ const destructive = destructiveFindings(command);
18
+ if (destructive.length) return { status: 'refused', reason: destructive.map((item) => item.reason).join('; ') };
19
+ const started = Date.now();
20
+ const result = proof.shell === false
21
+ ? spawnSync(command.split(/\s+/)[0], command.split(/\s+/).slice(1), { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs })
22
+ : spawnSync(command, { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs, shell: true });
23
+ const durationMs = Date.now() - started;
24
+ if (result.error?.code === 'ETIMEDOUT' || (result.signal && durationMs >= timeoutMs - 50)) {
25
+ return { status: 'timed_out', durationMs };
26
+ }
27
+ if (result.error) return { status: 'refused', reason: result.error.message, durationMs };
28
+ return {
29
+ status: result.status === 0 ? 'not_reproduced' : 'reproduced',
30
+ exitCode: result.status,
31
+ durationMs,
32
+ stdoutTail: tail(result.stdout),
33
+ stderrTail: tail(result.stderr)
34
+ };
35
+ }
36
+
37
+ function runFindingProofs(rootDir, run, { timeoutMs = PROOF_TIMEOUT_MS } = {}) {
38
+ const runDir = path.join(rootDir, '.ai-engineering-loop', 'runs', run.runId);
39
+ const ledgerPath = path.join(runDir, 'findings.json');
40
+ const counts = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
41
+ let ledger;
42
+ try {
43
+ ledger = JSON.parse(fs.readFileSync(ledgerPath, 'utf8'));
44
+ } catch {
45
+ return counts;
46
+ }
47
+ if (!Array.isArray(ledger.findings)) return counts;
48
+ const records = [];
49
+ let changed = false;
50
+ ledger.findings = ledger.findings.map((finding) => {
51
+ const next = { ...finding };
52
+ if ('proofStatus' in next) {
53
+ delete next.proofStatus;
54
+ changed = true;
55
+ }
56
+ const runnable = next.validity === 'VALID' && ['BLOCKER', 'HIGH'].includes(next.severity) && next.proof && typeof next.proof === 'object';
57
+ if (!runnable) return next;
58
+ const outcome = runProof(rootDir, next.proof, timeoutMs);
59
+ next.proofStatus = outcome.status;
60
+ counts[outcome.status] += 1;
61
+ records.push({ id: next.id, command: tail(next.proof.command), ...outcome });
62
+ changed = true;
63
+ return next;
64
+ });
65
+ if (changed) fs.writeFileSync(ledgerPath, `${JSON.stringify(ledger, null, 2)}\n`, { mode: 0o600 });
66
+ if (records.length) {
67
+ fs.writeFileSync(path.join(runDir, 'proofs.json'), `${JSON.stringify({ schemaVersion: 1, runId: run.runId, diffHash: ledger.diffHash, proofs: records }, null, 2)}\n`, { mode: 0o600 });
68
+ }
69
+ return counts;
70
+ }
71
+
72
+ module.exports = { PROOF_TIMEOUT_MS, runFindingProofs };
package/lib/gates.js CHANGED
@@ -545,6 +545,19 @@ function mergeCommentJustifications(rootDir, run) {
545
545
  return merged.added;
546
546
  }
547
547
 
548
+ const ITERATION_EVIDENCE = ['verification.json', 'verification-summary.json', 'final-verification.json'];
549
+
550
+ function archiveIterationEvidence(rootDir, run) {
551
+ const runDir = path.dirname(artifactPath(rootDir, run.runId, 'verification.json'));
552
+ const target = path.join(runDir, 'iterations', String(run.iteration || 1));
553
+ for (const name of ITERATION_EVIDENCE) {
554
+ const from = path.join(runDir, name);
555
+ if (!fs.existsSync(from)) continue;
556
+ fs.mkdirSync(target, { recursive: true, mode: 0o700 });
557
+ fs.renameSync(from, path.join(target, name));
558
+ }
559
+ }
560
+
548
561
  function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
549
562
  if (!Object.hasOwn(GATE_FILES, gate)) {
550
563
  throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`, [], { why: 'Only the listed stages exist.', fix: `Use one of: ${Object.keys(GATE_FILES).join(', ')}.` });
@@ -673,6 +686,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
673
686
  mergeImpactDrift(rootDir, run);
674
687
  mergeVerificationIntegrity(rootDir, run);
675
688
  mergeCommentJustifications(rootDir, run);
689
+ if (!require('./ablation.js').ablated(rootDir, 'proofs', run.runId)) require('./finding-proofs.js').runFindingProofs(rootDir, run);
676
690
  const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
677
691
  requireRunMetadata(value, run, 'Finding Ledger');
678
692
  if (value.diffHash !== run.artifacts.diff?.sha256) {
@@ -720,6 +734,14 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
720
734
  }
721
735
  const blocking = expected.blockingFindings || [];
722
736
  const cause = value.cause || inferCause(expected.verdict, blocking);
737
+ if (expected.verdict === 'ITERATE' && value.cause) {
738
+ const previous = require('./no-progress.js').readIterations(rootDir, run.runId).filter((entry) => entry.verdict === 'ITERATE').pop();
739
+ const premise = value.premise || {};
740
+ if (previous && !previous.causeInferred && previous.cause === cause && !(String(premise.statement || '').trim() && String(premise.test || '').trim())) {
741
+ throw gateError(`Judge ITERATE repeats cause "${cause}": state and test the shared premise first (PREMISE_REQUIRED)`, [`iteration ${previous.iteration || run.iteration - 1} also failed on ${cause}`],
742
+ { why: 'When two fixes that share one premise fail the same gate, the premise is the likely bug; a third fix on the same premise usually fails too.', fix: 'Add premise: { statement: "<the one sentence every failed fix assumed>", test: "<the observation that would differ if it were false, and what it showed>" } to verdict.json, then retry `gate judge`.' });
743
+ }
744
+ }
723
745
  recordIteration(rootDir, run, {
724
746
  verdict: expected.verdict,
725
747
  diffHash: run.artifacts?.diff?.sha256,
@@ -734,6 +756,18 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
734
756
  throw gateError('Judge PASS refused: rubric criteria are not scored or failed (AC_RUBRIC)', rubric,
735
757
  { why: 'Subjective ACs are judged generously by default; a rubric scored with evidence per criterion makes the judgment reproducible.', fix: 'Have the Devil\'s Advocate add rubricScores [{ac, criterion, pass, evidence}] for every criterion (screenshot path, DOM snapshot, measured value), re-ingest, then retry; or issue ITERATE for failed criteria.' });
736
758
  }
759
+ if (tierOf(run) === 'T2') {
760
+ const safety = require('./safety-fact.js').safetyFactProblem(rootDir, run);
761
+ if (safety) {
762
+ throw gateError('Judge PASS refused: T2 needs a proven safety fact (SAFETY_FACT_UNPROVEN)', [safety],
763
+ { why: 'A high-risk change is safe because of one fact; a writeup that sounds right proves nothing until real code runs.', fix: `Run \`safety record --run ${run.runId} --fact "<the one fact>" --cmd "<script or test that passes only if the fact holds>"\`, then retry \`gate judge\`.` });
764
+ }
765
+ }
766
+ const measured = require('./measurements.js').measurementProblems(rootDir, run, loadFrozenGoal(rootDir, run));
767
+ if (measured.length) {
768
+ throw gateError('Judge PASS refused: a performance number is not explained (MEASUREMENT_UNEXPLAINED)', measured,
769
+ { why: 'A single run or a gap smaller than the noise says nothing; the limiter tells you whether the number measured the change at all.', fix: `Measure at least 3 runs, find what bounds the result, then \`measurement record --run ${run.runId} --ac <AC-n> --runs <n> --spread <pct> --effect <pct> --limiter "<what bounds it>"\` and retry.` });
770
+ }
737
771
  const unproven = unprovenCriteria(rootDir, run);
738
772
  if (unproven.length) {
739
773
  throw gateError('Judge PASS refused: acceptance criteria are not proven (AC_NOT_PROVEN)',
@@ -749,9 +783,11 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
749
783
  reason: expected.reason
750
784
  });
751
785
  }
786
+ archiveIterationEvidence(rootDir, run);
752
787
  return transitionRun(rootDir, run.runId, RUN_STATES.GOAL_FROZEN, {
753
788
  gate,
754
789
  artifacts,
790
+ dropArtifacts: ['verification'],
755
791
  reason: expected.reason,
756
792
  incrementIteration: true
757
793
  });
@@ -92,6 +92,7 @@ function auditRules(rootDir, now = new Date()) {
92
92
  else if (backlog.clean) items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff) has no legacy violations left. Switch it to scope repo so the whole codebase stays clean.` });
93
93
  else items.push({ id: 'rule-backlog', level: 'info', detail: `${rule.id} (scope diff): legacy violations remain (${backlog.outputLines} output line(s), e.g. ${backlog.sample[0] || 'n/a'}). Clean them up over time; new ones are already blocked.` });
94
94
  }
95
+ if (!rule.provenBy) items.push({ id: 'rule-unproven', level: 'info', detail: `${rule.id} was never shown to catch a real mistake. FIX: re-promote it with \`learn promote ${rule.from || '<learning>'} ... --prove <commit that introduced the mistake>\`.` });
95
96
  const age = (now - new Date(rule.createdAt || 0)) / 86400000;
96
97
  if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
97
98
  }
@@ -127,10 +128,29 @@ function auditFeatureMap(rootDir) {
127
128
  return [{ id: 'feature-map', level: 'warn', detail: 'e2e/smoke is configured but .ai-engineering-loop/feature-map.md is missing or empty. FIX: list user-facing features (Feature | Where | Paths | How to reach | Driver) so agents and reviewers share one vocabulary for runtime checks.' }];
128
129
  }
129
130
 
131
+ function auditFeatureMapDrift(rootDir) {
132
+ const entries = require('./feature-map.js').loadFeatureMap(rootDir);
133
+ if (!entries.length) return [];
134
+ let files = [];
135
+ try {
136
+ files = require('child_process').execFileSync('git', ['ls-files'], { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 }).split('\n').filter(Boolean);
137
+ } catch {
138
+ return [];
139
+ }
140
+ const { matchesGlob } = require('./impact-map.js');
141
+ const items = [];
142
+ for (const entry of entries) {
143
+ const dead = entry.paths.filter((glob) => !files.some((file) => matchesGlob(file, glob)));
144
+ if (dead.length) items.push({ id: 'feature-map-drift', level: 'warn', detail: `Feature "${entry.id}" lists paths that match no tracked file: ${dead.join(', ')}. FIX: update the Paths column in .ai-engineering-loop/feature-map.md, or remove the feature if it is gone.` });
145
+ }
146
+ return items;
147
+ }
148
+
130
149
  function auditHarness(rootDir, { budget = true } = {}) {
131
150
  const items = [
132
151
  ...auditEntryFile(rootDir),
133
152
  ...auditFeatureMap(rootDir),
153
+ ...auditFeatureMapDrift(rootDir),
134
154
  ...auditLearnings(rootDir),
135
155
  ...auditRules(rootDir),
136
156
  ...(budget ? auditBudget(rootDir) : [])
@@ -0,0 +1,56 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+
6
+ const lines = (file) => {
7
+ try {
8
+ return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).flatMap((line) => {
9
+ try {
10
+ return [JSON.parse(line)];
11
+ } catch {
12
+ return [];
13
+ }
14
+ });
15
+ } catch {
16
+ return [];
17
+ }
18
+ };
19
+
20
+ const readJson = (file) => {
21
+ try {
22
+ return JSON.parse(fs.readFileSync(file, 'utf8'));
23
+ } catch {
24
+ return null;
25
+ }
26
+ };
27
+
28
+ function runInterventions(runDir) {
29
+ const answers = lines(path.join(runDir, 'interactions.jsonl')).filter((entry) => entry.type === 'ANSWER').length;
30
+ const state = readJson(path.join(runDir, 'state.json'));
31
+ const escalated = state?.state === 'ESCALATED' || (state?.history || []).some((entry) => entry.to === 'ESCALATED') ? 1 : 0;
32
+ const outcomes = lines(path.join(runDir, 'outcomes.jsonl')).length;
33
+ const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
34
+ for (const proof of readJson(path.join(runDir, 'proofs.json'))?.proofs || []) {
35
+ if (Object.hasOwn(proofs, proof.status)) proofs[proof.status] += 1;
36
+ }
37
+ return { answers, escalated, outcomes, total: answers + escalated + outcomes, proofs };
38
+ }
39
+
40
+ function summarizeInterventions(perRun) {
41
+ const sum = (key) => perRun.reduce((n, item) => n + item[key], 0);
42
+ const proofs = { reproduced: 0, not_reproduced: 0, refused: 0, timed_out: 0 };
43
+ for (const item of perRun) for (const key of Object.keys(proofs)) proofs[key] += item.proofs[key];
44
+ const ran = proofs.reproduced + proofs.not_reproduced;
45
+ return {
46
+ runs: perRun.length,
47
+ answers: sum('answers'),
48
+ escalations: sum('escalated'),
49
+ outcomes: sum('outcomes'),
50
+ perRun: perRun.length ? Math.round((sum('total') / perRun.length) * 100) / 100 : null,
51
+ proofs,
52
+ reproducedRate: ran ? Math.round((proofs.reproduced / ran) * 1000) / 1000 : null
53
+ };
54
+ }
55
+
56
+ module.exports = { runInterventions, summarizeInterventions };
@@ -0,0 +1,51 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+
6
+ const FILE = 'measurements.json';
7
+ const MIN_RUNS = 3;
8
+
9
+ const fail = (message) => Object.assign(new Error(message), { code: 'INVALID_USAGE' });
10
+ const filePath = (rootDir, runId) => path.join(rootDir, '.ai-engineering-loop', 'runs', runId, FILE);
11
+ const percent = (value, flag) => {
12
+ const number = Number(String(value ?? '').replace(/%$/, ''));
13
+ if (!Number.isFinite(number) || number < 0) throw fail(`${flag} must be a non-negative percentage, such as 4 or 4%`);
14
+ return number;
15
+ };
16
+
17
+ function loadMeasurements(rootDir, runId) {
18
+ try {
19
+ return JSON.parse(fs.readFileSync(filePath(rootDir, runId), 'utf8')).measurements || [];
20
+ } catch {
21
+ return [];
22
+ }
23
+ }
24
+
25
+ function recordMeasurement(rootDir, runId, { ac, runs, spread, effect, limiter, now = new Date() }) {
26
+ if (!/^AC-\d+$/.test(String(ac || ''))) throw fail('--ac must be an AC id such as AC-2');
27
+ const count = Number(runs);
28
+ if (!Number.isInteger(count) || count < 1) throw fail('--runs must be a whole number of measured runs');
29
+ const reason = String(limiter || '').trim();
30
+ if (!reason) throw fail('--limiter is required: the resource or code path that bounds the number, from a profile or counters');
31
+ const entry = { ac, runs: count, spread: percent(spread, '--spread'), effect: percent(effect, '--effect'), limiter: reason, recordedAt: now.toISOString() };
32
+ const measurements = loadMeasurements(rootDir, runId).filter((item) => item.ac !== ac).concat(entry);
33
+ fs.writeFileSync(filePath(rootDir, runId), `${JSON.stringify({ schemaVersion: 1, runId, measurements }, null, 2)}\n`, { mode: 0o600 });
34
+ return entry;
35
+ }
36
+
37
+ function measurementProblems(rootDir, run, goal) {
38
+ const needed = (goal?.acceptanceCriteria || []).filter((ac) => ac.measure === true);
39
+ if (!needed.length) return [];
40
+ const recorded = new Map(loadMeasurements(rootDir, run.runId).map((item) => [item.ac, item]));
41
+ const problems = [];
42
+ for (const ac of needed) {
43
+ const m = recorded.get(ac.id);
44
+ if (!m) problems.push(`${ac.id}: no measurement recorded`);
45
+ else if (m.runs < MIN_RUNS) problems.push(`${ac.id}: ${m.runs} run(s); at least ${MIN_RUNS} are needed to see the noise`);
46
+ else if (m.spread >= m.effect) problems.push(`${ac.id}: run-to-run spread ${m.spread}% is not smaller than the effect ${m.effect}%`);
47
+ }
48
+ return problems;
49
+ }
50
+
51
+ module.exports = { MIN_RUNS, loadMeasurements, measurementProblems, recordMeasurement };
@@ -151,6 +151,7 @@ function validateFindingLedger(ledger) {
151
151
  function isBlockingFinding(finding) {
152
152
  if (!finding || finding.validity !== 'VALID') return false;
153
153
  if (finding.severity !== 'BLOCKER' && finding.severity !== 'HIGH') return false;
154
+ if (finding.proofStatus === 'not_reproduced') return false;
154
155
  const axis = finding.axis || 'spec';
155
156
  if (axis === 'standards' && finding.hardConvention !== true) return false;
156
157
  return true;
package/lib/recap-html.js CHANGED
@@ -195,11 +195,11 @@ function renderRecap(data, { audience = 'me', now = new Date(), lang = null } =
195
195
  const contentLang = lang ? ` lang="${esc(lang)}"` : '';
196
196
  const technical = runs.filter((run) => run.delivered && run.clientTextSource === 'objective');
197
197
  const empty = `<p class="empty">No runs in the last ${esc(data.window.days)} days. Try a longer window: <code>ael recap --days ${esc(data.window.days * 4)}</code></p>`;
198
- const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}</div>
198
+ const me = runs.length ? `<div class="metrics">${metric('Runs', metrics.runs, prev?.runs)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Times you had to step in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>
199
199
  <div class="panel"><h2>Activity per day</h2>${heatmap(data)}</div>
200
200
  <div class="panel"><h2>Where the time went</h2>${stageBars(runs)}</div>
201
201
  <div class="panel"><h2>Why runs had to iterate</h2>${causes(metrics)}</div>` : `<div class="panel">${empty}</div>`;
202
- const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
202
+ const lead = runs.length ? `<div class="metrics">${metric('Delivered', metrics.delivered, prev?.delivered)}${metric('Passed on the first try', metrics.firstPassRate, prev?.firstPassRate, { kind: 'pct' })}${metric('Median cycle time', metrics.medianCycleMs, prev?.medianCycleMs, { kind: 'dur', lowerIsBetter: true })}${metric('High-risk (T2) runs', metrics.t2, prev?.t2, { lowerIsBetter: true })}${metric('Times a human stepped in, per run', metrics.interventionsPerRun, prev?.interventionsPerRun, { lowerIsBetter: true })}</div>${technical.length ? `<div class="panel"><h2>Before you share the client tab</h2><p>${technical.length} delivered run${technical.length === 1 ? ' has' : 's have'} no <code>plainSummary</code>, so the client tab shows the technical objective: ${technical.map((run) => `<b>${esc(run.title)}</b>`).join(', ')}. Add a one-sentence <code>plainSummary</code> to the Goal and rerun the recap.</p></div>` : ''}<h2>Work and its evidence</h2><div${contentLang}>${runs.slice().reverse().map(featureCard).join('')}</div>` : `<div class="panel">${empty}</div>`;
203
203
  const tab = (view, text) => `<button type="button" aria-controls="${view}" data-view="${view}" aria-pressed="${view === audience}">${text}</button>`;
204
204
  const section = (view, html) => `<section class="view${view === audience ? ' on' : ''}" id="${view}" aria-label="${esc({ me: 'For me', lead: 'For the lead', client: 'For the client' }[view])}">${html}</section>`;
205
205
  const range = `${fmtDate(data.window.from)} – ${fmtDate(new Date(data.window.to.getTime() - 1))}`;
package/lib/recap.js CHANGED
@@ -110,6 +110,7 @@ function loadRun(projectPath, runId) {
110
110
  outOfMap: Array.isArray(receipt.impact?.outOfMap) ? receipt.impact.outOfMap : [],
111
111
  openFindings: Array.isArray(summary.openFindings) ? summary.openFindings.length : 0,
112
112
  stages: stageDurations(history),
113
+ interventions: require('./interventions.js').runInterventions(dir),
113
114
  files
114
115
  };
115
116
  }
@@ -146,6 +147,7 @@ function metricsOf(runs) {
146
147
  firstPassRate: judged.length ? Math.round((judged.filter((run) => run.iterations[0].verdict === 'PASS').length / judged.length) * 100) : null,
147
148
  medianCycleMs: median(runs.filter((run) => run.delivered).map(cycle).filter(Boolean)),
148
149
  t2: runs.filter((run) => run.tier === 'T2').length,
150
+ interventionsPerRun: require('./interventions.js').summarizeInterventions(runs.map((run) => run.interventions)).perRun,
149
151
  causes
150
152
  };
151
153
  }
package/lib/rules.js CHANGED
@@ -36,7 +36,7 @@ function loadLearningEntries(rootDir) {
36
36
 
37
37
  const SCOPES = Object.freeze(['repo', 'diff']);
38
38
 
39
- function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, now = new Date() } = {}) {
39
+ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shell = true, scope = 'repo', forbidImport = null, forbidApi = null, except = null, prove = null, now = new Date() } = {}) {
40
40
  if (!SCOPES.includes(scope)) throw rulesError(`--scope must be repo or diff (got ${scope}).`, 'INVALID_USAGE');
41
41
  // C-20 built-in kinds: no hand-written check, always judged on added lines.
42
42
  let builtin = null;
@@ -79,10 +79,59 @@ function promoteLearning(rootDir, learningId, { check, fix, why = null, by, shel
79
79
  createdBy: String(by).trim(),
80
80
  createdAt: now.toISOString()
81
81
  };
82
+ if (prove) rule.provenBy = proveRule(rootDir, rule, prove, { now });
82
83
  atomicWriteJson(path.join(rootDir, RULES_FILE), { schemaVersion: 1, rules: [...document.rules, rule] });
83
84
  return rule;
84
85
  }
85
86
 
87
+ function proveRule(rootDir, rule, commit, { now = new Date() } = {}) {
88
+ const { execFileSync, spawnSync } = require('child_process');
89
+ const os = require('os');
90
+ const git = (...args) => execFileSync('git', args, { cwd: rootDir, encoding: 'utf8', maxBuffer: 64 * 1024 * 1024 });
91
+ let sha;
92
+ try {
93
+ sha = git('rev-parse', '--verify', `${commit}^{commit}`).trim();
94
+ } catch {
95
+ throw rulesError(`--prove ${commit} is not a commit in this repository.`, 'INVALID_USAGE');
96
+ }
97
+ let failed;
98
+ if (rule.scope === 'diff') {
99
+ const { parseDiff } = require('./verification-integrity.js');
100
+ const files = parseDiff(git('show', '--format=', '--no-color', sha, '--', '.', ':(exclude).ai-engineering-loop')).filter((file) => !file.deleted);
101
+ const lines = files.flatMap((file) => file.added.map((added) => ({ file: file.path, line: added.line, text: added.text })));
102
+ if (rule.kind) {
103
+ failed = require('./rule-check.js').checkRule(rule, lines).length > 0;
104
+ } else {
105
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-'));
106
+ const addedPath = path.join(tmp, 'added-lines.txt');
107
+ fs.writeFileSync(addedPath, lines.map((l) => `${l.file}:${l.line}:${l.text}`).join('\n'));
108
+ const env = { ...process.env, AEL_CHANGED_FILES: files.map((file) => file.path).join('\n'), AEL_ADDED_LINES_FILE: addedPath };
109
+ const result = rule.shell !== false
110
+ ? spawnSync(rule.check, { cwd: rootDir, shell: true, env, encoding: 'utf8' })
111
+ : spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: rootDir, env, encoding: 'utf8' });
112
+ fs.rmSync(tmp, { recursive: true, force: true });
113
+ failed = result.status !== 0;
114
+ }
115
+ } else {
116
+ const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ael-rule-proof-wt-'));
117
+ const tree = path.join(tmp, 'tree');
118
+ git('worktree', 'add', '--detach', '--quiet', tree, sha);
119
+ try {
120
+ const result = rule.shell !== false
121
+ ? spawnSync(rule.check, { cwd: tree, shell: true, encoding: 'utf8' })
122
+ : spawnSync(rule.check.split(/\s+/)[0], rule.check.split(/\s+/).slice(1), { cwd: tree, encoding: 'utf8' });
123
+ failed = result.status !== 0;
124
+ } finally {
125
+ git('worktree', 'remove', '--force', tree);
126
+ fs.rmSync(tmp, { recursive: true, force: true });
127
+ }
128
+ }
129
+ if (!failed) {
130
+ throw rulesError(`The check did not fail on ${sha.slice(0, 12)}, so it would not have caught that mistake. Pick the commit that introduced the mistake, or fix the check.`, 'RULE_NOT_PROVEN');
131
+ }
132
+ return { commit: sha, at: now.toISOString() };
133
+ }
134
+
86
135
  // Learnings seen in >= PROMOTE_AFTER_RUNS Runs and not promoted yet.
87
136
  function promotionCandidates(rootDir, entries = loadLearningEntries(rootDir)) {
88
137
  const promoted = new Set(loadRules(rootDir).rules.map((rule) => rule.from));
@@ -189,6 +238,7 @@ function missingRules(rootDir, bundle) {
189
238
  }
190
239
 
191
240
  module.exports = {
241
+ proveRule,
192
242
  SCOPES,
193
243
  commandForRule,
194
244
  findRule,
package/lib/run-state.js CHANGED
@@ -396,6 +396,7 @@ function transitionRunLocked(
396
396
  incrementIteration = false,
397
397
  statePatch = {},
398
398
  historyMetadata = {},
399
+ dropArtifacts = [],
399
400
  now = new Date()
400
401
  } = {}
401
402
  ) {
@@ -419,7 +420,7 @@ function transitionRunLocked(
419
420
  state: to,
420
421
  iteration: incrementIteration ? run.iteration + 1 : run.iteration,
421
422
  updatedAt: timestamp,
422
- artifacts: { ...run.artifacts, ...artifacts },
423
+ artifacts: Object.fromEntries(Object.entries({ ...run.artifacts, ...artifacts }).filter(([key]) => !dropArtifacts.includes(key))),
423
424
  history: [
424
425
  ...run.history,
425
426
  {