ai-engineering-loop 2.6.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/.agents/skills/ael/SKILL.md +1 -1
  2. package/.agents/skills/ael/references/loop-commands.md +13 -1
  3. package/.agents/skills/ael/references/loop-deliver.md +1 -1
  4. package/.agents/skills/ael/references/loop-goal.md +2 -0
  5. package/.agents/skills/ael/references/loop-maker.md +1 -6
  6. package/.agents/skills/ael/references/loop-report.md +8 -0
  7. package/.agents/skills/ael/references/loop-verify.md +11 -9
  8. package/.claude/agents/devil-advocate.md +7 -2
  9. package/.claude/agents/judge.md +3 -1
  10. package/.grok/agents/devil-advocate.md +7 -2
  11. package/.grok/agents/judge.md +3 -1
  12. package/CHANGELOG.md +31 -0
  13. package/agents/devil-advocate.md +8 -0
  14. package/agents/judge.md +1 -0
  15. package/agents/shared/devil-advocate.body.md +7 -2
  16. package/agents/shared/judge.body.md +3 -1
  17. package/bin/ai-engineering-loop.js +326 -21
  18. package/core/bench.md +15 -0
  19. package/core/formal-statechart.md +49 -0
  20. package/core/goal-contract.md +53 -0
  21. package/core/verification-loop.md +106 -1
  22. package/lib/ablation.js +52 -0
  23. package/lib/ac-status.js +95 -0
  24. package/lib/baseline.js +138 -0
  25. package/lib/bench.js +24 -1
  26. package/lib/budget.js +9 -1
  27. package/lib/calibration.js +129 -0
  28. package/lib/claimed-vs-reality.js +106 -12
  29. package/lib/clean-state.js +85 -0
  30. package/lib/cli-args.js +5 -3
  31. package/lib/comment-findings.js +111 -0
  32. package/lib/debt.js +110 -0
  33. package/lib/feature-map.js +57 -0
  34. package/lib/features.js +95 -0
  35. package/lib/flow.js +86 -0
  36. package/lib/gates.js +257 -41
  37. package/lib/harness-audit.js +141 -0
  38. package/lib/learn.js +24 -1
  39. package/lib/no-progress.js +6 -1
  40. package/lib/receipt.js +33 -2
  41. package/lib/reverts.js +77 -0
  42. package/lib/review-ingest.js +54 -2
  43. package/lib/rule-check.js +90 -0
  44. package/lib/rules.js +205 -0
  45. package/lib/stats.js +110 -0
  46. package/lib/sync-hosts.js +1 -0
  47. package/lib/verification-integrity.js +164 -0
  48. package/lib/verification-lanes.js +96 -0
  49. package/lib/verification-recorder.js +20 -8
  50. package/package.json +1 -1
  51. package/schemas/finding-ledger.schema.json +14 -0
  52. package/schemas/goal-contract.schema.json +50 -0
  53. package/schemas/verdict.schema.json +2 -1
@@ -113,3 +113,56 @@ Every single item listed under `Acceptance Criteria` must map to at least one co
113
113
  - **The Wrong Artifact**: Tests pass on a `/tmp` build while the sample the user opens omits the feature.
114
114
  - **The Happy-Path Contract**: Only the sunny case is an AC. Coverage % is not a substitute for a failure table.
115
115
  - **The Sloppy Ship**: Implementing from a vague prompt without a drafted failure table in the Goal Contract file.
116
+
117
+ ## Per-AC verify command (C-07)
118
+
119
+ An AC may name the command that proves it:
120
+
121
+ ```json
122
+ { "id": "AC-2", "statement": "page 0 is rejected", "evidenceRequired": "unit test", "failureCases": ["page 0 returns items"],
123
+ "verify": { "command": "npm test -- list.test.js", "layer": "L2" } }
124
+ ```
125
+
126
+ The harness owns the status: `proven` when that exact command passed on the gated diff (`verification.json` or `final-verification.json`), `failed` when it ran and failed, `pending` when it never ran. `verify` prints each status, and `gate judge` refuses PASS while any AC with a verify command is `pending` or `failed` (`AC_NOT_PROVEN`). ACs without `verify` keep the existing evidence rules.
127
+
128
+ ## Rubric for subjective ACs (C-09)
129
+
130
+ For ACs that are judged rather than computed (layout, copy, visual design), add a rubric:
131
+
132
+ ```json
133
+ { "id": "AC-3", "statement": "The empty state guides the user", "evidenceRequired": "screenshot", "failureCases": ["blank panel"],
134
+ "rubric": [ { "criterion": "contrast", "pass": ">= 4.5:1 (WCAG AA)" }, { "criterion": "guidance", "pass": "explains how to import a document" } ] }
135
+ ```
136
+
137
+ The Devil's Advocate scores every criterion in the ledger as `rubricScores: [{ac, criterion, pass, evidence}]` with observed evidence (screenshot path, DOM snapshot, measured value). `gate judge` refuses PASS while any criterion is unscored or `pass: false` (`AC_RUBRIC`).
138
+
139
+ ## Project feature ledger (C-15)
140
+
141
+ Optional `.ai-engineering-loop/features.json` lists what the project must do, one triple per feature:
142
+
143
+ ```json
144
+ { "schemaVersion": 1, "features": [
145
+ { "id": "F-1", "title": "Import", "behavior": "user imports a .md file and sees it in the list", "verify": "npm run e2e -- import", "status": "not_started", "priority": 1, "dependsOn": [] }
146
+ ] }
147
+ ```
148
+
149
+ Status moves only through Runs: a Goal Contract with `"featureId": "F-1"` makes it `active` at the Goal gate, and delivery makes it `passing` with the Run as evidence (passing is final). `ael next` prints the highest-priority unblocked feature, VCR (verified / activated), and flags any `passing` feature without a delivered Run behind it. `run --new` warns when an unfinished Run exists and VCR is below 1 (WIP=1).
150
+
151
+ ## Intended flow (C-21)
152
+
153
+ For UI or multi-step work, add the flow the user should experience as transitions:
154
+
155
+ ```json
156
+ "flow": [
157
+ { "from": "List", "event": "click Export", "to": "Exporting" },
158
+ { "from": "Exporting", "event": "file ready", "to": "Done" },
159
+ { "from": "Exporting", "event": "disk full", "to": "Error" }
160
+ ]
161
+ ```
162
+
163
+ `goal draft` returns it as a `From | Event | To` table and a Mermaid `stateDiagram-v2` (`flow.mermaid`), with warnings (a state nothing reaches, a state with no way out that is not Done/Error), so the human checks the intent before freeze. A malformed `flow` fails the Goal gate. The PR receipt shows the same diagram (GitHub renders it) and table under "Intended flow", after "Verification artifacts" and before the acceptance criteria.
164
+
165
+ ## Verification artifacts in the receipt (C-21)
166
+
167
+ The receipt lists `--evidence AC-N=<https-url>` links first, typed as video, screenshot or diagram from the URL. An AC with `crossComponent: true` or a `rubric` is `MISSING` (❌) until it has at least one such link, because a reviewer should be able to look at the behaviour rather than read the diff.
168
+
@@ -53,10 +53,59 @@ Verification terminates through three explicit layers of evidence. Unit tests al
53
53
  | **L2: Behavioral / Runtime** | Functions, units, modules, regressions | `test_unit`, `test_related`, `test_all` | Fast behavioral feedback at named seams and regression verification. |
54
54
  | **L3: End-to-End** | Cross-component flows, integrations, browser/API | `e2e` (optional) | Realistic integrated proof across boundary seams. |
55
55
 
56
- When `e2e` is configured in `verification.md`, cross-component acceptance criteria must cite L3 command evidence. If `e2e` is empty, Maker must document in Reality why L3 is not configured.
56
+ Unit tests isolate and mock by design, which is exactly why they miss interface mismatches, state propagation errors, resource leaks, and environment dependencies between components.
57
+
58
+ | Tier | Required layers |
59
+ |---|---|
60
+ | T0 | L1-L2 (L3 when an AC is `crossComponent: true`) |
61
+ | T1 | L1-L2, plus L3 for every `crossComponent: true` AC |
62
+ | T2 | L1-L3 on every AC |
63
+
64
+ Enforcement (`lib/claimed-vs-reality.js`, `gate review`):
65
+ - The Layer cell accepts `L1`, `L2`, `L3`, combos such as `L2+L3`, and `L3 waived: <reason>` (at least 15 characters). Unknown tokens fail the gate.
66
+ - An AC that needs L3 without L3 or a reasoned waiver fails `gate review` on every tier, with a FIX line that says what to run.
67
+ - Legacy 3-column files stay valid when no AC needs L3.
68
+ - Tags are bound to recorded evidence (C-01): `L1` needs a passing L1 command in `verification.json`/`final-verification.json`; `L3` needs a passing `e2e` command, or Reality that describes a manual end-to-end flow (steps, URL, screenshot path). Commands not mapped in `verification.md` count as L2. Waivers need no command.
69
+ - When `e2e` is empty in `verification.md`, run the AC's real user flow and cite the observation, or write the waiver. Devil's Advocate and Judge challenge waivers like any other claim.
57
70
 
58
71
  ---
59
72
 
73
+ ## 3a. Baseline and Default Plan
74
+
75
+ **Baseline (C-10).** After the Goal gate and before the Maker gate, `verify --baseline --run <id>` runs the default plan on the untouched tree and writes `runs/<id>/baseline.json` with status `GREEN`, `RED`, or `BROKEN`.
76
+ - `BROKEN` (a command exits 127, i.e. it does not exist): the command fails with a FIX to correct `verification.md`. Verification cannot prove anything with a broken command.
77
+ - `RED`: the tree was already failing. Later, when `verify` fails, each failing command is labelled `pre-existing` (every failing test name was already failing at baseline) or new. Unnamed failures are always treated as new. The gate still blocks; the label tells the Maker whether the change caused it.
78
+
79
+ **Default plan (C-02).** `verify` without `--cmd` runs `build -> typecheck -> lint -> test_unit` (whichever are configured), serially, and stops at the first failure. Commands that rewrite files (`--fix`, `--write`) are never run by default. Each record carries its layer (`L1`/`L2`/`L3`).
80
+
81
+ **Verification catalog (C-23).** AEL does not ship test tools; it recognises the ones a project has. Any of these keys in `verification.md` is mapped to a layer and recorded with it: `security` (L1); `property`, `fuzz`, `mutation`, `coverage`, `benchmark` (L2); `contract`, `pentest` (L3); `formal` (LF, a separate layer for model checking and proofs). Only `property` joins the default plan (after `test_unit`); the rest run when an AC requires them with `"verify": { "key": "<key>" }`, or with `--cmd`. An AC that requires only a layer (`"layer": "LF"`) makes default `verify` add the first configured catalog key of that layer when no step covers it (for `LF`, `formal`; see `docs formal-statechart` for model checking with pstack `/statechart`). `gate judge` refuses PASS until each required key passed on the gated diff (`AC_NOT_PROVEN`). Claimed-vs-reality accepts `LF` as a Layer tag. Testing in production (canary, shadowing, monitoring) stays outside AEL; its outcomes come back through `outcome record` (C-13).
82
+
83
+ **Feature map (C-22).** Optional `.ai-engineering-loop/feature-map.md` is a site map of what users can do, one Markdown table: `| Feature | Where (UI or endpoint) | Paths (globs) | How to reach | Driver |`. `review-pack` appends the entries whose `Paths` the diff touches, so the Devil's Advocate and L3 evidence use the same names ("export-csv", not "the button on the left"). `Driver` is an optional reusable command that runs the flow (cheaper and more repeatable than an agent clicking around). `doctor --harness` warns when `e2e` or `smoke` is configured but the map is empty.
84
+
85
+ **L1 at T1/T2.** When `verification.md` configures `typecheck` or `lint`, the verification gate at T1/T2 refuses a full-scope bundle with no L1 record (`L1_MISSING`). The related lane is exempt; `verify --final` runs the full plan.
86
+
87
+ `init` writes only commands the project really has: `typecheck` from a script or a TypeScript setup, `lint` and `build` only from package scripts.
88
+
89
+ **Smoke (C-03).** An optional `smoke` key in `verification.md` is a command that starts the app, probes readiness (a health URL, a golden journey), stops it, and exits. It counts as L2. When the gated diff touches startup-risky files (migrations, `.env*`, `config/`, `*.config.*`, Docker/Procfile, `main|index|server|app` entrypoints, dependency manifests), default `verify` runs it after `test_unit`, and the verification gate fails with `SMOKE_MISSING` when it did not run. A smoke command that times out or exits non-zero blocks the gate; on timeout the recorder kills its process group.
90
+
91
+ ## 3b. Verification Integrity (C-11)
92
+
93
+ At `gate review` the CLI scans the gated diff and adds `source: "verification-integrity"` findings (HIGH; BLOCKER at T2, VALID, with a concrete fix) for:
94
+ - a skip/only/todo marker added in a test file (`it.skip`, `xit`, `fit`, `@Disabled`, `pytest.mark.skip`, `t.Skip`, ...);
95
+ - a test file deleted without a replacement test in the same directory;
96
+ - assertions removed from a test file without replacement;
97
+ - a change to verification config (`.ai-engineering-loop/verification.md`, jest/vitest/playwright/cypress config, `pytest.ini`, coverage config) that the frozen Goal Contract does not name.
98
+
99
+ These findings cannot be dropped or downgraded by the reviewer. Only a human dismissal (`dismissal: {by, evidence}`) can downgrade them; otherwise the review gate fails.
100
+
101
+ ## 3c. Justification Comments and Suppressions (C-18)
102
+
103
+ Agents write comments to justify not doing the right thing, and comments cannot be enforced. At `gate review` the CLI scans comments added in non-test product files and adds `source: "comment-justification"` findings (`axis: "standards"`):
104
+ - a suppression (`eslint-disable*`, `@ts-ignore`, `@ts-expect-error`, `# type: ignore`, `# noqa`, `//nolint`, `NOSONAR`, `istanbul/c8 ignore`): **HIGH** without a reason, **MEDIUM** with one (eslint `-- reason`, or text after the directive);
105
+ - a comment that justifies a workaround (`workaround`, `quick fix`, `hack`, `temporary`, `for now`, `band-aid`, `fix later`, ...): **MEDIUM**. Projects add patterns in `.ai-engineering-loop/rules.json` as `"justificationPatterns": ["<regex>"]`.
106
+
107
+ Docs, changelogs, data files and tests are skipped. Unlike C-11, a reviewer may downgrade these with `dismissal: {by, evidence}` (for example generated code); dropping or downgrading one without evidence fails the review gate.
108
+
60
109
  ## 4. The 4 Verification Gates
61
110
 
62
111
  1. **Gate 1: Unit & Integration Tests**: All unit and edge-case tests must pass with 0 failures.
@@ -104,3 +153,59 @@ Do **not** spawn Devil's Advocate if:
104
153
  - Reality is only `seems green`
105
154
 
106
155
  Parser: `lib/claimed-vs-reality.js` (`readyForDa`). Pass that path to DA/Judge with the verification log.
156
+
157
+ ## 7. Clean State at Delivery (C-06)
158
+
159
+ `gate delivery` also checks that the next session starts from a tree it can trust:
160
+ - **Always blocks:** a `debugger` statement, merge-conflict marker or interactive breakpoint added by the change; a secret-like file in the change (`.env`, `.env.local`, keys; `.env.example` is fine); a `_templateOnly` scaffold left in the Run directory or `.ai-engineering-loop/tasks/`.
161
+ - **Untracked files that are not part of the reviewed diff** (logs, coverage, scratch files): block in UNATTENDED, warn in ASSISTED. Warnings are printed and kept in the Run history as `cleanStateWarnings`.
162
+
163
+ ## 8. Debt Ledger (C-14)
164
+
165
+ At `gate delivery` the CLI copies two kinds of deferred work into `.ai-engineering-loop/debt.json` (tracked in git, like `learnings.json`):
166
+ - VALID `MEDIUM`/`LOW` findings the Judge accepted as trade-offs (file, scenario, risk, "revisit on the next change touching <file>");
167
+ - claimed-vs-reality rows with `L3 waived: <reason>` ("revisit when an e2e command exists").
168
+
169
+ `review-pack` appends open debt on the files the new diff touches to the Devil's Advocate pack and lists it. `ael debt` lists open items (`--file <path>`); `ael debt close <id> --by <name> [--note <text>]` closes one.
170
+
171
+ ## 9. Reviewer Inputs Are Logged (C-16)
172
+
173
+ "Model-visible means logged": `review-pack` writes `runs/<id>/review-inputs.json` with the sha256 of the review pack, the gated diff, and the Devil's Advocate prompt body. `review ingest` copies them into `findings.json` (`reviewer.inputs`, `reviewer.promptSha256`). `gate review` fails the ledger binding when any recorded input changed after the reviewer saw it, so a missed defect can always be traced to exactly what the reviewer was shown.
174
+
175
+ ## 9b. Quality Anchors from Git and the Budget Ledger (C-24)
176
+
177
+ `ael stats` also reports:
178
+ - **Revert rate**: git `Revert "<subject>"` commits matched to DELIVERED Runs, by the reverted sha (the receipt's head) or by subject (squash merges keep it). A stable, low revert rate while throughput rises is the signal that quality held.
179
+ - **Verification token share**: tokens recorded with `budget record --role reviewer|judge|verification` over all role-labelled tokens (`maker`, `other` are the rest).
180
+
181
+ `calibrate` treats each detected revert as an outcome (`by: git`) unless one was already recorded by hand.
182
+
183
+ ## 10. Reviewer Calibration (C-13)
184
+
185
+ Gates prove the change matches the Goal; only reality proves the reviewers were right. After delivery, record problems people found:
186
+
187
+ ```
188
+ ael outcome record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name> [--evidence <path|url>]
189
+ ael calibrate [--json]
190
+ ```
191
+
192
+ `calibrate` compares each outcome with that Run's Finding Ledger: `DA_MISS` (no finding on that file), `FALSE_DISMISS` (a finding on that file was marked INVALID or dismissed), `CAUGHT_BUT_SHIPPED` (a VALID finding was accepted as a trade-off). Results are grouped by the reviewer prompt hash (C-16), so after you tighten `agents/shared/devil-advocate.body.md` for a recurring miss you can see whether the next prompt version misses less. Plan for several tuning rounds; record each prompt change in its commit message.
193
+
194
+ ## 11. Promoting Review Feedback to Rules (C-05)
195
+
196
+ `learn` lists **promotion candidates**: learnings seen in 3 or more Runs. A human turns one into an executable check:
197
+
198
+ ```
199
+ ael learn promote L-abc123 --check "! grep -rq \"require('fs')\" src/renderer" --fix "Move file access to src/preload/file-ops.ts and call window.api" --by <name> [--why <text>]
200
+ ```
201
+
202
+ Rules live in `.ai-engineering-loop/rules.json` (tracked). Default `verify` runs every rule as an L1 check after the static checks; a violation prints `Rule <id> violated. WHY: ... FIX: ...`. The verification gate refuses a full-scope bundle in which any rule has no passing record (`RULES_MISSING`). The check must exit non-zero on a violation and is run through the shell unless `--no-shell` is given.
203
+
204
+ **Ratchet rules (C-17).** `--scope diff` judges only what the change adds, so a rule can be promoted on a codebase that still has old violations. Its check receives `AEL_CHANGED_FILES` (one path per line) and `AEL_ADDED_LINES_FILE` (`path:line:text` for every added line, the `grep -n` format), for example `! grep -E "useEffect\(" "$AEL_ADDED_LINES_FILE"` (POSIX shell) or a small Node script for Windows. New occurrences fail with their `file:line`; old ones do not. `doctor --harness` runs each diff rule against every tracked line and reports the remaining legacy backlog, and says when it is clean enough to switch the rule to `--scope repo`.
205
+
206
+ **Built-in rule kinds (C-20).** The two most common corrections need no hand-written check:
207
+ - `--forbid-import "<fromGlob> -> <toGlob>"`: files under `fromGlob` may not import modules under `toGlob`. Relative specifiers are resolved against the importing file (with common extensions); bare names such as `fs` or `react` match as written, so `"src/renderer/** -> fs"` forbids Node's `fs` in the renderer.
208
+ - `--forbid-api "<regex>" [--except "<glob>"]`: the API may appear only inside the excepted paths, for example `useEffect` only in `src/framework/**`.
209
+
210
+ Both are stored with `kind`, always use `scope: "diff"`, and run as `rules check <ruleId>` through the same CLI (so `rules.json` holds no machine-specific path). Violations print as `file:line: <message>` and the rule's FIX.
211
+
@@ -0,0 +1,52 @@
1
+ 'use strict';
2
+
3
+ // Harness ablation for bench (C-08, learn-harness-engineering L02/L12):
4
+ // switch one gate component off inside a bench scenario repo to measure
5
+ // whether it still earns its cost. Honoured only when bench set it up (env
6
+ // AEL_BENCH_MODE + AEL_ABLATE and a marker file bench wrote into the repo),
7
+ // and every skip is logged to the Run so it can never pass silently.
8
+
9
+ const fs = require('fs');
10
+ const path = require('path');
11
+ const { runsDir } = require('./run-state.js');
12
+
13
+ const COMPONENTS = Object.freeze(['integrity', 'comments', 'l3', 'l1', 'smoke', 'rules']);
14
+ const MARKER = path.join('.ai-engineering-loop', 'bench-ablation.json');
15
+
16
+ function requested(env = process.env) {
17
+ return String(env.AEL_ABLATE || '').split(',').map((item) => item.trim()).filter(Boolean);
18
+ }
19
+
20
+ function writeMarker(repo, components) {
21
+ fs.mkdirSync(path.join(repo, '.ai-engineering-loop'), { recursive: true });
22
+ fs.writeFileSync(path.join(repo, MARKER), `${JSON.stringify({ schemaVersion: 1, components }, null, 2)}\n`);
23
+ }
24
+
25
+ function ablated(rootDir, component, runId = null, env = process.env) {
26
+ if (!env.AEL_BENCH_MODE || !requested(env).includes(component)) return false;
27
+ let marker = null;
28
+ try {
29
+ marker = JSON.parse(fs.readFileSync(path.join(rootDir, MARKER), 'utf8'));
30
+ } catch {
31
+ return false;
32
+ }
33
+ if (!Array.isArray(marker?.components) || !marker.components.includes(component)) return false;
34
+ if (runId) {
35
+ try {
36
+ fs.appendFileSync(path.join(runsDir(rootDir), runId, 'ablation.log'), `${JSON.stringify({ component, at: new Date().toISOString() })}\n`);
37
+ } catch {
38
+ // Run directory missing; the bench result still records the ablation.
39
+ }
40
+ }
41
+ return true;
42
+ }
43
+
44
+ function validateComponents(list) {
45
+ const unknown = list.filter((item) => !COMPONENTS.includes(item));
46
+ if (unknown.length) {
47
+ throw Object.assign(new Error(`Unknown --without component(s): ${unknown.join(', ')}. Use: ${COMPONENTS.join(', ')}`), { code: 'BENCH_USAGE' });
48
+ }
49
+ return list;
50
+ }
51
+
52
+ module.exports = { COMPONENTS, MARKER, ablated, requested, validateComponents, writeMarker };
@@ -0,0 +1,95 @@
1
+ 'use strict';
2
+
3
+ // Harness-owned AC status (C-07, learn-harness-engineering L08): an AC that
4
+ // names its own verify command is `proven` only when that exact command
5
+ // passed on the gated diff. Agents cannot set the status; evidence does.
6
+
7
+ const fs = require('fs');
8
+ const path = require('path');
9
+ const { runsDir } = require('./run-state.js');
10
+
11
+ const norm = (text) => String(text || '').trim().replace(/\s+/g, ' ');
12
+
13
+ function recordedCommands(rootDir, runId) {
14
+ const records = [];
15
+ for (const file of ['verification.json', 'final-verification.json']) {
16
+ try {
17
+ const bundle = JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, file), 'utf8'));
18
+ records.push(...(bundle.commands || []));
19
+ } catch {
20
+ // Not recorded yet.
21
+ }
22
+ }
23
+ return records;
24
+ }
25
+
26
+ // [{ id, command, status: 'proven'|'failed'|'pending'|'unbound' }]
27
+ // `verify` may name an exact `command`, a catalog `key` (C-23, e.g.
28
+ // "pentest"), or a `layer` (e.g. "LF"). Records carry `key`/`layer` when
29
+ // annotated by runAcStatuses.
30
+ function acStatuses(goal, records) {
31
+ return (goal?.acceptanceCriteria || []).map((criterion) => {
32
+ const verify = criterion.verify || {};
33
+ const command = norm(verify.command);
34
+ let label;
35
+ let matches;
36
+ if (command) {
37
+ label = command;
38
+ matches = (records || []).filter((record) => norm(record.command) === command);
39
+ } else if (verify.key) {
40
+ label = `key:${verify.key}`;
41
+ matches = (records || []).filter((record) => record.key === verify.key);
42
+ } else if (verify.layer) {
43
+ label = `layer:${verify.layer}`;
44
+ matches = (records || []).filter((record) => record.layer === verify.layer);
45
+ } else {
46
+ return { id: criterion.id, command: null, status: 'unbound' };
47
+ }
48
+ if (matches.some((record) => record.exitCode === 0 && record.timeoutStatus === 'COMPLETED')) {
49
+ return { id: criterion.id, command: label, status: 'proven' };
50
+ }
51
+ return { id: criterion.id, command: label, status: matches.length ? 'failed' : 'pending' };
52
+ });
53
+ }
54
+
55
+ function runAcStatuses(rootDir, run, goal) {
56
+ const lanes = require('./verification-lanes.js');
57
+ const records = recordedCommands(rootDir, run.runId).map((record) => {
58
+ const key = record.key || lanes.keyOfCommand(rootDir, record.command);
59
+ return { ...record, key, layer: record.layer || (key ? lanes.layerOfKey(key) : null) };
60
+ });
61
+ return acStatuses(goal, records);
62
+ }
63
+
64
+ // Catalog keys the frozen Goal's ACs require (C-23), so default verify runs them.
65
+ function requiredKeys(goal) {
66
+ return [...new Set((goal?.acceptanceCriteria || []).map((criterion) => criterion.verify?.key).filter(Boolean))];
67
+ }
68
+
69
+ // Layers the frozen Goal's ACs require without naming a key (e.g. "LF").
70
+ function requiredLayers(goal) {
71
+ return [...new Set((goal?.acceptanceCriteria || []).filter((criterion) => !criterion.verify?.key && !criterion.verify?.command)
72
+ .map((criterion) => criterion.verify?.layer).filter(Boolean))];
73
+ }
74
+
75
+ // C-09 (learn-harness-engineering L09/L11): subjective ACs (UI, copy,
76
+ // design) carry a rubric. The ledger must score every criterion with
77
+ // evidence; an unscored or failed criterion blocks Judge PASS.
78
+ function rubricProblems(goal, ledger) {
79
+ const problems = [];
80
+ const scores = Array.isArray(ledger?.rubricScores) ? ledger.rubricScores : [];
81
+ for (const criterion of goal?.acceptanceCriteria || []) {
82
+ for (const item of Array.isArray(criterion.rubric) ? criterion.rubric : []) {
83
+ const name = String(item.criterion || '').trim();
84
+ const score = scores.find((entry) => entry.ac === criterion.id && String(entry.criterion || '').trim() === name);
85
+ if (!score || typeof score.pass !== 'boolean' || !String(score.evidence || '').trim()) {
86
+ problems.push(`${criterion.id} rubric "${name}" has no score with evidence in the Finding Ledger (rubricScores)`);
87
+ } else if (!score.pass) {
88
+ problems.push(`${criterion.id} rubric "${name}" failed: ${score.evidence}`);
89
+ }
90
+ }
91
+ }
92
+ return problems;
93
+ }
94
+
95
+ module.exports = { acStatuses, recordedCommands, requiredKeys, requiredLayers, rubricProblems, runAcStatuses };
@@ -0,0 +1,138 @@
1
+ 'use strict';
2
+
3
+ // Baseline verification (C-10, learn-harness-engineering L06 + startup flow):
4
+ // before the Maker edits anything, run the project's own verification plan
5
+ // once on the frozen starting state. A broken command is caught before work
6
+ // starts, and failures that already existed are told apart from regressions.
7
+
8
+ const fs = require('fs');
9
+ const path = require('path');
10
+ const { atomicWriteJson, getCurrentRun, getGitRevision, loadRun, runsDir } = require('./run-state.js');
11
+
12
+ const BASELINE_FILE = 'baseline.json';
13
+ const COMMAND_NOT_FOUND = 127;
14
+
15
+ function baselineError(message, code, fix) {
16
+ return Object.assign(new Error(fix ? `${message} FIX: ${fix}` : message), { code });
17
+ }
18
+
19
+ function resolveRun(rootDir, runId) {
20
+ const run = runId ? loadRun(rootDir, runId) : getCurrentRun(rootDir);
21
+ if (!run) throw baselineError('No current run.', 'NO_CURRENT_RUN', 'pass --run <id>.');
22
+ return run;
23
+ }
24
+
25
+ // Failing test names from common runner output (node:test, TAP, jest, vitest,
26
+ // pytest). Best effort: an empty list means "could not tell", never "none".
27
+ function extractFailingTests(text) {
28
+ const names = new Set();
29
+ const patterns = [
30
+ /^\s*✖\s+(.+?)\s+\(\d+(?:\.\d+)?m?s\)\s*$/gm, // node:test spec reporter
31
+ /^not ok \d+ - (.+)$/gm, // TAP
32
+ /^\s*●\s+(.+?)\s*$/gm, // jest
33
+ /^\s*[×✗]\s+(.+?)(?:\s+\d+m?s)?\s*$/gm, // vitest
34
+ /^FAILED\s+(\S+)/gm // pytest -rf
35
+ ];
36
+ const value = String(text || '');
37
+ for (const pattern of patterns) {
38
+ for (const match of value.matchAll(pattern)) {
39
+ const name = match[1].trim();
40
+ if (name && !/^failing tests:?$/i.test(name)) names.add(name);
41
+ }
42
+ }
43
+ return [...names].sort();
44
+ }
45
+
46
+ const passedRecord = (record) => record.exitCode === 0 && record.timeoutStatus === 'COMPLETED';
47
+
48
+ // Run `plan` ([{ key, layer, command, args, shell }]) serially on the current
49
+ // tree and write runs/<id>/baseline.json. Must run before the Maker gate, so
50
+ // the tree is the frozen starting point.
51
+ async function recordBaseline(rootDir, { runId, plan, timeoutMs, runCommand } = {}) {
52
+ const run = resolveRun(rootDir, runId);
53
+ if (run.artifacts?.diff) {
54
+ throw baselineError(`Run ${run.runId} already has a gated Maker diff; a baseline must describe the tree before the change.`,
55
+ 'BASELINE_TOO_LATE', 'start a new Run, or skip the baseline for this one.');
56
+ }
57
+ if (!Array.isArray(plan) || plan.length === 0) {
58
+ throw baselineError('No baseline commands.', 'NO_BASELINE_COMMANDS',
59
+ 'add build/typecheck/lint/test_unit to .ai-engineering-loop/verification.md, or pass --cmd.');
60
+ }
61
+ const recorder = require('./verification-recorder.js');
62
+ const exec = runCommand || recorder.runCommand;
63
+ const limit = Number.isInteger(timeoutMs) && timeoutMs > 0 ? timeoutMs : recorder.DEFAULT_TIMEOUT_MS;
64
+ const records = [];
65
+ const problems = [];
66
+ for (const item of plan) {
67
+ const result = await exec(item.command, item.args || [], { cwd: rootDir, timeoutMs: limit, shell: Boolean(item.shell) });
68
+ const record = {
69
+ key: item.key || null,
70
+ layer: item.layer || null,
71
+ command: result.command,
72
+ exitCode: result.exitCode,
73
+ timeoutStatus: result.timeoutStatus,
74
+ failingTests: passedRecord(result) ? [] : extractFailingTests(`${result.stdout}\n${result.stderr}`)
75
+ };
76
+ records.push(record);
77
+ if (result.exitCode === COMMAND_NOT_FOUND || result.launchError) {
78
+ problems.push(`${record.command}: command not found (exit ${result.exitCode}). FIX: correct the ${record.key || 'command'} entry in .ai-engineering-loop/verification.md; verification cannot prove anything with it.`);
79
+ break;
80
+ }
81
+ }
82
+ const broken = problems.length > 0;
83
+ const red = records.some((record) => !passedRecord(record));
84
+ if (!broken && red) {
85
+ for (const record of records.filter((r) => !passedRecord(r))) {
86
+ problems.push(`${record.command}: already failing before any change (exit ${record.exitCode}${record.failingTests.length ? `; ${record.failingTests.length} failing test(s)` : ''}). FIX: repair the baseline in a separate Run first, or continue knowing these failures are pre-existing.`);
87
+ }
88
+ }
89
+ const status = broken ? 'BROKEN' : (red ? 'RED' : 'GREEN');
90
+ const document = {
91
+ schemaVersion: 1,
92
+ runId: run.runId,
93
+ gitRevision: getGitRevision(rootDir),
94
+ recordedAt: new Date().toISOString(),
95
+ status,
96
+ commands: records
97
+ };
98
+ const filePath = path.join(runsDir(rootDir), run.runId, BASELINE_FILE);
99
+ atomicWriteJson(filePath, document);
100
+ return {
101
+ runId: run.runId,
102
+ path: path.relative(rootDir, filePath).split(path.sep).join('/'),
103
+ status,
104
+ commands: records,
105
+ problems
106
+ };
107
+ }
108
+
109
+ function loadBaseline(rootDir, runId) {
110
+ try {
111
+ return JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), runId, BASELINE_FILE), 'utf8'));
112
+ } catch {
113
+ return null;
114
+ }
115
+ }
116
+
117
+ // Label failing verification records as pre-existing (every failing test was
118
+ // already failing at baseline) or new. Records without a baseline counterpart,
119
+ // or whose failures cannot be named, stay "new": unknown is never excused.
120
+ function classifyAgainstBaseline(baseline, records) {
121
+ const byCommand = new Map((baseline?.commands || []).map((record) => [record.command, record]));
122
+ return (records || []).filter((record) => !passedRecord(record)).map((record) => {
123
+ const before = byCommand.get(record.command);
124
+ const failing = extractFailingTests(`${record.stdout || ''}\n${record.stderr || ''}`);
125
+ const known = new Set(before?.failingTests || []);
126
+ const fresh = failing.filter((name) => !known.has(name));
127
+ const preExisting = Boolean(before && !passedRecord(before) && failing.length > 0 && fresh.length === 0);
128
+ return { command: record.command, preExisting, newFailures: preExisting ? [] : (fresh.length ? fresh : failing) };
129
+ });
130
+ }
131
+
132
+ module.exports = {
133
+ BASELINE_FILE,
134
+ classifyAgainstBaseline,
135
+ extractFailingTests,
136
+ loadBaseline,
137
+ recordBaseline
138
+ };
package/lib/bench.js CHANGED
@@ -817,6 +817,9 @@ function prepareScenarioRepo(repo, scenario, { host, tokenCap, extraEnv, cli, pa
817
817
  const env = extraEnv;
818
818
  const init = spawnNode([cli, 'init'], { cwd: repo, env });
819
819
  if (init.status !== 0) throw benchError(`init failed: ${init.stdout}\n${init.stderr}`, 'BENCH_SETUP');
820
+ const ablation = require('./ablation.js');
821
+ const components = ablation.requested(env || {});
822
+ if (components.length) ablation.writeMarker(repo, components);
820
823
  installProjectHostSkills(repo, host, packageRoot);
821
824
  installLocalCli(repo, cli);
822
825
  if (host.hooks) {
@@ -1085,7 +1088,9 @@ function runBench(options = {}) {
1085
1088
  const selected = selectScenarios(all, options.scenarioIds || []);
1086
1089
  const benchId = options.benchId || newBenchId();
1087
1090
  const startedAt = new Date().toISOString();
1088
- const extraEnv = options.env || {};
1091
+ // C-08 ablation: switch gate components off inside the scenario repos.
1092
+ const without = require('./ablation.js').validateComponents(options.without || []);
1093
+ const extraEnv = { ...(options.env || {}), ...(without.length ? { AEL_ABLATE: without.join(',') } : {}) };
1089
1094
  const cli = options.cli || cliPath(options.packageRoot);
1090
1095
 
1091
1096
  let scenarioScores;
@@ -1151,14 +1156,32 @@ function runBench(options = {}) {
1151
1156
  status,
1152
1157
  unavailableReason,
1153
1158
  repeat,
1159
+ ...(without.length ? { ablation: { without } } : {}),
1154
1160
  summary: summarize(scenarioScores),
1155
1161
  scenarios: scenarioScores,
1156
1162
  instructionViolations
1157
1163
  });
1158
1164
  const directory = writeBenchOutputs(rootDir, benchId, result);
1165
+ if (without.length) appendSimplificationLog(rootDir, { benchId, without, summary: result.summary });
1159
1166
  return { ...result, directory };
1160
1167
  }
1161
1168
 
1169
+ // Simplification log (OpenAI pack QUALITY_SCORE.md): every ablation bench is
1170
+ // recorded; a human fills `decision` (keep | remove | replace) after
1171
+ // comparing it with a baseline bench (`bench compare`).
1172
+ function appendSimplificationLog(rootDir, { benchId, without, summary }) {
1173
+ const filePath = path.join(rootDir, '.ai-engineering-loop', 'simplification-log.json');
1174
+ let log = { schemaVersion: 1, entries: [] };
1175
+ try {
1176
+ log = JSON.parse(fs.readFileSync(filePath, 'utf8'));
1177
+ } catch {
1178
+ // first entry
1179
+ }
1180
+ log.entries = [...(log.entries || []), { at: new Date().toISOString(), benchId, without, complianceRate: summary?.complianceRate ?? null, decision: null }];
1181
+ fs.mkdirSync(path.dirname(filePath), { recursive: true });
1182
+ fs.writeFileSync(filePath, `${JSON.stringify(log, null, 2)}\n`);
1183
+ }
1184
+
1162
1185
  function loadBenchResult(rootDir, benchId) {
1163
1186
  const filePath = path.join(benchDir(rootDir, benchId), 'results.json');
1164
1187
  if (!fs.existsSync(filePath)) throw benchError(`No bench result ${benchId}`, 'BENCH_NOT_FOUND');
package/lib/budget.js CHANGED
@@ -72,6 +72,7 @@ function readUsageFile(filePath) {
72
72
  if (entry.nodeId != null && !/^[a-z][a-z0-9-]{0,63}$/.test(entry.nodeId)) {
73
73
  throw new Error('nodeId is invalid');
74
74
  }
75
+ if (entry.role != null && !USAGE_ROLES.includes(entry.role)) throw new Error('role is invalid');
75
76
  if (typeof entry.recordedAt !== 'string' || Number.isNaN(Date.parse(entry.recordedAt))) {
76
77
  throw new Error('recordedAt is invalid');
77
78
  }
@@ -248,10 +249,15 @@ function appendPrivateLine(rootDir, filePath, line) {
248
249
  }
249
250
  }
250
251
 
252
+ // C-24: who spent the tokens, so `stats` can show the verification share
253
+ // ("70% of her daily tokens go into verification", talk 11:46).
254
+ const USAGE_ROLES = Object.freeze(['maker', 'reviewer', 'judge', 'verification', 'other']);
255
+
251
256
  function recordTokenUsage(
252
257
  rootDir,
253
- { runId, nodeId, inputTokens, outputTokens, model, now = new Date() } = {}
258
+ { runId, nodeId, inputTokens, outputTokens, model, role = null, now = new Date() } = {}
254
259
  ) {
260
+ if (role != null && !USAGE_ROLES.includes(role)) throw usageError(`role must be one of ${USAGE_ROLES.join(', ')}`);
255
261
  const run = resolveRun(rootDir, runId);
256
262
  validateTokenCount('inputTokens', inputTokens);
257
263
  validateTokenCount('outputTokens', outputTokens);
@@ -266,6 +272,7 @@ function recordTokenUsage(
266
272
  totalTokens,
267
273
  model: normalizedModel,
268
274
  ...(nodeId ? { nodeId } : {}),
275
+ ...(role ? { role } : {}),
269
276
  recordedAt: now.toISOString()
270
277
  };
271
278
  if (nodeId) budgetStatus(rootDir, { runId: run.runId, nodeId, now });
@@ -341,6 +348,7 @@ function readUnavailableUsage(rootDir) {
341
348
  }
342
349
 
343
350
  module.exports = {
351
+ USAGE_ROLES,
344
352
  usageDir,
345
353
  utcDay,
346
354
  usagePath,