ai-engineering-loop 2.7.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/references/loop-commands.md +4 -3
- package/.agents/skills/ael/references/loop-deliver.md +1 -1
- package/.agents/skills/ael/references/loop-goal.md +1 -1
- package/.claude/agents/devil-advocate.md +1 -1
- package/.grok/agents/devil-advocate.md +1 -1
- package/CHANGELOG.md +15 -0
- package/agents/shared/devil-advocate.body.md +1 -1
- package/bin/ai-engineering-loop.js +69 -6
- package/core/formal-statechart.md +49 -0
- package/core/goal-contract.md +18 -0
- package/core/verification-loop.md +28 -0
- package/lib/ablation.js +1 -1
- package/lib/ac-status.js +39 -7
- package/lib/budget.js +9 -1
- package/lib/calibration.js +17 -0
- package/lib/claimed-vs-reality.js +1 -1
- package/lib/cli-args.js +1 -1
- package/lib/comment-findings.js +111 -0
- package/lib/feature-map.js +57 -0
- package/lib/flow.js +86 -0
- package/lib/gates.js +23 -0
- package/lib/harness-audit.js +17 -0
- package/lib/learn.js +24 -1
- package/lib/receipt.js +33 -2
- package/lib/reverts.js +77 -0
- package/lib/rule-check.js +90 -0
- package/lib/rules.js +99 -6
- package/lib/stats.js +20 -0
- package/lib/verification-lanes.js +39 -7
- package/lib/verification-recorder.js +3 -3
- package/package.json +1 -1
- package/schemas/goal-contract.schema.json +22 -3
|
@@ -10,11 +10,12 @@
|
|
|
10
10
|
## Harness health and history
|
|
11
11
|
|
|
12
12
|
Run these when the user asks how the loop is doing, before planning work, or when a gate points at them. All read local Run files; none starts a Run.
|
|
13
|
-
- `stats [--last <n>] [--json]`: failure causes by harness layer (`task_spec`, `context`, `environment`, `verification`, `state`, `model`), false-victory rate per tier, iterations to PASS. Fix the bottleneck layer first.
|
|
13
|
+
- `stats [--last <n>] [--json]`: failure causes by harness layer (`task_spec`, `context`, `environment`, `verification`, `state`, `model`), false-victory rate per tier, iterations to PASS, revert rate of delivered Runs (from git) and the verification share of tokens. Fix the bottleneck layer first.
|
|
14
14
|
- `outcome record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name>`: a problem people found after delivery. Only record what the human reported; never invent one. Then `calibrate` lists DA misses and wrong dismissals per reviewer prompt version.
|
|
15
15
|
- `debt [--file <path>]` lists trade-offs the Judge accepted and L3 waivers; `debt close <id> --by <name>` only on the human's word.
|
|
16
16
|
- `next`: the next unblocked feature in `.ai-engineering-loop/features.json` and VCR (verified/activated). Finish the active feature before starting another.
|
|
17
|
-
- `learn` lists lessons seen in 3+ Runs; a human promotes one with `learn promote <id> --check "<cmd>" --fix "<instruction>" --by <name>`. Default `verify` then runs it as an L1 rule.
|
|
17
|
+
- `learn` lists lessons seen in 3+ Runs; a human promotes one with `learn promote <id> --check "<cmd>" --fix "<instruction>" --by <name>`. Default `verify` then runs it as an L1 rule. On a codebase that already has old violations use `--scope diff`: the check reads only added lines from `$AEL_ADDED_LINES_FILE`, so new ones fail and the old ones become a backlog (`doctor --harness`). Common cases need no script: `--forbid-import "src/renderer/** -> src/main/**"` or `--forbid-api "\buseEffect\(" --except "src/framework/**"`.
|
|
18
|
+
- `.ai-engineering-loop/feature-map.md` (Feature | Where | Paths | How to reach | Driver): `review-pack` adds the features a diff touches; use those names and drivers for L3 evidence.
|
|
18
19
|
- `doctor --harness`: harness debt (entry-file size and structure, stale learnings, rule provenance, skill budget). Report findings; do not mass-delete.
|
|
19
20
|
- `bench --without <integrity|l3|l1|smoke|rules>` measures whether a gate component still earns its cost (`docs bench`). Never use it on a real task.
|
|
20
21
|
|
|
@@ -28,7 +29,7 @@ Stage commands (`run`, `goal`, `gate`, `verify`, `review-pack`, `deliver`, `node
|
|
|
28
29
|
|
|
29
30
|
## Budget
|
|
30
31
|
|
|
31
|
-
`run --json` already reports the budget; run `budget status --run <runId>` only before each new iteration. After each child, `budget record --input <n> --output <n> --model <id> --run <runId>` from provider usage. No usage shown: `budget record --unavailable --run <runId>` once; ASSISTED continues and discloses it at delivery (do not ask the human); UNATTENDED fails that command, so stop and escalate. `budget pause` is the kill switch. The ledger mode is immutable. UNATTENDED or budget errors: `docs runtime-safety`.
|
|
32
|
+
`run --json` already reports the budget; run `budget status --run <runId>` only before each new iteration. After each child, `budget record --input <n> --output <n> --model <id> --role maker|reviewer|judge|verification --run <runId>` from provider usage. No usage shown: `budget record --unavailable --run <runId>` once; ASSISTED continues and discloses it at delivery (do not ask the human); UNATTENDED fails that command, so stop and escalate. `budget pause` is the kill switch. The ledger mode is immutable. UNATTENDED or budget errors: `docs runtime-safety`.
|
|
32
33
|
|
|
33
34
|
## Resuming
|
|
34
35
|
|
|
@@ -6,5 +6,5 @@
|
|
|
6
6
|
- Preview remote payloads; never include context packs, usage ledgers, raw logs, or credentials.
|
|
7
7
|
- ASSISTED: wait for explicit human approval and never infer it. Then one call: `npx ai-engineering-loop deliver --run <runId> --destination <where> --summary <text> --approve --by <name>` (approves READY approval nodes, writes `RUN_DIR/delivery.json` with `humanApproved: true`, runs `gate delivery`). Without `--approve` it refuses in ASSISTED.
|
|
8
8
|
- Long form: `node status --run <runId>`, `node approve delivery-approval --yes --by <name> --run <runId>` for each READY approval node, write `RUN_DIR/delivery.json`, then `gate delivery`. If the gate says a node is not READY, follow its prerequisite hint; never bypass approval.
|
|
9
|
-
- Commit before `deliver` (its receipt binds `HEAD`; screenshots: `--evidence AC-N=<https-url
|
|
9
|
+
- Commit before `deliver` (its receipt binds `HEAD`; videos/screenshots: `--evidence AC-N=<https-url>`, required for `crossComponent` and rubric ACs). Paste `RUN_DIR/receipt.md` into the PR/MR Receipt section unchanged; ❌ stays ❌ (`docs pr-receipt`).
|
|
10
10
|
- Only the delivery gate completes the run. It also mines lessons for the next Run.
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
- If the user prompt has no numbered AC, draft the contract (`docs goal-contract` if needed), list it, and wait. Chat agreement is not freeze.
|
|
6
6
|
- AC is a failure table: each row has a statement, required evidence, and failure cases. Do not freeze sunny-path-only. T0 may use 1-3 rows that still list failure cases.
|
|
7
7
|
- Boundary-crossing AC (UI-API, service-DB, queue, IPC): `crossComponent: true`; needs L3.
|
|
8
|
-
- Optional per AC: `verify.command` (Judge PASS needs it green), `rubric` for subjective ACs; Goal `featureId` links `features.json` (`docs goal-contract`).
|
|
8
|
+
- Optional per AC: `verify.command` (Judge PASS needs it green), `rubric` for subjective ACs; Goal `featureId` links `features.json`; Goal `flow` (from/event/to) for UI work: show its table with the draft (`docs goal-contract`).
|
|
9
9
|
- Write the contract JSON to a repository-local file, then `npx ai-engineering-loop goal draft --run <runId> --file <file> --revision <currentRevision>` (publishes the draft and the freeze question to Studio). Show the human the draft and its `impactPreview` in one message.
|
|
10
10
|
- Wait for the explicit human freeze, then `npx ai-engineering-loop goal freeze --latest --run <runId> --by <name>` (or `--revision <r> --hash <h>`). Add `--confirm-impact` when that answer also accepts the blast radius. T0: `quick start`.
|
|
11
11
|
- Otherwise the freeze writes a DRAFT blast-radius map: `impact show --run <runId>`. ASSISTED: the human confirms or edits it (`impact confirm --by <name> [--allow <glob>] [--remove <path>]`); copy/CSS/comment-only goals: `impact skip --by <name> --reason <text>` (`docs blast-radius-map`).
|
|
@@ -38,7 +38,7 @@ Report Spec and Standards as separate findings. Do not rerank one axis with the
|
|
|
38
38
|
|
|
39
39
|
**Standards** (`axis: "standards"`): `.ai-engineering-loop/conventions.md` plus the smell baseline below. Judgement calls: severity MEDIUM or LOW. Set `hardConvention: true` only when conventions.md states a hard rule that this hunk violates. A smell that hides an AC defect is Spec, not Standards.
|
|
40
40
|
|
|
41
|
-
Skip style nits unless they hide a defect.
|
|
41
|
+
Skip style nits unless they hide a defect. The CLI adds `comment-justification` findings for new suppressions and workaround comments: decide whether the root cause should be fixed instead; dismiss one only with evidence. Learnings in the pack carry `confidence`; never raise BLOCKER or HIGH only because a `tentative` (one-Run) learning was not followed.
|
|
42
42
|
|
|
43
43
|
Rubric ACs: when an AC in the Goal Contract has `rubric`, add `rubricScores: [{ac, criterion, pass, evidence}]` to the ledger, one per criterion. `evidence` must be observed (screenshot path, DOM snapshot, measured value), never an impression. A criterion you cannot observe is `pass: false`.
|
|
44
44
|
|
|
@@ -39,7 +39,7 @@ Report Spec and Standards as separate findings. Do not rerank one axis with the
|
|
|
39
39
|
|
|
40
40
|
**Standards** (`axis: "standards"`): `.ai-engineering-loop/conventions.md` plus the smell baseline below. Judgement calls: severity MEDIUM or LOW. Set `hardConvention: true` only when conventions.md states a hard rule that this hunk violates. A smell that hides an AC defect is Spec, not Standards.
|
|
41
41
|
|
|
42
|
-
Skip style nits unless they hide a defect.
|
|
42
|
+
Skip style nits unless they hide a defect. The CLI adds `comment-justification` findings for new suppressions and workaround comments: decide whether the root cause should be fixed instead; dismiss one only with evidence. Learnings in the pack carry `confidence`; never raise BLOCKER or HIGH only because a `tentative` (one-Run) learning was not followed.
|
|
43
43
|
|
|
44
44
|
Rubric ACs: when an AC in the Goal Contract has `rubric`, add `rubricScores: [{ac, criterion, pass, evidence}]` to the ledger, one per criterion. `evidence` must be observed (screenshot path, DOM snapshot, measured value), never an impression. A criterion you cannot observe is `pass: false`.
|
|
45
45
|
|
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,21 @@ All notable changes to this project are documented here. Versions follow [Semant
|
|
|
6
6
|
|
|
7
7
|
_No changes yet._
|
|
8
8
|
|
|
9
|
+
## 2.8.0 - 2026-11-16
|
|
10
|
+
|
|
11
|
+
From "All Code Is Slop (Except Mine)" (cards C-17..C-24 in `docs/cards/slop-to-gold-2026-11-16.md`).
|
|
12
|
+
|
|
13
|
+
- **Added:** ratchet rules: `learn promote --scope diff` judges only lines the change adds (`$AEL_ADDED_LINES_FILE`, `$AEL_CHANGED_FILES`), so a rule can be promoted on a codebase with old violations; `doctor --harness` reports each diff rule's legacy backlog (C-17).
|
|
14
|
+
- **Added:** suppressions (`eslint-disable`, `@ts-ignore`, `# noqa`, ...) and workaround comments added in product code become `comment-justification` review findings (HIGH without a reason, otherwise MEDIUM); reviewers may dismiss them only with evidence (C-18).
|
|
15
|
+
- **Changed:** learnings in context packs carry `confidence` (`tentative` 1 Run, `recurring` 2, `rule-candidate` 3+); a one-Run lesson reads "Consider ... not a rule" and the DA may not raise BLOCKER/HIGH on it alone (C-19).
|
|
16
|
+
- **Added:** built-in rule kinds: `learn promote --forbid-import "<fromGlob> -> <toGlob>"` and `--forbid-api "<regex>" [--except <glob>]`, run by `rules check <id>` on added lines only, reporting `file:line` (C-20).
|
|
17
|
+
- **Added:** verification catalog: `security`, `property`, `fuzz`, `mutation`, `coverage`, `benchmark`, `contract`, `pentest` and `formal` (new layer `LF`) are recognised in `verification.md` and mapped to layers; `property` runs in the default plan; an AC can require any key (`verify.key`) or layer (`verify.layer`), default `verify` runs required keys, and Judge PASS waits for them (C-23).
|
|
18
|
+
- **Added:** optional feature map (`.ai-engineering-loop/feature-map.md`): `review-pack` includes the user-facing features a diff touches with how to reach them and an optional driver command; `doctor --harness` warns when e2e/smoke exists without one (C-22).
|
|
19
|
+
- **Added:** Goal `flow` (from/event/to transitions) rendered as a table with dead-end warnings by `goal draft` and in the receipt; the receipt now opens with verification artifacts (video, screenshot, diagram) and marks crossComponent/rubric ACs without one as missing (C-21).
|
|
20
|
+
- **Added:** `stats` reports the revert rate of delivered Runs from git and the verification share of tokens; `budget record --role maker|reviewer|judge|verification`; `calibrate` counts detected reverts as outcomes (C-24).
|
|
21
|
+
- **Added:** Goal `flow` also renders as a Mermaid `stateDiagram-v2` in `goal draft` (`flow.mermaid`) and in the PR receipt; an AC requiring only a layer (e.g. `verify.layer: "LF"`) makes default `verify` run the first configured key of that layer; new `docs formal-statechart` recipe for model checking with pstack `/statechart` as the `formal` key (C-25).
|
|
22
|
+
- **Fixed:** the `verification.md` template no longer suggests `npx eslint --fix` and describes the current default plan.
|
|
23
|
+
|
|
9
24
|
## 2.7.0 - 2026-11-16
|
|
10
25
|
|
|
11
26
|
Harness gaps from a full review against learn-harness-engineering (cards C-01..C-16 in `docs/cards/harness-gaps-2026-11-16.md`).
|
|
@@ -31,7 +31,7 @@ Report Spec and Standards as separate findings. Do not rerank one axis with the
|
|
|
31
31
|
|
|
32
32
|
**Standards** (`axis: "standards"`): `.ai-engineering-loop/conventions.md` plus the smell baseline below. Judgement calls: severity MEDIUM or LOW. Set `hardConvention: true` only when conventions.md states a hard rule that this hunk violates. A smell that hides an AC defect is Spec, not Standards.
|
|
33
33
|
|
|
34
|
-
Skip style nits unless they hide a defect.
|
|
34
|
+
Skip style nits unless they hide a defect. The CLI adds `comment-justification` findings for new suppressions and workaround comments: decide whether the root cause should be fixed instead; dismiss one only with evidence. Learnings in the pack carry `confidence`; never raise BLOCKER or HIGH only because a `tentative` (one-Run) learning was not followed.
|
|
35
35
|
|
|
36
36
|
Rubric ACs: when an AC in the Goal Contract has `rubric`, add `rubricScores: [{ac, criterion, pass, evidence}]` to the ledger, one per criterion. `evidence` must be observed (screenshot path, DOM snapshot, measured value), never an impression. A criterion you cannot observe is `pass: false`.
|
|
37
37
|
|
|
@@ -1698,7 +1698,8 @@ function handleBudget() {
|
|
|
1698
1698
|
nodeId,
|
|
1699
1699
|
inputTokens: parseNonNegativeIntegerArg('--input', argValue('--input')),
|
|
1700
1700
|
outputTokens: parseNonNegativeIntegerArg('--output', argValue('--output')),
|
|
1701
|
-
model: argValue('--model')
|
|
1701
|
+
model: argValue('--model'),
|
|
1702
|
+
role: argValue('--role')
|
|
1702
1703
|
});
|
|
1703
1704
|
} else if (action === 'pause' || action === 'resume') {
|
|
1704
1705
|
result = setKillSwitch(CWD, action === 'pause');
|
|
@@ -3162,6 +3163,11 @@ function handleGoal() {
|
|
|
3162
3163
|
expectedRevision: requiredIntegerArg('--revision')
|
|
3163
3164
|
});
|
|
3164
3165
|
extra.impactPreview = impactPreview(runId, draftGoal);
|
|
3166
|
+
// C-21: show the intended user flow as a transition table before freeze.
|
|
3167
|
+
if (draftGoal && draftGoal.flow != null) {
|
|
3168
|
+
const flowLib = require('../lib/flow.js');
|
|
3169
|
+
extra.flow = { table: flowLib.renderFlow(draftGoal.flow), mermaid: flowLib.renderMermaid(draftGoal.flow), problems: flowLib.flowProblems(draftGoal.flow), warnings: flowLib.flowWarnings(draftGoal.flow) };
|
|
3170
|
+
}
|
|
3165
3171
|
appendRunLifecycle(CWD, runId, {
|
|
3166
3172
|
type: 'GOAL_DRAFT_UPDATED',
|
|
3167
3173
|
phase: 'goal',
|
|
@@ -3433,18 +3439,39 @@ async function handleVerify() {
|
|
|
3433
3439
|
// then L2 test_unit) and stops at the first failure.
|
|
3434
3440
|
const plan = lanes.defaultCommandPlan(CWD);
|
|
3435
3441
|
// C-05: promoted review rules run as L1 checks, after the static checks.
|
|
3436
|
-
const rules = require('../lib/rules.js').rulePlan(CWD);
|
|
3442
|
+
const rules = require('../lib/rules.js').rulePlan(CWD, loadRun(CWD, runId));
|
|
3437
3443
|
if (rules.length) {
|
|
3438
3444
|
const firstL2 = plan.findIndex((step) => step.layer !== 'L1');
|
|
3439
3445
|
plan.splice(firstL2 === -1 ? plan.length : firstL2, 0, ...rules);
|
|
3440
3446
|
}
|
|
3447
|
+
// C-23: catalog keys an AC requires (verify.key, e.g. pentest, formal)
|
|
3448
|
+
// run too when the project configures them.
|
|
3449
|
+
if (!baselineMode) {
|
|
3450
|
+
try {
|
|
3451
|
+
const target = loadRun(CWD, runId);
|
|
3452
|
+
const goal = require('../lib/claimed-vs-reality.js').loadFrozenGoal(CWD, target);
|
|
3453
|
+
const configured = lanes.verificationCommands(CWD);
|
|
3454
|
+
const acStatusLib = require('../lib/ac-status.js');
|
|
3455
|
+
for (const key of acStatusLib.requiredKeys(goal)) {
|
|
3456
|
+
if (configured[key] && !plan.some((step) => step.key === key)) plan.push({ key, layer: lanes.layerOfKey(key), text: configured[key] });
|
|
3457
|
+
}
|
|
3458
|
+
// A required layer with no step yet: the first configured catalog key of that layer.
|
|
3459
|
+
for (const layer of acStatusLib.requiredLayers(goal)) {
|
|
3460
|
+
if (plan.some((step) => step.layer === layer)) continue;
|
|
3461
|
+
const key = Object.keys(lanes.VERIFICATION_CATALOG).find((name) => lanes.VERIFICATION_CATALOG[name].layer === layer && configured[name]);
|
|
3462
|
+
if (key) plan.push({ key, layer, text: configured[key] });
|
|
3463
|
+
}
|
|
3464
|
+
} catch {
|
|
3465
|
+
// No frozen Goal yet; nothing extra to run.
|
|
3466
|
+
}
|
|
3467
|
+
}
|
|
3441
3468
|
// C-03: startup-risky diffs also run the configured smoke check (L2).
|
|
3442
3469
|
const smokeText = lanes.verificationCommands(CWD).smoke;
|
|
3443
3470
|
if (smokeText && !baselineMode && lanes.startupRiskFiles(CWD, loadRun(CWD, runId)).length) {
|
|
3444
3471
|
plan.push({ key: 'smoke', layer: 'L2', text: smokeText });
|
|
3445
3472
|
}
|
|
3446
3473
|
if (!plan.length) throw Object.assign(new Error('No --cmd given and no build/typecheck/lint/test_unit command in .ai-engineering-loop/verification.md. FIX: add them to verification.md or pass --cmd.'), { code: 'INVALID_COMMAND' });
|
|
3447
|
-
commandsToRun = plan.map((step) => ({ ...splitCommand(step.text, shell || Boolean(step.shell)), key: step.key, layer: step.layer }));
|
|
3474
|
+
commandsToRun = plan.map((step) => ({ ...splitCommand(step.text, shell || Boolean(step.shell)), key: step.key, layer: step.layer, ...(step.env ? { env: step.env } : {}) }));
|
|
3448
3475
|
defaultPlan = true;
|
|
3449
3476
|
} else {
|
|
3450
3477
|
commandsToRun = texts.map((text) => {
|
|
@@ -3518,8 +3545,9 @@ async function handleVerify() {
|
|
|
3518
3545
|
log.error('Verification failed; the gate was not attempted.');
|
|
3519
3546
|
for (const c of commands) console.log(`- ${c.command}: exit ${c.exitCode} (${c.timeoutStatus})`);
|
|
3520
3547
|
for (const skippedCommand of recorded.skipped) console.log(`- ${skippedCommand}: skipped (an earlier, cheaper check failed)`);
|
|
3521
|
-
for (const c of commands.filter((item) => item.exitCode !== 0)) {
|
|
3522
|
-
const
|
|
3548
|
+
for (const c of recorded.commands.filter((item) => item.exitCode !== 0)) {
|
|
3549
|
+
const rulesLib = require('../lib/rules.js');
|
|
3550
|
+
const rule = (c.key && c.key.startsWith('rule:') ? rulesLib.findRule(CWD, c.key.slice(5)) : null) || rulesLib.ruleForCommand(CWD, c.command);
|
|
3523
3551
|
if (rule) console.log(`! Rule ${rule.id} violated. WHY: ${rule.why} FIX: ${rule.fix}`);
|
|
3524
3552
|
}
|
|
3525
3553
|
for (const f of failures) {
|
|
@@ -3660,6 +3688,24 @@ function handleNext() {
|
|
|
3660
3688
|
}
|
|
3661
3689
|
}
|
|
3662
3690
|
|
|
3691
|
+
// C-20: `rules check <id>` runs a built-in rule kind against the lines the
|
|
3692
|
+
// change adds ($AEL_ADDED_LINES_FILE); violations print as file:line.
|
|
3693
|
+
function handleRules() {
|
|
3694
|
+
const json = process.argv.includes('--json');
|
|
3695
|
+
try {
|
|
3696
|
+
if (args[1] !== 'check' || !args[2]) throw Object.assign(new Error('rules supports: check <ruleId>'), { code: 'INVALID_USAGE' });
|
|
3697
|
+
const rule = require('../lib/rules.js').findRule(CWD, args[2]);
|
|
3698
|
+
if (!rule) throw Object.assign(new Error(`No rule ${args[2]} in .ai-engineering-loop/rules.json`), { code: 'NOT_FOUND' });
|
|
3699
|
+
const ruleCheck = require('../lib/rule-check.js');
|
|
3700
|
+
const violations = ruleCheck.checkRule(rule, ruleCheck.readAddedLines());
|
|
3701
|
+
if (json) console.log(JSON.stringify({ ok: violations.length === 0, rule: rule.id, violations }));
|
|
3702
|
+
else for (const v of violations) console.log(`${v.file}:${v.line}: ${v.message}`);
|
|
3703
|
+
if (violations.length) process.exitCode = 1;
|
|
3704
|
+
} catch (err) {
|
|
3705
|
+
stageJsonError(err, json, 'RULES_FAILED');
|
|
3706
|
+
}
|
|
3707
|
+
}
|
|
3708
|
+
|
|
3663
3709
|
function handleStats() {
|
|
3664
3710
|
const json = process.argv.includes('--json');
|
|
3665
3711
|
try {
|
|
@@ -3820,6 +3866,7 @@ function handleReviewPack() {
|
|
|
3820
3866
|
const tier = run.tier ? run.tier.current : null;
|
|
3821
3867
|
// C-14: open debt on the files this change touches goes into the pack.
|
|
3822
3868
|
let openDebt = [];
|
|
3869
|
+
let touchedFeatures = [];
|
|
3823
3870
|
try {
|
|
3824
3871
|
const changed = require('../lib/verification-lanes.js').relatedFiles(CWD, run);
|
|
3825
3872
|
openDebt = require('../lib/debt.js').openDebtFor(CWD, changed);
|
|
@@ -3827,6 +3874,10 @@ function handleReviewPack() {
|
|
|
3827
3874
|
const lines = ['', '## Open debt on changed files', '', ...openDebt.map((item) => `- ${item.id} ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; risk ${item.risk})`)];
|
|
3828
3875
|
fs.appendFileSync(path.join(CWD, pack.path), `${lines.join('\n')}\n`);
|
|
3829
3876
|
}
|
|
3877
|
+
// C-22: user-facing features this diff touches, with how to reach them.
|
|
3878
|
+
const featureMap = require('../lib/feature-map.js');
|
|
3879
|
+
touchedFeatures = featureMap.featuresForFiles(CWD, changed);
|
|
3880
|
+
if (touchedFeatures.length && pack.path) fs.appendFileSync(path.join(CWD, pack.path), `${featureMap.renderForPack(touchedFeatures)}\n`);
|
|
3830
3881
|
} catch {
|
|
3831
3882
|
openDebt = [];
|
|
3832
3883
|
}
|
|
@@ -3846,6 +3897,7 @@ function handleReviewPack() {
|
|
|
3846
3897
|
unresolvedFindings: pack.unresolvedFindings,
|
|
3847
3898
|
impactDrift: pack.impactDrift,
|
|
3848
3899
|
openDebt: openDebt.map((item) => ({ id: item.id, location: item.location, debt: item.debt })),
|
|
3900
|
+
features: touchedFeatures.map((entry) => ({ id: entry.id, where: entry.where, driver: entry.driver })),
|
|
3849
3901
|
reviewInputs: { inputs: reviewInputs.inputs, promptSha256: reviewInputs.promptSha256 },
|
|
3850
3902
|
fastPath: { eligible: fastPath.eligible, reasons: fastPath.reasons },
|
|
3851
3903
|
escalation: { requiredTier: escalation.requiredTier, reasons: escalation.reasons.map((r) => r.code) }
|
|
@@ -3860,6 +3912,7 @@ function handleReviewPack() {
|
|
|
3860
3912
|
console.log(`- Outside the impact map: ${result.impactDrift.outOfMap.join(', ')} (check their side effects)`);
|
|
3861
3913
|
}
|
|
3862
3914
|
for (const line of result.impactDrift?.design || []) console.log(`- ${line}`);
|
|
3915
|
+
if (touchedFeatures.length) console.log(`- Features touched: ${touchedFeatures.map((entry) => entry.id).join(', ')} (in the pack; use them for L3 evidence)`);
|
|
3863
3916
|
if (openDebt.length) console.log(`- Open debt on changed files: ${openDebt.map((item) => item.id).join(', ')} (in the pack; resolve or keep consciously)`);
|
|
3864
3917
|
}
|
|
3865
3918
|
} catch (err) {
|
|
@@ -4134,6 +4187,10 @@ function handleLearn() {
|
|
|
4134
4187
|
fix: argValue('--fix'),
|
|
4135
4188
|
why: argValue('--why'),
|
|
4136
4189
|
by: argValue('--by'),
|
|
4190
|
+
scope: argValue('--scope') || 'repo',
|
|
4191
|
+
forbidImport: argValue('--forbid-import'),
|
|
4192
|
+
forbidApi: argValue('--forbid-api'),
|
|
4193
|
+
except: argValue('--except'),
|
|
4137
4194
|
shell: !process.argv.includes('--no-shell')
|
|
4138
4195
|
});
|
|
4139
4196
|
if (json) console.log(JSON.stringify({ ok: true, rule }));
|
|
@@ -4346,7 +4403,10 @@ Commands:
|
|
|
4346
4403
|
writes RUN_DIR/receipt.json + receipt.md (paste into the PR/MR body); digest stored in delivery.json
|
|
4347
4404
|
verify-receipt Re-check a delivery receipt: artifact hashes, frozen Goal, AC evidence, verdict, head SHA
|
|
4348
4405
|
[--run <id> | <repository-relative-receipt.json>] [--head <sha>] [--json] exit 1 on mismatch (CI)
|
|
4349
|
-
learn promote <learningId> --check "<command>" --fix "<instruction>" --by <name> [--why <text>] [--no-shell]
|
|
4406
|
+
learn promote <learningId> --check "<command>" --fix "<instruction>" --by <name> [--why <text>] [--scope repo|diff] [--no-shell]
|
|
4407
|
+
--scope diff: the check sees only lines this change adds ($AEL_ADDED_LINES_FILE, $AEL_CHANGED_FILES)
|
|
4408
|
+
built-in instead of --check: --forbid-import "<fromGlob> -> <toGlob>" | --forbid-api "<regex>" [--except <glob>]
|
|
4409
|
+
rules check <ruleId> run a built-in rule on the added lines (used by verify)
|
|
4350
4410
|
turn a recurring review lesson into an executable L1 rule (.ai-engineering-loop/rules.json)
|
|
4351
4411
|
learn Mine past Runs' VALID findings into .ai-engineering-loop/learnings.json (+ .md) for context packs
|
|
4352
4412
|
[--dry-run] [--prune] [--max-age-runs <n>] [--json]
|
|
@@ -4525,6 +4585,9 @@ switch (command) {
|
|
|
4525
4585
|
case 'stats':
|
|
4526
4586
|
handleStats();
|
|
4527
4587
|
break;
|
|
4588
|
+
case 'rules':
|
|
4589
|
+
handleRules();
|
|
4590
|
+
break;
|
|
4528
4591
|
case 'next':
|
|
4529
4592
|
handleNext();
|
|
4530
4593
|
break;
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Formal flows with pstack `/statechart`
|
|
2
|
+
|
|
3
|
+
How to prove a risky user flow with a model checker and feed the result into AEL's gates. AEL ships none of this tooling; it only recognises the evidence (card C-25).
|
|
4
|
+
|
|
5
|
+
## When it is worth it
|
|
6
|
+
|
|
7
|
+
Use it for flows where a wrong state is expensive and hard to see in unit tests: payments and refunds, sync across devices, retries with leases, notification fan-out, multi-step approvals. For an ordinary screen flow, the Goal `flow` diagram (below) is enough.
|
|
8
|
+
|
|
9
|
+
## The tool
|
|
10
|
+
|
|
11
|
+
[`statechart`](https://github.com/cursor/plugins/tree/main/pstack/skills/statechart) is a skill in the pstack plugin by Lauren Tan (MIT). You write a TLA+ spec in which each rule the reader agrees to is an `INVARIANT`. TLC model-checks it, and the skill draws the result as a Mermaid statechart. Its report gives the shortest path to every state that breaks a rule.
|
|
12
|
+
|
|
13
|
+
Requirements: Java and TLC (installed with `mise` on first run, which needs the network) and a runtime that executes TypeScript. Install pstack as a Cursor plugin, or follow its Claude Code guide (`docs/harness/claude-code.md` in pstack) and call it as `/pstack:statechart`. It runs only when asked (`disable-model-invocation`).
|
|
14
|
+
|
|
15
|
+
## Wire it into AEL
|
|
16
|
+
|
|
17
|
+
1. Keep the spec in the repository, for example `spec/checkout.tla` and `spec/checkout.cfg`. Keep constants small, or add a `CONSTRAINT`: TLC keeps going past a broken rule, so an unbounded model can run without end.
|
|
18
|
+
2. `statechart` requires its `--out` folder to be new or empty, so a fixed folder fails the second `verify` (exit 2). Wrap it in a small script that clears the folder first, and add the folder to `.gitignore` (otherwise delivery reports it as an untracked file, C-06):
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
#!/usr/bin/env sh
|
|
22
|
+
# scripts/formal.sh
|
|
23
|
+
set -e
|
|
24
|
+
rm -rf .tmp/statechart
|
|
25
|
+
exec "$PSTACK/skills/statechart/scripts/statechart" spec/checkout.tla --config spec/checkout.cfg --out .tmp/statechart
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
3. Register the wrapper as the `formal` key in `.ai-engineering-loop/verification.md`. `formal` is layer `LF` and never runs in the default plan:
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
- **formal**: `sh scripts/formal.sh`
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The statechart command exits with TLC's code: 0 when every rule holds, 1 when a rule breaks, 2 when TLC stops with an error. So a broken rule fails the verification gate like any other failing command.
|
|
35
|
+
4. Require it on the AC that the model covers:
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{ "id": "AC-2", "statement": "A refund never leaves the order both paid and refunded", "evidenceRequired": "TLC model check", "failureCases": ["double refund"], "verify": { "layer": "LF" } }
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Default `verify` then runs `formal` (C-23), and `gate judge` refuses PASS until it passed on the gated diff (`AC_NOT_PROVEN`).
|
|
42
|
+
5. Attach the chart to the PR receipt: publish `chart.mmd` (or a rendered image) and pass `deliver --evidence AC-2=<https-url>`. The receipt lists it under "Verification artifacts" as a diagram.
|
|
43
|
+
6. When the model describes code, replay each bug path from the report against the real code, as a test or a live repro. A path that does not reproduce means the model is wrong until shown otherwise.
|
|
44
|
+
|
|
45
|
+
## Without TLA+: the Goal `flow` diagram
|
|
46
|
+
|
|
47
|
+
A Goal Contract `flow` (`[{ "from", "event", "to" }]`, see `docs goal-contract`) is rendered by `goal draft` and in the PR receipt as a Mermaid `stateDiagram-v2` plus a `From | Event | To` table. GitHub renders the diagram in the PR. It lets a reviewer check that the agent understood the intended flow; it proves nothing about the code. For proof, use `formal` as above.
|
|
48
|
+
|
|
49
|
+
Skill text never contains Mermaid (some hosts cannot render it); diagrams appear only in command output and PR receipts.
|
package/core/goal-contract.md
CHANGED
|
@@ -148,3 +148,21 @@ Optional `.ai-engineering-loop/features.json` lists what the project must do, on
|
|
|
148
148
|
|
|
149
149
|
Status moves only through Runs: a Goal Contract with `"featureId": "F-1"` makes it `active` at the Goal gate, and delivery makes it `passing` with the Run as evidence (passing is final). `ael next` prints the highest-priority unblocked feature, VCR (verified / activated), and flags any `passing` feature without a delivered Run behind it. `run --new` warns when an unfinished Run exists and VCR is below 1 (WIP=1).
|
|
150
150
|
|
|
151
|
+
## Intended flow (C-21)
|
|
152
|
+
|
|
153
|
+
For UI or multi-step work, add the flow the user should experience as transitions:
|
|
154
|
+
|
|
155
|
+
```json
|
|
156
|
+
"flow": [
|
|
157
|
+
{ "from": "List", "event": "click Export", "to": "Exporting" },
|
|
158
|
+
{ "from": "Exporting", "event": "file ready", "to": "Done" },
|
|
159
|
+
{ "from": "Exporting", "event": "disk full", "to": "Error" }
|
|
160
|
+
]
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
`goal draft` returns it as a `From | Event | To` table and a Mermaid `stateDiagram-v2` (`flow.mermaid`), with warnings (a state nothing reaches, a state with no way out that is not Done/Error), so the human checks the intent before freeze. A malformed `flow` fails the Goal gate. The PR receipt shows the same diagram (GitHub renders it) and table under "Intended flow", after "Verification artifacts" and before the acceptance criteria.
|
|
164
|
+
|
|
165
|
+
## Verification artifacts in the receipt (C-21)
|
|
166
|
+
|
|
167
|
+
The receipt lists `--evidence AC-N=<https-url>` links first, typed as video, screenshot or diagram from the URL. An AC with `crossComponent: true` or a `rubric` is `MISSING` (❌) until it has at least one such link, because a reviewer should be able to look at the behaviour rather than read the diff.
|
|
168
|
+
|
|
@@ -78,6 +78,10 @@ Enforcement (`lib/claimed-vs-reality.js`, `gate review`):
|
|
|
78
78
|
|
|
79
79
|
**Default plan (C-02).** `verify` without `--cmd` runs `build -> typecheck -> lint -> test_unit` (whichever are configured), serially, and stops at the first failure. Commands that rewrite files (`--fix`, `--write`) are never run by default. Each record carries its layer (`L1`/`L2`/`L3`).
|
|
80
80
|
|
|
81
|
+
**Verification catalog (C-23).** AEL does not ship test tools; it recognises the ones a project has. Any of these keys in `verification.md` is mapped to a layer and recorded with it: `security` (L1); `property`, `fuzz`, `mutation`, `coverage`, `benchmark` (L2); `contract`, `pentest` (L3); `formal` (LF, a separate layer for model checking and proofs). Only `property` joins the default plan (after `test_unit`); the rest run when an AC requires them with `"verify": { "key": "<key>" }`, or with `--cmd`. An AC that requires only a layer (`"layer": "LF"`) makes default `verify` add the first configured catalog key of that layer when no step covers it (for `LF`, `formal`; see `docs formal-statechart` for model checking with pstack `/statechart`). `gate judge` refuses PASS until each required key passed on the gated diff (`AC_NOT_PROVEN`). Claimed-vs-reality accepts `LF` as a Layer tag. Testing in production (canary, shadowing, monitoring) stays outside AEL; its outcomes come back through `outcome record` (C-13).
|
|
82
|
+
|
|
83
|
+
**Feature map (C-22).** Optional `.ai-engineering-loop/feature-map.md` is a site map of what users can do, one Markdown table: `| Feature | Where (UI or endpoint) | Paths (globs) | How to reach | Driver |`. `review-pack` appends the entries whose `Paths` the diff touches, so the Devil's Advocate and L3 evidence use the same names ("export-csv", not "the button on the left"). `Driver` is an optional reusable command that runs the flow (cheaper and more repeatable than an agent clicking around). `doctor --harness` warns when `e2e` or `smoke` is configured but the map is empty.
|
|
84
|
+
|
|
81
85
|
**L1 at T1/T2.** When `verification.md` configures `typecheck` or `lint`, the verification gate at T1/T2 refuses a full-scope bundle with no L1 record (`L1_MISSING`). The related lane is exempt; `verify --final` runs the full plan.
|
|
82
86
|
|
|
83
87
|
`init` writes only commands the project really has: `typecheck` from a script or a TypeScript setup, `lint` and `build` only from package scripts.
|
|
@@ -94,6 +98,14 @@ At `gate review` the CLI scans the gated diff and adds `source: "verification-in
|
|
|
94
98
|
|
|
95
99
|
These findings cannot be dropped or downgraded by the reviewer. Only a human dismissal (`dismissal: {by, evidence}`) can downgrade them; otherwise the review gate fails.
|
|
96
100
|
|
|
101
|
+
## 3c. Justification Comments and Suppressions (C-18)
|
|
102
|
+
|
|
103
|
+
Agents write comments to justify not doing the right thing, and comments cannot be enforced. At `gate review` the CLI scans comments added in non-test product files and adds `source: "comment-justification"` findings (`axis: "standards"`):
|
|
104
|
+
- a suppression (`eslint-disable*`, `@ts-ignore`, `@ts-expect-error`, `# type: ignore`, `# noqa`, `//nolint`, `NOSONAR`, `istanbul/c8 ignore`): **HIGH** without a reason, **MEDIUM** with one (eslint `-- reason`, or text after the directive);
|
|
105
|
+
- a comment that justifies a workaround (`workaround`, `quick fix`, `hack`, `temporary`, `for now`, `band-aid`, `fix later`, ...): **MEDIUM**. Projects add patterns in `.ai-engineering-loop/rules.json` as `"justificationPatterns": ["<regex>"]`.
|
|
106
|
+
|
|
107
|
+
Docs, changelogs, data files and tests are skipped. Unlike C-11, a reviewer may downgrade these with `dismissal: {by, evidence}` (for example generated code); dropping or downgrading one without evidence fails the review gate.
|
|
108
|
+
|
|
97
109
|
## 4. The 4 Verification Gates
|
|
98
110
|
|
|
99
111
|
1. **Gate 1: Unit & Integration Tests**: All unit and edge-case tests must pass with 0 failures.
|
|
@@ -160,6 +172,14 @@ At `gate delivery` the CLI copies two kinds of deferred work into `.ai-engineeri
|
|
|
160
172
|
|
|
161
173
|
"Model-visible means logged": `review-pack` writes `runs/<id>/review-inputs.json` with the sha256 of the review pack, the gated diff, and the Devil's Advocate prompt body. `review ingest` copies them into `findings.json` (`reviewer.inputs`, `reviewer.promptSha256`). `gate review` fails the ledger binding when any recorded input changed after the reviewer saw it, so a missed defect can always be traced to exactly what the reviewer was shown.
|
|
162
174
|
|
|
175
|
+
## 9b. Quality Anchors from Git and the Budget Ledger (C-24)
|
|
176
|
+
|
|
177
|
+
`ael stats` also reports:
|
|
178
|
+
- **Revert rate**: git `Revert "<subject>"` commits matched to DELIVERED Runs, by the reverted sha (the receipt's head) or by subject (squash merges keep it). A stable, low revert rate while throughput rises is the signal that quality held.
|
|
179
|
+
- **Verification token share**: tokens recorded with `budget record --role reviewer|judge|verification` over all role-labelled tokens (`maker`, `other` are the rest).
|
|
180
|
+
|
|
181
|
+
`calibrate` treats each detected revert as an outcome (`by: git`) unless one was already recorded by hand.
|
|
182
|
+
|
|
163
183
|
## 10. Reviewer Calibration (C-13)
|
|
164
184
|
|
|
165
185
|
Gates prove the change matches the Goal; only reality proves the reviewers were right. After delivery, record problems people found:
|
|
@@ -181,3 +201,11 @@ ael learn promote L-abc123 --check "! grep -rq \"require('fs')\" src/renderer" -
|
|
|
181
201
|
|
|
182
202
|
Rules live in `.ai-engineering-loop/rules.json` (tracked). Default `verify` runs every rule as an L1 check after the static checks; a violation prints `Rule <id> violated. WHY: ... FIX: ...`. The verification gate refuses a full-scope bundle in which any rule has no passing record (`RULES_MISSING`). The check must exit non-zero on a violation and is run through the shell unless `--no-shell` is given.
|
|
183
203
|
|
|
204
|
+
**Ratchet rules (C-17).** `--scope diff` judges only what the change adds, so a rule can be promoted on a codebase that still has old violations. Its check receives `AEL_CHANGED_FILES` (one path per line) and `AEL_ADDED_LINES_FILE` (`path:line:text` for every added line, the `grep -n` format), for example `! grep -E "useEffect\(" "$AEL_ADDED_LINES_FILE"` (POSIX shell) or a small Node script for Windows. New occurrences fail with their `file:line`; old ones do not. `doctor --harness` runs each diff rule against every tracked line and reports the remaining legacy backlog, and says when it is clean enough to switch the rule to `--scope repo`.
|
|
205
|
+
|
|
206
|
+
**Built-in rule kinds (C-20).** The two most common corrections need no hand-written check:
|
|
207
|
+
- `--forbid-import "<fromGlob> -> <toGlob>"`: files under `fromGlob` may not import modules under `toGlob`. Relative specifiers are resolved against the importing file (with common extensions); bare names such as `fs` or `react` match as written, so `"src/renderer/** -> fs"` forbids Node's `fs` in the renderer.
|
|
208
|
+
- `--forbid-api "<regex>" [--except "<glob>"]`: the API may appear only inside the excepted paths, for example `useEffect` only in `src/framework/**`.
|
|
209
|
+
|
|
210
|
+
Both are stored with `kind`, always use `scope: "diff"`, and run as `rules check <ruleId>` through the same CLI (so `rules.json` holds no machine-specific path). Violations print as `file:line: <message>` and the rule's FIX.
|
|
211
|
+
|
package/lib/ablation.js
CHANGED
|
@@ -10,7 +10,7 @@ const fs = require('fs');
|
|
|
10
10
|
const path = require('path');
|
|
11
11
|
const { runsDir } = require('./run-state.js');
|
|
12
12
|
|
|
13
|
-
const COMPONENTS = Object.freeze(['integrity', 'l3', 'l1', 'smoke', 'rules']);
|
|
13
|
+
const COMPONENTS = Object.freeze(['integrity', 'comments', 'l3', 'l1', 'smoke', 'rules']);
|
|
14
14
|
const MARKER = path.join('.ai-engineering-loop', 'bench-ablation.json');
|
|
15
15
|
|
|
16
16
|
function requested(env = process.env) {
|
package/lib/ac-status.js
CHANGED
|
@@ -24,20 +24,52 @@ function recordedCommands(rootDir, runId) {
|
|
|
24
24
|
}
|
|
25
25
|
|
|
26
26
|
// [{ id, command, status: 'proven'|'failed'|'pending'|'unbound' }]
|
|
27
|
+
// `verify` may name an exact `command`, a catalog `key` (C-23, e.g.
|
|
28
|
+
// "pentest"), or a `layer` (e.g. "LF"). Records carry `key`/`layer` when
|
|
29
|
+
// annotated by runAcStatuses.
|
|
27
30
|
function acStatuses(goal, records) {
|
|
28
31
|
return (goal?.acceptanceCriteria || []).map((criterion) => {
|
|
29
|
-
const
|
|
30
|
-
|
|
31
|
-
|
|
32
|
+
const verify = criterion.verify || {};
|
|
33
|
+
const command = norm(verify.command);
|
|
34
|
+
let label;
|
|
35
|
+
let matches;
|
|
36
|
+
if (command) {
|
|
37
|
+
label = command;
|
|
38
|
+
matches = (records || []).filter((record) => norm(record.command) === command);
|
|
39
|
+
} else if (verify.key) {
|
|
40
|
+
label = `key:${verify.key}`;
|
|
41
|
+
matches = (records || []).filter((record) => record.key === verify.key);
|
|
42
|
+
} else if (verify.layer) {
|
|
43
|
+
label = `layer:${verify.layer}`;
|
|
44
|
+
matches = (records || []).filter((record) => record.layer === verify.layer);
|
|
45
|
+
} else {
|
|
46
|
+
return { id: criterion.id, command: null, status: 'unbound' };
|
|
47
|
+
}
|
|
32
48
|
if (matches.some((record) => record.exitCode === 0 && record.timeoutStatus === 'COMPLETED')) {
|
|
33
|
-
return { id: criterion.id, command, status: 'proven' };
|
|
49
|
+
return { id: criterion.id, command: label, status: 'proven' };
|
|
34
50
|
}
|
|
35
|
-
return { id: criterion.id, command, status: matches.length ? 'failed' : 'pending' };
|
|
51
|
+
return { id: criterion.id, command: label, status: matches.length ? 'failed' : 'pending' };
|
|
36
52
|
});
|
|
37
53
|
}
|
|
38
54
|
|
|
39
55
|
function runAcStatuses(rootDir, run, goal) {
|
|
40
|
-
|
|
56
|
+
const lanes = require('./verification-lanes.js');
|
|
57
|
+
const records = recordedCommands(rootDir, run.runId).map((record) => {
|
|
58
|
+
const key = record.key || lanes.keyOfCommand(rootDir, record.command);
|
|
59
|
+
return { ...record, key, layer: record.layer || (key ? lanes.layerOfKey(key) : null) };
|
|
60
|
+
});
|
|
61
|
+
return acStatuses(goal, records);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Catalog keys the frozen Goal's ACs require (C-23), so default verify runs them.
|
|
65
|
+
function requiredKeys(goal) {
|
|
66
|
+
return [...new Set((goal?.acceptanceCriteria || []).map((criterion) => criterion.verify?.key).filter(Boolean))];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Layers the frozen Goal's ACs require without naming a key (e.g. "LF").
|
|
70
|
+
function requiredLayers(goal) {
|
|
71
|
+
return [...new Set((goal?.acceptanceCriteria || []).filter((criterion) => !criterion.verify?.key && !criterion.verify?.command)
|
|
72
|
+
.map((criterion) => criterion.verify?.layer).filter(Boolean))];
|
|
41
73
|
}
|
|
42
74
|
|
|
43
75
|
// C-09 (learn-harness-engineering L09/L11): subjective ACs (UI, copy,
|
|
@@ -60,4 +92,4 @@ function rubricProblems(goal, ledger) {
|
|
|
60
92
|
return problems;
|
|
61
93
|
}
|
|
62
94
|
|
|
63
|
-
module.exports = { acStatuses, recordedCommands, rubricProblems, runAcStatuses };
|
|
95
|
+
module.exports = { acStatuses, recordedCommands, requiredKeys, requiredLayers, rubricProblems, runAcStatuses };
|
package/lib/budget.js
CHANGED
|
@@ -72,6 +72,7 @@ function readUsageFile(filePath) {
|
|
|
72
72
|
if (entry.nodeId != null && !/^[a-z][a-z0-9-]{0,63}$/.test(entry.nodeId)) {
|
|
73
73
|
throw new Error('nodeId is invalid');
|
|
74
74
|
}
|
|
75
|
+
if (entry.role != null && !USAGE_ROLES.includes(entry.role)) throw new Error('role is invalid');
|
|
75
76
|
if (typeof entry.recordedAt !== 'string' || Number.isNaN(Date.parse(entry.recordedAt))) {
|
|
76
77
|
throw new Error('recordedAt is invalid');
|
|
77
78
|
}
|
|
@@ -248,10 +249,15 @@ function appendPrivateLine(rootDir, filePath, line) {
|
|
|
248
249
|
}
|
|
249
250
|
}
|
|
250
251
|
|
|
252
|
+
// C-24: who spent the tokens, so `stats` can show the verification share
|
|
253
|
+
// ("70% of her daily tokens go into verification", talk 11:46).
|
|
254
|
+
const USAGE_ROLES = Object.freeze(['maker', 'reviewer', 'judge', 'verification', 'other']);
|
|
255
|
+
|
|
251
256
|
function recordTokenUsage(
|
|
252
257
|
rootDir,
|
|
253
|
-
{ runId, nodeId, inputTokens, outputTokens, model, now = new Date() } = {}
|
|
258
|
+
{ runId, nodeId, inputTokens, outputTokens, model, role = null, now = new Date() } = {}
|
|
254
259
|
) {
|
|
260
|
+
if (role != null && !USAGE_ROLES.includes(role)) throw usageError(`role must be one of ${USAGE_ROLES.join(', ')}`);
|
|
255
261
|
const run = resolveRun(rootDir, runId);
|
|
256
262
|
validateTokenCount('inputTokens', inputTokens);
|
|
257
263
|
validateTokenCount('outputTokens', outputTokens);
|
|
@@ -266,6 +272,7 @@ function recordTokenUsage(
|
|
|
266
272
|
totalTokens,
|
|
267
273
|
model: normalizedModel,
|
|
268
274
|
...(nodeId ? { nodeId } : {}),
|
|
275
|
+
...(role ? { role } : {}),
|
|
269
276
|
recordedAt: now.toISOString()
|
|
270
277
|
};
|
|
271
278
|
if (nodeId) budgetStatus(rootDir, { runId: run.runId, nodeId, now });
|
|
@@ -341,6 +348,7 @@ function readUnavailableUsage(rootDir) {
|
|
|
341
348
|
}
|
|
342
349
|
|
|
343
350
|
module.exports = {
|
|
351
|
+
USAGE_ROLES,
|
|
344
352
|
usageDir,
|
|
345
353
|
utcDay,
|
|
346
354
|
usagePath,
|
package/lib/calibration.js
CHANGED
|
@@ -84,6 +84,23 @@ function calibrate(rootDir) {
|
|
|
84
84
|
bucket[result.verdict] += 1;
|
|
85
85
|
}
|
|
86
86
|
}
|
|
87
|
+
// C-24: reverts found in git count as outcomes too, unless already recorded.
|
|
88
|
+
for (const item of require('./reverts.js').detectReverts(rootDir).reverted) {
|
|
89
|
+
if (rows.some((row) => row.runId === item.runId && row.source === 'revert')) continue;
|
|
90
|
+
let ledger = null;
|
|
91
|
+
try {
|
|
92
|
+
ledger = JSON.parse(fs.readFileSync(path.join(runsDir(rootDir), item.runId, 'findings.json'), 'utf8'));
|
|
93
|
+
} catch {
|
|
94
|
+
ledger = null;
|
|
95
|
+
}
|
|
96
|
+
const outcome = { source: 'revert', location: item.location, summary: `reverted by ${String(item.revertSha).slice(0, 12)}`, by: 'git' };
|
|
97
|
+
const result = classifyOutcome(outcome, ledger?.findings);
|
|
98
|
+
const prompt = ledger?.reviewer?.promptSha256 ? ledger.reviewer.promptSha256.slice(0, 12) : 'unknown';
|
|
99
|
+
rows.push({ runId: item.runId, ...outcome, ...result, prompt });
|
|
100
|
+
const bucket = byPrompt[prompt] || (byPrompt[prompt] = { outcomes: 0, DA_MISS: 0, FALSE_DISMISS: 0, CAUGHT_BUT_SHIPPED: 0 });
|
|
101
|
+
bucket.outcomes += 1;
|
|
102
|
+
bucket[result.verdict] += 1;
|
|
103
|
+
}
|
|
87
104
|
const count = (verdict) => rows.filter((row) => row.verdict === verdict).length;
|
|
88
105
|
return {
|
|
89
106
|
outcomes: rows.length,
|
|
@@ -161,7 +161,7 @@ function parseClaimedVsReality(markdown) {
|
|
|
161
161
|
// L1 static (typecheck/lint/build), L2 runtime (unit/related/all tests),
|
|
162
162
|
// L3 end-to-end (e2e or the AC's real user flow). Unit tests isolate and mock
|
|
163
163
|
// by design, so they cannot prove a cross-component AC.
|
|
164
|
-
const LAYER_TOKEN = /^L[
|
|
164
|
+
const LAYER_TOKEN = /^L[123F]$/; // LF = formal (C-23)
|
|
165
165
|
const WAIVER = /^L3\s+waived\s*:\s*(.+)$/i;
|
|
166
166
|
const MIN_WAIVER_REASON = 15;
|
|
167
167
|
|
package/lib/cli-args.js
CHANGED
|
@@ -30,7 +30,7 @@ const WRITER_COMMANDS = Object.freeze({
|
|
|
30
30
|
verify: { flags: ['--json', '--serial', '--shell', '--related', '--final', '--fresh', '--baseline'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
|
|
31
31
|
quick: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms', '--mode'], positionals: null },
|
|
32
32
|
review: { flags: ['--json'], valued: ['--run', '--child-id', '--file'], positionals: ['ingest'] },
|
|
33
|
-
learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by'], positionals: null },
|
|
33
|
+
learn: { flags: ['--json', '--dry-run', '--prune', '--no-shell'], valued: ['--max-age-runs', '--check', '--fix', '--why', '--by', '--scope', '--forbid-import', '--forbid-api', '--except'], positionals: null },
|
|
34
34
|
guard: { flags: ['--json'], valued: ['--run', '--tool', '--input', '--path', '--command', '--hook'], positionals: ['check'] },
|
|
35
35
|
hooks: { flags: ['--json'], valued: ['--host'], positionals: ['install', 'uninstall', 'status'] },
|
|
36
36
|
impact: { flags: ['--json', '--force'], valued: ['--run', '--by', '--reason', '--allow', '--remove', '--limit'], positionals: ['map', 'show', 'confirm', 'skip', 'drift'] },
|