ai-engineering-loop 2.6.0 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/.agents/skills/ael/SKILL.md +1 -1
  2. package/.agents/skills/ael/references/loop-commands.md +13 -1
  3. package/.agents/skills/ael/references/loop-deliver.md +1 -1
  4. package/.agents/skills/ael/references/loop-goal.md +2 -0
  5. package/.agents/skills/ael/references/loop-maker.md +1 -6
  6. package/.agents/skills/ael/references/loop-report.md +8 -0
  7. package/.agents/skills/ael/references/loop-verify.md +11 -9
  8. package/.claude/agents/devil-advocate.md +7 -2
  9. package/.claude/agents/judge.md +3 -1
  10. package/.grok/agents/devil-advocate.md +7 -2
  11. package/.grok/agents/judge.md +3 -1
  12. package/CHANGELOG.md +31 -0
  13. package/agents/devil-advocate.md +8 -0
  14. package/agents/judge.md +1 -0
  15. package/agents/shared/devil-advocate.body.md +7 -2
  16. package/agents/shared/judge.body.md +3 -1
  17. package/bin/ai-engineering-loop.js +326 -21
  18. package/core/bench.md +15 -0
  19. package/core/formal-statechart.md +49 -0
  20. package/core/goal-contract.md +53 -0
  21. package/core/verification-loop.md +106 -1
  22. package/lib/ablation.js +52 -0
  23. package/lib/ac-status.js +95 -0
  24. package/lib/baseline.js +138 -0
  25. package/lib/bench.js +24 -1
  26. package/lib/budget.js +9 -1
  27. package/lib/calibration.js +129 -0
  28. package/lib/claimed-vs-reality.js +106 -12
  29. package/lib/clean-state.js +85 -0
  30. package/lib/cli-args.js +5 -3
  31. package/lib/comment-findings.js +111 -0
  32. package/lib/debt.js +110 -0
  33. package/lib/feature-map.js +57 -0
  34. package/lib/features.js +95 -0
  35. package/lib/flow.js +86 -0
  36. package/lib/gates.js +257 -41
  37. package/lib/harness-audit.js +141 -0
  38. package/lib/learn.js +24 -1
  39. package/lib/no-progress.js +6 -1
  40. package/lib/receipt.js +33 -2
  41. package/lib/reverts.js +77 -0
  42. package/lib/review-ingest.js +54 -2
  43. package/lib/rule-check.js +90 -0
  44. package/lib/rules.js +205 -0
  45. package/lib/stats.js +110 -0
  46. package/lib/sync-hosts.js +1 -0
  47. package/lib/verification-integrity.js +164 -0
  48. package/lib/verification-lanes.js +96 -0
  49. package/lib/verification-recorder.js +20 -8
  50. package/package.json +1 -1
  51. package/schemas/finding-ledger.schema.json +14 -0
  52. package/schemas/goal-contract.schema.json +50 -0
  53. package/schemas/verdict.schema.json +2 -1
@@ -425,9 +425,9 @@ function analyzeRepository(rootDir) {
425
425
  scripts: {
426
426
  testUnit: 'npm test',
427
427
  testAll: 'npm test',
428
- typecheck: 'npx tsc --noEmit',
429
- lint: 'npx eslint --fix',
430
- build: 'npm run build',
428
+ typecheck: null,
429
+ lint: null,
430
+ build: null,
431
431
  e2e: null
432
432
  },
433
433
  adapter: {
@@ -506,12 +506,16 @@ function analyzeRepository(rootDir) {
506
506
  discovery.scripts.testRelated = viaScript('--findRelatedTests --passWithNoTests {files}') || 'npx jest --findRelatedTests --passWithNoTests {files}';
507
507
  }
508
508
 
509
+ // Only commands the project really has: a guessed `npx tsc` can fetch an
510
+ // unrelated package, and `eslint --fix` would edit files during verify.
511
+ const hasTypeScript = Boolean(allDeps['typescript']) || fs.existsSync(path.join(rootDir, 'tsconfig.json'));
509
512
  if (scripts['typecheck']) discovery.scripts.typecheck = `${pm} run typecheck`;
510
513
  else if (scripts['type-check']) discovery.scripts.typecheck = `${pm} run type-check`;
511
514
  else if (scripts['check']) discovery.scripts.typecheck = `${pm} run check`;
515
+ else discovery.scripts.typecheck = hasTypeScript ? 'npx tsc --noEmit' : null;
512
516
 
513
- if (scripts['lint']) discovery.scripts.lint = `${pm} run lint`;
514
- if (scripts['build']) discovery.scripts.build = `${pm} run build`;
517
+ discovery.scripts.lint = scripts['lint'] ? `${pm} run lint` : null;
518
+ discovery.scripts.build = scripts['build'] ? `${pm} run build` : null;
515
519
  if (scripts['test:e2e']) discovery.scripts.e2e = `${pm} run test:e2e`;
516
520
  } catch (e) {}
517
521
  }
@@ -664,9 +668,9 @@ ${discovery.topLevelDirs.map((d) => `- \`${d}/\``).join('\n') || '- Flat directo
664
668
 
665
669
  ## Discovered Verification Commands
666
670
  - **test_unit**: \`${discovery.scripts.testUnit}\`
667
- - **typecheck**: \`${discovery.scripts.typecheck}\`
668
- - **lint**: \`${discovery.scripts.lint}\`
669
- - **build**: \`${discovery.scripts.build}\`
671
+ ${discovery.scripts.typecheck ? `- **typecheck**: \`${discovery.scripts.typecheck}\`` : ''}
672
+ ${discovery.scripts.lint ? `- **lint**: \`${discovery.scripts.lint}\`` : ''}
673
+ ${discovery.scripts.build ? `- **build**: \`${discovery.scripts.build}\`` : ''}
670
674
  ${discovery.scripts.e2e ? `- **e2e**: \`${discovery.scripts.e2e}\`` : ''}
671
675
  ${discovery.scripts.testRelated ? `- **test_related**: \`${discovery.scripts.testRelated}\`` : ''}
672
676
 
@@ -1390,6 +1394,15 @@ function handleRun() {
1390
1394
  const current = getCurrentRun(CWD);
1391
1395
  const policy = loadPolicy(CWD);
1392
1396
  const startsNewRun = forceNew || !current || TERMINAL_STATES.has(current.state);
1397
+ if (forceNew && current && !TERMINAL_STATES.has(current.state)) {
1398
+ // C-15 (L07 WIP=1): starting another Run while one is unfinished.
1399
+ try {
1400
+ const ratio = require('../lib/features.js').vcr(require('../lib/features.js').loadFeatures(CWD));
1401
+ if (ratio.value < 1) console.error(`! VCR ${ratio.verified}/${ratio.activated}: Run ${current.runId} is still ${current.state}. Finishing it first usually beats starting another (WIP=1).`);
1402
+ } catch {
1403
+ // No feature ledger.
1404
+ }
1405
+ }
1393
1406
  let mode = modeArg ? normalizeMode(modeArg) : (startsNewRun ? policy.defaultMode : null);
1394
1407
  let workflow = null;
1395
1408
  let requestedRunId = null;
@@ -1685,7 +1698,8 @@ function handleBudget() {
1685
1698
  nodeId,
1686
1699
  inputTokens: parseNonNegativeIntegerArg('--input', argValue('--input')),
1687
1700
  outputTokens: parseNonNegativeIntegerArg('--output', argValue('--output')),
1688
- model: argValue('--model')
1701
+ model: argValue('--model'),
1702
+ role: argValue('--role')
1689
1703
  });
1690
1704
  } else if (action === 'pause' || action === 'resume') {
1691
1705
  result = setKillSwitch(CWD, action === 'pause');
@@ -2015,6 +2029,7 @@ function handleGate() {
2015
2029
  console.log(`- Run: ${run.runId}`);
2016
2030
  console.log(`- State: ${run.state}, iteration ${run.iteration}`);
2017
2031
  if (run.tier) console.log(`- Tier: ${run.tier.current}`);
2032
+ for (const warning of (run.history || []).at(-1)?.cleanStateWarnings || []) console.log(`! Clean state: ${warning}`);
2018
2033
  if (guardEvents.warn || guardEvents.block) {
2019
2034
  console.log(`- Guard: ${guardEvents.block} blocked, ${guardEvents.warn} warned (${guardEvents.rules.join(', ')}); see guard-events.json`);
2020
2035
  }
@@ -2037,10 +2052,14 @@ function handleGate() {
2037
2052
  ok: false,
2038
2053
  code: err.code || 'GATE_FAILED',
2039
2054
  error: err.message,
2055
+ ...(err.why ? { why: err.why } : {}),
2056
+ ...(err.fix ? { fix: err.fix } : {}),
2040
2057
  details
2041
2058
  }));
2042
2059
  } else {
2043
2060
  log.error(err.message);
2061
+ if (err.why) console.error(`WHY: ${err.why}`);
2062
+ if (err.fix) console.error(`FIX: ${err.fix}`);
2044
2063
  for (const detail of details) console.error(`- ${detail}`);
2045
2064
  }
2046
2065
  process.exit(1);
@@ -2373,6 +2392,16 @@ function handleState() {
2373
2392
 
2374
2393
  function handleDoctor() {
2375
2394
  const json = process.argv.includes('--json');
2395
+ if (process.argv.includes('--harness')) {
2396
+ // C-08: harness debt in this project (warnings never block).
2397
+ const audit = require('../lib/harness-audit.js').auditHarness(CWD);
2398
+ if (json) console.log(JSON.stringify(audit));
2399
+ else {
2400
+ console.log(`Harness audit: ${audit.ok ? 'no debt found' : 'debt found'}`);
2401
+ for (const item of audit.items) console.log(`${item.level === 'warn' ? '!' : '-'} ${item.id}: ${item.detail}`);
2402
+ }
2403
+ return;
2404
+ }
2376
2405
  const result = runDoctor(path.join(__dirname, '..'), { projectDir: CWD });
2377
2406
  if (json) console.log(JSON.stringify(result));
2378
2407
  else {
@@ -3134,6 +3163,11 @@ function handleGoal() {
3134
3163
  expectedRevision: requiredIntegerArg('--revision')
3135
3164
  });
3136
3165
  extra.impactPreview = impactPreview(runId, draftGoal);
3166
+ // C-21: show the intended user flow as a transition table before freeze.
3167
+ if (draftGoal && draftGoal.flow != null) {
3168
+ const flowLib = require('../lib/flow.js');
3169
+ extra.flow = { table: flowLib.renderFlow(draftGoal.flow), mermaid: flowLib.renderMermaid(draftGoal.flow), problems: flowLib.flowProblems(draftGoal.flow), warnings: flowLib.flowWarnings(draftGoal.flow) };
3170
+ }
3137
3171
  appendRunLifecycle(CWD, runId, {
3138
3172
  type: 'GOAL_DRAFT_UPDATED',
3139
3173
  phase: 'goal',
@@ -3363,9 +3397,11 @@ function splitCommand(text, shell) {
3363
3397
 
3364
3398
  function stageJsonError(err, json, code) {
3365
3399
  if (json) {
3366
- console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, details: err.details || [] }));
3400
+ console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, ...(err.why ? { why: err.why } : {}), ...(err.fix ? { fix: err.fix } : {}), details: err.details || [] }));
3367
3401
  } else {
3368
3402
  log.error(err.message);
3403
+ if (err.why) console.error(`WHY: ${err.why}`);
3404
+ if (err.fix) console.error(`FIX: ${err.fix}`);
3369
3405
  for (const detail of err.details || []) console.error(`- ${detail}`);
3370
3406
  }
3371
3407
  process.exit(1);
@@ -3391,19 +3427,76 @@ async function handleVerify() {
3391
3427
  if (related && final) throw Object.assign(new Error('--related and --final cannot be combined'), { code: 'INVALID_USAGE' });
3392
3428
  let texts = repeatedArg('--cmd');
3393
3429
  const lanes = require('../lib/verification-lanes.js');
3430
+ const baselineMode = process.argv.includes('--baseline');
3431
+ if (baselineMode && (related || final)) throw Object.assign(new Error('--baseline cannot be combined with --related or --final'), { code: 'INVALID_USAGE' });
3394
3432
  let commandsToRun;
3433
+ let defaultPlan = false;
3395
3434
  if (related && texts.length === 0) {
3396
3435
  const target = loadRun(CWD, runId);
3397
- commandsToRun = [lanes.relatedCommand(CWD, target, { split: splitCommand, shell })];
3398
- } else {
3399
- if (texts.length === 0) {
3400
- const fallback = defaultVerificationCommand();
3401
- if (!fallback) throw Object.assign(new Error('No --cmd given and no test_unit command in .ai-engineering-loop/verification.md'), { code: 'INVALID_COMMAND' });
3402
- texts = [fallback];
3436
+ commandsToRun = [{ ...lanes.relatedCommand(CWD, target, { split: splitCommand, shell }), layer: 'L2' }];
3437
+ } else if (texts.length === 0) {
3438
+ // C-02: no --cmd runs the cheapest-first plan (L1 build/typecheck/lint,
3439
+ // then L2 test_unit) and stops at the first failure.
3440
+ const plan = lanes.defaultCommandPlan(CWD);
3441
+ // C-05: promoted review rules run as L1 checks, after the static checks.
3442
+ const rules = require('../lib/rules.js').rulePlan(CWD, loadRun(CWD, runId));
3443
+ if (rules.length) {
3444
+ const firstL2 = plan.findIndex((step) => step.layer !== 'L1');
3445
+ plan.splice(firstL2 === -1 ? plan.length : firstL2, 0, ...rules);
3446
+ }
3447
+ // C-23: catalog keys an AC requires (verify.key, e.g. pentest, formal)
3448
+ // run too when the project configures them.
3449
+ if (!baselineMode) {
3450
+ try {
3451
+ const target = loadRun(CWD, runId);
3452
+ const goal = require('../lib/claimed-vs-reality.js').loadFrozenGoal(CWD, target);
3453
+ const configured = lanes.verificationCommands(CWD);
3454
+ const acStatusLib = require('../lib/ac-status.js');
3455
+ for (const key of acStatusLib.requiredKeys(goal)) {
3456
+ if (configured[key] && !plan.some((step) => step.key === key)) plan.push({ key, layer: lanes.layerOfKey(key), text: configured[key] });
3457
+ }
3458
+ // A required layer with no step yet: the first configured catalog key of that layer.
3459
+ for (const layer of acStatusLib.requiredLayers(goal)) {
3460
+ if (plan.some((step) => step.layer === layer)) continue;
3461
+ const key = Object.keys(lanes.VERIFICATION_CATALOG).find((name) => lanes.VERIFICATION_CATALOG[name].layer === layer && configured[name]);
3462
+ if (key) plan.push({ key, layer, text: configured[key] });
3463
+ }
3464
+ } catch {
3465
+ // No frozen Goal yet; nothing extra to run.
3466
+ }
3467
+ }
3468
+ // C-03: startup-risky diffs also run the configured smoke check (L2).
3469
+ const smokeText = lanes.verificationCommands(CWD).smoke;
3470
+ if (smokeText && !baselineMode && lanes.startupRiskFiles(CWD, loadRun(CWD, runId)).length) {
3471
+ plan.push({ key: 'smoke', layer: 'L2', text: smokeText });
3403
3472
  }
3404
- commandsToRun = texts.map((text) => splitCommand(text, shell));
3473
+ if (!plan.length) throw Object.assign(new Error('No --cmd given and no build/typecheck/lint/test_unit command in .ai-engineering-loop/verification.md. FIX: add them to verification.md or pass --cmd.'), { code: 'INVALID_COMMAND' });
3474
+ commandsToRun = plan.map((step) => ({ ...splitCommand(step.text, shell || Boolean(step.shell)), key: step.key, layer: step.layer, ...(step.env ? { env: step.env } : {}) }));
3475
+ defaultPlan = true;
3476
+ } else {
3477
+ commandsToRun = texts.map((text) => {
3478
+ const layer = lanes.layerOfCommand(CWD, text);
3479
+ return { ...splitCommand(text, shell), ...(layer ? { layer } : {}) };
3480
+ });
3405
3481
  }
3406
3482
  const timeoutValue = argValue('--timeout-ms');
3483
+ if (baselineMode) {
3484
+ const outcome = await require('../lib/baseline.js').recordBaseline(CWD, {
3485
+ runId,
3486
+ plan: commandsToRun,
3487
+ ...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
3488
+ });
3489
+ const result = { ok: outcome.status !== 'BROKEN', runId, baseline: outcome };
3490
+ if (json) console.log(JSON.stringify(result));
3491
+ else {
3492
+ const say = outcome.status === 'GREEN' ? log.success : (outcome.status === 'RED' ? log.warn : log.error);
3493
+ say(`Baseline ${outcome.status}: ${outcome.path}`);
3494
+ for (const c of outcome.commands) console.log(`- ${c.command}: exit ${c.exitCode}`);
3495
+ for (const problem of outcome.problems) console.log(`! ${problem}`);
3496
+ }
3497
+ if (!result.ok) process.exitCode = 1;
3498
+ return;
3499
+ }
3407
3500
  if (final) {
3408
3501
  const target = loadRun(CWD, runId);
3409
3502
  if (target.state !== 'JUDGED_PASS') throw Object.assign(new Error(`verify --final needs a Judge PASS (state is ${target.state})`), { code: 'INVALID_STATE' });
@@ -3437,17 +3530,31 @@ async function handleVerify() {
3437
3530
  commands: commandsToRun,
3438
3531
  serial: process.argv.includes('--serial'),
3439
3532
  fresh: process.argv.includes('--fresh'),
3533
+ stopOnFailure: defaultPlan,
3440
3534
  ...(related ? { scope: 'related' } : {}),
3441
3535
  ...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
3442
3536
  });
3443
- const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
3537
+ const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.layer ? { layer: c.layer } : {}), ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
3444
3538
  if (!recorded.passed) {
3445
3539
  autoHeartbeat(runId, 'verification', 'Verification failed; Maker must fix before review');
3446
- const result = { ok: false, code: 'VERIFICATION_FAILED', runId, commands, gate: null };
3540
+ const baselineModule = require('../lib/baseline.js');
3541
+ const failures = baselineModule.classifyAgainstBaseline(baselineModule.loadBaseline(CWD, runId), recorded.commands);
3542
+ const result = { ok: false, code: 'VERIFICATION_FAILED', runId, commands, skipped: recorded.skipped, failures, gate: null };
3447
3543
  if (json) console.log(JSON.stringify(result));
3448
3544
  else {
3449
3545
  log.error('Verification failed; the gate was not attempted.');
3450
3546
  for (const c of commands) console.log(`- ${c.command}: exit ${c.exitCode} (${c.timeoutStatus})`);
3547
+ for (const skippedCommand of recorded.skipped) console.log(`- ${skippedCommand}: skipped (an earlier, cheaper check failed)`);
3548
+ for (const c of recorded.commands.filter((item) => item.exitCode !== 0)) {
3549
+ const rulesLib = require('../lib/rules.js');
3550
+ const rule = (c.key && c.key.startsWith('rule:') ? rulesLib.findRule(CWD, c.key.slice(5)) : null) || rulesLib.ruleForCommand(CWD, c.command);
3551
+ if (rule) console.log(`! Rule ${rule.id} violated. WHY: ${rule.why} FIX: ${rule.fix}`);
3552
+ }
3553
+ for (const f of failures) {
3554
+ console.log(f.preExisting
3555
+ ? `! ${f.command}: every failure was already failing at baseline (pre-existing). FIX: repair it in a separate Run, or narrow --cmd to the tests this change owns.`
3556
+ : `! ${f.command}: new failure(s)${f.newFailures.length ? `: ${f.newFailures.slice(0, 5).join('; ')}` : ''}. FIX: fix the change before review.`);
3557
+ }
3451
3558
  }
3452
3559
  process.exitCode = 1;
3453
3560
  return;
@@ -3471,6 +3578,14 @@ async function handleVerify() {
3471
3578
  impactDrift: { mapStatus: drift.mapStatus, outOfMap: drift.outOfMap },
3472
3579
  compact: compact.paths,
3473
3580
  claimedVsReality: claimed,
3581
+ acStatus: (() => {
3582
+ try {
3583
+ return require('../lib/ac-status.js').runAcStatuses(CWD, run, require('../lib/claimed-vs-reality.js').loadFrozenGoal(CWD, run))
3584
+ .filter((ac) => ac.status !== 'unbound');
3585
+ } catch {
3586
+ return [];
3587
+ }
3588
+ })(),
3474
3589
  ...(related ? { scope: 'related', finalRequired: 'Run `verify --final` after Judge PASS, before delivery.' } : {}),
3475
3590
  next: run.tier && run.tier.current === 'T0'
3476
3591
  ? 'Fill claimed-vs-reality, then gate review and gate judge'
@@ -3487,6 +3602,7 @@ async function handleVerify() {
3487
3602
  console.log('! No impact map for this Run; drift was not checked');
3488
3603
  }
3489
3604
  console.log(`- Claimed-vs-reality: ${claimed.path}${claimed.created ? ' (scaffolded; fill it from the command output)' : ''}`);
3605
+ for (const ac of result.acStatus) console.log(`- ${ac.id}: ${ac.status} (\`${ac.command}\`)`);
3490
3606
  console.log(`- Next: ${result.next}`);
3491
3607
  }
3492
3608
  } catch (err) {
@@ -3494,6 +3610,115 @@ async function handleVerify() {
3494
3610
  }
3495
3611
  }
3496
3612
 
3613
+ function handleOutcome() {
3614
+ const json = process.argv.includes('--json');
3615
+ try {
3616
+ const action = args[1] && !args[1].startsWith('--') ? args[1] : null;
3617
+ if (action !== 'record') throw Object.assign(new Error('outcome supports: record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name> [--evidence <path|url>]'), { code: 'INVALID_USAGE' });
3618
+ const entry = require('../lib/calibration.js').recordOutcome(CWD, {
3619
+ runId: argValue('--run'),
3620
+ source: argValue('--source'),
3621
+ location: argValue('--location'),
3622
+ summary: argValue('--summary'),
3623
+ evidence: argValue('--evidence'),
3624
+ by: argValue('--by')
3625
+ });
3626
+ if (json) console.log(JSON.stringify({ ok: true, outcome: entry }));
3627
+ else log.success(`✓ outcome recorded for ${argValue('--run')}; run \`calibrate\` to compare it with the review`);
3628
+ } catch (err) {
3629
+ stageJsonError(err, json, 'OUTCOME_FAILED');
3630
+ }
3631
+ }
3632
+
3633
+ function handleCalibrate() {
3634
+ const json = process.argv.includes('--json');
3635
+ try {
3636
+ const calibration = require('../lib/calibration.js');
3637
+ const report = calibration.calibrate(CWD);
3638
+ if (json) console.log(JSON.stringify({ ok: true, ...report }));
3639
+ else console.log(calibration.formatCalibration(report));
3640
+ } catch (err) {
3641
+ stageJsonError(err, json, 'CALIBRATE_FAILED');
3642
+ }
3643
+ }
3644
+
3645
+ function handleDebt() {
3646
+ const json = process.argv.includes('--json');
3647
+ try {
3648
+ const debt = require('../lib/debt.js');
3649
+ const action = args[1] && !args[1].startsWith('--') ? args[1] : 'list';
3650
+ if (action === 'close') {
3651
+ const item = debt.closeDebt(CWD, args[2], { by: argValue('--by'), note: argValue('--note') });
3652
+ if (json) console.log(JSON.stringify({ ok: true, item }));
3653
+ else log.success(`✓ closed ${item.id}`);
3654
+ return;
3655
+ }
3656
+ if (action !== 'list') throw Object.assign(new Error('debt supports: list, close <id> --by <name>'), { code: 'INVALID_USAGE' });
3657
+ const file = argValue('--file');
3658
+ const items = debt.loadDebt(CWD).items.filter((item) => item.status === 'open' && (!file || item.file === file));
3659
+ if (json) console.log(JSON.stringify({ ok: true, items }));
3660
+ else if (!items.length) console.log('No open debt.');
3661
+ else for (const item of items) console.log(`- ${item.id} [${item.risk}] ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; revisit: ${item.nextTrigger})`);
3662
+ } catch (err) {
3663
+ stageJsonError(err, json, 'DEBT_FAILED');
3664
+ }
3665
+ }
3666
+
3667
+ function handleNext() {
3668
+ const json = process.argv.includes('--json');
3669
+ try {
3670
+ const features = require('../lib/features.js');
3671
+ const document = features.loadFeatures(CWD);
3672
+ if (!document) throw Object.assign(new Error('No .ai-engineering-loop/features.json. FIX: create it with {"schemaVersion":1,"features":[{"id","title","behavior","verify","status":"not_started","priority","dependsOn":[]}]}.'), { code: 'NO_FEATURES' });
3673
+ const next = features.nextFeature(document);
3674
+ const ratio = features.vcr(document);
3675
+ const problems = features.auditFeatures(CWD, document);
3676
+ const active = document.features.filter((item) => item.status === 'active').map((item) => item.id);
3677
+ const result = { ok: problems.length === 0, next, vcr: ratio, active, problems };
3678
+ if (json) console.log(JSON.stringify(result));
3679
+ else {
3680
+ console.log(`VCR: ${ratio.verified}/${ratio.activated} (${ratio.value})${active.length ? `; active: ${active.join(', ')}` : ''}`);
3681
+ if (ratio.value < 1 && active.length) console.log(`! Finish ${active[0]} before starting another feature (WIP=1).`);
3682
+ console.log(next ? `Next: ${next.id} - ${next.title || next.behavior}${next.verify ? ` (verify: ${next.verify})` : ''}. Start it with a Goal whose featureId is "${next.id}".` : 'Next: nothing unblocked.');
3683
+ for (const problem of problems) console.log(`! ${problem}`);
3684
+ }
3685
+ if (problems.length) process.exitCode = 1;
3686
+ } catch (err) {
3687
+ stageJsonError(err, json, 'NEXT_FAILED');
3688
+ }
3689
+ }
3690
+
3691
+ // C-20: `rules check <id>` runs a built-in rule kind against the lines the
3692
+ // change adds ($AEL_ADDED_LINES_FILE); violations print as file:line.
3693
+ function handleRules() {
3694
+ const json = process.argv.includes('--json');
3695
+ try {
3696
+ if (args[1] !== 'check' || !args[2]) throw Object.assign(new Error('rules supports: check <ruleId>'), { code: 'INVALID_USAGE' });
3697
+ const rule = require('../lib/rules.js').findRule(CWD, args[2]);
3698
+ if (!rule) throw Object.assign(new Error(`No rule ${args[2]} in .ai-engineering-loop/rules.json`), { code: 'NOT_FOUND' });
3699
+ const ruleCheck = require('../lib/rule-check.js');
3700
+ const violations = ruleCheck.checkRule(rule, ruleCheck.readAddedLines());
3701
+ if (json) console.log(JSON.stringify({ ok: violations.length === 0, rule: rule.id, violations }));
3702
+ else for (const v of violations) console.log(`${v.file}:${v.line}: ${v.message}`);
3703
+ if (violations.length) process.exitCode = 1;
3704
+ } catch (err) {
3705
+ stageJsonError(err, json, 'RULES_FAILED');
3706
+ }
3707
+ }
3708
+
3709
+ function handleStats() {
3710
+ const json = process.argv.includes('--json');
3711
+ try {
3712
+ const lastValue = argValue('--last');
3713
+ const stats = require('../lib/stats.js');
3714
+ const result = stats.computeStats(CWD, lastValue == null ? {} : { last: parsePositiveIntegerArg('--last', lastValue) });
3715
+ if (json) console.log(JSON.stringify({ ok: true, ...result }));
3716
+ else console.log(stats.formatStats(result));
3717
+ } catch (err) {
3718
+ stageJsonError(err, json, 'STATS_FAILED');
3719
+ }
3720
+ }
3721
+
3497
3722
  function readGuardInput() {
3498
3723
  const source = argValue('--input') || (process.argv.includes('--hook') ? '-' : null);
3499
3724
  if (source == null) {
@@ -3639,6 +3864,25 @@ function handleReviewPack() {
3639
3864
  const pack = smartContextSummary(createDiffHunkPack(CWD, { runId, base, profile: argValue('--profile') }));
3640
3865
  const escalation = assessRunEscalation(CWD, { runId });
3641
3866
  const tier = run.tier ? run.tier.current : null;
3867
+ // C-14: open debt on the files this change touches goes into the pack.
3868
+ let openDebt = [];
3869
+ let touchedFeatures = [];
3870
+ try {
3871
+ const changed = require('../lib/verification-lanes.js').relatedFiles(CWD, run);
3872
+ openDebt = require('../lib/debt.js').openDebtFor(CWD, changed);
3873
+ if (openDebt.length && pack.path) {
3874
+ const lines = ['', '## Open debt on changed files', '', ...openDebt.map((item) => `- ${item.id} ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; risk ${item.risk})`)];
3875
+ fs.appendFileSync(path.join(CWD, pack.path), `${lines.join('\n')}\n`);
3876
+ }
3877
+ // C-22: user-facing features this diff touches, with how to reach them.
3878
+ const featureMap = require('../lib/feature-map.js');
3879
+ touchedFeatures = featureMap.featuresForFiles(CWD, changed);
3880
+ if (touchedFeatures.length && pack.path) fs.appendFileSync(path.join(CWD, pack.path), `${featureMap.renderForPack(touchedFeatures)}\n`);
3881
+ } catch {
3882
+ openDebt = [];
3883
+ }
3884
+ const diffRelative = path.relative(CWD, path.join(runsDir(CWD), runId, 'diff.patch')).split(path.sep).join('/');
3885
+ const reviewInputs = require('../lib/review-ingest.js').recordReviewInputs(CWD, runId, { packPath: pack.path, diffPath: diffRelative });
3642
3886
  autoHeartbeat(runId, 'review', 'Review pack built');
3643
3887
  const result = {
3644
3888
  ok: true,
@@ -3652,6 +3896,9 @@ function handleReviewPack() {
3652
3896
  truncated: pack.truncated,
3653
3897
  unresolvedFindings: pack.unresolvedFindings,
3654
3898
  impactDrift: pack.impactDrift,
3899
+ openDebt: openDebt.map((item) => ({ id: item.id, location: item.location, debt: item.debt })),
3900
+ features: touchedFeatures.map((entry) => ({ id: entry.id, where: entry.where, driver: entry.driver })),
3901
+ reviewInputs: { inputs: reviewInputs.inputs, promptSha256: reviewInputs.promptSha256 },
3655
3902
  fastPath: { eligible: fastPath.eligible, reasons: fastPath.reasons },
3656
3903
  escalation: { requiredTier: escalation.requiredTier, reasons: escalation.reasons.map((r) => r.code) }
3657
3904
  };
@@ -3665,6 +3912,8 @@ function handleReviewPack() {
3665
3912
  console.log(`- Outside the impact map: ${result.impactDrift.outOfMap.join(', ')} (check their side effects)`);
3666
3913
  }
3667
3914
  for (const line of result.impactDrift?.design || []) console.log(`- ${line}`);
3915
+ if (touchedFeatures.length) console.log(`- Features touched: ${touchedFeatures.map((entry) => entry.id).join(', ')} (in the pack; use them for L3 evidence)`);
3916
+ if (openDebt.length) console.log(`- Open debt on changed files: ${openDebt.map((item) => item.id).join(', ')} (in the pack; resolve or keep consciously)`);
3668
3917
  }
3669
3918
  } catch (err) {
3670
3919
  stageJsonError(err, json, 'REVIEW_PACK_FAILED');
@@ -3825,6 +4074,7 @@ function handleBench() {
3825
4074
  sandboxHome: process.argv.includes('--sandbox-home'),
3826
4075
  hostArgs: repeatedArg('--host-arg'),
3827
4076
  scenarioIds: repeatedArg('--scenario'),
4077
+ without: repeatedArg('--without').flatMap((value) => value.split(',')).map((value) => value.trim()).filter(Boolean),
3828
4078
  repeat: repeatArg == null ? 1 : parsePositiveIntegerArg('--repeat', repeatArg),
3829
4079
  timeoutMs: timeoutArg == null ? undefined : parseBenchTimeoutSeconds(timeoutArg),
3830
4080
  mode: modeArg == null ? 'ASSISTED' : normalizeMode(modeArg),
@@ -3930,6 +4180,26 @@ function handleQuick() {
3930
4180
 
3931
4181
  function handleLearn() {
3932
4182
  const json = process.argv.includes('--json');
4183
+ if (args[1] === 'promote') {
4184
+ try {
4185
+ const rule = require('../lib/rules.js').promoteLearning(CWD, args[2], {
4186
+ check: argValue('--check'),
4187
+ fix: argValue('--fix'),
4188
+ why: argValue('--why'),
4189
+ by: argValue('--by'),
4190
+ scope: argValue('--scope') || 'repo',
4191
+ forbidImport: argValue('--forbid-import'),
4192
+ forbidApi: argValue('--forbid-api'),
4193
+ except: argValue('--except'),
4194
+ shell: !process.argv.includes('--no-shell')
4195
+ });
4196
+ if (json) console.log(JSON.stringify({ ok: true, rule }));
4197
+ else log.success(`✓ ${rule.from} promoted to rule ${rule.id}; default \`verify\` now runs: ${rule.check}`);
4198
+ } catch (err) {
4199
+ stageJsonError(err, json, 'LEARN_PROMOTE_FAILED');
4200
+ }
4201
+ return;
4202
+ }
3933
4203
  try {
3934
4204
  const maxAge = argValue('--max-age-runs');
3935
4205
  const result = learn(CWD, {
@@ -3938,8 +4208,9 @@ function handleLearn() {
3938
4208
  ...(maxAge == null ? {} : { maxAgeRuns: parsePositiveIntegerArg('--max-age-runs', maxAge) })
3939
4209
  });
3940
4210
  const { document } = result;
4211
+ const candidates = require('../lib/rules.js').promotionCandidates(CWD, document.entries);
3941
4212
  if (json) {
3942
- console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, learnings: document }));
4213
+ console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, promotionCandidates: candidates, learnings: document }));
3943
4214
  return;
3944
4215
  }
3945
4216
  log.success(`✓ ${document.entries.length} learning(s) from ${document.sourceRuns} Run(s)${result.written ? ` → ${result.path}` : ' (dry run, nothing written)'}`);
@@ -3947,6 +4218,9 @@ function handleLearn() {
3947
4218
  console.log(`- ${entry.id} [${entry.axis}, ${entry.severity}, ×${entry.occurrences}] ${entry.lesson}`);
3948
4219
  }
3949
4220
  for (const item of result.pruned) console.log(` pruned ${item.id} (${item.reason})`);
4221
+ for (const item of candidates) {
4222
+ console.log(`! Promote candidate ${item.id} (seen in ${item.runs} Runs): ${item.lesson}. Make it a check: learn promote ${item.id} --check "<command>" --fix "<instruction>" --by <name>`);
4223
+ }
3950
4224
  } catch (err) {
3951
4225
  stageJsonError(err, json, 'LEARN_FAILED');
3952
4226
  }
@@ -4083,6 +4357,12 @@ Commands:
4083
4357
  A missing binary is UNAVAILABLE, not a model failure.
4084
4358
  Progress lines go to stderr. Writes .ai-engineering-loop/bench/<benchId>/{results.json,report.md,hardening.md}
4085
4359
  bench compare <benchIdA> <benchIdB> [--json] read-only side-by-side table
4360
+ outcome record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name>
4361
+ a post-delivery problem traced to a Run (input for calibrate)
4362
+ calibrate Compare recorded outcomes with each Run's Finding Ledger: DA misses, wrong dismissals, per reviewer prompt
4363
+ debt Accepted trade-offs and L3 waivers recorded at delivery: list [--file <path>] | close <id> --by <name> [--note <text>]
4364
+ next Next unblocked feature from .ai-engineering-loop/features.json, VCR (verified/activated), evidence audit [--json]
4365
+ stats Failure causes by harness layer, false-victory rate, iterations to PASS [--last <n>] [--json]
4086
4366
  escalation Compute the minimum Judge model tier from deterministic risk signals
4087
4367
  --run <id> target a non-current run
4088
4368
  --json print machine-readable reasons
@@ -4094,7 +4374,8 @@ Commands:
4094
4374
  start "<task>" [--mode assisted|unattended] | finish [--run <id>] [--cmd "<command>"]...
4095
4375
  verify Record verification commands, gate them, compact, and scaffold claimed-vs-reality
4096
4376
  [--run <id>] [--cmd "<command>"]... [--shell] [--serial] [--timeout-ms <n>] [--fresh] [--json]
4097
- default command: test_unit from .ai-engineering-loop/verification.md; commands run in parallel
4377
+ default (no --cmd): build -> typecheck -> lint -> test_unit from verification.md, serial, stops at first failure
4378
+ --baseline before the Maker gate: run the plan on the untouched tree, write baseline.json
4098
4379
  --related fast lane: test_related with {files} = changed files of the gated diff
4099
4380
  --final after Judge PASS: full run delivery needs when verification was related-only
4100
4381
  --fresh re-run commands that already passed for the same diff and revision
@@ -4122,6 +4403,11 @@ Commands:
4122
4403
  writes RUN_DIR/receipt.json + receipt.md (paste into the PR/MR body); digest stored in delivery.json
4123
4404
  verify-receipt Re-check a delivery receipt: artifact hashes, frozen Goal, AC evidence, verdict, head SHA
4124
4405
  [--run <id> | <repository-relative-receipt.json>] [--head <sha>] [--json] exit 1 on mismatch (CI)
4406
+ learn promote <learningId> --check "<command>" --fix "<instruction>" --by <name> [--why <text>] [--scope repo|diff] [--no-shell]
4407
+ --scope diff: the check sees only lines this change adds ($AEL_ADDED_LINES_FILE, $AEL_CHANGED_FILES)
4408
+ built-in instead of --check: --forbid-import "<fromGlob> -> <toGlob>" | --forbid-api "<regex>" [--except <glob>]
4409
+ rules check <ruleId> run a built-in rule on the added lines (used by verify)
4410
+ turn a recurring review lesson into an executable L1 rule (.ai-engineering-loop/rules.json)
4125
4411
  learn Mine past Runs' VALID findings into .ai-engineering-loop/learnings.json (+ .md) for context packs
4126
4412
  [--dry-run] [--prune] [--max-age-runs <n>] [--json]
4127
4413
  deterministic and deduped with runId provenance; --prune drops lessons whose files are gone
@@ -4130,6 +4416,7 @@ Commands:
4130
4416
  list [--json]
4131
4417
  <name> [--section <heading>] [--json] e.g. docs grill-policy --section "When to grill"
4132
4418
  doctor Check runtime, schemas, host assets, package contents, and eval catalog
4419
+ --harness audit harness debt instead: entry-file size/structure, stale learnings, rule provenance, skill budget
4133
4420
  --json print machine-readable diagnostics
4134
4421
  update Check or apply the package update workflow
4135
4422
  check|plan [--json]
@@ -4295,6 +4582,24 @@ switch (command) {
4295
4582
  case 'escalation':
4296
4583
  handleEscalation();
4297
4584
  break;
4585
+ case 'stats':
4586
+ handleStats();
4587
+ break;
4588
+ case 'rules':
4589
+ handleRules();
4590
+ break;
4591
+ case 'next':
4592
+ handleNext();
4593
+ break;
4594
+ case 'debt':
4595
+ handleDebt();
4596
+ break;
4597
+ case 'outcome':
4598
+ handleOutcome();
4599
+ break;
4600
+ case 'calibrate':
4601
+ handleCalibrate();
4602
+ break;
4298
4603
  case 'docs':
4299
4604
  handleDocs();
4300
4605
  break;
package/core/bench.md CHANGED
@@ -75,3 +75,18 @@ Bench results do not feed `ael learn`. Learnings stay findings mined from Runs;
75
75
  `bench compare <a> <b>` prints a side-by-side table and does not write.
76
76
 
77
77
  A live run against your own host is manual evidence for a change review. CI uses the fake host only.
78
+
79
+ ## Ablation and the simplification log (C-08)
80
+
81
+ Every harness component encodes an assumption about what the model cannot do alone; as models improve, some assumptions go stale. Measure instead of guessing:
82
+
83
+ ```bash
84
+ npx ai-engineering-loop bench --host <host> --model <model> # baseline
85
+ npx ai-engineering-loop bench --host <host> --model <model> --without integrity # one component off
86
+ npx ai-engineering-loop bench compare <baselineId> <ablationId>
87
+ ```
88
+
89
+ `--without` accepts `integrity`, `l3`, `l1`, `smoke`, `rules` (repeat or comma-separate). It only takes effect inside bench scenario repos: bench sets `AEL_ABLATE` and writes `.ai-engineering-loop/bench-ablation.json`; the gates honour the switch only when both are present together with `AEL_BENCH_MODE`, and every skip is logged to the Run's `ablation.log`. The result carries `ablation.without`, and an entry is appended to `.ai-engineering-loop/simplification-log.json` (`decision: null` until a human records keep / remove / replace).
90
+
91
+ `doctor --harness` audits harness debt in the project: entry-file size (50-200 lines) and structure (only headings, lists, tables and code count, so keywords in prose cannot game it), learnings not seen in the last 10 Runs, rules without provenance or older than 90 days, and, inside the AEL repo, T1 skill-context headroom from `scripts/measure.js`.
92
+
@@ -0,0 +1,49 @@
1
+ # Formal flows with pstack `/statechart`
2
+
3
+ How to prove a risky user flow with a model checker and feed the result into AEL's gates. AEL ships none of this tooling; it only recognises the evidence (card C-25).
4
+
5
+ ## When it is worth it
6
+
7
+ Use it for flows where a wrong state is expensive and hard to see in unit tests: payments and refunds, sync across devices, retries with leases, notification fan-out, multi-step approvals. For an ordinary screen flow, the Goal `flow` diagram (below) is enough.
8
+
9
+ ## The tool
10
+
11
+ [`statechart`](https://github.com/cursor/plugins/tree/main/pstack/skills/statechart) is a skill in the pstack plugin by Lauren Tan (MIT). You write a TLA+ spec in which each rule the reader agrees to is an `INVARIANT`. TLC model-checks it, and the skill draws the result as a Mermaid statechart. Its report gives the shortest path to every state that breaks a rule.
12
+
13
+ Requirements: Java and TLC (installed with `mise` on first run, which needs the network) and a runtime that executes TypeScript. Install pstack as a Cursor plugin, or follow its Claude Code guide (`docs/harness/claude-code.md` in pstack) and call it as `/pstack:statechart`. It runs only when asked (`disable-model-invocation`).
14
+
15
+ ## Wire it into AEL
16
+
17
+ 1. Keep the spec in the repository, for example `spec/checkout.tla` and `spec/checkout.cfg`. Keep constants small, or add a `CONSTRAINT`: TLC keeps going past a broken rule, so an unbounded model can run without end.
18
+ 2. `statechart` requires its `--out` folder to be new or empty, so a fixed folder fails the second `verify` (exit 2). Wrap it in a small script that clears the folder first, and add the folder to `.gitignore` (otherwise delivery reports it as an untracked file, C-06):
19
+
20
+ ```sh
21
+ #!/usr/bin/env sh
22
+ # scripts/formal.sh
23
+ set -e
24
+ rm -rf .tmp/statechart
25
+ exec "$PSTACK/skills/statechart/scripts/statechart" spec/checkout.tla --config spec/checkout.cfg --out .tmp/statechart
26
+ ```
27
+
28
+ 3. Register the wrapper as the `formal` key in `.ai-engineering-loop/verification.md`. `formal` is layer `LF` and never runs in the default plan:
29
+
30
+ ```
31
+ - **formal**: `sh scripts/formal.sh`
32
+ ```
33
+
34
+ The statechart command exits with TLC's code: 0 when every rule holds, 1 when a rule breaks, 2 when TLC stops with an error. So a broken rule fails the verification gate like any other failing command.
35
+ 4. Require it on the AC that the model covers:
36
+
37
+ ```json
38
+ { "id": "AC-2", "statement": "A refund never leaves the order both paid and refunded", "evidenceRequired": "TLC model check", "failureCases": ["double refund"], "verify": { "layer": "LF" } }
39
+ ```
40
+
41
+ Default `verify` then runs `formal` (C-23), and `gate judge` refuses PASS until it passed on the gated diff (`AC_NOT_PROVEN`).
42
+ 5. Attach the chart to the PR receipt: publish `chart.mmd` (or a rendered image) and pass `deliver --evidence AC-2=<https-url>`. The receipt lists it under "Verification artifacts" as a diagram.
43
+ 6. When the model describes code, replay each bug path from the report against the real code, as a test or a live repro. A path that does not reproduce means the model is wrong until shown otherwise.
44
+
45
+ ## Without TLA+: the Goal `flow` diagram
46
+
47
+ A Goal Contract `flow` (`[{ "from", "event", "to" }]`, see `docs goal-contract`) is rendered by `goal draft` and in the PR receipt as a Mermaid `stateDiagram-v2` plus a `From | Event | To` table. GitHub renders the diagram in the PR. It lets a reviewer check that the agent understood the intended flow; it proves nothing about the code. For proof, use `formal` as above.
48
+
49
+ Skill text never contains Mermaid (some hosts cannot render it); diagrams appear only in command output and PR receipts.