ai-engineering-loop 2.5.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.agents/skills/ael/SKILL.md +1 -1
  2. package/.agents/skills/ael/references/loop-commands.md +11 -0
  3. package/.agents/skills/ael/references/loop-goal.md +2 -0
  4. package/.agents/skills/ael/references/loop-maker.md +1 -6
  5. package/.agents/skills/ael/references/loop-report.md +8 -0
  6. package/.agents/skills/ael/references/loop-verify.md +21 -11
  7. package/.claude/agents/devil-advocate.md +6 -1
  8. package/.claude/agents/judge.md +3 -1
  9. package/.grok/agents/devil-advocate.md +6 -1
  10. package/.grok/agents/judge.md +3 -1
  11. package/CHANGELOG.md +22 -0
  12. package/agents/devil-advocate.md +8 -0
  13. package/agents/judge.md +1 -0
  14. package/agents/shared/devil-advocate.body.md +6 -1
  15. package/agents/shared/judge.body.md +3 -1
  16. package/bin/ai-engineering-loop.js +262 -20
  17. package/core/bench.md +15 -0
  18. package/core/goal-contract.md +35 -0
  19. package/core/verification-loop.md +96 -7
  20. package/lib/ablation.js +52 -0
  21. package/lib/ac-status.js +63 -0
  22. package/lib/baseline.js +138 -0
  23. package/lib/bench.js +24 -1
  24. package/lib/calibration.js +112 -0
  25. package/lib/claimed-vs-reality.js +114 -16
  26. package/lib/clean-state.js +85 -0
  27. package/lib/cli-args.js +5 -3
  28. package/lib/debt.js +110 -0
  29. package/lib/features.js +95 -0
  30. package/lib/gates.js +234 -41
  31. package/lib/harness-audit.js +124 -0
  32. package/lib/no-progress.js +6 -1
  33. package/lib/review-ingest.js +54 -2
  34. package/lib/rules.js +112 -0
  35. package/lib/stats.js +90 -0
  36. package/lib/sync-hosts.js +1 -0
  37. package/lib/verification-integrity.js +164 -0
  38. package/lib/verification-lanes.js +64 -0
  39. package/lib/verification-recorder.js +17 -5
  40. package/package.json +1 -1
  41. package/schemas/finding-ledger.schema.json +14 -0
  42. package/schemas/goal-contract.schema.json +31 -0
  43. package/schemas/verdict.schema.json +2 -1
@@ -425,9 +425,9 @@ function analyzeRepository(rootDir) {
425
425
  scripts: {
426
426
  testUnit: 'npm test',
427
427
  testAll: 'npm test',
428
- typecheck: 'npx tsc --noEmit',
429
- lint: 'npx eslint --fix',
430
- build: 'npm run build',
428
+ typecheck: null,
429
+ lint: null,
430
+ build: null,
431
431
  e2e: null
432
432
  },
433
433
  adapter: {
@@ -506,12 +506,16 @@ function analyzeRepository(rootDir) {
506
506
  discovery.scripts.testRelated = viaScript('--findRelatedTests --passWithNoTests {files}') || 'npx jest --findRelatedTests --passWithNoTests {files}';
507
507
  }
508
508
 
509
+ // Only commands the project really has: a guessed `npx tsc` can fetch an
510
+ // unrelated package, and `eslint --fix` would edit files during verify.
511
+ const hasTypeScript = Boolean(allDeps['typescript']) || fs.existsSync(path.join(rootDir, 'tsconfig.json'));
509
512
  if (scripts['typecheck']) discovery.scripts.typecheck = `${pm} run typecheck`;
510
513
  else if (scripts['type-check']) discovery.scripts.typecheck = `${pm} run type-check`;
511
514
  else if (scripts['check']) discovery.scripts.typecheck = `${pm} run check`;
515
+ else discovery.scripts.typecheck = hasTypeScript ? 'npx tsc --noEmit' : null;
512
516
 
513
- if (scripts['lint']) discovery.scripts.lint = `${pm} run lint`;
514
- if (scripts['build']) discovery.scripts.build = `${pm} run build`;
517
+ discovery.scripts.lint = scripts['lint'] ? `${pm} run lint` : null;
518
+ discovery.scripts.build = scripts['build'] ? `${pm} run build` : null;
515
519
  if (scripts['test:e2e']) discovery.scripts.e2e = `${pm} run test:e2e`;
516
520
  } catch (e) {}
517
521
  }
@@ -664,9 +668,9 @@ ${discovery.topLevelDirs.map((d) => `- \`${d}/\``).join('\n') || '- Flat directo
664
668
 
665
669
  ## Discovered Verification Commands
666
670
  - **test_unit**: \`${discovery.scripts.testUnit}\`
667
- - **typecheck**: \`${discovery.scripts.typecheck}\`
668
- - **lint**: \`${discovery.scripts.lint}\`
669
- - **build**: \`${discovery.scripts.build}\`
671
+ ${discovery.scripts.typecheck ? `- **typecheck**: \`${discovery.scripts.typecheck}\`` : ''}
672
+ ${discovery.scripts.lint ? `- **lint**: \`${discovery.scripts.lint}\`` : ''}
673
+ ${discovery.scripts.build ? `- **build**: \`${discovery.scripts.build}\`` : ''}
670
674
  ${discovery.scripts.e2e ? `- **e2e**: \`${discovery.scripts.e2e}\`` : ''}
671
675
  ${discovery.scripts.testRelated ? `- **test_related**: \`${discovery.scripts.testRelated}\`` : ''}
672
676
 
@@ -1390,6 +1394,15 @@ function handleRun() {
1390
1394
  const current = getCurrentRun(CWD);
1391
1395
  const policy = loadPolicy(CWD);
1392
1396
  const startsNewRun = forceNew || !current || TERMINAL_STATES.has(current.state);
1397
+ if (forceNew && current && !TERMINAL_STATES.has(current.state)) {
1398
+ // C-15 (L07 WIP=1): starting another Run while one is unfinished.
1399
+ try {
1400
+ const ratio = require('../lib/features.js').vcr(require('../lib/features.js').loadFeatures(CWD));
1401
+ if (ratio.value < 1) console.error(`! VCR ${ratio.verified}/${ratio.activated}: Run ${current.runId} is still ${current.state}. Finishing it first usually beats starting another (WIP=1).`);
1402
+ } catch {
1403
+ // No feature ledger.
1404
+ }
1405
+ }
1393
1406
  let mode = modeArg ? normalizeMode(modeArg) : (startsNewRun ? policy.defaultMode : null);
1394
1407
  let workflow = null;
1395
1408
  let requestedRunId = null;
@@ -2015,6 +2028,7 @@ function handleGate() {
2015
2028
  console.log(`- Run: ${run.runId}`);
2016
2029
  console.log(`- State: ${run.state}, iteration ${run.iteration}`);
2017
2030
  if (run.tier) console.log(`- Tier: ${run.tier.current}`);
2031
+ for (const warning of (run.history || []).at(-1)?.cleanStateWarnings || []) console.log(`! Clean state: ${warning}`);
2018
2032
  if (guardEvents.warn || guardEvents.block) {
2019
2033
  console.log(`- Guard: ${guardEvents.block} blocked, ${guardEvents.warn} warned (${guardEvents.rules.join(', ')}); see guard-events.json`);
2020
2034
  }
@@ -2037,10 +2051,14 @@ function handleGate() {
2037
2051
  ok: false,
2038
2052
  code: err.code || 'GATE_FAILED',
2039
2053
  error: err.message,
2054
+ ...(err.why ? { why: err.why } : {}),
2055
+ ...(err.fix ? { fix: err.fix } : {}),
2040
2056
  details
2041
2057
  }));
2042
2058
  } else {
2043
2059
  log.error(err.message);
2060
+ if (err.why) console.error(`WHY: ${err.why}`);
2061
+ if (err.fix) console.error(`FIX: ${err.fix}`);
2044
2062
  for (const detail of details) console.error(`- ${detail}`);
2045
2063
  }
2046
2064
  process.exit(1);
@@ -2373,6 +2391,16 @@ function handleState() {
2373
2391
 
2374
2392
  function handleDoctor() {
2375
2393
  const json = process.argv.includes('--json');
2394
+ if (process.argv.includes('--harness')) {
2395
+ // C-08: harness debt in this project (warnings never block).
2396
+ const audit = require('../lib/harness-audit.js').auditHarness(CWD);
2397
+ if (json) console.log(JSON.stringify(audit));
2398
+ else {
2399
+ console.log(`Harness audit: ${audit.ok ? 'no debt found' : 'debt found'}`);
2400
+ for (const item of audit.items) console.log(`${item.level === 'warn' ? '!' : '-'} ${item.id}: ${item.detail}`);
2401
+ }
2402
+ return;
2403
+ }
2376
2404
  const result = runDoctor(path.join(__dirname, '..'), { projectDir: CWD });
2377
2405
  if (json) console.log(JSON.stringify(result));
2378
2406
  else {
@@ -3363,9 +3391,11 @@ function splitCommand(text, shell) {
3363
3391
 
3364
3392
  function stageJsonError(err, json, code) {
3365
3393
  if (json) {
3366
- console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, details: err.details || [] }));
3394
+ console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, ...(err.why ? { why: err.why } : {}), ...(err.fix ? { fix: err.fix } : {}), details: err.details || [] }));
3367
3395
  } else {
3368
3396
  log.error(err.message);
3397
+ if (err.why) console.error(`WHY: ${err.why}`);
3398
+ if (err.fix) console.error(`FIX: ${err.fix}`);
3369
3399
  for (const detail of err.details || []) console.error(`- ${detail}`);
3370
3400
  }
3371
3401
  process.exit(1);
@@ -3391,19 +3421,55 @@ async function handleVerify() {
3391
3421
  if (related && final) throw Object.assign(new Error('--related and --final cannot be combined'), { code: 'INVALID_USAGE' });
3392
3422
  let texts = repeatedArg('--cmd');
3393
3423
  const lanes = require('../lib/verification-lanes.js');
3424
+ const baselineMode = process.argv.includes('--baseline');
3425
+ if (baselineMode && (related || final)) throw Object.assign(new Error('--baseline cannot be combined with --related or --final'), { code: 'INVALID_USAGE' });
3394
3426
  let commandsToRun;
3427
+ let defaultPlan = false;
3395
3428
  if (related && texts.length === 0) {
3396
3429
  const target = loadRun(CWD, runId);
3397
- commandsToRun = [lanes.relatedCommand(CWD, target, { split: splitCommand, shell })];
3398
- } else {
3399
- if (texts.length === 0) {
3400
- const fallback = defaultVerificationCommand();
3401
- if (!fallback) throw Object.assign(new Error('No --cmd given and no test_unit command in .ai-engineering-loop/verification.md'), { code: 'INVALID_COMMAND' });
3402
- texts = [fallback];
3430
+ commandsToRun = [{ ...lanes.relatedCommand(CWD, target, { split: splitCommand, shell }), layer: 'L2' }];
3431
+ } else if (texts.length === 0) {
3432
+ // C-02: no --cmd runs the cheapest-first plan (L1 build/typecheck/lint,
3433
+ // then L2 test_unit) and stops at the first failure.
3434
+ const plan = lanes.defaultCommandPlan(CWD);
3435
+ // C-05: promoted review rules run as L1 checks, after the static checks.
3436
+ const rules = require('../lib/rules.js').rulePlan(CWD);
3437
+ if (rules.length) {
3438
+ const firstL2 = plan.findIndex((step) => step.layer !== 'L1');
3439
+ plan.splice(firstL2 === -1 ? plan.length : firstL2, 0, ...rules);
3403
3440
  }
3404
- commandsToRun = texts.map((text) => splitCommand(text, shell));
3441
+ // C-03: startup-risky diffs also run the configured smoke check (L2).
3442
+ const smokeText = lanes.verificationCommands(CWD).smoke;
3443
+ if (smokeText && !baselineMode && lanes.startupRiskFiles(CWD, loadRun(CWD, runId)).length) {
3444
+ plan.push({ key: 'smoke', layer: 'L2', text: smokeText });
3445
+ }
3446
+ if (!plan.length) throw Object.assign(new Error('No --cmd given and no build/typecheck/lint/test_unit command in .ai-engineering-loop/verification.md. FIX: add them to verification.md or pass --cmd.'), { code: 'INVALID_COMMAND' });
3447
+ commandsToRun = plan.map((step) => ({ ...splitCommand(step.text, shell || Boolean(step.shell)), key: step.key, layer: step.layer }));
3448
+ defaultPlan = true;
3449
+ } else {
3450
+ commandsToRun = texts.map((text) => {
3451
+ const layer = lanes.layerOfCommand(CWD, text);
3452
+ return { ...splitCommand(text, shell), ...(layer ? { layer } : {}) };
3453
+ });
3405
3454
  }
3406
3455
  const timeoutValue = argValue('--timeout-ms');
3456
+ if (baselineMode) {
3457
+ const outcome = await require('../lib/baseline.js').recordBaseline(CWD, {
3458
+ runId,
3459
+ plan: commandsToRun,
3460
+ ...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
3461
+ });
3462
+ const result = { ok: outcome.status !== 'BROKEN', runId, baseline: outcome };
3463
+ if (json) console.log(JSON.stringify(result));
3464
+ else {
3465
+ const say = outcome.status === 'GREEN' ? log.success : (outcome.status === 'RED' ? log.warn : log.error);
3466
+ say(`Baseline ${outcome.status}: ${outcome.path}`);
3467
+ for (const c of outcome.commands) console.log(`- ${c.command}: exit ${c.exitCode}`);
3468
+ for (const problem of outcome.problems) console.log(`! ${problem}`);
3469
+ }
3470
+ if (!result.ok) process.exitCode = 1;
3471
+ return;
3472
+ }
3407
3473
  if (final) {
3408
3474
  const target = loadRun(CWD, runId);
3409
3475
  if (target.state !== 'JUDGED_PASS') throw Object.assign(new Error(`verify --final needs a Judge PASS (state is ${target.state})`), { code: 'INVALID_STATE' });
@@ -3437,17 +3503,30 @@ async function handleVerify() {
3437
3503
  commands: commandsToRun,
3438
3504
  serial: process.argv.includes('--serial'),
3439
3505
  fresh: process.argv.includes('--fresh'),
3506
+ stopOnFailure: defaultPlan,
3440
3507
  ...(related ? { scope: 'related' } : {}),
3441
3508
  ...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
3442
3509
  });
3443
- const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
3510
+ const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.layer ? { layer: c.layer } : {}), ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
3444
3511
  if (!recorded.passed) {
3445
3512
  autoHeartbeat(runId, 'verification', 'Verification failed; Maker must fix before review');
3446
- const result = { ok: false, code: 'VERIFICATION_FAILED', runId, commands, gate: null };
3513
+ const baselineModule = require('../lib/baseline.js');
3514
+ const failures = baselineModule.classifyAgainstBaseline(baselineModule.loadBaseline(CWD, runId), recorded.commands);
3515
+ const result = { ok: false, code: 'VERIFICATION_FAILED', runId, commands, skipped: recorded.skipped, failures, gate: null };
3447
3516
  if (json) console.log(JSON.stringify(result));
3448
3517
  else {
3449
3518
  log.error('Verification failed; the gate was not attempted.');
3450
3519
  for (const c of commands) console.log(`- ${c.command}: exit ${c.exitCode} (${c.timeoutStatus})`);
3520
+ for (const skippedCommand of recorded.skipped) console.log(`- ${skippedCommand}: skipped (an earlier, cheaper check failed)`);
3521
+ for (const c of commands.filter((item) => item.exitCode !== 0)) {
3522
+ const rule = require('../lib/rules.js').ruleForCommand(CWD, c.command);
3523
+ if (rule) console.log(`! Rule ${rule.id} violated. WHY: ${rule.why} FIX: ${rule.fix}`);
3524
+ }
3525
+ for (const f of failures) {
3526
+ console.log(f.preExisting
3527
+ ? `! ${f.command}: every failure was already failing at baseline (pre-existing). FIX: repair it in a separate Run, or narrow --cmd to the tests this change owns.`
3528
+ : `! ${f.command}: new failure(s)${f.newFailures.length ? `: ${f.newFailures.slice(0, 5).join('; ')}` : ''}. FIX: fix the change before review.`);
3529
+ }
3451
3530
  }
3452
3531
  process.exitCode = 1;
3453
3532
  return;
@@ -3471,6 +3550,14 @@ async function handleVerify() {
3471
3550
  impactDrift: { mapStatus: drift.mapStatus, outOfMap: drift.outOfMap },
3472
3551
  compact: compact.paths,
3473
3552
  claimedVsReality: claimed,
3553
+ acStatus: (() => {
3554
+ try {
3555
+ return require('../lib/ac-status.js').runAcStatuses(CWD, run, require('../lib/claimed-vs-reality.js').loadFrozenGoal(CWD, run))
3556
+ .filter((ac) => ac.status !== 'unbound');
3557
+ } catch {
3558
+ return [];
3559
+ }
3560
+ })(),
3474
3561
  ...(related ? { scope: 'related', finalRequired: 'Run `verify --final` after Judge PASS, before delivery.' } : {}),
3475
3562
  next: run.tier && run.tier.current === 'T0'
3476
3563
  ? 'Fill claimed-vs-reality, then gate review and gate judge'
@@ -3487,6 +3574,7 @@ async function handleVerify() {
3487
3574
  console.log('! No impact map for this Run; drift was not checked');
3488
3575
  }
3489
3576
  console.log(`- Claimed-vs-reality: ${claimed.path}${claimed.created ? ' (scaffolded; fill it from the command output)' : ''}`);
3577
+ for (const ac of result.acStatus) console.log(`- ${ac.id}: ${ac.status} (\`${ac.command}\`)`);
3490
3578
  console.log(`- Next: ${result.next}`);
3491
3579
  }
3492
3580
  } catch (err) {
@@ -3494,6 +3582,97 @@ async function handleVerify() {
3494
3582
  }
3495
3583
  }
3496
3584
 
3585
+ function handleOutcome() {
3586
+ const json = process.argv.includes('--json');
3587
+ try {
3588
+ const action = args[1] && !args[1].startsWith('--') ? args[1] : null;
3589
+ if (action !== 'record') throw Object.assign(new Error('outcome supports: record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name> [--evidence <path|url>]'), { code: 'INVALID_USAGE' });
3590
+ const entry = require('../lib/calibration.js').recordOutcome(CWD, {
3591
+ runId: argValue('--run'),
3592
+ source: argValue('--source'),
3593
+ location: argValue('--location'),
3594
+ summary: argValue('--summary'),
3595
+ evidence: argValue('--evidence'),
3596
+ by: argValue('--by')
3597
+ });
3598
+ if (json) console.log(JSON.stringify({ ok: true, outcome: entry }));
3599
+ else log.success(`✓ outcome recorded for ${argValue('--run')}; run \`calibrate\` to compare it with the review`);
3600
+ } catch (err) {
3601
+ stageJsonError(err, json, 'OUTCOME_FAILED');
3602
+ }
3603
+ }
3604
+
3605
+ function handleCalibrate() {
3606
+ const json = process.argv.includes('--json');
3607
+ try {
3608
+ const calibration = require('../lib/calibration.js');
3609
+ const report = calibration.calibrate(CWD);
3610
+ if (json) console.log(JSON.stringify({ ok: true, ...report }));
3611
+ else console.log(calibration.formatCalibration(report));
3612
+ } catch (err) {
3613
+ stageJsonError(err, json, 'CALIBRATE_FAILED');
3614
+ }
3615
+ }
3616
+
3617
+ function handleDebt() {
3618
+ const json = process.argv.includes('--json');
3619
+ try {
3620
+ const debt = require('../lib/debt.js');
3621
+ const action = args[1] && !args[1].startsWith('--') ? args[1] : 'list';
3622
+ if (action === 'close') {
3623
+ const item = debt.closeDebt(CWD, args[2], { by: argValue('--by'), note: argValue('--note') });
3624
+ if (json) console.log(JSON.stringify({ ok: true, item }));
3625
+ else log.success(`✓ closed ${item.id}`);
3626
+ return;
3627
+ }
3628
+ if (action !== 'list') throw Object.assign(new Error('debt supports: list, close <id> --by <name>'), { code: 'INVALID_USAGE' });
3629
+ const file = argValue('--file');
3630
+ const items = debt.loadDebt(CWD).items.filter((item) => item.status === 'open' && (!file || item.file === file));
3631
+ if (json) console.log(JSON.stringify({ ok: true, items }));
3632
+ else if (!items.length) console.log('No open debt.');
3633
+ else for (const item of items) console.log(`- ${item.id} [${item.risk}] ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; revisit: ${item.nextTrigger})`);
3634
+ } catch (err) {
3635
+ stageJsonError(err, json, 'DEBT_FAILED');
3636
+ }
3637
+ }
3638
+
3639
+ function handleNext() {
3640
+ const json = process.argv.includes('--json');
3641
+ try {
3642
+ const features = require('../lib/features.js');
3643
+ const document = features.loadFeatures(CWD);
3644
+ if (!document) throw Object.assign(new Error('No .ai-engineering-loop/features.json. FIX: create it with {"schemaVersion":1,"features":[{"id","title","behavior","verify","status":"not_started","priority","dependsOn":[]}]}.'), { code: 'NO_FEATURES' });
3645
+ const next = features.nextFeature(document);
3646
+ const ratio = features.vcr(document);
3647
+ const problems = features.auditFeatures(CWD, document);
3648
+ const active = document.features.filter((item) => item.status === 'active').map((item) => item.id);
3649
+ const result = { ok: problems.length === 0, next, vcr: ratio, active, problems };
3650
+ if (json) console.log(JSON.stringify(result));
3651
+ else {
3652
+ console.log(`VCR: ${ratio.verified}/${ratio.activated} (${ratio.value})${active.length ? `; active: ${active.join(', ')}` : ''}`);
3653
+ if (ratio.value < 1 && active.length) console.log(`! Finish ${active[0]} before starting another feature (WIP=1).`);
3654
+ console.log(next ? `Next: ${next.id} - ${next.title || next.behavior}${next.verify ? ` (verify: ${next.verify})` : ''}. Start it with a Goal whose featureId is "${next.id}".` : 'Next: nothing unblocked.');
3655
+ for (const problem of problems) console.log(`! ${problem}`);
3656
+ }
3657
+ if (problems.length) process.exitCode = 1;
3658
+ } catch (err) {
3659
+ stageJsonError(err, json, 'NEXT_FAILED');
3660
+ }
3661
+ }
3662
+
3663
+ function handleStats() {
3664
+ const json = process.argv.includes('--json');
3665
+ try {
3666
+ const lastValue = argValue('--last');
3667
+ const stats = require('../lib/stats.js');
3668
+ const result = stats.computeStats(CWD, lastValue == null ? {} : { last: parsePositiveIntegerArg('--last', lastValue) });
3669
+ if (json) console.log(JSON.stringify({ ok: true, ...result }));
3670
+ else console.log(stats.formatStats(result));
3671
+ } catch (err) {
3672
+ stageJsonError(err, json, 'STATS_FAILED');
3673
+ }
3674
+ }
3675
+
3497
3676
  function readGuardInput() {
3498
3677
  const source = argValue('--input') || (process.argv.includes('--hook') ? '-' : null);
3499
3678
  if (source == null) {
@@ -3639,6 +3818,20 @@ function handleReviewPack() {
3639
3818
  const pack = smartContextSummary(createDiffHunkPack(CWD, { runId, base, profile: argValue('--profile') }));
3640
3819
  const escalation = assessRunEscalation(CWD, { runId });
3641
3820
  const tier = run.tier ? run.tier.current : null;
3821
+ // C-14: open debt on the files this change touches goes into the pack.
3822
+ let openDebt = [];
3823
+ try {
3824
+ const changed = require('../lib/verification-lanes.js').relatedFiles(CWD, run);
3825
+ openDebt = require('../lib/debt.js').openDebtFor(CWD, changed);
3826
+ if (openDebt.length && pack.path) {
3827
+ const lines = ['', '## Open debt on changed files', '', ...openDebt.map((item) => `- ${item.id} ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; risk ${item.risk})`)];
3828
+ fs.appendFileSync(path.join(CWD, pack.path), `${lines.join('\n')}\n`);
3829
+ }
3830
+ } catch {
3831
+ openDebt = [];
3832
+ }
3833
+ const diffRelative = path.relative(CWD, path.join(runsDir(CWD), runId, 'diff.patch')).split(path.sep).join('/');
3834
+ const reviewInputs = require('../lib/review-ingest.js').recordReviewInputs(CWD, runId, { packPath: pack.path, diffPath: diffRelative });
3642
3835
  autoHeartbeat(runId, 'review', 'Review pack built');
3643
3836
  const result = {
3644
3837
  ok: true,
@@ -3652,6 +3845,8 @@ function handleReviewPack() {
3652
3845
  truncated: pack.truncated,
3653
3846
  unresolvedFindings: pack.unresolvedFindings,
3654
3847
  impactDrift: pack.impactDrift,
3848
+ openDebt: openDebt.map((item) => ({ id: item.id, location: item.location, debt: item.debt })),
3849
+ reviewInputs: { inputs: reviewInputs.inputs, promptSha256: reviewInputs.promptSha256 },
3655
3850
  fastPath: { eligible: fastPath.eligible, reasons: fastPath.reasons },
3656
3851
  escalation: { requiredTier: escalation.requiredTier, reasons: escalation.reasons.map((r) => r.code) }
3657
3852
  };
@@ -3665,6 +3860,7 @@ function handleReviewPack() {
3665
3860
  console.log(`- Outside the impact map: ${result.impactDrift.outOfMap.join(', ')} (check their side effects)`);
3666
3861
  }
3667
3862
  for (const line of result.impactDrift?.design || []) console.log(`- ${line}`);
3863
+ if (openDebt.length) console.log(`- Open debt on changed files: ${openDebt.map((item) => item.id).join(', ')} (in the pack; resolve or keep consciously)`);
3668
3864
  }
3669
3865
  } catch (err) {
3670
3866
  stageJsonError(err, json, 'REVIEW_PACK_FAILED');
@@ -3825,6 +4021,7 @@ function handleBench() {
3825
4021
  sandboxHome: process.argv.includes('--sandbox-home'),
3826
4022
  hostArgs: repeatedArg('--host-arg'),
3827
4023
  scenarioIds: repeatedArg('--scenario'),
4024
+ without: repeatedArg('--without').flatMap((value) => value.split(',')).map((value) => value.trim()).filter(Boolean),
3828
4025
  repeat: repeatArg == null ? 1 : parsePositiveIntegerArg('--repeat', repeatArg),
3829
4026
  timeoutMs: timeoutArg == null ? undefined : parseBenchTimeoutSeconds(timeoutArg),
3830
4027
  mode: modeArg == null ? 'ASSISTED' : normalizeMode(modeArg),
@@ -3930,6 +4127,22 @@ function handleQuick() {
3930
4127
 
3931
4128
  function handleLearn() {
3932
4129
  const json = process.argv.includes('--json');
4130
+ if (args[1] === 'promote') {
4131
+ try {
4132
+ const rule = require('../lib/rules.js').promoteLearning(CWD, args[2], {
4133
+ check: argValue('--check'),
4134
+ fix: argValue('--fix'),
4135
+ why: argValue('--why'),
4136
+ by: argValue('--by'),
4137
+ shell: !process.argv.includes('--no-shell')
4138
+ });
4139
+ if (json) console.log(JSON.stringify({ ok: true, rule }));
4140
+ else log.success(`✓ ${rule.from} promoted to rule ${rule.id}; default \`verify\` now runs: ${rule.check}`);
4141
+ } catch (err) {
4142
+ stageJsonError(err, json, 'LEARN_PROMOTE_FAILED');
4143
+ }
4144
+ return;
4145
+ }
3933
4146
  try {
3934
4147
  const maxAge = argValue('--max-age-runs');
3935
4148
  const result = learn(CWD, {
@@ -3938,8 +4151,9 @@ function handleLearn() {
3938
4151
  ...(maxAge == null ? {} : { maxAgeRuns: parsePositiveIntegerArg('--max-age-runs', maxAge) })
3939
4152
  });
3940
4153
  const { document } = result;
4154
+ const candidates = require('../lib/rules.js').promotionCandidates(CWD, document.entries);
3941
4155
  if (json) {
3942
- console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, learnings: document }));
4156
+ console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, promotionCandidates: candidates, learnings: document }));
3943
4157
  return;
3944
4158
  }
3945
4159
  log.success(`✓ ${document.entries.length} learning(s) from ${document.sourceRuns} Run(s)${result.written ? ` → ${result.path}` : ' (dry run, nothing written)'}`);
@@ -3947,6 +4161,9 @@ function handleLearn() {
3947
4161
  console.log(`- ${entry.id} [${entry.axis}, ${entry.severity}, ×${entry.occurrences}] ${entry.lesson}`);
3948
4162
  }
3949
4163
  for (const item of result.pruned) console.log(` pruned ${item.id} (${item.reason})`);
4164
+ for (const item of candidates) {
4165
+ console.log(`! Promote candidate ${item.id} (seen in ${item.runs} Runs): ${item.lesson}. Make it a check: learn promote ${item.id} --check "<command>" --fix "<instruction>" --by <name>`);
4166
+ }
3950
4167
  } catch (err) {
3951
4168
  stageJsonError(err, json, 'LEARN_FAILED');
3952
4169
  }
@@ -4083,6 +4300,12 @@ Commands:
4083
4300
  A missing binary is UNAVAILABLE, not a model failure.
4084
4301
  Progress lines go to stderr. Writes .ai-engineering-loop/bench/<benchId>/{results.json,report.md,hardening.md}
4085
4302
  bench compare <benchIdA> <benchIdB> [--json] read-only side-by-side table
4303
+ outcome record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name>
4304
+ a post-delivery problem traced to a Run (input for calibrate)
4305
+ calibrate Compare recorded outcomes with each Run's Finding Ledger: DA misses, wrong dismissals, per reviewer prompt
4306
+ debt Accepted trade-offs and L3 waivers recorded at delivery: list [--file <path>] | close <id> --by <name> [--note <text>]
4307
+ next Next unblocked feature from .ai-engineering-loop/features.json, VCR (verified/activated), evidence audit [--json]
4308
+ stats Failure causes by harness layer, false-victory rate, iterations to PASS [--last <n>] [--json]
4086
4309
  escalation Compute the minimum Judge model tier from deterministic risk signals
4087
4310
  --run <id> target a non-current run
4088
4311
  --json print machine-readable reasons
@@ -4094,7 +4317,8 @@ Commands:
4094
4317
  start "<task>" [--mode assisted|unattended] | finish [--run <id>] [--cmd "<command>"]...
4095
4318
  verify Record verification commands, gate them, compact, and scaffold claimed-vs-reality
4096
4319
  [--run <id>] [--cmd "<command>"]... [--shell] [--serial] [--timeout-ms <n>] [--fresh] [--json]
4097
- default command: test_unit from .ai-engineering-loop/verification.md; commands run in parallel
4320
+ default (no --cmd): build -> typecheck -> lint -> test_unit from verification.md, serial, stops at first failure
4321
+ --baseline before the Maker gate: run the plan on the untouched tree, write baseline.json
4098
4322
  --related fast lane: test_related with {files} = changed files of the gated diff
4099
4323
  --final after Judge PASS: full run delivery needs when verification was related-only
4100
4324
  --fresh re-run commands that already passed for the same diff and revision
@@ -4122,6 +4346,8 @@ Commands:
4122
4346
  writes RUN_DIR/receipt.json + receipt.md (paste into the PR/MR body); digest stored in delivery.json
4123
4347
  verify-receipt Re-check a delivery receipt: artifact hashes, frozen Goal, AC evidence, verdict, head SHA
4124
4348
  [--run <id> | <repository-relative-receipt.json>] [--head <sha>] [--json] exit 1 on mismatch (CI)
4349
+ learn promote <learningId> --check "<command>" --fix "<instruction>" --by <name> [--why <text>] [--no-shell]
4350
+ turn a recurring review lesson into an executable L1 rule (.ai-engineering-loop/rules.json)
4125
4351
  learn Mine past Runs' VALID findings into .ai-engineering-loop/learnings.json (+ .md) for context packs
4126
4352
  [--dry-run] [--prune] [--max-age-runs <n>] [--json]
4127
4353
  deterministic and deduped with runId provenance; --prune drops lessons whose files are gone
@@ -4130,6 +4356,7 @@ Commands:
4130
4356
  list [--json]
4131
4357
  <name> [--section <heading>] [--json] e.g. docs grill-policy --section "When to grill"
4132
4358
  doctor Check runtime, schemas, host assets, package contents, and eval catalog
4359
+ --harness audit harness debt instead: entry-file size/structure, stale learnings, rule provenance, skill budget
4133
4360
  --json print machine-readable diagnostics
4134
4361
  update Check or apply the package update workflow
4135
4362
  check|plan [--json]
@@ -4295,6 +4522,21 @@ switch (command) {
4295
4522
  case 'escalation':
4296
4523
  handleEscalation();
4297
4524
  break;
4525
+ case 'stats':
4526
+ handleStats();
4527
+ break;
4528
+ case 'next':
4529
+ handleNext();
4530
+ break;
4531
+ case 'debt':
4532
+ handleDebt();
4533
+ break;
4534
+ case 'outcome':
4535
+ handleOutcome();
4536
+ break;
4537
+ case 'calibrate':
4538
+ handleCalibrate();
4539
+ break;
4298
4540
  case 'docs':
4299
4541
  handleDocs();
4300
4542
  break;
package/core/bench.md CHANGED
@@ -75,3 +75,18 @@ Bench results do not feed `ael learn`. Learnings stay findings mined from Runs;
75
75
  `bench compare <a> <b>` prints a side-by-side table and does not write.
76
76
 
77
77
  A live run against your own host is manual evidence for a change review. CI uses the fake host only.
78
+
79
+ ## Ablation and the simplification log (C-08)
80
+
81
+ Every harness component encodes an assumption about what the model cannot do alone; as models improve, some assumptions go stale. Measure instead of guessing:
82
+
83
+ ```bash
84
+ npx ai-engineering-loop bench --host <host> --model <model> # baseline
85
+ npx ai-engineering-loop bench --host <host> --model <model> --without integrity # one component off
86
+ npx ai-engineering-loop bench compare <baselineId> <ablationId>
87
+ ```
88
+
89
+ `--without` accepts `integrity`, `l3`, `l1`, `smoke`, `rules` (repeat or comma-separate). It only takes effect inside bench scenario repos: bench sets `AEL_ABLATE` and writes `.ai-engineering-loop/bench-ablation.json`; the gates honour the switch only when both are present together with `AEL_BENCH_MODE`, and every skip is logged to the Run's `ablation.log`. The result carries `ablation.without`, and an entry is appended to `.ai-engineering-loop/simplification-log.json` (`decision: null` until a human records keep / remove / replace).
90
+
91
+ `doctor --harness` audits harness debt in the project: entry-file size (50-200 lines) and structure (only headings, lists, tables and code count, so keywords in prose cannot game it), learnings not seen in the last 10 Runs, rules without provenance or older than 90 days, and, inside the AEL repo, T1 skill-context headroom from `scripts/measure.js`.
92
+
@@ -113,3 +113,38 @@ Every single item listed under `Acceptance Criteria` must map to at least one co
113
113
  - **The Wrong Artifact**: Tests pass on a `/tmp` build while the sample the user opens omits the feature.
114
114
  - **The Happy-Path Contract**: Only the sunny case is an AC. Coverage % is not a substitute for a failure table.
115
115
  - **The Sloppy Ship**: Implementing from a vague prompt without a drafted failure table in the Goal Contract file.
116
+
117
+ ## Per-AC verify command (C-07)
118
+
119
+ An AC may name the command that proves it:
120
+
121
+ ```json
122
+ { "id": "AC-2", "statement": "page 0 is rejected", "evidenceRequired": "unit test", "failureCases": ["page 0 returns items"],
123
+ "verify": { "command": "npm test -- list.test.js", "layer": "L2" } }
124
+ ```
125
+
126
+ The harness owns the status: `proven` when that exact command passed on the gated diff (`verification.json` or `final-verification.json`), `failed` when it ran and failed, `pending` when it never ran. `verify` prints each status, and `gate judge` refuses PASS while any AC with a verify command is `pending` or `failed` (`AC_NOT_PROVEN`). ACs without `verify` keep the existing evidence rules.
127
+
128
+ ## Rubric for subjective ACs (C-09)
129
+
130
+ For ACs that are judged rather than computed (layout, copy, visual design), add a rubric:
131
+
132
+ ```json
133
+ { "id": "AC-3", "statement": "The empty state guides the user", "evidenceRequired": "screenshot", "failureCases": ["blank panel"],
134
+ "rubric": [ { "criterion": "contrast", "pass": ">= 4.5:1 (WCAG AA)" }, { "criterion": "guidance", "pass": "explains how to import a document" } ] }
135
+ ```
136
+
137
+ The Devil's Advocate scores every criterion in the ledger as `rubricScores: [{ac, criterion, pass, evidence}]` with observed evidence (screenshot path, DOM snapshot, measured value). `gate judge` refuses PASS while any criterion is unscored or `pass: false` (`AC_RUBRIC`).
138
+
139
+ ## Project feature ledger (C-15)
140
+
141
+ Optional `.ai-engineering-loop/features.json` lists what the project must do, one triple per feature:
142
+
143
+ ```json
144
+ { "schemaVersion": 1, "features": [
145
+ { "id": "F-1", "title": "Import", "behavior": "user imports a .md file and sees it in the list", "verify": "npm run e2e -- import", "status": "not_started", "priority": 1, "dependsOn": [] }
146
+ ] }
147
+ ```
148
+
149
+ Status moves only through Runs: a Goal Contract with `"featureId": "F-1"` makes it `active` at the Goal gate, and delivery makes it `passing` with the Run as evidence (passing is final). `ael next` prints the highest-priority unblocked feature, VCR (verified / activated), and flags any `passing` feature without a delivered Run behind it. `run --new` warns when an unfinished Run exists and VCR is below 1 (WIP=1).
150
+