ai-engineering-loop 2.6.0 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/.agents/skills/ael/SKILL.md +1 -1
  2. package/.agents/skills/ael/references/loop-commands.md +11 -0
  3. package/.agents/skills/ael/references/loop-goal.md +2 -0
  4. package/.agents/skills/ael/references/loop-maker.md +1 -6
  5. package/.agents/skills/ael/references/loop-report.md +8 -0
  6. package/.agents/skills/ael/references/loop-verify.md +11 -9
  7. package/.claude/agents/devil-advocate.md +6 -1
  8. package/.claude/agents/judge.md +3 -1
  9. package/.grok/agents/devil-advocate.md +6 -1
  10. package/.grok/agents/judge.md +3 -1
  11. package/CHANGELOG.md +16 -0
  12. package/agents/devil-advocate.md +8 -0
  13. package/agents/judge.md +1 -0
  14. package/agents/shared/devil-advocate.body.md +6 -1
  15. package/agents/shared/judge.body.md +3 -1
  16. package/bin/ai-engineering-loop.js +262 -20
  17. package/core/bench.md +15 -0
  18. package/core/goal-contract.md +35 -0
  19. package/core/verification-loop.md +78 -1
  20. package/lib/ablation.js +52 -0
  21. package/lib/ac-status.js +63 -0
  22. package/lib/baseline.js +138 -0
  23. package/lib/bench.js +24 -1
  24. package/lib/calibration.js +112 -0
  25. package/lib/claimed-vs-reality.js +106 -12
  26. package/lib/clean-state.js +85 -0
  27. package/lib/cli-args.js +5 -3
  28. package/lib/debt.js +110 -0
  29. package/lib/features.js +95 -0
  30. package/lib/gates.js +234 -41
  31. package/lib/harness-audit.js +124 -0
  32. package/lib/no-progress.js +6 -1
  33. package/lib/review-ingest.js +54 -2
  34. package/lib/rules.js +112 -0
  35. package/lib/stats.js +90 -0
  36. package/lib/sync-hosts.js +1 -0
  37. package/lib/verification-integrity.js +164 -0
  38. package/lib/verification-lanes.js +64 -0
  39. package/lib/verification-recorder.js +17 -5
  40. package/package.json +1 -1
  41. package/schemas/finding-ledger.schema.json +14 -0
  42. package/schemas/goal-contract.schema.json +31 -0
  43. package/schemas/verdict.schema.json +2 -1
package/lib/gates.js CHANGED
@@ -51,7 +51,7 @@ const {
51
51
  enforceVerdictTier
52
52
  } = require('./escalation.js');
53
53
  const { raiseTierFromDiff } = require('./tier.js');
54
- const { readyForDa } = require('./claimed-vs-reality.js');
54
+ const { checkClaimedVsReality, loadFrozenGoal } = require('./claimed-vs-reality.js');
55
55
  const { runMaxIterations, detectNoProgress, recordIteration } = require('./no-progress.js');
56
56
  const { finalVerificationStatus } = require('./verification-lanes.js');
57
57
  const { checkLedgerBinding } = require('./review-ingest.js');
@@ -68,10 +68,14 @@ const GATE_FILES = Object.freeze({
68
68
  delivery: 'delivery.json'
69
69
  });
70
70
 
71
- function gateError(message, details = []) {
71
+ // C-04: every gate failure says what failed (message), why the gate cares
72
+ // (why), and the next concrete step (fix), so an agent can self-correct.
73
+ function gateError(message, details = [], { why = null, fix = null } = {}) {
72
74
  const err = new Error(message);
73
75
  err.code = 'GATE_FAILED';
74
76
  err.details = details;
77
+ if (why) err.why = why;
78
+ if (fix) err.fix = fix;
75
79
  return err;
76
80
  }
77
81
 
@@ -85,7 +89,7 @@ function readJsonArtifact(rootDir, runId, fileName) {
85
89
  try {
86
90
  value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
87
91
  } catch (cause) {
88
- throw gateError(`Missing or invalid ${fileName}: ${cause.message}`);
92
+ throw gateError(`Missing or invalid ${fileName}: ${cause.message}`, [], { why: 'Every gate reads its artifact from the Run directory; without it there is nothing to check.', fix: `Write ${fileName} in the Run directory (\`scaffold\` can create a template), then retry the gate.` });
89
93
  }
90
94
  return { filePath, value };
91
95
  }
@@ -94,7 +98,7 @@ function hashFile(filePath) {
94
98
  try {
95
99
  return hashContent(fs.readFileSync(filePath));
96
100
  } catch (cause) {
97
- throw gateError(`Unable to hash ${filePath}: ${cause.message}`);
101
+ throw gateError(`Unable to hash ${filePath}: ${cause.message}`, [], { why: 'Gates bind artifacts by sha256 so later edits are detected.', fix: `Check that ${filePath} exists and is readable, then retry.` });
98
102
  }
99
103
  }
100
104
 
@@ -108,11 +112,11 @@ function descriptor(rootDir, filePath) {
108
112
  function assertArtifactCurrent(rootDir, run, name) {
109
113
  const artifact = run.artifacts[name];
110
114
  if (!artifact?.path || !artifact.sha256) {
111
- throw gateError(`Required ${name} artifact has not passed its gate`);
115
+ throw gateError(`Required ${name} artifact has not passed its gate`, [], { why: 'Stages run in order; a later gate cannot trust an earlier artifact that never passed.', fix: `Pass the ${name} gate first (\`gate <stage> --run <id>\`).` });
112
116
  }
113
117
  const filePath = path.join(rootDir, artifact.path);
114
118
  if (hashFile(filePath) !== artifact.sha256) {
115
- throw gateError(`${name} artifact changed after its gate; rerun the owning stage`);
119
+ throw gateError(`${name} artifact changed after its gate; rerun the owning stage`, [], { why: 'Gated artifacts are frozen by hash; a silent edit would invalidate later evidence.', fix: `Re-run the stage that owns ${name}, or restore the gated version.` });
116
120
  }
117
121
  }
118
122
 
@@ -120,7 +124,7 @@ function requireRunMetadata(document, run, label) {
120
124
  const errors = [];
121
125
  if (document.schemaVersion !== 1) errors.push('schemaVersion must be 1');
122
126
  if (document.runId !== run.runId) errors.push(`runId must be ${run.runId}`);
123
- if (errors.length) throw gateError(`${label} metadata is invalid`, errors);
127
+ if (errors.length) throw gateError(`${label} metadata is invalid`, errors, { why: 'Artifacts must name the Run they belong to so evidence cannot be copied across Runs.', fix: 'Set schemaVersion, runId and the other listed fields to this Run\'s values.' });
124
128
  }
125
129
 
126
130
  function validateGoalContract(contract) {
@@ -200,6 +204,18 @@ function validateVerificationBundle(bundle, run, diffHash, currentRevision) {
200
204
  return { valid: errors.length === 0, errors };
201
205
  }
202
206
 
207
+ // C-12 (learn-harness-engineering L01/L02): which harness layer a failed
208
+ // iteration points at. `model` = an implementation defect the harness could
209
+ // not have prevented.
210
+ const FAILURE_CAUSES = Object.freeze(['task_spec', 'context', 'environment', 'verification', 'state', 'model']);
211
+
212
+ function inferCause(verdict, blockingFindings = []) {
213
+ if (verdict === 'PASS') return null;
214
+ if (blockingFindings.some((finding) => finding.source === 'verification-integrity')) return 'verification';
215
+ if (verdict === 'ESCALATE') return 'task_spec';
216
+ return 'model';
217
+ }
218
+
203
219
  function validateVerdictDocument(verdict, run) {
204
220
  const errors = [];
205
221
  requireRunMetadata(verdict, run, 'verdict');
@@ -214,6 +230,9 @@ function validateVerdictDocument(verdict, run) {
214
230
  if (verdict.humanApproved != null && typeof verdict.humanApproved !== 'boolean') {
215
231
  errors.push('humanApproved must be a boolean');
216
232
  }
233
+ if (verdict.cause != null && !FAILURE_CAUSES.includes(verdict.cause)) {
234
+ errors.push(`cause must be one of ${FAILURE_CAUSES.join(', ')}`);
235
+ }
217
236
  return { valid: errors.length === 0, errors };
218
237
  }
219
238
 
@@ -228,20 +247,117 @@ function verificationForJudge(bundle) {
228
247
  function resolveRun(rootDir, runId) {
229
248
  if (runId) return loadRun(rootDir, runId);
230
249
  const current = getCurrentRun(rootDir);
231
- if (!current) throw gateError('No current run. Start one with `ai-engineering-loop run <task>`.');
250
+ if (!current) throw gateError('No current run. Start one with `ai-engineering-loop run <task>`.', [], { why: 'Gates act on a Run.', fix: 'Start a Run with `run <task>` or pass --run <id>.' });
232
251
  return current;
233
252
  }
234
253
 
254
+ // C-02: at T1/T2, when the project configures typecheck or lint, the gated
255
+ // bundle must contain at least one L1 (static) command. Unit tests alone do
256
+ // not prove the change compiles or passes lint.
257
+ function missingStaticLayer(rootDir, run, bundle) {
258
+ const tier = tierOf(run);
259
+ if (tier !== 'T1' && tier !== 'T2') return null;
260
+ if (require('./ablation.js').ablated(rootDir, 'l1', run.runId)) return null;
261
+ const lanes = require('./verification-lanes.js');
262
+ const configured = lanes.verificationCommands(rootDir);
263
+ const staticKeys = ['typecheck', 'lint'].filter((key) => configured[key] && !lanes.isMutatingCommand(configured[key]));
264
+ if (!staticKeys.length) return null;
265
+ // The related lane is the iteration fast path; `verify --final` runs the full plan.
266
+ if ((bundle.commands || []).length && bundle.commands.every((record) => record.scope === 'related')) return null;
267
+ const hasStatic = (bundle.commands || []).some((record) =>
268
+ (record.layer || lanes.layerOfCommand(rootDir, record.command)) === 'L1');
269
+ if (hasStatic) return null;
270
+ return `No L1 static check in the verification bundle, but verification.md configures ${staticKeys.join(' and ')}. FIX: run \`verify --run ${run.runId}\` without --cmd (it runs L1 first), or add --cmd "${configured[staticKeys[0]]}".`;
271
+ }
272
+
273
+ // C-03: when verification.md configures `smoke` and the diff touches
274
+ // startup-risky files, a passing smoke record must be in the bundle.
275
+ function missingSmoke(rootDir, run, bundle) {
276
+ const lanes = require('./verification-lanes.js');
277
+ const smokeText = lanes.verificationCommands(rootDir).smoke;
278
+ if (!smokeText) return null;
279
+ if (require('./ablation.js').ablated(rootDir, 'smoke', run.runId)) return null;
280
+ const risky = lanes.startupRiskFiles(rootDir, run);
281
+ if (!risky.length) return null;
282
+ const base = smokeText.trim();
283
+ if ((bundle.commands || []).some((record) => String(record.command || '').trim() === base)) return null;
284
+ return `The diff touches startup-risky files (${risky.slice(0, 3).join(', ')}${risky.length > 3 ? ', ...' : ''}) but the smoke check did not run. FIX: run \`verify --run ${run.runId}\` without --cmd, or add --cmd "${base}".`;
285
+ }
286
+
287
+ function rubricGaps(rootDir, run) {
288
+ let goal;
289
+ try {
290
+ goal = loadFrozenGoal(rootDir, run);
291
+ } catch {
292
+ return [];
293
+ }
294
+ let ledger = null;
295
+ try {
296
+ ledger = readJsonArtifact(rootDir, run.runId, GATE_FILES.review).value;
297
+ } catch {
298
+ ledger = null;
299
+ }
300
+ return require('./ac-status.js').rubricProblems(goal, ledger);
301
+ }
302
+
303
+ // C-07: ACs with verify.command must be proven by a passing record.
304
+ function unprovenCriteria(rootDir, run) {
305
+ let goal;
306
+ try {
307
+ goal = loadFrozenGoal(rootDir, run);
308
+ } catch {
309
+ return [];
310
+ }
311
+ return require('./ac-status.js').runAcStatuses(rootDir, run, goal)
312
+ .filter((ac) => ac.status === 'pending' || ac.status === 'failed');
313
+ }
314
+
235
315
  function tierOf(run) {
236
316
  return run?.tier?.current || null;
237
317
  }
238
318
 
239
- function claimedVsRealityReady(rootDir) {
319
+ // Layers proven by recorded verification (gated bundle + final run). A
320
+ // command not mapped in verification.md counts as L2 (a test run).
321
+ function recordedLayers(rootDir, run) {
322
+ if (!run) return null;
323
+ const lanes = require('./verification-lanes.js');
324
+ const layers = new Set();
325
+ for (const file of ['verification.json', 'final-verification.json']) {
326
+ let bundle;
327
+ try {
328
+ bundle = JSON.parse(fs.readFileSync(artifactPath(rootDir, run.runId, file), 'utf8'));
329
+ } catch {
330
+ continue;
331
+ }
332
+ for (const record of bundle.commands || []) {
333
+ if (record.exitCode !== 0) continue;
334
+ layers.add(record.layer || lanes.layerOfCommand(rootDir, record.command) || 'L2');
335
+ }
336
+ }
337
+ return layers.size ? layers : null;
338
+ }
339
+
340
+ // Status of claimed-vs-reality.md for this run, including the three-layer
341
+ // (L1/L2/L3) requirement from the frozen Goal Contract and the run tier.
342
+ function claimedVsRealityStatus(rootDir, run) {
343
+ let markdown;
240
344
  try {
241
- return readyForDa(fs.readFileSync(path.join(rootDir, CLAIMED_VS_REALITY_PATH), 'utf8'));
345
+ markdown = fs.readFileSync(path.join(rootDir, CLAIMED_VS_REALITY_PATH), 'utf8');
242
346
  } catch {
243
- return false;
347
+ return { ready: false, problems: [`${CLAIMED_VS_REALITY_PATH} is missing. FIX: scaffold it with \`context claimed-vs-reality --run ${run?.runId || '<id>'}\` after \`gate verification\`, then fill every row.`] };
244
348
  }
349
+ let goal = null;
350
+ try {
351
+ goal = run ? loadFrozenGoal(rootDir, run) : null;
352
+ } catch {
353
+ goal = null;
354
+ }
355
+ if (require('./ablation.js').ablated(rootDir, 'l3', run?.runId)) return checkClaimedVsReality(markdown, {});
356
+ return checkClaimedVsReality(markdown, { goal, tier: tierOf(run), layers: recordedLayers(rootDir, run) });
357
+ }
358
+
359
+ function claimedVsRealityReady(rootDir, run) {
360
+ return claimedVsRealityStatus(rootDir, run).ready;
245
361
  }
246
362
 
247
363
  // T0 has no Devil's Advocate. The CLI writes an explicit empty, diff-bound
@@ -250,10 +366,9 @@ function claimedVsRealityReady(rootDir) {
250
366
  function writeT0FindingLedger(rootDir, run) {
251
367
  const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
252
368
  if (fs.existsSync(filePath)) return false;
253
- if (!claimedVsRealityReady(rootDir)) {
254
- throw gateError('T0 review needs a completed claimed-vs-reality file', [
255
- `Fill every Claimed and Reality cell in ${CLAIMED_VS_REALITY_PATH} (scaffold: context claimed-vs-reality)`
256
- ]);
369
+ const status = claimedVsRealityStatus(rootDir, run);
370
+ if (!status.ready) {
371
+ throw gateError('T0 review needs a completed claimed-vs-reality file', status.problems, { why: 'T0 has no Devil\'s Advocate, so claimed-vs-reality is the only check that claims match evidence.', fix: 'Fill every Claimed and Reality cell (scaffold: `context claimed-vs-reality`), then retry `gate review`.' });
257
372
  }
258
373
  fs.writeFileSync(filePath, `${JSON.stringify({
259
374
  schemaVersion: 1,
@@ -294,7 +409,7 @@ function judgeRequirement(rootDir, run) {
294
409
  } catch (cause) {
295
410
  reasons.push(`ESCALATION_UNAVAILABLE: ${cause.message}`);
296
411
  }
297
- if (!claimedVsRealityReady(rootDir)) reasons.push('CLAIMED_VS_REALITY_INCOMPLETE');
412
+ if (!claimedVsRealityReady(rootDir, run)) reasons.push('CLAIMED_VS_REALITY_INCOMPLETE');
298
413
  return { required: reasons.length > 0, tier, reasons };
299
414
  }
300
415
 
@@ -335,9 +450,9 @@ function expectedJudgeVerdict(rootDir, run, { judgeFindings = [] } = {}) {
335
450
 
336
451
  function validateJudgeFindings(value) {
337
452
  if (value == null) return [];
338
- if (!Array.isArray(value)) throw gateError('Judge gate failed', ['judgeFindings must be an array']);
453
+ if (!Array.isArray(value)) throw gateError('Judge gate failed', ['judgeFindings must be an array'], { why: 'Judge-added findings use the Finding Ledger shape so the gate can recompute the verdict.', fix: 'Make judgeFindings an array of ledger findings (or remove it).' });
339
454
  const check = validateFindingLedger({ findings: value });
340
- if (!check.valid) throw gateError('Judge gate failed', [`judgeFindings: ${check.reason}`]);
455
+ if (!check.valid) throw gateError('Judge gate failed', [`judgeFindings: ${check.reason}`], { why: 'Judge-added findings must be complete enough to act on.', fix: 'Give each judgeFinding location, failureScenario, evidence, severity, validity and (for VALID BLOCKER/HIGH) concreteAlternativeDiff.' });
341
456
  return value.map((finding) => ({ ...finding, source: 'judge' }));
342
457
  }
343
458
 
@@ -348,7 +463,7 @@ function writeDeterministicVerdictIfEligible(rootDir, run) {
348
463
  if (!tier) return false;
349
464
  const requirement = judgeRequirement(rootDir, run);
350
465
  if (requirement.required) {
351
- throw gateError(`Judge subagent required for tier ${tier}; write ${GATE_FILES.judge} from its verdict`, requirement.reasons);
466
+ throw gateError(`Judge subagent required for tier ${tier}; write ${GATE_FILES.judge} from its verdict`, requirement.reasons, { why: 'This Run has signals (tier, serious findings, escalation, incomplete evidence) that a deterministic verdict must not decide alone.', fix: `Spawn the Judge subagent with the ledger and Goal, write its verdict to ${GATE_FILES.judge}, then retry \`gate judge\`.` });
352
467
  }
353
468
  const expected = expectedJudgeVerdict(rootDir, run);
354
469
  fs.writeFileSync(filePath, `${JSON.stringify({
@@ -382,9 +497,29 @@ function mergeImpactDrift(rootDir, run) {
382
497
  return merged.added;
383
498
  }
384
499
 
500
+ // C-11: weakened proof in the gated diff (skips, deleted tests, dropped
501
+ // assertions, edited verification config) becomes VALID ledger findings.
502
+ function mergeVerificationIntegrity(rootDir, run) {
503
+ if (require('./ablation.js').ablated(rootDir, 'integrity', run.runId)) return 0;
504
+ const { scanRun, mergeIntegrityIntoLedger } = require('./verification-integrity.js');
505
+ const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
506
+ if (!fs.existsSync(filePath)) return 0;
507
+ const findings = scanRun(rootDir, run);
508
+ let ledger;
509
+ try {
510
+ ledger = JSON.parse(fs.readFileSync(filePath, 'utf8'));
511
+ } catch {
512
+ return 0;
513
+ }
514
+ const merged = mergeIntegrityIntoLedger(ledger, findings);
515
+ if (merged.errors.length) throw gateError('Review gate failed (VERIFICATION_WEAKENED finding dropped)', merged.errors, { why: 'Weakened tests make a green gate meaningless, so only a human may excuse them.', fix: 'Restore the integrity finding, or record a human dismissal {by, evidence}.' });
516
+ if (merged.added) fs.writeFileSync(filePath, `${JSON.stringify(merged.ledger, null, 2)}\n`, { mode: 0o600 });
517
+ return merged.added;
518
+ }
519
+
385
520
  function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
386
521
  if (!Object.hasOwn(GATE_FILES, gate)) {
387
- throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`);
522
+ throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`, [], { why: 'Only the listed stages exist.', fix: `Use one of: ${Object.keys(GATE_FILES).join(', ')}.` });
388
523
  }
389
524
  const run = resolveRun(rootDir, runId);
390
525
  const mode = run.mode || RUN_MODES.ASSISTED;
@@ -394,17 +529,23 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
394
529
  const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.goal);
395
530
  requireRunMetadata(value, run, 'goal contract');
396
531
  const check = validateGoalContract(value);
397
- if (!check.valid) throw gateError('Goal Contract gate failed', check.errors);
398
- return transitionRun(rootDir, run.runId, RUN_STATES.GOAL_FROZEN, {
532
+ if (!check.valid) throw gateError('Goal Contract gate failed', check.errors, { why: 'Work may not start until acceptance criteria, evidence and failure cases are explicit.', fix: 'Fix the listed Goal Contract fields (every AC needs statement, evidenceRequired, failureCases), then retry `gate goal`.' });
533
+ const frozen = transitionRun(rootDir, run.runId, RUN_STATES.GOAL_FROZEN, {
399
534
  gate,
400
535
  artifacts: { goalContract: descriptor(rootDir, filePath) },
401
536
  ...transitionOptions
402
537
  });
538
+ try {
539
+ require('./features.js').onGoalFrozen(rootDir, run, value);
540
+ } catch {
541
+ // The feature ledger is optional; never block the Goal gate on it.
542
+ }
543
+ return frozen;
403
544
  }
404
545
 
405
546
  if (gate === 'report') {
406
547
  if (mode !== RUN_MODES.REPORT_ONLY) {
407
- throw gateError('Report gate is only available in REPORT_ONLY mode');
548
+ throw gateError('Report gate is only available in REPORT_ONLY mode', [], { why: 'Delivery runs must go through Maker, verification and review.', fix: 'Use the Maker gate for this Run, or start a REPORT_ONLY Run for a report.' });
408
549
  }
409
550
  const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.report);
410
551
  requireRunMetadata(value, run, 'report');
@@ -415,7 +556,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
415
556
  reportErrors.push('recommendations must contain non-empty strings');
416
557
  }
417
558
  reportErrors.push(...validateReportReview(value.review));
418
- if (reportErrors.length) throw gateError('Report gate failed', reportErrors);
559
+ if (reportErrors.length) throw gateError('Report gate failed', reportErrors, { why: 'A report must stand on cited evidence.', fix: 'Fix the listed report fields, then retry `gate report`.' });
419
560
  return transitionRun(rootDir, run.runId, RUN_STATES.REPORTED, {
420
561
  gate,
421
562
  artifacts: { report: descriptor(rootDir, filePath) }
@@ -424,16 +565,16 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
424
565
 
425
566
  if (gate === 'maker') {
426
567
  if (mode === RUN_MODES.REPORT_ONLY) {
427
- throw gateError('REPORT_ONLY mode cannot enter the Maker stage; finish with the report gate');
568
+ throw gateError('REPORT_ONLY mode cannot enter the Maker stage; finish with the report gate', [], { why: 'REPORT_ONLY Runs never change code.', fix: 'Finish with `gate report`, or start a normal Run to make changes.' });
428
569
  }
429
570
  const filePath = artifactPath(rootDir, run.runId, GATE_FILES.maker);
430
571
  let diff;
431
572
  try {
432
573
  diff = fs.readFileSync(filePath, 'utf8');
433
574
  } catch (cause) {
434
- throw gateError(`Missing ${GATE_FILES.maker}: ${cause.message}`);
575
+ throw gateError(`Missing ${GATE_FILES.maker}: ${cause.message}`, [], { why: 'The Maker gate binds review to an exact diff.', fix: `Run \`gate maker --from-git --run <id>\` to capture the diff, or write ${GATE_FILES.maker}.` });
435
576
  }
436
- if (!diff.trim()) throw gateError('Maker gate failed: diff.patch is empty');
577
+ if (!diff.trim()) throw gateError('Maker gate failed: diff.patch is empty', [], { why: 'There is no change to verify or review.', fix: 'Make the change first, then run `gate maker --from-git`.' });
437
578
  const artifacts = { diff: descriptor(rootDir, filePath) };
438
579
  if (mode === RUN_MODES.UNATTENDED && policy.requireSandboxForUnattended) {
439
580
  const { filePath: sandboxFile, value: evidence } = readJsonArtifact(
@@ -453,7 +594,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
453
594
  if (evidence.diffSha256 !== hashFile(filePath)) {
454
595
  sandboxErrors.push('diffSha256 does not match the captured Maker diff');
455
596
  }
456
- if (sandboxErrors.length) throw gateError('Unattended Maker sandbox evidence failed', sandboxErrors);
597
+ if (sandboxErrors.length) throw gateError('Unattended Maker sandbox evidence failed', sandboxErrors, { why: 'UNATTENDED Runs must prove the Maker ran inside the sandbox.', fix: 'Re-run the Maker inside the configured sandbox so its evidence is recorded, then retry.' });
457
598
  artifacts.sandboxEvidence = descriptor(rootDir, sandboxFile);
458
599
  }
459
600
  const raisedTier = raiseTierFromDiff(run.tier, { task: run.task, diff });
@@ -468,11 +609,20 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
468
609
  const diffPath = artifactPath(rootDir, run.runId, GATE_FILES.maker);
469
610
  const diffHash = hashFile(diffPath);
470
611
  if (run.artifacts.diff?.sha256 !== diffHash) {
471
- throw gateError('Maker diff changed after its gate; run the Maker gate again');
612
+ throw gateError('Maker diff changed after its gate; run the Maker gate again', [], { why: 'Verification evidence is bound to the gated diff hash; a new diff needs new evidence.', fix: 'Run `gate maker --from-git --run <id>`, then `verify` again.' });
472
613
  }
473
614
  const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.verification);
474
615
  const check = validateVerificationBundle(value, run, diffHash, getGitRevision(rootDir));
475
- if (!check.valid) throw gateError('Verification gate failed', check.errors);
616
+ if (!check.valid) throw gateError('Verification gate failed', check.errors, { why: 'Review only starts on green, diff-bound, complete command evidence.', fix: 'Fix the listed problems (re-run `verify` on the current diff), then retry.' });
617
+ const layerProblem = missingStaticLayer(rootDir, run, value);
618
+ if (layerProblem) throw gateError('Verification gate failed (L1_MISSING)', [layerProblem], { why: 'Unit tests do not prove the change compiles or passes lint.', fix: 'Run `verify` without --cmd so L1 runs first.' });
619
+ const missingRuleChecks = require('./ablation.js').ablated(rootDir, 'rules', run.runId) ? [] : require('./rules.js').missingRules(rootDir, value);
620
+ if (missingRuleChecks.length) {
621
+ throw gateError('Verification gate failed (RULES_MISSING)', missingRuleChecks.map((rule) => `${rule.id}: \`${rule.check}\` has no passing record`),
622
+ { why: 'Promoted rules encode review feedback that kept recurring; skipping them lets the same defect back in.', fix: `Run \`verify --run ${run.runId}\` without --cmd so every rule runs, and fix any violation it reports.` });
623
+ }
624
+ const smokeProblem = missingSmoke(rootDir, run, value);
625
+ if (smokeProblem) throw gateError('Verification gate failed (SMOKE_MISSING)', [smokeProblem], { why: 'Migrations, config, env and entrypoint changes can break startup while unit tests stay green.', fix: 'Run `verify` without --cmd so the smoke check runs.' });
476
626
  return transitionRun(rootDir, run.runId, RUN_STATES.VERIFIED, {
477
627
  gate,
478
628
  artifacts: { verification: descriptor(rootDir, filePath) }
@@ -484,22 +634,31 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
484
634
  assertArtifactCurrent(rootDir, run, 'diff');
485
635
  assertArtifactCurrent(rootDir, run, 'verification');
486
636
  if (tierOf(run) === 'T0') writeT0FindingLedger(rootDir, run);
637
+ else if (tierOf(run)) {
638
+ // T1/T2: the Devil's Advocate must have reviewed against completed,
639
+ // layer-tagged evidence (three-layer termination check).
640
+ const status = claimedVsRealityStatus(rootDir, run);
641
+ if (!status.ready) {
642
+ throw gateError(`${tierOf(run)} review needs a completed claimed-vs-reality file (CLAIMED_VS_REALITY_INCOMPLETE)`, status.problems, { why: 'The Devil\'s Advocate reviews claims against evidence; missing or untagged evidence hides what was really proven.', fix: 'Fix each listed row (Reality from command output, Layer backed by a recorded command), then retry `gate review`.' });
643
+ }
644
+ }
487
645
  mergeImpactDrift(rootDir, run);
646
+ mergeVerificationIntegrity(rootDir, run);
488
647
  const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
489
648
  requireRunMetadata(value, run, 'Finding Ledger');
490
649
  if (value.diffHash !== run.artifacts.diff?.sha256) {
491
- throw gateError('Finding Ledger diffHash does not match the gated Maker diff');
650
+ throw gateError('Finding Ledger diffHash does not match the gated Maker diff', [], { why: 'A review of a different diff says nothing about this one.', fix: 'Re-run the Devil\'s Advocate on the current review pack, then `review ingest`.' });
492
651
  }
493
652
  const check = validateFindingLedger(value);
494
- if (!check.valid) throw gateError('Review gate failed', [check.reason]);
653
+ if (!check.valid) throw gateError('Review gate failed', [check.reason], { why: 'The Judge recomputes the verdict from the ledger, so every finding must be well-formed.', fix: 'Fix the listed ledger field, then retry `gate review`.' });
495
654
  const binding = checkLedgerBinding(rootDir, run, value);
496
655
  if (binding.errors.length) {
497
- throw gateError('Finding Ledger does not match the reviewer output (LEDGER_TAMPERED)', binding.errors);
656
+ throw gateError('Finding Ledger does not match the reviewer output (LEDGER_TAMPERED)', binding.errors, { why: 'Findings may not be dropped or silently downgraded between the reviewer and the gate.', fix: 'Restore the reviewer\'s findings, or add dismissal {by, evidence} for each downgrade.' });
498
657
  }
499
658
  if (binding.binding === 'UNBOUND' && mode === RUN_MODES.UNATTENDED) {
500
659
  throw gateError('UNATTENDED review needs a ledger bound to the reviewer output', [
501
660
  'Run `review ingest --run <id> --child-id <id> --file <child-output>` instead of writing findings.json'
502
- ]);
661
+ ], { why: 'Without a human watching, the ledger must be provably the reviewer\'s own output.', fix: 'Use `review ingest --run <id> --child-id <id> --file <child-output>`.' });
503
662
  }
504
663
  return transitionRun(rootDir, run.runId, RUN_STATES.REVIEWED, {
505
664
  gate,
@@ -515,28 +674,43 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
515
674
  writeDeterministicVerdictIfEligible(rootDir, run);
516
675
  const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.judge);
517
676
  const verdictCheck = validateVerdictDocument(value, run);
518
- if (!verdictCheck.valid) throw gateError('Judge gate failed', verdictCheck.errors);
677
+ if (!verdictCheck.valid) throw gateError('Judge gate failed', verdictCheck.errors, { why: 'The verdict must name its decision, reason and action.', fix: 'Fix the listed verdict fields, then retry `gate judge`.' });
519
678
  if (tierOf(run) === 'T2' && value.verdict === 'PASS'
520
679
  && ![MODEL_TIERS.STRONG_MODEL, MODEL_TIERS.HUMAN].includes(value.modelTier)) {
521
- throw gateError('T2 Judge PASS requires modelTier STRONG_MODEL or HUMAN', ['TIER_T2_STRONG_JUDGE']);
680
+ throw gateError('T2 Judge PASS requires modelTier STRONG_MODEL or HUMAN', ['TIER_T2_STRONG_JUDGE'], { why: 'T2 changes (auth, money, data, security) need the strongest judgment.', fix: 'Re-run the Judge on STRONG_MODEL (or a human) and set modelTier accordingly.' });
522
681
  }
523
682
  try {
524
683
  enforceVerdictTier(rootDir, run, value);
525
684
  } catch (cause) {
526
- throw gateError(cause.message, [cause.code]);
685
+ throw gateError(cause.message, [cause.code], { why: 'The verdict contradicts the escalation policy.', fix: 'Follow the escalation output (`escalation --json`) and re-issue the verdict.' });
527
686
  }
528
687
  const judgeFindings = validateJudgeFindings(value.judgeFindings);
529
688
  const expected = expectedJudgeVerdict(rootDir, run, { judgeFindings });
530
689
  if (value.verdict !== expected.verdict) {
531
- throw gateError(`Judge verdict is unsupported: expected ${expected.verdict}, received ${value.verdict}`, [expected.reason]);
690
+ throw gateError(`Judge verdict is unsupported: expected ${expected.verdict}, received ${value.verdict}`, [expected.reason], { why: 'The gate recomputes the verdict from Validity and Severity; prose cannot override it.', fix: `Issue ${expected.verdict}, or add the blocking problem as a judgeFinding so the ledger supports your verdict.` });
532
691
  }
692
+ const blocking = expected.blockingFindings || [];
693
+ const cause = value.cause || inferCause(expected.verdict, blocking);
533
694
  recordIteration(rootDir, run, {
534
695
  verdict: expected.verdict,
535
696
  diffHash: run.artifacts?.diff?.sha256,
536
- blockingFindings: expected.blockingFindings || []
697
+ blockingFindings: blocking,
698
+ cause,
699
+ causeInferred: Boolean(cause && !value.cause)
537
700
  });
538
701
  const artifacts = { verdict: descriptor(rootDir, filePath) };
539
702
  if (expected.verdict === 'PASS') {
703
+ const rubric = rubricGaps(rootDir, run);
704
+ if (rubric.length) {
705
+ throw gateError('Judge PASS refused: rubric criteria are not scored or failed (AC_RUBRIC)', rubric,
706
+ { why: 'Subjective ACs are judged generously by default; a rubric scored with evidence per criterion makes the judgment reproducible.', fix: 'Have the Devil\'s Advocate add rubricScores [{ac, criterion, pass, evidence}] for every criterion (screenshot path, DOM snapshot, measured value), re-ingest, then retry; or issue ITERATE for failed criteria.' });
707
+ }
708
+ const unproven = unprovenCriteria(rootDir, run);
709
+ if (unproven.length) {
710
+ throw gateError('Judge PASS refused: acceptance criteria are not proven (AC_NOT_PROVEN)',
711
+ unproven.map((ac) => `${ac.id}: verify command \`${ac.command}\` is ${ac.status}`),
712
+ { why: 'An AC that names its verify command is proven only when that command passed on the gated diff.', fix: `Run \`verify --run ${run.runId} --cmd "<the AC's command>"\` for each listed AC (or fix the change), then retry \`gate judge\`.` });
713
+ }
540
714
  return transitionRun(rootDir, run.runId, RUN_STATES.JUDGED_PASS, { gate, artifacts });
541
715
  }
542
716
  if (expected.verdict === 'ESCALATE') {
@@ -569,14 +743,33 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
569
743
  }
570
744
  const finalCheck = finalVerificationStatus(rootDir, run);
571
745
  if (!finalCheck.satisfied) deliveryErrors.push(`Full verification required before delivery: ${finalCheck.reason}`);
572
- if (deliveryErrors.length) throw gateError('Delivery gate failed', deliveryErrors);
746
+ const cleanState = require('./clean-state.js').checkCleanState(rootDir, run, { mode });
747
+ deliveryErrors.push(...cleanState.errors);
748
+ if (deliveryErrors.length) throw gateError('Delivery gate failed', deliveryErrors, { why: 'Delivery is the last point where missing evidence can be caught before people rely on the change.', fix: 'Fix each listed delivery problem, then retry `gate delivery`.' });
749
+ // C-15: the linked feature becomes passing with this Run as evidence.
750
+ try {
751
+ require('./features.js').onDelivered(rootDir, run, loadFrozenGoal(rootDir, run));
752
+ } catch {
753
+ // Optional ledger.
754
+ }
755
+ // C-14: accepted trade-offs and L3 waivers outlive the Run.
756
+ let debtAdded = 0;
757
+ try {
758
+ debtAdded = require('./debt.js').recordRunDebt(rootDir, run.runId).added;
759
+ } catch {
760
+ debtAdded = 0;
761
+ }
573
762
  return transitionRun(rootDir, run.runId, RUN_STATES.DELIVERED, {
574
763
  gate,
575
- artifacts: { delivery: descriptor(rootDir, filePath) }
764
+ artifacts: { delivery: descriptor(rootDir, filePath) },
765
+ ...(debtAdded ? { reason: `${debtAdded} debt item(s) recorded in .ai-engineering-loop/debt.json` } : {}),
766
+ ...(cleanState.warnings.length ? { historyMetadata: { cleanStateWarnings: cleanState.warnings } } : {})
576
767
  });
577
768
  }
578
769
 
579
770
  module.exports = {
771
+ FAILURE_CAUSES,
772
+ inferCause,
580
773
  GATE_FILES,
581
774
  validateReportReview,
582
775
  artifactPath,
@@ -0,0 +1,124 @@
1
+ 'use strict';
2
+
3
+ // `doctor --harness` (C-08, learn-harness-engineering L03/L04/L12): harness
4
+ // debt in the project, reported as candidates to fix or delete. Harness rots
5
+ // like code: entry files bloat, learnings go stale, rules outlive their need.
6
+
7
+ const fs = require('fs');
8
+ const path = require('path');
9
+ const { runsDir } = require('./run-state.js');
10
+
11
+ const ENTRY_MIN = 50;
12
+ const ENTRY_MAX = 200;
13
+ const STALE_LEARNING_RUNS = 10;
14
+ const RULE_REVIEW_DAYS = 90;
15
+ const BUDGET_HEADROOM = 0.02;
16
+
17
+ // Only structured lines count (headings, lists, tables, code, bold leads), so
18
+ // sprinkling keywords into prose cannot game the score (Goodhart, L14).
19
+ // Adapted from harness-creator's structuredText (MIT).
20
+ function structuredText(markdown) {
21
+ const kept = [];
22
+ let inFence = false;
23
+ for (const raw of String(markdown || '').split(/\r?\n/)) {
24
+ const line = raw.trim();
25
+ if (/^(```|~~~)/.test(line)) {
26
+ inFence = !inFence;
27
+ continue;
28
+ }
29
+ if (inFence || /^#{1,6}\s/.test(line) || /^([-*+]|\d+\.)\s/.test(line) || line.startsWith('|') || /^\*\*[^*]+\*\*/.test(line)) {
30
+ if (line) kept.push(line);
31
+ }
32
+ }
33
+ return kept.join('\n').toLowerCase();
34
+ }
35
+
36
+ function auditEntryFile(rootDir) {
37
+ const name = ['AGENTS.md', 'CLAUDE.md'].find((file) => fs.existsSync(path.join(rootDir, file)));
38
+ if (!name) return [{ id: 'entry-file', level: 'warn', detail: 'No AGENTS.md or CLAUDE.md. FIX: add a 50-200 line router: overview, run/verify commands, hard rules, links to docs.' }];
39
+ const text = fs.readFileSync(path.join(rootDir, name), 'utf8');
40
+ const lines = text.split(/\r?\n/).length;
41
+ const items = [];
42
+ if (lines > ENTRY_MAX) items.push({ id: 'entry-file-size', level: 'warn', detail: `${name} has ${lines} lines (> ${ENTRY_MAX}). FIX: keep it a router; move topic detail to docs/ and link it.` });
43
+ if (lines < ENTRY_MIN) items.push({ id: 'entry-file-size', level: 'info', detail: `${name} has ${lines} lines; fine if it still answers what/run/verify.` });
44
+ const structured = structuredText(text);
45
+ const needs = [
46
+ ['verification', /verif|test|check/],
47
+ ['definition of done', /definition of done|done when|done only when/],
48
+ ['startup', /startup|start here|quick start|before writing|setup/]
49
+ ];
50
+ for (const [label, pattern] of needs) {
51
+ if (!pattern.test(structured)) items.push({ id: 'entry-file-structure', level: 'warn', detail: `${name} has no structured ${label} section (heading or list). FIX: add one; prose mentions do not count.` });
52
+ }
53
+ return items;
54
+ }
55
+
56
+ function listRunIds(rootDir) {
57
+ try {
58
+ return fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
59
+ } catch {
60
+ return [];
61
+ }
62
+ }
63
+
64
+ function auditLearnings(rootDir) {
65
+ let entries = [];
66
+ try {
67
+ entries = JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'learnings.json'), 'utf8')).entries || [];
68
+ } catch {
69
+ return [];
70
+ }
71
+ const runs = listRunIds(rootDir);
72
+ const recent = new Set(runs.slice(-STALE_LEARNING_RUNS));
73
+ if (runs.length <= STALE_LEARNING_RUNS) return [];
74
+ return entries
75
+ .filter((entry) => !(entry.runIds || []).some((runId) => recent.has(runId)))
76
+ .map((entry) => ({ id: 'stale-learning', level: 'warn', detail: `${entry.id} not seen in the last ${STALE_LEARNING_RUNS} Runs: ${String(entry.lesson || '').slice(0, 80)}. FIX: \`learn --prune\` or delete it if the problem is gone.` }));
77
+ }
78
+
79
+ function auditRules(rootDir, now = new Date()) {
80
+ let rules = [];
81
+ try {
82
+ rules = JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'rules.json'), 'utf8')).rules || [];
83
+ } catch {
84
+ return [];
85
+ }
86
+ const items = [];
87
+ for (const rule of rules) {
88
+ if (!rule.from || !rule.why || !rule.fix) items.push({ id: 'rule-provenance', level: 'warn', detail: `${rule.id} lacks a source (from), why or fix. FIX: record where it came from and how to repair a violation.` });
89
+ const age = (now - new Date(rule.createdAt || 0)) / 86400000;
90
+ if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
91
+ }
92
+ return items;
93
+ }
94
+
95
+ // Inside the AEL package repo: T1 skill-context headroom from scripts/measure.js.
96
+ function auditBudget(rootDir) {
97
+ const script = path.join(rootDir, 'scripts', 'measure.js');
98
+ if (!fs.existsSync(script)) return [];
99
+ try {
100
+ const { measureAll, BASELINE } = require(script);
101
+ const t1 = measureAll().results.find((row) => row.tier === 'T1');
102
+ const limit = BASELINE.parentContextChars / 2;
103
+ const headroom = (limit - t1.parentContextChars) / limit;
104
+ return [{
105
+ id: 'skill-budget',
106
+ level: headroom < BUDGET_HEADROOM ? 'warn' : 'info',
107
+ detail: `T1 skill context ${t1.parentContextChars}/${limit} chars (${(headroom * 100).toFixed(2)}% headroom).${headroom < BUDGET_HEADROOM ? ' FIX: trim skill-src references before adding text.' : ''}`
108
+ }];
109
+ } catch (cause) {
110
+ return [{ id: 'skill-budget', level: 'warn', detail: `measure failed: ${cause.message}` }];
111
+ }
112
+ }
113
+
114
+ function auditHarness(rootDir, { budget = true } = {}) {
115
+ const items = [
116
+ ...auditEntryFile(rootDir),
117
+ ...auditLearnings(rootDir),
118
+ ...auditRules(rootDir),
119
+ ...(budget ? auditBudget(rootDir) : [])
120
+ ];
121
+ return { ok: !items.some((item) => item.level === 'warn'), items };
122
+ }
123
+
124
+ module.exports = { auditHarness, structuredText };
@@ -87,12 +87,17 @@ function detectNoProgress(rootDir, run, { diffHash, blockingFindings = [] }) {
87
87
  return null;
88
88
  }
89
89
 
90
- function recordIteration(rootDir, run, { verdict, diffHash, blockingFindings = [], now = new Date() }) {
90
+ function recordIteration(rootDir, run, { verdict, diffHash, blockingFindings = [], cause = null, causeInferred = false, now = new Date() }) {
91
91
  const entry = {
92
92
  iteration: run.iteration,
93
93
  verdict,
94
94
  diffHash: diffHash || null,
95
95
  blockingSignatures: blockingFindings.map(findingSignature),
96
+ // C-12: verification was green when the Judge ran, so blocking findings
97
+ // here mean the Maker's "done" was premature (a false victory).
98
+ blockingCount: blockingFindings.length,
99
+ ...(cause ? { cause, causeInferred: Boolean(causeInferred) } : {}),
100
+ tier: run.tier?.current || null,
96
101
  at: now.toISOString()
97
102
  };
98
103
  const filePath = iterationsPath(rootDir, run.runId);