ai-engineering-loop 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +13 -1
- package/.agents/skills/ael/references/loop-deliver.md +1 -1
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +11 -9
- package/.claude/agents/devil-advocate.md +7 -2
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +7 -2
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +31 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +7 -2
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +326 -21
- package/core/bench.md +15 -0
- package/core/formal-statechart.md +49 -0
- package/core/goal-contract.md +53 -0
- package/core/verification-loop.md +106 -1
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +95 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/budget.js +9 -1
- package/lib/calibration.js +129 -0
- package/lib/claimed-vs-reality.js +106 -12
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/comment-findings.js +111 -0
- package/lib/debt.js +110 -0
- package/lib/feature-map.js +57 -0
- package/lib/features.js +95 -0
- package/lib/flow.js +86 -0
- package/lib/gates.js +257 -41
- package/lib/harness-audit.js +141 -0
- package/lib/learn.js +24 -1
- package/lib/no-progress.js +6 -1
- package/lib/receipt.js +33 -2
- package/lib/reverts.js +77 -0
- package/lib/review-ingest.js +54 -2
- package/lib/rule-check.js +90 -0
- package/lib/rules.js +205 -0
- package/lib/stats.js +110 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +96 -0
- package/lib/verification-recorder.js +20 -8
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +50 -0
- package/schemas/verdict.schema.json +2 -1
package/lib/gates.js
CHANGED
|
@@ -51,7 +51,7 @@ const {
|
|
|
51
51
|
enforceVerdictTier
|
|
52
52
|
} = require('./escalation.js');
|
|
53
53
|
const { raiseTierFromDiff } = require('./tier.js');
|
|
54
|
-
const {
|
|
54
|
+
const { checkClaimedVsReality, loadFrozenGoal } = require('./claimed-vs-reality.js');
|
|
55
55
|
const { runMaxIterations, detectNoProgress, recordIteration } = require('./no-progress.js');
|
|
56
56
|
const { finalVerificationStatus } = require('./verification-lanes.js');
|
|
57
57
|
const { checkLedgerBinding } = require('./review-ingest.js');
|
|
@@ -68,10 +68,14 @@ const GATE_FILES = Object.freeze({
|
|
|
68
68
|
delivery: 'delivery.json'
|
|
69
69
|
});
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
// C-04: every gate failure says what failed (message), why the gate cares
|
|
72
|
+
// (why), and the next concrete step (fix), so an agent can self-correct.
|
|
73
|
+
function gateError(message, details = [], { why = null, fix = null } = {}) {
|
|
72
74
|
const err = new Error(message);
|
|
73
75
|
err.code = 'GATE_FAILED';
|
|
74
76
|
err.details = details;
|
|
77
|
+
if (why) err.why = why;
|
|
78
|
+
if (fix) err.fix = fix;
|
|
75
79
|
return err;
|
|
76
80
|
}
|
|
77
81
|
|
|
@@ -85,7 +89,7 @@ function readJsonArtifact(rootDir, runId, fileName) {
|
|
|
85
89
|
try {
|
|
86
90
|
value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
87
91
|
} catch (cause) {
|
|
88
|
-
throw gateError(`Missing or invalid ${fileName}: ${cause.message}`);
|
|
92
|
+
throw gateError(`Missing or invalid ${fileName}: ${cause.message}`, [], { why: 'Every gate reads its artifact from the Run directory; without it there is nothing to check.', fix: `Write ${fileName} in the Run directory (\`scaffold\` can create a template), then retry the gate.` });
|
|
89
93
|
}
|
|
90
94
|
return { filePath, value };
|
|
91
95
|
}
|
|
@@ -94,7 +98,7 @@ function hashFile(filePath) {
|
|
|
94
98
|
try {
|
|
95
99
|
return hashContent(fs.readFileSync(filePath));
|
|
96
100
|
} catch (cause) {
|
|
97
|
-
throw gateError(`Unable to hash ${filePath}: ${cause.message}`);
|
|
101
|
+
throw gateError(`Unable to hash ${filePath}: ${cause.message}`, [], { why: 'Gates bind artifacts by sha256 so later edits are detected.', fix: `Check that ${filePath} exists and is readable, then retry.` });
|
|
98
102
|
}
|
|
99
103
|
}
|
|
100
104
|
|
|
@@ -108,11 +112,11 @@ function descriptor(rootDir, filePath) {
|
|
|
108
112
|
function assertArtifactCurrent(rootDir, run, name) {
|
|
109
113
|
const artifact = run.artifacts[name];
|
|
110
114
|
if (!artifact?.path || !artifact.sha256) {
|
|
111
|
-
throw gateError(`Required ${name} artifact has not passed its gate`);
|
|
115
|
+
throw gateError(`Required ${name} artifact has not passed its gate`, [], { why: 'Stages run in order; a later gate cannot trust an earlier artifact that never passed.', fix: `Pass the ${name} gate first (\`gate <stage> --run <id>\`).` });
|
|
112
116
|
}
|
|
113
117
|
const filePath = path.join(rootDir, artifact.path);
|
|
114
118
|
if (hashFile(filePath) !== artifact.sha256) {
|
|
115
|
-
throw gateError(`${name} artifact changed after its gate; rerun the owning stage`);
|
|
119
|
+
throw gateError(`${name} artifact changed after its gate; rerun the owning stage`, [], { why: 'Gated artifacts are frozen by hash; a silent edit would invalidate later evidence.', fix: `Re-run the stage that owns ${name}, or restore the gated version.` });
|
|
116
120
|
}
|
|
117
121
|
}
|
|
118
122
|
|
|
@@ -120,13 +124,14 @@ function requireRunMetadata(document, run, label) {
|
|
|
120
124
|
const errors = [];
|
|
121
125
|
if (document.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
122
126
|
if (document.runId !== run.runId) errors.push(`runId must be ${run.runId}`);
|
|
123
|
-
if (errors.length) throw gateError(`${label} metadata is invalid`, errors);
|
|
127
|
+
if (errors.length) throw gateError(`${label} metadata is invalid`, errors, { why: 'Artifacts must name the Run they belong to so evidence cannot be copied across Runs.', fix: 'Set schemaVersion, runId and the other listed fields to this Run\'s values.' });
|
|
124
128
|
}
|
|
125
129
|
|
|
126
130
|
function validateGoalContract(contract) {
|
|
127
131
|
const errors = [];
|
|
128
132
|
if (!contract || typeof contract !== 'object') return { valid: false, errors: ['contract must be an object'] };
|
|
129
133
|
if (!String(contract.objective || '').trim()) errors.push('objective is required');
|
|
134
|
+
errors.push(...require('./flow.js').flowProblems(contract.flow)); // C-21
|
|
130
135
|
if (!Array.isArray(contract.acceptanceCriteria) || contract.acceptanceCriteria.length === 0) {
|
|
131
136
|
errors.push('acceptanceCriteria must contain at least one item');
|
|
132
137
|
} else {
|
|
@@ -200,6 +205,18 @@ function validateVerificationBundle(bundle, run, diffHash, currentRevision) {
|
|
|
200
205
|
return { valid: errors.length === 0, errors };
|
|
201
206
|
}
|
|
202
207
|
|
|
208
|
+
// C-12 (learn-harness-engineering L01/L02): which harness layer a failed
|
|
209
|
+
// iteration points at. `model` = an implementation defect the harness could
|
|
210
|
+
// not have prevented.
|
|
211
|
+
const FAILURE_CAUSES = Object.freeze(['task_spec', 'context', 'environment', 'verification', 'state', 'model']);
|
|
212
|
+
|
|
213
|
+
function inferCause(verdict, blockingFindings = []) {
|
|
214
|
+
if (verdict === 'PASS') return null;
|
|
215
|
+
if (blockingFindings.some((finding) => finding.source === 'verification-integrity')) return 'verification';
|
|
216
|
+
if (verdict === 'ESCALATE') return 'task_spec';
|
|
217
|
+
return 'model';
|
|
218
|
+
}
|
|
219
|
+
|
|
203
220
|
function validateVerdictDocument(verdict, run) {
|
|
204
221
|
const errors = [];
|
|
205
222
|
requireRunMetadata(verdict, run, 'verdict');
|
|
@@ -214,6 +231,9 @@ function validateVerdictDocument(verdict, run) {
|
|
|
214
231
|
if (verdict.humanApproved != null && typeof verdict.humanApproved !== 'boolean') {
|
|
215
232
|
errors.push('humanApproved must be a boolean');
|
|
216
233
|
}
|
|
234
|
+
if (verdict.cause != null && !FAILURE_CAUSES.includes(verdict.cause)) {
|
|
235
|
+
errors.push(`cause must be one of ${FAILURE_CAUSES.join(', ')}`);
|
|
236
|
+
}
|
|
217
237
|
return { valid: errors.length === 0, errors };
|
|
218
238
|
}
|
|
219
239
|
|
|
@@ -228,20 +248,117 @@ function verificationForJudge(bundle) {
|
|
|
228
248
|
function resolveRun(rootDir, runId) {
|
|
229
249
|
if (runId) return loadRun(rootDir, runId);
|
|
230
250
|
const current = getCurrentRun(rootDir);
|
|
231
|
-
if (!current) throw gateError('No current run. Start one with `ai-engineering-loop run <task>`.');
|
|
251
|
+
if (!current) throw gateError('No current run. Start one with `ai-engineering-loop run <task>`.', [], { why: 'Gates act on a Run.', fix: 'Start a Run with `run <task>` or pass --run <id>.' });
|
|
232
252
|
return current;
|
|
233
253
|
}
|
|
234
254
|
|
|
255
|
+
// C-02: at T1/T2, when the project configures typecheck or lint, the gated
|
|
256
|
+
// bundle must contain at least one L1 (static) command. Unit tests alone do
|
|
257
|
+
// not prove the change compiles or passes lint.
|
|
258
|
+
function missingStaticLayer(rootDir, run, bundle) {
|
|
259
|
+
const tier = tierOf(run);
|
|
260
|
+
if (tier !== 'T1' && tier !== 'T2') return null;
|
|
261
|
+
if (require('./ablation.js').ablated(rootDir, 'l1', run.runId)) return null;
|
|
262
|
+
const lanes = require('./verification-lanes.js');
|
|
263
|
+
const configured = lanes.verificationCommands(rootDir);
|
|
264
|
+
const staticKeys = ['typecheck', 'lint'].filter((key) => configured[key] && !lanes.isMutatingCommand(configured[key]));
|
|
265
|
+
if (!staticKeys.length) return null;
|
|
266
|
+
// The related lane is the iteration fast path; `verify --final` runs the full plan.
|
|
267
|
+
if ((bundle.commands || []).length && bundle.commands.every((record) => record.scope === 'related')) return null;
|
|
268
|
+
const hasStatic = (bundle.commands || []).some((record) =>
|
|
269
|
+
(record.layer || lanes.layerOfCommand(rootDir, record.command)) === 'L1');
|
|
270
|
+
if (hasStatic) return null;
|
|
271
|
+
return `No L1 static check in the verification bundle, but verification.md configures ${staticKeys.join(' and ')}. FIX: run \`verify --run ${run.runId}\` without --cmd (it runs L1 first), or add --cmd "${configured[staticKeys[0]]}".`;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// C-03: when verification.md configures `smoke` and the diff touches
|
|
275
|
+
// startup-risky files, a passing smoke record must be in the bundle.
|
|
276
|
+
function missingSmoke(rootDir, run, bundle) {
|
|
277
|
+
const lanes = require('./verification-lanes.js');
|
|
278
|
+
const smokeText = lanes.verificationCommands(rootDir).smoke;
|
|
279
|
+
if (!smokeText) return null;
|
|
280
|
+
if (require('./ablation.js').ablated(rootDir, 'smoke', run.runId)) return null;
|
|
281
|
+
const risky = lanes.startupRiskFiles(rootDir, run);
|
|
282
|
+
if (!risky.length) return null;
|
|
283
|
+
const base = smokeText.trim();
|
|
284
|
+
if ((bundle.commands || []).some((record) => String(record.command || '').trim() === base)) return null;
|
|
285
|
+
return `The diff touches startup-risky files (${risky.slice(0, 3).join(', ')}${risky.length > 3 ? ', ...' : ''}) but the smoke check did not run. FIX: run \`verify --run ${run.runId}\` without --cmd, or add --cmd "${base}".`;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
function rubricGaps(rootDir, run) {
|
|
289
|
+
let goal;
|
|
290
|
+
try {
|
|
291
|
+
goal = loadFrozenGoal(rootDir, run);
|
|
292
|
+
} catch {
|
|
293
|
+
return [];
|
|
294
|
+
}
|
|
295
|
+
let ledger = null;
|
|
296
|
+
try {
|
|
297
|
+
ledger = readJsonArtifact(rootDir, run.runId, GATE_FILES.review).value;
|
|
298
|
+
} catch {
|
|
299
|
+
ledger = null;
|
|
300
|
+
}
|
|
301
|
+
return require('./ac-status.js').rubricProblems(goal, ledger);
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
// C-07: ACs with verify.command must be proven by a passing record.
|
|
305
|
+
function unprovenCriteria(rootDir, run) {
|
|
306
|
+
let goal;
|
|
307
|
+
try {
|
|
308
|
+
goal = loadFrozenGoal(rootDir, run);
|
|
309
|
+
} catch {
|
|
310
|
+
return [];
|
|
311
|
+
}
|
|
312
|
+
return require('./ac-status.js').runAcStatuses(rootDir, run, goal)
|
|
313
|
+
.filter((ac) => ac.status === 'pending' || ac.status === 'failed');
|
|
314
|
+
}
|
|
315
|
+
|
|
235
316
|
function tierOf(run) {
|
|
236
317
|
return run?.tier?.current || null;
|
|
237
318
|
}
|
|
238
319
|
|
|
239
|
-
|
|
320
|
+
// Layers proven by recorded verification (gated bundle + final run). A
|
|
321
|
+
// command not mapped in verification.md counts as L2 (a test run).
|
|
322
|
+
function recordedLayers(rootDir, run) {
|
|
323
|
+
if (!run) return null;
|
|
324
|
+
const lanes = require('./verification-lanes.js');
|
|
325
|
+
const layers = new Set();
|
|
326
|
+
for (const file of ['verification.json', 'final-verification.json']) {
|
|
327
|
+
let bundle;
|
|
328
|
+
try {
|
|
329
|
+
bundle = JSON.parse(fs.readFileSync(artifactPath(rootDir, run.runId, file), 'utf8'));
|
|
330
|
+
} catch {
|
|
331
|
+
continue;
|
|
332
|
+
}
|
|
333
|
+
for (const record of bundle.commands || []) {
|
|
334
|
+
if (record.exitCode !== 0) continue;
|
|
335
|
+
layers.add(record.layer || lanes.layerOfCommand(rootDir, record.command) || 'L2');
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
return layers.size ? layers : null;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
// Status of claimed-vs-reality.md for this run, including the three-layer
|
|
342
|
+
// (L1/L2/L3) requirement from the frozen Goal Contract and the run tier.
|
|
343
|
+
function claimedVsRealityStatus(rootDir, run) {
|
|
344
|
+
let markdown;
|
|
240
345
|
try {
|
|
241
|
-
|
|
346
|
+
markdown = fs.readFileSync(path.join(rootDir, CLAIMED_VS_REALITY_PATH), 'utf8');
|
|
242
347
|
} catch {
|
|
243
|
-
return false;
|
|
348
|
+
return { ready: false, problems: [`${CLAIMED_VS_REALITY_PATH} is missing. FIX: scaffold it with \`context claimed-vs-reality --run ${run?.runId || '<id>'}\` after \`gate verification\`, then fill every row.`] };
|
|
244
349
|
}
|
|
350
|
+
let goal = null;
|
|
351
|
+
try {
|
|
352
|
+
goal = run ? loadFrozenGoal(rootDir, run) : null;
|
|
353
|
+
} catch {
|
|
354
|
+
goal = null;
|
|
355
|
+
}
|
|
356
|
+
if (require('./ablation.js').ablated(rootDir, 'l3', run?.runId)) return checkClaimedVsReality(markdown, {});
|
|
357
|
+
return checkClaimedVsReality(markdown, { goal, tier: tierOf(run), layers: recordedLayers(rootDir, run) });
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
function claimedVsRealityReady(rootDir, run) {
|
|
361
|
+
return claimedVsRealityStatus(rootDir, run).ready;
|
|
245
362
|
}
|
|
246
363
|
|
|
247
364
|
// T0 has no Devil's Advocate. The CLI writes an explicit empty, diff-bound
|
|
@@ -250,10 +367,9 @@ function claimedVsRealityReady(rootDir) {
|
|
|
250
367
|
function writeT0FindingLedger(rootDir, run) {
|
|
251
368
|
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
|
|
252
369
|
if (fs.existsSync(filePath)) return false;
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
]);
|
|
370
|
+
const status = claimedVsRealityStatus(rootDir, run);
|
|
371
|
+
if (!status.ready) {
|
|
372
|
+
throw gateError('T0 review needs a completed claimed-vs-reality file', status.problems, { why: 'T0 has no Devil\'s Advocate, so claimed-vs-reality is the only check that claims match evidence.', fix: 'Fill every Claimed and Reality cell (scaffold: `context claimed-vs-reality`), then retry `gate review`.' });
|
|
257
373
|
}
|
|
258
374
|
fs.writeFileSync(filePath, `${JSON.stringify({
|
|
259
375
|
schemaVersion: 1,
|
|
@@ -294,7 +410,7 @@ function judgeRequirement(rootDir, run) {
|
|
|
294
410
|
} catch (cause) {
|
|
295
411
|
reasons.push(`ESCALATION_UNAVAILABLE: ${cause.message}`);
|
|
296
412
|
}
|
|
297
|
-
if (!claimedVsRealityReady(rootDir)) reasons.push('CLAIMED_VS_REALITY_INCOMPLETE');
|
|
413
|
+
if (!claimedVsRealityReady(rootDir, run)) reasons.push('CLAIMED_VS_REALITY_INCOMPLETE');
|
|
298
414
|
return { required: reasons.length > 0, tier, reasons };
|
|
299
415
|
}
|
|
300
416
|
|
|
@@ -335,9 +451,9 @@ function expectedJudgeVerdict(rootDir, run, { judgeFindings = [] } = {}) {
|
|
|
335
451
|
|
|
336
452
|
function validateJudgeFindings(value) {
|
|
337
453
|
if (value == null) return [];
|
|
338
|
-
if (!Array.isArray(value)) throw gateError('Judge gate failed', ['judgeFindings must be an array']);
|
|
454
|
+
if (!Array.isArray(value)) throw gateError('Judge gate failed', ['judgeFindings must be an array'], { why: 'Judge-added findings use the Finding Ledger shape so the gate can recompute the verdict.', fix: 'Make judgeFindings an array of ledger findings (or remove it).' });
|
|
339
455
|
const check = validateFindingLedger({ findings: value });
|
|
340
|
-
if (!check.valid) throw gateError('Judge gate failed', [`judgeFindings: ${check.reason}`]);
|
|
456
|
+
if (!check.valid) throw gateError('Judge gate failed', [`judgeFindings: ${check.reason}`], { why: 'Judge-added findings must be complete enough to act on.', fix: 'Give each judgeFinding location, failureScenario, evidence, severity, validity and (for VALID BLOCKER/HIGH) concreteAlternativeDiff.' });
|
|
341
457
|
return value.map((finding) => ({ ...finding, source: 'judge' }));
|
|
342
458
|
}
|
|
343
459
|
|
|
@@ -348,7 +464,7 @@ function writeDeterministicVerdictIfEligible(rootDir, run) {
|
|
|
348
464
|
if (!tier) return false;
|
|
349
465
|
const requirement = judgeRequirement(rootDir, run);
|
|
350
466
|
if (requirement.required) {
|
|
351
|
-
throw gateError(`Judge subagent required for tier ${tier}; write ${GATE_FILES.judge} from its verdict`, requirement.reasons);
|
|
467
|
+
throw gateError(`Judge subagent required for tier ${tier}; write ${GATE_FILES.judge} from its verdict`, requirement.reasons, { why: 'This Run has signals (tier, serious findings, escalation, incomplete evidence) that a deterministic verdict must not decide alone.', fix: `Spawn the Judge subagent with the ledger and Goal, write its verdict to ${GATE_FILES.judge}, then retry \`gate judge\`.` });
|
|
352
468
|
}
|
|
353
469
|
const expected = expectedJudgeVerdict(rootDir, run);
|
|
354
470
|
fs.writeFileSync(filePath, `${JSON.stringify({
|
|
@@ -382,9 +498,50 @@ function mergeImpactDrift(rootDir, run) {
|
|
|
382
498
|
return merged.added;
|
|
383
499
|
}
|
|
384
500
|
|
|
501
|
+
// C-11: weakened proof in the gated diff (skips, deleted tests, dropped
|
|
502
|
+
// assertions, edited verification config) becomes VALID ledger findings.
|
|
503
|
+
function mergeVerificationIntegrity(rootDir, run) {
|
|
504
|
+
if (require('./ablation.js').ablated(rootDir, 'integrity', run.runId)) return 0;
|
|
505
|
+
const { scanRun, mergeIntegrityIntoLedger } = require('./verification-integrity.js');
|
|
506
|
+
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
|
|
507
|
+
if (!fs.existsSync(filePath)) return 0;
|
|
508
|
+
const findings = scanRun(rootDir, run);
|
|
509
|
+
let ledger;
|
|
510
|
+
try {
|
|
511
|
+
ledger = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
512
|
+
} catch {
|
|
513
|
+
return 0;
|
|
514
|
+
}
|
|
515
|
+
const merged = mergeIntegrityIntoLedger(ledger, findings);
|
|
516
|
+
if (merged.errors.length) throw gateError('Review gate failed (VERIFICATION_WEAKENED finding dropped)', merged.errors, { why: 'Weakened tests make a green gate meaningless, so only a human may excuse them.', fix: 'Restore the integrity finding, or record a human dismissal {by, evidence}.' });
|
|
517
|
+
if (merged.added) fs.writeFileSync(filePath, `${JSON.stringify(merged.ledger, null, 2)}\n`, { mode: 0o600 });
|
|
518
|
+
return merged.added;
|
|
519
|
+
}
|
|
520
|
+
|
|
521
|
+
// C-18: suppressions and justification comments added by the change.
|
|
522
|
+
function mergeCommentJustifications(rootDir, run) {
|
|
523
|
+
if (require('./ablation.js').ablated(rootDir, 'comments', run.runId)) return 0;
|
|
524
|
+
const { scanRunComments, mergeCommentFindings } = require('./comment-findings.js');
|
|
525
|
+
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
|
|
526
|
+
if (!fs.existsSync(filePath)) return 0;
|
|
527
|
+
let ledger;
|
|
528
|
+
try {
|
|
529
|
+
ledger = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
530
|
+
} catch {
|
|
531
|
+
return 0;
|
|
532
|
+
}
|
|
533
|
+
const merged = mergeCommentFindings(ledger, scanRunComments(rootDir, run));
|
|
534
|
+
if (merged.errors.length) {
|
|
535
|
+
throw gateError('Review gate failed (comment-justification finding dropped)', merged.errors,
|
|
536
|
+
{ why: 'Comments and suppressions are how agents justify skipping the root cause; dropping the finding silently hides that decision.', fix: 'Restore the finding, or record dismissal: {by, evidence} for each one.' });
|
|
537
|
+
}
|
|
538
|
+
if (merged.added) fs.writeFileSync(filePath, `${JSON.stringify(merged.ledger, null, 2)}\n`, { mode: 0o600 });
|
|
539
|
+
return merged.added;
|
|
540
|
+
}
|
|
541
|
+
|
|
385
542
|
function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
386
543
|
if (!Object.hasOwn(GATE_FILES, gate)) {
|
|
387
|
-
throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`);
|
|
544
|
+
throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`, [], { why: 'Only the listed stages exist.', fix: `Use one of: ${Object.keys(GATE_FILES).join(', ')}.` });
|
|
388
545
|
}
|
|
389
546
|
const run = resolveRun(rootDir, runId);
|
|
390
547
|
const mode = run.mode || RUN_MODES.ASSISTED;
|
|
@@ -394,17 +551,23 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
394
551
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.goal);
|
|
395
552
|
requireRunMetadata(value, run, 'goal contract');
|
|
396
553
|
const check = validateGoalContract(value);
|
|
397
|
-
if (!check.valid) throw gateError('Goal Contract gate failed', check.errors);
|
|
398
|
-
|
|
554
|
+
if (!check.valid) throw gateError('Goal Contract gate failed', check.errors, { why: 'Work may not start until acceptance criteria, evidence and failure cases are explicit.', fix: 'Fix the listed Goal Contract fields (every AC needs statement, evidenceRequired, failureCases), then retry `gate goal`.' });
|
|
555
|
+
const frozen = transitionRun(rootDir, run.runId, RUN_STATES.GOAL_FROZEN, {
|
|
399
556
|
gate,
|
|
400
557
|
artifacts: { goalContract: descriptor(rootDir, filePath) },
|
|
401
558
|
...transitionOptions
|
|
402
559
|
});
|
|
560
|
+
try {
|
|
561
|
+
require('./features.js').onGoalFrozen(rootDir, run, value);
|
|
562
|
+
} catch {
|
|
563
|
+
// The feature ledger is optional; never block the Goal gate on it.
|
|
564
|
+
}
|
|
565
|
+
return frozen;
|
|
403
566
|
}
|
|
404
567
|
|
|
405
568
|
if (gate === 'report') {
|
|
406
569
|
if (mode !== RUN_MODES.REPORT_ONLY) {
|
|
407
|
-
throw gateError('Report gate is only available in REPORT_ONLY mode');
|
|
570
|
+
throw gateError('Report gate is only available in REPORT_ONLY mode', [], { why: 'Delivery runs must go through Maker, verification and review.', fix: 'Use the Maker gate for this Run, or start a REPORT_ONLY Run for a report.' });
|
|
408
571
|
}
|
|
409
572
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.report);
|
|
410
573
|
requireRunMetadata(value, run, 'report');
|
|
@@ -415,7 +578,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
415
578
|
reportErrors.push('recommendations must contain non-empty strings');
|
|
416
579
|
}
|
|
417
580
|
reportErrors.push(...validateReportReview(value.review));
|
|
418
|
-
if (reportErrors.length) throw gateError('Report gate failed', reportErrors);
|
|
581
|
+
if (reportErrors.length) throw gateError('Report gate failed', reportErrors, { why: 'A report must stand on cited evidence.', fix: 'Fix the listed report fields, then retry `gate report`.' });
|
|
419
582
|
return transitionRun(rootDir, run.runId, RUN_STATES.REPORTED, {
|
|
420
583
|
gate,
|
|
421
584
|
artifacts: { report: descriptor(rootDir, filePath) }
|
|
@@ -424,16 +587,16 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
424
587
|
|
|
425
588
|
if (gate === 'maker') {
|
|
426
589
|
if (mode === RUN_MODES.REPORT_ONLY) {
|
|
427
|
-
throw gateError('REPORT_ONLY mode cannot enter the Maker stage; finish with the report gate');
|
|
590
|
+
throw gateError('REPORT_ONLY mode cannot enter the Maker stage; finish with the report gate', [], { why: 'REPORT_ONLY Runs never change code.', fix: 'Finish with `gate report`, or start a normal Run to make changes.' });
|
|
428
591
|
}
|
|
429
592
|
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.maker);
|
|
430
593
|
let diff;
|
|
431
594
|
try {
|
|
432
595
|
diff = fs.readFileSync(filePath, 'utf8');
|
|
433
596
|
} catch (cause) {
|
|
434
|
-
throw gateError(`Missing ${GATE_FILES.maker}: ${cause.message}`);
|
|
597
|
+
throw gateError(`Missing ${GATE_FILES.maker}: ${cause.message}`, [], { why: 'The Maker gate binds review to an exact diff.', fix: `Run \`gate maker --from-git --run <id>\` to capture the diff, or write ${GATE_FILES.maker}.` });
|
|
435
598
|
}
|
|
436
|
-
if (!diff.trim()) throw gateError('Maker gate failed: diff.patch is empty');
|
|
599
|
+
if (!diff.trim()) throw gateError('Maker gate failed: diff.patch is empty', [], { why: 'There is no change to verify or review.', fix: 'Make the change first, then run `gate maker --from-git`.' });
|
|
437
600
|
const artifacts = { diff: descriptor(rootDir, filePath) };
|
|
438
601
|
if (mode === RUN_MODES.UNATTENDED && policy.requireSandboxForUnattended) {
|
|
439
602
|
const { filePath: sandboxFile, value: evidence } = readJsonArtifact(
|
|
@@ -453,7 +616,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
453
616
|
if (evidence.diffSha256 !== hashFile(filePath)) {
|
|
454
617
|
sandboxErrors.push('diffSha256 does not match the captured Maker diff');
|
|
455
618
|
}
|
|
456
|
-
if (sandboxErrors.length) throw gateError('Unattended Maker sandbox evidence failed', sandboxErrors);
|
|
619
|
+
if (sandboxErrors.length) throw gateError('Unattended Maker sandbox evidence failed', sandboxErrors, { why: 'UNATTENDED Runs must prove the Maker ran inside the sandbox.', fix: 'Re-run the Maker inside the configured sandbox so its evidence is recorded, then retry.' });
|
|
457
620
|
artifacts.sandboxEvidence = descriptor(rootDir, sandboxFile);
|
|
458
621
|
}
|
|
459
622
|
const raisedTier = raiseTierFromDiff(run.tier, { task: run.task, diff });
|
|
@@ -468,11 +631,20 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
468
631
|
const diffPath = artifactPath(rootDir, run.runId, GATE_FILES.maker);
|
|
469
632
|
const diffHash = hashFile(diffPath);
|
|
470
633
|
if (run.artifacts.diff?.sha256 !== diffHash) {
|
|
471
|
-
throw gateError('Maker diff changed after its gate; run the Maker gate again');
|
|
634
|
+
throw gateError('Maker diff changed after its gate; run the Maker gate again', [], { why: 'Verification evidence is bound to the gated diff hash; a new diff needs new evidence.', fix: 'Run `gate maker --from-git --run <id>`, then `verify` again.' });
|
|
472
635
|
}
|
|
473
636
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.verification);
|
|
474
637
|
const check = validateVerificationBundle(value, run, diffHash, getGitRevision(rootDir));
|
|
475
|
-
if (!check.valid) throw gateError('Verification gate failed', check.errors);
|
|
638
|
+
if (!check.valid) throw gateError('Verification gate failed', check.errors, { why: 'Review only starts on green, diff-bound, complete command evidence.', fix: 'Fix the listed problems (re-run `verify` on the current diff), then retry.' });
|
|
639
|
+
const layerProblem = missingStaticLayer(rootDir, run, value);
|
|
640
|
+
if (layerProblem) throw gateError('Verification gate failed (L1_MISSING)', [layerProblem], { why: 'Unit tests do not prove the change compiles or passes lint.', fix: 'Run `verify` without --cmd so L1 runs first.' });
|
|
641
|
+
const missingRuleChecks = require('./ablation.js').ablated(rootDir, 'rules', run.runId) ? [] : require('./rules.js').missingRules(rootDir, value);
|
|
642
|
+
if (missingRuleChecks.length) {
|
|
643
|
+
throw gateError('Verification gate failed (RULES_MISSING)', missingRuleChecks.map((rule) => `${rule.id}: \`${rule.check}\` has no passing record`),
|
|
644
|
+
{ why: 'Promoted rules encode review feedback that kept recurring; skipping them lets the same defect back in.', fix: `Run \`verify --run ${run.runId}\` without --cmd so every rule runs, and fix any violation it reports.` });
|
|
645
|
+
}
|
|
646
|
+
const smokeProblem = missingSmoke(rootDir, run, value);
|
|
647
|
+
if (smokeProblem) throw gateError('Verification gate failed (SMOKE_MISSING)', [smokeProblem], { why: 'Migrations, config, env and entrypoint changes can break startup while unit tests stay green.', fix: 'Run `verify` without --cmd so the smoke check runs.' });
|
|
476
648
|
return transitionRun(rootDir, run.runId, RUN_STATES.VERIFIED, {
|
|
477
649
|
gate,
|
|
478
650
|
artifacts: { verification: descriptor(rootDir, filePath) }
|
|
@@ -484,22 +656,32 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
484
656
|
assertArtifactCurrent(rootDir, run, 'diff');
|
|
485
657
|
assertArtifactCurrent(rootDir, run, 'verification');
|
|
486
658
|
if (tierOf(run) === 'T0') writeT0FindingLedger(rootDir, run);
|
|
659
|
+
else if (tierOf(run)) {
|
|
660
|
+
// T1/T2: the Devil's Advocate must have reviewed against completed,
|
|
661
|
+
// layer-tagged evidence (three-layer termination check).
|
|
662
|
+
const status = claimedVsRealityStatus(rootDir, run);
|
|
663
|
+
if (!status.ready) {
|
|
664
|
+
throw gateError(`${tierOf(run)} review needs a completed claimed-vs-reality file (CLAIMED_VS_REALITY_INCOMPLETE)`, status.problems, { why: 'The Devil\'s Advocate reviews claims against evidence; missing or untagged evidence hides what was really proven.', fix: 'Fix each listed row (Reality from command output, Layer backed by a recorded command), then retry `gate review`.' });
|
|
665
|
+
}
|
|
666
|
+
}
|
|
487
667
|
mergeImpactDrift(rootDir, run);
|
|
668
|
+
mergeVerificationIntegrity(rootDir, run);
|
|
669
|
+
mergeCommentJustifications(rootDir, run);
|
|
488
670
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
|
|
489
671
|
requireRunMetadata(value, run, 'Finding Ledger');
|
|
490
672
|
if (value.diffHash !== run.artifacts.diff?.sha256) {
|
|
491
|
-
throw gateError('Finding Ledger diffHash does not match the gated Maker diff');
|
|
673
|
+
throw gateError('Finding Ledger diffHash does not match the gated Maker diff', [], { why: 'A review of a different diff says nothing about this one.', fix: 'Re-run the Devil\'s Advocate on the current review pack, then `review ingest`.' });
|
|
492
674
|
}
|
|
493
675
|
const check = validateFindingLedger(value);
|
|
494
|
-
if (!check.valid) throw gateError('Review gate failed', [check.reason]);
|
|
676
|
+
if (!check.valid) throw gateError('Review gate failed', [check.reason], { why: 'The Judge recomputes the verdict from the ledger, so every finding must be well-formed.', fix: 'Fix the listed ledger field, then retry `gate review`.' });
|
|
495
677
|
const binding = checkLedgerBinding(rootDir, run, value);
|
|
496
678
|
if (binding.errors.length) {
|
|
497
|
-
throw gateError('Finding Ledger does not match the reviewer output (LEDGER_TAMPERED)', binding.errors);
|
|
679
|
+
throw gateError('Finding Ledger does not match the reviewer output (LEDGER_TAMPERED)', binding.errors, { why: 'Findings may not be dropped or silently downgraded between the reviewer and the gate.', fix: 'Restore the reviewer\'s findings, or add dismissal {by, evidence} for each downgrade.' });
|
|
498
680
|
}
|
|
499
681
|
if (binding.binding === 'UNBOUND' && mode === RUN_MODES.UNATTENDED) {
|
|
500
682
|
throw gateError('UNATTENDED review needs a ledger bound to the reviewer output', [
|
|
501
683
|
'Run `review ingest --run <id> --child-id <id> --file <child-output>` instead of writing findings.json'
|
|
502
|
-
]);
|
|
684
|
+
], { why: 'Without a human watching, the ledger must be provably the reviewer\'s own output.', fix: 'Use `review ingest --run <id> --child-id <id> --file <child-output>`.' });
|
|
503
685
|
}
|
|
504
686
|
return transitionRun(rootDir, run.runId, RUN_STATES.REVIEWED, {
|
|
505
687
|
gate,
|
|
@@ -515,28 +697,43 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
515
697
|
writeDeterministicVerdictIfEligible(rootDir, run);
|
|
516
698
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.judge);
|
|
517
699
|
const verdictCheck = validateVerdictDocument(value, run);
|
|
518
|
-
if (!verdictCheck.valid) throw gateError('Judge gate failed', verdictCheck.errors);
|
|
700
|
+
if (!verdictCheck.valid) throw gateError('Judge gate failed', verdictCheck.errors, { why: 'The verdict must name its decision, reason and action.', fix: 'Fix the listed verdict fields, then retry `gate judge`.' });
|
|
519
701
|
if (tierOf(run) === 'T2' && value.verdict === 'PASS'
|
|
520
702
|
&& ![MODEL_TIERS.STRONG_MODEL, MODEL_TIERS.HUMAN].includes(value.modelTier)) {
|
|
521
|
-
throw gateError('T2 Judge PASS requires modelTier STRONG_MODEL or HUMAN', ['TIER_T2_STRONG_JUDGE']);
|
|
703
|
+
throw gateError('T2 Judge PASS requires modelTier STRONG_MODEL or HUMAN', ['TIER_T2_STRONG_JUDGE'], { why: 'T2 changes (auth, money, data, security) need the strongest judgment.', fix: 'Re-run the Judge on STRONG_MODEL (or a human) and set modelTier accordingly.' });
|
|
522
704
|
}
|
|
523
705
|
try {
|
|
524
706
|
enforceVerdictTier(rootDir, run, value);
|
|
525
707
|
} catch (cause) {
|
|
526
|
-
throw gateError(cause.message, [cause.code]);
|
|
708
|
+
throw gateError(cause.message, [cause.code], { why: 'The verdict contradicts the escalation policy.', fix: 'Follow the escalation output (`escalation --json`) and re-issue the verdict.' });
|
|
527
709
|
}
|
|
528
710
|
const judgeFindings = validateJudgeFindings(value.judgeFindings);
|
|
529
711
|
const expected = expectedJudgeVerdict(rootDir, run, { judgeFindings });
|
|
530
712
|
if (value.verdict !== expected.verdict) {
|
|
531
|
-
throw gateError(`Judge verdict is unsupported: expected ${expected.verdict}, received ${value.verdict}`, [expected.reason]);
|
|
713
|
+
throw gateError(`Judge verdict is unsupported: expected ${expected.verdict}, received ${value.verdict}`, [expected.reason], { why: 'The gate recomputes the verdict from Validity and Severity; prose cannot override it.', fix: `Issue ${expected.verdict}, or add the blocking problem as a judgeFinding so the ledger supports your verdict.` });
|
|
532
714
|
}
|
|
715
|
+
const blocking = expected.blockingFindings || [];
|
|
716
|
+
const cause = value.cause || inferCause(expected.verdict, blocking);
|
|
533
717
|
recordIteration(rootDir, run, {
|
|
534
718
|
verdict: expected.verdict,
|
|
535
719
|
diffHash: run.artifacts?.diff?.sha256,
|
|
536
|
-
blockingFindings:
|
|
720
|
+
blockingFindings: blocking,
|
|
721
|
+
cause,
|
|
722
|
+
causeInferred: Boolean(cause && !value.cause)
|
|
537
723
|
});
|
|
538
724
|
const artifacts = { verdict: descriptor(rootDir, filePath) };
|
|
539
725
|
if (expected.verdict === 'PASS') {
|
|
726
|
+
const rubric = rubricGaps(rootDir, run);
|
|
727
|
+
if (rubric.length) {
|
|
728
|
+
throw gateError('Judge PASS refused: rubric criteria are not scored or failed (AC_RUBRIC)', rubric,
|
|
729
|
+
{ why: 'Subjective ACs are judged generously by default; a rubric scored with evidence per criterion makes the judgment reproducible.', fix: 'Have the Devil\'s Advocate add rubricScores [{ac, criterion, pass, evidence}] for every criterion (screenshot path, DOM snapshot, measured value), re-ingest, then retry; or issue ITERATE for failed criteria.' });
|
|
730
|
+
}
|
|
731
|
+
const unproven = unprovenCriteria(rootDir, run);
|
|
732
|
+
if (unproven.length) {
|
|
733
|
+
throw gateError('Judge PASS refused: acceptance criteria are not proven (AC_NOT_PROVEN)',
|
|
734
|
+
unproven.map((ac) => `${ac.id}: verify command \`${ac.command}\` is ${ac.status}`),
|
|
735
|
+
{ why: 'An AC that names its verify command is proven only when that command passed on the gated diff.', fix: `Run \`verify --run ${run.runId} --cmd "<the AC's command>"\` for each listed AC (or fix the change), then retry \`gate judge\`.` });
|
|
736
|
+
}
|
|
540
737
|
return transitionRun(rootDir, run.runId, RUN_STATES.JUDGED_PASS, { gate, artifacts });
|
|
541
738
|
}
|
|
542
739
|
if (expected.verdict === 'ESCALATE') {
|
|
@@ -569,14 +766,33 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
569
766
|
}
|
|
570
767
|
const finalCheck = finalVerificationStatus(rootDir, run);
|
|
571
768
|
if (!finalCheck.satisfied) deliveryErrors.push(`Full verification required before delivery: ${finalCheck.reason}`);
|
|
572
|
-
|
|
769
|
+
const cleanState = require('./clean-state.js').checkCleanState(rootDir, run, { mode });
|
|
770
|
+
deliveryErrors.push(...cleanState.errors);
|
|
771
|
+
if (deliveryErrors.length) throw gateError('Delivery gate failed', deliveryErrors, { why: 'Delivery is the last point where missing evidence can be caught before people rely on the change.', fix: 'Fix each listed delivery problem, then retry `gate delivery`.' });
|
|
772
|
+
// C-15: the linked feature becomes passing with this Run as evidence.
|
|
773
|
+
try {
|
|
774
|
+
require('./features.js').onDelivered(rootDir, run, loadFrozenGoal(rootDir, run));
|
|
775
|
+
} catch {
|
|
776
|
+
// Optional ledger.
|
|
777
|
+
}
|
|
778
|
+
// C-14: accepted trade-offs and L3 waivers outlive the Run.
|
|
779
|
+
let debtAdded = 0;
|
|
780
|
+
try {
|
|
781
|
+
debtAdded = require('./debt.js').recordRunDebt(rootDir, run.runId).added;
|
|
782
|
+
} catch {
|
|
783
|
+
debtAdded = 0;
|
|
784
|
+
}
|
|
573
785
|
return transitionRun(rootDir, run.runId, RUN_STATES.DELIVERED, {
|
|
574
786
|
gate,
|
|
575
|
-
artifacts: { delivery: descriptor(rootDir, filePath) }
|
|
787
|
+
artifacts: { delivery: descriptor(rootDir, filePath) },
|
|
788
|
+
...(debtAdded ? { reason: `${debtAdded} debt item(s) recorded in .ai-engineering-loop/debt.json` } : {}),
|
|
789
|
+
...(cleanState.warnings.length ? { historyMetadata: { cleanStateWarnings: cleanState.warnings } } : {})
|
|
576
790
|
});
|
|
577
791
|
}
|
|
578
792
|
|
|
579
793
|
module.exports = {
|
|
794
|
+
FAILURE_CAUSES,
|
|
795
|
+
inferCause,
|
|
580
796
|
GATE_FILES,
|
|
581
797
|
validateReportReview,
|
|
582
798
|
artifactPath,
|