ai-engineering-loop 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +11 -0
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +21 -11
- package/.claude/agents/devil-advocate.md +6 -1
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +6 -1
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +22 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +6 -1
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +262 -20
- package/core/bench.md +15 -0
- package/core/goal-contract.md +35 -0
- package/core/verification-loop.md +96 -7
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +63 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/calibration.js +112 -0
- package/lib/claimed-vs-reality.js +114 -16
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/debt.js +110 -0
- package/lib/features.js +95 -0
- package/lib/gates.js +234 -41
- package/lib/harness-audit.js +124 -0
- package/lib/no-progress.js +6 -1
- package/lib/review-ingest.js +54 -2
- package/lib/rules.js +112 -0
- package/lib/stats.js +90 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +64 -0
- package/lib/verification-recorder.js +17 -5
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +31 -0
- package/schemas/verdict.schema.json +2 -1
package/lib/gates.js
CHANGED
|
@@ -51,7 +51,7 @@ const {
|
|
|
51
51
|
enforceVerdictTier
|
|
52
52
|
} = require('./escalation.js');
|
|
53
53
|
const { raiseTierFromDiff } = require('./tier.js');
|
|
54
|
-
const {
|
|
54
|
+
const { checkClaimedVsReality, loadFrozenGoal } = require('./claimed-vs-reality.js');
|
|
55
55
|
const { runMaxIterations, detectNoProgress, recordIteration } = require('./no-progress.js');
|
|
56
56
|
const { finalVerificationStatus } = require('./verification-lanes.js');
|
|
57
57
|
const { checkLedgerBinding } = require('./review-ingest.js');
|
|
@@ -68,10 +68,14 @@ const GATE_FILES = Object.freeze({
|
|
|
68
68
|
delivery: 'delivery.json'
|
|
69
69
|
});
|
|
70
70
|
|
|
71
|
-
|
|
71
|
+
// C-04: every gate failure says what failed (message), why the gate cares
|
|
72
|
+
// (why), and the next concrete step (fix), so an agent can self-correct.
|
|
73
|
+
function gateError(message, details = [], { why = null, fix = null } = {}) {
|
|
72
74
|
const err = new Error(message);
|
|
73
75
|
err.code = 'GATE_FAILED';
|
|
74
76
|
err.details = details;
|
|
77
|
+
if (why) err.why = why;
|
|
78
|
+
if (fix) err.fix = fix;
|
|
75
79
|
return err;
|
|
76
80
|
}
|
|
77
81
|
|
|
@@ -85,7 +89,7 @@ function readJsonArtifact(rootDir, runId, fileName) {
|
|
|
85
89
|
try {
|
|
86
90
|
value = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
87
91
|
} catch (cause) {
|
|
88
|
-
throw gateError(`Missing or invalid ${fileName}: ${cause.message}`);
|
|
92
|
+
throw gateError(`Missing or invalid ${fileName}: ${cause.message}`, [], { why: 'Every gate reads its artifact from the Run directory; without it there is nothing to check.', fix: `Write ${fileName} in the Run directory (\`scaffold\` can create a template), then retry the gate.` });
|
|
89
93
|
}
|
|
90
94
|
return { filePath, value };
|
|
91
95
|
}
|
|
@@ -94,7 +98,7 @@ function hashFile(filePath) {
|
|
|
94
98
|
try {
|
|
95
99
|
return hashContent(fs.readFileSync(filePath));
|
|
96
100
|
} catch (cause) {
|
|
97
|
-
throw gateError(`Unable to hash ${filePath}: ${cause.message}`);
|
|
101
|
+
throw gateError(`Unable to hash ${filePath}: ${cause.message}`, [], { why: 'Gates bind artifacts by sha256 so later edits are detected.', fix: `Check that ${filePath} exists and is readable, then retry.` });
|
|
98
102
|
}
|
|
99
103
|
}
|
|
100
104
|
|
|
@@ -108,11 +112,11 @@ function descriptor(rootDir, filePath) {
|
|
|
108
112
|
function assertArtifactCurrent(rootDir, run, name) {
|
|
109
113
|
const artifact = run.artifacts[name];
|
|
110
114
|
if (!artifact?.path || !artifact.sha256) {
|
|
111
|
-
throw gateError(`Required ${name} artifact has not passed its gate`);
|
|
115
|
+
throw gateError(`Required ${name} artifact has not passed its gate`, [], { why: 'Stages run in order; a later gate cannot trust an earlier artifact that never passed.', fix: `Pass the ${name} gate first (\`gate <stage> --run <id>\`).` });
|
|
112
116
|
}
|
|
113
117
|
const filePath = path.join(rootDir, artifact.path);
|
|
114
118
|
if (hashFile(filePath) !== artifact.sha256) {
|
|
115
|
-
throw gateError(`${name} artifact changed after its gate; rerun the owning stage`);
|
|
119
|
+
throw gateError(`${name} artifact changed after its gate; rerun the owning stage`, [], { why: 'Gated artifacts are frozen by hash; a silent edit would invalidate later evidence.', fix: `Re-run the stage that owns ${name}, or restore the gated version.` });
|
|
116
120
|
}
|
|
117
121
|
}
|
|
118
122
|
|
|
@@ -120,7 +124,7 @@ function requireRunMetadata(document, run, label) {
|
|
|
120
124
|
const errors = [];
|
|
121
125
|
if (document.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
122
126
|
if (document.runId !== run.runId) errors.push(`runId must be ${run.runId}`);
|
|
123
|
-
if (errors.length) throw gateError(`${label} metadata is invalid`, errors);
|
|
127
|
+
if (errors.length) throw gateError(`${label} metadata is invalid`, errors, { why: 'Artifacts must name the Run they belong to so evidence cannot be copied across Runs.', fix: 'Set schemaVersion, runId and the other listed fields to this Run\'s values.' });
|
|
124
128
|
}
|
|
125
129
|
|
|
126
130
|
function validateGoalContract(contract) {
|
|
@@ -200,6 +204,18 @@ function validateVerificationBundle(bundle, run, diffHash, currentRevision) {
|
|
|
200
204
|
return { valid: errors.length === 0, errors };
|
|
201
205
|
}
|
|
202
206
|
|
|
207
|
+
// C-12 (learn-harness-engineering L01/L02): which harness layer a failed
|
|
208
|
+
// iteration points at. `model` = an implementation defect the harness could
|
|
209
|
+
// not have prevented.
|
|
210
|
+
const FAILURE_CAUSES = Object.freeze(['task_spec', 'context', 'environment', 'verification', 'state', 'model']);
|
|
211
|
+
|
|
212
|
+
function inferCause(verdict, blockingFindings = []) {
|
|
213
|
+
if (verdict === 'PASS') return null;
|
|
214
|
+
if (blockingFindings.some((finding) => finding.source === 'verification-integrity')) return 'verification';
|
|
215
|
+
if (verdict === 'ESCALATE') return 'task_spec';
|
|
216
|
+
return 'model';
|
|
217
|
+
}
|
|
218
|
+
|
|
203
219
|
function validateVerdictDocument(verdict, run) {
|
|
204
220
|
const errors = [];
|
|
205
221
|
requireRunMetadata(verdict, run, 'verdict');
|
|
@@ -214,6 +230,9 @@ function validateVerdictDocument(verdict, run) {
|
|
|
214
230
|
if (verdict.humanApproved != null && typeof verdict.humanApproved !== 'boolean') {
|
|
215
231
|
errors.push('humanApproved must be a boolean');
|
|
216
232
|
}
|
|
233
|
+
if (verdict.cause != null && !FAILURE_CAUSES.includes(verdict.cause)) {
|
|
234
|
+
errors.push(`cause must be one of ${FAILURE_CAUSES.join(', ')}`);
|
|
235
|
+
}
|
|
217
236
|
return { valid: errors.length === 0, errors };
|
|
218
237
|
}
|
|
219
238
|
|
|
@@ -228,20 +247,117 @@ function verificationForJudge(bundle) {
|
|
|
228
247
|
function resolveRun(rootDir, runId) {
|
|
229
248
|
if (runId) return loadRun(rootDir, runId);
|
|
230
249
|
const current = getCurrentRun(rootDir);
|
|
231
|
-
if (!current) throw gateError('No current run. Start one with `ai-engineering-loop run <task>`.');
|
|
250
|
+
if (!current) throw gateError('No current run. Start one with `ai-engineering-loop run <task>`.', [], { why: 'Gates act on a Run.', fix: 'Start a Run with `run <task>` or pass --run <id>.' });
|
|
232
251
|
return current;
|
|
233
252
|
}
|
|
234
253
|
|
|
254
|
+
// C-02: at T1/T2, when the project configures typecheck or lint, the gated
|
|
255
|
+
// bundle must contain at least one L1 (static) command. Unit tests alone do
|
|
256
|
+
// not prove the change compiles or passes lint.
|
|
257
|
+
function missingStaticLayer(rootDir, run, bundle) {
|
|
258
|
+
const tier = tierOf(run);
|
|
259
|
+
if (tier !== 'T1' && tier !== 'T2') return null;
|
|
260
|
+
if (require('./ablation.js').ablated(rootDir, 'l1', run.runId)) return null;
|
|
261
|
+
const lanes = require('./verification-lanes.js');
|
|
262
|
+
const configured = lanes.verificationCommands(rootDir);
|
|
263
|
+
const staticKeys = ['typecheck', 'lint'].filter((key) => configured[key] && !lanes.isMutatingCommand(configured[key]));
|
|
264
|
+
if (!staticKeys.length) return null;
|
|
265
|
+
// The related lane is the iteration fast path; `verify --final` runs the full plan.
|
|
266
|
+
if ((bundle.commands || []).length && bundle.commands.every((record) => record.scope === 'related')) return null;
|
|
267
|
+
const hasStatic = (bundle.commands || []).some((record) =>
|
|
268
|
+
(record.layer || lanes.layerOfCommand(rootDir, record.command)) === 'L1');
|
|
269
|
+
if (hasStatic) return null;
|
|
270
|
+
return `No L1 static check in the verification bundle, but verification.md configures ${staticKeys.join(' and ')}. FIX: run \`verify --run ${run.runId}\` without --cmd (it runs L1 first), or add --cmd "${configured[staticKeys[0]]}".`;
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// C-03: when verification.md configures `smoke` and the diff touches
|
|
274
|
+
// startup-risky files, a passing smoke record must be in the bundle.
|
|
275
|
+
function missingSmoke(rootDir, run, bundle) {
|
|
276
|
+
const lanes = require('./verification-lanes.js');
|
|
277
|
+
const smokeText = lanes.verificationCommands(rootDir).smoke;
|
|
278
|
+
if (!smokeText) return null;
|
|
279
|
+
if (require('./ablation.js').ablated(rootDir, 'smoke', run.runId)) return null;
|
|
280
|
+
const risky = lanes.startupRiskFiles(rootDir, run);
|
|
281
|
+
if (!risky.length) return null;
|
|
282
|
+
const base = smokeText.trim();
|
|
283
|
+
if ((bundle.commands || []).some((record) => String(record.command || '').trim() === base)) return null;
|
|
284
|
+
return `The diff touches startup-risky files (${risky.slice(0, 3).join(', ')}${risky.length > 3 ? ', ...' : ''}) but the smoke check did not run. FIX: run \`verify --run ${run.runId}\` without --cmd, or add --cmd "${base}".`;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
function rubricGaps(rootDir, run) {
|
|
288
|
+
let goal;
|
|
289
|
+
try {
|
|
290
|
+
goal = loadFrozenGoal(rootDir, run);
|
|
291
|
+
} catch {
|
|
292
|
+
return [];
|
|
293
|
+
}
|
|
294
|
+
let ledger = null;
|
|
295
|
+
try {
|
|
296
|
+
ledger = readJsonArtifact(rootDir, run.runId, GATE_FILES.review).value;
|
|
297
|
+
} catch {
|
|
298
|
+
ledger = null;
|
|
299
|
+
}
|
|
300
|
+
return require('./ac-status.js').rubricProblems(goal, ledger);
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
// C-07: ACs with verify.command must be proven by a passing record.
|
|
304
|
+
function unprovenCriteria(rootDir, run) {
|
|
305
|
+
let goal;
|
|
306
|
+
try {
|
|
307
|
+
goal = loadFrozenGoal(rootDir, run);
|
|
308
|
+
} catch {
|
|
309
|
+
return [];
|
|
310
|
+
}
|
|
311
|
+
return require('./ac-status.js').runAcStatuses(rootDir, run, goal)
|
|
312
|
+
.filter((ac) => ac.status === 'pending' || ac.status === 'failed');
|
|
313
|
+
}
|
|
314
|
+
|
|
235
315
|
function tierOf(run) {
|
|
236
316
|
return run?.tier?.current || null;
|
|
237
317
|
}
|
|
238
318
|
|
|
239
|
-
|
|
319
|
+
// Layers proven by recorded verification (gated bundle + final run). A
|
|
320
|
+
// command not mapped in verification.md counts as L2 (a test run).
|
|
321
|
+
function recordedLayers(rootDir, run) {
|
|
322
|
+
if (!run) return null;
|
|
323
|
+
const lanes = require('./verification-lanes.js');
|
|
324
|
+
const layers = new Set();
|
|
325
|
+
for (const file of ['verification.json', 'final-verification.json']) {
|
|
326
|
+
let bundle;
|
|
327
|
+
try {
|
|
328
|
+
bundle = JSON.parse(fs.readFileSync(artifactPath(rootDir, run.runId, file), 'utf8'));
|
|
329
|
+
} catch {
|
|
330
|
+
continue;
|
|
331
|
+
}
|
|
332
|
+
for (const record of bundle.commands || []) {
|
|
333
|
+
if (record.exitCode !== 0) continue;
|
|
334
|
+
layers.add(record.layer || lanes.layerOfCommand(rootDir, record.command) || 'L2');
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
return layers.size ? layers : null;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
// Status of claimed-vs-reality.md for this run, including the three-layer
|
|
341
|
+
// (L1/L2/L3) requirement from the frozen Goal Contract and the run tier.
|
|
342
|
+
function claimedVsRealityStatus(rootDir, run) {
|
|
343
|
+
let markdown;
|
|
240
344
|
try {
|
|
241
|
-
|
|
345
|
+
markdown = fs.readFileSync(path.join(rootDir, CLAIMED_VS_REALITY_PATH), 'utf8');
|
|
242
346
|
} catch {
|
|
243
|
-
return false;
|
|
347
|
+
return { ready: false, problems: [`${CLAIMED_VS_REALITY_PATH} is missing. FIX: scaffold it with \`context claimed-vs-reality --run ${run?.runId || '<id>'}\` after \`gate verification\`, then fill every row.`] };
|
|
244
348
|
}
|
|
349
|
+
let goal = null;
|
|
350
|
+
try {
|
|
351
|
+
goal = run ? loadFrozenGoal(rootDir, run) : null;
|
|
352
|
+
} catch {
|
|
353
|
+
goal = null;
|
|
354
|
+
}
|
|
355
|
+
if (require('./ablation.js').ablated(rootDir, 'l3', run?.runId)) return checkClaimedVsReality(markdown, {});
|
|
356
|
+
return checkClaimedVsReality(markdown, { goal, tier: tierOf(run), layers: recordedLayers(rootDir, run) });
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function claimedVsRealityReady(rootDir, run) {
|
|
360
|
+
return claimedVsRealityStatus(rootDir, run).ready;
|
|
245
361
|
}
|
|
246
362
|
|
|
247
363
|
// T0 has no Devil's Advocate. The CLI writes an explicit empty, diff-bound
|
|
@@ -250,10 +366,9 @@ function claimedVsRealityReady(rootDir) {
|
|
|
250
366
|
function writeT0FindingLedger(rootDir, run) {
|
|
251
367
|
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
|
|
252
368
|
if (fs.existsSync(filePath)) return false;
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
]);
|
|
369
|
+
const status = claimedVsRealityStatus(rootDir, run);
|
|
370
|
+
if (!status.ready) {
|
|
371
|
+
throw gateError('T0 review needs a completed claimed-vs-reality file', status.problems, { why: 'T0 has no Devil\'s Advocate, so claimed-vs-reality is the only check that claims match evidence.', fix: 'Fill every Claimed and Reality cell (scaffold: `context claimed-vs-reality`), then retry `gate review`.' });
|
|
257
372
|
}
|
|
258
373
|
fs.writeFileSync(filePath, `${JSON.stringify({
|
|
259
374
|
schemaVersion: 1,
|
|
@@ -294,7 +409,7 @@ function judgeRequirement(rootDir, run) {
|
|
|
294
409
|
} catch (cause) {
|
|
295
410
|
reasons.push(`ESCALATION_UNAVAILABLE: ${cause.message}`);
|
|
296
411
|
}
|
|
297
|
-
if (!claimedVsRealityReady(rootDir)) reasons.push('CLAIMED_VS_REALITY_INCOMPLETE');
|
|
412
|
+
if (!claimedVsRealityReady(rootDir, run)) reasons.push('CLAIMED_VS_REALITY_INCOMPLETE');
|
|
298
413
|
return { required: reasons.length > 0, tier, reasons };
|
|
299
414
|
}
|
|
300
415
|
|
|
@@ -335,9 +450,9 @@ function expectedJudgeVerdict(rootDir, run, { judgeFindings = [] } = {}) {
|
|
|
335
450
|
|
|
336
451
|
function validateJudgeFindings(value) {
|
|
337
452
|
if (value == null) return [];
|
|
338
|
-
if (!Array.isArray(value)) throw gateError('Judge gate failed', ['judgeFindings must be an array']);
|
|
453
|
+
if (!Array.isArray(value)) throw gateError('Judge gate failed', ['judgeFindings must be an array'], { why: 'Judge-added findings use the Finding Ledger shape so the gate can recompute the verdict.', fix: 'Make judgeFindings an array of ledger findings (or remove it).' });
|
|
339
454
|
const check = validateFindingLedger({ findings: value });
|
|
340
|
-
if (!check.valid) throw gateError('Judge gate failed', [`judgeFindings: ${check.reason}`]);
|
|
455
|
+
if (!check.valid) throw gateError('Judge gate failed', [`judgeFindings: ${check.reason}`], { why: 'Judge-added findings must be complete enough to act on.', fix: 'Give each judgeFinding location, failureScenario, evidence, severity, validity and (for VALID BLOCKER/HIGH) concreteAlternativeDiff.' });
|
|
341
456
|
return value.map((finding) => ({ ...finding, source: 'judge' }));
|
|
342
457
|
}
|
|
343
458
|
|
|
@@ -348,7 +463,7 @@ function writeDeterministicVerdictIfEligible(rootDir, run) {
|
|
|
348
463
|
if (!tier) return false;
|
|
349
464
|
const requirement = judgeRequirement(rootDir, run);
|
|
350
465
|
if (requirement.required) {
|
|
351
|
-
throw gateError(`Judge subagent required for tier ${tier}; write ${GATE_FILES.judge} from its verdict`, requirement.reasons);
|
|
466
|
+
throw gateError(`Judge subagent required for tier ${tier}; write ${GATE_FILES.judge} from its verdict`, requirement.reasons, { why: 'This Run has signals (tier, serious findings, escalation, incomplete evidence) that a deterministic verdict must not decide alone.', fix: `Spawn the Judge subagent with the ledger and Goal, write its verdict to ${GATE_FILES.judge}, then retry \`gate judge\`.` });
|
|
352
467
|
}
|
|
353
468
|
const expected = expectedJudgeVerdict(rootDir, run);
|
|
354
469
|
fs.writeFileSync(filePath, `${JSON.stringify({
|
|
@@ -382,9 +497,29 @@ function mergeImpactDrift(rootDir, run) {
|
|
|
382
497
|
return merged.added;
|
|
383
498
|
}
|
|
384
499
|
|
|
500
|
+
// C-11: weakened proof in the gated diff (skips, deleted tests, dropped
|
|
501
|
+
// assertions, edited verification config) becomes VALID ledger findings.
|
|
502
|
+
function mergeVerificationIntegrity(rootDir, run) {
|
|
503
|
+
if (require('./ablation.js').ablated(rootDir, 'integrity', run.runId)) return 0;
|
|
504
|
+
const { scanRun, mergeIntegrityIntoLedger } = require('./verification-integrity.js');
|
|
505
|
+
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.review);
|
|
506
|
+
if (!fs.existsSync(filePath)) return 0;
|
|
507
|
+
const findings = scanRun(rootDir, run);
|
|
508
|
+
let ledger;
|
|
509
|
+
try {
|
|
510
|
+
ledger = JSON.parse(fs.readFileSync(filePath, 'utf8'));
|
|
511
|
+
} catch {
|
|
512
|
+
return 0;
|
|
513
|
+
}
|
|
514
|
+
const merged = mergeIntegrityIntoLedger(ledger, findings);
|
|
515
|
+
if (merged.errors.length) throw gateError('Review gate failed (VERIFICATION_WEAKENED finding dropped)', merged.errors, { why: 'Weakened tests make a green gate meaningless, so only a human may excuse them.', fix: 'Restore the integrity finding, or record a human dismissal {by, evidence}.' });
|
|
516
|
+
if (merged.added) fs.writeFileSync(filePath, `${JSON.stringify(merged.ledger, null, 2)}\n`, { mode: 0o600 });
|
|
517
|
+
return merged.added;
|
|
518
|
+
}
|
|
519
|
+
|
|
385
520
|
function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
386
521
|
if (!Object.hasOwn(GATE_FILES, gate)) {
|
|
387
|
-
throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`);
|
|
522
|
+
throw gateError(`Unknown gate: ${gate}. Use ${Object.keys(GATE_FILES).join('|')}`, [], { why: 'Only the listed stages exist.', fix: `Use one of: ${Object.keys(GATE_FILES).join(', ')}.` });
|
|
388
523
|
}
|
|
389
524
|
const run = resolveRun(rootDir, runId);
|
|
390
525
|
const mode = run.mode || RUN_MODES.ASSISTED;
|
|
@@ -394,17 +529,23 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
394
529
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.goal);
|
|
395
530
|
requireRunMetadata(value, run, 'goal contract');
|
|
396
531
|
const check = validateGoalContract(value);
|
|
397
|
-
if (!check.valid) throw gateError('Goal Contract gate failed', check.errors);
|
|
398
|
-
|
|
532
|
+
if (!check.valid) throw gateError('Goal Contract gate failed', check.errors, { why: 'Work may not start until acceptance criteria, evidence and failure cases are explicit.', fix: 'Fix the listed Goal Contract fields (every AC needs statement, evidenceRequired, failureCases), then retry `gate goal`.' });
|
|
533
|
+
const frozen = transitionRun(rootDir, run.runId, RUN_STATES.GOAL_FROZEN, {
|
|
399
534
|
gate,
|
|
400
535
|
artifacts: { goalContract: descriptor(rootDir, filePath) },
|
|
401
536
|
...transitionOptions
|
|
402
537
|
});
|
|
538
|
+
try {
|
|
539
|
+
require('./features.js').onGoalFrozen(rootDir, run, value);
|
|
540
|
+
} catch {
|
|
541
|
+
// The feature ledger is optional; never block the Goal gate on it.
|
|
542
|
+
}
|
|
543
|
+
return frozen;
|
|
403
544
|
}
|
|
404
545
|
|
|
405
546
|
if (gate === 'report') {
|
|
406
547
|
if (mode !== RUN_MODES.REPORT_ONLY) {
|
|
407
|
-
throw gateError('Report gate is only available in REPORT_ONLY mode');
|
|
548
|
+
throw gateError('Report gate is only available in REPORT_ONLY mode', [], { why: 'Delivery runs must go through Maker, verification and review.', fix: 'Use the Maker gate for this Run, or start a REPORT_ONLY Run for a report.' });
|
|
408
549
|
}
|
|
409
550
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.report);
|
|
410
551
|
requireRunMetadata(value, run, 'report');
|
|
@@ -415,7 +556,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
415
556
|
reportErrors.push('recommendations must contain non-empty strings');
|
|
416
557
|
}
|
|
417
558
|
reportErrors.push(...validateReportReview(value.review));
|
|
418
|
-
if (reportErrors.length) throw gateError('Report gate failed', reportErrors);
|
|
559
|
+
if (reportErrors.length) throw gateError('Report gate failed', reportErrors, { why: 'A report must stand on cited evidence.', fix: 'Fix the listed report fields, then retry `gate report`.' });
|
|
419
560
|
return transitionRun(rootDir, run.runId, RUN_STATES.REPORTED, {
|
|
420
561
|
gate,
|
|
421
562
|
artifacts: { report: descriptor(rootDir, filePath) }
|
|
@@ -424,16 +565,16 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
424
565
|
|
|
425
566
|
if (gate === 'maker') {
|
|
426
567
|
if (mode === RUN_MODES.REPORT_ONLY) {
|
|
427
|
-
throw gateError('REPORT_ONLY mode cannot enter the Maker stage; finish with the report gate');
|
|
568
|
+
throw gateError('REPORT_ONLY mode cannot enter the Maker stage; finish with the report gate', [], { why: 'REPORT_ONLY Runs never change code.', fix: 'Finish with `gate report`, or start a normal Run to make changes.' });
|
|
428
569
|
}
|
|
429
570
|
const filePath = artifactPath(rootDir, run.runId, GATE_FILES.maker);
|
|
430
571
|
let diff;
|
|
431
572
|
try {
|
|
432
573
|
diff = fs.readFileSync(filePath, 'utf8');
|
|
433
574
|
} catch (cause) {
|
|
434
|
-
throw gateError(`Missing ${GATE_FILES.maker}: ${cause.message}`);
|
|
575
|
+
throw gateError(`Missing ${GATE_FILES.maker}: ${cause.message}`, [], { why: 'The Maker gate binds review to an exact diff.', fix: `Run \`gate maker --from-git --run <id>\` to capture the diff, or write ${GATE_FILES.maker}.` });
|
|
435
576
|
}
|
|
436
|
-
if (!diff.trim()) throw gateError('Maker gate failed: diff.patch is empty');
|
|
577
|
+
if (!diff.trim()) throw gateError('Maker gate failed: diff.patch is empty', [], { why: 'There is no change to verify or review.', fix: 'Make the change first, then run `gate maker --from-git`.' });
|
|
437
578
|
const artifacts = { diff: descriptor(rootDir, filePath) };
|
|
438
579
|
if (mode === RUN_MODES.UNATTENDED && policy.requireSandboxForUnattended) {
|
|
439
580
|
const { filePath: sandboxFile, value: evidence } = readJsonArtifact(
|
|
@@ -453,7 +594,7 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
453
594
|
if (evidence.diffSha256 !== hashFile(filePath)) {
|
|
454
595
|
sandboxErrors.push('diffSha256 does not match the captured Maker diff');
|
|
455
596
|
}
|
|
456
|
-
if (sandboxErrors.length) throw gateError('Unattended Maker sandbox evidence failed', sandboxErrors);
|
|
597
|
+
if (sandboxErrors.length) throw gateError('Unattended Maker sandbox evidence failed', sandboxErrors, { why: 'UNATTENDED Runs must prove the Maker ran inside the sandbox.', fix: 'Re-run the Maker inside the configured sandbox so its evidence is recorded, then retry.' });
|
|
457
598
|
artifacts.sandboxEvidence = descriptor(rootDir, sandboxFile);
|
|
458
599
|
}
|
|
459
600
|
const raisedTier = raiseTierFromDiff(run.tier, { task: run.task, diff });
|
|
@@ -468,11 +609,20 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
468
609
|
const diffPath = artifactPath(rootDir, run.runId, GATE_FILES.maker);
|
|
469
610
|
const diffHash = hashFile(diffPath);
|
|
470
611
|
if (run.artifacts.diff?.sha256 !== diffHash) {
|
|
471
|
-
throw gateError('Maker diff changed after its gate; run the Maker gate again');
|
|
612
|
+
throw gateError('Maker diff changed after its gate; run the Maker gate again', [], { why: 'Verification evidence is bound to the gated diff hash; a new diff needs new evidence.', fix: 'Run `gate maker --from-git --run <id>`, then `verify` again.' });
|
|
472
613
|
}
|
|
473
614
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.verification);
|
|
474
615
|
const check = validateVerificationBundle(value, run, diffHash, getGitRevision(rootDir));
|
|
475
|
-
if (!check.valid) throw gateError('Verification gate failed', check.errors);
|
|
616
|
+
if (!check.valid) throw gateError('Verification gate failed', check.errors, { why: 'Review only starts on green, diff-bound, complete command evidence.', fix: 'Fix the listed problems (re-run `verify` on the current diff), then retry.' });
|
|
617
|
+
const layerProblem = missingStaticLayer(rootDir, run, value);
|
|
618
|
+
if (layerProblem) throw gateError('Verification gate failed (L1_MISSING)', [layerProblem], { why: 'Unit tests do not prove the change compiles or passes lint.', fix: 'Run `verify` without --cmd so L1 runs first.' });
|
|
619
|
+
const missingRuleChecks = require('./ablation.js').ablated(rootDir, 'rules', run.runId) ? [] : require('./rules.js').missingRules(rootDir, value);
|
|
620
|
+
if (missingRuleChecks.length) {
|
|
621
|
+
throw gateError('Verification gate failed (RULES_MISSING)', missingRuleChecks.map((rule) => `${rule.id}: \`${rule.check}\` has no passing record`),
|
|
622
|
+
{ why: 'Promoted rules encode review feedback that kept recurring; skipping them lets the same defect back in.', fix: `Run \`verify --run ${run.runId}\` without --cmd so every rule runs, and fix any violation it reports.` });
|
|
623
|
+
}
|
|
624
|
+
const smokeProblem = missingSmoke(rootDir, run, value);
|
|
625
|
+
if (smokeProblem) throw gateError('Verification gate failed (SMOKE_MISSING)', [smokeProblem], { why: 'Migrations, config, env and entrypoint changes can break startup while unit tests stay green.', fix: 'Run `verify` without --cmd so the smoke check runs.' });
|
|
476
626
|
return transitionRun(rootDir, run.runId, RUN_STATES.VERIFIED, {
|
|
477
627
|
gate,
|
|
478
628
|
artifacts: { verification: descriptor(rootDir, filePath) }
|
|
@@ -484,22 +634,31 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
484
634
|
assertArtifactCurrent(rootDir, run, 'diff');
|
|
485
635
|
assertArtifactCurrent(rootDir, run, 'verification');
|
|
486
636
|
if (tierOf(run) === 'T0') writeT0FindingLedger(rootDir, run);
|
|
637
|
+
else if (tierOf(run)) {
|
|
638
|
+
// T1/T2: the Devil's Advocate must have reviewed against completed,
|
|
639
|
+
// layer-tagged evidence (three-layer termination check).
|
|
640
|
+
const status = claimedVsRealityStatus(rootDir, run);
|
|
641
|
+
if (!status.ready) {
|
|
642
|
+
throw gateError(`${tierOf(run)} review needs a completed claimed-vs-reality file (CLAIMED_VS_REALITY_INCOMPLETE)`, status.problems, { why: 'The Devil\'s Advocate reviews claims against evidence; missing or untagged evidence hides what was really proven.', fix: 'Fix each listed row (Reality from command output, Layer backed by a recorded command), then retry `gate review`.' });
|
|
643
|
+
}
|
|
644
|
+
}
|
|
487
645
|
mergeImpactDrift(rootDir, run);
|
|
646
|
+
mergeVerificationIntegrity(rootDir, run);
|
|
488
647
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.review);
|
|
489
648
|
requireRunMetadata(value, run, 'Finding Ledger');
|
|
490
649
|
if (value.diffHash !== run.artifacts.diff?.sha256) {
|
|
491
|
-
throw gateError('Finding Ledger diffHash does not match the gated Maker diff');
|
|
650
|
+
throw gateError('Finding Ledger diffHash does not match the gated Maker diff', [], { why: 'A review of a different diff says nothing about this one.', fix: 'Re-run the Devil\'s Advocate on the current review pack, then `review ingest`.' });
|
|
492
651
|
}
|
|
493
652
|
const check = validateFindingLedger(value);
|
|
494
|
-
if (!check.valid) throw gateError('Review gate failed', [check.reason]);
|
|
653
|
+
if (!check.valid) throw gateError('Review gate failed', [check.reason], { why: 'The Judge recomputes the verdict from the ledger, so every finding must be well-formed.', fix: 'Fix the listed ledger field, then retry `gate review`.' });
|
|
495
654
|
const binding = checkLedgerBinding(rootDir, run, value);
|
|
496
655
|
if (binding.errors.length) {
|
|
497
|
-
throw gateError('Finding Ledger does not match the reviewer output (LEDGER_TAMPERED)', binding.errors);
|
|
656
|
+
throw gateError('Finding Ledger does not match the reviewer output (LEDGER_TAMPERED)', binding.errors, { why: 'Findings may not be dropped or silently downgraded between the reviewer and the gate.', fix: 'Restore the reviewer\'s findings, or add dismissal {by, evidence} for each downgrade.' });
|
|
498
657
|
}
|
|
499
658
|
if (binding.binding === 'UNBOUND' && mode === RUN_MODES.UNATTENDED) {
|
|
500
659
|
throw gateError('UNATTENDED review needs a ledger bound to the reviewer output', [
|
|
501
660
|
'Run `review ingest --run <id> --child-id <id> --file <child-output>` instead of writing findings.json'
|
|
502
|
-
]);
|
|
661
|
+
], { why: 'Without a human watching, the ledger must be provably the reviewer\'s own output.', fix: 'Use `review ingest --run <id> --child-id <id> --file <child-output>`.' });
|
|
503
662
|
}
|
|
504
663
|
return transitionRun(rootDir, run.runId, RUN_STATES.REVIEWED, {
|
|
505
664
|
gate,
|
|
@@ -515,28 +674,43 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
515
674
|
writeDeterministicVerdictIfEligible(rootDir, run);
|
|
516
675
|
const { filePath, value } = readJsonArtifact(rootDir, run.runId, GATE_FILES.judge);
|
|
517
676
|
const verdictCheck = validateVerdictDocument(value, run);
|
|
518
|
-
if (!verdictCheck.valid) throw gateError('Judge gate failed', verdictCheck.errors);
|
|
677
|
+
if (!verdictCheck.valid) throw gateError('Judge gate failed', verdictCheck.errors, { why: 'The verdict must name its decision, reason and action.', fix: 'Fix the listed verdict fields, then retry `gate judge`.' });
|
|
519
678
|
if (tierOf(run) === 'T2' && value.verdict === 'PASS'
|
|
520
679
|
&& ![MODEL_TIERS.STRONG_MODEL, MODEL_TIERS.HUMAN].includes(value.modelTier)) {
|
|
521
|
-
throw gateError('T2 Judge PASS requires modelTier STRONG_MODEL or HUMAN', ['TIER_T2_STRONG_JUDGE']);
|
|
680
|
+
throw gateError('T2 Judge PASS requires modelTier STRONG_MODEL or HUMAN', ['TIER_T2_STRONG_JUDGE'], { why: 'T2 changes (auth, money, data, security) need the strongest judgment.', fix: 'Re-run the Judge on STRONG_MODEL (or a human) and set modelTier accordingly.' });
|
|
522
681
|
}
|
|
523
682
|
try {
|
|
524
683
|
enforceVerdictTier(rootDir, run, value);
|
|
525
684
|
} catch (cause) {
|
|
526
|
-
throw gateError(cause.message, [cause.code]);
|
|
685
|
+
throw gateError(cause.message, [cause.code], { why: 'The verdict contradicts the escalation policy.', fix: 'Follow the escalation output (`escalation --json`) and re-issue the verdict.' });
|
|
527
686
|
}
|
|
528
687
|
const judgeFindings = validateJudgeFindings(value.judgeFindings);
|
|
529
688
|
const expected = expectedJudgeVerdict(rootDir, run, { judgeFindings });
|
|
530
689
|
if (value.verdict !== expected.verdict) {
|
|
531
|
-
throw gateError(`Judge verdict is unsupported: expected ${expected.verdict}, received ${value.verdict}`, [expected.reason]);
|
|
690
|
+
throw gateError(`Judge verdict is unsupported: expected ${expected.verdict}, received ${value.verdict}`, [expected.reason], { why: 'The gate recomputes the verdict from Validity and Severity; prose cannot override it.', fix: `Issue ${expected.verdict}, or add the blocking problem as a judgeFinding so the ledger supports your verdict.` });
|
|
532
691
|
}
|
|
692
|
+
const blocking = expected.blockingFindings || [];
|
|
693
|
+
const cause = value.cause || inferCause(expected.verdict, blocking);
|
|
533
694
|
recordIteration(rootDir, run, {
|
|
534
695
|
verdict: expected.verdict,
|
|
535
696
|
diffHash: run.artifacts?.diff?.sha256,
|
|
536
|
-
blockingFindings:
|
|
697
|
+
blockingFindings: blocking,
|
|
698
|
+
cause,
|
|
699
|
+
causeInferred: Boolean(cause && !value.cause)
|
|
537
700
|
});
|
|
538
701
|
const artifacts = { verdict: descriptor(rootDir, filePath) };
|
|
539
702
|
if (expected.verdict === 'PASS') {
|
|
703
|
+
const rubric = rubricGaps(rootDir, run);
|
|
704
|
+
if (rubric.length) {
|
|
705
|
+
throw gateError('Judge PASS refused: rubric criteria are not scored or failed (AC_RUBRIC)', rubric,
|
|
706
|
+
{ why: 'Subjective ACs are judged generously by default; a rubric scored with evidence per criterion makes the judgment reproducible.', fix: 'Have the Devil\'s Advocate add rubricScores [{ac, criterion, pass, evidence}] for every criterion (screenshot path, DOM snapshot, measured value), re-ingest, then retry; or issue ITERATE for failed criteria.' });
|
|
707
|
+
}
|
|
708
|
+
const unproven = unprovenCriteria(rootDir, run);
|
|
709
|
+
if (unproven.length) {
|
|
710
|
+
throw gateError('Judge PASS refused: acceptance criteria are not proven (AC_NOT_PROVEN)',
|
|
711
|
+
unproven.map((ac) => `${ac.id}: verify command \`${ac.command}\` is ${ac.status}`),
|
|
712
|
+
{ why: 'An AC that names its verify command is proven only when that command passed on the gated diff.', fix: `Run \`verify --run ${run.runId} --cmd "<the AC's command>"\` for each listed AC (or fix the change), then retry \`gate judge\`.` });
|
|
713
|
+
}
|
|
540
714
|
return transitionRun(rootDir, run.runId, RUN_STATES.JUDGED_PASS, { gate, artifacts });
|
|
541
715
|
}
|
|
542
716
|
if (expected.verdict === 'ESCALATE') {
|
|
@@ -569,14 +743,33 @@ function applyGate(rootDir, gate, { runId, transitionOptions = {} } = {}) {
|
|
|
569
743
|
}
|
|
570
744
|
const finalCheck = finalVerificationStatus(rootDir, run);
|
|
571
745
|
if (!finalCheck.satisfied) deliveryErrors.push(`Full verification required before delivery: ${finalCheck.reason}`);
|
|
572
|
-
|
|
746
|
+
const cleanState = require('./clean-state.js').checkCleanState(rootDir, run, { mode });
|
|
747
|
+
deliveryErrors.push(...cleanState.errors);
|
|
748
|
+
if (deliveryErrors.length) throw gateError('Delivery gate failed', deliveryErrors, { why: 'Delivery is the last point where missing evidence can be caught before people rely on the change.', fix: 'Fix each listed delivery problem, then retry `gate delivery`.' });
|
|
749
|
+
// C-15: the linked feature becomes passing with this Run as evidence.
|
|
750
|
+
try {
|
|
751
|
+
require('./features.js').onDelivered(rootDir, run, loadFrozenGoal(rootDir, run));
|
|
752
|
+
} catch {
|
|
753
|
+
// Optional ledger.
|
|
754
|
+
}
|
|
755
|
+
// C-14: accepted trade-offs and L3 waivers outlive the Run.
|
|
756
|
+
let debtAdded = 0;
|
|
757
|
+
try {
|
|
758
|
+
debtAdded = require('./debt.js').recordRunDebt(rootDir, run.runId).added;
|
|
759
|
+
} catch {
|
|
760
|
+
debtAdded = 0;
|
|
761
|
+
}
|
|
573
762
|
return transitionRun(rootDir, run.runId, RUN_STATES.DELIVERED, {
|
|
574
763
|
gate,
|
|
575
|
-
artifacts: { delivery: descriptor(rootDir, filePath) }
|
|
764
|
+
artifacts: { delivery: descriptor(rootDir, filePath) },
|
|
765
|
+
...(debtAdded ? { reason: `${debtAdded} debt item(s) recorded in .ai-engineering-loop/debt.json` } : {}),
|
|
766
|
+
...(cleanState.warnings.length ? { historyMetadata: { cleanStateWarnings: cleanState.warnings } } : {})
|
|
576
767
|
});
|
|
577
768
|
}
|
|
578
769
|
|
|
579
770
|
module.exports = {
|
|
771
|
+
FAILURE_CAUSES,
|
|
772
|
+
inferCause,
|
|
580
773
|
GATE_FILES,
|
|
581
774
|
validateReportReview,
|
|
582
775
|
artifactPath,
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// `doctor --harness` (C-08, learn-harness-engineering L03/L04/L12): harness
|
|
4
|
+
// debt in the project, reported as candidates to fix or delete. Harness rots
|
|
5
|
+
// like code: entry files bloat, learnings go stale, rules outlive their need.
|
|
6
|
+
|
|
7
|
+
const fs = require('fs');
|
|
8
|
+
const path = require('path');
|
|
9
|
+
const { runsDir } = require('./run-state.js');
|
|
10
|
+
|
|
11
|
+
const ENTRY_MIN = 50;
|
|
12
|
+
const ENTRY_MAX = 200;
|
|
13
|
+
const STALE_LEARNING_RUNS = 10;
|
|
14
|
+
const RULE_REVIEW_DAYS = 90;
|
|
15
|
+
const BUDGET_HEADROOM = 0.02;
|
|
16
|
+
|
|
17
|
+
// Only structured lines count (headings, lists, tables, code, bold leads), so
|
|
18
|
+
// sprinkling keywords into prose cannot game the score (Goodhart, L14).
|
|
19
|
+
// Adapted from harness-creator's structuredText (MIT).
|
|
20
|
+
function structuredText(markdown) {
|
|
21
|
+
const kept = [];
|
|
22
|
+
let inFence = false;
|
|
23
|
+
for (const raw of String(markdown || '').split(/\r?\n/)) {
|
|
24
|
+
const line = raw.trim();
|
|
25
|
+
if (/^(```|~~~)/.test(line)) {
|
|
26
|
+
inFence = !inFence;
|
|
27
|
+
continue;
|
|
28
|
+
}
|
|
29
|
+
if (inFence || /^#{1,6}\s/.test(line) || /^([-*+]|\d+\.)\s/.test(line) || line.startsWith('|') || /^\*\*[^*]+\*\*/.test(line)) {
|
|
30
|
+
if (line) kept.push(line);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return kept.join('\n').toLowerCase();
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function auditEntryFile(rootDir) {
|
|
37
|
+
const name = ['AGENTS.md', 'CLAUDE.md'].find((file) => fs.existsSync(path.join(rootDir, file)));
|
|
38
|
+
if (!name) return [{ id: 'entry-file', level: 'warn', detail: 'No AGENTS.md or CLAUDE.md. FIX: add a 50-200 line router: overview, run/verify commands, hard rules, links to docs.' }];
|
|
39
|
+
const text = fs.readFileSync(path.join(rootDir, name), 'utf8');
|
|
40
|
+
const lines = text.split(/\r?\n/).length;
|
|
41
|
+
const items = [];
|
|
42
|
+
if (lines > ENTRY_MAX) items.push({ id: 'entry-file-size', level: 'warn', detail: `${name} has ${lines} lines (> ${ENTRY_MAX}). FIX: keep it a router; move topic detail to docs/ and link it.` });
|
|
43
|
+
if (lines < ENTRY_MIN) items.push({ id: 'entry-file-size', level: 'info', detail: `${name} has ${lines} lines; fine if it still answers what/run/verify.` });
|
|
44
|
+
const structured = structuredText(text);
|
|
45
|
+
const needs = [
|
|
46
|
+
['verification', /verif|test|check/],
|
|
47
|
+
['definition of done', /definition of done|done when|done only when/],
|
|
48
|
+
['startup', /startup|start here|quick start|before writing|setup/]
|
|
49
|
+
];
|
|
50
|
+
for (const [label, pattern] of needs) {
|
|
51
|
+
if (!pattern.test(structured)) items.push({ id: 'entry-file-structure', level: 'warn', detail: `${name} has no structured ${label} section (heading or list). FIX: add one; prose mentions do not count.` });
|
|
52
|
+
}
|
|
53
|
+
return items;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function listRunIds(rootDir) {
|
|
57
|
+
try {
|
|
58
|
+
return fs.readdirSync(runsDir(rootDir), { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort();
|
|
59
|
+
} catch {
|
|
60
|
+
return [];
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function auditLearnings(rootDir) {
|
|
65
|
+
let entries = [];
|
|
66
|
+
try {
|
|
67
|
+
entries = JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'learnings.json'), 'utf8')).entries || [];
|
|
68
|
+
} catch {
|
|
69
|
+
return [];
|
|
70
|
+
}
|
|
71
|
+
const runs = listRunIds(rootDir);
|
|
72
|
+
const recent = new Set(runs.slice(-STALE_LEARNING_RUNS));
|
|
73
|
+
if (runs.length <= STALE_LEARNING_RUNS) return [];
|
|
74
|
+
return entries
|
|
75
|
+
.filter((entry) => !(entry.runIds || []).some((runId) => recent.has(runId)))
|
|
76
|
+
.map((entry) => ({ id: 'stale-learning', level: 'warn', detail: `${entry.id} not seen in the last ${STALE_LEARNING_RUNS} Runs: ${String(entry.lesson || '').slice(0, 80)}. FIX: \`learn --prune\` or delete it if the problem is gone.` }));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function auditRules(rootDir, now = new Date()) {
|
|
80
|
+
let rules = [];
|
|
81
|
+
try {
|
|
82
|
+
rules = JSON.parse(fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'rules.json'), 'utf8')).rules || [];
|
|
83
|
+
} catch {
|
|
84
|
+
return [];
|
|
85
|
+
}
|
|
86
|
+
const items = [];
|
|
87
|
+
for (const rule of rules) {
|
|
88
|
+
if (!rule.from || !rule.why || !rule.fix) items.push({ id: 'rule-provenance', level: 'warn', detail: `${rule.id} lacks a source (from), why or fix. FIX: record where it came from and how to repair a violation.` });
|
|
89
|
+
const age = (now - new Date(rule.createdAt || 0)) / 86400000;
|
|
90
|
+
if (age > RULE_REVIEW_DAYS) items.push({ id: 'rule-review', level: 'info', detail: `${rule.id} is ${Math.floor(age)} days old. Re-check it is still needed (bench --without rules).` });
|
|
91
|
+
}
|
|
92
|
+
return items;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Inside the AEL package repo: T1 skill-context headroom from scripts/measure.js.
|
|
96
|
+
function auditBudget(rootDir) {
|
|
97
|
+
const script = path.join(rootDir, 'scripts', 'measure.js');
|
|
98
|
+
if (!fs.existsSync(script)) return [];
|
|
99
|
+
try {
|
|
100
|
+
const { measureAll, BASELINE } = require(script);
|
|
101
|
+
const t1 = measureAll().results.find((row) => row.tier === 'T1');
|
|
102
|
+
const limit = BASELINE.parentContextChars / 2;
|
|
103
|
+
const headroom = (limit - t1.parentContextChars) / limit;
|
|
104
|
+
return [{
|
|
105
|
+
id: 'skill-budget',
|
|
106
|
+
level: headroom < BUDGET_HEADROOM ? 'warn' : 'info',
|
|
107
|
+
detail: `T1 skill context ${t1.parentContextChars}/${limit} chars (${(headroom * 100).toFixed(2)}% headroom).${headroom < BUDGET_HEADROOM ? ' FIX: trim skill-src references before adding text.' : ''}`
|
|
108
|
+
}];
|
|
109
|
+
} catch (cause) {
|
|
110
|
+
return [{ id: 'skill-budget', level: 'warn', detail: `measure failed: ${cause.message}` }];
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function auditHarness(rootDir, { budget = true } = {}) {
|
|
115
|
+
const items = [
|
|
116
|
+
...auditEntryFile(rootDir),
|
|
117
|
+
...auditLearnings(rootDir),
|
|
118
|
+
...auditRules(rootDir),
|
|
119
|
+
...(budget ? auditBudget(rootDir) : [])
|
|
120
|
+
];
|
|
121
|
+
return { ok: !items.some((item) => item.level === 'warn'), items };
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
module.exports = { auditHarness, structuredText };
|
package/lib/no-progress.js
CHANGED
|
@@ -87,12 +87,17 @@ function detectNoProgress(rootDir, run, { diffHash, blockingFindings = [] }) {
|
|
|
87
87
|
return null;
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
-
function recordIteration(rootDir, run, { verdict, diffHash, blockingFindings = [], now = new Date() }) {
|
|
90
|
+
function recordIteration(rootDir, run, { verdict, diffHash, blockingFindings = [], cause = null, causeInferred = false, now = new Date() }) {
|
|
91
91
|
const entry = {
|
|
92
92
|
iteration: run.iteration,
|
|
93
93
|
verdict,
|
|
94
94
|
diffHash: diffHash || null,
|
|
95
95
|
blockingSignatures: blockingFindings.map(findingSignature),
|
|
96
|
+
// C-12: verification was green when the Judge ran, so blocking findings
|
|
97
|
+
// here mean the Maker's "done" was premature (a false victory).
|
|
98
|
+
blockingCount: blockingFindings.length,
|
|
99
|
+
...(cause ? { cause, causeInferred: Boolean(causeInferred) } : {}),
|
|
100
|
+
tier: run.tier?.current || null,
|
|
96
101
|
at: now.toISOString()
|
|
97
102
|
};
|
|
98
103
|
const filePath = iterationsPath(rootDir, run.runId);
|