ai-engineering-loop 2.5.0 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +11 -0
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +21 -11
- package/.claude/agents/devil-advocate.md +6 -1
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +6 -1
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +22 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +6 -1
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +262 -20
- package/core/bench.md +15 -0
- package/core/goal-contract.md +35 -0
- package/core/verification-loop.md +96 -7
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +63 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/calibration.js +112 -0
- package/lib/claimed-vs-reality.js +114 -16
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/debt.js +110 -0
- package/lib/features.js +95 -0
- package/lib/gates.js +234 -41
- package/lib/harness-audit.js +124 -0
- package/lib/no-progress.js +6 -1
- package/lib/review-ingest.js +54 -2
- package/lib/rules.js +112 -0
- package/lib/stats.js +90 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +64 -0
- package/lib/verification-recorder.js +17 -5
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +31 -0
- package/schemas/verdict.schema.json +2 -1
|
@@ -425,9 +425,9 @@ function analyzeRepository(rootDir) {
|
|
|
425
425
|
scripts: {
|
|
426
426
|
testUnit: 'npm test',
|
|
427
427
|
testAll: 'npm test',
|
|
428
|
-
typecheck:
|
|
429
|
-
lint:
|
|
430
|
-
build:
|
|
428
|
+
typecheck: null,
|
|
429
|
+
lint: null,
|
|
430
|
+
build: null,
|
|
431
431
|
e2e: null
|
|
432
432
|
},
|
|
433
433
|
adapter: {
|
|
@@ -506,12 +506,16 @@ function analyzeRepository(rootDir) {
|
|
|
506
506
|
discovery.scripts.testRelated = viaScript('--findRelatedTests --passWithNoTests {files}') || 'npx jest --findRelatedTests --passWithNoTests {files}';
|
|
507
507
|
}
|
|
508
508
|
|
|
509
|
+
// Only commands the project really has: a guessed `npx tsc` can fetch an
|
|
510
|
+
// unrelated package, and `eslint --fix` would edit files during verify.
|
|
511
|
+
const hasTypeScript = Boolean(allDeps['typescript']) || fs.existsSync(path.join(rootDir, 'tsconfig.json'));
|
|
509
512
|
if (scripts['typecheck']) discovery.scripts.typecheck = `${pm} run typecheck`;
|
|
510
513
|
else if (scripts['type-check']) discovery.scripts.typecheck = `${pm} run type-check`;
|
|
511
514
|
else if (scripts['check']) discovery.scripts.typecheck = `${pm} run check`;
|
|
515
|
+
else discovery.scripts.typecheck = hasTypeScript ? 'npx tsc --noEmit' : null;
|
|
512
516
|
|
|
513
|
-
|
|
514
|
-
|
|
517
|
+
discovery.scripts.lint = scripts['lint'] ? `${pm} run lint` : null;
|
|
518
|
+
discovery.scripts.build = scripts['build'] ? `${pm} run build` : null;
|
|
515
519
|
if (scripts['test:e2e']) discovery.scripts.e2e = `${pm} run test:e2e`;
|
|
516
520
|
} catch (e) {}
|
|
517
521
|
}
|
|
@@ -664,9 +668,9 @@ ${discovery.topLevelDirs.map((d) => `- \`${d}/\``).join('\n') || '- Flat directo
|
|
|
664
668
|
|
|
665
669
|
## Discovered Verification Commands
|
|
666
670
|
- **test_unit**: \`${discovery.scripts.testUnit}\`
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
671
|
+
${discovery.scripts.typecheck ? `- **typecheck**: \`${discovery.scripts.typecheck}\`` : ''}
|
|
672
|
+
${discovery.scripts.lint ? `- **lint**: \`${discovery.scripts.lint}\`` : ''}
|
|
673
|
+
${discovery.scripts.build ? `- **build**: \`${discovery.scripts.build}\`` : ''}
|
|
670
674
|
${discovery.scripts.e2e ? `- **e2e**: \`${discovery.scripts.e2e}\`` : ''}
|
|
671
675
|
${discovery.scripts.testRelated ? `- **test_related**: \`${discovery.scripts.testRelated}\`` : ''}
|
|
672
676
|
|
|
@@ -1390,6 +1394,15 @@ function handleRun() {
|
|
|
1390
1394
|
const current = getCurrentRun(CWD);
|
|
1391
1395
|
const policy = loadPolicy(CWD);
|
|
1392
1396
|
const startsNewRun = forceNew || !current || TERMINAL_STATES.has(current.state);
|
|
1397
|
+
if (forceNew && current && !TERMINAL_STATES.has(current.state)) {
|
|
1398
|
+
// C-15 (L07 WIP=1): starting another Run while one is unfinished.
|
|
1399
|
+
try {
|
|
1400
|
+
const ratio = require('../lib/features.js').vcr(require('../lib/features.js').loadFeatures(CWD));
|
|
1401
|
+
if (ratio.value < 1) console.error(`! VCR ${ratio.verified}/${ratio.activated}: Run ${current.runId} is still ${current.state}. Finishing it first usually beats starting another (WIP=1).`);
|
|
1402
|
+
} catch {
|
|
1403
|
+
// No feature ledger.
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1393
1406
|
let mode = modeArg ? normalizeMode(modeArg) : (startsNewRun ? policy.defaultMode : null);
|
|
1394
1407
|
let workflow = null;
|
|
1395
1408
|
let requestedRunId = null;
|
|
@@ -2015,6 +2028,7 @@ function handleGate() {
|
|
|
2015
2028
|
console.log(`- Run: ${run.runId}`);
|
|
2016
2029
|
console.log(`- State: ${run.state}, iteration ${run.iteration}`);
|
|
2017
2030
|
if (run.tier) console.log(`- Tier: ${run.tier.current}`);
|
|
2031
|
+
for (const warning of (run.history || []).at(-1)?.cleanStateWarnings || []) console.log(`! Clean state: ${warning}`);
|
|
2018
2032
|
if (guardEvents.warn || guardEvents.block) {
|
|
2019
2033
|
console.log(`- Guard: ${guardEvents.block} blocked, ${guardEvents.warn} warned (${guardEvents.rules.join(', ')}); see guard-events.json`);
|
|
2020
2034
|
}
|
|
@@ -2037,10 +2051,14 @@ function handleGate() {
|
|
|
2037
2051
|
ok: false,
|
|
2038
2052
|
code: err.code || 'GATE_FAILED',
|
|
2039
2053
|
error: err.message,
|
|
2054
|
+
...(err.why ? { why: err.why } : {}),
|
|
2055
|
+
...(err.fix ? { fix: err.fix } : {}),
|
|
2040
2056
|
details
|
|
2041
2057
|
}));
|
|
2042
2058
|
} else {
|
|
2043
2059
|
log.error(err.message);
|
|
2060
|
+
if (err.why) console.error(`WHY: ${err.why}`);
|
|
2061
|
+
if (err.fix) console.error(`FIX: ${err.fix}`);
|
|
2044
2062
|
for (const detail of details) console.error(`- ${detail}`);
|
|
2045
2063
|
}
|
|
2046
2064
|
process.exit(1);
|
|
@@ -2373,6 +2391,16 @@ function handleState() {
|
|
|
2373
2391
|
|
|
2374
2392
|
function handleDoctor() {
|
|
2375
2393
|
const json = process.argv.includes('--json');
|
|
2394
|
+
if (process.argv.includes('--harness')) {
|
|
2395
|
+
// C-08: harness debt in this project (warnings never block).
|
|
2396
|
+
const audit = require('../lib/harness-audit.js').auditHarness(CWD);
|
|
2397
|
+
if (json) console.log(JSON.stringify(audit));
|
|
2398
|
+
else {
|
|
2399
|
+
console.log(`Harness audit: ${audit.ok ? 'no debt found' : 'debt found'}`);
|
|
2400
|
+
for (const item of audit.items) console.log(`${item.level === 'warn' ? '!' : '-'} ${item.id}: ${item.detail}`);
|
|
2401
|
+
}
|
|
2402
|
+
return;
|
|
2403
|
+
}
|
|
2376
2404
|
const result = runDoctor(path.join(__dirname, '..'), { projectDir: CWD });
|
|
2377
2405
|
if (json) console.log(JSON.stringify(result));
|
|
2378
2406
|
else {
|
|
@@ -3363,9 +3391,11 @@ function splitCommand(text, shell) {
|
|
|
3363
3391
|
|
|
3364
3392
|
function stageJsonError(err, json, code) {
|
|
3365
3393
|
if (json) {
|
|
3366
|
-
console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, details: err.details || [] }));
|
|
3394
|
+
console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, ...(err.why ? { why: err.why } : {}), ...(err.fix ? { fix: err.fix } : {}), details: err.details || [] }));
|
|
3367
3395
|
} else {
|
|
3368
3396
|
log.error(err.message);
|
|
3397
|
+
if (err.why) console.error(`WHY: ${err.why}`);
|
|
3398
|
+
if (err.fix) console.error(`FIX: ${err.fix}`);
|
|
3369
3399
|
for (const detail of err.details || []) console.error(`- ${detail}`);
|
|
3370
3400
|
}
|
|
3371
3401
|
process.exit(1);
|
|
@@ -3391,19 +3421,55 @@ async function handleVerify() {
|
|
|
3391
3421
|
if (related && final) throw Object.assign(new Error('--related and --final cannot be combined'), { code: 'INVALID_USAGE' });
|
|
3392
3422
|
let texts = repeatedArg('--cmd');
|
|
3393
3423
|
const lanes = require('../lib/verification-lanes.js');
|
|
3424
|
+
const baselineMode = process.argv.includes('--baseline');
|
|
3425
|
+
if (baselineMode && (related || final)) throw Object.assign(new Error('--baseline cannot be combined with --related or --final'), { code: 'INVALID_USAGE' });
|
|
3394
3426
|
let commandsToRun;
|
|
3427
|
+
let defaultPlan = false;
|
|
3395
3428
|
if (related && texts.length === 0) {
|
|
3396
3429
|
const target = loadRun(CWD, runId);
|
|
3397
|
-
commandsToRun = [lanes.relatedCommand(CWD, target, { split: splitCommand, shell })];
|
|
3398
|
-
} else {
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3430
|
+
commandsToRun = [{ ...lanes.relatedCommand(CWD, target, { split: splitCommand, shell }), layer: 'L2' }];
|
|
3431
|
+
} else if (texts.length === 0) {
|
|
3432
|
+
// C-02: no --cmd runs the cheapest-first plan (L1 build/typecheck/lint,
|
|
3433
|
+
// then L2 test_unit) and stops at the first failure.
|
|
3434
|
+
const plan = lanes.defaultCommandPlan(CWD);
|
|
3435
|
+
// C-05: promoted review rules run as L1 checks, after the static checks.
|
|
3436
|
+
const rules = require('../lib/rules.js').rulePlan(CWD);
|
|
3437
|
+
if (rules.length) {
|
|
3438
|
+
const firstL2 = plan.findIndex((step) => step.layer !== 'L1');
|
|
3439
|
+
plan.splice(firstL2 === -1 ? plan.length : firstL2, 0, ...rules);
|
|
3403
3440
|
}
|
|
3404
|
-
|
|
3441
|
+
// C-03: startup-risky diffs also run the configured smoke check (L2).
|
|
3442
|
+
const smokeText = lanes.verificationCommands(CWD).smoke;
|
|
3443
|
+
if (smokeText && !baselineMode && lanes.startupRiskFiles(CWD, loadRun(CWD, runId)).length) {
|
|
3444
|
+
plan.push({ key: 'smoke', layer: 'L2', text: smokeText });
|
|
3445
|
+
}
|
|
3446
|
+
if (!plan.length) throw Object.assign(new Error('No --cmd given and no build/typecheck/lint/test_unit command in .ai-engineering-loop/verification.md. FIX: add them to verification.md or pass --cmd.'), { code: 'INVALID_COMMAND' });
|
|
3447
|
+
commandsToRun = plan.map((step) => ({ ...splitCommand(step.text, shell || Boolean(step.shell)), key: step.key, layer: step.layer }));
|
|
3448
|
+
defaultPlan = true;
|
|
3449
|
+
} else {
|
|
3450
|
+
commandsToRun = texts.map((text) => {
|
|
3451
|
+
const layer = lanes.layerOfCommand(CWD, text);
|
|
3452
|
+
return { ...splitCommand(text, shell), ...(layer ? { layer } : {}) };
|
|
3453
|
+
});
|
|
3405
3454
|
}
|
|
3406
3455
|
const timeoutValue = argValue('--timeout-ms');
|
|
3456
|
+
if (baselineMode) {
|
|
3457
|
+
const outcome = await require('../lib/baseline.js').recordBaseline(CWD, {
|
|
3458
|
+
runId,
|
|
3459
|
+
plan: commandsToRun,
|
|
3460
|
+
...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
|
|
3461
|
+
});
|
|
3462
|
+
const result = { ok: outcome.status !== 'BROKEN', runId, baseline: outcome };
|
|
3463
|
+
if (json) console.log(JSON.stringify(result));
|
|
3464
|
+
else {
|
|
3465
|
+
const say = outcome.status === 'GREEN' ? log.success : (outcome.status === 'RED' ? log.warn : log.error);
|
|
3466
|
+
say(`Baseline ${outcome.status}: ${outcome.path}`);
|
|
3467
|
+
for (const c of outcome.commands) console.log(`- ${c.command}: exit ${c.exitCode}`);
|
|
3468
|
+
for (const problem of outcome.problems) console.log(`! ${problem}`);
|
|
3469
|
+
}
|
|
3470
|
+
if (!result.ok) process.exitCode = 1;
|
|
3471
|
+
return;
|
|
3472
|
+
}
|
|
3407
3473
|
if (final) {
|
|
3408
3474
|
const target = loadRun(CWD, runId);
|
|
3409
3475
|
if (target.state !== 'JUDGED_PASS') throw Object.assign(new Error(`verify --final needs a Judge PASS (state is ${target.state})`), { code: 'INVALID_STATE' });
|
|
@@ -3437,17 +3503,30 @@ async function handleVerify() {
|
|
|
3437
3503
|
commands: commandsToRun,
|
|
3438
3504
|
serial: process.argv.includes('--serial'),
|
|
3439
3505
|
fresh: process.argv.includes('--fresh'),
|
|
3506
|
+
stopOnFailure: defaultPlan,
|
|
3440
3507
|
...(related ? { scope: 'related' } : {}),
|
|
3441
3508
|
...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
|
|
3442
3509
|
});
|
|
3443
|
-
const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
|
|
3510
|
+
const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.layer ? { layer: c.layer } : {}), ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
|
|
3444
3511
|
if (!recorded.passed) {
|
|
3445
3512
|
autoHeartbeat(runId, 'verification', 'Verification failed; Maker must fix before review');
|
|
3446
|
-
const
|
|
3513
|
+
const baselineModule = require('../lib/baseline.js');
|
|
3514
|
+
const failures = baselineModule.classifyAgainstBaseline(baselineModule.loadBaseline(CWD, runId), recorded.commands);
|
|
3515
|
+
const result = { ok: false, code: 'VERIFICATION_FAILED', runId, commands, skipped: recorded.skipped, failures, gate: null };
|
|
3447
3516
|
if (json) console.log(JSON.stringify(result));
|
|
3448
3517
|
else {
|
|
3449
3518
|
log.error('Verification failed; the gate was not attempted.');
|
|
3450
3519
|
for (const c of commands) console.log(`- ${c.command}: exit ${c.exitCode} (${c.timeoutStatus})`);
|
|
3520
|
+
for (const skippedCommand of recorded.skipped) console.log(`- ${skippedCommand}: skipped (an earlier, cheaper check failed)`);
|
|
3521
|
+
for (const c of commands.filter((item) => item.exitCode !== 0)) {
|
|
3522
|
+
const rule = require('../lib/rules.js').ruleForCommand(CWD, c.command);
|
|
3523
|
+
if (rule) console.log(`! Rule ${rule.id} violated. WHY: ${rule.why} FIX: ${rule.fix}`);
|
|
3524
|
+
}
|
|
3525
|
+
for (const f of failures) {
|
|
3526
|
+
console.log(f.preExisting
|
|
3527
|
+
? `! ${f.command}: every failure was already failing at baseline (pre-existing). FIX: repair it in a separate Run, or narrow --cmd to the tests this change owns.`
|
|
3528
|
+
: `! ${f.command}: new failure(s)${f.newFailures.length ? `: ${f.newFailures.slice(0, 5).join('; ')}` : ''}. FIX: fix the change before review.`);
|
|
3529
|
+
}
|
|
3451
3530
|
}
|
|
3452
3531
|
process.exitCode = 1;
|
|
3453
3532
|
return;
|
|
@@ -3471,6 +3550,14 @@ async function handleVerify() {
|
|
|
3471
3550
|
impactDrift: { mapStatus: drift.mapStatus, outOfMap: drift.outOfMap },
|
|
3472
3551
|
compact: compact.paths,
|
|
3473
3552
|
claimedVsReality: claimed,
|
|
3553
|
+
acStatus: (() => {
|
|
3554
|
+
try {
|
|
3555
|
+
return require('../lib/ac-status.js').runAcStatuses(CWD, run, require('../lib/claimed-vs-reality.js').loadFrozenGoal(CWD, run))
|
|
3556
|
+
.filter((ac) => ac.status !== 'unbound');
|
|
3557
|
+
} catch {
|
|
3558
|
+
return [];
|
|
3559
|
+
}
|
|
3560
|
+
})(),
|
|
3474
3561
|
...(related ? { scope: 'related', finalRequired: 'Run `verify --final` after Judge PASS, before delivery.' } : {}),
|
|
3475
3562
|
next: run.tier && run.tier.current === 'T0'
|
|
3476
3563
|
? 'Fill claimed-vs-reality, then gate review and gate judge'
|
|
@@ -3487,6 +3574,7 @@ async function handleVerify() {
|
|
|
3487
3574
|
console.log('! No impact map for this Run; drift was not checked');
|
|
3488
3575
|
}
|
|
3489
3576
|
console.log(`- Claimed-vs-reality: ${claimed.path}${claimed.created ? ' (scaffolded; fill it from the command output)' : ''}`);
|
|
3577
|
+
for (const ac of result.acStatus) console.log(`- ${ac.id}: ${ac.status} (\`${ac.command}\`)`);
|
|
3490
3578
|
console.log(`- Next: ${result.next}`);
|
|
3491
3579
|
}
|
|
3492
3580
|
} catch (err) {
|
|
@@ -3494,6 +3582,97 @@ async function handleVerify() {
|
|
|
3494
3582
|
}
|
|
3495
3583
|
}
|
|
3496
3584
|
|
|
3585
|
+
function handleOutcome() {
|
|
3586
|
+
const json = process.argv.includes('--json');
|
|
3587
|
+
try {
|
|
3588
|
+
const action = args[1] && !args[1].startsWith('--') ? args[1] : null;
|
|
3589
|
+
if (action !== 'record') throw Object.assign(new Error('outcome supports: record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name> [--evidence <path|url>]'), { code: 'INVALID_USAGE' });
|
|
3590
|
+
const entry = require('../lib/calibration.js').recordOutcome(CWD, {
|
|
3591
|
+
runId: argValue('--run'),
|
|
3592
|
+
source: argValue('--source'),
|
|
3593
|
+
location: argValue('--location'),
|
|
3594
|
+
summary: argValue('--summary'),
|
|
3595
|
+
evidence: argValue('--evidence'),
|
|
3596
|
+
by: argValue('--by')
|
|
3597
|
+
});
|
|
3598
|
+
if (json) console.log(JSON.stringify({ ok: true, outcome: entry }));
|
|
3599
|
+
else log.success(`✓ outcome recorded for ${argValue('--run')}; run \`calibrate\` to compare it with the review`);
|
|
3600
|
+
} catch (err) {
|
|
3601
|
+
stageJsonError(err, json, 'OUTCOME_FAILED');
|
|
3602
|
+
}
|
|
3603
|
+
}
|
|
3604
|
+
|
|
3605
|
+
function handleCalibrate() {
|
|
3606
|
+
const json = process.argv.includes('--json');
|
|
3607
|
+
try {
|
|
3608
|
+
const calibration = require('../lib/calibration.js');
|
|
3609
|
+
const report = calibration.calibrate(CWD);
|
|
3610
|
+
if (json) console.log(JSON.stringify({ ok: true, ...report }));
|
|
3611
|
+
else console.log(calibration.formatCalibration(report));
|
|
3612
|
+
} catch (err) {
|
|
3613
|
+
stageJsonError(err, json, 'CALIBRATE_FAILED');
|
|
3614
|
+
}
|
|
3615
|
+
}
|
|
3616
|
+
|
|
3617
|
+
function handleDebt() {
|
|
3618
|
+
const json = process.argv.includes('--json');
|
|
3619
|
+
try {
|
|
3620
|
+
const debt = require('../lib/debt.js');
|
|
3621
|
+
const action = args[1] && !args[1].startsWith('--') ? args[1] : 'list';
|
|
3622
|
+
if (action === 'close') {
|
|
3623
|
+
const item = debt.closeDebt(CWD, args[2], { by: argValue('--by'), note: argValue('--note') });
|
|
3624
|
+
if (json) console.log(JSON.stringify({ ok: true, item }));
|
|
3625
|
+
else log.success(`✓ closed ${item.id}`);
|
|
3626
|
+
return;
|
|
3627
|
+
}
|
|
3628
|
+
if (action !== 'list') throw Object.assign(new Error('debt supports: list, close <id> --by <name>'), { code: 'INVALID_USAGE' });
|
|
3629
|
+
const file = argValue('--file');
|
|
3630
|
+
const items = debt.loadDebt(CWD).items.filter((item) => item.status === 'open' && (!file || item.file === file));
|
|
3631
|
+
if (json) console.log(JSON.stringify({ ok: true, items }));
|
|
3632
|
+
else if (!items.length) console.log('No open debt.');
|
|
3633
|
+
else for (const item of items) console.log(`- ${item.id} [${item.risk}] ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; revisit: ${item.nextTrigger})`);
|
|
3634
|
+
} catch (err) {
|
|
3635
|
+
stageJsonError(err, json, 'DEBT_FAILED');
|
|
3636
|
+
}
|
|
3637
|
+
}
|
|
3638
|
+
|
|
3639
|
+
function handleNext() {
|
|
3640
|
+
const json = process.argv.includes('--json');
|
|
3641
|
+
try {
|
|
3642
|
+
const features = require('../lib/features.js');
|
|
3643
|
+
const document = features.loadFeatures(CWD);
|
|
3644
|
+
if (!document) throw Object.assign(new Error('No .ai-engineering-loop/features.json. FIX: create it with {"schemaVersion":1,"features":[{"id","title","behavior","verify","status":"not_started","priority","dependsOn":[]}]}.'), { code: 'NO_FEATURES' });
|
|
3645
|
+
const next = features.nextFeature(document);
|
|
3646
|
+
const ratio = features.vcr(document);
|
|
3647
|
+
const problems = features.auditFeatures(CWD, document);
|
|
3648
|
+
const active = document.features.filter((item) => item.status === 'active').map((item) => item.id);
|
|
3649
|
+
const result = { ok: problems.length === 0, next, vcr: ratio, active, problems };
|
|
3650
|
+
if (json) console.log(JSON.stringify(result));
|
|
3651
|
+
else {
|
|
3652
|
+
console.log(`VCR: ${ratio.verified}/${ratio.activated} (${ratio.value})${active.length ? `; active: ${active.join(', ')}` : ''}`);
|
|
3653
|
+
if (ratio.value < 1 && active.length) console.log(`! Finish ${active[0]} before starting another feature (WIP=1).`);
|
|
3654
|
+
console.log(next ? `Next: ${next.id} - ${next.title || next.behavior}${next.verify ? ` (verify: ${next.verify})` : ''}. Start it with a Goal whose featureId is "${next.id}".` : 'Next: nothing unblocked.');
|
|
3655
|
+
for (const problem of problems) console.log(`! ${problem}`);
|
|
3656
|
+
}
|
|
3657
|
+
if (problems.length) process.exitCode = 1;
|
|
3658
|
+
} catch (err) {
|
|
3659
|
+
stageJsonError(err, json, 'NEXT_FAILED');
|
|
3660
|
+
}
|
|
3661
|
+
}
|
|
3662
|
+
|
|
3663
|
+
function handleStats() {
|
|
3664
|
+
const json = process.argv.includes('--json');
|
|
3665
|
+
try {
|
|
3666
|
+
const lastValue = argValue('--last');
|
|
3667
|
+
const stats = require('../lib/stats.js');
|
|
3668
|
+
const result = stats.computeStats(CWD, lastValue == null ? {} : { last: parsePositiveIntegerArg('--last', lastValue) });
|
|
3669
|
+
if (json) console.log(JSON.stringify({ ok: true, ...result }));
|
|
3670
|
+
else console.log(stats.formatStats(result));
|
|
3671
|
+
} catch (err) {
|
|
3672
|
+
stageJsonError(err, json, 'STATS_FAILED');
|
|
3673
|
+
}
|
|
3674
|
+
}
|
|
3675
|
+
|
|
3497
3676
|
function readGuardInput() {
|
|
3498
3677
|
const source = argValue('--input') || (process.argv.includes('--hook') ? '-' : null);
|
|
3499
3678
|
if (source == null) {
|
|
@@ -3639,6 +3818,20 @@ function handleReviewPack() {
|
|
|
3639
3818
|
const pack = smartContextSummary(createDiffHunkPack(CWD, { runId, base, profile: argValue('--profile') }));
|
|
3640
3819
|
const escalation = assessRunEscalation(CWD, { runId });
|
|
3641
3820
|
const tier = run.tier ? run.tier.current : null;
|
|
3821
|
+
// C-14: open debt on the files this change touches goes into the pack.
|
|
3822
|
+
let openDebt = [];
|
|
3823
|
+
try {
|
|
3824
|
+
const changed = require('../lib/verification-lanes.js').relatedFiles(CWD, run);
|
|
3825
|
+
openDebt = require('../lib/debt.js').openDebtFor(CWD, changed);
|
|
3826
|
+
if (openDebt.length && pack.path) {
|
|
3827
|
+
const lines = ['', '## Open debt on changed files', '', ...openDebt.map((item) => `- ${item.id} ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; risk ${item.risk})`)];
|
|
3828
|
+
fs.appendFileSync(path.join(CWD, pack.path), `${lines.join('\n')}\n`);
|
|
3829
|
+
}
|
|
3830
|
+
} catch {
|
|
3831
|
+
openDebt = [];
|
|
3832
|
+
}
|
|
3833
|
+
const diffRelative = path.relative(CWD, path.join(runsDir(CWD), runId, 'diff.patch')).split(path.sep).join('/');
|
|
3834
|
+
const reviewInputs = require('../lib/review-ingest.js').recordReviewInputs(CWD, runId, { packPath: pack.path, diffPath: diffRelative });
|
|
3642
3835
|
autoHeartbeat(runId, 'review', 'Review pack built');
|
|
3643
3836
|
const result = {
|
|
3644
3837
|
ok: true,
|
|
@@ -3652,6 +3845,8 @@ function handleReviewPack() {
|
|
|
3652
3845
|
truncated: pack.truncated,
|
|
3653
3846
|
unresolvedFindings: pack.unresolvedFindings,
|
|
3654
3847
|
impactDrift: pack.impactDrift,
|
|
3848
|
+
openDebt: openDebt.map((item) => ({ id: item.id, location: item.location, debt: item.debt })),
|
|
3849
|
+
reviewInputs: { inputs: reviewInputs.inputs, promptSha256: reviewInputs.promptSha256 },
|
|
3655
3850
|
fastPath: { eligible: fastPath.eligible, reasons: fastPath.reasons },
|
|
3656
3851
|
escalation: { requiredTier: escalation.requiredTier, reasons: escalation.reasons.map((r) => r.code) }
|
|
3657
3852
|
};
|
|
@@ -3665,6 +3860,7 @@ function handleReviewPack() {
|
|
|
3665
3860
|
console.log(`- Outside the impact map: ${result.impactDrift.outOfMap.join(', ')} (check their side effects)`);
|
|
3666
3861
|
}
|
|
3667
3862
|
for (const line of result.impactDrift?.design || []) console.log(`- ${line}`);
|
|
3863
|
+
if (openDebt.length) console.log(`- Open debt on changed files: ${openDebt.map((item) => item.id).join(', ')} (in the pack; resolve or keep consciously)`);
|
|
3668
3864
|
}
|
|
3669
3865
|
} catch (err) {
|
|
3670
3866
|
stageJsonError(err, json, 'REVIEW_PACK_FAILED');
|
|
@@ -3825,6 +4021,7 @@ function handleBench() {
|
|
|
3825
4021
|
sandboxHome: process.argv.includes('--sandbox-home'),
|
|
3826
4022
|
hostArgs: repeatedArg('--host-arg'),
|
|
3827
4023
|
scenarioIds: repeatedArg('--scenario'),
|
|
4024
|
+
without: repeatedArg('--without').flatMap((value) => value.split(',')).map((value) => value.trim()).filter(Boolean),
|
|
3828
4025
|
repeat: repeatArg == null ? 1 : parsePositiveIntegerArg('--repeat', repeatArg),
|
|
3829
4026
|
timeoutMs: timeoutArg == null ? undefined : parseBenchTimeoutSeconds(timeoutArg),
|
|
3830
4027
|
mode: modeArg == null ? 'ASSISTED' : normalizeMode(modeArg),
|
|
@@ -3930,6 +4127,22 @@ function handleQuick() {
|
|
|
3930
4127
|
|
|
3931
4128
|
function handleLearn() {
|
|
3932
4129
|
const json = process.argv.includes('--json');
|
|
4130
|
+
if (args[1] === 'promote') {
|
|
4131
|
+
try {
|
|
4132
|
+
const rule = require('../lib/rules.js').promoteLearning(CWD, args[2], {
|
|
4133
|
+
check: argValue('--check'),
|
|
4134
|
+
fix: argValue('--fix'),
|
|
4135
|
+
why: argValue('--why'),
|
|
4136
|
+
by: argValue('--by'),
|
|
4137
|
+
shell: !process.argv.includes('--no-shell')
|
|
4138
|
+
});
|
|
4139
|
+
if (json) console.log(JSON.stringify({ ok: true, rule }));
|
|
4140
|
+
else log.success(`✓ ${rule.from} promoted to rule ${rule.id}; default \`verify\` now runs: ${rule.check}`);
|
|
4141
|
+
} catch (err) {
|
|
4142
|
+
stageJsonError(err, json, 'LEARN_PROMOTE_FAILED');
|
|
4143
|
+
}
|
|
4144
|
+
return;
|
|
4145
|
+
}
|
|
3933
4146
|
try {
|
|
3934
4147
|
const maxAge = argValue('--max-age-runs');
|
|
3935
4148
|
const result = learn(CWD, {
|
|
@@ -3938,8 +4151,9 @@ function handleLearn() {
|
|
|
3938
4151
|
...(maxAge == null ? {} : { maxAgeRuns: parsePositiveIntegerArg('--max-age-runs', maxAge) })
|
|
3939
4152
|
});
|
|
3940
4153
|
const { document } = result;
|
|
4154
|
+
const candidates = require('../lib/rules.js').promotionCandidates(CWD, document.entries);
|
|
3941
4155
|
if (json) {
|
|
3942
|
-
console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, learnings: document }));
|
|
4156
|
+
console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, promotionCandidates: candidates, learnings: document }));
|
|
3943
4157
|
return;
|
|
3944
4158
|
}
|
|
3945
4159
|
log.success(`✓ ${document.entries.length} learning(s) from ${document.sourceRuns} Run(s)${result.written ? ` → ${result.path}` : ' (dry run, nothing written)'}`);
|
|
@@ -3947,6 +4161,9 @@ function handleLearn() {
|
|
|
3947
4161
|
console.log(`- ${entry.id} [${entry.axis}, ${entry.severity}, ×${entry.occurrences}] ${entry.lesson}`);
|
|
3948
4162
|
}
|
|
3949
4163
|
for (const item of result.pruned) console.log(` pruned ${item.id} (${item.reason})`);
|
|
4164
|
+
for (const item of candidates) {
|
|
4165
|
+
console.log(`! Promote candidate ${item.id} (seen in ${item.runs} Runs): ${item.lesson}. Make it a check: learn promote ${item.id} --check "<command>" --fix "<instruction>" --by <name>`);
|
|
4166
|
+
}
|
|
3950
4167
|
} catch (err) {
|
|
3951
4168
|
stageJsonError(err, json, 'LEARN_FAILED');
|
|
3952
4169
|
}
|
|
@@ -4083,6 +4300,12 @@ Commands:
|
|
|
4083
4300
|
A missing binary is UNAVAILABLE, not a model failure.
|
|
4084
4301
|
Progress lines go to stderr. Writes .ai-engineering-loop/bench/<benchId>/{results.json,report.md,hardening.md}
|
|
4085
4302
|
bench compare <benchIdA> <benchIdB> [--json] read-only side-by-side table
|
|
4303
|
+
outcome record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name>
|
|
4304
|
+
a post-delivery problem traced to a Run (input for calibrate)
|
|
4305
|
+
calibrate Compare recorded outcomes with each Run's Finding Ledger: DA misses, wrong dismissals, per reviewer prompt
|
|
4306
|
+
debt Accepted trade-offs and L3 waivers recorded at delivery: list [--file <path>] | close <id> --by <name> [--note <text>]
|
|
4307
|
+
next Next unblocked feature from .ai-engineering-loop/features.json, VCR (verified/activated), evidence audit [--json]
|
|
4308
|
+
stats Failure causes by harness layer, false-victory rate, iterations to PASS [--last <n>] [--json]
|
|
4086
4309
|
escalation Compute the minimum Judge model tier from deterministic risk signals
|
|
4087
4310
|
--run <id> target a non-current run
|
|
4088
4311
|
--json print machine-readable reasons
|
|
@@ -4094,7 +4317,8 @@ Commands:
|
|
|
4094
4317
|
start "<task>" [--mode assisted|unattended] | finish [--run <id>] [--cmd "<command>"]...
|
|
4095
4318
|
verify Record verification commands, gate them, compact, and scaffold claimed-vs-reality
|
|
4096
4319
|
[--run <id>] [--cmd "<command>"]... [--shell] [--serial] [--timeout-ms <n>] [--fresh] [--json]
|
|
4097
|
-
default
|
|
4320
|
+
default (no --cmd): build -> typecheck -> lint -> test_unit from verification.md, serial, stops at first failure
|
|
4321
|
+
--baseline before the Maker gate: run the plan on the untouched tree, write baseline.json
|
|
4098
4322
|
--related fast lane: test_related with {files} = changed files of the gated diff
|
|
4099
4323
|
--final after Judge PASS: full run delivery needs when verification was related-only
|
|
4100
4324
|
--fresh re-run commands that already passed for the same diff and revision
|
|
@@ -4122,6 +4346,8 @@ Commands:
|
|
|
4122
4346
|
writes RUN_DIR/receipt.json + receipt.md (paste into the PR/MR body); digest stored in delivery.json
|
|
4123
4347
|
verify-receipt Re-check a delivery receipt: artifact hashes, frozen Goal, AC evidence, verdict, head SHA
|
|
4124
4348
|
[--run <id> | <repository-relative-receipt.json>] [--head <sha>] [--json] exit 1 on mismatch (CI)
|
|
4349
|
+
learn promote <learningId> --check "<command>" --fix "<instruction>" --by <name> [--why <text>] [--no-shell]
|
|
4350
|
+
turn a recurring review lesson into an executable L1 rule (.ai-engineering-loop/rules.json)
|
|
4125
4351
|
learn Mine past Runs' VALID findings into .ai-engineering-loop/learnings.json (+ .md) for context packs
|
|
4126
4352
|
[--dry-run] [--prune] [--max-age-runs <n>] [--json]
|
|
4127
4353
|
deterministic and deduped with runId provenance; --prune drops lessons whose files are gone
|
|
@@ -4130,6 +4356,7 @@ Commands:
|
|
|
4130
4356
|
list [--json]
|
|
4131
4357
|
<name> [--section <heading>] [--json] e.g. docs grill-policy --section "When to grill"
|
|
4132
4358
|
doctor Check runtime, schemas, host assets, package contents, and eval catalog
|
|
4359
|
+
--harness audit harness debt instead: entry-file size/structure, stale learnings, rule provenance, skill budget
|
|
4133
4360
|
--json print machine-readable diagnostics
|
|
4134
4361
|
update Check or apply the package update workflow
|
|
4135
4362
|
check|plan [--json]
|
|
@@ -4295,6 +4522,21 @@ switch (command) {
|
|
|
4295
4522
|
case 'escalation':
|
|
4296
4523
|
handleEscalation();
|
|
4297
4524
|
break;
|
|
4525
|
+
case 'stats':
|
|
4526
|
+
handleStats();
|
|
4527
|
+
break;
|
|
4528
|
+
case 'next':
|
|
4529
|
+
handleNext();
|
|
4530
|
+
break;
|
|
4531
|
+
case 'debt':
|
|
4532
|
+
handleDebt();
|
|
4533
|
+
break;
|
|
4534
|
+
case 'outcome':
|
|
4535
|
+
handleOutcome();
|
|
4536
|
+
break;
|
|
4537
|
+
case 'calibrate':
|
|
4538
|
+
handleCalibrate();
|
|
4539
|
+
break;
|
|
4298
4540
|
case 'docs':
|
|
4299
4541
|
handleDocs();
|
|
4300
4542
|
break;
|
package/core/bench.md
CHANGED
|
@@ -75,3 +75,18 @@ Bench results do not feed `ael learn`. Learnings stay findings mined from Runs;
|
|
|
75
75
|
`bench compare <a> <b>` prints a side-by-side table and does not write.
|
|
76
76
|
|
|
77
77
|
A live run against your own host is manual evidence for a change review. CI uses the fake host only.
|
|
78
|
+
|
|
79
|
+
## Ablation and the simplification log (C-08)
|
|
80
|
+
|
|
81
|
+
Every harness component encodes an assumption about what the model cannot do alone; as models improve, some assumptions go stale. Measure instead of guessing:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
npx ai-engineering-loop bench --host <host> --model <model> # baseline
|
|
85
|
+
npx ai-engineering-loop bench --host <host> --model <model> --without integrity # one component off
|
|
86
|
+
npx ai-engineering-loop bench compare <baselineId> <ablationId>
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`--without` accepts `integrity`, `l3`, `l1`, `smoke`, `rules` (repeat or comma-separate). It only takes effect inside bench scenario repos: bench sets `AEL_ABLATE` and writes `.ai-engineering-loop/bench-ablation.json`; the gates honour the switch only when both are present together with `AEL_BENCH_MODE`, and every skip is logged to the Run's `ablation.log`. The result carries `ablation.without`, and an entry is appended to `.ai-engineering-loop/simplification-log.json` (`decision: null` until a human records keep / remove / replace).
|
|
90
|
+
|
|
91
|
+
`doctor --harness` audits harness debt in the project: entry-file size (50-200 lines) and structure (only headings, lists, tables and code count, so keywords in prose cannot game it), learnings not seen in the last 10 Runs, rules without provenance or older than 90 days, and, inside the AEL repo, T1 skill-context headroom from `scripts/measure.js`.
|
|
92
|
+
|
package/core/goal-contract.md
CHANGED
|
@@ -113,3 +113,38 @@ Every single item listed under `Acceptance Criteria` must map to at least one co
|
|
|
113
113
|
- **The Wrong Artifact**: Tests pass on a `/tmp` build while the sample the user opens omits the feature.
|
|
114
114
|
- **The Happy-Path Contract**: Only the sunny case is an AC. Coverage % is not a substitute for a failure table.
|
|
115
115
|
- **The Sloppy Ship**: Implementing from a vague prompt without a drafted failure table in the Goal Contract file.
|
|
116
|
+
|
|
117
|
+
## Per-AC verify command (C-07)
|
|
118
|
+
|
|
119
|
+
An AC may name the command that proves it:
|
|
120
|
+
|
|
121
|
+
```json
|
|
122
|
+
{ "id": "AC-2", "statement": "page 0 is rejected", "evidenceRequired": "unit test", "failureCases": ["page 0 returns items"],
|
|
123
|
+
"verify": { "command": "npm test -- list.test.js", "layer": "L2" } }
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The harness owns the status: `proven` when that exact command passed on the gated diff (`verification.json` or `final-verification.json`), `failed` when it ran and failed, `pending` when it never ran. `verify` prints each status, and `gate judge` refuses PASS while any AC with a verify command is `pending` or `failed` (`AC_NOT_PROVEN`). ACs without `verify` keep the existing evidence rules.
|
|
127
|
+
|
|
128
|
+
## Rubric for subjective ACs (C-09)
|
|
129
|
+
|
|
130
|
+
For ACs that are judged rather than computed (layout, copy, visual design), add a rubric:
|
|
131
|
+
|
|
132
|
+
```json
|
|
133
|
+
{ "id": "AC-3", "statement": "The empty state guides the user", "evidenceRequired": "screenshot", "failureCases": ["blank panel"],
|
|
134
|
+
"rubric": [ { "criterion": "contrast", "pass": ">= 4.5:1 (WCAG AA)" }, { "criterion": "guidance", "pass": "explains how to import a document" } ] }
|
|
135
|
+
```
|
|
136
|
+
|
|
137
|
+
The Devil's Advocate scores every criterion in the ledger as `rubricScores: [{ac, criterion, pass, evidence}]` with observed evidence (screenshot path, DOM snapshot, measured value). `gate judge` refuses PASS while any criterion is unscored or `pass: false` (`AC_RUBRIC`).
|
|
138
|
+
|
|
139
|
+
## Project feature ledger (C-15)
|
|
140
|
+
|
|
141
|
+
Optional `.ai-engineering-loop/features.json` lists what the project must do, one triple per feature:
|
|
142
|
+
|
|
143
|
+
```json
|
|
144
|
+
{ "schemaVersion": 1, "features": [
|
|
145
|
+
{ "id": "F-1", "title": "Import", "behavior": "user imports a .md file and sees it in the list", "verify": "npm run e2e -- import", "status": "not_started", "priority": 1, "dependsOn": [] }
|
|
146
|
+
] }
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
Status moves only through Runs: a Goal Contract with `"featureId": "F-1"` makes it `active` at the Goal gate, and delivery makes it `passing` with the Run as evidence (passing is final). `ael next` prints the highest-priority unblocked feature, VCR (verified / activated), and flags any `passing` feature without a delivered Run behind it. `run --new` warns when an unfinished Run exists and VCR is below 1 (WIP=1).
|
|
150
|
+
|