ai-engineering-loop 2.6.0 → 2.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ael/SKILL.md +1 -1
- package/.agents/skills/ael/references/loop-commands.md +13 -1
- package/.agents/skills/ael/references/loop-deliver.md +1 -1
- package/.agents/skills/ael/references/loop-goal.md +2 -0
- package/.agents/skills/ael/references/loop-maker.md +1 -6
- package/.agents/skills/ael/references/loop-report.md +8 -0
- package/.agents/skills/ael/references/loop-verify.md +11 -9
- package/.claude/agents/devil-advocate.md +7 -2
- package/.claude/agents/judge.md +3 -1
- package/.grok/agents/devil-advocate.md +7 -2
- package/.grok/agents/judge.md +3 -1
- package/CHANGELOG.md +31 -0
- package/agents/devil-advocate.md +8 -0
- package/agents/judge.md +1 -0
- package/agents/shared/devil-advocate.body.md +7 -2
- package/agents/shared/judge.body.md +3 -1
- package/bin/ai-engineering-loop.js +326 -21
- package/core/bench.md +15 -0
- package/core/formal-statechart.md +49 -0
- package/core/goal-contract.md +53 -0
- package/core/verification-loop.md +106 -1
- package/lib/ablation.js +52 -0
- package/lib/ac-status.js +95 -0
- package/lib/baseline.js +138 -0
- package/lib/bench.js +24 -1
- package/lib/budget.js +9 -1
- package/lib/calibration.js +129 -0
- package/lib/claimed-vs-reality.js +106 -12
- package/lib/clean-state.js +85 -0
- package/lib/cli-args.js +5 -3
- package/lib/comment-findings.js +111 -0
- package/lib/debt.js +110 -0
- package/lib/feature-map.js +57 -0
- package/lib/features.js +95 -0
- package/lib/flow.js +86 -0
- package/lib/gates.js +257 -41
- package/lib/harness-audit.js +141 -0
- package/lib/learn.js +24 -1
- package/lib/no-progress.js +6 -1
- package/lib/receipt.js +33 -2
- package/lib/reverts.js +77 -0
- package/lib/review-ingest.js +54 -2
- package/lib/rule-check.js +90 -0
- package/lib/rules.js +205 -0
- package/lib/stats.js +110 -0
- package/lib/sync-hosts.js +1 -0
- package/lib/verification-integrity.js +164 -0
- package/lib/verification-lanes.js +96 -0
- package/lib/verification-recorder.js +20 -8
- package/package.json +1 -1
- package/schemas/finding-ledger.schema.json +14 -0
- package/schemas/goal-contract.schema.json +50 -0
- package/schemas/verdict.schema.json +2 -1
|
@@ -425,9 +425,9 @@ function analyzeRepository(rootDir) {
|
|
|
425
425
|
scripts: {
|
|
426
426
|
testUnit: 'npm test',
|
|
427
427
|
testAll: 'npm test',
|
|
428
|
-
typecheck:
|
|
429
|
-
lint:
|
|
430
|
-
build:
|
|
428
|
+
typecheck: null,
|
|
429
|
+
lint: null,
|
|
430
|
+
build: null,
|
|
431
431
|
e2e: null
|
|
432
432
|
},
|
|
433
433
|
adapter: {
|
|
@@ -506,12 +506,16 @@ function analyzeRepository(rootDir) {
|
|
|
506
506
|
discovery.scripts.testRelated = viaScript('--findRelatedTests --passWithNoTests {files}') || 'npx jest --findRelatedTests --passWithNoTests {files}';
|
|
507
507
|
}
|
|
508
508
|
|
|
509
|
+
// Only commands the project really has: a guessed `npx tsc` can fetch an
|
|
510
|
+
// unrelated package, and `eslint --fix` would edit files during verify.
|
|
511
|
+
const hasTypeScript = Boolean(allDeps['typescript']) || fs.existsSync(path.join(rootDir, 'tsconfig.json'));
|
|
509
512
|
if (scripts['typecheck']) discovery.scripts.typecheck = `${pm} run typecheck`;
|
|
510
513
|
else if (scripts['type-check']) discovery.scripts.typecheck = `${pm} run type-check`;
|
|
511
514
|
else if (scripts['check']) discovery.scripts.typecheck = `${pm} run check`;
|
|
515
|
+
else discovery.scripts.typecheck = hasTypeScript ? 'npx tsc --noEmit' : null;
|
|
512
516
|
|
|
513
|
-
|
|
514
|
-
|
|
517
|
+
discovery.scripts.lint = scripts['lint'] ? `${pm} run lint` : null;
|
|
518
|
+
discovery.scripts.build = scripts['build'] ? `${pm} run build` : null;
|
|
515
519
|
if (scripts['test:e2e']) discovery.scripts.e2e = `${pm} run test:e2e`;
|
|
516
520
|
} catch (e) {}
|
|
517
521
|
}
|
|
@@ -664,9 +668,9 @@ ${discovery.topLevelDirs.map((d) => `- \`${d}/\``).join('\n') || '- Flat directo
|
|
|
664
668
|
|
|
665
669
|
## Discovered Verification Commands
|
|
666
670
|
- **test_unit**: \`${discovery.scripts.testUnit}\`
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
671
|
+
${discovery.scripts.typecheck ? `- **typecheck**: \`${discovery.scripts.typecheck}\`` : ''}
|
|
672
|
+
${discovery.scripts.lint ? `- **lint**: \`${discovery.scripts.lint}\`` : ''}
|
|
673
|
+
${discovery.scripts.build ? `- **build**: \`${discovery.scripts.build}\`` : ''}
|
|
670
674
|
${discovery.scripts.e2e ? `- **e2e**: \`${discovery.scripts.e2e}\`` : ''}
|
|
671
675
|
${discovery.scripts.testRelated ? `- **test_related**: \`${discovery.scripts.testRelated}\`` : ''}
|
|
672
676
|
|
|
@@ -1390,6 +1394,15 @@ function handleRun() {
|
|
|
1390
1394
|
const current = getCurrentRun(CWD);
|
|
1391
1395
|
const policy = loadPolicy(CWD);
|
|
1392
1396
|
const startsNewRun = forceNew || !current || TERMINAL_STATES.has(current.state);
|
|
1397
|
+
if (forceNew && current && !TERMINAL_STATES.has(current.state)) {
|
|
1398
|
+
// C-15 (L07 WIP=1): starting another Run while one is unfinished.
|
|
1399
|
+
try {
|
|
1400
|
+
const ratio = require('../lib/features.js').vcr(require('../lib/features.js').loadFeatures(CWD));
|
|
1401
|
+
if (ratio.value < 1) console.error(`! VCR ${ratio.verified}/${ratio.activated}: Run ${current.runId} is still ${current.state}. Finishing it first usually beats starting another (WIP=1).`);
|
|
1402
|
+
} catch {
|
|
1403
|
+
// No feature ledger.
|
|
1404
|
+
}
|
|
1405
|
+
}
|
|
1393
1406
|
let mode = modeArg ? normalizeMode(modeArg) : (startsNewRun ? policy.defaultMode : null);
|
|
1394
1407
|
let workflow = null;
|
|
1395
1408
|
let requestedRunId = null;
|
|
@@ -1685,7 +1698,8 @@ function handleBudget() {
|
|
|
1685
1698
|
nodeId,
|
|
1686
1699
|
inputTokens: parseNonNegativeIntegerArg('--input', argValue('--input')),
|
|
1687
1700
|
outputTokens: parseNonNegativeIntegerArg('--output', argValue('--output')),
|
|
1688
|
-
model: argValue('--model')
|
|
1701
|
+
model: argValue('--model'),
|
|
1702
|
+
role: argValue('--role')
|
|
1689
1703
|
});
|
|
1690
1704
|
} else if (action === 'pause' || action === 'resume') {
|
|
1691
1705
|
result = setKillSwitch(CWD, action === 'pause');
|
|
@@ -2015,6 +2029,7 @@ function handleGate() {
|
|
|
2015
2029
|
console.log(`- Run: ${run.runId}`);
|
|
2016
2030
|
console.log(`- State: ${run.state}, iteration ${run.iteration}`);
|
|
2017
2031
|
if (run.tier) console.log(`- Tier: ${run.tier.current}`);
|
|
2032
|
+
for (const warning of (run.history || []).at(-1)?.cleanStateWarnings || []) console.log(`! Clean state: ${warning}`);
|
|
2018
2033
|
if (guardEvents.warn || guardEvents.block) {
|
|
2019
2034
|
console.log(`- Guard: ${guardEvents.block} blocked, ${guardEvents.warn} warned (${guardEvents.rules.join(', ')}); see guard-events.json`);
|
|
2020
2035
|
}
|
|
@@ -2037,10 +2052,14 @@ function handleGate() {
|
|
|
2037
2052
|
ok: false,
|
|
2038
2053
|
code: err.code || 'GATE_FAILED',
|
|
2039
2054
|
error: err.message,
|
|
2055
|
+
...(err.why ? { why: err.why } : {}),
|
|
2056
|
+
...(err.fix ? { fix: err.fix } : {}),
|
|
2040
2057
|
details
|
|
2041
2058
|
}));
|
|
2042
2059
|
} else {
|
|
2043
2060
|
log.error(err.message);
|
|
2061
|
+
if (err.why) console.error(`WHY: ${err.why}`);
|
|
2062
|
+
if (err.fix) console.error(`FIX: ${err.fix}`);
|
|
2044
2063
|
for (const detail of details) console.error(`- ${detail}`);
|
|
2045
2064
|
}
|
|
2046
2065
|
process.exit(1);
|
|
@@ -2373,6 +2392,16 @@ function handleState() {
|
|
|
2373
2392
|
|
|
2374
2393
|
function handleDoctor() {
|
|
2375
2394
|
const json = process.argv.includes('--json');
|
|
2395
|
+
if (process.argv.includes('--harness')) {
|
|
2396
|
+
// C-08: harness debt in this project (warnings never block).
|
|
2397
|
+
const audit = require('../lib/harness-audit.js').auditHarness(CWD);
|
|
2398
|
+
if (json) console.log(JSON.stringify(audit));
|
|
2399
|
+
else {
|
|
2400
|
+
console.log(`Harness audit: ${audit.ok ? 'no debt found' : 'debt found'}`);
|
|
2401
|
+
for (const item of audit.items) console.log(`${item.level === 'warn' ? '!' : '-'} ${item.id}: ${item.detail}`);
|
|
2402
|
+
}
|
|
2403
|
+
return;
|
|
2404
|
+
}
|
|
2376
2405
|
const result = runDoctor(path.join(__dirname, '..'), { projectDir: CWD });
|
|
2377
2406
|
if (json) console.log(JSON.stringify(result));
|
|
2378
2407
|
else {
|
|
@@ -3134,6 +3163,11 @@ function handleGoal() {
|
|
|
3134
3163
|
expectedRevision: requiredIntegerArg('--revision')
|
|
3135
3164
|
});
|
|
3136
3165
|
extra.impactPreview = impactPreview(runId, draftGoal);
|
|
3166
|
+
// C-21: show the intended user flow as a transition table before freeze.
|
|
3167
|
+
if (draftGoal && draftGoal.flow != null) {
|
|
3168
|
+
const flowLib = require('../lib/flow.js');
|
|
3169
|
+
extra.flow = { table: flowLib.renderFlow(draftGoal.flow), mermaid: flowLib.renderMermaid(draftGoal.flow), problems: flowLib.flowProblems(draftGoal.flow), warnings: flowLib.flowWarnings(draftGoal.flow) };
|
|
3170
|
+
}
|
|
3137
3171
|
appendRunLifecycle(CWD, runId, {
|
|
3138
3172
|
type: 'GOAL_DRAFT_UPDATED',
|
|
3139
3173
|
phase: 'goal',
|
|
@@ -3363,9 +3397,11 @@ function splitCommand(text, shell) {
|
|
|
3363
3397
|
|
|
3364
3398
|
function stageJsonError(err, json, code) {
|
|
3365
3399
|
if (json) {
|
|
3366
|
-
console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, details: err.details || [] }));
|
|
3400
|
+
console.log(JSON.stringify({ ok: false, code: err.code || code, error: err.message, ...(err.why ? { why: err.why } : {}), ...(err.fix ? { fix: err.fix } : {}), details: err.details || [] }));
|
|
3367
3401
|
} else {
|
|
3368
3402
|
log.error(err.message);
|
|
3403
|
+
if (err.why) console.error(`WHY: ${err.why}`);
|
|
3404
|
+
if (err.fix) console.error(`FIX: ${err.fix}`);
|
|
3369
3405
|
for (const detail of err.details || []) console.error(`- ${detail}`);
|
|
3370
3406
|
}
|
|
3371
3407
|
process.exit(1);
|
|
@@ -3391,19 +3427,76 @@ async function handleVerify() {
|
|
|
3391
3427
|
if (related && final) throw Object.assign(new Error('--related and --final cannot be combined'), { code: 'INVALID_USAGE' });
|
|
3392
3428
|
let texts = repeatedArg('--cmd');
|
|
3393
3429
|
const lanes = require('../lib/verification-lanes.js');
|
|
3430
|
+
const baselineMode = process.argv.includes('--baseline');
|
|
3431
|
+
if (baselineMode && (related || final)) throw Object.assign(new Error('--baseline cannot be combined with --related or --final'), { code: 'INVALID_USAGE' });
|
|
3394
3432
|
let commandsToRun;
|
|
3433
|
+
let defaultPlan = false;
|
|
3395
3434
|
if (related && texts.length === 0) {
|
|
3396
3435
|
const target = loadRun(CWD, runId);
|
|
3397
|
-
commandsToRun = [lanes.relatedCommand(CWD, target, { split: splitCommand, shell })];
|
|
3398
|
-
} else {
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3436
|
+
commandsToRun = [{ ...lanes.relatedCommand(CWD, target, { split: splitCommand, shell }), layer: 'L2' }];
|
|
3437
|
+
} else if (texts.length === 0) {
|
|
3438
|
+
// C-02: no --cmd runs the cheapest-first plan (L1 build/typecheck/lint,
|
|
3439
|
+
// then L2 test_unit) and stops at the first failure.
|
|
3440
|
+
const plan = lanes.defaultCommandPlan(CWD);
|
|
3441
|
+
// C-05: promoted review rules run as L1 checks, after the static checks.
|
|
3442
|
+
const rules = require('../lib/rules.js').rulePlan(CWD, loadRun(CWD, runId));
|
|
3443
|
+
if (rules.length) {
|
|
3444
|
+
const firstL2 = plan.findIndex((step) => step.layer !== 'L1');
|
|
3445
|
+
plan.splice(firstL2 === -1 ? plan.length : firstL2, 0, ...rules);
|
|
3446
|
+
}
|
|
3447
|
+
// C-23: catalog keys an AC requires (verify.key, e.g. pentest, formal)
|
|
3448
|
+
// run too when the project configures them.
|
|
3449
|
+
if (!baselineMode) {
|
|
3450
|
+
try {
|
|
3451
|
+
const target = loadRun(CWD, runId);
|
|
3452
|
+
const goal = require('../lib/claimed-vs-reality.js').loadFrozenGoal(CWD, target);
|
|
3453
|
+
const configured = lanes.verificationCommands(CWD);
|
|
3454
|
+
const acStatusLib = require('../lib/ac-status.js');
|
|
3455
|
+
for (const key of acStatusLib.requiredKeys(goal)) {
|
|
3456
|
+
if (configured[key] && !plan.some((step) => step.key === key)) plan.push({ key, layer: lanes.layerOfKey(key), text: configured[key] });
|
|
3457
|
+
}
|
|
3458
|
+
// A required layer with no step yet: the first configured catalog key of that layer.
|
|
3459
|
+
for (const layer of acStatusLib.requiredLayers(goal)) {
|
|
3460
|
+
if (plan.some((step) => step.layer === layer)) continue;
|
|
3461
|
+
const key = Object.keys(lanes.VERIFICATION_CATALOG).find((name) => lanes.VERIFICATION_CATALOG[name].layer === layer && configured[name]);
|
|
3462
|
+
if (key) plan.push({ key, layer, text: configured[key] });
|
|
3463
|
+
}
|
|
3464
|
+
} catch {
|
|
3465
|
+
// No frozen Goal yet; nothing extra to run.
|
|
3466
|
+
}
|
|
3467
|
+
}
|
|
3468
|
+
// C-03: startup-risky diffs also run the configured smoke check (L2).
|
|
3469
|
+
const smokeText = lanes.verificationCommands(CWD).smoke;
|
|
3470
|
+
if (smokeText && !baselineMode && lanes.startupRiskFiles(CWD, loadRun(CWD, runId)).length) {
|
|
3471
|
+
plan.push({ key: 'smoke', layer: 'L2', text: smokeText });
|
|
3403
3472
|
}
|
|
3404
|
-
|
|
3473
|
+
if (!plan.length) throw Object.assign(new Error('No --cmd given and no build/typecheck/lint/test_unit command in .ai-engineering-loop/verification.md. FIX: add them to verification.md or pass --cmd.'), { code: 'INVALID_COMMAND' });
|
|
3474
|
+
commandsToRun = plan.map((step) => ({ ...splitCommand(step.text, shell || Boolean(step.shell)), key: step.key, layer: step.layer, ...(step.env ? { env: step.env } : {}) }));
|
|
3475
|
+
defaultPlan = true;
|
|
3476
|
+
} else {
|
|
3477
|
+
commandsToRun = texts.map((text) => {
|
|
3478
|
+
const layer = lanes.layerOfCommand(CWD, text);
|
|
3479
|
+
return { ...splitCommand(text, shell), ...(layer ? { layer } : {}) };
|
|
3480
|
+
});
|
|
3405
3481
|
}
|
|
3406
3482
|
const timeoutValue = argValue('--timeout-ms');
|
|
3483
|
+
if (baselineMode) {
|
|
3484
|
+
const outcome = await require('../lib/baseline.js').recordBaseline(CWD, {
|
|
3485
|
+
runId,
|
|
3486
|
+
plan: commandsToRun,
|
|
3487
|
+
...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
|
|
3488
|
+
});
|
|
3489
|
+
const result = { ok: outcome.status !== 'BROKEN', runId, baseline: outcome };
|
|
3490
|
+
if (json) console.log(JSON.stringify(result));
|
|
3491
|
+
else {
|
|
3492
|
+
const say = outcome.status === 'GREEN' ? log.success : (outcome.status === 'RED' ? log.warn : log.error);
|
|
3493
|
+
say(`Baseline ${outcome.status}: ${outcome.path}`);
|
|
3494
|
+
for (const c of outcome.commands) console.log(`- ${c.command}: exit ${c.exitCode}`);
|
|
3495
|
+
for (const problem of outcome.problems) console.log(`! ${problem}`);
|
|
3496
|
+
}
|
|
3497
|
+
if (!result.ok) process.exitCode = 1;
|
|
3498
|
+
return;
|
|
3499
|
+
}
|
|
3407
3500
|
if (final) {
|
|
3408
3501
|
const target = loadRun(CWD, runId);
|
|
3409
3502
|
if (target.state !== 'JUDGED_PASS') throw Object.assign(new Error(`verify --final needs a Judge PASS (state is ${target.state})`), { code: 'INVALID_STATE' });
|
|
@@ -3437,17 +3530,31 @@ async function handleVerify() {
|
|
|
3437
3530
|
commands: commandsToRun,
|
|
3438
3531
|
serial: process.argv.includes('--serial'),
|
|
3439
3532
|
fresh: process.argv.includes('--fresh'),
|
|
3533
|
+
stopOnFailure: defaultPlan,
|
|
3440
3534
|
...(related ? { scope: 'related' } : {}),
|
|
3441
3535
|
...(timeoutValue == null ? {} : { timeoutMs: parsePositiveIntegerArg('--timeout-ms', timeoutValue) })
|
|
3442
3536
|
});
|
|
3443
|
-
const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
|
|
3537
|
+
const commands = recorded.commands.map((c) => ({ command: c.command, exitCode: c.exitCode, timeoutStatus: c.timeoutStatus, ...(c.layer ? { layer: c.layer } : {}), ...(c.scope ? { scope: c.scope } : {}), ...(c.reused ? { reused: true } : {}) }));
|
|
3444
3538
|
if (!recorded.passed) {
|
|
3445
3539
|
autoHeartbeat(runId, 'verification', 'Verification failed; Maker must fix before review');
|
|
3446
|
-
const
|
|
3540
|
+
const baselineModule = require('../lib/baseline.js');
|
|
3541
|
+
const failures = baselineModule.classifyAgainstBaseline(baselineModule.loadBaseline(CWD, runId), recorded.commands);
|
|
3542
|
+
const result = { ok: false, code: 'VERIFICATION_FAILED', runId, commands, skipped: recorded.skipped, failures, gate: null };
|
|
3447
3543
|
if (json) console.log(JSON.stringify(result));
|
|
3448
3544
|
else {
|
|
3449
3545
|
log.error('Verification failed; the gate was not attempted.');
|
|
3450
3546
|
for (const c of commands) console.log(`- ${c.command}: exit ${c.exitCode} (${c.timeoutStatus})`);
|
|
3547
|
+
for (const skippedCommand of recorded.skipped) console.log(`- ${skippedCommand}: skipped (an earlier, cheaper check failed)`);
|
|
3548
|
+
for (const c of recorded.commands.filter((item) => item.exitCode !== 0)) {
|
|
3549
|
+
const rulesLib = require('../lib/rules.js');
|
|
3550
|
+
const rule = (c.key && c.key.startsWith('rule:') ? rulesLib.findRule(CWD, c.key.slice(5)) : null) || rulesLib.ruleForCommand(CWD, c.command);
|
|
3551
|
+
if (rule) console.log(`! Rule ${rule.id} violated. WHY: ${rule.why} FIX: ${rule.fix}`);
|
|
3552
|
+
}
|
|
3553
|
+
for (const f of failures) {
|
|
3554
|
+
console.log(f.preExisting
|
|
3555
|
+
? `! ${f.command}: every failure was already failing at baseline (pre-existing). FIX: repair it in a separate Run, or narrow --cmd to the tests this change owns.`
|
|
3556
|
+
: `! ${f.command}: new failure(s)${f.newFailures.length ? `: ${f.newFailures.slice(0, 5).join('; ')}` : ''}. FIX: fix the change before review.`);
|
|
3557
|
+
}
|
|
3451
3558
|
}
|
|
3452
3559
|
process.exitCode = 1;
|
|
3453
3560
|
return;
|
|
@@ -3471,6 +3578,14 @@ async function handleVerify() {
|
|
|
3471
3578
|
impactDrift: { mapStatus: drift.mapStatus, outOfMap: drift.outOfMap },
|
|
3472
3579
|
compact: compact.paths,
|
|
3473
3580
|
claimedVsReality: claimed,
|
|
3581
|
+
acStatus: (() => {
|
|
3582
|
+
try {
|
|
3583
|
+
return require('../lib/ac-status.js').runAcStatuses(CWD, run, require('../lib/claimed-vs-reality.js').loadFrozenGoal(CWD, run))
|
|
3584
|
+
.filter((ac) => ac.status !== 'unbound');
|
|
3585
|
+
} catch {
|
|
3586
|
+
return [];
|
|
3587
|
+
}
|
|
3588
|
+
})(),
|
|
3474
3589
|
...(related ? { scope: 'related', finalRequired: 'Run `verify --final` after Judge PASS, before delivery.' } : {}),
|
|
3475
3590
|
next: run.tier && run.tier.current === 'T0'
|
|
3476
3591
|
? 'Fill claimed-vs-reality, then gate review and gate judge'
|
|
@@ -3487,6 +3602,7 @@ async function handleVerify() {
|
|
|
3487
3602
|
console.log('! No impact map for this Run; drift was not checked');
|
|
3488
3603
|
}
|
|
3489
3604
|
console.log(`- Claimed-vs-reality: ${claimed.path}${claimed.created ? ' (scaffolded; fill it from the command output)' : ''}`);
|
|
3605
|
+
for (const ac of result.acStatus) console.log(`- ${ac.id}: ${ac.status} (\`${ac.command}\`)`);
|
|
3490
3606
|
console.log(`- Next: ${result.next}`);
|
|
3491
3607
|
}
|
|
3492
3608
|
} catch (err) {
|
|
@@ -3494,6 +3610,115 @@ async function handleVerify() {
|
|
|
3494
3610
|
}
|
|
3495
3611
|
}
|
|
3496
3612
|
|
|
3613
|
+
function handleOutcome() {
|
|
3614
|
+
const json = process.argv.includes('--json');
|
|
3615
|
+
try {
|
|
3616
|
+
const action = args[1] && !args[1].startsWith('--') ? args[1] : null;
|
|
3617
|
+
if (action !== 'record') throw Object.assign(new Error('outcome supports: record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name> [--evidence <path|url>]'), { code: 'INVALID_USAGE' });
|
|
3618
|
+
const entry = require('../lib/calibration.js').recordOutcome(CWD, {
|
|
3619
|
+
runId: argValue('--run'),
|
|
3620
|
+
source: argValue('--source'),
|
|
3621
|
+
location: argValue('--location'),
|
|
3622
|
+
summary: argValue('--summary'),
|
|
3623
|
+
evidence: argValue('--evidence'),
|
|
3624
|
+
by: argValue('--by')
|
|
3625
|
+
});
|
|
3626
|
+
if (json) console.log(JSON.stringify({ ok: true, outcome: entry }));
|
|
3627
|
+
else log.success(`✓ outcome recorded for ${argValue('--run')}; run \`calibrate\` to compare it with the review`);
|
|
3628
|
+
} catch (err) {
|
|
3629
|
+
stageJsonError(err, json, 'OUTCOME_FAILED');
|
|
3630
|
+
}
|
|
3631
|
+
}
|
|
3632
|
+
|
|
3633
|
+
function handleCalibrate() {
|
|
3634
|
+
const json = process.argv.includes('--json');
|
|
3635
|
+
try {
|
|
3636
|
+
const calibration = require('../lib/calibration.js');
|
|
3637
|
+
const report = calibration.calibrate(CWD);
|
|
3638
|
+
if (json) console.log(JSON.stringify({ ok: true, ...report }));
|
|
3639
|
+
else console.log(calibration.formatCalibration(report));
|
|
3640
|
+
} catch (err) {
|
|
3641
|
+
stageJsonError(err, json, 'CALIBRATE_FAILED');
|
|
3642
|
+
}
|
|
3643
|
+
}
|
|
3644
|
+
|
|
3645
|
+
function handleDebt() {
|
|
3646
|
+
const json = process.argv.includes('--json');
|
|
3647
|
+
try {
|
|
3648
|
+
const debt = require('../lib/debt.js');
|
|
3649
|
+
const action = args[1] && !args[1].startsWith('--') ? args[1] : 'list';
|
|
3650
|
+
if (action === 'close') {
|
|
3651
|
+
const item = debt.closeDebt(CWD, args[2], { by: argValue('--by'), note: argValue('--note') });
|
|
3652
|
+
if (json) console.log(JSON.stringify({ ok: true, item }));
|
|
3653
|
+
else log.success(`✓ closed ${item.id}`);
|
|
3654
|
+
return;
|
|
3655
|
+
}
|
|
3656
|
+
if (action !== 'list') throw Object.assign(new Error('debt supports: list, close <id> --by <name>'), { code: 'INVALID_USAGE' });
|
|
3657
|
+
const file = argValue('--file');
|
|
3658
|
+
const items = debt.loadDebt(CWD).items.filter((item) => item.status === 'open' && (!file || item.file === file));
|
|
3659
|
+
if (json) console.log(JSON.stringify({ ok: true, items }));
|
|
3660
|
+
else if (!items.length) console.log('No open debt.');
|
|
3661
|
+
else for (const item of items) console.log(`- ${item.id} [${item.risk}] ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; revisit: ${item.nextTrigger})`);
|
|
3662
|
+
} catch (err) {
|
|
3663
|
+
stageJsonError(err, json, 'DEBT_FAILED');
|
|
3664
|
+
}
|
|
3665
|
+
}
|
|
3666
|
+
|
|
3667
|
+
function handleNext() {
|
|
3668
|
+
const json = process.argv.includes('--json');
|
|
3669
|
+
try {
|
|
3670
|
+
const features = require('../lib/features.js');
|
|
3671
|
+
const document = features.loadFeatures(CWD);
|
|
3672
|
+
if (!document) throw Object.assign(new Error('No .ai-engineering-loop/features.json. FIX: create it with {"schemaVersion":1,"features":[{"id","title","behavior","verify","status":"not_started","priority","dependsOn":[]}]}.'), { code: 'NO_FEATURES' });
|
|
3673
|
+
const next = features.nextFeature(document);
|
|
3674
|
+
const ratio = features.vcr(document);
|
|
3675
|
+
const problems = features.auditFeatures(CWD, document);
|
|
3676
|
+
const active = document.features.filter((item) => item.status === 'active').map((item) => item.id);
|
|
3677
|
+
const result = { ok: problems.length === 0, next, vcr: ratio, active, problems };
|
|
3678
|
+
if (json) console.log(JSON.stringify(result));
|
|
3679
|
+
else {
|
|
3680
|
+
console.log(`VCR: ${ratio.verified}/${ratio.activated} (${ratio.value})${active.length ? `; active: ${active.join(', ')}` : ''}`);
|
|
3681
|
+
if (ratio.value < 1 && active.length) console.log(`! Finish ${active[0]} before starting another feature (WIP=1).`);
|
|
3682
|
+
console.log(next ? `Next: ${next.id} - ${next.title || next.behavior}${next.verify ? ` (verify: ${next.verify})` : ''}. Start it with a Goal whose featureId is "${next.id}".` : 'Next: nothing unblocked.');
|
|
3683
|
+
for (const problem of problems) console.log(`! ${problem}`);
|
|
3684
|
+
}
|
|
3685
|
+
if (problems.length) process.exitCode = 1;
|
|
3686
|
+
} catch (err) {
|
|
3687
|
+
stageJsonError(err, json, 'NEXT_FAILED');
|
|
3688
|
+
}
|
|
3689
|
+
}
|
|
3690
|
+
|
|
3691
|
+
// C-20: `rules check <id>` runs a built-in rule kind against the lines the
|
|
3692
|
+
// change adds ($AEL_ADDED_LINES_FILE); violations print as file:line.
|
|
3693
|
+
function handleRules() {
|
|
3694
|
+
const json = process.argv.includes('--json');
|
|
3695
|
+
try {
|
|
3696
|
+
if (args[1] !== 'check' || !args[2]) throw Object.assign(new Error('rules supports: check <ruleId>'), { code: 'INVALID_USAGE' });
|
|
3697
|
+
const rule = require('../lib/rules.js').findRule(CWD, args[2]);
|
|
3698
|
+
if (!rule) throw Object.assign(new Error(`No rule ${args[2]} in .ai-engineering-loop/rules.json`), { code: 'NOT_FOUND' });
|
|
3699
|
+
const ruleCheck = require('../lib/rule-check.js');
|
|
3700
|
+
const violations = ruleCheck.checkRule(rule, ruleCheck.readAddedLines());
|
|
3701
|
+
if (json) console.log(JSON.stringify({ ok: violations.length === 0, rule: rule.id, violations }));
|
|
3702
|
+
else for (const v of violations) console.log(`${v.file}:${v.line}: ${v.message}`);
|
|
3703
|
+
if (violations.length) process.exitCode = 1;
|
|
3704
|
+
} catch (err) {
|
|
3705
|
+
stageJsonError(err, json, 'RULES_FAILED');
|
|
3706
|
+
}
|
|
3707
|
+
}
|
|
3708
|
+
|
|
3709
|
+
function handleStats() {
|
|
3710
|
+
const json = process.argv.includes('--json');
|
|
3711
|
+
try {
|
|
3712
|
+
const lastValue = argValue('--last');
|
|
3713
|
+
const stats = require('../lib/stats.js');
|
|
3714
|
+
const result = stats.computeStats(CWD, lastValue == null ? {} : { last: parsePositiveIntegerArg('--last', lastValue) });
|
|
3715
|
+
if (json) console.log(JSON.stringify({ ok: true, ...result }));
|
|
3716
|
+
else console.log(stats.formatStats(result));
|
|
3717
|
+
} catch (err) {
|
|
3718
|
+
stageJsonError(err, json, 'STATS_FAILED');
|
|
3719
|
+
}
|
|
3720
|
+
}
|
|
3721
|
+
|
|
3497
3722
|
function readGuardInput() {
|
|
3498
3723
|
const source = argValue('--input') || (process.argv.includes('--hook') ? '-' : null);
|
|
3499
3724
|
if (source == null) {
|
|
@@ -3639,6 +3864,25 @@ function handleReviewPack() {
|
|
|
3639
3864
|
const pack = smartContextSummary(createDiffHunkPack(CWD, { runId, base, profile: argValue('--profile') }));
|
|
3640
3865
|
const escalation = assessRunEscalation(CWD, { runId });
|
|
3641
3866
|
const tier = run.tier ? run.tier.current : null;
|
|
3867
|
+
// C-14: open debt on the files this change touches goes into the pack.
|
|
3868
|
+
let openDebt = [];
|
|
3869
|
+
let touchedFeatures = [];
|
|
3870
|
+
try {
|
|
3871
|
+
const changed = require('../lib/verification-lanes.js').relatedFiles(CWD, run);
|
|
3872
|
+
openDebt = require('../lib/debt.js').openDebtFor(CWD, changed);
|
|
3873
|
+
if (openDebt.length && pack.path) {
|
|
3874
|
+
const lines = ['', '## Open debt on changed files', '', ...openDebt.map((item) => `- ${item.id} ${item.location}: ${item.debt} (deferred: ${item.whyDeferred}; risk ${item.risk})`)];
|
|
3875
|
+
fs.appendFileSync(path.join(CWD, pack.path), `${lines.join('\n')}\n`);
|
|
3876
|
+
}
|
|
3877
|
+
// C-22: user-facing features this diff touches, with how to reach them.
|
|
3878
|
+
const featureMap = require('../lib/feature-map.js');
|
|
3879
|
+
touchedFeatures = featureMap.featuresForFiles(CWD, changed);
|
|
3880
|
+
if (touchedFeatures.length && pack.path) fs.appendFileSync(path.join(CWD, pack.path), `${featureMap.renderForPack(touchedFeatures)}\n`);
|
|
3881
|
+
} catch {
|
|
3882
|
+
openDebt = [];
|
|
3883
|
+
}
|
|
3884
|
+
const diffRelative = path.relative(CWD, path.join(runsDir(CWD), runId, 'diff.patch')).split(path.sep).join('/');
|
|
3885
|
+
const reviewInputs = require('../lib/review-ingest.js').recordReviewInputs(CWD, runId, { packPath: pack.path, diffPath: diffRelative });
|
|
3642
3886
|
autoHeartbeat(runId, 'review', 'Review pack built');
|
|
3643
3887
|
const result = {
|
|
3644
3888
|
ok: true,
|
|
@@ -3652,6 +3896,9 @@ function handleReviewPack() {
|
|
|
3652
3896
|
truncated: pack.truncated,
|
|
3653
3897
|
unresolvedFindings: pack.unresolvedFindings,
|
|
3654
3898
|
impactDrift: pack.impactDrift,
|
|
3899
|
+
openDebt: openDebt.map((item) => ({ id: item.id, location: item.location, debt: item.debt })),
|
|
3900
|
+
features: touchedFeatures.map((entry) => ({ id: entry.id, where: entry.where, driver: entry.driver })),
|
|
3901
|
+
reviewInputs: { inputs: reviewInputs.inputs, promptSha256: reviewInputs.promptSha256 },
|
|
3655
3902
|
fastPath: { eligible: fastPath.eligible, reasons: fastPath.reasons },
|
|
3656
3903
|
escalation: { requiredTier: escalation.requiredTier, reasons: escalation.reasons.map((r) => r.code) }
|
|
3657
3904
|
};
|
|
@@ -3665,6 +3912,8 @@ function handleReviewPack() {
|
|
|
3665
3912
|
console.log(`- Outside the impact map: ${result.impactDrift.outOfMap.join(', ')} (check their side effects)`);
|
|
3666
3913
|
}
|
|
3667
3914
|
for (const line of result.impactDrift?.design || []) console.log(`- ${line}`);
|
|
3915
|
+
if (touchedFeatures.length) console.log(`- Features touched: ${touchedFeatures.map((entry) => entry.id).join(', ')} (in the pack; use them for L3 evidence)`);
|
|
3916
|
+
if (openDebt.length) console.log(`- Open debt on changed files: ${openDebt.map((item) => item.id).join(', ')} (in the pack; resolve or keep consciously)`);
|
|
3668
3917
|
}
|
|
3669
3918
|
} catch (err) {
|
|
3670
3919
|
stageJsonError(err, json, 'REVIEW_PACK_FAILED');
|
|
@@ -3825,6 +4074,7 @@ function handleBench() {
|
|
|
3825
4074
|
sandboxHome: process.argv.includes('--sandbox-home'),
|
|
3826
4075
|
hostArgs: repeatedArg('--host-arg'),
|
|
3827
4076
|
scenarioIds: repeatedArg('--scenario'),
|
|
4077
|
+
without: repeatedArg('--without').flatMap((value) => value.split(',')).map((value) => value.trim()).filter(Boolean),
|
|
3828
4078
|
repeat: repeatArg == null ? 1 : parsePositiveIntegerArg('--repeat', repeatArg),
|
|
3829
4079
|
timeoutMs: timeoutArg == null ? undefined : parseBenchTimeoutSeconds(timeoutArg),
|
|
3830
4080
|
mode: modeArg == null ? 'ASSISTED' : normalizeMode(modeArg),
|
|
@@ -3930,6 +4180,26 @@ function handleQuick() {
|
|
|
3930
4180
|
|
|
3931
4181
|
function handleLearn() {
|
|
3932
4182
|
const json = process.argv.includes('--json');
|
|
4183
|
+
if (args[1] === 'promote') {
|
|
4184
|
+
try {
|
|
4185
|
+
const rule = require('../lib/rules.js').promoteLearning(CWD, args[2], {
|
|
4186
|
+
check: argValue('--check'),
|
|
4187
|
+
fix: argValue('--fix'),
|
|
4188
|
+
why: argValue('--why'),
|
|
4189
|
+
by: argValue('--by'),
|
|
4190
|
+
scope: argValue('--scope') || 'repo',
|
|
4191
|
+
forbidImport: argValue('--forbid-import'),
|
|
4192
|
+
forbidApi: argValue('--forbid-api'),
|
|
4193
|
+
except: argValue('--except'),
|
|
4194
|
+
shell: !process.argv.includes('--no-shell')
|
|
4195
|
+
});
|
|
4196
|
+
if (json) console.log(JSON.stringify({ ok: true, rule }));
|
|
4197
|
+
else log.success(`✓ ${rule.from} promoted to rule ${rule.id}; default \`verify\` now runs: ${rule.check}`);
|
|
4198
|
+
} catch (err) {
|
|
4199
|
+
stageJsonError(err, json, 'LEARN_PROMOTE_FAILED');
|
|
4200
|
+
}
|
|
4201
|
+
return;
|
|
4202
|
+
}
|
|
3933
4203
|
try {
|
|
3934
4204
|
const maxAge = argValue('--max-age-runs');
|
|
3935
4205
|
const result = learn(CWD, {
|
|
@@ -3938,8 +4208,9 @@ function handleLearn() {
|
|
|
3938
4208
|
...(maxAge == null ? {} : { maxAgeRuns: parsePositiveIntegerArg('--max-age-runs', maxAge) })
|
|
3939
4209
|
});
|
|
3940
4210
|
const { document } = result;
|
|
4211
|
+
const candidates = require('../lib/rules.js').promotionCandidates(CWD, document.entries);
|
|
3941
4212
|
if (json) {
|
|
3942
|
-
console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, learnings: document }));
|
|
4213
|
+
console.log(JSON.stringify({ ok: true, written: result.written, path: result.path, pruned: result.pruned, promotionCandidates: candidates, learnings: document }));
|
|
3943
4214
|
return;
|
|
3944
4215
|
}
|
|
3945
4216
|
log.success(`✓ ${document.entries.length} learning(s) from ${document.sourceRuns} Run(s)${result.written ? ` → ${result.path}` : ' (dry run, nothing written)'}`);
|
|
@@ -3947,6 +4218,9 @@ function handleLearn() {
|
|
|
3947
4218
|
console.log(`- ${entry.id} [${entry.axis}, ${entry.severity}, ×${entry.occurrences}] ${entry.lesson}`);
|
|
3948
4219
|
}
|
|
3949
4220
|
for (const item of result.pruned) console.log(` pruned ${item.id} (${item.reason})`);
|
|
4221
|
+
for (const item of candidates) {
|
|
4222
|
+
console.log(`! Promote candidate ${item.id} (seen in ${item.runs} Runs): ${item.lesson}. Make it a check: learn promote ${item.id} --check "<command>" --fix "<instruction>" --by <name>`);
|
|
4223
|
+
}
|
|
3950
4224
|
} catch (err) {
|
|
3951
4225
|
stageJsonError(err, json, 'LEARN_FAILED');
|
|
3952
4226
|
}
|
|
@@ -4083,6 +4357,12 @@ Commands:
|
|
|
4083
4357
|
A missing binary is UNAVAILABLE, not a model failure.
|
|
4084
4358
|
Progress lines go to stderr. Writes .ai-engineering-loop/bench/<benchId>/{results.json,report.md,hardening.md}
|
|
4085
4359
|
bench compare <benchIdA> <benchIdB> [--json] read-only side-by-side table
|
|
4360
|
+
outcome record --run <id> --source mr-comment|revert|bug|incident --location <file[:line]> --summary <text> --by <name>
|
|
4361
|
+
a post-delivery problem traced to a Run (input for calibrate)
|
|
4362
|
+
calibrate Compare recorded outcomes with each Run's Finding Ledger: DA misses, wrong dismissals, per reviewer prompt
|
|
4363
|
+
debt Accepted trade-offs and L3 waivers recorded at delivery: list [--file <path>] | close <id> --by <name> [--note <text>]
|
|
4364
|
+
next Next unblocked feature from .ai-engineering-loop/features.json, VCR (verified/activated), evidence audit [--json]
|
|
4365
|
+
stats Failure causes by harness layer, false-victory rate, iterations to PASS [--last <n>] [--json]
|
|
4086
4366
|
escalation Compute the minimum Judge model tier from deterministic risk signals
|
|
4087
4367
|
--run <id> target a non-current run
|
|
4088
4368
|
--json print machine-readable reasons
|
|
@@ -4094,7 +4374,8 @@ Commands:
|
|
|
4094
4374
|
start "<task>" [--mode assisted|unattended] | finish [--run <id>] [--cmd "<command>"]...
|
|
4095
4375
|
verify Record verification commands, gate them, compact, and scaffold claimed-vs-reality
|
|
4096
4376
|
[--run <id>] [--cmd "<command>"]... [--shell] [--serial] [--timeout-ms <n>] [--fresh] [--json]
|
|
4097
|
-
default
|
|
4377
|
+
default (no --cmd): build -> typecheck -> lint -> test_unit from verification.md, serial, stops at first failure
|
|
4378
|
+
--baseline before the Maker gate: run the plan on the untouched tree, write baseline.json
|
|
4098
4379
|
--related fast lane: test_related with {files} = changed files of the gated diff
|
|
4099
4380
|
--final after Judge PASS: full run delivery needs when verification was related-only
|
|
4100
4381
|
--fresh re-run commands that already passed for the same diff and revision
|
|
@@ -4122,6 +4403,11 @@ Commands:
|
|
|
4122
4403
|
writes RUN_DIR/receipt.json + receipt.md (paste into the PR/MR body); digest stored in delivery.json
|
|
4123
4404
|
verify-receipt Re-check a delivery receipt: artifact hashes, frozen Goal, AC evidence, verdict, head SHA
|
|
4124
4405
|
[--run <id> | <repository-relative-receipt.json>] [--head <sha>] [--json] exit 1 on mismatch (CI)
|
|
4406
|
+
learn promote <learningId> --check "<command>" --fix "<instruction>" --by <name> [--why <text>] [--scope repo|diff] [--no-shell]
|
|
4407
|
+
--scope diff: the check sees only lines this change adds ($AEL_ADDED_LINES_FILE, $AEL_CHANGED_FILES)
|
|
4408
|
+
built-in instead of --check: --forbid-import "<fromGlob> -> <toGlob>" | --forbid-api "<regex>" [--except <glob>]
|
|
4409
|
+
rules check <ruleId> run a built-in rule on the added lines (used by verify)
|
|
4410
|
+
turn a recurring review lesson into an executable L1 rule (.ai-engineering-loop/rules.json)
|
|
4125
4411
|
learn Mine past Runs' VALID findings into .ai-engineering-loop/learnings.json (+ .md) for context packs
|
|
4126
4412
|
[--dry-run] [--prune] [--max-age-runs <n>] [--json]
|
|
4127
4413
|
deterministic and deduped with runId provenance; --prune drops lessons whose files are gone
|
|
@@ -4130,6 +4416,7 @@ Commands:
|
|
|
4130
4416
|
list [--json]
|
|
4131
4417
|
<name> [--section <heading>] [--json] e.g. docs grill-policy --section "When to grill"
|
|
4132
4418
|
doctor Check runtime, schemas, host assets, package contents, and eval catalog
|
|
4419
|
+
--harness audit harness debt instead: entry-file size/structure, stale learnings, rule provenance, skill budget
|
|
4133
4420
|
--json print machine-readable diagnostics
|
|
4134
4421
|
update Check or apply the package update workflow
|
|
4135
4422
|
check|plan [--json]
|
|
@@ -4295,6 +4582,24 @@ switch (command) {
|
|
|
4295
4582
|
case 'escalation':
|
|
4296
4583
|
handleEscalation();
|
|
4297
4584
|
break;
|
|
4585
|
+
case 'stats':
|
|
4586
|
+
handleStats();
|
|
4587
|
+
break;
|
|
4588
|
+
case 'rules':
|
|
4589
|
+
handleRules();
|
|
4590
|
+
break;
|
|
4591
|
+
case 'next':
|
|
4592
|
+
handleNext();
|
|
4593
|
+
break;
|
|
4594
|
+
case 'debt':
|
|
4595
|
+
handleDebt();
|
|
4596
|
+
break;
|
|
4597
|
+
case 'outcome':
|
|
4598
|
+
handleOutcome();
|
|
4599
|
+
break;
|
|
4600
|
+
case 'calibrate':
|
|
4601
|
+
handleCalibrate();
|
|
4602
|
+
break;
|
|
4298
4603
|
case 'docs':
|
|
4299
4604
|
handleDocs();
|
|
4300
4605
|
break;
|
package/core/bench.md
CHANGED
|
@@ -75,3 +75,18 @@ Bench results do not feed `ael learn`. Learnings stay findings mined from Runs;
|
|
|
75
75
|
`bench compare <a> <b>` prints a side-by-side table and does not write.
|
|
76
76
|
|
|
77
77
|
A live run against your own host is manual evidence for a change review. CI uses the fake host only.
|
|
78
|
+
|
|
79
|
+
## Ablation and the simplification log (C-08)
|
|
80
|
+
|
|
81
|
+
Every harness component encodes an assumption about what the model cannot do alone; as models improve, some assumptions go stale. Measure instead of guessing:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
npx ai-engineering-loop bench --host <host> --model <model> # baseline
|
|
85
|
+
npx ai-engineering-loop bench --host <host> --model <model> --without integrity # one component off
|
|
86
|
+
npx ai-engineering-loop bench compare <baselineId> <ablationId>
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
`--without` accepts `integrity`, `l3`, `l1`, `smoke`, `rules` (repeat or comma-separate). It only takes effect inside bench scenario repos: bench sets `AEL_ABLATE` and writes `.ai-engineering-loop/bench-ablation.json`; the gates honour the switch only when both are present together with `AEL_BENCH_MODE`, and every skip is logged to the Run's `ablation.log`. The result carries `ablation.without`, and an entry is appended to `.ai-engineering-loop/simplification-log.json` (`decision: null` until a human records keep / remove / replace).
|
|
90
|
+
|
|
91
|
+
`doctor --harness` audits harness debt in the project: entry-file size (50-200 lines) and structure (only headings, lists, tables and code count, so keywords in prose cannot game it), learnings not seen in the last 10 Runs, rules without provenance or older than 90 days, and, inside the AEL repo, T1 skill-context headroom from `scripts/measure.js`.
|
|
92
|
+
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# Formal flows with pstack `/statechart`
|
|
2
|
+
|
|
3
|
+
How to prove a risky user flow with a model checker and feed the result into AEL's gates. AEL ships none of this tooling; it only recognises the evidence (card C-25).
|
|
4
|
+
|
|
5
|
+
## When it is worth it
|
|
6
|
+
|
|
7
|
+
Use it for flows where a wrong state is expensive and hard to see in unit tests: payments and refunds, sync across devices, retries with leases, notification fan-out, multi-step approvals. For an ordinary screen flow, the Goal `flow` diagram (below) is enough.
|
|
8
|
+
|
|
9
|
+
## The tool
|
|
10
|
+
|
|
11
|
+
[`statechart`](https://github.com/cursor/plugins/tree/main/pstack/skills/statechart) is a skill in the pstack plugin by Lauren Tan (MIT). You write a TLA+ spec in which each rule the reader agrees to is an `INVARIANT`. TLC model-checks it, and the skill draws the result as a Mermaid statechart. Its report gives the shortest path to every state that breaks a rule.
|
|
12
|
+
|
|
13
|
+
Requirements: Java and TLC (installed with `mise` on first run, which needs the network) and a runtime that executes TypeScript. Install pstack as a Cursor plugin, or follow its Claude Code guide (`docs/harness/claude-code.md` in pstack) and call it as `/pstack:statechart`. It runs only when asked (`disable-model-invocation`).
|
|
14
|
+
|
|
15
|
+
## Wire it into AEL
|
|
16
|
+
|
|
17
|
+
1. Keep the spec in the repository, for example `spec/checkout.tla` and `spec/checkout.cfg`. Keep constants small, or add a `CONSTRAINT`: TLC keeps going past a broken rule, so an unbounded model can run without end.
|
|
18
|
+
2. `statechart` requires its `--out` folder to be new or empty, so a fixed folder fails the second `verify` (exit 2). Wrap it in a small script that clears the folder first, and add the folder to `.gitignore` (otherwise delivery reports it as an untracked file, C-06):
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
#!/usr/bin/env sh
|
|
22
|
+
# scripts/formal.sh
|
|
23
|
+
set -e
|
|
24
|
+
rm -rf .tmp/statechart
|
|
25
|
+
exec "$PSTACK/skills/statechart/scripts/statechart" spec/checkout.tla --config spec/checkout.cfg --out .tmp/statechart
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
3. Register the wrapper as the `formal` key in `.ai-engineering-loop/verification.md`. `formal` is layer `LF` and never runs in the default plan:
|
|
29
|
+
|
|
30
|
+
```
|
|
31
|
+
- **formal**: `sh scripts/formal.sh`
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
The statechart command exits with TLC's code: 0 when every rule holds, 1 when a rule breaks, 2 when TLC stops with an error. So a broken rule fails the verification gate like any other failing command.
|
|
35
|
+
4. Require it on the AC that the model covers:
|
|
36
|
+
|
|
37
|
+
```json
|
|
38
|
+
{ "id": "AC-2", "statement": "A refund never leaves the order both paid and refunded", "evidenceRequired": "TLC model check", "failureCases": ["double refund"], "verify": { "layer": "LF" } }
|
|
39
|
+
```
|
|
40
|
+
|
|
41
|
+
Default `verify` then runs `formal` (C-23), and `gate judge` refuses PASS until it passed on the gated diff (`AC_NOT_PROVEN`).
|
|
42
|
+
5. Attach the chart to the PR receipt: publish `chart.mmd` (or a rendered image) and pass `deliver --evidence AC-2=<https-url>`. The receipt lists it under "Verification artifacts" as a diagram.
|
|
43
|
+
6. When the model describes code, replay each bug path from the report against the real code, as a test or a live repro. A path that does not reproduce means the model is wrong until shown otherwise.
|
|
44
|
+
|
|
45
|
+
## Without TLA+: the Goal `flow` diagram
|
|
46
|
+
|
|
47
|
+
A Goal Contract `flow` (`[{ "from", "event", "to" }]`, see `docs goal-contract`) is rendered by `goal draft` and in the PR receipt as a Mermaid `stateDiagram-v2` plus a `From | Event | To` table. GitHub renders the diagram in the PR. It lets a reviewer check that the agent understood the intended flow; it proves nothing about the code. For proof, use `formal` as above.
|
|
48
|
+
|
|
49
|
+
Skill text never contains Mermaid (some hosts cannot render it); diagrams appear only in command output and PR receipts.
|