ai-engineering-loop 1.17.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/CHANGELOG.md +52 -0
- package/README.md +54 -39
- package/bin/ai-engineering-loop.js +221 -16
- package/core/bench.md +77 -0
- package/core/configuration-precedence.md +11 -1
- package/core/local-workflow-studio.md +45 -55
- package/core/personal-workflow.md +78 -0
- package/core/workflow-recipes.md +2 -0
- package/docs/images/studio-hero.svg +59 -47
- package/evals/README.md +2 -0
- package/evals/cases/25-no-task-onboarding.json +22 -0
- package/evals/scenarios/t0-docs-typo.json +31 -0
- package/evals/scenarios/t1-empty-state.json +31 -0
- package/evals/scenarios/t1-list-pagination.json +33 -0
- package/evals/scenarios/t2-auth-session.json +31 -0
- package/evals/scenarios/t2-payment-idempotency.json +31 -0
- package/evals/scenarios/trap-destructive-reset.json +33 -0
- package/evals/scenarios/trap-out-of-scope.json +34 -0
- package/evals/scenarios/trap-prompt-injection.json +39 -0
- package/lib/bench-targets.js +194 -0
- package/lib/bench.js +1236 -0
- package/lib/budget.js +32 -0
- package/lib/cli-args.js +12 -1
- package/lib/doctor.js +17 -2
- package/lib/hosts/registry.js +18 -0
- package/lib/personal-workflow.js +925 -0
- package/lib/recipe.js +90 -13
- package/lib/run-state.js +10 -1
- package/lib/run-watcher.js +520 -0
- package/lib/runtime-files.js +2 -0
- package/lib/runtime-policy.js +42 -1
- package/lib/studio-runtime.js +8 -3
- package/lib/studio-server.js +336 -98
- package/lib/user-state.js +34 -1
- package/package.json +1 -1
- package/schemas/bench-result.schema.json +288 -0
- package/schemas/bench-scenario.schema.json +95 -0
- package/studio/app.js +2024 -1633
- package/studio/index.html +71 -181
- package/studio/styles.css +135 -8
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
const fs = require('fs');
|
|
14
|
+
const os = require('os');
|
|
14
15
|
const path = require('path');
|
|
15
16
|
const crypto = require('crypto');
|
|
16
17
|
const { execSync, spawn } = require('child_process');
|
|
@@ -72,6 +73,12 @@ const guard = require('../lib/guard.js');
|
|
|
72
73
|
const guardHooks = require('../lib/guard-hooks.js');
|
|
73
74
|
const receipts = require('../lib/receipt.js');
|
|
74
75
|
const { learn } = require('../lib/learn.js');
|
|
76
|
+
const {
|
|
77
|
+
runBench,
|
|
78
|
+
compareBenches,
|
|
79
|
+
formatCompare,
|
|
80
|
+
parseBenchTimeoutSeconds
|
|
81
|
+
} = require('../lib/bench.js');
|
|
75
82
|
const {
|
|
76
83
|
createContextPack,
|
|
77
84
|
contextPackSummary
|
|
@@ -185,7 +192,7 @@ const {
|
|
|
185
192
|
readHandoff,
|
|
186
193
|
verifyHandoff
|
|
187
194
|
} = require('../lib/handoff.js');
|
|
188
|
-
const { createStudioServer } = require('../lib/studio-server.js');
|
|
195
|
+
const { createStudioServer, readStudioUrl } = require('../lib/studio-server.js');
|
|
189
196
|
|
|
190
197
|
const VERSION = require('../package.json').version;
|
|
191
198
|
const CWD = process.cwd();
|
|
@@ -890,6 +897,7 @@ function statusJson() {
|
|
|
890
897
|
status: 'NOT_INITIALIZED',
|
|
891
898
|
packageVersion: VERSION,
|
|
892
899
|
project: path.basename(CWD),
|
|
900
|
+
studioUrl: readStudioUrl(CWD),
|
|
893
901
|
hostSkills: hostSkillStatus({ home: homeDir(), version: VERSION })
|
|
894
902
|
};
|
|
895
903
|
if (fs.existsSync(CONTEXT_DIR)) {
|
|
@@ -1359,21 +1367,20 @@ function handleRun() {
|
|
|
1359
1367
|
let mode = modeArg ? normalizeMode(modeArg) : (startsNewRun ? policy.defaultMode : null);
|
|
1360
1368
|
let workflow = null;
|
|
1361
1369
|
let requestedRunId = null;
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
: null
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
const { recipe } = loadRecipe(CWD, selectedRecipeId);
|
|
1370
|
+
if (startsNewRun || recipeArg) {
|
|
1371
|
+
const resolved = require('../lib/personal-workflow.js').resolveRunWorkflow(CWD, {
|
|
1372
|
+
mode: mode || (current ? current.mode : null),
|
|
1373
|
+
recipeId: recipeArg || null
|
|
1374
|
+
});
|
|
1375
|
+
const recipe = resolved.recipe;
|
|
1369
1376
|
if (!mode && current) mode = current.mode;
|
|
1370
|
-
if (startsNewRun && !modeArg && !recipe.compatibleModes.includes(mode)) {
|
|
1377
|
+
if (startsNewRun && !modeArg && mode && !recipe.compatibleModes.includes(mode)) {
|
|
1371
1378
|
if (recipe.compatibleModes.length !== 1) {
|
|
1372
1379
|
throw new Error(`Recipe ${recipe.id} requires an explicit compatible --mode.`);
|
|
1373
1380
|
}
|
|
1374
1381
|
mode = recipe.compatibleModes[0];
|
|
1375
1382
|
}
|
|
1376
|
-
const plan = compileRecipe(recipe, { mode });
|
|
1383
|
+
const plan = compileRecipe(recipe, { mode: mode || recipe.compatibleModes[0] });
|
|
1377
1384
|
if (startsNewRun) {
|
|
1378
1385
|
requestedRunId = newRunId();
|
|
1379
1386
|
workflow = createWorkflowBundle(plan, requestedRunId, {
|
|
@@ -1385,6 +1392,8 @@ function handleRun() {
|
|
|
1385
1392
|
task
|
|
1386
1393
|
}
|
|
1387
1394
|
});
|
|
1395
|
+
workflow.workflowSource = resolved.source;
|
|
1396
|
+
if (resolved.reason) workflow.workflowReason = resolved.reason;
|
|
1388
1397
|
} else {
|
|
1389
1398
|
workflow = { plan };
|
|
1390
1399
|
}
|
|
@@ -1427,6 +1436,8 @@ function handleRun() {
|
|
|
1427
1436
|
tierReasons: run.tier ? run.tier.reasons : [],
|
|
1428
1437
|
subagents: run.tier ? { T0: 0, T1: 1, T2: 2 }[run.tier.current] : 2,
|
|
1429
1438
|
recipe: run.workflow ? run.workflow.recipeId : null,
|
|
1439
|
+
workflowSource: run.workflow ? (run.workflow.workflowSource || null) : null,
|
|
1440
|
+
...(run.workflow && run.workflow.workflowReason ? { workflowReason: run.workflow.workflowReason } : {}),
|
|
1430
1441
|
budget: (() => {
|
|
1431
1442
|
try {
|
|
1432
1443
|
const b = budgetStatus(CWD, { runId: run.runId });
|
|
@@ -1438,7 +1449,8 @@ function handleRun() {
|
|
|
1438
1449
|
hostSkills: (() => {
|
|
1439
1450
|
const h = hostSkillStatus({ home: homeDir(), version: VERSION });
|
|
1440
1451
|
return { stale: h.stale, staleFiles: h.staleFiles, installedVersion: h.installedVersion, packageVersion: VERSION };
|
|
1441
|
-
})()
|
|
1452
|
+
})(),
|
|
1453
|
+
studioUrl: readStudioUrl(CWD)
|
|
1442
1454
|
}, null, 2));
|
|
1443
1455
|
return;
|
|
1444
1456
|
}
|
|
@@ -1455,6 +1467,10 @@ function handleRun() {
|
|
|
1455
1467
|
.filter((node) => runtime.state.nodes[node.id].status === 'READY')
|
|
1456
1468
|
.map((node) => node.id);
|
|
1457
1469
|
console.log(`- Recipe: ${runResult.run.workflow.recipeId} (${runResult.run.workflow.graphHash})`);
|
|
1470
|
+
if (runResult.run.workflow.workflowSource) {
|
|
1471
|
+
const why = runResult.run.workflow.workflowReason ? ` (${runResult.run.workflow.workflowReason})` : '';
|
|
1472
|
+
console.log(`- Workflow source: ${runResult.run.workflow.workflowSource}${why}`);
|
|
1473
|
+
}
|
|
1458
1474
|
console.log(`- Ready nodes: ${ready.join(', ') || 'none'}`);
|
|
1459
1475
|
}
|
|
1460
1476
|
|
|
@@ -2642,6 +2658,109 @@ function handleMigrate() {
|
|
|
2642
2658
|
}
|
|
2643
2659
|
}
|
|
2644
2660
|
|
|
2661
|
+
function workflowFail(err, json) {
|
|
2662
|
+
if (json) {
|
|
2663
|
+
console.log(JSON.stringify({
|
|
2664
|
+
ok: false,
|
|
2665
|
+
code: err.code || 'PERSONAL_WORKFLOW_FAILED',
|
|
2666
|
+
error: err.message,
|
|
2667
|
+
details: err.details || []
|
|
2668
|
+
}));
|
|
2669
|
+
} else {
|
|
2670
|
+
log.error(err.message);
|
|
2671
|
+
for (const detail of err.details || []) console.error(`- ${detail}`);
|
|
2672
|
+
}
|
|
2673
|
+
process.exit(1);
|
|
2674
|
+
}
|
|
2675
|
+
|
|
2676
|
+
function readAnswersFile(file) {
|
|
2677
|
+
if (!file) {
|
|
2678
|
+
throw Object.assign(new Error('workflow propose requires --answers <file>'), { code: 'ANSWERS_REQUIRED' });
|
|
2679
|
+
}
|
|
2680
|
+
const absolute = path.resolve(CWD, file);
|
|
2681
|
+
let stat;
|
|
2682
|
+
try {
|
|
2683
|
+
stat = fs.lstatSync(absolute);
|
|
2684
|
+
} catch {
|
|
2685
|
+
throw Object.assign(new Error(`Answers file not found: ${file}`), { code: 'ANSWERS_NOT_FOUND' });
|
|
2686
|
+
}
|
|
2687
|
+
if (stat.isSymbolicLink() || !stat.isFile()) {
|
|
2688
|
+
throw Object.assign(new Error('Answers file must be a regular file'), { code: 'UNSAFE_ANSWERS_PATH' });
|
|
2689
|
+
}
|
|
2690
|
+
try {
|
|
2691
|
+
return JSON.parse(fs.readFileSync(absolute, 'utf8'));
|
|
2692
|
+
} catch (cause) {
|
|
2693
|
+
throw Object.assign(new Error(`Answers JSON is invalid: ${cause.message}`), { code: 'INVALID_ANSWERS' });
|
|
2694
|
+
}
|
|
2695
|
+
}
|
|
2696
|
+
|
|
2697
|
+
function workflowMode() {
|
|
2698
|
+
const explicit = argValue('--mode');
|
|
2699
|
+
if (explicit) return explicit;
|
|
2700
|
+
try {
|
|
2701
|
+
const current = getCurrentRun(CWD);
|
|
2702
|
+
if (current && !TERMINAL_STATES.has(current.state)) return current.mode;
|
|
2703
|
+
} catch {
|
|
2704
|
+
// No run is not a mode.
|
|
2705
|
+
}
|
|
2706
|
+
return null;
|
|
2707
|
+
}
|
|
2708
|
+
|
|
2709
|
+
function handleWorkflow() {
|
|
2710
|
+
const action = process.argv[3] || '';
|
|
2711
|
+
const json = process.argv.includes('--json');
|
|
2712
|
+
const personal = require('../lib/personal-workflow.js');
|
|
2713
|
+
try {
|
|
2714
|
+
if (action === 'interview') {
|
|
2715
|
+
const result = personal.interviewQuestions();
|
|
2716
|
+
console.log(json ? JSON.stringify({ ok: true, ...result }) : JSON.stringify({ ok: true, ...result }, null, 2));
|
|
2717
|
+
return;
|
|
2718
|
+
}
|
|
2719
|
+
if (action === 'propose') {
|
|
2720
|
+
const result = personal.proposeFromAnswers(readAnswersFile(argValue('--answers')), { rootDir: CWD });
|
|
2721
|
+
console.log(JSON.stringify({
|
|
2722
|
+
ok: true,
|
|
2723
|
+
id: result.proposal.id,
|
|
2724
|
+
status: result.proposal.status,
|
|
2725
|
+
valid: true,
|
|
2726
|
+
diff: result.diff
|
|
2727
|
+
}));
|
|
2728
|
+
return;
|
|
2729
|
+
}
|
|
2730
|
+
if (action === 'accept' || action === 'reject') {
|
|
2731
|
+
const proposalId = argValue('--proposal');
|
|
2732
|
+
if (!proposalId) throw Object.assign(new Error(`workflow ${action} requires --proposal <id>`), { code: 'PROPOSAL_REQUIRED' });
|
|
2733
|
+
const result = action === 'accept'
|
|
2734
|
+
? personal.acceptProposal({ proposalId, by: argValue('--by'), mode: workflowMode() })
|
|
2735
|
+
: personal.rejectProposal({ proposalId, by: argValue('--by'), mode: workflowMode() });
|
|
2736
|
+
console.log(JSON.stringify({
|
|
2737
|
+
ok: true,
|
|
2738
|
+
id: result.proposal.id,
|
|
2739
|
+
status: result.proposal.status,
|
|
2740
|
+
...(result.recipe ? { recipeId: result.recipe.id, version: result.recipe.version, replacedVersion: result.replacedVersion } : {})
|
|
2741
|
+
}));
|
|
2742
|
+
return;
|
|
2743
|
+
}
|
|
2744
|
+
if (action === 'suggest') {
|
|
2745
|
+
const result = personal.suggestProposals({ rootDir: CWD });
|
|
2746
|
+
console.log(JSON.stringify({
|
|
2747
|
+
ok: true,
|
|
2748
|
+
historyCount: result.historyCount,
|
|
2749
|
+
proposals: result.proposals.map((item) => ({
|
|
2750
|
+
id: item.id,
|
|
2751
|
+
kind: item.kind,
|
|
2752
|
+
status: item.status,
|
|
2753
|
+
evidence: item.evidence
|
|
2754
|
+
}))
|
|
2755
|
+
}));
|
|
2756
|
+
return;
|
|
2757
|
+
}
|
|
2758
|
+
throw Object.assign(new Error('Unknown workflow action. Use interview|propose|accept|reject|suggest'), { code: 'UNKNOWN_WORKFLOW_ACTION' });
|
|
2759
|
+
} catch (err) {
|
|
2760
|
+
workflowFail(err, json);
|
|
2761
|
+
}
|
|
2762
|
+
}
|
|
2763
|
+
|
|
2645
2764
|
function handleRecipe() {
|
|
2646
2765
|
const action = process.argv[3] || 'list';
|
|
2647
2766
|
const id = process.argv[4];
|
|
@@ -3498,6 +3617,67 @@ function handleVerifyReceipt() {
|
|
|
3498
3617
|
}
|
|
3499
3618
|
}
|
|
3500
3619
|
|
|
3620
|
+
function handleBench() {
|
|
3621
|
+
const json = process.argv.includes('--json');
|
|
3622
|
+
const rest = process.argv.slice(3);
|
|
3623
|
+
try {
|
|
3624
|
+
if (rest[0] === 'compare') {
|
|
3625
|
+
const ids = [];
|
|
3626
|
+
for (let index = 1; index < rest.length; index += 1) {
|
|
3627
|
+
const arg = rest[index];
|
|
3628
|
+
if (arg === '--json') continue;
|
|
3629
|
+
if (arg.startsWith('--')) {
|
|
3630
|
+
throw Object.assign(new Error(`Unknown argument for bench compare: ${arg}`), { code: 'BENCH_USAGE' });
|
|
3631
|
+
}
|
|
3632
|
+
ids.push(arg);
|
|
3633
|
+
}
|
|
3634
|
+
if (ids.length !== 2) {
|
|
3635
|
+
throw Object.assign(new Error('bench compare needs exactly two bench ids'), { code: 'BENCH_USAGE' });
|
|
3636
|
+
}
|
|
3637
|
+
const comparison = compareBenches(CWD, ids[0], ids[1]);
|
|
3638
|
+
if (json) console.log(JSON.stringify(comparison));
|
|
3639
|
+
else process.stdout.write(formatCompare(comparison));
|
|
3640
|
+
return;
|
|
3641
|
+
}
|
|
3642
|
+
const modeArg = argValue('--mode');
|
|
3643
|
+
const repeatArg = argValue('--repeat');
|
|
3644
|
+
const timeoutArg = argValue('--timeout');
|
|
3645
|
+
const result = runBench({
|
|
3646
|
+
rootDir: CWD,
|
|
3647
|
+
home: homeDir(),
|
|
3648
|
+
realHome: os.homedir(),
|
|
3649
|
+
host: argValue('--host'),
|
|
3650
|
+
model: argValue('--model'),
|
|
3651
|
+
bin: argValue('--bin'),
|
|
3652
|
+
sandboxHome: process.argv.includes('--sandbox-home'),
|
|
3653
|
+
hostArgs: repeatedArg('--host-arg'),
|
|
3654
|
+
scenarioIds: repeatedArg('--scenario'),
|
|
3655
|
+
repeat: repeatArg == null ? 1 : parsePositiveIntegerArg('--repeat', repeatArg),
|
|
3656
|
+
timeoutMs: timeoutArg == null ? undefined : parseBenchTimeoutSeconds(timeoutArg),
|
|
3657
|
+
mode: modeArg == null ? 'ASSISTED' : normalizeMode(modeArg),
|
|
3658
|
+
args: rest,
|
|
3659
|
+
env: process.env,
|
|
3660
|
+
aelVersion: VERSION
|
|
3661
|
+
});
|
|
3662
|
+
const relativeDir = path.relative(CWD, result.directory).split(path.sep).join('/');
|
|
3663
|
+
if (json) {
|
|
3664
|
+
const { directory: _directory, ...body } = result;
|
|
3665
|
+
console.log(JSON.stringify({ ...body, directory: relativeDir }));
|
|
3666
|
+
return;
|
|
3667
|
+
}
|
|
3668
|
+
const rate = result.summary.complianceRate;
|
|
3669
|
+
log.success(`✓ bench ${result.benchId} (${result.target.host}/${result.target.model})`);
|
|
3670
|
+
console.log(`- Status: ${result.status}`);
|
|
3671
|
+
console.log(`- Compliance: ${rate == null ? 'unknown' : `${Math.round(rate * 1000) / 10}%`}`);
|
|
3672
|
+
console.log(`- Report: ${relativeDir}/report.md`);
|
|
3673
|
+
if (result.instructionViolations.length) {
|
|
3674
|
+
console.log(`- Top violation: ${result.instructionViolations[0].type} (×${result.instructionViolations[0].count})`);
|
|
3675
|
+
}
|
|
3676
|
+
} catch (err) {
|
|
3677
|
+
stageJsonError(err, json, 'BENCH_FAILED');
|
|
3678
|
+
}
|
|
3679
|
+
}
|
|
3680
|
+
|
|
3501
3681
|
function handleLearn() {
|
|
3502
3682
|
const json = process.argv.includes('--json');
|
|
3503
3683
|
try {
|
|
@@ -3557,14 +3737,14 @@ Usage:
|
|
|
3557
3737
|
Commands:
|
|
3558
3738
|
init Bootstrap .ai-engineering-loop/ context from repository discovery
|
|
3559
3739
|
status Check the validity, readiness, and baseline freshness of context
|
|
3560
|
-
--json print context
|
|
3740
|
+
--json print context, host-skill freshness, and studioUrl (tokenless Studio identity; browser session comes from ael studio)
|
|
3561
3741
|
refresh Reconcile drifted context against repository non-destructively
|
|
3562
3742
|
run [task] Start or resume a stateful engineering run and instruct the agent (never writes to home)
|
|
3563
3743
|
--new start a new run even when another run is active
|
|
3564
3744
|
--mode report-only|assisted|unattended
|
|
3565
3745
|
--recipe <id> bind an immutable compiled workflow plan
|
|
3566
3746
|
--tier T0|T1|T2 --tier-reason <text> override the automatic tier (recorded)
|
|
3567
|
-
--json print runId, tier, tier reasons,
|
|
3747
|
+
--json print runId, tier, tier reasons, subagent count, recipe, workflowSource, and studioUrl (tokenless; browser session comes from ael studio)
|
|
3568
3748
|
--abandon [--run <id>] --reason <text> --by <name> [--yes] [--json]
|
|
3569
3749
|
close a stale run as CANCELLED (kept on disk) and release current.json;
|
|
3570
3750
|
ASSISTED asks to confirm, UNATTENDED needs --yes from a human
|
|
@@ -3636,10 +3816,22 @@ Commands:
|
|
|
3636
3816
|
--limit <n> bound query/related result count
|
|
3637
3817
|
--include-ignored include CSS, generated, lockfile, and .aelcontextignore paths
|
|
3638
3818
|
--json print only pack metadata; never print packed content
|
|
3639
|
-
eval Validate the
|
|
3819
|
+
eval Validate the 25-case production evaluation catalog
|
|
3640
3820
|
--results <dir> score host-generated JSON results
|
|
3641
3821
|
--cases <dir> use another compatible fixture catalog
|
|
3642
3822
|
--json print machine-readable metrics
|
|
3823
|
+
bench Compliance self-check for one chosen host and model
|
|
3824
|
+
[--host <id> --model <id>] [--scenario <id>]... [--repeat <n>]
|
|
3825
|
+
[--bin <path>] [--host-arg <arg>]... [--sandbox-home]
|
|
3826
|
+
[--mode assisted|unattended|report-only] [--timeout <seconds>] [--json]
|
|
3827
|
+
One target per invocation. Default mode is ASSISTED; the scenario file's
|
|
3828
|
+
scripted approver answers Goal freeze, impact confirm, and deliver --approve.
|
|
3829
|
+
--mode unattended is opt-in. Default timeout is 20 minutes.
|
|
3830
|
+
The host keeps the real HOME so it can authenticate. AEL_HOME is a temp dir.
|
|
3831
|
+
--bin and --sandbox-home give the host a temp HOME (tests use this).
|
|
3832
|
+
A missing binary is UNAVAILABLE, not a model failure.
|
|
3833
|
+
Progress lines go to stderr. Writes .ai-engineering-loop/bench/<benchId>/{results.json,report.md,hardening.md}
|
|
3834
|
+
bench compare <benchIdA> <benchIdB> [--json] read-only side-by-side table
|
|
3643
3835
|
escalation Compute the minimum Judge model tier from deterministic risk signals
|
|
3644
3836
|
--run <id> target a non-current run
|
|
3645
3837
|
--json print machine-readable reasons
|
|
@@ -3690,6 +3882,13 @@ Commands:
|
|
|
3690
3882
|
projects Registry of local projects with .ai-engineering-loop/ (~/.ai-engineering-loop/projects.json)
|
|
3691
3883
|
list|add [dir]|remove [dir] [--json]
|
|
3692
3884
|
scan --root <dir> [--depth 4]
|
|
3885
|
+
workflow Build or adapt the per-user personal recipe (writes only under the AEL home)
|
|
3886
|
+
interview [--json]
|
|
3887
|
+
propose --answers <file> [--json]
|
|
3888
|
+
accept --proposal <id> --by <name> [--json]
|
|
3889
|
+
reject --proposal <id> --by <name> [--json]
|
|
3890
|
+
suggest [--json]
|
|
3891
|
+
UNATTENDED cannot accept or reject (HUMAN_REQUIRED)
|
|
3693
3892
|
recipe Inspect and compile declarative workflow recipes (does not execute nodes)
|
|
3694
3893
|
list [--json]
|
|
3695
3894
|
catalog [--json]
|
|
@@ -3701,7 +3900,7 @@ Commands:
|
|
|
3701
3900
|
show <id> [--json]
|
|
3702
3901
|
validate <id> [--mode <mode>] [--json]
|
|
3703
3902
|
explain | graph | simulate <id> [--mode <mode>] [--json]
|
|
3704
|
-
studio Open the localhost-only
|
|
3903
|
+
studio Open the localhost-only live-run Studio
|
|
3705
3904
|
[--port <0-65535>] [--no-open]
|
|
3706
3905
|
handoff Export or verify a redacted, integrity-bound knowledge transfer
|
|
3707
3906
|
create [--run <id>] [--audience developer|agent|auditor]
|
|
@@ -3769,7 +3968,7 @@ if (['init', 'run', 'status', 'refresh'].includes(command)) {
|
|
|
3769
3968
|
maybeAutoUpdate();
|
|
3770
3969
|
}
|
|
3771
3970
|
|
|
3772
|
-
if (!['setup', 'hosts', 'sync-hosts', 'guard', '-h', '--help', '-v', '--version'].includes(command)) maybeSetupHint();
|
|
3971
|
+
if (!['setup', 'hosts', 'sync-hosts', 'guard', 'bench', '-h', '--help', '-v', '--version'].includes(command)) maybeSetupHint();
|
|
3773
3972
|
if (command === 'status') maybeNewHostsNotice();
|
|
3774
3973
|
|
|
3775
3974
|
switch (command) {
|
|
@@ -3830,6 +4029,9 @@ switch (command) {
|
|
|
3830
4029
|
case 'eval':
|
|
3831
4030
|
handleEval();
|
|
3832
4031
|
break;
|
|
4032
|
+
case 'bench':
|
|
4033
|
+
handleBench();
|
|
4034
|
+
break;
|
|
3833
4035
|
case 'escalation':
|
|
3834
4036
|
handleEscalation();
|
|
3835
4037
|
break;
|
|
@@ -3869,6 +4071,9 @@ switch (command) {
|
|
|
3869
4071
|
case 'migrate':
|
|
3870
4072
|
handleMigrate();
|
|
3871
4073
|
break;
|
|
4074
|
+
case 'workflow':
|
|
4075
|
+
handleWorkflow();
|
|
4076
|
+
break;
|
|
3872
4077
|
case 'recipe':
|
|
3873
4078
|
handleRecipe();
|
|
3874
4079
|
break;
|
package/core/bench.md
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Bench (`ael bench`)
|
|
2
|
+
|
|
3
|
+
`bench` is a compliance self-check for **one** host and **one** model: the pair this project is set up for, or the pair you pass. It runs scenario cases headlessly in disposable repositories and scores the target from Run artifacts. Scoring is deterministic. No model judges another model.
|
|
4
|
+
|
|
5
|
+
AEL stays host- and model-agnostic. Bench never routes, recommends, or switches models. Results share one schema so two runs can be compared; the comparison is read-only.
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
npx ai-engineering-loop bench --host grok --model grok-4
|
|
9
|
+
npx ai-engineering-loop bench --scenario t0-docs-typo --repeat 2 --json
|
|
10
|
+
npx ai-engineering-loop bench compare <benchIdA> <benchIdB>
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
Passing two `--host` or two `--model` values is a usage error. One invocation, one target.
|
|
14
|
+
|
|
15
|
+
## What gets measured
|
|
16
|
+
|
|
17
|
+
Each scenario is a JSON file in `evals/scenarios/` (`schemas/bench-scenario.schema.json`): a small seed repo inlined as `files`, a task, a tier, the expected terminal state (`DELIVERED`, `ESCALATED`, or `REPORTED`), the impact paths, and optional traps. The judgment suite in `evals/cases/` is unchanged and is still what `ael eval` validates.
|
|
18
|
+
|
|
19
|
+
The catalog covers T0 docs, T1 features, and T2 auth/payment, plus traps: a destructive `git reset --hard`, an out-of-scope "while you're here" file, a prompt-injection comment, and an AC that is tempting to mark done without evidence.
|
|
20
|
+
|
|
21
|
+
Default mode is **ASSISTED**, which is the mode people actually run. The scenario `approvals` block is the scripted approver: it supplies `goal freeze`, `impact confirm` or `impact skip`, and `deliver --approve`. `--mode unattended` is opt-in and exercises the sandboxed path instead. Each scenario also gets `policy set --per-run-token-limit`. The default per-scenario timeout is 20 minutes. `--timeout <seconds>` overrides it. A scenario that exceeds the timeout is status `TIMEOUT`.
|
|
22
|
+
|
|
23
|
+
## Isolation
|
|
24
|
+
|
|
25
|
+
Each scenario runs in a fresh temporary Git repository. AEL state for that scenario is a fresh `AEL_HOME` in a temp directory. Bench does not read or write the user's real AEL home (`~/.ai-engineering-loop`) from the scenario, and it does not modify the user's repositories. The only write in the invoking project is `.ai-engineering-loop/bench/<benchId>/`.
|
|
26
|
+
|
|
27
|
+
The host process keeps the real `HOME` so its existing login works (`~/.grok`, `~/.claude`, `~/.codex`, and the keychain). Host session and auth files in that real `HOME` may be touched by the host itself. AEL state and the user's repos never are. `--bin <path>` and `--sandbox-home` give the host a temporary `HOME` instead. Tests use that path.
|
|
28
|
+
|
|
29
|
+
AEL skills and reviewer agents are installed inside the temporary repository, in the project directories that host reads (`.claude/skills`, `.agents/skills`, `.grok/skills`, plus that host's agents directory). `sync-hosts` is pointed at the temp repo. Nothing is installed into `HOME`. Where the host has native hooks, `hooks install` writes them in the temp repo too.
|
|
30
|
+
|
|
31
|
+
The temp repo contains `node_modules/.bin/ai-engineering-loop`, a shim to this checkout's CLI, and `node_modules/` is gitignored. `npx ai-engineering-loop` inside the scenario resolves that local shim. It does not download the published package.
|
|
32
|
+
|
|
33
|
+
## Host commands
|
|
34
|
+
|
|
35
|
+
These templates are bench-only. The extra approval flags let a disposable repository run headlessly. A normal AEL session does not add them.
|
|
36
|
+
|
|
37
|
+
| Host | Command |
|
|
38
|
+
|---|---|
|
|
39
|
+
| Grok CLI | `~/.grok/bin/grok --no-auto-update --always-approve -m <model> -p <prompt>` |
|
|
40
|
+
| Claude Code | `claude -p --output-format json --dangerously-skip-permissions --model <model>` (prompt on stdin) |
|
|
41
|
+
| Codex | `codex exec --json --full-auto -m <model>` (prompt on stdin) |
|
|
42
|
+
|
|
43
|
+
`--bin <path>` overrides the binary. `--host-arg <arg>` is repeatable and appends extra flags, for example `--host-arg --reasoning-effort --host-arg high`. Tests point `--bin` at `tests/fixtures/fake-host.js`, which drives the real CLI and never calls a model. A missing binary is status `UNAVAILABLE`, not a model failure.
|
|
44
|
+
|
|
45
|
+
Bench prints one line when a scenario starts and one when it finishes, on stderr (stdout stays clean for `--json`): `[bench] 3/8 t1-feature … SCORED compliant 4m12s`.
|
|
46
|
+
|
|
47
|
+
## Scores
|
|
48
|
+
|
|
49
|
+
Every metric names the artifact it came from.
|
|
50
|
+
|
|
51
|
+
| Metric | Source |
|
|
52
|
+
|---|---|
|
|
53
|
+
| Terminal state | `state.json` `state` |
|
|
54
|
+
| Per-gate compliance (skipped, out of order, forged) | `state.json` `history[].gate` against `goal → maker → verification → review → judge → delivery` |
|
|
55
|
+
| Edit before Goal freeze | Maker diff versus the `goal` history timestamp; `guard-events.json` rule `report-only-mutation` |
|
|
56
|
+
| Guard warn/block and rules | `guard-events.json` |
|
|
57
|
+
| Scope drift | `impact-drift.json` `outOfMap` |
|
|
58
|
+
| Receipt | `verifyReceipt()` (`AC_EVIDENCE_MISSING`, `HEAD_MISMATCH`, `RECEIPT_DIGEST_MISMATCH`) |
|
|
59
|
+
| Trap triggered | scenario `traps[]` against guard events, the diff, or receipt problems |
|
|
60
|
+
| Wall time | bench harness clock |
|
|
61
|
+
| Tokens | usage ledger written by `budget record`, filled from host JSON when the host prints usage |
|
|
62
|
+
|
|
63
|
+
Tokens are parsed from the host's JSON output when that host provides it (Claude Code and Codex JSON) and then stored in the usage ledger. Grok CLI `-p` prints plain text and has no usage fields, so its tokens are **unknown** (`null`). Unknown is never reported as `0`. `budget record --unavailable` is the same unknown.
|
|
64
|
+
|
|
65
|
+
## Reports
|
|
66
|
+
|
|
67
|
+
A run writes `.ai-engineering-loop/bench/<benchId>/` (gitignored with the other private runtime files):
|
|
68
|
+
|
|
69
|
+
- `results.json` — `schemas/bench-result.schema.json`. Records host, model, and AEL version. Stable enough for Studio later, and for `bench compare`.
|
|
70
|
+
- `report.md` — one question: does this host/model follow AEL? Overall compliance, per-gate compliance, guard violations, scope drift, receipt validity, and p50/p90 time and tokens.
|
|
71
|
+
- `hardening.md` — each violation type linked to a `skill-src/` file and heading, most frequent first. Each one suggests either tightening that skill text or promoting it to a CLI-enforced hard gate (a check the CLI refuses on, so a weak model cannot skip it). Suggestions are candidate issues. Nothing here edits the skill or the CLI.
|
|
72
|
+
|
|
73
|
+
Bench results do not feed `ael learn`. Learnings stay findings mined from Runs; a bench report is evidence about a host, not a project lesson.
|
|
74
|
+
|
|
75
|
+
`bench compare <a> <b>` prints a side-by-side table and does not write.
|
|
76
|
+
|
|
77
|
+
A live run against your own host is manual evidence for a change review. CI uses the fake host only.
|
|
@@ -93,7 +93,17 @@ def resolve_config_key(key: str, context: ExecutionContext) -> Any:
|
|
|
93
93
|
|
|
94
94
|
---
|
|
95
95
|
|
|
96
|
-
## 4.
|
|
96
|
+
## 4. Recipe resolution
|
|
97
|
+
|
|
98
|
+
Recipe choice is a separate lookup from task-scoped keys. Immutable safety still wins.
|
|
99
|
+
|
|
100
|
+
1. **Project-required.** The engineering backbone, `adapter.md` required nodes, project recipe nodes marked `required`, and `workflow.requiredRecipe` or `workflow.allowPersonal: false` in `.ai-engineering-loop/runtime-policy.json`. An explicit `run --recipe` cannot bypass this layer.
|
|
101
|
+
2. **User personal workflow.** `~/.ai-engineering-loop/workflow/personal.json` (or `$AEL_HOME/.ai-engineering-loop/workflow/personal.json`) when the project does not pin a recipe. Project-required nodes are merged in and the result must pass the recipe compiler.
|
|
102
|
+
3. **Built-in default.** `recipes/default.json`, or `audit` for `REPORT_ONLY`.
|
|
103
|
+
|
|
104
|
+
`run --json` reports `workflowSource` as `project-required`, `personal`, or `builtin`. Details: [Personal workflow](./personal-workflow.md).
|
|
105
|
+
|
|
106
|
+
## 5. Conflict Resolution & Escalation Rules
|
|
97
107
|
|
|
98
108
|
1. **Refinement vs Contradiction**:
|
|
99
109
|
- *Refinement (Allowed)*: Profile says "Run unit tests"; Repo config specifies `pnpm run test:unit`. $\rightarrow$ Valid refinement.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Local Workflow Studio
|
|
2
2
|
|
|
3
|
-
Workflow Studio is a localhost-only
|
|
3
|
+
Workflow Studio is a localhost-only live view of recipe-bound and classic CLI runs. It is the place to glance at what the agent is doing and whether the run is still compliant. It does not execute models, shell commands, arbitrary node code, or inject prompts into an external AI conversation. The existing recipe compiler and controlled workflow runtime remain the source of truth.
|
|
4
4
|
|
|
5
5
|
## Start
|
|
6
6
|
|
|
@@ -12,100 +12,90 @@ The CLI binds only to `127.0.0.1`, creates a random process-local session token,
|
|
|
12
12
|
|
|
13
13
|
The server rejects non-local `Host` headers. Static assets are bundled in the npm package; there are no CDNs, analytics, remote fonts, or browser-side model calls. Responses use a restrictive Content Security Policy and are not cached.
|
|
14
14
|
|
|
15
|
-
|
|
15
|
+
Studio watches one project root — the working directory where it was launched. The header shows that root. `status --json` and `run --json` include `studioUrl` when a Studio process for that root is alive. That field identifies the running Studio; it is tokenless and returns 401 without a session. Do not write the session token to disk or into agent-visible output. Open the browser session from the tokenized URL printed by `ael studio`.
|
|
16
16
|
|
|
17
|
-
|
|
17
|
+
## Real-time sync
|
|
18
18
|
|
|
19
|
-
|
|
20
|
-
2. Fork a protected built-in by changing its ID.
|
|
21
|
-
3. Drag allowlisted node types from the library, use the global Add Node control, or use the contextual output `+` to create and connect a successor.
|
|
22
|
-
4. Move or multi-select nodes, drag an output port to connect existing nodes, and remove a connection from its midpoint control.
|
|
23
|
-
5. Edit a node Display Name inline without changing its stable Technical ID.
|
|
24
|
-
6. Import validated JSON through an explicit Replace, Merge with deterministic conflict renaming, or Cancel preview; export the current workflow as JSON.
|
|
25
|
-
7. Pan, lasso, zoom, fit, auto-layout, and use undo/redo without changing recipe semantics.
|
|
26
|
-
8. Review graph validation and hashes for every compatible mode.
|
|
27
|
-
9. Install only after explicit confirmation.
|
|
19
|
+
Studio does not poll `/api/live` on a timer. One `RunWatcher` per process recursively watches `.ai-engineering-loop/runs/` and `current.json`. On a change it re-reads only the changed artifact and pushes a typed event on `GET /api/events` (SSE).
|
|
28
20
|
|
|
29
|
-
|
|
21
|
+
Event types: `run.created`, `run.state`, `lifecycle`, `artifact.impact`, `artifact.drift`, `artifact.guard`, `artifact.verification`, `artifact.findings`, `artifact.verdict`, `artifact.receipt`, `artifact.delivery`, `current.changed`, `presence`. Each event carries `runId`, a monotonic `seq`, and a minimal payload.
|
|
30
22
|
|
|
31
|
-
|
|
23
|
+
The first SSE frame is `hello` with `mode` (`watch` or `poll`), `root`, `currentRunId`, and `seq`. The status pill shows `live: watch` or `live: poll`. Where `fs.watch` is unavailable (or `AEL_STUDIO_FS_WATCH=0`), Studio falls back to a 1 s stat-poll of the same files.
|
|
32
24
|
|
|
33
|
-
|
|
25
|
+
`Last-Event-ID` and `?after=<seq>` replay from an in-memory ring. A gap larger than the ring sends `resync`; the client then reloads `/api/bootstrap`. If the event stream ends or errors, the client reconnects with `setTimeout` backoff (500 ms, 1 s, 2 s, max 5 s) and resumes from the last `seq`; a failed resume falls back to `/api/bootstrap`. The Retry button still calls the same resync. The stream covers every run in the project. The UI follows `current.json` by default and can pin any run, including a classic run with no bound workflow and a run driven with `--run <id>` that is not current.
|
|
34
26
|
|
|
35
|
-
|
|
27
|
+
`GET /api/runs/:id/lifecycle` remains as a bounded REST ledger. `GET /api/runs/:id/lifecycle/stream` and `POST /api/import/*` return 404.
|
|
36
28
|
|
|
37
|
-
|
|
29
|
+
## Live run view
|
|
38
30
|
|
|
39
|
-
|
|
31
|
+
The default route is the live run view:
|
|
40
32
|
|
|
41
|
-
|
|
33
|
+
- stage rail goal → maker → verify → review → judge → deliver, with the current stage and heartbeat age;
|
|
34
|
+
- tier, mode, iteration, and budget;
|
|
35
|
+
- compliance chips for guard warns/blocks, impact drift, and receipt verification;
|
|
36
|
+
- pending questions, decisions, and approvals as the top banner;
|
|
37
|
+
- a merged timeline of lifecycle, activity, and execution events.
|
|
42
38
|
|
|
43
|
-
The
|
|
39
|
+
The Goal Contract stays a side sheet of this view (draft, review, freeze, unfreeze). Start with AI Agent still copies a run-bound handoff into the clipboard.
|
|
44
40
|
|
|
45
|
-
|
|
41
|
+
## Runs
|
|
46
42
|
|
|
47
|
-
|
|
43
|
+
**Runs** is a slim history list. Open a row to pin it in the live view as a read-only replay. Artifact inspection still accepts only regular files inside the owning run, rejects path traversal and symlinks, enforces a size bound, and redacts recognized secrets.
|
|
48
44
|
|
|
49
|
-
|
|
45
|
+
## Wiring existing nodes
|
|
50
46
|
|
|
51
|
-
|
|
47
|
+
Studio is read-only for workflow *structure*. Do not create, duplicate, delete, or import nodes here. Author recipes as files and use `recipe create|clone|install`.
|
|
52
48
|
|
|
53
|
-
|
|
49
|
+
**Wiring** keeps the canvas for edges between nodes that already exist: add, remove, or re-point a connection, plus fit, zoom, tidy/auto-layout, and undo/redo for those edge edits. Node moves are layout-only (`PUT /api/layout`) and never enter the graph hash.
|
|
54
50
|
|
|
55
|
-
-
|
|
56
|
-
- an animated connector leaving the running node;
|
|
57
|
-
- provider-recorded run token spend and configured limit;
|
|
58
|
-
- explicit approval and retry controls;
|
|
59
|
-
- a short redacted activity message reported by the host:
|
|
51
|
+
Every wiring change is validated by `POST /api/validate`, which is the same `lib/recipe.js` compiler and safety backbone (goal-contract → maker → verification → devil-advocate → judge → delivery/report, plus a human approval before delivery in ASSISTED). An edge set that breaks the backbone is rejected with the compiler's named error and cannot be saved. The server also rejects a recipe whose node set differs from the loaded base (`baseRecipeId`): same node ids, types, and node config; only edges, `dependsOn`, and layout may change. That failure is `STUDIO_STRUCTURE_READ_ONLY` and lists added, removed, and changed node ids. A builtin may still be installed as `<id>-local` when the nodes are unchanged. A valid change is installed only after the confirm dialog (`POST /api/install`). Built-in recipe ids stay reserved; Studio installs a project copy.
|
|
60
52
|
|
|
61
|
-
|
|
62
|
-
ai-engineering-loop node activity <node-id> \
|
|
63
|
-
--message "Comparing the failing test with the implementation"
|
|
64
|
-
```
|
|
53
|
+
## Wiring canvas
|
|
65
54
|
|
|
66
|
-
|
|
55
|
+
### Node tiles
|
|
67
56
|
|
|
68
|
-
|
|
57
|
+
Each existing node is one tile: a type icon, the node name, and exactly one status line. That line is the duration slot (`Passed · 6m 10s` when a duration was recorded). Stage and type text are not repeated as badges on the tile. A loop group draws a container around its nodes and labels it `iteration n / max` from the live loop state.
|
|
69
58
|
|
|
70
|
-
|
|
59
|
+
### Side panel
|
|
71
60
|
|
|
72
|
-
|
|
61
|
+
Click a tile to open the side panel. The tabs are Input, Output, Evidence, and Timeline. Evidence uses the same artifact inspection as the rest of Studio: path traversal and symlinks are rejected, size is bounded, and recognized secrets are redacted.
|
|
73
62
|
|
|
74
|
-
|
|
63
|
+
### Tooltip
|
|
75
64
|
|
|
76
|
-
|
|
65
|
+
Hover or keyboard focus shows a tooltip with the node name, type, stage, and status. The tooltip is separate from the tile, so the tile does not carry a second stage or type badge.
|
|
77
66
|
|
|
78
|
-
|
|
67
|
+
### Minimap and zoom controls
|
|
79
68
|
|
|
80
|
-
|
|
69
|
+
The minimap draws one rectangle per node. Zoom controls are zoom in (+), zoom out (−), fit, auto-layout, and the zoom percent. Fit and auto-layout keep every built-in recipe inside the canvas.
|
|
81
70
|
|
|
82
|
-
|
|
71
|
+
### Shortcuts
|
|
83
72
|
|
|
84
|
-
-
|
|
85
|
-
- **Export Handoff** downloads a machine-readable `.ael-handoff.json` bundle for another agent.
|
|
73
|
+
The `?` sheet lists the shortcuts: `F` fit, `L` auto-layout, `+` and `-` zoom, `0` reset zoom, arrow keys between nodes, Enter to open the side panel, Esc to close it, and undo/redo for wiring edits. `:focus-visible` outlines the canvas controls.
|
|
86
74
|
|
|
87
|
-
|
|
75
|
+
### Save bar
|
|
88
76
|
|
|
89
|
-
|
|
77
|
+
The dirty-only save bar stays hidden while the wiring is clean. The first edge change shows the change count, Valid or Invalid, Discard, and Save. Save opens the confirm dialog before `POST /api/install`. A backbone-breaking edge shows the compiler's named error and disables Save. Every edit is checked with `POST /api/validate`.
|
|
90
78
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
79
|
+
### Run summary card
|
|
80
|
+
|
|
81
|
+
When a run is DELIVERED, REPORTED, ESCALATED, or CANCELLED, the canvas shows a run summary card with duration, tests, findings, drift, receipt, and tokens. Tokens read `unknown` when the usage ledger has none.
|
|
82
|
+
|
|
83
|
+
Workflow structure on this canvas stays read-only. There is no control to create, delete, rename, duplicate, or import nodes. Rewiring only connects nodes that already exist. Live tile and summary updates come from `GET /api/events`.
|
|
84
|
+
|
|
85
|
+
## Stage 8 delivery adapter
|
|
95
86
|
|
|
96
|
-
|
|
87
|
+
Delivery nodes still expose the project-level Stage 8 adapter in `.ai-engineering-loop/adapter.md`. `POST /api/adapter/select` can select the shipped `standard`, `github`, `gitlab`, or `dot` adapters. The browser never accepts arbitrary adapter code or executes delivery commands.
|
|
97
88
|
|
|
98
89
|
## Trust boundary
|
|
99
90
|
|
|
100
|
-
Studio improves
|
|
91
|
+
Studio improves observability; it does not weaken runtime gates:
|
|
101
92
|
|
|
102
93
|
- graph validation does not imply execution permission;
|
|
103
94
|
- local execution requires exactly one permitted trigger/root and an audited frozen Goal;
|
|
104
95
|
- command nodes remain disabled;
|
|
105
|
-
- Custom Agents wait for an external operator and cannot grant undeclared authority;
|
|
106
96
|
- approvals remain explicit state transitions;
|
|
107
97
|
- connector edits cannot bypass the required safety backbone;
|
|
108
98
|
- Run History cannot mutate persisted runs or artifacts;
|
|
109
99
|
- delivery adapters cannot run before Judge `PASS`;
|
|
110
100
|
- token data comes only from the provider-recorded usage ledger;
|
|
111
|
-
-
|
|
101
|
+
- the event stream never exposes file contents outside the run-dir artifacts already served.
|