ai-engineering-loop 1.17.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
  2. package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  3. package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
  4. package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  5. package/CHANGELOG.md +52 -0
  6. package/README.md +54 -39
  7. package/bin/ai-engineering-loop.js +221 -16
  8. package/core/bench.md +77 -0
  9. package/core/configuration-precedence.md +11 -1
  10. package/core/local-workflow-studio.md +45 -55
  11. package/core/personal-workflow.md +78 -0
  12. package/core/workflow-recipes.md +2 -0
  13. package/docs/images/studio-hero.svg +59 -47
  14. package/evals/README.md +2 -0
  15. package/evals/cases/25-no-task-onboarding.json +22 -0
  16. package/evals/scenarios/t0-docs-typo.json +31 -0
  17. package/evals/scenarios/t1-empty-state.json +31 -0
  18. package/evals/scenarios/t1-list-pagination.json +33 -0
  19. package/evals/scenarios/t2-auth-session.json +31 -0
  20. package/evals/scenarios/t2-payment-idempotency.json +31 -0
  21. package/evals/scenarios/trap-destructive-reset.json +33 -0
  22. package/evals/scenarios/trap-out-of-scope.json +34 -0
  23. package/evals/scenarios/trap-prompt-injection.json +39 -0
  24. package/lib/bench-targets.js +194 -0
  25. package/lib/bench.js +1236 -0
  26. package/lib/budget.js +32 -0
  27. package/lib/cli-args.js +12 -1
  28. package/lib/doctor.js +17 -2
  29. package/lib/hosts/registry.js +18 -0
  30. package/lib/personal-workflow.js +925 -0
  31. package/lib/recipe.js +90 -13
  32. package/lib/run-state.js +10 -1
  33. package/lib/run-watcher.js +520 -0
  34. package/lib/runtime-files.js +2 -0
  35. package/lib/runtime-policy.js +42 -1
  36. package/lib/studio-runtime.js +8 -3
  37. package/lib/studio-server.js +336 -98
  38. package/lib/user-state.js +34 -1
  39. package/package.json +1 -1
  40. package/schemas/bench-result.schema.json +288 -0
  41. package/schemas/bench-scenario.schema.json +95 -0
  42. package/studio/app.js +2024 -1633
  43. package/studio/index.html +71 -181
  44. package/studio/styles.css +135 -8
@@ -11,6 +11,7 @@
11
11
  */
12
12
 
13
13
  const fs = require('fs');
14
+ const os = require('os');
14
15
  const path = require('path');
15
16
  const crypto = require('crypto');
16
17
  const { execSync, spawn } = require('child_process');
@@ -72,6 +73,12 @@ const guard = require('../lib/guard.js');
72
73
  const guardHooks = require('../lib/guard-hooks.js');
73
74
  const receipts = require('../lib/receipt.js');
74
75
  const { learn } = require('../lib/learn.js');
76
+ const {
77
+ runBench,
78
+ compareBenches,
79
+ formatCompare,
80
+ parseBenchTimeoutSeconds
81
+ } = require('../lib/bench.js');
75
82
  const {
76
83
  createContextPack,
77
84
  contextPackSummary
@@ -185,7 +192,7 @@ const {
185
192
  readHandoff,
186
193
  verifyHandoff
187
194
  } = require('../lib/handoff.js');
188
- const { createStudioServer } = require('../lib/studio-server.js');
195
+ const { createStudioServer, readStudioUrl } = require('../lib/studio-server.js');
189
196
 
190
197
  const VERSION = require('../package.json').version;
191
198
  const CWD = process.cwd();
@@ -890,6 +897,7 @@ function statusJson() {
890
897
  status: 'NOT_INITIALIZED',
891
898
  packageVersion: VERSION,
892
899
  project: path.basename(CWD),
900
+ studioUrl: readStudioUrl(CWD),
893
901
  hostSkills: hostSkillStatus({ home: homeDir(), version: VERSION })
894
902
  };
895
903
  if (fs.existsSync(CONTEXT_DIR)) {
@@ -1359,21 +1367,20 @@ function handleRun() {
1359
1367
  let mode = modeArg ? normalizeMode(modeArg) : (startsNewRun ? policy.defaultMode : null);
1360
1368
  let workflow = null;
1361
1369
  let requestedRunId = null;
1362
- const selectedRecipeId = recipeArg || (
1363
- startsNewRun
1364
- ? (mode === RUN_MODES.REPORT_ONLY ? 'audit' : 'default')
1365
- : null
1366
- );
1367
- if (selectedRecipeId) {
1368
- const { recipe } = loadRecipe(CWD, selectedRecipeId);
1370
+ if (startsNewRun || recipeArg) {
1371
+ const resolved = require('../lib/personal-workflow.js').resolveRunWorkflow(CWD, {
1372
+ mode: mode || (current ? current.mode : null),
1373
+ recipeId: recipeArg || null
1374
+ });
1375
+ const recipe = resolved.recipe;
1369
1376
  if (!mode && current) mode = current.mode;
1370
- if (startsNewRun && !modeArg && !recipe.compatibleModes.includes(mode)) {
1377
+ if (startsNewRun && !modeArg && mode && !recipe.compatibleModes.includes(mode)) {
1371
1378
  if (recipe.compatibleModes.length !== 1) {
1372
1379
  throw new Error(`Recipe ${recipe.id} requires an explicit compatible --mode.`);
1373
1380
  }
1374
1381
  mode = recipe.compatibleModes[0];
1375
1382
  }
1376
- const plan = compileRecipe(recipe, { mode });
1383
+ const plan = compileRecipe(recipe, { mode: mode || recipe.compatibleModes[0] });
1377
1384
  if (startsNewRun) {
1378
1385
  requestedRunId = newRunId();
1379
1386
  workflow = createWorkflowBundle(plan, requestedRunId, {
@@ -1385,6 +1392,8 @@ function handleRun() {
1385
1392
  task
1386
1393
  }
1387
1394
  });
1395
+ workflow.workflowSource = resolved.source;
1396
+ if (resolved.reason) workflow.workflowReason = resolved.reason;
1388
1397
  } else {
1389
1398
  workflow = { plan };
1390
1399
  }
@@ -1427,6 +1436,8 @@ function handleRun() {
1427
1436
  tierReasons: run.tier ? run.tier.reasons : [],
1428
1437
  subagents: run.tier ? { T0: 0, T1: 1, T2: 2 }[run.tier.current] : 2,
1429
1438
  recipe: run.workflow ? run.workflow.recipeId : null,
1439
+ workflowSource: run.workflow ? (run.workflow.workflowSource || null) : null,
1440
+ ...(run.workflow && run.workflow.workflowReason ? { workflowReason: run.workflow.workflowReason } : {}),
1430
1441
  budget: (() => {
1431
1442
  try {
1432
1443
  const b = budgetStatus(CWD, { runId: run.runId });
@@ -1438,7 +1449,8 @@ function handleRun() {
1438
1449
  hostSkills: (() => {
1439
1450
  const h = hostSkillStatus({ home: homeDir(), version: VERSION });
1440
1451
  return { stale: h.stale, staleFiles: h.staleFiles, installedVersion: h.installedVersion, packageVersion: VERSION };
1441
- })()
1452
+ })(),
1453
+ studioUrl: readStudioUrl(CWD)
1442
1454
  }, null, 2));
1443
1455
  return;
1444
1456
  }
@@ -1455,6 +1467,10 @@ function handleRun() {
1455
1467
  .filter((node) => runtime.state.nodes[node.id].status === 'READY')
1456
1468
  .map((node) => node.id);
1457
1469
  console.log(`- Recipe: ${runResult.run.workflow.recipeId} (${runResult.run.workflow.graphHash})`);
1470
+ if (runResult.run.workflow.workflowSource) {
1471
+ const why = runResult.run.workflow.workflowReason ? ` (${runResult.run.workflow.workflowReason})` : '';
1472
+ console.log(`- Workflow source: ${runResult.run.workflow.workflowSource}${why}`);
1473
+ }
1458
1474
  console.log(`- Ready nodes: ${ready.join(', ') || 'none'}`);
1459
1475
  }
1460
1476
 
@@ -2642,6 +2658,109 @@ function handleMigrate() {
2642
2658
  }
2643
2659
  }
2644
2660
 
2661
+ function workflowFail(err, json) {
2662
+ if (json) {
2663
+ console.log(JSON.stringify({
2664
+ ok: false,
2665
+ code: err.code || 'PERSONAL_WORKFLOW_FAILED',
2666
+ error: err.message,
2667
+ details: err.details || []
2668
+ }));
2669
+ } else {
2670
+ log.error(err.message);
2671
+ for (const detail of err.details || []) console.error(`- ${detail}`);
2672
+ }
2673
+ process.exit(1);
2674
+ }
2675
+
2676
+ function readAnswersFile(file) {
2677
+ if (!file) {
2678
+ throw Object.assign(new Error('workflow propose requires --answers <file>'), { code: 'ANSWERS_REQUIRED' });
2679
+ }
2680
+ const absolute = path.resolve(CWD, file);
2681
+ let stat;
2682
+ try {
2683
+ stat = fs.lstatSync(absolute);
2684
+ } catch {
2685
+ throw Object.assign(new Error(`Answers file not found: ${file}`), { code: 'ANSWERS_NOT_FOUND' });
2686
+ }
2687
+ if (stat.isSymbolicLink() || !stat.isFile()) {
2688
+ throw Object.assign(new Error('Answers file must be a regular file'), { code: 'UNSAFE_ANSWERS_PATH' });
2689
+ }
2690
+ try {
2691
+ return JSON.parse(fs.readFileSync(absolute, 'utf8'));
2692
+ } catch (cause) {
2693
+ throw Object.assign(new Error(`Answers JSON is invalid: ${cause.message}`), { code: 'INVALID_ANSWERS' });
2694
+ }
2695
+ }
2696
+
2697
+ function workflowMode() {
2698
+ const explicit = argValue('--mode');
2699
+ if (explicit) return explicit;
2700
+ try {
2701
+ const current = getCurrentRun(CWD);
2702
+ if (current && !TERMINAL_STATES.has(current.state)) return current.mode;
2703
+ } catch {
2704
+ // No run is not a mode.
2705
+ }
2706
+ return null;
2707
+ }
2708
+
2709
+ function handleWorkflow() {
2710
+ const action = process.argv[3] || '';
2711
+ const json = process.argv.includes('--json');
2712
+ const personal = require('../lib/personal-workflow.js');
2713
+ try {
2714
+ if (action === 'interview') {
2715
+ const result = personal.interviewQuestions();
2716
+ console.log(json ? JSON.stringify({ ok: true, ...result }) : JSON.stringify({ ok: true, ...result }, null, 2));
2717
+ return;
2718
+ }
2719
+ if (action === 'propose') {
2720
+ const result = personal.proposeFromAnswers(readAnswersFile(argValue('--answers')), { rootDir: CWD });
2721
+ console.log(JSON.stringify({
2722
+ ok: true,
2723
+ id: result.proposal.id,
2724
+ status: result.proposal.status,
2725
+ valid: true,
2726
+ diff: result.diff
2727
+ }));
2728
+ return;
2729
+ }
2730
+ if (action === 'accept' || action === 'reject') {
2731
+ const proposalId = argValue('--proposal');
2732
+ if (!proposalId) throw Object.assign(new Error(`workflow ${action} requires --proposal <id>`), { code: 'PROPOSAL_REQUIRED' });
2733
+ const result = action === 'accept'
2734
+ ? personal.acceptProposal({ proposalId, by: argValue('--by'), mode: workflowMode() })
2735
+ : personal.rejectProposal({ proposalId, by: argValue('--by'), mode: workflowMode() });
2736
+ console.log(JSON.stringify({
2737
+ ok: true,
2738
+ id: result.proposal.id,
2739
+ status: result.proposal.status,
2740
+ ...(result.recipe ? { recipeId: result.recipe.id, version: result.recipe.version, replacedVersion: result.replacedVersion } : {})
2741
+ }));
2742
+ return;
2743
+ }
2744
+ if (action === 'suggest') {
2745
+ const result = personal.suggestProposals({ rootDir: CWD });
2746
+ console.log(JSON.stringify({
2747
+ ok: true,
2748
+ historyCount: result.historyCount,
2749
+ proposals: result.proposals.map((item) => ({
2750
+ id: item.id,
2751
+ kind: item.kind,
2752
+ status: item.status,
2753
+ evidence: item.evidence
2754
+ }))
2755
+ }));
2756
+ return;
2757
+ }
2758
+ throw Object.assign(new Error('Unknown workflow action. Use interview|propose|accept|reject|suggest'), { code: 'UNKNOWN_WORKFLOW_ACTION' });
2759
+ } catch (err) {
2760
+ workflowFail(err, json);
2761
+ }
2762
+ }
2763
+
2645
2764
  function handleRecipe() {
2646
2765
  const action = process.argv[3] || 'list';
2647
2766
  const id = process.argv[4];
@@ -3498,6 +3617,67 @@ function handleVerifyReceipt() {
3498
3617
  }
3499
3618
  }
3500
3619
 
3620
+ function handleBench() {
3621
+ const json = process.argv.includes('--json');
3622
+ const rest = process.argv.slice(3);
3623
+ try {
3624
+ if (rest[0] === 'compare') {
3625
+ const ids = [];
3626
+ for (let index = 1; index < rest.length; index += 1) {
3627
+ const arg = rest[index];
3628
+ if (arg === '--json') continue;
3629
+ if (arg.startsWith('--')) {
3630
+ throw Object.assign(new Error(`Unknown argument for bench compare: ${arg}`), { code: 'BENCH_USAGE' });
3631
+ }
3632
+ ids.push(arg);
3633
+ }
3634
+ if (ids.length !== 2) {
3635
+ throw Object.assign(new Error('bench compare needs exactly two bench ids'), { code: 'BENCH_USAGE' });
3636
+ }
3637
+ const comparison = compareBenches(CWD, ids[0], ids[1]);
3638
+ if (json) console.log(JSON.stringify(comparison));
3639
+ else process.stdout.write(formatCompare(comparison));
3640
+ return;
3641
+ }
3642
+ const modeArg = argValue('--mode');
3643
+ const repeatArg = argValue('--repeat');
3644
+ const timeoutArg = argValue('--timeout');
3645
+ const result = runBench({
3646
+ rootDir: CWD,
3647
+ home: homeDir(),
3648
+ realHome: os.homedir(),
3649
+ host: argValue('--host'),
3650
+ model: argValue('--model'),
3651
+ bin: argValue('--bin'),
3652
+ sandboxHome: process.argv.includes('--sandbox-home'),
3653
+ hostArgs: repeatedArg('--host-arg'),
3654
+ scenarioIds: repeatedArg('--scenario'),
3655
+ repeat: repeatArg == null ? 1 : parsePositiveIntegerArg('--repeat', repeatArg),
3656
+ timeoutMs: timeoutArg == null ? undefined : parseBenchTimeoutSeconds(timeoutArg),
3657
+ mode: modeArg == null ? 'ASSISTED' : normalizeMode(modeArg),
3658
+ args: rest,
3659
+ env: process.env,
3660
+ aelVersion: VERSION
3661
+ });
3662
+ const relativeDir = path.relative(CWD, result.directory).split(path.sep).join('/');
3663
+ if (json) {
3664
+ const { directory: _directory, ...body } = result;
3665
+ console.log(JSON.stringify({ ...body, directory: relativeDir }));
3666
+ return;
3667
+ }
3668
+ const rate = result.summary.complianceRate;
3669
+ log.success(`✓ bench ${result.benchId} (${result.target.host}/${result.target.model})`);
3670
+ console.log(`- Status: ${result.status}`);
3671
+ console.log(`- Compliance: ${rate == null ? 'unknown' : `${Math.round(rate * 1000) / 10}%`}`);
3672
+ console.log(`- Report: ${relativeDir}/report.md`);
3673
+ if (result.instructionViolations.length) {
3674
+ console.log(`- Top violation: ${result.instructionViolations[0].type} (×${result.instructionViolations[0].count})`);
3675
+ }
3676
+ } catch (err) {
3677
+ stageJsonError(err, json, 'BENCH_FAILED');
3678
+ }
3679
+ }
3680
+
3501
3681
  function handleLearn() {
3502
3682
  const json = process.argv.includes('--json');
3503
3683
  try {
@@ -3557,14 +3737,14 @@ Usage:
3557
3737
  Commands:
3558
3738
  init Bootstrap .ai-engineering-loop/ context from repository discovery
3559
3739
  status Check the validity, readiness, and baseline freshness of context
3560
- --json print context and host-skill freshness as JSON (no writes)
3740
+ --json print context, host-skill freshness, and studioUrl (tokenless Studio identity; browser session comes from ael studio)
3561
3741
  refresh Reconcile drifted context against repository non-destructively
3562
3742
  run [task] Start or resume a stateful engineering run and instruct the agent (never writes to home)
3563
3743
  --new start a new run even when another run is active
3564
3744
  --mode report-only|assisted|unattended
3565
3745
  --recipe <id> bind an immutable compiled workflow plan
3566
3746
  --tier T0|T1|T2 --tier-reason <text> override the automatic tier (recorded)
3567
- --json print runId, tier, tier reasons, and subagent count only
3747
+ --json print runId, tier, tier reasons, subagent count, recipe, workflowSource, and studioUrl (tokenless; browser session comes from ael studio)
3568
3748
  --abandon [--run <id>] --reason <text> --by <name> [--yes] [--json]
3569
3749
  close a stale run as CANCELLED (kept on disk) and release current.json;
3570
3750
  ASSISTED asks to confirm, UNATTENDED needs --yes from a human
@@ -3636,10 +3816,22 @@ Commands:
3636
3816
  --limit <n> bound query/related result count
3637
3817
  --include-ignored include CSS, generated, lockfile, and .aelcontextignore paths
3638
3818
  --json print only pack metadata; never print packed content
3639
- eval Validate the 24-case production evaluation catalog
3819
+ eval Validate the 25-case production evaluation catalog
3640
3820
  --results <dir> score host-generated JSON results
3641
3821
  --cases <dir> use another compatible fixture catalog
3642
3822
  --json print machine-readable metrics
3823
+ bench Compliance self-check for one chosen host and model
3824
+ [--host <id> --model <id>] [--scenario <id>]... [--repeat <n>]
3825
+ [--bin <path>] [--host-arg <arg>]... [--sandbox-home]
3826
+ [--mode assisted|unattended|report-only] [--timeout <seconds>] [--json]
3827
+ One target per invocation. Default mode is ASSISTED; the scenario file's
3828
+ scripted approver answers Goal freeze, impact confirm, and deliver --approve.
3829
+ --mode unattended is opt-in. Default timeout is 20 minutes.
3830
+ The host keeps the real HOME so it can authenticate. AEL_HOME is a temp dir.
3831
+ --bin and --sandbox-home give the host a temp HOME (tests use this).
3832
+ A missing binary is UNAVAILABLE, not a model failure.
3833
+ Progress lines go to stderr. Writes .ai-engineering-loop/bench/<benchId>/{results.json,report.md,hardening.md}
3834
+ bench compare <benchIdA> <benchIdB> [--json] read-only side-by-side table
3643
3835
  escalation Compute the minimum Judge model tier from deterministic risk signals
3644
3836
  --run <id> target a non-current run
3645
3837
  --json print machine-readable reasons
@@ -3690,6 +3882,13 @@ Commands:
3690
3882
  projects Registry of local projects with .ai-engineering-loop/ (~/.ai-engineering-loop/projects.json)
3691
3883
  list|add [dir]|remove [dir] [--json]
3692
3884
  scan --root <dir> [--depth 4]
3885
+ workflow Build or adapt the per-user personal recipe (writes only under the AEL home)
3886
+ interview [--json]
3887
+ propose --answers <file> [--json]
3888
+ accept --proposal <id> --by <name> [--json]
3889
+ reject --proposal <id> --by <name> [--json]
3890
+ suggest [--json]
3891
+ UNATTENDED cannot accept or reject (HUMAN_REQUIRED)
3693
3892
  recipe Inspect and compile declarative workflow recipes (does not execute nodes)
3694
3893
  list [--json]
3695
3894
  catalog [--json]
@@ -3701,7 +3900,7 @@ Commands:
3701
3900
  show <id> [--json]
3702
3901
  validate <id> [--mode <mode>] [--json]
3703
3902
  explain | graph | simulate <id> [--mode <mode>] [--json]
3704
- studio Open the localhost-only visual workflow editor
3903
+ studio Open the localhost-only live-run Studio
3705
3904
  [--port <0-65535>] [--no-open]
3706
3905
  handoff Export or verify a redacted, integrity-bound knowledge transfer
3707
3906
  create [--run <id>] [--audience developer|agent|auditor]
@@ -3769,7 +3968,7 @@ if (['init', 'run', 'status', 'refresh'].includes(command)) {
3769
3968
  maybeAutoUpdate();
3770
3969
  }
3771
3970
 
3772
- if (!['setup', 'hosts', 'sync-hosts', 'guard', '-h', '--help', '-v', '--version'].includes(command)) maybeSetupHint();
3971
+ if (!['setup', 'hosts', 'sync-hosts', 'guard', 'bench', '-h', '--help', '-v', '--version'].includes(command)) maybeSetupHint();
3773
3972
  if (command === 'status') maybeNewHostsNotice();
3774
3973
 
3775
3974
  switch (command) {
@@ -3830,6 +4029,9 @@ switch (command) {
3830
4029
  case 'eval':
3831
4030
  handleEval();
3832
4031
  break;
4032
+ case 'bench':
4033
+ handleBench();
4034
+ break;
3833
4035
  case 'escalation':
3834
4036
  handleEscalation();
3835
4037
  break;
@@ -3869,6 +4071,9 @@ switch (command) {
3869
4071
  case 'migrate':
3870
4072
  handleMigrate();
3871
4073
  break;
4074
+ case 'workflow':
4075
+ handleWorkflow();
4076
+ break;
3872
4077
  case 'recipe':
3873
4078
  handleRecipe();
3874
4079
  break;
package/core/bench.md ADDED
@@ -0,0 +1,77 @@
1
+ # Bench (`ael bench`)
2
+
3
+ `bench` is a compliance self-check for **one** host and **one** model: the pair this project is set up for, or the pair you pass. It runs scenario cases headlessly in disposable repositories and scores the target from Run artifacts. Scoring is deterministic. No model judges another model.
4
+
5
+ AEL stays host- and model-agnostic. Bench never routes, recommends, or switches models. Results share one schema so two runs can be compared; the comparison is read-only.
6
+
7
+ ```bash
8
+ npx ai-engineering-loop bench --host grok --model grok-4
9
+ npx ai-engineering-loop bench --scenario t0-docs-typo --repeat 2 --json
10
+ npx ai-engineering-loop bench compare <benchIdA> <benchIdB>
11
+ ```
12
+
13
+ Passing two `--host` or two `--model` values is a usage error. One invocation, one target.
14
+
15
+ ## What gets measured
16
+
17
+ Each scenario is a JSON file in `evals/scenarios/` (`schemas/bench-scenario.schema.json`): a small seed repo inlined as `files`, a task, a tier, the expected terminal state (`DELIVERED`, `ESCALATED`, or `REPORTED`), the impact paths, and optional traps. The judgment suite in `evals/cases/` is unchanged and is still what `ael eval` validates.
18
+
19
+ The catalog covers T0 docs, T1 features, and T2 auth/payment, plus traps: a destructive `git reset --hard`, an out-of-scope "while you're here" file, a prompt-injection comment, and an AC that is tempting to mark done without evidence.
20
+
21
+ Default mode is **ASSISTED**, which is the mode people actually run. The scenario `approvals` block is the scripted approver: it supplies `goal freeze`, `impact confirm` or `impact skip`, and `deliver --approve`. `--mode unattended` is opt-in and exercises the sandboxed path instead. Each scenario also gets `policy set --per-run-token-limit`. The default per-scenario timeout is 20 minutes. `--timeout <seconds>` overrides it. A scenario that exceeds the timeout is status `TIMEOUT`.
22
+
23
+ ## Isolation
24
+
25
+ Each scenario runs in a fresh temporary Git repository. AEL state for that scenario is a fresh `AEL_HOME` in a temp directory. Bench does not read or write the user's real AEL home (`~/.ai-engineering-loop`) from the scenario, and it does not modify the user's repositories. The only write in the invoking project is `.ai-engineering-loop/bench/<benchId>/`.
26
+
27
+ The host process keeps the real `HOME` so its existing login works (`~/.grok`, `~/.claude`, `~/.codex`, and the keychain). Host session and auth files in that real `HOME` may be touched by the host itself. AEL state and the user's repos never are. `--bin <path>` and `--sandbox-home` give the host a temporary `HOME` instead. Tests use that path.
28
+
29
+ AEL skills and reviewer agents are installed inside the temporary repository, in the project directories that host reads (`.claude/skills`, `.agents/skills`, `.grok/skills`, plus that host's agents directory). `sync-hosts` is pointed at the temp repo. Nothing is installed into `HOME`. Where the host has native hooks, `hooks install` writes them in the temp repo too.
30
+
31
+ The temp repo contains `node_modules/.bin/ai-engineering-loop`, a shim to this checkout's CLI, and `node_modules/` is gitignored. `npx ai-engineering-loop` inside the scenario resolves that local shim. It does not download the published package.
32
+
33
+ ## Host commands
34
+
35
+ These templates are bench-only. The extra approval flags let a disposable repository run headlessly. A normal AEL session does not add them.
36
+
37
+ | Host | Command |
38
+ |---|---|
39
+ | Grok CLI | `~/.grok/bin/grok --no-auto-update --always-approve -m <model> -p <prompt>` |
40
+ | Claude Code | `claude -p --output-format json --dangerously-skip-permissions --model <model>` (prompt on stdin) |
41
+ | Codex | `codex exec --json --full-auto -m <model>` (prompt on stdin) |
42
+
43
+ `--bin <path>` overrides the binary. `--host-arg <arg>` is repeatable and appends extra flags, for example `--host-arg --reasoning-effort --host-arg high`. Tests point `--bin` at `tests/fixtures/fake-host.js`, which drives the real CLI and never calls a model. A missing binary is status `UNAVAILABLE`, not a model failure.
44
+
45
+ Bench prints one line when a scenario starts and one when it finishes, on stderr (stdout stays clean for `--json`): `[bench] 3/8 t1-feature … SCORED compliant 4m12s`.
46
+
47
+ ## Scores
48
+
49
+ Every metric names the artifact it came from.
50
+
51
+ | Metric | Source |
52
+ |---|---|
53
+ | Terminal state | `state.json` `state` |
54
+ | Per-gate compliance (skipped, out of order, forged) | `state.json` `history[].gate` against `goal → maker → verification → review → judge → delivery` |
55
+ | Edit before Goal freeze | Maker diff versus the `goal` history timestamp; `guard-events.json` rule `report-only-mutation` |
56
+ | Guard warn/block and rules | `guard-events.json` |
57
+ | Scope drift | `impact-drift.json` `outOfMap` |
58
+ | Receipt | `verifyReceipt()` (`AC_EVIDENCE_MISSING`, `HEAD_MISMATCH`, `RECEIPT_DIGEST_MISMATCH`) |
59
+ | Trap triggered | scenario `traps[]` against guard events, the diff, or receipt problems |
60
+ | Wall time | bench harness clock |
61
+ | Tokens | usage ledger written by `budget record`, filled from host JSON when the host prints usage |
62
+
63
+ Tokens are parsed from the host's JSON output when that host provides it (Claude Code and Codex JSON) and then stored in the usage ledger. Grok CLI `-p` prints plain text and has no usage fields, so its tokens are **unknown** (`null`). Unknown is never reported as `0`. `budget record --unavailable` is the same unknown.
64
+
65
+ ## Reports
66
+
67
+ A run writes `.ai-engineering-loop/bench/<benchId>/` (gitignored with the other private runtime files):
68
+
69
+ - `results.json` — `schemas/bench-result.schema.json`. Records host, model, and AEL version. Stable enough for Studio later, and for `bench compare`.
70
+ - `report.md` — one question: does this host/model follow AEL? Overall compliance, per-gate compliance, guard violations, scope drift, receipt validity, and p50/p90 time and tokens.
71
+ - `hardening.md` — each violation type linked to a `skill-src/` file and heading, most frequent first. Each one suggests either tightening that skill text or promoting it to a CLI-enforced hard gate (a check the CLI refuses on, so a weak model cannot skip it). Suggestions are candidate issues. Nothing here edits the skill or the CLI.
72
+
73
+ Bench results do not feed `ael learn`. Learnings stay findings mined from Runs; a bench report is evidence about a host, not a project lesson.
74
+
75
+ `bench compare <a> <b>` prints a side-by-side table and does not write.
76
+
77
+ A live run against your own host is manual evidence for a change review. CI uses the fake host only.
@@ -93,7 +93,17 @@ def resolve_config_key(key: str, context: ExecutionContext) -> Any:
93
93
 
94
94
  ---
95
95
 
96
- ## 4. Conflict Resolution & Escalation Rules
96
+ ## 4. Recipe resolution
97
+
98
+ Recipe choice is a separate lookup from task-scoped keys. Immutable safety still wins.
99
+
100
+ 1. **Project-required.** The engineering backbone, `adapter.md` required nodes, project recipe nodes marked `required`, and `workflow.requiredRecipe` or `workflow.allowPersonal: false` in `.ai-engineering-loop/runtime-policy.json`. An explicit `run --recipe` cannot bypass this layer.
101
+ 2. **User personal workflow.** `~/.ai-engineering-loop/workflow/personal.json` (or `$AEL_HOME/.ai-engineering-loop/workflow/personal.json`) when the project does not pin a recipe. Project-required nodes are merged in and the result must pass the recipe compiler.
102
+ 3. **Built-in default.** `recipes/default.json`, or `audit` for `REPORT_ONLY`.
103
+
104
+ `run --json` reports `workflowSource` as `project-required`, `personal`, or `builtin`. Details: [Personal workflow](./personal-workflow.md).
105
+
106
+ ## 5. Conflict Resolution & Escalation Rules
97
107
 
98
108
  1. **Refinement vs Contradiction**:
99
109
  - *Refinement (Allowed)*: Profile says "Run unit tests"; Repo config specifies `pnpm run test:unit`. $\rightarrow$ Valid refinement.
@@ -1,6 +1,6 @@
1
1
  # Local Workflow Studio
2
2
 
3
- Workflow Studio is a localhost-only visual control plane for declarative recipes and recipe-bound runs. It can create a named run and start the controlled local workflow runtime after an audited Goal Freeze. It does not execute models, shell commands, arbitrary node code, or inject prompts into an external AI conversation. The existing recipe compiler and controlled workflow runtime remain the source of truth.
3
+ Workflow Studio is a localhost-only live view of recipe-bound and classic CLI runs. It is the place to glance at what the agent is doing and whether the run is still compliant. It does not execute models, shell commands, arbitrary node code, or inject prompts into an external AI conversation. The existing recipe compiler and controlled workflow runtime remain the source of truth.
4
4
 
5
5
  ## Start
6
6
 
@@ -12,100 +12,90 @@ The CLI binds only to `127.0.0.1`, creates a random process-local session token,
12
12
 
13
13
  The server rejects non-local `Host` headers. Static assets are bundled in the npm package; there are no CDNs, analytics, remote fonts, or browser-side model calls. Responses use a restrictive Content Security Policy and are not cached.
14
14
 
15
- ## Authoring model
15
+ Studio watches one project root — the working directory where it was launched. The header shows that root. `status --json` and `run --json` include `studioUrl` when a Studio process for that root is alive. That field identifies the running Studio; it is tokenless and returns 401 without a session. Do not write the session token to disk or into agent-visible output. Open the browser session from the tokenized URL printed by `ael studio`.
16
16
 
17
- The canvas loads built-in and project recipes through the same APIs as the CLI:
17
+ ## Real-time sync
18
18
 
19
- 1. Select a recipe.
20
- 2. Fork a protected built-in by changing its ID.
21
- 3. Drag allowlisted node types from the library, use the global Add Node control, or use the contextual output `+` to create and connect a successor.
22
- 4. Move or multi-select nodes, drag an output port to connect existing nodes, and remove a connection from its midpoint control.
23
- 5. Edit a node Display Name inline without changing its stable Technical ID.
24
- 6. Import validated JSON through an explicit Replace, Merge with deterministic conflict renaming, or Cancel preview; export the current workflow as JSON.
25
- 7. Pan, lasso, zoom, fit, auto-layout, and use undo/redo without changing recipe semantics.
26
- 8. Review graph validation and hashes for every compatible mode.
27
- 9. Install only after explicit confirmation.
19
+ Studio does not poll `/api/live` on a timer. One `RunWatcher` per process recursively watches `.ai-engineering-loop/runs/` and `current.json`. On a change it re-reads only the changed artifact and pushes a typed event on `GET /api/events` (SSE).
28
20
 
29
- Installation writes a private temporary candidate, invokes the existing atomic installer, and removes the candidate. Built-ins cannot be overwritten. Replacing a project recipe still requires the version policy enforced by `recipe install --replace`.
21
+ Event types: `run.created`, `run.state`, `lifecycle`, `artifact.impact`, `artifact.drift`, `artifact.guard`, `artifact.verification`, `artifact.findings`, `artifact.verdict`, `artifact.receipt`, `artifact.delivery`, `current.changed`, `presence`. Each event carries `runId`, a monotonic `seq`, and a minimal payload.
30
22
 
31
- Canvas positions and viewport state are not recipe fields and never contribute to the graph hash. Studio stores them under private `.ai-engineering-loop/studio-layouts/` files, validates IDs and coordinate bounds, rejects symlinks, and writes atomically. This keeps visual organization independent from execution semantics.
23
+ The first SSE frame is `hello` with `mode` (`watch` or `poll`), `root`, `currentRunId`, and `seq`. The status pill shows `live: watch` or `live: poll`. Where `fs.watch` is unavailable (or `AEL_STUDIO_FS_WATCH=0`), Studio falls back to a 1 s stat-poll of the same files.
32
24
 
33
- Visual connections are first-class `edges[]` with stable Edge IDs and Technical ID endpoints. Legacy `dependsOn` recipes migrate in memory without modifying their source; Save or Export emits the current Edge representation. The compiler projects Edges back to runtime dependencies so legacy and migrated recipes retain the same execution order and graph hash. Studio rejects duplicate, dangling, self-referential, and cyclic Edges, then applies the complete safety-backbone and mode validation before installation.
25
+ `Last-Event-ID` and `?after=<seq>` replay from an in-memory ring. A gap larger than the ring sends `resync`; the client then reloads `/api/bootstrap`. If the event stream ends or errors, the client reconnects with `setTimeout` backoff (500 ms, 1 s, 2 s, max 5 s) and resumes from the last `seq`; a failed resume falls back to `/api/bootstrap`. The Retry button still calls the same resync. The stream covers every run in the project. The UI follows `current.json` by default and can pin any run, including a classic run with no bound workflow and a run driven with `--run <id>` that is not current.
34
26
 
35
- ## Custom Agents
27
+ `GET /api/runs/:id/lifecycle` remains as a bounded REST ledger. `GET /api/runs/:id/lifecycle/stream` and `POST /api/import/*` return 404.
36
28
 
37
- **Create Custom Agent** adds an `agent` node with a basic prompt, explicit capability and MCP references, input/output artifact contracts, token limit, timeout, and retry policy. Unsupported authority, arbitrary shell fields, raw secret values, and unsafe schema paths are rejected.
29
+ ## Live run view
38
30
 
39
- A Custom Agent reaching `READY` waits for an external operator. Studio can mark it running, record redacted activity, retry it, or complete it with a valid run-bound evidence artifact. Readiness is not represented as independent model execution.
31
+ The default route is the live run view:
40
32
 
41
- ## Stage 8 delivery adapter
33
+ - stage rail goal → maker → verify → review → judge → deliver, with the current stage and heartbeat age;
34
+ - tier, mode, iteration, and budget;
35
+ - compliance chips for guard warns/blocks, impact drift, and receipt verification;
36
+ - pending questions, decisions, and approvals as the top banner;
37
+ - a merged timeline of lifecycle, activity, and execution events.
42
38
 
43
- The default canvas labels the lifecycle coverage from Stage 1 through Stage 8; Maker contains the Stage 2–4 diagnosis, planning, and implementation sequence. Delivery nodes expose the project-level Stage 8 adapter selected in `.ai-engineering-loop/adapter.md`. Studio can select the shipped `standard`, `github`, `gitlab`, or `dot` adapters and shows the adapter inferred from the repository remote.
39
+ The Goal Contract stays a side sheet of this view (draft, review, freeze, unfreeze). Start with AI Agent still copies a run-bound handoff into the clipboard.
44
40
 
45
- An adapter belongs to the project, not to one delivery node. For a custom adapter, **Custom with AI** copies a bounded builder request that tells an agent to use the existing `generate-adapter` Q1–Q5 protocol, preserve Stages 0–7, require Judge `PASS`, and keep credentials out of configuration. Once that agent writes `.ai-engineering-loop/adapter.md`, Studio discovers the custom adapter on refresh. The browser never accepts arbitrary adapter code or executes delivery commands.
41
+ ## Runs
46
42
 
47
- ## Live execution
43
+ **Runs** is a slim history list. Open a row to pin it in the live view as a read-only replay. Artifact inspection still accepts only regular files inside the owning run, rejects path traversal and symlinks, enforces a size bound, and redacts recognized secrets.
48
44
 
49
- The trigger/root node owns the task input and local Execute control. Studio first creates a Run with an immutable Technical ID and human-readable Display Name. A deterministic task-derived name is used when no safe proposal is available. The Goal Contract must contain numbered acceptance criteria with evidence and failure cases and must pass an explicit Goal Freeze before Execute can start the local runtime.
45
+ ## Wiring existing nodes
50
46
 
51
- Goal Freeze records actor, timestamp, version, and content hash. Unfreeze requires an actor and reason, increments the Goal version, resets the workflow to its Goal gate, and leaves previous evidence bound to the previous version.
47
+ Studio is read-only for workflow *structure*. Do not create, duplicate, delete, or import nodes here. Author recipes as files and use `recipe create|clone|install`.
52
48
 
53
- For a recipe-bound current run, Studio polls a small live snapshot rather than reloading the recipe catalog. It displays:
49
+ **Wiring** keeps the canvas for edges between nodes that already exist: add, remove, or re-point a connection, plus fit, zoom, tidy/auto-layout, and undo/redo for those edge edits. Node moves are layout-only (`PUT /api/layout`) and never enter the graph hash.
54
50
 
55
- - current node states and the append-only event stream;
56
- - an animated connector leaving the running node;
57
- - provider-recorded run token spend and configured limit;
58
- - explicit approval and retry controls;
59
- - a short redacted activity message reported by the host:
51
+ Every wiring change is validated by `POST /api/validate`, which is the same `lib/recipe.js` compiler and safety backbone (goal-contract → maker → verification → devil-advocate → judge → delivery/report, plus a human approval before delivery in ASSISTED). An edge set that breaks the backbone is rejected with the compiler's named error and cannot be saved. The server also rejects a recipe whose node set differs from the loaded base (`baseRecipeId`): same node ids, types, and node config; only edges, `dependsOn`, and layout may change. That failure is `STUDIO_STRUCTURE_READ_ONLY` and lists added, removed, and changed node ids. A builtin may still be installed as `<id>-local` when the nodes are unchanged. A valid change is installed only after the confirm dialog (`POST /api/install`). Built-in recipe ids stay reserved; Studio installs a project copy.
60
52
 
61
- ```bash
62
- ai-engineering-loop node activity <node-id> \
63
- --message "Comparing the failing test with the implementation"
64
- ```
53
+ ## Wiring canvas
65
54
 
66
- Activity can only be attached to a `RUNNING` node. It is capped at 240 printable characters, secret-redacted before persistence, and added to the integrity-bound workflow event chain.
55
+ ### Node tiles
67
56
 
68
- ## Run History
57
+ Each existing node is one tile: a type icon, the node name, and exactly one status line. That line is the duration slot (`Passed · 6m 10s` when a duration was recorded). Stage and type text are not repeated as badges on the tile. A loop group draws a container around its nodes and labels it `iteration n / max` from the live loop state.
69
58
 
70
- The separate **Runs** workspace reads every valid persisted Run rather than replacing Workflows with execution records. It supports text, status, mode, and date filters and shows human-readable names, technical IDs, Goal state, decisions, events, evidence presence, and bounded artifacts. Corrupt Run records are isolated so valid siblings remain available.
59
+ ### Side panel
71
60
 
72
- Run History is read-only. Artifact inspection accepts only regular files inside the owning Run, rejects path traversal and symlinks, enforces a size bound, and redacts recognized secrets before returning content.
61
+ Click a tile to open the side panel. The tabs are Input, Output, Evidence, and Timeline. Evidence uses the same artifact inspection as the rest of Studio: path traversal and symlinks are rejected, size is bounded, and recognized secrets are redacted.
73
62
 
74
- ## Decision memory
63
+ ### Tooltip
75
64
 
76
- Studio can record a concise decision and rationale against the current run. Decision memory is private runtime data under the run directory, is bounded to 100 entries, and is redacted before persistence. It becomes part of exported handoffs so the next developer or agent receives the “why,” not only the final state.
65
+ Hover or keyboard focus shows a tooltip with the node name, type, stage, and status. The tooltip is separate from the tile, so the tile does not carry a second stage or type badge.
77
66
 
78
- Do not use decision memory as a credential store or as a replacement for a repository ADR when a durable architectural decision belongs in version control.
67
+ ### Minimap and zoom controls
79
68
 
80
- ## Knowledge handoff
69
+ The minimap draws one rectangle per node. Zoom controls are zoom in (+), zoom out (−), fit, auto-layout, and the zoom percent. Fit and auto-layout keep every built-in recipe inside the canvas.
81
70
 
82
- The top bar provides two forms:
71
+ ### Shortcuts
83
72
 
84
- - **Copy Brief** creates a human-readable developer summary.
85
- - **Export Handoff** downloads a machine-readable `.ael-handoff.json` bundle for another agent.
73
+ The `?` sheet lists the shortcuts: `F` fit, `L` auto-layout, `+` and `-` zoom, `0` reset zoom, arrow keys between nodes, Enter to open the side panel, Esc to close it, and undo/redo for wiring edits. `:focus-visible` outlines the canvas controls.
86
74
 
87
- The bundle contains objective, workflow state, artifact descriptors, decisions, transitions, and ready next actions. It deliberately excludes source contents. Strings pass through secret and home-path redaction, and the complete canonical manifest is bound to `bundleHash`.
75
+ ### Save bar
88
76
 
89
- Verify a received bundle before trusting it:
77
+ The dirty-only save bar stays hidden while the wiring is clean. The first edge change shows the change count, Valid or Invalid, Discard, and Save. Save opens the confirm dialog before `POST /api/install`. A backbone-breaking edge shows the compiler's named error and disables Save. Every edit is checked with `POST /api/validate`.
90
78
 
91
- ```bash
92
- ai-engineering-loop handoff inspect run-id.ael-handoff.json
93
- ai-engineering-loop handoff brief run-id.ael-handoff.json
94
- ```
79
+ ### Run summary card
80
+
81
+ When a run is DELIVERED, REPORTED, ESCALATED, or CANCELLED, the canvas shows a run summary card with duration, tests, findings, drift, receipt, and tokens. Tokens read `unknown` when the usage ledger has none.
82
+
83
+ Workflow structure on this canvas stays read-only. There is no control to create, delete, rename, duplicate, or import nodes. Rewiring only connects nodes that already exist. Live tile and summary updates come from `GET /api/events`.
84
+
85
+ ## Stage 8 delivery adapter
95
86
 
96
- Inspection accepts only regular files smaller than 1 MiB inside the repository, rejects symlinks and path escapes, and fails if any field no longer matches the bundle hash.
87
+ Delivery nodes still expose the project-level Stage 8 adapter in `.ai-engineering-loop/adapter.md`. `POST /api/adapter/select` can select the shipped `standard`, `github`, `gitlab`, or `dot` adapters. The browser never accepts arbitrary adapter code or executes delivery commands.
97
88
 
98
89
  ## Trust boundary
99
90
 
100
- Studio improves authoring and observability; it does not weaken runtime gates:
91
+ Studio improves observability; it does not weaken runtime gates:
101
92
 
102
93
  - graph validation does not imply execution permission;
103
94
  - local execution requires exactly one permitted trigger/root and an audited frozen Goal;
104
95
  - command nodes remain disabled;
105
- - Custom Agents wait for an external operator and cannot grant undeclared authority;
106
96
  - approvals remain explicit state transitions;
107
97
  - connector edits cannot bypass the required safety backbone;
108
98
  - Run History cannot mutate persisted runs or artifacts;
109
99
  - delivery adapters cannot run before Judge `PASS`;
110
100
  - token data comes only from the provider-recorded usage ledger;
111
- - handoff integrity proves the bundle has not changed after export, not that every original claim was true.
101
+ - the event stream never exposes file contents outside the run-dir artifacts already served.