ai-engineering-loop 1.17.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
  2. package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  3. package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
  4. package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  5. package/CHANGELOG.md +52 -0
  6. package/README.md +54 -39
  7. package/bin/ai-engineering-loop.js +221 -16
  8. package/core/bench.md +77 -0
  9. package/core/configuration-precedence.md +11 -1
  10. package/core/local-workflow-studio.md +45 -55
  11. package/core/personal-workflow.md +78 -0
  12. package/core/workflow-recipes.md +2 -0
  13. package/docs/images/studio-hero.svg +59 -47
  14. package/evals/README.md +2 -0
  15. package/evals/cases/25-no-task-onboarding.json +22 -0
  16. package/evals/scenarios/t0-docs-typo.json +31 -0
  17. package/evals/scenarios/t1-empty-state.json +31 -0
  18. package/evals/scenarios/t1-list-pagination.json +33 -0
  19. package/evals/scenarios/t2-auth-session.json +31 -0
  20. package/evals/scenarios/t2-payment-idempotency.json +31 -0
  21. package/evals/scenarios/trap-destructive-reset.json +33 -0
  22. package/evals/scenarios/trap-out-of-scope.json +34 -0
  23. package/evals/scenarios/trap-prompt-injection.json +39 -0
  24. package/lib/bench-targets.js +194 -0
  25. package/lib/bench.js +1236 -0
  26. package/lib/budget.js +32 -0
  27. package/lib/cli-args.js +12 -1
  28. package/lib/doctor.js +17 -2
  29. package/lib/hosts/registry.js +18 -0
  30. package/lib/personal-workflow.js +925 -0
  31. package/lib/recipe.js +90 -13
  32. package/lib/run-state.js +10 -1
  33. package/lib/run-watcher.js +520 -0
  34. package/lib/runtime-files.js +2 -0
  35. package/lib/runtime-policy.js +42 -1
  36. package/lib/studio-runtime.js +8 -3
  37. package/lib/studio-server.js +336 -98
  38. package/lib/user-state.js +34 -1
  39. package/package.json +1 -1
  40. package/schemas/bench-result.schema.json +288 -0
  41. package/schemas/bench-scenario.schema.json +95 -0
  42. package/studio/app.js +2024 -1633
  43. package/studio/index.html +71 -181
  44. package/studio/styles.css +135 -8
package/lib/budget.js CHANGED
@@ -309,12 +309,44 @@ function setKillSwitch(rootDir, active) {
309
309
  return updatePolicy(rootDir, { killSwitch: Boolean(active) });
310
310
  }
311
311
 
312
+ function readUnavailableUsage(rootDir) {
313
+ const directory = usageDir(rootDir);
314
+ if (!fs.existsSync(directory)) return [];
315
+ let names;
316
+ try {
317
+ names = fs.readdirSync(directory)
318
+ .filter((name) => /^unavailable-\d{4}-\d{2}-\d{2}\.jsonl$/.test(name))
319
+ .sort();
320
+ } catch (cause) {
321
+ throw usageError(`Unable to enumerate usage ledgers: ${cause.message}`, 'INVALID_USAGE_LEDGER');
322
+ }
323
+ const entries = [];
324
+ for (const name of names) {
325
+ const filePath = path.join(directory, name);
326
+ let lines;
327
+ try {
328
+ lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
329
+ } catch (cause) {
330
+ throw usageError(`Unable to read usage ledger: ${cause.message}`, 'INVALID_USAGE_LEDGER');
331
+ }
332
+ for (const line of lines) {
333
+ try {
334
+ entries.push(JSON.parse(line));
335
+ } catch {
336
+ // Skip a corrupt unavailable line; token scoring must not invent counts.
337
+ }
338
+ }
339
+ }
340
+ return entries;
341
+ }
342
+
312
343
  module.exports = {
313
344
  usageDir,
314
345
  utcDay,
315
346
  usagePath,
316
347
  readUsage,
317
348
  readAllUsage,
349
+ readUnavailableUsage,
318
350
  budgetStatus,
319
351
  assertBudgetAvailable,
320
352
  recordTokenUsage,
package/lib/cli-args.js CHANGED
@@ -19,6 +19,11 @@ const WRITER_COMMANDS = Object.freeze({
19
19
  projects: { flags: ['--json'], valued: ['--root', '--depth'], positionals: null },
20
20
  // `run --abandon` (issue #46) is a writer; plain `run` takes free text.
21
21
  'run:abandon': { flags: ['--abandon', '--yes', '--json'], valued: ['--run', '--reason', '--by'], positionals: [] },
22
+ bench: {
23
+ flags: ['--json', '--sandbox-home'],
24
+ valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg'],
25
+ positionals: []
26
+ },
22
27
  migrate: { flags: ['--json', '--dry-run'], valued: ['--from'], positionals: [] },
23
28
  verify: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
24
29
  learn: { flags: ['--json', '--dry-run', '--prune'], valued: ['--max-age-runs'], positionals: [] },
@@ -28,7 +33,12 @@ const WRITER_COMMANDS = Object.freeze({
28
33
  'review-pack': { flags: ['--json'], valued: ['--run', '--base', '--profile'], positionals: [] },
29
34
  deliver: { flags: ['--json', '--approve'], valued: ['--run', '--destination', '--summary', '--by', '--host', '--model', '--head', '--evidence'], positionals: [] },
30
35
  'generate-adapter': { flags: [], valued: ['--type'], positionals: [] },
31
- 'generate-workflow': { flags: ['--write'], valued: [], positionals: [] }
36
+ 'generate-workflow': { flags: ['--write'], valued: [], positionals: [] },
37
+ workflow: {
38
+ flags: ['--json'],
39
+ valued: ['--answers', '--proposal', '--by', '--mode'],
40
+ positionals: ['interview', 'propose', 'accept', 'reject', 'suggest']
41
+ }
32
42
  });
33
43
 
34
44
  function beforeSeparator(args) {
@@ -39,6 +49,7 @@ function beforeSeparator(args) {
39
49
  // `run --abandon ...` is checked as its own writer command.
40
50
  function effectiveCommand(command, rest) {
41
51
  if (command === 'run' && beforeSeparator(rest).includes('--abandon')) return 'run:abandon';
52
+ if (command === 'bench' && beforeSeparator(rest)[0] === 'compare') return 'bench:compare';
42
53
  return command;
43
54
  }
44
55
 
package/lib/doctor.js CHANGED
@@ -3,6 +3,7 @@
3
3
  const fs = require('fs');
4
4
  const path = require('path');
5
5
  const { loadEvaluationCases } = require('./evaluation.js');
6
+ const { loadBenchScenarios } = require('./bench.js');
6
7
  const { auditPackage } = require('./package-audit.js');
7
8
  const { listRecipes, loadRecipe, validateRecipe } = require('./recipe.js');
8
9
  const { checkAdapter } = require('./adapter-check.js');
@@ -47,7 +48,9 @@ const REQUIRED_SCHEMAS = [
47
48
  'handoff.schema.json',
48
49
  'impact-map.schema.json',
49
50
  'receipt.schema.json',
50
- 'learnings.schema.json'
51
+ 'learnings.schema.json',
52
+ 'bench-scenario.schema.json',
53
+ 'bench-result.schema.json'
51
54
  ];
52
55
 
53
56
  function check(id, passed, detail) {
@@ -108,11 +111,23 @@ function runDoctor(rootDir = path.join(__dirname, '..'), { projectDir = null, ru
108
111
 
109
112
  try {
110
113
  const fixtures = loadEvaluationCases(path.join(rootDir, 'evals', 'cases'));
111
- checks.push(check('evaluation-catalog', fixtures.length === 24, `${fixtures.length} valid cases; required 24`));
114
+ checks.push(check('evaluation-catalog', fixtures.length === 25, `${fixtures.length} valid cases; required 25`));
112
115
  } catch (cause) {
113
116
  checks.push(check('evaluation-catalog', false, cause.message));
114
117
  }
115
118
 
119
+ try {
120
+ const scenarios = loadBenchScenarios(path.join(rootDir, 'evals', 'scenarios'));
121
+ const trapCount = scenarios.filter((item) => (item.traps || []).length > 0).length;
122
+ checks.push(check(
123
+ 'bench-catalog',
124
+ scenarios.length >= 6 && scenarios.length <= 8 && trapCount >= 2,
125
+ `${scenarios.length} bench scenarios; ${trapCount} with traps`
126
+ ));
127
+ } catch (cause) {
128
+ checks.push(check('bench-catalog', false, cause.message));
129
+ }
130
+
116
131
  const packageFiles = new Set(pkg.files || []);
117
132
  const requiredPackageEntries = ['bin/', 'lib/', 'core/', 'policies/', 'recipes/', 'schemas/', 'evals/', 'studio/'];
118
133
  const missingPackageEntries = requiredPackageEntries.filter((entry) => !packageFiles.has(entry));
@@ -38,6 +38,12 @@ const HOSTS = Object.freeze([
38
38
  reads: ['.claude/skills'],
39
39
  agents: { format: 'claude-md', dir: '.claude/agents' },
40
40
  hooks: { format: 'claude-pretooluse', settings: '.claude/settings.json' },
41
+ bench: {
42
+ bin: 'claude',
43
+ args: ['-p', '--output-format', 'json', '--dangerously-skip-permissions', '--model', '{{model}}'],
44
+ promptStdin: true,
45
+ usageParser: 'json-usage'
46
+ },
41
47
  docs: ['https://code.claude.com/docs/en/skills', 'https://code.claude.com/docs/en/sub-agents', 'https://code.claude.com/docs/en/hooks'],
42
48
  verified: '2026-10-11'
43
49
  },
@@ -50,6 +56,12 @@ const HOSTS = Object.freeze([
50
56
  skills: 'shared',
51
57
  reads: ['.agents/skills'],
52
58
  agents: { format: 'codex-toml', dir: '.codex/agents' },
59
+ bench: {
60
+ bin: 'codex',
61
+ args: ['exec', '--json', '--full-auto', '-m', '{{model}}'],
62
+ promptStdin: true,
63
+ usageParser: 'json-usage'
64
+ },
53
65
  docs: ['https://developers.openai.com/codex/skills', 'https://developers.openai.com/codex/subagents'],
54
66
  verified: '2026-10-11'
55
67
  },
@@ -98,6 +110,12 @@ const HOSTS = Object.freeze([
98
110
  skills: 'shared',
99
111
  reads: ['.grok/skills', '.claude/skills', '.agents/skills'],
100
112
  agents: { format: 'grok-md', dir: '.grok/agents' },
113
+ bench: {
114
+ bin: '~/.grok/bin/grok',
115
+ args: ['--no-auto-update', '--always-approve', '-m', '{{model}}', '-p', '{{prompt}}'],
116
+ // `grok -p` prints plain text. There is no usage object to parse; tokens stay unknown.
117
+ usageParser: 'none'
118
+ },
101
119
  docs: ['https://docs.x.ai/build/features/skills-plugins-marketplaces', 'https://docs.x.ai/build/features/subagents'],
102
120
  verified: '2026-10-11'
103
121
  },