ai-engineering-loop 1.17.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/CHANGELOG.md +37 -0
- package/bin/ai-engineering-loop.js +221 -16
- package/core/bench.md +77 -0
- package/core/configuration-precedence.md +11 -1
- package/core/local-workflow-studio.md +26 -68
- package/core/personal-workflow.md +78 -0
- package/core/workflow-recipes.md +2 -0
- package/evals/README.md +2 -0
- package/evals/cases/25-no-task-onboarding.json +22 -0
- package/evals/scenarios/t0-docs-typo.json +31 -0
- package/evals/scenarios/t1-empty-state.json +31 -0
- package/evals/scenarios/t1-list-pagination.json +33 -0
- package/evals/scenarios/t2-auth-session.json +31 -0
- package/evals/scenarios/t2-payment-idempotency.json +31 -0
- package/evals/scenarios/trap-destructive-reset.json +33 -0
- package/evals/scenarios/trap-out-of-scope.json +34 -0
- package/evals/scenarios/trap-prompt-injection.json +39 -0
- package/lib/bench-targets.js +194 -0
- package/lib/bench.js +1236 -0
- package/lib/budget.js +32 -0
- package/lib/cli-args.js +12 -1
- package/lib/doctor.js +17 -2
- package/lib/hosts/registry.js +18 -0
- package/lib/personal-workflow.js +925 -0
- package/lib/recipe.js +90 -13
- package/lib/run-state.js +10 -1
- package/lib/run-watcher.js +511 -0
- package/lib/runtime-files.js +2 -0
- package/lib/runtime-policy.js +42 -1
- package/lib/studio-runtime.js +8 -3
- package/lib/studio-server.js +336 -98
- package/lib/user-state.js +34 -1
- package/package.json +1 -1
- package/schemas/bench-result.schema.json +288 -0
- package/schemas/bench-scenario.schema.json +95 -0
- package/studio/app.js +599 -1621
- package/studio/index.html +43 -178
- package/studio/styles.css +17 -2
package/lib/budget.js
CHANGED
|
@@ -309,12 +309,44 @@ function setKillSwitch(rootDir, active) {
|
|
|
309
309
|
return updatePolicy(rootDir, { killSwitch: Boolean(active) });
|
|
310
310
|
}
|
|
311
311
|
|
|
312
|
+
function readUnavailableUsage(rootDir) {
|
|
313
|
+
const directory = usageDir(rootDir);
|
|
314
|
+
if (!fs.existsSync(directory)) return [];
|
|
315
|
+
let names;
|
|
316
|
+
try {
|
|
317
|
+
names = fs.readdirSync(directory)
|
|
318
|
+
.filter((name) => /^unavailable-\d{4}-\d{2}-\d{2}\.jsonl$/.test(name))
|
|
319
|
+
.sort();
|
|
320
|
+
} catch (cause) {
|
|
321
|
+
throw usageError(`Unable to enumerate usage ledgers: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
322
|
+
}
|
|
323
|
+
const entries = [];
|
|
324
|
+
for (const name of names) {
|
|
325
|
+
const filePath = path.join(directory, name);
|
|
326
|
+
let lines;
|
|
327
|
+
try {
|
|
328
|
+
lines = fs.readFileSync(filePath, 'utf8').split('\n').filter(Boolean);
|
|
329
|
+
} catch (cause) {
|
|
330
|
+
throw usageError(`Unable to read usage ledger: ${cause.message}`, 'INVALID_USAGE_LEDGER');
|
|
331
|
+
}
|
|
332
|
+
for (const line of lines) {
|
|
333
|
+
try {
|
|
334
|
+
entries.push(JSON.parse(line));
|
|
335
|
+
} catch {
|
|
336
|
+
// Skip a corrupt unavailable line; token scoring must not invent counts.
|
|
337
|
+
}
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
return entries;
|
|
341
|
+
}
|
|
342
|
+
|
|
312
343
|
module.exports = {
|
|
313
344
|
usageDir,
|
|
314
345
|
utcDay,
|
|
315
346
|
usagePath,
|
|
316
347
|
readUsage,
|
|
317
348
|
readAllUsage,
|
|
349
|
+
readUnavailableUsage,
|
|
318
350
|
budgetStatus,
|
|
319
351
|
assertBudgetAvailable,
|
|
320
352
|
recordTokenUsage,
|
package/lib/cli-args.js
CHANGED
|
@@ -19,6 +19,11 @@ const WRITER_COMMANDS = Object.freeze({
|
|
|
19
19
|
projects: { flags: ['--json'], valued: ['--root', '--depth'], positionals: null },
|
|
20
20
|
// `run --abandon` (issue #46) is a writer; plain `run` takes free text.
|
|
21
21
|
'run:abandon': { flags: ['--abandon', '--yes', '--json'], valued: ['--run', '--reason', '--by'], positionals: [] },
|
|
22
|
+
bench: {
|
|
23
|
+
flags: ['--json', '--sandbox-home'],
|
|
24
|
+
valued: ['--host', '--model', '--scenario', '--repeat', '--bin', '--mode', '--timeout', '--host-arg'],
|
|
25
|
+
positionals: []
|
|
26
|
+
},
|
|
22
27
|
migrate: { flags: ['--json', '--dry-run'], valued: ['--from'], positionals: [] },
|
|
23
28
|
verify: { flags: ['--json', '--serial', '--shell'], valued: ['--run', '--cmd', '--timeout-ms'], positionals: [] },
|
|
24
29
|
learn: { flags: ['--json', '--dry-run', '--prune'], valued: ['--max-age-runs'], positionals: [] },
|
|
@@ -28,7 +33,12 @@ const WRITER_COMMANDS = Object.freeze({
|
|
|
28
33
|
'review-pack': { flags: ['--json'], valued: ['--run', '--base', '--profile'], positionals: [] },
|
|
29
34
|
deliver: { flags: ['--json', '--approve'], valued: ['--run', '--destination', '--summary', '--by', '--host', '--model', '--head', '--evidence'], positionals: [] },
|
|
30
35
|
'generate-adapter': { flags: [], valued: ['--type'], positionals: [] },
|
|
31
|
-
'generate-workflow': { flags: ['--write'], valued: [], positionals: [] }
|
|
36
|
+
'generate-workflow': { flags: ['--write'], valued: [], positionals: [] },
|
|
37
|
+
workflow: {
|
|
38
|
+
flags: ['--json'],
|
|
39
|
+
valued: ['--answers', '--proposal', '--by', '--mode'],
|
|
40
|
+
positionals: ['interview', 'propose', 'accept', 'reject', 'suggest']
|
|
41
|
+
}
|
|
32
42
|
});
|
|
33
43
|
|
|
34
44
|
function beforeSeparator(args) {
|
|
@@ -39,6 +49,7 @@ function beforeSeparator(args) {
|
|
|
39
49
|
// `run --abandon ...` is checked as its own writer command.
|
|
40
50
|
function effectiveCommand(command, rest) {
|
|
41
51
|
if (command === 'run' && beforeSeparator(rest).includes('--abandon')) return 'run:abandon';
|
|
52
|
+
if (command === 'bench' && beforeSeparator(rest)[0] === 'compare') return 'bench:compare';
|
|
42
53
|
return command;
|
|
43
54
|
}
|
|
44
55
|
|
package/lib/doctor.js
CHANGED
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
const fs = require('fs');
|
|
4
4
|
const path = require('path');
|
|
5
5
|
const { loadEvaluationCases } = require('./evaluation.js');
|
|
6
|
+
const { loadBenchScenarios } = require('./bench.js');
|
|
6
7
|
const { auditPackage } = require('./package-audit.js');
|
|
7
8
|
const { listRecipes, loadRecipe, validateRecipe } = require('./recipe.js');
|
|
8
9
|
const { checkAdapter } = require('./adapter-check.js');
|
|
@@ -47,7 +48,9 @@ const REQUIRED_SCHEMAS = [
|
|
|
47
48
|
'handoff.schema.json',
|
|
48
49
|
'impact-map.schema.json',
|
|
49
50
|
'receipt.schema.json',
|
|
50
|
-
'learnings.schema.json'
|
|
51
|
+
'learnings.schema.json',
|
|
52
|
+
'bench-scenario.schema.json',
|
|
53
|
+
'bench-result.schema.json'
|
|
51
54
|
];
|
|
52
55
|
|
|
53
56
|
function check(id, passed, detail) {
|
|
@@ -108,11 +111,23 @@ function runDoctor(rootDir = path.join(__dirname, '..'), { projectDir = null, ru
|
|
|
108
111
|
|
|
109
112
|
try {
|
|
110
113
|
const fixtures = loadEvaluationCases(path.join(rootDir, 'evals', 'cases'));
|
|
111
|
-
checks.push(check('evaluation-catalog', fixtures.length ===
|
|
114
|
+
checks.push(check('evaluation-catalog', fixtures.length === 25, `${fixtures.length} valid cases; required 25`));
|
|
112
115
|
} catch (cause) {
|
|
113
116
|
checks.push(check('evaluation-catalog', false, cause.message));
|
|
114
117
|
}
|
|
115
118
|
|
|
119
|
+
try {
|
|
120
|
+
const scenarios = loadBenchScenarios(path.join(rootDir, 'evals', 'scenarios'));
|
|
121
|
+
const trapCount = scenarios.filter((item) => (item.traps || []).length > 0).length;
|
|
122
|
+
checks.push(check(
|
|
123
|
+
'bench-catalog',
|
|
124
|
+
scenarios.length >= 6 && scenarios.length <= 8 && trapCount >= 2,
|
|
125
|
+
`${scenarios.length} bench scenarios; ${trapCount} with traps`
|
|
126
|
+
));
|
|
127
|
+
} catch (cause) {
|
|
128
|
+
checks.push(check('bench-catalog', false, cause.message));
|
|
129
|
+
}
|
|
130
|
+
|
|
116
131
|
const packageFiles = new Set(pkg.files || []);
|
|
117
132
|
const requiredPackageEntries = ['bin/', 'lib/', 'core/', 'policies/', 'recipes/', 'schemas/', 'evals/', 'studio/'];
|
|
118
133
|
const missingPackageEntries = requiredPackageEntries.filter((entry) => !packageFiles.has(entry));
|
package/lib/hosts/registry.js
CHANGED
|
@@ -38,6 +38,12 @@ const HOSTS = Object.freeze([
|
|
|
38
38
|
reads: ['.claude/skills'],
|
|
39
39
|
agents: { format: 'claude-md', dir: '.claude/agents' },
|
|
40
40
|
hooks: { format: 'claude-pretooluse', settings: '.claude/settings.json' },
|
|
41
|
+
bench: {
|
|
42
|
+
bin: 'claude',
|
|
43
|
+
args: ['-p', '--output-format', 'json', '--dangerously-skip-permissions', '--model', '{{model}}'],
|
|
44
|
+
promptStdin: true,
|
|
45
|
+
usageParser: 'json-usage'
|
|
46
|
+
},
|
|
41
47
|
docs: ['https://code.claude.com/docs/en/skills', 'https://code.claude.com/docs/en/sub-agents', 'https://code.claude.com/docs/en/hooks'],
|
|
42
48
|
verified: '2026-10-11'
|
|
43
49
|
},
|
|
@@ -50,6 +56,12 @@ const HOSTS = Object.freeze([
|
|
|
50
56
|
skills: 'shared',
|
|
51
57
|
reads: ['.agents/skills'],
|
|
52
58
|
agents: { format: 'codex-toml', dir: '.codex/agents' },
|
|
59
|
+
bench: {
|
|
60
|
+
bin: 'codex',
|
|
61
|
+
args: ['exec', '--json', '--full-auto', '-m', '{{model}}'],
|
|
62
|
+
promptStdin: true,
|
|
63
|
+
usageParser: 'json-usage'
|
|
64
|
+
},
|
|
53
65
|
docs: ['https://developers.openai.com/codex/skills', 'https://developers.openai.com/codex/subagents'],
|
|
54
66
|
verified: '2026-10-11'
|
|
55
67
|
},
|
|
@@ -98,6 +110,12 @@ const HOSTS = Object.freeze([
|
|
|
98
110
|
skills: 'shared',
|
|
99
111
|
reads: ['.grok/skills', '.claude/skills', '.agents/skills'],
|
|
100
112
|
agents: { format: 'grok-md', dir: '.grok/agents' },
|
|
113
|
+
bench: {
|
|
114
|
+
bin: '~/.grok/bin/grok',
|
|
115
|
+
args: ['--no-auto-update', '--always-approve', '-m', '{{model}}', '-p', '{{prompt}}'],
|
|
116
|
+
// `grok -p` prints plain text. There is no usage object to parse; tokens stay unknown.
|
|
117
|
+
usageParser: 'none'
|
|
118
|
+
},
|
|
101
119
|
docs: ['https://docs.x.ai/build/features/skills-plugins-marketplaces', 'https://docs.x.ai/build/features/subagents'],
|
|
102
120
|
verified: '2026-10-11'
|
|
103
121
|
},
|