ai-engineering-loop 1.17.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
  2. package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  3. package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
  4. package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  5. package/CHANGELOG.md +52 -0
  6. package/README.md +54 -39
  7. package/bin/ai-engineering-loop.js +221 -16
  8. package/core/bench.md +77 -0
  9. package/core/configuration-precedence.md +11 -1
  10. package/core/local-workflow-studio.md +45 -55
  11. package/core/personal-workflow.md +78 -0
  12. package/core/workflow-recipes.md +2 -0
  13. package/docs/images/studio-hero.svg +59 -47
  14. package/evals/README.md +2 -0
  15. package/evals/cases/25-no-task-onboarding.json +22 -0
  16. package/evals/scenarios/t0-docs-typo.json +31 -0
  17. package/evals/scenarios/t1-empty-state.json +31 -0
  18. package/evals/scenarios/t1-list-pagination.json +33 -0
  19. package/evals/scenarios/t2-auth-session.json +31 -0
  20. package/evals/scenarios/t2-payment-idempotency.json +31 -0
  21. package/evals/scenarios/trap-destructive-reset.json +33 -0
  22. package/evals/scenarios/trap-out-of-scope.json +34 -0
  23. package/evals/scenarios/trap-prompt-injection.json +39 -0
  24. package/lib/bench-targets.js +194 -0
  25. package/lib/bench.js +1236 -0
  26. package/lib/budget.js +32 -0
  27. package/lib/cli-args.js +12 -1
  28. package/lib/doctor.js +17 -2
  29. package/lib/hosts/registry.js +18 -0
  30. package/lib/personal-workflow.js +925 -0
  31. package/lib/recipe.js +90 -13
  32. package/lib/run-state.js +10 -1
  33. package/lib/run-watcher.js +520 -0
  34. package/lib/runtime-files.js +2 -0
  35. package/lib/runtime-policy.js +42 -1
  36. package/lib/studio-runtime.js +8 -3
  37. package/lib/studio-server.js +336 -98
  38. package/lib/user-state.js +34 -1
  39. package/package.json +1 -1
  40. package/schemas/bench-result.schema.json +288 -0
  41. package/schemas/bench-scenario.schema.json +95 -0
  42. package/studio/app.js +2024 -1633
  43. package/studio/index.html +71 -181
  44. package/studio/styles.css +135 -8
@@ -0,0 +1,194 @@
1
+ 'use strict';
2
+
3
+ // One host+model per `ael bench` invocation (#79). Command templates live on
4
+ // each host's `bench` block. A missing binary is UNAVAILABLE, never a model
5
+ // failure. `--bin` is the test hook that points at tests/fixtures/fake-host.js.
6
+
7
+ const fs = require('fs');
8
+ const os = require('os');
9
+ const path = require('path');
10
+ const { HOSTS, HOST_IDS, getHost } = require('./hosts/registry.js');
11
+ const { findOnPath } = require('./hosts/detect.js');
12
+ const { readUserConfig } = require('./user-state.js');
13
+
14
+ function benchError(message, code = 'BENCH_FAILED', details = []) {
15
+ const error = new Error(message);
16
+ error.code = code;
17
+ error.details = details;
18
+ return error;
19
+ }
20
+
21
+ function countFlag(args, name) {
22
+ let count = 0;
23
+ for (let index = 0; index < args.length; index += 1) {
24
+ const arg = args[index];
25
+ if (arg === name) {
26
+ count += 1;
27
+ if (!String(args[index + 1] || '').startsWith('-')) index += 1;
28
+ continue;
29
+ }
30
+ if (arg.startsWith(`${name}=`)) count += 1;
31
+ }
32
+ return count;
33
+ }
34
+
35
+ function flagValues(args, name) {
36
+ const values = [];
37
+ for (let index = 0; index < args.length; index += 1) {
38
+ const arg = args[index];
39
+ if (arg === name && args[index + 1] != null) {
40
+ values.push(args[index + 1]);
41
+ index += 1;
42
+ continue;
43
+ }
44
+ if (arg.startsWith(`${name}=`)) values.push(arg.slice(name.length + 1));
45
+ }
46
+ return values;
47
+ }
48
+
49
+ function benchSpec(host) {
50
+ if (host?.bench?.bin && Array.isArray(host.bench.args)) return host.bench;
51
+ const bin = (host?.detect?.bins || [])[0];
52
+ if (!bin) return null;
53
+ return { bin, args: ['-m', '{{model}}', '-p', '{{prompt}}'], usageParser: 'json-usage' };
54
+ }
55
+
56
+ function expandHome(value, home) {
57
+ const raw = String(value || '');
58
+ if (raw === '~') return home;
59
+ if (raw.startsWith('~/')) return path.join(home, raw.slice(2));
60
+ return raw;
61
+ }
62
+
63
+ function resolveBenchBinary(host, { binOverride, home, env = process.env } = {}) {
64
+ if (binOverride) {
65
+ const resolved = path.resolve(String(binOverride));
66
+ try {
67
+ fs.statSync(resolved);
68
+ return { status: 'ok', bin: resolved, override: true };
69
+ } catch {
70
+ return { status: 'UNAVAILABLE', bin: resolved, override: true };
71
+ }
72
+ }
73
+ const spec = benchSpec(host);
74
+ if (!spec) return { status: 'UNAVAILABLE', bin: null, override: false };
75
+ const expanded = expandHome(spec.bin, home || os.homedir());
76
+ try {
77
+ if (fs.statSync(expanded).isFile()) return { status: 'ok', bin: expanded, override: false };
78
+ } catch {
79
+ // Fall through to PATH.
80
+ }
81
+ const names = [path.basename(spec.bin), ...(host.detect.bins || [])];
82
+ for (const name of names) {
83
+ const found = findOnPath(name, env);
84
+ if (found) return { status: 'ok', bin: found, override: false };
85
+ }
86
+ return { status: 'UNAVAILABLE', bin: expanded, override: false };
87
+ }
88
+
89
+ function buildBenchCommand(host, { model, prompt, bin, spec, extraArgs = [] }) {
90
+ const template = spec || benchSpec(host);
91
+ if (!template) throw benchError(`Host ${host?.id || '(unknown)'} has no bench command template`, 'UNAVAILABLE');
92
+ const args = template.args.map((part) => String(part)
93
+ .replace(/\{\{model\}\}/g, String(model))
94
+ .replace(/\{\{prompt\}\}/g, String(prompt)));
95
+ for (const extra of extraArgs) args.push(String(extra));
96
+ const usesPromptArg = template.args.some((part) => String(part).includes('{{prompt}}'));
97
+ return {
98
+ bin,
99
+ args,
100
+ input: usesPromptArg || template.promptStdin === false ? null : prompt,
101
+ usageParser: template.usageParser || 'json-usage'
102
+ };
103
+ }
104
+
105
+ function extractJsonObjects(text) {
106
+ const objects = [];
107
+ const raw = String(text || '');
108
+ for (let index = 0; index < raw.length; index += 1) {
109
+ if (raw[index] !== '{') continue;
110
+ let depth = 0;
111
+ for (let end = index; end < raw.length; end += 1) {
112
+ if (raw[end] === '{') depth += 1;
113
+ else if (raw[end] === '}') {
114
+ depth -= 1;
115
+ if (depth === 0) {
116
+ try {
117
+ objects.push(JSON.parse(raw.slice(index, end + 1)));
118
+ } catch {
119
+ // Not a complete JSON object; keep scanning.
120
+ }
121
+ index = end;
122
+ break;
123
+ }
124
+ }
125
+ }
126
+ }
127
+ return objects;
128
+ }
129
+
130
+ function parseHostUsage(text, parser = 'json-usage') {
131
+ if (!parser || parser === 'none') return null;
132
+ for (const obj of extractJsonObjects(text)) {
133
+ const usage = obj.usage && typeof obj.usage === 'object' ? obj.usage : obj;
134
+ const input = usage.input_tokens ?? usage.inputTokens ?? usage.prompt_tokens;
135
+ const output = usage.output_tokens ?? usage.outputTokens ?? usage.completion_tokens;
136
+ if (Number.isFinite(Number(input)) || Number.isFinite(Number(output))) {
137
+ const inputTokens = Math.max(0, Number(input) || 0);
138
+ const outputTokens = Math.max(0, Number(output) || 0);
139
+ return { inputTokens, outputTokens, totalTokens: inputTokens + outputTokens };
140
+ }
141
+ }
142
+ return null;
143
+ }
144
+
145
+ function defaultTarget({ host, model, home, env = process.env } = {}) {
146
+ const config = readUserConfig(home);
147
+ const saved = config.hosts && typeof config.hosts === 'object' ? config.hosts : {};
148
+ const enabled = Array.isArray(saved.enabled) ? saved.enabled.filter((id) => HOST_IDS.includes(id)) : [];
149
+ let hostId = host;
150
+ if (!hostId) {
151
+ if (enabled.length === 1) hostId = enabled[0];
152
+ else if (enabled.length > 1) {
153
+ throw benchError('Multiple hosts are enabled. Pass --host <id> (one target per bench).', 'BENCH_USAGE');
154
+ } else {
155
+ const detected = HOSTS.filter((item) => {
156
+ try {
157
+ return (item.detect.dirs || []).some((dir) => fs.statSync(path.join(home, dir)).isDirectory());
158
+ } catch {
159
+ return false;
160
+ }
161
+ });
162
+ if (detected.length === 1) hostId = detected[0].id;
163
+ }
164
+ }
165
+ if (!hostId) throw benchError('Pass --host <id> --model <id>. No project host is configured.', 'BENCH_USAGE');
166
+ if (!HOST_IDS.includes(hostId)) {
167
+ throw benchError(`Unknown host "${hostId}". Known: ${HOST_IDS.join(', ')}`, 'UNKNOWN_HOST');
168
+ }
169
+ const modelId = model || saved.model || saved.benchModel;
170
+ if (!modelId) throw benchError('Pass --model <id>. No project model is configured.', 'BENCH_USAGE');
171
+ return { host: getHost(hostId), model: String(modelId), env };
172
+ }
173
+
174
+ function assertSingleTarget(args) {
175
+ const hosts = countFlag(args, '--host');
176
+ const models = countFlag(args, '--model');
177
+ if (hosts > 1 || models > 1) {
178
+ throw benchError('ael bench accepts one --host and one --model per invocation', 'BENCH_USAGE');
179
+ }
180
+ }
181
+
182
+ module.exports = {
183
+ benchError,
184
+ countFlag,
185
+ flagValues,
186
+ benchSpec,
187
+ expandHome,
188
+ resolveBenchBinary,
189
+ buildBenchCommand,
190
+ parseHostUsage,
191
+ extractJsonObjects,
192
+ defaultTarget,
193
+ assertSingleTarget
194
+ };