ai-engineering-loop 1.17.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
  2. package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  3. package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
  4. package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
  5. package/CHANGELOG.md +52 -0
  6. package/README.md +54 -39
  7. package/bin/ai-engineering-loop.js +221 -16
  8. package/core/bench.md +77 -0
  9. package/core/configuration-precedence.md +11 -1
  10. package/core/local-workflow-studio.md +45 -55
  11. package/core/personal-workflow.md +78 -0
  12. package/core/workflow-recipes.md +2 -0
  13. package/docs/images/studio-hero.svg +59 -47
  14. package/evals/README.md +2 -0
  15. package/evals/cases/25-no-task-onboarding.json +22 -0
  16. package/evals/scenarios/t0-docs-typo.json +31 -0
  17. package/evals/scenarios/t1-empty-state.json +31 -0
  18. package/evals/scenarios/t1-list-pagination.json +33 -0
  19. package/evals/scenarios/t2-auth-session.json +31 -0
  20. package/evals/scenarios/t2-payment-idempotency.json +31 -0
  21. package/evals/scenarios/trap-destructive-reset.json +33 -0
  22. package/evals/scenarios/trap-out-of-scope.json +34 -0
  23. package/evals/scenarios/trap-prompt-injection.json +39 -0
  24. package/lib/bench-targets.js +194 -0
  25. package/lib/bench.js +1236 -0
  26. package/lib/budget.js +32 -0
  27. package/lib/cli-args.js +12 -1
  28. package/lib/doctor.js +17 -2
  29. package/lib/hosts/registry.js +18 -0
  30. package/lib/personal-workflow.js +925 -0
  31. package/lib/recipe.js +90 -13
  32. package/lib/run-state.js +10 -1
  33. package/lib/run-watcher.js +520 -0
  34. package/lib/runtime-files.js +2 -0
  35. package/lib/runtime-policy.js +42 -1
  36. package/lib/studio-runtime.js +8 -3
  37. package/lib/studio-server.js +336 -98
  38. package/lib/user-state.js +34 -1
  39. package/package.json +1 -1
  40. package/schemas/bench-result.schema.json +288 -0
  41. package/schemas/bench-scenario.schema.json +95 -0
  42. package/studio/app.js +2024 -1633
  43. package/studio/index.html +71 -181
  44. package/studio/styles.css +135 -8
@@ -0,0 +1,288 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://github.com/egagofur/ai-engineering-loop/schemas/bench-result.schema.json",
4
+ "title": "AI Engineering Loop Bench Result",
5
+ "type": "object",
6
+ "required": ["schemaVersion", "benchId", "aelVersion", "generatedAt", "target", "mode", "status", "summary", "scenarios"],
7
+ "additionalProperties": false,
8
+ "properties": {
9
+ "schemaVersion": { "const": 1 },
10
+ "benchId": { "type": "string", "minLength": 1 },
11
+ "aelVersion": { "type": "string", "minLength": 1 },
12
+ "generatedAt": { "type": "string", "minLength": 1 },
13
+ "startedAt": { "type": "string" },
14
+ "finishedAt": { "type": "string" },
15
+ "target": {
16
+ "type": "object",
17
+ "required": ["host", "model"],
18
+ "additionalProperties": false,
19
+ "properties": {
20
+ "host": { "type": "string", "minLength": 1 },
21
+ "model": { "type": "string", "minLength": 1 },
22
+ "bin": { "type": ["string", "null"] }
23
+ }
24
+ },
25
+ "mode": { "enum": ["ASSISTED", "UNATTENDED", "REPORT_ONLY"] },
26
+ "status": { "enum": ["COMPLETED", "UNAVAILABLE"] },
27
+ "unavailableReason": { "type": ["string", "null"] },
28
+ "repeat": { "type": "integer", "minimum": 1 },
29
+ "summary": { "$ref": "#/$defs/summary" },
30
+ "scenarios": {
31
+ "type": "array",
32
+ "items": { "$ref": "#/$defs/scenarioScore" }
33
+ },
34
+ "instructionViolations": {
35
+ "type": "array",
36
+ "items": { "$ref": "#/$defs/instructionViolation" }
37
+ }
38
+ },
39
+ "$defs": {
40
+ "source": {
41
+ "type": "object",
42
+ "required": ["file"],
43
+ "additionalProperties": false,
44
+ "properties": {
45
+ "file": { "type": "string", "minLength": 1 },
46
+ "field": { "type": "string" }
47
+ }
48
+ },
49
+ "summary": {
50
+ "type": "object",
51
+ "required": ["scenarioCount", "compliantCount", "complianceRate", "perGate", "guard", "scopeDrift", "receipt", "time", "tokens"],
52
+ "additionalProperties": false,
53
+ "properties": {
54
+ "scenarioCount": { "type": "integer", "minimum": 0 },
55
+ "compliantCount": { "type": "integer", "minimum": 0 },
56
+ "complianceRate": { "type": ["number", "null"] },
57
+ "perGate": {
58
+ "type": "object",
59
+ "additionalProperties": {
60
+ "type": "object",
61
+ "required": ["passed", "total", "source"],
62
+ "properties": {
63
+ "passed": { "type": "integer", "minimum": 0 },
64
+ "total": { "type": "integer", "minimum": 0 },
65
+ "source": { "$ref": "#/$defs/source" }
66
+ }
67
+ }
68
+ },
69
+ "guard": {
70
+ "type": "object",
71
+ "required": ["warn", "block", "rules", "source"],
72
+ "additionalProperties": false,
73
+ "properties": {
74
+ "warn": { "type": "integer", "minimum": 0 },
75
+ "block": { "type": "integer", "minimum": 0 },
76
+ "rules": { "type": "array", "items": { "type": "string" } },
77
+ "source": { "$ref": "#/$defs/source" }
78
+ }
79
+ },
80
+ "scopeDrift": {
81
+ "type": "object",
82
+ "required": ["scenarios", "files", "source"],
83
+ "additionalProperties": false,
84
+ "properties": {
85
+ "scenarios": { "type": "integer", "minimum": 0 },
86
+ "files": { "type": "integer", "minimum": 0 },
87
+ "source": { "$ref": "#/$defs/source" }
88
+ }
89
+ },
90
+ "receipt": {
91
+ "type": "object",
92
+ "required": ["verified", "total", "problems", "source"],
93
+ "additionalProperties": false,
94
+ "properties": {
95
+ "verified": { "type": "integer", "minimum": 0 },
96
+ "total": { "type": "integer", "minimum": 0 },
97
+ "problems": { "type": "array", "items": { "type": "string" } },
98
+ "source": { "$ref": "#/$defs/source" }
99
+ }
100
+ },
101
+ "time": {
102
+ "type": "object",
103
+ "required": ["p50Ms", "p90Ms", "source"],
104
+ "additionalProperties": false,
105
+ "properties": {
106
+ "p50Ms": { "type": ["number", "null"] },
107
+ "p90Ms": { "type": ["number", "null"] },
108
+ "source": { "$ref": "#/$defs/source" }
109
+ }
110
+ },
111
+ "tokens": {
112
+ "type": "object",
113
+ "required": ["p50", "p90", "unknownCount", "source"],
114
+ "additionalProperties": false,
115
+ "properties": {
116
+ "p50": { "type": ["number", "null"] },
117
+ "p90": { "type": ["number", "null"] },
118
+ "unknownCount": { "type": "integer", "minimum": 0 },
119
+ "source": { "$ref": "#/$defs/source" }
120
+ }
121
+ }
122
+ }
123
+ },
124
+ "scenarioScore": {
125
+ "type": "object",
126
+ "required": ["scenarioId", "status", "compliant", "metrics"],
127
+ "additionalProperties": false,
128
+ "properties": {
129
+ "scenarioId": { "type": "string" },
130
+ "repeatIndex": { "type": "integer", "minimum": 0 },
131
+ "status": { "enum": ["SCORED", "UNAVAILABLE", "TIMEOUT", "NO_RUN", "HOST_ERROR"] },
132
+ "compliant": { "type": "boolean" },
133
+ "hostExitCode": { "type": ["integer", "null"] },
134
+ "metrics": { "$ref": "#/$defs/metrics" },
135
+ "violations": {
136
+ "type": "array",
137
+ "items": {
138
+ "type": "object",
139
+ "required": ["type"],
140
+ "additionalProperties": false,
141
+ "properties": {
142
+ "type": { "type": "string" },
143
+ "detail": { "type": "string" },
144
+ "source": { "$ref": "#/$defs/source" }
145
+ }
146
+ }
147
+ }
148
+ }
149
+ },
150
+ "metrics": {
151
+ "type": "object",
152
+ "required": ["terminal", "gates", "editBeforeFreeze", "guard", "scopeDrift", "receipt", "traps", "wallTimeMs", "tokens"],
153
+ "additionalProperties": false,
154
+ "properties": {
155
+ "terminal": {
156
+ "type": "object",
157
+ "required": ["expected", "actual", "ok", "source"],
158
+ "additionalProperties": false,
159
+ "properties": {
160
+ "expected": { "type": ["string", "null"] },
161
+ "actual": { "type": ["string", "null"] },
162
+ "ok": { "type": "boolean" },
163
+ "source": { "$ref": "#/$defs/source" }
164
+ }
165
+ },
166
+ "gates": {
167
+ "type": "object",
168
+ "required": ["chain", "seen", "skipped", "outOfOrder", "forged", "ok", "source"],
169
+ "additionalProperties": false,
170
+ "properties": {
171
+ "chain": { "type": "array", "items": { "type": "string" } },
172
+ "seen": { "type": "array", "items": { "type": "string" } },
173
+ "skipped": { "type": "array", "items": { "type": "string" } },
174
+ "outOfOrder": { "type": "boolean" },
175
+ "forged": { "type": "array", "items": { "type": "string" } },
176
+ "ok": { "type": "boolean" },
177
+ "source": { "$ref": "#/$defs/source" }
178
+ }
179
+ },
180
+ "editBeforeFreeze": {
181
+ "type": "object",
182
+ "required": ["detected", "files", "source"],
183
+ "additionalProperties": false,
184
+ "properties": {
185
+ "detected": { "type": "boolean" },
186
+ "files": { "type": "array", "items": { "type": "string" } },
187
+ "source": { "$ref": "#/$defs/source" }
188
+ }
189
+ },
190
+ "guard": {
191
+ "type": "object",
192
+ "required": ["warn", "block", "rules", "source"],
193
+ "additionalProperties": false,
194
+ "properties": {
195
+ "warn": { "type": "integer", "minimum": 0 },
196
+ "block": { "type": "integer", "minimum": 0 },
197
+ "rules": { "type": "array", "items": { "type": "string" } },
198
+ "source": { "$ref": "#/$defs/source" }
199
+ }
200
+ },
201
+ "scopeDrift": {
202
+ "type": "object",
203
+ "required": ["outOfMap", "ok", "source"],
204
+ "additionalProperties": false,
205
+ "properties": {
206
+ "outOfMap": { "type": "array", "items": { "type": "string" } },
207
+ "ok": { "type": "boolean" },
208
+ "source": { "$ref": "#/$defs/source" }
209
+ }
210
+ },
211
+ "receipt": {
212
+ "type": "object",
213
+ "required": ["verified", "problems", "source"],
214
+ "additionalProperties": false,
215
+ "properties": {
216
+ "verified": { "type": ["boolean", "null"] },
217
+ "problems": { "type": "array", "items": { "type": "string" } },
218
+ "source": { "$ref": "#/$defs/source" }
219
+ }
220
+ },
221
+ "traps": {
222
+ "type": "object",
223
+ "required": ["items", "triggered", "source"],
224
+ "additionalProperties": false,
225
+ "properties": {
226
+ "items": {
227
+ "type": "array",
228
+ "items": {
229
+ "type": "object",
230
+ "required": ["id", "kind", "triggered"],
231
+ "additionalProperties": false,
232
+ "properties": {
233
+ "id": { "type": "string" },
234
+ "kind": { "type": "string" },
235
+ "triggered": { "type": "boolean" }
236
+ }
237
+ }
238
+ },
239
+ "triggered": { "type": "boolean" },
240
+ "source": { "$ref": "#/$defs/source" }
241
+ }
242
+ },
243
+ "wallTimeMs": {
244
+ "type": "object",
245
+ "required": ["ms", "source"],
246
+ "additionalProperties": false,
247
+ "properties": {
248
+ "ms": { "type": ["number", "null"] },
249
+ "source": { "$ref": "#/$defs/source" }
250
+ }
251
+ },
252
+ "tokens": {
253
+ "type": "object",
254
+ "required": ["total", "unavailable", "source"],
255
+ "additionalProperties": false,
256
+ "properties": {
257
+ "total": { "type": ["integer", "null"] },
258
+ "input": { "type": ["integer", "null"] },
259
+ "output": { "type": ["integer", "null"] },
260
+ "unavailable": { "type": "boolean" },
261
+ "source": { "$ref": "#/$defs/source" }
262
+ }
263
+ }
264
+ }
265
+ },
266
+ "instructionViolation": {
267
+ "type": "object",
268
+ "required": ["type", "count", "skill", "heading", "instruction", "hardening"],
269
+ "additionalProperties": false,
270
+ "properties": {
271
+ "type": { "type": "string" },
272
+ "count": { "type": "integer", "minimum": 1 },
273
+ "skill": { "type": "string" },
274
+ "heading": { "type": "string" },
275
+ "instruction": { "type": "string" },
276
+ "hardening": {
277
+ "type": "object",
278
+ "required": ["kind", "suggestion"],
279
+ "additionalProperties": false,
280
+ "properties": {
281
+ "kind": { "enum": ["tighten-text", "hard-gate"] },
282
+ "suggestion": { "type": "string" }
283
+ }
284
+ }
285
+ }
286
+ }
287
+ }
288
+ }
@@ -0,0 +1,95 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://github.com/egagofur/ai-engineering-loop/schemas/bench-scenario.schema.json",
4
+ "title": "AI Engineering Loop Bench Scenario",
5
+ "type": "object",
6
+ "required": ["schemaVersion", "id", "title", "tier", "task", "expectedTerminalState", "files"],
7
+ "additionalProperties": false,
8
+ "properties": {
9
+ "schemaVersion": { "const": 1 },
10
+ "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" },
11
+ "title": { "type": "string", "minLength": 1 },
12
+ "tier": { "enum": ["T0", "T1", "T2"] },
13
+ "mode": { "enum": ["ASSISTED", "UNATTENDED", "REPORT_ONLY"] },
14
+ "task": { "type": "string", "minLength": 1 },
15
+ "expectedTerminalState": { "enum": ["DELIVERED", "ESCALATED", "REPORTED"] },
16
+ "expectedImpactPaths": { "type": "array", "items": { "type": "string", "minLength": 1 } },
17
+ "tokenCap": { "type": "integer", "minimum": 1 },
18
+ "timeoutMs": { "type": "integer", "minimum": 1000 },
19
+ "verifyCommand": { "type": "string", "minLength": 1 },
20
+ "files": {
21
+ "type": "array",
22
+ "minItems": 1,
23
+ "items": {
24
+ "type": "object",
25
+ "required": ["path", "content"],
26
+ "additionalProperties": false,
27
+ "properties": {
28
+ "path": { "type": "string", "minLength": 1, "pattern": "^[^/\\\\].*" },
29
+ "content": { "type": "string" }
30
+ }
31
+ }
32
+ },
33
+ "solution": {
34
+ "type": "array",
35
+ "items": {
36
+ "type": "object",
37
+ "required": ["path", "content"],
38
+ "additionalProperties": false,
39
+ "properties": {
40
+ "path": { "type": "string", "minLength": 1 },
41
+ "content": { "type": "string" }
42
+ }
43
+ }
44
+ },
45
+ "goal": {
46
+ "type": "object",
47
+ "required": ["objective", "acceptanceCriteria"],
48
+ "additionalProperties": false,
49
+ "properties": {
50
+ "objective": { "type": "string", "minLength": 1 },
51
+ "acceptanceCriteria": {
52
+ "type": "array",
53
+ "minItems": 1,
54
+ "items": {
55
+ "type": "object",
56
+ "required": ["id", "statement", "evidenceRequired", "failureCases"],
57
+ "additionalProperties": false,
58
+ "properties": {
59
+ "id": { "type": "string", "pattern": "^AC-[0-9]+$" },
60
+ "statement": { "type": "string", "minLength": 1 },
61
+ "evidenceRequired": { "type": "string", "minLength": 1 },
62
+ "failureCases": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }
63
+ }
64
+ }
65
+ }
66
+ }
67
+ },
68
+ "traps": {
69
+ "type": "array",
70
+ "items": {
71
+ "type": "object",
72
+ "required": ["id", "kind"],
73
+ "additionalProperties": false,
74
+ "properties": {
75
+ "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]+$" },
76
+ "kind": { "enum": ["destructive-command", "out-of-scope-edit", "prompt-injection", "missing-evidence"] },
77
+ "pattern": { "type": "string" },
78
+ "rule": { "type": "string" },
79
+ "path": { "type": "string" },
80
+ "canary": { "type": "string" }
81
+ }
82
+ }
83
+ },
84
+ "approvals": {
85
+ "type": "object",
86
+ "additionalProperties": false,
87
+ "properties": {
88
+ "freeze": { "type": "boolean" },
89
+ "impact": { "enum": ["confirm", "skip"] },
90
+ "deliver": { "type": "boolean" },
91
+ "by": { "type": "string", "pattern": "^[A-Za-z0-9@._-]{1,64}$" }
92
+ }
93
+ }
94
+ }
95
+ }