ai-engineering-loop 1.17.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.agents/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/.claude/skills/ai-engineering-loop/SKILL.md +4 -4
- package/.claude/skills/ai-engineering-loop/references/loop-commands.md +5 -1
- package/CHANGELOG.md +52 -0
- package/README.md +54 -39
- package/bin/ai-engineering-loop.js +221 -16
- package/core/bench.md +77 -0
- package/core/configuration-precedence.md +11 -1
- package/core/local-workflow-studio.md +45 -55
- package/core/personal-workflow.md +78 -0
- package/core/workflow-recipes.md +2 -0
- package/docs/images/studio-hero.svg +59 -47
- package/evals/README.md +2 -0
- package/evals/cases/25-no-task-onboarding.json +22 -0
- package/evals/scenarios/t0-docs-typo.json +31 -0
- package/evals/scenarios/t1-empty-state.json +31 -0
- package/evals/scenarios/t1-list-pagination.json +33 -0
- package/evals/scenarios/t2-auth-session.json +31 -0
- package/evals/scenarios/t2-payment-idempotency.json +31 -0
- package/evals/scenarios/trap-destructive-reset.json +33 -0
- package/evals/scenarios/trap-out-of-scope.json +34 -0
- package/evals/scenarios/trap-prompt-injection.json +39 -0
- package/lib/bench-targets.js +194 -0
- package/lib/bench.js +1236 -0
- package/lib/budget.js +32 -0
- package/lib/cli-args.js +12 -1
- package/lib/doctor.js +17 -2
- package/lib/hosts/registry.js +18 -0
- package/lib/personal-workflow.js +925 -0
- package/lib/recipe.js +90 -13
- package/lib/run-state.js +10 -1
- package/lib/run-watcher.js +520 -0
- package/lib/runtime-files.js +2 -0
- package/lib/runtime-policy.js +42 -1
- package/lib/studio-runtime.js +8 -3
- package/lib/studio-server.js +336 -98
- package/lib/user-state.js +34 -1
- package/package.json +1 -1
- package/schemas/bench-result.schema.json +288 -0
- package/schemas/bench-scenario.schema.json +95 -0
- package/studio/app.js +2024 -1633
- package/studio/index.html +71 -181
- package/studio/styles.css +135 -8
package/lib/recipe.js
CHANGED
|
@@ -23,6 +23,40 @@ const RECIPE_ID = /^[a-z][a-z0-9-]{0,63}$/;
|
|
|
23
23
|
const NODE_ID = /^[a-z][a-z0-9-]{0,63}$/;
|
|
24
24
|
const MAX_NODES = 64;
|
|
25
25
|
const BUILTIN_RECIPE_DIR = path.join(__dirname, '..', 'recipes');
|
|
26
|
+
const MODEL_TIER_RANK = Object.freeze({
|
|
27
|
+
CHEAP_MODEL: 0,
|
|
28
|
+
STRONG_MODEL: 1,
|
|
29
|
+
HUMAN: 2
|
|
30
|
+
});
|
|
31
|
+
const BACKBONE_CODES = Object.freeze({
|
|
32
|
+
GOAL_CONTRACT: 'BACKBONE_GOAL_CONTRACT_REQUIRED',
|
|
33
|
+
MAKER: 'BACKBONE_MAKER_REQUIRED',
|
|
34
|
+
VERIFICATION: 'BACKBONE_VERIFICATION_REQUIRED',
|
|
35
|
+
DEVIL_ADVOCATE: 'BACKBONE_DEVIL_ADVOCATE_REQUIRED',
|
|
36
|
+
JUDGE: 'BACKBONE_JUDGE_REQUIRED',
|
|
37
|
+
DELIVERY: 'BACKBONE_DELIVERY_REQUIRED',
|
|
38
|
+
REPORT: 'BACKBONE_REPORT_REQUIRED',
|
|
39
|
+
CONDITIONAL_SKIP: 'BACKBONE_CONDITIONAL_SKIP',
|
|
40
|
+
ASSISTED_APPROVAL: 'BACKBONE_ASSISTED_APPROVAL_REQUIRED',
|
|
41
|
+
T2_JUDGE_MODEL: 'BACKBONE_T2_JUDGE_MODEL'
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
function backboneCodeForType(type) {
|
|
45
|
+
return {
|
|
46
|
+
'goal-contract': BACKBONE_CODES.GOAL_CONTRACT,
|
|
47
|
+
maker: BACKBONE_CODES.MAKER,
|
|
48
|
+
verification: BACKBONE_CODES.VERIFICATION,
|
|
49
|
+
'devil-advocate': BACKBONE_CODES.DEVIL_ADVOCATE,
|
|
50
|
+
judge: BACKBONE_CODES.JUDGE,
|
|
51
|
+
delivery: BACKBONE_CODES.DELIVERY,
|
|
52
|
+
report: BACKBONE_CODES.REPORT
|
|
53
|
+
}[type] || null;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function pushCoded(errors, codes, message, code) {
|
|
57
|
+
errors.push(message);
|
|
58
|
+
if (code) codes.push(code);
|
|
59
|
+
}
|
|
26
60
|
|
|
27
61
|
function recipeError(message, details = [], code = 'INVALID_RECIPE') {
|
|
28
62
|
const err = new Error(message);
|
|
@@ -100,14 +134,38 @@ function forbiddenAuthorityPaths(value, location, result = []) {
|
|
|
100
134
|
return result;
|
|
101
135
|
}
|
|
102
136
|
|
|
103
|
-
function
|
|
137
|
+
function validateJudgeModelTier(node, location, errors, codes) {
|
|
138
|
+
if (node.type !== 'judge') return;
|
|
139
|
+
const tier = node.modelTier;
|
|
140
|
+
if (tier == null || tier === '') return;
|
|
141
|
+
if (!Object.hasOwn(MODEL_TIER_RANK, tier)) {
|
|
142
|
+
errors.push(`${location}.modelTier must be CHEAP_MODEL, STRONG_MODEL, or HUMAN`);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (MODEL_TIER_RANK[tier] < MODEL_TIER_RANK.STRONG_MODEL) {
|
|
146
|
+
pushCoded(
|
|
147
|
+
errors,
|
|
148
|
+
codes,
|
|
149
|
+
'T2 judge cannot use a model below STRONG_MODEL',
|
|
150
|
+
BACKBONE_CODES.T2_JUDGE_MODEL
|
|
151
|
+
);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
function validateNodeConfiguration(node, index, errors, warnings, codes = []) {
|
|
104
156
|
const location = `nodes[${index}]`;
|
|
105
157
|
const definition = getNodeType(node.type);
|
|
106
158
|
if (!definition) return;
|
|
107
159
|
if (node.when != null) validateCondition(node.when, `${location}.when`, errors);
|
|
108
160
|
if (definition.builtIn && node.when != null) {
|
|
109
|
-
|
|
161
|
+
pushCoded(
|
|
162
|
+
errors,
|
|
163
|
+
codes,
|
|
164
|
+
`${location}.when cannot conditionally skip a required safety node`,
|
|
165
|
+
BACKBONE_CODES.CONDITIONAL_SKIP
|
|
166
|
+
);
|
|
110
167
|
}
|
|
168
|
+
validateJudgeModelTier(node, location, errors, codes);
|
|
111
169
|
|
|
112
170
|
if (node.type === 'condition') {
|
|
113
171
|
validateCondition(node.expression, `${location}.expression`, errors);
|
|
@@ -270,7 +328,7 @@ function firstNodeOfType(nodes, type) {
|
|
|
270
328
|
return nodes.find((node) => node.type === type);
|
|
271
329
|
}
|
|
272
330
|
|
|
273
|
-
function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
331
|
+
function validateSafetyBackbone(recipe, mode, byId, errors, codes = []) {
|
|
274
332
|
const nodes = recipe.nodes || [];
|
|
275
333
|
for (const type of ['goal-contract', 'maker', 'verification', 'devil-advocate', 'judge', 'report', 'delivery']) {
|
|
276
334
|
if (nodes.filter((node) => node.type === type).length > 1) {
|
|
@@ -278,11 +336,15 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
278
336
|
}
|
|
279
337
|
}
|
|
280
338
|
const goal = firstNodeOfType(nodes, 'goal-contract');
|
|
281
|
-
if (!goal)
|
|
339
|
+
if (!goal) {
|
|
340
|
+
pushCoded(errors, codes, 'a goal-contract node is required', BACKBONE_CODES.GOAL_CONTRACT);
|
|
341
|
+
}
|
|
282
342
|
|
|
283
343
|
if (mode === RUN_MODES.REPORT_ONLY) {
|
|
284
344
|
const report = firstNodeOfType(nodes, 'report');
|
|
285
|
-
if (!report)
|
|
345
|
+
if (!report) {
|
|
346
|
+
pushCoded(errors, codes, 'REPORT_ONLY requires a report node', BACKBONE_CODES.REPORT);
|
|
347
|
+
}
|
|
286
348
|
if (nodes.some((node) => getNodeType(node.type)?.mutation)) {
|
|
287
349
|
errors.push('REPORT_ONLY cannot contain mutation nodes');
|
|
288
350
|
}
|
|
@@ -301,7 +363,10 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
301
363
|
const chainTypes = ['goal-contract', 'maker', 'verification', 'devil-advocate', 'judge', 'delivery'];
|
|
302
364
|
const chain = chainTypes.map((type) => firstNodeOfType(nodes, type));
|
|
303
365
|
chain.forEach((node, index) => {
|
|
304
|
-
if (!node)
|
|
366
|
+
if (!node) {
|
|
367
|
+
const type = chainTypes[index];
|
|
368
|
+
pushCoded(errors, codes, `${type} node is required for ${mode}`, backboneCodeForType(type));
|
|
369
|
+
}
|
|
305
370
|
});
|
|
306
371
|
for (let index = 1; index < chain.length; index++) {
|
|
307
372
|
if (chain[index] && chain[index - 1] && !dependsTransitively(byId, chain[index].id, chain[index - 1].id)) {
|
|
@@ -318,9 +383,11 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
318
383
|
)) && judge && dependsTransitively(byId, node.id, judge.id)
|
|
319
384
|
));
|
|
320
385
|
const delivery = firstNodeOfType(nodes, 'delivery');
|
|
321
|
-
if (!approval)
|
|
386
|
+
if (!approval) {
|
|
387
|
+
pushCoded(errors, codes, 'ASSISTED requires a human approval node', BACKBONE_CODES.ASSISTED_APPROVAL);
|
|
388
|
+
}
|
|
322
389
|
if (delivery && approval && !dependsTransitively(byId, delivery.id, approval.id)) {
|
|
323
|
-
errors
|
|
390
|
+
pushCoded(errors, codes, 'delivery must depend on ASSISTED approval', BACKBONE_CODES.ASSISTED_APPROVAL);
|
|
324
391
|
}
|
|
325
392
|
}
|
|
326
393
|
}
|
|
@@ -328,8 +395,9 @@ function validateSafetyBackbone(recipe, mode, byId, errors) {
|
|
|
328
395
|
function validateRecipe(recipe, { mode } = {}) {
|
|
329
396
|
const errors = [];
|
|
330
397
|
const warnings = [];
|
|
398
|
+
const codes = [];
|
|
331
399
|
if (!recipe || typeof recipe !== 'object' || Array.isArray(recipe)) {
|
|
332
|
-
return { valid: false, errors: ['recipe must be a JSON object'], warnings, order: [] };
|
|
400
|
+
return { valid: false, errors: ['recipe must be a JSON object'], warnings, order: [], codes };
|
|
333
401
|
}
|
|
334
402
|
recipe = migrateLegacyRecipe(recipe);
|
|
335
403
|
if (recipe.schemaVersion !== 1) errors.push('schemaVersion must be 1');
|
|
@@ -371,7 +439,7 @@ function validateRecipe(recipe, { mode } = {}) {
|
|
|
371
439
|
)) {
|
|
372
440
|
errors.push(`${location}.dependsOn must be an array of node ids`);
|
|
373
441
|
}
|
|
374
|
-
validateNodeConfiguration(node, index, errors, warnings);
|
|
442
|
+
validateNodeConfiguration(node, index, errors, warnings, codes);
|
|
375
443
|
});
|
|
376
444
|
if (!Array.isArray(recipe.edges)) {
|
|
377
445
|
errors.push('edges must be an array');
|
|
@@ -426,9 +494,15 @@ function validateRecipe(recipe, { mode } = {}) {
|
|
|
426
494
|
errors.push(`recipe ${recipe.id || '(unknown)'} is not compatible with ${selectedMode}`);
|
|
427
495
|
continue;
|
|
428
496
|
}
|
|
429
|
-
validateSafetyBackbone(recipe, selectedMode, byId, errors);
|
|
497
|
+
validateSafetyBackbone(recipe, selectedMode, byId, errors, codes);
|
|
430
498
|
}
|
|
431
|
-
return {
|
|
499
|
+
return {
|
|
500
|
+
valid: errors.length === 0,
|
|
501
|
+
errors: [...new Set(errors)],
|
|
502
|
+
warnings: [...new Set(warnings)],
|
|
503
|
+
codes: [...new Set(codes)],
|
|
504
|
+
order
|
|
505
|
+
};
|
|
432
506
|
}
|
|
433
507
|
|
|
434
508
|
function compileRecipe(recipe, { mode } = {}) {
|
|
@@ -436,7 +510,9 @@ function compileRecipe(recipe, { mode } = {}) {
|
|
|
436
510
|
const selectedMode = normalizeRunMode(mode || recipe?.compatibleModes?.[0]);
|
|
437
511
|
const validation = validateRecipe(recipe, { mode: selectedMode });
|
|
438
512
|
if (!validation.valid) {
|
|
439
|
-
|
|
513
|
+
const code = (validation.codes || []).find((item) => String(item).startsWith('BACKBONE_'))
|
|
514
|
+
|| 'INVALID_RECIPE';
|
|
515
|
+
throw recipeError(`Recipe ${recipe?.id || '(unknown)'} is invalid`, validation.errors, code);
|
|
440
516
|
}
|
|
441
517
|
const byId = new Map(recipe.nodes.map((node) => [node.id, node]));
|
|
442
518
|
const compiledNodes = validation.order.map((id, index) => {
|
|
@@ -615,6 +691,7 @@ module.exports = {
|
|
|
615
691
|
MAX_NODES,
|
|
616
692
|
MAX_EDGES,
|
|
617
693
|
BUILTIN_RECIPE_DIR,
|
|
694
|
+
BACKBONE_CODES,
|
|
618
695
|
canonicalize,
|
|
619
696
|
canonicalJson,
|
|
620
697
|
sha256,
|
package/lib/run-state.js
CHANGED
|
@@ -163,7 +163,9 @@ function writeWorkflowBundle(rootDir, runId, workflow) {
|
|
|
163
163
|
recipeId: plan.recipe.id,
|
|
164
164
|
recipeVersion: plan.recipe.version,
|
|
165
165
|
sourceHash: plan.recipe.sourceHash,
|
|
166
|
-
graphHash: plan.graphHash
|
|
166
|
+
graphHash: plan.graphHash,
|
|
167
|
+
...(workflow.workflowSource ? { workflowSource: workflow.workflowSource } : {}),
|
|
168
|
+
...(workflow.workflowReason ? { workflowReason: workflow.workflowReason } : {})
|
|
167
169
|
};
|
|
168
170
|
}
|
|
169
171
|
|
|
@@ -383,6 +385,13 @@ function transitionRun(
|
|
|
383
385
|
};
|
|
384
386
|
atomicWriteJson(runStatePath(rootDir, runId), next);
|
|
385
387
|
atomicWriteJson(currentRunPath(rootDir), { schemaVersion: 1, runId });
|
|
388
|
+
if (TERMINAL_STATES.has(to)) {
|
|
389
|
+
try {
|
|
390
|
+
require('./personal-workflow.js').appendRunSummary(rootDir, next, { now });
|
|
391
|
+
} catch {
|
|
392
|
+
// The run summary is advisory history; it must never undo a terminal transition.
|
|
393
|
+
}
|
|
394
|
+
}
|
|
386
395
|
return next;
|
|
387
396
|
}
|
|
388
397
|
|