@sweny-ai/core 0.1.58 → 0.1.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/schema.test.js +1 -1
- package/dist/__tests__/spec-conformance.test.d.ts +18 -0
- package/dist/__tests__/spec-conformance.test.js +1074 -0
- package/dist/schema.d.ts +3 -3
- package/dist/schema.js +3 -3
- package/dist/studio.d.ts +3 -0
- package/dist/studio.js +5 -2
- package/package.json +1 -1
|
@@ -410,7 +410,7 @@ describe("validateWorkflow", () => {
|
|
|
410
410
|
// ─── JSON Schema shape tests ─────────────────────────────────────
|
|
411
411
|
describe("workflowJsonSchema", () => {
|
|
412
412
|
it("has the correct $id", () => {
|
|
413
|
-
expect(workflowJsonSchema.$id).toBe("https://sweny.ai/schemas/workflow.json");
|
|
413
|
+
expect(workflowJsonSchema.$id).toBe("https://spec.sweny.ai/schemas/workflow.json");
|
|
414
414
|
});
|
|
415
415
|
it("requires all top-level fields", () => {
|
|
416
416
|
expect(workflowJsonSchema.required).toContain("id");
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spec Conformance Tests
|
|
3
|
+
*
|
|
4
|
+
* These tests validate that the @sweny-ai/core implementation conforms to the
|
|
5
|
+
* SWEny Workflow Specification v1.0 (spec.sweny.ai). Each test references
|
|
6
|
+
* the relevant spec section.
|
|
7
|
+
*
|
|
8
|
+
* Sections covered:
|
|
9
|
+
* - Workflow structural validation
|
|
10
|
+
* - Node input augmentation (rules, context, skill instructions)
|
|
11
|
+
* - Edge routing algorithm
|
|
12
|
+
* - Dry-run semantics
|
|
13
|
+
* - Execution events ordering
|
|
14
|
+
* - Execution trace structure
|
|
15
|
+
* - Source resolution
|
|
16
|
+
* - JSON Schema correctness
|
|
17
|
+
*/
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,1074 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Spec Conformance Tests
|
|
3
|
+
*
|
|
4
|
+
* These tests validate that the @sweny-ai/core implementation conforms to the
|
|
5
|
+
* SWEny Workflow Specification v1.0 (spec.sweny.ai). Each test references
|
|
6
|
+
* the relevant spec section.
|
|
7
|
+
*
|
|
8
|
+
* Sections covered:
|
|
9
|
+
* - Workflow structural validation
|
|
10
|
+
* - Node input augmentation (rules, context, skill instructions)
|
|
11
|
+
* - Edge routing algorithm
|
|
12
|
+
* - Dry-run semantics
|
|
13
|
+
* - Execution events ordering
|
|
14
|
+
* - Execution trace structure
|
|
15
|
+
* - Source resolution
|
|
16
|
+
* - JSON Schema correctness
|
|
17
|
+
*/
|
|
18
|
+
import { describe, it, expect } from "vitest";
|
|
19
|
+
import { mkdirSync, writeFileSync, rmSync } from "node:fs";
|
|
20
|
+
import * as path from "node:path";
|
|
21
|
+
import { tmpdir } from "node:os";
|
|
22
|
+
import { execute } from "../executor.js";
|
|
23
|
+
import { validateWorkflow, workflowJsonSchema, parseWorkflow } from "../schema.js";
|
|
24
|
+
import { createSkillMap } from "../skills/index.js";
|
|
25
|
+
// ─── Helpers ────────────────────────────────────────────────────
|
|
26
|
+
const tmpBase = path.join(tmpdir(), "sweny-spec-conformance");
|
|
27
|
+
function freshDir(name) {
|
|
28
|
+
const dir = path.join(tmpBase, `${name}-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`);
|
|
29
|
+
rmSync(dir, { recursive: true, force: true });
|
|
30
|
+
mkdirSync(dir, { recursive: true });
|
|
31
|
+
return dir;
|
|
32
|
+
}
|
|
33
|
+
/** A Claude mock that captures instructions and contexts for assertion. */
|
|
34
|
+
function capturingClaude(opts) {
|
|
35
|
+
const captured = { instructions: {}, contexts: {}, runCalls: 0, evaluateCalls: 0 };
|
|
36
|
+
const claude = {
|
|
37
|
+
async run(runOpts) {
|
|
38
|
+
captured.runCalls++;
|
|
39
|
+
// Determine which node this is from the instruction
|
|
40
|
+
const nodeId = Object.keys(opts.data ?? {}).find((k) => runOpts.instruction.includes(k) || runOpts.instruction.includes(opts.data[k]?.__instruction));
|
|
41
|
+
// Store by run order since we may not know nodeId
|
|
42
|
+
const key = `run-${captured.runCalls}`;
|
|
43
|
+
captured.instructions[key] = runOpts.instruction;
|
|
44
|
+
captured.contexts[key] = runOpts.context;
|
|
45
|
+
return {
|
|
46
|
+
status: "success",
|
|
47
|
+
data: opts.data?.[nodeId ?? ""] ?? { step: captured.runCalls },
|
|
48
|
+
toolCalls: [],
|
|
49
|
+
};
|
|
50
|
+
},
|
|
51
|
+
async evaluate(evalOpts) {
|
|
52
|
+
captured.evaluateCalls++;
|
|
53
|
+
const from = evalOpts.choices[0]?.id;
|
|
54
|
+
// Find the route from the choices
|
|
55
|
+
for (const [_from, to] of Object.entries(opts.routes ?? {})) {
|
|
56
|
+
const match = evalOpts.choices.find((c) => c.id === to);
|
|
57
|
+
if (match)
|
|
58
|
+
return match.id;
|
|
59
|
+
}
|
|
60
|
+
return evalOpts.choices[0].id;
|
|
61
|
+
},
|
|
62
|
+
};
|
|
63
|
+
return { claude, captured };
|
|
64
|
+
}
|
|
65
|
+
/** Simple mock claude that just returns data per node based on instruction matching. */
|
|
66
|
+
function simpleClaude(nodeData, routes) {
|
|
67
|
+
const nodeInstructions = new Map();
|
|
68
|
+
let currentNodeId = "";
|
|
69
|
+
return {
|
|
70
|
+
async run(opts) {
|
|
71
|
+
// Find which node this is by checking if instruction ends with node text
|
|
72
|
+
const matchedNode = Object.entries(nodeData).find(([_id, _data]) => true);
|
|
73
|
+
return {
|
|
74
|
+
status: "success",
|
|
75
|
+
data: nodeData[currentNodeId] ?? {},
|
|
76
|
+
toolCalls: [],
|
|
77
|
+
};
|
|
78
|
+
},
|
|
79
|
+
async evaluate(opts) {
|
|
80
|
+
// Use routes map based on choices available
|
|
81
|
+
if (routes) {
|
|
82
|
+
for (const [from, to] of Object.entries(routes)) {
|
|
83
|
+
const match = opts.choices.find((c) => c.id === to);
|
|
84
|
+
if (match)
|
|
85
|
+
return match.id;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
return opts.choices[0].id;
|
|
89
|
+
},
|
|
90
|
+
// Hook to track current node
|
|
91
|
+
_setNodeId(id) {
|
|
92
|
+
currentNodeId = id;
|
|
93
|
+
},
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
// ─── Fixtures ───────────────────────────────────────────────────
|
|
97
|
+
const triageWorkflow = {
|
|
98
|
+
id: "triage",
|
|
99
|
+
name: "Alert Triage",
|
|
100
|
+
description: "Investigate and act on a production alert.",
|
|
101
|
+
entry: "gather",
|
|
102
|
+
nodes: {
|
|
103
|
+
gather: { name: "Gather Context", instruction: "Gather error details.", skills: [] },
|
|
104
|
+
investigate: { name: "Investigate", instruction: "Classify issues.", skills: [] },
|
|
105
|
+
create_issue: { name: "Create Issue", instruction: "Create issues.", skills: [] },
|
|
106
|
+
skip: { name: "Skip", instruction: "Log and skip.", skills: [] },
|
|
107
|
+
notify: { name: "Notify", instruction: "Notify the team.", skills: [] },
|
|
108
|
+
},
|
|
109
|
+
edges: [
|
|
110
|
+
{ from: "gather", to: "investigate" },
|
|
111
|
+
{ from: "investigate", to: "create_issue", when: "novel_count > 0 AND severity is medium or higher" },
|
|
112
|
+
{ from: "investigate", to: "skip", when: "novel_count is 0 OR severity is low" },
|
|
113
|
+
{ from: "create_issue", to: "notify" },
|
|
114
|
+
{ from: "skip", to: "notify" },
|
|
115
|
+
],
|
|
116
|
+
};
|
|
117
|
+
// ─── Spec Section: Workflow Structural Validation ───────────────
|
|
118
|
+
describe("spec: Workflow structural validation", () => {
|
|
119
|
+
it("MISSING_ENTRY — entry node must exist", () => {
|
|
120
|
+
const w = {
|
|
121
|
+
id: "t",
|
|
122
|
+
name: "T",
|
|
123
|
+
description: "",
|
|
124
|
+
entry: "ghost",
|
|
125
|
+
nodes: { a: { name: "A", instruction: "x", skills: [] } },
|
|
126
|
+
edges: [],
|
|
127
|
+
};
|
|
128
|
+
const errs = validateWorkflow(w);
|
|
129
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "MISSING_ENTRY" }));
|
|
130
|
+
});
|
|
131
|
+
it("UNKNOWN_EDGE_SOURCE — edge from must reference existing node", () => {
|
|
132
|
+
const w = {
|
|
133
|
+
id: "t",
|
|
134
|
+
name: "T",
|
|
135
|
+
description: "",
|
|
136
|
+
entry: "a",
|
|
137
|
+
nodes: { a: { name: "A", instruction: "x", skills: [] } },
|
|
138
|
+
edges: [{ from: "ghost", to: "a" }],
|
|
139
|
+
};
|
|
140
|
+
const errs = validateWorkflow(w);
|
|
141
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "UNKNOWN_EDGE_SOURCE" }));
|
|
142
|
+
});
|
|
143
|
+
it("UNKNOWN_EDGE_TARGET — edge to must reference existing node", () => {
|
|
144
|
+
const w = {
|
|
145
|
+
id: "t",
|
|
146
|
+
name: "T",
|
|
147
|
+
description: "",
|
|
148
|
+
entry: "a",
|
|
149
|
+
nodes: { a: { name: "A", instruction: "x", skills: [] } },
|
|
150
|
+
edges: [{ from: "a", to: "ghost" }],
|
|
151
|
+
};
|
|
152
|
+
const errs = validateWorkflow(w);
|
|
153
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "UNKNOWN_EDGE_TARGET" }));
|
|
154
|
+
});
|
|
155
|
+
it("UNREACHABLE_NODE — all nodes must be reachable from entry via BFS", () => {
|
|
156
|
+
const w = {
|
|
157
|
+
id: "t",
|
|
158
|
+
name: "T",
|
|
159
|
+
description: "",
|
|
160
|
+
entry: "a",
|
|
161
|
+
nodes: {
|
|
162
|
+
a: { name: "A", instruction: "x", skills: [] },
|
|
163
|
+
b: { name: "B", instruction: "x", skills: [] },
|
|
164
|
+
island: { name: "Island", instruction: "x", skills: [] },
|
|
165
|
+
},
|
|
166
|
+
edges: [{ from: "a", to: "b" }],
|
|
167
|
+
};
|
|
168
|
+
const errs = validateWorkflow(w);
|
|
169
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "UNREACHABLE_NODE", nodeId: "island" }));
|
|
170
|
+
});
|
|
171
|
+
it("SELF_LOOP — self-loop without max_iterations is invalid", () => {
|
|
172
|
+
const w = {
|
|
173
|
+
id: "t",
|
|
174
|
+
name: "T",
|
|
175
|
+
description: "",
|
|
176
|
+
entry: "a",
|
|
177
|
+
nodes: { a: { name: "A", instruction: "x", skills: [] } },
|
|
178
|
+
edges: [{ from: "a", to: "a" }],
|
|
179
|
+
};
|
|
180
|
+
const errs = validateWorkflow(w);
|
|
181
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "SELF_LOOP" }));
|
|
182
|
+
});
|
|
183
|
+
it("self-loop with max_iterations is valid", () => {
|
|
184
|
+
const w = {
|
|
185
|
+
id: "t",
|
|
186
|
+
name: "T",
|
|
187
|
+
description: "",
|
|
188
|
+
entry: "a",
|
|
189
|
+
nodes: { a: { name: "A", instruction: "x", skills: [] } },
|
|
190
|
+
edges: [{ from: "a", to: "a", when: "retry", max_iterations: 3 }],
|
|
191
|
+
};
|
|
192
|
+
const errs = validateWorkflow(w);
|
|
193
|
+
expect(errs.filter((e) => e.code === "SELF_LOOP")).toHaveLength(0);
|
|
194
|
+
});
|
|
195
|
+
it("UNBOUNDED_CYCLE — cycle with no max_iterations is invalid", () => {
|
|
196
|
+
const w = {
|
|
197
|
+
id: "t",
|
|
198
|
+
name: "T",
|
|
199
|
+
description: "",
|
|
200
|
+
entry: "a",
|
|
201
|
+
nodes: {
|
|
202
|
+
a: { name: "A", instruction: "x", skills: [] },
|
|
203
|
+
b: { name: "B", instruction: "x", skills: [] },
|
|
204
|
+
},
|
|
205
|
+
edges: [
|
|
206
|
+
{ from: "a", to: "b" },
|
|
207
|
+
{ from: "b", to: "a" },
|
|
208
|
+
],
|
|
209
|
+
};
|
|
210
|
+
const errs = validateWorkflow(w);
|
|
211
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "UNBOUNDED_CYCLE" }));
|
|
212
|
+
});
|
|
213
|
+
it("bounded cycle with max_iterations is valid", () => {
|
|
214
|
+
const w = {
|
|
215
|
+
id: "t",
|
|
216
|
+
name: "T",
|
|
217
|
+
description: "",
|
|
218
|
+
entry: "a",
|
|
219
|
+
nodes: {
|
|
220
|
+
a: { name: "A", instruction: "x", skills: [] },
|
|
221
|
+
b: { name: "B", instruction: "x", skills: [] },
|
|
222
|
+
done: { name: "Done", instruction: "x", skills: [] },
|
|
223
|
+
},
|
|
224
|
+
edges: [
|
|
225
|
+
{ from: "a", to: "b" },
|
|
226
|
+
{ from: "b", to: "a", when: "retry", max_iterations: 3 },
|
|
227
|
+
{ from: "b", to: "done", when: "done" },
|
|
228
|
+
],
|
|
229
|
+
};
|
|
230
|
+
const errs = validateWorkflow(w);
|
|
231
|
+
expect(errs.filter((e) => e.code === "UNBOUNDED_CYCLE")).toHaveLength(0);
|
|
232
|
+
});
|
|
233
|
+
it("UNKNOWN_SKILL — node references unknown skill (when knownSkills provided)", () => {
|
|
234
|
+
const w = {
|
|
235
|
+
id: "t",
|
|
236
|
+
name: "T",
|
|
237
|
+
description: "",
|
|
238
|
+
entry: "a",
|
|
239
|
+
nodes: { a: { name: "A", instruction: "x", skills: ["nonexistent"] } },
|
|
240
|
+
edges: [],
|
|
241
|
+
};
|
|
242
|
+
const errs = validateWorkflow(w, new Set(["github"]));
|
|
243
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "UNKNOWN_SKILL" }));
|
|
244
|
+
});
|
|
245
|
+
it("INVALID_INLINE_SKILL — inline skill without instruction or mcp", () => {
|
|
246
|
+
const w = {
|
|
247
|
+
id: "t",
|
|
248
|
+
name: "T",
|
|
249
|
+
description: "",
|
|
250
|
+
entry: "a",
|
|
251
|
+
nodes: { a: { name: "A", instruction: "x", skills: [] } },
|
|
252
|
+
edges: [],
|
|
253
|
+
skills: { "bad-skill": { name: "Bad" } },
|
|
254
|
+
};
|
|
255
|
+
const errs = validateWorkflow(w);
|
|
256
|
+
expect(errs).toContainEqual(expect.objectContaining({ code: "INVALID_INLINE_SKILL" }));
|
|
257
|
+
});
|
|
258
|
+
it("valid workflow with inline skills passes validation", () => {
|
|
259
|
+
const w = {
|
|
260
|
+
id: "t",
|
|
261
|
+
name: "T",
|
|
262
|
+
description: "",
|
|
263
|
+
entry: "a",
|
|
264
|
+
nodes: { a: { name: "A", instruction: "x", skills: ["rubric"] } },
|
|
265
|
+
edges: [],
|
|
266
|
+
skills: {
|
|
267
|
+
rubric: { instruction: "Score each dimension 1-5." },
|
|
268
|
+
},
|
|
269
|
+
};
|
|
270
|
+
const errs = validateWorkflow(w, new Set());
|
|
271
|
+
expect(errs).toHaveLength(0);
|
|
272
|
+
});
|
|
273
|
+
});
|
|
274
|
+
// ─── Spec Section: Dry-Run Semantics ────────────────────────────
|
|
275
|
+
describe("spec: Dry-run semantics", () => {
|
|
276
|
+
it("stops at first node with conditional outgoing edges", async () => {
|
|
277
|
+
const claude = {
|
|
278
|
+
async run() {
|
|
279
|
+
return { status: "success", data: { severity: "high" }, toolCalls: [] };
|
|
280
|
+
},
|
|
281
|
+
async evaluate() {
|
|
282
|
+
throw new Error("evaluate should NOT be called in dry-run");
|
|
283
|
+
},
|
|
284
|
+
};
|
|
285
|
+
const events = [];
|
|
286
|
+
const { results, trace } = await execute(triageWorkflow, { dryRun: true }, { skills: createSkillMap([]), claude, observer: (e) => events.push(e), config: {} });
|
|
287
|
+
// gather (unconditional → investigate) should execute
|
|
288
|
+
expect(results.has("gather")).toBe(true);
|
|
289
|
+
expect(results.has("investigate")).toBe(true);
|
|
290
|
+
// investigate has conditional edges → dry-run stops here
|
|
291
|
+
expect(results.has("create_issue")).toBe(false);
|
|
292
|
+
expect(results.has("skip")).toBe(false);
|
|
293
|
+
expect(results.has("notify")).toBe(false);
|
|
294
|
+
// Route event should show "dry run" as reason
|
|
295
|
+
const routeEvents = events.filter((e) => e.type === "route");
|
|
296
|
+
const dryRunRoute = routeEvents.find((e) => e.type === "route" && e.to === "(end)" && e.reason === "dry run");
|
|
297
|
+
expect(dryRunRoute).toBeDefined();
|
|
298
|
+
});
|
|
299
|
+
it("continues through unconditional edges in dry-run", async () => {
|
|
300
|
+
const linearWorkflow = {
|
|
301
|
+
id: "linear",
|
|
302
|
+
name: "Linear",
|
|
303
|
+
description: "",
|
|
304
|
+
entry: "a",
|
|
305
|
+
nodes: {
|
|
306
|
+
a: { name: "A", instruction: "Do A", skills: [] },
|
|
307
|
+
b: { name: "B", instruction: "Do B", skills: [] },
|
|
308
|
+
c: { name: "C", instruction: "Do C", skills: [] },
|
|
309
|
+
},
|
|
310
|
+
edges: [
|
|
311
|
+
{ from: "a", to: "b" },
|
|
312
|
+
{ from: "b", to: "c" },
|
|
313
|
+
],
|
|
314
|
+
};
|
|
315
|
+
const claude = {
|
|
316
|
+
async run() {
|
|
317
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
318
|
+
},
|
|
319
|
+
async evaluate() {
|
|
320
|
+
throw new Error("should not evaluate");
|
|
321
|
+
},
|
|
322
|
+
};
|
|
323
|
+
const { results } = await execute(linearWorkflow, { dryRun: true }, { skills: createSkillMap([]), claude, config: {} });
|
|
324
|
+
// All nodes in a linear (unconditional) workflow execute even in dry-run
|
|
325
|
+
expect(results.size).toBe(3);
|
|
326
|
+
});
|
|
327
|
+
});
|
|
328
|
+
// ─── Spec Section: Input Augmentation ───────────────────────────
|
|
329
|
+
describe("spec: Node input augmentation", () => {
|
|
330
|
+
it("prepends rules with correct heading", async () => {
|
|
331
|
+
let capturedInstruction = "";
|
|
332
|
+
const claude = {
|
|
333
|
+
async run(opts) {
|
|
334
|
+
capturedInstruction = opts.instruction;
|
|
335
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
336
|
+
},
|
|
337
|
+
async evaluate() {
|
|
338
|
+
return "a";
|
|
339
|
+
},
|
|
340
|
+
};
|
|
341
|
+
const w = {
|
|
342
|
+
id: "t",
|
|
343
|
+
name: "T",
|
|
344
|
+
description: "",
|
|
345
|
+
entry: "a",
|
|
346
|
+
nodes: { a: { name: "A", instruction: "Do the thing.", skills: [] } },
|
|
347
|
+
edges: [],
|
|
348
|
+
};
|
|
349
|
+
await execute(w, { rules: "Never use deprecated APIs." }, { skills: createSkillMap([]), claude, config: {} });
|
|
350
|
+
expect(capturedInstruction).toContain("## Rules — You MUST Follow These");
|
|
351
|
+
expect(capturedInstruction).toContain("Never use deprecated APIs.");
|
|
352
|
+
expect(capturedInstruction).toContain("Do the thing.");
|
|
353
|
+
});
|
|
354
|
+
it("prepends context with correct heading", async () => {
|
|
355
|
+
let capturedInstruction = "";
|
|
356
|
+
const claude = {
|
|
357
|
+
async run(opts) {
|
|
358
|
+
capturedInstruction = opts.instruction;
|
|
359
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
360
|
+
},
|
|
361
|
+
async evaluate() {
|
|
362
|
+
return "a";
|
|
363
|
+
},
|
|
364
|
+
};
|
|
365
|
+
const w = {
|
|
366
|
+
id: "t",
|
|
367
|
+
name: "T",
|
|
368
|
+
description: "",
|
|
369
|
+
entry: "a",
|
|
370
|
+
nodes: { a: { name: "A", instruction: "Investigate.", skills: [] } },
|
|
371
|
+
edges: [],
|
|
372
|
+
};
|
|
373
|
+
await execute(w, { context: "Server crashed at 3am." }, { skills: createSkillMap([]), claude, config: {} });
|
|
374
|
+
expect(capturedInstruction).toContain("## Background Context");
|
|
375
|
+
expect(capturedInstruction).toContain("Server crashed at 3am.");
|
|
376
|
+
});
|
|
377
|
+
it("assembles in correct order: rules → context → skill instructions → base instruction", async () => {
|
|
378
|
+
let capturedInstruction = "";
|
|
379
|
+
const claude = {
|
|
380
|
+
async run(opts) {
|
|
381
|
+
capturedInstruction = opts.instruction;
|
|
382
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
383
|
+
},
|
|
384
|
+
async evaluate() {
|
|
385
|
+
return "a";
|
|
386
|
+
},
|
|
387
|
+
};
|
|
388
|
+
const instructionSkill = {
|
|
389
|
+
id: "rubric",
|
|
390
|
+
name: "Severity Rubric",
|
|
391
|
+
description: "Scoring rubric",
|
|
392
|
+
category: "general",
|
|
393
|
+
config: {},
|
|
394
|
+
tools: [],
|
|
395
|
+
instruction: "Score severity 1-5 based on impact.",
|
|
396
|
+
};
|
|
397
|
+
const w = {
|
|
398
|
+
id: "t",
|
|
399
|
+
name: "T",
|
|
400
|
+
description: "",
|
|
401
|
+
entry: "a",
|
|
402
|
+
nodes: { a: { name: "A", instruction: "Classify the alert.", skills: ["rubric"] } },
|
|
403
|
+
edges: [],
|
|
404
|
+
};
|
|
405
|
+
await execute(w, { rules: "Be thorough.", context: "Prod is down." }, { skills: createSkillMap([instructionSkill]), claude, config: {} });
|
|
406
|
+
// Verify order: rules first, then context, then skill, then base instruction
|
|
407
|
+
const rulesIdx = capturedInstruction.indexOf("## Rules — You MUST Follow These");
|
|
408
|
+
const contextIdx = capturedInstruction.indexOf("## Background Context");
|
|
409
|
+
const skillIdx = capturedInstruction.indexOf("## Skill: Severity Rubric");
|
|
410
|
+
const baseIdx = capturedInstruction.indexOf("Classify the alert.");
|
|
411
|
+
expect(rulesIdx).toBeGreaterThanOrEqual(0);
|
|
412
|
+
expect(contextIdx).toBeGreaterThan(rulesIdx);
|
|
413
|
+
expect(skillIdx).toBeGreaterThan(contextIdx);
|
|
414
|
+
expect(baseIdx).toBeGreaterThan(skillIdx);
|
|
415
|
+
});
|
|
416
|
+
it("sections are separated by ---", async () => {
|
|
417
|
+
let capturedInstruction = "";
|
|
418
|
+
const claude = {
|
|
419
|
+
async run(opts) {
|
|
420
|
+
capturedInstruction = opts.instruction;
|
|
421
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
422
|
+
},
|
|
423
|
+
async evaluate() {
|
|
424
|
+
return "a";
|
|
425
|
+
},
|
|
426
|
+
};
|
|
427
|
+
const w = {
|
|
428
|
+
id: "t",
|
|
429
|
+
name: "T",
|
|
430
|
+
description: "",
|
|
431
|
+
entry: "a",
|
|
432
|
+
nodes: { a: { name: "A", instruction: "Base instruction.", skills: [] } },
|
|
433
|
+
edges: [],
|
|
434
|
+
};
|
|
435
|
+
await execute(w, { rules: "Rule one.", context: "Background info." }, { skills: createSkillMap([]), claude, config: {} });
|
|
436
|
+
expect(capturedInstruction).toContain("---");
|
|
437
|
+
// Count separators: rules + context + base = 2 separators
|
|
438
|
+
const separatorCount = capturedInstruction.split("---").length - 1;
|
|
439
|
+
expect(separatorCount).toBe(2);
|
|
440
|
+
});
|
|
441
|
+
});
|
|
442
|
+
// ─── Spec Section: Context Accumulation ─────────────────────────
|
|
443
|
+
describe("spec: Context accumulation", () => {
|
|
444
|
+
it("context includes input and all prior node results", async () => {
|
|
445
|
+
const contexts = [];
|
|
446
|
+
const claude = {
|
|
447
|
+
async run(opts) {
|
|
448
|
+
contexts.push(structuredClone(opts.context));
|
|
449
|
+
return { status: "success", data: { step: contexts.length }, toolCalls: [] };
|
|
450
|
+
},
|
|
451
|
+
async evaluate() {
|
|
452
|
+
return "a";
|
|
453
|
+
},
|
|
454
|
+
};
|
|
455
|
+
const w = {
|
|
456
|
+
id: "t",
|
|
457
|
+
name: "T",
|
|
458
|
+
description: "",
|
|
459
|
+
entry: "a",
|
|
460
|
+
nodes: {
|
|
461
|
+
a: { name: "A", instruction: "Step A", skills: [] },
|
|
462
|
+
b: { name: "B", instruction: "Step B", skills: [] },
|
|
463
|
+
c: { name: "C", instruction: "Step C", skills: [] },
|
|
464
|
+
},
|
|
465
|
+
edges: [
|
|
466
|
+
{ from: "a", to: "b" },
|
|
467
|
+
{ from: "b", to: "c" },
|
|
468
|
+
],
|
|
469
|
+
};
|
|
470
|
+
await execute(w, { alert: "test" }, { skills: createSkillMap([]), claude, config: {} });
|
|
471
|
+
// Node A: context has only input
|
|
472
|
+
expect(contexts[0]).toEqual({ input: { alert: "test" } });
|
|
473
|
+
// Node B: context has input + A's result
|
|
474
|
+
expect(contexts[1]).toEqual({ input: { alert: "test" }, a: { step: 1 } });
|
|
475
|
+
// Node C: context has input + A + B results
|
|
476
|
+
expect(contexts[2]).toEqual({ input: { alert: "test" }, a: { step: 1 }, b: { step: 2 } });
|
|
477
|
+
});
|
|
478
|
+
it("bounded cycle overwrites context with most recent result", async () => {
|
|
479
|
+
const contexts = [];
|
|
480
|
+
let runCount = 0;
|
|
481
|
+
const claude = {
|
|
482
|
+
async run(opts) {
|
|
483
|
+
runCount++;
|
|
484
|
+
contexts.push(structuredClone(opts.context));
|
|
485
|
+
return { status: "success", data: { attempt: runCount }, toolCalls: [] };
|
|
486
|
+
},
|
|
487
|
+
async evaluate(opts) {
|
|
488
|
+
// First time: loop back. Second time: go to done.
|
|
489
|
+
if (runCount <= 2)
|
|
490
|
+
return opts.choices.find((c) => c.id === "a")?.id ?? opts.choices[0].id;
|
|
491
|
+
return opts.choices.find((c) => c.id === "done")?.id ?? opts.choices[0].id;
|
|
492
|
+
},
|
|
493
|
+
};
|
|
494
|
+
const w = {
|
|
495
|
+
id: "t",
|
|
496
|
+
name: "T",
|
|
497
|
+
description: "",
|
|
498
|
+
entry: "a",
|
|
499
|
+
nodes: {
|
|
500
|
+
a: { name: "A", instruction: "Step A", skills: [] },
|
|
501
|
+
b: { name: "B", instruction: "Step B", skills: [] },
|
|
502
|
+
done: { name: "Done", instruction: "Done", skills: [] },
|
|
503
|
+
},
|
|
504
|
+
edges: [
|
|
505
|
+
{ from: "a", to: "b" },
|
|
506
|
+
{ from: "b", to: "a", when: "retry", max_iterations: 1 },
|
|
507
|
+
{ from: "b", to: "done", when: "finished" },
|
|
508
|
+
],
|
|
509
|
+
};
|
|
510
|
+
const { results } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
511
|
+
// A ran twice — results map should have the LAST result (attempt 3 = second A run)
|
|
512
|
+
expect(results.get("a")?.data.attempt).toBe(3);
|
|
513
|
+
});
|
|
514
|
+
});
|
|
515
|
+
// ─── Spec Section: Execution Events ─────────────────────────────
|
|
516
|
+
describe("spec: Execution events", () => {
|
|
517
|
+
it("emits events in correct order per spec", async () => {
|
|
518
|
+
const claude = {
|
|
519
|
+
async run() {
|
|
520
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
521
|
+
},
|
|
522
|
+
async evaluate() {
|
|
523
|
+
return "a";
|
|
524
|
+
},
|
|
525
|
+
};
|
|
526
|
+
const w = {
|
|
527
|
+
id: "test",
|
|
528
|
+
name: "Test",
|
|
529
|
+
description: "",
|
|
530
|
+
entry: "a",
|
|
531
|
+
nodes: {
|
|
532
|
+
a: { name: "A", instruction: "Do A", skills: [] },
|
|
533
|
+
b: { name: "B", instruction: "Do B", skills: [] },
|
|
534
|
+
},
|
|
535
|
+
edges: [{ from: "a", to: "b" }],
|
|
536
|
+
};
|
|
537
|
+
const events = [];
|
|
538
|
+
await execute(w, {}, { skills: createSkillMap([]), claude, observer: (e) => events.push(e), config: {} });
|
|
539
|
+
// First event must be workflow:start
|
|
540
|
+
expect(events[0].type).toBe("workflow:start");
|
|
541
|
+
// Second must be sources:resolved
|
|
542
|
+
expect(events[1].type).toBe("sources:resolved");
|
|
543
|
+
// Last must be workflow:end
|
|
544
|
+
expect(events[events.length - 1].type).toBe("workflow:end");
|
|
545
|
+
// For each node: node:enter comes before node:exit
|
|
546
|
+
const types = events.map((e) => e.type);
|
|
547
|
+
const aEnterIdx = types.indexOf("node:enter");
|
|
548
|
+
const aExitIdx = types.indexOf("node:exit");
|
|
549
|
+
expect(aEnterIdx).toBeLessThan(aExitIdx);
|
|
550
|
+
// Route event comes after node:exit
|
|
551
|
+
const routeIdx = types.indexOf("route");
|
|
552
|
+
expect(routeIdx).toBeGreaterThan(aExitIdx);
|
|
553
|
+
});
|
|
554
|
+
it("observer errors are silently swallowed", async () => {
|
|
555
|
+
const claude = {
|
|
556
|
+
async run() {
|
|
557
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
558
|
+
},
|
|
559
|
+
async evaluate() {
|
|
560
|
+
return "a";
|
|
561
|
+
},
|
|
562
|
+
};
|
|
563
|
+
const w = {
|
|
564
|
+
id: "t",
|
|
565
|
+
name: "T",
|
|
566
|
+
description: "",
|
|
567
|
+
entry: "a",
|
|
568
|
+
nodes: { a: { name: "A", instruction: "x", skills: [] } },
|
|
569
|
+
edges: [],
|
|
570
|
+
};
|
|
571
|
+
// Observer that throws every time
|
|
572
|
+
const { results } = await execute(w, {}, {
|
|
573
|
+
skills: createSkillMap([]),
|
|
574
|
+
claude,
|
|
575
|
+
observer: () => {
|
|
576
|
+
throw new Error("boom");
|
|
577
|
+
},
|
|
578
|
+
config: {},
|
|
579
|
+
});
|
|
580
|
+
// Workflow still completes
|
|
581
|
+
expect(results.size).toBe(1);
|
|
582
|
+
});
|
|
583
|
+
});
|
|
584
|
+
// ─── Spec Section: Execution Trace ──────────────────────────────
|
|
585
|
+
describe("spec: Execution trace", () => {
|
|
586
|
+
it("trace.steps records node, status, and iteration", async () => {
|
|
587
|
+
const claude = {
|
|
588
|
+
async run() {
|
|
589
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
590
|
+
},
|
|
591
|
+
async evaluate() {
|
|
592
|
+
return "a";
|
|
593
|
+
},
|
|
594
|
+
};
|
|
595
|
+
const w = {
|
|
596
|
+
id: "t",
|
|
597
|
+
name: "T",
|
|
598
|
+
description: "",
|
|
599
|
+
entry: "a",
|
|
600
|
+
nodes: {
|
|
601
|
+
a: { name: "A", instruction: "Do A", skills: [] },
|
|
602
|
+
b: { name: "B", instruction: "Do B", skills: [] },
|
|
603
|
+
},
|
|
604
|
+
edges: [{ from: "a", to: "b" }],
|
|
605
|
+
};
|
|
606
|
+
const { trace } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
607
|
+
expect(trace.steps).toHaveLength(2);
|
|
608
|
+
expect(trace.steps[0]).toEqual({ node: "a", status: "success", iteration: 1 });
|
|
609
|
+
expect(trace.steps[1]).toEqual({ node: "b", status: "success", iteration: 1 });
|
|
610
|
+
});
|
|
611
|
+
it("trace.edges records from, to, and reason", async () => {
|
|
612
|
+
const claude = {
|
|
613
|
+
async run() {
|
|
614
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
615
|
+
},
|
|
616
|
+
async evaluate() {
|
|
617
|
+
return "a";
|
|
618
|
+
},
|
|
619
|
+
};
|
|
620
|
+
const w = {
|
|
621
|
+
id: "t",
|
|
622
|
+
name: "T",
|
|
623
|
+
description: "",
|
|
624
|
+
entry: "a",
|
|
625
|
+
nodes: {
|
|
626
|
+
a: { name: "A", instruction: "Do A", skills: [] },
|
|
627
|
+
b: { name: "B", instruction: "Do B", skills: [] },
|
|
628
|
+
},
|
|
629
|
+
edges: [{ from: "a", to: "b" }],
|
|
630
|
+
};
|
|
631
|
+
const { trace } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
632
|
+
expect(trace.edges).toHaveLength(1);
|
|
633
|
+
expect(trace.edges[0]).toEqual({ from: "a", to: "b", reason: "only path" });
|
|
634
|
+
});
|
|
635
|
+
it("trace records iteration count for retry loops", async () => {
|
|
636
|
+
let runCount = 0;
|
|
637
|
+
const claude = {
|
|
638
|
+
async run() {
|
|
639
|
+
runCount++;
|
|
640
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
641
|
+
},
|
|
642
|
+
async evaluate(opts) {
|
|
643
|
+
// Loop once, then go to done
|
|
644
|
+
if (runCount <= 2)
|
|
645
|
+
return opts.choices.find((c) => c.id === "a")?.id ?? opts.choices[0].id;
|
|
646
|
+
return opts.choices.find((c) => c.id === "done")?.id ?? opts.choices[0].id;
|
|
647
|
+
},
|
|
648
|
+
};
|
|
649
|
+
const w = {
|
|
650
|
+
id: "t",
|
|
651
|
+
name: "T",
|
|
652
|
+
description: "",
|
|
653
|
+
entry: "a",
|
|
654
|
+
nodes: {
|
|
655
|
+
a: { name: "A", instruction: "Do A", skills: [] },
|
|
656
|
+
b: { name: "B", instruction: "Do B", skills: [] },
|
|
657
|
+
done: { name: "Done", instruction: "Finish", skills: [] },
|
|
658
|
+
},
|
|
659
|
+
edges: [
|
|
660
|
+
{ from: "a", to: "b" },
|
|
661
|
+
{ from: "b", to: "a", when: "retry", max_iterations: 1 },
|
|
662
|
+
{ from: "b", to: "done", when: "done" },
|
|
663
|
+
],
|
|
664
|
+
};
|
|
665
|
+
const { trace } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
666
|
+
// A runs twice: iteration 1 and iteration 2
|
|
667
|
+
const aSteps = trace.steps.filter((s) => s.node === "a");
|
|
668
|
+
expect(aSteps).toHaveLength(2);
|
|
669
|
+
expect(aSteps[0].iteration).toBe(1);
|
|
670
|
+
expect(aSteps[1].iteration).toBe(2);
|
|
671
|
+
});
|
|
672
|
+
it("trace.sources contains resolved source entries", async () => {
|
|
673
|
+
const claude = {
|
|
674
|
+
async run() {
|
|
675
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
676
|
+
},
|
|
677
|
+
async evaluate() {
|
|
678
|
+
return "a";
|
|
679
|
+
},
|
|
680
|
+
};
|
|
681
|
+
const w = {
|
|
682
|
+
id: "t",
|
|
683
|
+
name: "T",
|
|
684
|
+
description: "",
|
|
685
|
+
entry: "a",
|
|
686
|
+
nodes: { a: { name: "A", instruction: "Do something.", skills: [] } },
|
|
687
|
+
edges: [],
|
|
688
|
+
};
|
|
689
|
+
const { trace } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
690
|
+
// trace.sources should have an entry for nodes.a.instruction
|
|
691
|
+
expect(trace.sources).toHaveProperty("nodes.a.instruction");
|
|
692
|
+
const resolved = trace.sources["nodes.a.instruction"];
|
|
693
|
+
expect(resolved.content).toBe("Do something.");
|
|
694
|
+
expect(resolved.kind).toBe("inline");
|
|
695
|
+
expect(resolved.resolver).toBe("inline");
|
|
696
|
+
expect(resolved.hash).toMatch(/^[0-9a-f]{16}$/);
|
|
697
|
+
});
|
|
698
|
+
it("trace.sources records file-based instructions", async () => {
|
|
699
|
+
const dir = freshDir("file-source");
|
|
700
|
+
writeFileSync(path.join(dir, "prompt.md"), "Investigate the production alert.");
|
|
701
|
+
const claude = {
|
|
702
|
+
async run() {
|
|
703
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
704
|
+
},
|
|
705
|
+
async evaluate() {
|
|
706
|
+
return "a";
|
|
707
|
+
},
|
|
708
|
+
};
|
|
709
|
+
const w = {
|
|
710
|
+
id: "t",
|
|
711
|
+
name: "T",
|
|
712
|
+
description: "",
|
|
713
|
+
entry: "a",
|
|
714
|
+
nodes: { a: { name: "A", instruction: "./prompt.md", skills: [] } },
|
|
715
|
+
edges: [],
|
|
716
|
+
};
|
|
717
|
+
const { trace } = await execute(w, {}, { skills: createSkillMap([]), claude, cwd: dir, config: {} });
|
|
718
|
+
const resolved = trace.sources["nodes.a.instruction"];
|
|
719
|
+
expect(resolved.content).toBe("Investigate the production alert.");
|
|
720
|
+
expect(resolved.kind).toBe("file");
|
|
721
|
+
expect(resolved.resolver).toBe("file");
|
|
722
|
+
expect(resolved.sourcePath).toContain("prompt.md");
|
|
723
|
+
});
|
|
724
|
+
});
|
|
725
|
+
// ─── Spec Section: Edge Routing Algorithm ───────────────────────
|
|
726
|
+
describe("spec: Edge routing algorithm", () => {
|
|
727
|
+
it("single unconditional edge — follows without AI evaluation", async () => {
|
|
728
|
+
let evaluateCalled = false;
|
|
729
|
+
const claude = {
|
|
730
|
+
async run() {
|
|
731
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
732
|
+
},
|
|
733
|
+
async evaluate() {
|
|
734
|
+
evaluateCalled = true;
|
|
735
|
+
return "b";
|
|
736
|
+
},
|
|
737
|
+
};
|
|
738
|
+
const w = {
|
|
739
|
+
id: "t",
|
|
740
|
+
name: "T",
|
|
741
|
+
description: "",
|
|
742
|
+
entry: "a",
|
|
743
|
+
nodes: {
|
|
744
|
+
a: { name: "A", instruction: "Do A", skills: [] },
|
|
745
|
+
b: { name: "B", instruction: "Do B", skills: [] },
|
|
746
|
+
},
|
|
747
|
+
edges: [{ from: "a", to: "b" }],
|
|
748
|
+
};
|
|
749
|
+
const { results } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
750
|
+
expect(results.size).toBe(2);
|
|
751
|
+
// evaluate() should NOT have been called — unconditional single edge
|
|
752
|
+
expect(evaluateCalled).toBe(false);
|
|
753
|
+
});
|
|
754
|
+
it("zero outgoing edges — node is terminal", async () => {
|
|
755
|
+
const claude = {
|
|
756
|
+
async run() {
|
|
757
|
+
return { status: "success", data: { done: true }, toolCalls: [] };
|
|
758
|
+
},
|
|
759
|
+
async evaluate() {
|
|
760
|
+
throw new Error("should not evaluate");
|
|
761
|
+
},
|
|
762
|
+
};
|
|
763
|
+
const w = {
|
|
764
|
+
id: "t",
|
|
765
|
+
name: "T",
|
|
766
|
+
description: "",
|
|
767
|
+
entry: "a",
|
|
768
|
+
nodes: { a: { name: "A", instruction: "Do A", skills: [] } },
|
|
769
|
+
edges: [],
|
|
770
|
+
};
|
|
771
|
+
const { results } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
772
|
+
expect(results.size).toBe(1);
|
|
773
|
+
expect(results.get("a")?.data.done).toBe(true);
|
|
774
|
+
});
|
|
775
|
+
it("multiple conditional edges — AI model evaluates", async () => {
|
|
776
|
+
let evaluateCalled = false;
|
|
777
|
+
const claude = {
|
|
778
|
+
async run() {
|
|
779
|
+
return { status: "success", data: { severity: "high" }, toolCalls: [] };
|
|
780
|
+
},
|
|
781
|
+
async evaluate(opts) {
|
|
782
|
+
evaluateCalled = true;
|
|
783
|
+
return opts.choices.find((c) => c.id === "high")?.id ?? opts.choices[0].id;
|
|
784
|
+
},
|
|
785
|
+
};
|
|
786
|
+
const w = {
|
|
787
|
+
id: "t",
|
|
788
|
+
name: "T",
|
|
789
|
+
description: "",
|
|
790
|
+
entry: "check",
|
|
791
|
+
nodes: {
|
|
792
|
+
check: { name: "Check", instruction: "Check it", skills: [] },
|
|
793
|
+
high: { name: "High", instruction: "High path", skills: [] },
|
|
794
|
+
low: { name: "Low", instruction: "Low path", skills: [] },
|
|
795
|
+
},
|
|
796
|
+
edges: [
|
|
797
|
+
{ from: "check", to: "high", when: "severity is high" },
|
|
798
|
+
{ from: "check", to: "low", when: "severity is low" },
|
|
799
|
+
],
|
|
800
|
+
};
|
|
801
|
+
const { results } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
802
|
+
expect(evaluateCalled).toBe(true);
|
|
803
|
+
expect(results.has("high")).toBe(true);
|
|
804
|
+
expect(results.has("low")).toBe(false);
|
|
805
|
+
});
|
|
806
|
+
it("exhausted max_iterations edge is excluded from routing", async () => {
|
|
807
|
+
let judgeCount = 0;
|
|
808
|
+
const claude = {
|
|
809
|
+
async run() {
|
|
810
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
811
|
+
},
|
|
812
|
+
async evaluate(opts) {
|
|
813
|
+
judgeCount++;
|
|
814
|
+
// Always try to retry
|
|
815
|
+
const retry = opts.choices.find((c) => c.id === "work");
|
|
816
|
+
if (retry)
|
|
817
|
+
return "work";
|
|
818
|
+
return opts.choices[0].id;
|
|
819
|
+
},
|
|
820
|
+
};
|
|
821
|
+
const w = {
|
|
822
|
+
id: "t",
|
|
823
|
+
name: "T",
|
|
824
|
+
description: "",
|
|
825
|
+
entry: "work",
|
|
826
|
+
nodes: {
|
|
827
|
+
work: { name: "Work", instruction: "Do work", skills: [] },
|
|
828
|
+
review: { name: "Review", instruction: "Review", skills: [] },
|
|
829
|
+
done: { name: "Done", instruction: "Done", skills: [] },
|
|
830
|
+
},
|
|
831
|
+
edges: [
|
|
832
|
+
{ from: "work", to: "review" },
|
|
833
|
+
{ from: "review", to: "work", when: "needs rework", max_iterations: 2 },
|
|
834
|
+
{ from: "review", to: "done", when: "looks good" },
|
|
835
|
+
],
|
|
836
|
+
};
|
|
837
|
+
const { results } = await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
838
|
+
// After 2 retries, the review→work edge exhausts and falls through to done
|
|
839
|
+
expect(results.has("done")).toBe(true);
|
|
840
|
+
});
|
|
841
|
+
});
|
|
842
|
+
// ─── Spec Section: JSON Schema Correctness ──────────────────────
|
|
843
|
+
describe("spec: JSON Schema", () => {
|
|
844
|
+
it("workflow schema $id matches spec canonical URL", () => {
|
|
845
|
+
expect(workflowJsonSchema.$id).toBe("https://spec.sweny.ai/schemas/workflow.json");
|
|
846
|
+
});
|
|
847
|
+
it("workflow schema requires all mandatory fields", () => {
|
|
848
|
+
expect(workflowJsonSchema.required).toContain("id");
|
|
849
|
+
expect(workflowJsonSchema.required).toContain("name");
|
|
850
|
+
expect(workflowJsonSchema.required).toContain("nodes");
|
|
851
|
+
expect(workflowJsonSchema.required).toContain("edges");
|
|
852
|
+
expect(workflowJsonSchema.required).toContain("entry");
|
|
853
|
+
});
|
|
854
|
+
it("node requires name and instruction", () => {
|
|
855
|
+
const nodeSchema = workflowJsonSchema.properties.nodes.additionalProperties;
|
|
856
|
+
expect(nodeSchema.required).toContain("name");
|
|
857
|
+
expect(nodeSchema.required).toContain("instruction");
|
|
858
|
+
});
|
|
859
|
+
it("edge requires from and to", () => {
|
|
860
|
+
const edgeSchema = workflowJsonSchema.properties.edges.items;
|
|
861
|
+
expect(edgeSchema.required).toContain("from");
|
|
862
|
+
expect(edgeSchema.required).toContain("to");
|
|
863
|
+
});
|
|
864
|
+
it("spec minimal example parses successfully", () => {
|
|
865
|
+
const minimal = {
|
|
866
|
+
id: "hello",
|
|
867
|
+
name: "Hello World",
|
|
868
|
+
entry: "greet",
|
|
869
|
+
nodes: {
|
|
870
|
+
greet: { name: "Greet", instruction: "Say hello to the user." },
|
|
871
|
+
},
|
|
872
|
+
edges: [],
|
|
873
|
+
};
|
|
874
|
+
const result = parseWorkflow(minimal);
|
|
875
|
+
expect(result.id).toBe("hello");
|
|
876
|
+
expect(result.nodes.greet.skills).toEqual([]); // default
|
|
877
|
+
});
|
|
878
|
+
it("spec full triage example parses and validates", () => {
|
|
879
|
+
const triage = {
|
|
880
|
+
id: "triage",
|
|
881
|
+
name: "Alert Triage",
|
|
882
|
+
description: "Investigate a production alert and take action based on findings.",
|
|
883
|
+
entry: "gather",
|
|
884
|
+
skills: {
|
|
885
|
+
"triage-rubric": {
|
|
886
|
+
name: "Triage Severity Rubric",
|
|
887
|
+
instruction: "When assessing severity: critical = customer-facing outage.",
|
|
888
|
+
},
|
|
889
|
+
},
|
|
890
|
+
nodes: {
|
|
891
|
+
gather: {
|
|
892
|
+
name: "Gather Context",
|
|
893
|
+
instruction: "Investigate the production alert.",
|
|
894
|
+
skills: ["github", "sentry", "datadog"],
|
|
895
|
+
},
|
|
896
|
+
investigate: {
|
|
897
|
+
name: "Root Cause Analysis",
|
|
898
|
+
instruction: "Classify each issue found as novel or duplicate.",
|
|
899
|
+
skills: ["github", "linear", "triage-rubric"],
|
|
900
|
+
output: {
|
|
901
|
+
type: "object",
|
|
902
|
+
properties: {
|
|
903
|
+
findings: { type: "array" },
|
|
904
|
+
novel_count: { type: "number" },
|
|
905
|
+
highest_severity: { type: "string" },
|
|
906
|
+
},
|
|
907
|
+
required: ["findings", "novel_count", "highest_severity"],
|
|
908
|
+
},
|
|
909
|
+
},
|
|
910
|
+
create_issue: {
|
|
911
|
+
name: "Create Issues",
|
|
912
|
+
instruction: "Create issues for novel findings.",
|
|
913
|
+
skills: ["linear", "github"],
|
|
914
|
+
},
|
|
915
|
+
skip: {
|
|
916
|
+
name: "Skip",
|
|
917
|
+
instruction: "All findings were duplicates or low priority.",
|
|
918
|
+
skills: ["linear"],
|
|
919
|
+
},
|
|
920
|
+
notify: {
|
|
921
|
+
name: "Notify Team",
|
|
922
|
+
instruction: "Send a notification summarizing the triage result.",
|
|
923
|
+
skills: ["slack"],
|
|
924
|
+
},
|
|
925
|
+
},
|
|
926
|
+
edges: [
|
|
927
|
+
{ from: "gather", to: "investigate" },
|
|
928
|
+
{ from: "investigate", to: "create_issue", when: "novel_count is greater than 0" },
|
|
929
|
+
{ from: "investigate", to: "skip", when: "novel_count is 0" },
|
|
930
|
+
{ from: "create_issue", to: "notify" },
|
|
931
|
+
{ from: "skip", to: "notify" },
|
|
932
|
+
],
|
|
933
|
+
};
|
|
934
|
+
const result = parseWorkflow(triage);
|
|
935
|
+
expect(result.id).toBe("triage");
|
|
936
|
+
expect(result.skills?.["triage-rubric"]).toBeDefined();
|
|
937
|
+
const errors = validateWorkflow(result);
|
|
938
|
+
expect(errors).toHaveLength(0);
|
|
939
|
+
});
|
|
940
|
+
});
|
|
941
|
+
// ─── Spec Section: Source Types ──────────────────────────────────
|
|
942
|
+
describe("spec: Source type classification", () => {
|
|
943
|
+
it("string starting with ./ is classified as file", () => {
|
|
944
|
+
const result = parseWorkflow({
|
|
945
|
+
id: "t",
|
|
946
|
+
name: "T",
|
|
947
|
+
entry: "a",
|
|
948
|
+
nodes: { a: { name: "A", instruction: "./prompts/gather.md" } },
|
|
949
|
+
edges: [],
|
|
950
|
+
});
|
|
951
|
+
// The instruction is stored as-is — classification happens at resolution time
|
|
952
|
+
expect(result.nodes.a.instruction).toBe("./prompts/gather.md");
|
|
953
|
+
});
|
|
954
|
+
it("string starting with https:// is classified as url", () => {
|
|
955
|
+
const result = parseWorkflow({
|
|
956
|
+
id: "t",
|
|
957
|
+
name: "T",
|
|
958
|
+
entry: "a",
|
|
959
|
+
nodes: { a: { name: "A", instruction: "https://example.com/prompt.md" } },
|
|
960
|
+
edges: [],
|
|
961
|
+
});
|
|
962
|
+
expect(result.nodes.a.instruction).toBe("https://example.com/prompt.md");
|
|
963
|
+
});
|
|
964
|
+
it("tagged object form {inline:} is accepted", () => {
|
|
965
|
+
const result = parseWorkflow({
|
|
966
|
+
id: "t",
|
|
967
|
+
name: "T",
|
|
968
|
+
entry: "a",
|
|
969
|
+
nodes: { a: { name: "A", instruction: { inline: "./not-a-file" } } },
|
|
970
|
+
edges: [],
|
|
971
|
+
});
|
|
972
|
+
expect(result.nodes.a.instruction.inline).toBe("./not-a-file");
|
|
973
|
+
});
|
|
974
|
+
it("tagged object form {file:} is accepted", () => {
|
|
975
|
+
const result = parseWorkflow({
|
|
976
|
+
id: "t",
|
|
977
|
+
name: "T",
|
|
978
|
+
entry: "a",
|
|
979
|
+
nodes: { a: { name: "A", instruction: { file: "./prompts/gather.md" } } },
|
|
980
|
+
edges: [],
|
|
981
|
+
});
|
|
982
|
+
expect(result.nodes.a.instruction.file).toBe("./prompts/gather.md");
|
|
983
|
+
});
|
|
984
|
+
it("tagged object form {url:} is accepted", () => {
|
|
985
|
+
const result = parseWorkflow({
|
|
986
|
+
id: "t",
|
|
987
|
+
name: "T",
|
|
988
|
+
entry: "a",
|
|
989
|
+
nodes: { a: { name: "A", instruction: { url: "https://example.com/p.md" } } },
|
|
990
|
+
edges: [],
|
|
991
|
+
});
|
|
992
|
+
expect(result.nodes.a.instruction.url).toBe("https://example.com/p.md");
|
|
993
|
+
});
|
|
994
|
+
it("empty string is rejected", () => {
|
|
995
|
+
expect(() => parseWorkflow({
|
|
996
|
+
id: "t",
|
|
997
|
+
name: "T",
|
|
998
|
+
entry: "a",
|
|
999
|
+
nodes: { a: { name: "A", instruction: "" } },
|
|
1000
|
+
edges: [],
|
|
1001
|
+
})).toThrow();
|
|
1002
|
+
});
|
|
1003
|
+
});
|
|
1004
|
+
// ─── Spec Section: Inline Skills ────────────────────────────────
|
|
1005
|
+
describe("spec: Inline skill definitions", () => {
|
|
1006
|
+
it("inline skill instructions are injected into node prompt", async () => {
|
|
1007
|
+
let capturedInstruction = "";
|
|
1008
|
+
const claude = {
|
|
1009
|
+
async run(opts) {
|
|
1010
|
+
capturedInstruction = opts.instruction;
|
|
1011
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
1012
|
+
},
|
|
1013
|
+
async evaluate() {
|
|
1014
|
+
return "a";
|
|
1015
|
+
},
|
|
1016
|
+
};
|
|
1017
|
+
const w = {
|
|
1018
|
+
id: "t",
|
|
1019
|
+
name: "T",
|
|
1020
|
+
description: "",
|
|
1021
|
+
entry: "a",
|
|
1022
|
+
nodes: { a: { name: "A", instruction: "Score the postmortem.", skills: ["rubric"] } },
|
|
1023
|
+
edges: [],
|
|
1024
|
+
skills: {
|
|
1025
|
+
rubric: {
|
|
1026
|
+
name: "SRE Rubric",
|
|
1027
|
+
instruction: "Score completeness 1-5.",
|
|
1028
|
+
},
|
|
1029
|
+
},
|
|
1030
|
+
};
|
|
1031
|
+
await execute(w, {}, { skills: createSkillMap([]), claude, config: {} });
|
|
1032
|
+
expect(capturedInstruction).toContain("## Skill: SRE Rubric");
|
|
1033
|
+
expect(capturedInstruction).toContain("Score completeness 1-5.");
|
|
1034
|
+
expect(capturedInstruction).toContain("Score the postmortem.");
|
|
1035
|
+
});
|
|
1036
|
+
it("caller skill map takes precedence over inline skills", async () => {
|
|
1037
|
+
let capturedInstruction = "";
|
|
1038
|
+
const claude = {
|
|
1039
|
+
async run(opts) {
|
|
1040
|
+
capturedInstruction = opts.instruction;
|
|
1041
|
+
return { status: "success", data: {}, toolCalls: [] };
|
|
1042
|
+
},
|
|
1043
|
+
async evaluate() {
|
|
1044
|
+
return "a";
|
|
1045
|
+
},
|
|
1046
|
+
};
|
|
1047
|
+
const callerSkill = {
|
|
1048
|
+
id: "rubric",
|
|
1049
|
+
name: "Caller Rubric",
|
|
1050
|
+
description: "From caller",
|
|
1051
|
+
category: "general",
|
|
1052
|
+
config: {},
|
|
1053
|
+
tools: [],
|
|
1054
|
+
instruction: "Caller instruction wins.",
|
|
1055
|
+
};
|
|
1056
|
+
const w = {
|
|
1057
|
+
id: "t",
|
|
1058
|
+
name: "T",
|
|
1059
|
+
description: "",
|
|
1060
|
+
entry: "a",
|
|
1061
|
+
nodes: { a: { name: "A", instruction: "Score it.", skills: ["rubric"] } },
|
|
1062
|
+
edges: [],
|
|
1063
|
+
skills: {
|
|
1064
|
+
rubric: {
|
|
1065
|
+
name: "Inline Rubric",
|
|
1066
|
+
instruction: "Inline instruction loses.",
|
|
1067
|
+
},
|
|
1068
|
+
},
|
|
1069
|
+
};
|
|
1070
|
+
await execute(w, {}, { skills: createSkillMap([callerSkill]), claude, config: {} });
|
|
1071
|
+
expect(capturedInstruction).toContain("Caller instruction wins.");
|
|
1072
|
+
expect(capturedInstruction).not.toContain("Inline instruction loses.");
|
|
1073
|
+
});
|
|
1074
|
+
});
|
package/dist/schema.d.ts
CHANGED
|
@@ -697,9 +697,9 @@ export interface WorkflowError {
|
|
|
697
697
|
export declare function validateWorkflow(workflow: Workflow | z.infer<typeof workflowZ>, knownSkills?: Set<string>): WorkflowError[];
|
|
698
698
|
export declare const workflowJsonSchema: {
|
|
699
699
|
readonly $schema: "https://json-schema.org/draft/2020-12/schema";
|
|
700
|
-
readonly $id: "https://sweny.ai/schemas/workflow.json";
|
|
700
|
+
readonly $id: "https://spec.sweny.ai/schemas/workflow.json";
|
|
701
701
|
readonly title: "SWEny Workflow";
|
|
702
|
-
readonly description: "A
|
|
702
|
+
readonly description: "A declarative YAML format for AI agent orchestration as a directed graph with natural language routing.";
|
|
703
703
|
readonly type: "object";
|
|
704
704
|
readonly required: readonly ["id", "name", "nodes", "edges", "entry"];
|
|
705
705
|
readonly additionalProperties: false;
|
|
@@ -774,7 +774,7 @@ export declare const workflowJsonSchema: {
|
|
|
774
774
|
};
|
|
775
775
|
readonly instruction: {
|
|
776
776
|
readonly $ref: "#/$defs/Source";
|
|
777
|
-
readonly description: "
|
|
777
|
+
readonly description: "Natural language instruction for the AI model.";
|
|
778
778
|
};
|
|
779
779
|
readonly skills: {
|
|
780
780
|
readonly type: "array";
|
package/dist/schema.js
CHANGED
|
@@ -262,9 +262,9 @@ const sourceJsonSchema = {
|
|
|
262
262
|
};
|
|
263
263
|
export const workflowJsonSchema = {
|
|
264
264
|
$schema: "https://json-schema.org/draft/2020-12/schema",
|
|
265
|
-
$id: "https://sweny.ai/schemas/workflow.json",
|
|
265
|
+
$id: "https://spec.sweny.ai/schemas/workflow.json",
|
|
266
266
|
title: "SWEny Workflow",
|
|
267
|
-
description: "A
|
|
267
|
+
description: "A declarative YAML format for AI agent orchestration as a directed graph with natural language routing.",
|
|
268
268
|
type: "object",
|
|
269
269
|
required: ["id", "name", "nodes", "edges", "entry"],
|
|
270
270
|
additionalProperties: false,
|
|
@@ -282,7 +282,7 @@ export const workflowJsonSchema = {
|
|
|
282
282
|
additionalProperties: false,
|
|
283
283
|
properties: {
|
|
284
284
|
name: { type: "string", minLength: 1 },
|
|
285
|
-
instruction: { $ref: "#/$defs/Source", description: "
|
|
285
|
+
instruction: { $ref: "#/$defs/Source", description: "Natural language instruction for the AI model." },
|
|
286
286
|
skills: {
|
|
287
287
|
type: "array",
|
|
288
288
|
items: { type: "string" },
|
package/dist/studio.d.ts
CHANGED
|
@@ -71,8 +71,11 @@ export interface FlowEdge {
|
|
|
71
71
|
source: string;
|
|
72
72
|
target: string;
|
|
73
73
|
type: "conditionEdge";
|
|
74
|
+
/** Array index of this edge in workflow.edges — used for stable lookups. */
|
|
75
|
+
edgeIndex: number;
|
|
74
76
|
data: {
|
|
75
77
|
when?: string;
|
|
78
|
+
max_iterations?: number;
|
|
76
79
|
isConditional: boolean;
|
|
77
80
|
};
|
|
78
81
|
}
|
package/dist/studio.js
CHANGED
|
@@ -77,13 +77,15 @@ export function workflowToFlow(workflow, skillCatalog = builtinSkills) {
|
|
|
77
77
|
exec: { status: "pending" },
|
|
78
78
|
},
|
|
79
79
|
}));
|
|
80
|
-
const edges = workflowEdges.map((edge) => ({
|
|
81
|
-
id:
|
|
80
|
+
const edges = workflowEdges.map((edge, i) => ({
|
|
81
|
+
id: `edge-${i}`,
|
|
82
82
|
source: edge.from,
|
|
83
83
|
target: edge.to,
|
|
84
84
|
type: "conditionEdge",
|
|
85
|
+
edgeIndex: i,
|
|
85
86
|
data: {
|
|
86
87
|
when: edge.when,
|
|
88
|
+
max_iterations: edge.max_iterations,
|
|
87
89
|
isConditional: !!edge.when,
|
|
88
90
|
},
|
|
89
91
|
}));
|
|
@@ -146,6 +148,7 @@ export function flowToWorkflow(meta, nodes, edges) {
|
|
|
146
148
|
from: e.source,
|
|
147
149
|
to: e.target,
|
|
148
150
|
...(e.data.when ? { when: e.data.when } : {}),
|
|
151
|
+
...(e.data.max_iterations ? { max_iterations: e.data.max_iterations } : {}),
|
|
149
152
|
})),
|
|
150
153
|
};
|
|
151
154
|
}
|