@sweny-ai/core 0.1.77 → 0.1.79

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/dist/browser.d.ts +11 -5
  2. package/dist/browser.js +11 -11
  3. package/dist/claude.d.ts +16 -0
  4. package/dist/claude.js +98 -9
  5. package/dist/cli/config.d.ts +22 -0
  6. package/dist/cli/config.js +56 -9
  7. package/dist/cli/main.d.ts +1 -1
  8. package/dist/cli/main.js +56 -57
  9. package/dist/cli/marketplace.js +35 -9
  10. package/dist/cli/publish.d.ts +10 -0
  11. package/dist/cli/publish.js +22 -14
  12. package/dist/loader.d.ts +40 -0
  13. package/dist/loader.js +73 -0
  14. package/dist/mcp-catalog.d.ts +47 -0
  15. package/dist/mcp-catalog.js +304 -0
  16. package/dist/mcp.d.ts +6 -9
  17. package/dist/mcp.js +24 -314
  18. package/dist/schema.d.ts +183 -151
  19. package/dist/schema.js +49 -7
  20. package/dist/skills/custom-loader.d.ts +40 -0
  21. package/dist/skills/custom-loader.js +107 -32
  22. package/dist/skills/index.d.ts +13 -1
  23. package/dist/skills/index.js +12 -3
  24. package/dist/studio.js +4 -1
  25. package/dist/templates.d.ts +29 -8
  26. package/dist/templates.js +30 -20
  27. package/dist/types.d.ts +12 -0
  28. package/dist/verify.js +96 -7
  29. package/package.json +8 -4
  30. package/dist/__tests__/cascade.test.d.ts +0 -11
  31. package/dist/__tests__/cascade.test.js +0 -452
  32. package/dist/__tests__/claude.test.d.ts +0 -1
  33. package/dist/__tests__/claude.test.js +0 -427
  34. package/dist/__tests__/custom-loader.test.d.ts +0 -1
  35. package/dist/__tests__/custom-loader.test.js +0 -116
  36. package/dist/__tests__/executor.test.d.ts +0 -1
  37. package/dist/__tests__/executor.test.js +0 -1635
  38. package/dist/__tests__/integration/datadog.integration.test.d.ts +0 -1
  39. package/dist/__tests__/integration/datadog.integration.test.js +0 -23
  40. package/dist/__tests__/integration/e2e-workflow.integration.test.d.ts +0 -1
  41. package/dist/__tests__/integration/e2e-workflow.integration.test.js +0 -75
  42. package/dist/__tests__/integration/github.integration.test.d.ts +0 -1
  43. package/dist/__tests__/integration/github.integration.test.js +0 -37
  44. package/dist/__tests__/integration/harness.d.ts +0 -24
  45. package/dist/__tests__/integration/harness.js +0 -34
  46. package/dist/__tests__/integration/linear.integration.test.d.ts +0 -1
  47. package/dist/__tests__/integration/linear.integration.test.js +0 -15
  48. package/dist/__tests__/integration/sentry.integration.test.d.ts +0 -1
  49. package/dist/__tests__/integration/sentry.integration.test.js +0 -20
  50. package/dist/__tests__/integration/slack.integration.test.d.ts +0 -1
  51. package/dist/__tests__/integration/slack.integration.test.js +0 -22
  52. package/dist/__tests__/integration/sources.integration.test.d.ts +0 -1
  53. package/dist/__tests__/integration/sources.integration.test.js +0 -123
  54. package/dist/__tests__/publish.test.d.ts +0 -1
  55. package/dist/__tests__/publish.test.js +0 -180
  56. package/dist/__tests__/requires.test.d.ts +0 -1
  57. package/dist/__tests__/requires.test.js +0 -82
  58. package/dist/__tests__/retry.test.d.ts +0 -1
  59. package/dist/__tests__/retry.test.js +0 -213
  60. package/dist/__tests__/schema.test.d.ts +0 -1
  61. package/dist/__tests__/schema.test.js +0 -600
  62. package/dist/__tests__/skill-config-integration.test.d.ts +0 -1
  63. package/dist/__tests__/skill-config-integration.test.js +0 -204
  64. package/dist/__tests__/skills-index.test.d.ts +0 -1
  65. package/dist/__tests__/skills-index.test.js +0 -119
  66. package/dist/__tests__/skills.test.d.ts +0 -1
  67. package/dist/__tests__/skills.test.js +0 -565
  68. package/dist/__tests__/spec-conformance.test.d.ts +0 -18
  69. package/dist/__tests__/spec-conformance.test.js +0 -1084
  70. package/dist/__tests__/studio.test.d.ts +0 -1
  71. package/dist/__tests__/studio.test.js +0 -403
  72. package/dist/__tests__/templates.test.d.ts +0 -1
  73. package/dist/__tests__/templates.test.js +0 -49
  74. package/dist/__tests__/testing.test.d.ts +0 -1
  75. package/dist/__tests__/testing.test.js +0 -242
  76. package/dist/__tests__/verify.test.d.ts +0 -1
  77. package/dist/__tests__/verify.test.js +0 -517
  78. package/dist/__tests__/version-consistency.test.d.ts +0 -1
  79. package/dist/__tests__/version-consistency.test.js +0 -312
  80. package/dist/__tests__/workflow-builder.test.d.ts +0 -1
  81. package/dist/__tests__/workflow-builder.test.js +0 -110
  82. package/dist/__tests__/workflow-yaml.test.d.ts +0 -1
  83. package/dist/__tests__/workflow-yaml.test.js +0 -201
  84. package/dist/cli/__tests__/diagram.test.d.ts +0 -1
  85. package/dist/cli/__tests__/diagram.test.js +0 -225
  86. package/dist/cli/__tests__/report-cloud.test.d.ts +0 -1
  87. package/dist/cli/__tests__/report-cloud.test.js +0 -165
  88. package/dist/cli/__tests__/upgrade.test.d.ts +0 -1
  89. package/dist/cli/__tests__/upgrade.test.js +0 -317
  90. package/dist/cli/__tests__/version-check.test.d.ts +0 -1
  91. package/dist/cli/__tests__/version-check.test.js +0 -209
  92. package/dist/cli/config.test.d.ts +0 -1
  93. package/dist/cli/config.test.js +0 -200
  94. package/dist/cli/credentials.test.d.ts +0 -1
  95. package/dist/cli/credentials.test.js +0 -172
  96. package/dist/cli/e2e.test.d.ts +0 -1
  97. package/dist/cli/e2e.test.js +0 -690
  98. package/dist/cli/marketplace.test.d.ts +0 -1
  99. package/dist/cli/marketplace.test.js +0 -355
  100. package/dist/cli/new.test.d.ts +0 -1
  101. package/dist/cli/new.test.js +0 -1043
  102. package/dist/cli/output.test.d.ts +0 -1
  103. package/dist/cli/output.test.js +0 -180
  104. package/dist/cli/publish.test.d.ts +0 -1
  105. package/dist/cli/publish.test.js +0 -610
  106. package/dist/cli/renderer.test.d.ts +0 -1
  107. package/dist/cli/renderer.test.js +0 -302
  108. package/dist/executor.test.d.ts +0 -1
  109. package/dist/executor.test.js +0 -618
  110. package/dist/mcp.test.d.ts +0 -1
  111. package/dist/mcp.test.js +0 -735
  112. package/dist/mermaid.test.d.ts +0 -1
  113. package/dist/mermaid.test.js +0 -256
  114. package/dist/sources.test.d.ts +0 -1
  115. package/dist/sources.test.js +0 -265
  116. package/dist/sources.validation.test.d.ts +0 -12
  117. package/dist/sources.validation.test.js +0 -380
  118. package/dist/workflow-builder.test.d.ts +0 -1
  119. package/dist/workflow-builder.test.js +0 -123
@@ -1,1635 +0,0 @@
1
- import { describe, it, expect, beforeEach } from "vitest";
2
- import { mkdirSync, writeFileSync, existsSync, rmSync } from "node:fs";
3
- import * as path from "node:path";
4
- import { tmpdir } from "node:os";
5
- import { execute } from "../executor.js";
6
- import { MockClaude, createFileSkill } from "../testing.js";
7
- import { createSkillMap } from "../skills/index.js";
8
- // ─── Fixtures ────────────────────────────────────────────────────
9
- const tmpBase = path.join(tmpdir(), "sweny-executor-test");
10
- function freshDir(name) {
11
- const dir = path.join(tmpBase, `${name}-${Date.now()}`);
12
- rmSync(dir, { recursive: true, force: true });
13
- mkdirSync(dir, { recursive: true });
14
- return dir;
15
- }
16
- const linearWorkflow = {
17
- id: "linear",
18
- name: "Linear",
19
- description: "A→B→C",
20
- entry: "a",
21
- nodes: {
22
- a: { name: "A", instruction: "Do A", skills: ["filesystem"] },
23
- b: { name: "B", instruction: "Do B", skills: ["filesystem"] },
24
- c: { name: "C", instruction: "Do C", skills: ["filesystem"] },
25
- },
26
- edges: [
27
- { from: "a", to: "b" },
28
- { from: "b", to: "c" },
29
- ],
30
- };
31
- /** Same shape as linearWorkflow but with no skill requirements — for tests that
32
- * don't exercise tool usage and just need a 3-node DAG. */
33
- const skilllessWorkflow = {
34
- ...linearWorkflow,
35
- id: "linear-no-skills",
36
- nodes: {
37
- a: { name: "A", instruction: "Do A", skills: [] },
38
- b: { name: "B", instruction: "Do B", skills: [] },
39
- c: { name: "C", instruction: "Do C", skills: [] },
40
- },
41
- };
42
- const branchingWorkflow = {
43
- id: "branching",
44
- name: "Branching",
45
- description: "A → B or C",
46
- entry: "check",
47
- nodes: {
48
- check: { name: "Check", instruction: "Check severity", skills: [] },
49
- high: { name: "High", instruction: "Handle high", skills: [] },
50
- low: { name: "Low", instruction: "Handle low", skills: [] },
51
- },
52
- edges: [
53
- { from: "check", to: "high", when: "severity is high" },
54
- { from: "check", to: "low", when: "severity is low" },
55
- ],
56
- };
57
- // ─── Tests ───────────────────────────────────────────────────────
58
- describe("executor", () => {
59
- let outputDir;
60
- beforeEach(() => {
61
- outputDir = freshDir("exec");
62
- });
63
- it("executes a linear workflow end-to-end", async () => {
64
- const fileSkill = createFileSkill(outputDir);
65
- writeFileSync(path.join(outputDir, "input.json"), JSON.stringify({ alert: "test" }));
66
- const claude = new MockClaude({
67
- responses: {
68
- a: {
69
- toolCalls: [{ tool: "fs_read_json", input: { path: "input.json" } }],
70
- data: { context: "loaded" },
71
- },
72
- b: { data: { processed: true } },
73
- c: {
74
- toolCalls: [{ tool: "fs_write_json", input: { path: "out.json", data: { done: true } } }],
75
- data: { written: true },
76
- },
77
- },
78
- });
79
- const events = [];
80
- const { results } = await execute(linearWorkflow, { alert: "test" }, {
81
- skills: createSkillMap([fileSkill]),
82
- claude,
83
- observer: (e) => events.push(e),
84
- config: {},
85
- });
86
- expect(results.size).toBe(3);
87
- expect(results.get("a")?.status).toBe("success");
88
- expect(results.get("b")?.status).toBe("success");
89
- expect(results.get("c")?.status).toBe("success");
90
- // Verify file was written
91
- expect(existsSync(path.join(outputDir, "out.json"))).toBe(true);
92
- // Events — workflow:start fires first, then sources:resolved (per spec)
93
- expect(events[0]).toEqual({ type: "workflow:start", workflow: "linear" });
94
- expect(events[1].type).toBe("sources:resolved");
95
- expect(events.filter((e) => e.type === "node:enter")).toHaveLength(3);
96
- expect(events[events.length - 1].type).toBe("workflow:end");
97
- });
98
- it("executes conditional branches", async () => {
99
- const claude = new MockClaude({
100
- workflow: branchingWorkflow,
101
- responses: {
102
- check: { data: { severity: "high" } },
103
- high: { data: { handled: true } },
104
- },
105
- routes: { check: "high" },
106
- });
107
- const { results } = await execute(branchingWorkflow, { alert: "cpu" }, {
108
- skills: createSkillMap([]),
109
- claude,
110
- config: {},
111
- });
112
- expect(results.size).toBe(2);
113
- expect(results.has("check")).toBe(true);
114
- expect(results.has("high")).toBe(true);
115
- expect(results.has("low")).toBe(false);
116
- });
117
- it("routes to alternative branch", async () => {
118
- const claude = new MockClaude({
119
- workflow: branchingWorkflow,
120
- responses: {
121
- check: { data: { severity: "low" } },
122
- low: { data: { skipped: true } },
123
- },
124
- routes: { check: "low" },
125
- });
126
- const { results } = await execute(branchingWorkflow, { alert: "minor" }, {
127
- skills: createSkillMap([]),
128
- claude,
129
- config: {},
130
- });
131
- expect(results.size).toBe(2);
132
- expect(results.has("low")).toBe(true);
133
- expect(results.has("high")).toBe(false);
134
- });
135
- it("emits tool:call and tool:result events", async () => {
136
- const fileSkill = createFileSkill(outputDir);
137
- writeFileSync(path.join(outputDir, "data.json"), '{"key": "value"}');
138
- const claude = new MockClaude({
139
- responses: {
140
- a: {
141
- toolCalls: [{ tool: "fs_read_json", input: { path: "data.json" } }],
142
- data: { read: true },
143
- },
144
- b: { data: {} },
145
- c: { data: {} },
146
- },
147
- });
148
- const events = [];
149
- await execute(linearWorkflow, {}, {
150
- skills: createSkillMap([fileSkill]),
151
- claude,
152
- observer: (e) => events.push(e),
153
- config: {},
154
- });
155
- const toolCalls = events.filter((e) => e.type === "tool:call");
156
- const toolResults = events.filter((e) => e.type === "tool:result");
157
- expect(toolCalls).toHaveLength(1);
158
- expect(toolResults).toHaveLength(1);
159
- expect(toolCalls[0]).toMatchObject({ type: "tool:call", node: "a", tool: "fs_read_json" });
160
- });
161
- it("observer errors do not crash the workflow", async () => {
162
- const claude = new MockClaude({
163
- responses: {
164
- a: { data: {} },
165
- b: { data: {} },
166
- c: { data: {} },
167
- },
168
- });
169
- const { results } = await execute(skilllessWorkflow, {}, {
170
- skills: createSkillMap([]),
171
- claude,
172
- observer: () => {
173
- throw new Error("observer boom");
174
- },
175
- config: {},
176
- });
177
- // Workflow completed despite observer throwing
178
- expect(results.size).toBe(3);
179
- });
180
- it("throws on missing entry node", async () => {
181
- const bad = { ...linearWorkflow, entry: "nonexistent" };
182
- const claude = new MockClaude({ responses: {} });
183
- await expect(execute(bad, {}, { skills: createSkillMap([]), claude, config: {} })).rejects.toThrow("Entry node");
184
- });
185
- it("throws on invalid edge reference", async () => {
186
- const bad = {
187
- ...linearWorkflow,
188
- edges: [
189
- { from: "a", to: "ghost" },
190
- { from: "a", to: "b" },
191
- ],
192
- };
193
- const claude = new MockClaude({ responses: {} });
194
- await expect(execute(bad, {}, { skills: createSkillMap([]), claude, config: {} })).rejects.toThrow("unknown node");
195
- });
196
- it("handles single-node workflow (entry = terminal)", async () => {
197
- const singleNode = {
198
- id: "single",
199
- name: "Single",
200
- description: "",
201
- entry: "only",
202
- nodes: { only: { name: "Only", instruction: "Do it", skills: [] } },
203
- edges: [],
204
- };
205
- const claude = new MockClaude({
206
- responses: { only: { data: { result: "done" } } },
207
- });
208
- const { results } = await execute(singleNode, {}, {
209
- skills: createSkillMap([]),
210
- claude,
211
- config: {},
212
- });
213
- expect(results.size).toBe(1);
214
- expect(results.get("only")?.data.result).toBe("done");
215
- });
216
- it("passes prior node results as context", async () => {
217
- let capturedContext = {};
218
- const mockClaude = {
219
- async run(opts) {
220
- capturedContext = opts.context;
221
- return { status: "success", data: { step: "done" }, toolCalls: [] };
222
- },
223
- async evaluate(opts) {
224
- return opts.choices[0].id;
225
- },
226
- };
227
- const twoStep = {
228
- id: "two",
229
- name: "Two",
230
- description: "",
231
- entry: "first",
232
- nodes: {
233
- first: { name: "First", instruction: "First step", skills: [] },
234
- second: { name: "Second", instruction: "Second step", skills: [] },
235
- },
236
- edges: [{ from: "first", to: "second" }],
237
- };
238
- await execute(twoStep, { original: true }, {
239
- skills: createSkillMap([]),
240
- claude: mockClaude,
241
- config: {},
242
- });
243
- // Second node should have first node's result in context
244
- expect(capturedContext).toHaveProperty("first");
245
- expect(capturedContext).toHaveProperty("input");
246
- });
247
- it("resolves config from env vars and overrides", async () => {
248
- // This tests that config resolution works (skill with required config)
249
- const skillWithConfig = {
250
- id: "needs-config",
251
- name: "Needs Config",
252
- description: "test",
253
- config: {
254
- MY_KEY: { description: "A key", required: true, env: "MY_KEY_ENV" },
255
- },
256
- tools: [],
257
- };
258
- const claude = new MockClaude({
259
- responses: { a: { data: {} }, b: { data: {} }, c: { data: {} } },
260
- });
261
- // Should throw without config (uses skilllessWorkflow — no skill requirements on nodes)
262
- await expect(execute(skilllessWorkflow, {}, {
263
- skills: createSkillMap([skillWithConfig]),
264
- claude,
265
- })).rejects.toThrow("Missing required config");
266
- // Should succeed with override
267
- const { results } = await execute(skilllessWorkflow, {}, {
268
- skills: createSkillMap([skillWithConfig]),
269
- claude,
270
- config: { MY_KEY: "provided" },
271
- });
272
- expect(results.size).toBe(3);
273
- });
274
- it("handles failed node results", async () => {
275
- const claude = new MockClaude({
276
- responses: {
277
- a: { status: "failed", data: { error: "something broke" } },
278
- b: { data: {} },
279
- c: { data: {} },
280
- },
281
- });
282
- const { results } = await execute(skilllessWorkflow, {}, {
283
- skills: createSkillMap([]),
284
- claude,
285
- config: {},
286
- });
287
- // All nodes still execute (executor doesn't short-circuit on failure)
288
- expect(results.get("a")?.status).toBe("failed");
289
- });
290
- it("executes a retry loop with max_iterations", async () => {
291
- const retryWorkflow = {
292
- id: "retry",
293
- name: "Retry",
294
- description: "A→B→A (retry once)→B→C",
295
- entry: "extract",
296
- nodes: {
297
- extract: { name: "Extract", instruction: "Extract data", skills: [] },
298
- judge: { name: "Judge", instruction: "Judge quality", skills: [] },
299
- done: { name: "Done", instruction: "Finish", skills: [] },
300
- },
301
- edges: [
302
- { from: "extract", to: "judge" },
303
- { from: "judge", to: "extract", when: "quality is low", max_iterations: 1 },
304
- { from: "judge", to: "done", when: "quality is high or retries exhausted" },
305
- ],
306
- };
307
- // Track how many times each node runs
308
- let extractCount = 0;
309
- let judgeCount = 0;
310
- const mockClaude = {
311
- async run(opts) {
312
- if (opts.instruction.includes("Extract")) {
313
- extractCount++;
314
- return { status: "success", data: { extracted: true, attempt: extractCount }, toolCalls: [] };
315
- }
316
- if (opts.instruction.includes("Judge")) {
317
- judgeCount++;
318
- // First judge: low quality (triggers retry). Second judge: still low but edge exhausted.
319
- return { status: "success", data: { quality: "low" }, toolCalls: [] };
320
- }
321
- return { status: "success", data: {}, toolCalls: [] };
322
- },
323
- async evaluate(opts) {
324
- // Always try to route to extract (retry), but max_iterations will block it on 2nd attempt
325
- const retryChoice = opts.choices.find((c) => c.id === "extract");
326
- if (retryChoice)
327
- return "extract";
328
- return opts.choices[0].id;
329
- },
330
- };
331
- const { results, trace } = await execute(retryWorkflow, {}, { skills: createSkillMap([]), claude: mockClaude, config: {} });
332
- // extract ran twice (initial + 1 retry), judge ran twice, done ran once
333
- expect(extractCount).toBe(2);
334
- expect(judgeCount).toBe(2);
335
- expect(results.has("done")).toBe(true);
336
- // The second extract result overwrites the first
337
- expect(results.get("extract")?.data.attempt).toBe(2);
338
- // Trace captures the full execution path including loops
339
- expect(trace.steps).toHaveLength(5); // extract, judge, extract(retry), judge, done
340
- expect(trace.steps[0]).toEqual({ node: "extract", status: "success", iteration: 1 });
341
- expect(trace.steps[2]).toEqual({ node: "extract", status: "success", iteration: 2 });
342
- // Trace edges include the retry loop
343
- const retryEdge = trace.edges.find((e) => e.from === "judge" && e.to === "extract");
344
- expect(retryEdge).toBeDefined();
345
- });
346
- it("respects max_iterations=1 as single retry", async () => {
347
- const singleRetry = {
348
- id: "single-retry",
349
- name: "Single Retry",
350
- description: "A↔B with max 1 loop",
351
- entry: "a",
352
- nodes: {
353
- a: { name: "A", instruction: "Step A", skills: [] },
354
- b: { name: "B", instruction: "Step B", skills: [] },
355
- end: { name: "End", instruction: "End", skills: [] },
356
- },
357
- edges: [
358
- { from: "a", to: "b" },
359
- { from: "b", to: "a", when: "retry", max_iterations: 1 },
360
- { from: "b", to: "end", when: "done" },
361
- ],
362
- };
363
- let aCount = 0;
364
- const mockClaude = {
365
- async run(opts) {
366
- if (opts.instruction.includes("Step A"))
367
- aCount++;
368
- return { status: "success", data: {}, toolCalls: [] };
369
- },
370
- async evaluate(opts) {
371
- // Always try to loop back
372
- const retry = opts.choices.find((c) => c.id === "a");
373
- if (retry)
374
- return "a";
375
- return opts.choices[0].id;
376
- },
377
- };
378
- const { results } = await execute(singleRetry, {}, { skills: createSkillMap([]), claude: mockClaude, config: {} });
379
- // A runs twice (once initially, once via retry), then edge is exhausted → falls to "end"
380
- expect(aCount).toBe(2);
381
- expect(results.has("end")).toBe(true);
382
- });
383
- it("throws when a node with required skills is reached but none are configured", async () => {
384
- const workflowWithSkills = {
385
- id: "needs-linear",
386
- name: "Needs Linear",
387
- description: "A→B where B needs linear skill",
388
- entry: "a",
389
- nodes: {
390
- a: { name: "A", instruction: "Do A", skills: [] },
391
- b: { name: "B", instruction: "Create a Linear issue", skills: ["linear"] },
392
- },
393
- edges: [{ from: "a", to: "b" }],
394
- };
395
- const claude = new MockClaude({
396
- responses: {
397
- a: { data: { done: true } },
398
- b: { data: {} },
399
- },
400
- });
401
- // No linear skill in the map — should throw when node B is reached
402
- await expect(execute(workflowWithSkills, {}, { skills: createSkillMap([]), claude, config: {} })).rejects.toThrow('Node "b" requires skills [linear] but none are configured');
403
- });
404
- it("does not throw for unreachable nodes with missing skills", async () => {
405
- const workflowWithUnreachable = {
406
- id: "unreachable-skill",
407
- name: "Unreachable Skill",
408
- description: "A→C, B needs linear but is never reached",
409
- entry: "a",
410
- nodes: {
411
- a: { name: "A", instruction: "Do A", skills: [] },
412
- b: { name: "B", instruction: "Create issue", skills: ["linear"] },
413
- c: { name: "C", instruction: "Do C", skills: [] },
414
- },
415
- edges: [
416
- { from: "a", to: "c" },
417
- { from: "a", to: "b", when: "needs escalation" },
418
- ],
419
- };
420
- const claude = new MockClaude({
421
- responses: {
422
- a: { data: {} },
423
- c: { data: {} },
424
- },
425
- routes: { a: "c" },
426
- });
427
- // Node B is never reached, so missing linear skill should not cause a throw
428
- const { results } = await execute(workflowWithUnreachable, {}, { skills: createSkillMap([]), claude, config: {} });
429
- expect(results.has("a")).toBe(true);
430
- expect(results.has("c")).toBe(true);
431
- expect(results.has("b")).toBe(false);
432
- });
433
- it("validates route choice falls back on invalid Claude response", async () => {
434
- // Create a claude mock that returns an invalid route
435
- const badRouteClaude = {
436
- async run() {
437
- return { status: "success", data: {}, toolCalls: [] };
438
- },
439
- async evaluate() {
440
- return "nonexistent_node"; // invalid target
441
- },
442
- };
443
- const { results } = await execute(branchingWorkflow, {}, {
444
- skills: createSkillMap([]),
445
- claude: badRouteClaude,
446
- config: {},
447
- });
448
- // Should still complete — falls back to first valid edge target
449
- expect(results.size).toBe(2);
450
- });
451
- describe("node.verify", () => {
452
- // Context: a create_issue node can satisfy its output schema (issueIdentifier
453
- // present) while never actually calling the Linear/GitHub create-issue tool.
454
- // The verify block catches this by requiring a real, successful tool call.
455
- const verifyWorkflow = {
456
- id: "verify-create-issue",
457
- name: "Verify Create Issue",
458
- description: "Single node that must call linear_create_issue",
459
- entry: "create",
460
- nodes: {
461
- create: {
462
- name: "Create",
463
- instruction: "Create an issue",
464
- skills: [],
465
- verify: { any_tool_called: ["linear_create_issue", "github_create_issue"] },
466
- },
467
- },
468
- edges: [],
469
- };
470
- it("marks node failed when required tool was never called", async () => {
471
- const claude = {
472
- async run() {
473
- return {
474
- status: "success",
475
- data: { issueIdentifier: "OFF-9999", issueTitle: "hallucinated", issueUrl: "https://" },
476
- toolCalls: [{ tool: "linear_search_issues", input: {}, output: [] }],
477
- };
478
- },
479
- async evaluate(opts) {
480
- return opts.choices[0]?.id;
481
- },
482
- };
483
- const { results } = await execute(verifyWorkflow, {}, { skills: createSkillMap([]), claude, config: {} });
484
- const r = results.get("create");
485
- expect(r.status).toBe("failed");
486
- expect(r.data.error).toMatch(/verify failed:.*any_tool_called/s);
487
- expect(r.data.error).toMatch(/linear_create_issue/);
488
- });
489
- it("marks node failed when required tool erred (output.error set)", async () => {
490
- const claude = {
491
- async run() {
492
- return {
493
- status: "success",
494
- data: { issueIdentifier: "OFF-9999" },
495
- toolCalls: [{ tool: "linear_create_issue", input: {}, output: { error: "Linear API 403" } }],
496
- };
497
- },
498
- async evaluate(opts) {
499
- return opts.choices[0]?.id;
500
- },
501
- };
502
- const { results } = await execute(verifyWorkflow, {}, { skills: createSkillMap([]), claude, config: {} });
503
- expect(results.get("create").status).toBe("failed");
504
- });
505
- it("passes when a required tool was invoked successfully", async () => {
506
- const claude = {
507
- async run() {
508
- return {
509
- status: "success",
510
- data: { issueIdentifier: "OFF-1234" },
511
- toolCalls: [
512
- {
513
- tool: "linear_create_issue",
514
- input: { title: "t" },
515
- output: { issueCreate: { success: true, issue: { identifier: "OFF-1234" } } },
516
- },
517
- ],
518
- };
519
- },
520
- async evaluate(opts) {
521
- return opts.choices[0]?.id;
522
- },
523
- };
524
- const { results } = await execute(verifyWorkflow, {}, { skills: createSkillMap([]), claude, config: {} });
525
- expect(results.get("create").status).toBe("success");
526
- });
527
- it("leaves already-failed nodes alone (does not overwrite Claude errors)", async () => {
528
- const claude = {
529
- async run() {
530
- return { status: "failed", data: { error: "max_turns" }, toolCalls: [] };
531
- },
532
- async evaluate(opts) {
533
- return opts.choices[0]?.id;
534
- },
535
- };
536
- const { results } = await execute(verifyWorkflow, {}, { skills: createSkillMap([]), claude, config: {} });
537
- expect(results.get("create").status).toBe("failed");
538
- expect(results.get("create").data.error).toBe("max_turns");
539
- });
540
- it("marks node failed when all_tools_called missing a required tool", async () => {
541
- const wf = {
542
- id: "wf",
543
- name: "wf",
544
- description: "",
545
- entry: "n",
546
- nodes: {
547
- n: {
548
- name: "N",
549
- instruction: "Do",
550
- skills: [],
551
- verify: { all_tools_called: ["a", "b"] },
552
- },
553
- },
554
- edges: [],
555
- };
556
- const claude = {
557
- async run() {
558
- return {
559
- status: "success",
560
- data: {},
561
- toolCalls: [{ tool: "a", input: {}, output: { ok: true } }],
562
- };
563
- },
564
- async evaluate(opts) {
565
- return opts.choices[0]?.id;
566
- },
567
- };
568
- const { results } = await execute(wf, {}, { skills: createSkillMap([]), claude, config: {} });
569
- const r = results.get("n");
570
- expect(r.status).toBe("failed");
571
- expect(r.data.error).toMatch(/verify failed.*all_tools_called.*\[b\]/s);
572
- });
573
- it("marks node failed when no_tool_called was violated", async () => {
574
- const wf = {
575
- id: "wf",
576
- name: "wf",
577
- description: "",
578
- entry: "n",
579
- nodes: {
580
- n: {
581
- name: "N",
582
- instruction: "Do",
583
- skills: [],
584
- verify: { no_tool_called: ["force_push"] },
585
- },
586
- },
587
- edges: [],
588
- };
589
- const claude = {
590
- async run() {
591
- return {
592
- status: "success",
593
- data: {},
594
- toolCalls: [{ tool: "force_push", input: {}, output: { ok: true } }],
595
- };
596
- },
597
- async evaluate(opts) {
598
- return opts.choices[0]?.id;
599
- },
600
- };
601
- const { results } = await execute(wf, {}, { skills: createSkillMap([]), claude, config: {} });
602
- const r = results.get("n");
603
- expect(r.status).toBe("failed");
604
- expect(r.data.error).toMatch(/verify failed.*no_tool_called.*force_push/s);
605
- });
606
- it("marks node failed when output_required is missing", async () => {
607
- const wf = {
608
- id: "wf",
609
- name: "wf",
610
- description: "",
611
- entry: "n",
612
- nodes: {
613
- n: {
614
- name: "N",
615
- instruction: "Do",
616
- skills: [],
617
- verify: { output_required: ["prUrl"] },
618
- },
619
- },
620
- edges: [],
621
- };
622
- const claude = {
623
- async run() {
624
- return { status: "success", data: { branch: "main" }, toolCalls: [] };
625
- },
626
- async evaluate(opts) {
627
- return opts.choices[0]?.id;
628
- },
629
- };
630
- const { results } = await execute(wf, {}, { skills: createSkillMap([]), claude, config: {} });
631
- const r = results.get("n");
632
- expect(r.status).toBe("failed");
633
- expect(r.data.error).toMatch(/verify failed.*output_required.*'prUrl'/s);
634
- });
635
- it("marks node failed when output_matches assertion fails", async () => {
636
- const wf = {
637
- id: "wf",
638
- name: "wf",
639
- description: "",
640
- entry: "n",
641
- nodes: {
642
- n: {
643
- name: "N",
644
- instruction: "Do",
645
- skills: [],
646
- verify: { output_matches: [{ path: "branch", matches: "^sweny/" }] },
647
- },
648
- },
649
- edges: [],
650
- };
651
- const claude = {
652
- async run() {
653
- return { status: "success", data: { branch: "main" }, toolCalls: [] };
654
- },
655
- async evaluate(opts) {
656
- return opts.choices[0]?.id;
657
- },
658
- };
659
- const { results } = await execute(wf, {}, { skills: createSkillMap([]), claude, config: {} });
660
- const r = results.get("n");
661
- expect(r.status).toBe("failed");
662
- expect(r.data.error).toMatch(/verify failed.*output_matches.*'branch'/s);
663
- });
664
- it("aggregates multiple verify failures into one error string", async () => {
665
- const wf = {
666
- id: "wf",
667
- name: "wf",
668
- description: "",
669
- entry: "n",
670
- nodes: {
671
- n: {
672
- name: "N",
673
- instruction: "Do",
674
- skills: [],
675
- verify: {
676
- all_tools_called: ["create_pr"],
677
- output_required: ["prUrl"],
678
- },
679
- },
680
- },
681
- edges: [],
682
- };
683
- const claude = {
684
- async run() {
685
- return { status: "success", data: {}, toolCalls: [] };
686
- },
687
- async evaluate(opts) {
688
- return opts.choices[0]?.id;
689
- },
690
- };
691
- const { results } = await execute(wf, {}, { skills: createSkillMap([]), claude, config: {} });
692
- const r = results.get("n");
693
- expect(r.status).toBe("failed");
694
- expect(r.data.error).toMatch(/all_tools_called/);
695
- expect(r.data.error).toMatch(/output_required/);
696
- });
697
- it("passes a node when every declared check passes", async () => {
698
- const wf = {
699
- id: "wf",
700
- name: "wf",
701
- description: "",
702
- entry: "n",
703
- nodes: {
704
- n: {
705
- name: "N",
706
- instruction: "Do",
707
- skills: [],
708
- verify: {
709
- all_tools_called: ["create_pr"],
710
- no_tool_called: ["force_push"],
711
- output_required: ["prUrl"],
712
- output_matches: [{ path: "prUrl", matches: "^https://" }],
713
- },
714
- },
715
- },
716
- edges: [],
717
- };
718
- const claude = {
719
- async run() {
720
- return {
721
- status: "success",
722
- data: { prUrl: "https://github.com/x/y/pull/1" },
723
- toolCalls: [{ tool: "create_pr", input: {}, output: { ok: true } }],
724
- };
725
- },
726
- async evaluate(opts) {
727
- return opts.choices[0]?.id;
728
- },
729
- };
730
- const { results } = await execute(wf, {}, { skills: createSkillMap([]), claude, config: {} });
731
- const r = results.get("n");
732
- expect(r.status).toBe("success");
733
- expect(r.data.error).toBeUndefined();
734
- });
735
- });
736
- describe("requires (pre-conditions)", () => {
737
- it("fails the node and skips the LLM when output_required is missing", async () => {
738
- const workflow = {
739
- id: "req-fail",
740
- name: "Req Fail",
741
- description: "",
742
- entry: "a",
743
- nodes: {
744
- a: {
745
- name: "A",
746
- instruction: "Do A",
747
- skills: [],
748
- requires: { output_required: ["input.missing"] },
749
- },
750
- },
751
- edges: [],
752
- };
753
- const claude = new MockClaude({
754
- responses: { a: { data: { ran: true } } },
755
- workflow,
756
- });
757
- const { results } = await execute(workflow, { other: 1 }, {
758
- skills: createSkillMap([]),
759
- claude,
760
- });
761
- const a = results.get("a");
762
- expect(a.status).toBe("failed");
763
- expect(a.data.error).toMatch(/^requires failed:/);
764
- expect(a.data.error).toMatch(/'input\.missing'/);
765
- expect(claude.executedNodes).toEqual([]); // LLM never ran
766
- });
767
- it("skips the node when on_fail: 'skip' and requires fails", async () => {
768
- const workflow = {
769
- id: "req-skip",
770
- name: "Req Skip",
771
- description: "",
772
- entry: "a",
773
- nodes: {
774
- a: {
775
- name: "A",
776
- instruction: "Do A",
777
- skills: [],
778
- requires: { output_required: ["input.missing"], on_fail: "skip" },
779
- },
780
- },
781
- edges: [],
782
- };
783
- const claude = new MockClaude({ responses: { a: { data: {} } }, workflow });
784
- const { results } = await execute(workflow, {}, {
785
- skills: createSkillMap([]),
786
- claude,
787
- });
788
- const a = results.get("a");
789
- expect(a.status).toBe("skipped");
790
- expect(a.data.skipped_reason).toMatch(/requires not met/);
791
- expect(claude.executedNodes).toEqual([]);
792
- });
793
- it("runs the LLM when requires passes", async () => {
794
- const workflow = {
795
- id: "req-pass",
796
- name: "Req Pass",
797
- description: "",
798
- entry: "a",
799
- nodes: {
800
- a: {
801
- name: "A",
802
- instruction: "Do A",
803
- skills: [],
804
- requires: { output_required: ["input.x"] },
805
- },
806
- },
807
- edges: [],
808
- };
809
- const claude = new MockClaude({ responses: { a: { data: { ran: true } } }, workflow });
810
- const { results } = await execute(workflow, { x: 1 }, {
811
- skills: createSkillMap([]),
812
- claude,
813
- });
814
- expect(results.get("a").status).toBe("success");
815
- expect(claude.executedNodes).toEqual(["a"]);
816
- });
817
- it("resolves cross-node paths against prior node data", async () => {
818
- const workflow = {
819
- id: "req-cross",
820
- name: "Req Cross",
821
- description: "",
822
- entry: "a",
823
- nodes: {
824
- a: { name: "A", instruction: "Do A", skills: [] },
825
- b: {
826
- name: "B",
827
- instruction: "Do B",
828
- skills: [],
829
- requires: { output_required: ["a.handle"] },
830
- },
831
- },
832
- edges: [{ from: "a", to: "b" }],
833
- };
834
- const claude = new MockClaude({
835
- responses: {
836
- a: { data: { handle: "ok" } },
837
- b: { data: { ran: true } },
838
- },
839
- workflow,
840
- });
841
- const { results } = await execute(workflow, {}, {
842
- skills: createSkillMap([]),
843
- claude,
844
- });
845
- expect(results.get("b").status).toBe("success");
846
- });
847
- });
848
- describe("retry on verify failure", () => {
849
- it("retries with default preamble and succeeds on second attempt", async () => {
850
- const workflow = {
851
- id: "retry-default",
852
- name: "Retry Default",
853
- description: "",
854
- entry: "a",
855
- nodes: {
856
- a: {
857
- name: "A",
858
- instruction: "Do A",
859
- skills: [],
860
- verify: { output_required: ["done"] },
861
- retry: { max: 1 },
862
- },
863
- },
864
- edges: [],
865
- };
866
- let callCount = 0;
867
- const claude = {
868
- run: async (opts) => {
869
- callCount++;
870
- if (callCount === 1)
871
- return { status: "success", data: {}, toolCalls: [] };
872
- // Second call sees the retry preamble in the instruction
873
- expect(opts.instruction).toMatch(/Previous attempt failed verification/);
874
- return { status: "success", data: { done: true }, toolCalls: [] };
875
- },
876
- evaluate: async () => "x",
877
- ask: async () => "",
878
- };
879
- const { results } = await execute(workflow, {}, {
880
- skills: createSkillMap([]),
881
- claude,
882
- });
883
- expect(callCount).toBe(2);
884
- const a = results.get("a");
885
- expect(a.status).toBe("success");
886
- expect(a.data.done).toBe(true);
887
- });
888
- it("marks failed and stops after retry exhaustion", async () => {
889
- const workflow = {
890
- id: "retry-exhaust",
891
- name: "Retry Exhaust",
892
- description: "",
893
- entry: "a",
894
- nodes: {
895
- a: {
896
- name: "A",
897
- instruction: "Do A",
898
- skills: [],
899
- verify: { output_required: ["done"] },
900
- retry: { max: 2 },
901
- },
902
- },
903
- edges: [],
904
- };
905
- let callCount = 0;
906
- const claude = {
907
- run: async () => {
908
- callCount++;
909
- return { status: "success", data: {}, toolCalls: [] };
910
- },
911
- evaluate: async () => "x",
912
- ask: async () => "",
913
- };
914
- const { results } = await execute(workflow, {}, {
915
- skills: createSkillMap([]),
916
- claude,
917
- });
918
- expect(callCount).toBe(3); // initial + 2 retries
919
- const a = results.get("a");
920
- expect(a.status).toBe("failed");
921
- expect(a.data.error).toMatch(/output_required/);
922
- });
923
- it("includes the static preamble text in the retry instruction", async () => {
924
- const workflow = {
925
- id: "retry-static",
926
- name: "Retry Static",
927
- description: "",
928
- entry: "a",
929
- nodes: {
930
- a: {
931
- name: "A",
932
- instruction: "Do A",
933
- skills: [],
934
- verify: { output_required: ["done"] },
935
- retry: { max: 1, instruction: "Try harder this time." },
936
- },
937
- },
938
- edges: [],
939
- };
940
- const seenInstructions = [];
941
- const claude = {
942
- run: async (opts) => {
943
- seenInstructions.push(opts.instruction);
944
- return seenInstructions.length === 2
945
- ? { status: "success", data: { done: true }, toolCalls: [] }
946
- : { status: "success", data: {}, toolCalls: [] };
947
- },
948
- evaluate: async () => "x",
949
- ask: async () => "",
950
- };
951
- await execute(workflow, {}, { skills: createSkillMap([]), claude });
952
- expect(seenInstructions[1]).toContain("Try harder this time.");
953
- expect(seenInstructions[1]).toContain("Previous attempt failed verification");
954
- });
955
- it("does not retry when claude.run itself fails (non-verify failure)", async () => {
956
- const workflow = {
957
- id: "retry-no-trigger",
958
- name: "No Trigger",
959
- description: "",
960
- entry: "a",
961
- nodes: {
962
- a: {
963
- name: "A",
964
- instruction: "Do A",
965
- skills: [],
966
- verify: { output_required: ["done"] },
967
- retry: { max: 3 },
968
- },
969
- },
970
- edges: [],
971
- };
972
- let callCount = 0;
973
- const claude = {
974
- run: async () => {
975
- callCount++;
976
- return { status: "failed", data: { error: "API down" }, toolCalls: [] };
977
- },
978
- evaluate: async () => "x",
979
- ask: async () => "",
980
- };
981
- const { results } = await execute(workflow, {}, {
982
- skills: createSkillMap([]),
983
- claude,
984
- });
985
- expect(callCount).toBe(1);
986
- expect(results.get("a").status).toBe("failed");
987
- expect(results.get("a").data.error).toBe("API down");
988
- });
989
- });
990
- describe("retry — autonomous reflection", () => {
991
- it("calls claude.ask and injects reflection into the retry preamble", async () => {
992
- const workflow = {
993
- id: "retry-auto",
994
- name: "Retry Auto",
995
- description: "",
996
- entry: "a",
997
- nodes: {
998
- a: {
999
- name: "A",
1000
- instruction: "Open the PR",
1001
- skills: [],
1002
- verify: { output_required: ["done"] },
1003
- retry: { max: 1, instruction: { auto: true } },
1004
- },
1005
- },
1006
- edges: [],
1007
- };
1008
- const askCalls = [];
1009
- const seenInstructions = [];
1010
- const claude = {
1011
- run: async (opts) => {
1012
- seenInstructions.push(opts.instruction);
1013
- return seenInstructions.length === 2
1014
- ? { status: "success", data: { done: true }, toolCalls: [] }
1015
- : { status: "success", data: {}, toolCalls: [] };
1016
- },
1017
- evaluate: async () => "x",
1018
- ask: async (opts) => {
1019
- askCalls.push(opts);
1020
- return "Diagnosis: missing the create_pr tool. Strategy: call it before returning.";
1021
- },
1022
- };
1023
- await execute(workflow, {}, { skills: createSkillMap([]), claude });
1024
- expect(askCalls).toHaveLength(1);
1025
- expect(askCalls[0].instruction).toContain("Open the PR");
1026
- expect(askCalls[0].instruction).toMatch(/Briefly diagnose/);
1027
- expect(seenInstructions[1]).toContain("Diagnosis: missing the create_pr tool");
1028
- });
1029
- it("falls back to default preamble when claude.ask throws", async () => {
1030
- const workflow = {
1031
- id: "retry-auto-fallback",
1032
- name: "Retry Auto Fallback",
1033
- description: "",
1034
- entry: "a",
1035
- nodes: {
1036
- a: {
1037
- name: "A",
1038
- instruction: "Open the PR",
1039
- skills: [],
1040
- verify: { output_required: ["done"] },
1041
- retry: { max: 1, instruction: { auto: true } },
1042
- },
1043
- },
1044
- edges: [],
1045
- };
1046
- const seenInstructions = [];
1047
- const claude = {
1048
- run: async (opts) => {
1049
- seenInstructions.push(opts.instruction);
1050
- return seenInstructions.length === 2
1051
- ? { status: "success", data: { done: true }, toolCalls: [] }
1052
- : { status: "success", data: {}, toolCalls: [] };
1053
- },
1054
- evaluate: async () => "x",
1055
- ask: async () => {
1056
- throw new Error("network down");
1057
- },
1058
- };
1059
- await execute(workflow, {}, { skills: createSkillMap([]), claude });
1060
- expect(seenInstructions[1]).toMatch(/Previous attempt failed verification/);
1061
- });
1062
- it("uses the author's reflect prompt when instruction.reflect is set", async () => {
1063
- const workflow = {
1064
- id: "retry-reflect",
1065
- name: "Retry Reflect",
1066
- description: "",
1067
- entry: "a",
1068
- nodes: {
1069
- a: {
1070
- name: "A",
1071
- instruction: "Open the PR",
1072
- skills: [],
1073
- verify: { output_required: ["done"] },
1074
- retry: { max: 1, instruction: { reflect: "Focus on tool selection only." } },
1075
- },
1076
- },
1077
- edges: [],
1078
- };
1079
- const askCalls = [];
1080
- const claude = {
1081
- run: async () => ({ status: "success", data: { done: true }, toolCalls: [] }),
1082
- evaluate: async () => "x",
1083
- ask: async (opts) => {
1084
- askCalls.push(opts);
1085
- return "diagnosis";
1086
- },
1087
- };
1088
- // First call deliberately fails verify (output_required missing) to trigger ask.
1089
- let callCount = 0;
1090
- claude.run = async () => {
1091
- callCount++;
1092
- return callCount === 1
1093
- ? { status: "success", data: {}, toolCalls: [] }
1094
- : { status: "success", data: { done: true }, toolCalls: [] };
1095
- };
1096
- await execute(workflow, {}, { skills: createSkillMap([]), claude });
1097
- expect(askCalls).toHaveLength(1);
1098
- expect(askCalls[0].instruction).toContain("Focus on tool selection only.");
1099
- });
1100
- });
1101
- describe("retry — trace and observer", () => {
1102
- it("records each retry attempt as its own TraceStep with retryAttempt", async () => {
1103
- const workflow = {
1104
- id: "retry-trace",
1105
- name: "Retry Trace",
1106
- description: "",
1107
- entry: "a",
1108
- nodes: {
1109
- a: {
1110
- name: "A",
1111
- instruction: "Do A",
1112
- skills: [],
1113
- verify: { output_required: ["done"] },
1114
- retry: { max: 2 },
1115
- },
1116
- },
1117
- edges: [],
1118
- };
1119
- let callCount = 0;
1120
- const claude = {
1121
- run: async () => {
1122
- callCount++;
1123
- return callCount === 3
1124
- ? { status: "success", data: { done: true }, toolCalls: [] }
1125
- : { status: "success", data: {}, toolCalls: [] };
1126
- },
1127
- evaluate: async () => "x",
1128
- ask: async () => "",
1129
- };
1130
- const { trace } = await execute(workflow, {}, { skills: createSkillMap([]), claude });
1131
- const aSteps = trace.steps.filter((s) => s.node === "a");
1132
- expect(aSteps).toHaveLength(3);
1133
- expect(aSteps[0]).toMatchObject({ node: "a", status: "failed", iteration: 1, retryAttempt: 0 });
1134
- expect(aSteps[1]).toMatchObject({ node: "a", status: "failed", iteration: 1, retryAttempt: 1 });
1135
- expect(aSteps[2]).toMatchObject({ node: "a", status: "success", iteration: 1, retryAttempt: 2 });
1136
- });
1137
- it("emits node:retry observer events with attempt and preamble", async () => {
1138
- const workflow = {
1139
- id: "retry-observer",
1140
- name: "Retry Observer",
1141
- description: "",
1142
- entry: "a",
1143
- nodes: {
1144
- a: {
1145
- name: "A",
1146
- instruction: "Do A",
1147
- skills: [],
1148
- verify: { output_required: ["done"] },
1149
- retry: { max: 1 },
1150
- },
1151
- },
1152
- edges: [],
1153
- };
1154
- const events = [];
1155
- let callCount = 0;
1156
- const claude = {
1157
- run: async () => {
1158
- callCount++;
1159
- return callCount === 2
1160
- ? { status: "success", data: { done: true }, toolCalls: [] }
1161
- : { status: "success", data: {}, toolCalls: [] };
1162
- },
1163
- evaluate: async () => "x",
1164
- ask: async () => "",
1165
- };
1166
- await execute(workflow, {}, {
1167
- skills: createSkillMap([]),
1168
- claude,
1169
- observer: (e) => events.push(e),
1170
- });
1171
- const retryEvents = events.filter((e) => e.type === "node:retry");
1172
- expect(retryEvents).toHaveLength(1);
1173
- const evt = retryEvents[0];
1174
- expect(evt.node).toBe("a");
1175
- expect(evt.attempt).toBe(1);
1176
- expect(evt.preamble).toMatch(/Previous attempt failed verification/);
1177
- expect(evt.reason).toMatch(/output_required/);
1178
- });
1179
- it("omits retryAttempt on TraceStep when no retry fires", async () => {
1180
- const workflow = {
1181
- id: "no-retry",
1182
- name: "No Retry",
1183
- description: "",
1184
- entry: "a",
1185
- nodes: { a: { name: "A", instruction: "Do A", skills: [] } },
1186
- edges: [],
1187
- };
1188
- const claude = new MockClaude({ responses: { a: { data: { ok: true } } }, workflow });
1189
- const { trace } = await execute(workflow, {}, { skills: createSkillMap([]), claude });
1190
- expect(trace.steps[0].retryAttempt).toBeUndefined();
1191
- });
1192
- });
1193
- // ── Test 1: retry max>1 autonomous — ask sees the *latest* verify error ──
1194
- describe("retry — latest error in autonomous ask prompt", () => {
1195
- it("injects the latest verify error (not accumulated history) into claude.ask on each retry", async () => {
1196
- // Use output_matches so different run data produces different error messages
1197
- // (the error includes the actual got value, so "got [\"missing field foo\"]" vs "got [\"missing field bar\"]")
1198
- const workflow = {
1199
- id: "retry-latest-error",
1200
- name: "Retry Latest Error",
1201
- description: "",
1202
- entry: "a",
1203
- nodes: {
1204
- a: {
1205
- name: "A",
1206
- instruction: "Do A",
1207
- skills: [],
1208
- verify: { output_matches: [{ path: "message", equals: "ok" }] },
1209
- retry: { max: 3, instruction: { auto: true } },
1210
- },
1211
- },
1212
- edges: [],
1213
- };
1214
- let callCount = 0;
1215
- const askInstructions = [];
1216
- const claude = {
1217
- run: async () => {
1218
- callCount++;
1219
- if (callCount === 1)
1220
- return { status: "success", data: { message: "missing field foo" }, toolCalls: [] };
1221
- if (callCount === 2)
1222
- return { status: "success", data: { message: "missing field bar" }, toolCalls: [] };
1223
- // Third attempt succeeds verify
1224
- return { status: "success", data: { message: "ok" }, toolCalls: [] };
1225
- },
1226
- evaluate: async () => "x",
1227
- ask: async (opts) => {
1228
- askInstructions.push(opts.instruction);
1229
- return "I will fix it.";
1230
- },
1231
- };
1232
- const { results } = await execute(workflow, {}, { skills: createSkillMap([]), claude });
1233
- expect(results.get("a").status).toBe("success");
1234
- // Two ask calls: one after attempt 1, one after attempt 2
1235
- expect(askInstructions).toHaveLength(2);
1236
- // Second ask must reference the LATEST error (bar), not the first (foo)
1237
- expect(askInstructions[1]).toContain("missing field bar");
1238
- expect(askInstructions[1]).not.toContain("missing field foo");
1239
- // First ask referenced foo
1240
- expect(askInstructions[0]).toContain("missing field foo");
1241
- });
1242
- });
1243
- // ── Test 2: requires failure inside a max_iterations graph cycle ──
1244
- describe("requires — fires on every iteration through a cyclic node", () => {
1245
- it("halts with requires failure on second cycle iteration when upstream data disappears", async () => {
1246
- // Workflow: a → b → a (max_iterations: 1) → end
1247
- // Node a has requires checking b.proceed === "yes".
1248
- // First pass through a: b hasn't run yet — requires checks `b.proceed` which is
1249
- // absent, so requires fails immediately on the first pass.
1250
- //
1251
- // To make it pass on iter 1 and fail on iter 2, we check `input.go` (always present)
1252
- // for the first requires and `b.proceed` (set by b) for the cyclic check.
1253
- // Simplest: requires checks b.proceed. On iter 1 b hasn't run → fails.
1254
- // That's not what we want (fail on iter 2, not iter 1).
1255
- //
1256
- // Instead: check output_matches on b.proceed === "yes".
1257
- // Iter 1 of a: b.proceed not in context → output_required fails.
1258
- // So we need b to run BEFORE a's second iteration fails.
1259
- //
1260
- // Topology: a→b (b runs then routes back to a). On second a, requires on a
1261
- // checks b.proceed. B returns "no" on its second run → requires fails.
1262
- const workflow = {
1263
- id: "cycle-requires",
1264
- name: "Cycle Requires",
1265
- description: "",
1266
- entry: "a",
1267
- nodes: {
1268
- a: {
1269
- name: "A",
1270
- instruction: "Step A",
1271
- skills: [],
1272
- // Requires b.proceed === "yes" — fails when b says "no"
1273
- requires: { output_matches: [{ path: "b.proceed", equals: "yes" }] },
1274
- },
1275
- b: {
1276
- name: "B",
1277
- instruction: "Step B",
1278
- skills: [],
1279
- },
1280
- end: {
1281
- name: "End",
1282
- instruction: "End",
1283
- skills: [],
1284
- },
1285
- },
1286
- edges: [
1287
- { from: "a", to: "b" },
1288
- { from: "b", to: "a", when: "loop back", max_iterations: 1 },
1289
- { from: "b", to: "end", when: "done" },
1290
- ],
1291
- };
1292
- let aCount = 0;
1293
- let bCount = 0;
1294
- const claude = {
1295
- run: async (opts) => {
1296
- if (opts.instruction.includes("Step A")) {
1297
- aCount++;
1298
- return { status: "success", data: { aRan: true }, toolCalls: [] };
1299
- }
1300
- if (opts.instruction.includes("Step B")) {
1301
- bCount++;
1302
- // First B run: proceed = "yes" (allows second A iteration requires to pass)
1303
- // Wait — on second A, requires checks b.proceed. If B always returns "yes",
1304
- // requires passes on iter 2 as well. We need B to return "no" on iter 2.
1305
- const proceed = bCount === 1 ? "yes" : "no";
1306
- return { status: "success", data: { proceed }, toolCalls: [] };
1307
- }
1308
- return { status: "success", data: {}, toolCalls: [] };
1309
- },
1310
- evaluate: async (opts) => {
1311
- // Always try to loop back; max_iterations will block it after 1 use
1312
- const loopChoice = opts.choices.find((c) => c.id === "a");
1313
- if (loopChoice)
1314
- return "a";
1315
- return opts.choices[0].id;
1316
- },
1317
- };
1318
- const { results, trace } = await execute(workflow, {}, { skills: createSkillMap([]), claude });
1319
- // First iteration of a: requires fails (b hasn't run yet, b.proceed absent)
1320
- // But wait — on FIRST iteration of A, b hasn't run, so b.proceed is absent.
1321
- // The requires uses output_matches which calls checkOutputMatches.
1322
- // That means A fails on iteration 1 due to requires (before B ever runs).
1323
- // So: a fails (iter 1) → routing from a → b runs → b returns proceed="yes"
1324
- // → routing loops back → a iter 2: b.proceed="yes" → requires PASSES → a runs → success
1325
- // → b iter 2: proceed="no" → routing tries to loop but max_iterations=1 exhausted → end
1326
- // Actually let's assert what actually happens. Iter 1 of A: requires fails (b.proceed absent).
1327
- // After requires failure, advanceFromNode still routes to b. B runs (bCount=1, proceed="yes").
1328
- // Routing: loop back to A (iter 2). A iter 2: requires checks b.proceed="yes" → passes! A LLM runs.
1329
- // Then b iter 2: proceed="no". Routing tries loop-back but max_iterations=1 exhausted → end.
1330
- // So this test as written doesn't exercise "requires fails on iter 2" — it fails on iter 1.
1331
- //
1332
- // Rethink: use a requires check that passes on iter 1 (using input.go) and fails on iter 2 via b.proceed.
1333
- // But output_matches on a path that's absent returns a failure. We need to check a path
1334
- // that is PRESENT on iter 1 and WRONG on iter 2.
1335
- //
1336
- // Result: a iter 1 requires fails because b.proceed is absent. B then runs.
1337
- // a iter 2 requires checks b.proceed = "yes" — passes. B iter 2 returns "no".
1338
- // There's no third A iteration. So requires gates every iteration of A.
1339
- // Lock: requires gates fire on every visit to the node, not just the first.
1340
- expect(aCount).toBeGreaterThanOrEqual(1); // at least one LLM run of A
1341
- // Trace should include a requires-failure step for node A
1342
- const aSteps = trace.steps.filter((s) => s.node === "a");
1343
- expect(aSteps.length).toBeGreaterThanOrEqual(1);
1344
- // The first step through A must be a requires failure (status: failed)
1345
- expect(aSteps[0].status).toBe("failed");
1346
- // B ran (requires failure on A still routes to B)
1347
- expect(bCount).toBeGreaterThanOrEqual(1);
1348
- });
1349
- });
1350
- // ── Test 3: requires on_fail:skip — downstream node sees skipped data ──
1351
- describe("requires on_fail:skip — downstream sees skipped node data", () => {
1352
- it("downstream node can read the skipped_reason from a requires-skipped node", async () => {
1353
- const workflow = {
1354
- id: "skip-downstream",
1355
- name: "Skip Downstream",
1356
- description: "",
1357
- entry: "a",
1358
- nodes: {
1359
- a: {
1360
- name: "A",
1361
- instruction: "Do A",
1362
- skills: [],
1363
- requires: { output_required: ["input.missing"], on_fail: "skip" },
1364
- },
1365
- b: {
1366
- name: "B",
1367
- instruction: "Do B",
1368
- skills: [],
1369
- },
1370
- },
1371
- edges: [{ from: "a", to: "b" }],
1372
- };
1373
- let capturedContext = {};
1374
- const claude = {
1375
- run: async (opts) => {
1376
- if (opts.instruction.includes("Do B")) {
1377
- capturedContext = opts.context;
1378
- }
1379
- return { status: "success", data: { ran: true }, toolCalls: [] };
1380
- },
1381
- evaluate: async (opts) => opts.choices[0].id,
1382
- };
1383
- const { results } = await execute(workflow, {}, { skills: createSkillMap([]), claude });
1384
- // A was skipped due to requires failure
1385
- expect(results.get("a").status).toBe("skipped");
1386
- const skippedReason = results.get("a").data.skipped_reason;
1387
- expect(skippedReason).toMatch(/requires not met/);
1388
- // B ran successfully
1389
- expect(results.get("b").status).toBe("success");
1390
- // B's context contains a.skipped_reason — downstream can read skipped node data
1391
- // The context passed to B is { input, a: <a's result.data> }
1392
- const aContextData = capturedContext["a"];
1393
- expect(aContextData).toBeDefined();
1394
- expect(typeof aContextData.skipped_reason).toBe("string");
1395
- expect(aContextData.skipped_reason).toMatch(/requires not met/);
1396
- });
1397
- });
1398
- // ── Test 4: skipped status does NOT trigger retry ──
1399
- describe("retry — skipped status does not trigger retry loop", () => {
1400
- it("does not retry when claude.run returns status: skipped", async () => {
1401
- // The retry loop breaks on any non-success status (line 257: `if (result.status !== "success") break`).
1402
- // This locks that skipped ≠ failure — retry never fires for skipped runs.
1403
- const workflow = {
1404
- id: "retry-skipped",
1405
- name: "Retry Skipped",
1406
- description: "",
1407
- entry: "a",
1408
- nodes: {
1409
- a: {
1410
- name: "A",
1411
- instruction: "Do A",
1412
- skills: [],
1413
- verify: { output_required: ["done"] },
1414
- retry: { max: 2 },
1415
- },
1416
- },
1417
- edges: [],
1418
- };
1419
- let callCount = 0;
1420
- const events = [];
1421
- const askCallCount = { n: 0 };
1422
- const claude = {
1423
- run: async () => {
1424
- callCount++;
1425
- // Return skipped — should not trigger retry
1426
- return { status: "skipped", data: { reason: "agent chose to skip" }, toolCalls: [] };
1427
- },
1428
- evaluate: async () => "x",
1429
- ask: async () => {
1430
- askCallCount.n++;
1431
- return "";
1432
- },
1433
- };
1434
- const { results } = await execute(workflow, {}, {
1435
- skills: createSkillMap([]),
1436
- claude,
1437
- observer: (e) => events.push(e),
1438
- });
1439
- // Claude ran exactly once — no retry
1440
- expect(callCount).toBe(1);
1441
- // Node is marked skipped (not failed)
1442
- expect(results.get("a").status).toBe("skipped");
1443
- // No node:retry event emitted
1444
- expect(events.filter((e) => e.type === "node:retry")).toHaveLength(0);
1445
- // claude.ask was never called (reflection not invoked for skipped)
1446
- expect(askCallCount.n).toBe(0);
1447
- });
1448
- });
1449
- // ── Test 8: results map contains only the final (successful) attempt data ──
1450
- describe("retry — results map reflects only the final attempt", () => {
1451
- it("downstream nodes see only the successful attempt's data, not the failed attempt's data", async () => {
1452
- const workflow = {
1453
- id: "retry-results-final",
1454
- name: "Retry Results Final",
1455
- description: "",
1456
- entry: "a",
1457
- nodes: {
1458
- a: {
1459
- name: "A",
1460
- instruction: "Do A",
1461
- skills: [],
1462
- verify: { output_required: ["done"] },
1463
- retry: { max: 1 },
1464
- },
1465
- b: {
1466
- name: "B",
1467
- instruction: "Do B",
1468
- skills: [],
1469
- },
1470
- },
1471
- edges: [{ from: "a", to: "b" }],
1472
- };
1473
- let capturedContext = {};
1474
- let callCount = 0;
1475
- const claude = {
1476
- run: async (opts) => {
1477
- if (opts.instruction.includes("Do B")) {
1478
- capturedContext = opts.context;
1479
- return { status: "success", data: { bRan: true }, toolCalls: [] };
1480
- }
1481
- callCount++;
1482
- if (callCount === 1) {
1483
- // First attempt: returns no `done` → verify fails
1484
- return { status: "success", data: { partialField: "from-attempt-1" }, toolCalls: [] };
1485
- }
1486
- // Second attempt: returns `done` → verify passes
1487
- return { status: "success", data: { done: true, partialField: "from-attempt-2" }, toolCalls: [] };
1488
- },
1489
- evaluate: async () => "b",
1490
- ask: async () => "",
1491
- };
1492
- const { results, trace } = await execute(workflow, {}, { skills: createSkillMap([]), claude });
1493
- // A succeeded after retry
1494
- expect(results.get("a").status).toBe("success");
1495
- expect(results.get("a").data.done).toBe(true);
1496
- // The results map for A contains the FINAL attempt's data only
1497
- expect(results.get("a").data.partialField).toBe("from-attempt-2");
1498
- // B's context for "a" shows only the final successful data
1499
- const aInContext = capturedContext["a"];
1500
- expect(aInContext.done).toBe(true);
1501
- expect(aInContext.partialField).toBe("from-attempt-2");
1502
- // Trace has both attempts — failed attempt (retryAttempt: 0) + final (retryAttempt: 1)
1503
- const aSteps = trace.steps.filter((s) => s.node === "a");
1504
- expect(aSteps).toHaveLength(2);
1505
- expect(aSteps[0]).toMatchObject({ node: "a", status: "failed", retryAttempt: 0 });
1506
- expect(aSteps[1]).toMatchObject({ node: "a", status: "success", retryAttempt: 1 });
1507
- });
1508
- });
1509
- // ── Test 9: observer event ordering — node:enter once, node:retry before next run, node:exit after ──
1510
- describe("retry — observer event ordering", () => {
1511
- it("emits events in order: node:enter → node:retry → node:exit (enter fires once per node visit)", async () => {
1512
- const workflow = {
1513
- id: "retry-event-order",
1514
- name: "Retry Event Order",
1515
- description: "",
1516
- entry: "a",
1517
- nodes: {
1518
- a: {
1519
- name: "A",
1520
- instruction: "Do A",
1521
- skills: [],
1522
- verify: { output_required: ["done"] },
1523
- retry: { max: 1 },
1524
- },
1525
- },
1526
- edges: [],
1527
- };
1528
- let callCount = 0;
1529
- const events = [];
1530
- const claude = {
1531
- run: async () => {
1532
- callCount++;
1533
- return callCount === 2
1534
- ? { status: "success", data: { done: true }, toolCalls: [] }
1535
- : { status: "success", data: {}, toolCalls: [] };
1536
- },
1537
- evaluate: async () => "x",
1538
- ask: async () => "",
1539
- };
1540
- await execute(workflow, {}, {
1541
- skills: createSkillMap([]),
1542
- claude,
1543
- observer: (e) => events.push(e),
1544
- });
1545
- // Extract just the event types for node A (and workflow-level events)
1546
- const relevantTypes = events.filter((e) => ("node" in e ? e.node === "a" : true)).map((e) => e.type);
1547
- // node:enter fires exactly ONCE per node visit (not once per retry attempt)
1548
- expect(events.filter((e) => e.type === "node:enter" && e.node === "a")).toHaveLength(1);
1549
- // node:retry fires exactly once (one retry attempt)
1550
- expect(events.filter((e) => e.type === "node:retry" && e.node === "a")).toHaveLength(1);
1551
- // node:exit fires exactly once
1552
- expect(events.filter((e) => e.type === "node:exit" && e.node === "a")).toHaveLength(1);
1553
- // Ordering: node:enter < node:retry < node:exit (relative positions in full event stream)
1554
- const enterIdx = events.findIndex((e) => e.type === "node:enter" && e.node === "a");
1555
- const retryIdx = events.findIndex((e) => e.type === "node:retry" && e.node === "a");
1556
- const exitIdx = events.findIndex((e) => e.type === "node:exit" && e.node === "a");
1557
- expect(enterIdx).toBeLessThan(retryIdx);
1558
- expect(retryIdx).toBeLessThan(exitIdx);
1559
- });
1560
- });
1561
- describe("requires + retry interaction", () => {
1562
- it("does not trigger retry when requires fails (LLM never runs)", async () => {
1563
- const workflow = {
1564
- id: "req-no-retry",
1565
- name: "Req No Retry",
1566
- description: "",
1567
- entry: "a",
1568
- nodes: {
1569
- a: {
1570
- name: "A",
1571
- instruction: "Do A",
1572
- skills: [],
1573
- requires: { output_required: ["input.missing"] },
1574
- verify: { output_required: ["done"] },
1575
- retry: { max: 5 },
1576
- },
1577
- },
1578
- edges: [],
1579
- };
1580
- let callCount = 0;
1581
- const claude = {
1582
- run: async () => {
1583
- callCount++;
1584
- return { status: "success", data: { done: true }, toolCalls: [] };
1585
- },
1586
- evaluate: async () => "x",
1587
- ask: async () => "",
1588
- };
1589
- const { results } = await execute(workflow, {}, {
1590
- skills: createSkillMap([]),
1591
- claude,
1592
- });
1593
- expect(callCount).toBe(0);
1594
- const a = results.get("a");
1595
- expect(a.status).toBe("failed");
1596
- expect(a.data.error).toMatch(/^requires failed:/);
1597
- });
1598
- it("requires passes → verify fails → retry runs as normal", async () => {
1599
- const workflow = {
1600
- id: "req-pass-retry",
1601
- name: "Req Pass Retry",
1602
- description: "",
1603
- entry: "a",
1604
- nodes: {
1605
- a: {
1606
- name: "A",
1607
- instruction: "Do A",
1608
- skills: [],
1609
- requires: { output_required: ["input.x"] },
1610
- verify: { output_required: ["done"] },
1611
- retry: { max: 1 },
1612
- },
1613
- },
1614
- edges: [],
1615
- };
1616
- let callCount = 0;
1617
- const claude = {
1618
- run: async () => {
1619
- callCount++;
1620
- return callCount === 2
1621
- ? { status: "success", data: { done: true }, toolCalls: [] }
1622
- : { status: "success", data: {}, toolCalls: [] };
1623
- },
1624
- evaluate: async () => "x",
1625
- ask: async () => "",
1626
- };
1627
- const { results } = await execute(workflow, { x: 1 }, {
1628
- skills: createSkillMap([]),
1629
- claude,
1630
- });
1631
- expect(callCount).toBe(2);
1632
- expect(results.get("a").status).toBe("success");
1633
- });
1634
- });
1635
- });