@mhingston5/lasso 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/README.md +707 -0
  2. package/docs/agent-wrangling.png +0 -0
  3. package/package.json +26 -0
  4. package/src/capabilities/matcher.ts +25 -0
  5. package/src/capabilities/registry.ts +103 -0
  6. package/src/capabilities/types.ts +15 -0
  7. package/src/cir/lower.ts +253 -0
  8. package/src/cir/optimize.ts +251 -0
  9. package/src/cir/types.ts +131 -0
  10. package/src/cir/validate.ts +265 -0
  11. package/src/compiler/compile.ts +601 -0
  12. package/src/compiler/feedback.ts +471 -0
  13. package/src/compiler/runtime-helpers.ts +455 -0
  14. package/src/composition/chain.ts +58 -0
  15. package/src/composition/conditional.ts +76 -0
  16. package/src/composition/parallel.ts +75 -0
  17. package/src/composition/types.ts +105 -0
  18. package/src/environment/analyzer.ts +56 -0
  19. package/src/environment/discovery.ts +179 -0
  20. package/src/environment/types.ts +68 -0
  21. package/src/failures/classifiers.ts +134 -0
  22. package/src/failures/generator.ts +421 -0
  23. package/src/failures/map-reference-failures.ts +23 -0
  24. package/src/failures/ontology.ts +210 -0
  25. package/src/failures/recovery.ts +214 -0
  26. package/src/failures/types.ts +14 -0
  27. package/src/index.ts +67 -0
  28. package/src/memory/advisor.ts +132 -0
  29. package/src/memory/extractor.ts +166 -0
  30. package/src/memory/store.ts +107 -0
  31. package/src/memory/types.ts +53 -0
  32. package/src/metaharness/engine.ts +256 -0
  33. package/src/metaharness/predictor.ts +168 -0
  34. package/src/metaharness/types.ts +40 -0
  35. package/src/mutation/derive.ts +308 -0
  36. package/src/mutation/diff.ts +52 -0
  37. package/src/mutation/engine.ts +256 -0
  38. package/src/mutation/types.ts +84 -0
  39. package/src/pi/command-input.ts +209 -0
  40. package/src/pi/commands.ts +351 -0
  41. package/src/pi/extension.ts +16 -0
  42. package/src/planner/synthesize.ts +83 -0
  43. package/src/planner/template-rules.ts +183 -0
  44. package/src/planner/types.ts +42 -0
  45. package/src/reference/catalog.ts +128 -0
  46. package/src/reference/patch-validation-strategies.ts +170 -0
  47. package/src/reference/patch-validation.ts +174 -0
  48. package/src/reference/pr-review-merge.ts +155 -0
  49. package/src/reference/strategies.ts +126 -0
  50. package/src/reference/types.ts +33 -0
  51. package/src/replanner/risk-rules.ts +161 -0
  52. package/src/replanner/runtime.ts +308 -0
  53. package/src/replanner/synthesize.ts +619 -0
  54. package/src/replanner/types.ts +73 -0
  55. package/src/spec/schema.ts +254 -0
  56. package/src/spec/types.ts +319 -0
  57. package/src/spec/validate.ts +296 -0
  58. package/src/state/snapshots.ts +43 -0
  59. package/src/state/types.ts +12 -0
  60. package/src/synthesis/graph-builder.ts +267 -0
  61. package/src/synthesis/harness-builder.ts +113 -0
  62. package/src/synthesis/intent-ir.ts +63 -0
  63. package/src/synthesis/policy-builder.ts +320 -0
  64. package/src/synthesis/risk-analyzer.ts +182 -0
  65. package/src/synthesis/skill-parser.ts +441 -0
  66. package/src/verification/engine.ts +230 -0
  67. package/src/versioning/file-store.ts +103 -0
  68. package/src/versioning/history.ts +43 -0
  69. package/src/versioning/store.ts +16 -0
  70. package/src/versioning/types.ts +31 -0
  71. package/test/capabilities/matcher.test.ts +67 -0
  72. package/test/capabilities/registry.test.ts +136 -0
  73. package/test/capabilities/synthesis.test.ts +264 -0
  74. package/test/cir/lower.test.ts +417 -0
  75. package/test/cir/optimize.test.ts +266 -0
  76. package/test/cir/validate.test.ts +368 -0
  77. package/test/compiler/adaptive-runtime.test.ts +157 -0
  78. package/test/compiler/compile.test.ts +1198 -0
  79. package/test/compiler/feedback.test.ts +784 -0
  80. package/test/compiler/guardrails.test.ts +191 -0
  81. package/test/compiler/trace.test.ts +404 -0
  82. package/test/composition/chain.test.ts +328 -0
  83. package/test/composition/conditional.test.ts +241 -0
  84. package/test/composition/parallel.test.ts +215 -0
  85. package/test/environment/analyzer.test.ts +204 -0
  86. package/test/environment/discovery.test.ts +149 -0
  87. package/test/failures/classifiers.test.ts +287 -0
  88. package/test/failures/generator.test.ts +203 -0
  89. package/test/failures/ontology.test.ts +439 -0
  90. package/test/failures/recovery.test.ts +300 -0
  91. package/test/helpers/createFixtureRepo.ts +84 -0
  92. package/test/helpers/createPatchValidationFixture.ts +144 -0
  93. package/test/helpers/runCompiledWorkflow.ts +208 -0
  94. package/test/memory/advisor.test.ts +332 -0
  95. package/test/memory/extractor.test.ts +295 -0
  96. package/test/memory/store.test.ts +244 -0
  97. package/test/metaharness/engine.test.ts +575 -0
  98. package/test/metaharness/predictor.test.ts +436 -0
  99. package/test/mutation/derive-failure.test.ts +209 -0
  100. package/test/mutation/engine.test.ts +622 -0
  101. package/test/package-smoke.test.ts +29 -0
  102. package/test/pi/command-input.test.ts +153 -0
  103. package/test/pi/commands.test.ts +623 -0
  104. package/test/planner/classify-template.test.ts +32 -0
  105. package/test/planner/synthesize.test.ts +901 -0
  106. package/test/reference/PatchValidation.failures.test.ts +137 -0
  107. package/test/reference/PatchValidation.test.ts +326 -0
  108. package/test/reference/PrReviewMerge.failures.test.ts +121 -0
  109. package/test/reference/PrReviewMerge.test.ts +55 -0
  110. package/test/reference/catalog-open.test.ts +70 -0
  111. package/test/replanner/runtime.test.ts +207 -0
  112. package/test/replanner/synthesize.test.ts +303 -0
  113. package/test/spec/validate.test.ts +1056 -0
  114. package/test/state/snapshots.test.ts +264 -0
  115. package/test/synthesis/custom-workflow.test.ts +264 -0
  116. package/test/synthesis/graph-builder.test.ts +370 -0
  117. package/test/synthesis/harness-builder.test.ts +128 -0
  118. package/test/synthesis/policy-builder.test.ts +149 -0
  119. package/test/synthesis/risk-analyzer.test.ts +230 -0
  120. package/test/synthesis/skill-parser.test.ts +796 -0
  121. package/test/verification/engine.test.ts +509 -0
  122. package/test/versioning/history.test.ts +144 -0
  123. package/test/versioning/store.test.ts +254 -0
  124. package/vitest.config.ts +9 -0
@@ -0,0 +1,1056 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import { validateHarnessSpec } from "../../src/spec/validate.js";
3
+ import type { HarnessSpec } from "../../src/spec/types.js";
4
+
5
+ describe("validateHarnessSpec", () => {
6
+ it("accepts a valid minimal spec", () => {
7
+ const spec: HarnessSpec = {
8
+ name: "pr-review-merge",
9
+ graph: {
10
+ entryNodeId: "load-pr",
11
+ nodes: [
12
+ {
13
+ id: "load-pr",
14
+ kind: "tool",
15
+ tool: "gh",
16
+ args: ["pr", "view", "123"]
17
+ }
18
+ ],
19
+ edges: []
20
+ }
21
+ };
22
+
23
+ const result = validateHarnessSpec(spec);
24
+ expect(result.valid).toBe(true);
25
+ });
26
+
27
+ it("rejects spec with missing node ID", () => {
28
+ const spec = {
29
+ name: "test-workflow",
30
+ graph: {
31
+ entryNodeId: "start",
32
+ nodes: [
33
+ {
34
+ // missing id
35
+ kind: "tool",
36
+ tool: "echo",
37
+ args: ["hello"]
38
+ }
39
+ ],
40
+ edges: []
41
+ }
42
+ } as any;
43
+
44
+ const result = validateHarnessSpec(spec);
45
+ expect(result.valid).toBe(false);
46
+ if (!result.valid) {
47
+ expect(result.errors.length).toBeGreaterThan(0);
48
+ expect(result.errors.some(e => e.includes("id"))).toBe(true);
49
+ }
50
+ });
51
+
52
+ it("rejects spec with duplicate node IDs", () => {
53
+ const spec: HarnessSpec = {
54
+ name: "test-workflow",
55
+ graph: {
56
+ entryNodeId: "node1",
57
+ nodes: [
58
+ {
59
+ id: "node1",
60
+ kind: "tool",
61
+ tool: "echo",
62
+ args: ["first"]
63
+ },
64
+ {
65
+ id: "node1",
66
+ kind: "tool",
67
+ tool: "echo",
68
+ args: ["second"]
69
+ }
70
+ ],
71
+ edges: []
72
+ }
73
+ };
74
+
75
+ const result = validateHarnessSpec(spec);
76
+ expect(result.valid).toBe(false);
77
+ if (!result.valid) {
78
+ expect(result.errors.some(e => e.includes("duplicate") || e.includes("node1"))).toBe(true);
79
+ }
80
+ });
81
+
82
+ it("rejects spec with unreachable node", () => {
83
+ const spec: HarnessSpec = {
84
+ name: "test-workflow",
85
+ graph: {
86
+ entryNodeId: "start",
87
+ nodes: [
88
+ {
89
+ id: "start",
90
+ kind: "tool",
91
+ tool: "echo",
92
+ args: ["hello"]
93
+ },
94
+ {
95
+ id: "orphan",
96
+ kind: "tool",
97
+ tool: "echo",
98
+ args: ["unreachable"]
99
+ }
100
+ ],
101
+ edges: []
102
+ }
103
+ };
104
+
105
+ const result = validateHarnessSpec(spec);
106
+ expect(result.valid).toBe(false);
107
+ if (!result.valid) {
108
+ expect(result.errors.some(e => e.includes("unreachable") || e.includes("orphan"))).toBe(true);
109
+ }
110
+ });
111
+
112
+ it("rejects spec with invalid edge target", () => {
113
+ const spec: HarnessSpec = {
114
+ name: "test-workflow",
115
+ graph: {
116
+ entryNodeId: "start",
117
+ nodes: [
118
+ {
119
+ id: "start",
120
+ kind: "tool",
121
+ tool: "echo",
122
+ args: ["hello"]
123
+ }
124
+ ],
125
+ edges: [
126
+ {
127
+ from: "start",
128
+ to: "nonexistent"
129
+ }
130
+ ]
131
+ }
132
+ };
133
+
134
+ const result = validateHarnessSpec(spec);
135
+ expect(result.valid).toBe(false);
136
+ if (!result.valid) {
137
+ expect(result.errors.some(e => e.includes("nonexistent") || e.includes("invalid"))).toBe(true);
138
+ }
139
+ });
140
+
141
+ it("rejects retry policy applied to unsupported node kinds", () => {
142
+ const spec: HarnessSpec = {
143
+ name: "test-workflow",
144
+ graph: {
145
+ entryNodeId: "merge",
146
+ nodes: [
147
+ {
148
+ id: "merge",
149
+ kind: "merge",
150
+ waitFor: ["branch1", "branch2"],
151
+ retryPolicy: {
152
+ maxAttempts: 3,
153
+ backoff: "exponential"
154
+ }
155
+ },
156
+ {
157
+ id: "branch1",
158
+ kind: "tool",
159
+ tool: "echo",
160
+ args: ["1"]
161
+ },
162
+ {
163
+ id: "branch2",
164
+ kind: "tool",
165
+ tool: "echo",
166
+ args: ["2"]
167
+ }
168
+ ],
169
+ edges: []
170
+ }
171
+ };
172
+
173
+ const result = validateHarnessSpec(spec);
174
+ expect(result.valid).toBe(false);
175
+ if (!result.valid) {
176
+ expect(result.errors.some(e => e.includes("retry") && e.includes("merge"))).toBe(true);
177
+ }
178
+ });
179
+
180
+ it("rejects verification rule referencing a missing node", () => {
181
+ const spec: HarnessSpec = {
182
+ name: "test-workflow",
183
+ graph: {
184
+ entryNodeId: "start",
185
+ nodes: [
186
+ {
187
+ id: "start",
188
+ kind: "tool",
189
+ tool: "echo",
190
+ args: ["hello"],
191
+ verificationPolicy: {
192
+ rules: [
193
+ {
194
+ checkNodeId: "nonexistent-check",
195
+ onFail: "block"
196
+ }
197
+ ]
198
+ }
199
+ }
200
+ ],
201
+ edges: []
202
+ }
203
+ };
204
+
205
+ const result = validateHarnessSpec(spec);
206
+ expect(result.valid).toBe(false);
207
+ if (!result.valid) {
208
+ expect(result.errors.some(e => e.includes("nonexistent-check") || e.includes("verification"))).toBe(true);
209
+ }
210
+ });
211
+
212
+ // Issue 1: Reject edges with condition property
213
+ it("rejects edges with condition property", () => {
214
+ const spec = {
215
+ name: "test-workflow",
216
+ graph: {
217
+ entryNodeId: "start",
218
+ nodes: [
219
+ {
220
+ id: "start",
221
+ kind: "tool",
222
+ tool: "echo",
223
+ args: ["hello"]
224
+ },
225
+ {
226
+ id: "next",
227
+ kind: "tool",
228
+ tool: "echo",
229
+ args: ["world"]
230
+ }
231
+ ],
232
+ edges: [
233
+ {
234
+ from: "start",
235
+ to: "next",
236
+ condition: "some.expr"
237
+ }
238
+ ]
239
+ }
240
+ } as any;
241
+
242
+ const result = validateHarnessSpec(spec);
243
+ expect(result.valid).toBe(false);
244
+ if (!result.valid) {
245
+ expect(result.errors.some(e => e.includes("additional properties") && e.includes("edges"))).toBe(true);
246
+ }
247
+ });
248
+
249
+ // Issue 2: Reject nodes with arbitrary extra properties
250
+ it("rejects nodes with unsupported properties", () => {
251
+ const spec = {
252
+ name: "test-workflow",
253
+ graph: {
254
+ entryNodeId: "start",
255
+ nodes: [
256
+ {
257
+ id: "start",
258
+ kind: "tool",
259
+ tool: "echo",
260
+ args: ["hello"],
261
+ unsupportedField: "should be rejected"
262
+ }
263
+ ],
264
+ edges: []
265
+ }
266
+ } as any;
267
+
268
+ const result = validateHarnessSpec(spec);
269
+ expect(result.valid).toBe(false);
270
+ if (!result.valid) {
271
+ expect(result.errors.some(e => e.includes("additional properties"))).toBe(true);
272
+ }
273
+ });
274
+
275
+ // Issue 3: Reject verification self-reference
276
+ it("rejects verification rule that references itself", () => {
277
+ const spec: HarnessSpec = {
278
+ name: "test-workflow",
279
+ graph: {
280
+ entryNodeId: "start",
281
+ nodes: [
282
+ {
283
+ id: "start",
284
+ kind: "tool",
285
+ tool: "echo",
286
+ args: ["hello"],
287
+ verificationPolicy: {
288
+ rules: [
289
+ {
290
+ checkNodeId: "start",
291
+ onFail: "block"
292
+ }
293
+ ]
294
+ }
295
+ }
296
+ ],
297
+ edges: []
298
+ }
299
+ };
300
+
301
+ const result = validateHarnessSpec(spec);
302
+ expect(result.valid).toBe(false);
303
+ if (!result.valid) {
304
+ expect(result.errors.some(e => e.includes("cannot reference itself"))).toBe(true);
305
+ }
306
+ });
307
+
308
+ // Issue 3: Reject circular verification dependencies
309
+ it("rejects circular verification dependencies", () => {
310
+ const spec: HarnessSpec = {
311
+ name: "test-workflow",
312
+ graph: {
313
+ entryNodeId: "nodeA",
314
+ nodes: [
315
+ {
316
+ id: "nodeA",
317
+ kind: "tool",
318
+ tool: "echo",
319
+ args: ["A"],
320
+ verificationPolicy: {
321
+ rules: [
322
+ {
323
+ checkNodeId: "nodeB",
324
+ onFail: "block"
325
+ }
326
+ ]
327
+ }
328
+ },
329
+ {
330
+ id: "nodeB",
331
+ kind: "tool",
332
+ tool: "echo",
333
+ args: ["B"],
334
+ verificationPolicy: {
335
+ rules: [
336
+ {
337
+ checkNodeId: "nodeA",
338
+ onFail: "block"
339
+ }
340
+ ]
341
+ }
342
+ }
343
+ ],
344
+ edges: []
345
+ }
346
+ };
347
+
348
+ const result = validateHarnessSpec(spec);
349
+ expect(result.valid).toBe(false);
350
+ if (!result.valid) {
351
+ expect(result.errors.some(e => e.includes("Circular verification dependency"))).toBe(true);
352
+ }
353
+ });
354
+
355
+ // Issue 1: Reject 3-node verification cycle
356
+ it("rejects 3-node verification cycle", () => {
357
+ const spec: HarnessSpec = {
358
+ name: "test-workflow",
359
+ graph: {
360
+ entryNodeId: "nodeA",
361
+ nodes: [
362
+ {
363
+ id: "nodeA",
364
+ kind: "tool",
365
+ tool: "echo",
366
+ args: ["A"],
367
+ verificationPolicy: {
368
+ rules: [
369
+ {
370
+ checkNodeId: "nodeB",
371
+ onFail: "block"
372
+ }
373
+ ]
374
+ }
375
+ },
376
+ {
377
+ id: "nodeB",
378
+ kind: "tool",
379
+ tool: "echo",
380
+ args: ["B"],
381
+ verificationPolicy: {
382
+ rules: [
383
+ {
384
+ checkNodeId: "nodeC",
385
+ onFail: "block"
386
+ }
387
+ ]
388
+ }
389
+ },
390
+ {
391
+ id: "nodeC",
392
+ kind: "tool",
393
+ tool: "echo",
394
+ args: ["C"],
395
+ verificationPolicy: {
396
+ rules: [
397
+ {
398
+ checkNodeId: "nodeA",
399
+ onFail: "block"
400
+ }
401
+ ]
402
+ }
403
+ }
404
+ ],
405
+ edges: []
406
+ }
407
+ };
408
+
409
+ const result = validateHarnessSpec(spec);
410
+ expect(result.valid).toBe(false);
411
+ if (!result.valid) {
412
+ expect(result.errors.some(e => e.includes("Circular verification dependency"))).toBe(true);
413
+ }
414
+ });
415
+
416
+ // Issue 2: Reject empty string identifiers
417
+ it("rejects empty node id", () => {
418
+ const spec: HarnessSpec = {
419
+ name: "test-workflow",
420
+ graph: {
421
+ entryNodeId: "start",
422
+ nodes: [
423
+ {
424
+ id: "",
425
+ kind: "tool",
426
+ tool: "echo",
427
+ args: ["hello"]
428
+ }
429
+ ],
430
+ edges: []
431
+ }
432
+ };
433
+
434
+ const result = validateHarnessSpec(spec);
435
+ expect(result.valid).toBe(false);
436
+ if (!result.valid) {
437
+ expect(result.errors.some(e => e.includes("fewer than 1 characters"))).toBe(true);
438
+ }
439
+ });
440
+
441
+ it("rejects empty entryNodeId", () => {
442
+ const spec: HarnessSpec = {
443
+ name: "test-workflow",
444
+ graph: {
445
+ entryNodeId: "",
446
+ nodes: [
447
+ {
448
+ id: "start",
449
+ kind: "tool",
450
+ tool: "echo",
451
+ args: ["hello"]
452
+ }
453
+ ],
454
+ edges: []
455
+ }
456
+ };
457
+
458
+ const result = validateHarnessSpec(spec);
459
+ expect(result.valid).toBe(false);
460
+ if (!result.valid) {
461
+ expect(result.errors.some(e => e.includes("fewer than 1 characters"))).toBe(true);
462
+ }
463
+ });
464
+
465
+ it("rejects empty edge from/to", () => {
466
+ const spec: HarnessSpec = {
467
+ name: "test-workflow",
468
+ graph: {
469
+ entryNodeId: "start",
470
+ nodes: [
471
+ {
472
+ id: "start",
473
+ kind: "tool",
474
+ tool: "echo",
475
+ args: ["hello"]
476
+ },
477
+ {
478
+ id: "next",
479
+ kind: "tool",
480
+ tool: "echo",
481
+ args: ["world"]
482
+ }
483
+ ],
484
+ edges: [
485
+ {
486
+ from: "",
487
+ to: "next"
488
+ }
489
+ ]
490
+ }
491
+ };
492
+
493
+ const result = validateHarnessSpec(spec);
494
+ expect(result.valid).toBe(false);
495
+ if (!result.valid) {
496
+ expect(result.errors.some(e => e.includes("fewer than 1 characters"))).toBe(true);
497
+ }
498
+ });
499
+
500
+ it("rejects empty tool name", () => {
501
+ const spec: HarnessSpec = {
502
+ name: "test-workflow",
503
+ graph: {
504
+ entryNodeId: "start",
505
+ nodes: [
506
+ {
507
+ id: "start",
508
+ kind: "tool",
509
+ tool: "",
510
+ args: ["hello"]
511
+ }
512
+ ],
513
+ edges: []
514
+ }
515
+ };
516
+
517
+ const result = validateHarnessSpec(spec);
518
+ expect(result.valid).toBe(false);
519
+ if (!result.valid) {
520
+ expect(result.errors.some(e => e.includes("fewer than 1 characters"))).toBe(true);
521
+ }
522
+ });
523
+
524
+ it("rejects empty LLM provider/model/prompt", () => {
525
+ const spec: HarnessSpec = {
526
+ name: "test-workflow",
527
+ graph: {
528
+ entryNodeId: "start",
529
+ nodes: [
530
+ {
531
+ id: "start",
532
+ kind: "llm",
533
+ provider: "",
534
+ model: "gpt-4",
535
+ prompt: "test"
536
+ }
537
+ ],
538
+ edges: []
539
+ }
540
+ };
541
+
542
+ const result = validateHarnessSpec(spec);
543
+ expect(result.valid).toBe(false);
544
+ if (!result.valid) {
545
+ expect(result.errors.some(e => e.includes("fewer than 1 characters"))).toBe(true);
546
+ }
547
+ });
548
+
549
+ it("rejects empty condition fields", () => {
550
+ const spec: HarnessSpec = {
551
+ name: "test-workflow",
552
+ graph: {
553
+ entryNodeId: "start",
554
+ nodes: [
555
+ {
556
+ id: "start",
557
+ kind: "condition",
558
+ condition: "",
559
+ thenNodeId: "then",
560
+ elseNodeId: "else"
561
+ },
562
+ {
563
+ id: "then",
564
+ kind: "tool",
565
+ tool: "echo",
566
+ args: ["then"]
567
+ },
568
+ {
569
+ id: "else",
570
+ kind: "tool",
571
+ tool: "echo",
572
+ args: ["else"]
573
+ }
574
+ ],
575
+ edges: []
576
+ }
577
+ };
578
+
579
+ const result = validateHarnessSpec(spec);
580
+ expect(result.valid).toBe(false);
581
+ if (!result.valid) {
582
+ expect(result.errors.some(e => e.includes("fewer than 1 characters"))).toBe(true);
583
+ }
584
+ });
585
+
586
+ it("rejects empty checkNodeId in verification", () => {
587
+ const spec: HarnessSpec = {
588
+ name: "test-workflow",
589
+ graph: {
590
+ entryNodeId: "start",
591
+ nodes: [
592
+ {
593
+ id: "start",
594
+ kind: "tool",
595
+ tool: "echo",
596
+ args: ["hello"],
597
+ verificationPolicy: {
598
+ rules: [
599
+ {
600
+ checkNodeId: "",
601
+ onFail: "block"
602
+ }
603
+ ]
604
+ }
605
+ }
606
+ ],
607
+ edges: []
608
+ }
609
+ };
610
+
611
+ const result = validateHarnessSpec(spec);
612
+ expect(result.valid).toBe(false);
613
+ if (!result.valid) {
614
+ expect(result.errors.some(e => e.includes("fewer than 1 characters"))).toBe(true);
615
+ }
616
+ });
617
+
618
+ // Issue 3: Reject non-string env values
619
+ it("rejects non-string env values", () => {
620
+ const spec = {
621
+ name: "test-workflow",
622
+ graph: {
623
+ entryNodeId: "start",
624
+ nodes: [
625
+ {
626
+ id: "start",
627
+ kind: "tool",
628
+ tool: "echo",
629
+ args: ["hello"],
630
+ env: {
631
+ VAR1: "string-value",
632
+ VAR2: 123
633
+ }
634
+ }
635
+ ],
636
+ edges: []
637
+ }
638
+ } as any;
639
+
640
+ const result = validateHarnessSpec(spec);
641
+ expect(result.valid).toBe(false);
642
+ if (!result.valid) {
643
+ expect(result.errors.some(e => e.includes("must be string") || e.includes("type"))).toBe(true);
644
+ }
645
+ });
646
+
647
+ // Issue 1: Reject orphan subgraph
648
+ it("rejects orphan subgraph disconnected from entry", () => {
649
+ const spec: HarnessSpec = {
650
+ name: "test-workflow",
651
+ graph: {
652
+ entryNodeId: "start",
653
+ nodes: [
654
+ {
655
+ id: "start",
656
+ kind: "tool",
657
+ tool: "echo",
658
+ args: ["connected"]
659
+ },
660
+ {
661
+ id: "orphan",
662
+ kind: "tool",
663
+ tool: "echo",
664
+ args: ["orphan1"]
665
+ },
666
+ {
667
+ id: "orphan2",
668
+ kind: "tool",
669
+ tool: "echo",
670
+ args: ["orphan2"]
671
+ }
672
+ ],
673
+ edges: [
674
+ {
675
+ from: "orphan",
676
+ to: "orphan2"
677
+ }
678
+ ]
679
+ }
680
+ };
681
+
682
+ const result = validateHarnessSpec(spec);
683
+ expect(result.valid).toBe(false);
684
+ if (!result.valid) {
685
+ expect(result.errors.some(e => e.includes("Unreachable") && (e.includes("orphan") || e.includes("orphan2")))).toBe(true);
686
+ }
687
+ });
688
+
689
+ // Issue 2: Reject choice interaction without options
690
+ it("rejects choice interaction without options", () => {
691
+ const spec: HarnessSpec = {
692
+ name: "test-workflow",
693
+ graph: {
694
+ entryNodeId: "ask",
695
+ nodes: [
696
+ {
697
+ id: "ask",
698
+ kind: "human",
699
+ prompt: "Choose an option",
700
+ interactionType: "choice"
701
+ }
702
+ ],
703
+ edges: []
704
+ }
705
+ };
706
+
707
+ const result = validateHarnessSpec(spec);
708
+ expect(result.valid).toBe(false);
709
+ if (!result.valid) {
710
+ expect(result.errors.some(e => e.includes("choice") && e.includes("options"))).toBe(true);
711
+ }
712
+ });
713
+
714
+ it("rejects choice interaction with empty options", () => {
715
+ const spec: HarnessSpec = {
716
+ name: "test-workflow",
717
+ graph: {
718
+ entryNodeId: "ask",
719
+ nodes: [
720
+ {
721
+ id: "ask",
722
+ kind: "human",
723
+ prompt: "Choose an option",
724
+ interactionType: "choice",
725
+ options: []
726
+ }
727
+ ],
728
+ edges: []
729
+ }
730
+ };
731
+
732
+ const result = validateHarnessSpec(spec);
733
+ expect(result.valid).toBe(false);
734
+ if (!result.valid) {
735
+ expect(result.errors.some(e => e.includes("choice") && e.includes("options"))).toBe(true);
736
+ }
737
+ });
738
+
739
+ // Issue 3: Reject empty waitFor array
740
+ it("rejects merge node with empty waitFor", () => {
741
+ const spec: HarnessSpec = {
742
+ name: "test-workflow",
743
+ graph: {
744
+ entryNodeId: "merge",
745
+ nodes: [
746
+ {
747
+ id: "merge",
748
+ kind: "merge",
749
+ waitFor: []
750
+ }
751
+ ],
752
+ edges: []
753
+ }
754
+ };
755
+
756
+ const result = validateHarnessSpec(spec);
757
+ expect(result.valid).toBe(false);
758
+ if (!result.valid) {
759
+ expect(result.errors.some(e => e.includes("waitFor") || e.includes("minItems"))).toBe(true);
760
+ }
761
+ });
762
+
763
+ describe("verification rule kind validation", () => {
764
+ it("accepts tool verification rule pointing to tool node", () => {
765
+ const spec: HarnessSpec = {
766
+ name: "test-workflow",
767
+ graph: {
768
+ entryNodeId: "action",
769
+ nodes: [
770
+ {
771
+ id: "action",
772
+ kind: "tool",
773
+ tool: "echo",
774
+ args: ["test"],
775
+ verificationPolicy: {
776
+ rules: [
777
+ {
778
+ kind: "tool",
779
+ checkNodeId: "verifier",
780
+ onFail: "block",
781
+ },
782
+ ],
783
+ },
784
+ },
785
+ {
786
+ id: "verifier",
787
+ kind: "tool",
788
+ tool: "test",
789
+ args: ["-f", "output.txt"],
790
+ },
791
+ ],
792
+ edges: [],
793
+ },
794
+ };
795
+
796
+ const result = validateHarnessSpec(spec);
797
+ expect(result.valid).toBe(true);
798
+ });
799
+
800
+ it("accepts llm verification rule pointing to llm node", () => {
801
+ const spec: HarnessSpec = {
802
+ name: "test-workflow",
803
+ graph: {
804
+ entryNodeId: "action",
805
+ nodes: [
806
+ {
807
+ id: "action",
808
+ kind: "tool",
809
+ tool: "echo",
810
+ args: ["test"],
811
+ verificationPolicy: {
812
+ rules: [
813
+ {
814
+ kind: "llm",
815
+ checkNodeId: "verifier",
816
+ onFail: "warn",
817
+ },
818
+ ],
819
+ },
820
+ },
821
+ {
822
+ id: "verifier",
823
+ kind: "llm",
824
+ provider: "openai",
825
+ model: "gpt-4",
826
+ prompt: "Verify the output",
827
+ },
828
+ ],
829
+ edges: [],
830
+ },
831
+ };
832
+
833
+ const result = validateHarnessSpec(spec);
834
+ expect(result.valid).toBe(true);
835
+ });
836
+
837
+ it("accepts expression verification rule pointing to condition node", () => {
838
+ const spec: HarnessSpec = {
839
+ name: "test-workflow",
840
+ graph: {
841
+ entryNodeId: "action",
842
+ nodes: [
843
+ {
844
+ id: "action",
845
+ kind: "tool",
846
+ tool: "echo",
847
+ args: ["test"],
848
+ verificationPolicy: {
849
+ rules: [
850
+ {
851
+ kind: "expression",
852
+ checkNodeId: "check-expr",
853
+ onFail: "retry",
854
+ maxAttempts: 3,
855
+ },
856
+ ],
857
+ },
858
+ },
859
+ {
860
+ id: "check-expr",
861
+ kind: "condition",
862
+ condition: "outputs.action.exitCode === 0",
863
+ thenNodeId: "success",
864
+ elseNodeId: "failure",
865
+ },
866
+ {
867
+ id: "success",
868
+ kind: "tool",
869
+ tool: "echo",
870
+ args: ["success"],
871
+ },
872
+ {
873
+ id: "failure",
874
+ kind: "tool",
875
+ tool: "echo",
876
+ args: ["failure"],
877
+ },
878
+ ],
879
+ edges: [],
880
+ },
881
+ };
882
+
883
+ const result = validateHarnessSpec(spec);
884
+ expect(result.valid).toBe(true);
885
+ });
886
+
887
+ it("rejects verification rule with missing kind", () => {
888
+ const spec = {
889
+ name: "test-workflow",
890
+ graph: {
891
+ entryNodeId: "action",
892
+ nodes: [
893
+ {
894
+ id: "action",
895
+ kind: "tool",
896
+ tool: "echo",
897
+ args: ["test"],
898
+ verificationPolicy: {
899
+ rules: [
900
+ {
901
+ // missing kind
902
+ checkNodeId: "verifier",
903
+ onFail: "block",
904
+ },
905
+ ],
906
+ },
907
+ },
908
+ {
909
+ id: "verifier",
910
+ kind: "tool",
911
+ tool: "test",
912
+ args: ["-f", "output.txt"],
913
+ },
914
+ ],
915
+ edges: [],
916
+ },
917
+ } as any;
918
+
919
+ const result = validateHarnessSpec(spec);
920
+ expect(result.valid).toBe(false);
921
+ if (!result.valid) {
922
+ expect(result.errors.some((e) => e.includes("kind") && e.includes("verification"))).toBe(true);
923
+ }
924
+ });
925
+
926
+ it("rejects tool verification rule pointing to llm node", () => {
927
+ const spec: HarnessSpec = {
928
+ name: "test-workflow",
929
+ graph: {
930
+ entryNodeId: "action",
931
+ nodes: [
932
+ {
933
+ id: "action",
934
+ kind: "tool",
935
+ tool: "echo",
936
+ args: ["test"],
937
+ verificationPolicy: {
938
+ rules: [
939
+ {
940
+ kind: "tool",
941
+ checkNodeId: "verifier",
942
+ onFail: "block",
943
+ },
944
+ ],
945
+ },
946
+ },
947
+ {
948
+ id: "verifier",
949
+ kind: "llm",
950
+ provider: "openai",
951
+ model: "gpt-4",
952
+ prompt: "Check",
953
+ },
954
+ ],
955
+ edges: [],
956
+ },
957
+ };
958
+
959
+ const result = validateHarnessSpec(spec);
960
+ expect(result.valid).toBe(false);
961
+ if (!result.valid) {
962
+ expect(
963
+ result.errors.some(
964
+ (e) => e.includes("verifier") && e.includes("kind") && e.includes("tool")
965
+ )
966
+ ).toBe(true);
967
+ }
968
+ });
969
+
970
+ it("rejects llm verification rule pointing to tool node", () => {
971
+ const spec: HarnessSpec = {
972
+ name: "test-workflow",
973
+ graph: {
974
+ entryNodeId: "action",
975
+ nodes: [
976
+ {
977
+ id: "action",
978
+ kind: "tool",
979
+ tool: "echo",
980
+ args: ["test"],
981
+ verificationPolicy: {
982
+ rules: [
983
+ {
984
+ kind: "llm",
985
+ checkNodeId: "verifier",
986
+ onFail: "block",
987
+ },
988
+ ],
989
+ },
990
+ },
991
+ {
992
+ id: "verifier",
993
+ kind: "tool",
994
+ tool: "test",
995
+ args: ["-f", "output.txt"],
996
+ },
997
+ ],
998
+ edges: [],
999
+ },
1000
+ };
1001
+
1002
+ const result = validateHarnessSpec(spec);
1003
+ expect(result.valid).toBe(false);
1004
+ if (!result.valid) {
1005
+ expect(
1006
+ result.errors.some(
1007
+ (e) => e.includes("verifier") && e.includes("kind") && e.includes("llm")
1008
+ )
1009
+ ).toBe(true);
1010
+ }
1011
+ });
1012
+
1013
+ it("rejects expression verification rule pointing to tool node", () => {
1014
+ const spec: HarnessSpec = {
1015
+ name: "test-workflow",
1016
+ graph: {
1017
+ entryNodeId: "action",
1018
+ nodes: [
1019
+ {
1020
+ id: "action",
1021
+ kind: "tool",
1022
+ tool: "echo",
1023
+ args: ["test"],
1024
+ verificationPolicy: {
1025
+ rules: [
1026
+ {
1027
+ kind: "expression",
1028
+ checkNodeId: "verifier",
1029
+ onFail: "block",
1030
+ },
1031
+ ],
1032
+ },
1033
+ },
1034
+ {
1035
+ id: "verifier",
1036
+ kind: "tool",
1037
+ tool: "test",
1038
+ args: ["-f", "output.txt"],
1039
+ },
1040
+ ],
1041
+ edges: [],
1042
+ },
1043
+ };
1044
+
1045
+ const result = validateHarnessSpec(spec);
1046
+ expect(result.valid).toBe(false);
1047
+ if (!result.valid) {
1048
+ expect(
1049
+ result.errors.some(
1050
+ (e) => e.includes("verifier") && e.includes("kind") && e.includes("expression") && e.includes("condition")
1051
+ )
1052
+ ).toBe(true);
1053
+ }
1054
+ });
1055
+ });
1056
+ });