@sweny-ai/core 0.1.84 → 0.1.86

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/schema.js CHANGED
@@ -90,7 +90,9 @@ export const outputMatchZ = z
90
90
  const operators = [m.equals !== undefined, m.in !== undefined, m.matches !== undefined];
91
91
  return operators.filter(Boolean).length === 1;
92
92
  }, { message: "output_matches entry must declare exactly one of: equals, in, matches" });
93
- export const nodeVerifyZ = z
93
+ export const evaluatorKindZ = z.enum(["value", "function", "judge"]);
94
+ export const evalPolicyZ = z.enum(["all_pass", "any_pass", "weighted"]);
95
+ export const evaluatorRuleZ = z
94
96
  .object({
95
97
  any_tool_called: z.array(z.string().min(1)).min(1).optional(),
96
98
  all_tools_called: z.array(z.string().min(1)).min(1).optional(),
@@ -98,18 +100,71 @@ export const nodeVerifyZ = z
98
100
  output_required: z.array(z.string().min(1)).min(1).optional(),
99
101
  output_matches: z.array(outputMatchZ).min(1).optional(),
100
102
  })
101
- // Fix #4 gap: the exported JSON Schema sets additionalProperties: false.
102
- // Without .strict() here Zod would silently drop unknown keys and then
103
- // the refine would run on the stripped object, producing a different
104
- // verdict than ajv (which rejects the unknown key directly). Keep them
105
- // in sync.
106
103
  .strict()
107
- .refine((v) => v.any_tool_called !== undefined ||
108
- v.all_tools_called !== undefined ||
109
- v.no_tool_called !== undefined ||
110
- v.output_required !== undefined ||
111
- v.output_matches !== undefined, {
112
- message: "verify must declare at least one check (any_tool_called, all_tools_called, no_tool_called, output_required, or output_matches)",
104
+ .refine((r) => r.any_tool_called !== undefined ||
105
+ r.all_tools_called !== undefined ||
106
+ r.no_tool_called !== undefined ||
107
+ r.output_required !== undefined ||
108
+ r.output_matches !== undefined, {
109
+ message: "evaluator rule must declare at least one of: any_tool_called, all_tools_called, no_tool_called, output_required, output_matches",
110
+ });
111
+ /**
112
+ * A single evaluator. The `kind` field discriminates required-fields:
113
+ * `value` and `function` need `rule`; `judge` needs `rubric`.
114
+ *
115
+ * Strict object: unknown keys are rejected to match the public JSON Schema.
116
+ */
117
+ export const evaluatorZ = z
118
+ .object({
119
+ name: z.string().min(1),
120
+ kind: evaluatorKindZ,
121
+ rule: evaluatorRuleZ.optional(),
122
+ rubric: z.string().min(1).optional(),
123
+ // `pass_when` is parsed against a single VERDICT token from the model's
124
+ // response (default `yes`). Whitespace breaks the prompt format and the
125
+ // verdict comparison, so reject it at parse time rather than producing
126
+ // a confusing runtime mismatch.
127
+ pass_when: z
128
+ .string()
129
+ .min(1)
130
+ .refine((v) => !/\s/.test(v), { message: "pass_when must be a single whitespace-free token" })
131
+ .optional(),
132
+ model: z.string().min(1).optional(),
133
+ })
134
+ .strict()
135
+ .superRefine((e, ctx) => {
136
+ if (e.kind === "value" || e.kind === "function") {
137
+ if (!e.rule) {
138
+ ctx.addIssue({
139
+ code: z.ZodIssueCode.custom,
140
+ message: `evaluator '${e.name}' (kind: ${e.kind}) must declare a rule`,
141
+ path: ["rule"],
142
+ });
143
+ }
144
+ if (e.rubric !== undefined || e.pass_when !== undefined || e.model !== undefined) {
145
+ ctx.addIssue({
146
+ code: z.ZodIssueCode.custom,
147
+ message: `evaluator '${e.name}' (kind: ${e.kind}) must not declare rubric / pass_when / model (those are 'judge' only)`,
148
+ path: [],
149
+ });
150
+ }
151
+ }
152
+ else if (e.kind === "judge") {
153
+ if (!e.rubric) {
154
+ ctx.addIssue({
155
+ code: z.ZodIssueCode.custom,
156
+ message: `evaluator '${e.name}' (kind: judge) must declare a rubric`,
157
+ path: ["rubric"],
158
+ });
159
+ }
160
+ if (e.rule !== undefined) {
161
+ ctx.addIssue({
162
+ code: z.ZodIssueCode.custom,
163
+ message: `evaluator '${e.name}' (kind: judge) must not declare a rule`,
164
+ path: ["rule"],
165
+ });
166
+ }
167
+ }
113
168
  });
114
169
  export const nodeRequiresZ = z
115
170
  .object({
@@ -138,7 +193,9 @@ export const nodeZ = z
138
193
  max_turns: z.number().int().min(1).optional(),
139
194
  rules: nodeSourcesZ.optional(),
140
195
  context: nodeSourcesZ.optional(),
141
- verify: nodeVerifyZ.optional(),
196
+ eval: z.array(evaluatorZ).min(1).optional(),
197
+ eval_policy: evalPolicyZ.optional(),
198
+ judge_model: z.string().min(1).optional(),
142
199
  requires: nodeRequiresZ.optional(),
143
200
  retry: nodeRetryZ.optional(),
144
201
  })
@@ -167,9 +224,41 @@ export const workflowZ = z.object({
167
224
  skills: z.record(skillDefinitionZ).default({}),
168
225
  rules: z.array(sourceZ).optional(),
169
226
  context: z.array(sourceZ).optional(),
227
+ judge_model: z.string().min(1).optional(),
228
+ judge_budget: z.number().int().min(0).optional(),
170
229
  });
230
+ const LEGACY_VERIFY_MESSAGE = "uses the legacy 'verify:' field. It was renamed to 'eval:' in @sweny-ai/core v0.2.0 with a different shape (named evaluators with kind: value | function | judge). " +
231
+ "Migration guide: https://spec.sweny.ai/nodes/#eval";
232
+ /**
233
+ * Detect legacy `verify:` blocks before parse and throw a clear migration
234
+ * error. Without this preflight, the user gets Zod's generic "Unrecognized
235
+ * key(s)" message at the node level (and silent passthrough at the
236
+ * workflow level, since workflowZ allows extra top-level fields for
237
+ * marketplace metadata).
238
+ *
239
+ * Catches:
240
+ * - Top-level `workflow.verify` (typo / wrong scope).
241
+ * - Per-node `nodes[*].verify` (the most common shape pre-rename).
242
+ */
243
+ function preflightLegacyVerify(raw) {
244
+ if (!raw || typeof raw !== "object")
245
+ return;
246
+ const wf = raw;
247
+ if ("verify" in wf) {
248
+ throw new Error(`Workflow ${LEGACY_VERIFY_MESSAGE}`);
249
+ }
250
+ const nodes = wf.nodes;
251
+ if (!nodes || typeof nodes !== "object")
252
+ return;
253
+ for (const [id, node] of Object.entries(nodes)) {
254
+ if (node && typeof node === "object" && "verify" in node) {
255
+ throw new Error(`Node "${id}" ${LEGACY_VERIFY_MESSAGE}`);
256
+ }
257
+ }
258
+ }
171
259
  /** Parse + validate a raw object as a Workflow. Throws on invalid input. */
172
260
  export function parseWorkflow(raw) {
261
+ preflightLegacyVerify(raw);
173
262
  return workflowZ.parse(raw);
174
263
  }
175
264
  /**
@@ -353,7 +442,7 @@ export const workflowJsonSchema = {
353
442
  required: ["id", "name", "nodes", "edges", "entry"],
354
443
  // Top-level is intentionally open: marketplace workflows carry extra
355
444
  // metadata (author, category, tags) that publish.ts reads. Inner
356
- // objects (nodes, edges, skills, verify, requires, etc.) are strict.
445
+ // objects (nodes, edges, skills, eval, requires, etc.) are strict.
357
446
  additionalProperties: true,
358
447
  $defs: {
359
448
  Source: sourceJsonSchema,
@@ -371,6 +460,70 @@ export const workflowJsonSchema = {
371
460
  },
372
461
  ],
373
462
  },
463
+ OutputMatch: {
464
+ type: "object",
465
+ required: ["path"],
466
+ additionalProperties: false,
467
+ properties: {
468
+ path: { type: "string", minLength: 1 },
469
+ equals: {},
470
+ in: { type: "array" },
471
+ matches: { type: "string", minLength: 1 },
472
+ },
473
+ oneOf: [{ required: ["equals"] }, { required: ["in"] }, { required: ["matches"] }],
474
+ },
475
+ Evaluator: {
476
+ type: "object",
477
+ required: ["name", "kind"],
478
+ additionalProperties: false,
479
+ properties: {
480
+ name: {
481
+ type: "string",
482
+ minLength: 1,
483
+ description: "Stable identifier for this evaluator. Used in EvalResult and retry preambles.",
484
+ },
485
+ kind: { type: "string", enum: ["value", "function", "judge"] },
486
+ rule: {
487
+ type: "object",
488
+ description: "Required for value and function kinds. Shape depends on kind.",
489
+ additionalProperties: false,
490
+ minProperties: 1,
491
+ properties: {
492
+ any_tool_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
493
+ all_tools_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
494
+ no_tool_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
495
+ output_required: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
496
+ output_matches: { type: "array", items: { $ref: "#/$defs/OutputMatch" }, minItems: 1 },
497
+ },
498
+ },
499
+ rubric: {
500
+ type: "string",
501
+ minLength: 1,
502
+ description: "Required for judge kind. Natural-language criterion the judge model evaluates.",
503
+ },
504
+ pass_when: {
505
+ type: "string",
506
+ minLength: 1,
507
+ default: "yes",
508
+ description: "judge only. Verdict token that indicates pass. Default 'yes'.",
509
+ },
510
+ model: {
511
+ type: "string",
512
+ minLength: 1,
513
+ description: "judge only. Override the judge model for this evaluator.",
514
+ },
515
+ },
516
+ allOf: [
517
+ {
518
+ if: { properties: { kind: { enum: ["value", "function"] } } },
519
+ then: { required: ["rule"] },
520
+ },
521
+ {
522
+ if: { properties: { kind: { const: "judge" } } },
523
+ then: { required: ["rubric"] },
524
+ },
525
+ ],
526
+ },
374
527
  },
375
528
  properties: {
376
529
  id: { type: "string", minLength: 1 },
@@ -387,6 +540,18 @@ export const workflowJsonSchema = {
387
540
  items: { $ref: "#/$defs/Source" },
388
541
  description: "Background knowledge prepended to every node's instruction.",
389
542
  },
543
+ judge_model: {
544
+ type: "string",
545
+ minLength: 1,
546
+ default: "claude-haiku-4-5",
547
+ description: "Default model for judge evaluators across the workflow. Overridable per-node and per-evaluator.",
548
+ },
549
+ judge_budget: {
550
+ type: "integer",
551
+ minimum: 0,
552
+ default: 50,
553
+ description: "Soft cap on expected judge calls per workflow run. Executor warns at load time if exceeded; not a hard runtime cap in v1.",
554
+ },
390
555
  nodes: {
391
556
  type: "object",
392
557
  additionalProperties: {
@@ -418,42 +583,29 @@ export const workflowJsonSchema = {
418
583
  $ref: "#/$defs/NodeSources",
419
584
  description: "Per-node context. Additive by default; set { only: true } to block cascade.",
420
585
  },
421
- verify: {
422
- type: "object",
423
- description: "Machine-checked post-conditions evaluated after the LLM finishes.",
424
- additionalProperties: false,
425
- // Fix #4: at least one check must be declared.
426
- minProperties: 1,
427
- properties: {
428
- any_tool_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
429
- all_tools_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
430
- no_tool_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
431
- output_required: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
432
- output_matches: {
433
- type: "array",
434
- items: {
435
- type: "object",
436
- required: ["path"],
437
- additionalProperties: false,
438
- properties: {
439
- path: { type: "string", minLength: 1 },
440
- equals: {},
441
- in: { type: "array" },
442
- matches: { type: "string", minLength: 1 },
443
- },
444
- // Fix #4: exactly one operator must be declared.
445
- oneOf: [{ required: ["equals"] }, { required: ["in"] }, { required: ["matches"] }],
446
- },
447
- minItems: 1,
448
- },
449
- },
586
+ eval: {
587
+ type: "array",
588
+ description: "Named evaluators (value, function, judge) run after the LLM finishes the node.",
589
+ items: { $ref: "#/$defs/Evaluator" },
590
+ minItems: 1,
591
+ },
592
+ eval_policy: {
593
+ type: "string",
594
+ enum: ["all_pass", "any_pass", "weighted"],
595
+ default: "all_pass",
596
+ description: "How evaluator results aggregate. v1 implements all_pass; the others are reserved.",
597
+ },
598
+ judge_model: {
599
+ type: "string",
600
+ minLength: 1,
601
+ description: "Default model for judge evaluators on this node. Overrides workflow-level judge_model.",
450
602
  },
451
603
  requires: {
452
604
  type: "object",
453
605
  description: "Pre-condition checks evaluated before the LLM runs.",
454
606
  additionalProperties: false,
455
607
  // Fix #4: at least one of output_required / output_matches must be declared.
456
- // on_fail alone is not sufficient — it only tags how a failing check behaves.
608
+ // on_fail alone is not sufficient. It only tags how a failing check behaves.
457
609
  anyOf: [{ required: ["output_required"] }, { required: ["output_matches"] }],
458
610
  properties: {
459
611
  output_required: {
@@ -463,19 +615,7 @@ export const workflowJsonSchema = {
463
615
  },
464
616
  output_matches: {
465
617
  type: "array",
466
- items: {
467
- type: "object",
468
- required: ["path"],
469
- additionalProperties: false,
470
- properties: {
471
- path: { type: "string", minLength: 1 },
472
- equals: {},
473
- in: { type: "array" },
474
- matches: { type: "string", minLength: 1 },
475
- },
476
- // Fix #4: exactly one operator must be declared.
477
- oneOf: [{ required: ["equals"] }, { required: ["in"] }, { required: ["matches"] }],
478
- },
618
+ items: { $ref: "#/$defs/OutputMatch" },
479
619
  minItems: 1,
480
620
  },
481
621
  on_fail: { type: "string", enum: ["fail", "skip"] },
@@ -483,7 +623,7 @@ export const workflowJsonSchema = {
483
623
  },
484
624
  retry: {
485
625
  type: "object",
486
- description: "Node-local retry on verify failure.",
626
+ description: "Node-local retry on eval failure.",
487
627
  required: ["max"],
488
628
  additionalProperties: false,
489
629
  properties: {
@@ -519,11 +659,11 @@ export const workflowJsonSchema = {
519
659
  properties: {
520
660
  from: { type: "string", minLength: 1 },
521
661
  to: { type: "string", minLength: 1 },
522
- when: { type: "string", description: "Natural language condition — Claude evaluates at runtime" },
662
+ when: { type: "string", description: "Natural language condition. Claude evaluates at runtime." },
523
663
  max_iterations: {
524
664
  type: "integer",
525
665
  minimum: 1,
526
- description: "Max times this edge can be followed — enables controlled retry loops",
666
+ description: "Max times this edge can be followed. Enables controlled retry loops.",
527
667
  },
528
668
  },
529
669
  },
@@ -192,7 +192,7 @@ export function buildToolAliases(skills, logger) {
192
192
  for (const [name, claimers] of mcpClaims) {
193
193
  if (claimers.length > 1) {
194
194
  ambiguous.add(name);
195
- logger?.warn(`Tool alias "${name}" is declared by multiple skills (${claimers.join(", ")}); dropping from verify alias table to avoid cross-provider false positives.`);
195
+ logger?.warn(`Tool alias "${name}" is declared by multiple skills (${claimers.join(", ")}); dropping from eval alias table to avoid cross-provider false positives.`);
196
196
  }
197
197
  }
198
198
  // Second pass: union-find over the remaining pairs.
package/dist/types.d.ts CHANGED
@@ -37,20 +37,20 @@ export interface Skill {
37
37
  /** External MCP server definition wired for nodes referencing this skill. */
38
38
  mcp?: McpServerConfig;
39
39
  /**
40
- * Tool-name aliases recognized by `verify`.
40
+ * Tool-name aliases recognized by `function` evaluators.
41
41
  *
42
42
  * When a workflow references `any_tool_called: [linear_create_issue]` and
43
43
  * the agent instead calls the equivalent MCP tool exposed by this skill's
44
44
  * MCP server (e.g. `save_issue` on Linear's remote MCP), the two names
45
45
  * should count as equivalent. Each skill owns the mapping for its own
46
- * domain — core stays vendor-neutral.
46
+ * domain. Core stays vendor-neutral.
47
47
  *
48
48
  * Key: a canonical tool name (usually one of this skill's `tools[].name`,
49
49
  * but any name the skill wants to equate is valid).
50
50
  * Value: list of equivalent names, typically tool names exposed by this
51
51
  * skill's external MCP server.
52
52
  *
53
- * Aliases are symmetric: a verify rule naming either side matches a call
53
+ * Aliases are symmetric: a function rule naming either side matches a call
54
54
  * on either side. Omit names that are ambiguous across providers
55
55
  * (e.g. `get_issue` is exposed by both Linear and GitHub MCP servers).
56
56
  */
@@ -68,30 +68,74 @@ export type NodeSources = _Source[] | {
68
68
  only?: boolean;
69
69
  sources: _Source[];
70
70
  };
71
+ /** Kind of evaluator. See {@link Evaluator}. */
72
+ export type EvaluatorKind = "value" | "function" | "judge";
73
+ /** Aggregation policy for a node's evaluator results. v1 implements `all_pass`. */
74
+ export type EvalPolicy = "all_pass" | "any_pass" | "weighted";
71
75
  /**
72
- * Machine-checked post-condition for a node.
76
+ * The deterministic rule body for a `value` or `function` evaluator.
73
77
  *
74
- * Evaluated by the executor AFTER the LLM finishes. If any declared check
75
- * fails, the node is marked `failed` even if the LLM itself returned success.
76
- *
77
- * All declared checks are AND-ed. The executor runs every check (no fast-fail)
78
- * and concatenates failures into a single error string on `result.data.error`.
79
- *
80
- * Keep the shape small and declarative. Anything richer should be a skill or
81
- * a dedicated workflow node, not a verify clause.
78
+ * `value` rules use `output_required` and `output_matches` (data-shape).
79
+ * `function` rules use `any_tool_called` / `all_tools_called` / `no_tool_called`
80
+ * (trace-shape). Mixing the two on a single rule is allowed for compactness
81
+ * but discouraged: a single evaluator should test a single thing.
82
82
  */
83
- export interface NodeVerify {
84
- /** At least one of these tools was called and succeeded during this node. */
83
+ export interface EvaluatorRule {
84
+ /** At least one of these tools was called and succeeded. (`function` rules.) */
85
85
  any_tool_called?: string[];
86
- /** Every named tool was called and succeeded at least once during this node. */
86
+ /** Every named tool was called and succeeded at least once. (`function` rules.) */
87
87
  all_tools_called?: string[];
88
- /** None of these tools may have been invoked during this node. */
88
+ /** None of these tools may have been invoked. (`function` rules.) */
89
89
  no_tool_called?: string[];
90
- /** Listed paths must be present and non-null in `result.data` (see Path resolution in the spec). */
90
+ /** Listed paths must be present and non-null in `result.data`. (`value` rules.) */
91
91
  output_required?: string[];
92
- /** Each assertion must hold against `result.data`. */
92
+ /** Each assertion must hold against `result.data`. (`value` rules.) */
93
93
  output_matches?: OutputMatch[];
94
94
  }
95
+ /**
96
+ * A single evaluator on a node.
97
+ *
98
+ * - `value` and `function` use `rule`.
99
+ * - `judge` uses `rubric` (and optional `pass_when`, `model`).
100
+ *
101
+ * Each evaluator produces an {@link EvalResult}. The executor aggregates
102
+ * the results per the node's `eval_policy` (default `all_pass`).
103
+ */
104
+ export interface Evaluator {
105
+ /** Stable identifier. Used in EvalResult and retry preambles. */
106
+ name: string;
107
+ /** Discriminator. Determines which fields apply. */
108
+ kind: EvaluatorKind;
109
+ /** Required for `value` / `function`. */
110
+ rule?: EvaluatorRule;
111
+ /** Required for `judge`. Natural-language criterion. */
112
+ rubric?: string;
113
+ /** `judge` only. Verdict token that indicates pass. Default: `"yes"`. */
114
+ pass_when?: string;
115
+ /** `judge` only. Override the judge model for this evaluator. */
116
+ model?: string;
117
+ }
118
+ /**
119
+ * Per-evaluator result.
120
+ *
121
+ * The executor produces one EvalResult per declared evaluator. The list
122
+ * lands on {@link NodeResult.evals}.
123
+ */
124
+ export interface EvalResult {
125
+ /** Echoes the evaluator's name. */
126
+ name: string;
127
+ /** Echoes the evaluator's kind. */
128
+ kind: EvaluatorKind;
129
+ /** Whether this evaluator passed. */
130
+ pass: boolean;
131
+ /**
132
+ * Failure detail for value/function (formatted by the executor) or judge
133
+ * (returned by the model). Capped at ~500 characters when emitted.
134
+ */
135
+ reasoning?: string;
136
+ /** Reserved for `weighted` policies. Not populated in v1. */
137
+ score?: number;
138
+ }
95
139
  /**
96
140
  * Machine-checked pre-condition for a node.
97
141
  *
@@ -102,7 +146,7 @@ export interface NodeVerify {
102
146
  * Path roots resolve against the cross-node context map:
103
147
  * { input: <runtime input>, [priorNodeId]: <data of prior node>, ... }
104
148
  *
105
- * Reuses the same path grammar as `verify` (dotted segments, `[*]` wildcard,
149
+ * Reuses the same path grammar as `eval` (dotted segments, `[*]` wildcard,
106
150
  * optional `all:`/`any:` prefix).
107
151
  */
108
152
  export interface NodeRequires {
@@ -114,15 +158,15 @@ export interface NodeRequires {
114
158
  on_fail?: "fail" | "skip";
115
159
  }
116
160
  /**
117
- * Node-local retry on verify failure.
161
+ * Node-local retry on eval failure.
118
162
  *
119
163
  * Re-runs the LLM up to `max` additional times, prepending feedback derived
120
- * from the verify failure. Triggered ONLY by verify failure — not by tool/API
121
- * errors and not by `requires` failure.
164
+ * from the failing evaluators. Triggered ONLY by eval failure, not by tool
165
+ * / API errors and not by `requires` failure.
122
166
  *
123
167
  * `instruction` shapes the feedback preamble:
124
- * - omitted → default "## Previous attempt failed verification..."
125
- * - string → static text + verify error
168
+ * - omitted → default "## Previous attempt failed evaluation..."
169
+ * - string → static text + structured eval failure list
126
170
  * - { auto: true } → LLM-generated diagnosis from default reflection prompt
127
171
  * - { reflect: s } → LLM-generated diagnosis from author-provided prompt
128
172
  */
@@ -167,23 +211,27 @@ export interface Node {
167
211
  rules?: NodeSources;
168
212
  /** Per-node background knowledge. Additive by default; set `{ only: true, sources: [...] }` to block cascade. */
169
213
  context?: NodeSources;
170
- /** Machine-checked post-conditions. Enforced by the executor after the LLM finishes. */
171
- verify?: NodeVerify;
214
+ /** Named evaluators run after the LLM finishes. Each produces an {@link EvalResult}. */
215
+ eval?: Evaluator[];
216
+ /** How evaluator results aggregate. Default `all_pass`. v1 implements only `all_pass`. */
217
+ eval_policy?: EvalPolicy;
218
+ /** Default model for judge evaluators on this node. Overrides workflow-level `judge_model`. */
219
+ judge_model?: string;
172
220
  /** Machine-checked pre-conditions. Enforced by the executor before the LLM runs. */
173
221
  requires?: NodeRequires;
174
- /** Node-local retry on verify failure (with optional autonomous reflection). */
222
+ /** Node-local retry on eval failure (with optional autonomous reflection). */
175
223
  retry?: NodeRetry;
176
224
  }
177
225
  /** An edge connecting two nodes */
178
226
  export interface Edge {
179
227
  from: string;
180
228
  to: string;
181
- /** Natural language condition — Claude evaluates at runtime */
229
+ /** Natural language condition. Claude evaluates at runtime. */
182
230
  when?: string;
183
231
  /** Max times this edge can be followed (enables retry loops). Default: unlimited. */
184
232
  max_iterations?: number;
185
233
  }
186
- /** A complete workflow definition — pure data, fully serializable */
234
+ /** A complete workflow definition. Pure data, fully serializable. */
187
235
  export interface Workflow {
188
236
  id: string;
189
237
  name: string;
@@ -196,6 +244,10 @@ export interface Workflow {
196
244
  rules?: _Source[];
197
245
  /** Background knowledge prepended to every node's instruction. Cascade into per-node context. */
198
246
  context?: _Source[];
247
+ /** Default model for judge evaluators across the workflow. Overridable per-node and per-evaluator. */
248
+ judge_model?: string;
249
+ /** Soft cap on expected judge calls per workflow run. Warning at load time when exceeded. */
250
+ judge_budget?: number;
199
251
  }
200
252
  /**
201
253
  * Inline skill definition in a workflow's `skills` block.
@@ -215,6 +267,12 @@ export interface NodeResult {
215
267
  data: Record<string, unknown>;
216
268
  /** Tool calls made during this node's execution */
217
269
  toolCalls: ToolCall[];
270
+ /**
271
+ * Per-evaluator results from `node.eval` (one entry per declared evaluator,
272
+ * in declaration order). Absent when the node has no `eval` block or when
273
+ * eval was not run (e.g. node failed during execution).
274
+ */
275
+ evals?: EvalResult[];
218
276
  }
219
277
  export interface ToolCall {
220
278
  tool: string;
@@ -223,13 +281,13 @@ export interface ToolCall {
223
281
  /**
224
282
  * Authoritative outcome of the tool invocation.
225
283
  *
226
- * When present, this is the source of truth for `verify` checks — the
284
+ * When present, this is the source of truth for `function` evaluators. The
227
285
  * output-shape heuristic is a legacy fallback only. Set by the Claude
228
286
  * runtime on tool completion: "success" when the tool returned normally,
229
287
  * "error" when it threw or the MCP server returned is_error=true.
230
288
  *
231
- * Absent status indicates a legacy or hand-constructed ToolCall — verify
232
- * falls back to inspecting `output` for an `error` key.
289
+ * Absent status indicates a legacy or hand-constructed ToolCall. Function
290
+ * evaluators fall back to inspecting `output` for an `error` key.
233
291
  */
234
292
  status?: "success" | "error";
235
293
  }
@@ -334,11 +392,16 @@ export interface Claude {
334
392
  }): Promise<string>;
335
393
  /**
336
394
  * Single-completion free-text query. No tools, no output schema.
337
- * Used by the executor to generate retry strategies in autonomous reflection mode.
395
+ * Used by the executor to generate retry strategies in autonomous reflection mode
396
+ * and by judge evaluators to score node results against a rubric.
397
+ *
398
+ * `model` overrides the client's default for this call. Implementations
399
+ * SHOULD use it; mocks MAY ignore it.
338
400
  */
339
401
  ask(opts: {
340
402
  instruction: string;
341
403
  context: Record<string, unknown>;
404
+ model?: string;
342
405
  }): Promise<string>;
343
406
  }
344
407
  export interface McpServerConfig {