@sweny-ai/core 0.1.84 → 0.1.86
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claude.d.ts +1 -0
- package/dist/claude.js +5 -4
- package/dist/eval/function.d.ts +25 -0
- package/dist/eval/function.js +106 -0
- package/dist/eval/index.d.ts +41 -0
- package/dist/eval/index.js +91 -0
- package/dist/eval/judge.d.ts +43 -0
- package/dist/eval/judge.js +142 -0
- package/dist/eval/path.d.ts +9 -0
- package/dist/eval/path.js +72 -0
- package/dist/eval/value.d.ts +15 -0
- package/dist/eval/value.js +205 -0
- package/dist/executor.js +85 -18
- package/dist/requires.js +2 -2
- package/dist/retry.d.ts +17 -3
- package/dist/retry.js +29 -14
- package/dist/schema.d.ts +778 -316
- package/dist/schema.js +200 -60
- package/dist/skills/index.js +1 -1
- package/dist/types.d.ts +96 -33
- package/dist/workflows/browser.js +1 -1
- package/dist/workflows/triage.yml +68 -28
- package/package.json +1 -1
- package/dist/verify.d.ts +0 -25
- package/dist/verify.js +0 -357
package/dist/schema.js
CHANGED
|
@@ -90,7 +90,9 @@ export const outputMatchZ = z
|
|
|
90
90
|
const operators = [m.equals !== undefined, m.in !== undefined, m.matches !== undefined];
|
|
91
91
|
return operators.filter(Boolean).length === 1;
|
|
92
92
|
}, { message: "output_matches entry must declare exactly one of: equals, in, matches" });
|
|
93
|
-
export const
|
|
93
|
+
export const evaluatorKindZ = z.enum(["value", "function", "judge"]);
|
|
94
|
+
export const evalPolicyZ = z.enum(["all_pass", "any_pass", "weighted"]);
|
|
95
|
+
export const evaluatorRuleZ = z
|
|
94
96
|
.object({
|
|
95
97
|
any_tool_called: z.array(z.string().min(1)).min(1).optional(),
|
|
96
98
|
all_tools_called: z.array(z.string().min(1)).min(1).optional(),
|
|
@@ -98,18 +100,71 @@ export const nodeVerifyZ = z
|
|
|
98
100
|
output_required: z.array(z.string().min(1)).min(1).optional(),
|
|
99
101
|
output_matches: z.array(outputMatchZ).min(1).optional(),
|
|
100
102
|
})
|
|
101
|
-
// Fix #4 gap: the exported JSON Schema sets additionalProperties: false.
|
|
102
|
-
// Without .strict() here Zod would silently drop unknown keys and then
|
|
103
|
-
// the refine would run on the stripped object, producing a different
|
|
104
|
-
// verdict than ajv (which rejects the unknown key directly). Keep them
|
|
105
|
-
// in sync.
|
|
106
103
|
.strict()
|
|
107
|
-
.refine((
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
message: "
|
|
104
|
+
.refine((r) => r.any_tool_called !== undefined ||
|
|
105
|
+
r.all_tools_called !== undefined ||
|
|
106
|
+
r.no_tool_called !== undefined ||
|
|
107
|
+
r.output_required !== undefined ||
|
|
108
|
+
r.output_matches !== undefined, {
|
|
109
|
+
message: "evaluator rule must declare at least one of: any_tool_called, all_tools_called, no_tool_called, output_required, output_matches",
|
|
110
|
+
});
|
|
111
|
+
/**
|
|
112
|
+
* A single evaluator. The `kind` field discriminates required-fields:
|
|
113
|
+
* `value` and `function` need `rule`; `judge` needs `rubric`.
|
|
114
|
+
*
|
|
115
|
+
* Strict object: unknown keys are rejected to match the public JSON Schema.
|
|
116
|
+
*/
|
|
117
|
+
export const evaluatorZ = z
|
|
118
|
+
.object({
|
|
119
|
+
name: z.string().min(1),
|
|
120
|
+
kind: evaluatorKindZ,
|
|
121
|
+
rule: evaluatorRuleZ.optional(),
|
|
122
|
+
rubric: z.string().min(1).optional(),
|
|
123
|
+
// `pass_when` is parsed against a single VERDICT token from the model's
|
|
124
|
+
// response (default `yes`). Whitespace breaks the prompt format and the
|
|
125
|
+
// verdict comparison, so reject it at parse time rather than producing
|
|
126
|
+
// a confusing runtime mismatch.
|
|
127
|
+
pass_when: z
|
|
128
|
+
.string()
|
|
129
|
+
.min(1)
|
|
130
|
+
.refine((v) => !/\s/.test(v), { message: "pass_when must be a single whitespace-free token" })
|
|
131
|
+
.optional(),
|
|
132
|
+
model: z.string().min(1).optional(),
|
|
133
|
+
})
|
|
134
|
+
.strict()
|
|
135
|
+
.superRefine((e, ctx) => {
|
|
136
|
+
if (e.kind === "value" || e.kind === "function") {
|
|
137
|
+
if (!e.rule) {
|
|
138
|
+
ctx.addIssue({
|
|
139
|
+
code: z.ZodIssueCode.custom,
|
|
140
|
+
message: `evaluator '${e.name}' (kind: ${e.kind}) must declare a rule`,
|
|
141
|
+
path: ["rule"],
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
if (e.rubric !== undefined || e.pass_when !== undefined || e.model !== undefined) {
|
|
145
|
+
ctx.addIssue({
|
|
146
|
+
code: z.ZodIssueCode.custom,
|
|
147
|
+
message: `evaluator '${e.name}' (kind: ${e.kind}) must not declare rubric / pass_when / model (those are 'judge' only)`,
|
|
148
|
+
path: [],
|
|
149
|
+
});
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
else if (e.kind === "judge") {
|
|
153
|
+
if (!e.rubric) {
|
|
154
|
+
ctx.addIssue({
|
|
155
|
+
code: z.ZodIssueCode.custom,
|
|
156
|
+
message: `evaluator '${e.name}' (kind: judge) must declare a rubric`,
|
|
157
|
+
path: ["rubric"],
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
if (e.rule !== undefined) {
|
|
161
|
+
ctx.addIssue({
|
|
162
|
+
code: z.ZodIssueCode.custom,
|
|
163
|
+
message: `evaluator '${e.name}' (kind: judge) must not declare a rule`,
|
|
164
|
+
path: ["rule"],
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
}
|
|
113
168
|
});
|
|
114
169
|
export const nodeRequiresZ = z
|
|
115
170
|
.object({
|
|
@@ -138,7 +193,9 @@ export const nodeZ = z
|
|
|
138
193
|
max_turns: z.number().int().min(1).optional(),
|
|
139
194
|
rules: nodeSourcesZ.optional(),
|
|
140
195
|
context: nodeSourcesZ.optional(),
|
|
141
|
-
|
|
196
|
+
eval: z.array(evaluatorZ).min(1).optional(),
|
|
197
|
+
eval_policy: evalPolicyZ.optional(),
|
|
198
|
+
judge_model: z.string().min(1).optional(),
|
|
142
199
|
requires: nodeRequiresZ.optional(),
|
|
143
200
|
retry: nodeRetryZ.optional(),
|
|
144
201
|
})
|
|
@@ -167,9 +224,41 @@ export const workflowZ = z.object({
|
|
|
167
224
|
skills: z.record(skillDefinitionZ).default({}),
|
|
168
225
|
rules: z.array(sourceZ).optional(),
|
|
169
226
|
context: z.array(sourceZ).optional(),
|
|
227
|
+
judge_model: z.string().min(1).optional(),
|
|
228
|
+
judge_budget: z.number().int().min(0).optional(),
|
|
170
229
|
});
|
|
230
|
+
const LEGACY_VERIFY_MESSAGE = "uses the legacy 'verify:' field. It was renamed to 'eval:' in @sweny-ai/core v0.2.0 with a different shape (named evaluators with kind: value | function | judge). " +
|
|
231
|
+
"Migration guide: https://spec.sweny.ai/nodes/#eval";
|
|
232
|
+
/**
|
|
233
|
+
* Detect legacy `verify:` blocks before parse and throw a clear migration
|
|
234
|
+
* error. Without this preflight, the user gets Zod's generic "Unrecognized
|
|
235
|
+
* key(s)" message at the node level (and silent passthrough at the
|
|
236
|
+
* workflow level, since workflowZ allows extra top-level fields for
|
|
237
|
+
* marketplace metadata).
|
|
238
|
+
*
|
|
239
|
+
* Catches:
|
|
240
|
+
* - Top-level `workflow.verify` (typo / wrong scope).
|
|
241
|
+
* - Per-node `nodes[*].verify` (the most common shape pre-rename).
|
|
242
|
+
*/
|
|
243
|
+
function preflightLegacyVerify(raw) {
|
|
244
|
+
if (!raw || typeof raw !== "object")
|
|
245
|
+
return;
|
|
246
|
+
const wf = raw;
|
|
247
|
+
if ("verify" in wf) {
|
|
248
|
+
throw new Error(`Workflow ${LEGACY_VERIFY_MESSAGE}`);
|
|
249
|
+
}
|
|
250
|
+
const nodes = wf.nodes;
|
|
251
|
+
if (!nodes || typeof nodes !== "object")
|
|
252
|
+
return;
|
|
253
|
+
for (const [id, node] of Object.entries(nodes)) {
|
|
254
|
+
if (node && typeof node === "object" && "verify" in node) {
|
|
255
|
+
throw new Error(`Node "${id}" ${LEGACY_VERIFY_MESSAGE}`);
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
}
|
|
171
259
|
/** Parse + validate a raw object as a Workflow. Throws on invalid input. */
|
|
172
260
|
export function parseWorkflow(raw) {
|
|
261
|
+
preflightLegacyVerify(raw);
|
|
173
262
|
return workflowZ.parse(raw);
|
|
174
263
|
}
|
|
175
264
|
/**
|
|
@@ -353,7 +442,7 @@ export const workflowJsonSchema = {
|
|
|
353
442
|
required: ["id", "name", "nodes", "edges", "entry"],
|
|
354
443
|
// Top-level is intentionally open: marketplace workflows carry extra
|
|
355
444
|
// metadata (author, category, tags) that publish.ts reads. Inner
|
|
356
|
-
// objects (nodes, edges, skills,
|
|
445
|
+
// objects (nodes, edges, skills, eval, requires, etc.) are strict.
|
|
357
446
|
additionalProperties: true,
|
|
358
447
|
$defs: {
|
|
359
448
|
Source: sourceJsonSchema,
|
|
@@ -371,6 +460,70 @@ export const workflowJsonSchema = {
|
|
|
371
460
|
},
|
|
372
461
|
],
|
|
373
462
|
},
|
|
463
|
+
OutputMatch: {
|
|
464
|
+
type: "object",
|
|
465
|
+
required: ["path"],
|
|
466
|
+
additionalProperties: false,
|
|
467
|
+
properties: {
|
|
468
|
+
path: { type: "string", minLength: 1 },
|
|
469
|
+
equals: {},
|
|
470
|
+
in: { type: "array" },
|
|
471
|
+
matches: { type: "string", minLength: 1 },
|
|
472
|
+
},
|
|
473
|
+
oneOf: [{ required: ["equals"] }, { required: ["in"] }, { required: ["matches"] }],
|
|
474
|
+
},
|
|
475
|
+
Evaluator: {
|
|
476
|
+
type: "object",
|
|
477
|
+
required: ["name", "kind"],
|
|
478
|
+
additionalProperties: false,
|
|
479
|
+
properties: {
|
|
480
|
+
name: {
|
|
481
|
+
type: "string",
|
|
482
|
+
minLength: 1,
|
|
483
|
+
description: "Stable identifier for this evaluator. Used in EvalResult and retry preambles.",
|
|
484
|
+
},
|
|
485
|
+
kind: { type: "string", enum: ["value", "function", "judge"] },
|
|
486
|
+
rule: {
|
|
487
|
+
type: "object",
|
|
488
|
+
description: "Required for value and function kinds. Shape depends on kind.",
|
|
489
|
+
additionalProperties: false,
|
|
490
|
+
minProperties: 1,
|
|
491
|
+
properties: {
|
|
492
|
+
any_tool_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
|
|
493
|
+
all_tools_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
|
|
494
|
+
no_tool_called: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
|
|
495
|
+
output_required: { type: "array", items: { type: "string", minLength: 1 }, minItems: 1 },
|
|
496
|
+
output_matches: { type: "array", items: { $ref: "#/$defs/OutputMatch" }, minItems: 1 },
|
|
497
|
+
},
|
|
498
|
+
},
|
|
499
|
+
rubric: {
|
|
500
|
+
type: "string",
|
|
501
|
+
minLength: 1,
|
|
502
|
+
description: "Required for judge kind. Natural-language criterion the judge model evaluates.",
|
|
503
|
+
},
|
|
504
|
+
pass_when: {
|
|
505
|
+
type: "string",
|
|
506
|
+
minLength: 1,
|
|
507
|
+
default: "yes",
|
|
508
|
+
description: "judge only. Verdict token that indicates pass. Default 'yes'.",
|
|
509
|
+
},
|
|
510
|
+
model: {
|
|
511
|
+
type: "string",
|
|
512
|
+
minLength: 1,
|
|
513
|
+
description: "judge only. Override the judge model for this evaluator.",
|
|
514
|
+
},
|
|
515
|
+
},
|
|
516
|
+
allOf: [
|
|
517
|
+
{
|
|
518
|
+
if: { properties: { kind: { enum: ["value", "function"] } } },
|
|
519
|
+
then: { required: ["rule"] },
|
|
520
|
+
},
|
|
521
|
+
{
|
|
522
|
+
if: { properties: { kind: { const: "judge" } } },
|
|
523
|
+
then: { required: ["rubric"] },
|
|
524
|
+
},
|
|
525
|
+
],
|
|
526
|
+
},
|
|
374
527
|
},
|
|
375
528
|
properties: {
|
|
376
529
|
id: { type: "string", minLength: 1 },
|
|
@@ -387,6 +540,18 @@ export const workflowJsonSchema = {
|
|
|
387
540
|
items: { $ref: "#/$defs/Source" },
|
|
388
541
|
description: "Background knowledge prepended to every node's instruction.",
|
|
389
542
|
},
|
|
543
|
+
judge_model: {
|
|
544
|
+
type: "string",
|
|
545
|
+
minLength: 1,
|
|
546
|
+
default: "claude-haiku-4-5",
|
|
547
|
+
description: "Default model for judge evaluators across the workflow. Overridable per-node and per-evaluator.",
|
|
548
|
+
},
|
|
549
|
+
judge_budget: {
|
|
550
|
+
type: "integer",
|
|
551
|
+
minimum: 0,
|
|
552
|
+
default: 50,
|
|
553
|
+
description: "Soft cap on expected judge calls per workflow run. Executor warns at load time if exceeded; not a hard runtime cap in v1.",
|
|
554
|
+
},
|
|
390
555
|
nodes: {
|
|
391
556
|
type: "object",
|
|
392
557
|
additionalProperties: {
|
|
@@ -418,42 +583,29 @@ export const workflowJsonSchema = {
|
|
|
418
583
|
$ref: "#/$defs/NodeSources",
|
|
419
584
|
description: "Per-node context. Additive by default; set { only: true } to block cascade.",
|
|
420
585
|
},
|
|
421
|
-
|
|
422
|
-
type: "
|
|
423
|
-
description: "
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
additionalProperties: false,
|
|
438
|
-
properties: {
|
|
439
|
-
path: { type: "string", minLength: 1 },
|
|
440
|
-
equals: {},
|
|
441
|
-
in: { type: "array" },
|
|
442
|
-
matches: { type: "string", minLength: 1 },
|
|
443
|
-
},
|
|
444
|
-
// Fix #4: exactly one operator must be declared.
|
|
445
|
-
oneOf: [{ required: ["equals"] }, { required: ["in"] }, { required: ["matches"] }],
|
|
446
|
-
},
|
|
447
|
-
minItems: 1,
|
|
448
|
-
},
|
|
449
|
-
},
|
|
586
|
+
eval: {
|
|
587
|
+
type: "array",
|
|
588
|
+
description: "Named evaluators (value, function, judge) run after the LLM finishes the node.",
|
|
589
|
+
items: { $ref: "#/$defs/Evaluator" },
|
|
590
|
+
minItems: 1,
|
|
591
|
+
},
|
|
592
|
+
eval_policy: {
|
|
593
|
+
type: "string",
|
|
594
|
+
enum: ["all_pass", "any_pass", "weighted"],
|
|
595
|
+
default: "all_pass",
|
|
596
|
+
description: "How evaluator results aggregate. v1 implements all_pass; the others are reserved.",
|
|
597
|
+
},
|
|
598
|
+
judge_model: {
|
|
599
|
+
type: "string",
|
|
600
|
+
minLength: 1,
|
|
601
|
+
description: "Default model for judge evaluators on this node. Overrides workflow-level judge_model.",
|
|
450
602
|
},
|
|
451
603
|
requires: {
|
|
452
604
|
type: "object",
|
|
453
605
|
description: "Pre-condition checks evaluated before the LLM runs.",
|
|
454
606
|
additionalProperties: false,
|
|
455
607
|
// Fix #4: at least one of output_required / output_matches must be declared.
|
|
456
|
-
// on_fail alone is not sufficient
|
|
608
|
+
// on_fail alone is not sufficient. It only tags how a failing check behaves.
|
|
457
609
|
anyOf: [{ required: ["output_required"] }, { required: ["output_matches"] }],
|
|
458
610
|
properties: {
|
|
459
611
|
output_required: {
|
|
@@ -463,19 +615,7 @@ export const workflowJsonSchema = {
|
|
|
463
615
|
},
|
|
464
616
|
output_matches: {
|
|
465
617
|
type: "array",
|
|
466
|
-
items: {
|
|
467
|
-
type: "object",
|
|
468
|
-
required: ["path"],
|
|
469
|
-
additionalProperties: false,
|
|
470
|
-
properties: {
|
|
471
|
-
path: { type: "string", minLength: 1 },
|
|
472
|
-
equals: {},
|
|
473
|
-
in: { type: "array" },
|
|
474
|
-
matches: { type: "string", minLength: 1 },
|
|
475
|
-
},
|
|
476
|
-
// Fix #4: exactly one operator must be declared.
|
|
477
|
-
oneOf: [{ required: ["equals"] }, { required: ["in"] }, { required: ["matches"] }],
|
|
478
|
-
},
|
|
618
|
+
items: { $ref: "#/$defs/OutputMatch" },
|
|
479
619
|
minItems: 1,
|
|
480
620
|
},
|
|
481
621
|
on_fail: { type: "string", enum: ["fail", "skip"] },
|
|
@@ -483,7 +623,7 @@ export const workflowJsonSchema = {
|
|
|
483
623
|
},
|
|
484
624
|
retry: {
|
|
485
625
|
type: "object",
|
|
486
|
-
description: "Node-local retry on
|
|
626
|
+
description: "Node-local retry on eval failure.",
|
|
487
627
|
required: ["max"],
|
|
488
628
|
additionalProperties: false,
|
|
489
629
|
properties: {
|
|
@@ -519,11 +659,11 @@ export const workflowJsonSchema = {
|
|
|
519
659
|
properties: {
|
|
520
660
|
from: { type: "string", minLength: 1 },
|
|
521
661
|
to: { type: "string", minLength: 1 },
|
|
522
|
-
when: { type: "string", description: "Natural language condition
|
|
662
|
+
when: { type: "string", description: "Natural language condition. Claude evaluates at runtime." },
|
|
523
663
|
max_iterations: {
|
|
524
664
|
type: "integer",
|
|
525
665
|
minimum: 1,
|
|
526
|
-
description: "Max times this edge can be followed
|
|
666
|
+
description: "Max times this edge can be followed. Enables controlled retry loops.",
|
|
527
667
|
},
|
|
528
668
|
},
|
|
529
669
|
},
|
package/dist/skills/index.js
CHANGED
|
@@ -192,7 +192,7 @@ export function buildToolAliases(skills, logger) {
|
|
|
192
192
|
for (const [name, claimers] of mcpClaims) {
|
|
193
193
|
if (claimers.length > 1) {
|
|
194
194
|
ambiguous.add(name);
|
|
195
|
-
logger?.warn(`Tool alias "${name}" is declared by multiple skills (${claimers.join(", ")}); dropping from
|
|
195
|
+
logger?.warn(`Tool alias "${name}" is declared by multiple skills (${claimers.join(", ")}); dropping from eval alias table to avoid cross-provider false positives.`);
|
|
196
196
|
}
|
|
197
197
|
}
|
|
198
198
|
// Second pass: union-find over the remaining pairs.
|
package/dist/types.d.ts
CHANGED
|
@@ -37,20 +37,20 @@ export interface Skill {
|
|
|
37
37
|
/** External MCP server definition wired for nodes referencing this skill. */
|
|
38
38
|
mcp?: McpServerConfig;
|
|
39
39
|
/**
|
|
40
|
-
* Tool-name aliases recognized by `
|
|
40
|
+
* Tool-name aliases recognized by `function` evaluators.
|
|
41
41
|
*
|
|
42
42
|
* When a workflow references `any_tool_called: [linear_create_issue]` and
|
|
43
43
|
* the agent instead calls the equivalent MCP tool exposed by this skill's
|
|
44
44
|
* MCP server (e.g. `save_issue` on Linear's remote MCP), the two names
|
|
45
45
|
* should count as equivalent. Each skill owns the mapping for its own
|
|
46
|
-
* domain
|
|
46
|
+
* domain. Core stays vendor-neutral.
|
|
47
47
|
*
|
|
48
48
|
* Key: a canonical tool name (usually one of this skill's `tools[].name`,
|
|
49
49
|
* but any name the skill wants to equate is valid).
|
|
50
50
|
* Value: list of equivalent names, typically tool names exposed by this
|
|
51
51
|
* skill's external MCP server.
|
|
52
52
|
*
|
|
53
|
-
* Aliases are symmetric: a
|
|
53
|
+
* Aliases are symmetric: a function rule naming either side matches a call
|
|
54
54
|
* on either side. Omit names that are ambiguous across providers
|
|
55
55
|
* (e.g. `get_issue` is exposed by both Linear and GitHub MCP servers).
|
|
56
56
|
*/
|
|
@@ -68,30 +68,74 @@ export type NodeSources = _Source[] | {
|
|
|
68
68
|
only?: boolean;
|
|
69
69
|
sources: _Source[];
|
|
70
70
|
};
|
|
71
|
+
/** Kind of evaluator. See {@link Evaluator}. */
|
|
72
|
+
export type EvaluatorKind = "value" | "function" | "judge";
|
|
73
|
+
/** Aggregation policy for a node's evaluator results. v1 implements `all_pass`. */
|
|
74
|
+
export type EvalPolicy = "all_pass" | "any_pass" | "weighted";
|
|
71
75
|
/**
|
|
72
|
-
*
|
|
76
|
+
* The deterministic rule body for a `value` or `function` evaluator.
|
|
73
77
|
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
* and concatenates failures into a single error string on `result.data.error`.
|
|
79
|
-
*
|
|
80
|
-
* Keep the shape small and declarative. Anything richer should be a skill or
|
|
81
|
-
* a dedicated workflow node, not a verify clause.
|
|
78
|
+
* `value` rules use `output_required` and `output_matches` (data-shape).
|
|
79
|
+
* `function` rules use `any_tool_called` / `all_tools_called` / `no_tool_called`
|
|
80
|
+
* (trace-shape). Mixing the two on a single rule is allowed for compactness
|
|
81
|
+
* but discouraged: a single evaluator should test a single thing.
|
|
82
82
|
*/
|
|
83
|
-
export interface
|
|
84
|
-
/** At least one of these tools was called and succeeded
|
|
83
|
+
export interface EvaluatorRule {
|
|
84
|
+
/** At least one of these tools was called and succeeded. (`function` rules.) */
|
|
85
85
|
any_tool_called?: string[];
|
|
86
|
-
/** Every named tool was called and succeeded at least once
|
|
86
|
+
/** Every named tool was called and succeeded at least once. (`function` rules.) */
|
|
87
87
|
all_tools_called?: string[];
|
|
88
|
-
/** None of these tools may have been invoked
|
|
88
|
+
/** None of these tools may have been invoked. (`function` rules.) */
|
|
89
89
|
no_tool_called?: string[];
|
|
90
|
-
/** Listed paths must be present and non-null in `result.data
|
|
90
|
+
/** Listed paths must be present and non-null in `result.data`. (`value` rules.) */
|
|
91
91
|
output_required?: string[];
|
|
92
|
-
/** Each assertion must hold against `result.data`. */
|
|
92
|
+
/** Each assertion must hold against `result.data`. (`value` rules.) */
|
|
93
93
|
output_matches?: OutputMatch[];
|
|
94
94
|
}
|
|
95
|
+
/**
|
|
96
|
+
* A single evaluator on a node.
|
|
97
|
+
*
|
|
98
|
+
* - `value` and `function` use `rule`.
|
|
99
|
+
* - `judge` uses `rubric` (and optional `pass_when`, `model`).
|
|
100
|
+
*
|
|
101
|
+
* Each evaluator produces an {@link EvalResult}. The executor aggregates
|
|
102
|
+
* the results per the node's `eval_policy` (default `all_pass`).
|
|
103
|
+
*/
|
|
104
|
+
export interface Evaluator {
|
|
105
|
+
/** Stable identifier. Used in EvalResult and retry preambles. */
|
|
106
|
+
name: string;
|
|
107
|
+
/** Discriminator. Determines which fields apply. */
|
|
108
|
+
kind: EvaluatorKind;
|
|
109
|
+
/** Required for `value` / `function`. */
|
|
110
|
+
rule?: EvaluatorRule;
|
|
111
|
+
/** Required for `judge`. Natural-language criterion. */
|
|
112
|
+
rubric?: string;
|
|
113
|
+
/** `judge` only. Verdict token that indicates pass. Default: `"yes"`. */
|
|
114
|
+
pass_when?: string;
|
|
115
|
+
/** `judge` only. Override the judge model for this evaluator. */
|
|
116
|
+
model?: string;
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Per-evaluator result.
|
|
120
|
+
*
|
|
121
|
+
* The executor produces one EvalResult per declared evaluator. The list
|
|
122
|
+
* lands on {@link NodeResult.evals}.
|
|
123
|
+
*/
|
|
124
|
+
export interface EvalResult {
|
|
125
|
+
/** Echoes the evaluator's name. */
|
|
126
|
+
name: string;
|
|
127
|
+
/** Echoes the evaluator's kind. */
|
|
128
|
+
kind: EvaluatorKind;
|
|
129
|
+
/** Whether this evaluator passed. */
|
|
130
|
+
pass: boolean;
|
|
131
|
+
/**
|
|
132
|
+
* Failure detail for value/function (formatted by the executor) or judge
|
|
133
|
+
* (returned by the model). Capped at ~500 characters when emitted.
|
|
134
|
+
*/
|
|
135
|
+
reasoning?: string;
|
|
136
|
+
/** Reserved for `weighted` policies. Not populated in v1. */
|
|
137
|
+
score?: number;
|
|
138
|
+
}
|
|
95
139
|
/**
|
|
96
140
|
* Machine-checked pre-condition for a node.
|
|
97
141
|
*
|
|
@@ -102,7 +146,7 @@ export interface NodeVerify {
|
|
|
102
146
|
* Path roots resolve against the cross-node context map:
|
|
103
147
|
* { input: <runtime input>, [priorNodeId]: <data of prior node>, ... }
|
|
104
148
|
*
|
|
105
|
-
* Reuses the same path grammar as `
|
|
149
|
+
* Reuses the same path grammar as `eval` (dotted segments, `[*]` wildcard,
|
|
106
150
|
* optional `all:`/`any:` prefix).
|
|
107
151
|
*/
|
|
108
152
|
export interface NodeRequires {
|
|
@@ -114,15 +158,15 @@ export interface NodeRequires {
|
|
|
114
158
|
on_fail?: "fail" | "skip";
|
|
115
159
|
}
|
|
116
160
|
/**
|
|
117
|
-
* Node-local retry on
|
|
161
|
+
* Node-local retry on eval failure.
|
|
118
162
|
*
|
|
119
163
|
* Re-runs the LLM up to `max` additional times, prepending feedback derived
|
|
120
|
-
* from the
|
|
121
|
-
* errors and not by `requires` failure.
|
|
164
|
+
* from the failing evaluators. Triggered ONLY by eval failure, not by tool
|
|
165
|
+
* / API errors and not by `requires` failure.
|
|
122
166
|
*
|
|
123
167
|
* `instruction` shapes the feedback preamble:
|
|
124
|
-
* - omitted → default "## Previous attempt failed
|
|
125
|
-
* - string → static text +
|
|
168
|
+
* - omitted → default "## Previous attempt failed evaluation..."
|
|
169
|
+
* - string → static text + structured eval failure list
|
|
126
170
|
* - { auto: true } → LLM-generated diagnosis from default reflection prompt
|
|
127
171
|
* - { reflect: s } → LLM-generated diagnosis from author-provided prompt
|
|
128
172
|
*/
|
|
@@ -167,23 +211,27 @@ export interface Node {
|
|
|
167
211
|
rules?: NodeSources;
|
|
168
212
|
/** Per-node background knowledge. Additive by default; set `{ only: true, sources: [...] }` to block cascade. */
|
|
169
213
|
context?: NodeSources;
|
|
170
|
-
/**
|
|
171
|
-
|
|
214
|
+
/** Named evaluators run after the LLM finishes. Each produces an {@link EvalResult}. */
|
|
215
|
+
eval?: Evaluator[];
|
|
216
|
+
/** How evaluator results aggregate. Default `all_pass`. v1 implements only `all_pass`. */
|
|
217
|
+
eval_policy?: EvalPolicy;
|
|
218
|
+
/** Default model for judge evaluators on this node. Overrides workflow-level `judge_model`. */
|
|
219
|
+
judge_model?: string;
|
|
172
220
|
/** Machine-checked pre-conditions. Enforced by the executor before the LLM runs. */
|
|
173
221
|
requires?: NodeRequires;
|
|
174
|
-
/** Node-local retry on
|
|
222
|
+
/** Node-local retry on eval failure (with optional autonomous reflection). */
|
|
175
223
|
retry?: NodeRetry;
|
|
176
224
|
}
|
|
177
225
|
/** An edge connecting two nodes */
|
|
178
226
|
export interface Edge {
|
|
179
227
|
from: string;
|
|
180
228
|
to: string;
|
|
181
|
-
/** Natural language condition
|
|
229
|
+
/** Natural language condition. Claude evaluates at runtime. */
|
|
182
230
|
when?: string;
|
|
183
231
|
/** Max times this edge can be followed (enables retry loops). Default: unlimited. */
|
|
184
232
|
max_iterations?: number;
|
|
185
233
|
}
|
|
186
|
-
/** A complete workflow definition
|
|
234
|
+
/** A complete workflow definition. Pure data, fully serializable. */
|
|
187
235
|
export interface Workflow {
|
|
188
236
|
id: string;
|
|
189
237
|
name: string;
|
|
@@ -196,6 +244,10 @@ export interface Workflow {
|
|
|
196
244
|
rules?: _Source[];
|
|
197
245
|
/** Background knowledge prepended to every node's instruction. Cascade into per-node context. */
|
|
198
246
|
context?: _Source[];
|
|
247
|
+
/** Default model for judge evaluators across the workflow. Overridable per-node and per-evaluator. */
|
|
248
|
+
judge_model?: string;
|
|
249
|
+
/** Soft cap on expected judge calls per workflow run. Warning at load time when exceeded. */
|
|
250
|
+
judge_budget?: number;
|
|
199
251
|
}
|
|
200
252
|
/**
|
|
201
253
|
* Inline skill definition in a workflow's `skills` block.
|
|
@@ -215,6 +267,12 @@ export interface NodeResult {
|
|
|
215
267
|
data: Record<string, unknown>;
|
|
216
268
|
/** Tool calls made during this node's execution */
|
|
217
269
|
toolCalls: ToolCall[];
|
|
270
|
+
/**
|
|
271
|
+
* Per-evaluator results from `node.eval` (one entry per declared evaluator,
|
|
272
|
+
* in declaration order). Absent when the node has no `eval` block or when
|
|
273
|
+
* eval was not run (e.g. node failed during execution).
|
|
274
|
+
*/
|
|
275
|
+
evals?: EvalResult[];
|
|
218
276
|
}
|
|
219
277
|
export interface ToolCall {
|
|
220
278
|
tool: string;
|
|
@@ -223,13 +281,13 @@ export interface ToolCall {
|
|
|
223
281
|
/**
|
|
224
282
|
* Authoritative outcome of the tool invocation.
|
|
225
283
|
*
|
|
226
|
-
* When present, this is the source of truth for `
|
|
284
|
+
* When present, this is the source of truth for `function` evaluators. The
|
|
227
285
|
* output-shape heuristic is a legacy fallback only. Set by the Claude
|
|
228
286
|
* runtime on tool completion: "success" when the tool returned normally,
|
|
229
287
|
* "error" when it threw or the MCP server returned is_error=true.
|
|
230
288
|
*
|
|
231
|
-
* Absent status indicates a legacy or hand-constructed ToolCall
|
|
232
|
-
*
|
|
289
|
+
* Absent status indicates a legacy or hand-constructed ToolCall. Function
|
|
290
|
+
* evaluators fall back to inspecting `output` for an `error` key.
|
|
233
291
|
*/
|
|
234
292
|
status?: "success" | "error";
|
|
235
293
|
}
|
|
@@ -334,11 +392,16 @@ export interface Claude {
|
|
|
334
392
|
}): Promise<string>;
|
|
335
393
|
/**
|
|
336
394
|
* Single-completion free-text query. No tools, no output schema.
|
|
337
|
-
* Used by the executor to generate retry strategies in autonomous reflection mode
|
|
395
|
+
* Used by the executor to generate retry strategies in autonomous reflection mode
|
|
396
|
+
* and by judge evaluators to score node results against a rubric.
|
|
397
|
+
*
|
|
398
|
+
* `model` overrides the client's default for this call. Implementations
|
|
399
|
+
* SHOULD use it; mocks MAY ignore it.
|
|
338
400
|
*/
|
|
339
401
|
ask(opts: {
|
|
340
402
|
instruction: string;
|
|
341
403
|
context: Record<string, unknown>;
|
|
404
|
+
model?: string;
|
|
342
405
|
}): Promise<string>;
|
|
343
406
|
}
|
|
344
407
|
export interface McpServerConfig {
|