@sweny-ai/core 0.1.156 → 0.1.157
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eval/index.js +32 -32
- package/dist/executor.js +84 -5
- package/dist/schema.d.ts +9 -0
- package/dist/schema.js +7 -1
- package/dist/types.d.ts +10 -0
- package/dist/types.js +2 -0
- package/package.json +1 -1
package/dist/eval/index.js
CHANGED
|
@@ -35,42 +35,42 @@ function resolveJudgeModel(evaluator, node, workflow, harnessDefault) {
|
|
|
35
35
|
export async function evaluateAll(evaluators, result, opts = {}) {
|
|
36
36
|
if (!evaluators || evaluators.length === 0)
|
|
37
37
|
return [];
|
|
38
|
+
// A judge with no client is a wiring error: refuse before any model call.
|
|
39
|
+
const orphanJudge = evaluators.find((e) => e.kind === "judge" && !opts.claude);
|
|
40
|
+
if (orphanJudge) {
|
|
41
|
+
throw new Error(`evaluator '${orphanJudge.name}' is kind: judge but no Claude client was provided to evaluateAll`);
|
|
42
|
+
}
|
|
43
|
+
// Evaluators are independent, so they run concurrently (#337): N judges on
|
|
44
|
+
// a node cost one round-trip of wall time, not N. Results keep declaration
|
|
45
|
+
// order. Every evaluator settles before an error surfaces, and the first
|
|
46
|
+
// rejection in declaration order is rethrown, so the outcome never depends
|
|
47
|
+
// on which call finishes first.
|
|
48
|
+
const settled = await Promise.allSettled(evaluators.map((e) => evaluateOne(e, result, opts)));
|
|
38
49
|
const out = [];
|
|
39
|
-
for (const
|
|
40
|
-
if (
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
kind: "value",
|
|
45
|
-
pass: verdict.pass,
|
|
46
|
-
reasoning: capReasoning(verdict.reasoning),
|
|
47
|
-
});
|
|
48
|
-
}
|
|
49
|
-
else if (e.kind === "function") {
|
|
50
|
-
const verdict = evaluateFunctionRule(e.rule ?? {}, result.toolCalls, opts.aliases);
|
|
51
|
-
out.push({
|
|
52
|
-
name: e.name,
|
|
53
|
-
kind: "function",
|
|
54
|
-
pass: verdict.pass,
|
|
55
|
-
reasoning: capReasoning(verdict.reasoning),
|
|
56
|
-
});
|
|
57
|
-
}
|
|
58
|
-
else if (e.kind === "judge") {
|
|
59
|
-
if (!opts.claude) {
|
|
60
|
-
throw new Error(`evaluator '${e.name}' is kind: judge but no Claude client was provided to evaluateAll`);
|
|
61
|
-
}
|
|
62
|
-
const model = resolveJudgeModel(e, opts.node, opts.workflow, opts.claude.defaultJudgeModel);
|
|
63
|
-
const verdict = await evaluateJudge(e, result, opts.claude, { model });
|
|
64
|
-
out.push({
|
|
65
|
-
name: e.name,
|
|
66
|
-
kind: "judge",
|
|
67
|
-
pass: verdict.pass,
|
|
68
|
-
reasoning: capReasoning(verdict.reasoning),
|
|
69
|
-
});
|
|
70
|
-
}
|
|
50
|
+
for (const s of settled) {
|
|
51
|
+
if (s.status === "rejected")
|
|
52
|
+
throw s.reason;
|
|
53
|
+
if (s.value)
|
|
54
|
+
out.push(s.value);
|
|
71
55
|
}
|
|
72
56
|
return out;
|
|
73
57
|
}
|
|
58
|
+
async function evaluateOne(e, result, opts) {
|
|
59
|
+
if (e.kind === "value") {
|
|
60
|
+
const verdict = evaluateValueRule(e.rule ?? {}, result.data);
|
|
61
|
+
return { name: e.name, kind: "value", pass: verdict.pass, reasoning: capReasoning(verdict.reasoning) };
|
|
62
|
+
}
|
|
63
|
+
if (e.kind === "function") {
|
|
64
|
+
const verdict = evaluateFunctionRule(e.rule ?? {}, result.toolCalls, opts.aliases);
|
|
65
|
+
return { name: e.name, kind: "function", pass: verdict.pass, reasoning: capReasoning(verdict.reasoning) };
|
|
66
|
+
}
|
|
67
|
+
if (e.kind === "judge" && opts.claude) {
|
|
68
|
+
const model = resolveJudgeModel(e, opts.node, opts.workflow, opts.claude.defaultJudgeModel);
|
|
69
|
+
const verdict = await evaluateJudge(e, result, opts.claude, { model });
|
|
70
|
+
return { name: e.name, kind: "judge", pass: verdict.pass, reasoning: capReasoning(verdict.reasoning) };
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
74
|
/**
|
|
75
75
|
* Collapse a list of {@link EvalResult}s into a single pass/fail under the
|
|
76
76
|
* node's `eval_policy`. v1 implements `all_pass`. Other policies are
|
package/dist/executor.js
CHANGED
|
@@ -206,14 +206,22 @@ export async function execute(workflow, input, options) {
|
|
|
206
206
|
// (a Record<name, EvalResult> for downstream lookup like
|
|
207
207
|
// `priorNode.evals.tests_run_clean.pass`). This namespace is reserved
|
|
208
208
|
// for runtime verdicts; an agent-provided `data.evals` never overrides it.
|
|
209
|
-
|
|
209
|
+
// `requires` always reads this full map: it is a deterministic gate, not
|
|
210
|
+
// a prompt, so it costs no tokens and keeps every declared path working.
|
|
211
|
+
const requiresContext = {
|
|
210
212
|
input,
|
|
211
213
|
...Object.fromEntries([...results.entries()].map(([k, v]) => [k, buildPriorNodeContext(v)])),
|
|
212
214
|
};
|
|
215
|
+
// What the model sees (#337): only nodes this one can depend on, and a
|
|
216
|
+
// schema'd node's declared fields instead of its prose. `context_mode:
|
|
217
|
+
// full` keeps the old everything-prior map.
|
|
218
|
+
const context = workflow.context_mode === "full"
|
|
219
|
+
? requiresContext
|
|
220
|
+
: buildBoundedContext(workflow, currentId, results, input, resolvedInstruction);
|
|
213
221
|
// Pre-condition gate: evaluate `requires` against the cross-node context
|
|
214
222
|
// BEFORE invoking the LLM. Failure either marks the node failed (on_fail
|
|
215
223
|
// default) or skipped (on_fail: "skip") and skips execution entirely.
|
|
216
|
-
const requiresError = evaluateRequires(node.requires,
|
|
224
|
+
const requiresError = evaluateRequires(node.requires, requiresContext);
|
|
217
225
|
if (requiresError) {
|
|
218
226
|
const onFail = node.requires?.on_fail ?? "fail";
|
|
219
227
|
const result = onFail === "skip"
|
|
@@ -653,9 +661,9 @@ function buildPriorNodeContext(result) {
|
|
|
653
661
|
* evaluator. When no schema is declared we fall back to the full data
|
|
654
662
|
* (back-compat for workflows without structured outputs).
|
|
655
663
|
*
|
|
656
|
-
*
|
|
657
|
-
* `
|
|
658
|
-
*
|
|
664
|
+
* Downstream node prompts get a similar projection (see
|
|
665
|
+
* `buildBoundedNodeEntry`), minus the null-fill, unless the workflow sets
|
|
666
|
+
* `context_mode: full`.
|
|
659
667
|
*/
|
|
660
668
|
function buildRouteEvalEntry(result, sourceNode) {
|
|
661
669
|
const fullData = (result.data ?? {});
|
|
@@ -714,6 +722,77 @@ function getDeclaredOutputProperties(output) {
|
|
|
714
722
|
return null;
|
|
715
723
|
return new Set(keys);
|
|
716
724
|
}
|
|
725
|
+
// ─── Bounded node context (#337) ─────────────────────────────────
|
|
726
|
+
//
|
|
727
|
+
// Full mode hands every node the whole results map: node N re-sends nodes
|
|
728
|
+
// 1..N-1 including each one's prose `summary`, so prompt bytes per run grow
|
|
729
|
+
// quadratically and one verbose node inflates every later prompt. Bounded
|
|
730
|
+
// mode (the default) keeps the same keys a node can depend on, and sends a
|
|
731
|
+
// schema'd node's declared fields only.
|
|
732
|
+
/** Keys the executor itself writes into data; kept when a view is projected. */
|
|
733
|
+
const RUNTIME_DATA_KEYS = ["error", "fail_soft", "skipped_reason"];
|
|
734
|
+
/**
|
|
735
|
+
* The prompt-context entry for one prior node in bounded mode. A successful
|
|
736
|
+
* node with a declared `output.properties` block contributes those fields
|
|
737
|
+
* (plus executor-written error/fail_soft keys); everything else, including
|
|
738
|
+
* the free-text `summary`, is dropped. Nodes without a schema, and nodes
|
|
739
|
+
* that did not succeed, keep the full `buildPriorNodeContext` entry. The
|
|
740
|
+
* trusted `evals` and `safe_outputs` namespaces always come from the runtime.
|
|
741
|
+
*/
|
|
742
|
+
function buildBoundedNodeEntry(result, sourceNode) {
|
|
743
|
+
const full = buildPriorNodeContext(result);
|
|
744
|
+
const declared = getDeclaredOutputProperties(sourceNode?.output);
|
|
745
|
+
if (!declared || result.status !== "success")
|
|
746
|
+
return full;
|
|
747
|
+
const keep = new Set([...declared, ...RUNTIME_DATA_KEYS, "evals", "safe_outputs"]);
|
|
748
|
+
return Object.fromEntries(Object.entries(full).filter(([k]) => keep.has(k)));
|
|
749
|
+
}
|
|
750
|
+
/** First segment of a requires path (`any:triage.findings[*].x` -> `triage`). */
|
|
751
|
+
function pathRoot(path) {
|
|
752
|
+
return /^(?:all:|any:)?([^.[\]]+)/.exec(path)?.[1];
|
|
753
|
+
}
|
|
754
|
+
function mentionsNode(text, nodeId) {
|
|
755
|
+
const escaped = nodeId.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
756
|
+
return new RegExp(`(?<![A-Za-z0-9_-])${escaped}(?![A-Za-z0-9_-])`).test(text);
|
|
757
|
+
}
|
|
758
|
+
/**
|
|
759
|
+
* Node ids whose results `nodeId` may read: its graph ancestors (every node
|
|
760
|
+
* with an edge path into it, itself included when it sits on a cycle), any
|
|
761
|
+
* node a `requires` path names, and any node id its instruction mentions.
|
|
762
|
+
*/
|
|
763
|
+
function contextDependencies(workflow, nodeId, instruction = "") {
|
|
764
|
+
const deps = new Set();
|
|
765
|
+
const stack = [nodeId];
|
|
766
|
+
for (let id = stack.pop(); id !== undefined; id = stack.pop()) {
|
|
767
|
+
for (const e of workflow.edges) {
|
|
768
|
+
if (e.to === id && !deps.has(e.from)) {
|
|
769
|
+
deps.add(e.from);
|
|
770
|
+
stack.push(e.from);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
}
|
|
774
|
+
const requires = workflow.nodes[nodeId]?.requires;
|
|
775
|
+
for (const p of [...(requires?.output_required ?? []), ...(requires?.output_matches ?? []).map((m) => m.path)]) {
|
|
776
|
+
const root = pathRoot(p);
|
|
777
|
+
if (root && root !== "input")
|
|
778
|
+
deps.add(root);
|
|
779
|
+
}
|
|
780
|
+
for (const id of Object.keys(workflow.nodes)) {
|
|
781
|
+
if (id !== nodeId && mentionsNode(instruction, id))
|
|
782
|
+
deps.add(id);
|
|
783
|
+
}
|
|
784
|
+
return deps;
|
|
785
|
+
}
|
|
786
|
+
/** The bounded prompt context for one node: `input` plus its dependencies' entries. */
|
|
787
|
+
function buildBoundedContext(workflow, nodeId, results, input, instruction) {
|
|
788
|
+
const deps = contextDependencies(workflow, nodeId, instruction);
|
|
789
|
+
return {
|
|
790
|
+
input,
|
|
791
|
+
...Object.fromEntries([...results.entries()]
|
|
792
|
+
.filter(([k]) => deps.has(k))
|
|
793
|
+
.map(([k, v]) => [k, buildBoundedNodeEntry(v, workflow.nodes[k])])),
|
|
794
|
+
};
|
|
795
|
+
}
|
|
717
796
|
/**
|
|
718
797
|
* Return the declared `required` field names for a node's output schema,
|
|
719
798
|
* or an empty array when no required block is declared. Tolerant of
|
package/dist/schema.d.ts
CHANGED
|
@@ -2547,6 +2547,7 @@ export declare const workflowZ: z.ZodObject<{
|
|
|
2547
2547
|
judge_model: z.ZodOptional<z.ZodString>;
|
|
2548
2548
|
model: z.ZodOptional<z.ZodString>;
|
|
2549
2549
|
judge_budget: z.ZodOptional<z.ZodNumber>;
|
|
2550
|
+
context_mode: z.ZodOptional<z.ZodEnum<["bounded", "full"]>>;
|
|
2550
2551
|
inputs: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodEffects<z.ZodObject<{
|
|
2551
2552
|
type: z.ZodEnum<["string", "number", "boolean", "string[]"]>;
|
|
2552
2553
|
description: z.ZodOptional<z.ZodString>;
|
|
@@ -2785,6 +2786,7 @@ export declare const workflowZ: z.ZodObject<{
|
|
|
2785
2786
|
judge_model?: string | undefined;
|
|
2786
2787
|
workflow_type?: "pr_review" | "e2e_test" | "content_generation" | "monitor" | "data_sync" | "generic" | undefined;
|
|
2787
2788
|
judge_budget?: number | undefined;
|
|
2789
|
+
context_mode?: "full" | "bounded" | undefined;
|
|
2788
2790
|
inputs?: Record<string, {
|
|
2789
2791
|
type: "string" | "number" | "boolean" | "string[]";
|
|
2790
2792
|
description?: string | undefined;
|
|
@@ -2964,6 +2966,7 @@ export declare const workflowZ: z.ZodObject<{
|
|
|
2964
2966
|
judge_model?: string | undefined;
|
|
2965
2967
|
workflow_type?: "pr_review" | "e2e_test" | "content_generation" | "monitor" | "data_sync" | "generic" | undefined;
|
|
2966
2968
|
judge_budget?: number | undefined;
|
|
2969
|
+
context_mode?: "full" | "bounded" | undefined;
|
|
2967
2970
|
inputs?: Record<string, {
|
|
2968
2971
|
type: "string" | "number" | "boolean" | "string[]";
|
|
2969
2972
|
description?: string | undefined;
|
|
@@ -3157,6 +3160,7 @@ export declare function parseWorkflow(raw: unknown): {
|
|
|
3157
3160
|
judge_model?: string | undefined;
|
|
3158
3161
|
workflow_type?: "pr_review" | "e2e_test" | "content_generation" | "monitor" | "data_sync" | "generic" | undefined;
|
|
3159
3162
|
judge_budget?: number | undefined;
|
|
3163
|
+
context_mode?: "full" | "bounded" | undefined;
|
|
3160
3164
|
inputs?: Record<string, {
|
|
3161
3165
|
type: "string" | "number" | "boolean" | "string[]";
|
|
3162
3166
|
description?: string | undefined;
|
|
@@ -3610,6 +3614,11 @@ export declare const workflowJsonSchema: {
|
|
|
3610
3614
|
readonly minimum: 0;
|
|
3611
3615
|
readonly description: "Soft cap on expected judge calls per workflow run. Executor warns at load time if exceeded; not a hard runtime cap in v1.";
|
|
3612
3616
|
};
|
|
3617
|
+
readonly context_mode: {
|
|
3618
|
+
readonly type: "string";
|
|
3619
|
+
readonly enum: readonly ["bounded", "full"];
|
|
3620
|
+
readonly description: "What prior results a node's prompt receives. bounded (default): only nodes it can depend on, and a schema'd node's declared fields instead of its free-text summary. full: every prior node's complete data.";
|
|
3621
|
+
};
|
|
3613
3622
|
readonly inputs: {
|
|
3614
3623
|
readonly type: "object";
|
|
3615
3624
|
readonly description: "Declared per-run input contract. Each entry names a parameter the caller may supply via --input. The CLI validates against this declaration, applies defaults for omitted optional fields, and rejects unknown types before the executor runs. Optional; workflows without an inputs block accept any JSON object (back-compat).";
|
package/dist/schema.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* parsing untrusted input (JSON/YAML import, Studio editor).
|
|
8
8
|
*/
|
|
9
9
|
import { z } from "zod";
|
|
10
|
-
import { AUTHOR_ASSOCIATIONS, EVALUATOR_KINDS, EVAL_POLICIES, MCP_TRANSPORTS, NODE_ACCESS, NODE_ON_FAIL, REQUIRES_ON_FAIL, SAFE_OUTPUT_APPLIERS, SAFE_OUTPUT_EXPIRES_PATTERN, SAFE_OUTPUT_MAX_CEILING, SAFE_OUTPUT_STATES, SAFE_OUTPUT_TYPES, SKILL_CATEGORIES, SKILL_ID_MAX_LENGTH, SKILL_ID_PATTERN, TOOL_ACCESS, TOOL_CLASSES, WORKFLOW_TYPES, } from "./types.js";
|
|
10
|
+
import { AUTHOR_ASSOCIATIONS, CONTEXT_MODES, EVALUATOR_KINDS, EVAL_POLICIES, MCP_TRANSPORTS, NODE_ACCESS, NODE_ON_FAIL, REQUIRES_ON_FAIL, SAFE_OUTPUT_APPLIERS, SAFE_OUTPUT_EXPIRES_PATTERN, SAFE_OUTPUT_MAX_CEILING, SAFE_OUTPUT_STATES, SAFE_OUTPUT_TYPES, SKILL_CATEGORIES, SKILL_ID_MAX_LENGTH, SKILL_ID_PATTERN, TOOL_ACCESS, TOOL_CLASSES, WORKFLOW_TYPES, } from "./types.js";
|
|
11
11
|
import { sourceZ } from "./sources.js";
|
|
12
12
|
import { workflowInputsZ, WORKFLOW_INPUT_TYPES } from "./inputs.js";
|
|
13
13
|
export { sourceZ };
|
|
@@ -353,6 +353,7 @@ export const workflowZ = z.object({
|
|
|
353
353
|
judge_model: z.string().min(1).optional(),
|
|
354
354
|
model: z.string().min(1).optional(),
|
|
355
355
|
judge_budget: z.number().int().min(0).optional(),
|
|
356
|
+
context_mode: z.enum(CONTEXT_MODES).optional(),
|
|
356
357
|
inputs: workflowInputsZ.optional(),
|
|
357
358
|
permissions: nodePermissionsZ.optional(),
|
|
358
359
|
safe_outputs: safeOutputsPolicyZ.optional(),
|
|
@@ -983,6 +984,11 @@ export const workflowJsonSchema = {
|
|
|
983
984
|
// applies the soft-cap default at use-time. See judge_model above.
|
|
984
985
|
description: "Soft cap on expected judge calls per workflow run. Executor warns at load time if exceeded; not a hard runtime cap in v1.",
|
|
985
986
|
},
|
|
987
|
+
context_mode: {
|
|
988
|
+
type: "string",
|
|
989
|
+
enum: [...CONTEXT_MODES],
|
|
990
|
+
description: "What prior results a node's prompt receives. bounded (default): only nodes it can depend on, and a schema'd node's declared fields instead of its free-text summary. full: every prior node's complete data.",
|
|
991
|
+
},
|
|
986
992
|
inputs: {
|
|
987
993
|
type: "object",
|
|
988
994
|
description: "Declared per-run input contract. Each entry names a parameter the caller may supply via --input. The CLI validates against this declaration, applies defaults for omitted optional fields, and rejects unknown types before the executor runs. Optional; workflows without an inputs block accept any JSON object (back-compat).",
|
package/dist/types.d.ts
CHANGED
|
@@ -137,6 +137,9 @@ export type EvaluatorKind = (typeof EVALUATOR_KINDS)[number];
|
|
|
137
137
|
/** Aggregation policy for a node's evaluator results. v1 implements `all_pass`. */
|
|
138
138
|
export declare const EVAL_POLICIES: readonly ["all_pass", "any_pass", "weighted"];
|
|
139
139
|
export type EvalPolicy = (typeof EVAL_POLICIES)[number];
|
|
140
|
+
/** What prior results a node's prompt receives (#337). See {@link Workflow.context_mode}. */
|
|
141
|
+
export declare const CONTEXT_MODES: readonly ["bounded", "full"];
|
|
142
|
+
export type ContextMode = (typeof CONTEXT_MODES)[number];
|
|
140
143
|
/** Action when a `requires` precondition fails. */
|
|
141
144
|
export declare const REQUIRES_ON_FAIL: readonly ["fail", "skip"];
|
|
142
145
|
export type RequiresOnFail = (typeof REQUIRES_ON_FAIL)[number];
|
|
@@ -520,6 +523,13 @@ export interface Workflow {
|
|
|
520
523
|
model?: string;
|
|
521
524
|
/** Soft cap on expected judge calls per workflow run. Warning at load time when exceeded. */
|
|
522
525
|
judge_budget?: number;
|
|
526
|
+
/**
|
|
527
|
+
* What prior results a node's prompt receives (#337). `bounded` (default):
|
|
528
|
+
* only nodes it can depend on (graph ancestors, nodes named by `requires`
|
|
529
|
+
* or its instruction), and a schema'd node's declared fields instead of its
|
|
530
|
+
* free-text `summary`. `full`: every prior node's complete data.
|
|
531
|
+
*/
|
|
532
|
+
context_mode?: ContextMode;
|
|
523
533
|
/**
|
|
524
534
|
* Declared per-run input contract. When present, the CLI validates the
|
|
525
535
|
* caller-provided `--input` JSON against this declaration, applies defaults
|
package/dist/types.js
CHANGED
|
@@ -51,6 +51,8 @@ export function isValidSkillId(id) {
|
|
|
51
51
|
export const EVALUATOR_KINDS = ["value", "function", "judge"];
|
|
52
52
|
/** Aggregation policy for a node's evaluator results. v1 implements `all_pass`. */
|
|
53
53
|
export const EVAL_POLICIES = ["all_pass", "any_pass", "weighted"];
|
|
54
|
+
/** What prior results a node's prompt receives (#337). See {@link Workflow.context_mode}. */
|
|
55
|
+
export const CONTEXT_MODES = ["bounded", "full"];
|
|
54
56
|
/** Action when a `requires` precondition fails. */
|
|
55
57
|
export const REQUIRES_ON_FAIL = ["fail", "skip"];
|
|
56
58
|
/**
|