@sweny-ai/core 0.1.84 → 0.1.86

package/dist/eval/value.js

@@ -0,0 +1,205 @@
+// ─── Eval: value (data-shape) evaluator ──────────────────────────────
+//
+// Operates on `result.data`. Combines `output_required` and `output_matches`
+// checks. Pure, deterministic, fast.
+import { resolvePath } from "./path.js";
+// ─── Internals ───────────────────────────────────────────────────────
+function deepEqual(a, b) {
+    if (Object.is(a, b))
+        return true;
+    if (typeof a !== typeof b)
+        return false;
+    if (a === null || b === null)
+        return false;
+    if (typeof a !== "object")
+        return false;
+    if (Array.isArray(a)) {
+        if (!Array.isArray(b) || a.length !== b.length)
+            return false;
+        for (let i = 0; i < a.length; i++) {
+            if (!deepEqual(a[i], b[i]))
+                return false;
+        }
+        return true;
+    }
+    if (Array.isArray(b))
+        return false;
+    const ao = a;
+    const bo = b;
+    const aKeys = Object.keys(ao);
+    const bKeys = Object.keys(bo);
+    if (aKeys.length !== bKeys.length)
+        return false;
+    for (const k of aKeys) {
+        if (!Object.prototype.hasOwnProperty.call(bo, k))
+            return false;
+        if (!deepEqual(ao[k], bo[k]))
+            return false;
+    }
+    return true;
+}
+function formatValue(v) {
+    if (v === null)
+        return "null";
+    if (v === undefined)
+        return "undefined";
+    return JSON.stringify(v);
+}
+/**
+ * Cap on `output_matches[].matches` source length.
+ *
+ * `matches` is user-supplied through workflow YAML, including marketplace
+ * workflows. A pathological pattern like `(a+)+$` causes exponential
+ * backtracking in V8's regex engine. Capping the source length is a cheap,
+ * useful defense; well-formed regexes are tens of characters at most.
+ *
+ * Caveat: length alone does NOT prevent all ReDoS. A 12-char pattern like
+ * `(a+)+$` is still catastrophic against the right input. Future work:
+ * gate compilation through `safe-regex` or use the `re2` engine for
+ * marketplace inputs. The cap closes the obvious abuse surface without
+ * adding a runtime dep right now.
+ */
+const MAX_REGEX_SOURCE_LENGTH = 1000;
+function compileMatch(m) {
+    if (m.matches !== undefined) {
+        if (m.matches.length > MAX_REGEX_SOURCE_LENGTH) {
+            throw new Error(`regex source exceeds ${MAX_REGEX_SOURCE_LENGTH} characters (got ${m.matches.length}); ` +
+                `tighten the pattern to avoid catastrophic-backtracking risk`);
+        }
+        // Throws synchronously if the regex source is invalid; surfaced as a
+        // value-evaluator failure rather than a crash because the executor catches.
+        return { match: m, regex: new RegExp(m.matches) };
+    }
+    return { match: m };
+}
+function applyOperator(c, value) {
+    const m = c.match;
+    if (m.equals !== undefined) {
+        if (deepEqual(value, m.equals))
+            return { ok: true };
+        return { ok: false, reason: `equals ${formatValue(m.equals)}, got ${formatValue(value)}` };
+    }
+    if (m.in !== undefined) {
+        if (m.in.some((x) => deepEqual(value, x)))
+            return { ok: true };
+        return {
+            ok: false,
+            reason: `in [${m.in.map(formatValue).join(", ")}], got ${formatValue(value)}`,
+        };
+    }
+    if (c.regex) {
+        const s = typeof value === "string" ? value : JSON.stringify(value);
+        if (c.regex.test(s))
+            return { ok: true };
+        return { ok: false, reason: `matches /${m.matches}/, got ${formatValue(value)}` };
+    }
+    return { ok: false, reason: "no operator declared (zod should have caught this)" };
+}
+function describeOperator(c) {
+    const m = c.match;
+    if (m.equals !== undefined)
+        return `equals ${formatValue(m.equals)}`;
+    if (m.in !== undefined)
+        return `in [${m.in.map(formatValue).join(", ")}]`;
+    if (m.matches !== undefined)
+        return `matches /${m.matches}/`;
+    return "no operator";
+}
+// ─── Public check helpers ────────────────────────────────────────────
+export function checkOutputRequired(paths, data) {
+    const failures = [];
+    for (const path of paths) {
+        const r = resolvePath(data, path);
+        if (!r.ok) {
+            failures.push(`'${path}' ${r.reason}`);
+            continue;
+        }
+        if (r.mode === "all") {
+            if (r.values.length > 0 && !r.values.every((v) => v !== null && v !== undefined)) {
+                failures.push(`'${path}' resolved to null/undefined value(s)`);
+            }
+        }
+        else {
+            if (r.values.length === 0) {
+                failures.push(`'${path}' (any:) resolved to no elements`);
+            }
+            else if (!r.values.some((v) => v !== null && v !== undefined)) {
+                failures.push(`'${path}' (any:) all resolved values are null/undefined`);
+            }
+        }
+    }
+    if (failures.length === 0)
+        return null;
+    return `output_required: ${failures.join("; ")}`;
+}
+export function checkOutputMatches(matches, data) {
+    const failures = [];
+    for (const m of matches) {
+        let compiled;
+        try {
+            compiled = compileMatch(m);
+        }
+        catch (err) {
+            failures.push(`'${m.path}' invalid regex /${m.matches}/: ${err.message}`);
+            continue;
+        }
+        const r = resolvePath(data, m.path);
+        if (!r.ok) {
+            failures.push(`'${m.path}' ${r.reason}`);
+            continue;
+        }
+        if (r.mode === "all") {
+            if (r.values.length === 0)
+                continue;
+            const offenders = [];
+            for (const v of r.values) {
+                const res = applyOperator(compiled, v);
+                if (!res.ok)
+                    offenders.push(formatValue(v));
+            }
+            if (offenders.length > 0) {
+                const opDescription = describeOperator(compiled);
+                failures.push(`'${m.path}' ${opDescription}, got [${offenders.join(", ")}]`);
+            }
+        }
+        else {
+            if (r.values.length === 0) {
+                failures.push(`'${m.path}' (any:) resolved to no elements`);
+                continue;
+            }
+            const anyOk = r.values.some((v) => applyOperator(compiled, v).ok);
+            if (!anyOk) {
+                const opDescription = describeOperator(compiled);
+                failures.push(`'${m.path}' (any:) no element satisfies ${opDescription}`);
+            }
+        }
+    }
+    if (failures.length === 0)
+        return null;
+    return `output_matches: ${failures.join("; ")}`;
+}
+// ─── Evaluator entry point ───────────────────────────────────────────
+/**
+ * Evaluate a `value` rule against the node's `result.data`.
+ *
+ * Returns `{ pass: true }` when every declared check passes, otherwise
+ * `{ pass: false, reasoning: <semicolon-joined failure list> }`.
+ *
+ * Function-rule fields (`*_tool_called`) are ignored even if present.
+ */
+export function evaluateValueRule(rule, data) {
+    const failures = [];
+    if (rule.output_required && rule.output_required.length > 0) {
+        const e = checkOutputRequired(rule.output_required, data);
+        if (e)
+            failures.push(e);
+    }
+    if (rule.output_matches && rule.output_matches.length > 0) {
+        const e = checkOutputMatches(rule.output_matches, data);
+        if (e)
+            failures.push(e);
+    }
+    if (failures.length === 0)
+        return { pass: true };
+    return { pass: false, reasoning: failures.join("; ") };
+}

package/dist/executor.js

@@ -14,7 +14,7 @@
  */
 import { consoleLogger } from "./types.js";
 import { resolveSources } from "./source-resolver.js";
-import { evaluateVerify } from "./verify.js";
+import { evaluateAll, aggregateEval } from "./eval/index.js";
 import { evaluateRequires } from "./requires.js";
 import { buildRetryPreamble } from "./retry.js";
 import { buildToolAliases } from "./skills/index.js";
@@ -38,10 +38,15 @@ export async function execute(workflow, input, options) {
     const nodeRunCounts = new Map(); // node → times executed
     const trace = { steps: [], edges: [], sources: {} };
     validate(workflow, skills);
-    // Build a verify-time alias table from the loaded skills. Each skill owns
+    // Build an eval-time alias table from the loaded skills. Each skill owns
     // its own mapping between skill-tool names and equivalent MCP names. Core
-    // stays vendor-neutral — this call just unions what the skills declare.
+    // stays vendor-neutral; this call just unions what the skills declare.
     const toolAliases = buildToolAliases(skills.values(), logger);
+    // Soft-cap warning: count declared judge evaluators across all nodes and
+    // warn if the total exceeds workflow.judge_budget (default 50). Spec says
+    // this is informational, not a hard runtime cap; users hit the warning
+    // when they unintentionally fan out judges (e.g. 10 judges × 50 nodes).
+    warnOnJudgeBudget(workflow, logger);
     // ── Source resolution phase ─────────────────────────────────────
     // Resolve all Source values (node instructions + rules/context at every
     // level) into plain text before the main execution loop. This eagerly
@@ -98,10 +103,14 @@ export async function execute(workflow, input, options) {
             throw new Error(`Node "${currentId}" requires skills [${node.skills.join(", ")}] but none are configured. ` +
                 `Set the required environment variables and try again.`);
         }
-        // Build context: input + all prior node results
+        // Build context: input + all prior node results.
+        // Each prior node entry is the node's `data` augmented with `evals`
+        // (a Record<name, EvalResult> for downstream lookup like
+        // `priorNode.evals.tests_run_clean.pass`). When `data` already has an
+        // `evals` key, the data field wins (back-compat with existing workflows).
         const context = {
             input,
-            ...Object.fromEntries([...results.entries()].map(([k, v]) => [k, v.data])),
+            ...Object.fromEntries([...results.entries()].map(([k, v]) => [k, buildPriorNodeContext(v)])),
         };
         // Pre-condition gate: evaluate `requires` against the cross-node context
         // BEFORE invoking the LLM. Failure either marks the node failed (on_fail
@@ -145,7 +154,7 @@ export async function execute(workflow, input, options) {
         const effectiveRules = assembleCascaded("rules", currentId, workflow.nodes[currentId], inputRules.length, (workflow.rules ?? []).length, resolvedSources);
         const effectiveContext = assembleCascaded("context", currentId, workflow.nodes[currentId], inputContext.length, (workflow.context ?? []).length, resolvedSources);
         const instruction = buildNodeInstruction(resolvedInstruction, effectiveRules, effectiveContext, input, skillInstructions);
-        // Run Claude on this node, with optional verify-failure retry loop.
+        // Run Claude on this node, with optional eval-failure retry loop.
         let attempt = 0;
         let result;
         let currentInstruction = instruction;
@@ -161,25 +170,34 @@ export async function execute(workflow, input, options) {
                     safeObserve(observer, { type: "node:progress", node: currentId, message }, logger);
                 },
             });
-            // Retry only triggers on verify failure — bail on tool/API errors.
+            // Retry only triggers on eval failure. Bail on tool/API errors.
             if (result.status !== "success")
                 break;
-            const verifyError = evaluateVerify(node.verify, result, toolAliases);
-            if (!verifyError)
+            const evalResults = await evaluateAll(node.eval, result, {
+                aliases: toolAliases,
+                claude,
+                node,
+                workflow,
+            });
+            result.evals = evalResults;
+            const outcome = aggregateEval(evalResults, node.eval_policy ?? "all_pass");
+            if (outcome.pass)
                 break;
-            // Apply verify failure to the result.
+            // Apply eval failure to the result.
             result.status = "failed";
-            result.data = { ...result.data, error: verifyError };
+            result.data = { ...result.data, error: outcome.error };
             if (!retry || attempt >= retry.max) {
-                logger.warn(`  verify failed: ${verifyError}`, { node: currentId });
+                logger.warn(`  eval failed: ${outcome.error}`, { node: currentId });
                 break;
             }
             // Build retry preamble + record attempt in trace.
             trace.steps.push({ node: currentId, status: "failed", iteration, retryAttempt: attempt });
-            logger.warn(`  verify failed (attempt ${attempt + 1}/${retry.max + 1}): ${verifyError}`, { node: currentId });
+            logger.warn(`  eval failed (attempt ${attempt + 1}/${retry.max + 1}): ${outcome.error}`, {
+                node: currentId,
+            });
             const preamble = await buildRetryPreamble({
                 retry,
-                verifyError,
+                evalFailures: outcome.failures,
                 toolCalls: result.toolCalls,
                 nodeInstruction: resolvedInstruction,
                 claude,
@@ -187,7 +205,7 @@ export async function execute(workflow, input, options) {
                 context,
             });
             currentInstruction = `${preamble}\n\n---\n\n${instruction}`;
-            safeObserve(observer, { type: "node:retry", node: currentId, attempt: attempt + 1, reason: verifyError, preamble }, logger);
+            safeObserve(observer, { type: "node:retry", node: currentId, attempt: attempt + 1, reason: outcome.error ?? "eval failed", preamble }, logger);
             attempt++;
         }
         results.set(currentId, result);
@@ -249,6 +267,54 @@ function buildNodeInstruction(baseInstruction, effectiveRules, effectiveContext,
  * - Object form → use its `only` and `sources`
  * - undefined → empty additive
  */
+const DEFAULT_JUDGE_BUDGET = 50;
+/**
+ * Soft cap on judge calls per workflow run. Logs a warn when the count of
+ * declared `kind: judge` evaluators exceeds the configured budget. This is
+ * a load-time signal only; the executor does not refuse to run.
+ *
+ * Each judge counts once per node visit, but the executor counts judges
+ * once at declaration time (a fan-out of repeated visits via cycles is the
+ * author's responsibility, not core's).
+ */
+function warnOnJudgeBudget(workflow, logger) {
+    const budget = workflow.judge_budget ?? DEFAULT_JUDGE_BUDGET;
+    let total = 0;
+    for (const node of Object.values(workflow.nodes)) {
+        for (const evaluator of node.eval ?? []) {
+            if (evaluator.kind === "judge")
+                total++;
+        }
+    }
+    if (total > budget) {
+        logger.warn(`Workflow declares ${total} judge evaluators across all nodes (budget: ${budget}). ` +
+            `Consider reducing the count or raising 'judge_budget' on the workflow. ` +
+            `Each judge adds one model call per node attempt.`, { judgeCount: total, judgeBudget: budget });
+    }
+}
+/**
+ * Build the context-map entry for a single prior node.
+ *
+ * Spec contract (https://spec.sweny.ai/nodes/#evalresult-type): downstream
+ * nodes can read `priorNode.evals.<name>.pass` via the natural lookup path.
+ * To honor this without breaking back-compat with workflows that read
+ * `priorNode.<dataField>` directly, we spread `data` first and then attach
+ * an `evals` namespace keyed by evaluator name. If the agent's structured
+ * output happens to include a literal `evals` field, that field wins (and
+ * the lookup degrades to "evals not available" for downstream readers).
+ */
+function buildPriorNodeContext(result) {
+    const data = (result.data ?? {});
+    const evals = result.evals ?? [];
+    if (evals.length === 0)
+        return data;
+    const evalsByName = {};
+    for (const e of evals) {
+        evalsByName[e.name] = e;
+    }
+    // Spread evals first so a data-side `evals` key shadows it (back-compat).
+    return { evals: evalsByName, ...data };
+}
 function nodeSourcesToArray(ns) {
     if (!ns)
         return { sources: [], only: false };
@@ -443,11 +509,12 @@ async function resolveNext(workflow, current, results, input, claude, observer,
     // Check for a default (unconditional) edge among conditionals
     const defaultEdge = outEdges.find((e) => !e.when);
     const conditionalEdges = outEdges.filter((e) => e.when);
-    // Claude evaluates which condition matches — include input so conditions
-    // can reference workflow-level flags like dryRun
+    // Claude evaluates which condition matches. Include input so conditions
+    // can reference workflow-level flags like dryRun, and expose evals so
+    // routing edges can read `priorNode.evals.X.pass`.
     const context = {
         input,
-        ...Object.fromEntries([...results.entries()].map(([k, v]) => [k, v.data])),
+        ...Object.fromEntries([...results.entries()].map(([k, v]) => [k, buildPriorNodeContext(v)])),
     };
     const choices = conditionalEdges.map((e) => ({
         id: e.to,

package/dist/requires.js

@@ -1,9 +1,9 @@
 // ─── Requires: pre-condition gate ───────────────────────────────────
 //
 // Evaluates `node.requires` BEFORE the LLM runs. Same path grammar and
-// resolver as verify — the only differences are the data root (cross-node
+// resolver as eval. The only differences are the data root (cross-node
 // context map instead of result.data) and the error prefix.
-import { checkOutputRequired, checkOutputMatches } from "./verify.js";
+import { checkOutputRequired, checkOutputMatches } from "./eval/value.js";
 /**
  * Evaluate a node's `requires` block against the cross-node context map.
  * Returns null when all checks pass (or when `requires` is undefined),

package/dist/retry.d.ts

@@ -1,7 +1,12 @@
-import type { Claude, NodeRetry, ToolCall, Logger } from "./types.js";
+import type { Claude, EvalResult, NodeRetry, ToolCall, Logger } from "./types.js";
 export interface BuildRetryPreambleOptions {
     retry: NodeRetry;
-    verifyError: string;
+    /**
+     * Failing evaluator results from the previous attempt. Used both for the
+     * structured failure list shown to the agent and as input to autonomous
+     * reflection.
+     */
+    evalFailures: EvalResult[];
     toolCalls: ToolCall[];
     nodeInstruction: string;
     claude: Claude;
@@ -12,7 +17,7 @@ export interface BuildRetryPreambleOptions {
  * Build the retry preamble for a single retry attempt.
  *
  * Resolution order:
- *   1. retry.instruction is a string         → static preamble + verify error
+ *   1. retry.instruction is a string         → static preamble + structured failure list
  *   2. retry.instruction is { auto: true }   → claude.ask(default prompt)
  *   3. retry.instruction is { reflect: "..." } → claude.ask(author prompt)
  *   4. Otherwise (omitted)                   → default preamble
@@ -21,3 +26,12 @@ export interface BuildRetryPreambleOptions {
  * default static preamble and logs a warning.
  */
 export declare function buildRetryPreamble(opts: BuildRetryPreambleOptions): Promise<string>;
+/**
+ * Render a list of failing evaluator results as a structured bullet list
+ * suitable for prepending to a retry instruction.
+ *
+ * Each entry takes the form `- name (kind): reasoning`. The list is the
+ * same shape regardless of whether the originating policy is `all_pass`,
+ * `any_pass`, or `weighted`.
+ */
+export declare function formatEvalFailures(failures: EvalResult[]): string;

package/dist/retry.js

@@ -1,18 +1,18 @@
-// ─── Retry: preamble construction for verify-failure self-healing ──
+// ─── Retry: preamble construction for eval-failure self-healing ──────
 //
 // Builds the preamble that gets prepended to the node instruction on retry.
 // In autonomous modes ({ auto: true } or { reflect: ... }), this calls
 // claude.ask to generate a diagnosis. If reflection fails or returns empty,
-// falls back to the default static preamble — reflection failure never
+// falls back to the default static preamble. Reflection failure never
 // escalates to a workflow failure.
-import { isErrorOutput } from "./verify.js";
+import { isErrorOutput } from "./eval/function.js";
 const DEFAULT_REFLECTION_PROMPT = "Briefly diagnose the failure and state your strategy for the retry. Keep your response to 2-4 sentences.";
-const DEFAULT_PREAMBLE_HEADER = "## Previous attempt failed verification";
+const DEFAULT_PREAMBLE_HEADER = "## Previous attempt failed evaluation";
 /**
  * Build the retry preamble for a single retry attempt.
  *
  * Resolution order:
- *   1. retry.instruction is a string         → static preamble + verify error
+ *   1. retry.instruction is a string         → static preamble + structured failure list
  *   2. retry.instruction is { auto: true }   → claude.ask(default prompt)
  *   3. retry.instruction is { reflect: "..." } → claude.ask(author prompt)
  *   4. Otherwise (omitted)                   → default preamble
@@ -21,19 +21,20 @@ const DEFAULT_PREAMBLE_HEADER = "## Previous attempt failed verification";
  * default static preamble and logs a warning.
  */
 export async function buildRetryPreamble(opts) {
-    const { retry, verifyError, toolCalls, nodeInstruction, claude, logger, context } = opts;
+    const { retry, evalFailures, toolCalls, nodeInstruction, claude, logger, context } = opts;
+    const failureList = formatEvalFailures(evalFailures);
     const inst = retry.instruction;
     if (typeof inst === "string") {
-        return `## Retry guidance\n\n${inst}\n\n${DEFAULT_PREAMBLE_HEADER}\n\n${verifyError}`;
+        return `## Retry guidance\n\n${inst}\n\n${DEFAULT_PREAMBLE_HEADER}\n\n${failureList}`;
     }
     if (inst && typeof inst === "object") {
         const reflectPrompt = "reflect" in inst && typeof inst.reflect === "string" ? inst.reflect : DEFAULT_REFLECTION_PROMPT;
-        const askInstruction = buildReflectionPrompt(reflectPrompt, nodeInstruction, verifyError, toolCalls);
+        const askInstruction = buildReflectionPrompt(reflectPrompt, nodeInstruction, failureList, toolCalls);
         try {
             const diagnosis = await claude.ask({ instruction: askInstruction, context });
             const trimmed = diagnosis.trim();
             if (trimmed.length > 0) {
-                return `## Reflection on previous attempt\n\n${trimmed}\n\n${DEFAULT_PREAMBLE_HEADER}\n\n${verifyError}`;
+                return `## Reflection on previous attempt\n\n${trimmed}\n\n${DEFAULT_PREAMBLE_HEADER}\n\n${failureList}`;
             }
             logger.warn("Retry reflection returned empty response; falling back to default preamble.");
         }
@@ -42,17 +43,31 @@ export async function buildRetryPreamble(opts) {
             logger.warn(`Retry reflection threw: ${msg}; falling back to default preamble.`);
         }
     }
-    return defaultPreamble(verifyError);
+    return defaultPreamble(failureList);
 }
-function defaultPreamble(verifyError) {
-    return `${DEFAULT_PREAMBLE_HEADER}\n\n${verifyError}\n\nFix the issue and try again.`;
+/**
+ * Render a list of failing evaluator results as a structured bullet list
+ * suitable for prepending to a retry instruction.
+ *
+ * Each entry takes the form `- name (kind): reasoning`. The list is the
+ * same shape regardless of whether the originating policy is `all_pass`,
+ * `any_pass`, or `weighted`.
+ */
+export function formatEvalFailures(failures) {
+    if (failures.length === 0)
+        return "(no failing evaluators reported)";
+    return failures.map((f) => `  - ${f.name} (${f.kind}): ${f.reasoning ?? "no reasoning"}`).join("\n");
+}
+function defaultPreamble(failureList) {
+    return `${DEFAULT_PREAMBLE_HEADER}\n\n${failureList}\n\nFix the issue and try again.`;
 }
-function buildReflectionPrompt(reflectPrompt, nodeInstruction, verifyError, toolCalls) {
+function buildReflectionPrompt(reflectPrompt, nodeInstruction, failureList, toolCalls) {
     const summary = summarizeToolCalls(toolCalls);
     return [
         `You attempted to: ${nodeInstruction}`,
         "",
-        `Verification failed with: ${verifyError}`,
+        `Evaluation failed:`,
+        failureList,
         "",
         `You called these tools during the failed attempt:`,
         summary,