@sweny-ai/core 0.1.84 → 0.1.86

package/dist/claude.d.ts

@@ -56,6 +56,7 @@ export declare class ClaudeClient implements Claude {
     ask(opts: {
         instruction: string;
         context: Record<string, unknown>;
+        model?: string;
     }): Promise<string>;
 }
 /**

package/dist/claude.js

@@ -158,8 +158,8 @@ export class ClaudeClient {
                             const parsed = parseToolResultContent(block.content);
                             call.output = isError ? { error: parsed } : parsed;
                             // Surface tool errors in the CI log stream at warn level so
-                            // postmortems don't have to reconstruct them from verify-time
-                            // tool-call summaries. Without this, "verify failed: tool X
+                            // postmortems don't have to reconstruct them from eval-time
+                            // tool-call summaries. Without this, "eval failed: tool X
                             // did not succeed" never answers the WHY question, because
                             // the error body is buried in the per-call output captured
                             // only for in-memory verify evaluation.
@@ -263,7 +263,7 @@ export class ClaudeClient {
         return validIds[0];
     }
     async ask(opts) {
-        const { instruction, context } = opts;
+        const { instruction, context, model } = opts;
         const prompt = [
             instruction,
             Object.keys(context).length > 0 ? `\nContext:\n\`\`\`json\n${JSON.stringify(context, null, 2)}\n\`\`\`` : "",
@@ -272,6 +272,7 @@ export class ClaudeClient {
             .join("\n");
         const env = this.buildEnv();
         let response = "";
+        const effectiveModel = model ?? this.model;
         try {
             const stream = query({
                 prompt,
@@ -282,7 +283,7 @@ export class ClaudeClient {
                     permissionMode: "bypassPermissions",
                     allowDangerouslySkipPermissions: true,
                     stderr: (data) => this.logger.debug(`[claude-code] ${data}`),
-                    ...(this.model ? { model: this.model } : {}),
+                    ...(effectiveModel ? { model: effectiveModel } : {}),
                 },
             });
             for await (const message of stream) {

package/dist/eval/function.d.ts

@@ -0,0 +1,25 @@
+import type { EvaluatorRule, ToolCall } from "../types.js";
+/**
+ * Legacy output-shape heuristic: a tool call "succeeded" when its output
+ * does not carry a non-null `error` field. Kept as a fallback for ToolCall
+ * records that predate the explicit `status` field (hand-constructed in
+ * tests, older runtimes). The Claude runtime now sets `status` directly.
+ */
+export declare function isErrorOutput(output: unknown): boolean;
+export type ToolAliases = ReadonlyMap<string, ReadonlySet<string>>;
+export declare function checkAnyToolCalled(required: string[], toolCalls: ToolCall[], aliases?: ToolAliases): string | null;
+export declare function checkAllToolsCalled(required: string[], toolCalls: ToolCall[], aliases?: ToolAliases): string | null;
+export declare function checkNoToolCalled(forbidden: string[], toolCalls: ToolCall[], aliases?: ToolAliases): string | null;
+/**
+ * Evaluate a `function` rule against the node's tool-call trace.
+ *
+ * Returns `{ pass: true }` when every declared check passes, otherwise
+ * `{ pass: false, reasoning: <semicolon-joined failure list> }`.
+ *
+ * Value-rule fields (`output_required` / `output_matches`) are ignored
+ * even if present.
+ */
+export declare function evaluateFunctionRule(rule: EvaluatorRule, toolCalls: ToolCall[], aliases?: ToolAliases): {
+    pass: boolean;
+    reasoning?: string;
+};

package/dist/eval/function.js

@@ -0,0 +1,106 @@
+// ─── Eval: function (trace-shape) evaluator ──────────────────────────
+//
+// Operates on the node's tool-call trace. Accepts an alias map so a workflow
+// rule can reference a canonical tool name and match against an equivalent
+// MCP tool name. Each skill owns its own alias map; core stays vendor-neutral.
+// ─── Tool-call success heuristic ─────────────────────────────────────
+/**
+ * Legacy output-shape heuristic: a tool call "succeeded" when its output
+ * does not carry a non-null `error` field. Kept as a fallback for ToolCall
+ * records that predate the explicit `status` field (hand-constructed in
+ * tests, older runtimes). The Claude runtime now sets `status` directly.
+ */
+export function isErrorOutput(output) {
+    if (!output || typeof output !== "object")
+        return false;
+    const err = output.error;
+    return err !== undefined && err !== null && err !== false;
+}
+/**
+ * A tool call is considered successful when:
+ *   - explicit `status === "success"`, OR
+ *   - no explicit status AND output does not look like an error.
+ *
+ * `status` is authoritative. It's the only signal we have for external MCP
+ * tools whose structured output isn't exposed to the workflow runtime.
+ */
+function didSucceed(c) {
+    if (c.status === "error")
+        return false;
+    if (c.status === "success")
+        return true;
+    return !isErrorOutput(c.output);
+}
+function succeededTools(toolCalls) {
+    const names = new Set();
+    for (const c of toolCalls) {
+        if (didSucceed(c))
+            names.add(c.tool);
+    }
+    return names;
+}
+function calledList(toolCalls) {
+    return toolCalls.map((c) => c.tool).join(", ") || "none";
+}
+function expandAliases(name, aliases) {
+    return aliases?.get(name) ?? new Set([name]);
+}
+function anyAliasIn(name, set, aliases) {
+    for (const alias of expandAliases(name, aliases)) {
+        if (set.has(alias))
+            return true;
+    }
+    return false;
+}
+// ─── Public check helpers ────────────────────────────────────────────
+export function checkAnyToolCalled(required, toolCalls, aliases) {
+    const succeeded = succeededTools(toolCalls);
+    if (required.some((t) => anyAliasIn(t, succeeded, aliases)))
+        return null;
+    return `any_tool_called: required one of [${required.join(", ")}] to succeed, called: [${calledList(toolCalls)}]`;
+}
+export function checkAllToolsCalled(required, toolCalls, aliases) {
+    const succeeded = succeededTools(toolCalls);
+    const missing = required.filter((t) => !anyAliasIn(t, succeeded, aliases));
+    if (missing.length === 0)
+        return null;
+    return `all_tools_called: missing successful calls to [${missing.join(", ")}], called: [${calledList(toolCalls)}]`;
+}
+export function checkNoToolCalled(forbidden, toolCalls, aliases) {
+    const calledNames = new Set(toolCalls.map((c) => c.tool));
+    const violated = forbidden.filter((t) => anyAliasIn(t, calledNames, aliases));
+    if (violated.length === 0)
+        return null;
+    return `no_tool_called: forbidden tools were invoked: [${violated.join(", ")}], called: [${calledList(toolCalls)}]`;
+}
+// ─── Evaluator entry point ───────────────────────────────────────────
+/**
+ * Evaluate a `function` rule against the node's tool-call trace.
+ *
+ * Returns `{ pass: true }` when every declared check passes, otherwise
+ * `{ pass: false, reasoning: <semicolon-joined failure list> }`.
+ *
+ * Value-rule fields (`output_required` / `output_matches`) are ignored
+ * even if present.
+ */
+export function evaluateFunctionRule(rule, toolCalls, aliases) {
+    const failures = [];
+    if (rule.any_tool_called && rule.any_tool_called.length > 0) {
+        const e = checkAnyToolCalled(rule.any_tool_called, toolCalls, aliases);
+        if (e)
+            failures.push(e);
+    }
+    if (rule.all_tools_called && rule.all_tools_called.length > 0) {
+        const e = checkAllToolsCalled(rule.all_tools_called, toolCalls, aliases);
+        if (e)
+            failures.push(e);
+    }
+    if (rule.no_tool_called && rule.no_tool_called.length > 0) {
+        const e = checkNoToolCalled(rule.no_tool_called, toolCalls, aliases);
+        if (e)
+            failures.push(e);
+    }
+    if (failures.length === 0)
+        return { pass: true };
+    return { pass: false, reasoning: failures.join("; ") };
+}

package/dist/eval/index.d.ts

@@ -0,0 +1,41 @@
+import type { Claude, EvalPolicy, EvalResult, Evaluator, NodeResult, Workflow, Node } from "../types.js";
+import { type ToolAliases } from "./function.js";
+export type { ToolAliases } from "./function.js";
+export interface EvaluateAllOptions {
+    /** Tool-name aliases from the loaded skills. Used by `function` evaluators only. */
+    aliases?: ToolAliases;
+    /** Optional Claude client. Required when any evaluator has kind: "judge". */
+    claude?: Claude;
+    /** The node being evaluated. Used to resolve per-node `judge_model` overrides. */
+    node?: Node;
+    /** The workflow being executed. Used to resolve workflow-level `judge_model` overrides. */
+    workflow?: Workflow;
+}
+/**
+ * Run every declared evaluator against the node's result and return one
+ * {@link EvalResult} per evaluator, in declaration order.
+ *
+ * The function never throws on a failing rule (failures become
+ * `{ pass: false, reasoning }` results). It DOES throw if the underlying
+ * judge implementation throws (e.g. v1.0 judge stub) so the executor can
+ * surface the error clearly at the call site.
+ */
+export declare function evaluateAll(evaluators: Evaluator[] | undefined, result: NodeResult, opts?: EvaluateAllOptions): Promise<EvalResult[]>;
+export interface AggregateOutcome {
+    /** Whether the node passed under the configured policy. */
+    pass: boolean;
+    /**
+     * Structured failure message for the executor when `pass` is false.
+     * Format: header line + one bulleted line per failing evaluator.
+     * Suitable as input to {@link buildRetryPreamble}.
+     */
+    error?: string;
+    /** Direct access to the failed evaluator results, in declaration order. */
+    failures: EvalResult[];
+}
+/**
+ * Collapse a list of {@link EvalResult}s into a single pass/fail under the
+ * node's `eval_policy`. v1 implements `all_pass`. Other policies are
+ * accepted at parse time but rejected here so users get a clear error.
+ */
+export declare function aggregateEval(results: EvalResult[], policy?: EvalPolicy): AggregateOutcome;

package/dist/eval/index.js

@@ -0,0 +1,91 @@
+// ─── Eval: dispatcher + aggregation ──────────────────────────────────
+//
+// `evaluateAll(...)` runs every declared evaluator and returns a structured
+// list of EvalResults. `aggregateEval(...)` collapses that list into a
+// single pass/fail under the node's eval_policy. v1 implements `all_pass`.
+import { evaluateValueRule } from "./value.js";
+import { evaluateFunctionRule } from "./function.js";
+import { evaluateJudge } from "./judge.js";
+/** Reasoning cap, matching the spec's ~500-character ceiling. */
+const MAX_REASONING_LENGTH = 500;
+/** Default judge model when neither evaluator nor node nor workflow specify one. */
+const DEFAULT_JUDGE_MODEL = "claude-haiku-4-5";
+function capReasoning(reasoning) {
+    if (!reasoning)
+        return undefined;
+    if (reasoning.length <= MAX_REASONING_LENGTH)
+        return reasoning;
+    return reasoning.slice(0, MAX_REASONING_LENGTH - 1) + "…";
+}
+function resolveJudgeModel(evaluator, node, workflow) {
+    return evaluator.model ?? node?.judge_model ?? workflow?.judge_model ?? DEFAULT_JUDGE_MODEL;
+}
+/**
+ * Run every declared evaluator against the node's result and return one
+ * {@link EvalResult} per evaluator, in declaration order.
+ *
+ * The function never throws on a failing rule (failures become
+ * `{ pass: false, reasoning }` results). It DOES throw if the underlying
+ * judge implementation throws (e.g. v1.0 judge stub) so the executor can
+ * surface the error clearly at the call site.
+ */
+export async function evaluateAll(evaluators, result, opts = {}) {
+    if (!evaluators || evaluators.length === 0)
+        return [];
+    const out = [];
+    for (const e of evaluators) {
+        if (e.kind === "value") {
+            const verdict = evaluateValueRule(e.rule ?? {}, result.data);
+            out.push({
+                name: e.name,
+                kind: "value",
+                pass: verdict.pass,
+                reasoning: capReasoning(verdict.reasoning),
+            });
+        }
+        else if (e.kind === "function") {
+            const verdict = evaluateFunctionRule(e.rule ?? {}, result.toolCalls, opts.aliases);
+            out.push({
+                name: e.name,
+                kind: "function",
+                pass: verdict.pass,
+                reasoning: capReasoning(verdict.reasoning),
+            });
+        }
+        else if (e.kind === "judge") {
+            if (!opts.claude) {
+                throw new Error(`evaluator '${e.name}' is kind: judge but no Claude client was provided to evaluateAll`);
+            }
+            const model = resolveJudgeModel(e, opts.node, opts.workflow);
+            const verdict = await evaluateJudge(e, result, opts.claude, { model });
+            out.push({
+                name: e.name,
+                kind: "judge",
+                pass: verdict.pass,
+                reasoning: capReasoning(verdict.reasoning),
+            });
+        }
+    }
+    return out;
+}
+/**
+ * Collapse a list of {@link EvalResult}s into a single pass/fail under the
+ * node's `eval_policy`. v1 implements `all_pass`. Other policies are
+ * accepted at parse time but rejected here so users get a clear error.
+ */
+export function aggregateEval(results, policy = "all_pass") {
+    if (results.length === 0)
+        return { pass: true, failures: [] };
+    if (policy === "all_pass") {
+        const failures = results.filter((r) => !r.pass);
+        if (failures.length === 0)
+            return { pass: true, failures: [] };
+        const lines = failures.map((f) => `  - ${f.name} (${f.kind}): ${f.reasoning ?? "no reasoning"}`);
+        return {
+            pass: false,
+            error: `eval failed (policy: all_pass):\n${lines.join("\n")}`,
+            failures,
+        };
+    }
+    throw new Error(`eval_policy '${policy}' is reserved in v1.0 but not yet implemented; use 'all_pass' (default) for v1.0 workflows`);
+}

package/dist/eval/judge.d.ts

@@ -0,0 +1,43 @@
+import type { Claude, Evaluator, NodeResult } from "../types.js";
+export interface JudgeOptions {
+    /** Resolved judge model (evaluator > node > workflow > package default). */
+    model: string;
+}
+interface ParsedResponse {
+    verdict: string | null;
+    reasoning: string | null;
+}
+/**
+ * Build the structured judge prompt.
+ *
+ * The prompt format is fixed so the parser can rely on it. Model authors
+ * MUST NOT change the section markers without updating {@link parseJudgeResponse}
+ * in lockstep.
+ */
+export declare function buildJudgePrompt(evaluator: Evaluator, result: NodeResult): string;
+/**
+ * Parse a judge response into `{ verdict, reasoning }`.
+ *
+ * Format expected:
+ *   VERDICT: <token>
+ *   REASONING: <text...>
+ *
+ * Lines may have any leading whitespace. Reasoning may span multiple lines
+ * (everything after the `REASONING:` marker is consumed). Returns null
+ * fields when the markers are missing.
+ */
+export declare function parseJudgeResponse(text: string): ParsedResponse;
+/**
+ * Run a single judge evaluator against a node result.
+ *
+ * Resolves a verdict by calling the judge model, parsing VERDICT/REASONING,
+ * and comparing the verdict to `pass_when` (default `yes`). On parse
+ * failure, retries the call once. If the retry also fails to parse,
+ * returns `pass: false` with a "judge parse failure" reasoning instead of
+ * throwing, so a single flaky model response doesn't halt the workflow.
+ */
+export declare function evaluateJudge(evaluator: Evaluator, result: NodeResult, claude: Claude, opts: JudgeOptions): Promise<{
+    pass: boolean;
+    reasoning?: string;
+}>;
+export {};

package/dist/eval/judge.js

@@ -0,0 +1,142 @@
+// ─── Eval: judge (LLM-as-judge) evaluator ────────────────────────────
+//
+// Calls a small Claude model with the node's data, the node's tool-call
+// trace, and the author's rubric. Parses a verdict token and a short
+// reasoning paragraph from a structured response.
+//
+// Cost: one model call per judge evaluator per node attempt. Judges are
+// expected to use a small fast model (default `claude-haiku-4-5`); the
+// reasoning is capped at ~500 chars before it lands on EvalResult so a
+// chatty judge doesn't bloat the trace.
+//
+// Latency: a judge adds one round-trip per node beyond the main LLM call.
+// For a typical 4-node workflow with 2 judges per node, that's 8 extra
+// calls per run. The workflow-level `judge_budget` (default 50) gives
+// authors a load-time warning when the total stretches past expected.
+/**
+ * Build the structured judge prompt.
+ *
+ * The prompt format is fixed so the parser can rely on it. Model authors
+ * MUST NOT change the section markers without updating {@link parseJudgeResponse}
+ * in lockstep.
+ */
+export function buildJudgePrompt(evaluator, result) {
+    const passWhen = evaluator.pass_when ?? "yes";
+    const failWhen = passWhen === "yes" ? "no" : `not_${passWhen}`;
+    return [
+        `You are a judge evaluator. Read the rubric, the node's result data, and`,
+        `the tool-call trace, then decide whether the result satisfies the rubric.`,
+        ``,
+        `## Rubric`,
+        ``,
+        evaluator.rubric ?? "(no rubric provided)",
+        ``,
+        `## Node result data`,
+        ``,
+        "```json",
+        JSON.stringify(result.data ?? {}, null, 2),
+        "```",
+        ``,
+        `## Tool calls`,
+        ``,
+        summarizeToolCalls(result.toolCalls ?? []),
+        ``,
+        `## Output format`,
+        ``,
+        `Respond with EXACTLY two lines, in this order, and nothing else:`,
+        ``,
+        `VERDICT: <${passWhen}|${failWhen}>`,
+        `REASONING: <one paragraph, max 3 sentences, explaining the verdict>`,
+        ``,
+        `If the rubric passes vacuously (e.g. a conditional clause does not`,
+        `apply to this result), respond with VERDICT: ${passWhen} and a brief`,
+        `note in REASONING.`,
+    ].join("\n");
+}
+function summarizeToolCalls(calls) {
+    if (calls.length === 0)
+        return "(no tools were called)";
+    return calls
+        .map((c) => {
+        const status = c.status ?? (looksLikeError(c.output) ? "error" : "success");
+        const inputPreview = preview(c.input);
+        const outputPreview = preview(c.output);
+        return `- ${c.tool} [${status}] input=${inputPreview} output=${outputPreview}`;
+    })
+        .join("\n");
+}
+function looksLikeError(output) {
+    if (!output || typeof output !== "object")
+        return false;
+    const err = output.error;
+    return err !== undefined && err !== null && err !== false;
+}
+/** Compact JSON preview. Caps long values to keep the prompt short. */
+function preview(v) {
+    let s;
+    try {
+        s = JSON.stringify(v);
+    }
+    catch {
+        s = String(v);
+    }
+    if (!s)
+        return "<empty>";
+    return s.length > 200 ? s.slice(0, 200) + "…" : s;
+}
+/**
+ * Parse a judge response into `{ verdict, reasoning }`.
+ *
+ * Format expected:
+ *   VERDICT: <token>
+ *   REASONING: <text...>
+ *
+ * Lines may have any leading whitespace. Reasoning may span multiple lines
+ * (everything after the `REASONING:` marker is consumed). Returns null
+ * fields when the markers are missing.
+ */
+export function parseJudgeResponse(text) {
+    const verdictMatch = /VERDICT\s*:\s*(\S+)/i.exec(text);
+    const reasoningMatch = /REASONING\s*:\s*([\s\S]*)$/i.exec(text);
+    const verdict = verdictMatch ? verdictMatch[1].trim().toLowerCase() : null;
+    const reasoning = reasoningMatch ? reasoningMatch[1].trim() : null;
+    return { verdict, reasoning };
+}
+const PARSE_FAILURE_REASONING = "judge parse failure: model response did not contain a parseable VERDICT line";
+/**
+ * Run a single judge evaluator against a node result.
+ *
+ * Resolves a verdict by calling the judge model, parsing VERDICT/REASONING,
+ * and comparing the verdict to `pass_when` (default `yes`). On parse
+ * failure, retries the call once. If the retry also fails to parse,
+ * returns `pass: false` with a "judge parse failure" reasoning instead of
+ * throwing, so a single flaky model response doesn't halt the workflow.
+ */
+export async function evaluateJudge(evaluator, result, claude, opts) {
+    const passWhen = (evaluator.pass_when ?? "yes").toLowerCase();
+    const prompt = buildJudgePrompt(evaluator, result);
+    for (let attempt = 0; attempt < 2; attempt++) {
+        let response;
+        try {
+            response = await claude.ask({
+                instruction: prompt,
+                context: {},
+                model: opts.model,
+            });
+        }
+        catch {
+            // Treat throws as parse failure for retry purposes; the second attempt
+            // either succeeds or surfaces the same path through the same channel.
+            continue;
+        }
+        const parsed = parseJudgeResponse(response);
+        if (parsed.verdict === null)
+            continue;
+        const pass = parsed.verdict === passWhen;
+        return {
+            pass,
+            reasoning: parsed.reasoning ?? undefined,
+        };
+    }
+    return { pass: false, reasoning: PARSE_FAILURE_REASONING };
+}

package/dist/eval/path.d.ts

@@ -0,0 +1,9 @@
+export type Resolution = {
+    ok: true;
+    mode: "all" | "any";
+    values: unknown[];
+} | {
+    ok: false;
+    reason: string;
+};
+export declare function resolvePath(data: unknown, path: string): Resolution;

package/dist/eval/path.js

@@ -0,0 +1,72 @@
+// ─── Eval: path parser + resolver ────────────────────────────────────
+//
+// Shared by `value` evaluators (operate on `result.data`) and the `requires`
+// pre-condition gate (operates on the cross-node context map). The grammar
+// is documented at https://spec.sweny.ai/nodes/#path-grammar.
+const SEGMENT_RE = /^([a-zA-Z_][a-zA-Z0-9_]*)(\[\*\])?$/;
+const PREFIX_RE = /^([a-z]+):/;
+function parsePath(path) {
+    if (path.length === 0)
+        return "empty path";
+    let mode = "all";
+    let body = path;
+    const prefixMatch = PREFIX_RE.exec(path);
+    if (prefixMatch) {
+        const prefix = prefixMatch[1];
+        if (prefix !== "all" && prefix !== "any") {
+            return `unknown prefix '${prefix}' (expected 'all' or 'any')`;
+        }
+        mode = prefix;
+        body = path.slice(prefixMatch[0].length);
+        if (body.length === 0)
+            return "empty path after prefix";
+    }
+    const parts = body.split(".");
+    const segments = [];
+    for (const part of parts) {
+        const m = SEGMENT_RE.exec(part);
+        if (!m)
+            return `malformed segment '${part}'`;
+        segments.push({ name: m[1], wildcard: m[2] === "[*]" });
+    }
+    return { mode, segments };
+}
+export function resolvePath(data, path) {
+    const parsed = parsePath(path);
+    if (typeof parsed === "string")
+        return { ok: false, reason: parsed };
+    let current = [data];
+    for (const seg of parsed.segments) {
+        const next = [];
+        for (const v of current) {
+            if (v === null)
+                return { ok: false, reason: `null encountered before segment '${seg.name}'` };
+            if (typeof v !== "object") {
+                return { ok: false, reason: `expected object before segment '${seg.name}', got ${typeof v}` };
+            }
+            const obj = v;
+            if (!(seg.name in obj)) {
+                return { ok: false, reason: `missing segment '${seg.name}'` };
+            }
+            const child = obj[seg.name];
+            if (seg.wildcard) {
+                if (!Array.isArray(child)) {
+                    return {
+                        ok: false,
+                        reason: `expected array at '${seg.name}', got ${child === null ? "null" : typeof child}`,
+                    };
+                }
+                for (const el of child)
+                    next.push(el);
+            }
+            else {
+                next.push(child);
+            }
+        }
+        current = next;
+        if (current.length === 0) {
+            return { ok: true, mode: parsed.mode, values: [] };
+        }
+    }
+    return { ok: true, mode: parsed.mode, values: current };
+}

package/dist/eval/value.d.ts

@@ -0,0 +1,15 @@
+import type { EvaluatorRule, OutputMatch } from "../types.js";
+export declare function checkOutputRequired(paths: string[], data: unknown): string | null;
+export declare function checkOutputMatches(matches: OutputMatch[], data: unknown): string | null;
+/**
+ * Evaluate a `value` rule against the node's `result.data`.
+ *
+ * Returns `{ pass: true }` when every declared check passes, otherwise
+ * `{ pass: false, reasoning: <semicolon-joined failure list> }`.
+ *
+ * Function-rule fields (`*_tool_called`) are ignored even if present.
+ */
+export declare function evaluateValueRule(rule: EvaluatorRule, data: unknown): {
+    pass: boolean;
+    reasoning?: string;
+};