@nodetool-ai/agents 0.7.0-rc.23 → 0.8.1

package/dist/evals/goal-judge.d.ts

@@ -0,0 +1,51 @@
+/**
+ * LLM judge for end-to-end eval suites: given the goal a run was supposed to
+ * achieve and what the run actually produced, decide whether the goal was met.
+ *
+ * Structural checks (an output exists, it is non-empty, it matches a regex)
+ * cannot tell a real German translation from the English text echoed back, so
+ * the e2e suite pairs them with this judge. The judge answers in plain JSON
+ * rather than through a tool call, so it works identically on providers with
+ * weak or absent tool support, and an unparseable answer is reported as a
+ * judge error instead of being silently scored as a pass.
+ */
+import type { BaseProvider } from "@nodetool-ai/runtime";
+export interface GoalJudgeVerdict {
+    /** The run produced what the goal asked for. */
+    achieved: boolean;
+    /** How well, 0..1 — partial credit for a run that got some of it right. */
+    score: number;
+    /** One or two sentences, quoting the outputs it judged. */
+    reasoning: string;
+    /** Set when the judge could not be reached or its answer was unparseable. */
+    error?: string;
+}
+export interface JudgeGoalOptions {
+    provider: BaseProvider;
+    model: string;
+    /** What the run was asked to do, in the planner's words. */
+    objective: string;
+    /** What "achieved" means for this case, in the case author's words. */
+    goal: string;
+    /** Inputs the run was started with, keyed by input name. */
+    inputs?: Record<string, unknown>;
+    /** What the run produced, keyed by output name. */
+    outputs: Record<string, unknown>;
+    signal?: AbortSignal;
+}
+/** Render one output value for the judge, truncating long or binary payloads. */
+export declare function renderValueForJudge(value: unknown, maxChars?: number): string;
+/**
+ * Parse the judge's answer. Accepts a bare JSON object or one wrapped in prose
+ * or a fenced code block; anything else returns null so the caller can report
+ * a judge error rather than guess a verdict.
+ */
+export declare function parseJudgeVerdict(text: string): GoalJudgeVerdict | null;
+/**
+ * Ask the judge whether a run achieved its goal. Never throws: a provider
+ * failure or an unparseable answer comes back as `achieved: false` with
+ * `error` set, so a broken judge reads as a harness problem in the report
+ * rather than as a failing workflow.
+ */
+export declare function judgeGoalAchievement(opts: JudgeGoalOptions): Promise<GoalJudgeVerdict>;
+//# sourceMappingURL=goal-judge.d.ts.map

package/dist/evals/goal-judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"goal-judge.d.ts","sourceRoot":"","sources":["../../src/evals/goal-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAW,MAAM,sBAAsB,CAAC;AAQlE,MAAM,WAAW,gBAAgB;IAC/B,gDAAgD;IAChD,QAAQ,EAAE,OAAO,CAAC;IAClB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,2DAA2D;IAC3D,SAAS,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,uEAAuE;IACvE,IAAI,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,mDAAmD;IACnD,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB;AAwBD,iFAAiF;AACjF,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,OAAO,EACd,QAAQ,GAAE,MAAwB,GACjC,MAAM,CAQR;AAkBD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CAyBvE;AAcD;;;;;GAKG;AACH,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,gBAAgB,GACrB,OAAO,CAAC,gBAAgB,CAAC,CA0C3B"}

package/dist/evals/goal-judge.js

@@ -0,0 +1,152 @@
+/**
+ * LLM judge for end-to-end eval suites: given the goal a run was supposed to
+ * achieve and what the run actually produced, decide whether the goal was met.
+ *
+ * Structural checks (an output exists, it is non-empty, it matches a regex)
+ * cannot tell a real German translation from the English text echoed back, so
+ * the e2e suite pairs them with this judge. The judge answers in plain JSON
+ * rather than through a tool call, so it works identically on providers with
+ * weak or absent tool support, and an unparseable answer is reported as a
+ * judge error instead of being silently scored as a pass.
+ */
+import { isBoolean, isNumber, isObjectLike, isString } from "../utils/type-guards.js";
+const JUDGE_MAX_TOKENS = 700;
+const MAX_VALUE_CHARS = 4000;
+const SYSTEM_PROMPT = `You grade the output of an automated workflow run.
+
+You are given the objective the workflow was built for, a goal statement
+describing what a successful run must produce, the inputs the run started
+with, and the outputs it produced.
+
+Judge ONLY the outputs against the goal. Ignore how the workflow was built,
+how many steps it took, and whether you would have phrased the result
+differently. Wording, formatting, and length may vary freely — content is what
+counts. An output that is empty, an error message, an unfilled placeholder, or
+an echo of the input where new content was required does NOT achieve the goal.
+
+Answer with a single JSON object and nothing else:
+{"achieved": true|false, "score": 0.0-1.0, "reasoning": "one or two sentences"}
+
+"achieved" is your verdict. "score" is partial credit: 1.0 when the goal is
+fully met, 0.0 when nothing about it was met, in between when part of it was.
+Quote the deciding output in "reasoning".`;
+/** Render one output value for the judge, truncating long or binary payloads. */
+export function renderValueForJudge(value, maxChars = MAX_VALUE_CHARS) {
+    if (value === null || value === undefined)
+        return String(value);
+    if (value instanceof Uint8Array)
+        return `<binary ${value.byteLength} bytes>`;
+    const text = isString(value) ? value : safeStringify(value);
+    return text.length > maxChars
+        ? `${text.slice(0, maxChars)}…[${text.length} chars]`
+        : text;
+}
+function safeStringify(value) {
+    try {
+        return JSON.stringify(value, null, 2) ?? String(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function renderRecord(record) {
+    const entries = Object.entries(record);
+    if (entries.length === 0)
+        return "(none)";
+    return entries
+        .map(([name, value]) => `- ${name}: ${renderValueForJudge(value)}`)
+        .join("\n");
+}
+/**
+ * Parse the judge's answer. Accepts a bare JSON object or one wrapped in prose
+ * or a fenced code block; anything else returns null so the caller can report
+ * a judge error rather than guess a verdict.
+ */
+export function parseJudgeVerdict(text) {
+    const start = text.indexOf("{");
+    const end = text.lastIndexOf("}");
+    if (start === -1 || end <= start)
+        return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(text.slice(start, end + 1));
+    }
+    catch {
+        return null;
+    }
+    if (!isObjectLike(parsed))
+        return null;
+    const obj = parsed;
+    if (!isBoolean(obj.achieved))
+        return null;
+    const rawScore = isNumber(obj.score) ? obj.score : NaN;
+    const score = Number.isFinite(rawScore)
+        ? Math.min(1, Math.max(0, rawScore))
+        : obj.achieved
+            ? 1
+            : 0;
+    return {
+        achieved: obj.achieved,
+        score,
+        reasoning: isString(obj.reasoning) ? obj.reasoning : "(no reasoning given)"
+    };
+}
+function extractText(content) {
+    if (isString(content))
+        return content;
+    if (!Array.isArray(content))
+        return "";
+    return content
+        .map((part) => isObjectLike(part) && "text" in part
+        ? String(part.text ?? "")
+        : "")
+        .join("");
+}
+/**
+ * Ask the judge whether a run achieved its goal. Never throws: a provider
+ * failure or an unparseable answer comes back as `achieved: false` with
+ * `error` set, so a broken judge reads as a harness problem in the report
+ * rather than as a failing workflow.
+ */
+export async function judgeGoalAchievement(opts) {
+    const userPrompt = [
+        `OBJECTIVE:\n${opts.objective}`,
+        `GOAL (what a successful run must produce):\n${opts.goal}`,
+        `INPUTS:\n${renderRecord(opts.inputs ?? {})}`,
+        `OUTPUTS:\n${renderRecord(opts.outputs)}`
+    ].join("\n\n");
+    const messages = [
+        { role: "system", content: SYSTEM_PROMPT },
+        { role: "user", content: userPrompt }
+    ];
+    let reply;
+    try {
+        reply = await opts.provider.generateMessageTraced({
+            messages,
+            model: opts.model,
+            tools: [],
+            maxTokens: JUDGE_MAX_TOKENS,
+            temperature: 0,
+            signal: opts.signal
+        });
+    }
+    catch (err) {
+        return {
+            achieved: false,
+            score: 0,
+            reasoning: "",
+            error: `judge call failed: ${err instanceof Error ? err.message : String(err)}`
+        };
+    }
+    const verdict = parseJudgeVerdict(extractText(reply.content));
+    if (!verdict) {
+        return {
+            achieved: false,
+            score: 0,
+            reasoning: "",
+            error: "judge answer was not parseable JSON"
+        };
+    }
+    return verdict;
+}
+//# sourceMappingURL=goal-judge.js.map

package/dist/evals/goal-judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"goal-judge.js","sourceRoot":"","sources":["../../src/evals/goal-judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EACL,SAAS,EACT,QAAQ,EACR,YAAY,EACZ,QAAQ,EACT,MAAM,yBAAyB,CAAC;AA2BjC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;0CAiBoB,CAAC;AAE3C,iFAAiF;AACjF,MAAM,UAAU,mBAAmB,CACjC,KAAc,EACd,QAAQ,GAAW,eAAe;IAElC,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAChE,IAAI,KAAK,YAAY,UAAU;QAAE,OAAO,WAAW,KAAK,CAAC,UAAU,SAAS,CAAC;IAC7E,MAAM,IAAI,GACR,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;IACjD,OAAO,IAAI,CAAC,MAAM,GAAG,QAAQ;QAC3B,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,IAAI,CAAC,MAAM,SAAS;QACrD,CAAC,CAAC,IAAI,CAAC;AACX,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,MAA+B;IACnD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACvC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC1C,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;SAClE,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC5C,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK;QAAE,OAAO,IAAI,CAAC;IAC9C,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,GAAG,GAAG,MAAiC,CAAC;IAC9C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAC;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC;IACvD,MAAM,KAAK,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACrC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC,CAAC,GAAG,CAAC,QAAQ;YACZ,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,CAAC,CAAC;IACR,OAAO;QACL,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK;QACL,SAAS,EACP,QAAQ,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,sBAAsB;KACnE,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,OAA2B;IAC9C,IAAI,QAAQ,CAAC,OAAO,CAAC;QAAE,OAAO,OAAO,CAAC;IACtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACZ,YAAY,CAAC,IAAI,CAAC,IAAI,MAAM,IAAI,IAAI;QAClC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACzB,CAAC,CAAC,EAAE,CACP;SACA,IAAI,CAAC,EAAE,CAAC,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAAsB;IAEtB,MAAM,UAAU,GAAG;QACjB,eAAe,IAAI,CAAC,SAAS,EAAE;QAC/B,+CAA+C,IAAI,CAAC,IAAI,EAAE;QAC1D,YAAY,YAAY,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE;QAC7C,aAAa,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;KAC1C,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEf,MAAM,QAAQ,GAAc;QAC1B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;QAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE;KACtC,CAAC;IAEF,IAAI,KAAc,CAAC;IACnB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,qBAAqB,CAAC;YAChD,QAAQ;YACR,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,gBAAgB;YAC3B,WAAW,EAAE,CAAC;YACd,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,sBAAsB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;SAChF,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,iBAAiB,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9D,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO;YACL,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,EAAE;YACb,KAAK,EAAE,qCAAqC;SAC7C,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/evals/graph-e2e-cases.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Built-in cases for the end-to-end graph suite: plan a workflow, run it, judge
+ * the outputs.
+ *
+ * A case is an objective (what the planner builds), inputs (what the run starts
+ * with), and a goal (what the judge grades the outputs against). Expectations
+ * on the run are deliberately thin — an output exists, it is non-empty, it does
+ * or does not contain a literal — because the substantive question ("is this
+ * actually a German translation of the input?") is the judge's, not a regex's.
+ *
+ * Every case that plans an LLM step needs configured model providers: the
+ * planner picks a real model via `find_model`, and the run then calls it.
+ *
+ * A `nodetool.code.Code` step is the opposite: it is fully deterministic, so a
+ * Code case pins every output value (`expectedOutputs`), skips the judge, and
+ * needs no model provider for its run. The planner still needs one — it is the
+ * model that writes the graph.
+ */
+import type { GraphE2eEvalCase } from "./graph-e2e-eval.js";
+export declare const GRAPH_E2E_EVAL_CASES: readonly GraphE2eEvalCase[];
+//# sourceMappingURL=graph-e2e-cases.d.ts.map

package/dist/evals/graph-e2e-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-e2e-cases.d.ts","sourceRoot":"","sources":["../../src/evals/graph-e2e-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,eAAO,MAAM,oBAAoB,EAAE,SAAS,gBAAgB,EAwQ3D,CAAC"}

package/dist/evals/graph-e2e-cases.js

@@ -0,0 +1,262 @@
+/**
+ * Built-in cases for the end-to-end graph suite: plan a workflow, run it, judge
+ * the outputs.
+ *
+ * A case is an objective (what the planner builds), inputs (what the run starts
+ * with), and a goal (what the judge grades the outputs against). Expectations
+ * on the run are deliberately thin — an output exists, it is non-empty, it does
+ * or does not contain a literal — because the substantive question ("is this
+ * actually a German translation of the input?") is the judge's, not a regex's.
+ *
+ * Every case that plans an LLM step needs configured model providers: the
+ * planner picks a real model via `find_model`, and the run then calls it.
+ *
+ * A `nodetool.code.Code` step is the opposite: it is fully deterministic, so a
+ * Code case pins every output value (`expectedOutputs`), skips the judge, and
+ * needs no model provider for its run. The planner still needs one — it is the
+ * model that writes the graph.
+ */
+export const GRAPH_E2E_EVAL_CASES = [
+    {
+        id: "concat",
+        description: "Deterministic string mechanics — exact output, no judge",
+        objective: "Concatenate the two input strings first and second, in that order, and output the result. This is pure string mechanics — no reasoning or generation involved.",
+        goal: 'The output is exactly "Hello, world".',
+        inputs: { first: "Hello, ", second: "world" },
+        skipJudge: true,
+        expectGraph: {
+            requiredInputNames: ["first", "second"],
+            forbiddenNodeTypePatterns: ["^nodetool\\.agents\\."],
+            requireConnected: true,
+            requireOutputNode: true
+        },
+        expect: {
+            minOutputs: 1,
+            requireNonEmptyOutputs: true,
+            requiredOutputPatterns: ["^Hello, world$"]
+        }
+    },
+    {
+        id: "code-parse-json",
+        description: "Code node parses a JSON input and reads two fields out",
+        objective: 'The input payload is a JSON string. Parse it with one nodetool.code.Code node and surface two values: create an output node named "order_id" carrying the order id, and an output node named "city" carrying the customer city. This is plain data work — no LLM step, and no Python.',
+        goal: 'order_id is exactly "A-4417" and city is exactly "Porto".',
+        inputs: {
+            payload: '{"order":{"id":"A-4417","items":3},"customer":{"name":"Rui Alves","city":"Porto"}}'
+        },
+        skipJudge: true,
+        expectGraph: {
+            requiredInputNames: ["payload"],
+            requiredNodeTypePatterns: ["^nodetool\\.code\\.Code$"],
+            forbiddenNodeTypePatterns: ["^nodetool\\.agents\\."],
+            minOutputNodes: 2,
+            requireConnected: true
+        },
+        expect: {
+            requiredOutputNames: ["order_id", "city"],
+            expectedOutputs: { order_id: "A-4417", city: "Porto" }
+        }
+    },
+    {
+        id: "code-aggregate",
+        description: "Code node aggregates a structured input into exact numbers",
+        objective: 'The input line_items is a JSON string holding an array of objects, each with sku, qty and unit_price. With one nodetool.code.Code node, compute the order total (the sum of qty times unit_price over every item, rounded to two decimal places) and the number of distinct line items. Create an output node named "total" for the total and one named "item_count" for the count. This is arithmetic — no LLM step, and no Python.',
+        goal: "total is 44.85 and item_count is 3.",
+        inputs: {
+            line_items: '[{"sku":"PN-1","qty":3,"unit_price":4.25},{"sku":"PN-2","qty":1,"unit_price":12.6},{"sku":"PN-3","qty":2,"unit_price":9.75}]'
+        },
+        skipJudge: true,
+        expectGraph: {
+            requiredInputNames: ["line_items"],
+            requiredNodeTypePatterns: ["^nodetool\\.code\\.Code$"],
+            forbiddenNodeTypePatterns: ["^nodetool\\.agents\\."],
+            minOutputNodes: 2,
+            requireConnected: true
+        },
+        expect: {
+            requiredOutputNames: ["total", "item_count"],
+            expectedOutputs: { total: 44.85, item_count: 3 }
+        }
+    },
+    {
+        id: "code-format",
+        description: "Code node formats a list input into one pinned string",
+        // The wording pins every character the output must carry, because that is
+        // what the case scores. A formatting rule stated loosely ("tidy the
+        // names") has no single right answer, and nothing deterministic to check.
+        objective: 'The input names is a list of strings. With one nodetool.code.Code node, put each name in title case (first letter of every word upper case, the rest lower case), sort the formatted names alphabetically, and build one line: the number of names, then " attendees: ", then the names joined by "; ". For three names Ann Lee, Bo Fry and Cy Ray the line reads "3 attendees: Ann Lee; Bo Fry; Cy Ray". Create one output node named "roster" carrying that line. This is string work — no LLM step, and no Python.',
+        goal: 'roster is exactly "3 attendees: June Okafor; Omar Haddad; Priya Sharma".',
+        inputs: { names: ["priya sharma", "OMAR HADDAD", "june okafor"] },
+        skipJudge: true,
+        expectGraph: {
+            requiredInputNames: ["names"],
+            requiredNodeTypePatterns: ["^nodetool\\.code\\.Code$"],
+            forbiddenNodeTypePatterns: ["^nodetool\\.agents\\."],
+            requireConnected: true,
+            requireOutputNode: true
+        },
+        expect: {
+            requiredOutputNames: ["roster"],
+            expectedOutputs: {
+                roster: "3 attendees: June Okafor; Omar Haddad; Priya Sharma"
+            }
+        }
+    },
+    {
+        id: "summarize",
+        description: "One LLM step summarizes the input text",
+        objective: "Take the input text and produce a one-paragraph summary of it with an LLM step. Output the summary under the name summary.",
+        goal: "The summary output is a short prose summary, in English, of the input text — it must mention that NodeTool is a visual workflow platform for AI, and must not be a verbatim copy of the input.",
+        inputs: {
+            text: "NodeTool is a visual AI workflow platform. Users drag nodes onto a canvas, wire them together, and run the resulting graph. Nodes wrap LLMs, image models, file I/O, and plain code, so a workflow can mix generation with deterministic steps. Workflows run locally on a desktop app or on a server, and the same graph can be published as a mini app."
+        },
+        needsModelProviders: true,
+        expectGraph: {
+            requiredInputNames: ["text"],
+            minAgentSteps: 1,
+            requireConnected: true,
+            requireOutputNode: true
+        },
+        expect: {
+            requiredOutputNames: ["summary"],
+            requireNonEmptyOutputs: true
+        }
+    },
+    {
+        id: "translate-chain",
+        description: "Two chained LLM steps; both intermediate and final surfaced",
+        objective: "Take the input text, summarize it in one sentence with an LLM step, then translate that summary to German with a second LLM step. Output the summary under the name summary and the translation under the name translation.",
+        goal: "The translation output is German prose that says the same thing as the summary output, and the summary output is a one-sentence English summary of the input text. Both must be present and non-empty.",
+        inputs: {
+            text: "The harbour festival runs for three days. Boats are open to visitors on Saturday, there is live music each evening, and the fireworks close the festival on Sunday night."
+        },
+        needsModelProviders: true,
+        expectGraph: {
+            requiredInputNames: ["text"],
+            minAgentSteps: 2,
+            minOutputNodes: 2,
+            requireConnected: true
+        },
+        expect: {
+            requiredOutputNames: ["summary", "translation"],
+            requireNonEmptyOutputs: true
+        }
+    },
+    {
+        id: "extract-fields",
+        description: "Structured extraction from unstructured input text",
+        // "Output the name under the name person" reads as a riddle and the
+        // planner resolved it by naming the output `name`. Name the three output
+        // nodes explicitly instead — the case is about extraction, not about
+        // parsing the instruction.
+        objective: 'Extract three fields from the input text with an LLM step: the person\'s full name, their city, and their job title. Create exactly three output nodes, named "person", "city", and "role" — "person" carries the full name, "city" the city, "role" the job title.',
+        goal: 'The three outputs carry the values from the input text: person is "Amara Osei", city is "Lisbon", role is her job title (marine biologist). Each output holds only that value, not the whole sentence.',
+        inputs: {
+            text: "Amara Osei, a marine biologist who moved to Lisbon last spring, will present her findings at the conference."
+        },
+        needsModelProviders: true,
+        expectGraph: {
+            requiredInputNames: ["text"],
+            minAgentSteps: 1,
+            minOutputNodes: 3,
+            requireConnected: true
+        },
+        expect: {
+            requiredOutputNames: ["person", "city", "role"],
+            requireNonEmptyOutputs: true,
+            requiredOutputPatterns: ["Amara Osei", "Lisbon"]
+        }
+    },
+    {
+        id: "branch-language",
+        description: "Conditional routing — only the taken branch produces output",
+        objective: 'If the input language is exactly the string "de", translate the input text to German with an LLM step; otherwise translate it to French with an LLM step. Output the translation from each branch.',
+        goal: 'The run was given language "de", so the produced output must be a German translation of "Good morning, the meeting starts at nine." Any French output means the wrong branch ran.',
+        inputs: { language: "de", text: "Good morning, the meeting starts at nine." },
+        needsModelProviders: true,
+        expectGraph: {
+            requiredInputNames: ["language", "text"],
+            requiredNodeTypePatterns: ["^nodetool\\.control\\.If$"],
+            requiredSourceHandles: ["if_true", "if_false"],
+            minAgentSteps: 2,
+            requireConnected: true
+        },
+        expect: {
+            minOutputs: 1,
+            requireNonEmptyOutputs: true
+        }
+    },
+    {
+        id: "arithmetic",
+        description: "Deterministic compute the graph must get exactly right",
+        // Python nodes need a worker the eval host may not have, and the planner
+        // reached for one on some runs — so the constraint is stated, the way a
+        // user who knows their environment would state it.
+        objective: 'Compute the total of the input amount plus a tip of the input tip_percent percent of that amount, rounded to two decimal places, and output it from an output node named "total". Use a deterministic computation step, not an LLM. No Python interpreter is available — the computation must run in JavaScript.',
+        goal: "The total output is 101.4 (84.50 plus a 20% tip of 16.90).",
+        inputs: { amount: 84.5, tip_percent: 20 },
+        expectGraph: {
+            requiredInputNames: ["amount", "tip_percent"],
+            forbiddenNodeTypePatterns: ["^nodetool\\.agents\\."],
+            requireConnected: true,
+            requireOutputNode: true
+        },
+        expect: {
+            requiredOutputNames: ["total"],
+            requireNonEmptyOutputs: true,
+            requiredOutputPatterns: ["101\\.4"]
+        }
+    },
+    {
+        id: "agent-constraint-reasoning",
+        description: "One Agent step reasons over constraints to a single answer",
+        objective: 'Take the input constraints text and work out, with one nodetool.agents.Agent step, the one weekday the team can meet. Create an output node named "answer" carrying just the weekday, and an output node named "reasoning" carrying the short explanation that rules the other days out.',
+        goal: 'The answer output names Wednesday and nothing else. The reasoning output explains why each other weekday is ruled out: Dana is away Monday and Tuesday, the room is taken on Thursday, and Elias never meets on a Friday.',
+        inputs: {
+            constraints: "The team meets exactly once next week, on a weekday from Monday to Friday. Dana must attend and is away on Monday and Tuesday. The meeting room is already booked all day Thursday. Elias must attend and never meets on a Friday."
+        },
+        needsModelProviders: true,
+        expectGraph: {
+            requiredInputNames: ["constraints"],
+            requiredNodeTypePatterns: ["^nodetool\\.agents\\.Agent$"],
+            minAgentSteps: 1,
+            minOutputNodes: 2,
+            requireConnected: true
+        },
+        expect: {
+            requiredOutputNames: ["answer", "reasoning"],
+            requireNonEmptyOutputs: true,
+            requiredOutputPatterns: ["Wednesday"]
+        }
+    },
+    {
+        id: "code-then-agent",
+        description: "Deterministic Code prep feeds an Agent step; prep is pinned",
+        objective: 'The input reviews is a JSON string holding an array of objects, each with author, rating and text. Do two steps. First, with one nodetool.code.Code node, keep only the reviews whose rating is 2 or lower, in the order they appear, and build one block of text: one line per kept review, each line the position among the kept reviews starting at 1, then ". ", then the author, then ": ", then the review text, with the lines joined by a single newline. For two kept reviews by Ann and Bo the block reads "1. Ann: Too loud.\\n2. Bo: It leaks." Second, pass that block to one nodetool.agents.Agent step that writes a single paragraph summarizing what the unhappy customers complain about. Create an output node named "complaints" carrying the block from the Code node, and an output node named "summary" carrying the paragraph.',
+        goal: 'The complaints output is the two low-rated reviews, numbered 1 and 2, and the summary output is one paragraph of prose that names both problems — the cracked lid and the unit that stopped charging. The summary must be prose, not a copy of the numbered block.',
+        inputs: {
+            reviews: '[{"author":"Rui","rating":5,"text":"Fast and tidy."},{"author":"Mina","rating":2,"text":"The lid arrived cracked."},{"author":"Tomas","rating":4,"text":"Good value."},{"author":"Aya","rating":1,"text":"It stopped charging after a week."}]'
+        },
+        needsModelProviders: true,
+        expectGraph: {
+            requiredInputNames: ["reviews"],
+            requiredNodeTypePatterns: [
+                "^nodetool\\.code\\.Code$",
+                "^nodetool\\.agents\\.Agent$"
+            ],
+            minAgentSteps: 1,
+            minOutputNodes: 2,
+            requireConnected: true
+        },
+        expect: {
+            requiredOutputNames: ["complaints", "summary"],
+            requireNonEmptyOutputs: true,
+            // The Code half is deterministic, so it is pinned; the paragraph the
+            // Agent writes is the judge's half.
+            expectedOutputs: {
+                complaints: "1. Mina: The lid arrived cracked.\n2. Aya: It stopped charging after a week."
+            }
+        }
+    }
+];
+//# sourceMappingURL=graph-e2e-cases.js.map

package/dist/evals/graph-e2e-cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"graph-e2e-cases.js","sourceRoot":"","sources":["../../src/evals/graph-e2e-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAgC;IAC/D;QACE,EAAE,EAAE,QAAQ;QACZ,WAAW,EAAE,yDAAyD;QACtE,SAAS,EACP,gKAAgK;QAClK,IAAI,EAAE,uCAAuC;QAC7C,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE;QAC7C,SAAS,EAAE,IAAI;QACf,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC;YACvC,yBAAyB,EAAE,CAAC,uBAAuB,CAAC;YACpD,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;SACxB;QACD,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,sBAAsB,EAAE,IAAI;YAC5B,sBAAsB,EAAE,CAAC,gBAAgB,CAAC;SAC3C;KACF;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,wDAAwD;QACrE,SAAS,EACP,uRAAuR;QACzR,IAAI,EAAE,2DAA2D;QACjE,MAAM,EAAE;YACN,OAAO,EACL,oFAAoF;SACvF;QACD,SAAS,EAAE,IAAI;QACf,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,SAAS,CAAC;YAC/B,wBAAwB,EAAE,CAAC,0BAA0B,CAAC;YACtD,yBAAyB,EAAE,CAAC,uBAAuB,CAAC;YACpD,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,IAAI;SACvB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;YACzC,eAAe,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE;SACvD;KACF;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,WAAW,EAAE,4DAA4D;QACzE,SAAS,EACP,saAAsa;QACxa,IAAI,EAAE,qCAAqC;QAC3C,MAAM,EAAE;YACN,UAAU,EACR,8HAA8H;SACjI;QACD,SAAS,EAAE,IAAI;QACf,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,YAAY,CAAC;YAClC,wBAAwB,EAAE,CAAC,0BAA0B,CAAC;YACtD,yBAAyB,EAAE,CAAC,uBAAuB,CAAC;YACpD,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,IAAI;SACvB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC;YAC5C,eAAe,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,EAAE;SACjD;KACF;IACD;QACE,EAAE,EAAE,aAAa;QACjB,WAAW,EAAE,uDAAuD;QACpE,0EAA0E;QAC1E,oEAAoE;QACpE,0EAA0E;QAC1E,SAAS,EACP,ufAAuf;QACzf,IAAI,EAAE,0EAA0E;QAChF,MAAM,EAAE,EAAE,KAAK,EAAE,CAAC,cAAc,EAAE,aAAa,EAAE,aAAa,CAAC,EAAE;QACjE,SAAS,EAAE,IAAI;QACf,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,OAAO,CAAC;YAC7B,wBAAwB,EAAE,CAAC,0BAA0B,CAAC;YACtD,yBAAyB,EAAE,CAAC,uBAAuB,CAAC;YACpD,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;SACxB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,QAAQ,CAAC;YAC/B,eAAe,EAAE;gBACf,MAAM,EAAE,qDAAqD;aAC9D;SACF;KACF;IACD;QACE,EAAE,EAAE,WAAW;QACf,WAAW,EAAE,wCAAwC;QACrD,SAAS,EACP,4HAA4H;QAC9H,IAAI,EACF,gMAAgM;QAClM,MAAM,EAAE;YACN,IAAI,EAAE,2VAA2V;SAClW;QACD,mBAAmB,EAAE,IAAI;QACzB,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,MAAM,CAAC;YAC5B,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;SACxB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,SAAS,CAAC;YAChC,sBAAsB,EAAE,IAAI;SAC7B;KACF;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,6DAA6D;QAC1E,SAAS,EACP,6NAA6N;QAC/N,IAAI,EACF,wMAAwM;QAC1M,MAAM,EAAE;YACN,IAAI,EAAE,2KAA2K;SAClL;QACD,mBAAmB,EAAE,IAAI;QACzB,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,MAAM,CAAC;YAC5B,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,IAAI;SACvB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,SAAS,EAAE,aAAa,CAAC;YAC/C,sBAAsB,EAAE,IAAI;SAC7B;KACF;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,WAAW,EAAE,oDAAoD;QACjE,oEAAoE;QACpE,yEAAyE;QACzE,qEAAqE;QACrE,2BAA2B;QAC3B,SAAS,EACP,qQAAqQ;QACvQ,IAAI,EACF,wMAAwM;QAC1M,MAAM,EAAE;YACN,IAAI,EAAE,8GAA8G;SACrH;QACD,mBAAmB,EAAE,IAAI;QACzB,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,MAAM,CAAC;YAC5B,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,IAAI;SACvB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YAC/C,sBAAsB,EAAE,IAAI;YAC5B,sBAAsB,EAAE,CAAC,YAAY,EAAE,QAAQ,CAAC;SACjD;KACF;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,6DAA6D;QAC1E,SAAS,EACP,oMAAoM;QACtM,IAAI,EACF,mLAAmL;QACrL,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,2CAA2C,EAAE;QAC7E,mBAAmB,EAAE,IAAI;QACzB,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,UAAU,EAAE,MAAM,CAAC;YACxC,wBAAwB,EAAE,CAAC,2BAA2B,CAAC;YACvD,qBAAqB,EAAE,CAAC,SAAS,EAAE,UAAU,CAAC;YAC9C,aAAa,EAAE,CAAC;YAChB,gBAAgB,EAAE,IAAI;SACvB;QACD,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,sBAAsB,EAAE,IAAI;SAC7B;KACF;IACD;QACE,EAAE,EAAE,YAAY;QAChB,WAAW,EAAE,wDAAwD;QACrE,yEAAyE;QACzE,wEAAwE;QACxE,mDAAmD;QACnD,SAAS,EACP,kTAAkT;QACpT,IAAI,EAAE,4DAA4D;QAClE,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,EAAE;QACzC,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC;YAC7C,yBAAyB,EAAE,CAAC,uBAAuB,CAAC;YACpD,gBAAgB,EAAE,IAAI;YACtB,iBAAiB,EAAE,IAAI;SACxB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,OAAO,CAAC;YAC9B,sBAAsB,EAAE,IAAI;YAC5B,sBAAsB,EAAE,CAAC,SAAS,CAAC;SACpC;KACF;IACD;QACE,EAAE,EAAE,4BAA4B;QAChC,WAAW,EAAE,4DAA4D;QACzE,SAAS,EACP,0RAA0R;QAC5R,IAAI,EACF,2NAA2N;QAC7N,MAAM,EAAE;YACN,WAAW,EACT,oOAAoO;SACvO;QACD,mBAAmB,EAAE,IAAI;QACzB,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,aAAa,CAAC;YACnC,wBAAwB,EAAE,CAAC,6BAA6B,CAAC;YACzD,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,IAAI;SACvB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC;YAC5C,sBAAsB,EAAE,IAAI;YAC5B,sBAAsB,EAAE,CAAC,WAAW,CAAC;SACtC;KACF;IACD;QACE,EAAE,EAAE,iBAAiB;QACrB,WAAW,EAAE,6DAA6D;QAC1E,SAAS,EACP,wzBAAwzB;QAC1zB,IAAI,EACF,oQAAoQ;QACtQ,MAAM,EAAE;YACN,OAAO,EACL,gPAAgP;SACnP;QACD,mBAAmB,EAAE,IAAI;QACzB,WAAW,EAAE;YACX,kBAAkB,EAAE,CAAC,SAAS,CAAC;YAC/B,wBAAwB,EAAE;gBACxB,0BAA0B;gBAC1B,6BAA6B;aAC9B;YACD,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,CAAC;YACjB,gBAAgB,EAAE,IAAI;SACvB;QACD,MAAM,EAAE;YACN,mBAAmB,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC;YAC9C,sBAAsB,EAAE,IAAI;YAC5B,qEAAqE;YACrE,oCAAoC;YACpC,eAAe,EAAE;gBACf,UAAU,EACR,8EAA8E;aACjF;SACF;KACF;CACF,CAAC"}