@nodetool-ai/agents 0.7.0-rc.23 → 0.8.2

package/dist/evals/app-build-plan.js

@@ -0,0 +1,140 @@
+/**
+ * The plan-to-graph half of the `app-build` harness (PRD § 11.6, R6).
+ *
+ * The Workflow flow's build is not a mini app, but it is the same journey the
+ * app-build harness already runs — a plan, a graph built from it, a static
+ * check, a real run — so it grades here rather than in a harness of its own.
+ *
+ * What makes this case worth having is the last check. A plan whose steps name
+ * node types that exist builds a graph that passes `validate_workflow` and can
+ * still produce nothing: an output node fed by no step, a step wired to a
+ * handle that never carries a value. Validation cannot see that. So a case is
+ * green only when
+ *
+ *  1. `planToPlacement` reported no wiring it could not do,
+ *  2. the graph passes the same static check `validate_workflow` runs, and
+ *  3. **the run produced a value for every declared output**, matching what the
+ *     case expects.
+ *
+ * Check 3 is the one that fails on a plan that validates and does nothing.
+ * `packages/agents/tests/app-build-workflow-plan.test.ts` feeds exactly such a
+ * plan and asserts checks 1 and 3 go red while 2 stays green.
+ */
+import { planNodeShape, planToPlacement } from "@nodetool-ai/protocol";
+import { isNonEmptyString, isString } from "../utils/type-guards.js";
+const check = (name, pass, detail) => detail === undefined ? { name, pass } : { name, pass, detail };
+/** A value that reached an output but carries nothing a user would call a result. */
+const isEmptyResult = (value) => value === undefined ||
+    value === null ||
+    (isString(value) && value.trim().length === 0) ||
+    (Array.isArray(value) && value.length === 0);
+/**
+ * Build the case's plan into a graph, check it, run it, and grade what came
+ * out. Returns the three checks in the order they are earned, so a report reads
+ * as the journey: wired → validated → produced.
+ */
+export async function runPlanToGraphCase(planCase, opts) {
+    const { registry } = opts;
+    const shapes = planCase.shapes ?? {};
+    const placement = planToPlacement(planCase.plan, (nodeType) => {
+        const meta = registry.getMetadata(nodeType);
+        if (meta)
+            return planNodeShape(meta);
+        return shapes[nodeType] ?? null;
+    });
+    const checks = [
+        check("plan wiring", placement.issues.length === 0, placement.issues.join(" "))
+    ];
+    const graph = {
+        nodes: placement.nodes.map((node) => ({
+            id: node.id,
+            type: node.type,
+            data: node.properties,
+            ui_properties: {
+                position: node.position,
+                setup_step_id: node.setupStepId
+            },
+            dynamic_properties: node.dynamicProperties ?? {},
+            dynamic_outputs: {}
+        })),
+        edges: placement.edges.map((edge, index) => ({
+            id: `e${index + 1}`,
+            source: edge.source,
+            sourceHandle: edge.sourceHandle,
+            target: edge.target,
+            targetHandle: edge.targetHandle
+        }))
+    };
+    const { validateGraph } = await import("@nodetool-ai/node-sdk");
+    const report = validateGraph(graph, {
+        has: (type) => registry.has(type),
+        getMetadata: (type) => registry.getMetadata(type),
+        validateNode: (descriptor, connectedHandles) => registry.validateNode(descriptor, connectedHandles)
+    });
+    const validationIssues = report.issues
+        .filter((issue) => issue.severity === "error")
+        .map((issue) => issue.message);
+    checks.push(check("graph validates", report.ok, validationIssues.join(" ")));
+    // Sample values live on the input nodes the builder placed, so a run needs no
+    // params at all; a case may still override them.
+    const params = planCase.params ??
+        Object.fromEntries(planCase.plan.inputs
+            .filter((input) => input.sample !== undefined)
+            .map((input) => [input.name, input.sample]));
+    let produced = {};
+    let runError;
+    try {
+        const runInput = {
+            graph: graph,
+            workflowId: null,
+            params
+        };
+        if (opts.timeoutMs !== undefined)
+            runInput.timeoutMs = opts.timeoutMs;
+        const outcome = await opts.runOnServer(runInput);
+        // The output node's `name` property is what a user reads the result by, so
+        // key the produced map by it rather than by the node id.
+        const nameByNodeId = new Map(placement.nodes
+            .filter((node) => isNonEmptyString(node.properties["name"]))
+            .map((node) => [node.id, String(node.properties["name"])]));
+        for (const output of outcome.report.summary.outputs) {
+            const key = nameByNodeId.get(output.nodeId ?? "") ?? output.outputName;
+            produced[key] = output.value;
+        }
+        if (!outcome.report.ok) {
+            runError = outcome.report.error ?? outcome.report.status;
+        }
+    }
+    catch (cause) {
+        produced = {};
+        runError = cause instanceof Error ? cause.message : String(cause);
+    }
+    for (const expected of planCase.outputs) {
+        const value = produced[expected.name];
+        const detail = runError !== undefined
+            ? `run failed: ${runError}`
+            : `produced ${JSON.stringify(value)}`;
+        if (isEmptyResult(value)) {
+            checks.push(check(`output "${expected.name}" produced`, false, runError !== undefined
+                ? detail
+                : `the graph validated but produced nothing for "${expected.name}"`));
+            continue;
+        }
+        const matches = expected.equals !== undefined
+            ? JSON.stringify(value) === JSON.stringify(expected.equals)
+            : expected.contains !== undefined
+                ? String(value).includes(expected.contains)
+                : true;
+        checks.push(check(`output "${expected.name}" produced`, matches, detail));
+    }
+    const result = {
+        checks,
+        produced,
+        issues: placement.issues,
+        validationIssues
+    };
+    if (runError !== undefined)
+        result.error = runError;
+    return result;
+}
+//# sourceMappingURL=app-build-plan.js.map

package/dist/evals/app-build-plan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"app-build-plan.js","sourceRoot":"","sources":["../../src/evals/app-build-plan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAQvE,OAAO,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,yBAAyB,CAAC;AA4CrE,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,IAAa,EAAE,MAAe,EAAa,EAAE,CACxE,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAEjE,qFAAqF;AACrF,MAAM,aAAa,GAAG,CAAC,KAAc,EAAW,EAAE,CAChD,KAAK,KAAK,SAAS;IACnB,KAAK,KAAK,IAAI;IACd,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;IAC9C,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC;AAQ/C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAyB,EACzB,IAAwB;IAExB,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAC1B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC;IACrC,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,QAAQ,EAAE,EAAE;QAC5D,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,IAAI;YAAE,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;QACrC,OAAO,MAAM,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GAAgB;QAC1B,KAAK,CACH,aAAa,EACb,SAAS,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,EAC7B,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAC3B;KACF,CAAC;IAEF,MAAM,KAAK,GAAG;QACZ,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACpC,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,UAAU;YACrB,aAAa,EAAE;gBACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,aAAa,EAAE,IAAI,CAAC,WAAW;aAChC;YACD,kBAAkB,EAAE,IAAI,CAAC,iBAAiB,IAAI,EAAE;YAChD,eAAe,EAAE,EAAE;SACpB,CAAC,CAAC;QACH,KAAK,EAAE,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3C,EAAE,EAAE,IAAI,KAAK,GAAG,CAAC,EAAE;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,IAAI,CAAC,YAAY;YAC/B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC;KACJ,CAAC;IAEF,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,uBAAuB,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,aAAa,CAAC,KAAK,EAAE;QAClC,GAAG,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;QACzC,WAAW,EAAE,CAAC,IAAY,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC;QACzD,YAAY,EAAE,CACZ,UAAuD,EACvD,gBAA6D,EAC7D,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,UAAU,EAAE,gBAAgB,CAAC;KACzD,CAAC,CAAC;IACH,MAAM,gBAAgB,GAAG,MAAM,CAAC,MAAM;SACnC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,OAAO,CAAC;SAC7C,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,IAAI,CACT,KAAK,CAAC,iBAAiB,EAAE,MAAM,CAAC,EAAE,EAAE,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAChE,CAAC;IAEF,8EAA8E;IAC9E,iDAAiD;IACjD,MAAM,MAAM,GACV,QAAQ,CAAC,MAAM;QACf,MAAM,CAAC,WAAW,CAChB,QAAQ,CAAC,IAAI,CAAC,MAAM;aACjB,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,CAAC;aAC7C,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAC9C,CAAC;IAEJ,IAAI,QAAQ,GAA4B,EAAE,CAAC;IAC3C,IAAI,QAA4B,CAAC;IACjC,IAAI,CAAC;QACH,MAAM,QAAQ,GAAsB;YAClC,KAAK,EAAE,KAA8C;YACrD,UAAU,EAAE,IAAI;YAChB,MAAM;SACP,CAAC;QACF,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS;YAAE,QAAQ,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACtE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC;QACjD,2EAA2E;QAC3E,yDAAyD;QACzD,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,SAAS,CAAC,KAAK;aACZ,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,gBAAgB,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;aAC3D,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAC7D,CAAC;QACF,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YACpD,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,IAAI,MAAM,CAAC,UAAU,CAAC;YACvE,QAAQ,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC;QAC/B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACvB,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC;QAC3D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,QAAQ,GAAG,EAAE,CAAC;QACd,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,MAAM,GACV,QAAQ,KAAK,SAAS;YACpB,CAAC,CAAC,eAAe,QAAQ,EAAE;YAC3B,CAAC,CAAC,YAAY,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1C,IAAI,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CACT,KAAK,CACH,WAAW,QAAQ,CAAC,IAAI,YAAY,EACpC,KAAK,EACL,QAAQ,KAAK,SAAS;gBACpB,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,iDAAiD,QAAQ,CAAC,IAAI,GAAG,CACtE,CACF,CAAC;YACF,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GACX,QAAQ,CAAC,MAAM,KAAK,SAAS;YAC3B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC3D,CAAC,CAAC,QAAQ,CAAC,QAAQ,KAAK,SAAS;gBAC/B,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC3C,CAAC,CAAC,IAAI,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,QAAQ,CAAC,IAAI,YAAY,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,MAAM,GAAmB;QAC7B,MAAM;QACN,QAAQ;QACR,MAAM,EAAE,SAAS,CAAC,MAAM;QACxB,gBAAgB;KACjB,CAAC;IACF,IAAI,QAAQ,KAAK,SAAS;QAAE,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC;IACpD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/evals/code-gen-cases.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Built-in evaluation cases for Code node AI authoring (`CodePlanner`).
+ *
+ * One case per shape the feature is meant to cover — reshape, merge/join,
+ * compute, extract/parse, split, format, validate, seed — each an instruction
+ * plus structural expectations on the accepted submission. Expectations never
+ * pin exact code: many bodies are right answers, but all of them must declare
+ * the outputs the instruction asked for, keep to the inputs on offer, emit
+ * every output on every path, and stay inside the sandbox API.
+ */
+import type { CodeGenEvalCase } from "./code-gen-eval.js";
+export declare const CODE_GEN_EVAL_CASES: readonly CodeGenEvalCase[];
+//# sourceMappingURL=code-gen-cases.d.ts.map

package/dist/evals/code-gen-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-gen-cases.d.ts","sourceRoot":"","sources":["../../src/evals/code-gen-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAU1D,eAAO,MAAM,mBAAmB,EAAE,SAAS,eAAe,EAkHpB,CAAC"}

package/dist/evals/code-gen-cases.js

@@ -0,0 +1,124 @@
+/**
+ * Built-in evaluation cases for Code node AI authoring (`CodePlanner`).
+ *
+ * One case per shape the feature is meant to cover — reshape, merge/join,
+ * compute, extract/parse, split, format, validate, seed — each an instruction
+ * plus structural expectations on the accepted submission. Expectations never
+ * pin exact code: many bodies are right answers, but all of them must declare
+ * the outputs the instruction asked for, keep to the inputs on offer, emit
+ * every output on every path, and stay inside the sandbox API.
+ */
+const STR = { type: "str" };
+const FLOAT = { type: "float" };
+const DICT = { type: "dict" };
+const listOf = (item) => ({
+    type: "list",
+    type_args: [item]
+});
+export const CODE_GEN_EVAL_CASES = [
+    {
+        id: "reshape",
+        description: "Reshape a list of records into a keyed lookup",
+        instruction: "Turn the list of user records into an object keyed by each record's id, with the record as the value. Also report how many records were keyed.",
+        inputs: [
+            {
+                name: "users",
+                type: listOf(DICT),
+                description: "Records, each with an `id` field"
+            }
+        ],
+        expect: {
+            requiredOutputs: ["count"],
+            outputTypes: { count: ["int", "float"] },
+            minOutputs: 2,
+            maxOutputs: 3
+        }
+    },
+    {
+        id: "merge-join",
+        description: "Join two lists on a shared key",
+        instruction: "Join the orders to the customers on order.customer_id === customer.id. Return the joined rows, each order enriched with the customer's name, and the orders that matched no customer.",
+        inputs: [
+            { name: "orders", type: listOf(DICT) },
+            { name: "customers", type: listOf(DICT) }
+        ],
+        expect: {
+            minOutputs: 2,
+            requiredCodePatterns: ["orders", "customers"]
+        }
+    },
+    {
+        id: "compute-derive",
+        description: "Derive totals from line items",
+        instruction: "Compute the subtotal of the line items (each has price and quantity), the tax at the given rate, and the grand total.",
+        inputs: [
+            { name: "items", type: listOf(DICT) },
+            { name: "taxRate", type: FLOAT, default: 0.2 }
+        ],
+        expect: {
+            minOutputs: 3,
+            // Subtotal, tax and grand total are all money. An earlier version also
+            // demanded an int output, which nothing in the instruction asks for —
+            // the eval was wrong, not the model.
+            outputTypeKinds: ["float"]
+        }
+    },
+    {
+        id: "extract-parse",
+        description: "Parse a delimited text format by hand",
+        instruction: "Each line of the log text is `level|timestamp|message`. Parse it into one record per line, and also return how many records have level 'ERROR'.",
+        inputs: [{ name: "logText", type: STR }],
+        expect: {
+            minOutputs: 2,
+            // This authoring surface declares no sandbox packages, so `analyzeGeneratedCode`
+            // refuses a module declaration outright. Parsing has to be hand-rolled.
+            forbiddenCodePatterns: ["\\brequire\\(", "\\bimport\\s"]
+        }
+    },
+    {
+        id: "split",
+        description: "Split text and report the pieces plus a count",
+        instruction: "Split the text into sentences. Return the sentences and how many there are.",
+        inputs: [{ name: "text", type: STR }],
+        expect: {
+            minOutputs: 2,
+            outputTypeKinds: ["list", "int"]
+        }
+    },
+    {
+        id: "format-render",
+        description: "Format a number for display through the format bridge",
+        instruction: "Format the amount as a currency string in the given currency code using the sandbox's number formatter, and return the formatted string.",
+        inputs: [
+            { name: "amount", type: FLOAT },
+            { name: "currency", type: STR, default: "EUR" }
+        ],
+        expect: {
+            minOutputs: 1,
+            requiredCodePatterns: ["format\\.number"],
+            outputTypeKinds: ["str"]
+        }
+    },
+    {
+        id: "validate",
+        description: "Validate a record and emit both outputs on every path",
+        instruction: "Check the record: it must have a non-empty name and an email containing '@'. Return whether it is valid and the list of problems found.",
+        inputs: [{ name: "record", type: DICT }],
+        expect: {
+            minOutputs: 2,
+            outputTypeKinds: ["bool", "list"]
+        }
+    },
+    {
+        id: "seed",
+        description: "Produce a value with no inputs, seeded from a destination handle",
+        instruction: "Produce the twelve month names in English, in calendar order.",
+        expectedOutput: { name: "months", type: listOf(STR) },
+        expect: {
+            requiredOutputs: ["months"],
+            minOutputs: 1,
+            maxOutputs: 1
+        }
+    }
+];
+//# sourceMappingURL=code-gen-cases.js.map

package/dist/evals/code-gen-cases.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-gen-cases.js","sourceRoot":"","sources":["../../src/evals/code-gen-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAMH,MAAM,GAAG,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC5B,MAAM,KAAK,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AAChC,MAAM,IAAI,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC9B,MAAM,MAAM,GAAG,CAAC,IAAyB,EAAuB,EAAE,CAAC,CAAC;IAClE,IAAI,EAAE,MAAM;IACZ,SAAS,EAAE,CAAC,IAAI,CAAC;CAClB,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,mBAAmB,GAA+B;IAC7D;QACE,EAAE,EAAE,SAAS;QACb,WAAW,EAAE,+CAA+C;QAC5D,WAAW,EACT,gJAAgJ;QAClJ,MAAM,EAAE;YACN;gBACE,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC;gBAClB,WAAW,EAAE,kCAAkC;aAChD;SACF;QACD,MAAM,EAAE;YACN,eAAe,EAAE,CAAC,OAAO,CAAC;YAC1B,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE;YACxC,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,CAAC;SACd;KACF;IACD;QACE,EAAE,EAAE,YAAY;QAChB,WAAW,EAAE,gCAAgC;QAC7C,WAAW,EACT,uLAAuL;QACzL,MAAM,EAAE;YACN,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;YACtC,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;SAC1C;QACD,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,oBAAoB,EAAE,CAAC,QAAQ,EAAE,WAAW,CAAC;SAC9C;KACF;IACD;QACE,EAAE,EAAE,gBAAgB;QACpB,WAAW,EAAE,+BAA+B;QAC5C,WAAW,EACT,uHAAuH;QACzH,MAAM,EAAE;YACN,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;YACrC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE;SAC/C;QACD,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,uEAAuE;YACvE,sEAAsE;YACtE,qCAAqC;YACrC,eAAe,EAAE,CAAC,OAAO,CAAC;SAC3B;KACF;IACD;QACE,EAAE,EAAE,eAAe;QACnB,WAAW,EAAE,uCAAuC;QACpD,WAAW,EACT,iJAAiJ;QACnJ,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QACxC,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,iFAAiF;YACjF,wEAAwE;YACxE,qBAAqB,EAAE,CAAC,eAAe,EAAE,cAAc,CAAC;SACzD;KACF;IACD;QACE,EAAE,EAAE,OAAO;QACX,WAAW,EAAE,+CAA+C;QAC5D,WAAW,EACT,6EAA6E;QAC/E,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC;QACrC,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,eAAe,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;SACjC;KACF;IACD;QACE,EAAE,EAAE,eAAe;QACnB,WAAW,EAAE,uDAAuD;QACpE,WAAW,EACT,0IAA0I;QAC5I,MAAM,EAAE;YACN,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE;YAC/B,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE;SAChD;QACD,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,oBAAoB,EAAE,CAAC,iBAAiB,CAAC;YACzC,eAAe,EAAE,CAAC,KAAK,CAAC;SACzB;KACF;IACD;QACE,EAAE,EAAE,UAAU;QACd,WAAW,EAAE,uDAAuD;QACpE,WAAW,EACT,yIAAyI;QAC3I,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACxC,MAAM,EAAE;YACN,UAAU,EAAE,CAAC;YACb,eAAe,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SAClC;KACF;IACD;QACE,EAAE,EAAE,MAAM;QACV,WAAW,EACT,kEAAkE;QACpE,WAAW,EACT,+DAA+D;QACjE,cAAc,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE;QACrD,MAAM,EAAE;YACN,eAAe,EAAE,CAAC,QAAQ,CAAC;YAC3B,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,CAAC;SACd;KACF;CACmC,CAAC"}

package/dist/evals/code-gen-eval.d.ts

@@ -0,0 +1,109 @@
+/**
+ * Code node authoring (`CodePlanner`) evaluation harness — provider-agnostic.
+ *
+ * Runs {@link CodeGenEvalCase}s through `CodePlanner.plan()` with any
+ * `BaseProvider`, records what the feedback loop cost (submit rounds, tool
+ * calls, duration, cost), and scores each accepted submission structurally.
+ *
+ * Acceptance is reported twice, because they are different claims about a
+ * model: **first-pass** is a submission accepted on round 1 — the model wrote
+ * valid code from the prompt alone — and **post-repair** is a submission
+ * accepted at all within the round cap, after the tool fed its rejections back.
+ * A model can score badly on the first and well on the second; the rollout gate
+ * reads post-repair.
+ */
+import type { BaseProvider } from "@nodetool-ai/runtime";
+import type { CodeGenExpectedOutput, CodeGenInputPort, CodeGenSubmission } from "@nodetool-ai/protocol/api-schemas/code-gen.js";
+export interface CodeGenEvalExpectations {
+    /** Output names that must be declared (exact names the instruction implies). */
+    requiredOutputs?: string[];
+    /** Accepted `type.type` values per declared output name. */
+    outputTypes?: Record<string, string[]>;
+    /** Each kind must be the type of at least one declared output. */
+    outputTypeKinds?: string[];
+    minOutputs?: number;
+    maxOutputs?: number;
+    /** Regex sources; each must match the generated code. */
+    requiredCodePatterns?: string[];
+    /** Regex sources; none may match the generated code. */
+    forbiddenCodePatterns?: string[];
+}
+export interface CodeGenEvalCase {
+    id: string;
+    description: string;
+    /** What the user asked for, in their own words. */
+    instruction: string;
+    /** Slots the dialog offers, as if seeded from connected handles. */
+    inputs?: CodeGenInputPort[];
+    /** Set for the destination-handle entry point. */
+    expectedOutput?: CodeGenExpectedOutput;
+    expect: CodeGenEvalExpectations;
+}
+export interface CodeGenEvalCheck {
+    name: string;
+    pass: boolean;
+    detail?: string;
+}
+export interface CodeGenCaseResult {
+    caseId: string;
+    description: string;
+    /** A submission was accepted within the round cap. */
+    accepted: boolean;
+    /** Accepted on the first `submit_code` call — no repair round needed. */
+    firstPass: boolean;
+    /** Accepted only after at least one rejection was fed back. */
+    repaired: boolean;
+    /** Fraction of checks passed (0 when nothing was accepted). */
+    score: number;
+    checks: CodeGenEvalCheck[];
+    /** Tool calls the planner made, by tool name. */
+    toolCalls: Record<string, number>;
+    /** `submit_code` calls — 1 means the model got it right first try. */
+    submitRounds: number;
+    durationMs: number;
+    costUsd: number;
+    /** Declared output names of the accepted submission. */
+    outputs: string[];
+    codeChars: number;
+    /** Failure code from the planner, when nothing was accepted. */
+    error?: string;
+}
+export interface CodeGenEvalReport {
+    provider: string;
+    model: string;
+    startedAt: string;
+    cases: CodeGenCaseResult[];
+    summary: {
+        total: number;
+        /** Cases accepted on round 1. */
+        firstPass: number;
+        /** Cases accepted at all, round 1 or after repair. */
+        postRepair: number;
+        firstPassRate: number;
+        postRepairRate: number;
+        /** Fraction of accepted cases that needed a repair round. */
+        repairRate: number;
+        /** Mean expectation score over all cases. */
+        meanScore: number;
+        avgSubmitRounds: number;
+        avgDurationMs: number;
+        totalCostUsd: number;
+    };
+}
+export interface RunCodeGenEvalOptions {
+    provider: BaseProvider;
+    model: string;
+    /** Cases to run; defaults to the built-in suite. */
+    cases?: readonly CodeGenEvalCase[];
+    /** Feedback rounds per case. Defaults to the planner's own cap. */
+    maxRounds?: number;
+    signal?: AbortSignal;
+    /** Progress callback (one line per event, for CLI display). */
+    onEvent?: (line: string) => void;
+}
+/** Score an accepted submission against the case expectations. */
+export declare function checkCodeGenExpectations(submission: CodeGenSubmission, evalCase: CodeGenEvalCase): CodeGenEvalCheck[];
+export declare function runCodeGenEval(opts: RunCodeGenEvalOptions): Promise<CodeGenEvalReport>;
+/** Text summary table for terminal output. */
+export declare function formatCodeGenReport(report: CodeGenEvalReport): string;
+//# sourceMappingURL=code-gen-eval.d.ts.map

package/dist/evals/code-gen-eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-gen-eval.d.ts","sourceRoot":"","sources":["../../src/evals/code-gen-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAqB,MAAM,sBAAsB,CAAC;AAC5E,OAAO,KAAK,EACV,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EAClB,MAAM,+CAA+C,CAAC;AAWvD,MAAM,WAAW,uBAAuB;IACtC,gFAAgF;IAChF,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IACvC,kEAAkE;IAClE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;IAChC,wDAAwD;IACxD,qBAAqB,CAAC,EAAE,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,mDAAmD;IACnD,WAAW,EAAE,MAAM,CAAC;IACpB,oEAAoE;IACpE,MAAM,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAC5B,kDAAkD;IAClD,cAAc,CAAC,EAAE,qBAAqB,CAAC;IACvC,MAAM,EAAE,uBAAuB,CAAC;CACjC;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,OAAO,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,iBAAiB;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,QAAQ,EAAE,OAAO,CAAC;IAClB,yEAAyE;IACzE,SAAS,EAAE,OAAO,CAAC;IACnB,+DAA+D;IAC/D,QAAQ,EAAE,OAAO,CAAC;IAClB,+DAA+D;IAC/D,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,gBAAgB,EAAE,CAAC;IAC3B,iDAAiD;IACjD,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAClC,sEAAsE;IACtE,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,wDAAwD;IACxD,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,gEAAgE;IAChE,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,iBAAiB,EAAE,CAAC;IAC3B,OAAO,EAAE;QACP,KAAK,EAAE,MAAM,CAAC;QACd,iCAAiC;QACjC,SAAS,EAAE,MAAM,CAAC;QAClB,sDAAsD;QACtD,UAAU,EAAE,MAAM,CAAC;QACnB,aAAa,EAAE,MAAM,CAAC;QACtB,cAAc,EAAE,MAAM,CAAC;QACvB,6DAA6D;QAC7D,UAAU,EAAE,MAAM,CAAC;QACnB,6CAA6C;QAC7C,SAAS,EAAE,MAAM,CAAC;QAClB,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,qBAAqB;IACpC,QAAQ,EAAE,YAAY,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,KAAK,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IACnC,mEAAmE;IACnE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,+DAA+D;IAC/D,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;CAClC;AAuBD,kEAAkE;AAClE,wBAAgB,wBAAwB,CACtC,UAAU,EAAE,iBAAiB,EAC7B,QAAQ,EAAE,eAAe,GACxB,gBAAgB,EAAE,CA+HpB;AA0ED,wBAAsB,cAAc,CAClC,IAAI,EAAE,qBAAqB,GAC1B,OAAO,CAAC,iBAAiB,CAAC,CA+C5B;AAED,8CAA8C;AAC9C,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,iBAAiB,GAAG,MAAM,CAgDrE"}