@womp/spark-agent 6.1.1 → 6.3.0

package/dist/intent-bench-flow-catalog.js

@@ -0,0 +1,4 @@
+// Swan's Flow node catalog (`flowNodeCatalog()` in swan-frontend src/llm/flowbot/editableDocument.ts),
+// copied from crow-backend src/utils/agent/flow-node-catalog.generated.ts on 2026-10-08 for
+// intent-bench runs. Refresh it from there when the Flow node surface changes.
+export const INTENT_BENCH_FLOW_CATALOG = "Connect only the listed input and output handles. Set literal-only settings with values, never refs. generateImage exposes Image1 through ImageN according to :num_images (default 1); use Image1, not its hidden array outputs.\n\nai.llm\nConnectable inputs:\n  System: base.string\n  Prompt: base.string\n  Image1: res.bitmap\n  Image2: res.bitmap\n  Image3: res.bitmap\n  Image4: res.bitmap\nConnectable outputs:\n  Result: base.string\nLiteral-only settings (no connections):\n  Model: enum:ai.llmModel\n\nai.llmText\nConnectable inputs:\n  System: base.string\n  Prompt1: base.string\n  Prompt2: base.string\n  Prompt3: base.string\n  Prompt4: base.string\nConnectable outputs:\n  Result: base.string\nLiteral-only settings (no connections):\n  Model: enum:ai.llmModel\n\nai.removeImageBackground\nConnectable inputs:\n  Image: res.image\n  Model: enum:ai.processImage.removeBackgroundModel\n  Sensitivity: base.float\nConnectable outputs:\n  Image: res.bitmap\nLiteral-only settings (no connections):\n  Temporary: base.bool\n\nai.upscaleImage\nConnectable inputs:\n  Image: res.bitmap\n  Model: enum:ai.upscaleModel\n  Scale: base.float\nConnectable outputs:\n  Image: res.bitmap\nLiteral-only settings (no connections):\n  Temporary: base.bool\n\nbase.input.string\nConnectable inputs:\n  Value: base.string\nConnectable outputs:\n  Result: base.string\nLiteral-only settings (no connections):\n\n\ndb.scriptInstance\nConnectable inputs:\n\nConnectable outputs:\n  Result: db.scriptInstance\nLiteral-only settings (no connections):\n\n\next.ai.generateImage\nConnectable inputs:\n  Prompt: base.string\n  Image1: res.bitmap\n  Image2: res.bitmap\n  Image3: res.bitmap\n  Image4: res.bitmap\nConnectable outputs:\n  Image1: res.bitmap\n  Image2: res.bitmap - only with :num_images 2 or more\n  Image3: res.bitmap - only with :num_images 3 or more\n  Image4: res.bitmap - only with :num_images 4 or more\nLiteral-only settings (no connections):\n  PromptPreset: base.string\n  AiWorkflow: enum:db.aiWorkflowId\n  Seed: base.float\n  RandomSeed: base.bool\n  NumImages: base.float\n  Temporary: base.bool\n\next.ai.generateMesh\nConnectable inputs:\n  Image: res.bitmap\n  Back: res.bitmap\n  Left: res.bitmap\n  Right: res.bitmap\nConnectable outputs:\n  Mesh: res.mesh\nLiteral-only settings (no connections):\n  AiWorkflow: enum:db.aiWorkflowId\n  Seed: base.float\n  RandomSeed: base.bool\n\next.edit.paper\nConnectable inputs:\n  Background: res.image\n  Overlays: res.image[]\n  Width: base.float\n  Height: base.float\n  Title: base.string\nConnectable outputs:\n  Image: res.bitmap\nLiteral-only settings (no connections):\n\n\next.scene.area\nConnectable inputs:\n  Script: db.scriptInstance\nConnectable outputs:\n  AreaId: enum:scene.areaId\nLiteral-only settings (no connections):\n  ProjectId: enum:db.projectId\n  AreaId: enum:scene.areaId\n  Hide: base.bool\n\nscene.importAsset\nConnectable inputs:\n  Image: res.bitmap\n  Mesh: res.mesh\n  AreaId: enum:scene.areaId\nConnectable outputs:\n  AreaId: enum:scene.areaId\nLiteral-only settings (no connections):\n  ProjectId: enum:db.projectId\n  AssetType: enum:db.assetType\n  LibraryOnly: base.bool\n\nscene.printEstimate\nConnectable inputs:\n  Target: enum:scene.areaId | res.mesh[]\n  Material: base.string\n  Color: base.string\n  Scale: base.float\n  Thickness: base.float\n  DisableAIPreview: base.bool\nConnectable outputs:\n  Preview: res.image\nLiteral-only settings (no connections):\n  ProjectId: enum:db.projectId\n\nviewer.images\nConnectable inputs:\n  Images: res.bitmap[]\nConnectable outputs:\n  Image: res.bitmap\nLiteral-only settings (no connections):\n  ImageIndex: base.float\n\nviewer.meshes\nConnectable inputs:\n  Meshes: res.mesh[]\nConnectable outputs:\n  Mesh: res.mesh\nLiteral-only settings (no connections):\n  MeshIndex: base.float";

package/dist/intent-bench.d.ts

@@ -0,0 +1,66 @@
+import { type SparkAgentOptions } from "./agent.js";
+import type { IntentCase, IntentRoute } from "./intent-bench-cases.js";
+import type { SparkRunSettings } from "./server/types.js";
+export type IntentDecision = {
+    route: IntentRoute;
+    /** The first call that would have changed something; the run stopped there. */
+    call: string;
+    detail?: unknown;
+    /** The JavaScript it ran in. */
+    code?: string;
+};
+/** A scene edit made in the same call as another route's write belongs to that route. */
+export declare function intentRoute(decision: IntentDecision): IntentRoute;
+export type IntentRun = {
+    caseId: string;
+    repeat: number;
+    route: IntentRoute;
+    pass: boolean;
+    decision?: IntentDecision;
+    /** Skills the model loaded, in order. */
+    skills: string[];
+    /** Agent API calls the fixtures served, in order. */
+    calls: string[];
+    /** Fixture reads that could not answer; the run may not reflect production. */
+    fixtureErrors: string[];
+    text: string;
+    /** Thinking, skill loads, code and its outcome, and replies, in order. */
+    transcript: string[];
+    steps: number;
+    costUsd: number | null;
+    durationMs: number;
+    error?: string;
+};
+/** A solid-color PNG, so conversation cases need no image files. */
+export declare function solidPng([r, g, b]: [number, number, number], size?: number): Uint8Array;
+export type IntentModel = Pick<SparkAgentOptions, "model" | "modelId" | "providerOptions" | "maxOutputTokens">;
+/** The standalone server's provider settings for a Spark model. */
+export declare function intentBenchModel(settings: SparkRunSettings): IntentModel;
+export declare function runIntentCase(options: {
+    testCase: IntentCase;
+    repeat: number;
+    model: IntentModel;
+    maxSteps: number;
+    timeoutMs: number;
+    signal?: AbortSignal;
+}): Promise<IntentRun>;
+export type IntentCaseSummary = {
+    id: string;
+    expect: readonly IntentRoute[];
+    runs: number;
+    passed: number;
+    routes: Partial<Record<IntentRoute, number>>;
+    note?: string;
+};
+export declare function summarizeIntentRuns(cases: readonly IntentCase[], runs: readonly IntentRun[]): IntentCaseSummary[];
+/** Runs every case `repeat` times, at most `concurrency` at once. */
+export declare function runIntentBench(options: {
+    cases: readonly IntentCase[];
+    repeat: number;
+    concurrency: number;
+    model: IntentModel;
+    maxSteps: number;
+    timeoutMs: number;
+    signal?: AbortSignal;
+    onRun?(run: IntentRun): void;
+}): Promise<IntentRun[]>;

package/dist/intent-bench.js

@@ -0,0 +1,555 @@
+import { crc32, deflateSync } from "node:zlib";
+import { createAnthropic } from "@ai-sdk/anthropic";
+import { createGoogleGenerativeAI } from "@ai-sdk/google";
+import { createOpenAI } from "@ai-sdk/openai";
+import { runSparkAgent } from "./agent.js";
+import { composeCapabilities, createAccountCapability, createAssetsCapability, createFlowCapability, createGeometryCapability, createKnowledgeCapability, createMaterialsCapability, createOpenScadCapability, createOperationsCapability, createSceneCapability, } from "./capabilities.js";
+import { geometryReference } from "./geometry.js";
+import { INTENT_BENCH_FLOW_CATALOG } from "./intent-bench-flow-catalog.js";
+import { sparkModelOptions } from "./model-options.js";
+import { getSparkModel } from "./models.js";
+import { QuickJSJavaScriptExecutor } from "./quickjs-executor.js";
+import { createBenchMetrics } from "./run-metrics.js";
+import { createWompSceneApi } from "./womp-scene-api.js";
+/**
+ * Skills that edit the scene to prepare their own target in the same call: Geometry Nodes
+ * creates an Area in scene.access and reads it as its target, for example.
+ */
+const PREPARED_WRITES = [
+    [/\bwomp\.geometry\.(?:read|write)\s*\(/u, "geometry-nodes"],
+    [/\bwomp\.openscad\.(?:create|update)\s*\(/u, "openscad"],
+];
+/** A scene edit made in the same call as another route's write belongs to that route. */
+export function intentRoute(decision) {
+    const { route, code } = decision;
+    if (route !== "scene" || code === undefined)
+        return route;
+    return PREPARED_WRITES.find(([pattern]) => pattern.test(code))?.[1] ?? "scene";
+}
+const FLOW_CLOSED = "Open the Flow canvas before using flow-graphs.";
+const GENERATION_ROUTES = {
+    "text-to-image": "image",
+    "image-to-image": "image-edit",
+    "upscale-image": "image-edit",
+    "remove-image-background": "image-edit",
+    "text-to-mesh": "mesh",
+    "image-to-mesh": "image-to-mesh",
+};
+/** A solid-color PNG, so conversation cases need no image files. */
+export function solidPng([r, g, b], size = 64) {
+    const chunk = (type, data) => {
+        const body = Buffer.concat([Buffer.from(type, "ascii"), data]);
+        const length = Buffer.alloc(4);
+        length.writeUInt32BE(data.length);
+        const crc = Buffer.alloc(4);
+        crc.writeUInt32BE(crc32(body));
+        return Buffer.concat([length, body, crc]);
+    };
+    const header = Buffer.alloc(13);
+    header.writeUInt32BE(size, 0);
+    header.writeUInt32BE(size, 4);
+    header.set([8, 2, 0, 0, 0], 8);
+    const row = Buffer.concat([Buffer.from([0]), Buffer.from(Array.from({ length: size }, () => [r, g, b]).flat())]);
+    return new Uint8Array(Buffer.concat([
+        Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
+        chunk("IHDR", header),
+        chunk("IDAT", deflateSync(Buffer.concat(Array.from({ length: size }, () => row)))),
+        chunk("IEND", Buffer.alloc(0)),
+    ]));
+}
+// Each photo is downloaded once per process and shared by every run that attaches it.
+const downloads = new Map();
+async function download(url) {
+    const response = await fetch(url, { signal: AbortSignal.timeout(30_000) });
+    const mediaType = response.headers.get("content-type")?.split(";")[0]?.trim() ?? "";
+    if (!response.ok || !mediaType.startsWith("image/")) {
+        throw new Error(`Cannot download the case image ${url}: HTTP ${response.status} ${mediaType}`.trim());
+    }
+    return { name: decodeURIComponent(new URL(url).pathname.split("/").pop()), mediaType, data: new Uint8Array(await response.arrayBuffer()) };
+}
+async function loadImage(image) {
+    if ("color" in image)
+        return { name: image.name, mediaType: "image/png", data: solidPng(image.color) };
+    let loaded = downloads.get(image.url);
+    if (!loaded) {
+        loaded = download(image.url);
+        downloads.set(image.url, loaded);
+        loaded.catch(() => downloads.delete(image.url));
+    }
+    return loaded;
+}
+class IntentDecided extends Error {
+    decision;
+    constructor(decision) {
+        super(`Intent recorded: ${decision.route}`);
+        this.decision = decision;
+        this.name = "IntentDecided";
+    }
+}
+/**
+ * Swan's capability set over fixtures. Reads answer from an empty scene and the case's
+ * conversation; the first call that would change the scene, generate, write a graph or
+ * import is the decision, and stops the run before it takes effect.
+ */
+async function intentFixtures(testCase, controller) {
+    const calls = [];
+    const fixtureErrors = [];
+    let decision;
+    // Spark runs one execute_javascript call at a time.
+    let code;
+    const note = (call) => calls.push(call);
+    const decide = (route, call, detail) => {
+        note(call);
+        decision ??= { route, call, detail, ...(code === undefined ? {} : { code }) };
+        controller.abort(new IntentDecided(decision));
+        throw new IntentDecided(decision);
+    };
+    /** An error production would also return, such as a node that does not exist. */
+    const refuse = (call, message) => {
+        note(call);
+        throw new Error(message);
+    };
+    /** A read these fixtures cannot answer; the run may then differ from production. */
+    const missing = (call, message) => {
+        fixtureErrors.push(call);
+        return refuse(call, message);
+    };
+    const files = new Map();
+    const assets = [];
+    const messages = [];
+    const addFile = async (image, messageId, source) => {
+        const loaded = await loadImage(image);
+        const id = `${source}-${messageId}-${files.size + 1}`;
+        files.set(id, { artifact: { id, url: `fixture://${id}`, mediaType: loaded.mediaType, name: loaded.name }, data: loaded.data });
+        assets.push({ artifact: { id, mediaType: loaded.mediaType, name: loaded.name }, messageId, kind: "image", source });
+        return loaded;
+    };
+    for (const [index, turn] of testCase.turns.entries()) {
+        const messageId = 1001 + index;
+        if (turn.role === "assistant") {
+            for (const image of turn.generated ?? [])
+                await addFile(image, messageId, "generated");
+            messages.push({ role: "assistant", content: turn.text });
+            continue;
+        }
+        const content = [];
+        for (const image of turn.images ?? []) {
+            const loaded = await addFile(image, messageId, "uploaded");
+            content.push({ type: "file", data: loaded.data, mediaType: loaded.mediaType, filename: loaded.name });
+        }
+        if (turn.text)
+            content.push({ type: "text", text: turn.text });
+        messages.push({ role: "user", content });
+    }
+    const screenshot = { id: "scene-screenshot", mediaType: "image/png", name: "scene.png" };
+    files.set(screenshot.id, { artifact: { ...screenshot, url: "fixture://scene-screenshot" }, data: solidPng([228, 228, 228]) });
+    // An empty scene: the root group and the default material. Materials are prepared for
+    // whichever route follows (Geometry Nodes assigns them too), so material edits apply here
+    // and do not decide the route; a run that only edits materials is a scene edit.
+    const transform = { position: { x: 0, y: 0, z: 0 }, rotation: { x: 0, y: 0, z: 0 }, scale: { x: 1, y: 1, z: 1 } };
+    // The root, plus the case's OpenSCAD parts from earlier turns.
+    const nodes = [
+        { id: 0, kind: "group", name: "", parentId: null, pickable: true, transform },
+        ...(testCase.openScadParts ?? []).map((part) => ({
+            id: part.objectId, kind: "openScad", name: part.title, parentId: part.parentId, pickable: true, transform,
+        })),
+    ];
+    const node = (id) => nodes.find((item) => item.id === id) ?? refuse(`scene.getNode(${id})`, `Node ${id} does not exist.`);
+    const materials = new Map([
+        [1, { id: 1, data: { color: { x: 0.74, y: 0.74, z: 0.74 }, roughness: 0.22, metalness: 0 }, shaders: [] }],
+    ]);
+    const ext = { materialsExt: { 1: { name: "Default Material", tier: 0 } } };
+    const materialEdits = [];
+    const material = (id) => materials.get(id) ?? refuse(`scene.getMaterial(${id})`, `Material ${id} does not exist.`);
+    const sceneReads = {
+        getNode: (id) => structuredClone(node(id)),
+        getNodeChildren: (id) => nodes.filter((item) => item.id !== 0 && item.parentId === id).map((item) => structuredClone(item)),
+        listNodes: () => nodes.map((item) => structuredClone(item)),
+        listNodeSummaries: () => nodes.map(({ id, name, kind, parentId }) => ({ id, name, kind, parentId })),
+        getSceneExt: () => structuredClone(ext),
+        getSceneProperties: () => ({}),
+        listMaterials: () => [...materials.values()].map((item) => structuredClone(item)),
+        getMaterial: (id) => structuredClone(material(id)),
+        createMaterial: (data) => {
+            const id = Math.max(...materials.keys()) + 1;
+            materials.set(id, { id, data: structuredClone(data), shaders: [] });
+            return structuredClone(material(id));
+        },
+        updateMaterial: (id, patch) => {
+            const item = material(id);
+            item.data = { ...item.data, ...structuredClone(patch) };
+            return structuredClone(item);
+        },
+        deleteMaterial: (id) => {
+            const item = material(id);
+            materials.delete(id);
+            return item;
+        },
+        // Metadata: material names here; node metadata needs a node, which only a decision creates.
+        patchSceneExt: (operations) => {
+            for (const { op, path, value } of operations) {
+                const keys = path.slice(1).split("/");
+                const last = keys.pop();
+                let parent = ext;
+                for (const key of keys)
+                    parent = (parent[key] ??= {});
+                if (op === "remove")
+                    delete parent[last];
+                else
+                    parent[last] = structuredClone(value);
+            }
+            return undefined;
+        },
+    };
+    const scene = new Proxy(sceneReads, {
+        get: (target, key) => typeof key !== "string" ? undefined : target[key] ?? ((...args) => decide("scene", `scene.${key}`, args)),
+    });
+    const MATERIAL_EDITS = new Set(["createMaterial", "updateMaterial", "deleteMaterial", "materialImport"]);
+    const sceneCall = (method, args, action) => {
+        if (method === "getNodeBoundingBox")
+            return action();
+        if (!MATERIAL_EDITS.has(method))
+            return decide("scene", `scene.${method}`, args);
+        note(`scene.${method}`);
+        materialEdits.push({ call: `scene.${method}`, args });
+        return action();
+    };
+    const bounds = { min: { x: 0, y: 0, z: 0 }, max: { x: 0, y: 0, z: 0 }, center: { x: 0, y: 0, z: 0 }, size: { x: 0, y: 0, z: 0 } };
+    const draft = () => createWompSceneApi({
+        scene: scene,
+        engine: { getNodeBoundingBox: async () => bounds },
+        rootId: 0,
+        getDefaultMaterialId: () => 1,
+        fonts: [],
+        requireNode: (id) => {
+            node(id);
+        },
+        importLibraryMaterial: async (name) => {
+            const id = Math.max(...materials.keys()) + 1;
+            materials.set(id, { id, data: { name }, shaders: [] });
+            return id;
+        },
+        trackNode: () => undefined,
+        trackMaterial: () => undefined,
+        untrackMaterial: () => undefined,
+        listMaterialIds: () => [...materials.keys()],
+        requireMaterial: (id) => {
+            material(id);
+        },
+        call: sceneCall,
+        callAsync: sceneCall,
+        print: () => undefined,
+    });
+    const flow = testCase.flow ?? "closed";
+    const stage = testCase.stage ?? true;
+    const bindings = {
+        assets: createAssetsCapability({
+            list: async (options) => {
+                note("assets.list");
+                const items = assets.filter((item) => (options.kind === undefined || item.kind === options.kind) &&
+                    (options.source === undefined || item.source === options.source) &&
+                    (options.messageId === undefined || item.messageId === options.messageId));
+                return { items: items.slice(options.from, options.from + options.count), nextFrom: null };
+            },
+        }, stage
+            ? {
+                importMesh: async (request) => decide("import", "assets.importMesh", request),
+                importImage: async (request) => decide("import", "assets.importImage", request),
+                retryMeshImport: async (request) => decide("import", "assets.retryMeshImport", request),
+                readImport: async (key) => refuse("assets.readImport", `No import with key ${key}.`),
+                finishImport: async (key) => refuse("assets.finishImport", `No import with key ${key}.`),
+            }
+            : undefined, stage
+            ? {
+                importTexture: async (request) => decide("import", "assets.importTexture", request),
+                readTextureImport: async (key) => refuse("assets.readTextureImport", `No texture import with key ${key}.`),
+            }
+            : undefined),
+        operations: createOperationsCapability({
+            catalog: async () => {
+                note("operations.catalog");
+                return {
+                    models: [
+                        { id: "womp-image", name: "Womp Image", requiresSubscription: false, operations: ["text-to-image", "image-to-image", "upscale-image", "remove-image-background"] },
+                        { id: "womp-mesh", name: "Womp Mesh", requiresSubscription: false, operations: ["text-to-mesh", "image-to-mesh"] },
+                    ],
+                    defaults: { image: "womp-image", editing: "womp-image", mesh: "womp-mesh" },
+                    confirmationRequired: false,
+                };
+            },
+            list: async () => {
+                note("operations.list");
+                return { jobs: [], nextFrom: null };
+            },
+            start: async (request) => decide(GENERATION_ROUTES[request.type], `operations.generate:${request.type}`, request),
+            read: async (messageId) => missing("operations.read", `No job ${messageId}.`),
+            wait: async (messageId) => missing("operations.wait", `No job ${messageId}.`),
+        }),
+        account: createAccountCapability({ read: async () => missing("account.read", "Account information is unavailable right now.") }),
+        knowledge: createKnowledgeCapability({ search: async () => missing("knowledge.search", "Help search is unavailable right now.") }),
+    };
+    if (flow !== "unavailable") {
+        bindings.flow = createFlowCapability({
+            read: async () => {
+                if (flow === "closed")
+                    return refuse("flow.read", FLOW_CLOSED);
+                note("flow.read");
+                return { flowId: "intent-bench", source: "", revision: "1", preservedInputs: {}, diagnostics: [] };
+            },
+            write: async (options) => (flow === "closed" ? refuse("flow.write", FLOW_CLOSED) : decide("flow", "flow.write", options)),
+        }, INTENT_BENCH_FLOW_CATALOG);
+    }
+    if (stage) {
+        Object.assign(bindings, {
+            scene: createSceneCapability({
+                health: async () => {
+                    note("scene.health");
+                    return { status: "within_limits", freshness: "snapshot", revisionVerified: false, affectedAreaIds: [] };
+                },
+                access: async (callback) => {
+                    note("scene.access");
+                    return callback(draft());
+                },
+                screenshot: async () => {
+                    note("scene.screenshot");
+                    return screenshot;
+                },
+            }),
+            materials: createMaterialsCapability({
+                // An empty node catalog: enough to plan plain colors, not procedural layers.
+                catalog: async () => {
+                    note("materials.catalog");
+                    fixtureErrors.push("materials.catalog (no shader nodes)");
+                    return { operations: [], nodes: {}, textureCount: 0 };
+                },
+                read: async () => {
+                    note("materials.read");
+                    return {
+                        materials: [...materials.keys()].map((id) => ({ id, shaderIds: [], objectIds: [] })),
+                        material: { id: 1, data: structuredClone(material(1).data), layers: [] },
+                        diagnostics: [],
+                    };
+                },
+                write: async (options) => decide("scene", "materials.write", options),
+                status: async () => {
+                    note("materials.status");
+                    return { remaining: 0, pendingResources: {}, rendering: "unverified", explanation: "No material work is pending." };
+                },
+            }),
+            geometry: createGeometryCapability({
+                // The scene root is the creation container.
+                read: async (objectId) => {
+                    if (objectId !== 0)
+                        return refuse("geometry.read", `Object ${objectId} has no Geometry Nodes graph.`);
+                    note("geometry.read");
+                    return { kind: "container", objectId: 0, title: "Scene", source: null, instances: [], diagnostics: [] };
+                },
+                write: async (options) => decide("geometry-nodes", "geometry.write", options),
+            }, geometryReference()),
+            openscad: createOpenScadCapability({
+                read: async (objectId) => {
+                    const part = objectId === 0
+                        ? { kind: "container", objectId: 0, parentId: 0, title: "Scene", source: null, params: {} }
+                        : testCase.openScadParts?.find((item) => item.objectId === objectId);
+                    if (!part)
+                        return refuse("openscad.read", `Object ${objectId} is not an OpenSCAD part.`);
+                    note("openscad.read");
+                    return structuredClone(part);
+                },
+                create: async (options) => decide("openscad", "openscad.create", options),
+                update: async (options) => decide("openscad", "openscad.update", options),
+            }),
+        });
+    }
+    const executor = new QuickJSJavaScriptExecutor();
+    return {
+        capabilities: composeCapabilities(bindings),
+        artifacts: {
+            resolve: async (reference) => {
+                const file = files.get(reference.id);
+                if (!file)
+                    throw new Error(`Artifact unavailable: ${reference.id}`);
+                return file;
+            },
+        },
+        messages,
+        messageId: String(1000 + testCase.turns.length),
+        executor: {
+            execute: (...args) => {
+                code = args[0];
+                return executor.execute(...args);
+            },
+        },
+        result: () => ({
+            calls,
+            fixtureErrors,
+            decision: decision ?? (materialEdits[0] && { route: "scene", call: materialEdits[0].call, detail: materialEdits[0].args }),
+        }),
+    };
+}
+/** The standalone server's provider settings for a Spark model. */
+export function intentBenchModel(settings) {
+    const selected = getSparkModel(settings.modelId);
+    const key = { openai: "OPENAI_API_KEY", anthropic: "ANTHROPIC_API_KEY", google: "GEMINI_API_KEY" }[selected.provider];
+    const apiKey = process.env[key]?.trim();
+    if (!apiKey)
+        throw new Error(`${key} is required in the environment or .env.`);
+    const model = selected.provider === "anthropic" ? createAnthropic({ apiKey })(selected.id)
+        : selected.provider === "google" ? createGoogleGenerativeAI({ apiKey })(selected.id)
+            : createOpenAI({ apiKey }).responses(selected.id);
+    return {
+        model,
+        modelId: selected.id,
+        providerOptions: sparkModelOptions(selected.id, selected.provider, settings.reasoningEffort),
+        maxOutputTokens: 48_000,
+    };
+}
+export async function runIntentCase(options) {
+    const { testCase } = options;
+    const startedAt = Date.now();
+    const controller = new AbortController();
+    const onAbort = () => controller.abort(options.signal?.reason);
+    options.signal?.addEventListener("abort", onAbort, { once: true });
+    const timer = setTimeout(() => controller.abort(new Error(`Timed out after ${options.timeoutMs / 1000} s.`)), options.timeoutMs);
+    const fixtures = await intentFixtures(testCase, controller);
+    const metrics = createBenchMetrics();
+    const skills = [];
+    let streamed = "";
+    const transcript = [];
+    let open;
+    const append = (kind, delta) => {
+        if (open?.kind !== kind) {
+            if (open)
+                transcript.push(`[${open.kind}] ${open.text.trim()}`);
+            open = { kind, text: "" };
+        }
+        open.text += delta;
+    };
+    const entry = (line) => {
+        if (open)
+            transcript.push(`[${open.kind}] ${open.text.trim()}`);
+        open = undefined;
+        transcript.push(line);
+    };
+    let completed;
+    let error;
+    try {
+        const run = runSparkAgent({
+            ...options.model,
+            messages: fixtures.messages,
+            capabilities: fixtures.capabilities,
+            executor: fixtures.executor,
+            artifacts: fixtures.artifacts,
+            runId: `intent-${testCase.id}-${options.repeat}`,
+            messageId: fixtures.messageId,
+            signal: controller.signal,
+            maxSteps: options.maxSteps,
+        });
+        const reader = run.stream.getReader();
+        try {
+            for (;;) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                const chunk = value;
+                metrics.accept(chunk);
+                if (chunk.type === "text-delta") {
+                    streamed += chunk.delta;
+                    append("reply", chunk.delta);
+                }
+                if (chunk.type === "reasoning-delta")
+                    append("thinking", chunk.delta);
+                if (chunk.type === "tool-input-available") {
+                    const input = chunk.input;
+                    entry(chunk.toolName === "execute_javascript"
+                        ? `[run] ${String(input.purpose ?? "")}
+${String(input.code ?? "")}`
+                        : `[${chunk.toolName}] ${JSON.stringify(input)}`);
+                }
+                if (chunk.type === "tool-output-available") {
+                    const output = chunk.output;
+                    entry(`[result] ${output?.ok === false ? `error: ${String(output.error)}` : JSON.stringify(output?.logs ?? output).slice(0, 600)}`);
+                }
+                if (chunk.type === "tool-output-error")
+                    entry(`[result] error: ${chunk.errorText}`);
+                if (chunk.type === "tool-input-available" && chunk.toolName === "read_skill") {
+                    const names = chunk.input.names;
+                    if (Array.isArray(names))
+                        skills.push(...names.filter((name) => typeof name === "string"));
+                }
+            }
+        }
+        catch {
+            // The run's result reports why the stream stopped.
+        }
+        if (open)
+            transcript.push(`[${open.kind}] ${open.text.trim()}`);
+        const result = await run.result;
+        if (result.status === "completed")
+            completed = result.text;
+    }
+    catch (caught) {
+        if (!(controller.signal.reason instanceof IntentDecided)) {
+            const reason = controller.signal.aborted ? controller.signal.reason : caught;
+            error = reason instanceof Error ? reason.message : String(reason);
+        }
+    }
+    finally {
+        clearTimeout(timer);
+        options.signal?.removeEventListener("abort", onAbort);
+    }
+    const { calls, fixtureErrors, decision } = fixtures.result();
+    const route = decision
+        ? intentRoute(decision)
+        : error ? "error" : completed !== undefined ? "reply" : "undecided";
+    const snapshot = metrics.snapshot();
+    return {
+        caseId: testCase.id,
+        repeat: options.repeat,
+        route,
+        pass: testCase.expect.includes(route),
+        ...(decision ? { decision } : {}),
+        skills,
+        calls,
+        fixtureErrors,
+        text: (completed ?? streamed).trim(),
+        transcript,
+        steps: snapshot.steps,
+        costUsd: snapshot.costUsd,
+        durationMs: Date.now() - startedAt,
+        ...(error ? { error } : {}),
+    };
+}
+export function summarizeIntentRuns(cases, runs) {
+    return cases.map((testCase) => {
+        const own = runs.filter((run) => run.caseId === testCase.id);
+        const routes = {};
+        for (const run of own)
+            routes[run.route] = (routes[run.route] ?? 0) + 1;
+        return {
+            id: testCase.id,
+            expect: testCase.expect,
+            runs: own.length,
+            passed: own.filter((run) => run.pass).length,
+            routes,
+            ...(testCase.note ? { note: testCase.note } : {}),
+        };
+    });
+}
+/** Runs every case `repeat` times, at most `concurrency` at once. */
+export async function runIntentBench(options) {
+    const jobs = options.cases.flatMap((testCase) => Array.from({ length: options.repeat }, (_, repeat) => ({ testCase, repeat })));
+    const runs = [];
+    let next = 0;
+    const worker = async () => {
+        while (next < jobs.length && !options.signal?.aborted) {
+            const job = jobs[next++];
+            const run = await runIntentCase({ ...job, model: options.model, maxSteps: options.maxSteps, timeoutMs: options.timeoutMs, signal: options.signal });
+            runs.push(run);
+            options.onRun?.(run);
+        }
+    };
+    await Promise.all(Array.from({ length: Math.min(options.concurrency, jobs.length) }, worker));
+    const order = new Map(options.cases.map((testCase, index) => [testCase.id, index]));
+    return runs.sort((a, b) => order.get(a.caseId) - order.get(b.caseId) || a.repeat - b.repeat);
+}

package/dist/model-history.js

@@ -22,6 +22,16 @@ async function liveInput(tools, part) {
     const validated = await asSchema(tool.inputSchema).validate?.(input);
     return validated?.success ? { ...part, input: validated.value } : part;
 }
+/** How the worker reads the conditions the user confirmed for the goal a message starts. */
+function goalConditionsText({ conditions }) {
+    const list = (measurable) => conditions.filter((condition) => condition.measurable === measurable).map(({ text }) => `\n- ${text}`).join("");
+    const measured = list(true);
+    const reviewed = list(false);
+    return "[The user confirmed this request as a goal. Keep working until it is met; do not ask whether to continue."
+        + (measured ? `\nConditions an independent check will verify. Verify each with concrete tool results before claiming completion:${measured}` : "")
+        + (reviewed ? `\nPreferences the user will review themselves:${reviewed}` : "")
+        + "]";
+}
 /** UI artifact previews already represented by a tool result are not model output. */
 export async function convertSparkMessages(messages, tools) {
     const history = await Promise.all(messages.map(async (message) => {
@@ -53,6 +63,10 @@ export async function convertSparkMessages(messages, tools) {
     return convertToModelMessages(history, {
         tools,
         convertDataPart: (part) => {
+            if (part.type === "data-goal") {
+                const goal = sparkDataPartSchemas.goal.safeParse(part.data);
+                return goal.success ? { type: "text", text: goalConditionsText(goal.data) } : undefined;
+            }
             if (part.type !== "data-selection")
                 return undefined;
             const selection = sparkDataPartSchemas.selection.safeParse(part.data);