@honua/mcp-server 0.0.10-alpha.0 → 0.1.0-beta.0

package/dist/src/eval/drivers/bedrock.js

@@ -0,0 +1,178 @@
+/**
+ * Claude-via-Amazon-Bedrock driver for the cross-model eval (honua-server #1956).
+ *
+ * This is the "any client LLM" counterpart to the direct-Anthropic driver, but it
+ * reaches Claude through Amazon Bedrock's Converse API instead of the Anthropic
+ * API. It exists so the AI proof can run on infrastructure that has AWS Bedrock
+ * access (default `~/.aws` credential chain) but no ANTHROPIC_API_KEY — exactly
+ * the situation on the build host. The SAME corpus and grading are driven over
+ * the identical MCP catalog, so the artifact compares success / clarification /
+ * edit rates across vendors and transports.
+ *
+ * Auth uses the standard AWS SDK credential provider chain (env vars, shared
+ * config/credentials files, SSO, instance/role credentials) — no key is ever
+ * hardcoded. Because that chain is resolved asynchronously and cannot be sniffed
+ * synchronously, the driver opts in via the `HONUA_EVAL_BEDROCK` env flag (set by
+ * `resolveDrivers`) or by explicit selection (`--driver bedrock`).
+ *
+ * Model id: defaults to `anthropic.claude-sonnet-4-5-20250929-v1:0`, overridable
+ * via `HONUA_EVAL_BEDROCK_MODEL`. NOTE: most current Anthropic models on Bedrock
+ * are not enabled for on-demand throughput by the bare id and must be invoked
+ * through a cross-region inference profile — i.e. the `us.`-prefixed form
+ * `us.anthropic.claude-sonnet-4-5-20250929-v1:0` in us.* regions. Set
+ * `HONUA_EVAL_BEDROCK_MODEL` accordingly for live runs.
+ *
+ * Region: `AWS_REGION` if set, else `us-west-2`.
+ *
+ * `@aws-sdk/client-bedrock-runtime` is a declared dependency but is imported
+ * dynamically (variable specifier) so the offline CI path — which only runs the
+ * deterministic control — never needs to load it, and a graceful `driverError`
+ * is recorded if it is somehow absent.
+ *
+ * Live cross-model runs are tracked under honua-io/honua-server#1956.
+ */
+const ASK_USER_TOOL_NAME = "ask_user";
+const ASK_USER_TOOL = {
+    toolSpec: {
+        name: ASK_USER_TOOL_NAME,
+        description: "Call this ONLY if the request is too ambiguous to proceed. Provide the clarifying question you would ask.",
+        inputSchema: {
+            json: {
+                type: "object",
+                properties: { question: { type: "string" } },
+                required: ["question"],
+                additionalProperties: false,
+            },
+        },
+    },
+};
+const SYSTEM_PROMPT = "You are a GIS analyst working through the Honua MCP tools. Complete the user's task by calling the available tools. " +
+    "Ground every step in real tool results — do not invent service names, layers, or fields. " +
+    "If, and only if, the request is genuinely ambiguous and you cannot proceed, call ask_user. " +
+    "When finished, give a concise final answer that references what the tools returned.";
+const MAX_ITERATIONS = 8;
+const MAX_TOKENS = 4096;
+const DEFAULT_MODEL = "anthropic.claude-sonnet-4-5-20250929-v1:0";
+const DEFAULT_REGION = "us-west-2";
+export class BedrockDriver {
+    vendor = "bedrock";
+    id;
+    region;
+    enabled;
+    constructor(options = {}) {
+        this.id = options.model ?? DEFAULT_MODEL;
+        this.region = options.region ?? DEFAULT_REGION;
+        const flag = options.enabled ?? process.env.HONUA_EVAL_BEDROCK;
+        this.enabled = isTruthy(flag);
+    }
+    isAvailable() {
+        return this.enabled;
+    }
+    async runWorkflow(scenario, ctx) {
+        const steps = [];
+        const transcript = {
+            scenarioId: scenario.id,
+            modelId: this.id,
+            steps,
+            finalAnswer: "",
+            clarificationRequested: false,
+            errorCount: 0,
+        };
+        let runtime;
+        try {
+            // Variable specifier → not statically resolved; keeps the AWS SDK an
+            // optional load for live runs only and the build independent of it.
+            const specifier = "@aws-sdk/client-bedrock-runtime";
+            runtime = (await import(specifier));
+        }
+        catch {
+            transcript.driverError = "@aws-sdk/client-bedrock-runtime is not installed (required for live Bedrock eval runs)";
+            return transcript;
+        }
+        const { BedrockRuntimeClient, ConverseCommand } = runtime;
+        const client = new BedrockRuntimeClient({ region: this.region });
+        const toolConfig = { tools: [...toBedrockTools(ctx.tools), ASK_USER_TOOL] };
+        const messages = [{ role: "user", content: [{ text: scenario.prompt }] }];
+        try {
+            for (let i = 0; i < MAX_ITERATIONS; i++) {
+                const response = await client.send(new ConverseCommand({
+                    modelId: this.id,
+                    system: [{ text: SYSTEM_PROMPT }],
+                    messages,
+                    toolConfig,
+                    inferenceConfig: { maxTokens: MAX_TOKENS },
+                }));
+                const content = response.output?.message?.content ?? [];
+                const toolUses = extractToolUses(content);
+                const text = extractText(content);
+                if (response.stopReason !== "tool_use" || toolUses.length === 0) {
+                    transcript.finalAnswer = text;
+                    return transcript;
+                }
+                messages.push({ role: "assistant", content });
+                const toolResults = [];
+                for (const use of toolUses) {
+                    if (use.name === ASK_USER_TOOL_NAME) {
+                        transcript.clarificationRequested = true;
+                        transcript.finalAnswer = text;
+                        return transcript;
+                    }
+                    const args = (use.input ?? {});
+                    const result = await ctx.callTool(use.name, args);
+                    steps.push({ tool: use.name, args, isError: result.isError });
+                    if (result.isError) {
+                        transcript.errorCount++;
+                    }
+                    toolResults.push(toToolResultBlock(use.toolUseId, result.text, result.isError));
+                }
+                messages.push({ role: "user", content: toolResults });
+            }
+            transcript.driverError = `exceeded ${MAX_ITERATIONS} tool-use iterations without finishing`;
+            return transcript;
+        }
+        catch (err) {
+            transcript.driverError = err instanceof Error ? err.message : String(err);
+            return transcript;
+        }
+    }
+}
+function isTruthy(value) {
+    if (typeof value !== "string") {
+        return false;
+    }
+    const v = value.trim().toLowerCase();
+    return v.length > 0 && v !== "0" && v !== "false" && v !== "no" && v !== "off";
+}
+/** Map MCP tool descriptors to Bedrock Converse `toolSpec` entries. */
+export function toBedrockTools(tools) {
+    return tools.map((t) => ({
+        toolSpec: {
+            name: t.name,
+            description: t.description,
+            inputSchema: { json: (t.inputSchema ?? { type: "object" }) },
+        },
+    }));
+}
+/** Pull tool-use blocks out of a Converse content array. */
+export function extractToolUses(content) {
+    return content
+        .map((b) => b.toolUse)
+        .filter((u) => typeof u === "object" && u !== null && typeof u.name === "string");
+}
+/** Concatenate text blocks of a Converse content array. */
+export function extractText(content) {
+    return content
+        .map((b) => b.text)
+        .filter((t) => typeof t === "string")
+        .join("\n");
+}
+/** Build a Converse `toolResult` content block to feed back to the model. */
+export function toToolResultBlock(toolUseId, text, isError) {
+    return {
+        toolResult: {
+            toolUseId,
+            content: [{ text }],
+            status: isError ? "error" : "success",
+        },
+    };
+}

package/dist/src/eval/drivers/deterministic.d.ts

@@ -0,0 +1,17 @@
+import type { ModelDriver, Scenario, WorkflowContext, WorkflowTranscript } from "../types.js";
+/**
+ * Deterministic offline driver — the CI control for the cross-model eval.
+ *
+ * It does not call any model. Instead it executes the scenario's scripted "ideal
+ * client" tool trajectory against the REAL MCP surface (the same `tools/call`
+ * round-trips a live LLM would make), then synthesizes a final answer from the
+ * concatenated tool results. This makes the offline eval a genuine end-to-end
+ * exercise of the MCP surface — not a stub — while staying fully reproducible.
+ */
+export declare class DeterministicDriver implements ModelDriver {
+    readonly id = "deterministic";
+    readonly vendor: "deterministic";
+    isAvailable(): boolean;
+    runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
+}
+//# sourceMappingURL=deterministic.d.ts.map

package/dist/src/eval/drivers/deterministic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAkB,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAE9G;;;;;;;;GAQG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IACrD,QAAQ,CAAC,EAAE,mBAAmB;IAC9B,QAAQ,CAAC,MAAM,EAAG,eAAe,CAAU;IAE3C,WAAW,IAAI,OAAO;IAIhB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAoDzF"}

package/dist/src/eval/drivers/deterministic.js

@@ -0,0 +1,64 @@
+/**
+ * Deterministic offline driver — the CI control for the cross-model eval.
+ *
+ * It does not call any model. Instead it executes the scenario's scripted "ideal
+ * client" tool trajectory against the REAL MCP surface (the same `tools/call`
+ * round-trips a live LLM would make), then synthesizes a final answer from the
+ * concatenated tool results. This makes the offline eval a genuine end-to-end
+ * exercise of the MCP surface — not a stub — while staying fully reproducible.
+ */
+export class DeterministicDriver {
+    id = "deterministic";
+    vendor = "deterministic";
+    isAvailable() {
+        return true;
+    }
+    async runWorkflow(scenario, ctx) {
+        const advertised = new Set(ctx.tools.map((t) => t.name));
+        const steps = [];
+        const answerParts = [];
+        let errorCount = 0;
+        // Ambiguity/refusal scenarios: the ideal client asks for clarification rather
+        // than guessing. Honoring this deterministically lets the offline control
+        // exercise refusal/clarification behavior, not just happy-path trajectories.
+        if (scenario.clarify) {
+            return {
+                scenarioId: scenario.id,
+                modelId: this.id,
+                steps,
+                finalAnswer: scenario.clarify.question,
+                clarificationRequested: true,
+                errorCount,
+            };
+        }
+        for (const planned of scenario.script) {
+            if (!advertised.has(planned.tool)) {
+                // The scripted tool is not on the surface — record a driver-visible error
+                // (a real catalog regression would surface here, not silently pass).
+                return {
+                    scenarioId: scenario.id,
+                    modelId: this.id,
+                    steps,
+                    finalAnswer: answerParts.join("\n"),
+                    clarificationRequested: false,
+                    errorCount,
+                    driverError: `scripted tool not advertised by the MCP surface: ${planned.tool}`,
+                };
+            }
+            const result = await ctx.callTool(planned.tool, planned.args);
+            steps.push({ tool: planned.tool, args: planned.args, isError: result.isError });
+            if (result.isError) {
+                errorCount++;
+            }
+            answerParts.push(result.text);
+        }
+        return {
+            scenarioId: scenario.id,
+            modelId: this.id,
+            steps,
+            finalAnswer: answerParts.join("\n"),
+            clarificationRequested: false,
+            errorCount,
+        };
+    }
+}

package/dist/src/eval/drivers/index.d.ts

@@ -0,0 +1,34 @@
+import type { ModelDriver } from "../types.js";
+import { AnthropicDriver } from "./anthropic.js";
+import { BedrockDriver } from "./bedrock.js";
+import { DeterministicDriver } from "./deterministic.js";
+import { OpenAiDriver } from "./openai.js";
+export { AnthropicDriver, BedrockDriver, DeterministicDriver, OpenAiDriver };
+/** Names accepted by `selectDrivers` / the CLI `--driver` flag. */
+export type DriverName = "deterministic" | "anthropic" | "openai" | "bedrock";
+export interface ResolveDriversOptions {
+    env?: NodeJS.ProcessEnv;
+    /** Include live drivers even if their key/opt-in is absent (they record driverError). */
+    includeUnavailable?: boolean;
+}
+/**
+ * Resolve the set of model drivers to run.
+ *
+ * The deterministic control is always included. Claude (Anthropic API) and GPT
+ * drivers are added when their API keys are present; the Claude-via-Bedrock driver
+ * is added when `HONUA_EVAL_BEDROCK` is set (its auth comes from the AWS credential
+ * chain, which can't be sniffed from a single env var). This gates live runs behind
+ * env without hardcoding any credentials. CI runs the deterministic driver only —
+ * a green, reproducible artifact with no model/API calls.
+ */
+export declare function resolveDrivers(options?: ResolveDriversOptions): ModelDriver[];
+/**
+ * Build the driver set for an explicit `--driver` selection. The deterministic
+ * control is always included (it is the CI gate). Named live drivers are always
+ * constructed and run — even if their key/opt-in is absent — so an explicit
+ * request surfaces an honest `driverError` rather than being silently dropped.
+ */
+export declare function selectDrivers(names: DriverName[], options?: ResolveDriversOptions): ModelDriver[];
+/** Parse/validate a `--driver` value into a known driver name. */
+export declare function parseDriverName(value: string): DriverName;
+//# sourceMappingURL=index.d.ts.map

package/dist/src/eval/drivers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,mBAAmB,EAAE,YAAY,EAAE,CAAC;AAE7E,mEAAmE;AACnE,MAAM,MAAM,UAAU,GAAG,eAAe,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,qBAAqB;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,yFAAyF;IACzF,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,OAAO,GAAE,qBAA0B,GAAG,WAAW,EAAE,CAwBjF;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,OAAO,GAAE,qBAA0B,GAAG,WAAW,EAAE,CAuBrG;AAED,kEAAkE;AAClE,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,CAMzD"}

package/dist/src/eval/drivers/index.js

@@ -0,0 +1,74 @@
+import { AnthropicDriver } from "./anthropic.js";
+import { BedrockDriver } from "./bedrock.js";
+import { DeterministicDriver } from "./deterministic.js";
+import { OpenAiDriver } from "./openai.js";
+export { AnthropicDriver, BedrockDriver, DeterministicDriver, OpenAiDriver };
+/**
+ * Resolve the set of model drivers to run.
+ *
+ * The deterministic control is always included. Claude (Anthropic API) and GPT
+ * drivers are added when their API keys are present; the Claude-via-Bedrock driver
+ * is added when `HONUA_EVAL_BEDROCK` is set (its auth comes from the AWS credential
+ * chain, which can't be sniffed from a single env var). This gates live runs behind
+ * env without hardcoding any credentials. CI runs the deterministic driver only —
+ * a green, reproducible artifact with no model/API calls.
+ */
+export function resolveDrivers(options = {}) {
+    const env = options.env ?? process.env;
+    const drivers = [new DeterministicDriver()];
+    const anthropic = new AnthropicDriver({ apiKey: env.ANTHROPIC_API_KEY, model: env.HONUA_EVAL_ANTHROPIC_MODEL });
+    if (options.includeUnavailable || anthropic.isAvailable()) {
+        drivers.push(anthropic);
+    }
+    const openai = new OpenAiDriver({ apiKey: env.OPENAI_API_KEY, model: env.OPENAI_MODEL });
+    if (options.includeUnavailable || openai.isAvailable()) {
+        drivers.push(openai);
+    }
+    const bedrock = new BedrockDriver({
+        enabled: env.HONUA_EVAL_BEDROCK,
+        model: env.HONUA_EVAL_BEDROCK_MODEL,
+        region: env.AWS_REGION,
+    });
+    if (options.includeUnavailable || bedrock.isAvailable()) {
+        drivers.push(bedrock);
+    }
+    return drivers;
+}
+/**
+ * Build the driver set for an explicit `--driver` selection. The deterministic
+ * control is always included (it is the CI gate). Named live drivers are always
+ * constructed and run — even if their key/opt-in is absent — so an explicit
+ * request surfaces an honest `driverError` rather than being silently dropped.
+ */
+export function selectDrivers(names, options = {}) {
+    const env = options.env ?? process.env;
+    const drivers = [new DeterministicDriver()];
+    for (const name of names) {
+        switch (name) {
+            case "deterministic":
+                break; // already included
+            case "anthropic":
+                drivers.push(new AnthropicDriver({ apiKey: env.ANTHROPIC_API_KEY, model: env.HONUA_EVAL_ANTHROPIC_MODEL }));
+                break;
+            case "openai":
+                drivers.push(new OpenAiDriver({ apiKey: env.OPENAI_API_KEY, model: env.OPENAI_MODEL }));
+                break;
+            case "bedrock":
+                drivers.push(new BedrockDriver({ enabled: "1", model: env.HONUA_EVAL_BEDROCK_MODEL, region: env.AWS_REGION }));
+                break;
+            default: {
+                const exhaustive = name;
+                throw new Error(`unknown driver: ${String(exhaustive)}`);
+            }
+        }
+    }
+    return drivers;
+}
+/** Parse/validate a `--driver` value into a known driver name. */
+export function parseDriverName(value) {
+    const known = ["deterministic", "anthropic", "openai", "bedrock"];
+    if (known.includes(value)) {
+        return value;
+    }
+    throw new Error(`unknown --driver "${value}" (expected one of: ${known.join(", ")})`);
+}

package/dist/src/eval/drivers/openai.d.ts

@@ -0,0 +1,14 @@
+import type { ModelDriver, Scenario, WorkflowContext, WorkflowTranscript } from "../types.js";
+export interface OpenAiDriverOptions {
+    apiKey?: string | undefined;
+    model?: string;
+}
+export declare class OpenAiDriver implements ModelDriver {
+    readonly vendor: "openai";
+    readonly id: string;
+    private readonly apiKey;
+    constructor(options?: OpenAiDriverOptions);
+    isAvailable(): boolean;
+    runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
+}
+//# sourceMappingURL=openai.d.ts.map

package/dist/src/eval/drivers/openai.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/openai.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAkB,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAkD9G,MAAM,WAAW,mBAAmB;IAClC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,YAAa,YAAW,WAAW;IAC9C,QAAQ,CAAC,MAAM,EAAG,QAAQ,CAAU;IACpC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,GAAE,mBAAwB;IAK7C,WAAW,IAAI,OAAO;IAIhB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;CA4EzF"}

package/dist/src/eval/drivers/openai.js

@@ -0,0 +1,136 @@
+/**
+ * GPT client driver for the cross-model eval (honua-server #1956).
+ *
+ * Gated behind OPENAI_API_KEY and a dynamically-imported `openai` SDK — neither
+ * is a dependency of this package, so the offline CI path never needs them. This
+ * is the "any client" counterpart to the Claude driver: the SAME corpus and the
+ * SAME grading are driven by a GPT model over the identical MCP catalog, so the
+ * artifact can compare success / clarification / edit rates across vendors.
+ *
+ * Live cross-model runs are tracked under honua-io/honua-server#1956.
+ */
+const ASK_USER_FUNCTION = {
+    type: "function",
+    function: {
+        name: "ask_user",
+        description: "Call this ONLY if the request is too ambiguous to proceed. Provide the clarifying question you would ask.",
+        parameters: {
+            type: "object",
+            properties: { question: { type: "string" } },
+            required: ["question"],
+            additionalProperties: false,
+        },
+    },
+};
+const SYSTEM_PROMPT = "You are a GIS analyst working through the Honua MCP tools. Complete the user's task by calling the available tools. " +
+    "Ground every step in real tool results — do not invent service names, layers, or fields. " +
+    "If, and only if, the request is genuinely ambiguous and you cannot proceed, call ask_user. " +
+    "When finished, give a concise final answer that references what the tools returned.";
+const MAX_ITERATIONS = 8;
+/**
+ * Default GPT model for live cross-model runs.
+ *
+ * `gpt-5.5` is OpenAI's generally-available flagship reasoning model as of
+ * July 2026 (the apex of the public Chat Completions catalog; see
+ * developers.openai.com/api/docs/models/all). The newer GPT-5.6 "Sol" family was
+ * still preview-gated to a handful of orgs at that date and not callable by an
+ * ordinary API key, so it is NOT the default — set `OPENAI_MODEL=gpt-5.6-sol`
+ * (or any other id) to override once you have access. The resolved id is recorded
+ * per-driver in the results artifact (`models[].model`) so published comparisons
+ * are attributable to an exact model.
+ */
+const DEFAULT_MODEL = "gpt-5.5";
+export class OpenAiDriver {
+    vendor = "openai";
+    id;
+    apiKey;
+    constructor(options = {}) {
+        this.id = options.model ?? process.env.OPENAI_MODEL ?? DEFAULT_MODEL;
+        this.apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
+    }
+    isAvailable() {
+        return typeof this.apiKey === "string" && this.apiKey.length > 0;
+    }
+    async runWorkflow(scenario, ctx) {
+        const steps = [];
+        const transcript = {
+            scenarioId: scenario.id,
+            modelId: this.id,
+            steps,
+            finalAnswer: "",
+            clarificationRequested: false,
+            errorCount: 0,
+        };
+        let OpenAI;
+        try {
+            const specifier = "openai";
+            const mod = (await import(specifier));
+            OpenAI = mod.default;
+        }
+        catch {
+            transcript.driverError = "openai SDK is not installed (required for live GPT eval runs)";
+            return transcript;
+        }
+        if (!this.apiKey) {
+            transcript.driverError = "OPENAI_API_KEY is not set";
+            return transcript;
+        }
+        const client = new OpenAI({ apiKey: this.apiKey });
+        const tools = [
+            ...ctx.tools.map((t) => ({
+                type: "function",
+                function: { name: t.name, description: t.description, parameters: t.inputSchema },
+            })),
+            ASK_USER_FUNCTION,
+        ];
+        const messages = [
+            { role: "system", content: SYSTEM_PROMPT },
+            { role: "user", content: scenario.prompt },
+        ];
+        try {
+            for (let i = 0; i < MAX_ITERATIONS; i++) {
+                const response = await client.chat.completions.create({ model: this.id, messages, tools });
+                const message = response.choices[0]?.message;
+                if (!message) {
+                    transcript.driverError = "no choice returned by the model";
+                    return transcript;
+                }
+                const toolCalls = message.tool_calls ?? [];
+                if (toolCalls.length === 0) {
+                    transcript.finalAnswer = message.content ?? "";
+                    return transcript;
+                }
+                messages.push({ role: "assistant", content: message.content ?? null, tool_calls: toolCalls });
+                for (const call of toolCalls) {
+                    const args = safeParse(call.function.arguments);
+                    if (call.function.name === "ask_user") {
+                        transcript.clarificationRequested = true;
+                        transcript.finalAnswer = message.content ?? "";
+                        return transcript;
+                    }
+                    const result = await ctx.callTool(call.function.name, args);
+                    steps.push({ tool: call.function.name, args, isError: result.isError });
+                    if (result.isError) {
+                        transcript.errorCount++;
+                    }
+                    messages.push({ role: "tool", tool_call_id: call.id, content: result.text });
+                }
+            }
+            transcript.driverError = `exceeded ${MAX_ITERATIONS} tool-use iterations without finishing`;
+            return transcript;
+        }
+        catch (err) {
+            transcript.driverError = err instanceof Error ? err.message : String(err);
+            return transcript;
+        }
+    }
+}
+function safeParse(raw) {
+    try {
+        const parsed = JSON.parse(raw);
+        return typeof parsed === "object" && parsed !== null ? parsed : {};
+    }
+    catch {
+        return {};
+    }
+}

package/dist/src/eval/grade.d.ts

@@ -0,0 +1,14 @@
+import type { Scenario, ScenarioGrade, WorkflowTranscript } from "./types.js";
+/** True if `sequence` appears as an ordered (not necessarily contiguous) subsequence of `steps`. */
+export declare function isOrderedSubsequence(steps: string[], sequence: string[]): boolean;
+/**
+ * Grade a transcript against a scenario's declarative success criteria.
+ *
+ * Outcomes:
+ *  - `error`: the driver failed to run (unavailable model, exception).
+ *  - `clarified`: the driver asked a clarifying question instead of completing.
+ *  - `pass`: all criteria satisfied.
+ *  - `fail`: completed but violated one or more criteria.
+ */
+export declare function grade(scenario: Scenario, transcript: WorkflowTranscript): ScenarioGrade;
+//# sourceMappingURL=grade.d.ts.map

package/dist/src/eval/grade.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"grade.d.ts","sourceRoot":"","sources":["../../../src/eval/grade.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE9E,oGAAoG;AACpG,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAQjF;AAED;;;;;;;;GAQG;AACH,wBAAgB,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,kBAAkB,GAAG,aAAa,CAgEvF"}

package/dist/src/eval/grade.js

@@ -0,0 +1,75 @@
+/** True if `sequence` appears as an ordered (not necessarily contiguous) subsequence of `steps`. */
+export function isOrderedSubsequence(steps, sequence) {
+    let i = 0;
+    for (const step of steps) {
+        if (i < sequence.length && step === sequence[i]) {
+            i++;
+        }
+    }
+    return i === sequence.length;
+}
+/**
+ * Grade a transcript against a scenario's declarative success criteria.
+ *
+ * Outcomes:
+ *  - `error`: the driver failed to run (unavailable model, exception).
+ *  - `clarified`: the driver asked a clarifying question instead of completing.
+ *  - `pass`: all criteria satisfied.
+ *  - `fail`: completed but violated one or more criteria.
+ */
+export function grade(scenario, transcript) {
+    const base = {
+        scenarioId: scenario.id,
+        modelId: transcript.modelId,
+        errorCount: transcript.errorCount,
+    };
+    const { criteria } = scenario;
+    if (transcript.driverError) {
+        return { ...base, outcome: "error", violations: [transcript.driverError] };
+    }
+    if (transcript.clarificationRequested) {
+        // For an ambiguity/refusal scenario, clarifying IS the correct behavior.
+        if (criteria.expectClarification) {
+            return { ...base, outcome: "pass", violations: [] };
+        }
+        return { ...base, outcome: "clarified", violations: ["driver requested clarification"] };
+    }
+    if (criteria.expectClarification) {
+        // The prompt was ambiguous/unsupported but the driver answered anyway.
+        return { ...base, outcome: "fail", violations: ["expected a clarifying question but the driver completed"] };
+    }
+    const violations = [];
+    const calledTools = transcript.steps.map((s) => s.tool);
+    const calledSet = new Set(calledTools);
+    for (const required of criteria.requiredTools) {
+        if (!calledSet.has(required)) {
+            violations.push(`required tool not called: ${required}`);
+        }
+    }
+    if (criteria.expectedToolSequence && !isOrderedSubsequence(calledTools, criteria.expectedToolSequence)) {
+        violations.push(`expected tool sequence not observed: ${criteria.expectedToolSequence.join(" -> ")}`);
+    }
+    for (const forbidden of criteria.forbiddenTools ?? []) {
+        if (calledSet.has(forbidden)) {
+            violations.push(`forbidden tool called: ${forbidden}`);
+        }
+    }
+    const answer = transcript.finalAnswer.toLowerCase();
+    for (const fragment of criteria.answerMustInclude ?? []) {
+        if (!answer.includes(fragment.toLowerCase())) {
+            violations.push(`answer missing expected content: "${fragment}"`);
+        }
+    }
+    // Semantic assertions (issue #369): meaning, not just trajectory.
+    for (const fragment of criteria.answerMustNotInclude ?? []) {
+        if (answer.includes(fragment.toLowerCase())) {
+            violations.push(`answer contains forbidden content: "${fragment}"`);
+        }
+    }
+    for (const pattern of criteria.answerMustMatch ?? []) {
+        if (!new RegExp(pattern, "i").test(transcript.finalAnswer)) {
+            violations.push(`answer did not match required pattern: /${pattern}/i`);
+        }
+    }
+    return { ...base, outcome: violations.length === 0 ? "pass" : "fail", violations };
+}

package/dist/src/eval/northstar-corpus.d.ts

@@ -0,0 +1,3 @@
+import type { Scenario } from "./types.js";
+export declare const NORTHSTAR_CORPUS: Scenario[];
+//# sourceMappingURL=northstar-corpus.d.ts.map

package/dist/src/eval/northstar-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"northstar-corpus.d.ts","sourceRoot":"","sources":["../../../src/eval/northstar-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAiF3C,eAAO,MAAM,gBAAgB,EAAE,QAAQ,EAyHtC,CAAC"}