@honua/mcp-server 0.0.9-alpha.0 → 0.0.20-alpha.0

package/dist/src/eval/corpus.d.ts

@@ -0,0 +1,27 @@
+import type { Scenario } from "./types.js";
+/**
+ * Held-out corpus of GIS workflows for the cross-model eval (honua-server #1956).
+ *
+ * Each scenario is a natural-language GIS task plus declarative success criteria.
+ * The `script` is the "ideal client" tool trajectory the deterministic offline
+ * driver follows; live LLM drivers (Claude, GPT) plan their own calls from the
+ * prompt and the advertised tool catalog. Criteria are graded identically across
+ * all drivers, so the same corpus measures any client against any workflow.
+ *
+ * Tool/argument shapes target the canonical honua MCP catalog and resolve
+ * against the offline fixture backend (service "Parks", layer 0).
+ */
+export declare const CORPUS: Scenario[];
+/**
+ * Select the corpus to run from the environment (honua-server #1956).
+ *
+ * The default `analyst` corpus targets the in-process SDK fixture surface and is
+ * the CI control. `operator` selects {@link OPERATOR_CORPUS} for live runs against
+ * the honua-server operator MCP surface; `northstar` selects {@link NORTHSTAR_CORPUS},
+ * the three P1-gate workflows (also live-gated); `all` runs every corpus
+ * back-to-back. Selection is case-insensitive and falls back to the analyst corpus
+ * for any unset/unknown value so CI behavior is unchanged unless `HONUA_EVAL_CORPUS`
+ * is set explicitly.
+ */
+export declare function resolveCorpus(env?: NodeJS.ProcessEnv): Scenario[];
+//# sourceMappingURL=corpus.d.ts.map

package/dist/src/eval/corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus.d.ts","sourceRoot":"","sources":["../../../src/eval/corpus.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,MAAM,EAAE,QAAQ,EAkG5B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,wBAAgB,aAAa,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,QAAQ,EAAE,CAW9E"}

package/dist/src/eval/corpus.js

@@ -0,0 +1,134 @@
+import { NORTHSTAR_CORPUS } from "./northstar-corpus.js";
+import { OPERATOR_CORPUS } from "./operator-corpus.js";
+/**
+ * Held-out corpus of GIS workflows for the cross-model eval (honua-server #1956).
+ *
+ * Each scenario is a natural-language GIS task plus declarative success criteria.
+ * The `script` is the "ideal client" tool trajectory the deterministic offline
+ * driver follows; live LLM drivers (Claude, GPT) plan their own calls from the
+ * prompt and the advertised tool catalog. Criteria are graded identically across
+ * all drivers, so the same corpus measures any client against any workflow.
+ *
+ * Tool/argument shapes target the canonical honua MCP catalog and resolve
+ * against the offline fixture backend (service "Parks", layer 0).
+ */
+export const CORPUS = [
+    {
+        id: "discover-services",
+        title: "Discover available feature services",
+        category: "discovery",
+        prompt: "I'm new to this server. What feature services are available to query? Just list them.",
+        criteria: {
+            requiredTools: ["honua_list_services"],
+            answerMustInclude: ["Parks"],
+        },
+        script: [{ tool: "honua_list_services", args: {} }],
+    },
+    {
+        id: "inspect-layer-schema",
+        title: "Inspect a layer's schema before querying",
+        category: "grounding",
+        prompt: "Before I query the Parks service, show me the schema of its first layer (layer 0) — the fields and geometry type.",
+        criteria: {
+            requiredTools: ["honua_describe_layer"],
+            answerMustInclude: ["OBJECTID"],
+        },
+        script: [{ tool: "honua_describe_layer", args: { serviceId: "Parks", layerId: 0 } }],
+    },
+    {
+        id: "count-then-query",
+        title: "Check cardinality, then fetch a sample",
+        category: "query",
+        prompt: "How many features are in the Parks service layer 0? Then show me a couple of them.",
+        criteria: {
+            requiredTools: ["honua_count_features", "honua_query_features"],
+            expectedToolSequence: ["honua_count_features", "honua_query_features"],
+            answerMustInclude: ["Park A"],
+        },
+        script: [
+            { tool: "honua_count_features", args: { serviceId: "Parks", layerId: 0 } },
+            { tool: "honua_query_features", args: { serviceId: "Parks", layerId: 0, limit: 2 } },
+        ],
+    },
+    {
+        id: "discover-then-query",
+        title: "End-to-end discover → query",
+        category: "multi-step",
+        prompt: "I don't know what's on this server. Find a feature service and show me a few features from one of its layers.",
+        criteria: {
+            requiredTools: ["honua_list_services", "honua_query_features"],
+            expectedToolSequence: ["honua_list_services", "honua_query_features"],
+            answerMustInclude: ["Park"],
+        },
+        script: [
+            { tool: "honua_list_services", args: {} },
+            { tool: "honua_query_features", args: { serviceId: "Parks", layerId: 0, limit: 2 } },
+        ],
+    },
+    {
+        id: "aggregate-statistic",
+        title: "Compute an aggregate statistic",
+        category: "analysis",
+        prompt: "What is the average of the VALUE field for the Parks service, layer 0?",
+        criteria: {
+            requiredTools: ["honua_statistics"],
+            answerMustInclude: ["statistics"],
+        },
+        script: [
+            {
+                tool: "honua_statistics",
+                args: { serviceId: "Parks", layerId: 0, statisticType: "avg", onField: "VALUE" },
+            },
+        ],
+    },
+    {
+        id: "spatial-extent",
+        title: "Get the spatial extent of a layer",
+        category: "query",
+        prompt: "What is the bounding box / spatial extent of the Parks service, layer 0?",
+        criteria: {
+            requiredTools: ["honua_get_extent"],
+            answerMustInclude: ["extent"],
+        },
+        script: [{ tool: "honua_get_extent", args: { serviceId: "Parks", layerId: 0 } }],
+    },
+    {
+        id: "capability-gap",
+        title: "Explain a capability gap honestly",
+        category: "grounding",
+        prompt: "Can I run an attribute query against a WMTS source on this platform? If not, what should I do instead?",
+        criteria: {
+            requiredTools: ["honua_explain_capability_gap"],
+            answerMustInclude: ["explanation"],
+        },
+        script: [
+            {
+                tool: "honua_explain_capability_gap",
+                args: { capability: "query", protocol: "wmts" },
+            },
+        ],
+    },
+];
+/**
+ * Select the corpus to run from the environment (honua-server #1956).
+ *
+ * The default `analyst` corpus targets the in-process SDK fixture surface and is
+ * the CI control. `operator` selects {@link OPERATOR_CORPUS} for live runs against
+ * the honua-server operator MCP surface; `northstar` selects {@link NORTHSTAR_CORPUS},
+ * the three P1-gate workflows (also live-gated); `all` runs every corpus
+ * back-to-back. Selection is case-insensitive and falls back to the analyst corpus
+ * for any unset/unknown value so CI behavior is unchanged unless `HONUA_EVAL_CORPUS`
+ * is set explicitly.
+ */
+export function resolveCorpus(env = process.env) {
+    switch ((env.HONUA_EVAL_CORPUS ?? "").trim().toLowerCase()) {
+        case "operator":
+            return OPERATOR_CORPUS;
+        case "northstar":
+            return NORTHSTAR_CORPUS;
+        case "all":
+            return [...CORPUS, ...OPERATOR_CORPUS, ...NORTHSTAR_CORPUS];
+        default:
+            return CORPUS;
+    }
+}

package/dist/src/eval/drivers/anthropic.d.ts

@@ -0,0 +1,14 @@
+import type { ModelDriver, Scenario, WorkflowContext, WorkflowTranscript } from "../types.js";
+export interface AnthropicDriverOptions {
+    apiKey?: string | undefined;
+    model?: string;
+}
+export declare class AnthropicDriver implements ModelDriver {
+    readonly vendor: "anthropic";
+    readonly id: string;
+    private readonly apiKey;
+    constructor(options?: AnthropicDriverOptions);
+    isAvailable(): boolean;
+    runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
+}
+//# sourceMappingURL=anthropic.d.ts.map

package/dist/src/eval/drivers/anthropic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAkB,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAoC9G,MAAM,WAAW,sBAAsB;IACrC,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,qBAAa,eAAgB,YAAW,WAAW;IACjD,QAAQ,CAAC,MAAM,EAAG,WAAW,CAAU;IACvC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,GAAE,sBAA2B;IAKhD,WAAW,IAAI,OAAO;IAIhB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAoFzF"}

package/dist/src/eval/drivers/anthropic.js

@@ -0,0 +1,120 @@
+/**
+ * Claude client driver for the cross-model eval (honua-server #1956).
+ *
+ * Gated behind ANTHROPIC_API_KEY and a dynamically-imported `@anthropic-ai/sdk`
+ * — neither is a dependency of this package, so the offline CI path never needs
+ * them. Live runs use the latest Opus model (claude-opus-4-8) with a manual
+ * tool-use loop over the advertised MCP catalog. A synthetic `ask_user` tool
+ * lets the model signal that it needs clarification, which the harness records
+ * separately from success/failure.
+ *
+ * Live cross-model runs are tracked under honua-io/honua-server#1956.
+ */
+const ASK_USER_TOOL = {
+    name: "ask_user",
+    description: "Call this ONLY if the request is too ambiguous to proceed. Provide the clarifying question you would ask.",
+    input_schema: {
+        type: "object",
+        properties: { question: { type: "string" } },
+        required: ["question"],
+        additionalProperties: false,
+    },
+};
+const SYSTEM_PROMPT = "You are a GIS analyst working through the Honua MCP tools. Complete the user's task by calling the available tools. " +
+    "Ground every step in real tool results — do not invent service names, layers, or fields. " +
+    "If, and only if, the request is genuinely ambiguous and you cannot proceed, call ask_user. " +
+    "When finished, give a concise final answer that references what the tools returned.";
+const MAX_ITERATIONS = 8;
+const DEFAULT_MODEL = "claude-opus-4-8";
+export class AnthropicDriver {
+    vendor = "anthropic";
+    id;
+    apiKey;
+    constructor(options = {}) {
+        this.id = options.model ?? DEFAULT_MODEL;
+        this.apiKey = options.apiKey ?? process.env.ANTHROPIC_API_KEY;
+    }
+    isAvailable() {
+        return typeof this.apiKey === "string" && this.apiKey.length > 0;
+    }
+    async runWorkflow(scenario, ctx) {
+        const steps = [];
+        const transcript = {
+            scenarioId: scenario.id,
+            modelId: this.id,
+            steps,
+            finalAnswer: "",
+            clarificationRequested: false,
+            errorCount: 0,
+        };
+        let Anthropic;
+        try {
+            // Variable specifier → not statically resolved; keeps @anthropic-ai/sdk
+            // an optional, non-bundled dependency for live runs only.
+            const specifier = "@anthropic-ai/sdk";
+            const mod = (await import(specifier));
+            Anthropic = mod.default;
+        }
+        catch {
+            transcript.driverError = "@anthropic-ai/sdk is not installed (required for live Claude eval runs)";
+            return transcript;
+        }
+        if (!this.apiKey) {
+            transcript.driverError = "ANTHROPIC_API_KEY is not set";
+            return transcript;
+        }
+        const client = new Anthropic({ apiKey: this.apiKey });
+        const tools = [
+            ...ctx.tools.map((t) => ({ name: t.name, description: t.description, input_schema: t.inputSchema })),
+            ASK_USER_TOOL,
+        ];
+        const messages = [{ role: "user", content: scenario.prompt }];
+        try {
+            for (let i = 0; i < MAX_ITERATIONS; i++) {
+                const response = await client.messages.create({
+                    model: this.id,
+                    max_tokens: 4096,
+                    system: SYSTEM_PROMPT,
+                    tools,
+                    messages,
+                });
+                const toolUses = response.content.filter((b) => b.type === "tool_use");
+                const text = response.content
+                    .filter((b) => b.type === "text")
+                    .map((b) => b.text)
+                    .join("\n");
+                if (response.stop_reason !== "tool_use" || toolUses.length === 0) {
+                    transcript.finalAnswer = text;
+                    return transcript;
+                }
+                messages.push({ role: "assistant", content: response.content });
+                const toolResults = [];
+                for (const use of toolUses) {
+                    if (use.name === ASK_USER_TOOL.name) {
+                        transcript.clarificationRequested = true;
+                        transcript.finalAnswer = text;
+                        return transcript;
+                    }
+                    const result = await ctx.callTool(use.name, (use.input ?? {}));
+                    steps.push({ tool: use.name, args: (use.input ?? {}), isError: result.isError });
+                    if (result.isError) {
+                        transcript.errorCount++;
+                    }
+                    toolResults.push({
+                        type: "tool_result",
+                        tool_use_id: use.id,
+                        content: result.text,
+                        is_error: result.isError,
+                    });
+                }
+                messages.push({ role: "user", content: toolResults });
+            }
+            transcript.driverError = `exceeded ${MAX_ITERATIONS} tool-use iterations without finishing`;
+            return transcript;
+        }
+        catch (err) {
+            transcript.driverError = err instanceof Error ? err.message : String(err);
+            return transcript;
+        }
+    }
+}

package/dist/src/eval/drivers/bedrock.d.ts

@@ -0,0 +1,54 @@
+import type { ModelDriver, Scenario, ToolDescriptor, WorkflowContext, WorkflowTranscript } from "../types.js";
+export interface BedrockDriverOptions {
+    /** Bedrock model id or inference-profile id. Defaults to Sonnet 4.5. */
+    model?: string | undefined;
+    /** AWS region. Defaults to AWS_REGION, then us-west-2. */
+    region?: string | undefined;
+    /** Opt-in flag value (defaults to process.env.HONUA_EVAL_BEDROCK). */
+    enabled?: string | undefined;
+}
+export declare class BedrockDriver implements ModelDriver {
+    readonly vendor: "bedrock";
+    readonly id: string;
+    readonly region: string;
+    private readonly enabled;
+    constructor(options?: BedrockDriverOptions);
+    isAvailable(): boolean;
+    runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
+}
+/** Map MCP tool descriptors to Bedrock Converse `toolSpec` entries. */
+export declare function toBedrockTools(tools: ToolDescriptor[]): BedrockTool[];
+/** Pull tool-use blocks out of a Converse content array. */
+export declare function extractToolUses(content: BedrockContentBlock[]): BedrockToolUse[];
+/** Concatenate text blocks of a Converse content array. */
+export declare function extractText(content: BedrockContentBlock[]): string;
+/** Build a Converse `toolResult` content block to feed back to the model. */
+export declare function toToolResultBlock(toolUseId: string, text: string, isError: boolean): BedrockContentBlock;
+interface BedrockToolUse {
+    toolUseId: string;
+    name: string;
+    input?: unknown;
+}
+interface BedrockToolResult {
+    toolUseId: string;
+    content: {
+        text: string;
+    }[];
+    status: "error" | "success";
+}
+interface BedrockContentBlock {
+    text?: string;
+    toolUse?: BedrockToolUse;
+    toolResult?: BedrockToolResult;
+}
+interface BedrockTool {
+    toolSpec: {
+        name: string;
+        description: string;
+        inputSchema: {
+            json: Record<string, unknown>;
+        };
+    };
+}
+export {};
+//# sourceMappingURL=bedrock.d.ts.map

package/dist/src/eval/drivers/bedrock.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bedrock.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/bedrock.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,WAAW,EACX,QAAQ,EACR,cAAc,EAEd,eAAe,EACf,kBAAkB,EACnB,MAAM,aAAa,CAAC;AAgErB,MAAM,WAAW,oBAAoB;IACnC,wEAAwE;IACxE,KAAK,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC3B,0DAA0D;IAC1D,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC5B,sEAAsE;IACtE,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;CAC9B;AAED,qBAAa,aAAc,YAAW,WAAW;IAC/C,QAAQ,CAAC,MAAM,EAAG,SAAS,CAAU;IACrC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;gBAEtB,OAAO,GAAE,oBAAyB;IAO9C,WAAW,IAAI,OAAO;IAIhB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAyEzF;AAUD,uEAAuE;AACvE,wBAAgB,cAAc,CAAC,KAAK,EAAE,cAAc,EAAE,GAAG,WAAW,EAAE,CAQrE;AAED,4DAA4D;AAC5D,wBAAgB,eAAe,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,cAAc,EAAE,CAIhF;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,OAAO,EAAE,mBAAmB,EAAE,GAAG,MAAM,CAKlE;AAED,6EAA6E;AAC7E,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,mBAAmB,CAQxG;AAID,UAAU,cAAc;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AACD,UAAU,iBAAiB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5B,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC;CAC7B;AACD,UAAU,mBAAmB;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,OAAO,CAAC,EAAE,cAAc,CAAC;IACzB,UAAU,CAAC,EAAE,iBAAiB,CAAC;CAChC;AAKD,UAAU,WAAW;IACnB,QAAQ,EAAE;QACR,IAAI,EAAE,MAAM,CAAC;QACb,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;SAAE,CAAC;KAChD,CAAC;CACH"}

package/dist/src/eval/drivers/bedrock.js

@@ -0,0 +1,178 @@
+/**
+ * Claude-via-Amazon-Bedrock driver for the cross-model eval (honua-server #1956).
+ *
+ * This is the "any client LLM" counterpart to the direct-Anthropic driver, but it
+ * reaches Claude through Amazon Bedrock's Converse API instead of the Anthropic
+ * API. It exists so the AI proof can run on infrastructure that has AWS Bedrock
+ * access (default `~/.aws` credential chain) but no ANTHROPIC_API_KEY — exactly
+ * the situation on the build host. The SAME corpus and grading are driven over
+ * the identical MCP catalog, so the artifact compares success / clarification /
+ * edit rates across vendors and transports.
+ *
+ * Auth uses the standard AWS SDK credential provider chain (env vars, shared
+ * config/credentials files, SSO, instance/role credentials) — no key is ever
+ * hardcoded. Because that chain is resolved asynchronously and cannot be sniffed
+ * synchronously, the driver opts in via the `HONUA_EVAL_BEDROCK` env flag (set by
+ * `resolveDrivers`) or by explicit selection (`--driver bedrock`).
+ *
+ * Model id: defaults to `anthropic.claude-sonnet-4-5-20250929-v1:0`, overridable
+ * via `HONUA_EVAL_BEDROCK_MODEL`. NOTE: most current Anthropic models on Bedrock
+ * are not enabled for on-demand throughput by the bare id and must be invoked
+ * through a cross-region inference profile — i.e. the `us.`-prefixed form
+ * `us.anthropic.claude-sonnet-4-5-20250929-v1:0` in us.* regions. Set
+ * `HONUA_EVAL_BEDROCK_MODEL` accordingly for live runs.
+ *
+ * Region: `AWS_REGION` if set, else `us-west-2`.
+ *
+ * `@aws-sdk/client-bedrock-runtime` is a declared dependency but is imported
+ * dynamically (variable specifier) so the offline CI path — which only runs the
+ * deterministic control — never needs to load it, and a graceful `driverError`
+ * is recorded if it is somehow absent.
+ *
+ * Live cross-model runs are tracked under honua-io/honua-server#1956.
+ */
+const ASK_USER_TOOL_NAME = "ask_user";
+const ASK_USER_TOOL = {
+    toolSpec: {
+        name: ASK_USER_TOOL_NAME,
+        description: "Call this ONLY if the request is too ambiguous to proceed. Provide the clarifying question you would ask.",
+        inputSchema: {
+            json: {
+                type: "object",
+                properties: { question: { type: "string" } },
+                required: ["question"],
+                additionalProperties: false,
+            },
+        },
+    },
+};
+const SYSTEM_PROMPT = "You are a GIS analyst working through the Honua MCP tools. Complete the user's task by calling the available tools. " +
+    "Ground every step in real tool results — do not invent service names, layers, or fields. " +
+    "If, and only if, the request is genuinely ambiguous and you cannot proceed, call ask_user. " +
+    "When finished, give a concise final answer that references what the tools returned.";
+const MAX_ITERATIONS = 8;
+const MAX_TOKENS = 4096;
+const DEFAULT_MODEL = "anthropic.claude-sonnet-4-5-20250929-v1:0";
+const DEFAULT_REGION = "us-west-2";
+export class BedrockDriver {
+    vendor = "bedrock";
+    id;
+    region;
+    enabled;
+    constructor(options = {}) {
+        this.id = options.model ?? DEFAULT_MODEL;
+        this.region = options.region ?? DEFAULT_REGION;
+        const flag = options.enabled ?? process.env.HONUA_EVAL_BEDROCK;
+        this.enabled = isTruthy(flag);
+    }
+    isAvailable() {
+        return this.enabled;
+    }
+    async runWorkflow(scenario, ctx) {
+        const steps = [];
+        const transcript = {
+            scenarioId: scenario.id,
+            modelId: this.id,
+            steps,
+            finalAnswer: "",
+            clarificationRequested: false,
+            errorCount: 0,
+        };
+        let runtime;
+        try {
+            // Variable specifier → not statically resolved; keeps the AWS SDK an
+            // optional load for live runs only and the build independent of it.
+            const specifier = "@aws-sdk/client-bedrock-runtime";
+            runtime = (await import(specifier));
+        }
+        catch {
+            transcript.driverError = "@aws-sdk/client-bedrock-runtime is not installed (required for live Bedrock eval runs)";
+            return transcript;
+        }
+        const { BedrockRuntimeClient, ConverseCommand } = runtime;
+        const client = new BedrockRuntimeClient({ region: this.region });
+        const toolConfig = { tools: [...toBedrockTools(ctx.tools), ASK_USER_TOOL] };
+        const messages = [{ role: "user", content: [{ text: scenario.prompt }] }];
+        try {
+            for (let i = 0; i < MAX_ITERATIONS; i++) {
+                const response = await client.send(new ConverseCommand({
+                    modelId: this.id,
+                    system: [{ text: SYSTEM_PROMPT }],
+                    messages,
+                    toolConfig,
+                    inferenceConfig: { maxTokens: MAX_TOKENS },
+                }));
+                const content = response.output?.message?.content ?? [];
+                const toolUses = extractToolUses(content);
+                const text = extractText(content);
+                if (response.stopReason !== "tool_use" || toolUses.length === 0) {
+                    transcript.finalAnswer = text;
+                    return transcript;
+                }
+                messages.push({ role: "assistant", content });
+                const toolResults = [];
+                for (const use of toolUses) {
+                    if (use.name === ASK_USER_TOOL_NAME) {
+                        transcript.clarificationRequested = true;
+                        transcript.finalAnswer = text;
+                        return transcript;
+                    }
+                    const args = (use.input ?? {});
+                    const result = await ctx.callTool(use.name, args);
+                    steps.push({ tool: use.name, args, isError: result.isError });
+                    if (result.isError) {
+                        transcript.errorCount++;
+                    }
+                    toolResults.push(toToolResultBlock(use.toolUseId, result.text, result.isError));
+                }
+                messages.push({ role: "user", content: toolResults });
+            }
+            transcript.driverError = `exceeded ${MAX_ITERATIONS} tool-use iterations without finishing`;
+            return transcript;
+        }
+        catch (err) {
+            transcript.driverError = err instanceof Error ? err.message : String(err);
+            return transcript;
+        }
+    }
+}
+function isTruthy(value) {
+    if (typeof value !== "string") {
+        return false;
+    }
+    const v = value.trim().toLowerCase();
+    return v.length > 0 && v !== "0" && v !== "false" && v !== "no" && v !== "off";
+}
+/** Map MCP tool descriptors to Bedrock Converse `toolSpec` entries. */
+export function toBedrockTools(tools) {
+    return tools.map((t) => ({
+        toolSpec: {
+            name: t.name,
+            description: t.description,
+            inputSchema: { json: (t.inputSchema ?? { type: "object" }) },
+        },
+    }));
+}
+/** Pull tool-use blocks out of a Converse content array. */
+export function extractToolUses(content) {
+    return content
+        .map((b) => b.toolUse)
+        .filter((u) => typeof u === "object" && u !== null && typeof u.name === "string");
+}
+/** Concatenate text blocks of a Converse content array. */
+export function extractText(content) {
+    return content
+        .map((b) => b.text)
+        .filter((t) => typeof t === "string")
+        .join("\n");
+}
+/** Build a Converse `toolResult` content block to feed back to the model. */
+export function toToolResultBlock(toolUseId, text, isError) {
+    return {
+        toolResult: {
+            toolUseId,
+            content: [{ text }],
+            status: isError ? "error" : "success",
+        },
+    };
+}

package/dist/src/eval/drivers/deterministic.d.ts

@@ -0,0 +1,17 @@
+import type { ModelDriver, Scenario, WorkflowContext, WorkflowTranscript } from "../types.js";
+/**
+ * Deterministic offline driver — the CI control for the cross-model eval.
+ *
+ * It does not call any model. Instead it executes the scenario's scripted "ideal
+ * client" tool trajectory against the REAL MCP surface (the same `tools/call`
+ * round-trips a live LLM would make), then synthesizes a final answer from the
+ * concatenated tool results. This makes the offline eval a genuine end-to-end
+ * exercise of the MCP surface — not a stub — while staying fully reproducible.
+ */
+export declare class DeterministicDriver implements ModelDriver {
+    readonly id = "deterministic";
+    readonly vendor: "deterministic";
+    isAvailable(): boolean;
+    runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
+}
+//# sourceMappingURL=deterministic.d.ts.map

package/dist/src/eval/drivers/deterministic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"deterministic.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/deterministic.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAkB,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAE9G;;;;;;;;GAQG;AACH,qBAAa,mBAAoB,YAAW,WAAW;IACrD,QAAQ,CAAC,EAAE,mBAAmB;IAC9B,QAAQ,CAAC,MAAM,EAAG,eAAe,CAAU;IAE3C,WAAW,IAAI,OAAO;IAIhB,WAAW,CAAC,QAAQ,EAAE,QAAQ,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,kBAAkB,CAAC;CAsCzF"}

package/dist/src/eval/drivers/deterministic.js

@@ -0,0 +1,51 @@
+/**
+ * Deterministic offline driver — the CI control for the cross-model eval.
+ *
+ * It does not call any model. Instead it executes the scenario's scripted "ideal
+ * client" tool trajectory against the REAL MCP surface (the same `tools/call`
+ * round-trips a live LLM would make), then synthesizes a final answer from the
+ * concatenated tool results. This makes the offline eval a genuine end-to-end
+ * exercise of the MCP surface — not a stub — while staying fully reproducible.
+ */
+export class DeterministicDriver {
+    id = "deterministic";
+    vendor = "deterministic";
+    isAvailable() {
+        return true;
+    }
+    async runWorkflow(scenario, ctx) {
+        const advertised = new Set(ctx.tools.map((t) => t.name));
+        const steps = [];
+        const answerParts = [];
+        let errorCount = 0;
+        for (const planned of scenario.script) {
+            if (!advertised.has(planned.tool)) {
+                // The scripted tool is not on the surface — record a driver-visible error
+                // (a real catalog regression would surface here, not silently pass).
+                return {
+                    scenarioId: scenario.id,
+                    modelId: this.id,
+                    steps,
+                    finalAnswer: answerParts.join("\n"),
+                    clarificationRequested: false,
+                    errorCount,
+                    driverError: `scripted tool not advertised by the MCP surface: ${planned.tool}`,
+                };
+            }
+            const result = await ctx.callTool(planned.tool, planned.args);
+            steps.push({ tool: planned.tool, args: planned.args, isError: result.isError });
+            if (result.isError) {
+                errorCount++;
+            }
+            answerParts.push(result.text);
+        }
+        return {
+            scenarioId: scenario.id,
+            modelId: this.id,
+            steps,
+            finalAnswer: answerParts.join("\n"),
+            clarificationRequested: false,
+            errorCount,
+        };
+    }
+}

package/dist/src/eval/drivers/index.d.ts

@@ -0,0 +1,34 @@
+import type { ModelDriver } from "../types.js";
+import { AnthropicDriver } from "./anthropic.js";
+import { BedrockDriver } from "./bedrock.js";
+import { DeterministicDriver } from "./deterministic.js";
+import { OpenAiDriver } from "./openai.js";
+export { AnthropicDriver, BedrockDriver, DeterministicDriver, OpenAiDriver };
+/** Names accepted by `selectDrivers` / the CLI `--driver` flag. */
+export type DriverName = "deterministic" | "anthropic" | "openai" | "bedrock";
+export interface ResolveDriversOptions {
+    env?: NodeJS.ProcessEnv;
+    /** Include live drivers even if their key/opt-in is absent (they record driverError). */
+    includeUnavailable?: boolean;
+}
+/**
+ * Resolve the set of model drivers to run.
+ *
+ * The deterministic control is always included. Claude (Anthropic API) and GPT
+ * drivers are added when their API keys are present; the Claude-via-Bedrock driver
+ * is added when `HONUA_EVAL_BEDROCK` is set (its auth comes from the AWS credential
+ * chain, which can't be sniffed from a single env var). This gates live runs behind
+ * env without hardcoding any credentials. CI runs the deterministic driver only —
+ * a green, reproducible artifact with no model/API calls.
+ */
+export declare function resolveDrivers(options?: ResolveDriversOptions): ModelDriver[];
+/**
+ * Build the driver set for an explicit `--driver` selection. The deterministic
+ * control is always included (it is the CI gate). Named live drivers are always
+ * constructed and run — even if their key/opt-in is absent — so an explicit
+ * request surfaces an honest `driverError` rather than being silently dropped.
+ */
+export declare function selectDrivers(names: DriverName[], options?: ResolveDriversOptions): ModelDriver[];
+/** Parse/validate a `--driver` value into a known driver name. */
+export declare function parseDriverName(value: string): DriverName;
+//# sourceMappingURL=index.d.ts.map

package/dist/src/eval/drivers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/eval/drivers/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,mBAAmB,EAAE,YAAY,EAAE,CAAC;AAE7E,mEAAmE;AACnE,MAAM,MAAM,UAAU,GAAG,eAAe,GAAG,WAAW,GAAG,QAAQ,GAAG,SAAS,CAAC;AAE9E,MAAM,WAAW,qBAAqB;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,yFAAyF;IACzF,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,OAAO,GAAE,qBAA0B,GAAG,WAAW,EAAE,CAwBjF;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE,OAAO,GAAE,qBAA0B,GAAG,WAAW,EAAE,CAuBrG;AAED,kEAAkE;AAClE,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,CAMzD"}