@honua/mcp-server 0.0.9-alpha.0 → 0.0.20-alpha.0

package/dist/src/eval/operator-corpus.js

@@ -0,0 +1,193 @@
+/**
+ * Operator-surface corpus for the cross-model eval (honua-server #1956).
+ *
+ * The analyst corpus in `corpus.ts` targets the SDK `@honua/mcp-server` ANALYST
+ * tools (count_features, describe_layer, query_features, …). This corpus instead
+ * exercises the LIVE honua-server OPERATOR MCP surface — the agent-native control
+ * plane published at `https://demo.honua.io/mcp` (MCP streamable-HTTP, protocol
+ * 2025-03-26, serverInfo `honua.operator.mcp`). Its tools are the grounding →
+ * clarify → plan → validate → dry-run → (gated) execute lifecycle plus read-only
+ * data tools (list layers, query features) and Pro tools (geocode, route).
+ *
+ * Select it with `HONUA_EVAL_CORPUS=operator` (see `resolveCorpus` in corpus.ts)
+ * and run live with `--driver bedrock` + `HONUA_MCP_REMOTE_URL=<demo>/mcp` +
+ * `HONUA_API_KEY=<demo key>`. The same grading runs for every driver, so the
+ * deterministic control and Bedrock-Claude are measured against identical
+ * criteria over the identical live catalog.
+ *
+ * SAFETY: these scenarios are READ-MOSTLY and reversible. The end-to-end planning
+ * scenarios stop at validate / dry-run and FORBID the mutating lifecycle tools
+ * (`honua_execute_plan`, `honua_propose_operation`) — on the Community-edition
+ * demo `honua_propose_operation` executes directly (no approval gate), so a
+ * correct agent must NOT call it for a read-only analysis task. Forbidding those
+ * tools turns "did the agent stay read-only?" into a graded criterion.
+ *
+ * Grading is deliberately tool-centric: required-tool selection, ordered
+ * subsequence, and forbidden-tool avoidance — the verifiable signals of "the
+ * client composed the right workflow". Answer-content checks are used only where
+ * a data value (a layer name, a geometry word) is near-certain to be echoed, so a
+ * non-pass means a real capability/composition gap rather than a phrasing
+ * artifact. Tools gated by edition entitlements (geocode/route) are scored on
+ * correct tool selection alone (see `operator-geocode-gap`).
+ *
+ * Dataset shapes resolve against the seeded Maui demo content: the published
+ * service `svc-publish-maui-buildings` (layer 13, Overture building footprints).
+ */
+/** Seeded demo service/layer the scenarios bind to (from honua_list_layers). */
+const MAUI_BUILDINGS_SERVICE = "svc-publish-maui-buildings";
+const MAUI_BUILDINGS_LAYER = 13;
+/** Mutating lifecycle tools a read-only analysis agent must never call. */
+const MUTATING_TOOLS = ["honua_execute_plan", "honua_propose_operation"];
+/** A small, static, well-formed analysis plan reused by validate/dry-run. */
+const SAMPLE_PLAN = {
+    planId: "eval-operator-plan-1",
+    intentId: "eval-operator-intent-1",
+    steps: [
+        {
+            stepId: "s1",
+            kind: "QueryFeatures",
+            inputs: { serviceId: MAUI_BUILDINGS_SERVICE, layerId: String(MAUI_BUILDINGS_LAYER) },
+        },
+    ],
+    outputs: ["Scalar"],
+};
+export const OPERATOR_CORPUS = [
+    {
+        id: "operator-list-layers",
+        title: "Discover the published layers on the operator surface",
+        category: "discovery",
+        prompt: "I just connected to this Honua server. What map layers are published here that I could query or map? Just list what's available.",
+        criteria: {
+            requiredTools: ["honua_list_layers"],
+            answerMustInclude: ["maui"],
+        },
+        script: [{ tool: "honua_list_layers", args: {} }],
+    },
+    {
+        id: "operator-discover-then-query",
+        title: "Discover a layer, then query features from it",
+        category: "query",
+        prompt: "Find the Maui buildings layer on this server and show me a few of its building features so I can see what attributes they carry.",
+        criteria: {
+            requiredTools: ["honua_list_layers", "honua_query_features"],
+            expectedToolSequence: ["honua_list_layers", "honua_query_features"],
+            forbiddenTools: MUTATING_TOOLS,
+            answerMustInclude: ["building"],
+        },
+        script: [
+            { tool: "honua_list_layers", args: { filter: "maui-buildings" } },
+            {
+                tool: "honua_query_features",
+                args: { serviceId: MAUI_BUILDINGS_SERVICE, layerId: MAUI_BUILDINGS_LAYER, limit: 3 },
+            },
+        ],
+    },
+    {
+        id: "operator-ground-intent",
+        title: "Ground an ambiguous analysis goal to a workflow",
+        category: "grounding",
+        prompt: "I want to count how many Maui buildings fall within 100 meters of the coast. Help me get started — figure out what kind of workflow this is and what data and processes apply. Don't run anything yet.",
+        criteria: {
+            requiredTools: ["honua_ground_candidates"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            {
+                tool: "honua_ground_candidates",
+                args: {
+                    goal: "count how many Maui buildings fall within 100 meters of the coast",
+                    assumptionPolicy: "AskAlways",
+                },
+            },
+        ],
+    },
+    {
+        id: "operator-clarify-loop",
+        title: "Ground then answer the clarification to refine intent",
+        category: "multi-step",
+        prompt: "I want to buffer the Maui buildings by 100 meters and count how many features fall in the buffered area. Start the grounding pass, and when the server asks a clarifying question, answer it (the buffer distance is 100 meters, the input is the maui-buildings layer). Stop once the intent is refined — do not execute.",
+        criteria: {
+            requiredTools: ["honua_ground_candidates", "honua_clarify_intent"],
+            expectedToolSequence: ["honua_ground_candidates", "honua_clarify_intent"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            {
+                tool: "honua_ground_candidates",
+                args: {
+                    goal: "buffer the maui buildings by 100 meters and count how many fall in the area",
+                    assumptionPolicy: "AskAlways",
+                },
+            },
+            {
+                tool: "honua_clarify_intent",
+                args: {
+                    goal: "buffer the maui buildings by 100 meters and count how many fall in the area",
+                    intentId: "eval-operator-clarify-1",
+                    response: { answers: { "param.distance": ["100"], "param.input": ["maui-buildings"] } },
+                },
+            },
+        ],
+    },
+    {
+        id: "operator-plan-validate",
+        title: "End-to-end read-only planning: ground → plan → validate",
+        category: "multi-step",
+        prompt: "I need an executable analysis plan to count the Maui buildings, but I am NOT ready to run it. Ground the goal, compile it into an analysis plan, and validate that the plan is executable and what approvals it would need. Do not execute or propose any operation.",
+        criteria: {
+            requiredTools: ["honua_ground_candidates", "honua_plan_analysis", "honua_validate_plan"],
+            expectedToolSequence: ["honua_ground_candidates", "honua_plan_analysis", "honua_validate_plan"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            { tool: "honua_ground_candidates", args: { goal: "count the maui buildings", assumptionPolicy: "UseDefaults" } },
+            { tool: "honua_plan_analysis", args: { intent: "count the features in the maui-buildings layer" } },
+            { tool: "honua_validate_plan", args: { plan: SAMPLE_PLAN } },
+        ],
+    },
+    {
+        id: "operator-dry-run",
+        title: "Estimate a plan with a dry run before executing",
+        category: "analysis",
+        prompt: "Before I commit to running an analysis that queries the Maui buildings layer, give me a dry-run estimate of how long it would take, what kind of artifacts it would produce, and any side effects. Do not actually execute it.",
+        criteria: {
+            requiredTools: ["honua_dry_run_plan"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [{ tool: "honua_dry_run_plan", args: { plan: SAMPLE_PLAN } }],
+    },
+    {
+        id: "operator-validate-package",
+        title: "Validate a query package against the governance contract",
+        category: "governance",
+        prompt: 'Run a read-only validation/review of a query package and tell me whether it is ready to execute and whether it needs approval. The package family is \'query\' and its entire payload is just this layer binding: { "serviceId": "svc-publish-maui-buildings", "layerId": 13 }. Validate it exactly as given — do not ask me for anything more.',
+        criteria: {
+            requiredTools: ["honua_validate_package"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            {
+                tool: "honua_validate_package",
+                args: {
+                    packageFamily: "query",
+                    packagePayload: { serviceId: MAUI_BUILDINGS_SERVICE, layerId: MAUI_BUILDINGS_LAYER },
+                },
+            },
+        ],
+    },
+    {
+        id: "operator-geocode-gap",
+        title: "Route an address task to geocoding (entitlement-gated)",
+        category: "capability-gap",
+        prompt: "I have a street address — 'Kahului, Maui, Hawaii'. Get me the coordinates for it using this server. If the server can't do it, tell me honestly why instead of guessing coordinates.",
+        criteria: {
+            // Pro-gated on the Community demo: scored on correct tool SELECTION (the
+            // client must route an address task to the geocoder rather than fabricate
+            // coordinates). The honest gap report is asserted qualitatively, not via a
+            // brittle answer-substring match. forbiddenTools keeps it read-only.
+            requiredTools: ["honua_geocode_address"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [{ tool: "honua_geocode_address", args: { address: "Kahului, Maui, Hawaii", maxResults: 3 } }],
+    },
+];

package/dist/src/eval/report.d.ts

@@ -0,0 +1,104 @@
+import type { ModelDriver, Scenario, ScenarioGrade, WorkflowTranscript } from "./types.js";
+/** One graded (model, scenario) result plus a compact transcript view. */
+export interface EvalResult {
+    scenarioId: string;
+    modelId: string;
+    outcome: ScenarioGrade["outcome"];
+    violations: string[];
+    toolsCalled: string[];
+    errorCount: number;
+    driverError?: string | undefined;
+    /**
+     * Required tools THIS scenario needs that the live `tools/list` did not
+     * advertise — the per-workflow view of {@link EvalReport.catalog.unresolvedRequiredTools}.
+     * Empty when the surface advertises every tool the scenario requires. This is
+     * the P1 gate dashboard: it names, per north-star workflow, exactly which tools
+     * a target surface is still missing.
+     */
+    missingTools: string[];
+}
+/**
+ * Auth mode the eval used to reach the MCP surface.
+ *
+ *  - `bearer`    — `Authorization: Bearer <token>` (HONUA_MCP_AUTH_TOKEN).
+ *  - `api-key`   — `x-api-key: <key>` (HONUA_API_KEY).
+ *  - `anonymous` — a live remote surface reached with NO credential.
+ *  - `none`      — the in-process offline fixture surface (no transport auth).
+ *
+ * Recorded in the artifact so a published live run proves it was authenticated
+ * (not run through a dev-auth bypass).
+ */
+export type AuthMode = "bearer" | "api-key" | "anonymous" | "none";
+/** Per-model aggregate metrics — the measurable form of the north star. */
+export interface ModelScorecard {
+    id: string;
+    /** Resolved model id for live drivers (attributable); omitted for the control. */
+    model?: string;
+    vendor: ModelDriver["vendor"];
+    available: boolean;
+    scenarios: number;
+    pass: number;
+    fail: number;
+    clarified: number;
+    error: number;
+    /** pass / scenarios. */
+    successRate: number;
+    /** clarified / scenarios. */
+    clarificationRate: number;
+    /** scenarios with >=1 erroring tool call / scenarios. */
+    editRate: number;
+    totalToolErrors: number;
+}
+export interface EvalReport {
+    schemaVersion: 4;
+    generatedAt: string;
+    surface: {
+        backend: "fixture" | "live";
+        mcpTransport: string;
+        remoteUrl?: string;
+        /** How the eval authenticated to the surface (proves the run was authed). */
+        auth: AuthMode;
+    };
+    corpus: {
+        scenarios: number;
+        ids: string[];
+    };
+    /**
+     * Catalog-coverage audit: how the corpus's required tools resolve against the
+     * LIVE `tools/list` the surface advertised. `unresolvedRequiredTools` names any
+     * required tool absent from the live catalog — so a scenario that fails does so
+     * for a real capability gap, not a silent name-resolution bug.
+     */
+    catalog: {
+        advertisedToolCount: number;
+        requiredTools: string[];
+        unresolvedRequiredTools: string[];
+    };
+    models: ModelScorecard[];
+    results: EvalResult[];
+    summary: {
+        /** True iff the deterministic control passed every scenario (the CI gate). */
+        pass: boolean;
+        modelsEvaluated: number;
+        liveModelsEvaluated: number;
+        scenarios: number;
+    };
+}
+export interface AssembleInput {
+    backend: "fixture" | "live";
+    mcpTransport: string;
+    remoteUrl?: string | undefined;
+    auth: AuthMode;
+    /** Tool names advertised by the live `tools/list` (for the catalog audit). */
+    advertisedTools: string[];
+    corpus: Scenario[];
+    drivers: ModelDriver[];
+    graded: {
+        grade: ScenarioGrade;
+        transcript: WorkflowTranscript;
+    }[];
+}
+export declare function assembleReport(input: AssembleInput): EvalReport;
+/** Render a human-readable Markdown summary of the eval report. */
+export declare function renderMarkdown(report: EvalReport): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/src/eval/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE3F,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;OAMG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAEnE,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,kFAAkF;IAClF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,CAAC,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE;QACP,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,6EAA6E;QAC7E,IAAI,EAAE,QAAQ,CAAC;KAChB,CAAC;IACF,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC7C;;;;;OAKG;IACH,OAAO,EAAE;QACP,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,uBAAuB,EAAE,MAAM,EAAE,CAAC;KACnC,CAAC;IACF,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,EAAE;QACP,8EAA8E;QAC9E,IAAI,EAAE,OAAO,CAAC;QACd,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,IAAI,EAAE,QAAQ,CAAC;IACf,8EAA8E;IAC9E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,kBAAkB,CAAA;KAAE,EAAE,CAAC;CACpE;AAaD,wBAAgB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,UAAU,CA4E/D;AAED,mEAAmE;AACnE,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAuDzD"}

package/dist/src/eval/report.js

@@ -0,0 +1,130 @@
+/** Distinct required-tool names referenced across a corpus. */
+function corpusRequiredTools(corpus) {
+    const names = new Set();
+    for (const scenario of corpus) {
+        for (const tool of scenario.criteria.requiredTools) {
+            names.add(tool);
+        }
+    }
+    return [...names].sort();
+}
+export function assembleReport(input) {
+    const advertised = new Set(input.advertisedTools);
+    // Per-scenario required tools, so each result can name the tools THIS workflow
+    // is missing from the surface (the per-workflow gate view).
+    const requiredByScenario = new Map(input.corpus.map((s) => [s.id, s.criteria.requiredTools]));
+    const results = input.graded.map(({ grade, transcript }) => ({
+        scenarioId: grade.scenarioId,
+        modelId: grade.modelId,
+        outcome: grade.outcome,
+        violations: grade.violations,
+        toolsCalled: transcript.steps.map((s) => s.tool),
+        errorCount: grade.errorCount,
+        driverError: transcript.driverError,
+        missingTools: (requiredByScenario.get(grade.scenarioId) ?? []).filter((t) => !advertised.has(t)),
+    }));
+    const models = input.drivers.map((driver) => {
+        const own = results.filter((r) => r.modelId === driver.id);
+        const scenarios = own.length;
+        const pass = own.filter((r) => r.outcome === "pass").length;
+        const fail = own.filter((r) => r.outcome === "fail").length;
+        const clarified = own.filter((r) => r.outcome === "clarified").length;
+        const error = own.filter((r) => r.outcome === "error").length;
+        const withToolErrors = own.filter((r) => r.errorCount > 0).length;
+        const totalToolErrors = own.reduce((acc, r) => acc + r.errorCount, 0);
+        const rate = (n) => (scenarios === 0 ? 0 : Number((n / scenarios).toFixed(4)));
+        return {
+            id: driver.id,
+            // The control has no underlying model; live drivers' id IS the resolved model.
+            ...(driver.vendor === "deterministic" ? {} : { model: driver.id }),
+            vendor: driver.vendor,
+            available: driver.isAvailable(),
+            scenarios,
+            pass,
+            fail,
+            clarified,
+            error,
+            successRate: rate(pass),
+            clarificationRate: rate(clarified),
+            editRate: rate(withToolErrors),
+            totalToolErrors,
+        };
+    });
+    const control = models.find((m) => m.vendor === "deterministic");
+    const controlPass = control ? control.fail === 0 && control.error === 0 && control.scenarios > 0 : false;
+    const liveModelsEvaluated = models.filter((m) => m.vendor !== "deterministic" && m.scenarios > 0).length;
+    const requiredTools = corpusRequiredTools(input.corpus);
+    const unresolvedRequiredTools = requiredTools.filter((t) => !advertised.has(t));
+    return {
+        schemaVersion: 4,
+        generatedAt: new Date().toISOString(),
+        surface: {
+            backend: input.backend,
+            mcpTransport: input.mcpTransport,
+            remoteUrl: input.remoteUrl,
+            auth: input.auth,
+        },
+        corpus: { scenarios: input.corpus.length, ids: input.corpus.map((s) => s.id) },
+        catalog: {
+            advertisedToolCount: input.advertisedTools.length,
+            requiredTools,
+            unresolvedRequiredTools,
+        },
+        models,
+        results,
+        summary: {
+            pass: controlPass,
+            modelsEvaluated: models.filter((m) => m.scenarios > 0).length,
+            liveModelsEvaluated,
+            scenarios: input.corpus.length,
+        },
+    };
+}
+/** Render a human-readable Markdown summary of the eval report. */
+export function renderMarkdown(report) {
+    const lines = [];
+    const status = report.summary.pass ? "✅ PASS" : "❌ FAIL";
+    lines.push("# MCP Cross-Model Workflow Eval");
+    lines.push("");
+    lines.push(`**Result (deterministic control):** ${status}`);
+    lines.push("");
+    lines.push(`- Generated: \`${report.generatedAt}\``);
+    lines.push(`- MCP surface: \`${report.surface.backend}\` (transport: \`${report.surface.mcpTransport}\`${report.surface.remoteUrl ? `, remote: \`${report.surface.remoteUrl}\`` : ""})`);
+    lines.push(`- Auth mode: \`${report.surface.auth}\``);
+    lines.push(`- Corpus: ${report.corpus.scenarios} GIS workflows`);
+    lines.push(`- Live models evaluated: ${report.summary.liveModelsEvaluated}`);
+    const unresolved = report.catalog.unresolvedRequiredTools;
+    lines.push(unresolved.length === 0
+        ? `- Catalog coverage: all ${report.catalog.requiredTools.length} required tools resolve against the live catalog (${report.catalog.advertisedToolCount} advertised)`
+        : `- Catalog coverage: ⚠️ ${unresolved.length} required tool(s) NOT advertised by the surface: ${unresolved.map((t) => `\`${t}\``).join(", ")}`);
+    lines.push("");
+    lines.push("## Per-model scorecard");
+    lines.push("");
+    lines.push("| Model | Vendor | Avail | Pass | Fail | Clarified | Error | Success | Clarify | Edit |");
+    lines.push("| --- | --- | :---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |");
+    for (const m of report.models) {
+        lines.push(`| \`${m.id}\` | ${m.vendor} | ${m.available ? "yes" : "no"} | ${m.pass} | ${m.fail} | ${m.clarified} | ${m.error} | ${pct(m.successRate)} | ${pct(m.clarificationRate)} | ${pct(m.editRate)} |`);
+    }
+    lines.push("");
+    const failures = report.results.filter((r) => r.outcome === "fail" || r.outcome === "error");
+    if (failures.length > 0) {
+        lines.push("## Non-passing results");
+        lines.push("");
+        for (const r of failures) {
+            lines.push(`- \`${r.modelId}\` × \`${r.scenarioId}\` — **${r.outcome}**`);
+            if (r.missingTools.length > 0) {
+                lines.push(`  - missing required tool(s) on this surface: ${r.missingTools.map((t) => `\`${t}\``).join(", ")}`);
+            }
+            for (const v of r.violations) {
+                lines.push(`  - ${v}`);
+            }
+        }
+        lines.push("");
+    }
+    lines.push("> Live cross-model runs (Claude + GPT) are tracked under honua-io/honua-server#1956.");
+    lines.push("");
+    return `${lines.join("\n")}\n`;
+}
+function pct(rate) {
+    return `${(rate * 100).toFixed(0)}%`;
+}

package/dist/src/eval/runner.d.ts

@@ -0,0 +1,22 @@
+import { type EvalReport } from "./report.js";
+import type { ModelDriver, Scenario } from "./types.js";
+/**
+ * Cross-model eval runner (honua-server #1956).
+ *
+ * Backend selection mirrors the certifier: offline-first and deterministic.
+ *   - HONUA_MCP_REMOTE_URL set → drive the live remote /mcp over streamable HTTP
+ *     (the same surface the stdio proxy bridges).
+ *   - otherwise → the offline in-memory fixture server (no network, no models).
+ *
+ * Drivers default to whatever `resolveDrivers` returns: the deterministic control
+ * always, plus Claude/GPT when their API keys are present.
+ */
+export interface RunEvalOptions {
+    env?: NodeJS.ProcessEnv;
+    drivers?: ModelDriver[];
+    corpus?: Scenario[];
+    /** Force the in-memory fixture surface even if HONUA_MCP_REMOTE_URL is set. */
+    forceOffline?: boolean;
+}
+export declare function runEval(options?: RunEvalOptions): Promise<EvalReport>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/src/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/eval/runner.ts"],"names":[],"mappings":"AAQA,OAAO,EAAiB,KAAK,UAAU,EAAkB,MAAM,aAAa,CAAC;AAC7E,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAmC,MAAM,YAAY,CAAC;AAEzF;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACpB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAyFD,wBAAsB,OAAO,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,UAAU,CAAC,CAqC/E"}

package/dist/src/eval/runner.js

@@ -0,0 +1,116 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { createFixtureClient } from "../certification/fixture-client.js";
+import { createServer } from "../index.js";
+import { connectUpstream, resolveProxyOptions } from "../proxy.js";
+import { resolveCorpus } from "./corpus.js";
+import { resolveDrivers } from "./drivers/index.js";
+import { grade } from "./grade.js";
+import { assembleReport } from "./report.js";
+/** Truthy-flag parse shared with the driver opt-ins (0/false/no/off ⇒ false). */
+function isTruthy(value) {
+    if (typeof value !== "string") {
+        return false;
+    }
+    const v = value.trim().toLowerCase();
+    return v.length > 0 && v !== "0" && v !== "false" && v !== "no" && v !== "off";
+}
+async function connectSurface(env, forceOffline) {
+    const live = !forceOffline && typeof env.HONUA_MCP_REMOTE_URL === "string" && env.HONUA_MCP_REMOTE_URL.length > 0;
+    if (live) {
+        const options = resolveProxyOptions(env);
+        // The live lane authenticates via the proxy's real credential headers — no
+        // dev-auth bypass. `Bearer` takes precedence over `x-api-key`; anonymous means
+        // no credential was supplied at all.
+        const auth = options.authToken ? "bearer" : options.apiKey ? "api-key" : "anonymous";
+        if (auth === "anonymous" && isTruthy(env.HONUA_EVAL_REQUIRE_AUTH)) {
+            throw new Error("HONUA_EVAL_REQUIRE_AUTH is set but the live MCP surface would be reached anonymously " +
+                "(no HONUA_MCP_AUTH_TOKEN / HONUA_API_KEY). Refusing to run an unauthenticated live eval.");
+        }
+        const client = await connectUpstream(options);
+        return {
+            client,
+            backend: "live",
+            mcpTransport: "streamable-http",
+            remoteUrl: options.remoteUrl,
+            auth,
+            close: async () => {
+                await client.close().catch(() => { });
+            },
+        };
+    }
+    const server = createServer(createFixtureClient());
+    const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+    const client = new Client({ name: "honua-mcp-eval", version: "1.0.0" });
+    await server.connect(serverTransport);
+    await client.connect(clientTransport);
+    return {
+        client,
+        backend: "fixture",
+        mcpTransport: "in-memory",
+        auth: "none",
+        close: async () => {
+            await client.close().catch(() => { });
+            await server.close().catch(() => { });
+        },
+    };
+}
+function flattenToolResult(result) {
+    const r = result;
+    const isError = r.isError === true;
+    const texts = (r.content ?? [])
+        .filter((b) => b.type === "text" && typeof b.text === "string")
+        .map((b) => b.text);
+    const text = texts.length > 0 ? texts.join("\n") : JSON.stringify(r.content ?? result);
+    return { isError, text };
+}
+function buildContext(client, tools) {
+    return {
+        tools,
+        async callTool(name, args) {
+            try {
+                const result = await client.callTool({ name, arguments: args });
+                return flattenToolResult(result);
+            }
+            catch (err) {
+                return { isError: true, text: err instanceof Error ? err.message : String(err) };
+            }
+        },
+    };
+}
+export async function runEval(options = {}) {
+    const env = options.env ?? process.env;
+    const corpus = options.corpus ?? resolveCorpus(env);
+    const drivers = options.drivers ?? resolveDrivers({ env });
+    const forceOffline = options.forceOffline ?? false;
+    const surface = await connectSurface(env, forceOffline);
+    try {
+        const listed = await surface.client.listTools();
+        const tools = listed.tools.map((t) => ({
+            name: t.name,
+            description: t.description ?? "",
+            inputSchema: t.inputSchema,
+        }));
+        const ctx = buildContext(surface.client, tools);
+        const graded = [];
+        for (const driver of drivers) {
+            for (const scenario of corpus) {
+                const transcript = await driver.runWorkflow(scenario, ctx);
+                graded.push({ grade: grade(scenario, transcript), transcript });
+            }
+        }
+        return assembleReport({
+            backend: surface.backend,
+            mcpTransport: surface.mcpTransport,
+            remoteUrl: surface.remoteUrl,
+            auth: surface.auth,
+            advertisedTools: tools.map((t) => t.name),
+            corpus,
+            drivers,
+            graded,
+        });
+    }
+    finally {
+        await surface.close();
+    }
+}

package/dist/src/eval/types.d.ts

@@ -0,0 +1,93 @@
+/**
+ * Cross-model MCP workflow eval — shared types (honua-server #1956, WS-H).
+ *
+ * The eval proves the north-star claim "any client → any workflow": a held-out
+ * corpus of GIS workflows is driven through the honua MCP surface by DIFFERENT
+ * client LLMs (Claude AND a GPT model), and end-to-end success / clarification /
+ * edit rates are recorded per model. A deterministic (scripted) driver provides
+ * an offline control that runs in CI with no model/API calls.
+ */
+/** A tool as advertised by the MCP surface, handed to a model driver. */
+export interface ToolDescriptor {
+    name: string;
+    description: string;
+    inputSchema: unknown;
+}
+/** Result of a single `tools/call` over the MCP surface. */
+export interface ToolCallResult {
+    isError: boolean;
+    /** Flattened text content of the tool result (what a model would read back). */
+    text: string;
+}
+/** The MCP surface a driver may exercise while solving a workflow. */
+export interface WorkflowContext {
+    tools: ToolDescriptor[];
+    callTool(name: string, args: Record<string, unknown>): Promise<ToolCallResult>;
+}
+/** One step a driver took: a tool call and whether it errored. */
+export interface TranscriptStep {
+    tool: string;
+    args: Record<string, unknown>;
+    isError: boolean;
+}
+/** What a driver produced for one scenario. */
+export interface WorkflowTranscript {
+    scenarioId: string;
+    modelId: string;
+    steps: TranscriptStep[];
+    finalAnswer: string;
+    /** The driver asked the user a clarifying question instead of completing. */
+    clarificationRequested: boolean;
+    /** Tool calls that returned isError (a proxy for self-correction / edits). */
+    errorCount: number;
+    /** Driver-level failure (timeout, exception, model unavailable). */
+    driverError?: string;
+}
+/** Declarative success criteria graded against a transcript. */
+export interface SuccessCriteria {
+    /** Tools that must be called at least once (any order). */
+    requiredTools: string[];
+    /** An ordered subsequence of tool names that must appear in order. */
+    expectedToolSequence?: string[];
+    /** Tools that must never be called (e.g. write/destructive tools). */
+    forbiddenTools?: string[];
+    /** Case-insensitive substrings the final answer must contain. */
+    answerMustInclude?: string[];
+}
+/** One GIS workflow in the held-out corpus. */
+export interface Scenario {
+    id: string;
+    title: string;
+    category: string;
+    /** The natural-language task handed to the client LLM. */
+    prompt: string;
+    criteria: SuccessCriteria;
+    /**
+     * Scripted "ideal client" trajectory used by the deterministic offline driver.
+     * Live LLM drivers ignore this and plan their own tool calls.
+     */
+    script: {
+        tool: string;
+        args: Record<string, unknown>;
+    }[];
+}
+/** Per-scenario grade for one model. */
+export type GradeOutcome = "pass" | "fail" | "clarified" | "error";
+export interface ScenarioGrade {
+    scenarioId: string;
+    modelId: string;
+    outcome: GradeOutcome;
+    /** Human-readable reasons a grade was not `pass`. */
+    violations: string[];
+    errorCount: number;
+}
+/** A driver wraps a single client model (or the deterministic control). */
+export interface ModelDriver {
+    /** Stable identifier recorded in the artifact, e.g. "claude-opus-4-8". */
+    readonly id: string;
+    readonly vendor: "anthropic" | "openai" | "bedrock" | "deterministic";
+    /** Whether this driver can actually run (key + SDK present). */
+    isAvailable(): boolean;
+    runWorkflow(scenario: Scenario, ctx: WorkflowContext): Promise<WorkflowTranscript>;
+}
+//# sourceMappingURL=types.d.ts.map