@honua/mcp-server 0.0.10-alpha.0 → 0.1.0-beta.0

package/dist/src/eval/northstar-corpus.js

@@ -0,0 +1,194 @@
+/**
+ * NORTH-STAR corpus — the P1 gate measure (honua-server #1948, MCP initiative P1.7).
+ *
+ * These are the three canonical end-to-end workflows the Phase-1 gate is defined
+ * against: "all 3 north-star workflows pass via a vanilla MCP client". Encoding
+ * them as eval scenarios turns that gate into a mechanical pass/fail signal that
+ * runs on the exact same grading (required-tool selection, ordered subsequence,
+ * forbidden-tool avoidance, answer-substring evidence) as every other corpus.
+ *
+ * The tool names are the reference-implementation (`honua_*`) names for the
+ * standard geospatial-mcp tools (see geospatial-mcp `spec/schemas/index.json`).
+ * Some required tools are P1 deliverables that DO NOT yet ship on every surface:
+ *
+ *   - `honua_publish_result`   — standard `publish_result`, reference status
+ *                                `known-gap` (promote a job/result package to a
+ *                                hosted layer). Lands with P1.
+ *   - `honua_apply_style_preset` — standard `apply_style_preset`, `known-gap`
+ *                                (styling a published layer). Lands with P1.
+ *
+ * Against a server that predates the P1 tools these scenarios are EXPECTED to
+ * fail — that failure is the gate signal, not a harness bug. The report makes it
+ * explicit which required tools a surface is missing:
+ *   - `catalog.unresolvedRequiredTools` names every required tool the live
+ *     `tools/list` did not advertise, corpus-wide.
+ *   - each per-scenario `results[].missingTools` names the tools THAT scenario
+ *     requires but the surface lacks (the gate dashboard, per workflow).
+ * The deterministic offline driver degrades HONESTLY: when its scripted ideal
+ * trajectory reaches a tool the surface does not advertise it records a
+ * `driverError` ("scripted tool not advertised …") and the scenario grades as
+ * `error`, never a silent pass. No fixture responses are faked to make an
+ * unbuilt tool look green.
+ *
+ * LIVE-GATED like the operator corpus: selected only via
+ * `HONUA_EVAL_CORPUS=northstar` (or `--corpus northstar`), and included in
+ * `all`. It is NOT part of the default analyst corpus, so the offline CI gate
+ * (`test:eval`, analyst-only) is untouched. Run it live against a P1 surface
+ * with `--driver bedrock` + `HONUA_MCP_REMOTE_URL=<server>/mcp` + credentials.
+ */
+/**
+ * The generic mutation escape-hatch. Every north-star workflow must compose the
+ * structured lifecycle (plan → validate → execute → publish) rather than reach
+ * for `honua_propose_operation`, so forbidding it turns "did the client use the
+ * intended workflow?" into a graded criterion.
+ */
+const GENERIC_MUTATION_TOOL = "honua_propose_operation";
+/** A small, embedded address CSV for the geocode → publish workflow (10 rows). */
+const ADDRESS_CSV = [
+    "name,street,city,state,zip",
+    "City Hall,1 Dr Carlton B Goodlett Pl,San Francisco,CA,94102",
+    "Ferry Building,1 Ferry Building,San Francisco,CA,94111",
+    "Coit Tower,1 Telegraph Hill Blvd,San Francisco,CA,94133",
+    "Palace of Fine Arts,3601 Lyon St,San Francisco,CA,94123",
+    "SFMOMA,151 3rd St,San Francisco,CA,94103",
+    "Oracle Park,24 Willie Mays Plaza,San Francisco,CA,94107",
+    "Golden Gate Park,501 Stanyan St,San Francisco,CA,94117",
+    "Exploratorium,Pier 15 Embarcadero,San Francisco,CA,94111",
+    "Chase Center,1 Warriors Way,San Francisco,CA,94158",
+    "Sutro Baths,1004 Point Lobos Ave,San Francisco,CA,94121",
+].join("\n");
+/** Parsed address rows the geocoder batch call replays (without the header). */
+const ADDRESS_ROWS = ADDRESS_CSV.split("\n")
+    .slice(1)
+    .map((line) => {
+    const [, street, city, state, zip] = line.split(",");
+    return `${street}, ${city}, ${state} ${zip}`;
+});
+/** The natural-language prompt for the CSV → point-layer workflow (CSV embedded). */
+const CSV_GEOCODE_PROMPT = `Here is a CSV of addresses. Geocode them and publish a point layer from the results.
+
+\`\`\`csv
+${ADDRESS_CSV}
+\`\`\`
+
+Batch-geocode every row, ingest the geocoded points as a dataset, and publish that dataset as a hosted point service. Do not reach for a raw propose_operation.`;
+export const NORTHSTAR_CORPUS = [
+    {
+        id: "northstar-styled-map",
+        title: "Proximity analysis → published, styled map",
+        category: "north-star",
+        prompt: "Find all parcels within 500 m of a flood zone and produce a styled map of them. " +
+            "Resolve the parcels and flood-zone layers, plan and run the proximity analysis as a " +
+            "governed job, publish the result as a new layer, style it, and render a map. Use the " +
+            "structured analysis lifecycle — do not reach for a raw propose_operation.",
+        criteria: {
+            // Load-bearing arc of the workflow: ground the entities, execute the plan,
+            // publish the job artifact to a layer, and render a map of it.
+            requiredTools: ["honua_resolve_entity", "honua_execute_plan", "honua_publish_result", "honua_render_map"],
+            expectedToolSequence: ["honua_resolve_entity", "honua_execute_plan", "honua_publish_result", "honua_render_map"],
+            forbiddenTools: [GENERIC_MUTATION_TOOL],
+        },
+        script: [
+            { tool: "honua_resolve_entity", args: { text: "parcels", entityType: "layer" } },
+            { tool: "honua_resolve_entity", args: { text: "flood zones", entityType: "layer" } },
+            {
+                tool: "honua_plan_analysis",
+                args: { intent: "select parcels within 500 meters of a flood zone and produce a styled map" },
+            },
+            { tool: "honua_validate_plan", args: { plan: { planId: "northstar-styled-map-plan" } } },
+            { tool: "honua_execute_plan", args: { plan: { planId: "northstar-styled-map-plan" } } },
+            // P1 tools (reference known-gap today): promote the job artifact to a layer,
+            // then style it. On a pre-P1 surface these are unadvertised and the driver
+            // degrades to an honest driverError here rather than faking a response.
+            { tool: "honua_publish_result", args: { sourceId: "northstar-styled-map-result", targetKind: "layer" } },
+            {
+                tool: "honua_apply_style_preset",
+                args: { mapPackageId: "northstar-styled-map-package", styleId: "flood-risk" },
+            },
+            {
+                tool: "honua_render_map",
+                args: { layers: ["northstar-styled-map-layer"], bbox: [-122.55, 37.7, -122.35, 37.85] },
+            },
+        ],
+    },
+    {
+        id: "northstar-csv-geocode-publish",
+        title: "CSV of addresses → geocoded, published point layer",
+        category: "north-star",
+        prompt: CSV_GEOCODE_PROMPT,
+        criteria: {
+            // The CSV → point-layer arc as the standard defines it: batch geocode →
+            // ingest the resulting points → publish the source table as a service.
+            requiredTools: ["honua_geocode_addresses", "honua_ingest_dataset", "honua_publish_service"],
+            expectedToolSequence: ["honua_geocode_addresses", "honua_ingest_dataset", "honua_publish_service"],
+            forbiddenTools: [GENERIC_MUTATION_TOOL],
+        },
+        script: [
+            { tool: "honua_geocode_addresses", args: { addresses: ADDRESS_ROWS } },
+            {
+                tool: "honua_ingest_dataset",
+                args: {
+                    format: "csv",
+                    datasetName: "sf-landmarks",
+                    data: ADDRESS_CSV,
+                    addressColumn: "street",
+                },
+            },
+            {
+                tool: "honua_publish_service",
+                args: {
+                    connectionId: "default",
+                    schema: "public",
+                    table: "sf_landmarks",
+                    layerName: "SF Landmarks",
+                    geometryType: "point",
+                },
+            },
+        ],
+    },
+    {
+        id: "northstar-buffer-stats-job",
+        title: "Buffer → intersect → summary statistics, run as a job",
+        category: "north-star",
+        prompt: "Buffer highways by 100 m, intersect the buffers with wetlands, and give me summary " +
+            "statistics for the overlap. Run it as a job and report progress. Ground the goal into " +
+            "an analysis plan, validate it, then execute it as a governed job and report the results " +
+            "the job produces. Do not reach for a raw propose_operation.",
+        criteria: {
+            // Plan entry → validate → execute-as-job. The answer must carry evidence
+            // that the JOB path was used and its results read back — kept loose (the
+            // job reference token, not exact statistic values) so live variance in the
+            // numbers does not flip the grade.
+            requiredTools: ["honua_plan_analysis", "honua_validate_plan", "honua_execute_plan"],
+            expectedToolSequence: [
+                "honua_ground_candidates",
+                "honua_plan_analysis",
+                "honua_validate_plan",
+                "honua_execute_plan",
+            ],
+            forbiddenTools: [GENERIC_MUTATION_TOOL],
+            answerMustInclude: ["job"],
+        },
+        script: [
+            {
+                tool: "honua_ground_candidates",
+                args: {
+                    goal: "buffer highways by 100 meters, intersect with wetlands, and summarize the overlap",
+                    assumptionPolicy: "UseDefaults",
+                },
+            },
+            {
+                tool: "honua_plan_analysis",
+                args: { intent: "buffer highways by 100m, intersect with wetlands, and compute summary statistics" },
+            },
+            { tool: "honua_validate_plan", args: { plan: { planId: "northstar-buffer-stats-plan" } } },
+            // Optional dry-run estimate before committing (reference advertises it with
+            // the same partial-plan input schema as validate_plan).
+            { tool: "honua_dry_run_plan", args: { plan: { planId: "northstar-buffer-stats-plan" } } },
+            {
+                tool: "honua_execute_plan",
+                args: { plan: { planId: "northstar-buffer-stats-plan" }, idempotencyKey: "northstar-buffer-stats-1" },
+            },
+        ],
+    },
+];

package/dist/src/eval/operator-corpus.d.ts

@@ -0,0 +1,3 @@
+import type { Scenario } from "./types.js";
+export declare const OPERATOR_CORPUS: Scenario[];
+//# sourceMappingURL=operator-corpus.d.ts.map

package/dist/src/eval/operator-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"operator-corpus.d.ts","sourceRoot":"","sources":["../../../src/eval/operator-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AA2D3C,eAAO,MAAM,eAAe,EAAE,QAAQ,EAmJrC,CAAC"}

package/dist/src/eval/operator-corpus.js

@@ -0,0 +1,193 @@
+/**
+ * Operator-surface corpus for the cross-model eval (honua-server #1956).
+ *
+ * The analyst corpus in `corpus.ts` targets the SDK `@honua/mcp-server` ANALYST
+ * tools (count_features, describe_layer, query_features, …). This corpus instead
+ * exercises the LIVE honua-server OPERATOR MCP surface — the agent-native control
+ * plane published at `https://demo.honua.io/mcp` (MCP streamable-HTTP, protocol
+ * 2025-03-26, serverInfo `honua.operator.mcp`). Its tools are the grounding →
+ * clarify → plan → validate → dry-run → (gated) execute lifecycle plus read-only
+ * data tools (list layers, query features) and Pro tools (geocode, route).
+ *
+ * Select it with `HONUA_EVAL_CORPUS=operator` (see `resolveCorpus` in corpus.ts)
+ * and run live with `--driver bedrock` + `HONUA_MCP_REMOTE_URL=<demo>/mcp` +
+ * `HONUA_API_KEY=<demo key>`. The same grading runs for every driver, so the
+ * deterministic control and Bedrock-Claude are measured against identical
+ * criteria over the identical live catalog.
+ *
+ * SAFETY: these scenarios are READ-MOSTLY and reversible. The end-to-end planning
+ * scenarios stop at validate / dry-run and FORBID the mutating lifecycle tools
+ * (`honua_execute_plan`, `honua_propose_operation`) — on the Community-edition
+ * demo `honua_propose_operation` executes directly (no approval gate), so a
+ * correct agent must NOT call it for a read-only analysis task. Forbidding those
+ * tools turns "did the agent stay read-only?" into a graded criterion.
+ *
+ * Grading is deliberately tool-centric: required-tool selection, ordered
+ * subsequence, and forbidden-tool avoidance — the verifiable signals of "the
+ * client composed the right workflow". Answer-content checks are used only where
+ * a data value (a layer name, a geometry word) is near-certain to be echoed, so a
+ * non-pass means a real capability/composition gap rather than a phrasing
+ * artifact. Tools gated by edition entitlements (geocode/route) are scored on
+ * correct tool selection alone (see `operator-geocode-gap`).
+ *
+ * Dataset shapes resolve against the seeded Maui demo content: the published
+ * service `svc-publish-maui-buildings` (layer 13, Overture building footprints).
+ */
+/** Seeded demo service/layer the scenarios bind to (from honua_list_layers). */
+const MAUI_BUILDINGS_SERVICE = "svc-publish-maui-buildings";
+const MAUI_BUILDINGS_LAYER = 13;
+/** Mutating lifecycle tools a read-only analysis agent must never call. */
+const MUTATING_TOOLS = ["honua_execute_plan", "honua_propose_operation"];
+/** A small, static, well-formed analysis plan reused by validate/dry-run. */
+const SAMPLE_PLAN = {
+    planId: "eval-operator-plan-1",
+    intentId: "eval-operator-intent-1",
+    steps: [
+        {
+            stepId: "s1",
+            kind: "QueryFeatures",
+            inputs: { serviceId: MAUI_BUILDINGS_SERVICE, layerId: String(MAUI_BUILDINGS_LAYER) },
+        },
+    ],
+    outputs: ["Scalar"],
+};
+export const OPERATOR_CORPUS = [
+    {
+        id: "operator-list-layers",
+        title: "Discover the published layers on the operator surface",
+        category: "discovery",
+        prompt: "I just connected to this Honua server. What map layers are published here that I could query or map? Just list what's available.",
+        criteria: {
+            requiredTools: ["honua_list_layers"],
+            answerMustInclude: ["maui"],
+        },
+        script: [{ tool: "honua_list_layers", args: {} }],
+    },
+    {
+        id: "operator-discover-then-query",
+        title: "Discover a layer, then query features from it",
+        category: "query",
+        prompt: "Find the Maui buildings layer on this server and show me a few of its building features so I can see what attributes they carry.",
+        criteria: {
+            requiredTools: ["honua_list_layers", "honua_query_features"],
+            expectedToolSequence: ["honua_list_layers", "honua_query_features"],
+            forbiddenTools: MUTATING_TOOLS,
+            answerMustInclude: ["building"],
+        },
+        script: [
+            { tool: "honua_list_layers", args: { filter: "maui-buildings" } },
+            {
+                tool: "honua_query_features",
+                args: { serviceId: MAUI_BUILDINGS_SERVICE, layerId: MAUI_BUILDINGS_LAYER, limit: 3 },
+            },
+        ],
+    },
+    {
+        id: "operator-ground-intent",
+        title: "Ground an ambiguous analysis goal to a workflow",
+        category: "grounding",
+        prompt: "I want to count how many Maui buildings fall within 100 meters of the coast. Help me get started — figure out what kind of workflow this is and what data and processes apply. Don't run anything yet.",
+        criteria: {
+            requiredTools: ["honua_ground_candidates"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            {
+                tool: "honua_ground_candidates",
+                args: {
+                    goal: "count how many Maui buildings fall within 100 meters of the coast",
+                    assumptionPolicy: "AskAlways",
+                },
+            },
+        ],
+    },
+    {
+        id: "operator-clarify-loop",
+        title: "Ground then answer the clarification to refine intent",
+        category: "multi-step",
+        prompt: "I want to buffer the Maui buildings by 100 meters and count how many features fall in the buffered area. Start the grounding pass, and when the server asks a clarifying question, answer it (the buffer distance is 100 meters, the input is the maui-buildings layer). Stop once the intent is refined — do not execute.",
+        criteria: {
+            requiredTools: ["honua_ground_candidates", "honua_clarify_intent"],
+            expectedToolSequence: ["honua_ground_candidates", "honua_clarify_intent"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            {
+                tool: "honua_ground_candidates",
+                args: {
+                    goal: "buffer the maui buildings by 100 meters and count how many fall in the area",
+                    assumptionPolicy: "AskAlways",
+                },
+            },
+            {
+                tool: "honua_clarify_intent",
+                args: {
+                    goal: "buffer the maui buildings by 100 meters and count how many fall in the area",
+                    intentId: "eval-operator-clarify-1",
+                    response: { answers: { "param.distance": ["100"], "param.input": ["maui-buildings"] } },
+                },
+            },
+        ],
+    },
+    {
+        id: "operator-plan-validate",
+        title: "End-to-end read-only planning: ground → plan → validate",
+        category: "multi-step",
+        prompt: "I need an executable analysis plan to count the Maui buildings, but I am NOT ready to run it. Ground the goal, compile it into an analysis plan, and validate that the plan is executable and what approvals it would need. Do not execute or propose any operation.",
+        criteria: {
+            requiredTools: ["honua_ground_candidates", "honua_plan_analysis", "honua_validate_plan"],
+            expectedToolSequence: ["honua_ground_candidates", "honua_plan_analysis", "honua_validate_plan"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            { tool: "honua_ground_candidates", args: { goal: "count the maui buildings", assumptionPolicy: "UseDefaults" } },
+            { tool: "honua_plan_analysis", args: { intent: "count the features in the maui-buildings layer" } },
+            { tool: "honua_validate_plan", args: { plan: SAMPLE_PLAN } },
+        ],
+    },
+    {
+        id: "operator-dry-run",
+        title: "Estimate a plan with a dry run before executing",
+        category: "analysis",
+        prompt: "Before I commit to running an analysis that queries the Maui buildings layer, give me a dry-run estimate of how long it would take, what kind of artifacts it would produce, and any side effects. Do not actually execute it.",
+        criteria: {
+            requiredTools: ["honua_dry_run_plan"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [{ tool: "honua_dry_run_plan", args: { plan: SAMPLE_PLAN } }],
+    },
+    {
+        id: "operator-validate-package",
+        title: "Validate a query package against the governance contract",
+        category: "governance",
+        prompt: 'Run a read-only validation/review of a query package and tell me whether it is ready to execute and whether it needs approval. The package family is \'query\' and its entire payload is just this layer binding: { "serviceId": "svc-publish-maui-buildings", "layerId": 13 }. Validate it exactly as given — do not ask me for anything more.',
+        criteria: {
+            requiredTools: ["honua_validate_package"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [
+            {
+                tool: "honua_validate_package",
+                args: {
+                    packageFamily: "query",
+                    packagePayload: { serviceId: MAUI_BUILDINGS_SERVICE, layerId: MAUI_BUILDINGS_LAYER },
+                },
+            },
+        ],
+    },
+    {
+        id: "operator-geocode-gap",
+        title: "Route an address task to geocoding (entitlement-gated)",
+        category: "capability-gap",
+        prompt: "I have a street address — 'Kahului, Maui, Hawaii'. Get me the coordinates for it using this server. If the server can't do it, tell me honestly why instead of guessing coordinates.",
+        criteria: {
+            // Pro-gated on the Community demo: scored on correct tool SELECTION (the
+            // client must route an address task to the geocoder rather than fabricate
+            // coordinates). The honest gap report is asserted qualitatively, not via a
+            // brittle answer-substring match. forbiddenTools keeps it read-only.
+            requiredTools: ["honua_geocode_address"],
+            forbiddenTools: MUTATING_TOOLS,
+        },
+        script: [{ tool: "honua_geocode_address", args: { address: "Kahului, Maui, Hawaii", maxResults: 3 } }],
+    },
+];

package/dist/src/eval/report.d.ts

@@ -0,0 +1,109 @@
+import type { SuiteProvenance } from "../provenance.js";
+import type { ModelDriver, Scenario, ScenarioGrade, WorkflowTranscript } from "./types.js";
+/** One graded (model, scenario) result plus a compact transcript view. */
+export interface EvalResult {
+    scenarioId: string;
+    modelId: string;
+    outcome: ScenarioGrade["outcome"];
+    violations: string[];
+    toolsCalled: string[];
+    errorCount: number;
+    driverError?: string | undefined;
+    /**
+     * Required tools THIS scenario needs that the live `tools/list` did not
+     * advertise — the per-workflow view of {@link EvalReport.catalog.unresolvedRequiredTools}.
+     * Empty when the surface advertises every tool the scenario requires. This is
+     * the P1 gate dashboard: it names, per north-star workflow, exactly which tools
+     * a target surface is still missing.
+     */
+    missingTools: string[];
+}
+/**
+ * Auth mode the eval used to reach the MCP surface.
+ *
+ *  - `bearer`    — `Authorization: Bearer <token>` (HONUA_MCP_AUTH_TOKEN).
+ *  - `api-key`   — `x-api-key: <key>` (HONUA_API_KEY).
+ *  - `anonymous` — a live remote surface reached with NO credential.
+ *  - `none`      — the in-process offline fixture surface (no transport auth).
+ *
+ * Recorded in the artifact so a published live run proves it was authenticated
+ * (not run through a dev-auth bypass).
+ */
+export type AuthMode = "bearer" | "api-key" | "anonymous" | "none";
+/** Per-model aggregate metrics — the measurable form of the north star. */
+export interface ModelScorecard {
+    id: string;
+    /** Resolved model id for live drivers (attributable); omitted for the control. */
+    model?: string;
+    vendor: ModelDriver["vendor"];
+    available: boolean;
+    scenarios: number;
+    pass: number;
+    fail: number;
+    clarified: number;
+    error: number;
+    /** pass / scenarios. */
+    successRate: number;
+    /** clarified / scenarios. */
+    clarificationRate: number;
+    /** scenarios with >=1 erroring tool call / scenarios. */
+    editRate: number;
+    totalToolErrors: number;
+}
+export interface EvalReport {
+    schemaVersion: 4;
+    generatedAt: string;
+    /** Self-proving provenance: target, protocol, tool count, auth, suite git SHA. */
+    provenance: SuiteProvenance;
+    surface: {
+        backend: "fixture" | "live";
+        mcpTransport: string;
+        remoteUrl?: string;
+        /** How the eval authenticated to the surface (proves the run was authed). */
+        auth: AuthMode;
+    };
+    corpus: {
+        scenarios: number;
+        ids: string[];
+    };
+    /**
+     * Catalog-coverage audit: how the corpus's required tools resolve against the
+     * LIVE `tools/list` the surface advertised. `unresolvedRequiredTools` names any
+     * required tool absent from the live catalog — so a scenario that fails does so
+     * for a real capability gap, not a silent name-resolution bug.
+     */
+    catalog: {
+        advertisedToolCount: number;
+        requiredTools: string[];
+        unresolvedRequiredTools: string[];
+    };
+    models: ModelScorecard[];
+    results: EvalResult[];
+    summary: {
+        /** True iff the deterministic control passed every scenario (the CI gate). */
+        pass: boolean;
+        modelsEvaluated: number;
+        liveModelsEvaluated: number;
+        scenarios: number;
+    };
+}
+export interface AssembleInput {
+    backend: "fixture" | "live";
+    mcpTransport: string;
+    remoteUrl?: string | undefined;
+    auth: AuthMode;
+    /** Reproducibility fingerprint attached to the artifact (A5 provability). */
+    provenance: SuiteProvenance;
+    /** Tool names advertised by the live `tools/list` (for the catalog audit). */
+    advertisedTools: string[];
+    corpus: Scenario[];
+    drivers: ModelDriver[];
+    graded: {
+        grade: ScenarioGrade;
+        transcript: WorkflowTranscript;
+    }[];
+}
+export declare function assembleReport(input: AssembleInput): EvalReport;
+/** Render a human-readable Markdown summary of the eval report. */
+export declare function renderMarkdown(report: EvalReport): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/src/eval/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE3F,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;OAMG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAEnE,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,kFAAkF;IAClF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,CAAC,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kFAAkF;IAClF,UAAU,EAAE,eAAe,CAAC;IAC5B,OAAO,EAAE;QACP,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,6EAA6E;QAC7E,IAAI,EAAE,QAAQ,CAAC;KAChB,CAAC;IACF,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC7C;;;;;OAKG;IACH,OAAO,EAAE;QACP,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,uBAAuB,EAAE,MAAM,EAAE,CAAC;KACnC,CAAC;IACF,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,EAAE;QACP,8EAA8E;QAC9E,IAAI,EAAE,OAAO,CAAC;QACd,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,IAAI,EAAE,QAAQ,CAAC;IACf,6EAA6E;IAC7E,UAAU,EAAE,eAAe,CAAC;IAC5B,8EAA8E;IAC9E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,kBAAkB,CAAA;KAAE,EAAE,CAAC;CACpE;AAaD,wBAAgB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,UAAU,CA6E/D;AAED,mEAAmE;AACnE,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAyDzD"}

package/dist/src/eval/report.js

@@ -0,0 +1,133 @@
+/** Distinct required-tool names referenced across a corpus. */
+function corpusRequiredTools(corpus) {
+    const names = new Set();
+    for (const scenario of corpus) {
+        for (const tool of scenario.criteria.requiredTools) {
+            names.add(tool);
+        }
+    }
+    return [...names].sort();
+}
+export function assembleReport(input) {
+    const advertised = new Set(input.advertisedTools);
+    // Per-scenario required tools, so each result can name the tools THIS workflow
+    // is missing from the surface (the per-workflow gate view).
+    const requiredByScenario = new Map(input.corpus.map((s) => [s.id, s.criteria.requiredTools]));
+    const results = input.graded.map(({ grade, transcript }) => ({
+        scenarioId: grade.scenarioId,
+        modelId: grade.modelId,
+        outcome: grade.outcome,
+        violations: grade.violations,
+        toolsCalled: transcript.steps.map((s) => s.tool),
+        errorCount: grade.errorCount,
+        driverError: transcript.driverError,
+        missingTools: (requiredByScenario.get(grade.scenarioId) ?? []).filter((t) => !advertised.has(t)),
+    }));
+    const models = input.drivers.map((driver) => {
+        const own = results.filter((r) => r.modelId === driver.id);
+        const scenarios = own.length;
+        const pass = own.filter((r) => r.outcome === "pass").length;
+        const fail = own.filter((r) => r.outcome === "fail").length;
+        const clarified = own.filter((r) => r.outcome === "clarified").length;
+        const error = own.filter((r) => r.outcome === "error").length;
+        const withToolErrors = own.filter((r) => r.errorCount > 0).length;
+        const totalToolErrors = own.reduce((acc, r) => acc + r.errorCount, 0);
+        const rate = (n) => (scenarios === 0 ? 0 : Number((n / scenarios).toFixed(4)));
+        return {
+            id: driver.id,
+            // The control has no underlying model; live drivers' id IS the resolved model.
+            ...(driver.vendor === "deterministic" ? {} : { model: driver.id }),
+            vendor: driver.vendor,
+            available: driver.isAvailable(),
+            scenarios,
+            pass,
+            fail,
+            clarified,
+            error,
+            successRate: rate(pass),
+            clarificationRate: rate(clarified),
+            editRate: rate(withToolErrors),
+            totalToolErrors,
+        };
+    });
+    const control = models.find((m) => m.vendor === "deterministic");
+    const controlPass = control ? control.fail === 0 && control.error === 0 && control.scenarios > 0 : false;
+    const liveModelsEvaluated = models.filter((m) => m.vendor !== "deterministic" && m.scenarios > 0).length;
+    const requiredTools = corpusRequiredTools(input.corpus);
+    const unresolvedRequiredTools = requiredTools.filter((t) => !advertised.has(t));
+    return {
+        schemaVersion: 4,
+        generatedAt: new Date().toISOString(),
+        provenance: input.provenance,
+        surface: {
+            backend: input.backend,
+            mcpTransport: input.mcpTransport,
+            remoteUrl: input.remoteUrl,
+            auth: input.auth,
+        },
+        corpus: { scenarios: input.corpus.length, ids: input.corpus.map((s) => s.id) },
+        catalog: {
+            advertisedToolCount: input.advertisedTools.length,
+            requiredTools,
+            unresolvedRequiredTools,
+        },
+        models,
+        results,
+        summary: {
+            pass: controlPass,
+            modelsEvaluated: models.filter((m) => m.scenarios > 0).length,
+            liveModelsEvaluated,
+            scenarios: input.corpus.length,
+        },
+    };
+}
+/** Render a human-readable Markdown summary of the eval report. */
+export function renderMarkdown(report) {
+    const lines = [];
+    const status = report.summary.pass ? "✅ PASS" : "❌ FAIL";
+    lines.push("# MCP Cross-Model Workflow Eval");
+    lines.push("");
+    lines.push(`**Result (deterministic control):** ${status}`);
+    lines.push("");
+    lines.push(`- Generated: \`${report.generatedAt}\``);
+    lines.push(`- MCP surface: \`${report.surface.backend}\` (transport: \`${report.surface.mcpTransport}\`${report.surface.remoteUrl ? `, remote: \`${report.surface.remoteUrl}\`` : ""})`);
+    lines.push(`- Auth mode: \`${report.surface.auth}\``);
+    lines.push(`- Negotiated protocol: \`${report.provenance.protocolVersion ?? "n/a"}\``);
+    lines.push(`- Suite git SHA: \`${report.provenance.suiteGitSha}\` (${report.provenance.suiteGitShaSource})`);
+    lines.push(`- Corpus: ${report.corpus.scenarios} GIS workflows`);
+    lines.push(`- Live models evaluated: ${report.summary.liveModelsEvaluated}`);
+    const unresolved = report.catalog.unresolvedRequiredTools;
+    lines.push(unresolved.length === 0
+        ? `- Catalog coverage: all ${report.catalog.requiredTools.length} required tools resolve against the live catalog (${report.catalog.advertisedToolCount} advertised)`
+        : `- Catalog coverage: ⚠️ ${unresolved.length} required tool(s) NOT advertised by the surface: ${unresolved.map((t) => `\`${t}\``).join(", ")}`);
+    lines.push("");
+    lines.push("## Per-model scorecard");
+    lines.push("");
+    lines.push("| Model | Vendor | Avail | Pass | Fail | Clarified | Error | Success | Clarify | Edit |");
+    lines.push("| --- | --- | :---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |");
+    for (const m of report.models) {
+        lines.push(`| \`${m.id}\` | ${m.vendor} | ${m.available ? "yes" : "no"} | ${m.pass} | ${m.fail} | ${m.clarified} | ${m.error} | ${pct(m.successRate)} | ${pct(m.clarificationRate)} | ${pct(m.editRate)} |`);
+    }
+    lines.push("");
+    const failures = report.results.filter((r) => r.outcome === "fail" || r.outcome === "error");
+    if (failures.length > 0) {
+        lines.push("## Non-passing results");
+        lines.push("");
+        for (const r of failures) {
+            lines.push(`- \`${r.modelId}\` × \`${r.scenarioId}\` — **${r.outcome}**`);
+            if (r.missingTools.length > 0) {
+                lines.push(`  - missing required tool(s) on this surface: ${r.missingTools.map((t) => `\`${t}\``).join(", ")}`);
+            }
+            for (const v of r.violations) {
+                lines.push(`  - ${v}`);
+            }
+        }
+        lines.push("");
+    }
+    lines.push("> Live cross-model runs (Claude + GPT) are tracked under honua-io/honua-server#1956.");
+    lines.push("");
+    return `${lines.join("\n")}\n`;
+}
+function pct(rate) {
+    return `${(rate * 100).toFixed(0)}%`;
+}