@honua/mcp-server 0.0.20-alpha.0 → 0.1.1-beta.0

package/dist/src/eval/grade.js

@@ -23,16 +23,24 @@ export function grade(scenario, transcript) {
         modelId: transcript.modelId,
         errorCount: transcript.errorCount,
     };
+    const { criteria } = scenario;
     if (transcript.driverError) {
         return { ...base, outcome: "error", violations: [transcript.driverError] };
     }
     if (transcript.clarificationRequested) {
+        // For an ambiguity/refusal scenario, clarifying IS the correct behavior.
+        if (criteria.expectClarification) {
+            return { ...base, outcome: "pass", violations: [] };
+        }
         return { ...base, outcome: "clarified", violations: ["driver requested clarification"] };
     }
+    if (criteria.expectClarification) {
+        // The prompt was ambiguous/unsupported but the driver answered anyway.
+        return { ...base, outcome: "fail", violations: ["expected a clarifying question but the driver completed"] };
+    }
     const violations = [];
     const calledTools = transcript.steps.map((s) => s.tool);
     const calledSet = new Set(calledTools);
-    const { criteria } = scenario;
     for (const required of criteria.requiredTools) {
         if (!calledSet.has(required)) {
             violations.push(`required tool not called: ${required}`);
@@ -52,5 +60,16 @@ export function grade(scenario, transcript) {
             violations.push(`answer missing expected content: "${fragment}"`);
         }
     }
+    // Semantic assertions (issue #369): meaning, not just trajectory.
+    for (const fragment of criteria.answerMustNotInclude ?? []) {
+        if (answer.includes(fragment.toLowerCase())) {
+            violations.push(`answer contains forbidden content: "${fragment}"`);
+        }
+    }
+    for (const pattern of criteria.answerMustMatch ?? []) {
+        if (!new RegExp(pattern, "i").test(transcript.finalAnswer)) {
+            violations.push(`answer did not match required pattern: /${pattern}/i`);
+        }
+    }
     return { ...base, outcome: violations.length === 0 ? "pass" : "fail", violations };
 }

package/dist/src/eval/report.d.ts

@@ -1,3 +1,4 @@
+import type { SuiteProvenance } from "../provenance.js";
 import type { ModelDriver, Scenario, ScenarioGrade, WorkflowTranscript } from "./types.js";
 /** One graded (model, scenario) result plus a compact transcript view. */
 export interface EvalResult {
@@ -52,6 +53,8 @@ export interface ModelScorecard {
 export interface EvalReport {
     schemaVersion: 4;
     generatedAt: string;
+    /** Self-proving provenance: target, protocol, tool count, auth, suite git SHA. */
+    provenance: SuiteProvenance;
     surface: {
         backend: "fixture" | "live";
         mcpTransport: string;
@@ -89,6 +92,8 @@ export interface AssembleInput {
     mcpTransport: string;
     remoteUrl?: string | undefined;
     auth: AuthMode;
+    /** Reproducibility fingerprint attached to the artifact (A5 provability). */
+    provenance: SuiteProvenance;
     /** Tool names advertised by the live `tools/list` (for the catalog audit). */
     advertisedTools: string[];
     corpus: Scenario[];

package/dist/src/eval/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE3F,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;OAMG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAEnE,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,kFAAkF;IAClF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,CAAC,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE;QACP,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,6EAA6E;QAC7E,IAAI,EAAE,QAAQ,CAAC;KAChB,CAAC;IACF,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC7C;;;;;OAKG;IACH,OAAO,EAAE;QACP,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,uBAAuB,EAAE,MAAM,EAAE,CAAC;KACnC,CAAC;IACF,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,EAAE;QACP,8EAA8E;QAC9E,IAAI,EAAE,OAAO,CAAC;QACd,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,IAAI,EAAE,QAAQ,CAAC;IACf,8EAA8E;IAC9E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,kBAAkB,CAAA;KAAE,EAAE,CAAC;CACpE;AAaD,wBAAgB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,UAAU,CA4E/D;AAED,mEAAmE;AACnE,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAuDzD"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAE3F,0EAA0E;AAC1E,MAAM,WAAW,UAAU;IACzB,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,aAAa,CAAC,SAAS,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACjC;;;;;;OAMG;IACH,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAED;;;;;;;;;;GAUG;AACH,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,SAAS,GAAG,WAAW,GAAG,MAAM,CAAC;AAEnE,2EAA2E;AAC3E,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,kFAAkF;IAClF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC;IAC9B,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,wBAAwB;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,6BAA6B;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,yDAAyD;IACzD,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,UAAU;IACzB,aAAa,EAAE,CAAC,CAAC;IACjB,WAAW,EAAE,MAAM,CAAC;IACpB,kFAAkF;IAClF,UAAU,EAAE,eAAe,CAAC;IAC5B,OAAO,EAAE;QACP,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;QAC5B,YAAY,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,6EAA6E;QAC7E,IAAI,EAAE,QAAQ,CAAC;KAChB,CAAC;IACF,MAAM,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAC7C;;;;;OAKG;IACH,OAAO,EAAE;QACP,mBAAmB,EAAE,MAAM,CAAC;QAC5B,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,uBAAuB,EAAE,MAAM,EAAE,CAAC;KACnC,CAAC;IACF,MAAM,EAAE,cAAc,EAAE,CAAC;IACzB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,EAAE;QACP,8EAA8E;QAC9E,IAAI,EAAE,OAAO,CAAC;QACd,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC;CACH;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,SAAS,GAAG,MAAM,CAAC;IAC5B,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC/B,IAAI,EAAE,QAAQ,CAAC;IACf,6EAA6E;IAC7E,UAAU,EAAE,eAAe,CAAC;IAC5B,8EAA8E;IAC9E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,MAAM,EAAE,QAAQ,EAAE,CAAC;IACnB,OAAO,EAAE,WAAW,EAAE,CAAC;IACvB,MAAM,EAAE;QAAE,KAAK,EAAE,aAAa,CAAC;QAAC,UAAU,EAAE,kBAAkB,CAAA;KAAE,EAAE,CAAC;CACpE;AAaD,wBAAgB,cAAc,CAAC,KAAK,EAAE,aAAa,GAAG,UAAU,CA6E/D;AAED,mEAAmE;AACnE,wBAAgB,cAAc,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAyDzD"}

package/dist/src/eval/report.js

@@ -58,6 +58,7 @@ export function assembleReport(input) {
     return {
         schemaVersion: 4,
         generatedAt: new Date().toISOString(),
+        provenance: input.provenance,
         surface: {
             backend: input.backend,
             mcpTransport: input.mcpTransport,
@@ -91,6 +92,8 @@ export function renderMarkdown(report) {
     lines.push(`- Generated: \`${report.generatedAt}\``);
     lines.push(`- MCP surface: \`${report.surface.backend}\` (transport: \`${report.surface.mcpTransport}\`${report.surface.remoteUrl ? `, remote: \`${report.surface.remoteUrl}\`` : ""})`);
     lines.push(`- Auth mode: \`${report.surface.auth}\``);
+    lines.push(`- Negotiated protocol: \`${report.provenance.protocolVersion ?? "n/a"}\``);
+    lines.push(`- Suite git SHA: \`${report.provenance.suiteGitSha}\` (${report.provenance.suiteGitShaSource})`);
     lines.push(`- Corpus: ${report.corpus.scenarios} GIS workflows`);
     lines.push(`- Live models evaluated: ${report.summary.liveModelsEvaluated}`);
     const unresolved = report.catalog.unresolvedRequiredTools;

package/dist/src/eval/runner.d.ts

@@ -17,6 +17,12 @@ export interface RunEvalOptions {
     corpus?: Scenario[];
     /** Force the in-memory fixture surface even if HONUA_MCP_REMOTE_URL is set. */
     forceOffline?: boolean;
+    /**
+     * Use the PLATFORM-FREE standalone fixture surface (a plain public FeatureServer,
+     * recorded census layer) instead of the Honua operator fixture. Implies offline.
+     * Defaults to true when `HONUA_EVAL_CORPUS=standalone`.
+     */
+    forceStandaloneSurface?: boolean;
 }
 export declare function runEval(options?: RunEvalOptions): Promise<EvalReport>;
 //# sourceMappingURL=runner.d.ts.map

package/dist/src/eval/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/eval/runner.ts"],"names":[],"mappings":"AAQA,OAAO,EAAiB,KAAK,UAAU,EAAkB,MAAM,aAAa,CAAC;AAC7E,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAmC,MAAM,YAAY,CAAC;AAEzF;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACpB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,OAAO,CAAC;CACxB;AAyFD,wBAAsB,OAAO,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,UAAU,CAAC,CAqC/E"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/eval/runner.ts"],"names":[],"mappings":"AAUA,OAAO,EAAiB,KAAK,UAAU,EAAkB,MAAM,aAAa,CAAC;AAC7E,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAmC,MAAM,YAAY,CAAC;AAEzF;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACpB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AA+FD,wBAAsB,OAAO,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,UAAU,CAAC,CAiD/E"}

package/dist/src/eval/runner.js

@@ -1,9 +1,11 @@
 import { Client } from "@modelcontextprotocol/sdk/client/index.js";
 import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { createStandaloneFixtureClient } from "../certification/census-fixture-client.js";
 import { createFixtureClient } from "../certification/fixture-client.js";
 import { createServer } from "../index.js";
+import { readNegotiatedProtocolVersion, resolveSuiteGitSha } from "../provenance.js";
 import { connectUpstream, resolveProxyOptions } from "../proxy.js";
-import { resolveCorpus } from "./corpus.js";
+import { isStandaloneCorpus, resolveCorpus } from "./corpus.js";
 import { resolveDrivers } from "./drivers/index.js";
 import { grade } from "./grade.js";
 import { assembleReport } from "./report.js";
@@ -15,8 +17,8 @@ function isTruthy(value) {
     const v = value.trim().toLowerCase();
     return v.length > 0 && v !== "0" && v !== "false" && v !== "no" && v !== "off";
 }
-async function connectSurface(env, forceOffline) {
-    const live = !forceOffline && typeof env.HONUA_MCP_REMOTE_URL === "string" && env.HONUA_MCP_REMOTE_URL.length > 0;
+async function connectSurface(env, forceOffline, standalone) {
+    const live = !forceOffline && !standalone && typeof env.HONUA_MCP_REMOTE_URL === "string" && env.HONUA_MCP_REMOTE_URL.length > 0;
     if (live) {
         const options = resolveProxyOptions(env);
         // The live lane authenticates via the proxy's real credential headers — no
@@ -39,7 +41,8 @@ async function connectSurface(env, forceOffline) {
             },
         };
     }
-    const server = createServer(createFixtureClient());
+    const fixtureClient = standalone ? createStandaloneFixtureClient() : createFixtureClient();
+    const server = createServer(fixtureClient);
     const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
     const client = new Client({ name: "honua-mcp-eval", version: "1.0.0" });
     await server.connect(serverTransport);
@@ -47,7 +50,7 @@ async function connectSurface(env, forceOffline) {
     return {
         client,
         backend: "fixture",
-        mcpTransport: "in-memory",
+        mcpTransport: standalone ? "in-memory (standalone FeatureServer fixture)" : "in-memory",
         auth: "none",
         close: async () => {
             await client.close().catch(() => { });
@@ -82,8 +85,9 @@ export async function runEval(options = {}) {
     const env = options.env ?? process.env;
     const corpus = options.corpus ?? resolveCorpus(env);
     const drivers = options.drivers ?? resolveDrivers({ env });
-    const forceOffline = options.forceOffline ?? false;
-    const surface = await connectSurface(env, forceOffline);
+    const standalone = options.forceStandaloneSurface ?? isStandaloneCorpus(env);
+    const forceOffline = options.forceOffline ?? standalone;
+    const surface = await connectSurface(env, forceOffline, standalone);
     try {
         const listed = await surface.client.listTools();
         const tools = listed.tools.map((t) => ({
@@ -99,11 +103,21 @@ export async function runEval(options = {}) {
                 graded.push({ grade: grade(scenario, transcript), transcript });
             }
         }
+        const gitSha = resolveSuiteGitSha(env);
+        const provenance = {
+            suiteGitSha: gitSha.sha,
+            suiteGitShaSource: gitSha.source,
+            targetUrl: surface.remoteUrl ?? `${surface.backend} (${surface.mcpTransport})`,
+            protocolVersion: readNegotiatedProtocolVersion(surface.client),
+            toolCount: tools.length,
+            authMode: surface.auth,
+        };
         return assembleReport({
             backend: surface.backend,
             mcpTransport: surface.mcpTransport,
             remoteUrl: surface.remoteUrl,
             auth: surface.auth,
+            provenance,
             advertisedTools: tools.map((t) => t.name),
             corpus,
             drivers,

package/dist/src/eval/standalone-corpus.d.ts

@@ -0,0 +1,3 @@
+import type { Scenario } from "./types.js";
+export declare const STANDALONE_CORPUS: Scenario[];
+//# sourceMappingURL=standalone-corpus.d.ts.map

package/dist/src/eval/standalone-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"standalone-corpus.d.ts","sourceRoot":"","sources":["../../../src/eval/standalone-corpus.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAgC3C,eAAO,MAAM,iBAAiB,EAAE,QAAQ,EA2qBvC,CAAC"}