@honua/mcp-server 0.0.10-alpha.0 → 0.1.0-beta.0

package/dist/src/eval/runner.d.ts

@@ -0,0 +1,28 @@
+import { type EvalReport } from "./report.js";
+import type { ModelDriver, Scenario } from "./types.js";
+/**
+ * Cross-model eval runner (honua-server #1956).
+ *
+ * Backend selection mirrors the certifier: offline-first and deterministic.
+ *   - HONUA_MCP_REMOTE_URL set → drive the live remote /mcp over streamable HTTP
+ *     (the same surface the stdio proxy bridges).
+ *   - otherwise → the offline in-memory fixture server (no network, no models).
+ *
+ * Drivers default to whatever `resolveDrivers` returns: the deterministic control
+ * always, plus Claude/GPT when their API keys are present.
+ */
+export interface RunEvalOptions {
+    env?: NodeJS.ProcessEnv;
+    drivers?: ModelDriver[];
+    corpus?: Scenario[];
+    /** Force the in-memory fixture surface even if HONUA_MCP_REMOTE_URL is set. */
+    forceOffline?: boolean;
+    /**
+     * Use the PLATFORM-FREE standalone fixture surface (a plain public FeatureServer,
+     * recorded census layer) instead of the Honua operator fixture. Implies offline.
+     * Defaults to true when `HONUA_EVAL_CORPUS=standalone`.
+     */
+    forceStandaloneSurface?: boolean;
+}
+export declare function runEval(options?: RunEvalOptions): Promise<EvalReport>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/src/eval/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/eval/runner.ts"],"names":[],"mappings":"AAUA,OAAO,EAAiB,KAAK,UAAU,EAAkB,MAAM,aAAa,CAAC;AAC7E,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAmC,MAAM,YAAY,CAAC;AAEzF;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,cAAc;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACxB,OAAO,CAAC,EAAE,WAAW,EAAE,CAAC;IACxB,MAAM,CAAC,EAAE,QAAQ,EAAE,CAAC;IACpB,+EAA+E;IAC/E,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB;;;;OAIG;IACH,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AA+FD,wBAAsB,OAAO,CAAC,OAAO,GAAE,cAAmB,GAAG,OAAO,CAAC,UAAU,CAAC,CAiD/E"}

package/dist/src/eval/runner.js

@@ -0,0 +1,130 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { createStandaloneFixtureClient } from "../certification/census-fixture-client.js";
+import { createFixtureClient } from "../certification/fixture-client.js";
+import { createServer } from "../index.js";
+import { readNegotiatedProtocolVersion, resolveSuiteGitSha } from "../provenance.js";
+import { connectUpstream, resolveProxyOptions } from "../proxy.js";
+import { isStandaloneCorpus, resolveCorpus } from "./corpus.js";
+import { resolveDrivers } from "./drivers/index.js";
+import { grade } from "./grade.js";
+import { assembleReport } from "./report.js";
+/** Truthy-flag parse shared with the driver opt-ins (0/false/no/off ⇒ false). */
+function isTruthy(value) {
+    if (typeof value !== "string") {
+        return false;
+    }
+    const v = value.trim().toLowerCase();
+    return v.length > 0 && v !== "0" && v !== "false" && v !== "no" && v !== "off";
+}
+async function connectSurface(env, forceOffline, standalone) {
+    const live = !forceOffline && !standalone && typeof env.HONUA_MCP_REMOTE_URL === "string" && env.HONUA_MCP_REMOTE_URL.length > 0;
+    if (live) {
+        const options = resolveProxyOptions(env);
+        // The live lane authenticates via the proxy's real credential headers — no
+        // dev-auth bypass. `Bearer` takes precedence over `x-api-key`; anonymous means
+        // no credential was supplied at all.
+        const auth = options.authToken ? "bearer" : options.apiKey ? "api-key" : "anonymous";
+        if (auth === "anonymous" && isTruthy(env.HONUA_EVAL_REQUIRE_AUTH)) {
+            throw new Error("HONUA_EVAL_REQUIRE_AUTH is set but the live MCP surface would be reached anonymously " +
+                "(no HONUA_MCP_AUTH_TOKEN / HONUA_API_KEY). Refusing to run an unauthenticated live eval.");
+        }
+        const client = await connectUpstream(options);
+        return {
+            client,
+            backend: "live",
+            mcpTransport: "streamable-http",
+            remoteUrl: options.remoteUrl,
+            auth,
+            close: async () => {
+                await client.close().catch(() => { });
+            },
+        };
+    }
+    const fixtureClient = standalone ? createStandaloneFixtureClient() : createFixtureClient();
+    const server = createServer(fixtureClient);
+    const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+    const client = new Client({ name: "honua-mcp-eval", version: "1.0.0" });
+    await server.connect(serverTransport);
+    await client.connect(clientTransport);
+    return {
+        client,
+        backend: "fixture",
+        mcpTransport: standalone ? "in-memory (standalone FeatureServer fixture)" : "in-memory",
+        auth: "none",
+        close: async () => {
+            await client.close().catch(() => { });
+            await server.close().catch(() => { });
+        },
+    };
+}
+function flattenToolResult(result) {
+    const r = result;
+    const isError = r.isError === true;
+    const texts = (r.content ?? [])
+        .filter((b) => b.type === "text" && typeof b.text === "string")
+        .map((b) => b.text);
+    const text = texts.length > 0 ? texts.join("\n") : JSON.stringify(r.content ?? result);
+    return { isError, text };
+}
+function buildContext(client, tools) {
+    return {
+        tools,
+        async callTool(name, args) {
+            try {
+                const result = await client.callTool({ name, arguments: args });
+                return flattenToolResult(result);
+            }
+            catch (err) {
+                return { isError: true, text: err instanceof Error ? err.message : String(err) };
+            }
+        },
+    };
+}
+export async function runEval(options = {}) {
+    const env = options.env ?? process.env;
+    const corpus = options.corpus ?? resolveCorpus(env);
+    const drivers = options.drivers ?? resolveDrivers({ env });
+    const standalone = options.forceStandaloneSurface ?? isStandaloneCorpus(env);
+    const forceOffline = options.forceOffline ?? standalone;
+    const surface = await connectSurface(env, forceOffline, standalone);
+    try {
+        const listed = await surface.client.listTools();
+        const tools = listed.tools.map((t) => ({
+            name: t.name,
+            description: t.description ?? "",
+            inputSchema: t.inputSchema,
+        }));
+        const ctx = buildContext(surface.client, tools);
+        const graded = [];
+        for (const driver of drivers) {
+            for (const scenario of corpus) {
+                const transcript = await driver.runWorkflow(scenario, ctx);
+                graded.push({ grade: grade(scenario, transcript), transcript });
+            }
+        }
+        const gitSha = resolveSuiteGitSha(env);
+        const provenance = {
+            suiteGitSha: gitSha.sha,
+            suiteGitShaSource: gitSha.source,
+            targetUrl: surface.remoteUrl ?? `${surface.backend} (${surface.mcpTransport})`,
+            protocolVersion: readNegotiatedProtocolVersion(surface.client),
+            toolCount: tools.length,
+            authMode: surface.auth,
+        };
+        return assembleReport({
+            backend: surface.backend,
+            mcpTransport: surface.mcpTransport,
+            remoteUrl: surface.remoteUrl,
+            auth: surface.auth,
+            provenance,
+            advertisedTools: tools.map((t) => t.name),
+            corpus,
+            drivers,
+            graded,
+        });
+    }
+    finally {
+        await surface.close();
+    }
+}

package/dist/src/eval/standalone-corpus.d.ts

@@ -0,0 +1,3 @@
+import type { Scenario } from "./types.js";
+export declare const STANDALONE_CORPUS: Scenario[];
+//# sourceMappingURL=standalone-corpus.d.ts.map

package/dist/src/eval/standalone-corpus.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"standalone-corpus.d.ts","sourceRoot":"","sources":["../../../src/eval/standalone-corpus.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAgC3C,eAAO,MAAM,iBAAiB,EAAE,QAAQ,EA2qBvC,CAAC"}