@honua/mcp-server 0.0.9-alpha.0 → 0.0.20-alpha.0

package/dist/src/certification/certifier.js

@@ -0,0 +1,410 @@
+import { AUTH_ERROR_CODES, checkAuthContract, checkInvalidArgsContract, checkPaginationContract, } from "./contracts.js";
+import { buildInvalidArgsInputs, buildRoundTripInputs, resolveRoundTripEnv } from "./fixtures.js";
+import { checkConformance, checkWellFormed, validateAgainstSchema } from "./json-schema.js";
+import { buildReferenceToolLookup, loadSchemaFile, loadSchemaIndex, } from "./schema-index.js";
+const STANDARD_SOURCE = "geospatial-mcp@54cbd49 (spec/schemas)";
+/** Determine whether a discovered tool is safe to round-trip (read-only). */
+export function isReadOnlyTool(tool) {
+    if (tool.annotations && typeof tool.annotations.readOnlyHint === "boolean") {
+        return tool.annotations.readOnlyHint;
+    }
+    const mutating = /(create|update|delete|publish|apply_.*write|execute|edit|deploy|cancel|propose)/i;
+    return !mutating.test(tool.name);
+}
+/**
+ * Connect an MCP client to the certified surface and produce a deterministic
+ * certification report. The client is already connected (offline mock upstream,
+ * a live `/mcp`, or the stdio proxy); certify never owns its lifecycle.
+ */
+export async function certify(options) {
+    const { client } = options;
+    const roundTripEnv = resolveRoundTripEnv(options.env);
+    const roundTripInputs = buildRoundTripInputs(roundTripEnv);
+    const index = loadSchemaIndex();
+    const referenceLookup = buildReferenceToolLookup(index);
+    const serverInfo = client.getServerVersion() ?? { name: "unknown", version: "0.0.0" };
+    const { tools: advertisedTools, pages: toolPages } = await listAllTools(client);
+    const { resources: advertisedResources, pages: resourcePages } = await listAllResources(client);
+    const prompts = await listPromptsSafely(client);
+    const tools = [];
+    const advertisedNames = new Set();
+    for (const tool of advertisedTools) {
+        advertisedNames.add(tool.name);
+        tools.push(await certifyTool(client, tool, referenceLookup, roundTripInputs));
+    }
+    const resources = advertisedResources.map((r) => ({
+        name: r.name,
+        uri: r.uri,
+        discovered: true,
+    }));
+    const contracts = [];
+    contracts.push(paginationCheck("tools", toolPages));
+    contracts.push(paginationCheck("resources", resourcePages));
+    collectOutputSchemaContracts(tools, contracts);
+    contracts.push(await runErrorShapeContract(client, advertisedTools));
+    await runAuthContracts(options, advertisedTools, advertisedResources, contracts);
+    const knownGaps = collectKnownGaps(index, advertisedNames, referenceLookup);
+    return assembleReport({
+        serverInfo: serverInfo,
+        index,
+        options,
+        tools,
+        resources,
+        prompts,
+        contracts,
+        knownGaps,
+    });
+}
+async function listAllTools(client) {
+    const tools = [];
+    const pages = [];
+    let cursor;
+    do {
+        const resp = await client.listTools(cursor ? { cursor } : undefined);
+        tools.push(...resp.tools);
+        pages.push({ keys: resp.tools.map((t) => t.name), nextCursor: resp.nextCursor });
+        cursor = resp.nextCursor;
+    } while (cursor && pages.length < 100);
+    return { tools, pages };
+}
+async function listAllResources(client) {
+    const resources = [];
+    const pages = [];
+    let cursor;
+    do {
+        const resp = await client.listResources(cursor ? { cursor } : undefined);
+        resources.push(...resp.resources.map((r) => ({ name: r.name, uri: r.uri })));
+        pages.push({ keys: resp.resources.map((r) => r.uri), nextCursor: resp.nextCursor });
+        cursor = resp.nextCursor;
+    } while (cursor && pages.length < 100);
+    return { resources, pages };
+}
+async function listPromptsSafely(client) {
+    try {
+        const response = await client.listPrompts();
+        return response.prompts.map((p) => ({ name: p.name, discovered: true }));
+    }
+    catch {
+        return [];
+    }
+}
+async function certifyTool(client, tool, referenceLookup, roundTripInputs) {
+    const errors = [];
+    const notes = [];
+    const wellFormed = checkWellFormed(tool.inputSchema);
+    if (!wellFormed.wellFormed) {
+        errors.push(...wellFormed.errors.map((e) => `inputSchema not well-formed: ${e}`));
+    }
+    const hasOutputSchema = tool.outputSchema !== undefined && tool.outputSchema !== null;
+    if (hasOutputSchema) {
+        const outputWellFormed = checkWellFormed(tool.outputSchema);
+        if (!outputWellFormed.wellFormed) {
+            errors.push(...outputWellFormed.errors.map((e) => `outputSchema not well-formed: ${e}`));
+        }
+    }
+    else {
+        notes.push("no structured output schema advertised");
+    }
+    const standardEntry = referenceLookup.get(tool.name);
+    let conformant = null;
+    if (standardEntry) {
+        const standardSchema = loadSchemaFile(standardEntry.schema);
+        if (wellFormed.wellFormed) {
+            const result = checkConformance(tool.inputSchema, standardSchema);
+            conformant = result.conformant;
+            if (!result.conformant) {
+                errors.push(...result.violations.map((v) => `conformance: ${v}`));
+            }
+            notes.push(...result.notes);
+        }
+        else {
+            conformant = false;
+            errors.push("conformance: skipped — inputSchema is not well-formed");
+        }
+    }
+    const readOnly = isReadOnlyTool(tool);
+    let roundTrip = "skipped";
+    let structuredOutputValidated = null;
+    const fixtureInput = roundTripInputs[tool.name];
+    if (readOnly && fixtureInput !== undefined) {
+        const outcome = await roundTripTool(client, tool, fixtureInput, errors, notes);
+        roundTrip = outcome.roundTrip;
+        structuredOutputValidated = outcome.structuredOutputValidated;
+    }
+    else if (!readOnly) {
+        notes.push("round-trip skipped: tool is not read-only");
+    }
+    else {
+        notes.push("round-trip skipped: no fixture input");
+    }
+    return {
+        name: tool.name,
+        discovered: true,
+        schemaValid: wellFormed.wellFormed,
+        hasOutputSchema,
+        standardName: standardEntry?.standardName ?? null,
+        conformant,
+        readOnly,
+        roundTrip,
+        structuredOutputValidated,
+        errors,
+        notes,
+    };
+}
+async function roundTripTool(client, tool, input, errors, notes) {
+    try {
+        const result = (await client.callTool({ name: tool.name, arguments: input }));
+        if (result.isError) {
+            errors.push("round-trip: tool returned isError=true");
+            return { roundTrip: "failed", structuredOutputValidated: null };
+        }
+        if (tool.outputSchema && result.structuredContent !== undefined) {
+            const validation = validateAgainstSchema(tool.outputSchema, result.structuredContent);
+            if (!validation.valid) {
+                errors.push(...validation.errors.map((e) => `output-schema: structuredContent ${e}`));
+                return { roundTrip: "failed", structuredOutputValidated: false };
+            }
+            return { roundTrip: "passed", structuredOutputValidated: true };
+        }
+        if (tool.outputSchema && result.structuredContent === undefined) {
+            errors.push("output-schema: outputSchema advertised but no structuredContent returned");
+            return { roundTrip: "failed", structuredOutputValidated: false };
+        }
+        return { roundTrip: "passed", structuredOutputValidated: null };
+    }
+    catch (err) {
+        errors.push(`round-trip: ${err instanceof Error ? err.message : String(err)}`);
+        return { roundTrip: "failed", structuredOutputValidated: null };
+    }
+}
+function paginationCheck(target, pages) {
+    const paginated = pages.length > 1 || (pages[0]?.nextCursor !== undefined && pages[0]?.nextCursor !== null);
+    if (!paginated) {
+        return {
+            contract: "list-pagination",
+            target,
+            status: "skipped",
+            detail: `${target} list is a single page (no nextCursor advertised)`,
+        };
+    }
+    const outcome = checkPaginationContract(pages);
+    return {
+        contract: "list-pagination",
+        target,
+        status: outcome.ok ? "passed" : "failed",
+        detail: outcome.ok
+            ? `${target} list paginated across ${pages.length} pages; nextCursor honored and terminated`
+            : outcome.errors.join("; "),
+    };
+}
+function collectOutputSchemaContracts(tools, contracts) {
+    for (const tool of tools) {
+        if (tool.structuredOutputValidated === null) {
+            continue;
+        }
+        contracts.push({
+            contract: "output-schema",
+            target: tool.name,
+            status: tool.structuredOutputValidated ? "passed" : "failed",
+            detail: tool.structuredOutputValidated
+                ? "structuredContent validated against advertised outputSchema"
+                : "structuredContent did not validate against advertised outputSchema",
+        });
+    }
+}
+async function runErrorShapeContract(client, advertised) {
+    const invalidInputs = buildInvalidArgsInputs();
+    const advertisedNames = new Set(advertised.map((t) => t.name));
+    const target = Object.keys(invalidInputs).find((name) => advertisedNames.has(name));
+    if (!target) {
+        return {
+            contract: "error-shape",
+            target: "(none)",
+            status: "skipped",
+            detail: "no advertised tool has an invalid-args fixture",
+        };
+    }
+    try {
+        const result = (await client.callTool({ name: target, arguments: invalidInputs[target] }));
+        const outcome = checkInvalidArgsContract(result);
+        return contractFromOutcome("error-shape", target, outcome);
+    }
+    catch (err) {
+        return {
+            contract: "error-shape",
+            target,
+            status: "failed",
+            detail: `invalid arguments raised a protocol error instead of a structured tool error: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+}
+async function runAuthContracts(options, advertised, resources, contracts) {
+    if (!options.connectUnauthenticated) {
+        contracts.push({
+            contract: "auth-unauthenticated",
+            target: "tools/call",
+            status: "skipped",
+            detail: "target does not support an unauthenticated pass",
+        });
+        contracts.push({
+            contract: "auth-unauthenticated",
+            target: "resources/read",
+            status: "skipped",
+            detail: "target does not support an unauthenticated pass",
+        });
+        return;
+    }
+    let unauth;
+    try {
+        unauth = await options.connectUnauthenticated();
+        // tools/call without credentials.
+        const toolTarget = advertised.find((t) => isReadOnlyTool(t)) ?? advertised[0];
+        if (toolTarget) {
+            const roundTripInputs = buildRoundTripInputs(resolveRoundTripEnv(options.env));
+            const args = roundTripInputs[toolTarget.name] ?? {};
+            contracts.push(await authCheckToolsCall(unauth, toolTarget.name, args));
+        }
+        // resources/read without credentials.
+        if (resources[0]) {
+            contracts.push(await authCheckResourcesRead(unauth, resources[0].uri));
+        }
+    }
+    catch (err) {
+        contracts.push({
+            contract: "auth-unauthenticated",
+            target: "tools/call",
+            status: "failed",
+            detail: `unauthenticated pass could not be established: ${err instanceof Error ? err.message : String(err)}`,
+        });
+    }
+    finally {
+        if (unauth) {
+            await unauth.close().catch(() => { });
+        }
+    }
+}
+async function authCheckToolsCall(client, name, args) {
+    try {
+        const result = await client.callTool({ name, arguments: args });
+        if (result.isError !== true) {
+            return {
+                contract: "auth-unauthenticated",
+                target: "tools/call",
+                status: "failed",
+                detail: `unauthenticated tools/call on ${name} returned a non-error result — auth boundary not enforced`,
+            };
+        }
+        return contractFromOutcome("auth-unauthenticated", "tools/call", checkAuthContract(result));
+    }
+    catch (err) {
+        // A structured McpError carrying an auth code also satisfies the contract.
+        return contractFromOutcome("auth-unauthenticated", "tools/call", checkAuthContract(err));
+    }
+}
+async function authCheckResourcesRead(client, uri) {
+    try {
+        await client.readResource({ uri });
+        return {
+            contract: "auth-unauthenticated",
+            target: "resources/read",
+            status: "failed",
+            detail: `unauthenticated resources/read on ${uri} succeeded — auth boundary not enforced`,
+        };
+    }
+    catch (err) {
+        return contractFromOutcome("auth-unauthenticated", "resources/read", checkAuthContract(err));
+    }
+}
+function contractFromOutcome(contract, target, outcome) {
+    return {
+        contract,
+        target,
+        status: outcome.ok ? "passed" : "failed",
+        detail: outcome.ok ? "contract satisfied" : outcome.errors.join("; "),
+    };
+}
+function collectKnownGaps(index, advertisedNames, referenceLookup) {
+    const gaps = [];
+    // Advertised names that resolve to a standard entry through the SDK-local
+    // reference aliases (e.g. honua_dry_run_plan → validate_plan) also count as
+    // covering that standard tool.
+    const coveredStandardNames = new Set();
+    for (const name of advertisedNames) {
+        const entry = referenceLookup.get(name);
+        if (entry) {
+            coveredStandardNames.add(entry.standardName);
+        }
+    }
+    for (const tool of index.tools) {
+        if (!coveredStandardNames.has(tool.standardName)) {
+            gaps.push({
+                kind: "standard-tool",
+                name: tool.standardName,
+                family: tool.family,
+                detail: tool.implementationStatus === "known-gap"
+                    ? "standard family not yet implemented as a discrete tool"
+                    : `reference tool ${tool.referenceToolName ?? "(unnamed)"} not advertised by the certified surface`,
+            });
+        }
+    }
+    for (const name of advertisedNames) {
+        if (!referenceLookup.has(name)) {
+            gaps.push({
+                kind: "advertised-tool",
+                name,
+                detail: "advertised tool has no matching geospatial-mcp standard schema (validated for well-formedness only)",
+            });
+        }
+    }
+    return gaps;
+}
+function assembleReport(args) {
+    const { serverInfo, index, options, tools, resources, prompts, contracts, knownGaps } = args;
+    const toolsSchemaValid = tools.filter((t) => t.schemaValid).length;
+    const conformanceChecked = tools.filter((t) => t.conformant !== null);
+    const toolsConformant = conformanceChecked.filter((t) => t.conformant === true).length;
+    const toolsRoundTripped = tools.filter((t) => t.roundTrip === "passed").length;
+    const toolsOutputValidated = tools.filter((t) => t.structuredOutputValidated === true).length;
+    const contractsChecked = contracts.filter((c) => c.status !== "skipped").length;
+    const contractsPassed = contracts.filter((c) => c.status === "passed").length;
+    const contractsFailed = contracts.filter((c) => c.status === "failed").length;
+    const toolFailures = tools.reduce((acc, t) => acc + t.errors.length, 0);
+    const failures = toolFailures + contractsFailed;
+    const pass = failures === 0;
+    return {
+        schemaVersion: 2,
+        generatedAt: new Date().toISOString(),
+        server: serverInfo,
+        protocol: {
+            mcpTransport: options.targetMode === "stdio-proxy" ? "stdio→streamable-http" : "streamable-http",
+            honuaTransport: options.honuaTransport ?? null,
+            backend: options.backend,
+            targetMode: options.targetMode,
+            surface: options.surface,
+        },
+        standard: { source: STANDARD_SOURCE, indexDate: index.date, dialect: index.dialect },
+        summary: {
+            pass,
+            toolsDiscovered: tools.length,
+            toolsSchemaValid,
+            toolsConformanceChecked: conformanceChecked.length,
+            toolsConformant,
+            toolsRoundTripped,
+            toolsOutputValidated,
+            resourcesDiscovered: resources.length,
+            promptsDiscovered: prompts.length,
+            contractsChecked,
+            contractsPassed,
+            contractsFailed,
+            knownGaps: knownGaps.length,
+            failures,
+        },
+        tools,
+        resources,
+        prompts,
+        contracts,
+        knownGaps,
+    };
+}
+// Re-export for artifact/report consumers.
+export { AUTH_ERROR_CODES };

package/dist/src/certification/cli.d.ts

@@ -0,0 +1,3 @@
+#!/usr/bin/env node
+export {};
+//# sourceMappingURL=cli.d.ts.map

package/dist/src/certification/cli.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../../../src/certification/cli.ts"],"names":[],"mappings":""}

package/dist/src/certification/cli.js

@@ -0,0 +1,48 @@
+#!/usr/bin/env node
+import { resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { runCertification, writeArtifacts } from "./run.js";
+function parseArgs(argv) {
+    let artifactOnly = false;
+    // Default to the package root (two levels up from dist/src/certification/).
+    let outDir = fileURLToPath(new URL("../../../", import.meta.url));
+    for (let i = 0; i < argv.length; i++) {
+        const arg = argv[i];
+        if (arg === "--artifact-only") {
+            artifactOnly = true;
+        }
+        else if (arg === "--out-dir") {
+            const value = argv[++i];
+            if (!value) {
+                throw new Error("--out-dir requires a directory argument");
+            }
+            outDir = resolve(value);
+        }
+    }
+    return { artifactOnly, outDir };
+}
+function artifactPaths(outDir) {
+    return {
+        json: resolve(outDir, "mcp-certification-results.json"),
+        markdown: resolve(outDir, "mcp-certification-results.md"),
+    };
+}
+async function main() {
+    const { artifactOnly, outDir } = parseArgs(process.argv.slice(2));
+    const report = await runCertification();
+    const paths = artifactPaths(outDir);
+    writeArtifacts(report, paths);
+    const s = report.summary;
+    const status = s.pass ? "PASS" : "FAIL";
+    process.stdout.write(`MCP certification ${status} [${report.protocol.targetMode}]: ${s.toolsConformant}/${s.toolsConformanceChecked} conformant, ` +
+        `${s.toolsOutputValidated} output-validated, ${s.contractsPassed}/${s.contractsChecked} contracts, ` +
+        `${s.knownGaps} known gaps, ${s.failures} failures.\n`);
+    process.stdout.write(`Artifacts: ${paths.json}\n           ${paths.markdown}\n`);
+    if (!artifactOnly && !report.summary.pass) {
+        process.exitCode = 1;
+    }
+}
+main().catch((err) => {
+    process.stderr.write(`Certification error: ${err instanceof Error ? (err.stack ?? err.message) : String(err)}\n`);
+    process.exit(1);
+});

package/dist/src/certification/contracts.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Pure, deterministic contract-check analyzers for MCP certification.
+ *
+ * These operate on values already obtained from the certified surface (tool
+ * results, thrown errors, paginated list responses) so they are unit-testable
+ * without any transport. The certifier does the I/O; these judge the shape.
+ *
+ * Contracts enforced (§P0.2):
+ *   - output-schema : a tool's structuredContent validates against its outputSchema.
+ *   - error-shape   : invalid arguments yield an `isError` tool result carrying a
+ *                     structured, actionable code (validated against the vendored
+ *                     geoprocessing-error schema), never a protocol error.
+ *   - auth          : unauthenticated tools/call and resources/read surface a
+ *                     structured `unauthenticated` / `permission_denied` signal.
+ *   - pagination    : list `nextCursor` is honored and terminates.
+ */
+/** Machine codes that satisfy the unauthenticated / permission-denied contract. */
+export declare const AUTH_ERROR_CODES: readonly ["unauthenticated", "permission_denied"];
+/** geoprocessing-error `kind` values that denote an authorization failure. */
+export declare const AUTH_ERROR_KINDS: readonly ["AuthorizationDenied", "Unauthenticated", "PermissionDenied"];
+export interface StructuredErrorEnvelope {
+    code?: string;
+    error?: unknown;
+}
+/**
+ * Normalize a structured error envelope out of any of the places a surface may
+ * carry it: a tool result's `structuredContent`, an `McpError`'s `data`, or a
+ * bare geoprocessing-error object. Returns undefined when no structure is found.
+ */
+export declare function extractStructuredError(source: unknown): StructuredErrorEnvelope | undefined;
+export interface ContractOutcome {
+    ok: boolean;
+    errors: string[];
+}
+/** Validate a value against the vendored geoprocessing-error schema. */
+export declare function validateGeoprocessingError(error: unknown): ContractOutcome;
+/**
+ * error-shape contract: an invalid-argument tool call must return an `isError`
+ * result whose structuredContent carries an actionable code and a
+ * geoprocessing-error-shaped `ValidationFailed` envelope.
+ */
+export declare function checkInvalidArgsContract(result: {
+    isError?: boolean;
+    structuredContent?: unknown;
+}): ContractOutcome;
+/**
+ * auth contract: an unauthenticated call must surface a structured
+ * `unauthenticated` / `permission_denied` signal — either as a code on the
+ * envelope or as an AuthorizationDenied geoprocessing-error kind.
+ */
+export declare function checkAuthContract(source: unknown): ContractOutcome;
+export interface ListPage {
+    /** Distinct item identity keys observed on this page (e.g. tool names). */
+    keys: string[];
+    nextCursor: string | undefined;
+}
+/**
+ * pagination contract: following `nextCursor` yields additional distinct items,
+ * never repeats a cursor (terminates), and the final page has no cursor.
+ */
+export declare function checkPaginationContract(pages: ListPage[]): ContractOutcome;
+//# sourceMappingURL=contracts.d.ts.map

package/dist/src/certification/contracts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"contracts.d.ts","sourceRoot":"","sources":["../../../src/certification/contracts.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;GAeG;AAEH,mFAAmF;AACnF,eAAO,MAAM,gBAAgB,mDAAoD,CAAC;AAElF,8EAA8E;AAC9E,eAAO,MAAM,gBAAgB,yEAA0E,CAAC;AAUxG,MAAM,WAAW,uBAAuB;IACtC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAMD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,OAAO,GAAG,uBAAuB,GAAG,SAAS,CAgC3F;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,wEAAwE;AACxE,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,OAAO,GAAG,eAAe,CAG1E;AAED;;;;GAIG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,iBAAiB,CAAC,EAAE,OAAO,CAAC;CAC7B,GAAG,eAAe,CA8BlB;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,OAAO,GAAG,eAAe,CAoBlE;AAED,MAAM,WAAW,QAAQ;IACvB,2EAA2E;IAC3E,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;CAChC;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,eAAe,CA6B1E"}