@digital-science-dsl/dimensions-analytics-mcp 0.5.3

package/test/integration/harness.ts

@@ -0,0 +1,307 @@
+/**
+ * MCP eval harness runner.
+ *
+ * Creates a real MCP server with live API credentials, connects an MCP Client
+ * via in-memory transport, and executes eval suites against it.
+ * @module test/integration/harness
+ */
+
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
+import { createMcpServerAsync, type McpServerConfig } from "../../src/mcp/server.js";
+import { missingEnvVars } from "./env.js";
+import type {
+  AssertionResult,
+  CaseResult,
+  EvalCase,
+  EvalRunResult,
+  EvalSuite,
+  SuiteResult,
+} from "./types.js";
+
+const DEFAULT_TIMEOUT = 30_000;
+
+/**
+ * Configuration for the eval harness.
+ */
+export interface EvalHarnessConfig {
+  /** MCP server config (API keys, URLs). Defaults to env vars. */
+  readonly serverConfig?: McpServerConfig;
+  /** Only run suites whose names match these (empty = all). */
+  readonly suiteFilter?: readonly string[];
+  /** Only run cases tagged with one of these (empty = all). */
+  readonly tagFilter?: readonly string[];
+  /** Exclude cases tagged with one of these. */
+  readonly tagExclude?: readonly string[];
+  /** Max concurrent cases per suite (default: 1 — sequential). */
+  readonly concurrency?: number;
+}
+
+/**
+ * Parses the JSON text from an MCP tool call result.
+ * @param result - Raw MCP callTool result
+ * @returns Parsed JSON data
+ */
+function parseResult(result: { content: Array<{ type: string; text?: string }> }): unknown {
+  const textContent = result.content.find((c) => c.type === "text" && c.text);
+  if (!textContent || !("text" in textContent) || !textContent.text) {
+    throw new Error("Tool result has no text content");
+  }
+  return JSON.parse(textContent.text);
+}
+
+/**
+ * Reads a resource via the MCP client and returns it as a tool-like result.
+ * @param client - Connected MCP client
+ * @param uri - Resource URI
+ * @returns Parsed resource content
+ */
+async function readResource(client: Client, uri: string): Promise<unknown> {
+  const result = await client.readResource({ uri });
+  const textContent = result.contents.find((c) => c.mimeType === "application/json" || c.text);
+  if (!textContent || !("text" in textContent) || !textContent.text) {
+    throw new Error("Resource has no text content");
+  }
+  return JSON.parse(textContent.text as string);
+}
+
+/**
+ * Runs a single eval case against the MCP client.
+ * @param client - Connected MCP client
+ * @param suiteName - Parent suite name
+ * @param evalCase - The case to run
+ * @returns Case result
+ */
+async function runCase(client: Client, suiteName: string, evalCase: EvalCase): Promise<CaseResult> {
+  const start = performance.now();
+  const timeout = evalCase.timeout ?? DEFAULT_TIMEOUT;
+
+  try {
+    // Handle resource reads (sentinel tool name)
+    if (evalCase.tool === "__resource__") {
+      const uri = evalCase.args.uri as string;
+      const data = await Promise.race([
+        readResource(client, uri),
+        new Promise<never>((_, reject) =>
+          setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout),
+        ),
+      ]);
+      const assertions: AssertionResult[] = evalCase.assertions.map((assertion) => {
+        try {
+          const checkResult = assertion.check(data);
+          return checkResult === true
+            ? { label: assertion.label, passed: true }
+            : { label: assertion.label, passed: false, error: checkResult };
+        } catch (err) {
+          return {
+            label: assertion.label,
+            passed: false,
+            error: `Assertion threw: ${err instanceof Error ? err.message : String(err)}`,
+          };
+        }
+      });
+      return {
+        suiteName,
+        caseName: evalCase.name,
+        tool: `resource:${uri}`,
+        passed: assertions.every((a) => a.passed),
+        assertions,
+        durationMs: performance.now() - start,
+      };
+    }
+
+    const result = await Promise.race([
+      client.callTool({ name: evalCase.tool, arguments: evalCase.args }),
+      new Promise<never>((_, reject) =>
+        setTimeout(() => reject(new Error(`Timeout after ${timeout}ms`)), timeout),
+      ),
+    ]);
+
+    // Check if the tool returned an error
+    if ("isError" in result && result.isError) {
+      const parsed = parseResult(result as { content: Array<{ type: string; text?: string }> });
+      const errorMsg =
+        parsed && typeof parsed === "object" && "error" in parsed
+          ? String((parsed as Record<string, unknown>).error)
+          : "Unknown tool error";
+      return {
+        suiteName,
+        caseName: evalCase.name,
+        tool: evalCase.tool,
+        passed: false,
+        assertions: [],
+        durationMs: performance.now() - start,
+        toolError: errorMsg,
+      };
+    }
+
+    const data = parseResult(result as { content: Array<{ type: string; text?: string }> });
+
+    // Run assertions
+    const assertions: AssertionResult[] = evalCase.assertions.map((assertion) => {
+      try {
+        const checkResult = assertion.check(data);
+        return checkResult === true
+          ? { label: assertion.label, passed: true }
+          : { label: assertion.label, passed: false, error: checkResult };
+      } catch (err) {
+        return {
+          label: assertion.label,
+          passed: false,
+          error: `Assertion threw: ${err instanceof Error ? err.message : String(err)}`,
+        };
+      }
+    });
+
+    return {
+      suiteName,
+      caseName: evalCase.name,
+      tool: evalCase.tool,
+      passed: assertions.every((a) => a.passed),
+      assertions,
+      durationMs: performance.now() - start,
+    };
+  } catch (err) {
+    return {
+      suiteName,
+      caseName: evalCase.name,
+      tool: evalCase.tool,
+      passed: false,
+      assertions: [],
+      durationMs: performance.now() - start,
+      toolError: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+/** Checks whether required credentials for a suite are present. */
+function missingCredentials(suite: EvalSuite, serverConfig?: McpServerConfig): string[] {
+  if (!suite.requiredEnvVars) return [];
+  return missingEnvVars(suite.requiredEnvVars).filter((v) => {
+    if (v === "DIMENSIONS_API_KEY") {
+      return !(process.env.DIMENSIONS_API_KEY ?? serverConfig?.apiKey);
+    }
+    return true;
+  });
+}
+
+/**
+ * Filters cases by tag include/exclude rules.
+ * @param cases - All cases in a suite
+ * @param config - Harness config with tag filters
+ * @returns Filtered cases
+ */
+function filterCases(cases: readonly EvalCase[], config: EvalHarnessConfig): readonly EvalCase[] {
+  let filtered = [...cases];
+
+  if (config.tagFilter && config.tagFilter.length > 0) {
+    filtered = filtered.filter((c) => c.tags?.some((t) => config.tagFilter?.includes(t)));
+  }
+
+  if (config.tagExclude && config.tagExclude.length > 0) {
+    filtered = filtered.filter(
+      (c) => !c.tags || !c.tags.some((t) => config.tagExclude?.includes(t)),
+    );
+  }
+
+  return filtered;
+}
+
+/**
+ * Runs all eval suites against an already-connected MCP client.
+ * @param client - Connected MCP client
+ * @param suites - Array of eval suites to run
+ * @param config - Harness configuration
+ * @returns Full eval run result
+ */
+export async function runEvalsWithClient(
+  client: Client,
+  suites: readonly EvalSuite[],
+  config: EvalHarnessConfig = {},
+): Promise<EvalRunResult> {
+  const runStart = performance.now();
+
+  const activeSuites =
+    config.suiteFilter && config.suiteFilter.length > 0
+      ? suites.filter((s) => config.suiteFilter?.includes(s.name))
+      : [...suites];
+
+  const suiteResults: SuiteResult[] = [];
+
+  for (const suite of activeSuites) {
+    const suiteStart = performance.now();
+
+    const missing = missingCredentials(suite, config.serverConfig);
+    if (missing.length > 0) {
+      const skippedCases: CaseResult[] = suite.cases.map((c) => ({
+        suiteName: suite.name,
+        caseName: c.name,
+        tool: c.tool,
+        passed: false,
+        assertions: [],
+        durationMs: 0,
+        skipped: true,
+        skipReason: `Missing env vars: ${missing.join(", ")}`,
+      }));
+      suiteResults.push({
+        name: suite.name,
+        cases: skippedCases,
+        passed: 0,
+        failed: 0,
+        skipped: skippedCases.length,
+        durationMs: 0,
+      });
+      continue;
+    }
+
+    const cases = filterCases(suite.cases, config);
+    const caseResults: CaseResult[] = [];
+
+    for (const evalCase of cases) {
+      const result = await runCase(client, suite.name, evalCase);
+      caseResults.push(result);
+    }
+
+    suiteResults.push({
+      name: suite.name,
+      cases: caseResults,
+      passed: caseResults.filter((c) => c.passed).length,
+      failed: caseResults.filter((c) => !c.passed && !c.skipped).length,
+      skipped: caseResults.filter((c) => c.skipped).length,
+      durationMs: performance.now() - suiteStart,
+    });
+  }
+
+  return {
+    suites: suiteResults,
+    totalPassed: suiteResults.reduce((sum, s) => sum + s.passed, 0),
+    totalFailed: suiteResults.reduce((sum, s) => sum + s.failed, 0),
+    totalSkipped: suiteResults.reduce((sum, s) => sum + s.skipped, 0),
+    durationMs: performance.now() - runStart,
+  };
+}
+
+/**
+ * Runs all eval suites and returns aggregate results.
+ * @param suites - Array of eval suites to run
+ * @param config - Harness configuration
+ * @returns Full eval run result
+ */
+export async function runEvals(
+  suites: readonly EvalSuite[],
+  config: EvalHarnessConfig = {},
+): Promise<EvalRunResult> {
+  const { server } = await createMcpServerAsync(config.serverConfig ?? {});
+  const [clientTransport, serverTransport] = InMemoryTransport.createLinkedPair();
+  const client = new Client({ name: "eval-harness", version: "1.0.0" });
+
+  await server.connect(serverTransport);
+  await client.connect(clientTransport);
+
+  const result = await runEvalsWithClient(client, suites, config);
+
+  await client.close();
+  await server.close();
+
+  return result;
+}

package/test/integration/hosted-client.ts

@@ -0,0 +1,56 @@
+/**
+ * MCP client helpers for hosted Streamable HTTP transport.
+ * @module test/integration/hosted-client
+ */
+
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js";
+import { hostedMcpEndpoint } from "./env.js";
+
+export interface HostedMcpSession {
+  readonly client: Client;
+  readonly transport: StreamableHTTPClientTransport;
+  readonly close: () => Promise<void>;
+}
+
+export interface ConnectHostedMcpClientOptions {
+  /** Server base URL (with or without `/mcp` suffix). */
+  readonly baseUrl: string;
+  /** User API key sent as `Authorization: Bearer`. */
+  readonly apiKey: string;
+}
+
+/**
+ * Connects an MCP client to a hosted HTTP server via Streamable HTTP.
+ */
+export async function connectHostedMcpClient(
+  options: ConnectHostedMcpClientOptions,
+): Promise<HostedMcpSession> {
+  const url = new URL(hostedMcpEndpoint(options.baseUrl));
+  const client = new Client({ name: "hosted-integration", version: "1.0.0" });
+  const transport = new StreamableHTTPClientTransport(url, {
+    requestInit: {
+      headers: {
+        Authorization: `Bearer ${options.apiKey}`,
+      },
+    },
+  });
+
+  await client.connect(transport);
+
+  return {
+    client,
+    transport,
+    close: async () => {
+      await transport.close();
+    },
+  };
+}
+
+/**
+ * Performs a GET /health check against a hosted MCP server base URL.
+ */
+export async function checkHostedHealth(baseUrl: string): Promise<Response> {
+  const root = baseUrl.replace(/\/mcp\/?$/, "").replace(/\/$/, "");
+  return fetch(`${root}/health`);
+}

package/test/integration/hosted-mocks.ts

@@ -0,0 +1,81 @@
+/**
+ * Mock fetch handlers for hosted MCP e2e tests (auth.json + dsl-service).
+ * @module test/integration/hosted-mocks
+ */
+
+import describeFixture from "../fixtures/describe-schema.json";
+
+export interface HostedServiceMocksOptions {
+  readonly authUrl: string;
+  readonly dslServiceUrl: string;
+  /** API keys accepted by mock auth (default: any non-empty key). */
+  readonly validApiKeys?: readonly string[];
+  readonly userEmail?: string;
+}
+
+function jwtWithSub(sub: string): string {
+  const header = Buffer.from(JSON.stringify({ alg: "HS256" })).toString("base64url");
+  const payload = Buffer.from(JSON.stringify({ sub })).toString("base64url");
+  return `${header}.${payload}.signature`;
+}
+
+function resolveUrl(input: RequestInfo | URL): string {
+  if (typeof input === "string") return input;
+  if (input instanceof URL) return input.href;
+  return input.url;
+}
+
+function mockSearchPublicationsResponse(): Record<string, unknown> {
+  return {
+    publications: [{ id: "pub.test.1", title: "Mock CRISPR publication" }],
+    _stats: { total_count: 42 },
+  };
+}
+
+/**
+ * Installs global fetch mocks for Radar auth and dsl-service.
+ * @returns Function that restores the original fetch implementation
+ */
+export function installHostedServiceMocks(options: HostedServiceMocksOptions): () => void {
+  const originalFetch = globalThis.fetch;
+  const authBase = options.authUrl.replace(/\/$/, "");
+  const dslBase = options.dslServiceUrl.replace(/\/$/, "");
+  const userEmail = options.userEmail ?? "user@example.com";
+
+  globalThis.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+    const url = resolveUrl(input);
+
+    if (url.startsWith(`${authBase}/api/auth.json`) || url.endsWith("/api/auth.json")) {
+      const body = init?.body ? JSON.parse(String(init.body)) : {};
+      const key = body.key as string | undefined;
+      if (!key?.trim()) {
+        return Response.json({ error: "Invalid or revoked API key." }, { status: 401 });
+      }
+      if (options.validApiKeys && !options.validApiKeys.includes(key)) {
+        return Response.json({ error: "Invalid or revoked API key." }, { status: 401 });
+      }
+      return Response.json({ token: jwtWithSub(userEmail) });
+    }
+
+    if (url.startsWith(`${dslBase}/query`)) {
+      const body = init?.body ? JSON.parse(String(init.body)) : {};
+      const query = String(body.query ?? "");
+      if (query.includes("describe schema")) {
+        return Response.json(describeFixture);
+      }
+      if (query.includes("describe version")) {
+        return Response.json({ version: "2.15.0" });
+      }
+      if (query.includes("search publications")) {
+        return Response.json(mockSearchPublicationsResponse());
+      }
+      return Response.json({ error: `Unhandled mock query: ${query}` }, { status: 400 });
+    }
+
+    return originalFetch(input, init);
+  }) as typeof fetch;
+
+  return () => {
+    globalThis.fetch = originalFetch;
+  };
+}

package/test/integration/hosted-run.ts

@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+/**
+ * Live hosted MCP integration smoke test runner.
+ *
+ * Starts an in-process HTTP server (unless HOSTED_MCP_BASE_URL is set) and
+ * exercises hosted MCP over Streamable HTTP with a real API key and backends.
+ *
+ * Usage:
+ *   node --import tsx apps/mcp/test/integration/hosted-run.ts
+ *
+ * @module test/integration/hosted-run
+ */
+
+import type { HostedHttpServerHandle } from "../../src/mcp/http-server.js";
+import { startHostedHttpServer } from "../../src/mcp/http-server.js";
+import { HOSTED_INTEGRATION_ENV_VARS, loadHostedIntegrationConfig } from "./env.js";
+import { runEvalsWithClient } from "./harness.js";
+import { checkHostedHealth, connectHostedMcpClient } from "./hosted-client.js";
+import { exitCode, printReport } from "./reporter.js";
+import { hostedSmokeSuite } from "./suites/hosted-smoke.integration.js";
+
+async function resolveServerBaseUrl(
+  config: NonNullable<ReturnType<typeof loadHostedIntegrationConfig>>,
+): Promise<{ baseUrl: string; close?: () => Promise<void> }> {
+  if (config.remoteBaseUrl) {
+    console.error(`Using remote hosted MCP at ${config.remoteBaseUrl}`);
+    return { baseUrl: config.remoteBaseUrl };
+  }
+
+  const handle: HostedHttpServerHandle = startHostedHttpServer({
+    hosted: config.hosted,
+    port: 0,
+    bindHost: "127.0.0.1",
+  });
+  const port = await handle.ready;
+  const baseUrl = `http://127.0.0.1:${port}`;
+  console.error(`Started in-process hosted MCP at ${baseUrl}`);
+  return {
+    baseUrl,
+    close: () => handle.close(),
+  };
+}
+
+const config = loadHostedIntegrationConfig();
+if (!config) {
+  console.error(
+    `Hosted integration smoke requires env vars: ${HOSTED_INTEGRATION_ENV_VARS.join(", ")}`,
+  );
+  process.exit(1);
+}
+
+let serverClose: (() => Promise<void>) | undefined;
+let status = 1;
+
+try {
+  const { baseUrl, close } = await resolveServerBaseUrl(config);
+  serverClose = close;
+
+  const health = await checkHostedHealth(baseUrl);
+  if (!health.ok) {
+    throw new Error(`Health check failed: GET /health returned ${health.status}`);
+  }
+
+  const session = await connectHostedMcpClient({
+    baseUrl,
+    apiKey: config.apiKey,
+  });
+
+  try {
+    console.error("Running hosted MCP smoke suite...\n");
+    const result = await runEvalsWithClient(session.client, [hostedSmokeSuite]);
+    printReport(result);
+    status = exitCode(result);
+  } finally {
+    await session.close();
+  }
+} catch (error) {
+  console.error(error instanceof Error ? error.message : error);
+  status = 1;
+} finally {
+  if (serverClose) {
+    await serverClose();
+  }
+}
+
+process.exit(status);

package/test/integration/hosted.e2e.test.ts

@@ -0,0 +1,101 @@
+/**
+ * Hosted MCP end-to-end tests with mocked auth and dsl-service backends.
+ * Runs in default CI without secrets.
+ * @module test/integration/hosted.e2e
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+import type { HostedEnvConfig } from "../../src/client/deployment-config.js";
+import { clearSchemaCache } from "../../src/dsl/schema/index.js";
+import type { HostedHttpServerHandle } from "../../src/mcp/http-server.js";
+import { startHostedHttpServer } from "../../src/mcp/http-server.js";
+import { clearSharedSchemaStore } from "../../src/mcp/shared-schema.js";
+import { runEvalsWithClient } from "./harness.js";
+import { checkHostedHealth, connectHostedMcpClient } from "./hosted-client.js";
+import { installHostedServiceMocks } from "./hosted-mocks.js";
+import { hostedSmokeSuite } from "./suites/hosted-smoke.integration.js";
+
+const MOCK_HOSTED: HostedEnvConfig = {
+  deploymentMode: "hosted",
+  radarAuthUrl: "https://app.test.example",
+  internal: {
+    serviceUrl: "https://dsl.test.example",
+    username: "svc-user",
+    password: "svc-pass",
+    dslSchema: "external",
+    host: "app.test.example",
+    variant: "standard",
+  },
+  httpPort: 8080,
+};
+
+const TEST_API_KEY = "test-hosted-api-key";
+
+describe("Hosted MCP e2e (mocked backends)", () => {
+  let restoreFetch: (() => void) | undefined;
+  let server: HostedHttpServerHandle | undefined;
+  let baseUrl: string;
+
+  beforeEach(async () => {
+    clearSharedSchemaStore();
+    clearSchemaCache();
+
+    restoreFetch = installHostedServiceMocks({
+      authUrl: MOCK_HOSTED.radarAuthUrl,
+      dslServiceUrl: MOCK_HOSTED.internal.serviceUrl,
+      validApiKeys: [TEST_API_KEY],
+      userEmail: "user@test.example",
+    });
+
+    server = startHostedHttpServer({
+      hosted: MOCK_HOSTED,
+      port: 0,
+      bindHost: "127.0.0.1",
+    });
+    const port = await server.ready;
+    baseUrl = `http://127.0.0.1:${port}`;
+  });
+
+  afterEach(async () => {
+    await server?.close();
+    restoreFetch?.();
+  });
+
+  it("responds to GET /health", async () => {
+    const response = await checkHostedHealth(baseUrl);
+    expect(response.status).toBe(200);
+    expect(await response.json()).toEqual({ status: "ok" });
+  });
+
+  it("returns 401 for POST /mcp without Authorization", async () => {
+    const response = await fetch(`${baseUrl}/mcp`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ jsonrpc: "2.0", method: "initialize", id: 1, params: {} }),
+    });
+    expect(response.status).toBe(401);
+  });
+
+  it("omits fetch_search_pages from the hosted tool surface", async () => {
+    const session = await connectHostedMcpClient({ baseUrl, apiKey: TEST_API_KEY });
+    try {
+      const tools = await session.client.listTools();
+      const names = tools.tools.map((tool) => tool.name);
+      expect(names).not.toContain("fetch_search_pages");
+      expect(names).toContain("search_publications");
+    } finally {
+      await session.close();
+    }
+  });
+
+  it("runs hosted smoke eval cases over Streamable HTTP", async () => {
+    const session = await connectHostedMcpClient({ baseUrl, apiKey: TEST_API_KEY });
+    try {
+      const result = await runEvalsWithClient(session.client, [hostedSmokeSuite]);
+      expect(result.totalFailed).toBe(0);
+      expect(result.totalPassed).toBe(hostedSmokeSuite.cases.length);
+    } finally {
+      await session.close();
+    }
+  });
+});