@digital-science-dsl/dimensions-analytics-mcp 0.5.3

package/test/integration/reporter.ts

@@ -0,0 +1,100 @@
+/**
+ * Console reporter for MCP eval results.
+ * @module test/integration/reporter
+ */
+
+import type { CaseResult, EvalRunResult, SuiteResult } from "./types.js";
+
+const PASS = "\x1b[32m PASS \x1b[0m";
+const FAIL = "\x1b[31m FAIL \x1b[0m";
+const SKIP = "\x1b[33m SKIP \x1b[0m";
+
+/**
+ * Formats a duration in milliseconds to a human-readable string.
+ * @param ms - Duration in milliseconds
+ * @returns Formatted string (e.g., "1.2s", "450ms")
+ */
+function formatDuration(ms: number): string {
+  if (ms >= 1000) return `${(ms / 1000).toFixed(1)}s`;
+  return `${Math.round(ms)}ms`;
+}
+
+/**
+ * Prints a single case result to stderr.
+ * @param result - The case result to print
+ */
+function printCase(result: CaseResult): void {
+  if (result.skipped) {
+    console.error(`  ${SKIP} ${result.caseName} — ${result.skipReason}`);
+    return;
+  }
+
+  const badge = result.passed ? PASS : FAIL;
+  const duration = formatDuration(result.durationMs);
+  console.error(`  ${badge} ${result.caseName} (${duration})`);
+
+  if (result.toolError) {
+    console.error(`         Tool error: ${result.toolError}`);
+    return;
+  }
+
+  for (const assertion of result.assertions) {
+    if (!assertion.passed) {
+      console.error(`         \x1b[31m✗\x1b[0m ${assertion.label}`);
+      if (assertion.error) {
+        console.error(`           ${assertion.error}`);
+      }
+    }
+  }
+}
+
+/**
+ * Prints a suite result to stderr.
+ * @param suite - The suite result to print
+ */
+function printSuite(suite: SuiteResult): void {
+  const duration = formatDuration(suite.durationMs);
+  const counts = [
+    suite.passed > 0 ? `\x1b[32m${suite.passed} passed\x1b[0m` : null,
+    suite.failed > 0 ? `\x1b[31m${suite.failed} failed\x1b[0m` : null,
+    suite.skipped > 0 ? `\x1b[33m${suite.skipped} skipped\x1b[0m` : null,
+  ]
+    .filter(Boolean)
+    .join(", ");
+
+  console.error(`\n\x1b[1m${suite.name}\x1b[0m (${counts}) ${duration}`);
+
+  for (const c of suite.cases) {
+    printCase(c);
+  }
+}
+
+/**
+ * Prints the full eval run report to stderr.
+ * @param result - The eval run result
+ */
+export function printReport(result: EvalRunResult): void {
+  console.error("\n\x1b[1m═══ MCP Eval Report ═══\x1b[0m\n");
+
+  for (const suite of result.suites) {
+    printSuite(suite);
+  }
+
+  const total = result.totalPassed + result.totalFailed + result.totalSkipped;
+  const duration = formatDuration(result.durationMs);
+
+  console.error("\n\x1b[1m───────────────────────\x1b[0m");
+  console.error(
+    `Total: ${total} cases | \x1b[32m${result.totalPassed} passed\x1b[0m | \x1b[31m${result.totalFailed} failed\x1b[0m | \x1b[33m${result.totalSkipped} skipped\x1b[0m | ${duration}`,
+  );
+  console.error("");
+}
+
+/**
+ * Returns process exit code based on results: 0 if all passed/skipped, 1 if any failed.
+ * @param result - The eval run result
+ * @returns Exit code
+ */
+export function exitCode(result: EvalRunResult): number {
+  return result.totalFailed > 0 ? 1 : 0;
+}

package/test/integration/run.ts

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+
+/**
+ * CLI entry point for the MCP integration test harness.
+ *
+ * Usage:
+ *   node --import tsx apps/mcp/test/integration/run.ts [--suite <name>] [--tag <tag>] [--exclude <tag>]
+ *
+ * Reads API keys from .dimensions.config.json (cwd → home fallback),
+ * with env var overrides.
+ * @module test/integration/run
+ */
+
+import type { McpServerConfig } from "../../src/mcp/server.js";
+import { loadLocalIntegrationConfig } from "./env.js";
+import { type EvalHarnessConfig, runEvals } from "./harness.js";
+import { exitCode, printReport } from "./reporter.js";
+import { allSuites } from "./suites/index.js";
+
+/**
+ * Loads MCP server config from env / .dimensions.config.json.
+ */
+async function loadServerConfig(): Promise<McpServerConfig> {
+  const fromEnv = loadLocalIntegrationConfig();
+  if (fromEnv) {
+    return {
+      apiKey: fromEnv.apiKey,
+      baseUrl: fromEnv.baseUrl,
+    };
+  }
+
+  const { loadDimensionsConfig } = await import("../../src/client/config/node.js");
+  const fileConfig = await loadDimensionsConfig();
+  return {
+    apiKey: fileConfig?.dsl?.apiKey,
+    baseUrl: fileConfig?.dsl?.baseUrl,
+  };
+}
+
+function parseArgs(argv: string[]): Omit<EvalHarnessConfig, "serverConfig"> {
+  const suiteFilter: string[] = [];
+  const tagFilter: string[] = [];
+  const tagExclude: string[] = [];
+
+  for (let i = 0; i < argv.length; i++) {
+    const arg = argv[i];
+    const next = argv[i + 1];
+    if (arg === "--suite" && next) {
+      suiteFilter.push(next);
+      i++;
+    } else if (arg === "--tag" && next) {
+      tagFilter.push(next);
+      i++;
+    } else if (arg === "--exclude" && next) {
+      tagExclude.push(next);
+      i++;
+    }
+  }
+
+  return {
+    suiteFilter: suiteFilter.length > 0 ? suiteFilter : undefined,
+    tagFilter: tagFilter.length > 0 ? tagFilter : undefined,
+    tagExclude: tagExclude.length > 0 ? tagExclude : undefined,
+  };
+}
+
+const serverConfig = await loadServerConfig();
+const config: EvalHarnessConfig = { ...parseArgs(process.argv.slice(2)), serverConfig };
+
+console.error(`Running ${allSuites.length} integration test suites...\n`);
+
+const result = await runEvals(allSuites, config);
+printReport(result);
+process.exit(exitCode(result));

package/test/integration/suites/analytics.integration.ts

@@ -0,0 +1,72 @@
+/**
+ * Eval suite for analytics tools.
+ * Tests facet queries, aggregations, and trend analysis against the live API.
+ * @module test/integration/suites/analytics
+ */
+
+import { arrayMinLength, hasField, isSuccess } from "../assertions.js";
+import type { EvalSuite } from "../types.js";
+
+export const analyticsSuite: EvalSuite = {
+  name: "analytics",
+  description: "Facet queries, aggregations, and trend analysis",
+  cases: [
+    {
+      name: "facet query: publications by year",
+      tool: "facet_query",
+      args: {
+        entityType: "publications",
+        facetField: "year",
+        query: "artificial intelligence",
+        limit: 10,
+      },
+      assertions: [isSuccess(), arrayMinLength("buckets", 1)],
+    },
+    {
+      name: "facet query: grants by funder",
+      tool: "facet_query",
+      args: {
+        entityType: "grants",
+        facetField: "funder_orgs",
+        query: "climate change",
+        limit: 10,
+      },
+      assertions: [isSuccess(), arrayMinLength("buckets", 1)],
+      timeout: 60_000,
+    },
+    {
+      name: "aggregate query: publications by year with citation sum",
+      tool: "aggregate_query",
+      args: {
+        entityType: "publications",
+        facetField: "year",
+        indicators: ["citations_total"],
+        query: "deep learning",
+        limit: 10,
+      },
+      assertions: [isSuccess(), arrayMinLength("buckets", 1)],
+      timeout: 60_000,
+    },
+    {
+      name: "citation trend for COVID-19",
+      tool: "citation_trend",
+      args: {
+        query: "COVID-19",
+        startYear: 2020,
+        endYear: 2024,
+      },
+      assertions: [isSuccess(), hasField("citationsPerYear")],
+    },
+    {
+      name: "funding trend for AI research",
+      tool: "funding_trend",
+      args: {
+        query: "artificial intelligence",
+        startYear: 2019,
+        endYear: 2024,
+        currency: "USD",
+      },
+      assertions: [isSuccess(), hasField("fundingPerYear")],
+    },
+  ],
+};

package/test/integration/suites/functions.integration.ts

@@ -0,0 +1,37 @@
+/**
+ * Eval suite for entity resolution function tools.
+ * @module test/integration/suites/functions
+ */
+
+import { hasField, isSuccess } from "../assertions.js";
+import type { EvalSuite } from "../types.js";
+
+export const functionsSuite: EvalSuite = {
+  name: "functions",
+  description: "Entity resolution tools (affiliations, grants)",
+  cases: [
+    {
+      name: "extract affiliations from freetext",
+      tool: "extract_affiliations",
+      args: {
+        affiliations: [
+          { affiliation: "Department of Computer Science, Stanford University, CA, USA" },
+          { affiliation: "Max Planck Institute for Molecular Cell Biology, Dresden, Germany" },
+        ],
+      },
+      assertions: [isSuccess(), hasField("affiliations")],
+      tags: ["flaky"],
+    },
+    {
+      name: "extract grant from grant number",
+      tool: "extract_grants",
+      args: {
+        grant_number: "R01GM123456",
+        funder_name: "National Institutes of Health",
+      },
+      assertions: [isSuccess(), hasField("grants")],
+      tags: ["flaky"],
+      timeout: 60_000,
+    },
+  ],
+};

package/test/integration/suites/hosted-smoke.integration.ts

@@ -0,0 +1,38 @@
+/**
+ * Minimal eval suite for hosted MCP smoke tests (local stdio harness uses full suites).
+ * @module test/integration/suites/hosted-smoke
+ */
+
+import { arrayMinLength, fieldAtLeast, hasField, isSuccess } from "../assertions.js";
+import type { EvalSuite } from "../types.js";
+
+/** Smoke cases exercised over Streamable HTTP against hosted MCP. */
+export const hostedSmokeSuite: EvalSuite = {
+  name: "hosted-smoke",
+  description: "Hosted MCP protocol smoke (describe + search)",
+  cases: [
+    {
+      name: "describe_schema summary",
+      tool: "describe_schema",
+      args: {},
+      assertions: [isSuccess(), hasField("summary"), hasField("stats")],
+    },
+    {
+      name: "search publications for CRISPR",
+      tool: "search_publications",
+      args: { query: "CRISPR gene editing", limit: 5 },
+      assertions: [
+        isSuccess(),
+        hasField("totalCount"),
+        fieldAtLeast("totalCount", 1),
+        arrayMinLength("publications", 1),
+      ],
+    },
+    {
+      name: "read schema summary resource",
+      tool: "__resource__",
+      args: { uri: "dimensions://schema/summary" },
+      assertions: [isSuccess(), hasField("stats"), hasField("sources")],
+    },
+  ],
+};

package/test/integration/suites/index.ts

@@ -0,0 +1,22 @@
+/**
+ * Collects all integration test suites for the MCP smoke test harness.
+ * @module test/integration/suites/index
+ */
+
+import type { EvalSuite } from "../types.js";
+import { analyticsSuite } from "./analytics.integration.js";
+import { functionsSuite } from "./functions.integration.js";
+import { lookupSuite } from "./lookup.integration.js";
+import { querySuite } from "./query.integration.js";
+import { resourcesSuite } from "./resources.integration.js";
+import { searchSuite } from "./search.integration.js";
+
+/** All integration test suites, in execution order. */
+export const allSuites: readonly EvalSuite[] = [
+  searchSuite,
+  lookupSuite,
+  querySuite,
+  functionsSuite,
+  analyticsSuite,
+  resourcesSuite,
+];

package/test/integration/suites/lookup.integration.ts

@@ -0,0 +1,57 @@
+/**
+ * Eval suite for identifier lookup tools.
+ * Tests DOI, PMID, and entity ID lookups against the live API with well-known identifiers.
+ * @module test/integration/suites/lookup
+ */
+
+import { custom, fieldEquals, hasField, isSuccess } from "../assertions.js";
+import type { EvalSuite } from "../types.js";
+
+export const lookupSuite: EvalSuite = {
+  name: "lookup",
+  description: "Identifier-based lookups (DOI, PMID, entity ID)",
+  cases: [
+    {
+      name: "get publication by DOI (Watson & Crick 1953)",
+      tool: "get_by_doi",
+      args: { doi: "10.1038/171737a0" },
+      assertions: [
+        isSuccess(),
+        fieldEquals("found", true),
+        hasField("publication"),
+        custom("title mentions DNA", (data) => {
+          const d = data as Record<string, unknown>;
+          const pub = d.publication as Record<string, unknown>;
+          const title = String(pub?.title ?? "").toLowerCase();
+          return title.includes("nucleic acid") || title.includes("deoxyribose")
+            ? true
+            : `Expected DNA-related title, got: "${pub?.title}"`;
+        }),
+      ],
+      timeout: 15_000,
+    },
+    {
+      name: "get publication by DOI with custom fields",
+      tool: "get_by_doi",
+      args: { doi: "10.1038/171737a0", fields: ["title", "year", "times_cited"] },
+      assertions: [
+        isSuccess(),
+        fieldEquals("found", true),
+        hasField("publication.title"),
+        hasField("publication.year"),
+      ],
+    },
+    {
+      name: "get publication by PMID (aspirin and cancer)",
+      tool: "get_by_pmid",
+      args: { pmid: "17554120" },
+      assertions: [isSuccess(), fieldEquals("found", true), hasField("publication")],
+    },
+    {
+      name: "DOI not found returns found=false",
+      tool: "get_by_doi",
+      args: { doi: "10.9999/does-not-exist-12345" },
+      assertions: [isSuccess(), fieldEquals("found", false)],
+    },
+  ],
+};

package/test/integration/suites/query.integration.ts

@@ -0,0 +1,49 @@
+/**
+ * Eval suite for the execute_dsl tool.
+ * Tests raw DSL query execution against the live API.
+ * @module test/integration/suites/query
+ */
+
+import { custom, hasField, isSuccess } from "../assertions.js";
+import type { EvalSuite } from "../types.js";
+
+export const querySuite: EvalSuite = {
+  name: "query",
+  description: "Raw DSL query execution",
+  cases: [
+    {
+      name: "simple publication search DSL",
+      tool: "execute_dsl",
+      args: {
+        dsl: 'search publications for "CRISPR" return publications[title+year+times_cited] limit 5',
+      },
+      assertions: [
+        isSuccess(),
+        hasField("publications"),
+        custom("has publication results", (data) => {
+          const d = data as Record<string, unknown>;
+          const pubs = d.publications;
+          return Array.isArray(pubs) && pubs.length > 0
+            ? true
+            : "Expected non-empty publications array";
+        }),
+      ],
+    },
+    {
+      name: "aggregation DSL query",
+      tool: "execute_dsl",
+      args: {
+        dsl: 'search publications for "deep learning" where year = 2023 return year aggregate citations_total limit 5',
+      },
+      assertions: [isSuccess()],
+    },
+    {
+      name: "researcher search DSL",
+      tool: "execute_dsl",
+      args: {
+        dsl: 'search researchers for "machine learning" return researchers[first_name+last_name] limit 5',
+      },
+      assertions: [isSuccess(), hasField("researchers")],
+    },
+  ],
+};

package/test/integration/suites/resources.integration.ts

@@ -0,0 +1,68 @@
+/**
+ * Eval suite for MCP resources (fields metadata).
+ * Tests the describe_fields resource template against the live server.
+ * @module test/integration/suites/resources
+ */
+
+import { custom } from "../assertions.js";
+import type { EvalSuite } from "../types.js";
+
+/**
+ * Resources eval suite.
+ *
+ * Note: Resources are accessed via the MCP client's readResource method,
+ * not callTool. The harness runner handles this differently — resource
+ * cases use tool="__resource__" as a sentinel and the URI is in args.uri.
+ */
+export const resourcesSuite: EvalSuite = {
+  name: "resources",
+  description: "Field metadata resources",
+  cases: [
+    {
+      name: "list publications fields",
+      tool: "__resource__",
+      args: { uri: "dimensions://fields/publications" },
+      assertions: [
+        custom("returns field metadata object", (data) => {
+          if (typeof data !== "object" || data === null) {
+            return "Expected object response";
+          }
+          const d = data as Record<string, unknown>;
+          // Should have field descriptions
+          if (d.fields && typeof d.fields === "object") return true;
+          // Or be the field map itself
+          if (Object.keys(d).length > 0) return true;
+          return "Expected non-empty field metadata";
+        }),
+      ],
+    },
+    {
+      name: "list grants fields",
+      tool: "__resource__",
+      args: { uri: "dimensions://fields/grants" },
+      assertions: [
+        custom("returns field metadata", (data) => {
+          return typeof data === "object" && data !== null && Object.keys(data as object).length > 0
+            ? true
+            : "Expected non-empty field metadata";
+        }),
+      ],
+    },
+    {
+      name: "read publications DSL examples",
+      tool: "__resource__",
+      args: { uri: "dimensions://examples/publications" },
+      assertions: [
+        custom("returns per-source examples", (data) => {
+          if (typeof data !== "object" || data === null) return "Expected object response";
+          const d = data as Record<string, unknown>;
+          if (d.source !== "publications") return "Expected source publications";
+          if (!Array.isArray(d.examples) || d.examples.length === 0) {
+            return "Expected non-empty examples array";
+          }
+          return true;
+        }),
+      ],
+    },
+  ],
+};

package/test/integration/suites/search.integration.ts

@@ -0,0 +1,80 @@
+/**
+ * Eval suite for search_* tools.
+ * Tests keyword search across multiple entity types against the live API.
+ * @module test/integration/suites/search
+ */
+
+import { arrayMinLength, fieldAtLeast, hasField, isSuccess } from "../assertions.js";
+import type { EvalSuite } from "../types.js";
+
+export const searchSuite: EvalSuite = {
+  name: "search",
+  description: "Keyword search tools across entity types",
+  cases: [
+    {
+      name: "search publications for CRISPR",
+      tool: "search_publications",
+      args: { query: "CRISPR gene editing", limit: 10 },
+      assertions: [
+        isSuccess(),
+        hasField("totalCount"),
+        fieldAtLeast("totalCount", 1),
+        arrayMinLength("publications", 1),
+      ],
+    },
+    {
+      name: "search publications with year filter",
+      tool: "search_publications",
+      args: { query: "machine learning", yearFrom: 2023, yearTo: 2024, limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("publications", 1)],
+    },
+    {
+      name: "search publications with sort",
+      tool: "search_publications",
+      args: { query: "deep learning", sortBy: "times_cited", limit: 5 },
+      assertions: [isSuccess(), arrayMinLength("publications", 1)],
+    },
+    {
+      name: "search grants for climate research",
+      tool: "search_grants",
+      args: { query: "climate change adaptation", limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("grants", 1)],
+    },
+    {
+      name: "search researchers for a common name",
+      tool: "search_researchers",
+      args: { query: "John Smith", limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("researchers", 1)],
+    },
+    {
+      name: "search patents for mRNA",
+      tool: "search_patents",
+      args: { query: "mRNA vaccine delivery", limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("patents", 1)],
+    },
+    {
+      name: "search clinical trials for Alzheimer",
+      tool: "search_clinical_trials",
+      args: { query: "Alzheimer disease treatment", limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("clinicalTrials", 1)],
+    },
+    {
+      name: "search datasets for genomics",
+      tool: "search_datasets",
+      args: { query: "human genome sequencing", limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("datasets", 1)],
+    },
+    {
+      name: "search policy documents for pandemic",
+      tool: "search_policy_documents",
+      args: { query: "pandemic preparedness", limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("policyDocuments", 1)],
+    },
+    {
+      name: "search organizations for MIT",
+      tool: "search_organizations",
+      args: { query: "Massachusetts Institute of Technology", limit: 5 },
+      assertions: [isSuccess(), hasField("totalCount"), arrayMinLength("organizations", 1)],
+    },
+  ],
+};