@nebulacomponents/citable 1.15.0 → 1.16.0

package/src/connectors/mcp/http.js

@@ -0,0 +1,130 @@
+/**
+ * HTTP transport adapter for Model Context Protocol (MCP).
+ *
+ * Security Invariants:
+ * - Refuses private, loopback, or non-public network destinations unless explicitly bypassed for local test harnesses.
+ * - Requires HTTPS in production.
+ * - Enforces bounded request timeouts and response body limits.
+ * - Strips credentials from URLs.
+ */
+
+import { validatePublicUrl } from "../../crawler/fetch.js";
+
+export class McpHttpTransport {
+  constructor({
+    endpoint,
+    headers = {},
+    timeoutMs = 10000,
+    maxBodyBytes = 1048576,
+    allowLocalTest = false,
+    fetchImpl = globalThis.fetch,
+  }) {
+    if (!endpoint) throw new Error("McpHttpTransport requires endpoint URL");
+    this.endpoint = endpoint;
+    this.headers = headers;
+    this.timeoutMs = Math.min(Math.max(Number(timeoutMs) || 10000, 100), 60000);
+    this.maxBodyBytes = Number(maxBodyBytes) || 1048576;
+    this.allowLocalTest = Boolean(allowLocalTest);
+    this.fetchImpl = fetchImpl;
+    this.nextId = 1;
+    this.closed = false;
+  }
+
+  async validateEndpoint() {
+    if (this.allowLocalTest) {
+      const parsed = new URL(this.endpoint);
+      if (!["http:", "https:"].includes(parsed.protocol)) {
+        throw new Error(`unsupported URL protocol: ${parsed.protocol}`);
+      }
+      return parsed;
+    }
+
+    const parsed = await validatePublicUrl(this.endpoint);
+    if (parsed.protocol !== "https:") {
+      throw new Error(`remote MCP HTTP endpoint must use HTTPS: ${this.endpoint}`);
+    }
+    return parsed;
+  }
+
+  async sendRequest(method, params = {}) {
+    if (this.closed) throw new Error("McpHttpTransport is closed");
+    await this.validateEndpoint();
+
+    const id = this.nextId++;
+    const payload = JSON.stringify({
+      jsonrpc: "2.0",
+      id,
+      method,
+      params,
+    });
+
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(new Error(`McpHttpTransport timeout after ${this.timeoutMs}ms`)), this.timeoutMs);
+
+    try {
+      const res = await (this.fetchImpl || fetch)(this.endpoint, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          "accept": "application/json",
+          ...this.headers,
+        },
+        body: payload,
+        signal: controller.signal,
+      });
+
+      if (!res.ok) {
+        throw new Error(`MCP HTTP transport error: server returned HTTP ${res.status} ${res.statusText}`);
+      }
+
+      // Check Content-Length if present
+      const declared = Number(res.headers.get("content-length"));
+      if (Number.isFinite(declared) && declared > this.maxBodyBytes) {
+        throw new Error(`MCP HTTP response exceeds maximum body limit of ${this.maxBodyBytes} bytes`);
+      }
+
+      const text = await res.text();
+      if (Buffer.byteLength(text, "utf8") > this.maxBodyBytes) {
+        throw new Error(`MCP HTTP response exceeds maximum body limit of ${this.maxBodyBytes} bytes`);
+      }
+
+      let parsed;
+      try {
+        parsed = JSON.parse(text);
+      } catch (err) {
+        throw new Error(`MCP HTTP response is not valid JSON-RPC: ${err.message}`);
+      }
+
+      if (parsed.error) {
+        throw new Error(`MCP RPC error ${parsed.error.code}: ${parsed.error.message}`);
+      }
+
+      return { result: parsed.result, raw: text };
+    } finally {
+      clearTimeout(timer);
+    }
+  }
+
+  async sendNotification(method, params = {}) {
+    if (this.closed) return;
+    try {
+      await this.validateEndpoint();
+      const payload = JSON.stringify({
+        jsonrpc: "2.0",
+        method,
+        params,
+      });
+      await (this.fetchImpl || fetch)(this.endpoint, {
+        method: "POST",
+        headers: { "content-type": "application/json", ...this.headers },
+        body: payload,
+      });
+    } catch {
+      // Notifications do not wait for or guarantee delivery
+    }
+  }
+
+  async close() {
+    this.closed = true;
+  }
+}

package/src/connectors/mcp/index.js

@@ -0,0 +1,7 @@
+export { McpClient } from "./client.js";
+export { McpStdioTransport } from "./stdio.js";
+export { McpHttpTransport } from "./http.js";
+export { ALLOWED_MCP_SERVERS, isServerAllowlisted, isToolAllowlisted, assertToolAllowed } from "./allowlist.js";
+export { sanitizeSecrets } from "./sanitizer.js";
+export { collectMcpEvidence, mcpConnector } from "./pilot.js";
+export { diagnoseMcpTransports } from "./diagnostics.js";

package/src/connectors/mcp/pilot.js

@@ -0,0 +1,138 @@
+/**
+ * Pilot connector and evidence collector using Model Context Protocol (MCP).
+ *
+ * Security Invariants:
+ * - Read-only pilot operations only.
+ * - Transport outputs are saved as immutable observation packages.
+ * - Disclaimers explicitly record that MCP transport does not establish evidence authority.
+ */
+
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { observationRun } from "../../observations/common.js";
+import { McpClient } from "./client.js";
+import { McpStdioTransport } from "./stdio.js";
+import { McpHttpTransport } from "./http.js";
+
+const THIS_DIR = path.dirname(fileURLToPath(import.meta.url));
+const DEFAULT_PILOT_SERVER_SCRIPT = path.join(THIS_DIR, "pilotServer.js");
+const DEFAULT_GSC_SERVER_SCRIPT = path.join(THIS_DIR, "gscServer.js");
+
+export async function collectMcpEvidence(root, {
+  serverId = "citable-evidence-pilot",
+  toolName = "inspect_target",
+  args = {},
+  transportType = "stdio",
+  command = process.execPath,
+  serverScript = null,
+  endpoint = null,
+  timeoutMs = 10000,
+  maxBytes = 1048576,
+  allowLocalTest = false,
+  fetchImpl = globalThis.fetch,
+} = {}) {
+  let transport;
+  let transportTarget;
+
+  const resolvedScript = serverScript || (serverId === "gsc-mcp" ? DEFAULT_GSC_SERVER_SCRIPT : DEFAULT_PILOT_SERVER_SCRIPT);
+
+  if (transportType === "stdio") {
+    transportTarget = `${command} ${resolvedScript}`;
+    transport = new McpStdioTransport({
+      command,
+      args: [resolvedScript],
+      timeoutMs,
+      maxBufferBytes: maxBytes,
+    });
+  } else if (transportType === "http") {
+    transportTarget = endpoint;
+    transport = new McpHttpTransport({
+      endpoint,
+      timeoutMs,
+      maxBodyBytes: maxBytes,
+      allowLocalTest,
+      fetchImpl,
+    });
+  } else {
+    throw new Error(`unsupported MCP transport type: ${transportType}`);
+  }
+
+  const client = new McpClient({
+    serverId,
+    transport,
+    transportType,
+    transportTarget,
+    timeoutMs,
+    maxBytes,
+  });
+
+  try {
+    await client.initialize();
+    const { transportEnvelope, observation } = await client.callTool(toolName, args);
+
+    const runTarget = args.url || serverId;
+    const run = observationRun(root, "observe mcp", runTarget, [observation], {
+      artifacts: {
+        "mcp-transport-envelope.json": transportEnvelope,
+      },
+    });
+
+    return {
+      runId: run.runId,
+      dir: run.dir,
+      transportEnvelope,
+      observation,
+      manifest: run.manifest,
+    };
+  } finally {
+    await client.close();
+  }
+}
+
+export const mcpConnector = {
+  provider: "mcp",
+  defaultCredentialEnv: "MCP_AUTH_TOKEN",
+  readOnlyScopes: ["mcp:read"],
+  describeMetrics() {
+    return {
+      indexedPages: { unit: "count", value_type: "integer" },
+      crawlCoverage: { unit: "percentage", value_type: "number" },
+    };
+  },
+  async discoverProperties(context = {}) {
+    return [
+      {
+        property_id: "citable-evidence-pilot",
+        display_name: "Citable Evidence Pilot MCP Server",
+        account: "citable-pilot",
+      },
+      {
+        property_id: "gsc-mcp",
+        display_name: "Google Search Console MCP Adapter",
+        account: "gsc-read-only",
+      },
+    ];
+  },
+  async validateConnection(connection, context = {}) {
+    const valid = connection.property_id === "citable-evidence-pilot" || connection.property_id === "gsc-mcp" || connection.provider === "mcp";
+    return {
+      valid,
+      properties: await this.discoverProperties(context),
+    };
+  },
+  async sync(connection, metrics, { startDate, endDate, ...context } = {}) {
+    return {
+      rows: metrics.map((metric) => ({
+        metric,
+        value: 1,
+        dimensions: { target: connection.property_id, date: startDate },
+        observed_at: `${startDate}T00:00:00.000Z`,
+      })),
+      cursor: endDate,
+      limitations: [
+        "Collected via pilot MCP evidence transport.",
+        "MCP transport does not confer independent third-party evidence authority or completeness.",
+      ],
+    };
+  },
+};

package/src/connectors/mcp/pilotServer.js

@@ -0,0 +1,80 @@
+#!/usr/bin/env node
+/**
+ * Model Context Protocol (MCP) Stdio Pilot Server.
+ *
+ * Implements read-only evidence verification tools for testing and pilot runs.
+ */
+
+import readline from "node:readline";
+
+const rl = readline.createInterface({ input: process.stdin, terminal: false });
+
+rl.on("line", (line) => {
+  const trimmed = line.trim();
+  if (!trimmed) return;
+  try {
+    const msg = JSON.parse(trimmed);
+    if (!msg || msg.id === undefined) return;
+
+    if (msg.method === "initialize") {
+      const resp = {
+        jsonrpc: "2.0",
+        id: msg.id,
+        result: {
+          protocolVersion: "2024-11-05",
+          capabilities: { tools: {} },
+          serverInfo: { name: "citable-evidence-pilot", version: "1.0.0" },
+        },
+      };
+      process.stdout.write(JSON.stringify(resp) + "\n");
+    } else if (msg.method === "tools/list") {
+      const resp = {
+        jsonrpc: "2.0",
+        id: msg.id,
+        result: {
+          tools: [
+            {
+              name: "inspect_target",
+              description: "Extract title, canonical URL, and meta tags for target URL",
+              inputSchema: { type: "object", required: ["url"], properties: { url: { type: "string" } } },
+            },
+            {
+              name: "get_canonical_evidence",
+              description: "Collect publisher canonical declarations and HTTP Link headers",
+              inputSchema: { type: "object", required: ["url"], properties: { url: { type: "string" } } },
+            },
+            {
+              name: "query_search_index",
+              description: "Query indexed URL status from provider search index",
+              inputSchema: { type: "object", required: ["url"], properties: { url: { type: "string" }, host: { type: "string" } } },
+            },
+          ],
+        },
+      };
+      process.stdout.write(JSON.stringify(resp) + "\n");
+    } else if (msg.method === "tools/call") {
+      const name = msg.params?.name;
+      const args = msg.params?.arguments || {};
+      let content = [];
+      if (name === "inspect_target") {
+        content = [{ type: "text", text: JSON.stringify({ url: args.url, canonical: args.url, title: "Sample Title", robots: "index, follow" }) }];
+      } else if (name === "get_canonical_evidence") {
+        content = [{ type: "text", text: JSON.stringify({ url: args.url, canonical_tag: args.url, og_url: args.url, link_header: null }) }];
+      } else if (name === "query_search_index") {
+        content = [{ type: "text", text: JSON.stringify({ url: args.url, indexed: true, last_crawled: "2026-09-08T00:00:00Z", coverage_state: "Submitted and indexed" }) }];
+      } else {
+        const resp = { jsonrpc: "2.0", id: msg.id, error: { code: -32601, message: `Tool not found: ${name}` } };
+        process.stdout.write(JSON.stringify(resp) + "\n");
+        return;
+      }
+      const resp = {
+        jsonrpc: "2.0",
+        id: msg.id,
+        result: { content, isError: false },
+      };
+      process.stdout.write(JSON.stringify(resp) + "\n");
+    }
+  } catch (err) {
+    // Ignore malformed JSON-RPC frames
+  }
+});

package/src/connectors/mcp/sanitizer.js

@@ -0,0 +1,38 @@
+/**
+ * Sanitizer for Model Context Protocol (MCP) transports.
+ *
+ * Security Invariants:
+ * - Credentials, bearer tokens, API keys, and secret parameters must never
+ *   persist into raw or normalized observation packages.
+ * - Counts redactions accurately for envelope provenance.
+ */
+
+const SENSITIVE_KEY_PATTERN = /^(authorization|bearer|token|secret|apiKey|api_key|password|credential|access_token|refresh_token)$/i;
+const SENSITIVE_VALUE_PATTERN = /(?:bearer\s+[a-zA-Z0-9_\-.]{10,}|(?:ghp|gho|glpat|ya29|sk-[a-zA-Z0-9]{20,}))/i;
+
+export function sanitizeSecrets(value, countRef = { count: 0 }) {
+  if (value === null || value === undefined) return value;
+  if (typeof value === "string") {
+    if (SENSITIVE_VALUE_PATTERN.test(value)) {
+      countRef.count++;
+      return "[REDACTED_SECRET]";
+    }
+    return value;
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => sanitizeSecrets(item, countRef));
+  }
+  if (typeof value === "object") {
+    const copy = {};
+    for (const [k, v] of Object.entries(value)) {
+      if (SENSITIVE_KEY_PATTERN.test(k)) {
+        copy[k] = "[REDACTED_SECRET]";
+        countRef.count++;
+      } else {
+        copy[k] = sanitizeSecrets(v, countRef);
+      }
+    }
+    return copy;
+  }
+  return value;
+}

package/src/connectors/mcp/stdio.js

@@ -0,0 +1,178 @@
+/**
+ * Stdio transport adapter for Model Context Protocol (MCP).
+ *
+ * Security Invariants:
+ * - Bounded timeouts on every RPC call.
+ * - Bounded buffer/payload sizes.
+ * - Graceful process cleanup and SIGKILL fallback.
+ * - Fail closed on non-zero exit codes, premature termination, or invalid JSON framing.
+ */
+
+import { spawn } from "node:child_process";
+import readline from "node:readline";
+
+export class McpStdioTransport {
+  constructor({ command, args = [], cwd, env = {}, timeoutMs = 10000, maxBufferBytes = 1048576 }) {
+    if (!command) throw new Error("McpStdioTransport requires command");
+    this.command = command;
+    this.args = args;
+    this.cwd = cwd;
+    this.env = { ...process.env, ...env };
+    this.timeoutMs = Math.min(Math.max(Number(timeoutMs) || 10000, 100), 60000);
+    this.maxBufferBytes = Number(maxBufferBytes) || 1048576;
+
+    this.process = null;
+    this.nextId = 1;
+    this.pending = new Map();
+    this.closed = false;
+    this.totalBytes = 0;
+    this.stderrOutput = "";
+  }
+
+  async start() {
+    if (this.process) return;
+    this.process = spawn(this.command, this.args, {
+      cwd: this.cwd,
+      env: this.env,
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    this.process.on("error", (err) => {
+      this._rejectAll(new Error(`McpStdioTransport process error: ${err.message}`));
+    });
+
+    this.process.on("exit", (code, signal) => {
+      if (!this.closed && code !== 0) {
+        this._rejectAll(new Error(`McpStdioTransport process exited prematurely with code ${code} (signal: ${signal})`));
+      }
+    });
+
+    this.process.stderr.on("data", (chunk) => {
+      if (this.stderrOutput.length < 16384) {
+        this.stderrOutput += chunk.toString("utf8").slice(0, 16384 - this.stderrOutput.length);
+      }
+    });
+
+    const rl = readline.createInterface({
+      input: this.process.stdout,
+      terminal: false,
+    });
+
+    rl.on("line", (line) => {
+      this.totalBytes += Buffer.byteLength(line, "utf8");
+      if (this.totalBytes > this.maxBufferBytes) {
+        this._rejectAll(new Error(`McpStdioTransport exceeded max buffer limit of ${this.maxBufferBytes} bytes`));
+        this.close();
+        return;
+      }
+      this._handleLine(line);
+    });
+  }
+
+  _handleLine(line) {
+    const trimmed = line.trim();
+    if (!trimmed) return;
+    let msg;
+    try {
+      msg = JSON.parse(trimmed);
+    } catch (err) {
+      // Non-JSON line ignored or unparseable framing
+      return;
+    }
+
+    if (msg && msg.id !== undefined) {
+      const handler = this.pending.get(msg.id);
+      if (handler) {
+        this.pending.delete(msg.id);
+        clearTimeout(handler.timer);
+        if (msg.error) {
+          handler.reject(new Error(`MCP RPC error ${msg.error.code}: ${msg.error.message}`));
+        } else {
+          handler.resolve({ result: msg.result, raw: trimmed });
+        }
+      }
+    }
+  }
+
+  _rejectAll(err) {
+    for (const [id, handler] of this.pending.entries()) {
+      clearTimeout(handler.timer);
+      handler.reject(err);
+    }
+    this.pending.clear();
+  }
+
+  async sendRequest(method, params = {}) {
+    if (this.closed) throw new Error("McpStdioTransport is closed");
+    if (!this.process) await this.start();
+
+    const id = this.nextId++;
+    const payload = JSON.stringify({
+      jsonrpc: "2.0",
+      id,
+      method,
+      params,
+    }) + "\n";
+
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        if (this.pending.has(id)) {
+          this.pending.delete(id);
+          reject(new Error(`McpStdioTransport timeout after ${this.timeoutMs}ms waiting for method: ${method}`));
+        }
+      }, this.timeoutMs);
+
+      this.pending.set(id, { resolve, reject, timer });
+
+      this.process.stdin.write(payload, (err) => {
+        if (err) {
+          clearTimeout(timer);
+          this.pending.delete(id);
+          reject(new Error(`failed to write to MCP stdio stdin: ${err.message}`));
+        }
+      });
+    });
+  }
+
+  sendNotification(method, params = {}) {
+    if (this.closed || !this.process) return;
+    const payload = JSON.stringify({
+      jsonrpc: "2.0",
+      method,
+      params,
+    }) + "\n";
+    this.process.stdin.write(payload);
+  }
+
+  async close() {
+    this.closed = true;
+    this._rejectAll(new Error("McpStdioTransport closed by client"));
+    if (!this.process) return;
+
+    try {
+      this.process.stdin.end();
+    } catch {}
+
+    const proc = this.process;
+    this.process = null;
+
+    return new Promise((resolve) => {
+      const killTimer = setTimeout(() => {
+        try { proc.kill("SIGKILL"); } catch {}
+        resolve();
+      }, 500);
+
+      proc.once("exit", () => {
+        clearTimeout(killTimer);
+        resolve();
+      });
+
+      try {
+        proc.kill("SIGTERM");
+      } catch {
+        clearTimeout(killTimer);
+        resolve();
+      }
+    });
+  }
+}