mcp-scraper 0.38.2 → 0.40.1

package/dist/bin/mcp-stdio-server.d.cts

@@ -1 +0,0 @@
-#!/usr/bin/env node

package/dist/bin/mcp-stdio-server.d.ts

@@ -1 +0,0 @@
-#!/usr/bin/env node

package/dist/bin/mcp-stdio-server.js

@@ -1,135 +0,0 @@
-#!/usr/bin/env node
-import {
-  HttpMcpToolExecutor,
-  MEMORY_TOOL_SCHEMAS,
-  SERVER_INSTRUCTIONS,
-  registerBrowserAgentMcpTools,
-  registerMemoryMcpTools,
-  registerPaaExtractorMcpTools,
-  registerSerpIntelligenceCaptureTools
-} from "../chunk-NKCCGADE.js";
-import "../chunk-345BQXZH.js";
-import "../chunk-EGJKUB4Q.js";
-import {
-  renderInstallTerminal
-} from "../chunk-NNW3O6ZD.js";
-import "../chunk-CB5C3BPB.js";
-import {
-  PACKAGE_VERSION
-} from "../chunk-YODBNTTN.js";
-import "../chunk-AZRPG43B.js";
-import "../chunk-44HZLHDV.js";
-import "../chunk-N7KUTTCC.js";
-
-// bin/mcp-stdio-server.ts
-import { readFileSync } from "fs";
-import { homedir } from "os";
-import { join } from "path";
-import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
-
-// src/mcp/hosted-memory-mcp-tool-executor.ts
-var publicToolNameByUpstreamName = new Map(
-  MEMORY_TOOL_SCHEMAS.map((schema) => [schema.upstreamName, schema.id])
-);
-function parseJsonRpcEnvelope(text) {
-  const payloads = text.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).filter(Boolean);
-  if (!payloads.length) return JSON.parse(text);
-  for (const payload of payloads) {
-    const parsed = JSON.parse(payload);
-    if (parsed.result || parsed.error) return parsed;
-  }
-  throw new Error("hosted MCP returned no JSON-RPC result");
-}
-function errorResult(message) {
-  return {
-    content: [{ type: "text", text: JSON.stringify({ ok: false, error: message }) }],
-    isError: true
-  };
-}
-var HostedMemoryMcpToolExecutor = class {
-  baseUrl;
-  apiKey;
-  constructor(baseUrl2, apiKey2) {
-    this.baseUrl = baseUrl2.replace(/\/$/, "");
-    this.apiKey = apiKey2;
-  }
-  async callMemoryTool(upstreamName, args) {
-    const publicToolName = publicToolNameByUpstreamName.get(upstreamName);
-    if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`);
-    try {
-      const res = await fetch(`${this.baseUrl}/mcp`, {
-        method: "POST",
-        headers: {
-          "content-type": "application/json",
-          accept: "application/json, text/event-stream",
-          "x-api-key": this.apiKey
-        },
-        body: JSON.stringify({
-          jsonrpc: "2.0",
-          id: `memory:${publicToolName}`,
-          method: "tools/call",
-          params: { name: publicToolName, arguments: args }
-        })
-      });
-      const text = await res.text();
-      if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`);
-      const envelope = parseJsonRpcEnvelope(text);
-      if (envelope.error) {
-        return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`);
-      }
-      return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`);
-    } catch (err) {
-      return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`);
-    }
-  }
-};
-
-// bin/mcp-stdio-server.ts
-var forceStdio = process.argv.includes("--stdio") || process.env.MCP_SCRAPER_FORCE_STDIO === "1";
-var interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY);
-var wantsHelp = process.argv.includes("--help") || process.argv.includes("-h");
-if (!forceStdio && (interactiveTerminal || wantsHelp)) {
-  const noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
-  process.stdout.write(renderInstallTerminal({
-    version: PACKAGE_VERSION,
-    color: !noColor,
-    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim())
-  }));
-  process.exit(0);
-}
-function readApiKeyFile() {
-  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim();
-  const paths = [explicitPath, join(homedir(), ".mcp-scraper-key")].filter(Boolean);
-  for (const path of paths) {
-    try {
-      const value = readFileSync(path, "utf8").trim();
-      if (value) return value;
-    } catch {
-    }
-  }
-  return void 0;
-}
-var apiKey = (process.env.MCP_SCRAPER_API_KEY ?? process.env.MCP_SCRAPER_KEY ?? process.env.MCP_API_KEY ?? readApiKeyFile())?.trim();
-if (!apiKey) {
-  process.stderr.write("MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\n");
-  process.exit(1);
-}
-var baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
-var consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
-var server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
-var httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey);
-registerPaaExtractorMcpTools(server, httpExecutor);
-registerSerpIntelligenceCaptureTools(server, httpExecutor);
-registerBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl });
-registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, apiKey));
-var transport = new StdioServerTransport();
-async function main() {
-  await server.connect(transport);
-}
-main().catch((err) => {
-  process.stderr.write(`${err instanceof Error ? err.message : String(err)}
-`);
-  process.exit(1);
-});
-//# sourceMappingURL=mcp-stdio-server.js.map

package/dist/bin/mcp-stdio-server.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../../bin/mcp-stdio-server.ts","../../src/mcp/hosted-memory-mcp-tool-executor.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { registerMemoryMcpTools } from '../src/mcp/memory-mcp-server.js'\nimport { HostedMemoryMcpToolExecutor } from '../src/mcp/hosted-memory-mcp-tool-executor.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\n\nconst forceStdio =\n  process.argv.includes('--stdio') ||\n  process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n  const noColor =\n    process.argv.includes('--no-color') ||\n    process.env.NO_COLOR !== undefined ||\n    process.env.FORCE_COLOR === '0' ||\n    !process.stdout.isTTY\n\n  process.stdout.write(renderInstallTerminal({\n    version: PACKAGE_VERSION,\n    color: !noColor,\n    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n  }))\n  process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {\n      void 0\n    }\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })\n\nconst httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey)\nregisterPaaExtractorMcpTools(server, httpExecutor)\nregisterSerpIntelligenceCaptureTools(server, httpExecutor)\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\nregisterMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, apiKey))\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch(err => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n","import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'\nimport { MEMORY_TOOL_SCHEMAS } from './memory-tool-schemas.js'\n\nconst publicToolNameByUpstreamName = new Map(\n  MEMORY_TOOL_SCHEMAS.map(schema => [schema.upstreamName, schema.id]),\n)\n\ninterface JsonRpcEnvelope {\n  result?: CallToolResult\n  error?: { code?: number; message?: string }\n}\n\nfunction parseJsonRpcEnvelope(text: string): JsonRpcEnvelope {\n  const payloads = text\n    .split('\\n')\n    .filter(line => line.startsWith('data:'))\n    .map(line => line.slice(5).trim())\n    .filter(Boolean)\n\n  if (!payloads.length) return JSON.parse(text) as JsonRpcEnvelope\n\n  for (const payload of payloads) {\n    const parsed = JSON.parse(payload) as JsonRpcEnvelope\n    if (parsed.result || parsed.error) return parsed\n  }\n  throw new Error('hosted MCP returned no JSON-RPC result')\n}\n\nfunction errorResult(message: string): CallToolResult {\n  return {\n    content: [{ type: 'text', text: JSON.stringify({ ok: false, error: message }) }],\n    isError: true,\n  }\n}\n\n/**\n * Stdio-only bridge for memory tools. It deliberately calls the hosted aggregate\n * MCP surface so local clients execute the same registered tool implementation\n * and output contract as https://mcpscraper.dev/mcp.\n */\nexport class HostedMemoryMcpToolExecutor {\n  private readonly baseUrl: string\n  private readonly apiKey: string\n\n  constructor(baseUrl: string, apiKey: string) {\n    this.baseUrl = baseUrl.replace(/\\/$/, '')\n    this.apiKey = apiKey\n  }\n\n  async callMemoryTool(upstreamName: string, args: Record<string, unknown>): Promise<CallToolResult> {\n    const publicToolName = publicToolNameByUpstreamName.get(upstreamName)\n    if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`)\n\n    try {\n      const res = await fetch(`${this.baseUrl}/mcp`, {\n        method: 'POST',\n        headers: {\n          'content-type': 'application/json',\n          accept: 'application/json, text/event-stream',\n          'x-api-key': this.apiKey,\n        },\n        body: JSON.stringify({\n          jsonrpc: '2.0',\n          id: `memory:${publicToolName}`,\n          method: 'tools/call',\n          params: { name: publicToolName, arguments: args },\n        }),\n      })\n      const text = await res.text()\n      if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`)\n\n      const envelope = parseJsonRpcEnvelope(text)\n      if (envelope.error) {\n        return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`)\n      }\n      return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`)\n    } catch (err) {\n      return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`)\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;;;ACFrC,IAAM,+BAA+B,IAAI;AAAA,EACvC,oBAAoB,IAAI,YAAU,CAAC,OAAO,cAAc,OAAO,EAAE,CAAC;AACpE;AAOA,SAAS,qBAAqB,MAA+B;AAC3D,QAAM,WAAW,KACd,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,WAAW,OAAO,CAAC,EACvC,IAAI,UAAQ,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,EAChC,OAAO,OAAO;AAEjB,MAAI,CAAC,SAAS,OAAQ,QAAO,KAAK,MAAM,IAAI;AAE5C,aAAW,WAAW,UAAU;AAC9B,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,UAAU,OAAO,MAAO,QAAO;AAAA,EAC5C;AACA,QAAM,IAAI,MAAM,wCAAwC;AAC1D;AAEA,SAAS,YAAY,SAAiC;AACpD,SAAO;AAAA,IACL,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,KAAK,UAAU,EAAE,IAAI,OAAO,OAAO,QAAQ,CAAC,EAAE,CAAC;AAAA,IAC/E,SAAS;AAAA,EACX;AACF;AAOO,IAAM,8BAAN,MAAkC;AAAA,EACtB;AAAA,EACA;AAAA,EAEjB,YAAYA,UAAiBC,SAAgB;AAC3C,SAAK,UAAUD,SAAQ,QAAQ,OAAO,EAAE;AACxC,SAAK,SAASC;AAAA,EAChB;AAAA,EAEA,MAAM,eAAe,cAAsB,MAAwD;AACjG,UAAM,iBAAiB,6BAA6B,IAAI,YAAY;AACpE,QAAI,CAAC,eAAgB,QAAO,YAAY,wBAAwB,YAAY,EAAE;AAE9E,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAC7C,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,QAAQ;AAAA,UACR,aAAa,KAAK;AAAA,QACpB;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,SAAS;AAAA,UACT,IAAI,UAAU,cAAc;AAAA,UAC5B,QAAQ;AAAA,UACR,QAAQ,EAAE,MAAM,gBAAgB,WAAW,KAAK;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AACD,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI,CAAC,IAAI,GAAI,QAAO,YAAY,iBAAiB,cAAc,iBAAiB,IAAI,MAAM,GAAG;AAE7F,YAAM,WAAW,qBAAqB,IAAI;AAC1C,UAAI,SAAS,OAAO;AAClB,eAAO,YAAY,SAAS,MAAM,WAAW,iBAAiB,cAAc,SAAS;AAAA,MACvF;AACA,aAAO,SAAS,UAAU,YAAY,iBAAiB,cAAc,qBAAqB;AAAA,IAC5F,SAAS,KAAK;AACZ,aAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,iBAAiB,cAAc,cAAc;AAAA,IACvG;AAAA,EACF;AACF;;;ADjEA,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG,EAAE,cAAc,oBAAoB,CAAC;AAErH,IAAM,eAAe,IAAI,oBAAoB,SAAS,MAAM;AAC5D,6BAA6B,QAAQ,YAAY;AACjD,qCAAqC,QAAQ,YAAY;AACzD,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AACxE,uBAAuB,QAAQ,IAAI,4BAA4B,SAAS,MAAM,CAAC;AAE/E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["baseUrl","apiKey"]}