mcp-scraper 0.42.3 → 0.43.2

package/dist/bin/mcp-stdio-server.js

@@ -1,26 +1,27 @@
 #!/usr/bin/env node
 import {
   HttpMcpToolExecutor,
-  MEMORY_TOOL_SCHEMAS,
+  MEMORY_TOOL_REGISTRY,
   SERVER_INSTRUCTIONS,
   ScheduledResultsMcpExecutor,
+  permitsLocalNetworkAccess,
   registerBrowserAgentMcpTools,
   registerMemoryMcpTools,
   registerPaaExtractorMcpTools,
   registerScheduledResultsMcpTools,
-  registerSerpIntelligenceCaptureTools
-} from "../chunk-QKCDYMFT.js";
-import "../chunk-MJKVDLVR.js";
+  registerSerpIntelligenceCaptureTools,
+  resolveDeploymentProfile
+} from "../chunk-YJNGNMDO.js";
+import "../chunk-2K74LVV7.js";
 import "../chunk-BBI7RGOT.js";
 import "../chunk-DAS7L4SF.js";
 import {
   renderInstallTerminal
-} from "../chunk-ENJQ2NSX.js";
-import "../chunk-CB5C3BPB.js";
+} from "../chunk-TC4GDB2Q.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-2YFTFT3V.js";
-import "../chunk-IMUA3FJB.js";
+} from "../chunk-MD75M2L5.js";
+import "../chunk-X5WDJ7D7.js";
 import "../chunk-5UN33CGU.js";
 import "../chunk-FCFQ634B.js";
 
@@ -28,12 +29,12 @@ import "../chunk-FCFQ634B.js";
 import { readFileSync } from "fs";
 import { homedir } from "os";
 import { join } from "path";
-import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import { serveStdio } from "@modelcontextprotocol/server/stdio";
+import { McpServer } from "@modelcontextprotocol/server";
 
 // src/mcp/hosted-memory-mcp-tool-executor.ts
 var publicToolNameByUpstreamName = new Map(
-  MEMORY_TOOL_SCHEMAS.map((schema) => [schema.upstreamName, schema.id])
+  MEMORY_TOOL_REGISTRY.map((schema) => [schema.upstreamName, schema.id])
 );
 function parseJsonRpcEnvelope(text) {
   const payloads = text.split("\n").filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).filter(Boolean);
@@ -118,22 +119,48 @@ if (!apiKey) {
   process.stderr.write("MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\n");
   process.exit(1);
 }
+var requiredApiKey = apiKey;
 var baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || "https://mcpscraper.dev";
+var deploymentProfile = resolveDeploymentProfile();
+var localNetworkAccess = permitsLocalNetworkAccess({
+  deploymentProfile,
+  transportProfile: "stdio",
+  baseUrl,
+  explicitlyEnabled: process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === "1"
+});
 var consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl;
-var server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS });
-var httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey);
-registerPaaExtractorMcpTools(server, httpExecutor);
-registerSerpIntelligenceCaptureTools(server, httpExecutor);
-registerBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl });
-registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, apiKey));
-registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, apiKey));
-var transport = new StdioServerTransport();
-async function main() {
-  await server.connect(transport);
+function buildStdioServer() {
+  const server = new McpServer({ name: "mcp-scraper", version: PACKAGE_VERSION }, {
+    instructions: SERVER_INSTRUCTIONS,
+    cacheHints: {
+      "server/discover": { ttlMs: 3e5, cacheScope: "private" },
+      "tools/list": { ttlMs: 3e5, cacheScope: "private" },
+      "resources/list": { ttlMs: 3e5, cacheScope: "private" },
+      "resources/templates/list": { ttlMs: 3e5, cacheScope: "private" },
+      "resources/read": { ttlMs: 6e4, cacheScope: "private" }
+    }
+  });
+  const httpExecutor = new HttpMcpToolExecutor(baseUrl, requiredApiKey, { localNetworkAccess });
+  registerPaaExtractorMcpTools(server, httpExecutor, {
+    deploymentProfile,
+    transportProfile: "stdio",
+    baseUrl,
+    localNetworkAccess,
+    taskHandleSecret: requiredApiKey
+  });
+  registerSerpIntelligenceCaptureTools(server, httpExecutor, {
+    exposeDevelopmentDiagnostics: deploymentProfile === "development" || deploymentProfile === "test"
+  });
+  registerBrowserAgentMcpTools(server, { baseUrl, apiKey: requiredApiKey, consoleBaseUrl });
+  registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, requiredApiKey));
+  registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, requiredApiKey));
+  return server;
 }
-main().catch((err) => {
-  process.stderr.write(`${err instanceof Error ? err.message : String(err)}
+serveStdio(() => buildStdioServer(), {
+  legacy: "serve",
+  onerror(error) {
+    process.stderr.write(`${error.message}
 `);
-  process.exit(1);
+  }
 });
 //# sourceMappingURL=mcp-stdio-server.js.map

package/dist/bin/mcp-stdio-server.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../bin/mcp-stdio-server.ts","../../src/mcp/hosted-memory-mcp-tool-executor.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { registerMemoryMcpTools } from '../src/mcp/memory-mcp-server.js'\nimport { HostedMemoryMcpToolExecutor } from '../src/mcp/hosted-memory-mcp-tool-executor.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\nimport {\n  registerScheduledResultsMcpTools,\n  ScheduledResultsMcpExecutor,\n} from '../src/mcp/scheduled-results-mcp-server.js'\n\nconst forceStdio =\n  process.argv.includes('--stdio') ||\n  process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n  const noColor =\n    process.argv.includes('--no-color') ||\n    process.env.NO_COLOR !== undefined ||\n    process.env.FORCE_COLOR === '0' ||\n    !process.stdout.isTTY\n\n  process.stdout.write(renderInstallTerminal({\n    version: PACKAGE_VERSION,\n    color: !noColor,\n    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n  }))\n  process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {\n      void 0\n    }\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nconst server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, { instructions: SERVER_INSTRUCTIONS })\n\nconst httpExecutor = new HttpMcpToolExecutor(baseUrl, apiKey)\nregisterPaaExtractorMcpTools(server, httpExecutor)\nregisterSerpIntelligenceCaptureTools(server, httpExecutor)\nregisterBrowserAgentMcpTools(server, { baseUrl, apiKey, consoleBaseUrl })\nregisterScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, apiKey))\nregisterMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, apiKey))\n\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch(err => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n","import type { CallToolResult } from '@modelcontextprotocol/sdk/types.js'\nimport { MEMORY_TOOL_SCHEMAS } from './memory-tool-schemas.js'\n\nconst publicToolNameByUpstreamName = new Map(\n  MEMORY_TOOL_SCHEMAS.map(schema => [schema.upstreamName, schema.id]),\n)\n\ninterface JsonRpcEnvelope {\n  result?: CallToolResult\n  error?: { code?: number; message?: string }\n}\n\nfunction parseJsonRpcEnvelope(text: string): JsonRpcEnvelope {\n  const payloads = text\n    .split('\\n')\n    .filter(line => line.startsWith('data:'))\n    .map(line => line.slice(5).trim())\n    .filter(Boolean)\n\n  if (!payloads.length) return JSON.parse(text) as JsonRpcEnvelope\n\n  for (const payload of payloads) {\n    const parsed = JSON.parse(payload) as JsonRpcEnvelope\n    if (parsed.result || parsed.error) return parsed\n  }\n  throw new Error('hosted MCP returned no JSON-RPC result')\n}\n\nfunction errorResult(message: string): CallToolResult {\n  return {\n    content: [{ type: 'text', text: JSON.stringify({ ok: false, error: message }) }],\n    isError: true,\n  }\n}\n\n/**\n * Stdio-only bridge for memory tools. It deliberately calls the hosted aggregate\n * MCP surface so local clients execute the same registered tool implementation\n * and output contract as https://mcpscraper.dev/mcp.\n */\nexport class HostedMemoryMcpToolExecutor {\n  private readonly baseUrl: string\n  private readonly apiKey: string\n\n  constructor(baseUrl: string, apiKey: string) {\n    this.baseUrl = baseUrl.replace(/\\/$/, '')\n    this.apiKey = apiKey\n  }\n\n  async callMemoryTool(upstreamName: string, args: Record<string, unknown>): Promise<CallToolResult> {\n    const publicToolName = publicToolNameByUpstreamName.get(upstreamName)\n    if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`)\n\n    try {\n      const res = await fetch(`${this.baseUrl}/mcp`, {\n        method: 'POST',\n        headers: {\n          'content-type': 'application/json',\n          accept: 'application/json, text/event-stream',\n          'x-api-key': this.apiKey,\n        },\n        body: JSON.stringify({\n          jsonrpc: '2.0',\n          id: `memory:${publicToolName}`,\n          method: 'tools/call',\n          params: { name: publicToolName, arguments: args },\n        }),\n      })\n      const text = await res.text()\n      if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`)\n\n      const envelope = parseJsonRpcEnvelope(text)\n      if (envelope.error) {\n        return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`)\n      }\n      return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`)\n    } catch (err) {\n      return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`)\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,iBAAiB;AAC1B,SAAS,4BAA4B;;;ACFrC,IAAM,+BAA+B,IAAI;AAAA,EACvC,oBAAoB,IAAI,YAAU,CAAC,OAAO,cAAc,OAAO,EAAE,CAAC;AACpE;AAOA,SAAS,qBAAqB,MAA+B;AAC3D,QAAM,WAAW,KACd,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,WAAW,OAAO,CAAC,EACvC,IAAI,UAAQ,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,EAChC,OAAO,OAAO;AAEjB,MAAI,CAAC,SAAS,OAAQ,QAAO,KAAK,MAAM,IAAI;AAE5C,aAAW,WAAW,UAAU;AAC9B,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,UAAU,OAAO,MAAO,QAAO;AAAA,EAC5C;AACA,QAAM,IAAI,MAAM,wCAAwC;AAC1D;AAEA,SAAS,YAAY,SAAiC;AACpD,SAAO;AAAA,IACL,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,KAAK,UAAU,EAAE,IAAI,OAAO,OAAO,QAAQ,CAAC,EAAE,CAAC;AAAA,IAC/E,SAAS;AAAA,EACX;AACF;AAOO,IAAM,8BAAN,MAAkC;AAAA,EACtB;AAAA,EACA;AAAA,EAEjB,YAAYA,UAAiBC,SAAgB;AAC3C,SAAK,UAAUD,SAAQ,QAAQ,OAAO,EAAE;AACxC,SAAK,SAASC;AAAA,EAChB;AAAA,EAEA,MAAM,eAAe,cAAsB,MAAwD;AACjG,UAAM,iBAAiB,6BAA6B,IAAI,YAAY;AACpE,QAAI,CAAC,eAAgB,QAAO,YAAY,wBAAwB,YAAY,EAAE;AAE9E,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAC7C,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,QAAQ;AAAA,UACR,aAAa,KAAK;AAAA,QACpB;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,SAAS;AAAA,UACT,IAAI,UAAU,cAAc;AAAA,UAC5B,QAAQ;AAAA,UACR,QAAQ,EAAE,MAAM,gBAAgB,WAAW,KAAK;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AACD,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI,CAAC,IAAI,GAAI,QAAO,YAAY,iBAAiB,cAAc,iBAAiB,IAAI,MAAM,GAAG;AAE7F,YAAM,WAAW,qBAAqB,IAAI;AAC1C,UAAI,SAAS,OAAO;AAClB,eAAO,YAAY,SAAS,MAAM,WAAW,iBAAiB,cAAc,SAAS;AAAA,MACvF;AACA,aAAO,SAAS,UAAU,YAAY,iBAAiB,cAAc,qBAAqB;AAAA,IAC5F,SAAS,KAAK;AACZ,aAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,iBAAiB,cAAc,cAAc;AAAA,IACvG;AAAA,EACF;AACF;;;AD7DA,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,IAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG,EAAE,cAAc,oBAAoB,CAAC;AAErH,IAAM,eAAe,IAAI,oBAAoB,SAAS,MAAM;AAC5D,6BAA6B,QAAQ,YAAY;AACjD,qCAAqC,QAAQ,YAAY;AACzD,6BAA6B,QAAQ,EAAE,SAAS,QAAQ,eAAe,CAAC;AACxE,iCAAiC,QAAQ,IAAI,4BAA4B,SAAS,MAAM,CAAC;AACzF,uBAAuB,QAAQ,IAAI,4BAA4B,SAAS,MAAM,CAAC;AAE/E,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,SAAO;AAClB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":["baseUrl","apiKey"]}
+{"version":3,"sources":["../../bin/mcp-stdio-server.ts","../../src/mcp/hosted-memory-mcp-tool-executor.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { serveStdio } from '@modelcontextprotocol/server/stdio'\nimport { McpServer } from \"@modelcontextprotocol/server\";\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { registerMemoryMcpTools } from '../src/mcp/memory-mcp-server.js'\nimport { HostedMemoryMcpToolExecutor } from '../src/mcp/hosted-memory-mcp-tool-executor.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\nimport { permitsLocalNetworkAccess, resolveDeploymentProfile } from '../src/runtime/deployment-profile.js'\nimport {\n  registerScheduledResultsMcpTools,\n  ScheduledResultsMcpExecutor,\n} from '../src/mcp/scheduled-results-mcp-server.js'\n\nconst forceStdio =\n  process.argv.includes('--stdio') ||\n  process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n  const noColor =\n    process.argv.includes('--no-color') ||\n    process.env.NO_COLOR !== undefined ||\n    process.env.FORCE_COLOR === '0' ||\n    !process.stdout.isTTY\n\n  process.stdout.write(renderInstallTerminal({\n    version: PACKAGE_VERSION,\n    color: !noColor,\n    apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n  }))\n  process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {\n      void 0\n    }\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\nconst requiredApiKey = apiKey as string\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst deploymentProfile = resolveDeploymentProfile()\nconst localNetworkAccess = permitsLocalNetworkAccess({\n  deploymentProfile,\n  transportProfile: 'stdio',\n  baseUrl,\n  explicitlyEnabled: process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === '1',\n})\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nfunction buildStdioServer(): McpServer {\n  const server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, {\n    instructions: SERVER_INSTRUCTIONS,\n    cacheHints: {\n      'server/discover': { ttlMs: 300_000, cacheScope: 'private' },\n      'tools/list': { ttlMs: 300_000, cacheScope: 'private' },\n      'resources/list': { ttlMs: 300_000, cacheScope: 'private' },\n      'resources/templates/list': { ttlMs: 300_000, cacheScope: 'private' },\n      'resources/read': { ttlMs: 60_000, cacheScope: 'private' },\n    },\n  })\n  const httpExecutor = new HttpMcpToolExecutor(baseUrl, requiredApiKey, { localNetworkAccess })\n  registerPaaExtractorMcpTools(server, httpExecutor, {\n    deploymentProfile,\n    transportProfile: 'stdio',\n    baseUrl,\n    localNetworkAccess,\n    taskHandleSecret: requiredApiKey,\n  })\n  registerSerpIntelligenceCaptureTools(server, httpExecutor, {\n    exposeDevelopmentDiagnostics: deploymentProfile === 'development' || deploymentProfile === 'test',\n  })\n  registerBrowserAgentMcpTools(server, { baseUrl, apiKey: requiredApiKey, consoleBaseUrl })\n  registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, requiredApiKey))\n  registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, requiredApiKey))\n  return server\n}\n\nserveStdio(() => buildStdioServer(), {\n  legacy: 'serve',\n  onerror(error) {\n    process.stderr.write(`${error.message}\\n`)\n  },\n})\n","import type { CallToolResult } from \"@modelcontextprotocol/server\";\nimport { MEMORY_TOOL_REGISTRY } from './memory-tool-registry.js'\n\nconst publicToolNameByUpstreamName = new Map(\n  MEMORY_TOOL_REGISTRY.map(schema => [schema.upstreamName, schema.id]),\n)\n\ninterface JsonRpcEnvelope {\n  result?: CallToolResult\n  error?: { code?: number; message?: string }\n}\n\nfunction parseJsonRpcEnvelope(text: string): JsonRpcEnvelope {\n  const payloads = text\n    .split('\\n')\n    .filter(line => line.startsWith('data:'))\n    .map(line => line.slice(5).trim())\n    .filter(Boolean)\n\n  if (!payloads.length) return JSON.parse(text) as JsonRpcEnvelope\n\n  for (const payload of payloads) {\n    const parsed = JSON.parse(payload) as JsonRpcEnvelope\n    if (parsed.result || parsed.error) return parsed\n  }\n  throw new Error('hosted MCP returned no JSON-RPC result')\n}\n\nfunction errorResult(message: string): CallToolResult {\n  return {\n    content: [{ type: 'text', text: JSON.stringify({ ok: false, error: message }) }],\n    isError: true,\n  }\n}\n\n/**\n * Stdio-only bridge for memory tools. It deliberately calls the hosted aggregate\n * MCP surface so local clients execute the same registered tool implementation\n * and output contract as https://mcpscraper.dev/mcp.\n */\nexport class HostedMemoryMcpToolExecutor {\n  private readonly baseUrl: string\n  private readonly apiKey: string\n\n  constructor(baseUrl: string, apiKey: string) {\n    this.baseUrl = baseUrl.replace(/\\/$/, '')\n    this.apiKey = apiKey\n  }\n\n  async callMemoryTool(upstreamName: string, args: Record<string, unknown>): Promise<CallToolResult> {\n    const publicToolName = publicToolNameByUpstreamName.get(upstreamName)\n    if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`)\n\n    try {\n      const res = await fetch(`${this.baseUrl}/mcp`, {\n        method: 'POST',\n        headers: {\n          'content-type': 'application/json',\n          accept: 'application/json, text/event-stream',\n          'x-api-key': this.apiKey,\n        },\n        body: JSON.stringify({\n          jsonrpc: '2.0',\n          id: `memory:${publicToolName}`,\n          method: 'tools/call',\n          params: { name: publicToolName, arguments: args },\n        }),\n      })\n      const text = await res.text()\n      if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`)\n\n      const envelope = parseJsonRpcEnvelope(text)\n      if (envelope.error) {\n        return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`)\n      }\n      return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`)\n    } catch (err) {\n      return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`)\n    }\n  }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;;;ACF1B,IAAM,+BAA+B,IAAI;AAAA,EACvC,qBAAqB,IAAI,YAAU,CAAC,OAAO,cAAc,OAAO,EAAE,CAAC;AACrE;AAOA,SAAS,qBAAqB,MAA+B;AAC3D,QAAM,WAAW,KACd,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,WAAW,OAAO,CAAC,EACvC,IAAI,UAAQ,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,EAChC,OAAO,OAAO;AAEjB,MAAI,CAAC,SAAS,OAAQ,QAAO,KAAK,MAAM,IAAI;AAE5C,aAAW,WAAW,UAAU;AAC9B,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,UAAU,OAAO,MAAO,QAAO;AAAA,EAC5C;AACA,QAAM,IAAI,MAAM,wCAAwC;AAC1D;AAEA,SAAS,YAAY,SAAiC;AACpD,SAAO;AAAA,IACL,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,KAAK,UAAU,EAAE,IAAI,OAAO,OAAO,QAAQ,CAAC,EAAE,CAAC;AAAA,IAC/E,SAAS;AAAA,EACX;AACF;AAOO,IAAM,8BAAN,MAAkC;AAAA,EACtB;AAAA,EACA;AAAA,EAEjB,YAAYA,UAAiBC,SAAgB;AAC3C,SAAK,UAAUD,SAAQ,QAAQ,OAAO,EAAE;AACxC,SAAK,SAASC;AAAA,EAChB;AAAA,EAEA,MAAM,eAAe,cAAsB,MAAwD;AACjG,UAAM,iBAAiB,6BAA6B,IAAI,YAAY;AACpE,QAAI,CAAC,eAAgB,QAAO,YAAY,wBAAwB,YAAY,EAAE;AAE9E,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAC7C,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,QAAQ;AAAA,UACR,aAAa,KAAK;AAAA,QACpB;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,SAAS;AAAA,UACT,IAAI,UAAU,cAAc;AAAA,UAC5B,QAAQ;AAAA,UACR,QAAQ,EAAE,MAAM,gBAAgB,WAAW,KAAK;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AACD,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI,CAAC,IAAI,GAAI,QAAO,YAAY,iBAAiB,cAAc,iBAAiB,IAAI,MAAM,GAAG;AAE7F,YAAM,WAAW,qBAAqB,IAAI;AAC1C,UAAI,SAAS,OAAO;AAClB,eAAO,YAAY,SAAS,MAAM,WAAW,iBAAiB,cAAc,SAAS;AAAA,MACvF;AACA,aAAO,SAAS,UAAU,YAAY,iBAAiB,cAAc,qBAAqB;AAAA,IAC5F,SAAS,KAAK;AACZ,aAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,iBAAiB,cAAc,cAAc;AAAA,IACvG;AAAA,EACF;AACF;;;AD5DA,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AACA,IAAM,iBAAiB;AAEvB,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,oBAAoB,yBAAyB;AACnD,IAAM,qBAAqB,0BAA0B;AAAA,EACnD;AAAA,EACA,kBAAkB;AAAA,EAClB;AAAA,EACA,mBAAmB,QAAQ,IAAI,sCAAsC;AACvE,CAAC;AACD,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,SAAS,mBAA8B;AACrC,QAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG;AAAA,IAC9E,cAAc;AAAA,IACd,YAAY;AAAA,MACV,mBAAmB,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MAC3D,cAAc,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MACtD,kBAAkB,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MAC1D,4BAA4B,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MACpE,kBAAkB,EAAE,OAAO,KAAQ,YAAY,UAAU;AAAA,IAC3D;AAAA,EACF,CAAC;AACD,QAAM,eAAe,IAAI,oBAAoB,SAAS,gBAAgB,EAAE,mBAAmB,CAAC;AAC5F,+BAA6B,QAAQ,cAAc;AAAA,IACjD;AAAA,IACA,kBAAkB;AAAA,IAClB;AAAA,IACA;AAAA,IACA,kBAAkB;AAAA,EACpB,CAAC;AACD,uCAAqC,QAAQ,cAAc;AAAA,IACzD,8BAA8B,sBAAsB,iBAAiB,sBAAsB;AAAA,EAC7F,CAAC;AACD,+BAA6B,QAAQ,EAAE,SAAS,QAAQ,gBAAgB,eAAe,CAAC;AACxF,mCAAiC,QAAQ,IAAI,4BAA4B,SAAS,cAAc,CAAC;AACjG,yBAAuB,QAAQ,IAAI,4BAA4B,SAAS,cAAc,CAAC;AACvF,SAAO;AACT;AAEA,WAAW,MAAM,iBAAiB,GAAG;AAAA,EACnC,QAAQ;AAAA,EACR,QAAQ,OAAO;AACb,YAAQ,OAAO,MAAM,GAAG,MAAM,OAAO;AAAA,CAAI;AAAA,EAC3C;AACF,CAAC;","names":["baseUrl","apiKey"]}

package/dist/bin/paa-harvest.cjs

@@ -546,11 +546,21 @@ async function ensureKernelProfile(kernelClient, name) {
     if (!isKernelProfileConflict(err)) throw err;
   }
 }
-function rankCheckContextOptions(config) {
+async function detectRuntimeUserAgent(browser) {
+  try {
+    const page = browser.contexts()[0]?.pages()[0];
+    if (!page) return null;
+    const ua = await page.evaluate(() => navigator.userAgent);
+    return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
+  } catch {
+    return null;
+  }
+}
+function rankCheckContextOptions(config, runtimeUserAgent) {
   return {
     viewport: config.viewport,
     locale: config.locale,
-    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
     ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
     ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -647,7 +657,7 @@ var BrowserDriver = class {
       this.kernelHeadlessSent = useHeadless ? true : null;
       if (costCtx?.headlessSentOut) costCtx.headlessSentOut.value = this.kernelHeadlessSent;
       const kernelBrowser = await this.kernelClient.browsers.create({
-        stealth: true,
+        stealth: config.kernelStealth === true,
         timeout_seconds: timeoutSeconds,
         ...useHeadless ? { headless: true } : {},
         ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {},
@@ -712,7 +722,9 @@ var BrowserDriver = class {
         await this.populateKernelRetrieveDebug(kernelDebug, config.kernelProxyId);
       }
       this.browser = await import_playwright.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
-      this.context = await this.browser.newContext(rankCheckContextOptions(config));
+      const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
+      this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
+      this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
       await this.page.setViewportSize(config.viewport);
@@ -2845,7 +2857,6 @@ var import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
 var BACKUP_PROXY_HOST = "pr.oxylabs.io";
 var BACKUP_PROXY_PORT = 7777;
 var BACKUP_STICKY_SESSION_MINUTES = 10;
-var BACKUP_LAST_RESORT_ATTEMPTS = 2;
 function backupProxyUsername() {
   const value = process.env.OXYLABS_USERNAME?.trim();
   return value || void 0;
@@ -3199,15 +3210,6 @@ async function settleCreateExhaustion(options, target, createErrors) {
   }
   return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
 }
-async function resolveBackupKernelProxyId(options) {
-  if (!options.kernelApiKey || !backupProxyAvailable()) return null;
-  try {
-    const backupId = await createBackupProxyId(options.kernelApiKey, { attemptIndex: options.attemptIndex });
-    return resolution("backup_custom", options.proxyMode, backupId, null, null, true);
-  } catch {
-    return null;
-  }
-}
 async function createFreshLocationProxy(kernel, options, target) {
   const createErrors = [];
   for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
@@ -3567,7 +3569,7 @@ async function harvest(rawOptions) {
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
   const requestedMaxAttempts = raw.maxAttempts;
   const baseMaxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
-  const maxAttempts = baseMaxAttempts + (backupProxyAvailable() ? BACKUP_LAST_RESORT_ATTEMPTS : 0);
+  const maxAttempts = baseMaxAttempts;
   const softDeadlineMs = typeof raw.softDeadlineMs === "number" ? raw.softDeadlineMs : null;
   const serializer = new OutputSerializer();
   let lastError = null;
@@ -3578,14 +3580,11 @@ async function harvest(rawOptions) {
     const startedAtMs = Date.now();
     try {
       if (signal?.aborted) throw abortReason(signal);
-      const isBackupAttempt = i >= baseMaxAttempts;
-      if (isBackupAttempt && softDeadlineMs !== null && Date.now() >= softDeadlineMs) break;
-      const resolution2 = isBackupAttempt ? await resolveBackupKernelProxyId({ kernelApiKey, proxyMode: effectiveProxyMode, attemptIndex: i }) : await resolveKernelProxyId({
+      const resolution2 = await resolveKernelProxyId({
         ...proxyOpts,
         proxyMode: effectiveProxyMode,
         attemptIndex: i,
-        fresh: effectiveProxyMode === "location",
-        backupOnExhaustion: true
+        fresh: effectiveProxyMode === "location"
       });
       if (!resolution2) break;
       const mergedAttempt = {
@@ -3594,9 +3593,10 @@ async function harvest(rawOptions) {
         kernelProxyId: resolution2.kernelProxyId,
         kernelProxyResolution: resolution2.resolution,
         proxyMode: effectiveProxyMode,
+        keepDefaultProxy: i > 0,
         debug: requestedDebug || needsLocationEvidence
       };
-      if (effectiveProxyMode === "none" && !isBackupAttempt) mergedAttempt.kernelProxyId = void 0;
+      if (effectiveProxyMode === "none") mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",