mcp-scraper 0.43.1 → 0.43.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +7 -17
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +3 -4
- package/dist/bin/mcp-scraper-cli.js.map +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +2321 -2437
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +2 -3
- package/dist/bin/mcp-stdio-server.js.map +1 -1
- package/dist/bin/paa-harvest.js +2 -3
- package/dist/bin/paa-harvest.js.map +1 -1
- package/dist/{chunk-CB5C3BPB.js → chunk-3ZUBQQPQ.js} +23 -2
- package/dist/chunk-3ZUBQQPQ.js.map +1 -0
- package/dist/{chunk-LXVIQZ7Y.js → chunk-H7LQ6PTI.js} +3 -5
- package/dist/{chunk-LXVIQZ7Y.js.map → chunk-H7LQ6PTI.js.map} +1 -1
- package/dist/chunk-MD75M2L5.js +7 -0
- package/dist/chunk-MD75M2L5.js.map +1 -0
- package/dist/{chunk-HO4DOPEG.js → chunk-YJNGNMDO.js} +8 -21
- package/dist/chunk-YJNGNMDO.js.map +1 -0
- package/dist/{chunk-FQI5PFE7.js → chunk-ZSJHRZY5.js} +3 -5
- package/dist/{chunk-FQI5PFE7.js.map → chunk-ZSJHRZY5.js.map} +1 -1
- package/dist/index.js +2 -3
- package/dist/index.js.map +1 -1
- package/dist/{server-WAB6Z47N.js → server-BPEXEKA6.js} +7 -8
- package/dist/{server-WAB6Z47N.js.map → server-BPEXEKA6.js.map} +1 -1
- package/dist/{worker-E6MIVF33.js → worker-2LIOH744.js} +3 -4
- package/dist/{worker-E6MIVF33.js.map → worker-2LIOH744.js.map} +1 -1
- package/package.json +2 -2
- package/dist/chunk-3CQZ3DAT.js +0 -7
- package/dist/chunk-3CQZ3DAT.js.map +0 -1
- package/dist/chunk-CB5C3BPB.js.map +0 -1
- package/dist/chunk-HO4DOPEG.js.map +0 -1
- package/dist/chunk-K443GQY5.js +0 -24
- package/dist/chunk-K443GQY5.js.map +0 -1
|
@@ -11,17 +11,16 @@ import {
|
|
|
11
11
|
registerScheduledResultsMcpTools,
|
|
12
12
|
registerSerpIntelligenceCaptureTools,
|
|
13
13
|
resolveDeploymentProfile
|
|
14
|
-
} from "../chunk-
|
|
14
|
+
} from "../chunk-YJNGNMDO.js";
|
|
15
15
|
import "../chunk-2K74LVV7.js";
|
|
16
16
|
import "../chunk-BBI7RGOT.js";
|
|
17
17
|
import "../chunk-DAS7L4SF.js";
|
|
18
18
|
import {
|
|
19
19
|
renderInstallTerminal
|
|
20
20
|
} from "../chunk-TC4GDB2Q.js";
|
|
21
|
-
import "../chunk-CB5C3BPB.js";
|
|
22
21
|
import {
|
|
23
22
|
PACKAGE_VERSION
|
|
24
|
-
} from "../chunk-
|
|
23
|
+
} from "../chunk-MD75M2L5.js";
|
|
25
24
|
import "../chunk-X5WDJ7D7.js";
|
|
26
25
|
import "../chunk-5UN33CGU.js";
|
|
27
26
|
import "../chunk-FCFQ634B.js";
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../bin/mcp-stdio-server.ts","../../src/mcp/hosted-memory-mcp-tool-executor.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { serveStdio } from '@modelcontextprotocol/server/stdio'\nimport { McpServer } from \"@modelcontextprotocol/server\";\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { registerMemoryMcpTools } from '../src/mcp/memory-mcp-server.js'\nimport { HostedMemoryMcpToolExecutor } from '../src/mcp/hosted-memory-mcp-tool-executor.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\nimport { permitsLocalNetworkAccess, resolveDeploymentProfile } from '../src/runtime/deployment-profile.js'\nimport {\n registerScheduledResultsMcpTools,\n ScheduledResultsMcpExecutor,\n} from '../src/mcp/scheduled-results-mcp-server.js'\n\nconst forceStdio =\n process.argv.includes('--stdio') ||\n process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n const noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\n process.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n }))\n process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\nconst requiredApiKey = apiKey as string\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst deploymentProfile = resolveDeploymentProfile()\nconst localNetworkAccess = permitsLocalNetworkAccess({\n deploymentProfile,\n transportProfile: 'stdio',\n baseUrl,\n explicitlyEnabled: process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === '1',\n})\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nfunction buildStdioServer(): McpServer {\n const server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, {\n instructions: SERVER_INSTRUCTIONS,\n cacheHints: {\n 'server/discover': { ttlMs: 300_000, cacheScope: 'private' },\n 'tools/list': { ttlMs: 300_000, cacheScope: 'private' },\n 'resources/list': { ttlMs: 300_000, cacheScope: 'private' },\n 'resources/templates/list': { ttlMs: 300_000, cacheScope: 'private' },\n 'resources/read': { ttlMs: 60_000, cacheScope: 'private' },\n },\n })\n const httpExecutor = new HttpMcpToolExecutor(baseUrl, requiredApiKey, { localNetworkAccess })\n registerPaaExtractorMcpTools(server, httpExecutor, {\n deploymentProfile,\n transportProfile: 'stdio',\n baseUrl,\n localNetworkAccess,\n taskHandleSecret: requiredApiKey,\n })\n registerSerpIntelligenceCaptureTools(server, httpExecutor, {\n exposeDevelopmentDiagnostics: deploymentProfile === 'development' || deploymentProfile === 'test',\n })\n registerBrowserAgentMcpTools(server, { baseUrl, apiKey: requiredApiKey, consoleBaseUrl })\n registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, requiredApiKey))\n registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, requiredApiKey))\n return server\n}\n\nserveStdio(() => buildStdioServer(), {\n legacy: 'serve',\n onerror(error) {\n process.stderr.write(`${error.message}\\n`)\n },\n})\n","import type { CallToolResult } from \"@modelcontextprotocol/server\";\nimport { MEMORY_TOOL_REGISTRY } from './memory-tool-registry.js'\n\nconst publicToolNameByUpstreamName = new Map(\n MEMORY_TOOL_REGISTRY.map(schema => [schema.upstreamName, schema.id]),\n)\n\ninterface JsonRpcEnvelope {\n result?: CallToolResult\n error?: { code?: number; message?: string }\n}\n\nfunction parseJsonRpcEnvelope(text: string): JsonRpcEnvelope {\n const payloads = text\n .split('\\n')\n .filter(line => line.startsWith('data:'))\n .map(line => line.slice(5).trim())\n .filter(Boolean)\n\n if (!payloads.length) return JSON.parse(text) as JsonRpcEnvelope\n\n for (const payload of payloads) {\n const parsed = JSON.parse(payload) as JsonRpcEnvelope\n if (parsed.result || parsed.error) return parsed\n }\n throw new Error('hosted MCP returned no JSON-RPC result')\n}\n\nfunction errorResult(message: string): CallToolResult {\n return {\n content: [{ type: 'text', text: JSON.stringify({ ok: false, error: message }) }],\n isError: true,\n }\n}\n\n/**\n * Stdio-only bridge for memory tools. It deliberately calls the hosted aggregate\n * MCP surface so local clients execute the same registered tool implementation\n * and output contract as https://mcpscraper.dev/mcp.\n */\nexport class HostedMemoryMcpToolExecutor {\n private readonly baseUrl: string\n private readonly apiKey: string\n\n constructor(baseUrl: string, apiKey: string) {\n this.baseUrl = baseUrl.replace(/\\/$/, '')\n this.apiKey = apiKey\n }\n\n async callMemoryTool(upstreamName: string, args: Record<string, unknown>): Promise<CallToolResult> {\n const publicToolName = publicToolNameByUpstreamName.get(upstreamName)\n if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`)\n\n try {\n const res = await fetch(`${this.baseUrl}/mcp`, {\n method: 'POST',\n headers: {\n 'content-type': 'application/json',\n accept: 'application/json, text/event-stream',\n 'x-api-key': this.apiKey,\n },\n body: JSON.stringify({\n jsonrpc: '2.0',\n id: `memory:${publicToolName}`,\n method: 'tools/call',\n params: { name: publicToolName, arguments: args },\n }),\n })\n const text = await res.text()\n if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`)\n\n const envelope = parseJsonRpcEnvelope(text)\n if (envelope.error) {\n return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`)\n }\n return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`)\n } catch (err) {\n return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`)\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;;;ACF1B,IAAM,+BAA+B,IAAI;AAAA,EACvC,qBAAqB,IAAI,YAAU,CAAC,OAAO,cAAc,OAAO,EAAE,CAAC;AACrE;AAOA,SAAS,qBAAqB,MAA+B;AAC3D,QAAM,WAAW,KACd,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,WAAW,OAAO,CAAC,EACvC,IAAI,UAAQ,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,EAChC,OAAO,OAAO;AAEjB,MAAI,CAAC,SAAS,OAAQ,QAAO,KAAK,MAAM,IAAI;AAE5C,aAAW,WAAW,UAAU;AAC9B,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,UAAU,OAAO,MAAO,QAAO;AAAA,EAC5C;AACA,QAAM,IAAI,MAAM,wCAAwC;AAC1D;AAEA,SAAS,YAAY,SAAiC;AACpD,SAAO;AAAA,IACL,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,KAAK,UAAU,EAAE,IAAI,OAAO,OAAO,QAAQ,CAAC,EAAE,CAAC;AAAA,IAC/E,SAAS;AAAA,EACX;AACF;AAOO,IAAM,8BAAN,MAAkC;AAAA,EACtB;AAAA,EACA;AAAA,EAEjB,YAAYA,UAAiBC,SAAgB;AAC3C,SAAK,UAAUD,SAAQ,QAAQ,OAAO,EAAE;AACxC,SAAK,SAASC;AAAA,EAChB;AAAA,EAEA,MAAM,eAAe,cAAsB,MAAwD;AACjG,UAAM,iBAAiB,6BAA6B,IAAI,YAAY;AACpE,QAAI,CAAC,eAAgB,QAAO,YAAY,wBAAwB,YAAY,EAAE;AAE9E,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAC7C,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,QAAQ;AAAA,UACR,aAAa,KAAK;AAAA,QACpB;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,SAAS;AAAA,UACT,IAAI,UAAU,cAAc;AAAA,UAC5B,QAAQ;AAAA,UACR,QAAQ,EAAE,MAAM,gBAAgB,WAAW,KAAK;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AACD,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI,CAAC,IAAI,GAAI,QAAO,YAAY,iBAAiB,cAAc,iBAAiB,IAAI,MAAM,GAAG;AAE7F,YAAM,WAAW,qBAAqB,IAAI;AAC1C,UAAI,SAAS,OAAO;AAClB,eAAO,YAAY,SAAS,MAAM,WAAW,iBAAiB,cAAc,SAAS;AAAA,MACvF;AACA,aAAO,SAAS,UAAU,YAAY,iBAAiB,cAAc,qBAAqB;AAAA,IAC5F,SAAS,KAAK;AACZ,aAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,iBAAiB,cAAc,cAAc;AAAA,IACvG;AAAA,EACF;AACF;;;AD5DA,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AACA,IAAM,iBAAiB;AAEvB,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,oBAAoB,yBAAyB;AACnD,IAAM,qBAAqB,0BAA0B;AAAA,EACnD;AAAA,EACA,kBAAkB;AAAA,EAClB;AAAA,EACA,mBAAmB,QAAQ,IAAI,sCAAsC;AACvE,CAAC;AACD,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,SAAS,mBAA8B;AACrC,QAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG;AAAA,IAC9E,cAAc;AAAA,IACd,YAAY;AAAA,MACV,mBAAmB,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MAC3D,cAAc,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MACtD,kBAAkB,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MAC1D,4BAA4B,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MACpE,kBAAkB,EAAE,OAAO,KAAQ,YAAY,UAAU;AAAA,IAC3D;AAAA,EACF,CAAC;AACD,QAAM,eAAe,IAAI,oBAAoB,SAAS,gBAAgB,EAAE,mBAAmB,CAAC;AAC5F,+BAA6B,QAAQ,cAAc;AAAA,IACjD;AAAA,IACA,kBAAkB;AAAA,IAClB;AAAA,IACA;AAAA,IACA,kBAAkB;AAAA,EACpB,CAAC;AACD,uCAAqC,QAAQ,cAAc;AAAA,IACzD,8BAA8B,sBAAsB,iBAAiB,sBAAsB;AAAA,EAC7F,CAAC;AACD,+BAA6B,QAAQ,EAAE,SAAS,QAAQ,gBAAgB,eAAe,CAAC;AACxF,mCAAiC,QAAQ,IAAI,4BAA4B,SAAS,cAAc,CAAC;AACjG,yBAAuB,QAAQ,IAAI,4BAA4B,SAAS,cAAc,CAAC;AACvF,SAAO;AACT;AAEA,WAAW,MAAM,iBAAiB,GAAG;AAAA,EACnC,QAAQ;AAAA,EACR,QAAQ,OAAO;AACb,YAAQ,OAAO,MAAM,GAAG,MAAM,OAAO;AAAA,CAAI;AAAA,EAC3C;AACF,CAAC;","names":["baseUrl","apiKey"]}
|
|
1
|
+
{"version":3,"sources":["../../bin/mcp-stdio-server.ts","../../src/mcp/hosted-memory-mcp-tool-executor.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { serveStdio } from '@modelcontextprotocol/server/stdio'\nimport { McpServer } from \"@modelcontextprotocol/server\";\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { registerBrowserAgentMcpTools } from '../src/mcp/browser-agent-mcp-server.js'\nimport { registerPaaExtractorMcpTools, registerSerpIntelligenceCaptureTools } from '../src/mcp/paa-mcp-server.js'\nimport { registerMemoryMcpTools } from '../src/mcp/memory-mcp-server.js'\nimport { HostedMemoryMcpToolExecutor } from '../src/mcp/hosted-memory-mcp-tool-executor.js'\nimport { SERVER_INSTRUCTIONS } from '../src/mcp/server-instructions.js'\nimport { renderInstallTerminal } from '../src/install-terminal.js'\nimport { PACKAGE_VERSION } from '../src/version.js'\nimport { permitsLocalNetworkAccess, resolveDeploymentProfile } from '../src/runtime/deployment-profile.js'\nimport {\n registerScheduledResultsMcpTools,\n ScheduledResultsMcpExecutor,\n} from '../src/mcp/scheduled-results-mcp-server.js'\n\nconst forceStdio =\n process.argv.includes('--stdio') ||\n process.env.MCP_SCRAPER_FORCE_STDIO === '1'\nconst interactiveTerminal = Boolean(process.stdin.isTTY && process.stdout.isTTY)\nconst wantsHelp = process.argv.includes('--help') || process.argv.includes('-h')\n\nif (!forceStdio && (interactiveTerminal || wantsHelp)) {\n const noColor =\n process.argv.includes('--no-color') ||\n process.env.NO_COLOR !== undefined ||\n process.env.FORCE_COLOR === '0' ||\n !process.stdout.isTTY\n\n process.stdout.write(renderInstallTerminal({\n version: PACKAGE_VERSION,\n color: !noColor,\n apiKeyConfigured: Boolean(process.env.MCP_SCRAPER_API_KEY?.trim()),\n }))\n process.exit(0)\n}\n\nfunction readApiKeyFile(): string | undefined {\n const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n for (const path of paths) {\n try {\n const value = readFileSync(path, 'utf8').trim()\n if (value) return value\n } catch {\n void 0\n }\n }\n return undefined\n}\n\nconst apiKey = (\n process.env.MCP_SCRAPER_API_KEY ??\n process.env.MCP_SCRAPER_KEY ??\n process.env.MCP_API_KEY ??\n readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n process.exit(1)\n}\nconst requiredApiKey = apiKey as string\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() || process.env.MCP_BASE_URL?.trim() || 'https://mcpscraper.dev'\nconst deploymentProfile = resolveDeploymentProfile()\nconst localNetworkAccess = permitsLocalNetworkAccess({\n deploymentProfile,\n transportProfile: 'stdio',\n baseUrl,\n explicitlyEnabled: process.env.MCP_SCRAPER_ALLOW_PRIVATE_NETWORK === '1',\n})\nconst consoleBaseUrl = process.env.BROWSER_AGENT_CONSOLE_URL?.trim() || baseUrl\nfunction buildStdioServer(): McpServer {\n const server = new McpServer({ name: 'mcp-scraper', version: PACKAGE_VERSION }, {\n instructions: SERVER_INSTRUCTIONS,\n cacheHints: {\n 'server/discover': { ttlMs: 300_000, cacheScope: 'private' },\n 'tools/list': { ttlMs: 300_000, cacheScope: 'private' },\n 'resources/list': { ttlMs: 300_000, cacheScope: 'private' },\n 'resources/templates/list': { ttlMs: 300_000, cacheScope: 'private' },\n 'resources/read': { ttlMs: 60_000, cacheScope: 'private' },\n },\n })\n const httpExecutor = new HttpMcpToolExecutor(baseUrl, requiredApiKey, { localNetworkAccess })\n registerPaaExtractorMcpTools(server, httpExecutor, {\n deploymentProfile,\n transportProfile: 'stdio',\n baseUrl,\n localNetworkAccess,\n taskHandleSecret: requiredApiKey,\n })\n registerSerpIntelligenceCaptureTools(server, httpExecutor, {\n exposeDevelopmentDiagnostics: deploymentProfile === 'development' || deploymentProfile === 'test',\n })\n registerBrowserAgentMcpTools(server, { baseUrl, apiKey: requiredApiKey, consoleBaseUrl })\n registerScheduledResultsMcpTools(server, new ScheduledResultsMcpExecutor(baseUrl, requiredApiKey))\n registerMemoryMcpTools(server, new HostedMemoryMcpToolExecutor(baseUrl, requiredApiKey))\n return server\n}\n\nserveStdio(() => buildStdioServer(), {\n legacy: 'serve',\n onerror(error) {\n process.stderr.write(`${error.message}\\n`)\n },\n})\n","import type { CallToolResult } from \"@modelcontextprotocol/server\";\nimport { MEMORY_TOOL_REGISTRY } from './memory-tool-registry.js'\n\nconst publicToolNameByUpstreamName = new Map(\n MEMORY_TOOL_REGISTRY.map(schema => [schema.upstreamName, schema.id]),\n)\n\ninterface JsonRpcEnvelope {\n result?: CallToolResult\n error?: { code?: number; message?: string }\n}\n\nfunction parseJsonRpcEnvelope(text: string): JsonRpcEnvelope {\n const payloads = text\n .split('\\n')\n .filter(line => line.startsWith('data:'))\n .map(line => line.slice(5).trim())\n .filter(Boolean)\n\n if (!payloads.length) return JSON.parse(text) as JsonRpcEnvelope\n\n for (const payload of payloads) {\n const parsed = JSON.parse(payload) as JsonRpcEnvelope\n if (parsed.result || parsed.error) return parsed\n }\n throw new Error('hosted MCP returned no JSON-RPC result')\n}\n\nfunction errorResult(message: string): CallToolResult {\n return {\n content: [{ type: 'text', text: JSON.stringify({ ok: false, error: message }) }],\n isError: true,\n }\n}\n\n/**\n * Stdio-only bridge for memory tools. It deliberately calls the hosted aggregate\n * MCP surface so local clients execute the same registered tool implementation\n * and output contract as https://mcpscraper.dev/mcp.\n */\nexport class HostedMemoryMcpToolExecutor {\n private readonly baseUrl: string\n private readonly apiKey: string\n\n constructor(baseUrl: string, apiKey: string) {\n this.baseUrl = baseUrl.replace(/\\/$/, '')\n this.apiKey = apiKey\n }\n\n async callMemoryTool(upstreamName: string, args: Record<string, unknown>): Promise<CallToolResult> {\n const publicToolName = publicToolNameByUpstreamName.get(upstreamName)\n if (!publicToolName) return errorResult(`unknown memory tool: ${upstreamName}`)\n\n try {\n const res = await fetch(`${this.baseUrl}/mcp`, {\n method: 'POST',\n headers: {\n 'content-type': 'application/json',\n accept: 'application/json, text/event-stream',\n 'x-api-key': this.apiKey,\n },\n body: JSON.stringify({\n jsonrpc: '2.0',\n id: `memory:${publicToolName}`,\n method: 'tools/call',\n params: { name: publicToolName, arguments: args },\n }),\n })\n const text = await res.text()\n if (!res.ok) return errorResult(`hosted memory ${publicToolName} failed (HTTP ${res.status})`)\n\n const envelope = parseJsonRpcEnvelope(text)\n if (envelope.error) {\n return errorResult(envelope.error.message ?? `hosted memory ${publicToolName} failed`)\n }\n return envelope.result ?? errorResult(`hosted memory ${publicToolName} returned no result`)\n } catch (err) {\n return errorResult(err instanceof Error ? err.message : `hosted memory ${publicToolName} call failed`)\n }\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,kBAAkB;AAC3B,SAAS,iBAAiB;;;ACF1B,IAAM,+BAA+B,IAAI;AAAA,EACvC,qBAAqB,IAAI,YAAU,CAAC,OAAO,cAAc,OAAO,EAAE,CAAC;AACrE;AAOA,SAAS,qBAAqB,MAA+B;AAC3D,QAAM,WAAW,KACd,MAAM,IAAI,EACV,OAAO,UAAQ,KAAK,WAAW,OAAO,CAAC,EACvC,IAAI,UAAQ,KAAK,MAAM,CAAC,EAAE,KAAK,CAAC,EAChC,OAAO,OAAO;AAEjB,MAAI,CAAC,SAAS,OAAQ,QAAO,KAAK,MAAM,IAAI;AAE5C,aAAW,WAAW,UAAU;AAC9B,UAAM,SAAS,KAAK,MAAM,OAAO;AACjC,QAAI,OAAO,UAAU,OAAO,MAAO,QAAO;AAAA,EAC5C;AACA,QAAM,IAAI,MAAM,wCAAwC;AAC1D;AAEA,SAAS,YAAY,SAAiC;AACpD,SAAO;AAAA,IACL,SAAS,CAAC,EAAE,MAAM,QAAQ,MAAM,KAAK,UAAU,EAAE,IAAI,OAAO,OAAO,QAAQ,CAAC,EAAE,CAAC;AAAA,IAC/E,SAAS;AAAA,EACX;AACF;AAOO,IAAM,8BAAN,MAAkC;AAAA,EACtB;AAAA,EACA;AAAA,EAEjB,YAAYA,UAAiBC,SAAgB;AAC3C,SAAK,UAAUD,SAAQ,QAAQ,OAAO,EAAE;AACxC,SAAK,SAASC;AAAA,EAChB;AAAA,EAEA,MAAM,eAAe,cAAsB,MAAwD;AACjG,UAAM,iBAAiB,6BAA6B,IAAI,YAAY;AACpE,QAAI,CAAC,eAAgB,QAAO,YAAY,wBAAwB,YAAY,EAAE;AAE9E,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,GAAG,KAAK,OAAO,QAAQ;AAAA,QAC7C,QAAQ;AAAA,QACR,SAAS;AAAA,UACP,gBAAgB;AAAA,UAChB,QAAQ;AAAA,UACR,aAAa,KAAK;AAAA,QACpB;AAAA,QACA,MAAM,KAAK,UAAU;AAAA,UACnB,SAAS;AAAA,UACT,IAAI,UAAU,cAAc;AAAA,UAC5B,QAAQ;AAAA,UACR,QAAQ,EAAE,MAAM,gBAAgB,WAAW,KAAK;AAAA,QAClD,CAAC;AAAA,MACH,CAAC;AACD,YAAM,OAAO,MAAM,IAAI,KAAK;AAC5B,UAAI,CAAC,IAAI,GAAI,QAAO,YAAY,iBAAiB,cAAc,iBAAiB,IAAI,MAAM,GAAG;AAE7F,YAAM,WAAW,qBAAqB,IAAI;AAC1C,UAAI,SAAS,OAAO;AAClB,eAAO,YAAY,SAAS,MAAM,WAAW,iBAAiB,cAAc,SAAS;AAAA,MACvF;AACA,aAAO,SAAS,UAAU,YAAY,iBAAiB,cAAc,qBAAqB;AAAA,IAC5F,SAAS,KAAK;AACZ,aAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,iBAAiB,cAAc,cAAc;AAAA,IACvG;AAAA,EACF;AACF;;;AD5DA,IAAM,aACJ,QAAQ,KAAK,SAAS,SAAS,KAC/B,QAAQ,IAAI,4BAA4B;AAC1C,IAAM,sBAAsB,QAAQ,QAAQ,MAAM,SAAS,QAAQ,OAAO,KAAK;AAC/E,IAAM,YAAY,QAAQ,KAAK,SAAS,QAAQ,KAAK,QAAQ,KAAK,SAAS,IAAI;AAE/E,IAAI,CAAC,eAAe,uBAAuB,YAAY;AACrD,QAAM,UACJ,QAAQ,KAAK,SAAS,YAAY,KAClC,QAAQ,IAAI,aAAa,UACzB,QAAQ,IAAI,gBAAgB,OAC5B,CAAC,QAAQ,OAAO;AAElB,UAAQ,OAAO,MAAM,sBAAsB;AAAA,IACzC,SAAS;AAAA,IACT,OAAO,CAAC;AAAA,IACR,kBAAkB,QAAQ,QAAQ,IAAI,qBAAqB,KAAK,CAAC;AAAA,EACnE,CAAC,CAAC;AACF,UAAQ,KAAK,CAAC;AAChB;AAEA,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAER;AAAA,EACF;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AACA,IAAM,iBAAiB;AAEvB,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,oBAAoB,yBAAyB;AACnD,IAAM,qBAAqB,0BAA0B;AAAA,EACnD;AAAA,EACA,kBAAkB;AAAA,EAClB;AAAA,EACA,mBAAmB,QAAQ,IAAI,sCAAsC;AACvE,CAAC;AACD,IAAM,iBAAiB,QAAQ,IAAI,2BAA2B,KAAK,KAAK;AACxE,SAAS,mBAA8B;AACrC,QAAM,SAAS,IAAI,UAAU,EAAE,MAAM,eAAe,SAAS,gBAAgB,GAAG;AAAA,IAC9E,cAAc;AAAA,IACd,YAAY;AAAA,MACV,mBAAmB,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MAC3D,cAAc,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MACtD,kBAAkB,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MAC1D,4BAA4B,EAAE,OAAO,KAAS,YAAY,UAAU;AAAA,MACpE,kBAAkB,EAAE,OAAO,KAAQ,YAAY,UAAU;AAAA,IAC3D;AAAA,EACF,CAAC;AACD,QAAM,eAAe,IAAI,oBAAoB,SAAS,gBAAgB,EAAE,mBAAmB,CAAC;AAC5F,+BAA6B,QAAQ,cAAc;AAAA,IACjD;AAAA,IACA,kBAAkB;AAAA,IAClB;AAAA,IACA;AAAA,IACA,kBAAkB;AAAA,EACpB,CAAC;AACD,uCAAqC,QAAQ,cAAc;AAAA,IACzD,8BAA8B,sBAAsB,iBAAiB,sBAAsB;AAAA,EAC7F,CAAC;AACD,+BAA6B,QAAQ,EAAE,SAAS,QAAQ,gBAAgB,eAAe,CAAC;AACxF,mCAAiC,QAAQ,IAAI,4BAA4B,SAAS,cAAc,CAAC;AACjG,yBAAuB,QAAQ,IAAI,4BAA4B,SAAS,cAAc,CAAC;AACvF,SAAO;AACT;AAEA,WAAW,MAAM,iBAAiB,GAAG;AAAA,EACnC,QAAQ;AAAA,EACR,QAAQ,OAAO;AACb,YAAQ,OAAO,MAAM,GAAG,MAAM,OAAO;AAAA,CAAI;AAAA,EAC3C;AACF,CAAC;","names":["baseUrl","apiKey"]}
|
package/dist/bin/paa-harvest.js
CHANGED
|
@@ -1,12 +1,11 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
harvest
|
|
4
|
-
} from "../chunk-
|
|
4
|
+
} from "../chunk-H7LQ6PTI.js";
|
|
5
5
|
import {
|
|
6
6
|
browserServiceApiKey
|
|
7
7
|
} from "../chunk-DAS7L4SF.js";
|
|
8
|
-
import "../chunk-
|
|
9
|
-
import "../chunk-CB5C3BPB.js";
|
|
8
|
+
import "../chunk-3ZUBQQPQ.js";
|
|
10
9
|
import "../chunk-5UN33CGU.js";
|
|
11
10
|
import "../chunk-FCFQ634B.js";
|
|
12
11
|
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../../src/cli.ts","../../bin/paa-harvest.ts"],"sourcesContent":["import { Command } from 'commander'\nimport { browserServiceApiKey } from './lib/browser-service-env.js'\nimport { harvest } from './harvest.js'\n\nconst program = new Command()\n\nprogram\n .name('paa-harvest')\n .description('Recursively extract Google People Also Ask questions')\n .requiredOption('-q, --query <query>', 'Seed query')\n .option('-l, --location <location>', 'Location name (e.g. \"austin\" or \"Austin,Texas,United States\")')\n .option('--gl <gl>', 'Google country code', 'us')\n .option('--hl <hl>', 'Google language code', 'en')\n .option('-d, --depth <depth>', 'BFS depth (1-30)', '3')\n .option('-m, --max-questions <n>', 'Max questions to harvest', '100')\n .option('-o, --output <dir>', 'Output directory', './paa-output')\n .option('-f, --format <format>', 'Output format: json, csv, or both', 'both')\n .option('--headless', 'Run browser in headless mode', false)\n .option('--profile <dir>', 'Persistent browser profile directory')\n .option('--proxy <url>', 'Proxy server URL')\n .option('--kernel-api-key <key>', 'Browser service API key (or set BROWSER_SERVICE_API_KEY env var)')\n .action(async (opts) => {\n try {\n const result = await harvest({\n query: opts.query,\n location: opts.location,\n gl: opts.gl,\n hl: opts.hl,\n depth: parseInt(opts.depth, 10),\n maxQuestions: parseInt(opts.maxQuestions, 10),\n outputDir: opts.output,\n format: opts.format,\n headless: opts.headless,\n profileDir: opts.profile,\n proxy: opts.proxy,\n kernelApiKey: opts.kernelApiKey ?? browserServiceApiKey(),\n })\n console.log(JSON.stringify({ totalQuestions: result.totalQuestions, outputDir: result.stats.seed }))\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err))\n process.exit(1)\n }\n })\n\nexport async function runCli(): Promise<void> {\n await program.parseAsync()\n}\n","#!/usr/bin/env node\nimport { runCli } from '../src/cli.js'\nrunCli()\n"],"mappings":"
|
|
1
|
+
{"version":3,"sources":["../../src/cli.ts","../../bin/paa-harvest.ts"],"sourcesContent":["import { Command } from 'commander'\nimport { browserServiceApiKey } from './lib/browser-service-env.js'\nimport { harvest } from './harvest.js'\n\nconst program = new Command()\n\nprogram\n .name('paa-harvest')\n .description('Recursively extract Google People Also Ask questions')\n .requiredOption('-q, --query <query>', 'Seed query')\n .option('-l, --location <location>', 'Location name (e.g. \"austin\" or \"Austin,Texas,United States\")')\n .option('--gl <gl>', 'Google country code', 'us')\n .option('--hl <hl>', 'Google language code', 'en')\n .option('-d, --depth <depth>', 'BFS depth (1-30)', '3')\n .option('-m, --max-questions <n>', 'Max questions to harvest', '100')\n .option('-o, --output <dir>', 'Output directory', './paa-output')\n .option('-f, --format <format>', 'Output format: json, csv, or both', 'both')\n .option('--headless', 'Run browser in headless mode', false)\n .option('--profile <dir>', 'Persistent browser profile directory')\n .option('--proxy <url>', 'Proxy server URL')\n .option('--kernel-api-key <key>', 'Browser service API key (or set BROWSER_SERVICE_API_KEY env var)')\n .action(async (opts) => {\n try {\n const result = await harvest({\n query: opts.query,\n location: opts.location,\n gl: opts.gl,\n hl: opts.hl,\n depth: parseInt(opts.depth, 10),\n maxQuestions: parseInt(opts.maxQuestions, 10),\n outputDir: opts.output,\n format: opts.format,\n headless: opts.headless,\n profileDir: opts.profile,\n proxy: opts.proxy,\n kernelApiKey: opts.kernelApiKey ?? browserServiceApiKey(),\n })\n console.log(JSON.stringify({ totalQuestions: result.totalQuestions, outputDir: result.stats.seed }))\n } catch (err) {\n console.error(err instanceof Error ? err.message : String(err))\n process.exit(1)\n }\n })\n\nexport async function runCli(): Promise<void> {\n await program.parseAsync()\n}\n","#!/usr/bin/env node\nimport { runCli } from '../src/cli.js'\nrunCli()\n"],"mappings":";;;;;;;;;;;;AAAA,SAAS,eAAe;AAIxB,IAAM,UAAU,IAAI,QAAQ;AAE5B,QACG,KAAK,aAAa,EAClB,YAAY,sDAAsD,EAClE,eAAe,uBAAuB,YAAY,EAClD,OAAO,6BAA6B,+DAA+D,EACnG,OAAO,aAAa,uBAAuB,IAAI,EAC/C,OAAO,aAAa,wBAAwB,IAAI,EAChD,OAAO,uBAAuB,oBAAoB,GAAG,EACrD,OAAO,2BAA2B,4BAA4B,KAAK,EACnE,OAAO,sBAAsB,oBAAoB,cAAc,EAC/D,OAAO,yBAAyB,qCAAqC,MAAM,EAC3E,OAAO,cAAc,gCAAgC,KAAK,EAC1D,OAAO,mBAAmB,sCAAsC,EAChE,OAAO,iBAAiB,kBAAkB,EAC1C,OAAO,0BAA0B,kEAAkE,EACnG,OAAO,OAAO,SAAS;AACtB,MAAI;AACF,UAAM,SAAS,MAAM,QAAQ;AAAA,MAC3B,OAAO,KAAK;AAAA,MACZ,UAAU,KAAK;AAAA,MACf,IAAI,KAAK;AAAA,MACT,IAAI,KAAK;AAAA,MACT,OAAO,SAAS,KAAK,OAAO,EAAE;AAAA,MAC9B,cAAc,SAAS,KAAK,cAAc,EAAE;AAAA,MAC5C,WAAW,KAAK;AAAA,MAChB,QAAQ,KAAK;AAAA,MACb,UAAU,KAAK;AAAA,MACf,YAAY,KAAK;AAAA,MACjB,OAAO,KAAK;AAAA,MACZ,cAAc,KAAK,gBAAgB,qBAAqB;AAAA,IAC1D,CAAC;AACD,YAAQ,IAAI,KAAK,UAAU,EAAE,gBAAgB,OAAO,gBAAgB,WAAW,OAAO,MAAM,KAAK,CAAC,CAAC;AAAA,EACrG,SAAS,KAAK;AACZ,YAAQ,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAC9D,YAAQ,KAAK,CAAC;AAAA,EAChB;AACF,CAAC;AAEH,eAAsB,SAAwB;AAC5C,QAAM,QAAQ,WAAW;AAC3B;;;AC5CA,OAAO;","names":[]}
|
|
@@ -119,6 +119,26 @@ var RawMapsAboutAttributeSchema = z.object({
|
|
|
119
119
|
attribute: z.string()
|
|
120
120
|
});
|
|
121
121
|
|
|
122
|
+
// src/output/memory-library-sink.ts
|
|
123
|
+
async function postToMemoryLibrary(args) {
|
|
124
|
+
const url = process.env.MCP_MEMORY_INGEST_URL;
|
|
125
|
+
const key = process.env.MCP_MEMORY_KEY;
|
|
126
|
+
if (!url || !key) return;
|
|
127
|
+
try {
|
|
128
|
+
const res = await fetch(url, {
|
|
129
|
+
method: "POST",
|
|
130
|
+
headers: { "content-type": "application/json" },
|
|
131
|
+
body: JSON.stringify({ data: { apiKey: key, title: args.title, content: args.content, source: args.source } })
|
|
132
|
+
});
|
|
133
|
+
const body = await res.json().catch(() => null);
|
|
134
|
+
if (!res.ok || body?.result?.error) {
|
|
135
|
+
console.warn("[memory-library-sink] ingest not accepted:", res.status, body?.result?.message ?? "");
|
|
136
|
+
}
|
|
137
|
+
} catch (err) {
|
|
138
|
+
console.warn("[memory-library-sink] ingest failed:", err?.message);
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
122
142
|
export {
|
|
123
143
|
DEFAULT_PROXY_MODE,
|
|
124
144
|
DEFAULT_MAPS_PROXY_MODE,
|
|
@@ -130,6 +150,7 @@ export {
|
|
|
130
150
|
RawMapsOverviewSchema,
|
|
131
151
|
RawMapsHoursRowSchema,
|
|
132
152
|
RawMapsReviewStatsSchema,
|
|
133
|
-
RawMapsAboutAttributeSchema
|
|
153
|
+
RawMapsAboutAttributeSchema,
|
|
154
|
+
postToMemoryLibrary
|
|
134
155
|
};
|
|
135
|
-
//# sourceMappingURL=chunk-
|
|
156
|
+
//# sourceMappingURL=chunk-3ZUBQQPQ.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/schemas.ts","../src/output/memory-library-sink.ts"],"sourcesContent":["import { z } from 'zod'\n\nexport const DEFAULT_PROXY_MODE = 'none' as const\nexport const DEFAULT_MAPS_PROXY_MODE = 'none' as const\n\nexport const HarvestOptionsSchema = z.object({\n query: z.string().min(1),\n location: z.string().optional(),\n gl: z.string().length(2).default('us'),\n hl: z.string().length(2).default('en'),\n device: z.enum(['desktop', 'mobile']).default('desktop'),\n proxyMode: z.enum(['location', 'configured', 'none']).default(DEFAULT_PROXY_MODE),\n proxyZip: z.string().regex(/^\\d{5}$/).optional(),\n keepDefaultProxy: z.boolean().optional(),\n requireUsEgress: z.boolean().optional(),\n maxAttempts: z.number().int().min(1).max(12).optional(),\n debug: z.boolean().default(false),\n depth: z.number().int().min(1).max(30).default(3),\n maxQuestions: z.number().int().min(1).max(1000).default(100),\n headless: z.boolean().default(false),\n profileDir: z.string().optional(),\n proxy: z.string().url().optional(),\n kernelApiKey: z.string().optional(),\n kernelProxyId: z.string().optional(),\n kernelProxyResolution: z.unknown().optional(),\n outputDir: z.string().default('./paa-output'),\n format: z.enum(['json', 'csv', 'both']).default('both'),\n serpOnly: z.boolean().default(false),\n pages: z.number().int().min(1).max(2).default(1),\n recency: z.enum(['day', 'week', 'month', 'year']).optional(),\n softDeadlineMs: z.number().optional(),\n})\n\nexport function recencyToTbs(recency: 'day' | 'week' | 'month' | 'year' | undefined): string | undefined {\n switch (recency) {\n case 'day': return 'qdr:d'\n case 'week': return 'qdr:w'\n case 'month': return 'qdr:m'\n case 'year': return 'qdr:y'\n default: return undefined\n }\n}\n\nexport const MapsPlaceOptionsSchema = z.object({\n businessName: z.string().min(1),\n location: z.string().min(1),\n gl: z.string().length(2).default('us'),\n hl: z.string().length(2).default('en'),\n includeReviews: z.boolean().default(false),\n maxReviews: z.number().int().min(1).max(500).default(50),\n includeServices: z.boolean().default(false),\n kernelApiKey: z.string().optional(),\n kernelProxyId: z.string().optional(),\n headless: z.boolean().default(true),\n})\n\nexport const MapsSearchOptionsSchema = z.object({\n query: z.string().min(1),\n location: z.string().optional(),\n gl: z.string().length(2).default('us'),\n hl: z.string().length(2).default('en'),\n maxResults: z.number().int().min(1).max(50).default(10),\n includeServices: z.boolean().default(false),\n proxyMode: z.enum(['location', 'configured', 'none']).default(DEFAULT_MAPS_PROXY_MODE),\n proxyZip: z.string().regex(/^\\d{5}$/).optional(),\n serpRedirect: z.boolean().optional(),\n forceDirectEgress: z.boolean().optional(),\n debug: z.boolean().default(false),\n kernelApiKey: z.string().optional(),\n kernelProxyId: z.string().optional(),\n kernelProxyResolution: z.unknown().optional(),\n headless: z.boolean().default(true),\n})\n\nexport const RawPAAItemSchema = z.object({\n question: z.string().min(1),\n answer: z.string().optional(),\n sourceTitle: z.string().optional(),\n sourceSite: z.string().optional(),\n sourceCite: z.string().optional(),\n})\n\nexport const RawMapsOverviewSchema = z.object({\n name: z.string().nullable(),\n rating: z.string().nullable(),\n reviewCount: z.string().nullable(),\n category: z.string().nullable(),\n address: z.string().nullable(),\n hoursSummary: z.string().nullable(),\n phone: z.string().nullable(),\n phoneDisplay: z.string().nullable(),\n website: z.string().nullable(),\n plusCode: z.string().nullable(),\n bookingUrl: z.string().nullable(),\n})\n\nexport const RawMapsHoursRowSchema = z.object({\n day: z.string(),\n hours: z.string(),\n})\n\nexport const RawMapsReviewStatsSchema = z.object({\n reviewHistogram: z.array(z.object({\n stars: z.number(),\n count: z.string(),\n })),\n reviewTopics: z.array(z.object({\n label: z.string(),\n count: z.string(),\n })),\n})\n\nexport const RawMapsReviewCardSchema = z.object({\n reviewId: z.string(),\n author: z.string().nullable(),\n stars: z.string().nullable(),\n date: z.string().nullable(),\n text: z.string().nullable(),\n ownerResponse: z.string().nullable(),\n})\n\nexport const RawMapsAboutAttributeSchema = z.object({\n section: z.string(),\n attribute: z.string(),\n})\n","export async function postToMemoryLibrary(args: { title: string; content: string; source: string }): Promise<void> {\n const url = process.env.MCP_MEMORY_INGEST_URL\n const key = process.env.MCP_MEMORY_KEY\n if (!url || !key) return\n try {\n const res = await fetch(url, {\n method: 'POST',\n headers: { 'content-type': 'application/json' },\n body: JSON.stringify({ data: { apiKey: key, title: args.title, content: args.content, source: args.source } }),\n })\n const body = (await res.json().catch(() => null)) as { result?: { error?: boolean; message?: string } } | null\n if (!res.ok || body?.result?.error) {\n console.warn('[memory-library-sink] ingest not accepted:', res.status, body?.result?.message ?? '')\n }\n } catch (err) {\n console.warn('[memory-library-sink] ingest failed:', (err as Error)?.message)\n }\n}\n\nexport function memoryLibrarySinkEnabled(): boolean {\n return Boolean(process.env.MCP_MEMORY_INGEST_URL && process.env.MCP_MEMORY_KEY)\n}\n"],"mappings":";AAAA,SAAS,SAAS;AAEX,IAAM,qBAAqB;AAC3B,IAAM,0BAA0B;AAEhC,IAAM,uBAAuB,EAAE,OAAO;AAAA,EAC3C,OAAc,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC9B,UAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,IAAc,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAC/C,IAAc,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAC/C,QAAc,EAAE,KAAK,CAAC,WAAW,QAAQ,CAAC,EAAE,QAAQ,SAAS;AAAA,EAC7D,WAAc,EAAE,KAAK,CAAC,YAAY,cAAc,MAAM,CAAC,EAAE,QAAQ,kBAAkB;AAAA,EACnF,UAAc,EAAE,OAAO,EAAE,MAAM,SAAS,EAAE,SAAS;AAAA,EACnD,kBAAkB,EAAE,QAAQ,EAAE,SAAS;AAAA,EACvC,iBAAiB,EAAE,QAAQ,EAAE,SAAS;AAAA,EACtC,aAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS;AAAA,EACvD,OAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACvC,OAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,CAAC;AAAA,EACvD,cAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,GAAI,EAAE,QAAQ,GAAG;AAAA,EAC3D,UAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACvC,YAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,OAAc,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS;AAAA,EACxC,cAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,eAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,uBAAuB,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5C,WAAc,EAAE,OAAO,EAAE,QAAQ,cAAc;AAAA,EAC/C,QAAc,EAAE,KAAK,CAAC,QAAQ,OAAO,MAAM,CAAC,EAAE,QAAQ,MAAM;AAAA,EAC5D,UAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACvC,OAAc,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,EAAE,QAAQ,CAAC;AAAA,EACtD,SAAc,EAAE,KAAK,CAAC,OAAO,QAAQ,SAAS,MAAM,CAAC,EAAE,SAAS;AAAA,EAChE,gBAAgB,EAAE,OAAO,EAAE,SAAS;AACtC,CAAC;AAEM,SAAS,aAAa,SAA4E;AACvG,UAAQ,SAAS;AAAA,IACf,KAAK;AAAO,aAAO;AAAA,IACnB,KAAK;AAAQ,aAAO;AAAA,IACpB,KAAK;AAAS,aAAO;AAAA,IACrB,KAAK;AAAQ,aAAO;AAAA,IACpB;AAAS,aAAO;AAAA,EAClB;AACF;AAEO,IAAM,yBAAyB,EAAE,OAAO;AAAA,EAC7C,cAAe,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC/B,UAAe,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC/B,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,gBAAgB,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACzC,YAAe,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,GAAG,EAAE,QAAQ,EAAE;AAAA,EAC1D,iBAAiB,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EAC1C,cAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,eAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,UAAe,EAAE,QAAQ,EAAE,QAAQ,IAAI;AACzC,CAAC;AAEM,IAAM,0BAA0B,EAAE,OAAO;AAAA,EAC9C,OAAe,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC/B,UAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,IAAe,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,QAAQ,IAAI;AAAA,EAChD,YAAe,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE;AAAA,EACzD,iBAAiB,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EAC1C,WAAe,EAAE,KAAK,CAAC,YAAY,cAAc,MAAM,CAAC,EAAE,QAAQ,uBAAuB;AAAA,EACzF,UAAe,EAAE,OAAO,EAAE,MAAM,SAAS,EAAE,SAAS;AAAA,EACpD,cAAe,EAAE,QAAQ,EAAE,SAAS;AAAA,EACpC,mBAAmB,EAAE,QAAQ,EAAE,SAAS;AAAA,EACxC,OAAe,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACxC,cAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,eAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,uBAAuB,EAAE,QAAQ,EAAE,SAAS;AAAA,EAC5C,UAAe,EAAE,QAAQ,EAAE,QAAQ,IAAI;AACzC,CAAC;AAEM,IAAM,mBAAmB,EAAE,OAAO;AAAA,EACvC,UAAa,EAAE,OAAO,EAAE,IAAI,CAAC;AAAA,EAC7B,QAAa,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,aAAa,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,YAAa,EAAE,OAAO,EAAE,SAAS;AAAA,EACjC,YAAa,EAAE,OAAO,EAAE,SAAS;AACnC,CAAC;AAEM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,MAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,QAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,aAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,UAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,SAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,cAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,OAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,cAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,SAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,UAAc,EAAE,OAAO,EAAE,SAAS;AAAA,EAClC,YAAc,EAAE,OAAO,EAAE,SAAS;AACpC,CAAC;AAEM,IAAM,wBAAwB,EAAE,OAAO;AAAA,EAC5C,KAAO,EAAE,OAAO;AAAA,EAChB,OAAO,EAAE,OAAO;AAClB,CAAC;AAEM,IAAM,2BAA2B,EAAE,OAAO;AAAA,EAC/C,iBAAiB,EAAE,MAAM,EAAE,OAAO;AAAA,IAChC,OAAO,EAAE,OAAO;AAAA,IAChB,OAAO,EAAE,OAAO;AAAA,EAClB,CAAC,CAAC;AAAA,EACF,cAAc,EAAE,MAAM,EAAE,OAAO;AAAA,IAC7B,OAAO,EAAE,OAAO;AAAA,IAChB,OAAO,EAAE,OAAO;AAAA,EAClB,CAAC,CAAC;AACJ,CAAC;AAEM,IAAM,0BAA0B,EAAE,OAAO;AAAA,EAC9C,UAAe,EAAE,OAAO;AAAA,EACxB,QAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,OAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,MAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,MAAe,EAAE,OAAO,EAAE,SAAS;AAAA,EACnC,eAAe,EAAE,OAAO,EAAE,SAAS;AACrC,CAAC;AAEM,IAAM,8BAA8B,EAAE,OAAO;AAAA,EAClD,SAAW,EAAE,OAAO;AAAA,EACpB,WAAW,EAAE,OAAO;AACtB,CAAC;;;AC5HD,eAAsB,oBAAoB,MAAyE;AACjH,QAAM,MAAM,QAAQ,IAAI;AACxB,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,OAAO,CAAC,IAAK;AAClB,MAAI;AACF,UAAM,MAAM,MAAM,MAAM,KAAK;AAAA,MAC3B,QAAQ;AAAA,MACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,MAC9C,MAAM,KAAK,UAAU,EAAE,MAAM,EAAE,QAAQ,KAAK,OAAO,KAAK,OAAO,SAAS,KAAK,SAAS,QAAQ,KAAK,OAAO,EAAE,CAAC;AAAA,IAC/G,CAAC;AACD,UAAM,OAAQ,MAAM,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI;AAC/C,QAAI,CAAC,IAAI,MAAM,MAAM,QAAQ,OAAO;AAClC,cAAQ,KAAK,8CAA8C,IAAI,QAAQ,MAAM,QAAQ,WAAW,EAAE;AAAA,IACpG;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ,KAAK,wCAAyC,KAAe,OAAO;AAAA,EAC9E;AACF;","names":[]}
|
|
@@ -6,15 +6,13 @@ import {
|
|
|
6
6
|
recordKernelSession,
|
|
7
7
|
runWithCostContext
|
|
8
8
|
} from "./chunk-DAS7L4SF.js";
|
|
9
|
-
import {
|
|
10
|
-
postToMemoryLibrary
|
|
11
|
-
} from "./chunk-K443GQY5.js";
|
|
12
9
|
import {
|
|
13
10
|
DEFAULT_PROXY_MODE,
|
|
14
11
|
HarvestOptionsSchema,
|
|
15
12
|
RawPAAItemSchema,
|
|
13
|
+
postToMemoryLibrary,
|
|
16
14
|
recencyToTbs
|
|
17
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-3ZUBQQPQ.js";
|
|
18
16
|
import {
|
|
19
17
|
CaptchaError,
|
|
20
18
|
ExtractionError,
|
|
@@ -3449,4 +3447,4 @@ export {
|
|
|
3449
3447
|
resolveKernelProxyId,
|
|
3450
3448
|
harvest
|
|
3451
3449
|
};
|
|
3452
|
-
//# sourceMappingURL=chunk-
|
|
3450
|
+
//# sourceMappingURL=chunk-H7LQ6PTI.js.map
|