mcp-scraper 0.1.6 → 0.1.8

package/dist/bin/mcp-stdio-server.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../bin/mcp-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { buildPaaExtractorMcpServer } from '../src/mcp/paa-mcp-server.js'\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {}\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() ?? process.env.MCP_BASE_URL?.trim() ?? 'https://mcpscraper.dev'\nconst executor = new HttpMcpToolExecutor(baseUrl, apiKey)\nconst server = buildPaaExtractorMcpServer(executor)\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch((err) => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n"],"mappings":";;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,4BAA4B;AAIrC,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAAC;AAAA,EACX;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,WAAW,IAAI,oBAAoB,SAAS,MAAM;AACxD,IAAM,SAAS,2BAA2B,QAAQ;AAClD,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}
+{"version":3,"sources":["../../bin/mcp-stdio-server.ts"],"sourcesContent":["#!/usr/bin/env node\nimport { readFileSync } from 'node:fs'\nimport { homedir } from 'node:os'\nimport { join } from 'node:path'\nimport { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js'\nimport { HttpMcpToolExecutor } from '../src/mcp/http-mcp-tool-executor.js'\nimport { buildPaaExtractorMcpServer } from '../src/mcp/paa-mcp-server.js'\n\nfunction readApiKeyFile(): string | undefined {\n  const explicitPath = process.env.MCP_SCRAPER_KEY_PATH?.trim()\n  const paths = [explicitPath, join(homedir(), '.mcp-scraper-key')].filter(Boolean) as string[]\n  for (const path of paths) {\n    try {\n      const value = readFileSync(path, 'utf8').trim()\n      if (value) return value\n    } catch {}\n  }\n  return undefined\n}\n\nconst apiKey = (\n  process.env.MCP_SCRAPER_API_KEY ??\n  process.env.MCP_SCRAPER_KEY ??\n  process.env.MCP_API_KEY ??\n  readApiKeyFile()\n)?.trim()\nif (!apiKey) {\n  process.stderr.write('MCP_SCRAPER_API_KEY env var or ~/.mcp-scraper-key is required\\n')\n  process.exit(1)\n}\n\nconst baseUrl = process.env.MCP_SCRAPER_BASE_URL?.trim() ?? process.env.MCP_BASE_URL?.trim() ?? 'https://mcpscraper.dev'\nconst executor = new HttpMcpToolExecutor(baseUrl, apiKey)\nconst server = buildPaaExtractorMcpServer(executor)\nconst transport = new StdioServerTransport()\n\nasync function main() {\n  await server.connect(transport)\n}\n\nmain().catch((err) => {\n  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\\n`)\n  process.exit(1)\n})\n"],"mappings":";;;;;;;;AACA,SAAS,oBAAoB;AAC7B,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,4BAA4B;AAIrC,SAAS,iBAAqC;AAC5C,QAAM,eAAe,QAAQ,IAAI,sBAAsB,KAAK;AAC5D,QAAM,QAAQ,CAAC,cAAc,KAAK,QAAQ,GAAG,kBAAkB,CAAC,EAAE,OAAO,OAAO;AAChF,aAAW,QAAQ,OAAO;AACxB,QAAI;AACF,YAAM,QAAQ,aAAa,MAAM,MAAM,EAAE,KAAK;AAC9C,UAAI,MAAO,QAAO;AAAA,IACpB,QAAQ;AAAA,IAAC;AAAA,EACX;AACA,SAAO;AACT;AAEA,IAAM,UACJ,QAAQ,IAAI,uBACZ,QAAQ,IAAI,mBACZ,QAAQ,IAAI,eACZ,eAAe,IACd,KAAK;AACR,IAAI,CAAC,QAAQ;AACX,UAAQ,OAAO,MAAM,iEAAiE;AACtF,UAAQ,KAAK,CAAC;AAChB;AAEA,IAAM,UAAU,QAAQ,IAAI,sBAAsB,KAAK,KAAK,QAAQ,IAAI,cAAc,KAAK,KAAK;AAChG,IAAM,WAAW,IAAI,oBAAoB,SAAS,MAAM;AACxD,IAAM,SAAS,2BAA2B,QAAQ;AAClD,IAAM,YAAY,IAAI,qBAAqB;AAE3C,eAAe,OAAO;AACpB,QAAM,OAAO,QAAQ,SAAS;AAChC;AAEA,KAAK,EAAE,MAAM,CAAC,QAAQ;AACpB,UAAQ,OAAO,MAAM,GAAG,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,CAAC;AAAA,CAAI;AAC5E,UAAQ,KAAK,CAAC;AAChB,CAAC;","names":[]}

package/dist/bin/paa-harvest.cjs

@@ -26,6 +26,16 @@ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__ge
 // src/cli.ts
 var import_commander = require("commander");
 
+// src/lib/browser-service-env.ts
+function browserServiceApiKey() {
+  const value = (process.env.BROWSER_SERVICE_API_KEY ?? process.env.KERNEL_API_KEY)?.trim();
+  return value || void 0;
+}
+function browserServiceProxyId() {
+  const value = (process.env.BROWSER_SERVICE_PROXY_ID ?? process.env.KERNEL_PROXY_ID)?.trim();
+  return value || void 0;
+}
+
 // src/schemas.ts
 var import_zod = require("zod");
 var HarvestOptionsSchema = import_zod.z.object({
@@ -61,6 +71,16 @@ var MapsPlaceOptionsSchema = import_zod.z.object({
   kernelProxyId: import_zod.z.string().optional(),
   headless: import_zod.z.boolean().default(true)
 });
+var MapsSearchOptionsSchema = import_zod.z.object({
+  query: import_zod.z.string().min(1),
+  location: import_zod.z.string().optional(),
+  gl: import_zod.z.string().length(2).default("us"),
+  hl: import_zod.z.string().length(2).default("en"),
+  maxResults: import_zod.z.number().int().min(1).max(50).default(10),
+  kernelApiKey: import_zod.z.string().optional(),
+  kernelProxyId: import_zod.z.string().optional(),
+  headless: import_zod.z.boolean().default(true)
+});
 var RawPAAItemSchema = import_zod.z.object({
   question: import_zod.z.string().min(1),
   answer: import_zod.z.string().optional(),
@@ -924,8 +944,19 @@ function addCandidate(candidates, city, region, example) {
   }
   candidates.set(key, { city: normalizedCity, regionCode, count: 1, examples: [example] });
 }
+function decodeSerpText(text) {
+  try {
+    return decodeURIComponent(text);
+  } catch {
+  }
+  try {
+    return decodeURIComponent(text.replace(/%(?![0-9a-fA-F]{2})/g, "%25"));
+  } catch {
+    return text;
+  }
+}
 function scanText(candidates, text) {
-  const normalized = decodeURIComponent(text).replace(/[+/|_-]+/g, " ");
+  const normalized = decodeSerpText(text).replace(/[+/|_-]+/g, " ");
   for (const match of normalized.matchAll(CITY_STATE_RE)) {
     addCandidate(candidates, match[1] ?? "", match[2] ?? "", normalized.slice(0, 180));
   }
@@ -2593,8 +2624,8 @@ async function harvest(rawOptions) {
   const onAttemptEvent = getAttemptLogSink(rawOptions);
   const requestedProxyMode = raw.proxyMode;
   const proxyMode = requestedProxyMode === "none" ? "none" : requestedProxyMode === "configured" ? "configured" : "location";
-  const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : process.env.KERNEL_API_KEY?.trim();
-  const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : process.env.KERNEL_PROXY_ID?.trim();
+  const kernelApiKey = typeof raw.kernelApiKey === "string" ? raw.kernelApiKey.trim() : browserServiceApiKey();
+  const configuredKernelProxyId = typeof raw.kernelProxyId === "string" ? raw.kernelProxyId.trim() : browserServiceProxyId();
   const proxyOpts = {
     kernelApiKey,
     proxyMode,
@@ -2779,7 +2810,7 @@ async function harvest(rawOptions) {
 
 // src/cli.ts
 var program = new import_commander.Command();
-program.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>", "Seed query").option("-l, --location <location>", 'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>", "Google country code", "us").option("--hl <hl>", "Google language code", "en").option("-d, --depth <depth>", "BFS depth (1-30)", "3").option("-m, --max-questions <n>", "Max questions to harvest", "100").option("-o, --output <dir>", "Output directory", "./paa-output").option("-f, --format <format>", "Output format: json, csv, or both", "both").option("--headless", "Run browser in headless mode", false).option("--profile <dir>", "Persistent browser profile directory").option("--proxy <url>", "Proxy server URL").option("--kernel-api-key <key>", "Kernel.sh API key (or set KERNEL_API_KEY env var)").action(async (opts) => {
+program.name("paa-harvest").description("Recursively extract Google People Also Ask questions").requiredOption("-q, --query <query>", "Seed query").option("-l, --location <location>", 'Location name (e.g. "austin" or "Austin,Texas,United States")').option("--gl <gl>", "Google country code", "us").option("--hl <hl>", "Google language code", "en").option("-d, --depth <depth>", "BFS depth (1-30)", "3").option("-m, --max-questions <n>", "Max questions to harvest", "100").option("-o, --output <dir>", "Output directory", "./paa-output").option("-f, --format <format>", "Output format: json, csv, or both", "both").option("--headless", "Run browser in headless mode", false).option("--profile <dir>", "Persistent browser profile directory").option("--proxy <url>", "Proxy server URL").option("--kernel-api-key <key>", "Browser service API key (or set BROWSER_SERVICE_API_KEY env var)").action(async (opts) => {
   try {
     const result = await harvest({
       query: opts.query,
@@ -2793,7 +2824,7 @@ program.name("paa-harvest").description("Recursively extract Google People Also
       headless: opts.headless,
       profileDir: opts.profile,
       proxy: opts.proxy,
-      kernelApiKey: opts.kernelApiKey ?? process.env.KERNEL_API_KEY
+      kernelApiKey: opts.kernelApiKey ?? browserServiceApiKey()
     });
     console.log(JSON.stringify({ totalQuestions: result.totalQuestions, outputDir: result.stats.seed }));
   } catch (err) {