mcp-scraper 0.40.2 → 0.40.3

package/dist/{site-extract-repository-2SMMFKKL.js → site-extract-repository-6GX72U4N.js}

@@ -26,10 +26,10 @@ import {
   setExtractJobTotal,
   settleExtractJob,
   terminalExtractJobStatus
-} from "./chunk-X2LKCX6H.js";
+} from "./chunk-6J57U6HA.js";
 import "./chunk-SBLGBZZB.js";
-import "./chunk-44HZLHDV.js";
-import "./chunk-GUVKHCKE.js";
+import "./chunk-5UN33CGU.js";
+import "./chunk-IQTT7CAB.js";
 export {
   abandonExtractSettlement,
   claimFailedExtractJobForRefinalize,
@@ -59,4 +59,4 @@ export {
   settleExtractJob,
   terminalExtractJobStatus
 };
-//# sourceMappingURL=site-extract-repository-2SMMFKKL.js.map
+//# sourceMappingURL=site-extract-repository-6GX72U4N.js.map

package/dist/{worker-FXAGFYOE.js → worker-5RWLFSAJ.js}

@@ -1,23 +1,28 @@
 import {
   classifyHarvestProblem,
   createHarvestAttemptRecorder,
+  deliverWebhook,
   harvestProblemResponse,
   serializeHarvestProblem
-} from "./chunk-NVXNEOUQ.js";
+} from "./chunk-CS4HE6QY.js";
 import {
   harvest
-} from "./chunk-7XBBFBYY.js";
+} from "./chunk-JVIW4GK2.js";
+import {
+  sanitizeAttempts,
+  sanitizeHarvestResult
+} from "./chunk-BBI7RGOT.js";
 import {
   browserServiceApiKey,
   runWithCostContext
-} from "./chunk-3PIWJS6Y.js";
+} from "./chunk-EHES33KB.js";
 import "./chunk-K443GQY5.js";
 import "./chunk-CB5C3BPB.js";
 import {
   MC_COSTS,
   serpActualCostMc
 } from "./chunk-SBLGBZZB.js";
-import "./chunk-44HZLHDV.js";
+import "./chunk-5UN33CGU.js";
 import {
   claimPendingJob,
   completeJob,
@@ -25,27 +30,7 @@ import {
   debitMc,
   failJob,
   listHarvestAttempts
-} from "./chunk-GUVKHCKE.js";
-
-// src/api/webhook.ts
-async function deliverWebhook(url, payload, retries = 3) {
-  for (let attempt = 1; attempt <= retries; attempt++) {
-    try {
-      const res = await fetch(url, {
-        method: "POST",
-        headers: { "content-type": "application/json" },
-        body: JSON.stringify(payload),
-        signal: AbortSignal.timeout(1e4)
-      });
-      if (res.ok) return;
-      console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
-    } catch (err) {
-      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
-    }
-    if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
-  }
-  console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
-}
+} from "./chunk-IQTT7CAB.js";
 
 // src/api/worker.ts
 var MAX_CONCURRENT = 2;
@@ -89,9 +74,18 @@ async function processJob(job) {
       else if (diff < 0) await debitMc(job.user_id, -diff, "serp", opts.query ?? job.query);
     }
     if (job.callback_url) {
-      await deliverWebhook(job.callback_url, { job_id: job.id, status: "done", result, attempts });
+      await deliverWebhook(job.callback_url, {
+        job_id: job.id,
+        status: "done",
+        result: sanitizeHarvestResult(result),
+        attempts: sanitizeAttempts(attempts)
+      });
     }
   } catch (err) {
+    console.error("[harvest/worker] failed", {
+      jobId: job.id,
+      error: err instanceof Error ? err.message : String(err)
+    });
     const problem = classifyHarvestProblem(err);
     await failJob(job.id, serializeHarvestProblem(problem));
     const attempts = await listHarvestAttempts(job.id, job.user_id);
@@ -103,7 +97,12 @@ async function processJob(job) {
     } catch {
     }
     if (job.callback_url) {
-      await deliverWebhook(job.callback_url, { job_id: job.id, status: "failed", ...harvestProblemResponse(problem), attempts });
+      await deliverWebhook(job.callback_url, {
+        job_id: job.id,
+        status: "failed",
+        ...harvestProblemResponse(problem),
+        attempts: sanitizeAttempts(attempts)
+      });
     }
   } finally {
     running--;
@@ -139,4 +138,4 @@ export {
   startWorker,
   tickOnce
 };
-//# sourceMappingURL=worker-FXAGFYOE.js.map
+//# sourceMappingURL=worker-5RWLFSAJ.js.map

package/dist/worker-5RWLFSAJ.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/worker.ts"],"sourcesContent":["import { claimPendingJob, completeJob, failJob, creditMc, debitMc, listHarvestAttempts } from './db.js'\nimport { browserServiceApiKey } from '../lib/browser-service-env.js'\nimport { harvest } from '../harvest.js'\nimport { deliverWebhook } from './webhook.js'\nimport type { HarvestOptions } from '../types.js'\nimport { MC_COSTS, serpActualCostMc } from './rates.js'\nimport { classifyHarvestProblem, harvestProblemResponse, serializeHarvestProblem } from './harvest-problems.js'\nimport { createHarvestAttemptRecorder } from './harvest-attempt-events.js'\nimport { runWithCostContext } from './cost-context.js'\nimport { sanitizeAttempts, sanitizeHarvestResult } from './outbound-sanitize.js'\n\nexport type TickResult = {\n  claimed: boolean\n  jobId?: string\n  completed?: boolean\n  durationMs?: number\n}\nexport type DrainBudget = {\n  maxJobs: number\n  deadlineMs: number\n}\n\nconst MAX_CONCURRENT = 2\nlet running = 0\n\nfunction countPaaQuestions(result: unknown): number {\n  if (!result || typeof result !== 'object') return 0\n  const value = result as { totalQuestions?: unknown; flat?: unknown }\n  if (typeof value.totalQuestions === 'number') return value.totalQuestions\n  return Array.isArray(value.flat) ? value.flat.length : 0\n}\n\nfunction paaCostForQuestionCount(questionCount: number): number {\n  return MC_COSTS.paa_base + Math.max(1, questionCount) * MC_COSTS.paa\n}\n\nasync function processJob(job: Awaited<ReturnType<typeof claimPendingJob>> & object) {\n  running++\n  try {\n    const opts = typeof job.options === 'string' ? JSON.parse(job.options) as Partial<HarvestOptions> & { billingHoldMc?: number } : job.options as Partial<HarvestOptions> & { billingHoldMc?: number }\n    const headlessSentOut: { value: boolean | null } = { value: null }\n    const result = await runWithCostContext(\n      { op: opts.serpOnly ? 'serp' : 'paa', userId: Number(job.user_id), headlessSentOut },\n      () => harvest({\n        ...opts,\n        kernelApiKey: browserServiceApiKey(),\n        headless: true,\n        format: 'json',\n        outputDir: '/tmp/paa-output-api',\n        onAttemptEvent: createHarvestAttemptRecorder(job.id, job.user_id),\n      }),\n    )\n    await completeJob(job.id, result)\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    if (!opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = paaCostForQuestionCount(countPaaQuestions(result))\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'paa_refund', 'overestimate refund')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'paa', opts.query ?? job.query)\n    } else if (opts.serpOnly && typeof opts.billingHoldMc === 'number') {\n      const actualCost = serpActualCostMc(headlessSentOut.value)\n      const diff = opts.billingHoldMc - actualCost\n      if (diff > 0) await creditMc(job.user_id, diff, 'serp_refund', 'headless-mode pricing settle')\n      else if (diff < 0) await debitMc(job.user_id, -diff, 'serp', opts.query ?? job.query)\n    }\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, {\n        job_id: job.id,\n        status: 'done',\n        result: sanitizeHarvestResult(result as unknown as Record<string, unknown>),\n        attempts: sanitizeAttempts(attempts),\n      })\n    }\n  } catch (err) {\n    console.error('[harvest/worker] failed', {\n      jobId: job.id,\n      error: err instanceof Error ? err.message : String(err),\n    })\n    const problem = classifyHarvestProblem(err)\n    await failJob(job.id, serializeHarvestProblem(problem))\n    const attempts = await listHarvestAttempts(job.id, job.user_id)\n    try {\n      const opts = typeof job.options === 'string' ? JSON.parse(job.options) as { billingHoldMc?: number } : job.options as { billingHoldMc?: number }\n      if (typeof opts.billingHoldMc === 'number' && opts.billingHoldMc > 0) {\n        await creditMc(job.user_id, opts.billingHoldMc, 'refund', 'failed call')\n      }\n    } catch {}\n    if (job.callback_url) {\n      await deliverWebhook(job.callback_url, {\n        job_id: job.id,\n        status: 'failed',\n        ...harvestProblemResponse(problem),\n        attempts: sanitizeAttempts(attempts),\n      })\n    }\n  } finally {\n    running--\n  }\n}\n\nexport async function tickOnce(): Promise<TickResult> {\n  const job = await claimPendingJob()\n  if (!job) return { claimed: false }\n  const startedAt = Date.now()\n  await processJob(job as NonNullable<typeof job>)\n  return { claimed: true, jobId: (job as { id: string }).id, completed: true, durationMs: Date.now() - startedAt }\n}\n\nexport async function drainQueue(budget: DrainBudget): Promise<TickResult[]> {\n  const results: TickResult[] = []\n  for (let i = 0; i < budget.maxJobs; i++) {\n    if (Date.now() >= budget.deadlineMs) break\n    const r = await tickOnce()\n    results.push(r)\n    if (!r.claimed) break\n  }\n  return results\n}\n\nexport function startWorker(): void {\n  setInterval(async () => {\n    if (running >= MAX_CONCURRENT) return\n    const job = await claimPendingJob()\n    if (job) void processJob(job as NonNullable<typeof job>)\n  }, 2000)\n  console.log(`[worker] started — polling every 2s, max ${MAX_CONCURRENT} concurrent`)\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBA,IAAM,iBAAiB;AACvB,IAAI,UAAU;AAEd,SAAS,kBAAkB,QAAyB;AAClD,MAAI,CAAC,UAAU,OAAO,WAAW,SAAU,QAAO;AAClD,QAAM,QAAQ;AACd,MAAI,OAAO,MAAM,mBAAmB,SAAU,QAAO,MAAM;AAC3D,SAAO,MAAM,QAAQ,MAAM,IAAI,IAAI,MAAM,KAAK,SAAS;AACzD;AAEA,SAAS,wBAAwB,eAA+B;AAC9D,SAAO,SAAS,WAAW,KAAK,IAAI,GAAG,aAAa,IAAI,SAAS;AACnE;AAEA,eAAe,WAAW,KAA2D;AACnF;AACA,MAAI;AACF,UAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAA4D,IAAI;AACrI,UAAM,kBAA6C,EAAE,OAAO,KAAK;AACjE,UAAM,SAAS,MAAM;AAAA,MACnB,EAAE,IAAI,KAAK,WAAW,SAAS,OAAO,QAAQ,OAAO,IAAI,OAAO,GAAG,gBAAgB;AAAA,MACnF,MAAM,QAAQ;AAAA,QACZ,GAAG;AAAA,QACH,cAAc,qBAAqB;AAAA,QACnC,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,WAAW;AAAA,QACX,gBAAgB,6BAA6B,IAAI,IAAI,IAAI,OAAO;AAAA,MAClE,CAAC;AAAA,IACH;AACA,UAAM,YAAY,IAAI,IAAI,MAAM;AAChC,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI,CAAC,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAC5D,YAAM,aAAa,wBAAwB,kBAAkB,MAAM,CAAC;AACpE,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,cAAc,qBAAqB;AAAA,eAC1E,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,OAAO,KAAK,SAAS,IAAI,KAAK;AAAA,IACrF,WAAW,KAAK,YAAY,OAAO,KAAK,kBAAkB,UAAU;AAClE,YAAM,aAAa,iBAAiB,gBAAgB,KAAK;AACzD,YAAM,OAAO,KAAK,gBAAgB;AAClC,UAAI,OAAO,EAAG,OAAM,SAAS,IAAI,SAAS,MAAM,eAAe,8BAA8B;AAAA,eACpF,OAAO,EAAG,OAAM,QAAQ,IAAI,SAAS,CAAC,MAAM,QAAQ,KAAK,SAAS,IAAI,KAAK;AAAA,IACtF;AACA,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc;AAAA,QACrC,QAAQ,IAAI;AAAA,QACZ,QAAQ;AAAA,QACR,QAAQ,sBAAsB,MAA4C;AAAA,QAC1E,UAAU,iBAAiB,QAAQ;AAAA,MACrC,CAAC;AAAA,IACH;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ,MAAM,2BAA2B;AAAA,MACvC,OAAO,IAAI;AAAA,MACX,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,UAAM,UAAU,uBAAuB,GAAG;AAC1C,UAAM,QAAQ,IAAI,IAAI,wBAAwB,OAAO,CAAC;AACtD,UAAM,WAAW,MAAM,oBAAoB,IAAI,IAAI,IAAI,OAAO;AAC9D,QAAI;AACF,YAAM,OAAO,OAAO,IAAI,YAAY,WAAW,KAAK,MAAM,IAAI,OAAO,IAAkC,IAAI;AAC3G,UAAI,OAAO,KAAK,kBAAkB,YAAY,KAAK,gBAAgB,GAAG;AACpE,cAAM,SAAS,IAAI,SAAS,KAAK,eAAe,UAAU,aAAa;AAAA,MACzE;AAAA,IACF,QAAQ;AAAA,IAAC;AACT,QAAI,IAAI,cAAc;AACpB,YAAM,eAAe,IAAI,cAAc;AAAA,QACrC,QAAQ,IAAI;AAAA,QACZ,QAAQ;AAAA,QACR,GAAG,uBAAuB,OAAO;AAAA,QACjC,UAAU,iBAAiB,QAAQ;AAAA,MACrC,CAAC;AAAA,IACH;AAAA,EACF,UAAE;AACA;AAAA,EACF;AACF;AAEA,eAAsB,WAAgC;AACpD,QAAM,MAAM,MAAM,gBAAgB;AAClC,MAAI,CAAC,IAAK,QAAO,EAAE,SAAS,MAAM;AAClC,QAAM,YAAY,KAAK,IAAI;AAC3B,QAAM,WAAW,GAA8B;AAC/C,SAAO,EAAE,SAAS,MAAM,OAAQ,IAAuB,IAAI,WAAW,MAAM,YAAY,KAAK,IAAI,IAAI,UAAU;AACjH;AAEA,eAAsB,WAAW,QAA4C;AAC3E,QAAM,UAAwB,CAAC;AAC/B,WAAS,IAAI,GAAG,IAAI,OAAO,SAAS,KAAK;AACvC,QAAI,KAAK,IAAI,KAAK,OAAO,WAAY;AACrC,UAAM,IAAI,MAAM,SAAS;AACzB,YAAQ,KAAK,CAAC;AACd,QAAI,CAAC,EAAE,QAAS;AAAA,EAClB;AACA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,cAAY,YAAY;AACtB,QAAI,WAAW,eAAgB;AAC/B,UAAM,MAAM,MAAM,gBAAgB;AAClC,QAAI,IAAK,MAAK,WAAW,GAA8B;AAAA,EACzD,GAAG,GAAI;AACP,UAAQ,IAAI,iDAA4C,cAAc,aAAa;AACrF;","names":[]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-scraper",
-  "version": "0.40.2",
+  "version": "0.40.3",
   "description": "MCP server for MCP Scraper web intelligence tools",
   "type": "module",
   "main": "./dist/index.cjs",

package/dist/chunk-44HZLHDV.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/errors.ts"],"sourcesContent":["export const RECAPTCHA_INSTRUCTIONS = 'Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.'\n\nexport function sanitizeVendorName(message: string): string {\n  return message\n    .replace(/kernel\\.sh\\s+sessions?/gi, 'sessions')\n    .replace(/kernel\\.sh\\s+session/gi, 'this session')\n    .replace(/kernel\\.sh/gi, 'the service')\n    .replace(/kernel\\s+sessions?/gi, 'sessions')\n    .replace(/kernel\\s+session/gi, 'this session')\n    .replace(/\\bkernel\\b/gi, 'the service')\n    .replace(/  +/g, ' ')\n    .trim()\n}\n\nexport class CaptchaError extends Error {\n  readonly name = 'CaptchaError'\n  constructor(public readonly instructions: string) {\n    super(`CAPTCHA detected. ${instructions}`)\n  }\n}\n\nexport class ExtractionError extends Error {\n  readonly name = 'ExtractionError'\n  constructor(message: string, public readonly cause?: unknown) {\n    super(message)\n  }\n}\n\n/**\n * Google Maps sometimes finishes navigation without rendering either result\n * cards or its explicit no-results message. Treating that page as a genuine\n * empty search loses data, so callers should retry it with a fresh browser (and\n * a fresh proxy when location rotation is enabled).\n */\nexport class MapsSearchIncompletePageError extends Error {\n  readonly name = 'MapsSearchIncompletePageError'\n\n  constructor(public readonly pageState: string) {\n    super(`Google Maps search did not reach a complete result state (${pageState})`)\n  }\n}\n\nexport class RequestAbortedError extends Error {\n  readonly name = 'RequestAbortedError'\n  constructor(message = 'Request aborted before harvest completed') {\n    super(message)\n  }\n}\n\nexport class LocationMismatchError extends Error {\n  readonly name = 'LocationMismatchError'\n  constructor(message = 'Google returned results for a different location than requested') {\n    super(message)\n  }\n}\n"],"mappings":";AAAO,IAAM,yBAAyB;AAE/B,SAAS,mBAAmB,SAAyB;AAC1D,SAAO,QACJ,QAAQ,4BAA4B,UAAU,EAC9C,QAAQ,0BAA0B,cAAc,EAChD,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,wBAAwB,UAAU,EAC1C,QAAQ,sBAAsB,cAAc,EAC5C,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACV;AAEO,IAAM,eAAN,cAA2B,MAAM;AAAA,EAEtC,YAA4B,cAAsB;AAChD,UAAM,qBAAqB,YAAY,EAAE;AADf;AAAA,EAE5B;AAAA,EAF4B;AAAA,EADnB,OAAO;AAIlB;AAEO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EAEzC,YAAY,SAAiC,OAAiB;AAC5D,UAAM,OAAO;AAD8B;AAAA,EAE7C;AAAA,EAF6C;AAAA,EADpC,OAAO;AAIlB;AAQO,IAAM,gCAAN,cAA4C,MAAM;AAAA,EAGvD,YAA4B,WAAmB;AAC7C,UAAM,6DAA6D,SAAS,GAAG;AADrD;AAAA,EAE5B;AAAA,EAF4B;AAAA,EAFnB,OAAO;AAKlB;AAEO,IAAM,sBAAN,cAAkC,MAAM;AAAA,EACpC,OAAO;AAAA,EAChB,YAAY,UAAU,4CAA4C;AAChE,UAAM,OAAO;AAAA,EACf;AACF;AAEO,IAAM,wBAAN,cAAoC,MAAM;AAAA,EACtC,OAAO;AAAA,EAChB,YAAY,UAAU,mEAAmE;AACvF,UAAM,OAAO;AAAA,EACf;AACF;","names":[]}