mcp-scraper 0.40.2 → 0.41.0

package/dist/bin/paa-harvest.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-7XBBFBYY.js";
+} from "../chunk-JVIW4GK2.js";
 import {
   browserServiceApiKey
-} from "../chunk-3PIWJS6Y.js";
+} from "../chunk-EHES33KB.js";
 import "../chunk-K443GQY5.js";
 import "../chunk-CB5C3BPB.js";
-import "../chunk-44HZLHDV.js";
-import "../chunk-GUVKHCKE.js";
+import "../chunk-5UN33CGU.js";
+import "../chunk-IQTT7CAB.js";
 
 // src/cli.ts
 import { Command } from "commander";

package/dist/{chunk-XORPNO3Z.js → chunk-27DUCRAZ.js}

@@ -3,7 +3,7 @@ import {
 } from "./chunk-G3P3ZDB4.js";
 import {
   getDb
-} from "./chunk-GUVKHCKE.js";
+} from "./chunk-IQTT7CAB.js";
 
 // src/api/location-data-repository.ts
 import { createHash, randomUUID } from "crypto";
@@ -681,4 +681,4 @@ export {
   importHostedCensusPlacesCsv,
   queryHostedLocationMarkets
 };
-//# sourceMappingURL=chunk-XORPNO3Z.js.map
+//# sourceMappingURL=chunk-27DUCRAZ.js.map

package/dist/{chunk-44HZLHDV.js → chunk-5UN33CGU.js}

@@ -1,5 +1,39 @@
 // src/errors.ts
 var RECAPTCHA_INSTRUCTIONS = "Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.";
+var PUBLIC_SERVICE_UNAVAILABLE_MESSAGE = "This operation is temporarily unavailable. Please retry later.";
+var PUBLIC_ERROR_CODES = /* @__PURE__ */ new Set([
+  "request_aborted",
+  "captcha_exhausted",
+  "captcha_or_blocked",
+  "location_mismatch",
+  "proxy_tunnel_failed",
+  "proxy_unavailable",
+  "harvest_timeout",
+  "mcp_request_timeout",
+  "service_unavailable"
+]);
+function normalizePublicErrorCode(code) {
+  return typeof code === "string" && PUBLIC_ERROR_CODES.has(code) ? code : "service_unavailable";
+}
+function publicErrorMessage(code) {
+  switch (normalizePublicErrorCode(code)) {
+    case "request_aborted":
+      return "The request was cancelled before it completed.";
+    case "captcha_exhausted":
+    case "captcha_or_blocked":
+      return "The target site returned a verification challenge. Please retry later.";
+    case "location_mismatch":
+      return "The target site returned results for a different location. Please retry.";
+    case "proxy_tunnel_failed":
+    case "proxy_unavailable":
+      return "The web request could not establish a working connection. Please retry later.";
+    case "harvest_timeout":
+    case "mcp_request_timeout":
+      return "The request timed out before it completed. Please retry with a smaller request.";
+    default:
+      return PUBLIC_SERVICE_UNAVAILABLE_MESSAGE;
+  }
+}
 function sanitizeVendorName(message) {
   return message.replace(/kernel\.sh\s+sessions?/gi, "sessions").replace(/kernel\.sh\s+session/gi, "this session").replace(/kernel\.sh/gi, "the service").replace(/kernel\s+sessions?/gi, "sessions").replace(/kernel\s+session/gi, "this session").replace(/\bkernel\b/gi, "the service").replace(/  +/g, " ").trim();
 }
@@ -42,6 +76,9 @@ var LocationMismatchError = class extends Error {
 
 export {
   RECAPTCHA_INSTRUCTIONS,
+  PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
+  normalizePublicErrorCode,
+  publicErrorMessage,
   sanitizeVendorName,
   CaptchaError,
   ExtractionError,
@@ -49,4 +86,4 @@ export {
   RequestAbortedError,
   LocationMismatchError
 };
-//# sourceMappingURL=chunk-44HZLHDV.js.map
+//# sourceMappingURL=chunk-5UN33CGU.js.map

package/dist/chunk-5UN33CGU.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/errors.ts"],"sourcesContent":["export const RECAPTCHA_INSTRUCTIONS = 'Google returned a CAPTCHA. Run with --headless=false to re-warm the browser profile, then retry.'\nexport const PUBLIC_SERVICE_UNAVAILABLE_MESSAGE = 'This operation is temporarily unavailable. Please retry later.'\n\nconst PUBLIC_ERROR_CODES = new Set([\n  'request_aborted',\n  'captcha_exhausted',\n  'captcha_or_blocked',\n  'location_mismatch',\n  'proxy_tunnel_failed',\n  'proxy_unavailable',\n  'harvest_timeout',\n  'mcp_request_timeout',\n  'service_unavailable',\n])\n\nexport function normalizePublicErrorCode(code: unknown): string {\n  return typeof code === 'string' && PUBLIC_ERROR_CODES.has(code)\n    ? code\n    : 'service_unavailable'\n}\n\nexport function publicErrorMessage(code: unknown): string {\n  switch (normalizePublicErrorCode(code)) {\n    case 'request_aborted':\n      return 'The request was cancelled before it completed.'\n    case 'captcha_exhausted':\n    case 'captcha_or_blocked':\n      return 'The target site returned a verification challenge. Please retry later.'\n    case 'location_mismatch':\n      return 'The target site returned results for a different location. Please retry.'\n    case 'proxy_tunnel_failed':\n    case 'proxy_unavailable':\n      return 'The web request could not establish a working connection. Please retry later.'\n    case 'harvest_timeout':\n    case 'mcp_request_timeout':\n      return 'The request timed out before it completed. Please retry with a smaller request.'\n    default:\n      return PUBLIC_SERVICE_UNAVAILABLE_MESSAGE\n  }\n}\n\nexport function sanitizeVendorName(message: string): string {\n  return message\n    .replace(/kernel\\.sh\\s+sessions?/gi, 'sessions')\n    .replace(/kernel\\.sh\\s+session/gi, 'this session')\n    .replace(/kernel\\.sh/gi, 'the service')\n    .replace(/kernel\\s+sessions?/gi, 'sessions')\n    .replace(/kernel\\s+session/gi, 'this session')\n    .replace(/\\bkernel\\b/gi, 'the service')\n    .replace(/  +/g, ' ')\n    .trim()\n}\n\nexport class CaptchaError extends Error {\n  readonly name = 'CaptchaError'\n  constructor(public readonly instructions: string) {\n    super(`CAPTCHA detected. ${instructions}`)\n  }\n}\n\nexport class ExtractionError extends Error {\n  readonly name = 'ExtractionError'\n  constructor(message: string, public readonly cause?: unknown) {\n    super(message)\n  }\n}\n\n/**\n * Google Maps sometimes finishes navigation without rendering either result\n * cards or its explicit no-results message. Treating that page as a genuine\n * empty search loses data, so callers should retry it with a fresh browser (and\n * a fresh proxy when location rotation is enabled).\n */\nexport class MapsSearchIncompletePageError extends Error {\n  readonly name = 'MapsSearchIncompletePageError'\n\n  constructor(public readonly pageState: string) {\n    super(`Google Maps search did not reach a complete result state (${pageState})`)\n  }\n}\n\nexport class RequestAbortedError extends Error {\n  readonly name = 'RequestAbortedError'\n  constructor(message = 'Request aborted before harvest completed') {\n    super(message)\n  }\n}\n\nexport class LocationMismatchError extends Error {\n  readonly name = 'LocationMismatchError'\n  constructor(message = 'Google returned results for a different location than requested') {\n    super(message)\n  }\n}\n"],"mappings":";AAAO,IAAM,yBAAyB;AAC/B,IAAM,qCAAqC;AAElD,IAAM,qBAAqB,oBAAI,IAAI;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,SAAS,yBAAyB,MAAuB;AAC9D,SAAO,OAAO,SAAS,YAAY,mBAAmB,IAAI,IAAI,IAC1D,OACA;AACN;AAEO,SAAS,mBAAmB,MAAuB;AACxD,UAAQ,yBAAyB,IAAI,GAAG;AAAA,IACtC,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT,KAAK;AAAA,IACL,KAAK;AACH,aAAO;AAAA,IACT;AACE,aAAO;AAAA,EACX;AACF;AAEO,SAAS,mBAAmB,SAAyB;AAC1D,SAAO,QACJ,QAAQ,4BAA4B,UAAU,EAC9C,QAAQ,0BAA0B,cAAc,EAChD,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,wBAAwB,UAAU,EAC1C,QAAQ,sBAAsB,cAAc,EAC5C,QAAQ,gBAAgB,aAAa,EACrC,QAAQ,QAAQ,GAAG,EACnB,KAAK;AACV;AAEO,IAAM,eAAN,cAA2B,MAAM;AAAA,EAEtC,YAA4B,cAAsB;AAChD,UAAM,qBAAqB,YAAY,EAAE;AADf;AAAA,EAE5B;AAAA,EAF4B;AAAA,EADnB,OAAO;AAIlB;AAEO,IAAM,kBAAN,cAA8B,MAAM;AAAA,EAEzC,YAAY,SAAiC,OAAiB;AAC5D,UAAM,OAAO;AAD8B;AAAA,EAE7C;AAAA,EAF6C;AAAA,EADpC,OAAO;AAIlB;AAQO,IAAM,gCAAN,cAA4C,MAAM;AAAA,EAGvD,YAA4B,WAAmB;AAC7C,UAAM,6DAA6D,SAAS,GAAG;AADrD;AAAA,EAE5B;AAAA,EAF4B;AAAA,EAFnB,OAAO;AAKlB;AAEO,IAAM,sBAAN,cAAkC,MAAM;AAAA,EACpC,OAAO;AAAA,EAChB,YAAY,UAAU,4CAA4C;AAChE,UAAM,OAAO;AAAA,EACf;AACF;AAEO,IAAM,wBAAN,cAAoC,MAAM;AAAA,EACtC,OAAO;AAAA,EAChB,YAAY,UAAU,mEAAmE;AACvF,UAAM,OAAO;AAAA,EACf;AACF;","names":[]}

package/dist/{chunk-X2LKCX6H.js → chunk-6J57U6HA.js}

@@ -3,12 +3,12 @@ import {
 } from "./chunk-SBLGBZZB.js";
 import {
   sanitizeVendorName
-} from "./chunk-44HZLHDV.js";
+} from "./chunk-5UN33CGU.js";
 import {
   CREDIT_LOT_TTL,
   getDb,
   settleDebitMcIdempotent
-} from "./chunk-GUVKHCKE.js";
+} from "./chunk-IQTT7CAB.js";
 
 // src/api/site-extract-repository.ts
 function extractJobLimitInfo(job) {
@@ -496,4 +496,4 @@ export {
   failUnfundedExtractJob,
   settleExtractJob
 };
-//# sourceMappingURL=chunk-X2LKCX6H.js.map
+//# sourceMappingURL=chunk-6J57U6HA.js.map

package/dist/chunk-BBI7RGOT.js

@@ -0,0 +1,99 @@
+import {
+  publicErrorMessage
+} from "./chunk-5UN33CGU.js";
+
+// src/api/outbound-sanitize.ts
+var KEY_RENAMES = {
+  kernel: "browserRuntime"
+};
+var SANITIZED_VALUE_KEYS = /error|message/i;
+var INTERNAL_DIAGNOSTIC_KEYS = /* @__PURE__ */ new Set([
+  "kernel_session_id",
+  "browser_session_id",
+  "kernelSessionId",
+  "browserSessionId",
+  "sessionId",
+  "kernel_delete_started",
+  "session_cleanup_started",
+  "kernelDeleteStarted",
+  "sessionCleanupStarted",
+  "kernel_delete_succeeded",
+  "session_cleanup_succeeded",
+  "kernelDeleteSucceeded",
+  "sessionCleanupSucceeded",
+  "kernel_delete_error",
+  "session_cleanup_error",
+  "kernelDeleteError",
+  "sessionCleanupError",
+  "kernelProxyId",
+  "proxyId",
+  "requestedProxyIdSuffix",
+  "retrievedProxyIdSuffix",
+  "provider",
+  "providerName",
+  "service",
+  "serviceName",
+  "vendor",
+  "vendorName",
+  "organization",
+  "organizationId",
+  "accountId"
+]);
+var VENDOR_URL_KEYS = /* @__PURE__ */ new Set([
+  "hosted_url",
+  "hostedUrl",
+  "live_view_url",
+  "liveViewUrl",
+  "cdp_ws_url",
+  "cdpWsUrl",
+  "browser_live_view_url"
+]);
+var VENDOR_URL_RE = /\b(?:wss?|https?):\/\/[^\s"'<>]*\bonkernel\.com[^\s"'<>]*/gi;
+function redactVendorUrls(value) {
+  return value.replace(VENDOR_URL_RE, "[browser-service]");
+}
+function sanitizeOutboundDiagnostics(value, parentKey = "") {
+  if (typeof value === "string") {
+    let out = redactVendorUrls(value);
+    if (SANITIZED_VALUE_KEYS.test(parentKey)) {
+      out = publicErrorMessage("service_unavailable");
+    }
+    return out;
+  }
+  if (Array.isArray(value)) return value.map((v) => sanitizeOutboundDiagnostics(v, parentKey));
+  if (value !== null && typeof value === "object") {
+    const out = {};
+    for (const [key, val] of Object.entries(value)) {
+      if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue;
+      const renamed = KEY_RENAMES[key] ?? key;
+      if (VENDOR_URL_KEYS.has(key)) {
+        out[renamed] = null;
+        continue;
+      }
+      out[renamed] = sanitizeOutboundDiagnostics(val, key);
+    }
+    return out;
+  }
+  return value;
+}
+function sanitizeAttempts(attempts) {
+  return attempts.map((a) => sanitizeOutboundDiagnostics(a));
+}
+function sanitizeHarvestResult(result) {
+  const diagnostics = result?.diagnostics;
+  if (!diagnostics?.debug) return result;
+  return {
+    ...result,
+    diagnostics: {
+      ...diagnostics,
+      debug: sanitizeOutboundDiagnostics(diagnostics.debug)
+    }
+  };
+}
+
+export {
+  sanitizeOutboundDiagnostics,
+  sanitizeAttempts,
+  sanitizeHarvestResult
+};
+//# sourceMappingURL=chunk-BBI7RGOT.js.map

package/dist/chunk-BBI7RGOT.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/outbound-sanitize.ts"],"sourcesContent":["import { publicErrorMessage } from '../errors.js'\n\nconst KEY_RENAMES: Record<string, string> = {\n  kernel: 'browserRuntime',\n}\n\nconst SANITIZED_VALUE_KEYS = /error|message/i\nconst INTERNAL_DIAGNOSTIC_KEYS = new Set([\n  'kernel_session_id',\n  'browser_session_id',\n  'kernelSessionId',\n  'browserSessionId',\n  'sessionId',\n  'kernel_delete_started',\n  'session_cleanup_started',\n  'kernelDeleteStarted',\n  'sessionCleanupStarted',\n  'kernel_delete_succeeded',\n  'session_cleanup_succeeded',\n  'kernelDeleteSucceeded',\n  'sessionCleanupSucceeded',\n  'kernel_delete_error',\n  'session_cleanup_error',\n  'kernelDeleteError',\n  'sessionCleanupError',\n  'kernelProxyId',\n  'proxyId',\n  'requestedProxyIdSuffix',\n  'retrievedProxyIdSuffix',\n  'provider',\n  'providerName',\n  'service',\n  'serviceName',\n  'vendor',\n  'vendorName',\n  'organization',\n  'organizationId',\n  'accountId',\n])\n\nconst VENDOR_URL_KEYS = new Set([\n  'hosted_url',\n  'hostedUrl',\n  'live_view_url',\n  'liveViewUrl',\n  'cdp_ws_url',\n  'cdpWsUrl',\n  'browser_live_view_url',\n])\nconst VENDOR_URL_RE = /\\b(?:wss?|https?):\\/\\/[^\\s\"'<>]*\\bonkernel\\.com[^\\s\"'<>]*/gi\n\nfunction redactVendorUrls(value: string): string {\n  return value.replace(VENDOR_URL_RE, '[browser-service]')\n}\n\nexport function sanitizeOutboundDiagnostics<T>(value: T, parentKey = ''): T {\n  if (typeof value === 'string') {\n    let out = redactVendorUrls(value)\n    if (SANITIZED_VALUE_KEYS.test(parentKey)) {\n      out = publicErrorMessage('service_unavailable')\n    }\n    return out as T\n  }\n  if (Array.isArray(value)) return value.map(v => sanitizeOutboundDiagnostics(v, parentKey)) as T\n  if (value !== null && typeof value === 'object') {\n    const out: Record<string, unknown> = {}\n    for (const [key, val] of Object.entries(value as Record<string, unknown>)) {\n      if (INTERNAL_DIAGNOSTIC_KEYS.has(key)) continue\n      const renamed = KEY_RENAMES[key] ?? key\n      if (VENDOR_URL_KEYS.has(key)) {\n        out[renamed] = null\n        continue\n      }\n      out[renamed] = sanitizeOutboundDiagnostics(val, key)\n    }\n    return out as T\n  }\n  return value\n}\n\nexport function sanitizeAttempts<T>(attempts: T[]): T[] {\n  return attempts.map(a => sanitizeOutboundDiagnostics(a))\n}\n\nexport function sanitizeHarvestResult<T extends Record<string, unknown>>(result: T): T {\n  const diagnostics = result?.diagnostics as Record<string, unknown> | undefined\n  if (!diagnostics?.debug) return result\n  return {\n    ...result,\n    diagnostics: {\n      ...diagnostics,\n      debug: sanitizeOutboundDiagnostics(diagnostics.debug),\n    },\n  }\n}\n"],"mappings":";;;;;AAEA,IAAM,cAAsC;AAAA,EAC1C,QAAQ;AACV;AAEA,IAAM,uBAAuB;AAC7B,IAAM,2BAA2B,oBAAI,IAAI;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,kBAAkB,oBAAI,IAAI;AAAA,EAC9B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AACD,IAAM,gBAAgB;AAEtB,SAAS,iBAAiB,OAAuB;AAC/C,SAAO,MAAM,QAAQ,eAAe,mBAAmB;AACzD;AAEO,SAAS,4BAA+B,OAAU,YAAY,IAAO;AAC1E,MAAI,OAAO,UAAU,UAAU;AAC7B,QAAI,MAAM,iBAAiB,KAAK;AAChC,QAAI,qBAAqB,KAAK,SAAS,GAAG;AACxC,YAAM,mBAAmB,qBAAqB;AAAA,IAChD;AACA,WAAO;AAAA,EACT;AACA,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,IAAI,OAAK,4BAA4B,GAAG,SAAS,CAAC;AACzF,MAAI,UAAU,QAAQ,OAAO,UAAU,UAAU;AAC/C,UAAM,MAA+B,CAAC;AACtC,eAAW,CAAC,KAAK,GAAG,KAAK,OAAO,QAAQ,KAAgC,GAAG;AACzE,UAAI,yBAAyB,IAAI,GAAG,EAAG;AACvC,YAAM,UAAU,YAAY,GAAG,KAAK;AACpC,UAAI,gBAAgB,IAAI,GAAG,GAAG;AAC5B,YAAI,OAAO,IAAI;AACf;AAAA,MACF;AACA,UAAI,OAAO,IAAI,4BAA4B,KAAK,GAAG;AAAA,IACrD;AACA,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,SAAS,iBAAoB,UAAoB;AACtD,SAAO,SAAS,IAAI,OAAK,4BAA4B,CAAC,CAAC;AACzD;AAEO,SAAS,sBAAyD,QAAc;AACrF,QAAM,cAAc,QAAQ;AAC5B,MAAI,CAAC,aAAa,MAAO,QAAO;AAChC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,aAAa;AAAA,MACX,GAAG;AAAA,MACH,OAAO,4BAA4B,YAAY,KAAK;AAAA,IACtD;AAAA,EACF;AACF;","names":[]}

package/dist/{chunk-NVXNEOUQ.js → chunk-CS4HE6QY.js}

@@ -1,12 +1,34 @@
 import {
   CaptchaError,
   LocationMismatchError,
-  RequestAbortedError
-} from "./chunk-44HZLHDV.js";
+  PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
+  RequestAbortedError,
+  publicErrorMessage
+} from "./chunk-5UN33CGU.js";
 import {
   finishHarvestAttempt,
   startHarvestAttempt
-} from "./chunk-GUVKHCKE.js";
+} from "./chunk-IQTT7CAB.js";
+
+// src/api/webhook.ts
+async function deliverWebhook(url, payload, retries = 3) {
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      const res = await fetch(url, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(1e4)
+      });
+      if (res.ok) return;
+      console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
+    } catch (err) {
+      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
+    }
+    if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
+  }
+  console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
+}
 
 // src/api/harvest-problems.ts
 function errorMessage(err) {
@@ -25,13 +47,16 @@ function looksLikeProxyTunnelFailure(message) {
 function looksLikeProxyUnavailable(message) {
   return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message);
 }
+function looksLikeInternalServiceUnavailable(message) {
+  return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\/\/\S*dashboard/i.test(message);
+}
 function classifyHarvestProblem(err) {
   const message = errorMessage(err);
   if (err instanceof RequestAbortedError) {
     return {
       error_code: "request_aborted",
       error_type: "request_aborted",
-      message,
+      message: publicErrorMessage("request_aborted"),
       retryable: true,
       httpStatus: 408,
       terminalStatus: "cancelled"
@@ -41,7 +66,7 @@ function classifyHarvestProblem(err) {
     return {
       error_code: "captcha_exhausted",
       error_type: "captcha",
-      message,
+      message: publicErrorMessage("captcha_exhausted"),
       retryable: true,
       httpStatus: 503,
       terminalStatus: "failed"
@@ -51,7 +76,7 @@ function classifyHarvestProblem(err) {
     return {
       error_code: "location_mismatch",
       error_type: "location_mismatch",
-      message,
+      message: publicErrorMessage("location_mismatch"),
       retryable: true,
       httpStatus: 503,
       terminalStatus: "failed"
@@ -61,7 +86,7 @@ function classifyHarvestProblem(err) {
     return {
       error_code: "proxy_tunnel_failed",
       error_type: "proxy_tunnel_failed",
-      message,
+      message: publicErrorMessage("proxy_tunnel_failed"),
       retryable: true,
       httpStatus: 503,
       terminalStatus: "failed"
@@ -71,7 +96,7 @@ function classifyHarvestProblem(err) {
     return {
       error_code: "proxy_unavailable",
       error_type: "proxy_unavailable",
-      message,
+      message: publicErrorMessage("proxy_unavailable"),
       retryable: true,
       httpStatus: 503,
       terminalStatus: "failed"
@@ -81,16 +106,26 @@ function classifyHarvestProblem(err) {
     return {
       error_code: "harvest_timeout",
       error_type: "timeout",
-      message,
+      message: publicErrorMessage("harvest_timeout"),
       retryable: true,
       httpStatus: 504,
       terminalStatus: "failed"
     };
   }
+  if (looksLikeInternalServiceUnavailable(message)) {
+    return {
+      error_code: "service_unavailable",
+      error_type: "service_unavailable",
+      message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
+      retryable: true,
+      httpStatus: 503,
+      terminalStatus: "failed"
+    };
+  }
   return {
     error_code: "extraction_failed",
     error_type: "extraction",
-    message,
+    message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,
     retryable: false,
     httpStatus: 500,
     terminalStatus: "failed"
@@ -150,9 +185,10 @@ function createHarvestAttemptRecorder(jobId, userId) {
 }
 
 export {
+  deliverWebhook,
   classifyHarvestProblem,
   serializeHarvestProblem,
   harvestProblemResponse,
   createHarvestAttemptRecorder
 };
-//# sourceMappingURL=chunk-NVXNEOUQ.js.map
+//# sourceMappingURL=chunk-CS4HE6QY.js.map

package/dist/chunk-CS4HE6QY.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/api/webhook.ts","../src/api/harvest-problems.ts","../src/api/harvest-attempt-events.ts"],"sourcesContent":["export async function deliverWebhook(url: string, payload: object, retries = 3): Promise<void> {\n  for (let attempt = 1; attempt <= retries; attempt++) {\n    try {\n      const res = await fetch(url, {\n        method: 'POST',\n        headers: { 'content-type': 'application/json' },\n        body: JSON.stringify(payload),\n        signal: AbortSignal.timeout(10_000),\n      })\n      if (res.ok) return\n      console.warn(`[webhook] attempt ${attempt} → ${res.status} from ${url}`)\n    } catch (err) {\n      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err)\n    }\n    if (attempt < retries) await new Promise((r) => setTimeout(r, 1000 * attempt * 2))\n  }\n  console.error(`[webhook] gave up after ${retries} attempts for ${url}`)\n}\n","import {\n  CaptchaError,\n  LocationMismatchError,\n  PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,\n  RequestAbortedError,\n  publicErrorMessage,\n} from '../errors.js'\n\nexport type HarvestProblemCode =\n  | 'request_aborted'\n  | 'captcha_exhausted'\n  | 'location_mismatch'\n  | 'proxy_tunnel_failed'\n  | 'proxy_unavailable'\n  | 'harvest_timeout'\n  | 'service_unavailable'\n  | 'extraction_failed'\n\nexport interface HarvestProblem {\n  error_code: HarvestProblemCode\n  error_type: string\n  message: string\n  retryable: boolean\n  httpStatus: number\n  terminalStatus: 'cancelled' | 'failed'\n}\n\nfunction errorMessage(err: unknown): string {\n  return err instanceof Error ? err.message : String(err)\n}\n\nfunction looksLikeTimeout(err: unknown, message: string): boolean {\n  if (err instanceof DOMException && (err.name === 'TimeoutError' || err.name === 'AbortError')) return true\n  return /timeout|timed out|Timeout \\d+ms exceeded|deadline/i.test(message)\n}\n\nfunction looksLikeCaptcha(message: string): boolean {\n  return /captcha|recaptcha|unusual traffic|google\\.com\\/sorry|blocked/i.test(message)\n}\n\nfunction looksLikeProxyTunnelFailure(message: string): boolean {\n  return /ERR_TUNNEL_CONNECTION_FAILED|ERR_PROXY_CONNECTION_FAILED|ERR_SOCKS_CONNECTION_FAILED|tunnel connection failed|proxy connection failed|transport error: proxy/i.test(message)\n}\n\nfunction looksLikeProxyUnavailable(message: string): boolean {\n  return /proxy unavailable|proxy_unavailable|connection_test_failed|did not return a proxy id|configured fallback/i.test(message)\n}\n\nfunction looksLikeInternalServiceUnavailable(message: string): boolean {\n  return /(?:spending|billing|organization|account).{0,80}(?:cap|limit|blocked|disabled)|(?:quota|capacity).{0,40}(?:reached|exceeded)|https?:\\/\\/\\S*dashboard/i.test(message)\n}\n\nexport function classifyHarvestProblem(err: unknown): HarvestProblem {\n  const message = errorMessage(err)\n\n  if (err instanceof RequestAbortedError) {\n    return {\n      error_code: 'request_aborted',\n      error_type: 'request_aborted',\n      message: publicErrorMessage('request_aborted'),\n      retryable: true,\n      httpStatus: 408,\n      terminalStatus: 'cancelled',\n    }\n  }\n\n  if (err instanceof CaptchaError || looksLikeCaptcha(message)) {\n    return {\n      error_code: 'captcha_exhausted',\n      error_type: 'captcha',\n      message: publicErrorMessage('captcha_exhausted'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (err instanceof LocationMismatchError) {\n    return {\n      error_code: 'location_mismatch',\n      error_type: 'location_mismatch',\n      message: publicErrorMessage('location_mismatch'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyTunnelFailure(message)) {\n    return {\n      error_code: 'proxy_tunnel_failed',\n      error_type: 'proxy_tunnel_failed',\n      message: publicErrorMessage('proxy_tunnel_failed'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeProxyUnavailable(message)) {\n    return {\n      error_code: 'proxy_unavailable',\n      error_type: 'proxy_unavailable',\n      message: publicErrorMessage('proxy_unavailable'),\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeTimeout(err, message)) {\n    return {\n      error_code: 'harvest_timeout',\n      error_type: 'timeout',\n      message: publicErrorMessage('harvest_timeout'),\n      retryable: true,\n      httpStatus: 504,\n      terminalStatus: 'failed',\n    }\n  }\n\n  if (looksLikeInternalServiceUnavailable(message)) {\n    return {\n      error_code: 'service_unavailable',\n      error_type: 'service_unavailable',\n      message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,\n      retryable: true,\n      httpStatus: 503,\n      terminalStatus: 'failed',\n    }\n  }\n\n  return {\n    error_code: 'extraction_failed',\n    error_type: 'extraction',\n    message: PUBLIC_SERVICE_UNAVAILABLE_MESSAGE,\n    retryable: false,\n    httpStatus: 500,\n    terminalStatus: 'failed',\n  }\n}\n\nexport function serializeHarvestProblem(problem: HarvestProblem): string {\n  return JSON.stringify({\n    error_code: problem.error_code,\n    error_type: problem.error_type,\n    message: problem.message,\n    retryable: problem.retryable,\n  })\n}\n\nexport function harvestProblemResponse(problem: HarvestProblem): {\n  error: string\n  error_code: HarvestProblemCode\n  error_type: string\n  retryable: boolean\n} {\n  return {\n    error: problem.message,\n    error_code: problem.error_code,\n    error_type: problem.error_type,\n    retryable: problem.retryable,\n  }\n}\n","import type { HarvestAttemptLogEvent } from '../harvest.js'\nimport { finishHarvestAttempt, startHarvestAttempt } from './db.js'\n\nexport function createHarvestAttemptRecorder(jobId: string, userId: number | bigint) {\n  return async (event: HarvestAttemptLogEvent): Promise<void> => {\n    if (event.type === 'started') {\n      await startHarvestAttempt({\n        jobId,\n        userId,\n        attemptNumber: event.attemptNumber,\n        maxAttempts: event.maxAttempts,\n        query: event.query,\n        location: event.location,\n        maxQuestions: event.maxQuestions,\n        startedAt: event.startedAt,\n      })\n      return\n    }\n\n    await finishHarvestAttempt({\n      jobId,\n      attemptNumber: event.attemptNumber,\n      outcome: event.outcome,\n      kernelSessionId: event.kernelSessionId,\n      questionCount: event.questionCount,\n      durationMs: event.durationMs,\n      error: event.error,\n      willRetry: event.willRetry,\n      kernelDeleteStarted: event.cleanup.kernelDeleteStarted,\n      kernelDeleteSucceeded: event.cleanup.kernelDeleteSucceeded,\n      kernelDeleteError: event.cleanup.kernelDeleteError,\n      browserCloseSucceeded: event.cleanup.browserCloseSucceeded,\n      browserCloseError: event.cleanup.browserCloseError,\n      debug: event.debug,\n      completedAt: event.completedAt,\n    })\n  }\n}\n"],"mappings":";;;;;;;;;;;;;AAAA,eAAsB,eAAe,KAAa,SAAiB,UAAU,GAAkB;AAC7F,WAAS,UAAU,GAAG,WAAW,SAAS,WAAW;AACnD,QAAI;AACF,YAAM,MAAM,MAAM,MAAM,KAAK;AAAA,QAC3B,QAAQ;AAAA,QACR,SAAS,EAAE,gBAAgB,mBAAmB;AAAA,QAC9C,MAAM,KAAK,UAAU,OAAO;AAAA,QAC5B,QAAQ,YAAY,QAAQ,GAAM;AAAA,MACpC,CAAC;AACD,UAAI,IAAI,GAAI;AACZ,cAAQ,KAAK,qBAAqB,OAAO,WAAM,IAAI,MAAM,SAAS,GAAG,EAAE;AAAA,IACzE,SAAS,KAAK;AACZ,cAAQ,KAAK,qBAAqB,OAAO,YAAY,eAAe,QAAQ,IAAI,UAAU,GAAG;AAAA,IAC/F;AACA,QAAI,UAAU,QAAS,OAAM,IAAI,QAAQ,CAAC,MAAM,WAAW,GAAG,MAAO,UAAU,CAAC,CAAC;AAAA,EACnF;AACA,UAAQ,MAAM,2BAA2B,OAAO,iBAAiB,GAAG,EAAE;AACxE;;;ACUA,SAAS,aAAa,KAAsB;AAC1C,SAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AACxD;AAEA,SAAS,iBAAiB,KAAc,SAA0B;AAChE,MAAI,eAAe,iBAAiB,IAAI,SAAS,kBAAkB,IAAI,SAAS,cAAe,QAAO;AACtG,SAAO,qDAAqD,KAAK,OAAO;AAC1E;AAEA,SAAS,iBAAiB,SAA0B;AAClD,SAAO,gEAAgE,KAAK,OAAO;AACrF;AAEA,SAAS,4BAA4B,SAA0B;AAC7D,SAAO,gKAAgK,KAAK,OAAO;AACrL;AAEA,SAAS,0BAA0B,SAA0B;AAC3D,SAAO,4GAA4G,KAAK,OAAO;AACjI;AAEA,SAAS,oCAAoC,SAA0B;AACrE,SAAO,wJAAwJ,KAAK,OAAO;AAC7K;AAEO,SAAS,uBAAuB,KAA8B;AACnE,QAAM,UAAU,aAAa,GAAG;AAEhC,MAAI,eAAe,qBAAqB;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,iBAAiB;AAAA,MAC7C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,gBAAgB,iBAAiB,OAAO,GAAG;AAC5D,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,mBAAmB;AAAA,MAC/C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,eAAe,uBAAuB;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,mBAAmB;AAAA,MAC/C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,4BAA4B,OAAO,GAAG;AACxC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,qBAAqB;AAAA,MACjD,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,0BAA0B,OAAO,GAAG;AACtC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,mBAAmB;AAAA,MAC/C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,iBAAiB,KAAK,OAAO,GAAG;AAClC,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS,mBAAmB,iBAAiB;AAAA,MAC7C,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,MAAI,oCAAoC,OAAO,GAAG;AAChD,WAAO;AAAA,MACL,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,SAAS;AAAA,MACT,WAAW;AAAA,MACX,YAAY;AAAA,MACZ,gBAAgB;AAAA,IAClB;AAAA,EACF;AAEA,SAAO;AAAA,IACL,YAAY;AAAA,IACZ,YAAY;AAAA,IACZ,SAAS;AAAA,IACT,WAAW;AAAA,IACX,YAAY;AAAA,IACZ,gBAAgB;AAAA,EAClB;AACF;AAEO,SAAS,wBAAwB,SAAiC;AACvE,SAAO,KAAK,UAAU;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,SAAS,QAAQ;AAAA,IACjB,WAAW,QAAQ;AAAA,EACrB,CAAC;AACH;AAEO,SAAS,uBAAuB,SAKrC;AACA,SAAO;AAAA,IACL,OAAO,QAAQ;AAAA,IACf,YAAY,QAAQ;AAAA,IACpB,YAAY,QAAQ;AAAA,IACpB,WAAW,QAAQ;AAAA,EACrB;AACF;;;AChKO,SAAS,6BAA6B,OAAe,QAAyB;AACnF,SAAO,OAAO,UAAiD;AAC7D,QAAI,MAAM,SAAS,WAAW;AAC5B,YAAM,oBAAoB;AAAA,QACxB;AAAA,QACA;AAAA,QACA,eAAe,MAAM;AAAA,QACrB,aAAa,MAAM;AAAA,QACnB,OAAO,MAAM;AAAA,QACb,UAAU,MAAM;AAAA,QAChB,cAAc,MAAM;AAAA,QACpB,WAAW,MAAM;AAAA,MACnB,CAAC;AACD;AAAA,IACF;AAEA,UAAM,qBAAqB;AAAA,MACzB;AAAA,MACA,eAAe,MAAM;AAAA,MACrB,SAAS,MAAM;AAAA,MACf,iBAAiB,MAAM;AAAA,MACvB,eAAe,MAAM;AAAA,MACrB,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,WAAW,MAAM;AAAA,MACjB,qBAAqB,MAAM,QAAQ;AAAA,MACnC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,uBAAuB,MAAM,QAAQ;AAAA,MACrC,mBAAmB,MAAM,QAAQ;AAAA,MACjC,OAAO,MAAM;AAAA,MACb,aAAa,MAAM;AAAA,IACrB,CAAC;AAAA,EACH;AACF;","names":[]}

package/dist/{chunk-3PIWJS6Y.js → chunk-EHES33KB.js}

@@ -1,6 +1,6 @@
 import {
   getDb
-} from "./chunk-GUVKHCKE.js";
+} from "./chunk-IQTT7CAB.js";
 
 // src/lib/browser-service-env.ts
 function browserServiceApiKey() {
@@ -273,4 +273,4 @@ export {
   recordKernelSession,
   recordVendorUsage
 };
-//# sourceMappingURL=chunk-3PIWJS6Y.js.map
+//# sourceMappingURL=chunk-EHES33KB.js.map

package/dist/{chunk-GUVKHCKE.js → chunk-IQTT7CAB.js}

@@ -1389,11 +1389,15 @@ function deserialize(raw) {
 }
 async function createJob(userId, query, options, callbackUrl) {
   const id = randomUUID();
+  await createJobWithId(id, userId, query, options, callbackUrl);
+  return id;
+}
+async function createJobWithId(id, userId, query, options, callbackUrl) {
+  if (!/^[a-zA-Z0-9-]{1,100}$/.test(id)) throw new Error("job id is invalid");
   await getDb().execute({
     sql: "INSERT INTO jobs (id, user_id, query, options, callback_url) VALUES (?, ?, ?, ?, ?)",
     args: [id, userId, query, JSON.stringify(options), callbackUrl ?? null]
   });
-  return id;
 }
 async function createRunningJob(userId, query, options) {
   const id = randomUUID();
@@ -1924,7 +1928,13 @@ async function countActiveJobsForUser(userId) {
 }
 async function claimPendingJob() {
   const db = getDb();
-  const res = await db.execute(`SELECT * FROM jobs WHERE status = 'pending' ORDER BY created_at LIMIT 1`);
+  const res = await db.execute(`
+    SELECT * FROM jobs
+    WHERE status = 'pending'
+      AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') != 'inngest'
+    ORDER BY created_at
+    LIMIT 1
+  `);
   const job = res.rows[0] ? rowToRawJob(res.rows[0]) : void 0;
   if (!job) return void 0;
   const upd = await db.execute({
@@ -1933,6 +1943,80 @@ async function claimPendingJob() {
   });
   return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
 }
+async function claimInngestPaaJob(id) {
+  const db = getDb();
+  const current = await getJob(id);
+  if (!current || current.options.executionOwner !== "inngest" || current.options.serpOnly === true) return void 0;
+  if (current.status === "running") return void 0;
+  if (current.status === "pending") {
+    const updated = await db.execute({
+      sql: `UPDATE jobs
+            SET status = 'running', started_at = COALESCE(started_at, datetime('now')), error = NULL
+            WHERE id = ? AND status = 'pending'`,
+      args: [id]
+    });
+    if (updated.rowsAffected === 0) return getJob(id);
+    return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
+  }
+  return current;
+}
+async function checkpointJob(id, result) {
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET result = ? WHERE id = ? AND status = 'running'`,
+    args: [JSON.stringify(result), id]
+  });
+  return updated.rowsAffected === 1;
+}
+async function listPendingInngestPaaJobs(limit = 25) {
+  const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
+  const rows = await getDb().execute({
+    sql: `SELECT * FROM jobs
+          WHERE status = 'pending'
+            AND json_extract(options, '$.executionOwner') = 'inngest'
+            AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
+          ORDER BY created_at
+          LIMIT ?`,
+    args: [safeLimit]
+  });
+  return rows.rows.map((row) => deserialize(rowToRawJob(row)));
+}
+async function listRecentTerminalInngestPaaJobs(limit = 25) {
+  const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
+  const rows = await getDb().execute({
+    sql: `SELECT * FROM jobs
+          WHERE status IN ('done', 'failed', 'cancelled')
+            AND json_extract(options, '$.executionOwner') = 'inngest'
+            AND json_extract(options, '$.billingDebitKey') IS NOT NULL
+            AND json_extract(options, '$.billingSettledAt') IS NULL
+          ORDER BY completed_at DESC
+          LIMIT ?`,
+    args: [safeLimit]
+  });
+  return rows.rows.map((row) => deserialize(rowToRawJob(row)));
+}
+async function listStaleRunningInngestPaaJobs(limit = 25) {
+  const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
+  const rows = await getDb().execute({
+    sql: `SELECT * FROM jobs
+          WHERE status = 'running'
+            AND started_at <= datetime('now', '-10 minutes')
+            AND json_extract(options, '$.executionOwner') = 'inngest'
+          ORDER BY started_at
+          LIMIT ?`,
+    args: [safeLimit]
+  });
+  return rows.rows.map((row) => deserialize(rowToRawJob(row)));
+}
+async function markJobBillingSettled(id, billedMc, settledAt = (/* @__PURE__ */ new Date()).toISOString()) {
+  if (!Number.isSafeInteger(billedMc) || billedMc < 0) throw new Error("job billed amount must be a non-negative integer");
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs
+          SET options = json_set(options, '$.billingSettledAt', ?, '$.billedMc', ?)
+          WHERE id = ?`,
+    args: [settledAt, billedMc, id]
+  });
+  return updated.rowsAffected === 1;
+}
 async function completeJob(id, result) {
   await getDb().execute({ sql: `UPDATE jobs SET status = 'done', result = ?, completed_at = datetime('now') WHERE id = ?`, args: [JSON.stringify(result), id] });
 }
@@ -2965,6 +3049,7 @@ export {
   listUsers,
   deactivateUser,
   createJob,
+  createJobWithId,
   createRunningJob,
   startHarvestAttempt,
   finishHarvestAttempt,
@@ -2999,6 +3084,12 @@ export {
   countActiveUsers,
   countActiveJobsForUser,
   claimPendingJob,
+  claimInngestPaaJob,
+  checkpointJob,
+  listPendingInngestPaaJobs,
+  listRecentTerminalInngestPaaJobs,
+  listStaleRunningInngestPaaJobs,
+  markJobBillingSettled,
   completeJob,
   failJob,
   cancelJob,
@@ -3047,4 +3138,4 @@ export {
   insertInboxMessage,
   listActiveUserIds
 };
-//# sourceMappingURL=chunk-GUVKHCKE.js.map
+//# sourceMappingURL=chunk-IQTT7CAB.js.map