mcp-scraper 0.66.0 → 0.66.2

package/dist/bin/mcp-stdio-server.js

@@ -13,21 +13,21 @@ import {
   registerScheduledResultsMcpTools,
   registerSerpIntelligenceCaptureTools,
   resolveDeploymentProfile
-} from "../chunk-SRRI2HJL.js";
-import "../chunk-X7UG2JCF.js";
-import "../chunk-VIWT35S6.js";
-import "../chunk-4LINSJNI.js";
+} from "../chunk-YKL3O7IU.js";
+import "../chunk-PGJQDMC2.js";
+import "../chunk-3EYALCYK.js";
+import "../chunk-3TKKJQB4.js";
 import {
   renderInstallTerminal
-} from "../chunk-DCWXVAQT.js";
-import "../chunk-WD73OFL3.js";
-import "../chunk-3YSEJTMC.js";
+} from "../chunk-OZDCTNUV.js";
+import "../chunk-VXLU74YZ.js";
+import "../chunk-GGZEC22A.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-HDYOVGC6.js";
-import "../chunk-EKJ65E3Z.js";
-import "../chunk-2XUXVKT4.js";
-import "../chunk-4VG3NUUY.js";
+} from "../chunk-ALEBMPOW.js";
+import "../chunk-DNM65UCK.js";
+import "../chunk-AYJHI2FK.js";
+import "../chunk-55B3T5L2.js";
 
 // bin/mcp-stdio-server.ts
 import { readFileSync } from "fs";
@@ -169,4 +169,3 @@ serveStdio(() => buildStdioServer(), {
 `);
   }
 });
-//# sourceMappingURL=mcp-stdio-server.js.map

package/dist/bin/paa-harvest.cjs

@@ -4457,4 +4457,3 @@ async function runCli() {
 
 // bin/paa-harvest.ts
 runCli();
-//# sourceMappingURL=paa-harvest.cjs.map

package/dist/bin/paa-harvest.js

@@ -1,12 +1,12 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-AJ4YGHG2.js";
+} from "../chunk-K7Y27ND3.js";
 import {
   browserServiceApiKey
-} from "../chunk-4LINSJNI.js";
-import "../chunk-FG6LJGJ7.js";
-import "../chunk-4VG3NUUY.js";
+} from "../chunk-3TKKJQB4.js";
+import "../chunk-77LQV73B.js";
+import "../chunk-55B3T5L2.js";
 
 // src/cli.ts
 import { Command } from "commander";
@@ -39,4 +39,3 @@ async function runCli() {
 
 // bin/paa-harvest.ts
 runCli();
-//# sourceMappingURL=paa-harvest.js.map

package/dist/{chunk-VIWT35S6.js → chunk-3EYALCYK.js}

@@ -1,7 +1,7 @@
 import {
   publicErrorMessage,
   sanitizeVendorName
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/api/outbound-sanitize.ts
 var KEY_RENAMES = {
@@ -101,4 +101,3 @@ export {
   sanitizeAttempts,
   sanitizeHarvestResult
 };
-//# sourceMappingURL=chunk-VIWT35S6.js.map

package/dist/{chunk-4LINSJNI.js → chunk-3TKKJQB4.js}

@@ -1,6 +1,6 @@
 import {
   getDb
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/lib/browser-service-env.ts
 function browserServiceApiKey() {
@@ -275,4 +275,3 @@ export {
   recordKernelSession,
   recordVendorUsage
 };
-//# sourceMappingURL=chunk-4LINSJNI.js.map

package/dist/{chunk-4VG3NUUY.js → chunk-55B3T5L2.js}

@@ -186,8 +186,9 @@ function publicErrorMessage(code) {
     case "proxy_unavailable":
       return "The web request could not establish a working connection. Please retry later.";
     case "harvest_timeout":
-    case "mcp_request_timeout":
       return "The request timed out before it completed. Please retry with a smaller request.";
+    case "mcp_request_timeout":
+      return "The MCP caller stopped waiting before a final response arrived. Durable work may still be running; continue with the returned Task or job handle instead of restarting it.";
     case "concurrency_limit_exceeded":
       return "The service is available, but this account has reached its parallel-operation limit. Wait for an active operation to finish or add capacity.";
     case "unauthorized":
@@ -2331,7 +2332,7 @@ async function claimPendingJob() {
   const res = await db.execute(`
     SELECT * FROM jobs
     WHERE status = 'pending'
-      AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') != 'inngest'
+      AND COALESCE(json_extract(options, '$.executionOwner'), 'cron') NOT IN ('direct', 'inngest')
     ORDER BY created_at
     LIMIT 1
   `);
@@ -2343,10 +2344,10 @@ async function claimPendingJob() {
   });
   return upd.rowsAffected === 0 ? void 0 : { ...job, status: "running" };
 }
-async function claimInngestPaaJob(id) {
+async function claimDurablePaaJob(id) {
   const db = getDb();
   const current = await getJob(id);
-  if (!current || current.options.executionOwner !== "inngest" || current.options.serpOnly === true) return void 0;
+  if (!current || current.options.executionOwner !== "direct" || current.options.serpOnly === true) return void 0;
   if (current.status === "running") return void 0;
   if (current.status === "pending") {
     const updated = await db.execute({
@@ -2355,7 +2356,7 @@ async function claimInngestPaaJob(id) {
             WHERE id = ? AND status = 'pending'`,
       args: [id]
     });
-    if (updated.rowsAffected === 0) return getJob(id);
+    if (updated.rowsAffected === 0) return void 0;
     return { ...current, status: "running", started_at: current.started_at ?? (/* @__PURE__ */ new Date()).toISOString() };
   }
   return current;
@@ -2367,25 +2368,12 @@ async function checkpointJob(id, result) {
   });
   return updated.rowsAffected === 1;
 }
-async function listPendingInngestPaaJobs(limit = 25) {
-  const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
-  const rows = await getDb().execute({
-    sql: `SELECT * FROM jobs
-          WHERE status = 'pending'
-            AND json_extract(options, '$.executionOwner') = 'inngest'
-            AND COALESCE(json_extract(options, '$.serpOnly'), 0) = 0
-          ORDER BY created_at
-          LIMIT ?`,
-    args: [safeLimit]
-  });
-  return rows.rows.map((row) => deserialize(rowToRawJob(row)));
-}
-async function listRecentTerminalInngestPaaJobs(limit = 25) {
+async function listRecentTerminalDurablePaaJobs(limit = 25) {
   const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
   const rows = await getDb().execute({
     sql: `SELECT * FROM jobs
           WHERE status IN ('done', 'failed', 'cancelled')
-            AND json_extract(options, '$.executionOwner') = 'inngest'
+            AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
             AND json_extract(options, '$.billingDebitKey') IS NOT NULL
             AND json_extract(options, '$.billingSettledAt') IS NULL
           ORDER BY completed_at DESC
@@ -2394,13 +2382,13 @@ async function listRecentTerminalInngestPaaJobs(limit = 25) {
   });
   return rows.rows.map((row) => deserialize(rowToRawJob(row)));
 }
-async function listStaleRunningInngestPaaJobs(limit = 25) {
+async function listStaleRunningDurablePaaJobs(limit = 25) {
   const safeLimit = Math.max(1, Math.min(100, Math.floor(limit)));
   const rows = await getDb().execute({
     sql: `SELECT * FROM jobs
           WHERE status = 'running'
-            AND started_at <= datetime('now', '-10 minutes')
-            AND json_extract(options, '$.executionOwner') = 'inngest'
+            AND started_at <= datetime('now', '-15 minutes')
+            AND json_extract(options, '$.executionOwner') IN ('direct', 'inngest')
           ORDER BY started_at
           LIMIT ?`,
     args: [safeLimit]
@@ -2429,6 +2417,14 @@ async function failJob(id, error, publicError) {
     args: [error, serializePublicErrorEnvelope(publicError), id]
   });
 }
+async function failRunningJob(id, error, publicError) {
+  const updated = await getDb().execute({
+    sql: `UPDATE jobs SET status = 'failed', error = ?, public_error_json = ?, completed_at = datetime('now')
+          WHERE id = ? AND status = 'running'`,
+    args: [error, serializePublicErrorEnvelope(publicError), id]
+  });
+  return updated.rowsAffected === 1;
+}
 async function cancelJob(id, reason, publicError) {
   await getDb().execute({
     sql: `UPDATE jobs SET status = 'cancelled', error = ?, public_error_json = ?, completed_at = datetime('now') WHERE id = ?`,
@@ -3518,14 +3514,14 @@ export {
   countActiveUsers,
   countActiveJobsForUser,
   claimPendingJob,
-  claimInngestPaaJob,
+  claimDurablePaaJob,
   checkpointJob,
-  listPendingInngestPaaJobs,
-  listRecentTerminalInngestPaaJobs,
-  listStaleRunningInngestPaaJobs,
+  listRecentTerminalDurablePaaJobs,
+  listStaleRunningDurablePaaJobs,
   markJobBillingSettled,
   completeJob,
   failJob,
+  failRunningJob,
   cancelJob,
   createKpoJob,
   claimPendingKpoJob,
@@ -3573,4 +3569,3 @@ export {
   insertInboxMessage,
   listActiveUserIds
 };
-//# sourceMappingURL=chunk-4VG3NUUY.js.map

package/dist/{chunk-FG6LJGJ7.js → chunk-77LQV73B.js}

@@ -170,4 +170,3 @@ export {
   RawMapsAboutAttributeSchema,
   postToMemoryLibrary
 };
-//# sourceMappingURL=chunk-FG6LJGJ7.js.map

package/dist/{chunk-N54JS5WP.js → chunk-ABYBPODM.js}

@@ -1,6 +1,6 @@
 import {
   getDb
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/api/lead-list-enrichment-repository.ts
 import { createHash, randomUUID } from "crypto";
@@ -590,4 +590,3 @@ export {
   markLeadListEnrichmentOutboxDispatched,
   markLeadListEnrichmentOutboxFailed
 };
-//# sourceMappingURL=chunk-N54JS5WP.js.map

package/dist/chunk-ALEBMPOW.js

@@ -0,0 +1,6 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.66.2";
+
+export {
+  PACKAGE_VERSION
+};

package/dist/{chunk-2XUXVKT4.js → chunk-AYJHI2FK.js}

@@ -1011,4 +1011,3 @@ export {
   MEDIA_TRANSCRIBE_HOLD_MC,
   MC_PER_USD
 };
-//# sourceMappingURL=chunk-2XUXVKT4.js.map

package/dist/{chunk-AAGVOPP5.js → chunk-CETDPNMZ.js}

@@ -1,6 +1,6 @@
 import {
   isVendorUnavailableError
-} from "./chunk-BIUQ6O2Y.js";
+} from "./chunk-OZJMVCDK.js";
 import {
   CaptchaError,
   LocationMismatchError,
@@ -10,27 +10,7 @@ import {
   finishHarvestAttempt,
   publicErrorMessage,
   startHarvestAttempt
-} from "./chunk-4VG3NUUY.js";
-
-// src/api/webhook.ts
-async function deliverWebhook(url, payload, retries = 3) {
-  for (let attempt = 1; attempt <= retries; attempt++) {
-    try {
-      const res = await fetch(url, {
-        method: "POST",
-        headers: { "content-type": "application/json" },
-        body: JSON.stringify(payload),
-        signal: AbortSignal.timeout(1e4)
-      });
-      if (res.ok) return;
-      console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
-    } catch (err) {
-      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
-    }
-    if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
-  }
-  console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
-}
+} from "./chunk-55B3T5L2.js";
 
 // src/api/harvest-problems.ts
 function errorMessage(err) {
@@ -226,13 +206,32 @@ function createHarvestAttemptRecorder(jobId, userId) {
   };
 }
 
+// src/api/webhook.ts
+async function deliverWebhook(url, payload, retries = 3) {
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      const res = await fetch(url, {
+        method: "POST",
+        headers: { "content-type": "application/json" },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(1e4)
+      });
+      if (res.ok) return;
+      console.warn(`[webhook] attempt ${attempt} \u2192 ${res.status} from ${url}`);
+    } catch (err) {
+      console.warn(`[webhook] attempt ${attempt} failed:`, err instanceof Error ? err.message : err);
+    }
+    if (attempt < retries) await new Promise((r) => setTimeout(r, 1e3 * attempt * 2));
+  }
+  console.error(`[webhook] gave up after ${retries} attempts for ${url}`);
+}
+
 export {
-  deliverWebhook,
   classifyHarvestProblem,
   serializeHarvestProblem,
   harvestProblemResponse,
   harvestProblemEnvelope,
   publicErrorBoundary,
-  createHarvestAttemptRecorder
+  createHarvestAttemptRecorder,
+  deliverWebhook
 };
-//# sourceMappingURL=chunk-AAGVOPP5.js.map

package/dist/{chunk-5BEPOFKG.js → chunk-CNLLTD5B.js}

@@ -3007,4 +3007,3 @@ export {
   refreshAnalyticsDailyRollupsIfDue,
   createAnalyticsExport
 };
-//# sourceMappingURL=chunk-5BEPOFKG.js.map

package/dist/{chunk-P7YFUNKR.js → chunk-DBWWIK7O.js}

@@ -1,14 +1,14 @@
 import {
   isVendorUnavailableError
-} from "./chunk-BIUQ6O2Y.js";
+} from "./chunk-OZJMVCDK.js";
 import {
   loadHtmlDocument,
   validatePublicHttpUrl
-} from "./chunk-3YSEJTMC.js";
+} from "./chunk-GGZEC22A.js";
 import {
   buildPublicErrorEnvelope,
   getDb
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/lib/media-extractor.ts
 import { createWriteStream, mkdirSync, rmSync } from "fs";
@@ -730,4 +730,3 @@ export {
   queuedCommonsDocumentCount,
   semanticCommonsEntityScores
 };
-//# sourceMappingURL=chunk-P7YFUNKR.js.map

package/dist/{chunk-EKJ65E3Z.js → chunk-DNM65UCK.js}

@@ -297,4 +297,3 @@ export {
   readPrivateArtifactWindow,
   readPrivateArtifactBuffer
 };
-//# sourceMappingURL=chunk-EKJ65E3Z.js.map

package/dist/{chunk-3YSEJTMC.js → chunk-GGZEC22A.js}

@@ -213,4 +213,3 @@ export {
   validatePublicHttpUrl,
   loadHtmlDocument
 };
-//# sourceMappingURL=chunk-3YSEJTMC.js.map

package/dist/{chunk-AJ4YGHG2.js → chunk-K7Y27ND3.js}

@@ -6,14 +6,14 @@ import {
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-4LINSJNI.js";
+} from "./chunk-3TKKJQB4.js";
 import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
   RawPAAItemSchema,
   postToMemoryLibrary,
   recencyToTbs
-} from "./chunk-FG6LJGJ7.js";
+} from "./chunk-77LQV73B.js";
 import {
   CaptchaError,
   ExtractionError,
@@ -21,7 +21,7 @@ import {
   RECAPTCHA_INSTRUCTIONS,
   RequestAbortedError,
   sanitizeVendorName
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/driver/BrowserDriver.ts
 import { chromium } from "playwright-extra";
@@ -3988,14 +3988,6 @@ async function harvest(rawOptions) {
 }
 
 export {
-  PAASelectors,
-  VideoSelectors,
-  ForumSelectors,
-  WhatPeopleSayingSelectors,
-  AIOverviewSelectors,
-  AIModeSelectors,
-  OrganicSelectors,
-  LocalPackSelectors,
   MapsSelectors,
   detectPageUserAgent,
   detectRuntimeUserAgent,
@@ -4004,8 +3996,6 @@ export {
   buildYouTubeChannelVideosUrl,
   BrowserDriver,
   brightDataSerpEnabled,
-  encodeUule,
-  normalizeLocation,
   BACKUP_LAST_RESORT_ATTEMPTS,
   backupProxyAvailable,
   createBackupProxyIdSafe,
@@ -4015,4 +4005,3 @@ export {
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-AJ4YGHG2.js.map

package/dist/{chunk-XPFRP5QT.js → chunk-KBTUPRTF.js}

@@ -1,17 +1,17 @@
 import {
   rowsToCsv
-} from "./chunk-G3P3ZDB4.js";
+} from "./chunk-RJ7JVYKU.js";
 import {
   DEFAULT_MAPS_PROXY_MODE,
   postToMemoryLibrary
-} from "./chunk-FG6LJGJ7.js";
+} from "./chunk-77LQV73B.js";
 import {
   buildSerpEmailQuery,
   contactAttemptUrls,
   extractContactEvidence,
   extractSerpEmailEvidence,
   selectPrimaryEmail
-} from "./chunk-WD73OFL3.js";
+} from "./chunk-VXLU74YZ.js";
 
 // src/lib/slugify.ts
 function slugify(s) {
@@ -1913,4 +1913,3 @@ export {
   runWorkflow,
   runWorkflowStep
 };
-//# sourceMappingURL=chunk-XPFRP5QT.js.map

package/dist/{chunk-5PIO7QBG.js → chunk-KXRWY7DZ.js}

@@ -4,10 +4,10 @@ import {
   privateArtifactOwnerId,
   readPrivateArtifactBuffer,
   renewPrivateArtifactDownload
-} from "./chunk-EKJ65E3Z.js";
+} from "./chunk-DNM65UCK.js";
 import {
   LedgerOperation
-} from "./chunk-2XUXVKT4.js";
+} from "./chunk-AYJHI2FK.js";
 import {
   CREDIT_LOT_TTL,
   getDb,
@@ -15,7 +15,7 @@ import {
   sanitizeVendorName,
   serializePublicErrorEnvelope,
   settleDebitMcIdempotent
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/api/site-extract-content-store.ts
 import { createHash } from "crypto";
@@ -797,4 +797,3 @@ export {
   setExtractJobPublicError,
   settleExtractJob
 };
-//# sourceMappingURL=chunk-5PIO7QBG.js.map

package/dist/{chunk-DCWXVAQT.js → chunk-OZDCTNUV.js}

@@ -1,3 +1,8 @@
+// release-message.json
+var release_message_default = {
+  message: "PAA work now starts immediately on the direct runtime, and every MCP Scraper install exposes the same complete typed tool contract."
+};
+
 // src/install-terminal.ts
 var CODES = {
   reset: "\x1B[0m",
@@ -48,7 +53,7 @@ function renderInstallTerminal(options) {
     colorize(ascii, "amber", color),
     `${colorize("MCP Scraper Agent", "cyan", color)}  .  v${options.version}  .  mcpscraper.dev`,
     "1/1 install surfaces ready",
-    colorize("Newest: any approved connection read can become an indexed Memory snapshot in one call. OAuth stays tenant-isolated and provider content is redacted and marked untrusted.", "lime", color),
+    colorize(`Newest in v${options.version}: ${release_message_default.message}`, "lime", color),
     "",
     `${colorize("Tools", "cyan", color)}  ${colorize("(259 MCP tools)", "muted", color)}`,
     toolRow("search", ["harvest_paa", "search_serp", "maps_search", "maps_place_intel"], color),
@@ -106,4 +111,3 @@ function renderInstallTerminal(options) {
 export {
   renderInstallTerminal
 };
-//# sourceMappingURL=chunk-DCWXVAQT.js.map

package/dist/{chunk-BIUQ6O2Y.js → chunk-OZJMVCDK.js}

@@ -14,4 +14,3 @@ function isVendorUnavailableError(err, message) {
 export {
   isVendorUnavailableError
 };
-//# sourceMappingURL=chunk-BIUQ6O2Y.js.map

package/dist/{chunk-X7UG2JCF.js → chunk-PGJQDMC2.js}

@@ -1,6 +1,6 @@
 import {
   validatePublicHttpUrl
-} from "./chunk-3YSEJTMC.js";
+} from "./chunk-GGZEC22A.js";
 
 // src/api/image-audit.ts
 var OVER_BYTES = 100 * 1024;
@@ -381,4 +381,3 @@ export {
   computeIssues,
   renderIssueReport
 };
-//# sourceMappingURL=chunk-X7UG2JCF.js.map

package/dist/{chunk-DKIG2TEE.js → chunk-PMCT2YQM.js}

@@ -1,9 +1,9 @@
 import {
   csvRecords
-} from "./chunk-G3P3ZDB4.js";
+} from "./chunk-RJ7JVYKU.js";
 import {
   getDb
-} from "./chunk-4VG3NUUY.js";
+} from "./chunk-55B3T5L2.js";
 
 // src/api/location-data-repository.ts
 import { createHash, randomUUID } from "crypto";
@@ -681,4 +681,3 @@ export {
   importHostedCensusPlacesCsv,
   queryHostedLocationMarkets
 };
-//# sourceMappingURL=chunk-DKIG2TEE.js.map

package/dist/{chunk-G3P3ZDB4.js → chunk-RJ7JVYKU.js}

@@ -66,4 +66,3 @@ export {
   csvRecords,
   rowsToCsv
 };
-//# sourceMappingURL=chunk-G3P3ZDB4.js.map

package/dist/{chunk-WD73OFL3.js → chunk-VXLU74YZ.js}

@@ -1,7 +1,7 @@
 import {
   isPrivateIpAddress,
   loadHtmlDocument
-} from "./chunk-3YSEJTMC.js";
+} from "./chunk-GGZEC22A.js";
 
 // src/lib/contact-evidence.ts
 var MAX_EMAILS = 20;
@@ -291,4 +291,3 @@ export {
   buildSerpEmailQuery,
   extractSerpEmailEvidence
 };
-//# sourceMappingURL=chunk-WD73OFL3.js.map