mcp-scraper 0.38.0 → 0.38.2

package/dist/{chunk-FSAXLDB3.js → chunk-SFRMFGQ6.js}

@@ -6,7 +6,7 @@ import {
 import {
   finishHarvestAttempt,
   startHarvestAttempt
-} from "./chunk-CSCD2HNS.js";
+} from "./chunk-N7KUTTCC.js";
 
 // src/api/harvest-problems.ts
 function errorMessage(err) {
@@ -155,4 +155,4 @@ export {
   harvestProblemResponse,
   createHarvestAttemptRecorder
 };
-//# sourceMappingURL=chunk-FSAXLDB3.js.map
+//# sourceMappingURL=chunk-SFRMFGQ6.js.map

package/dist/{chunk-Q35WZJJK.js → chunk-YCI2PNCS.js}

@@ -1,6 +1,6 @@
 import {
   LedgerOperation
-} from "./chunk-PWPUKR5U.js";
+} from "./chunk-AZRPG43B.js";
 import {
   sanitizeVendorName
 } from "./chunk-44HZLHDV.js";
@@ -8,7 +8,7 @@ import {
   CREDIT_LOT_TTL,
   getDb,
   settleDebitMcIdempotent
-} from "./chunk-CSCD2HNS.js";
+} from "./chunk-N7KUTTCC.js";
 
 // src/api/site-extract-repository.ts
 function extractJobLimitInfo(job) {
@@ -496,4 +496,4 @@ export {
   failUnfundedExtractJob,
   settleExtractJob
 };
-//# sourceMappingURL=chunk-Q35WZJJK.js.map
+//# sourceMappingURL=chunk-YCI2PNCS.js.map

package/dist/chunk-YODBNTTN.js

@@ -0,0 +1,7 @@
+// src/version.ts
+var PACKAGE_VERSION = "0.38.2";
+
+export {
+  PACKAGE_VERSION
+};
+//# sourceMappingURL=chunk-YODBNTTN.js.map

package/dist/chunk-YODBNTTN.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/version.ts"],"sourcesContent":["export const PACKAGE_VERSION = '0.38.2'\n"],"mappings":";AAAO,IAAM,kBAAkB;","names":[]}

package/dist/{db-N3YECFWR.js → db-C5KVCOYT.js}

@@ -1,5 +1,6 @@
 import {
   CORE_SCHEMA_VERSION,
+  CREDIT_LOTS_EPOCH,
   CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema,
@@ -10,8 +11,10 @@ import {
   claimMonthlyFreeRefresh,
   claimPendingJob,
   claimPendingKpoJob,
+  claimStripeEvent,
   completeJob,
   completeKpoJob,
+  completeStripeEvent,
   completeWorkflowRunRecord,
   consumeCode,
   consumePasswordResetToken,
@@ -33,6 +36,7 @@ import {
   expireOldLots,
   failJob,
   failKpoJob,
+  failStripeEvent,
   failWorkflowRunRecord,
   finishHarvestAttempt,
   generateApiKey,
@@ -87,7 +91,6 @@ import {
   putCode,
   putRefresh,
   reconcileBalanceMc,
-  recordStripeEvent,
   recordWorkflowWebhookDelivery,
   registerClient,
   replaceWorkflowArtifacts,
@@ -113,9 +116,10 @@ import {
   updateKpoJobState,
   upsertPageSnapshot,
   verifyPassword
-} from "./chunk-CSCD2HNS.js";
+} from "./chunk-N7KUTTCC.js";
 export {
   CORE_SCHEMA_VERSION,
+  CREDIT_LOTS_EPOCH,
   CREDIT_LOT_TTL,
   SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema,
@@ -126,8 +130,10 @@ export {
   claimMonthlyFreeRefresh,
   claimPendingJob,
   claimPendingKpoJob,
+  claimStripeEvent,
   completeJob,
   completeKpoJob,
+  completeStripeEvent,
   completeWorkflowRunRecord,
   consumeCode,
   consumePasswordResetToken,
@@ -149,6 +155,7 @@ export {
   expireOldLots,
   failJob,
   failKpoJob,
+  failStripeEvent,
   failWorkflowRunRecord,
   finishHarvestAttempt,
   generateApiKey,
@@ -203,7 +210,6 @@ export {
   putCode,
   putRefresh,
   reconcileBalanceMc,
-  recordStripeEvent,
   recordWorkflowWebhookDelivery,
   registerClient,
   replaceWorkflowArtifacts,
@@ -230,4 +236,4 @@ export {
   upsertPageSnapshot,
   verifyPassword
 };
-//# sourceMappingURL=db-N3YECFWR.js.map
+//# sourceMappingURL=db-C5KVCOYT.js.map

package/dist/{extract-bundle-346R6MXD.js → extract-bundle-KUBX6N6Z.js}

@@ -10,12 +10,12 @@ import {
 } from "./chunk-345BQXZH.js";
 import {
   extractJobLimitInfo
-} from "./chunk-Q35WZJJK.js";
-import "./chunk-PWPUKR5U.js";
+} from "./chunk-YCI2PNCS.js";
+import "./chunk-AZRPG43B.js";
 import "./chunk-44HZLHDV.js";
 import {
   getDb
-} from "./chunk-CSCD2HNS.js";
+} from "./chunk-N7KUTTCC.js";
 
 // src/api/extract-bundle.ts
 import { createWriteStream, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "fs";
@@ -565,4 +565,4 @@ export {
   SITE_EXTRACT_PAGE_CHUNK,
   assembleExtractArtifacts
 };
-//# sourceMappingURL=extract-bundle-346R6MXD.js.map
+//# sourceMappingURL=extract-bundle-KUBX6N6Z.js.map

package/dist/index.cjs

@@ -337,7 +337,7 @@ function currentCostContext() {
 // src/api/cost-rates.ts
 var KERNEL_HEADLESS_USD_PER_SEC = 166667e-10;
 var KERNEL_HEADFUL_USD_PER_SEC = 1333336e-10;
-var HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "maps_search", "yt_channel", "yt_transcription", "fb_search", "fb_ad", "instagram"]);
+var HEADLESS_OPS = /* @__PURE__ */ new Set(["serp", "fb_search", "fb_ad", "instagram"]);
 var FAL_WIZPER_USD_PER_COMPUTE_SEC = 111e-5;
 var FAL_WIZPER_WALLCLOCK_TO_COMPUTE_SEC_FACTOR = 4;
 var DEEPINFRA_QWEN_USD_PER_1K_OUT_TOKENS = 4e-4;
@@ -1020,8 +1020,10 @@ var BrowserDriver = class {
     return this.kernelClient ? "Google returned a CAPTCHA on this session \u2014 retrying with a fresh session." : RECAPTCHA_INSTRUCTIONS;
   }
   async navigateTo(url) {
+    const arrivalBase = process.env.ADS_ARRIVAL_REDIRECT_BASE?.trim();
+    const navUrl = arrivalBase ? serpArrivalUrl(url, arrivalBase) : url;
     try {
-      await this.page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
+      await this.page.goto(navUrl, { waitUntil: "domcontentloaded", timeout: 45e3 });
       if (/google\.[^/]+\/search\?/i.test(url) && await this.isSerpChallengeVisible()) {
         throw new CaptchaError(this.captchaMessage());
       }
@@ -1736,6 +1738,8 @@ var PAAExtractor = class {
     aioShareCaptured: null
   };
   aioShareUrlEarly = null;
+  collectedRows = [];
+  partialContext = null;
   normalizeQuestion(q) {
     return q.toLowerCase().replace(/[^\w\s]/g, "").replace(/\s+/g, " ").trim();
   }
@@ -1840,11 +1844,50 @@ var PAAExtractor = class {
       extracted_at: (/* @__PURE__ */ new Date()).toISOString()
     };
   }
+  getPartialResult() {
+    const ctx = this.partialContext;
+    if (!ctx || this.collectedRows.length === 0) return null;
+    const flat = [...this.collectedRows];
+    const stats = {
+      seed: ctx.seed,
+      totalQuestions: flat.length,
+      maxDepthReached: flat.reduce((m, r) => Math.max(m, r.depth), 0),
+      durationMs: Date.now() - ctx.startMs,
+      errorCount: 0
+    };
+    return {
+      seed: ctx.seed,
+      location: ctx.location,
+      extractedAt: (/* @__PURE__ */ new Date()).toISOString(),
+      diagnostics: {
+        completionStatus: "paa_found",
+        problem: null,
+        completeness: { ...this.completeness },
+        warnings: [{
+          code: "paa_partial",
+          surface: "paa",
+          message: `Harvest stopped before finishing; returning the ${flat.length} question(s) collected up to that point.`,
+          retryable: true
+        }]
+      },
+      totalQuestions: flat.length,
+      whatPeopleSaying: [],
+      tree: this.buildTree(flat, ctx.seed),
+      flat,
+      videos: [],
+      forums: [],
+      organicResults: [],
+      localPack: [],
+      entityIds: {},
+      stats
+    };
+  }
   async runBFS(page, options, signal) {
     const seenKeys = /* @__PURE__ */ new Set();
     const seenQs = /* @__PURE__ */ new Set();
     const orderedQs = [];
-    const results = [];
+    this.collectedRows = [];
+    const results = this.collectedRows;
     const abandonedNeverReclick = /* @__PURE__ */ new Set();
     const clickedOnceEver = /* @__PURE__ */ new Set();
     const itemArgs = {
@@ -2456,6 +2499,8 @@ var PAAExtractor = class {
     const startMs = Date.now();
     this.completeness = { paaWithoutAnswer: 0, paaWithoutSource: 0, paaAnswersRecovered: 0, aioShareCaptured: null };
     this.aioShareUrlEarly = null;
+    this.collectedRows = [];
+    this.partialContext = { seed: options.query, location: options.location ?? null, startMs };
     const isMobile = options.device === "mobile";
     const config = {
       headless: options.headless,
@@ -3517,7 +3562,23 @@ async function extractOnce(options, signal) {
     } : null);
     cleanup = await driver.close();
   }
-  return error ? { result: null, error, cleanup, debug } : { result, error: null, cleanup, debug };
+  if (error) {
+    const outcome = classifyAttemptError(error);
+    const budgetExhausted = outcome === "timeout" || outcome === "request_aborted";
+    let salvaged = null;
+    if (budgetExhausted) {
+      try {
+        salvaged = typeof extractor.getPartialResult === "function" ? extractor.getPartialResult() : null;
+      } catch {
+        salvaged = null;
+      }
+    }
+    if (salvaged) {
+      return { result: salvaged, error: null, cleanup, debug, salvagedFrom: error };
+    }
+    return { result: null, error, cleanup, debug };
+  }
+  return { result, error: null, cleanup, debug };
 }
 async function harvest(rawOptions) {
   const raw = typeof rawOptions === "object" && rawOptions !== null ? rawOptions : {};