mcp-scraper 0.33.6 → 0.34.0

package/dist/bin/mcp-stdio-server.js

@@ -7,19 +7,19 @@ import {
   registerMemoryMcpTools,
   registerPaaExtractorMcpTools,
   registerSerpIntelligenceCaptureTools
-} from "../chunk-6ZUK4H66.js";
+} from "../chunk-J32XSMJI.js";
 import "../chunk-R7EETU7Z.js";
-import "../chunk-WJJTGRK4.js";
+import "../chunk-V36LS5YV.js";
 import {
   renderInstallTerminal
-} from "../chunk-DUS4ENOY.js";
-import "../chunk-V73MPRU6.js";
+} from "../chunk-2EDFOQD7.js";
+import "../chunk-CB5C3BPB.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-F65IIUHI.js";
-import "../chunk-LKIP7VDN.js";
+} from "../chunk-NFMIHEYE.js";
+import "../chunk-JBSGBSGT.js";
 import "../chunk-M2S27J6Z.js";
-import "../chunk-AIK7WQD5.js";
+import "../chunk-62DQAWPF.js";
 
 // bin/mcp-stdio-server.ts
 import { readFileSync } from "fs";

package/dist/bin/paa-harvest.cjs

@@ -63,8 +63,23 @@ var HarvestOptionsSchema = import_zod.z.object({
   format: import_zod.z.enum(["json", "csv", "both"]).default("both"),
   serpOnly: import_zod.z.boolean().default(false),
   pages: import_zod.z.number().int().min(1).max(2).default(1),
+  recency: import_zod.z.enum(["day", "week", "month", "year"]).optional(),
   softDeadlineMs: import_zod.z.number().optional()
 });
+function recencyToTbs(recency) {
+  switch (recency) {
+    case "day":
+      return "qdr:d";
+    case "week":
+      return "qdr:w";
+    case "month":
+      return "qdr:m";
+    case "year":
+      return "qdr:y";
+    default:
+      return void 0;
+  }
+}
 var MapsPlaceOptionsSchema = import_zod.z.object({
   businessName: import_zod.z.string().min(1),
   location: import_zod.z.string().min(1),
@@ -836,6 +851,7 @@ var BrowserDriver = class {
   async navigateToSERP(query, uule, gl, hl, options) {
     const params = new URLSearchParams({ q: query, gl, hl, pws: "0" });
     if (options?.num) params.set("num", String(options.num));
+    if (options?.tbs) params.set("tbs", options.tbs);
     if (uule) params.set("uule", uule);
     const url = "https://www.google.com/search?" + params.toString();
     const navUrl = serpArrivalUrl(url);
@@ -2402,6 +2418,7 @@ var PAAExtractor = class {
         options.hl,
         {
           ...options.serpOnly ? { num: 100 } : {},
+          ...recencyToTbs(options.recency) ? { tbs: recencyToTbs(options.recency) } : {},
           debug: options.debug
         }
       );
@@ -2421,6 +2438,8 @@ var PAAExtractor = class {
         let allOrganic2 = organicResults2;
         if ((options.pages ?? 1) >= 2) {
           const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
           await this.throwIfCaptcha(page, "Google SERP page 2");
@@ -2480,6 +2499,8 @@ var PAAExtractor = class {
         let locationEvidence2 = initialLocationEvidence;
         if ((options.pages ?? 1) >= 2) {
           const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
+          if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
           if (uule) p2params.set("uule", uule);
           await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
           await this.throwIfCaptcha(page, "Google SERP page 2");
@@ -2546,6 +2567,7 @@ var PAAExtractor = class {
       let locationEvidence = initialLocationEvidence;
       if ((options.pages ?? 1) >= 2) {
         const p2params = new URLSearchParams({ q: options.query, gl: options.gl, hl: options.hl, pws: "0", start: "10" });
+        if (recencyToTbs(options.recency)) p2params.set("tbs", recencyToTbs(options.recency));
         if (uule) p2params.set("uule", uule);
         await this.driver.navigateTo("https://www.google.com/search?" + p2params.toString());
         await this.throwIfCaptcha(page, "Google SERP page 2");
@@ -2731,7 +2753,76 @@ var ProgressReporter = class {
 };
 
 // src/kernel-proxy-resolver.ts
+var import_sdk3 = __toESM(require("@onkernel/sdk"), 1);
+
+// src/backup-proxy.ts
+var import_node_crypto3 = require("crypto");
 var import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
+var BACKUP_PROXY_HOST = "pr.oxylabs.io";
+var BACKUP_PROXY_PORT = 7777;
+var BACKUP_STICKY_SESSION_MINUTES = 10;
+var BACKUP_LAST_RESORT_ATTEMPTS = 2;
+function backupProxyUsername() {
+  const value = process.env.OXYLABS_USERNAME?.trim();
+  return value || void 0;
+}
+function backupProxyPassword() {
+  const value = process.env.OXYLABS_PASSWORD?.trim();
+  return value || void 0;
+}
+function backupProxyCountry() {
+  const value = process.env.OXYLABS_COUNTRY?.trim();
+  return (value || "US").toUpperCase();
+}
+function backupProxyAvailable() {
+  return Boolean(backupProxyUsername() && backupProxyPassword());
+}
+function randomStickySessionId() {
+  let digits = "";
+  for (let i = 0; i < 10; i++) digits += String((0, import_node_crypto3.randomInt)(0, 10));
+  return digits;
+}
+function buildBackupUsername(opts = {}) {
+  const user = backupProxyUsername();
+  if (!user) return void 0;
+  const base = user.startsWith("customer-") ? user : `customer-${user}`;
+  const country = (opts.country?.trim() || backupProxyCountry()).toUpperCase();
+  const sticky = opts.sticky ? `-sessid-${randomStickySessionId()}-sesstime-${BACKUP_STICKY_SESSION_MINUTES}` : "";
+  return `${base}-cc-${country}${sticky}`;
+}
+function scrubBackupProxyText(message) {
+  return message.replace(/pr\.oxylabs\.io(:\d+)?/gi, "backup proxy").replace(/oxylabs/gi, "backup proxy");
+}
+function backupProxyName(attemptIndex) {
+  const stamp = `${Date.now()}-${attemptIndex ?? 0}-${Math.random().toString(36).slice(2, 8)}`;
+  return `mcp-backup-custom-${stamp}`;
+}
+async function createBackupProxyId(kernelApiKey, opts = {}) {
+  const username = buildBackupUsername({ sticky: opts.sticky });
+  const password = backupProxyPassword();
+  if (!username || !password) throw new Error("backup proxy pool is not configured");
+  const kernel = new import_sdk2.default({ apiKey: kernelApiKey });
+  try {
+    const created = await kernel.proxies.create({
+      type: "custom",
+      name: backupProxyName(opts.attemptIndex),
+      protocol: "http",
+      config: {
+        host: BACKUP_PROXY_HOST,
+        port: BACKUP_PROXY_PORT,
+        username,
+        password
+      }
+    });
+    if (!created.id) throw new Error("backup proxy creation did not return a proxy id");
+    return created.id;
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    throw new Error(scrubBackupProxyText(message));
+  }
+}
+
+// src/kernel-proxy-resolver.ts
 var US_STATE_CODES = {
   alabama: "AL",
   alaska: "AK",
@@ -2829,7 +2920,7 @@ function proxyIdSuffix2(proxyId) {
   return proxyId ? proxyId.slice(-6) : null;
 }
 function resolution(source, proxyMode, proxyId, target, error, disposable = false) {
-  const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
+  const proxyType = source === "disabled" ? "none" : source === "configured_fallback" ? "configured" : source === "isp_created" ? "isp" : source === "backup_custom" ? "custom" : source === "location_created" || source === "location_reused" ? "residential" : "unknown";
   return {
     kernelProxyId: proxyId,
     ...disposable && proxyId ? { disposableProxyId: proxyId } : {},
@@ -3013,6 +3104,26 @@ function freshTargetCandidates(target, explicitZip, attemptIndex) {
   out.push(withProxyName(fallbackTarget, freshProxyName(fallbackTarget.proxyName, attemptIndex)));
   return out;
 }
+async function settleCreateExhaustion(options, target, createErrors) {
+  if (options.backupOnExhaustion && createErrors.length > 0 && options.kernelApiKey && backupProxyAvailable()) {
+    try {
+      const backupId = await createBackupProxyId(options.kernelApiKey, { attemptIndex: options.attemptIndex });
+      return resolution("backup_custom", options.proxyMode, backupId, target, null, true);
+    } catch (err) {
+      createErrors.push(`backup: ${scrubBackupProxyText(errorText2(err))}`);
+    }
+  }
+  return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+}
+async function resolveBackupKernelProxyId(options) {
+  if (!options.kernelApiKey || !backupProxyAvailable()) return null;
+  try {
+    const backupId = await createBackupProxyId(options.kernelApiKey, { attemptIndex: options.attemptIndex });
+    return resolution("backup_custom", options.proxyMode, backupId, null, null, true);
+  } catch {
+    return null;
+  }
+}
 async function createFreshLocationProxy(kernel, options, target) {
   const createErrors = [];
   for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
@@ -3046,11 +3157,11 @@ async function createFreshLocationProxy(kernel, options, target) {
       createErrors.push(`isp: ${errorText2(err)}`);
     }
   }
-  return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+  return settleCreateExhaustion(options, target, createErrors);
 }
 async function deleteKernelProxyId(kernelApiKey, proxyId) {
   if (!kernelApiKey || !proxyId) return;
-  const kernel = new import_sdk2.default({ apiKey: kernelApiKey });
+  const kernel = new import_sdk3.default({ apiKey: kernelApiKey });
   await kernel.proxies.delete(proxyId);
 }
 async function resolveKernelProxyId(options) {
@@ -3064,7 +3175,7 @@ async function resolveKernelProxyId(options) {
   if (!target || !options.kernelApiKey) {
     return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, target ? null : "location could not be normalized to a US city/state proxy target");
   }
-  const kernel = new import_sdk2.default({ apiKey: options.kernelApiKey });
+  const kernel = new import_sdk3.default({ apiKey: options.kernelApiKey });
   try {
     const attemptIndex = options.attemptIndex ?? 0;
     if (options.fresh) {
@@ -3086,7 +3197,7 @@ async function resolveKernelProxyId(options) {
       } catch (err) {
         createErrors2.push(`${escalatedTarget.state}: ${errorText2(err)}`);
       }
-      return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, escalatedTarget, createErrors2.join(" | "));
+      return settleCreateExhaustion(options, escalatedTarget, createErrors2);
     }
     const proxies = await kernel.proxies.list();
     const zip = knownZipFor(target, options.proxyZip);
@@ -3165,7 +3276,7 @@ async function resolveKernelProxyId(options) {
     } catch (err) {
       createErrors.push(`${fallbackTarget.state}: ${errorText2(err)}`);
     }
-    return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
+    return settleCreateExhaustion(options, target, createErrors);
   } catch (err) {
     return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, errorText2(err));
   }
@@ -3330,7 +3441,9 @@ async function harvest(rawOptions) {
   const requestedDebug = typeof raw.debug === "boolean" ? raw.debug : false;
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
   const requestedMaxAttempts = raw.maxAttempts;
-  const maxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
+  const baseMaxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
+  const maxAttempts = baseMaxAttempts + (backupProxyAvailable() ? BACKUP_LAST_RESORT_ATTEMPTS : 0);
+  const softDeadlineMs = typeof raw.softDeadlineMs === "number" ? raw.softDeadlineMs : null;
   const serializer = new OutputSerializer();
   let lastError = null;
   let effectiveProxyMode = proxyMode;
@@ -3340,12 +3453,16 @@ async function harvest(rawOptions) {
     const startedAtMs = Date.now();
     try {
       if (signal?.aborted) throw abortReason(signal);
-      const resolution2 = await resolveKernelProxyId({
+      const isBackupAttempt = i >= baseMaxAttempts;
+      if (isBackupAttempt && softDeadlineMs !== null && Date.now() >= softDeadlineMs) break;
+      const resolution2 = isBackupAttempt ? await resolveBackupKernelProxyId({ kernelApiKey, proxyMode: effectiveProxyMode, attemptIndex: i }) : await resolveKernelProxyId({
         ...proxyOpts,
         proxyMode: effectiveProxyMode,
         attemptIndex: i,
-        fresh: effectiveProxyMode === "location"
+        fresh: effectiveProxyMode === "location",
+        backupOnExhaustion: true
       });
+      if (!resolution2) break;
       const mergedAttempt = {
         ...raw,
         kernelApiKey,
@@ -3354,7 +3471,7 @@ async function harvest(rawOptions) {
         proxyMode: effectiveProxyMode,
         debug: requestedDebug || needsLocationEvidence
       };
-      if (effectiveProxyMode === "none") mergedAttempt.kernelProxyId = void 0;
+      if (effectiveProxyMode === "none" && !isBackupAttempt) mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",
@@ -3373,10 +3490,8 @@ async function harvest(rawOptions) {
         location: attemptOptions.location ?? null,
         max_questions: attemptOptions.maxQuestions
       }));
-      const isHeadfulEscalation = i === maxAttempts - 1;
-      const needHeadful = isHeadfulEscalation || !attemptOptions.serpOnly;
       const baseCtx = currentCostContext();
-      const attempt = needHeadful && baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
+      const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome = classifyAttemptError(err);