mcp-scraper 0.42.2 → 0.43.1

package/dist/{chunk-DH42LIR3.js → chunk-LXVIQZ7Y.js}

@@ -170,11 +170,21 @@ async function ensureKernelProfile(kernelClient, name) {
     if (!isKernelProfileConflict(err)) throw err;
   }
 }
-function rankCheckContextOptions(config) {
+async function detectRuntimeUserAgent(browser) {
+  try {
+    const page = browser.contexts()[0]?.pages()[0];
+    if (!page) return null;
+    const ua = await page.evaluate(() => navigator.userAgent);
+    return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
+  } catch {
+    return null;
+  }
+}
+function rankCheckContextOptions(config, runtimeUserAgent) {
   return {
     viewport: config.viewport,
     locale: config.locale,
-    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
     ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
     ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -271,7 +281,7 @@ var BrowserDriver = class {
       this.kernelHeadlessSent = useHeadless ? true : null;
       if (costCtx?.headlessSentOut) costCtx.headlessSentOut.value = this.kernelHeadlessSent;
       const kernelBrowser = await this.kernelClient.browsers.create({
-        stealth: true,
+        stealth: config.kernelStealth === true,
         timeout_seconds: timeoutSeconds,
         ...useHeadless ? { headless: true } : {},
         ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {},
@@ -336,7 +346,9 @@ var BrowserDriver = class {
         await this.populateKernelRetrieveDebug(kernelDebug, config.kernelProxyId);
       }
       this.browser = await playwrightChromium.connectOverCDP(kernelBrowser.cdp_ws_url);
-      this.context = await this.browser.newContext(rankCheckContextOptions(config));
+      const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
+      this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
+      this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
       await this.page.setViewportSize(config.viewport);
@@ -3185,7 +3197,7 @@ async function harvest(rawOptions) {
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
   const requestedMaxAttempts = raw.maxAttempts;
   const baseMaxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
-  const maxAttempts = baseMaxAttempts + (backupProxyAvailable() ? BACKUP_LAST_RESORT_ATTEMPTS : 0);
+  const maxAttempts = baseMaxAttempts;
   const softDeadlineMs = typeof raw.softDeadlineMs === "number" ? raw.softDeadlineMs : null;
   const serializer = new OutputSerializer();
   let lastError = null;
@@ -3196,14 +3208,11 @@ async function harvest(rawOptions) {
     const startedAtMs = Date.now();
     try {
       if (signal?.aborted) throw abortReason(signal);
-      const isBackupAttempt = i >= baseMaxAttempts;
-      if (isBackupAttempt && softDeadlineMs !== null && Date.now() >= softDeadlineMs) break;
-      const resolution2 = isBackupAttempt ? await resolveBackupKernelProxyId({ kernelApiKey, proxyMode: effectiveProxyMode, attemptIndex: i }) : await resolveKernelProxyId({
+      const resolution2 = await resolveKernelProxyId({
         ...proxyOpts,
         proxyMode: effectiveProxyMode,
         attemptIndex: i,
-        fresh: effectiveProxyMode === "location",
-        backupOnExhaustion: true
+        fresh: effectiveProxyMode === "location"
       });
       if (!resolution2) break;
       const mergedAttempt = {
@@ -3212,9 +3221,10 @@ async function harvest(rawOptions) {
         kernelProxyId: resolution2.kernelProxyId,
         kernelProxyResolution: resolution2.resolution,
         proxyMode: effectiveProxyMode,
+        keepDefaultProxy: i > 0,
         debug: requestedDebug || needsLocationEvidence
       };
-      if (effectiveProxyMode === "none" && !isBackupAttempt) mergedAttempt.kernelProxyId = void 0;
+      if (effectiveProxyMode === "none") mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",
@@ -3439,4 +3449,4 @@ export {
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-DH42LIR3.js.map
+//# sourceMappingURL=chunk-LXVIQZ7Y.js.map