mcp-scraper 0.43.2 → 0.43.4

package/dist/bin/paa-harvest.js

@@ -1,10 +1,10 @@
 #!/usr/bin/env node
 import {
   harvest
-} from "../chunk-H7LQ6PTI.js";
+} from "../chunk-6TAQ2MXK.js";
 import {
   browserServiceApiKey
-} from "../chunk-DAS7L4SF.js";
+} from "../chunk-ZDVQARDQ.js";
 import "../chunk-3ZUBQQPQ.js";
 import "../chunk-5UN33CGU.js";
 import "../chunk-FCFQ634B.js";

package/dist/{chunk-H7LQ6PTI.js → chunk-6TAQ2MXK.js}

@@ -1,11 +1,12 @@
 import {
+  HEADLESS_HARVEST_TEST_ENV,
   HEADLESS_OPS,
   browserServiceApiKey,
   browserServiceProxyId,
   currentCostContext,
   recordKernelSession,
   runWithCostContext
-} from "./chunk-DAS7L4SF.js";
+} from "./chunk-ZDVQARDQ.js";
 import {
   DEFAULT_PROXY_MODE,
   HarvestOptionsSchema,
@@ -130,10 +131,38 @@ var MapsSelectors = {
   expandReview: '[data-review-id] button[aria-label*="See more"], [data-review-id] button.w8nwRe'
 };
 
+// src/lib/browser-user-agent.ts
+var RUNTIME_BROWSER_UA = /\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;
+async function detectPageUserAgent(page) {
+  try {
+    const value = await page.evaluate(() => navigator.userAgent);
+    return typeof value === "string" && RUNTIME_BROWSER_UA.test(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+async function detectRuntimeUserAgent(browser) {
+  let probeContext = null;
+  try {
+    let page = browser.contexts().flatMap((context) => context.pages())[0];
+    if (!page) {
+      probeContext = await browser.newContext();
+      page = probeContext.pages()[0] ?? await probeContext.newPage();
+    }
+    return await detectPageUserAgent(page);
+  } catch {
+    return null;
+  } finally {
+    if (probeContext) await probeContext.close().catch(() => {
+    });
+  }
+}
+function withRuntimeUserAgent(options, runtimeUserAgent) {
+  return runtimeUserAgent ? { ...options, userAgent: runtimeUserAgent } : options;
+}
+
 // src/driver/BrowserDriver.ts
 chromium.use(StealthPlugin());
-var DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
 var KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
 var KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
@@ -168,21 +197,11 @@ async function ensureKernelProfile(kernelClient, name) {
     if (!isKernelProfileConflict(err)) throw err;
   }
 }
-async function detectRuntimeUserAgent(browser) {
-  try {
-    const page = browser.contexts()[0]?.pages()[0];
-    if (!page) return null;
-    const ua = await page.evaluate(() => navigator.userAgent);
-    return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
-  } catch {
-    return null;
-  }
-}
 function rankCheckContextOptions(config, runtimeUserAgent) {
   return {
     viewport: config.viewport,
     locale: config.locale,
-    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
+    ...runtimeUserAgent ? { userAgent: runtimeUserAgent } : {},
     ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
     ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -257,7 +276,7 @@ var BrowserDriver = class {
         viewport: config.viewport,
         locale: config.locale,
         device,
-        userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+        userAgent: null,
         deviceScaleFactor: config.deviceScaleFactor ?? null,
         isMobile: config.isMobile === true,
         hasTouch: config.hasTouch === true
@@ -345,7 +364,7 @@ var BrowserDriver = class {
       }
       this.browser = await playwrightChromium.connectOverCDP(kernelBrowser.cdp_ws_url);
       const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
-      this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
+      this.debugSnapshot.context.userAgent = runtimeUserAgent;
       this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
@@ -367,17 +386,19 @@ var BrowserDriver = class {
       ...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
       ...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
     };
-    const ctxOpts = rankCheckContextOptions(config);
     if (config.profileDir) {
       this.context = await chromium.launchPersistentContext(config.profileDir, {
         ...launchOpts,
-        ...ctxOpts
+        ...rankCheckContextOptions(config)
       });
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
+      this.debugSnapshot.context.userAgent = await detectPageUserAgent(this.page);
     } else {
       this.browser = await chromium.launch(launchOpts);
-      this.context = await this.browser.newContext(ctxOpts);
+      const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
+      this.debugSnapshot.context.userAgent = runtimeUserAgent;
+      this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
     }
@@ -1304,8 +1325,6 @@ async function extractAISurfacesFromDocument(config) {
 }
 
 // src/extractor/PAAExtractor.ts
-var DESKTOP_USER_AGENT2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_USER_AGENT2 = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var PAAExtractor = class {
   constructor(driver, reporter) {
     this.driver = driver;
@@ -2115,7 +2134,6 @@ var PAAExtractor = class {
       requireUsEgress: options.requireUsEgress,
       viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
-      userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
       deviceScaleFactor: isMobile ? 3 : 1,
       isMobile,
       hasTouch: isMobile,
@@ -3242,7 +3260,12 @@ async function harvest(rawOptions) {
         max_questions: attemptOptions.maxQuestions
       }));
       const baseCtx = currentCostContext();
-      const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
+      const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
+      const attempt = baseCtx ? await runWithCostContext({
+        ...baseCtx,
+        forceHeadless: allowHeadlessTest,
+        forceHeadful: !allowHeadlessTest
+      }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);
@@ -3433,6 +3456,9 @@ export {
   OrganicSelectors,
   LocalPackSelectors,
   MapsSelectors,
+  detectPageUserAgent,
+  detectRuntimeUserAgent,
+  withRuntimeUserAgent,
   serpArrivalUrl,
   buildYouTubeChannelVideosUrl,
   BrowserDriver,
@@ -3447,4 +3473,4 @@ export {
   resolveKernelProxyId,
   harvest
 };
-//# sourceMappingURL=chunk-H7LQ6PTI.js.map
+//# sourceMappingURL=chunk-6TAQ2MXK.js.map