mcp-scraper 0.43.2 → 0.43.4

package/dist/bin/mcp-stdio-server.js

@@ -11,16 +11,16 @@ import {
   registerScheduledResultsMcpTools,
   registerSerpIntelligenceCaptureTools,
   resolveDeploymentProfile
-} from "../chunk-YJNGNMDO.js";
+} from "../chunk-SXLQZKWC.js";
 import "../chunk-2K74LVV7.js";
 import "../chunk-BBI7RGOT.js";
-import "../chunk-DAS7L4SF.js";
+import "../chunk-ZDVQARDQ.js";
 import {
   renderInstallTerminal
 } from "../chunk-TC4GDB2Q.js";
 import {
   PACKAGE_VERSION
-} from "../chunk-MD75M2L5.js";
+} from "../chunk-XQIVXIAL.js";
 import "../chunk-X5WDJ7D7.js";
 import "../chunk-5UN33CGU.js";
 import "../chunk-FCFQ634B.js";

package/dist/bin/paa-harvest.cjs

@@ -319,6 +319,7 @@ var SiteAuditPhaseLogRowSchema = import_zod2.z.object({
 
 // src/api/cost-context.ts
 var import_node_async_hooks = require("async_hooks");
+var HEADLESS_HARVEST_TEST_ENV = "MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST";
 var storage = new import_node_async_hooks.AsyncLocalStorage();
 function runWithCostContext(ctx, fn) {
   return storage.run(ctx, fn);
@@ -508,10 +509,35 @@ function boolToInt(v) {
   return v ? 1 : 0;
 }
 
+// src/lib/browser-user-agent.ts
+var RUNTIME_BROWSER_UA = /\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;
+async function detectPageUserAgent(page) {
+  try {
+    const value = await page.evaluate(() => navigator.userAgent);
+    return typeof value === "string" && RUNTIME_BROWSER_UA.test(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+async function detectRuntimeUserAgent(browser) {
+  let probeContext = null;
+  try {
+    let page = browser.contexts().flatMap((context) => context.pages())[0];
+    if (!page) {
+      probeContext = await browser.newContext();
+      page = probeContext.pages()[0] ?? await probeContext.newPage();
+    }
+    return await detectPageUserAgent(page);
+  } catch {
+    return null;
+  } finally {
+    if (probeContext) await probeContext.close().catch(() => {
+    });
+  }
+}
+
 // src/driver/BrowserDriver.ts
 import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
-var DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
 var KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
 var KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
@@ -546,21 +572,11 @@ async function ensureKernelProfile(kernelClient, name) {
     if (!isKernelProfileConflict(err)) throw err;
   }
 }
-async function detectRuntimeUserAgent(browser) {
-  try {
-    const page = browser.contexts()[0]?.pages()[0];
-    if (!page) return null;
-    const ua = await page.evaluate(() => navigator.userAgent);
-    return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
-  } catch {
-    return null;
-  }
-}
 function rankCheckContextOptions(config, runtimeUserAgent) {
   return {
     viewport: config.viewport,
     locale: config.locale,
-    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
+    ...runtimeUserAgent ? { userAgent: runtimeUserAgent } : {},
     ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
     ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -635,7 +651,7 @@ var BrowserDriver = class {
         viewport: config.viewport,
         locale: config.locale,
         device,
-        userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+        userAgent: null,
         deviceScaleFactor: config.deviceScaleFactor ?? null,
         isMobile: config.isMobile === true,
         hasTouch: config.hasTouch === true
@@ -723,7 +739,7 @@ var BrowserDriver = class {
       }
       this.browser = await import_playwright.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
       const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
-      this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
+      this.debugSnapshot.context.userAgent = runtimeUserAgent;
       this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
@@ -745,17 +761,19 @@ var BrowserDriver = class {
       ...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
       ...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
     };
-    const ctxOpts = rankCheckContextOptions(config);
     if (config.profileDir) {
       this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
         ...launchOpts,
-        ...ctxOpts
+        ...rankCheckContextOptions(config)
       });
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
+      this.debugSnapshot.context.userAgent = await detectPageUserAgent(this.page);
     } else {
       this.browser = await import_playwright_extra.chromium.launch(launchOpts);
-      this.context = await this.browser.newContext(ctxOpts);
+      const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
+      this.debugSnapshot.context.userAgent = runtimeUserAgent;
+      this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
     }
@@ -1682,8 +1700,6 @@ async function extractAISurfacesFromDocument(config) {
 }
 
 // src/extractor/PAAExtractor.ts
-var DESKTOP_USER_AGENT2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_USER_AGENT2 = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var PAAExtractor = class {
   constructor(driver, reporter) {
     this.driver = driver;
@@ -2493,7 +2509,6 @@ var PAAExtractor = class {
       requireUsEgress: options.requireUsEgress,
       viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
       locale: `${options.hl}-${options.gl.toUpperCase()}`,
-      userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
       deviceScaleFactor: isMobile ? 3 : 1,
       isMobile,
       hasTouch: isMobile,
@@ -3616,7 +3631,12 @@ async function harvest(rawOptions) {
         max_questions: attemptOptions.maxQuestions
       }));
       const baseCtx = currentCostContext();
-      const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
+      const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
+      const attempt = baseCtx ? await runWithCostContext({
+        ...baseCtx,
+        forceHeadless: allowHeadlessTest,
+        forceHeadful: !allowHeadlessTest
+      }, () => extractOnce(attemptOptions, signal)) : await extractOnce(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);