mcp-scraper 0.42.3 → 0.43.2

package/dist/{extract-bundle-FH734F3R.js → extract-bundle-IOM45SFL.js}

@@ -1,17 +1,17 @@
 import {
   createSiteExtractBundleArtifactStream,
   downloadAsset
-} from "./chunk-6Q7V4IQV.js";
+} from "./chunk-6WLNXYRG.js";
 import {
   computeIssues,
   renderImageSection,
   renderIssueReport,
   renderLinkReport
-} from "./chunk-MJKVDLVR.js";
+} from "./chunk-2K74LVV7.js";
 import {
   extractJobLimitInfo
-} from "./chunk-G5MHJGKE.js";
-import "./chunk-IMUA3FJB.js";
+} from "./chunk-GUI33Q27.js";
+import "./chunk-X5WDJ7D7.js";
 import "./chunk-5UN33CGU.js";
 import {
   getDb
@@ -565,4 +565,4 @@ export {
   SITE_EXTRACT_PAGE_CHUNK,
   assembleExtractArtifacts
 };
-//# sourceMappingURL=extract-bundle-FH734F3R.js.map
+//# sourceMappingURL=extract-bundle-IOM45SFL.js.map

package/dist/index.cjs

@@ -601,11 +601,21 @@ async function ensureKernelProfile(kernelClient, name) {
     if (!isKernelProfileConflict(err)) throw err;
   }
 }
-function rankCheckContextOptions(config) {
+async function detectRuntimeUserAgent(browser) {
+  try {
+    const page = browser.contexts()[0]?.pages()[0];
+    if (!page) return null;
+    const ua = await page.evaluate(() => navigator.userAgent);
+    return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
+  } catch {
+    return null;
+  }
+}
+function rankCheckContextOptions(config, runtimeUserAgent) {
   return {
     viewport: config.viewport,
     locale: config.locale,
-    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
     ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
     ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -702,7 +712,7 @@ var BrowserDriver = class {
       this.kernelHeadlessSent = useHeadless ? true : null;
       if (costCtx?.headlessSentOut) costCtx.headlessSentOut.value = this.kernelHeadlessSent;
       const kernelBrowser = await this.kernelClient.browsers.create({
-        stealth: true,
+        stealth: config.kernelStealth === true,
         timeout_seconds: timeoutSeconds,
         ...useHeadless ? { headless: true } : {},
         ...config.kernelProxyId ? { proxy_id: config.kernelProxyId } : {},
@@ -767,7 +777,9 @@ var BrowserDriver = class {
         await this.populateKernelRetrieveDebug(kernelDebug, config.kernelProxyId);
       }
       this.browser = await import_playwright.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
-      this.context = await this.browser.newContext(rankCheckContextOptions(config));
+      const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
+      this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
+      this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
       await this.installEsbuildHelperShims(this.context);
       this.page = await this.context.newPage();
       await this.page.setViewportSize(config.viewport);
@@ -2900,7 +2912,6 @@ var import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
 var BACKUP_PROXY_HOST = "pr.oxylabs.io";
 var BACKUP_PROXY_PORT = 7777;
 var BACKUP_STICKY_SESSION_MINUTES = 10;
-var BACKUP_LAST_RESORT_ATTEMPTS = 2;
 function backupProxyUsername() {
   const value = process.env.OXYLABS_USERNAME?.trim();
   return value || void 0;
@@ -3254,15 +3265,6 @@ async function settleCreateExhaustion(options, target, createErrors) {
   }
   return resolution("configured_fallback", options.proxyMode, options.configuredKernelProxyId, target, createErrors.join(" | "));
 }
-async function resolveBackupKernelProxyId(options) {
-  if (!options.kernelApiKey || !backupProxyAvailable()) return null;
-  try {
-    const backupId = await createBackupProxyId(options.kernelApiKey, { attemptIndex: options.attemptIndex });
-    return resolution("backup_custom", options.proxyMode, backupId, null, null, true);
-  } catch {
-    return null;
-  }
-}
 async function createFreshLocationProxy(kernel, options, target) {
   const createErrors = [];
   for (const candidate of freshTargetCandidates(target, options.proxyZip, options.attemptIndex)) {
@@ -3622,7 +3624,7 @@ async function harvest(rawOptions) {
   const needsLocationEvidence = proxyMode === "location" && Boolean(proxyOpts.location);
   const requestedMaxAttempts = raw.maxAttempts;
   const baseMaxAttempts = typeof requestedMaxAttempts === "number" && Number.isInteger(requestedMaxAttempts) && requestedMaxAttempts >= 1 ? requestedMaxAttempts : maxAttemptsForProxyMode(proxyMode) + 1;
-  const maxAttempts = baseMaxAttempts + (backupProxyAvailable() ? BACKUP_LAST_RESORT_ATTEMPTS : 0);
+  const maxAttempts = baseMaxAttempts;
   const softDeadlineMs = typeof raw.softDeadlineMs === "number" ? raw.softDeadlineMs : null;
   const serializer = new OutputSerializer();
   let lastError = null;
@@ -3633,14 +3635,11 @@ async function harvest(rawOptions) {
     const startedAtMs = Date.now();
     try {
       if (signal?.aborted) throw abortReason(signal);
-      const isBackupAttempt = i >= baseMaxAttempts;
-      if (isBackupAttempt && softDeadlineMs !== null && Date.now() >= softDeadlineMs) break;
-      const resolution2 = isBackupAttempt ? await resolveBackupKernelProxyId({ kernelApiKey, proxyMode: effectiveProxyMode, attemptIndex: i }) : await resolveKernelProxyId({
+      const resolution2 = await resolveKernelProxyId({
         ...proxyOpts,
         proxyMode: effectiveProxyMode,
         attemptIndex: i,
-        fresh: effectiveProxyMode === "location",
-        backupOnExhaustion: true
+        fresh: effectiveProxyMode === "location"
       });
       if (!resolution2) break;
       const mergedAttempt = {
@@ -3649,9 +3648,10 @@ async function harvest(rawOptions) {
         kernelProxyId: resolution2.kernelProxyId,
         kernelProxyResolution: resolution2.resolution,
         proxyMode: effectiveProxyMode,
+        keepDefaultProxy: i > 0,
         debug: requestedDebug || needsLocationEvidence
       };
-      if (effectiveProxyMode === "none" && !isBackupAttempt) mergedAttempt.kernelProxyId = void 0;
+      if (effectiveProxyMode === "none") mergedAttempt.kernelProxyId = void 0;
       const attemptOptions = HarvestOptionsSchema.parse(mergedAttempt);
       await emitAttemptEvent(onAttemptEvent, {
         type: "started",