mcp-scraper 0.43.2 → 0.43.4

package/dist/index.d.cts

@@ -150,7 +150,7 @@ interface BrowserContextDebug {
     };
     locale: string;
     device: SerpDevice;
-    userAgent: string;
+    userAgent: string | null;
     deviceScaleFactor: number | null;
     isMobile: boolean;
     hasTouch: boolean;

package/dist/index.d.ts

@@ -150,7 +150,7 @@ interface BrowserContextDebug {
     };
     locale: string;
     device: SerpDevice;
-    userAgent: string;
+    userAgent: string | null;
     deviceScaleFactor: number | null;
     isMobile: boolean;
     hasTouch: boolean;

package/dist/index.js

@@ -1,9 +1,9 @@
 import {
   harvest
-} from "./chunk-H7LQ6PTI.js";
+} from "./chunk-6TAQ2MXK.js";
 import {
   recordVendorUsage
-} from "./chunk-DAS7L4SF.js";
+} from "./chunk-ZDVQARDQ.js";
 import "./chunk-3ZUBQQPQ.js";
 import "./chunk-5UN33CGU.js";
 import "./chunk-FCFQ634B.js";

package/dist/{server-BPEXEKA6.js → server-K76DL4S7.js}

@@ -35,13 +35,16 @@ import {
   cleanupBackupProxyId,
   createBackupProxyIdSafe,
   deleteKernelProxyId,
+  detectPageUserAgent,
+  detectRuntimeUserAgent,
   encodeUule,
   harvest,
   normalizeLocation,
   resolveBackupKernelProxyId,
   resolveKernelProxyId,
-  serpArrivalUrl
-} from "./chunk-H7LQ6PTI.js";
+  serpArrivalUrl,
+  withRuntimeUserAgent
+} from "./chunk-6TAQ2MXK.js";
 import {
   ApiErrorResponseSchema,
   ArtifactTemplateConfigSchema,
@@ -83,7 +86,7 @@ import {
   renewDirectoryArtifactDownload,
   resolveDeploymentProfile,
   transcribeMediaUrl
-} from "./chunk-YJNGNMDO.js";
+} from "./chunk-SXLQZKWC.js";
 import {
   auditImageUrls,
   auditImages,
@@ -111,7 +114,7 @@ import {
   recordKernelSession,
   recordVendorUsage,
   runWithCostContext
-} from "./chunk-DAS7L4SF.js";
+} from "./chunk-ZDVQARDQ.js";
 import {
   listWorkflowDefinitions,
   runWorkflow,
@@ -139,7 +142,7 @@ import {
 } from "./chunk-3ZUBQQPQ.js";
 import {
   PACKAGE_VERSION
-} from "./chunk-MD75M2L5.js";
+} from "./chunk-XQIVXIAL.js";
 import {
   abandonExtractSettlement,
   countSuccessfulPages,
@@ -3749,10 +3752,10 @@ async function fetchWithKernel(url) {
   const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
   const browser = await chromium.connectOverCDP(kb.cdp_ws_url);
   try {
-    const context = await browser.newContext({
-      userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+    const context = await browser.newContext(withRuntimeUserAgent({
       serviceWorkers: "block"
-    });
+    }, runtimeUserAgent));
     const page = context.pages()[0] ?? await context.newPage();
     const guard = await installBrowserRequestGuard(page, auditedUrl);
     await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
@@ -4296,8 +4299,6 @@ async function extractBrandingFromPage(page) {
 // src/lib/screenshot.ts
 var DESKTOP_VIEWPORT = { width: 1440, height: 900 };
 var MOBILE_VIEWPORT = { width: 390, height: 844 };
-var DESKTOP_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-var MOBILE_UA = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
 var SCROLL_STEP = 900;
 var SETTLE_MS = 400;
 async function scrollAndStitch(page, device = "desktop") {
@@ -4348,14 +4349,13 @@ async function scrollAndStitch(page, device = "desktop") {
 }
 async function captureLocal(url, device, wantScreenshot, wantBranding) {
   const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
-  const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
   const browser = await chromium2.launch({ headless: true });
-  const context = await browser.newContext({
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  const context = await browser.newContext(withRuntimeUserAgent({
     viewport: vp,
     locale: "en-US",
-    userAgent: ua,
     isMobile: device === "mobile"
-  });
+  }, runtimeUserAgent));
   const page = await context.newPage();
   try {
     await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -4374,12 +4374,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
   const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
   const browser = await chromium2.connectOverCDP(session.cdp_ws_url);
   const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
-  const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
-  const context = browser.contexts()[0] ?? await browser.newContext({
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  const context = browser.contexts()[0] ?? await browser.newContext(withRuntimeUserAgent({
     viewport: vp,
-    userAgent: ua,
     isMobile: device === "mobile"
-  });
+  }, runtimeUserAgent));
   const page = context.pages()[0] ?? await context.newPage();
   try {
     await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -4885,6 +4884,7 @@ async function crawlWithRotation(urls, opts) {
     const worker = async () => {
       let kb = null;
       let browser = null;
+      let runtimeUserAgent = null;
       let inBrowser = 0;
       let kbOpenedAtMs = 0;
       const retire = async () => {
@@ -4892,6 +4892,7 @@ async function crawlWithRotation(urls, opts) {
         browser = null;
         kb = null;
         kbOpenedAtMs = 0;
+        runtimeUserAgent = null;
         if (b) await b.close().catch(() => {
         });
         if (k) {
@@ -4913,9 +4914,10 @@ async function crawlWithRotation(urls, opts) {
               kb = await createBrowserWithRetry(client2, headless);
               kbOpenedAtMs = Date.now();
               browser = await withTimeout(chromium3.connectOverCDP(kb.cdp_ws_url), 3e4);
+              runtimeUserAgent = await detectRuntimeUserAgent(browser);
               inBrowser = 0;
             }
-            const context = await browser.newContext({ serviceWorkers: "block" });
+            const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
             try {
               r = await crawlPage(context, url);
             } finally {
@@ -16681,7 +16683,6 @@ facebookAdApp.post("/media", createApiKeyAuth(), async (c) => {
   try {
     res = await fetch(parsedUrl.href, {
       headers: {
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
         "Referer": "https://www.facebook.com/"
       },
       signal: AbortSignal.timeout(6e4)
@@ -17998,10 +17999,10 @@ function blobKeyPrefix(shortcode, sourceUrl) {
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
   return `instagram/${stamp}-${slug}`;
 }
-async function fetchMediaBytes(url, referer) {
+async function fetchMediaBytes(url, referer, runtimeUserAgent) {
   const res = await fetch(url, {
     headers: {
-      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      ...runtimeUserAgent ? { "User-Agent": runtimeUserAgent } : {},
       "Referer": referer
     },
     signal: AbortSignal.timeout(9e4)
@@ -18121,6 +18122,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
     const launch = await resolveInstagramLaunch(body);
     await driver.launch(launch.config);
     const page = driver.getPage();
+    const runtimeUserAgent = await detectPageUserAgent(page);
     const capturedMediaUrls = [];
     page.on("response", (response) => {
       const url = normalizeInstagramMediaUrl(response.url());
@@ -18149,7 +18151,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
         downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
         if (mediaTypes.has("image") && extraction.imageUrl) {
           try {
-            const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
+            const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl, runtimeUserAgent);
             const ext = extFromMime(mimeType, "jpg");
             const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
             downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
@@ -18165,7 +18167,7 @@ instagramApp.post("/media-download", createApiKeyAuth(), async (c) => {
           if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
           const kind = track.streamType === "audio" ? "audio" : "video";
           try {
-            const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
+            const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl, runtimeUserAgent);
             const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
             downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
             const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
@@ -24041,6 +24043,12 @@ async function runFanoutCapture(page, input) {
 // src/services/browser-agent/browser-agent-service.ts
 function keepHostedBrowserAlive(_browser) {
 }
+async function currentOrRuntimeContext(browser) {
+  const current = browser.contexts()[0];
+  if (current) return current;
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  return browser.newContext(withRuntimeUserAgent({}, runtimeUserAgent));
+}
 var DEFAULT_TIMEOUT_SECONDS = 600;
 function client() {
   const apiKey = browserServiceApiKey();
@@ -24165,7 +24173,7 @@ async function pressKeys(runtimeSessionId, keys) {
 async function goto(cdpWsUrl, url) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
     return { url: page.url(), title: await page.title() };
@@ -24176,7 +24184,7 @@ async function goto(cdpWsUrl, url) {
 async function captureFanout(cdpWsUrl, input) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     return await runFanoutCapture(page, input);
   } finally {
@@ -24186,7 +24194,7 @@ async function captureFanout(cdpWsUrl, input) {
 async function readPage(cdpWsUrl) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     const url = page.url();
     const title = await page.title().catch(() => "");
@@ -24241,7 +24249,7 @@ async function readPage(cdpWsUrl) {
 async function locatePageTargets(cdpWsUrl, targets) {
   const browser = await playwrightChromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     const url = page.url();
     const title = await page.title().catch(() => "");
@@ -34346,8 +34354,7 @@ app.use("*", (c, next) => {
   const op = opForCostPath(new URL(c.req.url).pathname);
   if (!op) return next();
   const probeRunId = c.req.header("x-cost-probe-id") ?? null;
-  const forceHeadless = c.req.header("x-kernel-headless") === "true";
-  return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
+  return runWithCostContext({ op, probeRunId }, () => next());
 });
 app.post("/auth/register", requireAllowedOrigin, async (c) => {
   const { email, password } = await c.req.json();
@@ -37165,7 +37172,7 @@ app.get("/cron/tick", async (c) => {
   if (!process.env.CRON_SECRET || secret2 !== `Bearer ${process.env.CRON_SECRET}`) {
     return c.json({ error: "Unauthorized" }, 401);
   }
-  const { drainQueue } = await import("./worker-2LIOH744.js");
+  const { drainQueue } = await import("./worker-OB6WSWOD.js");
   const budget = { maxJobs: 10, deadlineMs: Date.now() + 28e4 };
   const workflowDispatchResult = await dispatchDueWorkflowSchedules(`${new URL(c.req.url).protocol}//${new URL(c.req.url).host}`);
   const [results, sweepResult, reapResult, expiredResult, blobCleanup, connectedDataArtifactCleanup, connectedAccountBilling, directoryWorkflowDispatch, directoryWorkflowReconciliation, directoryArtifactCleanup, siteExtractArtifactCleanup, siteExtractRedispatch, siteExtractReconciliation, paaHarvestRedispatch, paaHarvestReconciliation] = await Promise.all([
@@ -37425,4 +37432,4 @@ app.get("/blog/:slug/", (c) => {
 export {
   app
 };
-//# sourceMappingURL=server-BPEXEKA6.js.map
+//# sourceMappingURL=server-K76DL4S7.js.map