mcp-scraper 0.43.3 → 0.43.5

package/dist/bin/api-server.cjs

@@ -3671,6 +3671,7 @@ __export(db_exports, {
   CORE_SCHEMA_VERSION: () => CORE_SCHEMA_VERSION,
   CREDIT_LOTS_EPOCH: () => CREDIT_LOTS_EPOCH,
   CREDIT_LOT_TTL: () => CREDIT_LOT_TTL,
+  SITE_EXTRACT_COUNTER_REPAIR_VERSION: () => SITE_EXTRACT_COUNTER_REPAIR_VERSION,
   SiteAuditJobRowSchema: () => SiteAuditJobRowSchema,
   SiteAuditPhaseLogRowSchema: () => SiteAuditPhaseLogRowSchema,
   advanceWorkflowRunStep: () => advanceWorkflowRunStep,
@@ -3808,7 +3809,7 @@ async function hasCurrentCoreSchema(db) {
   const result = await db.execute(`SELECT ${checks.join(" AND ")} AS ready`);
   return Number(result.rows[0]?.ready ?? 0) === 1;
 }
-async function runIdempotentCoreDataBackfills(db) {
+async function runSiteExtractCounterRepair(db) {
   await db.execute(`
     UPDATE site_extract_jobs SET
       attempted_urls = done_urls,
@@ -3848,6 +3849,18 @@ async function runIdempotentCoreDataBackfills(db) {
       AND failed_urls = 0
   `);
 }
+async function runPendingDataRepairs(db) {
+  const applied = await db.execute({
+    sql: "SELECT 1 AS applied FROM schema_migrations WHERE version = ? LIMIT 1",
+    args: [SITE_EXTRACT_COUNTER_REPAIR_VERSION]
+  });
+  if (applied.rows.length > 0) return;
+  await runSiteExtractCounterRepair(db);
+  await db.execute({
+    sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
+    args: [SITE_EXTRACT_COUNTER_REPAIR_VERSION]
+  });
+}
 function getDb() {
   if (!_db) _db = (0, import_http.createClient)({ url: DB_URL, authToken: DB_TOKEN });
   return _db;
@@ -3855,7 +3868,7 @@ function getDb() {
 function setDbForTesting(client2) {
   _db = client2;
 }
-async function migrate() {
+async function migrate(options = {}) {
   const db = getDb();
   let applied;
   try {
@@ -3874,12 +3887,15 @@ async function migrate() {
     `);
     applied = { rows: [] };
   }
-  if (applied.rows.length > 0 && await hasCurrentCoreSchema(db)) {
-    await runIdempotentCoreDataBackfills(db);
-    return;
+  if (applied.rows.length > 0) {
+    if (!options.verifyCurrentSchema) return;
+    if (await hasCurrentCoreSchema(db)) {
+      if (options.runDataRepairs) await runPendingDataRepairs(db);
+      return;
+    }
   }
   if (await hasCurrentCoreSchema(db)) {
-    await runIdempotentCoreDataBackfills(db);
+    await runPendingDataRepairs(db);
     await db.execute({
       sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
       args: [CORE_SCHEMA_VERSION]
@@ -4692,7 +4708,7 @@ async function migrate() {
   if (!await hasCurrentCoreSchema(db)) {
     throw new Error("core schema migration finished without all required tables, indexes, and columns");
   }
-  await runIdempotentCoreDataBackfills(db);
+  await runPendingDataRepairs(db);
   await db.execute({
     sql: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (?)",
     args: [CORE_SCHEMA_VERSION]
@@ -6666,7 +6682,7 @@ async function listActiveUserIds() {
   const res = await getDb().execute("SELECT id FROM users WHERE active = 1");
   return res.rows.map((r) => Number(r.id));
 }
-var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_TRIGGERS, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
+var import_http, import_node_crypto, import_zod, DB_URL, DB_TOKEN, _db, _rateLimitSchemaReady, CORE_SCHEMA_VERSION, SITE_EXTRACT_COUNTER_REPAIR_VERSION, CORE_SCHEMA_TABLES, CORE_SCHEMA_INDEXES, CORE_SCHEMA_TRIGGERS, CORE_SCHEMA_COLUMNS, REQUEST_EVENT_RESULT_MAX_BYTES, CREDIT_LOT_TTL, SiteAuditJobRowSchema, SiteAuditPhaseLogRowSchema;
 var init_db = __esm({
   "src/api/db.ts"() {
     "use strict";
@@ -6680,6 +6696,7 @@ var init_db = __esm({
     _db = null;
     _rateLimitSchemaReady = false;
     CORE_SCHEMA_VERSION = "2026-07-30.1";
+    SITE_EXTRACT_COUNTER_REPAIR_VERSION = "data-2026-07-28-site-extract-counters.1";
     CORE_SCHEMA_TABLES = [
       "admin_credit_adjustment_events",
       "admin_credit_adjustments",
@@ -6842,11 +6859,12 @@ function runWithCostContext(ctx, fn) {
 function currentCostContext() {
   return storage.getStore();
 }
-var import_node_async_hooks, storage;
+var import_node_async_hooks, HEADLESS_HARVEST_TEST_ENV, storage;
 var init_cost_context = __esm({
   "src/api/cost-context.ts"() {
     "use strict";
     import_node_async_hooks = require("async_hooks");
+    HEADLESS_HARVEST_TEST_ENV = "MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST";
     storage = new import_node_async_hooks.AsyncLocalStorage();
   }
 });
@@ -7197,6 +7215,42 @@ var init_bounded_response = __esm({
   }
 });
 
+// src/lib/browser-user-agent.ts
+async function detectPageUserAgent(page) {
+  try {
+    const value = await page.evaluate(() => navigator.userAgent);
+    return typeof value === "string" && RUNTIME_BROWSER_UA.test(value) ? value : null;
+  } catch {
+    return null;
+  }
+}
+async function detectRuntimeUserAgent(browser) {
+  let probeContext = null;
+  try {
+    let page = browser.contexts().flatMap((context) => context.pages())[0];
+    if (!page) {
+      probeContext = await browser.newContext();
+      page = probeContext.pages()[0] ?? await probeContext.newPage();
+    }
+    return await detectPageUserAgent(page);
+  } catch {
+    return null;
+  } finally {
+    if (probeContext) await probeContext.close().catch(() => {
+    });
+  }
+}
+function withRuntimeUserAgent(options, runtimeUserAgent) {
+  return runtimeUserAgent ? { ...options, userAgent: runtimeUserAgent } : options;
+}
+var RUNTIME_BROWSER_UA;
+var init_browser_user_agent = __esm({
+  "src/lib/browser-user-agent.ts"() {
+    "use strict";
+    RUNTIME_BROWSER_UA = /\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;
+  }
+});
+
 // src/api/kernel-fetch.ts
 async function fetchWithKernel(url) {
   const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
@@ -7210,10 +7264,10 @@ async function fetchWithKernel(url) {
   const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
   const browser = await import_playwright.chromium.connectOverCDP(kb.cdp_ws_url);
   try {
-    const context = await browser.newContext({
-      userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+    const context = await browser.newContext(withRuntimeUserAgent({
       serviceWorkers: "block"
-    });
+    }, runtimeUserAgent));
     const page = context.pages()[0] ?? await context.newPage();
     const guard = await installBrowserRequestGuard(page, auditedUrl);
     await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
@@ -7244,6 +7298,7 @@ var init_kernel_fetch = __esm({
     init_browser_request_guard();
     init_url_utils();
     init_bounded_response();
+    init_browser_user_agent();
   }
 });
 
@@ -7827,14 +7882,13 @@ async function scrollAndStitch(page, device = "desktop") {
 }
 async function captureLocal(url, device, wantScreenshot, wantBranding) {
   const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
-  const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
   const browser = await import_playwright2.chromium.launch({ headless: true });
-  const context = await browser.newContext({
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  const context = await browser.newContext(withRuntimeUserAgent({
     viewport: vp,
     locale: "en-US",
-    userAgent: ua,
     isMobile: device === "mobile"
-  });
+  }, runtimeUserAgent));
   const page = await context.newPage();
   try {
     await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -7853,12 +7907,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
   const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
   const browser = await import_playwright2.chromium.connectOverCDP(session.cdp_ws_url);
   const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
-  const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
-  const context = browser.contexts()[0] ?? await browser.newContext({
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  const context = browser.contexts()[0] ?? await browser.newContext(withRuntimeUserAgent({
     viewport: vp,
-    userAgent: ua,
     isMobile: device === "mobile"
-  });
+  }, runtimeUserAgent));
   const page = context.pages()[0] ?? await context.newPage();
   try {
     await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -7892,7 +7945,7 @@ async function capturePageData(url, opts = {}) {
     return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding);
   }
 }
-var import_playwright2, import_sdk2, DESKTOP_VIEWPORT, MOBILE_VIEWPORT, DESKTOP_UA, MOBILE_UA, SCROLL_STEP, SETTLE_MS;
+var import_playwright2, import_sdk2, DESKTOP_VIEWPORT, MOBILE_VIEWPORT, SCROLL_STEP, SETTLE_MS;
 var init_screenshot = __esm({
   "src/lib/screenshot.ts"() {
     "use strict";
@@ -7900,10 +7953,9 @@ var init_screenshot = __esm({
     import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
     init_branding_extractor();
     init_cost_telemetry();
+    init_browser_user_agent();
     DESKTOP_VIEWPORT = { width: 1440, height: 900 };
     MOBILE_VIEWPORT = { width: 390, height: 844 };
-    DESKTOP_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-    MOBILE_UA = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
     SCROLL_STEP = 900;
     SETTLE_MS = 400;
   }
@@ -8715,6 +8767,7 @@ async function crawlWithRotation(urls, opts) {
     const worker = async () => {
       let kb = null;
       let browser = null;
+      let runtimeUserAgent = null;
       let inBrowser = 0;
       let kbOpenedAtMs = 0;
       const retire = async () => {
@@ -8722,6 +8775,7 @@ async function crawlWithRotation(urls, opts) {
         browser = null;
         kb = null;
         kbOpenedAtMs = 0;
+        runtimeUserAgent = null;
         if (b) await b.close().catch(() => {
         });
         if (k) {
@@ -8743,9 +8797,10 @@ async function crawlWithRotation(urls, opts) {
               kb = await createBrowserWithRetry(client2, headless);
               kbOpenedAtMs = Date.now();
               browser = await withTimeout(import_playwright3.chromium.connectOverCDP(kb.cdp_ws_url), 3e4);
+              runtimeUserAgent = await detectRuntimeUserAgent(browser);
               inBrowser = 0;
             }
-            const context = await browser.newContext({ serviceWorkers: "block" });
+            const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
             try {
               r = await crawlPage(context, url);
             } finally {
@@ -8812,6 +8867,7 @@ var init_rotating_proxy_crawl = __esm({
     init_cost_telemetry();
     init_browser_request_guard();
     init_bounded_response();
+    init_browser_user_agent();
     init_browser_request_guard();
     DEFAULT_CONCURRENCY = 1;
     DEFAULT_URLS_PER_BROWSER = 10;
@@ -18981,21 +19037,11 @@ async function ensureKernelProfile(kernelClient, name) {
     if (!isKernelProfileConflict(err)) throw err;
   }
 }
-async function detectRuntimeUserAgent(browser) {
-  try {
-    const page = browser.contexts()[0]?.pages()[0];
-    if (!page) return null;
-    const ua = await page.evaluate(() => navigator.userAgent);
-    return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
-  } catch {
-    return null;
-  }
-}
 function rankCheckContextOptions(config, runtimeUserAgent) {
   return {
     viewport: config.viewport,
     locale: config.locale,
-    userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
+    ...runtimeUserAgent ? { userAgent: runtimeUserAgent } : {},
     ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
     ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
     ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -19041,7 +19087,7 @@ function buildYouTubeChannelVideosUrl(channelInput) {
   }
   return `https://www.youtube.com/${handle}/videos`;
 }
-var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright4, import_sdk5, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, BrowserDriver;
+var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright4, import_sdk5, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, BrowserDriver;
 var init_BrowserDriver = __esm({
   "src/driver/BrowserDriver.ts"() {
     "use strict";
@@ -19054,9 +19100,9 @@ var init_BrowserDriver = __esm({
     init_cost_telemetry();
     init_cost_context();
     init_cost_rates();
+    init_browser_user_agent();
+    init_browser_user_agent();
     import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
-    DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-    MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
     DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
     KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
     KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
@@ -19089,7 +19135,7 @@ var init_BrowserDriver = __esm({
             viewport: config.viewport,
             locale: config.locale,
             device,
-            userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
+            userAgent: null,
             deviceScaleFactor: config.deviceScaleFactor ?? null,
             isMobile: config.isMobile === true,
             hasTouch: config.hasTouch === true
@@ -19177,7 +19223,7 @@ var init_BrowserDriver = __esm({
           }
           this.browser = await import_playwright4.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
           const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
-          this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
+          this.debugSnapshot.context.userAgent = runtimeUserAgent;
           this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
           await this.installEsbuildHelperShims(this.context);
           this.page = await this.context.newPage();
@@ -19199,17 +19245,19 @@ var init_BrowserDriver = __esm({
           ...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
           ...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
         };
-        const ctxOpts = rankCheckContextOptions(config);
         if (config.profileDir) {
           this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
             ...launchOpts,
-            ...ctxOpts
+            ...rankCheckContextOptions(config)
           });
           await this.installEsbuildHelperShims(this.context);
           this.page = await this.context.newPage();
+          this.debugSnapshot.context.userAgent = await detectPageUserAgent(this.page);
         } else {
           this.browser = await import_playwright_extra.chromium.launch(launchOpts);
-          this.context = await this.browser.newContext(ctxOpts);
+          const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
+          this.debugSnapshot.context.userAgent = runtimeUserAgent;
+          this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
           await this.installEsbuildHelperShims(this.context);
           this.page = await this.context.newPage();
         }
@@ -29553,7 +29601,6 @@ var init_facebook_ad_routes = __esm({
       try {
         res = await fetch(parsedUrl.href, {
           headers: {
-            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
             "Referer": "https://www.facebook.com/"
           },
           signal: AbortSignal.timeout(6e4)
@@ -30892,10 +30939,10 @@ function blobKeyPrefix(shortcode, sourceUrl) {
   const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
   return `instagram/${stamp}-${slug}`;
 }
-async function fetchMediaBytes(url, referer) {
+async function fetchMediaBytes(url, referer, runtimeUserAgent) {
   const res = await fetch(url, {
     headers: {
-      "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+      ...runtimeUserAgent ? { "User-Agent": runtimeUserAgent } : {},
       "Referer": referer
     },
     signal: AbortSignal.timeout(9e4)
@@ -30946,6 +30993,7 @@ var init_instagram_routes = __esm({
     import_node_os8 = require("os");
     import_node_path11 = require("path");
     import_node_child_process2 = require("child_process");
+    init_browser_user_agent();
     init_browser_service_env();
     init_BrowserDriver();
     init_api_auth();
@@ -31062,6 +31110,7 @@ var init_instagram_routes = __esm({
         const launch = await resolveInstagramLaunch(body);
         await driver.launch(launch.config);
         const page = driver.getPage();
+        const runtimeUserAgent = await detectPageUserAgent(page);
         const capturedMediaUrls = [];
         page.on("response", (response) => {
           const url = normalizeInstagramMediaUrl(response.url());
@@ -31090,7 +31139,7 @@ var init_instagram_routes = __esm({
             downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
             if (mediaTypes.has("image") && extraction.imageUrl) {
               try {
-                const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
+                const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl, runtimeUserAgent);
                 const ext = extFromMime(mimeType, "jpg");
                 const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
                 downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
@@ -31106,7 +31155,7 @@ var init_instagram_routes = __esm({
               if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
               const kind = track.streamType === "audio" ? "audio" : "video";
               try {
-                const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
+                const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl, runtimeUserAgent);
                 const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
                 downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
                 const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
@@ -31823,7 +31872,7 @@ var init_ai_surfaces = __esm({
 });
 
 // src/extractor/PAAExtractor.ts
-var DESKTOP_USER_AGENT2, MOBILE_USER_AGENT2, PAAExtractor;
+var PAAExtractor;
 var init_PAAExtractor = __esm({
   "src/extractor/PAAExtractor.ts"() {
     "use strict";
@@ -31835,8 +31884,6 @@ var init_PAAExtractor = __esm({
     init_paa_answer_cleanup();
     init_errors();
     init_ai_surfaces();
-    DESKTOP_USER_AGENT2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
-    MOBILE_USER_AGENT2 = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
     PAAExtractor = class {
       constructor(driver, reporter) {
         this.driver = driver;
@@ -32646,7 +32693,6 @@ var init_PAAExtractor = __esm({
           requireUsEgress: options.requireUsEgress,
           viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
           locale: `${options.hl}-${options.gl.toUpperCase()}`,
-          userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
           deviceScaleFactor: isMobile ? 3 : 1,
           isMobile,
           hasTouch: isMobile,
@@ -33265,7 +33311,12 @@ async function harvest(rawOptions) {
         max_questions: attemptOptions.maxQuestions
       }));
       const baseCtx = currentCostContext();
-      const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce2(attemptOptions, signal)) : await extractOnce2(attemptOptions, signal);
+      const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
+      const attempt = baseCtx ? await runWithCostContext({
+        ...baseCtx,
+        forceHeadless: allowHeadlessTest,
+        forceHeadful: !allowHeadlessTest
+      }, () => extractOnce2(attemptOptions, signal)) : await extractOnce2(attemptOptions, signal);
       if (attempt.error) {
         const err = attempt.error;
         const outcome2 = classifyAttemptError(err);
@@ -39483,7 +39534,7 @@ var PACKAGE_VERSION;
 var init_version = __esm({
   "src/version.ts"() {
     "use strict";
-    PACKAGE_VERSION = "0.43.3";
+    PACKAGE_VERSION = "0.43.5";
   }
 });
 
@@ -50524,6 +50575,12 @@ var init_run_capture = __esm({
 // src/services/browser-agent/browser-agent-service.ts
 function keepHostedBrowserAlive(_browser) {
 }
+async function currentOrRuntimeContext(browser) {
+  const current = browser.contexts()[0];
+  if (current) return current;
+  const runtimeUserAgent = await detectRuntimeUserAgent(browser);
+  return browser.newContext(withRuntimeUserAgent({}, runtimeUserAgent));
+}
 function client() {
   const apiKey = browserServiceApiKey();
   if (!apiKey) throw new Error("Browser backend API key is required");
@@ -50647,7 +50704,7 @@ async function pressKeys(runtimeSessionId, keys) {
 async function goto(cdpWsUrl, url) {
   const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
     return { url: page.url(), title: await page.title() };
@@ -50658,7 +50715,7 @@ async function goto(cdpWsUrl, url) {
 async function captureFanout(cdpWsUrl, input) {
   const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     return await runFanoutCapture(page, input);
   } finally {
@@ -50668,7 +50725,7 @@ async function captureFanout(cdpWsUrl, input) {
 async function readPage(cdpWsUrl) {
   const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     const url = page.url();
     const title = await page.title().catch(() => "");
@@ -50723,7 +50780,7 @@ async function readPage(cdpWsUrl) {
 async function locatePageTargets(cdpWsUrl, targets) {
   const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
   try {
-    const context = browser.contexts()[0] ?? await browser.newContext();
+    const context = await currentOrRuntimeContext(browser);
     const page = context.pages()[0] ?? await context.newPage();
     const url = page.url();
     const title = await page.title().catch(() => "");
@@ -50893,6 +50950,7 @@ var init_browser_agent_service = __esm({
     import_playwright5 = require("playwright");
     init_browser_service_env();
     init_run_capture();
+    init_browser_user_agent();
     DEFAULT_TIMEOUT_SECONDS = 600;
   }
 });
@@ -52055,6 +52113,22 @@ function isScraperResource(resource) {
   if (!resource) return false;
   return resource.replace(/\/$/, "") === SCRAPER_RESOURCE();
 }
+async function withOAuthDatabaseDeadline(stage, operation, timeoutMs = OAUTH_DATABASE_TIMEOUT_MS) {
+  const startedAt = Date.now();
+  let timeout;
+  try {
+    const value = await Promise.race([
+      operation,
+      new Promise((_, reject) => {
+        timeout = setTimeout(() => reject(new OAuthDatabaseUnavailableError(stage)), timeoutMs);
+      })
+    ]);
+    console.log("[oauth-db] stage=%s status=ok duration_ms=%d", stage, Date.now() - startedAt);
+    return value;
+  } finally {
+    if (timeout) clearTimeout(timeout);
+  }
+}
 async function getKeys() {
   if (cachedKeys) return cachedKeys;
   const raw = process.env.OAUTH_SIGNING_KEY;
@@ -52108,7 +52182,7 @@ async function sessionUser(token4) {
   if (!token4) return null;
   const userId = verifySession(token4);
   if (!userId) return null;
-  const user = await getUserById(userId);
+  const user = await withOAuthDatabaseDeadline("session-user", getUserById(userId));
   return user ?? null;
 }
 function escapeHtml2(value) {
@@ -52233,7 +52307,7 @@ function parseAuthParams(get) {
 async function validateAuthRequest(p) {
   if (p.response_type !== "code") return { ok: false, status: 400, message: "response_type must be code" };
   if (!p.client_id) return { ok: false, status: 400, message: "client_id is required" };
-  const client2 = await getClient(p.client_id);
+  const client2 = await withOAuthDatabaseDeadline("get-client", getClient(p.client_id));
   if (!client2) return { ok: false, status: 400, message: "unknown client_id" };
   if (!p.redirect_uri || !client2.redirect_uris.includes(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri does not match the registered set" };
   if (!isAllowedRedirect(p.redirect_uri)) return { ok: false, status: 400, message: "redirect_uri host not allowed" };
@@ -52252,7 +52326,7 @@ async function mintAccessToken(identity, scope, plan, audience) {
 function tokenErrorResponse(c, error, description, status) {
   return c.json({ error, error_description: description }, status);
 }
-var import_hono23, import_cookie, import_node_crypto28, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
+var import_hono23, import_cookie, import_node_crypto28, import_jose2, ISSUER, RESOURCE, SCRAPER_RESOURCE, MEMORY_SCOPES, SCRAPER_SCOPES, SUPPORTED_SCOPES, ACCESS_TTL_SECONDS, REFRESH_TTL_SECONDS, CODE_TTL_SECONDS, ROTATION_GRACE_SECONDS, OAUTH_DATABASE_TIMEOUT_MS, OAuthDatabaseUnavailableError, secureCookies, sessionCookieOptions, cachedKeys, oauthApp;
 var init_oauth_routes = __esm({
   "src/api/oauth-routes.ts"() {
     "use strict";
@@ -52274,6 +52348,15 @@ var init_oauth_routes = __esm({
     REFRESH_TTL_SECONDS = 60 * 60 * 24 * 30;
     CODE_TTL_SECONDS = 60;
     ROTATION_GRACE_SECONDS = 120;
+    OAUTH_DATABASE_TIMEOUT_MS = 5e3;
+    OAuthDatabaseUnavailableError = class extends Error {
+      stage;
+      constructor(stage) {
+        super("The authorization service database is temporarily unavailable.");
+        this.name = "OAuthDatabaseUnavailableError";
+        this.stage = stage;
+      }
+    };
     secureCookies = process.env.NODE_ENV === "production" || process.env.VERCEL === "1";
     sessionCookieOptions = {
       httpOnly: true,
@@ -52284,6 +52367,18 @@ var init_oauth_routes = __esm({
     };
     cachedKeys = null;
     oauthApp = new import_hono23.Hono();
+    oauthApp.onError((error, c) => {
+      if (error instanceof OAuthDatabaseUnavailableError) {
+        console.error("[oauth-db] stage=%s status=timeout timeout_ms=%d", error.stage, OAUTH_DATABASE_TIMEOUT_MS);
+        c.header("Retry-After", "5");
+        return c.json({
+          error: "temporarily_unavailable",
+          error_description: "Authorization is temporarily unavailable. Please retry shortly."
+        }, 503);
+      }
+      console.error("[oauth] unhandled error:", error);
+      return c.json({ error: "server_error", error_description: "Authorization could not be completed." }, 500);
+    });
     oauthApp.use("*", async (c, next) => {
       c.header("Access-Control-Allow-Origin", "*");
       c.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS");
@@ -52345,7 +52440,7 @@ var init_oauth_routes = __esm({
       }
       const clientName = typeof body.client_name === "string" ? body.client_name : null;
       const clientId = `client_${(0, import_node_crypto28.randomBytes)(16).toString("hex")}`;
-      await registerClient(clientId, redirectUris, clientName);
+      await withOAuthDatabaseDeadline("register-client", registerClient(clientId, redirectUris, clientName));
       console.log("[oauth-dcr] register OK client_id=%s redirect_uris=%s", clientId, JSON.stringify(redirectUris));
       return c.json({
         client_id: clientId,
@@ -52383,7 +52478,7 @@ var init_oauth_routes = __esm({
       if (action === "login") {
         const email = (get("email") ?? "").trim().toLowerCase();
         const password = get("password") ?? "";
-        const user2 = email ? await getUserByEmail(email) : void 0;
+        const user2 = email ? await withOAuthDatabaseDeadline("login-user", getUserByEmail(email)) : void 0;
         const okPw = !!(user2 && user2.password_hash && verifyPassword(password, user2.password_hash));
         console.log("[oauth-dcr] LOGIN email=%s found=%s hasHash=%s pwLen=%d ok=%s", email, !!user2, !!(user2 && user2.password_hash), password.length, okPw);
         if (!okPw) {
@@ -52399,7 +52494,7 @@ var init_oauth_routes = __esm({
       const scope = negotiateScope(p.scope, user, p.resource);
       const code = `code_${(0, import_node_crypto28.randomBytes)(32).toString("base64url")}`;
       const expiresAt = new Date(Date.now() + CODE_TTL_SECONDS * 1e3).toISOString();
-      await putCode({
+      await withOAuthDatabaseDeadline("put-authorization-code", putCode({
         code,
         client_id: p.client_id,
         redirect_uri: p.redirect_uri,
@@ -52408,7 +52503,7 @@ var init_oauth_routes = __esm({
         scope,
         resource: p.resource || RESOURCE(),
         expires_at: expiresAt
-      });
+      }));
       const u = new URL(p.redirect_uri);
       u.searchParams.set("code", code);
       if (p.state) u.searchParams.set("state", p.state);
@@ -52426,18 +52521,18 @@ var init_oauth_routes = __esm({
         const clientId = get("client_id") ?? "";
         const redirectUri = get("redirect_uri") ?? "";
         const verifier = get("code_verifier") ?? "";
-        const record = await consumeCode(code);
+        const record = await withOAuthDatabaseDeadline("consume-authorization-code", consumeCode(code));
         if (!record) return tokenErrorResponse(c, "invalid_grant", "Authorization code is invalid or already used", 400);
         if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Authorization code has expired", 400);
         if (record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
         if (record.redirect_uri !== redirectUri) return tokenErrorResponse(c, "invalid_grant", "redirect_uri mismatch", 400);
         if (!pkceMatches(verifier, record.code_challenge)) return tokenErrorResponse(c, "invalid_grant", "PKCE verification failed", 400);
-        const user = await getUserByEmail(record.identity);
+        const user = await withOAuthDatabaseDeadline("token-user", getUserByEmail(record.identity));
         const plan = user ? resolvePlan(user) : "free";
         const audience = record.resource ?? RESOURCE();
         const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
         const refreshToken = `rt_${(0, import_node_crypto28.randomBytes)(40).toString("base64url")}`;
-        await putRefresh({
+        await withOAuthDatabaseDeadline("put-refresh-token", putRefresh({
           refresh_token: refreshToken,
           client_id: clientId,
           identity: record.identity,
@@ -52445,15 +52540,15 @@ var init_oauth_routes = __esm({
           resource: audience,
           expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
           revoked: 0
-        });
+        }));
         return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: refreshToken, scope: record.scope });
       }
       if (grantType === "refresh_token") {
         const refreshToken = get("refresh_token") ?? "";
         const clientId = get("client_id") ?? "";
-        const record = await getRefresh(refreshToken);
+        const record = await withOAuthDatabaseDeadline("get-refresh-token", getRefresh(refreshToken));
         if (record && record.revoked && record.replaced_by && record.rotated_at && Date.now() - new Date(record.rotated_at).getTime() < ROTATION_GRACE_SECONDS * 1e3 && (!clientId || record.client_id === clientId)) {
-          const graceUser = await getUserByEmail(record.identity);
+          const graceUser = await withOAuthDatabaseDeadline("refresh-grace-user", getUserByEmail(record.identity));
           const gracePlan = graceUser ? resolvePlan(graceUser) : "free";
           const graceAudience = record.resource ?? RESOURCE();
           const graceAccess = await mintAccessToken(record.identity, record.scope, gracePlan, graceAudience);
@@ -52462,12 +52557,12 @@ var init_oauth_routes = __esm({
         if (!record || record.revoked) return tokenErrorResponse(c, "invalid_grant", "Refresh token is invalid or revoked", 400);
         if (new Date(record.expires_at).getTime() < Date.now()) return tokenErrorResponse(c, "invalid_grant", "Refresh token has expired", 400);
         if (clientId && record.client_id !== clientId) return tokenErrorResponse(c, "invalid_grant", "client_id mismatch", 400);
-        const user = await getUserByEmail(record.identity);
+        const user = await withOAuthDatabaseDeadline("refresh-user", getUserByEmail(record.identity));
         const plan = user ? resolvePlan(user) : "free";
         const audience = record.resource ?? RESOURCE();
         const accessToken = await mintAccessToken(record.identity, record.scope, plan, audience);
         const nextRefresh = `rt_${(0, import_node_crypto28.randomBytes)(40).toString("base64url")}`;
-        await rotateRefresh(refreshToken, {
+        await withOAuthDatabaseDeadline("rotate-refresh-token", rotateRefresh(refreshToken, {
           refresh_token: nextRefresh,
           client_id: record.client_id,
           identity: record.identity,
@@ -52475,7 +52570,7 @@ var init_oauth_routes = __esm({
           resource: audience,
           expires_at: new Date(Date.now() + REFRESH_TTL_SECONDS * 1e3).toISOString(),
           revoked: 0
-        });
+        }));
         return c.json({ access_token: accessToken, token_type: "Bearer", expires_in: ACCESS_TTL_SECONDS, refresh_token: nextRefresh, scope: record.scope });
       }
       return tokenErrorResponse(c, "unsupported_grant_type", "grant_type must be authorization_code or refresh_token", 400);
@@ -61698,8 +61793,7 @@ var init_server = __esm({
       const op = opForCostPath(new URL(c.req.url).pathname);
       if (!op) return next();
       const probeRunId = c.req.header("x-cost-probe-id") ?? null;
-      const forceHeadless = c.req.header("x-kernel-headless") === "true";
-      return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
+      return runWithCostContext({ op, probeRunId }, () => next());
     });
     app.post("/auth/register", requireAllowedOrigin, async (c) => {
       const { email, password } = await c.req.json();