mcp-scraper 0.43.3 → 0.43.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +1 -1
  2. package/dist/bin/api-server.cjs +92 -52
  3. package/dist/bin/api-server.cjs.map +1 -1
  4. package/dist/bin/api-server.js +2 -2
  5. package/dist/bin/mcp-scraper-cli.cjs +1 -1
  6. package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
  7. package/dist/bin/mcp-scraper-cli.js +1 -1
  8. package/dist/bin/mcp-scraper-install.cjs +1 -1
  9. package/dist/bin/mcp-scraper-install.cjs.map +1 -1
  10. package/dist/bin/mcp-scraper-install.js +1 -1
  11. package/dist/bin/mcp-stdio-server.cjs +1 -1
  12. package/dist/bin/mcp-stdio-server.cjs.map +1 -1
  13. package/dist/bin/mcp-stdio-server.js +3 -3
  14. package/dist/bin/paa-harvest.cjs +42 -22
  15. package/dist/bin/paa-harvest.cjs.map +1 -1
  16. package/dist/bin/paa-harvest.js +2 -2
  17. package/dist/{chunk-H7LQ6PTI.js → chunk-6TAQ2MXK.js} +50 -24
  18. package/dist/chunk-6TAQ2MXK.js.map +1 -0
  19. package/dist/{chunk-CHC3RCFA.js → chunk-SXLQZKWC.js} +3 -3
  20. package/dist/chunk-XQIVXIAL.js +7 -0
  21. package/dist/chunk-XQIVXIAL.js.map +1 -0
  22. package/dist/{chunk-DAS7L4SF.js → chunk-ZDVQARDQ.js} +3 -1
  23. package/dist/chunk-ZDVQARDQ.js.map +1 -0
  24. package/dist/index.cjs +42 -22
  25. package/dist/index.cjs.map +1 -1
  26. package/dist/index.d.cts +1 -1
  27. package/dist/index.d.ts +1 -1
  28. package/dist/index.js +2 -2
  29. package/dist/{server-7CLMCX42.js → server-K76DL4S7.js} +39 -32
  30. package/dist/server-K76DL4S7.js.map +1 -0
  31. package/dist/{worker-2LIOH744.js → worker-OB6WSWOD.js} +3 -3
  32. package/package.json +2 -1
  33. package/dist/chunk-4LQVW2EK.js +0 -7
  34. package/dist/chunk-4LQVW2EK.js.map +0 -1
  35. package/dist/chunk-DAS7L4SF.js.map +0 -1
  36. package/dist/chunk-H7LQ6PTI.js.map +0 -1
  37. package/dist/server-7CLMCX42.js.map +0 -1
  38. /package/dist/{chunk-CHC3RCFA.js.map → chunk-SXLQZKWC.js.map} +0 -0
  39. /package/dist/{worker-2LIOH744.js.map → worker-OB6WSWOD.js.map} +0 -0
package/README.md CHANGED
@@ -90,7 +90,7 @@ Build the branded one-click bundle:
90
90
  npm run build:mcpb
91
91
  ```
92
92
 
93
- The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.43.3`, SHA-256 `e9f3d7896517f57e3081a31cb5f3b2d5eb1a653684106f0ac9682b4947f2d16f`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
93
+ The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.43.4`, SHA-256 `f6181c189de21204e9c22c6b1e4cff322ee021ab6708c5271f02f8cc6b5f005f`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
94
94
 
95
95
  The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
96
96
 
@@ -6842,11 +6842,12 @@ function runWithCostContext(ctx, fn) {
6842
6842
  function currentCostContext() {
6843
6843
  return storage.getStore();
6844
6844
  }
6845
- var import_node_async_hooks, storage;
6845
+ var import_node_async_hooks, HEADLESS_HARVEST_TEST_ENV, storage;
6846
6846
  var init_cost_context = __esm({
6847
6847
  "src/api/cost-context.ts"() {
6848
6848
  "use strict";
6849
6849
  import_node_async_hooks = require("async_hooks");
6850
+ HEADLESS_HARVEST_TEST_ENV = "MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST";
6850
6851
  storage = new import_node_async_hooks.AsyncLocalStorage();
6851
6852
  }
6852
6853
  });
@@ -7197,6 +7198,42 @@ var init_bounded_response = __esm({
7197
7198
  }
7198
7199
  });
7199
7200
 
7201
+ // src/lib/browser-user-agent.ts
7202
+ async function detectPageUserAgent(page) {
7203
+ try {
7204
+ const value = await page.evaluate(() => navigator.userAgent);
7205
+ return typeof value === "string" && RUNTIME_BROWSER_UA.test(value) ? value : null;
7206
+ } catch {
7207
+ return null;
7208
+ }
7209
+ }
7210
+ async function detectRuntimeUserAgent(browser) {
7211
+ let probeContext = null;
7212
+ try {
7213
+ let page = browser.contexts().flatMap((context) => context.pages())[0];
7214
+ if (!page) {
7215
+ probeContext = await browser.newContext();
7216
+ page = probeContext.pages()[0] ?? await probeContext.newPage();
7217
+ }
7218
+ return await detectPageUserAgent(page);
7219
+ } catch {
7220
+ return null;
7221
+ } finally {
7222
+ if (probeContext) await probeContext.close().catch(() => {
7223
+ });
7224
+ }
7225
+ }
7226
+ function withRuntimeUserAgent(options, runtimeUserAgent) {
7227
+ return runtimeUserAgent ? { ...options, userAgent: runtimeUserAgent } : options;
7228
+ }
7229
+ var RUNTIME_BROWSER_UA;
7230
+ var init_browser_user_agent = __esm({
7231
+ "src/lib/browser-user-agent.ts"() {
7232
+ "use strict";
7233
+ RUNTIME_BROWSER_UA = /\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;
7234
+ }
7235
+ });
7236
+
7200
7237
  // src/api/kernel-fetch.ts
7201
7238
  async function fetchWithKernel(url) {
7202
7239
  const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
@@ -7210,10 +7247,10 @@ async function fetchWithKernel(url) {
7210
7247
  const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
7211
7248
  const browser = await import_playwright.chromium.connectOverCDP(kb.cdp_ws_url);
7212
7249
  try {
7213
- const context = await browser.newContext({
7214
- userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
7250
+ const runtimeUserAgent = await detectRuntimeUserAgent(browser);
7251
+ const context = await browser.newContext(withRuntimeUserAgent({
7215
7252
  serviceWorkers: "block"
7216
- });
7253
+ }, runtimeUserAgent));
7217
7254
  const page = context.pages()[0] ?? await context.newPage();
7218
7255
  const guard = await installBrowserRequestGuard(page, auditedUrl);
7219
7256
  await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
@@ -7244,6 +7281,7 @@ var init_kernel_fetch = __esm({
7244
7281
  init_browser_request_guard();
7245
7282
  init_url_utils();
7246
7283
  init_bounded_response();
7284
+ init_browser_user_agent();
7247
7285
  }
7248
7286
  });
7249
7287
 
@@ -7827,14 +7865,13 @@ async function scrollAndStitch(page, device = "desktop") {
7827
7865
  }
7828
7866
  async function captureLocal(url, device, wantScreenshot, wantBranding) {
7829
7867
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
7830
- const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
7831
7868
  const browser = await import_playwright2.chromium.launch({ headless: true });
7832
- const context = await browser.newContext({
7869
+ const runtimeUserAgent = await detectRuntimeUserAgent(browser);
7870
+ const context = await browser.newContext(withRuntimeUserAgent({
7833
7871
  viewport: vp,
7834
7872
  locale: "en-US",
7835
- userAgent: ua,
7836
7873
  isMobile: device === "mobile"
7837
- });
7874
+ }, runtimeUserAgent));
7838
7875
  const page = await context.newPage();
7839
7876
  try {
7840
7877
  await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -7853,12 +7890,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
7853
7890
  const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
7854
7891
  const browser = await import_playwright2.chromium.connectOverCDP(session.cdp_ws_url);
7855
7892
  const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
7856
- const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
7857
- const context = browser.contexts()[0] ?? await browser.newContext({
7893
+ const runtimeUserAgent = await detectRuntimeUserAgent(browser);
7894
+ const context = browser.contexts()[0] ?? await browser.newContext(withRuntimeUserAgent({
7858
7895
  viewport: vp,
7859
- userAgent: ua,
7860
7896
  isMobile: device === "mobile"
7861
- });
7897
+ }, runtimeUserAgent));
7862
7898
  const page = context.pages()[0] ?? await context.newPage();
7863
7899
  try {
7864
7900
  await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
@@ -7892,7 +7928,7 @@ async function capturePageData(url, opts = {}) {
7892
7928
  return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding);
7893
7929
  }
7894
7930
  }
7895
- var import_playwright2, import_sdk2, DESKTOP_VIEWPORT, MOBILE_VIEWPORT, DESKTOP_UA, MOBILE_UA, SCROLL_STEP, SETTLE_MS;
7931
+ var import_playwright2, import_sdk2, DESKTOP_VIEWPORT, MOBILE_VIEWPORT, SCROLL_STEP, SETTLE_MS;
7896
7932
  var init_screenshot = __esm({
7897
7933
  "src/lib/screenshot.ts"() {
7898
7934
  "use strict";
@@ -7900,10 +7936,9 @@ var init_screenshot = __esm({
7900
7936
  import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
7901
7937
  init_branding_extractor();
7902
7938
  init_cost_telemetry();
7939
+ init_browser_user_agent();
7903
7940
  DESKTOP_VIEWPORT = { width: 1440, height: 900 };
7904
7941
  MOBILE_VIEWPORT = { width: 390, height: 844 };
7905
- DESKTOP_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
7906
- MOBILE_UA = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
7907
7942
  SCROLL_STEP = 900;
7908
7943
  SETTLE_MS = 400;
7909
7944
  }
@@ -8715,6 +8750,7 @@ async function crawlWithRotation(urls, opts) {
8715
8750
  const worker = async () => {
8716
8751
  let kb = null;
8717
8752
  let browser = null;
8753
+ let runtimeUserAgent = null;
8718
8754
  let inBrowser = 0;
8719
8755
  let kbOpenedAtMs = 0;
8720
8756
  const retire = async () => {
@@ -8722,6 +8758,7 @@ async function crawlWithRotation(urls, opts) {
8722
8758
  browser = null;
8723
8759
  kb = null;
8724
8760
  kbOpenedAtMs = 0;
8761
+ runtimeUserAgent = null;
8725
8762
  if (b) await b.close().catch(() => {
8726
8763
  });
8727
8764
  if (k) {
@@ -8743,9 +8780,10 @@ async function crawlWithRotation(urls, opts) {
8743
8780
  kb = await createBrowserWithRetry(client2, headless);
8744
8781
  kbOpenedAtMs = Date.now();
8745
8782
  browser = await withTimeout(import_playwright3.chromium.connectOverCDP(kb.cdp_ws_url), 3e4);
8783
+ runtimeUserAgent = await detectRuntimeUserAgent(browser);
8746
8784
  inBrowser = 0;
8747
8785
  }
8748
- const context = await browser.newContext({ serviceWorkers: "block" });
8786
+ const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
8749
8787
  try {
8750
8788
  r = await crawlPage(context, url);
8751
8789
  } finally {
@@ -8812,6 +8850,7 @@ var init_rotating_proxy_crawl = __esm({
8812
8850
  init_cost_telemetry();
8813
8851
  init_browser_request_guard();
8814
8852
  init_bounded_response();
8853
+ init_browser_user_agent();
8815
8854
  init_browser_request_guard();
8816
8855
  DEFAULT_CONCURRENCY = 1;
8817
8856
  DEFAULT_URLS_PER_BROWSER = 10;
@@ -18981,21 +19020,11 @@ async function ensureKernelProfile(kernelClient, name) {
18981
19020
  if (!isKernelProfileConflict(err)) throw err;
18982
19021
  }
18983
19022
  }
18984
- async function detectRuntimeUserAgent(browser) {
18985
- try {
18986
- const page = browser.contexts()[0]?.pages()[0];
18987
- if (!page) return null;
18988
- const ua = await page.evaluate(() => navigator.userAgent);
18989
- return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
18990
- } catch {
18991
- return null;
18992
- }
18993
- }
18994
19023
  function rankCheckContextOptions(config, runtimeUserAgent) {
18995
19024
  return {
18996
19025
  viewport: config.viewport,
18997
19026
  locale: config.locale,
18998
- userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT),
19027
+ ...runtimeUserAgent ? { userAgent: runtimeUserAgent } : {},
18999
19028
  ...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
19000
19029
  ...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
19001
19030
  ...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
@@ -19041,7 +19070,7 @@ function buildYouTubeChannelVideosUrl(channelInput) {
19041
19070
  }
19042
19071
  return `https://www.youtube.com/${handle}/videos`;
19043
19072
  }
19044
- var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright4, import_sdk5, DESKTOP_USER_AGENT, MOBILE_USER_AGENT, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, BrowserDriver;
19073
+ var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright4, import_sdk5, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, BrowserDriver;
19045
19074
  var init_BrowserDriver = __esm({
19046
19075
  "src/driver/BrowserDriver.ts"() {
19047
19076
  "use strict";
@@ -19054,9 +19083,9 @@ var init_BrowserDriver = __esm({
19054
19083
  init_cost_telemetry();
19055
19084
  init_cost_context();
19056
19085
  init_cost_rates();
19086
+ init_browser_user_agent();
19087
+ init_browser_user_agent();
19057
19088
  import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
19058
- DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
19059
- MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
19060
19089
  DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
19061
19090
  KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
19062
19091
  KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
@@ -19089,7 +19118,7 @@ var init_BrowserDriver = __esm({
19089
19118
  viewport: config.viewport,
19090
19119
  locale: config.locale,
19091
19120
  device,
19092
- userAgent: config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : DESKTOP_USER_AGENT),
19121
+ userAgent: null,
19093
19122
  deviceScaleFactor: config.deviceScaleFactor ?? null,
19094
19123
  isMobile: config.isMobile === true,
19095
19124
  hasTouch: config.hasTouch === true
@@ -19177,7 +19206,7 @@ var init_BrowserDriver = __esm({
19177
19206
  }
19178
19207
  this.browser = await import_playwright4.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
19179
19208
  const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
19180
- this.debugSnapshot.context.userAgent = config.userAgent ?? (config.isMobile ? MOBILE_USER_AGENT : runtimeUserAgent ?? DESKTOP_USER_AGENT);
19209
+ this.debugSnapshot.context.userAgent = runtimeUserAgent;
19181
19210
  this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
19182
19211
  await this.installEsbuildHelperShims(this.context);
19183
19212
  this.page = await this.context.newPage();
@@ -19199,17 +19228,19 @@ var init_BrowserDriver = __esm({
19199
19228
  ...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
19200
19229
  ...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
19201
19230
  };
19202
- const ctxOpts = rankCheckContextOptions(config);
19203
19231
  if (config.profileDir) {
19204
19232
  this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
19205
19233
  ...launchOpts,
19206
- ...ctxOpts
19234
+ ...rankCheckContextOptions(config)
19207
19235
  });
19208
19236
  await this.installEsbuildHelperShims(this.context);
19209
19237
  this.page = await this.context.newPage();
19238
+ this.debugSnapshot.context.userAgent = await detectPageUserAgent(this.page);
19210
19239
  } else {
19211
19240
  this.browser = await import_playwright_extra.chromium.launch(launchOpts);
19212
- this.context = await this.browser.newContext(ctxOpts);
19241
+ const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
19242
+ this.debugSnapshot.context.userAgent = runtimeUserAgent;
19243
+ this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
19213
19244
  await this.installEsbuildHelperShims(this.context);
19214
19245
  this.page = await this.context.newPage();
19215
19246
  }
@@ -29553,7 +29584,6 @@ var init_facebook_ad_routes = __esm({
29553
29584
  try {
29554
29585
  res = await fetch(parsedUrl.href, {
29555
29586
  headers: {
29556
- "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
29557
29587
  "Referer": "https://www.facebook.com/"
29558
29588
  },
29559
29589
  signal: AbortSignal.timeout(6e4)
@@ -30892,10 +30922,10 @@ function blobKeyPrefix(shortcode, sourceUrl) {
30892
30922
  const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
30893
30923
  return `instagram/${stamp}-${slug}`;
30894
30924
  }
30895
- async function fetchMediaBytes(url, referer) {
30925
+ async function fetchMediaBytes(url, referer, runtimeUserAgent) {
30896
30926
  const res = await fetch(url, {
30897
30927
  headers: {
30898
- "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
30928
+ ...runtimeUserAgent ? { "User-Agent": runtimeUserAgent } : {},
30899
30929
  "Referer": referer
30900
30930
  },
30901
30931
  signal: AbortSignal.timeout(9e4)
@@ -30946,6 +30976,7 @@ var init_instagram_routes = __esm({
30946
30976
  import_node_os8 = require("os");
30947
30977
  import_node_path11 = require("path");
30948
30978
  import_node_child_process2 = require("child_process");
30979
+ init_browser_user_agent();
30949
30980
  init_browser_service_env();
30950
30981
  init_BrowserDriver();
30951
30982
  init_api_auth();
@@ -31062,6 +31093,7 @@ var init_instagram_routes = __esm({
31062
31093
  const launch = await resolveInstagramLaunch(body);
31063
31094
  await driver.launch(launch.config);
31064
31095
  const page = driver.getPage();
31096
+ const runtimeUserAgent = await detectPageUserAgent(page);
31065
31097
  const capturedMediaUrls = [];
31066
31098
  page.on("response", (response) => {
31067
31099
  const url = normalizeInstagramMediaUrl(response.url());
@@ -31090,7 +31122,7 @@ var init_instagram_routes = __esm({
31090
31122
  downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
31091
31123
  if (mediaTypes.has("image") && extraction.imageUrl) {
31092
31124
  try {
31093
- const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
31125
+ const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl, runtimeUserAgent);
31094
31126
  const ext = extFromMime(mimeType, "jpg");
31095
31127
  const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
31096
31128
  downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
@@ -31106,7 +31138,7 @@ var init_instagram_routes = __esm({
31106
31138
  if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
31107
31139
  const kind = track.streamType === "audio" ? "audio" : "video";
31108
31140
  try {
31109
- const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
31141
+ const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl, runtimeUserAgent);
31110
31142
  const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
31111
31143
  downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
31112
31144
  const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
@@ -31823,7 +31855,7 @@ var init_ai_surfaces = __esm({
31823
31855
  });
31824
31856
 
31825
31857
  // src/extractor/PAAExtractor.ts
31826
- var DESKTOP_USER_AGENT2, MOBILE_USER_AGENT2, PAAExtractor;
31858
+ var PAAExtractor;
31827
31859
  var init_PAAExtractor = __esm({
31828
31860
  "src/extractor/PAAExtractor.ts"() {
31829
31861
  "use strict";
@@ -31835,8 +31867,6 @@ var init_PAAExtractor = __esm({
31835
31867
  init_paa_answer_cleanup();
31836
31868
  init_errors();
31837
31869
  init_ai_surfaces();
31838
- DESKTOP_USER_AGENT2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
31839
- MOBILE_USER_AGENT2 = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
31840
31870
  PAAExtractor = class {
31841
31871
  constructor(driver, reporter) {
31842
31872
  this.driver = driver;
@@ -32646,7 +32676,6 @@ var init_PAAExtractor = __esm({
32646
32676
  requireUsEgress: options.requireUsEgress,
32647
32677
  viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
32648
32678
  locale: `${options.hl}-${options.gl.toUpperCase()}`,
32649
- userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
32650
32679
  deviceScaleFactor: isMobile ? 3 : 1,
32651
32680
  isMobile,
32652
32681
  hasTouch: isMobile,
@@ -33265,7 +33294,12 @@ async function harvest(rawOptions) {
33265
33294
  max_questions: attemptOptions.maxQuestions
33266
33295
  }));
33267
33296
  const baseCtx = currentCostContext();
33268
- const attempt = baseCtx ? await runWithCostContext({ ...baseCtx, forceHeadful: true }, () => extractOnce2(attemptOptions, signal)) : await extractOnce2(attemptOptions, signal);
33297
+ const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
33298
+ const attempt = baseCtx ? await runWithCostContext({
33299
+ ...baseCtx,
33300
+ forceHeadless: allowHeadlessTest,
33301
+ forceHeadful: !allowHeadlessTest
33302
+ }, () => extractOnce2(attemptOptions, signal)) : await extractOnce2(attemptOptions, signal);
33269
33303
  if (attempt.error) {
33270
33304
  const err = attempt.error;
33271
33305
  const outcome2 = classifyAttemptError(err);
@@ -39483,7 +39517,7 @@ var PACKAGE_VERSION;
39483
39517
  var init_version = __esm({
39484
39518
  "src/version.ts"() {
39485
39519
  "use strict";
39486
- PACKAGE_VERSION = "0.43.3";
39520
+ PACKAGE_VERSION = "0.43.4";
39487
39521
  }
39488
39522
  });
39489
39523
 
@@ -50524,6 +50558,12 @@ var init_run_capture = __esm({
50524
50558
  // src/services/browser-agent/browser-agent-service.ts
50525
50559
  function keepHostedBrowserAlive(_browser) {
50526
50560
  }
50561
+ async function currentOrRuntimeContext(browser) {
50562
+ const current = browser.contexts()[0];
50563
+ if (current) return current;
50564
+ const runtimeUserAgent = await detectRuntimeUserAgent(browser);
50565
+ return browser.newContext(withRuntimeUserAgent({}, runtimeUserAgent));
50566
+ }
50527
50567
  function client() {
50528
50568
  const apiKey = browserServiceApiKey();
50529
50569
  if (!apiKey) throw new Error("Browser backend API key is required");
@@ -50647,7 +50687,7 @@ async function pressKeys(runtimeSessionId, keys) {
50647
50687
  async function goto(cdpWsUrl, url) {
50648
50688
  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
50649
50689
  try {
50650
- const context = browser.contexts()[0] ?? await browser.newContext();
50690
+ const context = await currentOrRuntimeContext(browser);
50651
50691
  const page = context.pages()[0] ?? await context.newPage();
50652
50692
  await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
50653
50693
  return { url: page.url(), title: await page.title() };
@@ -50658,7 +50698,7 @@ async function goto(cdpWsUrl, url) {
50658
50698
  async function captureFanout(cdpWsUrl, input) {
50659
50699
  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
50660
50700
  try {
50661
- const context = browser.contexts()[0] ?? await browser.newContext();
50701
+ const context = await currentOrRuntimeContext(browser);
50662
50702
  const page = context.pages()[0] ?? await context.newPage();
50663
50703
  return await runFanoutCapture(page, input);
50664
50704
  } finally {
@@ -50668,7 +50708,7 @@ async function captureFanout(cdpWsUrl, input) {
50668
50708
  async function readPage(cdpWsUrl) {
50669
50709
  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
50670
50710
  try {
50671
- const context = browser.contexts()[0] ?? await browser.newContext();
50711
+ const context = await currentOrRuntimeContext(browser);
50672
50712
  const page = context.pages()[0] ?? await context.newPage();
50673
50713
  const url = page.url();
50674
50714
  const title = await page.title().catch(() => "");
@@ -50723,7 +50763,7 @@ async function readPage(cdpWsUrl) {
50723
50763
  async function locatePageTargets(cdpWsUrl, targets) {
50724
50764
  const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
50725
50765
  try {
50726
- const context = browser.contexts()[0] ?? await browser.newContext();
50766
+ const context = await currentOrRuntimeContext(browser);
50727
50767
  const page = context.pages()[0] ?? await context.newPage();
50728
50768
  const url = page.url();
50729
50769
  const title = await page.title().catch(() => "");
@@ -50893,6 +50933,7 @@ var init_browser_agent_service = __esm({
50893
50933
  import_playwright5 = require("playwright");
50894
50934
  init_browser_service_env();
50895
50935
  init_run_capture();
50936
+ init_browser_user_agent();
50896
50937
  DEFAULT_TIMEOUT_SECONDS = 600;
50897
50938
  }
50898
50939
  });
@@ -61698,8 +61739,7 @@ var init_server = __esm({
61698
61739
  const op = opForCostPath(new URL(c.req.url).pathname);
61699
61740
  if (!op) return next();
61700
61741
  const probeRunId = c.req.header("x-cost-probe-id") ?? null;
61701
- const forceHeadless = c.req.header("x-kernel-headless") === "true";
61702
- return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
61742
+ return runWithCostContext({ op, probeRunId }, () => next());
61703
61743
  });
61704
61744
  app.post("/auth/register", requireAllowedOrigin, async (c) => {
61705
61745
  const { email, password } = await c.req.json();