mcp-scraper 0.43.2 → 0.43.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bin/api-server.cjs +92 -52
- package/dist/bin/api-server.cjs.map +1 -1
- package/dist/bin/api-server.js +2 -2
- package/dist/bin/mcp-scraper-cli.cjs +1 -1
- package/dist/bin/mcp-scraper-cli.cjs.map +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-install.cjs +1 -1
- package/dist/bin/mcp-scraper-install.cjs.map +1 -1
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.cjs +1 -1
- package/dist/bin/mcp-stdio-server.cjs.map +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -3
- package/dist/bin/paa-harvest.cjs +42 -22
- package/dist/bin/paa-harvest.cjs.map +1 -1
- package/dist/bin/paa-harvest.js +2 -2
- package/dist/{chunk-H7LQ6PTI.js → chunk-6TAQ2MXK.js} +50 -24
- package/dist/chunk-6TAQ2MXK.js.map +1 -0
- package/dist/{chunk-YJNGNMDO.js → chunk-SXLQZKWC.js} +3 -3
- package/dist/chunk-XQIVXIAL.js +7 -0
- package/dist/chunk-XQIVXIAL.js.map +1 -0
- package/dist/{chunk-DAS7L4SF.js → chunk-ZDVQARDQ.js} +3 -1
- package/dist/chunk-ZDVQARDQ.js.map +1 -0
- package/dist/index.cjs +42 -22
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +2 -2
- package/dist/{server-BPEXEKA6.js → server-K76DL4S7.js} +39 -32
- package/dist/server-K76DL4S7.js.map +1 -0
- package/dist/{worker-2LIOH744.js → worker-OB6WSWOD.js} +3 -3
- package/package.json +2 -1
- package/dist/chunk-DAS7L4SF.js.map +0 -1
- package/dist/chunk-H7LQ6PTI.js.map +0 -1
- package/dist/chunk-MD75M2L5.js +0 -7
- package/dist/chunk-MD75M2L5.js.map +0 -1
- package/dist/server-BPEXEKA6.js.map +0 -1
- /package/dist/{chunk-YJNGNMDO.js.map → chunk-SXLQZKWC.js.map} +0 -0
- /package/dist/{worker-2LIOH744.js.map → worker-OB6WSWOD.js.map} +0 -0
package/README.md
CHANGED
|
@@ -90,7 +90,7 @@ Build the branded one-click bundle:
|
|
|
90
90
|
npm run build:mcpb
|
|
91
91
|
```
|
|
92
92
|
|
|
93
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.43.
|
|
93
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.43.4`, SHA-256 `f6181c189de21204e9c22c6b1e4cff322ee021ab6708c5271f02f8cc6b5f005f`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, and API-key configuration field from the bundle manifest.
|
|
94
94
|
|
|
95
95
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
96
96
|
|
package/dist/bin/api-server.cjs
CHANGED
|
@@ -6842,11 +6842,12 @@ function runWithCostContext(ctx, fn) {
|
|
|
6842
6842
|
function currentCostContext() {
|
|
6843
6843
|
return storage.getStore();
|
|
6844
6844
|
}
|
|
6845
|
-
var import_node_async_hooks, storage;
|
|
6845
|
+
var import_node_async_hooks, HEADLESS_HARVEST_TEST_ENV, storage;
|
|
6846
6846
|
var init_cost_context = __esm({
|
|
6847
6847
|
"src/api/cost-context.ts"() {
|
|
6848
6848
|
"use strict";
|
|
6849
6849
|
import_node_async_hooks = require("async_hooks");
|
|
6850
|
+
HEADLESS_HARVEST_TEST_ENV = "MCP_SCRAPER_ALLOW_HEADLESS_HARVEST_TEST";
|
|
6850
6851
|
storage = new import_node_async_hooks.AsyncLocalStorage();
|
|
6851
6852
|
}
|
|
6852
6853
|
});
|
|
@@ -7197,6 +7198,42 @@ var init_bounded_response = __esm({
|
|
|
7197
7198
|
}
|
|
7198
7199
|
});
|
|
7199
7200
|
|
|
7201
|
+
// src/lib/browser-user-agent.ts
|
|
7202
|
+
async function detectPageUserAgent(page) {
|
|
7203
|
+
try {
|
|
7204
|
+
const value = await page.evaluate(() => navigator.userAgent);
|
|
7205
|
+
return typeof value === "string" && RUNTIME_BROWSER_UA.test(value) ? value : null;
|
|
7206
|
+
} catch {
|
|
7207
|
+
return null;
|
|
7208
|
+
}
|
|
7209
|
+
}
|
|
7210
|
+
async function detectRuntimeUserAgent(browser) {
|
|
7211
|
+
let probeContext = null;
|
|
7212
|
+
try {
|
|
7213
|
+
let page = browser.contexts().flatMap((context) => context.pages())[0];
|
|
7214
|
+
if (!page) {
|
|
7215
|
+
probeContext = await browser.newContext();
|
|
7216
|
+
page = probeContext.pages()[0] ?? await probeContext.newPage();
|
|
7217
|
+
}
|
|
7218
|
+
return await detectPageUserAgent(page);
|
|
7219
|
+
} catch {
|
|
7220
|
+
return null;
|
|
7221
|
+
} finally {
|
|
7222
|
+
if (probeContext) await probeContext.close().catch(() => {
|
|
7223
|
+
});
|
|
7224
|
+
}
|
|
7225
|
+
}
|
|
7226
|
+
function withRuntimeUserAgent(options, runtimeUserAgent) {
|
|
7227
|
+
return runtimeUserAgent ? { ...options, userAgent: runtimeUserAgent } : options;
|
|
7228
|
+
}
|
|
7229
|
+
var RUNTIME_BROWSER_UA;
|
|
7230
|
+
var init_browser_user_agent = __esm({
|
|
7231
|
+
"src/lib/browser-user-agent.ts"() {
|
|
7232
|
+
"use strict";
|
|
7233
|
+
RUNTIME_BROWSER_UA = /\b(?:HeadlessChrome|Chrome|Chromium|Firefox)\/\d+|\bVersion\/\d+.*\bSafari\//;
|
|
7234
|
+
}
|
|
7235
|
+
});
|
|
7236
|
+
|
|
7200
7237
|
// src/api/kernel-fetch.ts
|
|
7201
7238
|
async function fetchWithKernel(url) {
|
|
7202
7239
|
const checked = await validatePublicHttpUrl(url, { field: "browser URL" });
|
|
@@ -7210,10 +7247,10 @@ async function fetchWithKernel(url) {
|
|
|
7210
7247
|
const kb = await client2.browsers.create({ stealth: true, ...useHeadless ? { headless: true } : {}, timeout_seconds: 60 });
|
|
7211
7248
|
const browser = await import_playwright.chromium.connectOverCDP(kb.cdp_ws_url);
|
|
7212
7249
|
try {
|
|
7213
|
-
const
|
|
7214
|
-
|
|
7250
|
+
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
7251
|
+
const context = await browser.newContext(withRuntimeUserAgent({
|
|
7215
7252
|
serviceWorkers: "block"
|
|
7216
|
-
});
|
|
7253
|
+
}, runtimeUserAgent));
|
|
7217
7254
|
const page = context.pages()[0] ?? await context.newPage();
|
|
7218
7255
|
const guard = await installBrowserRequestGuard(page, auditedUrl);
|
|
7219
7256
|
await page.goto(auditedUrl, { waitUntil: "domcontentloaded", timeout: 3e4 });
|
|
@@ -7244,6 +7281,7 @@ var init_kernel_fetch = __esm({
|
|
|
7244
7281
|
init_browser_request_guard();
|
|
7245
7282
|
init_url_utils();
|
|
7246
7283
|
init_bounded_response();
|
|
7284
|
+
init_browser_user_agent();
|
|
7247
7285
|
}
|
|
7248
7286
|
});
|
|
7249
7287
|
|
|
@@ -7827,14 +7865,13 @@ async function scrollAndStitch(page, device = "desktop") {
|
|
|
7827
7865
|
}
|
|
7828
7866
|
async function captureLocal(url, device, wantScreenshot, wantBranding) {
|
|
7829
7867
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
7830
|
-
const ua = device === "mobile" ? MOBILE_UA : DESKTOP_UA;
|
|
7831
7868
|
const browser = await import_playwright2.chromium.launch({ headless: true });
|
|
7832
|
-
const
|
|
7869
|
+
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
7870
|
+
const context = await browser.newContext(withRuntimeUserAgent({
|
|
7833
7871
|
viewport: vp,
|
|
7834
7872
|
locale: "en-US",
|
|
7835
|
-
userAgent: ua,
|
|
7836
7873
|
isMobile: device === "mobile"
|
|
7837
|
-
});
|
|
7874
|
+
}, runtimeUserAgent));
|
|
7838
7875
|
const page = await context.newPage();
|
|
7839
7876
|
try {
|
|
7840
7877
|
await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
|
|
@@ -7853,12 +7890,11 @@ async function captureKernel(url, kernelApiKey, device, wantScreenshot, wantBran
|
|
|
7853
7890
|
const session = await kernel.browsers.create({ stealth: true, timeout_seconds: 240 });
|
|
7854
7891
|
const browser = await import_playwright2.chromium.connectOverCDP(session.cdp_ws_url);
|
|
7855
7892
|
const vp = device === "mobile" ? MOBILE_VIEWPORT : DESKTOP_VIEWPORT;
|
|
7856
|
-
const
|
|
7857
|
-
const context = browser.contexts()[0] ?? await browser.newContext({
|
|
7893
|
+
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
7894
|
+
const context = browser.contexts()[0] ?? await browser.newContext(withRuntimeUserAgent({
|
|
7858
7895
|
viewport: vp,
|
|
7859
|
-
userAgent: ua,
|
|
7860
7896
|
isMobile: device === "mobile"
|
|
7861
|
-
});
|
|
7897
|
+
}, runtimeUserAgent));
|
|
7862
7898
|
const page = context.pages()[0] ?? await context.newPage();
|
|
7863
7899
|
try {
|
|
7864
7900
|
await page.goto(url, { waitUntil: "networkidle", timeout: 6e4 });
|
|
@@ -7892,7 +7928,7 @@ async function capturePageData(url, opts = {}) {
|
|
|
7892
7928
|
return await captureKernel(url, opts.kernelApiKey, device, wantShot, wantBranding);
|
|
7893
7929
|
}
|
|
7894
7930
|
}
|
|
7895
|
-
var import_playwright2, import_sdk2, DESKTOP_VIEWPORT, MOBILE_VIEWPORT,
|
|
7931
|
+
var import_playwright2, import_sdk2, DESKTOP_VIEWPORT, MOBILE_VIEWPORT, SCROLL_STEP, SETTLE_MS;
|
|
7896
7932
|
var init_screenshot = __esm({
|
|
7897
7933
|
"src/lib/screenshot.ts"() {
|
|
7898
7934
|
"use strict";
|
|
@@ -7900,10 +7936,9 @@ var init_screenshot = __esm({
|
|
|
7900
7936
|
import_sdk2 = __toESM(require("@onkernel/sdk"), 1);
|
|
7901
7937
|
init_branding_extractor();
|
|
7902
7938
|
init_cost_telemetry();
|
|
7939
|
+
init_browser_user_agent();
|
|
7903
7940
|
DESKTOP_VIEWPORT = { width: 1440, height: 900 };
|
|
7904
7941
|
MOBILE_VIEWPORT = { width: 390, height: 844 };
|
|
7905
|
-
DESKTOP_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
|
7906
|
-
MOBILE_UA = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
|
|
7907
7942
|
SCROLL_STEP = 900;
|
|
7908
7943
|
SETTLE_MS = 400;
|
|
7909
7944
|
}
|
|
@@ -8715,6 +8750,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
8715
8750
|
const worker = async () => {
|
|
8716
8751
|
let kb = null;
|
|
8717
8752
|
let browser = null;
|
|
8753
|
+
let runtimeUserAgent = null;
|
|
8718
8754
|
let inBrowser = 0;
|
|
8719
8755
|
let kbOpenedAtMs = 0;
|
|
8720
8756
|
const retire = async () => {
|
|
@@ -8722,6 +8758,7 @@ async function crawlWithRotation(urls, opts) {
|
|
|
8722
8758
|
browser = null;
|
|
8723
8759
|
kb = null;
|
|
8724
8760
|
kbOpenedAtMs = 0;
|
|
8761
|
+
runtimeUserAgent = null;
|
|
8725
8762
|
if (b) await b.close().catch(() => {
|
|
8726
8763
|
});
|
|
8727
8764
|
if (k) {
|
|
@@ -8743,9 +8780,10 @@ async function crawlWithRotation(urls, opts) {
|
|
|
8743
8780
|
kb = await createBrowserWithRetry(client2, headless);
|
|
8744
8781
|
kbOpenedAtMs = Date.now();
|
|
8745
8782
|
browser = await withTimeout(import_playwright3.chromium.connectOverCDP(kb.cdp_ws_url), 3e4);
|
|
8783
|
+
runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
8746
8784
|
inBrowser = 0;
|
|
8747
8785
|
}
|
|
8748
|
-
const context = await browser.newContext({ serviceWorkers: "block" });
|
|
8786
|
+
const context = await browser.newContext(withRuntimeUserAgent({ serviceWorkers: "block" }, runtimeUserAgent));
|
|
8749
8787
|
try {
|
|
8750
8788
|
r = await crawlPage(context, url);
|
|
8751
8789
|
} finally {
|
|
@@ -8812,6 +8850,7 @@ var init_rotating_proxy_crawl = __esm({
|
|
|
8812
8850
|
init_cost_telemetry();
|
|
8813
8851
|
init_browser_request_guard();
|
|
8814
8852
|
init_bounded_response();
|
|
8853
|
+
init_browser_user_agent();
|
|
8815
8854
|
init_browser_request_guard();
|
|
8816
8855
|
DEFAULT_CONCURRENCY = 1;
|
|
8817
8856
|
DEFAULT_URLS_PER_BROWSER = 10;
|
|
@@ -18981,21 +19020,11 @@ async function ensureKernelProfile(kernelClient, name) {
|
|
|
18981
19020
|
if (!isKernelProfileConflict(err)) throw err;
|
|
18982
19021
|
}
|
|
18983
19022
|
}
|
|
18984
|
-
async function detectRuntimeUserAgent(browser) {
|
|
18985
|
-
try {
|
|
18986
|
-
const page = browser.contexts()[0]?.pages()[0];
|
|
18987
|
-
if (!page) return null;
|
|
18988
|
-
const ua = await page.evaluate(() => navigator.userAgent);
|
|
18989
|
-
return typeof ua === "string" && /Chrome\/\d+/.test(ua) ? ua : null;
|
|
18990
|
-
} catch {
|
|
18991
|
-
return null;
|
|
18992
|
-
}
|
|
18993
|
-
}
|
|
18994
19023
|
function rankCheckContextOptions(config, runtimeUserAgent) {
|
|
18995
19024
|
return {
|
|
18996
19025
|
viewport: config.viewport,
|
|
18997
19026
|
locale: config.locale,
|
|
18998
|
-
|
|
19027
|
+
...runtimeUserAgent ? { userAgent: runtimeUserAgent } : {},
|
|
18999
19028
|
...config.deviceScaleFactor ? { deviceScaleFactor: config.deviceScaleFactor } : {},
|
|
19000
19029
|
...config.isMobile !== void 0 ? { isMobile: config.isMobile } : {},
|
|
19001
19030
|
...config.hasTouch !== void 0 ? { hasTouch: config.hasTouch } : {}
|
|
@@ -19041,7 +19070,7 @@ function buildYouTubeChannelVideosUrl(channelInput) {
|
|
|
19041
19070
|
}
|
|
19042
19071
|
return `https://www.youtube.com/${handle}/videos`;
|
|
19043
19072
|
}
|
|
19044
|
-
var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright4, import_sdk5,
|
|
19073
|
+
var import_playwright_extra, import_puppeteer_extra_plugin_stealth, import_playwright4, import_sdk5, DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS, KERNEL_BROWSER_CLOSE_TIMEOUT_MS, KERNEL_SESSION_DELETE_TIMEOUT_MS, BrowserDriver;
|
|
19045
19074
|
var init_BrowserDriver = __esm({
|
|
19046
19075
|
"src/driver/BrowserDriver.ts"() {
|
|
19047
19076
|
"use strict";
|
|
@@ -19054,9 +19083,9 @@ var init_BrowserDriver = __esm({
|
|
|
19054
19083
|
init_cost_telemetry();
|
|
19055
19084
|
init_cost_context();
|
|
19056
19085
|
init_cost_rates();
|
|
19086
|
+
init_browser_user_agent();
|
|
19087
|
+
init_browser_user_agent();
|
|
19057
19088
|
import_playwright_extra.chromium.use((0, import_puppeteer_extra_plugin_stealth.default)());
|
|
19058
|
-
DESKTOP_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
|
19059
|
-
MOBILE_USER_AGENT = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
|
|
19060
19089
|
DEFAULT_KERNEL_BROWSER_TIMEOUT_SECONDS = 180;
|
|
19061
19090
|
KERNEL_BROWSER_CLOSE_TIMEOUT_MS = 3e3;
|
|
19062
19091
|
KERNEL_SESSION_DELETE_TIMEOUT_MS = 5e3;
|
|
@@ -19089,7 +19118,7 @@ var init_BrowserDriver = __esm({
|
|
|
19089
19118
|
viewport: config.viewport,
|
|
19090
19119
|
locale: config.locale,
|
|
19091
19120
|
device,
|
|
19092
|
-
userAgent:
|
|
19121
|
+
userAgent: null,
|
|
19093
19122
|
deviceScaleFactor: config.deviceScaleFactor ?? null,
|
|
19094
19123
|
isMobile: config.isMobile === true,
|
|
19095
19124
|
hasTouch: config.hasTouch === true
|
|
@@ -19177,7 +19206,7 @@ var init_BrowserDriver = __esm({
|
|
|
19177
19206
|
}
|
|
19178
19207
|
this.browser = await import_playwright4.chromium.connectOverCDP(kernelBrowser.cdp_ws_url);
|
|
19179
19208
|
const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
|
|
19180
|
-
this.debugSnapshot.context.userAgent =
|
|
19209
|
+
this.debugSnapshot.context.userAgent = runtimeUserAgent;
|
|
19181
19210
|
this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
|
|
19182
19211
|
await this.installEsbuildHelperShims(this.context);
|
|
19183
19212
|
this.page = await this.context.newPage();
|
|
@@ -19199,17 +19228,19 @@ var init_BrowserDriver = __esm({
|
|
|
19199
19228
|
...config.browserExecutablePath ? { executablePath: config.browserExecutablePath } : {},
|
|
19200
19229
|
...config.ignoreDefaultArgs ? { ignoreDefaultArgs: config.ignoreDefaultArgs } : {}
|
|
19201
19230
|
};
|
|
19202
|
-
const ctxOpts = rankCheckContextOptions(config);
|
|
19203
19231
|
if (config.profileDir) {
|
|
19204
19232
|
this.context = await import_playwright_extra.chromium.launchPersistentContext(config.profileDir, {
|
|
19205
19233
|
...launchOpts,
|
|
19206
|
-
...
|
|
19234
|
+
...rankCheckContextOptions(config)
|
|
19207
19235
|
});
|
|
19208
19236
|
await this.installEsbuildHelperShims(this.context);
|
|
19209
19237
|
this.page = await this.context.newPage();
|
|
19238
|
+
this.debugSnapshot.context.userAgent = await detectPageUserAgent(this.page);
|
|
19210
19239
|
} else {
|
|
19211
19240
|
this.browser = await import_playwright_extra.chromium.launch(launchOpts);
|
|
19212
|
-
|
|
19241
|
+
const runtimeUserAgent = await detectRuntimeUserAgent(this.browser);
|
|
19242
|
+
this.debugSnapshot.context.userAgent = runtimeUserAgent;
|
|
19243
|
+
this.context = await this.browser.newContext(rankCheckContextOptions(config, runtimeUserAgent));
|
|
19213
19244
|
await this.installEsbuildHelperShims(this.context);
|
|
19214
19245
|
this.page = await this.context.newPage();
|
|
19215
19246
|
}
|
|
@@ -29553,7 +29584,6 @@ var init_facebook_ad_routes = __esm({
|
|
|
29553
29584
|
try {
|
|
29554
29585
|
res = await fetch(parsedUrl.href, {
|
|
29555
29586
|
headers: {
|
|
29556
|
-
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
|
29557
29587
|
"Referer": "https://www.facebook.com/"
|
|
29558
29588
|
},
|
|
29559
29589
|
signal: AbortSignal.timeout(6e4)
|
|
@@ -30892,10 +30922,10 @@ function blobKeyPrefix(shortcode, sourceUrl) {
|
|
|
30892
30922
|
const slug = shortcode ? `instagram-${shortcode}` : safeFilePart(sourceUrl);
|
|
30893
30923
|
return `instagram/${stamp}-${slug}`;
|
|
30894
30924
|
}
|
|
30895
|
-
async function fetchMediaBytes(url, referer) {
|
|
30925
|
+
async function fetchMediaBytes(url, referer, runtimeUserAgent) {
|
|
30896
30926
|
const res = await fetch(url, {
|
|
30897
30927
|
headers: {
|
|
30898
|
-
"User-Agent":
|
|
30928
|
+
...runtimeUserAgent ? { "User-Agent": runtimeUserAgent } : {},
|
|
30899
30929
|
"Referer": referer
|
|
30900
30930
|
},
|
|
30901
30931
|
signal: AbortSignal.timeout(9e4)
|
|
@@ -30946,6 +30976,7 @@ var init_instagram_routes = __esm({
|
|
|
30946
30976
|
import_node_os8 = require("os");
|
|
30947
30977
|
import_node_path11 = require("path");
|
|
30948
30978
|
import_node_child_process2 = require("child_process");
|
|
30979
|
+
init_browser_user_agent();
|
|
30949
30980
|
init_browser_service_env();
|
|
30950
30981
|
init_BrowserDriver();
|
|
30951
30982
|
init_api_auth();
|
|
@@ -31062,6 +31093,7 @@ var init_instagram_routes = __esm({
|
|
|
31062
31093
|
const launch = await resolveInstagramLaunch(body);
|
|
31063
31094
|
await driver.launch(launch.config);
|
|
31064
31095
|
const page = driver.getPage();
|
|
31096
|
+
const runtimeUserAgent = await detectPageUserAgent(page);
|
|
31065
31097
|
const capturedMediaUrls = [];
|
|
31066
31098
|
page.on("response", (response) => {
|
|
31067
31099
|
const url = normalizeInstagramMediaUrl(response.url());
|
|
@@ -31090,7 +31122,7 @@ var init_instagram_routes = __esm({
|
|
|
31090
31122
|
downloads.push({ kind: "text", url: null, savedPath: storedText.url, sizeBytes: storedText.bytes, mimeType: "text/plain", error: null });
|
|
31091
31123
|
if (mediaTypes.has("image") && extraction.imageUrl) {
|
|
31092
31124
|
try {
|
|
31093
|
-
const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl);
|
|
31125
|
+
const { bytes, mimeType } = await fetchMediaBytes(extraction.imageUrl, extraction.pageUrl, runtimeUserAgent);
|
|
31094
31126
|
const ext = extFromMime(mimeType, "jpg");
|
|
31095
31127
|
const stored = await store.put(`${prefix}/image.${ext}`, bytes, mimeType ?? "application/octet-stream");
|
|
31096
31128
|
downloads.push({ kind: "image", url: extraction.imageUrl, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
|
|
@@ -31106,7 +31138,7 @@ var init_instagram_routes = __esm({
|
|
|
31106
31138
|
if (track.streamType === "audio" && !mediaTypes.has("audio")) continue;
|
|
31107
31139
|
const kind = track.streamType === "audio" ? "audio" : "video";
|
|
31108
31140
|
try {
|
|
31109
|
-
const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl);
|
|
31141
|
+
const { bytes, mimeType } = await fetchMediaBytes(track.url, extraction.pageUrl, runtimeUserAgent);
|
|
31110
31142
|
const stored = await store.put(`${prefix}/${trackFilename(track, index, extraction.shortcode)}`, bytes, mimeType ?? "video/mp4");
|
|
31111
31143
|
downloads.push({ kind, url: track.url, savedPath: stored.url, sizeBytes: stored.bytes, mimeType, error: null });
|
|
31112
31144
|
const isMuxInput = muxEligible && (track.url === extraction.selectedVideoTrack.url || track.url === extraction.selectedAudioTrack.url);
|
|
@@ -31823,7 +31855,7 @@ var init_ai_surfaces = __esm({
|
|
|
31823
31855
|
});
|
|
31824
31856
|
|
|
31825
31857
|
// src/extractor/PAAExtractor.ts
|
|
31826
|
-
var
|
|
31858
|
+
var PAAExtractor;
|
|
31827
31859
|
var init_PAAExtractor = __esm({
|
|
31828
31860
|
"src/extractor/PAAExtractor.ts"() {
|
|
31829
31861
|
"use strict";
|
|
@@ -31835,8 +31867,6 @@ var init_PAAExtractor = __esm({
|
|
|
31835
31867
|
init_paa_answer_cleanup();
|
|
31836
31868
|
init_errors();
|
|
31837
31869
|
init_ai_surfaces();
|
|
31838
|
-
DESKTOP_USER_AGENT2 = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36";
|
|
31839
|
-
MOBILE_USER_AGENT2 = "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1";
|
|
31840
31870
|
PAAExtractor = class {
|
|
31841
31871
|
constructor(driver, reporter) {
|
|
31842
31872
|
this.driver = driver;
|
|
@@ -32646,7 +32676,6 @@ var init_PAAExtractor = __esm({
|
|
|
32646
32676
|
requireUsEgress: options.requireUsEgress,
|
|
32647
32677
|
viewport: isMobile ? { width: 390, height: 844 } : { width: 1280, height: 800 },
|
|
32648
32678
|
locale: `${options.hl}-${options.gl.toUpperCase()}`,
|
|
32649
|
-
userAgent: isMobile ? MOBILE_USER_AGENT2 : DESKTOP_USER_AGENT2,
|
|
32650
32679
|
deviceScaleFactor: isMobile ? 3 : 1,
|
|
32651
32680
|
isMobile,
|
|
32652
32681
|
hasTouch: isMobile,
|
|
@@ -33265,7 +33294,12 @@ async function harvest(rawOptions) {
|
|
|
33265
33294
|
max_questions: attemptOptions.maxQuestions
|
|
33266
33295
|
}));
|
|
33267
33296
|
const baseCtx = currentCostContext();
|
|
33268
|
-
const
|
|
33297
|
+
const allowHeadlessTest = baseCtx?.harvestBrowserMode === "headless-test" && process.env[HEADLESS_HARVEST_TEST_ENV] === "true";
|
|
33298
|
+
const attempt = baseCtx ? await runWithCostContext({
|
|
33299
|
+
...baseCtx,
|
|
33300
|
+
forceHeadless: allowHeadlessTest,
|
|
33301
|
+
forceHeadful: !allowHeadlessTest
|
|
33302
|
+
}, () => extractOnce2(attemptOptions, signal)) : await extractOnce2(attemptOptions, signal);
|
|
33269
33303
|
if (attempt.error) {
|
|
33270
33304
|
const err = attempt.error;
|
|
33271
33305
|
const outcome2 = classifyAttemptError(err);
|
|
@@ -39483,7 +39517,7 @@ var PACKAGE_VERSION;
|
|
|
39483
39517
|
var init_version = __esm({
|
|
39484
39518
|
"src/version.ts"() {
|
|
39485
39519
|
"use strict";
|
|
39486
|
-
PACKAGE_VERSION = "0.43.
|
|
39520
|
+
PACKAGE_VERSION = "0.43.4";
|
|
39487
39521
|
}
|
|
39488
39522
|
});
|
|
39489
39523
|
|
|
@@ -50524,6 +50558,12 @@ var init_run_capture = __esm({
|
|
|
50524
50558
|
// src/services/browser-agent/browser-agent-service.ts
|
|
50525
50559
|
function keepHostedBrowserAlive(_browser) {
|
|
50526
50560
|
}
|
|
50561
|
+
async function currentOrRuntimeContext(browser) {
|
|
50562
|
+
const current = browser.contexts()[0];
|
|
50563
|
+
if (current) return current;
|
|
50564
|
+
const runtimeUserAgent = await detectRuntimeUserAgent(browser);
|
|
50565
|
+
return browser.newContext(withRuntimeUserAgent({}, runtimeUserAgent));
|
|
50566
|
+
}
|
|
50527
50567
|
function client() {
|
|
50528
50568
|
const apiKey = browserServiceApiKey();
|
|
50529
50569
|
if (!apiKey) throw new Error("Browser backend API key is required");
|
|
@@ -50647,7 +50687,7 @@ async function pressKeys(runtimeSessionId, keys) {
|
|
|
50647
50687
|
async function goto(cdpWsUrl, url) {
|
|
50648
50688
|
const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
|
|
50649
50689
|
try {
|
|
50650
|
-
const context =
|
|
50690
|
+
const context = await currentOrRuntimeContext(browser);
|
|
50651
50691
|
const page = context.pages()[0] ?? await context.newPage();
|
|
50652
50692
|
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 45e3 });
|
|
50653
50693
|
return { url: page.url(), title: await page.title() };
|
|
@@ -50658,7 +50698,7 @@ async function goto(cdpWsUrl, url) {
|
|
|
50658
50698
|
async function captureFanout(cdpWsUrl, input) {
|
|
50659
50699
|
const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
|
|
50660
50700
|
try {
|
|
50661
|
-
const context =
|
|
50701
|
+
const context = await currentOrRuntimeContext(browser);
|
|
50662
50702
|
const page = context.pages()[0] ?? await context.newPage();
|
|
50663
50703
|
return await runFanoutCapture(page, input);
|
|
50664
50704
|
} finally {
|
|
@@ -50668,7 +50708,7 @@ async function captureFanout(cdpWsUrl, input) {
|
|
|
50668
50708
|
async function readPage(cdpWsUrl) {
|
|
50669
50709
|
const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
|
|
50670
50710
|
try {
|
|
50671
|
-
const context =
|
|
50711
|
+
const context = await currentOrRuntimeContext(browser);
|
|
50672
50712
|
const page = context.pages()[0] ?? await context.newPage();
|
|
50673
50713
|
const url = page.url();
|
|
50674
50714
|
const title = await page.title().catch(() => "");
|
|
@@ -50723,7 +50763,7 @@ async function readPage(cdpWsUrl) {
|
|
|
50723
50763
|
async function locatePageTargets(cdpWsUrl, targets) {
|
|
50724
50764
|
const browser = await import_playwright5.chromium.connectOverCDP(cdpWsUrl);
|
|
50725
50765
|
try {
|
|
50726
|
-
const context =
|
|
50766
|
+
const context = await currentOrRuntimeContext(browser);
|
|
50727
50767
|
const page = context.pages()[0] ?? await context.newPage();
|
|
50728
50768
|
const url = page.url();
|
|
50729
50769
|
const title = await page.title().catch(() => "");
|
|
@@ -50893,6 +50933,7 @@ var init_browser_agent_service = __esm({
|
|
|
50893
50933
|
import_playwright5 = require("playwright");
|
|
50894
50934
|
init_browser_service_env();
|
|
50895
50935
|
init_run_capture();
|
|
50936
|
+
init_browser_user_agent();
|
|
50896
50937
|
DEFAULT_TIMEOUT_SECONDS = 600;
|
|
50897
50938
|
}
|
|
50898
50939
|
});
|
|
@@ -61698,8 +61739,7 @@ var init_server = __esm({
|
|
|
61698
61739
|
const op = opForCostPath(new URL(c.req.url).pathname);
|
|
61699
61740
|
if (!op) return next();
|
|
61700
61741
|
const probeRunId = c.req.header("x-cost-probe-id") ?? null;
|
|
61701
|
-
|
|
61702
|
-
return runWithCostContext({ op, probeRunId, forceHeadless }, () => next());
|
|
61742
|
+
return runWithCostContext({ op, probeRunId }, () => next());
|
|
61703
61743
|
});
|
|
61704
61744
|
app.post("/auth/register", requireAllowedOrigin, async (c) => {
|
|
61705
61745
|
const { email, password } = await c.req.json();
|