mcp-scraper 0.86.2 → 0.86.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -1
- package/README.md +1 -1
- package/dist/bin/api-server.js +1 -1
- package/dist/bin/mcp-scraper-cli.js +1 -1
- package/dist/bin/mcp-scraper-core.js +3 -3
- package/dist/bin/mcp-scraper-install.js +1 -1
- package/dist/bin/mcp-stdio-server.js +3 -3
- package/dist/{chunk-KCKODNTH.js → chunk-CBVEBDQA.js} +2 -2
- package/dist/{chunk-ET646CIY.js → chunk-CJPHSGOF.js} +1 -1
- package/dist/{chunk-KBP335CI.js → chunk-IHW3HTO6.js} +4 -4
- package/dist/{server-ACZHVP65.js → server-JF6SAM4D.js} +106 -47
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,11 +4,17 @@ All notable changes to MCP Scraper are documented here. The format is based on [
|
|
|
4
4
|
|
|
5
5
|
## [Unreleased]
|
|
6
6
|
|
|
7
|
+
## [0.86.3] - 2026-09-02
|
|
8
|
+
|
|
9
|
+
### Fixed
|
|
10
|
+
|
|
11
|
+
- Applied 45-second Kernel and 35-second Bright Data deadlines to the complete Reddit browser-attempt lifecycle, and made known-thread primary attempts find and click the target through DuckDuckGo before the residential landing.
|
|
12
|
+
|
|
7
13
|
## [0.86.2] - 2026-09-02
|
|
8
14
|
|
|
9
15
|
### Fixed
|
|
10
16
|
|
|
11
|
-
- Bounded
|
|
17
|
+
- Bounded the Reddit DOM challenge-readability loop at 90 seconds; production verification showed the surrounding browser lifecycle still needed a complete attempt deadline, which is corrected in 0.86.3.
|
|
12
18
|
|
|
13
19
|
## [0.86.1] - 2026-09-02
|
|
14
20
|
|
package/README.md
CHANGED
|
@@ -175,7 +175,7 @@ Build the branded one-click bundle:
|
|
|
175
175
|
npm run build:mcpb
|
|
176
176
|
```
|
|
177
177
|
|
|
178
|
-
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.86.
|
|
178
|
+
The generated bundle is written to `build/mcpb/mcp-scraper-<version>.mcpb` and copied to `public/downloads/` for the hosted download. The current public bundle is `https://mcpscraper.dev/downloads/mcp-scraper.mcpb` (`0.86.3`, SHA-256 `f1643e34c7bb3152068b269e7610325a3d24f7a268787e8ab3364f31bc3afc90`). Install it by opening or dragging it into Claude Desktop. Claude displays the `MCP Scraper` install card, icon, API-key configuration field, and manually curated current-release message from the bundle manifest.
|
|
179
179
|
|
|
180
180
|
The MCPB install exposes every tool — web-intelligence plus all `browser_*` tools — through the one `mcp-scraper` server.
|
|
181
181
|
|
package/dist/bin/api-server.js
CHANGED
|
@@ -17,7 +17,7 @@ loadDotEnv();
|
|
|
17
17
|
async function main() {
|
|
18
18
|
const [{ serve }, { app, personalAssistantProductionStartup }, { startWorker }, { migrate }] = await Promise.all([
|
|
19
19
|
import("@hono/node-server"),
|
|
20
|
-
import("../server-
|
|
20
|
+
import("../server-JF6SAM4D.js"),
|
|
21
21
|
import("../worker-OQSFSLNG.js"),
|
|
22
22
|
import("../db-Z34LPZNR.js")
|
|
23
23
|
]);
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-CBVEBDQA.js";
|
|
5
|
+
import "../chunk-IHW3HTO6.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
@@ -13,7 +13,7 @@ import "../chunk-OPQIGAFB.js";
|
|
|
13
13
|
import "../chunk-3JRZZVWL.js";
|
|
14
14
|
import "../chunk-P7FWOMU7.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-CJPHSGOF.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// src/mcp/thorbit-restricted-tool-surface.ts
|
|
@@ -4,7 +4,7 @@ import {
|
|
|
4
4
|
} from "../chunk-3JRZZVWL.js";
|
|
5
5
|
import {
|
|
6
6
|
PACKAGE_VERSION
|
|
7
|
-
} from "../chunk-
|
|
7
|
+
} from "../chunk-CJPHSGOF.js";
|
|
8
8
|
|
|
9
9
|
// bin/mcp-scraper-install.ts
|
|
10
10
|
var noColor = process.argv.includes("--no-color") || process.env.NO_COLOR !== void 0 || process.env.FORCE_COLOR === "0" || !process.stdout.isTTY;
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
runMcpScraperStdio
|
|
4
|
-
} from "../chunk-
|
|
5
|
-
import "../chunk-
|
|
4
|
+
} from "../chunk-CBVEBDQA.js";
|
|
5
|
+
import "../chunk-IHW3HTO6.js";
|
|
6
6
|
import "../chunk-PGJQDMC2.js";
|
|
7
7
|
import "../chunk-DNM65UCK.js";
|
|
8
8
|
import "../chunk-T3MZISOF.js";
|
|
@@ -13,7 +13,7 @@ import "../chunk-OPQIGAFB.js";
|
|
|
13
13
|
import "../chunk-3JRZZVWL.js";
|
|
14
14
|
import "../chunk-P7FWOMU7.js";
|
|
15
15
|
import "../chunk-GGZEC22A.js";
|
|
16
|
-
import "../chunk-
|
|
16
|
+
import "../chunk-CJPHSGOF.js";
|
|
17
17
|
import "../chunk-YXNDOQXN.js";
|
|
18
18
|
|
|
19
19
|
// bin/mcp-stdio-server.ts
|
|
@@ -12,13 +12,13 @@ import {
|
|
|
12
12
|
registerScheduledResultsMcpTools,
|
|
13
13
|
registerSerpIntelligenceCaptureTools,
|
|
14
14
|
resolveDeploymentProfile
|
|
15
|
-
} from "./chunk-
|
|
15
|
+
} from "./chunk-IHW3HTO6.js";
|
|
16
16
|
import {
|
|
17
17
|
renderInstallTerminal
|
|
18
18
|
} from "./chunk-3JRZZVWL.js";
|
|
19
19
|
import {
|
|
20
20
|
PACKAGE_VERSION
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-CJPHSGOF.js";
|
|
22
22
|
|
|
23
23
|
// src/mcp/stdio-runtime.ts
|
|
24
24
|
import { readFileSync } from "fs";
|
|
@@ -50,7 +50,7 @@ import {
|
|
|
50
50
|
} from "./chunk-P7FWOMU7.js";
|
|
51
51
|
import {
|
|
52
52
|
PACKAGE_VERSION
|
|
53
|
-
} from "./chunk-
|
|
53
|
+
} from "./chunk-CJPHSGOF.js";
|
|
54
54
|
import {
|
|
55
55
|
PUBLIC_ERROR_CODES,
|
|
56
56
|
buildPublicErrorEnvelope,
|
|
@@ -5731,8 +5731,8 @@ seam is noted so you can chain them.
|
|
|
5731
5731
|
url the user gives).
|
|
5732
5732
|
|
|
5733
5733
|
## Reddit
|
|
5734
|
-
- Read ONE known reddit.com thread/post URL -> **reddit_thread**.
|
|
5735
|
-
residential browser
|
|
5734
|
+
- Read ONE known reddit.com thread/post URL -> **reddit_thread**. Each primary attempt finds and clicks the exact
|
|
5735
|
+
URL through a site-restricted web search before landing on current Reddit through a managed residential browser. It returns a bounded rendered comment tree and makes up to two fresh primary attempts,
|
|
5736
5736
|
then up to two managed-browser backup attempts that can recover the post and up to 25 comments.
|
|
5737
5737
|
- DISCOVER what a niche is talking about (topic, no known thread) -> **reddit_trending** (takes a topic, optional
|
|
5738
5738
|
subreddit and a week, 30-day, or all-time window; returns top threads ranked by engagement plus the questions people asked \u2014
|
|
@@ -17452,7 +17452,7 @@ function registerPaaExtractorMcpTools(server, executor, options = {}) {
|
|
|
17452
17452
|
}, async (input) => formatFacebookAdSearch(await executor.facebookAdSearch(input), input));
|
|
17453
17453
|
server.registerTool("reddit_thread", {
|
|
17454
17454
|
title: "Reddit Thread + Comments",
|
|
17455
|
-
description: "Read one known reddit.com post and its rendered comment tree on current Reddit.
|
|
17455
|
+
description: "Read one known reddit.com post and its rendered comment tree on current Reddit. Each of up to two primary attempts finds and clicks the exact URL through a site-restricted web search before landing with a managed residential browser; up to two managed-browser backup attempts can then recover the post and 25 comments. It stops on the first accepted result. Use reddit_trending when no thread URL is known. Costs one Reddit base lookup plus the per-comment rate for comments returned; pass maxComments to bound both output and cost.",
|
|
17456
17456
|
inputSchema: RedditThreadInputSchema,
|
|
17457
17457
|
outputSchema: recordOutputSchema("reddit_thread", RedditThreadOutputSchema),
|
|
17458
17458
|
annotations: liveWebToolAnnotations("Reddit Thread + Comments")
|
|
@@ -388,7 +388,7 @@ import {
|
|
|
388
388
|
transcribeMediaUrl,
|
|
389
389
|
transitionAnalyticsExperiment,
|
|
390
390
|
validateAnalyticsRestResponseForRequest
|
|
391
|
-
} from "./chunk-
|
|
391
|
+
} from "./chunk-IHW3HTO6.js";
|
|
392
392
|
import {
|
|
393
393
|
auditImageUrls,
|
|
394
394
|
auditImages,
|
|
@@ -509,7 +509,7 @@ import {
|
|
|
509
509
|
} from "./chunk-GGZEC22A.js";
|
|
510
510
|
import {
|
|
511
511
|
PACKAGE_VERSION
|
|
512
|
-
} from "./chunk-
|
|
512
|
+
} from "./chunk-CJPHSGOF.js";
|
|
513
513
|
import {
|
|
514
514
|
CaptchaError,
|
|
515
515
|
MapsSearchIncompletePageError,
|
|
@@ -20951,8 +20951,22 @@ function redditDiscoveryQuery(query, subreddit) {
|
|
|
20951
20951
|
const scope = subreddit ? `site:reddit.com/r/${subreddit.replace(/^r\//i, "")}` : "site:reddit.com";
|
|
20952
20952
|
return `${scope} ${query.trim()}`;
|
|
20953
20953
|
}
|
|
20954
|
+
function redditThreadLookupQuery(value) {
|
|
20955
|
+
const normalized = normalizeRedditThreadUrl(value);
|
|
20956
|
+
if (!normalized) throw new Error("url must be a modern reddit.com thread URL");
|
|
20957
|
+
const parts = new URL(normalized).pathname.split("/").filter(Boolean);
|
|
20958
|
+
const subreddit = parts[1] ?? "";
|
|
20959
|
+
const slug5 = (parts[4] ?? "").replace(/[_-]+/g, " ").trim();
|
|
20960
|
+
return [slug5, subreddit ? `r/${subreddit}` : ""].filter(Boolean).join(" ");
|
|
20961
|
+
}
|
|
20954
20962
|
|
|
20955
20963
|
// src/kernel-reddit/duckduckgo-search.ts
|
|
20964
|
+
function boundedTimeout(deadlineAt, maximumMs) {
|
|
20965
|
+
if (deadlineAt == null) return maximumMs;
|
|
20966
|
+
const remaining = deadlineAt - Date.now();
|
|
20967
|
+
if (remaining <= 0) throw new Error("Kernel Reddit browser attempt deadline exceeded");
|
|
20968
|
+
return Math.max(1, Math.min(maximumMs, remaining));
|
|
20969
|
+
}
|
|
20956
20970
|
function unwrapDuckDuckGoUrl(href) {
|
|
20957
20971
|
try {
|
|
20958
20972
|
const url2 = new URL(href, "https://duckduckgo.com");
|
|
@@ -20966,8 +20980,8 @@ async function searchDuckDuckGoForReddit(page3, query, maxResults, options = {})
|
|
|
20966
20980
|
searchUrl.searchParams.set("q", redditDiscoveryQuery(query, options.subreddit));
|
|
20967
20981
|
if (options.window === "week") searchUrl.searchParams.set("df", "w");
|
|
20968
20982
|
if (options.window === "month") searchUrl.searchParams.set("df", "m");
|
|
20969
|
-
await page3.goto(searchUrl.toString(), { waitUntil: "domcontentloaded", timeout:
|
|
20970
|
-
await page3.waitForSelector('a[data-testid="result-title-a"], article h2 a', { timeout:
|
|
20983
|
+
await page3.goto(searchUrl.toString(), { waitUntil: "domcontentloaded", timeout: boundedTimeout(options.deadlineAt, 1e4) });
|
|
20984
|
+
await page3.waitForSelector('a[data-testid="result-title-a"], article h2 a', { timeout: boundedTimeout(options.deadlineAt, 5e3) });
|
|
20971
20985
|
const seen = /* @__PURE__ */ new Set();
|
|
20972
20986
|
const results = [];
|
|
20973
20987
|
for (let pageNumber = 0; pageNumber < 4 && results.length < maxResults; pageNumber += 1) {
|
|
@@ -20987,25 +21001,25 @@ async function searchDuckDuckGoForReddit(page3, query, maxResults, options = {})
|
|
|
20987
21001
|
const previousArticleCount = await page3.locator("article").count();
|
|
20988
21002
|
const more = page3.getByRole("button", { name: /more results/i }).or(page3.getByRole("link", { name: /more results/i })).first();
|
|
20989
21003
|
if (!await more.count()) break;
|
|
20990
|
-
await more.click({ timeout:
|
|
20991
|
-
await page3.waitForFunction((count3) => document.querySelectorAll("article").length > count3, previousArticleCount, { timeout:
|
|
21004
|
+
await more.click({ timeout: boundedTimeout(options.deadlineAt, 3e3) }).catch(() => void 0);
|
|
21005
|
+
await page3.waitForFunction((count3) => document.querySelectorAll("article").length > count3, previousArticleCount, { timeout: boundedTimeout(options.deadlineAt, 3e3) }).catch(() => void 0);
|
|
20992
21006
|
}
|
|
20993
21007
|
return results;
|
|
20994
21008
|
}
|
|
20995
|
-
async function clickDuckDuckGoResult(page3, clickHref) {
|
|
21009
|
+
async function clickDuckDuckGoResult(page3, clickHref, deadlineAt) {
|
|
20996
21010
|
const exact = page3.locator(`a[href=${JSON.stringify(clickHref)}]`).first();
|
|
20997
21011
|
if (!await exact.count()) throw new Error("Selected DuckDuckGo result is no longer present");
|
|
20998
21012
|
try {
|
|
20999
|
-
await exact.click({ timeout:
|
|
21013
|
+
await exact.click({ timeout: boundedTimeout(deadlineAt, 3e3) });
|
|
21000
21014
|
} catch {
|
|
21001
|
-
await exact.click({ force: true, timeout:
|
|
21015
|
+
await exact.click({ force: true, timeout: boundedTimeout(deadlineAt, 3e3) });
|
|
21002
21016
|
}
|
|
21003
|
-
await page3.waitForLoadState("domcontentloaded", { timeout:
|
|
21017
|
+
await page3.waitForLoadState("domcontentloaded", { timeout: boundedTimeout(deadlineAt, 8e3) }).catch(() => void 0);
|
|
21004
21018
|
}
|
|
21005
21019
|
|
|
21006
21020
|
// src/kernel-reddit/reddit-thread.ts
|
|
21007
21021
|
var THREAD_POLL_MS = 2e3;
|
|
21008
|
-
var REDDIT_THREAD_CHALLENGE_WAIT_MS =
|
|
21022
|
+
var REDDIT_THREAD_CHALLENGE_WAIT_MS = 2e4;
|
|
21009
21023
|
var POST_CHALLENGE_RENDER_GRACE_MS = 2e4;
|
|
21010
21024
|
var COMMENT_EXPANSION_MAX_MS = 3e4;
|
|
21011
21025
|
var COMMENT_STABLE_POLLS = 5;
|
|
@@ -21028,16 +21042,19 @@ var MODERN_REDDIT_THREAD_SCRIPT = `(() => {
|
|
|
21028
21042
|
});
|
|
21029
21043
|
return { title, author, score, postBody, postPresent: Boolean(post), declaredCommentCount, comments, blocked, challenge };
|
|
21030
21044
|
})()`;
|
|
21031
|
-
async function readModernRedditThread(page3, maxComments) {
|
|
21045
|
+
async function readModernRedditThread(page3, maxComments, options = {}) {
|
|
21046
|
+
const maxWaitMs = Math.max(1, Math.min(REDDIT_THREAD_CHALLENGE_WAIT_MS, options.maxWaitMs ?? REDDIT_THREAD_CHALLENGE_WAIT_MS));
|
|
21032
21047
|
let renderReadyAt = null;
|
|
21033
21048
|
let expansionStartedAt = null;
|
|
21034
21049
|
let largestCommentCount = -1;
|
|
21035
21050
|
let stableCommentPolls = 0;
|
|
21036
21051
|
let bestThread = null;
|
|
21037
|
-
|
|
21052
|
+
const startedAt = Date.now();
|
|
21053
|
+
const deadlineAt = startedAt + maxWaitMs;
|
|
21054
|
+
for (; Date.now() - startedAt <= maxWaitMs; ) {
|
|
21038
21055
|
const data = await page3.evaluate(MODERN_REDDIT_THREAD_SCRIPT).catch(() => null);
|
|
21039
21056
|
if (!data) {
|
|
21040
|
-
await page3.waitForTimeout(THREAD_POLL_MS);
|
|
21057
|
+
await page3.waitForTimeout(Math.max(1, Math.min(THREAD_POLL_MS, deadlineAt - Date.now())));
|
|
21041
21058
|
continue;
|
|
21042
21059
|
}
|
|
21043
21060
|
if (data.challenge) renderReadyAt = null;
|
|
@@ -21058,8 +21075,8 @@ async function readModernRedditThread(page3, maxComments) {
|
|
|
21058
21075
|
if (Date.now() - expansionStartedAt >= COMMENT_EXPANSION_MAX_MS || stableCommentPolls >= COMMENT_STABLE_POLLS) {
|
|
21059
21076
|
return { ...bestThread, comments: bestThread.comments.slice(0, maxComments) };
|
|
21060
21077
|
}
|
|
21061
|
-
await expandRenderedComments(page3);
|
|
21062
|
-
await page3.waitForTimeout(THREAD_POLL_MS);
|
|
21078
|
+
await expandRenderedComments(page3, deadlineAt);
|
|
21079
|
+
await page3.waitForTimeout(Math.max(1, Math.min(THREAD_POLL_MS, deadlineAt - Date.now())));
|
|
21063
21080
|
continue;
|
|
21064
21081
|
}
|
|
21065
21082
|
if (data.blocked && !data.challenge) throw new Error("Reddit rejected the residential browser session");
|
|
@@ -21070,38 +21087,59 @@ async function readModernRedditThread(page3, maxComments) {
|
|
|
21070
21087
|
}
|
|
21071
21088
|
throw new Error("Reddit rendered no usable thread data");
|
|
21072
21089
|
}
|
|
21073
|
-
await page3.waitForTimeout(THREAD_POLL_MS);
|
|
21090
|
+
await page3.waitForTimeout(Math.max(1, Math.min(THREAD_POLL_MS, deadlineAt - Date.now())));
|
|
21074
21091
|
}
|
|
21075
21092
|
throw new Error("Reddit thread did not become readable before the challenge deadline");
|
|
21076
21093
|
}
|
|
21077
|
-
async function expandRenderedComments(page3) {
|
|
21094
|
+
async function expandRenderedComments(page3, deadlineAt) {
|
|
21078
21095
|
await page3.evaluate(() => window.scrollTo(0, document.documentElement.scrollHeight)).catch(() => void 0);
|
|
21079
21096
|
const buttons = page3.getByRole("button", {
|
|
21080
21097
|
name: /^(view more comments|load more comments|more replies|continue this thread)/i
|
|
21081
21098
|
});
|
|
21082
21099
|
const count3 = Math.min(await buttons.count().catch(() => 0), 4);
|
|
21083
21100
|
for (let index = 0; index < count3; index += 1) {
|
|
21084
|
-
|
|
21101
|
+
const remaining = deadlineAt - Date.now();
|
|
21102
|
+
if (remaining <= 0) return;
|
|
21103
|
+
await buttons.nth(index).click({ timeout: Math.max(1, Math.min(2e3, remaining)) }).catch(() => void 0);
|
|
21085
21104
|
}
|
|
21086
21105
|
}
|
|
21087
21106
|
|
|
21088
21107
|
// src/kernel-reddit/service.ts
|
|
21108
|
+
var KERNEL_REDDIT_THREAD_ATTEMPT_MS = 45e3;
|
|
21109
|
+
var KERNEL_CONTROL_TIMEOUT_MS = 5e3;
|
|
21110
|
+
var CLEANUP_TIMEOUT_MS = 3e3;
|
|
21111
|
+
function remainingTimeout(deadlineAt, maximumMs) {
|
|
21112
|
+
const remaining = deadlineAt - Date.now();
|
|
21113
|
+
if (remaining <= 0) throw new Error("Kernel Reddit browser attempt deadline exceeded");
|
|
21114
|
+
return Math.max(1, Math.min(maximumMs, remaining));
|
|
21115
|
+
}
|
|
21116
|
+
async function settleWithin(promise, timeoutMs = CLEANUP_TIMEOUT_MS) {
|
|
21117
|
+
let timer;
|
|
21118
|
+
await Promise.race([
|
|
21119
|
+
promise.then(() => void 0),
|
|
21120
|
+
new Promise((resolve3) => {
|
|
21121
|
+
timer = setTimeout(resolve3, timeoutMs);
|
|
21122
|
+
})
|
|
21123
|
+
]).finally(() => {
|
|
21124
|
+
if (timer) clearTimeout(timer);
|
|
21125
|
+
});
|
|
21126
|
+
}
|
|
21089
21127
|
function publicSearchRows(rows) {
|
|
21090
21128
|
return rows.map(({ position, title, url: url2, snippet }) => ({ position, title, url: url2, snippet }));
|
|
21091
21129
|
}
|
|
21092
|
-
async function createSearchSession(apiKey, hooks) {
|
|
21130
|
+
async function createSearchSession(apiKey, hooks, deadlineAt = Date.now() + KERNEL_REDDIT_THREAD_ATTEMPT_MS) {
|
|
21093
21131
|
const openedAtMs = Date.now();
|
|
21094
|
-
const kernel = new Kernel4({ apiKey, timeout:
|
|
21132
|
+
const kernel = new Kernel4({ apiKey, timeout: KERNEL_CONTROL_TIMEOUT_MS, maxRetries: 0 });
|
|
21095
21133
|
const created = await kernel.browsers.create({
|
|
21096
21134
|
stealth: true,
|
|
21097
21135
|
headless: false,
|
|
21098
|
-
timeout_seconds:
|
|
21136
|
+
timeout_seconds: 90,
|
|
21099
21137
|
viewport: { width: 1920, height: 1080 },
|
|
21100
21138
|
tags: { workflow: "kernel-reddit", phase: "duckduckgo-search" },
|
|
21101
21139
|
telemetry: { enabled: true, browser: { captcha: { enabled: true }, page: { enabled: true } } }
|
|
21102
21140
|
});
|
|
21103
21141
|
try {
|
|
21104
|
-
const browser = await chromium4.connectOverCDP(created.cdp_ws_url);
|
|
21142
|
+
const browser = await chromium4.connectOverCDP(created.cdp_ws_url, { timeout: remainingTimeout(deadlineAt, 8e3) });
|
|
21105
21143
|
const context = browser.contexts()[0] ?? await browser.newContext({ locale: "en-US" });
|
|
21106
21144
|
const page3 = context.pages()[0] ?? await context.newPage();
|
|
21107
21145
|
await hooks?.browserCreated?.(kernel, created.session_id);
|
|
@@ -21125,12 +21163,12 @@ async function createSearchSession(apiKey, hooks) {
|
|
|
21125
21163
|
async function closeSearchSession(session, hooks, options = {}) {
|
|
21126
21164
|
const cleanupErrors = [];
|
|
21127
21165
|
await hooks?.browserClosing?.(session.kernel, session.browserId).catch(() => void 0);
|
|
21128
|
-
await session.browser.close().catch((error) => {
|
|
21166
|
+
await settleWithin(session.browser.close().catch((error) => {
|
|
21129
21167
|
cleanupErrors.push(sanitizeProviderLocalMessage(error instanceof Error ? error.message : String(error)) ?? "Kernel browser close failed");
|
|
21130
|
-
});
|
|
21131
|
-
await session.kernel.browsers.deleteByID(session.browserId).catch((error) => {
|
|
21168
|
+
}));
|
|
21169
|
+
await settleWithin(session.kernel.browsers.deleteByID(session.browserId).catch((error) => {
|
|
21132
21170
|
cleanupErrors.push(sanitizeProviderLocalMessage(error instanceof Error ? error.message : String(error)) ?? "Kernel browser deletion failed");
|
|
21133
|
-
});
|
|
21171
|
+
}));
|
|
21134
21172
|
await recordKernelSession({
|
|
21135
21173
|
kernelSessionId: session.browserId,
|
|
21136
21174
|
source: "kernel_reddit",
|
|
@@ -21186,22 +21224,21 @@ async function attachResidentialProxy(session) {
|
|
|
21186
21224
|
}
|
|
21187
21225
|
}
|
|
21188
21226
|
async function runSelectedThread(apiKey, input) {
|
|
21227
|
+
const deadlineAt = Date.now() + KERNEL_REDDIT_THREAD_ATTEMPT_MS;
|
|
21189
21228
|
const targetUrl = normalizeRedditThreadUrl(input.url);
|
|
21190
21229
|
if (!targetUrl) throw new Error("url must be a modern reddit.com thread URL");
|
|
21191
|
-
const session = await createSearchSession(apiKey);
|
|
21230
|
+
const session = await createSearchSession(apiKey, void 0, deadlineAt);
|
|
21192
21231
|
let residentialProxyId2 = null;
|
|
21193
21232
|
let attemptError = null;
|
|
21194
21233
|
try {
|
|
21195
21234
|
let selection = null;
|
|
21196
21235
|
let results = [];
|
|
21197
21236
|
const phaseOrder = [];
|
|
21198
|
-
|
|
21199
|
-
|
|
21200
|
-
|
|
21201
|
-
|
|
21202
|
-
|
|
21203
|
-
selection = { clickHref: selected.clickHref, position: selected.position };
|
|
21204
|
-
}
|
|
21237
|
+
results = await searchDuckDuckGoForReddit(session.page, input.query ?? redditThreadLookupQuery(targetUrl), 10, { deadlineAt });
|
|
21238
|
+
phaseOrder.push("duckduckgo_searched");
|
|
21239
|
+
const selected = results.find((result3) => new URL(result3.url).pathname === new URL(targetUrl).pathname);
|
|
21240
|
+
if (!selected) throw new Error("Requested Reddit URL was not present in the DuckDuckGo result set");
|
|
21241
|
+
selection = { clickHref: selected.clickHref, position: selected.position };
|
|
21205
21242
|
const proxy = await attachResidentialProxy(session);
|
|
21206
21243
|
residentialProxyId2 = proxy.id;
|
|
21207
21244
|
phaseOrder.push("residential_proxy_attached");
|
|
@@ -21209,14 +21246,14 @@ async function runSelectedThread(apiKey, input) {
|
|
|
21209
21246
|
session.page.on("response", (response) => {
|
|
21210
21247
|
if (response.request().isNavigationRequest() && response.request().frame() === session.page.mainFrame()) documentStatuses.push(response.status());
|
|
21211
21248
|
});
|
|
21212
|
-
|
|
21213
|
-
|
|
21214
|
-
phaseOrder.push("result_clicked");
|
|
21215
|
-
} else await session.page.goto(targetUrl, { waitUntil: "domcontentloaded", timeout: 6e4 });
|
|
21249
|
+
await clickDuckDuckGoResult(session.page, selection.clickHref, deadlineAt);
|
|
21250
|
+
phaseOrder.push("result_clicked");
|
|
21216
21251
|
const landedUrl = normalizeRedditThreadUrl(session.page.url());
|
|
21217
21252
|
if (!landedUrl || new URL(landedUrl).pathname !== new URL(targetUrl).pathname) throw new Error("Reddit landing URL did not match the selected result");
|
|
21218
21253
|
phaseOrder.push("reddit_landed");
|
|
21219
|
-
const thread = await readModernRedditThread(session.page, input.maxComments
|
|
21254
|
+
const thread = await readModernRedditThread(session.page, input.maxComments, {
|
|
21255
|
+
maxWaitMs: remainingTimeout(deadlineAt, REDDIT_THREAD_CHALLENGE_WAIT_MS)
|
|
21256
|
+
});
|
|
21220
21257
|
phaseOrder.push("thread_extracted");
|
|
21221
21258
|
return {
|
|
21222
21259
|
thread,
|
|
@@ -21240,7 +21277,7 @@ async function runSelectedThread(apiKey, input) {
|
|
|
21240
21277
|
throw error;
|
|
21241
21278
|
} finally {
|
|
21242
21279
|
await closeSearchSession(session, void 0, { proxyUsed: residentialProxyId2 !== null, error: attemptError });
|
|
21243
|
-
if (residentialProxyId2) await session.kernel.proxies.delete(residentialProxyId2).catch(() => void 0);
|
|
21280
|
+
if (residentialProxyId2) await settleWithin(session.kernel.proxies.delete(residentialProxyId2).catch(() => void 0));
|
|
21244
21281
|
}
|
|
21245
21282
|
}
|
|
21246
21283
|
async function runKernelRedditThread(apiKey, input) {
|
|
@@ -21614,6 +21651,24 @@ var BLOCKED_HOSTS = /* @__PURE__ */ new Set(["styles.redditmedia.com", "preview.
|
|
|
21614
21651
|
var REQUIRED_HOSTS = /* @__PURE__ */ new Set(["external-preview.redd.it", "googletagmanager.com"]);
|
|
21615
21652
|
var MAX_COMMENTS = 25;
|
|
21616
21653
|
var TELEMETRY_RETRY_DELAYS_MS = [0, 1500];
|
|
21654
|
+
var BRIGHT_DATA_REDDIT_ATTEMPT_MS = 35e3;
|
|
21655
|
+
var CLEANUP_TIMEOUT_MS2 = 3e3;
|
|
21656
|
+
function remainingTimeout2(deadlineAt, maximumMs) {
|
|
21657
|
+
const remaining = deadlineAt - Date.now();
|
|
21658
|
+
if (remaining <= 0) throw new Error("Bright Data Reddit browser attempt deadline exceeded");
|
|
21659
|
+
return Math.max(1, Math.min(maximumMs, remaining));
|
|
21660
|
+
}
|
|
21661
|
+
async function settleWithin2(promise, timeoutMs = CLEANUP_TIMEOUT_MS2) {
|
|
21662
|
+
let timer;
|
|
21663
|
+
return Promise.race([
|
|
21664
|
+
promise,
|
|
21665
|
+
new Promise((resolve3) => {
|
|
21666
|
+
timer = setTimeout(() => resolve3(null), timeoutMs);
|
|
21667
|
+
})
|
|
21668
|
+
]).finally(() => {
|
|
21669
|
+
if (timer) clearTimeout(timer);
|
|
21670
|
+
});
|
|
21671
|
+
}
|
|
21617
21672
|
function hostnameMatches(hostname, expected) {
|
|
21618
21673
|
return hostname === expected || hostname.endsWith(`.${expected}`);
|
|
21619
21674
|
}
|
|
@@ -21659,6 +21714,7 @@ async function getProviderSessionId(context, page3) {
|
|
|
21659
21714
|
return { cdp, id: validateProviderSessionId(response?.sessionId) };
|
|
21660
21715
|
}
|
|
21661
21716
|
async function runBrightDataRedditThreadBackup(url2, requestedComments) {
|
|
21717
|
+
const deadlineAt = Date.now() + BRIGHT_DATA_REDDIT_ATTEMPT_MS;
|
|
21662
21718
|
const endpoint = brightDataBrowserEndpoint();
|
|
21663
21719
|
if (!endpoint) throw new Error("Bright Data Reddit backup is not configured");
|
|
21664
21720
|
const canonicalUrl = normalizeRedditThreadUrl(url2);
|
|
@@ -21675,10 +21731,11 @@ async function runBrightDataRedditThreadBackup(url2, requestedComments) {
|
|
|
21675
21731
|
let blockedRequestCount = 0;
|
|
21676
21732
|
let finalUrl = null;
|
|
21677
21733
|
try {
|
|
21678
|
-
browser = await chromium5.connectOverCDP(endpoint);
|
|
21734
|
+
browser = await chromium5.connectOverCDP(endpoint, { timeout: remainingTimeout2(deadlineAt, 8e3) });
|
|
21679
21735
|
context = browser.contexts()[0] ?? await browser.newContext();
|
|
21680
21736
|
page3 = context.pages()[0] ?? await context.newPage();
|
|
21681
|
-
const session = await getProviderSessionId(context, page3);
|
|
21737
|
+
const session = await settleWithin2(getProviderSessionId(context, page3), remainingTimeout2(deadlineAt, 4e3));
|
|
21738
|
+
if (!session) throw new Error("Bright Data Reddit browser session identifier timed out");
|
|
21682
21739
|
cdp = session.cdp;
|
|
21683
21740
|
providerSessionId = session.id;
|
|
21684
21741
|
await page3.route("**/*", async (route) => {
|
|
@@ -21690,12 +21747,14 @@ async function runBrightDataRedditThreadBackup(url2, requestedComments) {
|
|
|
21690
21747
|
}
|
|
21691
21748
|
await route.continue();
|
|
21692
21749
|
});
|
|
21693
|
-
await page3.goto(canonicalUrl, { waitUntil: "domcontentloaded", timeout:
|
|
21750
|
+
await page3.goto(canonicalUrl, { waitUntil: "domcontentloaded", timeout: remainingTimeout2(deadlineAt, 1e4) });
|
|
21694
21751
|
finalUrl = normalizeRedditThreadUrl(page3.url());
|
|
21695
21752
|
if (!finalUrl || new URL(finalUrl).pathname !== new URL(canonicalUrl).pathname) {
|
|
21696
21753
|
throw new Error("Bright Data Reddit backup landed on an unexpected URL");
|
|
21697
21754
|
}
|
|
21698
|
-
thread = await readModernRedditThread(page3, Math.min(requestedComments, MAX_COMMENTS)
|
|
21755
|
+
thread = await readModernRedditThread(page3, Math.min(requestedComments, MAX_COMMENTS), {
|
|
21756
|
+
maxWaitMs: remainingTimeout2(deadlineAt, 15e3)
|
|
21757
|
+
});
|
|
21699
21758
|
if (!isAcceptedRedditThread(thread, requestedComments)) {
|
|
21700
21759
|
throw new Error(`Bright Data Reddit backup returned ${thread.comments.length} comments below the ${redditThreadAcceptanceTarget(thread, requestedComments)} comment acceptance target`);
|
|
21701
21760
|
}
|
|
@@ -21703,8 +21762,8 @@ async function runBrightDataRedditThreadBackup(url2, requestedComments) {
|
|
|
21703
21762
|
attemptError = error instanceof Error ? error : new Error(String(error));
|
|
21704
21763
|
}
|
|
21705
21764
|
try {
|
|
21706
|
-
if (cdp) await cdp.send("Browser.close");
|
|
21707
|
-
else await browser
|
|
21765
|
+
if (cdp) await settleWithin2(cdp.send("Browser.close"));
|
|
21766
|
+
else if (browser) await settleWithin2(browser.close());
|
|
21708
21767
|
} catch (error) {
|
|
21709
21768
|
closeError = sanitizeProviderLocalMessage(error instanceof Error ? error.message : String(error)) ?? "Bright Data browser close failed";
|
|
21710
21769
|
}
|