dsh-web-fetch-playwright 0.2.22 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/README.zh-CN.md +4 -2
- package/lib/client.js +19 -19
- package/lib/index.js +386 -50
- package/lib/types/candidates.d.ts +48 -0
- package/lib/types/challenge.d.ts +2 -2
- package/lib/types/config.d.ts +30 -4
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +55 -0
- package/lib/types/types.d.ts +19 -0
- package/package.json +1 -1
- package/src/candidates.ts +162 -29
- package/src/challenge.ts +2 -2
- package/src/config.ts +37 -6
- package/src/index.ts +2 -0
- package/src/markdown.ts +13 -2
- package/src/provider.ts +292 -27
- package/src/types.ts +20 -0
package/lib/index.js
CHANGED
|
@@ -20,6 +20,14 @@ import { delimiter, dirname, join } from "node:path";
|
|
|
20
20
|
* skipped outright.
|
|
21
21
|
*/
|
|
22
22
|
const DEFAULT_SCROLL_BUDGET_MS = 2e4;
|
|
23
|
+
/**
|
|
24
|
+
* Default cap (ms) on the post-navigation settle — the wait for SPA
|
|
25
|
+
* rendering to finish once the document has loaded (see the settle in
|
|
26
|
+
* `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
|
|
27
|
+
* the page's network goes quiet. Code-level only (tests, demo scripts):
|
|
28
|
+
* not part of the settings schema. `0` disables the settle entirely.
|
|
29
|
+
*/
|
|
30
|
+
const DEFAULT_SETTLE_MS = 5e3;
|
|
23
31
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
24
32
|
const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
25
33
|
/**
|
|
@@ -84,7 +92,7 @@ function snapshotsOf(config) {
|
|
|
84
92
|
* @returns the resolved config.
|
|
85
93
|
*/
|
|
86
94
|
function resolveConfig(config) {
|
|
87
|
-
const { challengeWaitMs, scrollBudgetMs, ...settings } = config;
|
|
95
|
+
const { challengeWaitMs, scrollBudgetMs, settleMs, ...settings } = config;
|
|
88
96
|
const resolved = Config(settings);
|
|
89
97
|
return {
|
|
90
98
|
backend: resolved.backend.get(),
|
|
@@ -97,7 +105,8 @@ function resolveConfig(config) {
|
|
|
97
105
|
challengeRetries: resolved.challengeRetries.get(),
|
|
98
106
|
challengeHttpFallback: resolved.challengeHttpFallback.get(),
|
|
99
107
|
...challengeWaitMs === void 0 ? {} : { challengeWaitMs },
|
|
100
|
-
...scrollBudgetMs === void 0 ? {} : { scrollBudgetMs }
|
|
108
|
+
...scrollBudgetMs === void 0 ? {} : { scrollBudgetMs },
|
|
109
|
+
...settleMs === void 0 ? {} : { settleMs }
|
|
101
110
|
};
|
|
102
111
|
}
|
|
103
112
|
/**
|
|
@@ -156,6 +165,18 @@ function effectiveScrollBudgetMs(config) {
|
|
|
156
165
|
return DEFAULT_SCROLL_BUDGET_MS;
|
|
157
166
|
}
|
|
158
167
|
/**
|
|
168
|
+
* The settle cap a fetch actually runs with: an explicit value wins, else
|
|
169
|
+
* the built-in default. `0` means the settle is off (domcontentloaded
|
|
170
|
+
* content is read as-is).
|
|
171
|
+
*
|
|
172
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
173
|
+
* @returns the bounded settle cap in milliseconds.
|
|
174
|
+
*/
|
|
175
|
+
function effectiveSettleMs(config) {
|
|
176
|
+
if (typeof config.settleMs === "number") return config.settleMs;
|
|
177
|
+
return DEFAULT_SETTLE_MS;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
159
180
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
160
181
|
* browser's real profile) requires the CDP backend — a local launch has no
|
|
161
182
|
* meaningful shared profile — and the share option (which defaults on, so an
|
|
@@ -231,8 +252,8 @@ function normalizeCdpEndpoint(input) {
|
|
|
231
252
|
const CHALLENGE_POLL_INTERVAL_MS = 500;
|
|
232
253
|
/**
|
|
233
254
|
* Tail of the fetch budget held back from the challenge wait so a cleared
|
|
234
|
-
* challenge still has time for its settle (
|
|
235
|
-
* content read and the denoise pipeline.
|
|
255
|
+
* challenge still has time for its settle (the `DEFAULT_SETTLE_MS` cap)
|
|
256
|
+
* plus the content read and the denoise pipeline.
|
|
236
257
|
*/
|
|
237
258
|
const CHALLENGE_FINISH_RESERVE_MS = 8e3;
|
|
238
259
|
/**
|
|
@@ -2215,15 +2236,16 @@ const GATE_MIN_BODY_TEXT_CHARS = 1e3;
|
|
|
2215
2236
|
/** Minimum share of the page's visible text a trusted extraction covers. */
|
|
2216
2237
|
const GATE_MIN_ARTICLE_COVERAGE = .1;
|
|
2217
2238
|
/**
|
|
2218
|
-
*
|
|
2219
|
-
*
|
|
2220
|
-
*
|
|
2239
|
+
* {@link articleCoversPage} on an already-parsed article container, so the
|
|
2240
|
+
* generator's gate and trailing prune share ONE parse of the article — the
|
|
2241
|
+
* extraction is hundreds of KB on heavy pages (measured on the Angular docs
|
|
2242
|
+
* badcase: ~400ms per parse of Readability's 485KB pick).
|
|
2243
|
+
*
|
|
2244
|
+
* @param container - the parsed article (read, never mutated).
|
|
2245
|
+
* @param bodyText - the page body's visible text length, computed once.
|
|
2221
2246
|
*/
|
|
2222
|
-
function
|
|
2223
|
-
const bodyText = visibleTextLength(document.body);
|
|
2247
|
+
function articleCoversParsed(container, bodyText) {
|
|
2224
2248
|
if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true;
|
|
2225
|
-
const container = document.createElement("div");
|
|
2226
|
-
container.innerHTML = articleHtml;
|
|
2227
2249
|
const articleText = visibleTextLength(container);
|
|
2228
2250
|
return articleText >= 250 && articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE;
|
|
2229
2251
|
}
|
|
@@ -2288,8 +2310,18 @@ function removeTrailingLinkDenseRun(parent, articleText) {
|
|
|
2288
2310
|
}
|
|
2289
2311
|
/** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
|
|
2290
2312
|
function prunedArticleContent(articleHtml, document) {
|
|
2291
|
-
|
|
2292
|
-
|
|
2313
|
+
return pruneParsedArticle(htmlFragment(articleHtml, document), articleHtml);
|
|
2314
|
+
}
|
|
2315
|
+
/**
|
|
2316
|
+
* {@link prunedArticleContent} on an already-parsed article container (the
|
|
2317
|
+
* generator passes the gate's parse: the gate reads it first, then this
|
|
2318
|
+
* mutates it — the same order the two-string-parses version produced).
|
|
2319
|
+
*
|
|
2320
|
+
* @param container - the parsed article; pruned in place.
|
|
2321
|
+
* @param articleHtml - the string the container was parsed from (the
|
|
2322
|
+
* misfire-guard's verbatim return).
|
|
2323
|
+
*/
|
|
2324
|
+
function pruneParsedArticle(container, articleHtml) {
|
|
2293
2325
|
const before = visibleTextLength(container);
|
|
2294
2326
|
if (before === 0) return articleHtml;
|
|
2295
2327
|
pruneTrailingLinkDenseSections(container);
|
|
@@ -2297,13 +2329,24 @@ function prunedArticleContent(articleHtml, document) {
|
|
|
2297
2329
|
if (after === 0 || before >= 250 && after < 250) return articleHtml;
|
|
2298
2330
|
return container.innerHTML;
|
|
2299
2331
|
}
|
|
2300
|
-
/**
|
|
2332
|
+
/**
|
|
2333
|
+
* Link density at or above which a recommendation-named subtree is a rail.
|
|
2334
|
+
*/
|
|
2301
2335
|
const RAIL_LINK_DENSITY = .3;
|
|
2302
2336
|
const RAIL_MIN_LINKS = 4;
|
|
2303
2337
|
/** Prose (low-link text) the page must keep outside the rails. */
|
|
2304
2338
|
const RAIL_MIN_PROSE_CHARS = 80;
|
|
2305
2339
|
const RAIL_PROSE_LINK_DENSITY = .2;
|
|
2306
2340
|
/**
|
|
2341
|
+
* Necessary condition for {@link isRailName} on the raw class+id string: a
|
|
2342
|
+
* matching token (`recommended-posts`, `Related`, …) always leaves one of
|
|
2343
|
+
* these literals as a substring. Testing it first short-circuits the
|
|
2344
|
+
* per-token split on the pages where it matters — {@link markRecommendationRails}
|
|
2345
|
+
* walks every element, and on the measured 1.1MB Angular docs badcase the
|
|
2346
|
+
* split alone cost ~360ms of the candidate generation.
|
|
2347
|
+
*/
|
|
2348
|
+
const RAIL_NAME_HINT = /recommend|related|outbrain|taboola/;
|
|
2349
|
+
/**
|
|
2307
2350
|
* Class/id sub-tokens naming a recommendation / related-content rail: the
|
|
2308
2351
|
* English words every CMS and theme uses, plus the two syndicated
|
|
2309
2352
|
* recommendation widgets embedded across thousands of publishers (their
|
|
@@ -2314,7 +2357,9 @@ function isRailName(token) {
|
|
|
2314
2357
|
return token.includes("recommend") || token.startsWith("related") || token === "outbrain" || token === "taboola";
|
|
2315
2358
|
}
|
|
2316
2359
|
function isRecommendationRail(element) {
|
|
2317
|
-
|
|
2360
|
+
const names = `${element.getAttribute("class") ?? ""} ${element.id}`.toLowerCase();
|
|
2361
|
+
if (!RAIL_NAME_HINT.test(names)) return false;
|
|
2362
|
+
if (!names.split(/[\s_-]+/).some(isRailName)) return false;
|
|
2318
2363
|
if (element.querySelectorAll("a").length < RAIL_MIN_LINKS) return false;
|
|
2319
2364
|
const text = visibleTextLength(element);
|
|
2320
2365
|
return text > 0 && linkTextLength(element) / text >= RAIL_LINK_DENSITY;
|
|
@@ -2654,6 +2699,59 @@ function escapeHtml(text) {
|
|
|
2654
2699
|
/** Below this much noscript text the noscript-unwrapped paths are skipped. */
|
|
2655
2700
|
const NOSCRIPT_MIN_TEXT_CHARS = 200;
|
|
2656
2701
|
/**
|
|
2702
|
+
* Share of the body's visible text that sits inside custom-element subtrees
|
|
2703
|
+
* (tag names containing `-`: `<ms-app>`, `<my-doc>` — the HTML spec reserves
|
|
2704
|
+
* them for custom elements; no standard HTML or MathML tag and no tag the
|
|
2705
|
+
* denoise pipeline's DOMPurify step keeps contains a dash). DOMPurify's
|
|
2706
|
+
* allowlist removes such tags wholesale — `KEEP_CONTENT: false` takes their
|
|
2707
|
+
* subtrees with them — so once nearly ALL the page's text is custom-wrapped,
|
|
2708
|
+
* every extraction that would carry it sanitizes to a corpse. Measured on
|
|
2709
|
+
* the badcase (aistudio.google.com/docs/computer-use, an Angular app whose
|
|
2710
|
+
* entire DOM lives under `<ms-app>`): 99% of body text custom-wrapped;
|
|
2711
|
+
* Readability's pick sanitized to a root with ZERO words, the whole-document
|
|
2712
|
+
* candidate to 39 of ~4.5k words — while the three Readability passes over
|
|
2713
|
+
* the 1.1MB document cost ~6.5s of pure latency on candidates that could
|
|
2714
|
+
* never win. The DOM-backed fallbacks, the noscript paths and the block
|
|
2715
|
+
* classifier — the candidates that survive DOMPurify on this shape, and the
|
|
2716
|
+
* ones the selector picks on such pages anyway — carry the extraction.
|
|
2717
|
+
*
|
|
2718
|
+
* Noise subtrees (script/style/nav/…) count as neither wrapped nor unwrapped;
|
|
2719
|
+
* the share is over CONTENT-bearing text only. A partial share (a web
|
|
2720
|
+
* component widget inside a normal article) stays below the threshold and
|
|
2721
|
+
* keeps the full pipeline: only the shape where the family is provably dead
|
|
2722
|
+
* pays for it.
|
|
2723
|
+
*/
|
|
2724
|
+
function customWrappedTextShare(body) {
|
|
2725
|
+
let total = 0;
|
|
2726
|
+
let wrapped = 0;
|
|
2727
|
+
const visit = (node, insideCustom) => {
|
|
2728
|
+
if (node.nodeType === 3) {
|
|
2729
|
+
const length = (node.nodeValue ?? "").replace(/\s+/g, "").length;
|
|
2730
|
+
if (length === 0) return;
|
|
2731
|
+
total += length;
|
|
2732
|
+
if (insideCustom) wrapped += length;
|
|
2733
|
+
return;
|
|
2734
|
+
}
|
|
2735
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return;
|
|
2736
|
+
const custom = insideCustom || node.nodeName.includes("-");
|
|
2737
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, custom);
|
|
2738
|
+
};
|
|
2739
|
+
visit(body, false);
|
|
2740
|
+
return total === 0 ? 0 : wrapped / total;
|
|
2741
|
+
}
|
|
2742
|
+
/**
|
|
2743
|
+
* The title the gated path hands its candidates: Readability's own first
|
|
2744
|
+
* source is `og:title`, its second the `<title>` element — both remain
|
|
2745
|
+
* readable on custom-element pages, where the DOM wrapping is the framework's
|
|
2746
|
+
* but the metadata is the page's.
|
|
2747
|
+
*/
|
|
2748
|
+
function fallbackTitle(document) {
|
|
2749
|
+
const og = document.querySelector("meta[property=\"og:title\"]")?.getAttribute("content")?.trim();
|
|
2750
|
+
if (og !== void 0 && og !== "") return og;
|
|
2751
|
+
const title = document.querySelector("title")?.textContent?.trim();
|
|
2752
|
+
return title === void 0 || title === "" ? void 0 : title;
|
|
2753
|
+
}
|
|
2754
|
+
/**
|
|
2657
2755
|
* Propose every candidate extraction for one parsed page.
|
|
2658
2756
|
* @param document - the parsed page (data URIs already elided); not mutated.
|
|
2659
2757
|
*/
|
|
@@ -2661,48 +2759,59 @@ function generateCandidates(document) {
|
|
|
2661
2759
|
const candidates = [];
|
|
2662
2760
|
const body = document.body;
|
|
2663
2761
|
const bodyHtml = body?.innerHTML ?? "";
|
|
2664
|
-
const
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
content: rails.plain
|
|
2669
|
-
};
|
|
2670
|
-
const gatePass = art !== null && articleCoversPage(art.content, document);
|
|
2762
|
+
const skipReadability = body !== null && customWrappedTextShare(body) >= .9;
|
|
2763
|
+
let art = null;
|
|
2764
|
+
let lenient = null;
|
|
2765
|
+
let gatePass = false;
|
|
2671
2766
|
let pruneChanged = false;
|
|
2672
|
-
if (
|
|
2673
|
-
const
|
|
2674
|
-
|
|
2675
|
-
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2680
|
-
|
|
2681
|
-
|
|
2682
|
-
|
|
2683
|
-
|
|
2684
|
-
|
|
2767
|
+
if (!skipReadability) {
|
|
2768
|
+
const marked = runReadability(markRecommendationRails(document) ?? document);
|
|
2769
|
+
const rails = marked === null ? null : splitRails(marked.content, document);
|
|
2770
|
+
art = marked === null || rails === null ? null : {
|
|
2771
|
+
...marked,
|
|
2772
|
+
content: rails.plain
|
|
2773
|
+
};
|
|
2774
|
+
if (art !== null) {
|
|
2775
|
+
const bodyText = visibleTextLength(body);
|
|
2776
|
+
const parsed = htmlFragment(art.content, document);
|
|
2777
|
+
gatePass = articleCoversParsed(parsed, bodyText);
|
|
2778
|
+
const pruning = rails?.derailed ?? art.content;
|
|
2779
|
+
const pruned = pruning === art.content ? pruneParsedArticle(parsed, art.content) : prunedArticleContent(pruning, document);
|
|
2780
|
+
pruneChanged = pruned !== art.content;
|
|
2781
|
+
candidates.push({
|
|
2782
|
+
kind: "art",
|
|
2783
|
+
html: pruned,
|
|
2784
|
+
title: art.title,
|
|
2785
|
+
legacy: gatePass
|
|
2786
|
+
});
|
|
2787
|
+
if (pruneChanged) candidates.push({
|
|
2788
|
+
kind: "artu",
|
|
2789
|
+
html: art.content,
|
|
2790
|
+
title: art.title,
|
|
2791
|
+
legacy: false
|
|
2792
|
+
});
|
|
2793
|
+
}
|
|
2794
|
+
lenient = runReadability(document, { linkDensityModifier: .4 });
|
|
2795
|
+
if (lenient !== null && lenient.content !== art?.content) candidates.push({
|
|
2796
|
+
kind: "artx",
|
|
2797
|
+
html: lenient.content,
|
|
2798
|
+
title: lenient.title,
|
|
2685
2799
|
legacy: false
|
|
2686
2800
|
});
|
|
2687
2801
|
}
|
|
2688
|
-
const
|
|
2689
|
-
if (lenient !== null && lenient.content !== art?.content) candidates.push({
|
|
2690
|
-
kind: "artx",
|
|
2691
|
-
html: lenient.content,
|
|
2692
|
-
title: lenient.title,
|
|
2693
|
-
legacy: false
|
|
2694
|
-
});
|
|
2695
|
-
const title = art?.title ?? lenient?.title;
|
|
2802
|
+
const title = art?.title ?? lenient?.title ?? (skipReadability ? fallbackTitle(document) : void 0);
|
|
2696
2803
|
const main = body === null ? null : findMainContainer(document);
|
|
2697
2804
|
if (main !== null) candidates.push({
|
|
2698
2805
|
kind: "main",
|
|
2699
2806
|
html: main.innerHTML,
|
|
2807
|
+
node: main,
|
|
2700
2808
|
title,
|
|
2701
2809
|
legacy: false
|
|
2702
2810
|
});
|
|
2703
2811
|
candidates.push({
|
|
2704
2812
|
kind: "doc",
|
|
2705
2813
|
html: bodyHtml,
|
|
2814
|
+
node: body ?? void 0,
|
|
2706
2815
|
legacy: !gatePass
|
|
2707
2816
|
});
|
|
2708
2817
|
if (body !== null) {
|
|
@@ -2711,6 +2820,7 @@ function generateCandidates(document) {
|
|
|
2711
2820
|
candidates.push({
|
|
2712
2821
|
kind: "docc",
|
|
2713
2822
|
html: cleaned.innerHTML,
|
|
2823
|
+
node: cleaned,
|
|
2714
2824
|
legacy: false
|
|
2715
2825
|
});
|
|
2716
2826
|
}
|
|
@@ -2738,6 +2848,7 @@ function generateCandidates(document) {
|
|
|
2738
2848
|
candidates.push({
|
|
2739
2849
|
kind: "ns_doc",
|
|
2740
2850
|
html: unwrapped.body?.innerHTML ?? "",
|
|
2851
|
+
node: unwrapped.body ?? void 0,
|
|
2741
2852
|
legacy: false
|
|
2742
2853
|
});
|
|
2743
2854
|
}
|
|
@@ -2764,7 +2875,8 @@ function generateCandidates(document) {
|
|
|
2764
2875
|
record,
|
|
2765
2876
|
hasMain: main !== null,
|
|
2766
2877
|
hasPreloaded: preloaded !== null,
|
|
2767
|
-
main
|
|
2878
|
+
main,
|
|
2879
|
+
...skipReadability ? { readabilitySkipped: true } : {}
|
|
2768
2880
|
}
|
|
2769
2881
|
};
|
|
2770
2882
|
}
|
|
@@ -59714,7 +59826,7 @@ function analyze(html, url) {
|
|
|
59714
59826
|
};
|
|
59715
59827
|
}
|
|
59716
59828
|
const sanitize = (candidate) => {
|
|
59717
|
-
const root = purify.sanitize(candidate.html, {
|
|
59829
|
+
const root = purify.sanitize(candidate.node ?? candidate.html, {
|
|
59718
59830
|
FORBID_TAGS,
|
|
59719
59831
|
FORBID_ATTR,
|
|
59720
59832
|
KEEP_CONTENT: false,
|
|
@@ -59729,6 +59841,16 @@ function analyze(html, url) {
|
|
|
59729
59841
|
};
|
|
59730
59842
|
};
|
|
59731
59843
|
const sanitized = generated.candidates.map(sanitize);
|
|
59844
|
+
if (generated.facts.readabilitySkipped === true && !sanitized.some((c) => c.kind === "art")) {
|
|
59845
|
+
const empty = document.createElement("div");
|
|
59846
|
+
sanitized.push({
|
|
59847
|
+
kind: "art",
|
|
59848
|
+
title: void 0,
|
|
59849
|
+
legacy: false,
|
|
59850
|
+
root: empty,
|
|
59851
|
+
stats: measureCandidate("art", false, empty)
|
|
59852
|
+
});
|
|
59853
|
+
}
|
|
59732
59854
|
const page = pageFeatures(document, url, generated.facts, sanitized.find((c) => c.kind === "doc")?.stats, visibleTextLength(document.body));
|
|
59733
59855
|
let blocks = [];
|
|
59734
59856
|
let blockVectors = [];
|
|
@@ -60921,6 +61043,21 @@ function existsAsFile(path) {
|
|
|
60921
61043
|
* refetch through the page's API request context
|
|
60922
61044
|
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
60923
61045
|
*
|
|
61046
|
+
* Transient transport failures (issue: `net::ERR_NETWORK_CHANGED` and its
|
|
61047
|
+
* family) are retried, not terminal: Chromium's NetworkChangeNotifier kills
|
|
61048
|
+
* every in-flight request with `ERR_NETWORK_CHANGED` the moment the HOST's
|
|
61049
|
+
* network configuration shifts under the browser — a route/interface/DNS/
|
|
61050
|
+
* proxy change, a VPN or DHCP flap, a Docker embedded-DNS rewrite; the page
|
|
61051
|
+
* and its target are perfectly fine, only that one navigation died, and the
|
|
61052
|
+
* very next `goto` usually succeeds (measured live: source.android.com
|
|
61053
|
+
* failed once mid-navigation in one DSH session and fetched clean from the
|
|
61054
|
+
* same host seconds later). Each navigation therefore retries the
|
|
61055
|
+
* transport-blip family ({@link isTransientNavigationFailure} — deliberately
|
|
61056
|
+
* NOT `ERR_ABORTED`, which is overloaded with the deterministic shapes the
|
|
61057
|
+
* recoveries above already own) a bounded number of times with a short
|
|
61058
|
+
* backoff, inside the same page and the remaining waiting budget; a blip
|
|
61059
|
+
* that survives the attempt cap still fails as before.
|
|
61060
|
+
*
|
|
60924
61061
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
60925
61062
|
* nothing outlives the call. CDP session isolation keeps one connection per
|
|
60926
61063
|
* DSH session until its context becomes idle; this also isolates gateways that
|
|
@@ -61041,8 +61178,30 @@ const DEFAULT_TIMEOUT_MS = 25e3;
|
|
|
61041
61178
|
const EXTRACT_GRACE_MS = 1e4;
|
|
61042
61179
|
/** Bound (ms) on the best-effort `window.stop()` of a still-loading page. */
|
|
61043
61180
|
const STOP_LOADING_GRACE_MS = 1e3;
|
|
61044
|
-
/**
|
|
61045
|
-
|
|
61181
|
+
/**
|
|
61182
|
+
* Settle quiet window (ms): the post-load settle ends once no request has
|
|
61183
|
+
* STARTED for this long AND every still-open request is stale (see
|
|
61184
|
+
* {@link SETTLE_STALE_MS}) — the Lighthouse-style network-quiet threshold,
|
|
61185
|
+
* sized to match Playwright's own networkidle quiet window.
|
|
61186
|
+
*/
|
|
61187
|
+
const SETTLE_QUIET_MS = 500;
|
|
61188
|
+
/**
|
|
61189
|
+
* A request open longer than this (ms) stops gating the settle. On the
|
|
61190
|
+
* pages that motivated the custom detector, the long-lived request is a
|
|
61191
|
+
* worker-script fetch or a cross-origin auth frame whose completion events
|
|
61192
|
+
* are attributed to another CDP target and NEVER arrive on the page session
|
|
61193
|
+
* (measured live over CDP: platform.claude.com's `turbopack-worker-*.js`
|
|
61194
|
+
* and aistudio.google.com's `accounts.google.com/ServiceLogin` probe stayed
|
|
61195
|
+
* "open" for the page's whole life while the page itself completed at
|
|
61196
|
+
* `load`), so strict `networkidle` — Playwright's is officially discouraged
|
|
61197
|
+
* — is unreachable there and the legacy wait burned its whole cap on every
|
|
61198
|
+
* fetch. A genuine content request this old is the documented trade: the
|
|
61199
|
+
* settle stops waiting for it, and the scroll walk plus the live read still
|
|
61200
|
+
* capture whatever it renders when it lands.
|
|
61201
|
+
*/
|
|
61202
|
+
const SETTLE_STALE_MS = 3e3;
|
|
61203
|
+
/** How often the settle re-checks its quiet/stale conditions (ms). */
|
|
61204
|
+
const SETTLE_POLL_MS = 100;
|
|
61046
61205
|
/**
|
|
61047
61206
|
* Grace period (ms) for context/browser closes in the cleanup path. A wedged
|
|
61048
61207
|
* `close()` must never hold a concurrency slot hostage — a leaked slot would
|
|
@@ -61050,6 +61209,27 @@ const SETTLE_MS = 5e3;
|
|
|
61050
61209
|
*/
|
|
61051
61210
|
const CLOSE_GRACE_MS = 2e3;
|
|
61052
61211
|
/**
|
|
61212
|
+
* How many times a navigation may be re-issued after a TRANSIENT transport
|
|
61213
|
+
* failure (`net::ERR_NETWORK_CHANGED` and family — see
|
|
61214
|
+
* {@link isTransientNavigationFailure}) before the fetch gives up. Three
|
|
61215
|
+
* total navigations cover the single-blip badcase and the rare double-blip
|
|
61216
|
+
* while capping the worst-case latency a persistently flapping network can
|
|
61217
|
+
* add.
|
|
61218
|
+
*/
|
|
61219
|
+
const MAX_TRANSIENT_NAV_RETRIES = 2;
|
|
61220
|
+
/**
|
|
61221
|
+
* Backoff (ms) before each transient-navigation retry, indexed by the retry
|
|
61222
|
+
* number: short first (most blips clear in well under a second), longer
|
|
61223
|
+
* second. Bounded by the fetch deadline via an abortable sleep.
|
|
61224
|
+
*/
|
|
61225
|
+
const TRANSIENT_RETRY_BACKOFF_MS = [250, 750];
|
|
61226
|
+
/**
|
|
61227
|
+
* Waiting budget (ms) that must REMAIN before a transient-navigation retry
|
|
61228
|
+
* may start: the retry must not eat the tail the settle/decode path still
|
|
61229
|
+
* needs, and a re-navigation that cannot finish is pure waste.
|
|
61230
|
+
*/
|
|
61231
|
+
const TRANSIENT_RETRY_RESERVE_MS = 2e3;
|
|
61232
|
+
/**
|
|
61053
61233
|
* The fetch's two-stage deadline composed with the caller's signal. The SOFT
|
|
61054
61234
|
* stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
|
|
61055
61235
|
* it and {@link expired} reports it, so every wait ends there and the fetch
|
|
@@ -61208,6 +61388,12 @@ var PlaywrightFetchProvider = class {
|
|
|
61208
61388
|
fetchTimeoutMs = DEFAULT_TIMEOUT_MS;
|
|
61209
61389
|
/** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
|
|
61210
61390
|
extractGraceMs = EXTRACT_GRACE_MS;
|
|
61391
|
+
/** Settle quiet window (ms); code-level, overridable by subclasses (tests). */
|
|
61392
|
+
settleQuietMs = SETTLE_QUIET_MS;
|
|
61393
|
+
/** Request age (ms) past which it stops gating the settle; overridable by subclasses (tests). */
|
|
61394
|
+
settleStaleMs = SETTLE_STALE_MS;
|
|
61395
|
+
/** Settle re-check cadence (ms); overridable by subclasses (tests). */
|
|
61396
|
+
settlePollMs = SETTLE_POLL_MS;
|
|
61211
61397
|
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
61212
61398
|
cdpPool;
|
|
61213
61399
|
/**
|
|
@@ -61345,6 +61531,7 @@ var PlaywrightFetchProvider = class {
|
|
|
61345
61531
|
const page = session.page;
|
|
61346
61532
|
const challengeWaitMs = effectiveChallengeWaitMs(config);
|
|
61347
61533
|
const tracker = trackMainFrameResponses(page);
|
|
61534
|
+
const activity = trackRequestActivity(page);
|
|
61348
61535
|
const first = await this.navigate(page, url, tracker, deadline, config.denoise);
|
|
61349
61536
|
if (first.recovered !== void 0) return first.recovered;
|
|
61350
61537
|
let response = first.response;
|
|
@@ -61396,7 +61583,7 @@ var PlaywrightFetchProvider = class {
|
|
|
61396
61583
|
kind: "text",
|
|
61397
61584
|
content: partial ? await renderedText(page) : finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
61398
61585
|
}), partial);
|
|
61399
|
-
await
|
|
61586
|
+
await this.settleForContent(page, deadline, config, activity);
|
|
61400
61587
|
const scrollBudgetMs = deadline.expired ? 0 : Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs());
|
|
61401
61588
|
const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : void 0;
|
|
61402
61589
|
let html;
|
|
@@ -61464,13 +61651,18 @@ var PlaywrightFetchProvider = class {
|
|
|
61464
61651
|
* status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
|
|
61465
61652
|
* being a download the browser refuses to commit as a document
|
|
61466
61653
|
* (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
|
|
61654
|
+
* A TRANSIENT transport failure (`net::ERR_NETWORK_CHANGED` and family)
|
|
61655
|
+
* is neither: it carries no response and no render decision, only a
|
|
61656
|
+
* momentary host-network blip that killed this one navigation — so it is
|
|
61657
|
+
* retried on the SAME page, bounded by {@link MAX_TRANSIENT_NAV_RETRIES}
|
|
61658
|
+
* attempts, a short backoff, and the remaining waiting budget.
|
|
61467
61659
|
*
|
|
61468
61660
|
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
61469
61661
|
* navigation failure was recovered into a finished result the caller must
|
|
61470
61662
|
* return verbatim (the throw never escapes in that case).
|
|
61471
61663
|
*/
|
|
61472
61664
|
async navigate(page, url, tracker, deadline, denoise) {
|
|
61473
|
-
try {
|
|
61665
|
+
for (let attempt = 0;; attempt++) try {
|
|
61474
61666
|
return { response: await page.goto(url.toString(), {
|
|
61475
61667
|
waitUntil: "domcontentloaded",
|
|
61476
61668
|
timeout: deadline.remainingMs()
|
|
@@ -61495,6 +61687,10 @@ var PlaywrightFetchProvider = class {
|
|
|
61495
61687
|
partial: true
|
|
61496
61688
|
};
|
|
61497
61689
|
}
|
|
61690
|
+
if (attempt < MAX_TRANSIENT_NAV_RETRIES && isTransientNavigationFailure(error) && deadline.remainingMs() > TRANSIENT_RETRY_RESERVE_MS) {
|
|
61691
|
+
await abortableSleep(TRANSIENT_RETRY_BACKOFF_MS[Math.min(attempt, TRANSIENT_RETRY_BACKOFF_MS.length - 1)] ?? TRANSIENT_RETRY_BACKOFF_MS.at(-1) ?? 500, deadline.signal);
|
|
61692
|
+
if (!deadline.signal.aborted) continue;
|
|
61693
|
+
}
|
|
61498
61694
|
}
|
|
61499
61695
|
throw error;
|
|
61500
61696
|
}
|
|
@@ -61542,6 +61738,64 @@ var PlaywrightFetchProvider = class {
|
|
|
61542
61738
|
return true;
|
|
61543
61739
|
}
|
|
61544
61740
|
}
|
|
61741
|
+
/**
|
|
61742
|
+
* Best-effort settle for client-rendered content — a Lighthouse-style
|
|
61743
|
+
* bounded network-quiet window replacing Playwright's `networkidle`
|
|
61744
|
+
* (officially discouraged): worker-script fetches and cross-origin auth
|
|
61745
|
+
* frames hold one request open on the page session FOREVER (their
|
|
61746
|
+
* completion events are attributed to other CDP targets), so strict
|
|
61747
|
+
* networkidle never arrives on exactly the modern pages the settle exists
|
|
61748
|
+
* for, and the legacy wait burned its whole cap on them every fetch.
|
|
61749
|
+
*
|
|
61750
|
+
* Sequence: a `load` baseline (usually already past — domcontentloaded
|
|
61751
|
+
* resolved earlier), then poll until no request has STARTED within
|
|
61752
|
+
* {@link settleQuietMs} and every still-open request is older than
|
|
61753
|
+
* {@link settleStaleMs} — a request that old is an attribution artifact
|
|
61754
|
+
* or a long-poll whose completion no longer gates content. All waits
|
|
61755
|
+
* share the {@link effectiveSettleMs} cap and the fetch deadline; a page
|
|
61756
|
+
* that exposes no request events degrades to the legacy bounded
|
|
61757
|
+
* networkidle wait. Never throws: any exit path keeps what the document
|
|
61758
|
+
* already holds.
|
|
61759
|
+
*/
|
|
61760
|
+
async settleForContent(page, deadline, config, activity) {
|
|
61761
|
+
const beganAt = Date.now();
|
|
61762
|
+
const cap = effectiveSettleMs(config);
|
|
61763
|
+
if (cap <= 0) {
|
|
61764
|
+
this.noteSettleOutcome("off", 0);
|
|
61765
|
+
return;
|
|
61766
|
+
}
|
|
61767
|
+
const endsAt = beganAt + Math.min(cap, deadline.remainingMs());
|
|
61768
|
+
await page.waitForLoadState("load", { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {});
|
|
61769
|
+
if (activity === void 0) {
|
|
61770
|
+
await page.waitForLoadState("networkidle", { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {});
|
|
61771
|
+
this.noteSettleOutcome("fallback", Date.now() - beganAt);
|
|
61772
|
+
return;
|
|
61773
|
+
}
|
|
61774
|
+
for (;;) {
|
|
61775
|
+
if (deadline.signal.aborted) {
|
|
61776
|
+
this.noteSettleOutcome("aborted", Date.now() - beganAt);
|
|
61777
|
+
return;
|
|
61778
|
+
}
|
|
61779
|
+
if (Date.now() >= endsAt) {
|
|
61780
|
+
this.noteSettleOutcome("cap", Date.now() - beganAt);
|
|
61781
|
+
return;
|
|
61782
|
+
}
|
|
61783
|
+
if (activity.idle(this.settleQuietMs, this.settleStaleMs)) {
|
|
61784
|
+
this.noteSettleOutcome(activity.openCount() > 0 ? "stale-discounted" : "quiet", Date.now() - beganAt);
|
|
61785
|
+
return;
|
|
61786
|
+
}
|
|
61787
|
+
await sleep(Math.min(this.settlePollMs, Math.max(1, endsAt - Date.now())));
|
|
61788
|
+
}
|
|
61789
|
+
}
|
|
61790
|
+
/**
|
|
61791
|
+
* Settle outcome observation hook — silent in production by design (the
|
|
61792
|
+
* settle is best-effort), but subclasses (tests, diagnostics) override it
|
|
61793
|
+
* to see which exit path every fetch took: `off` (config), `quiet`
|
|
61794
|
+
* (network went quiet), `stale-discounted` (only stale requests remained
|
|
61795
|
+
* open), `cap` (window exhausted), `fallback` (no request events — the
|
|
61796
|
+
* legacy networkidle wait ran), `aborted` (fetch deadline fired).
|
|
61797
|
+
*/
|
|
61798
|
+
noteSettleOutcome(outcome, elapsedMs) {}
|
|
61545
61799
|
};
|
|
61546
61800
|
/** Resolve after `ms` — the bounded wait's inter-poll nap. */
|
|
61547
61801
|
function sleep(ms) {
|
|
@@ -61614,6 +61868,43 @@ function trackMainFrameResponses(page) {
|
|
|
61614
61868
|
};
|
|
61615
61869
|
}
|
|
61616
61870
|
/**
|
|
61871
|
+
* Watch the page for request lifecycle events so the settle can decide
|
|
61872
|
+
* network quietness itself. `undefined` when the page exposes no request
|
|
61873
|
+
* events (minimal fakes): the settle then falls back to the legacy bounded
|
|
61874
|
+
* networkidle wait.
|
|
61875
|
+
*
|
|
61876
|
+
* @param page - the fetch's tab.
|
|
61877
|
+
* @returns the open-request tracker, or undefined on an unsupported surface.
|
|
61878
|
+
*/
|
|
61879
|
+
function trackRequestActivity(page) {
|
|
61880
|
+
if (typeof page.on !== "function") return void 0;
|
|
61881
|
+
const opened = /* @__PURE__ */ new Map();
|
|
61882
|
+
let lastStart = Date.now();
|
|
61883
|
+
const onStarted = (request) => {
|
|
61884
|
+
opened.set(request, Date.now());
|
|
61885
|
+
lastStart = Date.now();
|
|
61886
|
+
};
|
|
61887
|
+
const onSettled = (request) => {
|
|
61888
|
+
opened.delete(request);
|
|
61889
|
+
};
|
|
61890
|
+
try {
|
|
61891
|
+
page.on("request", onStarted);
|
|
61892
|
+
page.on("requestfinished", onSettled);
|
|
61893
|
+
page.on("requestfailed", onSettled);
|
|
61894
|
+
} catch {
|
|
61895
|
+
return;
|
|
61896
|
+
}
|
|
61897
|
+
return {
|
|
61898
|
+
idle: (quietMs, staleMs) => {
|
|
61899
|
+
const now = Date.now();
|
|
61900
|
+
if (now - lastStart < quietMs) return false;
|
|
61901
|
+
for (const startedAt of opened.values()) if (now - startedAt < staleMs) return false;
|
|
61902
|
+
return true;
|
|
61903
|
+
},
|
|
61904
|
+
openCount: () => opened.size
|
|
61905
|
+
};
|
|
61906
|
+
}
|
|
61907
|
+
/**
|
|
61617
61908
|
* Whether a `goto` throw is Chromium refusing to RENDER a response it
|
|
61618
61909
|
* received — not a transport failure. Two net errors, both documented as
|
|
61619
61910
|
* rendering decisions rather than network ones:
|
|
@@ -61637,6 +61928,51 @@ function isUnrenderedStatusFailure(error, status) {
|
|
|
61637
61928
|
return message.includes("ERR_ABORTED") && (status === 204 || status === 205);
|
|
61638
61929
|
}
|
|
61639
61930
|
/**
|
|
61931
|
+
* Chromium net errors a plain re-navigation can plausibly fix — the
|
|
61932
|
+
* transport-layer BLIPS, not render decisions (those are recovered above)
|
|
61933
|
+
* and not `ERR_ABORTED` (deliberately excluded: that code is overloaded with
|
|
61934
|
+
* deterministic shapes — SPA cancels, 204/205 aborts, downloads — which the
|
|
61935
|
+
* recoveries already own). `ERR_NETWORK_CHANGED` is the measured badcase:
|
|
61936
|
+
* the host's network configuration shifted under the browser mid-navigation
|
|
61937
|
+
* (route/interface/DNS/proxy change, VPN or DHCP flap, a Docker
|
|
61938
|
+
* embedded-DNS rewrite) and Chromium's NetworkChangeNotifier killed the
|
|
61939
|
+
* in-flight request; the sibling codes are the same story on a different
|
|
61940
|
+
* layer (DNS blips, TCP resets and idle breaks, brief disconnects).
|
|
61941
|
+
* Playwright's own navigation timeouts never match: they reject with a
|
|
61942
|
+
* `TimeoutError` ("Timeout Nms exceeded"), not a `net::ERR_*` code.
|
|
61943
|
+
*/
|
|
61944
|
+
const TRANSIENT_NET_ERROR_RE = /net::ERR_(NETWORK_CHANGED|INTERNET_DISCONNECTED|CONNECTION_RESET|CONNECTION_CLOSED|CONNECTION_TIMED_OUT|TIMED_OUT|EMPTY_RESPONSE|NAME_NOT_RESOLVED|NAME_RESOLUTION_FAILED|ADDRESS_UNREACHABLE)\b/;
|
|
61945
|
+
/**
|
|
61946
|
+
* Whether a `goto` throw is one of the transient transport failures a
|
|
61947
|
+
* bounded same-page retry can recover (see {@link TRANSIENT_NET_ERROR_RE}).
|
|
61948
|
+
*
|
|
61949
|
+
* @param error - the value `page.goto` rejected with.
|
|
61950
|
+
* @returns true when the navigation died to a momentary network blip.
|
|
61951
|
+
*/
|
|
61952
|
+
function isTransientNavigationFailure(error) {
|
|
61953
|
+
return error instanceof Error && TRANSIENT_NET_ERROR_RE.test(error.message);
|
|
61954
|
+
}
|
|
61955
|
+
/**
|
|
61956
|
+
* Sleep for `ms`, ending EARLY when `signal` aborts: the transient-retry
|
|
61957
|
+
* backoff must never outlive the fetch — a deadline firing mid-backoff
|
|
61958
|
+
* resolves immediately so the caller re-checks the signal and rethrows the
|
|
61959
|
+
* original failure for the timeout/abort classification.
|
|
61960
|
+
*/
|
|
61961
|
+
function abortableSleep(ms, signal) {
|
|
61962
|
+
return new Promise((resolve) => {
|
|
61963
|
+
if (signal.aborted) return resolve();
|
|
61964
|
+
const onAbort = () => {
|
|
61965
|
+
clearTimeout(timer);
|
|
61966
|
+
resolve();
|
|
61967
|
+
};
|
|
61968
|
+
const timer = setTimeout(() => {
|
|
61969
|
+
signal.removeEventListener("abort", onAbort);
|
|
61970
|
+
resolve();
|
|
61971
|
+
}, ms);
|
|
61972
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
61973
|
+
});
|
|
61974
|
+
}
|
|
61975
|
+
/**
|
|
61640
61976
|
* Recover the response behind a status-only navigation failure into the
|
|
61641
61977
|
* result the web seam's contract demands — "a successful network fetch of a
|
|
61642
61978
|
* non-2xx response is a result, not an error: the status code is part of the
|
|
@@ -62022,4 +62358,4 @@ function apply(ctx, config) {
|
|
|
62022
62358
|
ctx.web.registerFetchProvider(provider);
|
|
62023
62359
|
}
|
|
62024
62360
|
//#endregion
|
|
62025
|
-
export { BLOCK_FEATURES, CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, FEATURE_NAMES, MAX_PDF_BYTES, MAX_PDF_PAGES, PAGE_FEATURES, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE, apply, blockTrainingRows, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, extractPdfMarkdown, htmlToMarkdown, inject, isChallengeCompatibleResponse, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, name, normalizeCdpEndpoint, readPdfDocument, renderCandidate, resolveConfig, scoreCandidates, setBlockModel, snapshotsOf };
|
|
62361
|
+
export { BLOCK_FEATURES, CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, DEFAULT_SETTLE_MS, FEATURE_NAMES, MAX_PDF_BYTES, MAX_PDF_PAGES, PAGE_FEATURES, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE, apply, blockTrainingRows, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, effectiveSettleMs, extractPdfMarkdown, htmlToMarkdown, inject, isChallengeCompatibleResponse, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, name, normalizeCdpEndpoint, readPdfDocument, renderCandidate, resolveConfig, scoreCandidates, setBlockModel, snapshotsOf };
|