dsh-web-fetch-playwright 0.2.22 → 0.2.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -20,6 +20,14 @@ import { delimiter, dirname, join } from "node:path";
20
20
  * skipped outright.
21
21
  */
22
22
  const DEFAULT_SCROLL_BUDGET_MS = 2e4;
23
+ /**
24
+ * Default cap (ms) on the post-navigation settle — the wait for SPA
25
+ * rendering to finish once the document has loaded (see the settle in
26
+ * `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
27
+ * the page's network goes quiet. Code-level only (tests, demo scripts):
28
+ * not part of the settings schema. `0` disables the settle entirely.
29
+ */
30
+ const DEFAULT_SETTLE_MS = 5e3;
23
31
  /** Default CDP endpoint when the settings section leaves it blank. */
24
32
  const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
25
33
  /**
@@ -84,7 +92,7 @@ function snapshotsOf(config) {
84
92
  * @returns the resolved config.
85
93
  */
86
94
  function resolveConfig(config) {
87
- const { challengeWaitMs, scrollBudgetMs, ...settings } = config;
95
+ const { challengeWaitMs, scrollBudgetMs, settleMs, ...settings } = config;
88
96
  const resolved = Config(settings);
89
97
  return {
90
98
  backend: resolved.backend.get(),
@@ -97,7 +105,8 @@ function resolveConfig(config) {
97
105
  challengeRetries: resolved.challengeRetries.get(),
98
106
  challengeHttpFallback: resolved.challengeHttpFallback.get(),
99
107
  ...challengeWaitMs === void 0 ? {} : { challengeWaitMs },
100
- ...scrollBudgetMs === void 0 ? {} : { scrollBudgetMs }
108
+ ...scrollBudgetMs === void 0 ? {} : { scrollBudgetMs },
109
+ ...settleMs === void 0 ? {} : { settleMs }
101
110
  };
102
111
  }
103
112
  /**
@@ -156,6 +165,18 @@ function effectiveScrollBudgetMs(config) {
156
165
  return DEFAULT_SCROLL_BUDGET_MS;
157
166
  }
158
167
  /**
168
+ * The settle cap a fetch actually runs with: an explicit value wins, else
169
+ * the built-in default. `0` means the settle is off (domcontentloaded
170
+ * content is read as-is).
171
+ *
172
+ * @param config - the resolved settings section (or any partial of it).
173
+ * @returns the bounded settle cap in milliseconds.
174
+ */
175
+ function effectiveSettleMs(config) {
176
+ if (typeof config.settleMs === "number") return config.settleMs;
177
+ return DEFAULT_SETTLE_MS;
178
+ }
179
+ /**
159
180
  * The context mode a fetch actually runs with: profile (a tab in the remote
160
181
  * browser's real profile) requires the CDP backend — a local launch has no
161
182
  * meaningful shared profile — and the share option (which defaults on, so an
@@ -231,8 +252,8 @@ function normalizeCdpEndpoint(input) {
231
252
  const CHALLENGE_POLL_INTERVAL_MS = 500;
232
253
  /**
233
254
  * Tail of the fetch budget held back from the challenge wait so a cleared
234
- * challenge still has time for its settle (5s networkidle cap) plus the
235
- * content read and the denoise pipeline.
255
+ * challenge still has time for its settle (the `DEFAULT_SETTLE_MS` cap)
256
+ * plus the content read and the denoise pipeline.
236
257
  */
237
258
  const CHALLENGE_FINISH_RESERVE_MS = 8e3;
238
259
  /**
@@ -2215,15 +2236,16 @@ const GATE_MIN_BODY_TEXT_CHARS = 1e3;
2215
2236
  /** Minimum share of the page's visible text a trusted extraction covers. */
2216
2237
  const GATE_MIN_ARTICLE_COVERAGE = .1;
2217
2238
  /**
2218
- * The 0.2.18 acceptance gate: distrust an extraction that is tiny in absolute
2219
- * terms or covers a sliver of the page's visible text. Kept as a selector
2220
- * feature and as the legacy-choice marker; it no longer decides alone.
2239
+ * {@link articleCoversPage} on an already-parsed article container, so the
2240
+ * generator's gate and trailing prune share ONE parse of the article — the
2241
+ * extraction is hundreds of KB on heavy pages (measured on the Angular docs
2242
+ * badcase: ~400ms per parse of Readability's 485KB pick).
2243
+ *
2244
+ * @param container - the parsed article (read, never mutated).
2245
+ * @param bodyText - the page body's visible text length, computed once.
2221
2246
  */
2222
- function articleCoversPage(articleHtml, document) {
2223
- const bodyText = visibleTextLength(document.body);
2247
+ function articleCoversParsed(container, bodyText) {
2224
2248
  if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true;
2225
- const container = document.createElement("div");
2226
- container.innerHTML = articleHtml;
2227
2249
  const articleText = visibleTextLength(container);
2228
2250
  return articleText >= 250 && articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE;
2229
2251
  }
@@ -2288,8 +2310,18 @@ function removeTrailingLinkDenseRun(parent, articleText) {
2288
2310
  }
2289
2311
  /** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
2290
2312
  function prunedArticleContent(articleHtml, document) {
2291
- const container = document.createElement("div");
2292
- container.innerHTML = articleHtml;
2313
+ return pruneParsedArticle(htmlFragment(articleHtml, document), articleHtml);
2314
+ }
2315
+ /**
2316
+ * {@link prunedArticleContent} on an already-parsed article container (the
2317
+ * generator passes the gate's parse: the gate reads it first, then this
2318
+ * mutates it — the same order the two-string-parses version produced).
2319
+ *
2320
+ * @param container - the parsed article; pruned in place.
2321
+ * @param articleHtml - the string the container was parsed from (the
2322
+ * misfire-guard's verbatim return).
2323
+ */
2324
+ function pruneParsedArticle(container, articleHtml) {
2293
2325
  const before = visibleTextLength(container);
2294
2326
  if (before === 0) return articleHtml;
2295
2327
  pruneTrailingLinkDenseSections(container);
@@ -2297,13 +2329,24 @@ function prunedArticleContent(articleHtml, document) {
2297
2329
  if (after === 0 || before >= 250 && after < 250) return articleHtml;
2298
2330
  return container.innerHTML;
2299
2331
  }
2300
- /** Link density at or above which a recommendation-named subtree is a rail. */
2332
+ /**
2333
+ * Link density at or above which a recommendation-named subtree is a rail.
2334
+ */
2301
2335
  const RAIL_LINK_DENSITY = .3;
2302
2336
  const RAIL_MIN_LINKS = 4;
2303
2337
  /** Prose (low-link text) the page must keep outside the rails. */
2304
2338
  const RAIL_MIN_PROSE_CHARS = 80;
2305
2339
  const RAIL_PROSE_LINK_DENSITY = .2;
2306
2340
  /**
2341
+ * Necessary condition for {@link isRailName} on the raw class+id string: a
2342
+ * matching token (`recommended-posts`, `Related`, …) always leaves one of
2343
+ * these literals as a substring. Testing it first short-circuits the
2344
+ * per-token split on the pages where it matters — {@link markRecommendationRails}
2345
+ * walks every element, and on the measured 1.1MB Angular docs badcase the
2346
+ * split alone cost ~360ms of the candidate generation.
2347
+ */
2348
+ const RAIL_NAME_HINT = /recommend|related|outbrain|taboola/;
2349
+ /**
2307
2350
  * Class/id sub-tokens naming a recommendation / related-content rail: the
2308
2351
  * English words every CMS and theme uses, plus the two syndicated
2309
2352
  * recommendation widgets embedded across thousands of publishers (their
@@ -2314,7 +2357,9 @@ function isRailName(token) {
2314
2357
  return token.includes("recommend") || token.startsWith("related") || token === "outbrain" || token === "taboola";
2315
2358
  }
2316
2359
  function isRecommendationRail(element) {
2317
- if (!`${element.getAttribute("class") ?? ""} ${element.id}`.toLowerCase().split(/[\s_-]+/).some(isRailName)) return false;
2360
+ const names = `${element.getAttribute("class") ?? ""} ${element.id}`.toLowerCase();
2361
+ if (!RAIL_NAME_HINT.test(names)) return false;
2362
+ if (!names.split(/[\s_-]+/).some(isRailName)) return false;
2318
2363
  if (element.querySelectorAll("a").length < RAIL_MIN_LINKS) return false;
2319
2364
  const text = visibleTextLength(element);
2320
2365
  return text > 0 && linkTextLength(element) / text >= RAIL_LINK_DENSITY;
@@ -2654,6 +2699,59 @@ function escapeHtml(text) {
2654
2699
  /** Below this much noscript text the noscript-unwrapped paths are skipped. */
2655
2700
  const NOSCRIPT_MIN_TEXT_CHARS = 200;
2656
2701
  /**
2702
+ * Share of the body's visible text that sits inside custom-element subtrees
2703
+ * (tag names containing `-`: `<ms-app>`, `<my-doc>` — the HTML spec reserves
2704
+ * them for custom elements; no standard HTML or MathML tag and no tag the
2705
+ * denoise pipeline's DOMPurify step keeps contains a dash). DOMPurify's
2706
+ * allowlist removes such tags wholesale — `KEEP_CONTENT: false` takes their
2707
+ * subtrees with them — so once nearly ALL the page's text is custom-wrapped,
2708
+ * every extraction that would carry it sanitizes to a corpse. Measured on
2709
+ * the badcase (aistudio.google.com/docs/computer-use, an Angular app whose
2710
+ * entire DOM lives under `<ms-app>`): 99% of body text custom-wrapped;
2711
+ * Readability's pick sanitized to a root with ZERO words, the whole-document
2712
+ * candidate to 39 of ~4.5k words — while the three Readability passes over
2713
+ * the 1.1MB document cost ~6.5s of pure latency on candidates that could
2714
+ * never win. The DOM-backed fallbacks, the noscript paths and the block
2715
+ * classifier — the candidates that survive DOMPurify on this shape, and the
2716
+ * ones the selector picks on such pages anyway — carry the extraction.
2717
+ *
2718
+ * Noise subtrees (script/style/nav/…) count as neither wrapped nor unwrapped;
2719
+ * the share is over CONTENT-bearing text only. A partial share (a web
2720
+ * component widget inside a normal article) stays below the threshold and
2721
+ * keeps the full pipeline: only the shape where the family is provably dead
2722
+ * pays for it.
2723
+ */
2724
+ function customWrappedTextShare(body) {
2725
+ let total = 0;
2726
+ let wrapped = 0;
2727
+ const visit = (node, insideCustom) => {
2728
+ if (node.nodeType === 3) {
2729
+ const length = (node.nodeValue ?? "").replace(/\s+/g, "").length;
2730
+ if (length === 0) return;
2731
+ total += length;
2732
+ if (insideCustom) wrapped += length;
2733
+ return;
2734
+ }
2735
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return;
2736
+ const custom = insideCustom || node.nodeName.includes("-");
2737
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, custom);
2738
+ };
2739
+ visit(body, false);
2740
+ return total === 0 ? 0 : wrapped / total;
2741
+ }
2742
+ /**
2743
+ * The title the gated path hands its candidates: Readability's own first
2744
+ * source is `og:title`, its second the `<title>` element — both remain
2745
+ * readable on custom-element pages, where the DOM wrapping is the framework's
2746
+ * but the metadata is the page's.
2747
+ */
2748
+ function fallbackTitle(document) {
2749
+ const og = document.querySelector("meta[property=\"og:title\"]")?.getAttribute("content")?.trim();
2750
+ if (og !== void 0 && og !== "") return og;
2751
+ const title = document.querySelector("title")?.textContent?.trim();
2752
+ return title === void 0 || title === "" ? void 0 : title;
2753
+ }
2754
+ /**
2657
2755
  * Propose every candidate extraction for one parsed page.
2658
2756
  * @param document - the parsed page (data URIs already elided); not mutated.
2659
2757
  */
@@ -2661,48 +2759,59 @@ function generateCandidates(document) {
2661
2759
  const candidates = [];
2662
2760
  const body = document.body;
2663
2761
  const bodyHtml = body?.innerHTML ?? "";
2664
- const marked = runReadability(markRecommendationRails(document) ?? document);
2665
- const rails = marked === null ? null : splitRails(marked.content, document);
2666
- const art = marked === null || rails === null ? null : {
2667
- ...marked,
2668
- content: rails.plain
2669
- };
2670
- const gatePass = art !== null && articleCoversPage(art.content, document);
2762
+ const skipReadability = body !== null && customWrappedTextShare(body) >= .9;
2763
+ let art = null;
2764
+ let lenient = null;
2765
+ let gatePass = false;
2671
2766
  let pruneChanged = false;
2672
- if (art !== null) {
2673
- const pruned = prunedArticleContent(rails?.derailed ?? art.content, document);
2674
- pruneChanged = pruned !== art.content;
2675
- candidates.push({
2676
- kind: "art",
2677
- html: pruned,
2678
- title: art.title,
2679
- legacy: gatePass
2680
- });
2681
- if (pruneChanged) candidates.push({
2682
- kind: "artu",
2683
- html: art.content,
2684
- title: art.title,
2767
+ if (!skipReadability) {
2768
+ const marked = runReadability(markRecommendationRails(document) ?? document);
2769
+ const rails = marked === null ? null : splitRails(marked.content, document);
2770
+ art = marked === null || rails === null ? null : {
2771
+ ...marked,
2772
+ content: rails.plain
2773
+ };
2774
+ if (art !== null) {
2775
+ const bodyText = visibleTextLength(body);
2776
+ const parsed = htmlFragment(art.content, document);
2777
+ gatePass = articleCoversParsed(parsed, bodyText);
2778
+ const pruning = rails?.derailed ?? art.content;
2779
+ const pruned = pruning === art.content ? pruneParsedArticle(parsed, art.content) : prunedArticleContent(pruning, document);
2780
+ pruneChanged = pruned !== art.content;
2781
+ candidates.push({
2782
+ kind: "art",
2783
+ html: pruned,
2784
+ title: art.title,
2785
+ legacy: gatePass
2786
+ });
2787
+ if (pruneChanged) candidates.push({
2788
+ kind: "artu",
2789
+ html: art.content,
2790
+ title: art.title,
2791
+ legacy: false
2792
+ });
2793
+ }
2794
+ lenient = runReadability(document, { linkDensityModifier: .4 });
2795
+ if (lenient !== null && lenient.content !== art?.content) candidates.push({
2796
+ kind: "artx",
2797
+ html: lenient.content,
2798
+ title: lenient.title,
2685
2799
  legacy: false
2686
2800
  });
2687
2801
  }
2688
- const lenient = runReadability(document, { linkDensityModifier: .4 });
2689
- if (lenient !== null && lenient.content !== art?.content) candidates.push({
2690
- kind: "artx",
2691
- html: lenient.content,
2692
- title: lenient.title,
2693
- legacy: false
2694
- });
2695
- const title = art?.title ?? lenient?.title;
2802
+ const title = art?.title ?? lenient?.title ?? (skipReadability ? fallbackTitle(document) : void 0);
2696
2803
  const main = body === null ? null : findMainContainer(document);
2697
2804
  if (main !== null) candidates.push({
2698
2805
  kind: "main",
2699
2806
  html: main.innerHTML,
2807
+ node: main,
2700
2808
  title,
2701
2809
  legacy: false
2702
2810
  });
2703
2811
  candidates.push({
2704
2812
  kind: "doc",
2705
2813
  html: bodyHtml,
2814
+ node: body ?? void 0,
2706
2815
  legacy: !gatePass
2707
2816
  });
2708
2817
  if (body !== null) {
@@ -2711,6 +2820,7 @@ function generateCandidates(document) {
2711
2820
  candidates.push({
2712
2821
  kind: "docc",
2713
2822
  html: cleaned.innerHTML,
2823
+ node: cleaned,
2714
2824
  legacy: false
2715
2825
  });
2716
2826
  }
@@ -2738,6 +2848,7 @@ function generateCandidates(document) {
2738
2848
  candidates.push({
2739
2849
  kind: "ns_doc",
2740
2850
  html: unwrapped.body?.innerHTML ?? "",
2851
+ node: unwrapped.body ?? void 0,
2741
2852
  legacy: false
2742
2853
  });
2743
2854
  }
@@ -2764,7 +2875,8 @@ function generateCandidates(document) {
2764
2875
  record,
2765
2876
  hasMain: main !== null,
2766
2877
  hasPreloaded: preloaded !== null,
2767
- main
2878
+ main,
2879
+ ...skipReadability ? { readabilitySkipped: true } : {}
2768
2880
  }
2769
2881
  };
2770
2882
  }
@@ -59714,7 +59826,7 @@ function analyze(html, url) {
59714
59826
  };
59715
59827
  }
59716
59828
  const sanitize = (candidate) => {
59717
- const root = purify.sanitize(candidate.html, {
59829
+ const root = purify.sanitize(candidate.node ?? candidate.html, {
59718
59830
  FORBID_TAGS,
59719
59831
  FORBID_ATTR,
59720
59832
  KEEP_CONTENT: false,
@@ -59729,6 +59841,16 @@ function analyze(html, url) {
59729
59841
  };
59730
59842
  };
59731
59843
  const sanitized = generated.candidates.map(sanitize);
59844
+ if (generated.facts.readabilitySkipped === true && !sanitized.some((c) => c.kind === "art")) {
59845
+ const empty = document.createElement("div");
59846
+ sanitized.push({
59847
+ kind: "art",
59848
+ title: void 0,
59849
+ legacy: false,
59850
+ root: empty,
59851
+ stats: measureCandidate("art", false, empty)
59852
+ });
59853
+ }
59732
59854
  const page = pageFeatures(document, url, generated.facts, sanitized.find((c) => c.kind === "doc")?.stats, visibleTextLength(document.body));
59733
59855
  let blocks = [];
59734
59856
  let blockVectors = [];
@@ -60921,6 +61043,21 @@ function existsAsFile(path) {
60921
61043
  * refetch through the page's API request context
60922
61044
  * ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
60923
61045
  *
61046
+ * Transient transport failures (issue: `net::ERR_NETWORK_CHANGED` and its
61047
+ * family) are retried, not terminal: Chromium's NetworkChangeNotifier kills
61048
+ * every in-flight request with `ERR_NETWORK_CHANGED` the moment the HOST's
61049
+ * network configuration shifts under the browser — a route/interface/DNS/
61050
+ * proxy change, a VPN or DHCP flap, a Docker embedded-DNS rewrite; the page
61051
+ * and its target are perfectly fine, only that one navigation died, and the
61052
+ * very next `goto` usually succeeds (measured live: source.android.com
61053
+ * failed once mid-navigation in one DSH session and fetched clean from the
61054
+ * same host seconds later). Each navigation therefore retries the
61055
+ * transport-blip family ({@link isTransientNavigationFailure} — deliberately
61056
+ * NOT `ERR_ABORTED`, which is overloaded with the deterministic shapes the
61057
+ * recoveries above already own) a bounded number of times with a short
61058
+ * backoff, inside the same page and the remaining waiting budget; a blip
61059
+ * that survives the attempt cap still fails as before.
61060
+ *
60924
61061
  * Lifecycle: the local backend launches a browser per fetch and closes it —
60925
61062
  * nothing outlives the call. CDP session isolation keeps one connection per
60926
61063
  * DSH session until its context becomes idle; this also isolates gateways that
@@ -61041,8 +61178,30 @@ const DEFAULT_TIMEOUT_MS = 25e3;
61041
61178
  const EXTRACT_GRACE_MS = 1e4;
61042
61179
  /** Bound (ms) on the best-effort `window.stop()` of a still-loading page. */
61043
61180
  const STOP_LOADING_GRACE_MS = 1e3;
61044
- /** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
61045
- const SETTLE_MS = 5e3;
61181
+ /**
61182
+ * Settle quiet window (ms): the post-load settle ends once no request has
61183
+ * STARTED for this long AND every still-open request is stale (see
61184
+ * {@link SETTLE_STALE_MS}) — the Lighthouse-style network-quiet threshold,
61185
+ * sized to match Playwright's own networkidle quiet window.
61186
+ */
61187
+ const SETTLE_QUIET_MS = 500;
61188
+ /**
61189
+ * A request open longer than this (ms) stops gating the settle. On the
61190
+ * pages that motivated the custom detector, the long-lived request is a
61191
+ * worker-script fetch or a cross-origin auth frame whose completion events
61192
+ * are attributed to another CDP target and NEVER arrive on the page session
61193
+ * (measured live over CDP: platform.claude.com's `turbopack-worker-*.js`
61194
+ * and aistudio.google.com's `accounts.google.com/ServiceLogin` probe stayed
61195
+ * "open" for the page's whole life while the page itself completed at
61196
+ * `load`), so strict `networkidle` — Playwright's is officially discouraged
61197
+ * — is unreachable there and the legacy wait burned its whole cap on every
61198
+ * fetch. A genuine content request this old is the documented trade: the
61199
+ * settle stops waiting for it, and the scroll walk plus the live read still
61200
+ * capture whatever it renders when it lands.
61201
+ */
61202
+ const SETTLE_STALE_MS = 3e3;
61203
+ /** How often the settle re-checks its quiet/stale conditions (ms). */
61204
+ const SETTLE_POLL_MS = 100;
61046
61205
  /**
61047
61206
  * Grace period (ms) for context/browser closes in the cleanup path. A wedged
61048
61207
  * `close()` must never hold a concurrency slot hostage — a leaked slot would
@@ -61050,6 +61209,27 @@ const SETTLE_MS = 5e3;
61050
61209
  */
61051
61210
  const CLOSE_GRACE_MS = 2e3;
61052
61211
  /**
61212
+ * How many times a navigation may be re-issued after a TRANSIENT transport
61213
+ * failure (`net::ERR_NETWORK_CHANGED` and family — see
61214
+ * {@link isTransientNavigationFailure}) before the fetch gives up. Three
61215
+ * total navigations cover the single-blip badcase and the rare double-blip
61216
+ * while capping the worst-case latency a persistently flapping network can
61217
+ * add.
61218
+ */
61219
+ const MAX_TRANSIENT_NAV_RETRIES = 2;
61220
+ /**
61221
+ * Backoff (ms) before each transient-navigation retry, indexed by the retry
61222
+ * number: short first (most blips clear in well under a second), longer
61223
+ * second. Bounded by the fetch deadline via an abortable sleep.
61224
+ */
61225
+ const TRANSIENT_RETRY_BACKOFF_MS = [250, 750];
61226
+ /**
61227
+ * Waiting budget (ms) that must REMAIN before a transient-navigation retry
61228
+ * may start: the retry must not eat the tail the settle/decode path still
61229
+ * needs, and a re-navigation that cannot finish is pure waste.
61230
+ */
61231
+ const TRANSIENT_RETRY_RESERVE_MS = 2e3;
61232
+ /**
61053
61233
  * The fetch's two-stage deadline composed with the caller's signal. The SOFT
61054
61234
  * stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
61055
61235
  * it and {@link expired} reports it, so every wait ends there and the fetch
@@ -61208,6 +61388,12 @@ var PlaywrightFetchProvider = class {
61208
61388
  fetchTimeoutMs = DEFAULT_TIMEOUT_MS;
61209
61389
  /** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
61210
61390
  extractGraceMs = EXTRACT_GRACE_MS;
61391
+ /** Settle quiet window (ms); code-level, overridable by subclasses (tests). */
61392
+ settleQuietMs = SETTLE_QUIET_MS;
61393
+ /** Request age (ms) past which it stops gating the settle; overridable by subclasses (tests). */
61394
+ settleStaleMs = SETTLE_STALE_MS;
61395
+ /** Settle re-check cadence (ms); overridable by subclasses (tests). */
61396
+ settlePollMs = SETTLE_POLL_MS;
61211
61397
  /** Shared CDP connection; injectable so the suite can substitute a fake. */
61212
61398
  cdpPool;
61213
61399
  /**
@@ -61345,6 +61531,7 @@ var PlaywrightFetchProvider = class {
61345
61531
  const page = session.page;
61346
61532
  const challengeWaitMs = effectiveChallengeWaitMs(config);
61347
61533
  const tracker = trackMainFrameResponses(page);
61534
+ const activity = trackRequestActivity(page);
61348
61535
  const first = await this.navigate(page, url, tracker, deadline, config.denoise);
61349
61536
  if (first.recovered !== void 0) return first.recovered;
61350
61537
  let response = first.response;
@@ -61396,7 +61583,7 @@ var PlaywrightFetchProvider = class {
61396
61583
  kind: "text",
61397
61584
  content: partial ? await renderedText(page) : finalResponse !== null ? await finalResponse.text() : await page.content()
61398
61585
  }), partial);
61399
- await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
61586
+ await this.settleForContent(page, deadline, config, activity);
61400
61587
  const scrollBudgetMs = deadline.expired ? 0 : Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs());
61401
61588
  const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : void 0;
61402
61589
  let html;
@@ -61464,13 +61651,18 @@ var PlaywrightFetchProvider = class {
61464
61651
  * status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
61465
61652
  * being a download the browser refuses to commit as a document
61466
61653
  * (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
61654
+ * A TRANSIENT transport failure (`net::ERR_NETWORK_CHANGED` and family)
61655
+ * is neither: it carries no response and no render decision, only a
61656
+ * momentary host-network blip that killed this one navigation — so it is
61657
+ * retried on the SAME page, bounded by {@link MAX_TRANSIENT_NAV_RETRIES}
61658
+ * attempts, a short backoff, and the remaining waiting budget.
61467
61659
  *
61468
61660
  * @returns the response `goto` resolved with, plus `recovered` when the
61469
61661
  * navigation failure was recovered into a finished result the caller must
61470
61662
  * return verbatim (the throw never escapes in that case).
61471
61663
  */
61472
61664
  async navigate(page, url, tracker, deadline, denoise) {
61473
- try {
61665
+ for (let attempt = 0;; attempt++) try {
61474
61666
  return { response: await page.goto(url.toString(), {
61475
61667
  waitUntil: "domcontentloaded",
61476
61668
  timeout: deadline.remainingMs()
@@ -61495,6 +61687,10 @@ var PlaywrightFetchProvider = class {
61495
61687
  partial: true
61496
61688
  };
61497
61689
  }
61690
+ if (attempt < MAX_TRANSIENT_NAV_RETRIES && isTransientNavigationFailure(error) && deadline.remainingMs() > TRANSIENT_RETRY_RESERVE_MS) {
61691
+ await abortableSleep(TRANSIENT_RETRY_BACKOFF_MS[Math.min(attempt, TRANSIENT_RETRY_BACKOFF_MS.length - 1)] ?? TRANSIENT_RETRY_BACKOFF_MS.at(-1) ?? 500, deadline.signal);
61692
+ if (!deadline.signal.aborted) continue;
61693
+ }
61498
61694
  }
61499
61695
  throw error;
61500
61696
  }
@@ -61542,6 +61738,64 @@ var PlaywrightFetchProvider = class {
61542
61738
  return true;
61543
61739
  }
61544
61740
  }
61741
+ /**
61742
+ * Best-effort settle for client-rendered content — a Lighthouse-style
61743
+ * bounded network-quiet window replacing Playwright's `networkidle`
61744
+ * (officially discouraged): worker-script fetches and cross-origin auth
61745
+ * frames hold one request open on the page session FOREVER (their
61746
+ * completion events are attributed to other CDP targets), so strict
61747
+ * networkidle never arrives on exactly the modern pages the settle exists
61748
+ * for, and the legacy wait burned its whole cap on them every fetch.
61749
+ *
61750
+ * Sequence: a `load` baseline (usually already past — domcontentloaded
61751
+ * resolved earlier), then poll until no request has STARTED within
61752
+ * {@link settleQuietMs} and every still-open request is older than
61753
+ * {@link settleStaleMs} — a request that old is an attribution artifact
61754
+ * or a long-poll whose completion no longer gates content. All waits
61755
+ * share the {@link effectiveSettleMs} cap and the fetch deadline; a page
61756
+ * that exposes no request events degrades to the legacy bounded
61757
+ * networkidle wait. Never throws: any exit path keeps what the document
61758
+ * already holds.
61759
+ */
61760
+ async settleForContent(page, deadline, config, activity) {
61761
+ const beganAt = Date.now();
61762
+ const cap = effectiveSettleMs(config);
61763
+ if (cap <= 0) {
61764
+ this.noteSettleOutcome("off", 0);
61765
+ return;
61766
+ }
61767
+ const endsAt = beganAt + Math.min(cap, deadline.remainingMs());
61768
+ await page.waitForLoadState("load", { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {});
61769
+ if (activity === void 0) {
61770
+ await page.waitForLoadState("networkidle", { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {});
61771
+ this.noteSettleOutcome("fallback", Date.now() - beganAt);
61772
+ return;
61773
+ }
61774
+ for (;;) {
61775
+ if (deadline.signal.aborted) {
61776
+ this.noteSettleOutcome("aborted", Date.now() - beganAt);
61777
+ return;
61778
+ }
61779
+ if (Date.now() >= endsAt) {
61780
+ this.noteSettleOutcome("cap", Date.now() - beganAt);
61781
+ return;
61782
+ }
61783
+ if (activity.idle(this.settleQuietMs, this.settleStaleMs)) {
61784
+ this.noteSettleOutcome(activity.openCount() > 0 ? "stale-discounted" : "quiet", Date.now() - beganAt);
61785
+ return;
61786
+ }
61787
+ await sleep(Math.min(this.settlePollMs, Math.max(1, endsAt - Date.now())));
61788
+ }
61789
+ }
61790
+ /**
61791
+ * Settle outcome observation hook — silent in production by design (the
61792
+ * settle is best-effort), but subclasses (tests, diagnostics) override it
61793
+ * to see which exit path every fetch took: `off` (config), `quiet`
61794
+ * (network went quiet), `stale-discounted` (only stale requests remained
61795
+ * open), `cap` (window exhausted), `fallback` (no request events — the
61796
+ * legacy networkidle wait ran), `aborted` (fetch deadline fired).
61797
+ */
61798
+ noteSettleOutcome(outcome, elapsedMs) {}
61545
61799
  };
61546
61800
  /** Resolve after `ms` — the bounded wait's inter-poll nap. */
61547
61801
  function sleep(ms) {
@@ -61614,6 +61868,43 @@ function trackMainFrameResponses(page) {
61614
61868
  };
61615
61869
  }
61616
61870
  /**
61871
+ * Watch the page for request lifecycle events so the settle can decide
61872
+ * network quietness itself. `undefined` when the page exposes no request
61873
+ * events (minimal fakes): the settle then falls back to the legacy bounded
61874
+ * networkidle wait.
61875
+ *
61876
+ * @param page - the fetch's tab.
61877
+ * @returns the open-request tracker, or undefined on an unsupported surface.
61878
+ */
61879
+ function trackRequestActivity(page) {
61880
+ if (typeof page.on !== "function") return void 0;
61881
+ const opened = /* @__PURE__ */ new Map();
61882
+ let lastStart = Date.now();
61883
+ const onStarted = (request) => {
61884
+ opened.set(request, Date.now());
61885
+ lastStart = Date.now();
61886
+ };
61887
+ const onSettled = (request) => {
61888
+ opened.delete(request);
61889
+ };
61890
+ try {
61891
+ page.on("request", onStarted);
61892
+ page.on("requestfinished", onSettled);
61893
+ page.on("requestfailed", onSettled);
61894
+ } catch {
61895
+ return;
61896
+ }
61897
+ return {
61898
+ idle: (quietMs, staleMs) => {
61899
+ const now = Date.now();
61900
+ if (now - lastStart < quietMs) return false;
61901
+ for (const startedAt of opened.values()) if (now - startedAt < staleMs) return false;
61902
+ return true;
61903
+ },
61904
+ openCount: () => opened.size
61905
+ };
61906
+ }
61907
+ /**
61617
61908
  * Whether a `goto` throw is Chromium refusing to RENDER a response it
61618
61909
  * received — not a transport failure. Two net errors, both documented as
61619
61910
  * rendering decisions rather than network ones:
@@ -61637,6 +61928,51 @@ function isUnrenderedStatusFailure(error, status) {
61637
61928
  return message.includes("ERR_ABORTED") && (status === 204 || status === 205);
61638
61929
  }
61639
61930
  /**
61931
+ * Chromium net errors a plain re-navigation can plausibly fix — the
61932
+ * transport-layer BLIPS, not render decisions (those are recovered above)
61933
+ * and not `ERR_ABORTED` (deliberately excluded: that code is overloaded with
61934
+ * deterministic shapes — SPA cancels, 204/205 aborts, downloads — which the
61935
+ * recoveries already own). `ERR_NETWORK_CHANGED` is the measured badcase:
61936
+ * the host's network configuration shifted under the browser mid-navigation
61937
+ * (route/interface/DNS/proxy change, VPN or DHCP flap, a Docker
61938
+ * embedded-DNS rewrite) and Chromium's NetworkChangeNotifier killed the
61939
+ * in-flight request; the sibling codes are the same story on a different
61940
+ * layer (DNS blips, TCP resets and idle breaks, brief disconnects).
61941
+ * Playwright's own navigation timeouts never match: they reject with a
61942
+ * `TimeoutError` ("Timeout Nms exceeded"), not a `net::ERR_*` code.
61943
+ */
61944
+ const TRANSIENT_NET_ERROR_RE = /net::ERR_(NETWORK_CHANGED|INTERNET_DISCONNECTED|CONNECTION_RESET|CONNECTION_CLOSED|CONNECTION_TIMED_OUT|TIMED_OUT|EMPTY_RESPONSE|NAME_NOT_RESOLVED|NAME_RESOLUTION_FAILED|ADDRESS_UNREACHABLE)\b/;
61945
+ /**
61946
+ * Whether a `goto` throw is one of the transient transport failures a
61947
+ * bounded same-page retry can recover (see {@link TRANSIENT_NET_ERROR_RE}).
61948
+ *
61949
+ * @param error - the value `page.goto` rejected with.
61950
+ * @returns true when the navigation died to a momentary network blip.
61951
+ */
61952
+ function isTransientNavigationFailure(error) {
61953
+ return error instanceof Error && TRANSIENT_NET_ERROR_RE.test(error.message);
61954
+ }
61955
+ /**
61956
+ * Sleep for `ms`, ending EARLY when `signal` aborts: the transient-retry
61957
+ * backoff must never outlive the fetch — a deadline firing mid-backoff
61958
+ * resolves immediately so the caller re-checks the signal and rethrows the
61959
+ * original failure for the timeout/abort classification.
61960
+ */
61961
+ function abortableSleep(ms, signal) {
61962
+ return new Promise((resolve) => {
61963
+ if (signal.aborted) return resolve();
61964
+ const onAbort = () => {
61965
+ clearTimeout(timer);
61966
+ resolve();
61967
+ };
61968
+ const timer = setTimeout(() => {
61969
+ signal.removeEventListener("abort", onAbort);
61970
+ resolve();
61971
+ }, ms);
61972
+ signal.addEventListener("abort", onAbort, { once: true });
61973
+ });
61974
+ }
61975
+ /**
61640
61976
  * Recover the response behind a status-only navigation failure into the
61641
61977
  * result the web seam's contract demands — "a successful network fetch of a
61642
61978
  * non-2xx response is a result, not an error: the status code is part of the
@@ -62022,4 +62358,4 @@ function apply(ctx, config) {
62022
62358
  ctx.web.registerFetchProvider(provider);
62023
62359
  }
62024
62360
  //#endregion
62025
- export { BLOCK_FEATURES, CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, FEATURE_NAMES, MAX_PDF_BYTES, MAX_PDF_PAGES, PAGE_FEATURES, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE, apply, blockTrainingRows, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, extractPdfMarkdown, htmlToMarkdown, inject, isChallengeCompatibleResponse, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, name, normalizeCdpEndpoint, readPdfDocument, renderCandidate, resolveConfig, scoreCandidates, setBlockModel, snapshotsOf };
62361
+ export { BLOCK_FEATURES, CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, DEFAULT_SETTLE_MS, FEATURE_NAMES, MAX_PDF_BYTES, MAX_PDF_PAGES, PAGE_FEATURES, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE, apply, blockTrainingRows, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, effectiveSettleMs, extractPdfMarkdown, htmlToMarkdown, inject, isChallengeCompatibleResponse, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, name, normalizeCdpEndpoint, readPdfDocument, renderCandidate, resolveConfig, scoreCandidates, setBlockModel, snapshotsOf };