dsh-web-fetch-playwright 0.2.20 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -17
- package/README.zh-CN.md +19 -17
- package/cordis.patch.yml +0 -4
- package/lib/client.js +18 -68
- package/lib/index.js +24049 -22754
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/candidates.d.ts +55 -0
- package/lib/types/challenge.d.ts +2 -2
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +80 -30
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +101 -9
- package/lib/types/types.d.ts +26 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +1 -1
- package/src/candidates.ts +184 -35
- package/src/challenge.ts +2 -2
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +97 -37
- package/src/index.ts +4 -1
- package/src/markdown.ts +13 -2
- package/src/provider.ts +482 -56
- package/src/selector-model.ts +1 -1
- package/src/selector.ts +18 -7
- package/src/types.ts +27 -7
package/src/provider.ts
CHANGED
|
@@ -18,6 +18,21 @@
|
|
|
18
18
|
* refetch through the page's API request context
|
|
19
19
|
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
20
20
|
*
|
|
21
|
+
* Transient transport failures (issue: `net::ERR_NETWORK_CHANGED` and its
|
|
22
|
+
* family) are retried, not terminal: Chromium's NetworkChangeNotifier kills
|
|
23
|
+
* every in-flight request with `ERR_NETWORK_CHANGED` the moment the HOST's
|
|
24
|
+
* network configuration shifts under the browser — a route/interface/DNS/
|
|
25
|
+
* proxy change, a VPN or DHCP flap, a Docker embedded-DNS rewrite; the page
|
|
26
|
+
* and its target are perfectly fine, only that one navigation died, and the
|
|
27
|
+
* very next `goto` usually succeeds (measured live: source.android.com
|
|
28
|
+
* failed once mid-navigation in one DSH session and fetched clean from the
|
|
29
|
+
* same host seconds later). Each navigation therefore retries the
|
|
30
|
+
* transport-blip family ({@link isTransientNavigationFailure} — deliberately
|
|
31
|
+
* NOT `ERR_ABORTED`, which is overloaded with the deterministic shapes the
|
|
32
|
+
* recoveries above already own) a bounded number of times with a short
|
|
33
|
+
* backoff, inside the same page and the remaining waiting budget; a blip
|
|
34
|
+
* that survives the attempt cap still fails as before.
|
|
35
|
+
*
|
|
21
36
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
22
37
|
* nothing outlives the call. CDP session isolation keeps one connection per
|
|
23
38
|
* DSH session until its context becomes idle; this also isolates gateways that
|
|
@@ -55,9 +70,14 @@
|
|
|
55
70
|
* behavior). The wait tracks the LAST main-frame navigation response (the
|
|
56
71
|
* real page reloads in after the challenge clears) and watches the live DOM
|
|
57
72
|
* so SPA-style clears are caught too. It never clicks, never injects
|
|
58
|
-
* answers, never touches cookies itself; when the budget runs out
|
|
59
|
-
*
|
|
60
|
-
*
|
|
73
|
+
* answers, never touches cookies itself; when the budget runs out, ONE
|
|
74
|
+
* plain-HTTP refetch of the same URL through the page's request context
|
|
75
|
+
* (same cookie jar, the browser's UA, none of Chromium's render decisions
|
|
76
|
+
* — `challengeHttpFallback`, default on) tries to retrieve the real
|
|
77
|
+
* document, gated by the same response-level and content-level challenge
|
|
78
|
+
* classifiers so an interstitial can never pass as content. When that too
|
|
79
|
+
* fails, the fetch fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of
|
|
80
|
+
* returning the interstitial as content.
|
|
61
81
|
*
|
|
62
82
|
* Deep capture (viewport-virtualized feeds and app-shell documents): feed-
|
|
63
83
|
* shaped pages — Discourse topic streams, comment threads, timelines —
|
|
@@ -84,6 +104,17 @@
|
|
|
84
104
|
* degrades to the legacy capture (see `src/autoscroll.ts` for the measured
|
|
85
105
|
* linux.do and Feishu badcases).
|
|
86
106
|
*
|
|
107
|
+
* Deadline (soft, then hard): each fetch has a {@link DEFAULT_TIMEOUT_MS}
|
|
108
|
+
* budget for WAITING — navigation, settle, challenge wait, scroll walk. When
|
|
109
|
+
* it runs out the page is NOT torn down: every wait is bounded by the same
|
|
110
|
+
* budget and simply ends, a navigation still loading is stopped
|
|
111
|
+
* (`window.stop()`) once its main-frame response has arrived, and whatever
|
|
112
|
+
* the page holds by then goes through the normal extraction — a slow or
|
|
113
|
+
* endless page still hands the agent the content it already rendered
|
|
114
|
+
* (flagged `truncated` when the document never finished loading). Only when
|
|
115
|
+
* nothing arrived at all, or extraction itself hangs past
|
|
116
|
+
* {@link EXTRACT_GRACE_MS}, does the fetch fail with `WEB_FETCH_TIMEOUT`.
|
|
117
|
+
*
|
|
87
118
|
* @module dsh-web-fetch-playwright/provider
|
|
88
119
|
*/
|
|
89
120
|
|
|
@@ -91,15 +122,15 @@ import { WebError } from '@deepseek-ai/dsh-web'
|
|
|
91
122
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
|
|
92
123
|
import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
93
124
|
import type { ChallengeVerdict } from './challenge.ts'
|
|
94
|
-
import { deepCaptureWalk
|
|
95
|
-
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
|
|
125
|
+
import { deepCaptureWalk } from './autoscroll.ts'
|
|
126
|
+
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, effectiveSettleMs, normalizeCdpEndpoint } from './config.ts'
|
|
96
127
|
import type { ResolvedConfig } from './config.ts'
|
|
97
128
|
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
98
129
|
import { htmlToMarkdown } from './markdown.ts'
|
|
99
130
|
import { extractPdfMarkdown, isPdfBytes, isPdfContentType, mayBePdf, MAX_PDF_BYTES, WEB_PDF_INVALID_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
|
|
100
131
|
import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
|
|
101
132
|
import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
|
|
102
|
-
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
|
|
133
|
+
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute, PlaywrightWebRequest } from './types.ts'
|
|
103
134
|
|
|
104
135
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
105
136
|
export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
|
|
@@ -128,13 +159,50 @@ const MAX_PIPELINE_INPUT_CHARS = 2_000_000
|
|
|
128
159
|
* budget to render after dequeue — and failing fast tells the caller to
|
|
129
160
|
* retry or raise `maxConcurrency` instead of hanging until an abort.
|
|
130
161
|
*/
|
|
131
|
-
const QUEUE_TIMEOUT_MS =
|
|
162
|
+
const QUEUE_TIMEOUT_MS = 10_000
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Per-fetch waiting budget (ms): when it runs out the fetch stops waiting
|
|
166
|
+
* for the page and extracts what it already holds (see the module docs).
|
|
167
|
+
*/
|
|
168
|
+
const DEFAULT_TIMEOUT_MS = 25_000
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Hard backstop (ms) after {@link DEFAULT_TIMEOUT_MS} for reading and
|
|
172
|
+
* decoding what the page holds; only past it is the tab closed and the fetch
|
|
173
|
+
* failed. Budget plus grace stays inside the tool layer's 60s.
|
|
174
|
+
*/
|
|
175
|
+
const EXTRACT_GRACE_MS = 10_000
|
|
176
|
+
|
|
177
|
+
/** Bound (ms) on the best-effort `window.stop()` of a still-loading page. */
|
|
178
|
+
const STOP_LOADING_GRACE_MS = 1_000
|
|
132
179
|
|
|
133
|
-
/**
|
|
134
|
-
|
|
180
|
+
/**
|
|
181
|
+
* Settle quiet window (ms): the post-load settle ends once no request has
|
|
182
|
+
* STARTED for this long AND every still-open request is stale (see
|
|
183
|
+
* {@link SETTLE_STALE_MS}) — the Lighthouse-style network-quiet threshold,
|
|
184
|
+
* sized to match Playwright's own networkidle quiet window.
|
|
185
|
+
*/
|
|
186
|
+
const SETTLE_QUIET_MS = 500
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* A request open longer than this (ms) stops gating the settle. On the
|
|
190
|
+
* pages that motivated the custom detector, the long-lived request is a
|
|
191
|
+
* worker-script fetch or a cross-origin auth frame whose completion events
|
|
192
|
+
* are attributed to another CDP target and NEVER arrive on the page session
|
|
193
|
+
* (measured live over CDP: platform.claude.com's `turbopack-worker-*.js`
|
|
194
|
+
* and aistudio.google.com's `accounts.google.com/ServiceLogin` probe stayed
|
|
195
|
+
* "open" for the page's whole life while the page itself completed at
|
|
196
|
+
* `load`), so strict `networkidle` — Playwright's is officially discouraged
|
|
197
|
+
* — is unreachable there and the legacy wait burned its whole cap on every
|
|
198
|
+
* fetch. A genuine content request this old is the documented trade: the
|
|
199
|
+
* settle stops waiting for it, and the scroll walk plus the live read still
|
|
200
|
+
* capture whatever it renders when it lands.
|
|
201
|
+
*/
|
|
202
|
+
const SETTLE_STALE_MS = 3_000
|
|
135
203
|
|
|
136
|
-
/**
|
|
137
|
-
const
|
|
204
|
+
/** How often the settle re-checks its quiet/stale conditions (ms). */
|
|
205
|
+
const SETTLE_POLL_MS = 100
|
|
138
206
|
|
|
139
207
|
/**
|
|
140
208
|
* Grace period (ms) for context/browser closes in the cleanup path. A wedged
|
|
@@ -143,6 +211,30 @@ const SETTLE_MS = 5_000
|
|
|
143
211
|
*/
|
|
144
212
|
const CLOSE_GRACE_MS = 2_000
|
|
145
213
|
|
|
214
|
+
/**
|
|
215
|
+
* How many times a navigation may be re-issued after a TRANSIENT transport
|
|
216
|
+
* failure (`net::ERR_NETWORK_CHANGED` and family — see
|
|
217
|
+
* {@link isTransientNavigationFailure}) before the fetch gives up. Three
|
|
218
|
+
* total navigations cover the single-blip badcase and the rare double-blip
|
|
219
|
+
* while capping the worst-case latency a persistently flapping network can
|
|
220
|
+
* add.
|
|
221
|
+
*/
|
|
222
|
+
const MAX_TRANSIENT_NAV_RETRIES = 2
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Backoff (ms) before each transient-navigation retry, indexed by the retry
|
|
226
|
+
* number: short first (most blips clear in well under a second), longer
|
|
227
|
+
* second. Bounded by the fetch deadline via an abortable sleep.
|
|
228
|
+
*/
|
|
229
|
+
const TRANSIENT_RETRY_BACKOFF_MS = [250, 750]
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Waiting budget (ms) that must REMAIN before a transient-navigation retry
|
|
233
|
+
* may start: the retry must not eat the tail the settle/decode path still
|
|
234
|
+
* needs, and a re-navigation that cannot finish is pure waste.
|
|
235
|
+
*/
|
|
236
|
+
const TRANSIENT_RETRY_RESERVE_MS = 2_000
|
|
237
|
+
|
|
146
238
|
/**
|
|
147
239
|
* One render session: a browser, the context this fetch works in, and the
|
|
148
240
|
* tab it owns. Local backends own the browser too; CDP sessions ride the
|
|
@@ -163,9 +255,13 @@ export interface BrowserSession {
|
|
|
163
255
|
}
|
|
164
256
|
|
|
165
257
|
/**
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
258
|
+
* The fetch's two-stage deadline composed with the caller's signal. The SOFT
|
|
259
|
+
* stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
|
|
260
|
+
* it and {@link expired} reports it, so every wait ends there and the fetch
|
|
261
|
+
* extracts what the page holds. The HARD stage (`timeoutMs + graceMs`) aborts
|
|
262
|
+
* {@link signal}, which closes the page. The timeout reason is kept apart
|
|
263
|
+
* from an outer abort so the error translation can pick `WEB_FETCH_TIMEOUT`
|
|
264
|
+
* over `WEB_ABORTED`.
|
|
169
265
|
*/
|
|
170
266
|
class Deadline {
|
|
171
267
|
readonly signal: AbortSignal
|
|
@@ -173,13 +269,13 @@ class Deadline {
|
|
|
173
269
|
private readonly expiresAt: number
|
|
174
270
|
private timedOut = false
|
|
175
271
|
|
|
176
|
-
constructor(outer: AbortSignal | undefined, timeoutMs: number) {
|
|
272
|
+
constructor(outer: AbortSignal | undefined, timeoutMs: number, graceMs = 0) {
|
|
177
273
|
this.signal = this.controller.signal
|
|
178
274
|
this.expiresAt = Date.now() + timeoutMs
|
|
179
275
|
const timer = setTimeout(() => {
|
|
180
276
|
this.timedOut = true
|
|
181
277
|
this.controller.abort(new Error('playwright fetch deadline'))
|
|
182
|
-
}, timeoutMs)
|
|
278
|
+
}, timeoutMs + graceMs)
|
|
183
279
|
if (outer !== undefined) {
|
|
184
280
|
if (outer.aborted) this.controller.abort(outer.reason)
|
|
185
281
|
else outer.addEventListener('abort', () => { this.controller.abort(outer.reason) }, { once: true })
|
|
@@ -187,12 +283,17 @@ class Deadline {
|
|
|
187
283
|
this.signal.addEventListener('abort', () => { clearTimeout(timer) }, { once: true })
|
|
188
284
|
}
|
|
189
285
|
|
|
190
|
-
/** Whether OUR timer fired (vs an outer cancellation). */
|
|
286
|
+
/** Whether OUR hard timer fired (vs an outer cancellation). */
|
|
191
287
|
get isTimeout(): boolean {
|
|
192
288
|
return this.timedOut
|
|
193
289
|
}
|
|
194
290
|
|
|
195
|
-
/**
|
|
291
|
+
/** Whether the soft (waiting) budget has run out. */
|
|
292
|
+
get expired(): boolean {
|
|
293
|
+
return Date.now() >= this.expiresAt
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/** Milliseconds left on the waiting budget, floored at 1 for Playwright options. */
|
|
196
297
|
remainingMs(): number {
|
|
197
298
|
return Math.max(1, this.expiresAt - Date.now())
|
|
198
299
|
}
|
|
@@ -324,6 +425,9 @@ function translateError(error: unknown, deadline: Deadline): WebError {
|
|
|
324
425
|
return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
|
|
325
426
|
}
|
|
326
427
|
if (error instanceof WebError) return error
|
|
428
|
+
// Past the waiting budget with nothing salvageable (no response arrived,
|
|
429
|
+
// or a bounded wait's own timeout surfaced): the budget is the cause.
|
|
430
|
+
if (deadline.expired) return new WebError('playwright web fetch timed out before the page delivered any content', 'WEB_FETCH_TIMEOUT', { cause: error })
|
|
327
431
|
return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
|
|
328
432
|
}
|
|
329
433
|
|
|
@@ -337,6 +441,21 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
337
441
|
|
|
338
442
|
private readonly semaphore = new Semaphore(DEFAULT_MAX_CONCURRENCY_LOCAL)
|
|
339
443
|
|
|
444
|
+
/** Waiting budget per fetch (ms); code-level, overridable by subclasses (tests). */
|
|
445
|
+
protected readonly fetchTimeoutMs: number = DEFAULT_TIMEOUT_MS
|
|
446
|
+
|
|
447
|
+
/** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
|
|
448
|
+
protected readonly extractGraceMs: number = EXTRACT_GRACE_MS
|
|
449
|
+
|
|
450
|
+
/** Settle quiet window (ms); code-level, overridable by subclasses (tests). */
|
|
451
|
+
protected readonly settleQuietMs: number = SETTLE_QUIET_MS
|
|
452
|
+
|
|
453
|
+
/** Request age (ms) past which it stops gating the settle; overridable by subclasses (tests). */
|
|
454
|
+
protected readonly settleStaleMs: number = SETTLE_STALE_MS
|
|
455
|
+
|
|
456
|
+
/** Settle re-check cadence (ms); overridable by subclasses (tests). */
|
|
457
|
+
protected readonly settlePollMs: number = SETTLE_POLL_MS
|
|
458
|
+
|
|
340
459
|
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
341
460
|
protected readonly cdpPool: CdpConnectionPool
|
|
342
461
|
|
|
@@ -382,7 +501,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
382
501
|
}
|
|
383
502
|
}
|
|
384
503
|
const url = validateFetchUrl(request.url)
|
|
385
|
-
const deadline = new Deadline(signal,
|
|
504
|
+
const deadline = new Deadline(signal, this.fetchTimeoutMs, this.extractGraceMs)
|
|
386
505
|
|
|
387
506
|
let session: BrowserSession | undefined
|
|
388
507
|
let acquired = false
|
|
@@ -394,6 +513,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
394
513
|
acquired = true
|
|
395
514
|
session = await this.openSession(config, deadline, sessionId)
|
|
396
515
|
if (deadline.signal.aborted) throw new Error('web fetch aborted during session setup')
|
|
516
|
+
if (deadline.expired) throw new Error('the fetch budget ran out during session setup')
|
|
397
517
|
// An aborted deadline must also interrupt Playwright's own waits:
|
|
398
518
|
// closing the page rejects every pending operation on it (and, for
|
|
399
519
|
// fetch-owned contexts, the context close that follows takes the rest).
|
|
@@ -501,15 +621,22 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
501
621
|
const page = session.page
|
|
502
622
|
const challengeWaitMs = effectiveChallengeWaitMs(config)
|
|
503
623
|
// The tracker is always on: the challenge wait needs the LAST main-frame
|
|
504
|
-
// response (feature switch below only gates the waiting),
|
|
505
|
-
// recovery needs whatever response arrived before a failed goto
|
|
624
|
+
// response (feature switch below only gates the waiting), status
|
|
625
|
+
// recovery needs whatever response arrived before a failed goto, and
|
|
626
|
+
// the settle needs the page's open-request bookkeeping from BEFORE
|
|
627
|
+
// navigation — a request that started before the settle begins still
|
|
628
|
+
// counts toward quiet and staleness.
|
|
506
629
|
const tracker = trackMainFrameResponses(page)
|
|
630
|
+
const activity = trackRequestActivity(page)
|
|
507
631
|
// Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
|
|
508
632
|
// first response decides, no waiting — an escape hatch and the A/B
|
|
509
633
|
// baseline every test proves the bug against.
|
|
510
634
|
const first = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
511
635
|
if (first.recovered !== undefined) return first.recovered
|
|
512
636
|
let response = first.response
|
|
637
|
+
// The document never finished loading inside the budget: extract what
|
|
638
|
+
// arrived and flag the result as incomplete.
|
|
639
|
+
let partial = first.partial === true
|
|
513
640
|
tracker.seed(response)
|
|
514
641
|
|
|
515
642
|
let challengeEntryResponse: PlaywrightResponse | null = null
|
|
@@ -536,7 +663,18 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
536
663
|
try { chained = classifyChallengeHtml(await page.content()) === 'challenge' } catch { chained = false }
|
|
537
664
|
if (!chained) break
|
|
538
665
|
}
|
|
539
|
-
|
|
666
|
+
// No budget left for another window: a re-navigation could only
|
|
667
|
+
// land on the interstitial again.
|
|
668
|
+
if (--attemptsLeft <= 0 || deadline.expired) {
|
|
669
|
+
// The dead-end refetch: many Cloudflare edges serve the real
|
|
670
|
+
// document to plain-HTTP clients (same cookies, no render
|
|
671
|
+
// decision) while withholding the interactive challenge from
|
|
672
|
+
// THIS browser session — measured live on v2ex.com, where the
|
|
673
|
+
// browser drew an empty widget frame and curl drew the article.
|
|
674
|
+
if (effectiveChallengeHttpFallback(config)) {
|
|
675
|
+
const fallback = await this.challengeHttpFallback(page, url, deadline, config.denoise)
|
|
676
|
+
if (fallback !== undefined) return fallback
|
|
677
|
+
}
|
|
540
678
|
const lastStatus = (tracker.last() ?? response)?.status()
|
|
541
679
|
throw new WebError(
|
|
542
680
|
`the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
|
|
@@ -549,6 +687,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
549
687
|
const retry = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
550
688
|
if (retry.recovered !== undefined) return retry.recovered
|
|
551
689
|
response = retry.response
|
|
690
|
+
partial = retry.partial === true
|
|
552
691
|
tracker.seed(response)
|
|
553
692
|
}
|
|
554
693
|
}
|
|
@@ -577,24 +716,29 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
577
716
|
|
|
578
717
|
// Non-HTML decodes straight from the response body; no denoise applies.
|
|
579
718
|
if (kind === 'text') {
|
|
580
|
-
|
|
581
|
-
|
|
719
|
+
// A stopped load has no complete response body to read; the page's
|
|
720
|
+
// rendered text is what arrived.
|
|
721
|
+
const text = partial
|
|
722
|
+
? await renderedText(page)
|
|
723
|
+
: finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
724
|
+
return flagPartial(capResult(finalUrl, statusCode, { kind: 'text', content: text }), partial)
|
|
582
725
|
}
|
|
583
726
|
|
|
584
727
|
// Best-effort settle for client-rendered content; a timeout just keeps
|
|
585
|
-
// what domcontentloaded already produced.
|
|
586
|
-
|
|
728
|
+
// what domcontentloaded already produced. See {@link settleForContent}
|
|
729
|
+
// for why this is a custom quiet window rather than Playwright's
|
|
730
|
+
// networkidle.
|
|
731
|
+
await this.settleForContent(page, deadline, config, activity)
|
|
587
732
|
|
|
588
733
|
// Deep capture (viewport-virtualized feeds and app-shell documents lose
|
|
589
734
|
// their off-window content at rest — see `src/autoscroll.ts`): a bounded
|
|
590
735
|
// walk stashes what each rendered window shows and, when it restored
|
|
591
736
|
// anything, hands back its own merged, already-stripped serialization.
|
|
592
|
-
// HTML documents only,
|
|
593
|
-
//
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
)
|
|
737
|
+
// HTML documents only, ending at the waiting budget at the latest — the
|
|
738
|
+
// walk then hands back what it stashed so far and extraction runs in
|
|
739
|
+
// the grace window — silently degrading to the legacy capture;
|
|
740
|
+
// `scrollBudgetMs: 0` skips it, as does a budget already spent.
|
|
741
|
+
const scrollBudgetMs = deadline.expired ? 0 : Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs())
|
|
598
742
|
const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : undefined
|
|
599
743
|
|
|
600
744
|
// Serialization safety for the live read: the document is about to be
|
|
@@ -613,14 +757,56 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
613
757
|
if (!config.denoise) {
|
|
614
758
|
// The tool layer's own turndown renders raw HTML; the checkbox only
|
|
615
759
|
// governs the Readability/DOMPurify stage this provider owns.
|
|
616
|
-
return capResult(finalUrl, statusCode, { kind: 'html', content: html })
|
|
760
|
+
return flagPartial(capResult(finalUrl, statusCode, { kind: 'html', content: html }), partial)
|
|
761
|
+
}
|
|
762
|
+
return flagPartial(denoiseResult(finalUrl, statusCode, html), partial)
|
|
763
|
+
}
|
|
764
|
+
|
|
765
|
+
/**
|
|
766
|
+
* The challenge dead-end's plain-HTTP refetch: one request through the
|
|
767
|
+
* page's own request context (the SAME cookie jar — any clearance the
|
|
768
|
+
* profile holds rides along — but none of Chromium's render decisions),
|
|
769
|
+
* wearing the browser's own User-Agent. Returns the finished result when
|
|
770
|
+
* the answer is a NON-challenge HTML document, undefined otherwise (the
|
|
771
|
+
* caller throws the honest WEB_FETCH_CHALLENGE). A challenge served to
|
|
772
|
+
* this request too, a non-HTML body, or any transport failure keeps the
|
|
773
|
+
* browser-only verdict.
|
|
774
|
+
*/
|
|
775
|
+
private async challengeHttpFallback(page: PlaywrightPage, url: URL, deadline: Deadline, denoise: boolean): Promise<WebFetchResult | undefined> {
|
|
776
|
+
if (page.request === undefined || deadline.expired || deadline.signal.aborted) return undefined
|
|
777
|
+
try {
|
|
778
|
+
// A CDP-attached browser's request context does not inherit the
|
|
779
|
+
// browser's UA — send it explicitly so the edge sees one client.
|
|
780
|
+
let userAgent: string | undefined
|
|
781
|
+
if (typeof page.evaluate === 'function') {
|
|
782
|
+
try {
|
|
783
|
+
const ua = await page.evaluate('navigator.userAgent')
|
|
784
|
+
if (typeof ua === 'string' && ua !== '') userAgent = ua
|
|
785
|
+
} catch {
|
|
786
|
+
userAgent = undefined
|
|
787
|
+
}
|
|
788
|
+
}
|
|
789
|
+
const response = await page.request.get(url.toString(), {
|
|
790
|
+
headers: {
|
|
791
|
+
...(userAgent === undefined ? {} : { 'user-agent': userAgent }),
|
|
792
|
+
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
793
|
+
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
|
794
|
+
},
|
|
795
|
+
timeout: Math.min(deadline.remainingMs(), 15_000),
|
|
796
|
+
signal: deadline.signal,
|
|
797
|
+
})
|
|
798
|
+
const headers = response.headers()
|
|
799
|
+
const status = response.status()
|
|
800
|
+
if (status < 200 || status >= 300) return undefined
|
|
801
|
+
if (classifyChallengeResponse(status, headers) === 'challenge') return undefined
|
|
802
|
+
if (classifyContentType(headers['content-type']) !== 'html') return undefined
|
|
803
|
+
const html = await response.text()
|
|
804
|
+
if (classifyChallengeHtml(html) !== 'none') return undefined
|
|
805
|
+
if (!denoise) return capResult(url.toString(), status, { kind: 'html', content: html })
|
|
806
|
+
return denoiseResult(url.toString(), status, html)
|
|
807
|
+
} catch {
|
|
808
|
+
return undefined
|
|
617
809
|
}
|
|
618
|
-
const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
|
|
619
|
-
const { markdown } = htmlToMarkdown(bounded, finalUrl)
|
|
620
|
-
const result = capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
621
|
-
return bounded !== html
|
|
622
|
-
? { ...result, truncated: true }
|
|
623
|
-
: result
|
|
624
810
|
}
|
|
625
811
|
|
|
626
812
|
/**
|
|
@@ -631,6 +817,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
631
817
|
* status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
|
|
632
818
|
* being a download the browser refuses to commit as a document
|
|
633
819
|
* (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
|
|
820
|
+
* A TRANSIENT transport failure (`net::ERR_NETWORK_CHANGED` and family)
|
|
821
|
+
* is neither: it carries no response and no render decision, only a
|
|
822
|
+
* momentary host-network blip that killed this one navigation — so it is
|
|
823
|
+
* retried on the SAME page, bounded by {@link MAX_TRANSIENT_NAV_RETRIES}
|
|
824
|
+
* attempts, a short backoff, and the remaining waiting budget.
|
|
634
825
|
*
|
|
635
826
|
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
636
827
|
* navigation failure was recovered into a finished result the caller must
|
|
@@ -642,20 +833,44 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
642
833
|
tracker: MainFrameTracker,
|
|
643
834
|
deadline: Deadline,
|
|
644
835
|
denoise: boolean,
|
|
645
|
-
): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult }> {
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
836
|
+
): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult; partial?: boolean }> {
|
|
837
|
+
for (let attempt = 0; ; attempt++) {
|
|
838
|
+
try {
|
|
839
|
+
const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
|
|
840
|
+
return { response }
|
|
841
|
+
} catch (error) {
|
|
842
|
+
// Our own deadline/abort closing the page mid-navigation is a
|
|
843
|
+
// cancellation (translateError classifies it) — never a status fact.
|
|
844
|
+
if (!deadline.signal.aborted) {
|
|
845
|
+
const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
|
|
846
|
+
if (recovered !== undefined) return { response: null, recovered }
|
|
847
|
+
const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
|
|
848
|
+
if (download !== undefined) return { response: null, recovered: download }
|
|
849
|
+
// The waiting budget ran out mid-load: once the document's response
|
|
850
|
+
// arrived the page already holds content, so stop the load and let
|
|
851
|
+
// the caller extract it instead of failing the whole fetch.
|
|
852
|
+
const arrived = tracker.last()
|
|
853
|
+
if (deadline.expired && arrived !== null) {
|
|
854
|
+
await stopLoading(page)
|
|
855
|
+
return { response: arrived, partial: true }
|
|
856
|
+
}
|
|
857
|
+
// The transport blip family: nothing about the PAGE died — the
|
|
858
|
+
// host's network shifted under the browser and Chromium aborted
|
|
859
|
+
// the in-flight navigation. Retry on the same page while the
|
|
860
|
+
// attempt cap, the backoff, and the remaining budget allow; the
|
|
861
|
+
// reserve keeps enough budget for the settle/decode tail.
|
|
862
|
+
if (
|
|
863
|
+
attempt < MAX_TRANSIENT_NAV_RETRIES
|
|
864
|
+
&& isTransientNavigationFailure(error)
|
|
865
|
+
&& deadline.remainingMs() > TRANSIENT_RETRY_RESERVE_MS
|
|
866
|
+
) {
|
|
867
|
+
const backoff = TRANSIENT_RETRY_BACKOFF_MS[Math.min(attempt, TRANSIENT_RETRY_BACKOFF_MS.length - 1)] ?? TRANSIENT_RETRY_BACKOFF_MS.at(-1) ?? 500
|
|
868
|
+
await abortableSleep(backoff, deadline.signal)
|
|
869
|
+
if (!deadline.signal.aborted) continue
|
|
870
|
+
}
|
|
871
|
+
}
|
|
872
|
+
throw error
|
|
657
873
|
}
|
|
658
|
-
throw error
|
|
659
874
|
}
|
|
660
875
|
}
|
|
661
876
|
|
|
@@ -715,6 +930,74 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
715
930
|
return true
|
|
716
931
|
}
|
|
717
932
|
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* Best-effort settle for client-rendered content — a Lighthouse-style
|
|
936
|
+
* bounded network-quiet window replacing Playwright's `networkidle`
|
|
937
|
+
* (officially discouraged): worker-script fetches and cross-origin auth
|
|
938
|
+
* frames hold one request open on the page session FOREVER (their
|
|
939
|
+
* completion events are attributed to other CDP targets), so strict
|
|
940
|
+
* networkidle never arrives on exactly the modern pages the settle exists
|
|
941
|
+
* for, and the legacy wait burned its whole cap on them every fetch.
|
|
942
|
+
*
|
|
943
|
+
* Sequence: a `load` baseline (usually already past — domcontentloaded
|
|
944
|
+
* resolved earlier), then poll until no request has STARTED within
|
|
945
|
+
* {@link settleQuietMs} and every still-open request is older than
|
|
946
|
+
* {@link settleStaleMs} — a request that old is an attribution artifact
|
|
947
|
+
* or a long-poll whose completion no longer gates content. All waits
|
|
948
|
+
* share the {@link effectiveSettleMs} cap and the fetch deadline; a page
|
|
949
|
+
* that exposes no request events degrades to the legacy bounded
|
|
950
|
+
* networkidle wait. Never throws: any exit path keeps what the document
|
|
951
|
+
* already holds.
|
|
952
|
+
*/
|
|
953
|
+
private async settleForContent(
|
|
954
|
+
page: PlaywrightPage,
|
|
955
|
+
deadline: Deadline,
|
|
956
|
+
config: ResolvedConfig,
|
|
957
|
+
activity: RequestActivity | undefined,
|
|
958
|
+
): Promise<void> {
|
|
959
|
+
const beganAt = Date.now()
|
|
960
|
+
const cap = effectiveSettleMs(config)
|
|
961
|
+
if (cap <= 0) {
|
|
962
|
+
this.noteSettleOutcome('off', 0)
|
|
963
|
+
return
|
|
964
|
+
}
|
|
965
|
+
const endsAt = beganAt + Math.min(cap, deadline.remainingMs())
|
|
966
|
+
// The load event is the baseline quiet is measured against; usually it
|
|
967
|
+
// has already fired by the time the challenge loop released the page.
|
|
968
|
+
await page.waitForLoadState('load', { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {})
|
|
969
|
+
if (activity === undefined) {
|
|
970
|
+
// No request events on this surface: the legacy bounded wait.
|
|
971
|
+
await page.waitForLoadState('networkidle', { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {})
|
|
972
|
+
this.noteSettleOutcome('fallback', Date.now() - beganAt)
|
|
973
|
+
return
|
|
974
|
+
}
|
|
975
|
+
for (;;) {
|
|
976
|
+
if (deadline.signal.aborted) {
|
|
977
|
+
this.noteSettleOutcome('aborted', Date.now() - beganAt)
|
|
978
|
+
return
|
|
979
|
+
}
|
|
980
|
+
if (Date.now() >= endsAt) {
|
|
981
|
+
this.noteSettleOutcome('cap', Date.now() - beganAt)
|
|
982
|
+
return
|
|
983
|
+
}
|
|
984
|
+
if (activity.idle(this.settleQuietMs, this.settleStaleMs)) {
|
|
985
|
+
this.noteSettleOutcome(activity.openCount() > 0 ? 'stale-discounted' : 'quiet', Date.now() - beganAt)
|
|
986
|
+
return
|
|
987
|
+
}
|
|
988
|
+
await sleep(Math.min(this.settlePollMs, Math.max(1, endsAt - Date.now())))
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* Settle outcome observation hook — silent in production by design (the
|
|
994
|
+
* settle is best-effort), but subclasses (tests, diagnostics) override it
|
|
995
|
+
* to see which exit path every fetch took: `off` (config), `quiet`
|
|
996
|
+
* (network went quiet), `stale-discounted` (only stale requests remained
|
|
997
|
+
* open), `cap` (window exhausted), `fallback` (no request events — the
|
|
998
|
+
* legacy networkidle wait ran), `aborted` (fetch deadline fired).
|
|
999
|
+
*/
|
|
1000
|
+
protected noteSettleOutcome(outcome: 'off' | 'quiet' | 'stale-discounted' | 'cap' | 'fallback' | 'aborted', elapsedMs: number): void {}
|
|
718
1001
|
}
|
|
719
1002
|
|
|
720
1003
|
/** Resolve after `ms` — the bounded wait's inter-poll nap. */
|
|
@@ -722,6 +1005,39 @@ function sleep(ms: number): Promise<void> {
|
|
|
722
1005
|
return new Promise(resolve => { setTimeout(resolve, ms) })
|
|
723
1006
|
}
|
|
724
1007
|
|
|
1008
|
+
/**
|
|
1009
|
+
* Best-effort `window.stop()` on a page whose load outran the fetch budget:
|
|
1010
|
+
* the parser and pending subresources halt, so the DOM read next is what
|
|
1011
|
+
* arrived rather than a moving target. Bounded; failures are ignored.
|
|
1012
|
+
*/
|
|
1013
|
+
async function stopLoading(page: PlaywrightPage): Promise<void> {
|
|
1014
|
+
if (page.evaluate === undefined) return
|
|
1015
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
1016
|
+
await Promise.race([
|
|
1017
|
+
page.evaluate('window.stop()').catch(() => {}),
|
|
1018
|
+
new Promise<void>(resolve => { timer = setTimeout(resolve, STOP_LOADING_GRACE_MS) }),
|
|
1019
|
+
])
|
|
1020
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
/** The rendered text of a stopped non-HTML document (its body's innerText). */
|
|
1024
|
+
async function renderedText(page: PlaywrightPage): Promise<string> {
|
|
1025
|
+
if (page.evaluate !== undefined) {
|
|
1026
|
+
try {
|
|
1027
|
+
const text = await page.evaluate('document.body ? document.body.innerText : ""')
|
|
1028
|
+
if (typeof text === 'string') return text
|
|
1029
|
+
} catch {
|
|
1030
|
+
// fall through to the serialized document
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
return await page.content()
|
|
1034
|
+
}
|
|
1035
|
+
|
|
1036
|
+
/** Mark a result built from a document that never finished loading. */
|
|
1037
|
+
function flagPartial(result: WebFetchResult, partial: boolean): WebFetchResult {
|
|
1038
|
+
return partial ? { ...result, truncated: true } : result
|
|
1039
|
+
}
|
|
1040
|
+
|
|
725
1041
|
/**
|
|
726
1042
|
* Whether a response event belongs to the page's main frame as a document
|
|
727
1043
|
* navigation — the filter the challenge wait uses to keep the LAST such
|
|
@@ -770,6 +1086,62 @@ function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
|
|
|
770
1086
|
}
|
|
771
1087
|
}
|
|
772
1088
|
|
|
1089
|
+
/**
|
|
1090
|
+
* The running record of the page's open requests, for the settle's
|
|
1091
|
+
* network-quiet detection. Attached BEFORE navigation so every request —
|
|
1092
|
+
* including ones that started before the settle begins — counts toward the
|
|
1093
|
+
* quiet window and staleness.
|
|
1094
|
+
*/
|
|
1095
|
+
interface RequestActivity {
|
|
1096
|
+
/**
|
|
1097
|
+
* Whether the page is settle-idle: no request started within `quietMs`,
|
|
1098
|
+
* and every still-open request has been open at least `staleMs`.
|
|
1099
|
+
*/
|
|
1100
|
+
idle(quietMs: number, staleMs: number): boolean
|
|
1101
|
+
/** How many requests are still open (labels the idle exit). */
|
|
1102
|
+
openCount(): number
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* Watch the page for request lifecycle events so the settle can decide
|
|
1107
|
+
* network quietness itself. `undefined` when the page exposes no request
|
|
1108
|
+
* events (minimal fakes): the settle then falls back to the legacy bounded
|
|
1109
|
+
* networkidle wait.
|
|
1110
|
+
*
|
|
1111
|
+
* @param page - the fetch's tab.
|
|
1112
|
+
* @returns the open-request tracker, or undefined on an unsupported surface.
|
|
1113
|
+
*/
|
|
1114
|
+
function trackRequestActivity(page: PlaywrightPage): RequestActivity | undefined {
|
|
1115
|
+
if (typeof page.on !== 'function') return undefined
|
|
1116
|
+
const opened = new Map<PlaywrightWebRequest, number>()
|
|
1117
|
+
let lastStart = Date.now()
|
|
1118
|
+
const onStarted = (request: PlaywrightWebRequest): void => {
|
|
1119
|
+
opened.set(request, Date.now())
|
|
1120
|
+
lastStart = Date.now()
|
|
1121
|
+
}
|
|
1122
|
+
const onSettled = (request: PlaywrightWebRequest): void => {
|
|
1123
|
+
opened.delete(request)
|
|
1124
|
+
}
|
|
1125
|
+
try {
|
|
1126
|
+
page.on('request', onStarted)
|
|
1127
|
+
page.on('requestfinished', onSettled)
|
|
1128
|
+
page.on('requestfailed', onSettled)
|
|
1129
|
+
} catch {
|
|
1130
|
+
return undefined
|
|
1131
|
+
}
|
|
1132
|
+
return {
|
|
1133
|
+
idle: (quietMs, staleMs) => {
|
|
1134
|
+
const now = Date.now()
|
|
1135
|
+
if (now - lastStart < quietMs) return false
|
|
1136
|
+
for (const startedAt of opened.values()) {
|
|
1137
|
+
if (now - startedAt < staleMs) return false
|
|
1138
|
+
}
|
|
1139
|
+
return true
|
|
1140
|
+
},
|
|
1141
|
+
openCount: () => opened.size,
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
|
|
773
1145
|
/**
|
|
774
1146
|
* Whether a `goto` throw is Chromium refusing to RENDER a response it
|
|
775
1147
|
* received — not a transport failure. Two net errors, both documented as
|
|
@@ -794,6 +1166,48 @@ function isUnrenderedStatusFailure(error: unknown, status: number): boolean {
|
|
|
794
1166
|
return message.includes('ERR_ABORTED') && (status === 204 || status === 205)
|
|
795
1167
|
}
|
|
796
1168
|
|
|
1169
|
+
/**
|
|
1170
|
+
* Chromium net errors a plain re-navigation can plausibly fix — the
|
|
1171
|
+
* transport-layer BLIPS, not render decisions (those are recovered above)
|
|
1172
|
+
* and not `ERR_ABORTED` (deliberately excluded: that code is overloaded with
|
|
1173
|
+
* deterministic shapes — SPA cancels, 204/205 aborts, downloads — which the
|
|
1174
|
+
* recoveries already own). `ERR_NETWORK_CHANGED` is the measured badcase:
|
|
1175
|
+
* the host's network configuration shifted under the browser mid-navigation
|
|
1176
|
+
* (route/interface/DNS/proxy change, VPN or DHCP flap, a Docker
|
|
1177
|
+
* embedded-DNS rewrite) and Chromium's NetworkChangeNotifier killed the
|
|
1178
|
+
* in-flight request; the sibling codes are the same story on a different
|
|
1179
|
+
* layer (DNS blips, TCP resets and idle breaks, brief disconnects).
|
|
1180
|
+
* Playwright's own navigation timeouts never match: they reject with a
|
|
1181
|
+
* `TimeoutError` ("Timeout Nms exceeded"), not a `net::ERR_*` code.
|
|
1182
|
+
*/
|
|
1183
|
+
const TRANSIENT_NET_ERROR_RE = /net::ERR_(NETWORK_CHANGED|INTERNET_DISCONNECTED|CONNECTION_RESET|CONNECTION_CLOSED|CONNECTION_TIMED_OUT|TIMED_OUT|EMPTY_RESPONSE|NAME_NOT_RESOLVED|NAME_RESOLUTION_FAILED|ADDRESS_UNREACHABLE)\b/
|
|
1184
|
+
|
|
1185
|
+
/**
|
|
1186
|
+
* Whether a `goto` throw is one of the transient transport failures a
|
|
1187
|
+
* bounded same-page retry can recover (see {@link TRANSIENT_NET_ERROR_RE}).
|
|
1188
|
+
*
|
|
1189
|
+
* @param error - the value `page.goto` rejected with.
|
|
1190
|
+
* @returns true when the navigation died to a momentary network blip.
|
|
1191
|
+
*/
|
|
1192
|
+
function isTransientNavigationFailure(error: unknown): boolean {
|
|
1193
|
+
return error instanceof Error && TRANSIENT_NET_ERROR_RE.test(error.message)
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
/**
|
|
1197
|
+
* Sleep for `ms`, ending EARLY when `signal` aborts: the transient-retry
|
|
1198
|
+
* backoff must never outlive the fetch — a deadline firing mid-backoff
|
|
1199
|
+
* resolves immediately so the caller re-checks the signal and rethrows the
|
|
1200
|
+
* original failure for the timeout/abort classification.
|
|
1201
|
+
*/
|
|
1202
|
+
function abortableSleep(ms: number, signal: AbortSignal): Promise<void> {
|
|
1203
|
+
return new Promise(resolve => {
|
|
1204
|
+
if (signal.aborted) return resolve()
|
|
1205
|
+
const onAbort = () => { clearTimeout(timer); resolve() }
|
|
1206
|
+
const timer = setTimeout(() => { signal.removeEventListener('abort', onAbort); resolve() }, ms)
|
|
1207
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
1208
|
+
})
|
|
1209
|
+
}
|
|
1210
|
+
|
|
797
1211
|
/**
|
|
798
1212
|
* Recover the response behind a status-only navigation failure into the
|
|
799
1213
|
* result the web seam's contract demands — "a successful network fetch of a
|
|
@@ -848,8 +1262,7 @@ async function recoverUnrenderedStatus(
|
|
|
848
1262
|
return capResult(finalUrl, status, { kind: 'text', content })
|
|
849
1263
|
}
|
|
850
1264
|
if (!denoise) return capResult(finalUrl, status, { kind: 'html', content })
|
|
851
|
-
|
|
852
|
-
return capResult(finalUrl, status, { kind: 'text', content: markdown })
|
|
1265
|
+
return denoiseResult(finalUrl, status, content)
|
|
853
1266
|
}
|
|
854
1267
|
|
|
855
1268
|
/**
|
|
@@ -961,8 +1374,7 @@ async function recoverDownloadResponse(
|
|
|
961
1374
|
return capResult(finalUrl, statusCode, { kind: 'text', content })
|
|
962
1375
|
}
|
|
963
1376
|
if (!denoise) return capResult(finalUrl, statusCode, { kind: 'html', content })
|
|
964
|
-
|
|
965
|
-
return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
1377
|
+
return denoiseResult(finalUrl, statusCode, content)
|
|
966
1378
|
}
|
|
967
1379
|
|
|
968
1380
|
/** Time kept back from the PDF page walk for layout and result assembly. */
|
|
@@ -1077,6 +1489,20 @@ async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
|
|
|
1077
1489
|
}
|
|
1078
1490
|
}
|
|
1079
1491
|
|
|
1492
|
+
/**
|
|
1493
|
+
* Denoise an HTML body into a capped text result. The synchronous pipeline
|
|
1494
|
+
* builds several DOMs per page, so its input is bounded on EVERY path that
|
|
1495
|
+
* reaches it (live render, unrenderable-loader fallback, refetch): an
|
|
1496
|
+
* uncapped 5MB API-reference page needs >4GB of heap. A cut input flags
|
|
1497
|
+
* the result truncated.
|
|
1498
|
+
*/
|
|
1499
|
+
function denoiseResult(url: string, statusCode: number, html: string): WebFetchResult {
|
|
1500
|
+
const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
|
|
1501
|
+
const { markdown } = htmlToMarkdown(bounded, url)
|
|
1502
|
+
const result = capResult(url, statusCode, { kind: 'text', content: markdown })
|
|
1503
|
+
return { ...result, truncated: result.truncated || bounded !== html }
|
|
1504
|
+
}
|
|
1505
|
+
|
|
1080
1506
|
/** Apply the decoded-body cap and flag the cut. */
|
|
1081
1507
|
function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text'; content: string }): WebFetchResult {
|
|
1082
1508
|
const truncated = body.content.length > MAX_BODY_CHARS
|