dsh-web-fetch-playwright 0.2.22 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/README.zh-CN.md +4 -2
- package/lib/client.js +19 -19
- package/lib/index.js +386 -50
- package/lib/types/candidates.d.ts +48 -0
- package/lib/types/challenge.d.ts +2 -2
- package/lib/types/config.d.ts +30 -4
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +55 -0
- package/lib/types/types.d.ts +19 -0
- package/package.json +1 -1
- package/src/candidates.ts +162 -29
- package/src/challenge.ts +2 -2
- package/src/config.ts +37 -6
- package/src/index.ts +2 -0
- package/src/markdown.ts +13 -2
- package/src/provider.ts +292 -27
- package/src/types.ts +20 -0
package/src/index.ts
CHANGED
|
@@ -28,11 +28,13 @@ export {
|
|
|
28
28
|
DEFAULT_CHALLENGE_RETRIES,
|
|
29
29
|
DEFAULT_CHALLENGE_WAIT_MS,
|
|
30
30
|
DEFAULT_SCROLL_BUDGET_MS,
|
|
31
|
+
DEFAULT_SETTLE_MS,
|
|
31
32
|
effectiveChallengeHttpFallback,
|
|
32
33
|
effectiveChallengeRetries,
|
|
33
34
|
effectiveChallengeWaitMs,
|
|
34
35
|
effectiveContextMode,
|
|
35
36
|
effectiveScrollBudgetMs,
|
|
37
|
+
effectiveSettleMs,
|
|
36
38
|
normalizeCdpEndpoint,
|
|
37
39
|
resolveConfig,
|
|
38
40
|
snapshotsOf,
|
package/src/markdown.ts
CHANGED
|
@@ -227,11 +227,22 @@ function analyze(html: string, url: string): Analysis {
|
|
|
227
227
|
const sanitize = (candidate: RawCandidate): Analysis['sanitized'][number] => {
|
|
228
228
|
// KEEP_CONTENT: false is the load-bearing half of the denoise: DOMPurify
|
|
229
229
|
// un-wraps forbidden tags but keeps their text by default; with it, the
|
|
230
|
-
// whole subtree of a forbidden element goes.
|
|
231
|
-
|
|
230
|
+
// whole subtree of a forbidden element goes. A generator-held NODE is
|
|
231
|
+
// sanitized by import (identical output, measured on the whole-body
|
|
232
|
+
// candidate of the Angular docs badcase); a string is re-parsed.
|
|
233
|
+
const root = purify.sanitize(candidate.node ?? candidate.html, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false, RETURN_DOM: true }) as HTMLElement
|
|
232
234
|
return { kind: candidate.kind, title: candidate.title, legacy: candidate.legacy, root, stats: measureCandidate(candidate.kind, candidate.legacy, root) }
|
|
233
235
|
}
|
|
234
236
|
const sanitized = generated.candidates.map(sanitize)
|
|
237
|
+
// The custom-wrap skip left no `art` candidate — but the selector's
|
|
238
|
+
// features read the art reference (`words_vs_art_log`, the block diffs),
|
|
239
|
+
// and a skipped Readability run would have measured a DEAD-empty one. Seed
|
|
240
|
+
// exactly that, so the surviving candidates' vectors stay what a dead pick
|
|
241
|
+
// yields (verified on the badcase: the sanitized art root holds no text).
|
|
242
|
+
if (generated.facts.readabilitySkipped === true && !sanitized.some(c => c.kind === 'art')) {
|
|
243
|
+
const empty = document.createElement('div')
|
|
244
|
+
sanitized.push({ kind: 'art', title: undefined, legacy: false, root: empty, stats: measureCandidate('art', false, empty) })
|
|
245
|
+
}
|
|
235
246
|
const page = pageFeatures(document, url, generated.facts, sanitized.find(c => c.kind === 'doc')?.stats, visibleTextLength(document.body))
|
|
236
247
|
|
|
237
248
|
let blocks: Block[] = []
|
package/src/provider.ts
CHANGED
|
@@ -18,6 +18,21 @@
|
|
|
18
18
|
* refetch through the page's API request context
|
|
19
19
|
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
20
20
|
*
|
|
21
|
+
* Transient transport failures (issue: `net::ERR_NETWORK_CHANGED` and its
|
|
22
|
+
* family) are retried, not terminal: Chromium's NetworkChangeNotifier kills
|
|
23
|
+
* every in-flight request with `ERR_NETWORK_CHANGED` the moment the HOST's
|
|
24
|
+
* network configuration shifts under the browser — a route/interface/DNS/
|
|
25
|
+
* proxy change, a VPN or DHCP flap, a Docker embedded-DNS rewrite; the page
|
|
26
|
+
* and its target are perfectly fine, only that one navigation died, and the
|
|
27
|
+
* very next `goto` usually succeeds (measured live: source.android.com
|
|
28
|
+
* failed once mid-navigation in one DSH session and fetched clean from the
|
|
29
|
+
* same host seconds later). Each navigation therefore retries the
|
|
30
|
+
* transport-blip family ({@link isTransientNavigationFailure} — deliberately
|
|
31
|
+
* NOT `ERR_ABORTED`, which is overloaded with the deterministic shapes the
|
|
32
|
+
* recoveries above already own) a bounded number of times with a short
|
|
33
|
+
* backoff, inside the same page and the remaining waiting budget; a blip
|
|
34
|
+
* that survives the attempt cap still fails as before.
|
|
35
|
+
*
|
|
21
36
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
22
37
|
* nothing outlives the call. CDP session isolation keeps one connection per
|
|
23
38
|
* DSH session until its context becomes idle; this also isolates gateways that
|
|
@@ -108,14 +123,14 @@ import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepsee
|
|
|
108
123
|
import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
109
124
|
import type { ChallengeVerdict } from './challenge.ts'
|
|
110
125
|
import { deepCaptureWalk } from './autoscroll.ts'
|
|
111
|
-
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
|
|
126
|
+
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, effectiveSettleMs, normalizeCdpEndpoint } from './config.ts'
|
|
112
127
|
import type { ResolvedConfig } from './config.ts'
|
|
113
128
|
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
114
129
|
import { htmlToMarkdown } from './markdown.ts'
|
|
115
130
|
import { extractPdfMarkdown, isPdfBytes, isPdfContentType, mayBePdf, MAX_PDF_BYTES, WEB_PDF_INVALID_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
|
|
116
131
|
import { STRIP_SCRIPTING_ELEMENTS } from './strip.ts'
|
|
117
132
|
import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
|
|
118
|
-
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute } from './types.ts'
|
|
133
|
+
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightDownload, PlaywrightPage, PlaywrightResponse, PlaywrightRoute, PlaywrightWebRequest } from './types.ts'
|
|
119
134
|
|
|
120
135
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
121
136
|
export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
|
|
@@ -162,8 +177,32 @@ const EXTRACT_GRACE_MS = 10_000
|
|
|
162
177
|
/** Bound (ms) on the best-effort `window.stop()` of a still-loading page. */
|
|
163
178
|
const STOP_LOADING_GRACE_MS = 1_000
|
|
164
179
|
|
|
165
|
-
/**
|
|
166
|
-
|
|
180
|
+
/**
|
|
181
|
+
* Settle quiet window (ms): the post-load settle ends once no request has
|
|
182
|
+
* STARTED for this long AND every still-open request is stale (see
|
|
183
|
+
* {@link SETTLE_STALE_MS}) — the Lighthouse-style network-quiet threshold,
|
|
184
|
+
* sized to match Playwright's own networkidle quiet window.
|
|
185
|
+
*/
|
|
186
|
+
const SETTLE_QUIET_MS = 500
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* A request open longer than this (ms) stops gating the settle. On the
|
|
190
|
+
* pages that motivated the custom detector, the long-lived request is a
|
|
191
|
+
* worker-script fetch or a cross-origin auth frame whose completion events
|
|
192
|
+
* are attributed to another CDP target and NEVER arrive on the page session
|
|
193
|
+
* (measured live over CDP: platform.claude.com's `turbopack-worker-*.js`
|
|
194
|
+
* and aistudio.google.com's `accounts.google.com/ServiceLogin` probe stayed
|
|
195
|
+
* "open" for the page's whole life while the page itself completed at
|
|
196
|
+
* `load`), so strict `networkidle` — Playwright's is officially discouraged
|
|
197
|
+
* — is unreachable there and the legacy wait burned its whole cap on every
|
|
198
|
+
* fetch. A genuine content request this old is the documented trade: the
|
|
199
|
+
* settle stops waiting for it, and the scroll walk plus the live read still
|
|
200
|
+
* capture whatever it renders when it lands.
|
|
201
|
+
*/
|
|
202
|
+
const SETTLE_STALE_MS = 3_000
|
|
203
|
+
|
|
204
|
+
/** How often the settle re-checks its quiet/stale conditions (ms). */
|
|
205
|
+
const SETTLE_POLL_MS = 100
|
|
167
206
|
|
|
168
207
|
/**
|
|
169
208
|
* Grace period (ms) for context/browser closes in the cleanup path. A wedged
|
|
@@ -172,6 +211,30 @@ const SETTLE_MS = 5_000
|
|
|
172
211
|
*/
|
|
173
212
|
const CLOSE_GRACE_MS = 2_000
|
|
174
213
|
|
|
214
|
+
/**
|
|
215
|
+
* How many times a navigation may be re-issued after a TRANSIENT transport
|
|
216
|
+
* failure (`net::ERR_NETWORK_CHANGED` and family — see
|
|
217
|
+
* {@link isTransientNavigationFailure}) before the fetch gives up. Three
|
|
218
|
+
* total navigations cover the single-blip badcase and the rare double-blip
|
|
219
|
+
* while capping the worst-case latency a persistently flapping network can
|
|
220
|
+
* add.
|
|
221
|
+
*/
|
|
222
|
+
const MAX_TRANSIENT_NAV_RETRIES = 2
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Backoff (ms) before each transient-navigation retry, indexed by the retry
|
|
226
|
+
* number: short first (most blips clear in well under a second), longer
|
|
227
|
+
* second. Bounded by the fetch deadline via an abortable sleep.
|
|
228
|
+
*/
|
|
229
|
+
const TRANSIENT_RETRY_BACKOFF_MS = [250, 750]
|
|
230
|
+
|
|
231
|
+
/**
|
|
232
|
+
* Waiting budget (ms) that must REMAIN before a transient-navigation retry
|
|
233
|
+
* may start: the retry must not eat the tail the settle/decode path still
|
|
234
|
+
* needs, and a re-navigation that cannot finish is pure waste.
|
|
235
|
+
*/
|
|
236
|
+
const TRANSIENT_RETRY_RESERVE_MS = 2_000
|
|
237
|
+
|
|
175
238
|
/**
|
|
176
239
|
* One render session: a browser, the context this fetch works in, and the
|
|
177
240
|
* tab it owns. Local backends own the browser too; CDP sessions ride the
|
|
@@ -384,6 +447,15 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
384
447
|
/** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
|
|
385
448
|
protected readonly extractGraceMs: number = EXTRACT_GRACE_MS
|
|
386
449
|
|
|
450
|
+
/** Settle quiet window (ms); code-level, overridable by subclasses (tests). */
|
|
451
|
+
protected readonly settleQuietMs: number = SETTLE_QUIET_MS
|
|
452
|
+
|
|
453
|
+
/** Request age (ms) past which it stops gating the settle; overridable by subclasses (tests). */
|
|
454
|
+
protected readonly settleStaleMs: number = SETTLE_STALE_MS
|
|
455
|
+
|
|
456
|
+
/** Settle re-check cadence (ms); overridable by subclasses (tests). */
|
|
457
|
+
protected readonly settlePollMs: number = SETTLE_POLL_MS
|
|
458
|
+
|
|
387
459
|
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
388
460
|
protected readonly cdpPool: CdpConnectionPool
|
|
389
461
|
|
|
@@ -549,9 +621,13 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
549
621
|
const page = session.page
|
|
550
622
|
const challengeWaitMs = effectiveChallengeWaitMs(config)
|
|
551
623
|
// The tracker is always on: the challenge wait needs the LAST main-frame
|
|
552
|
-
// response (feature switch below only gates the waiting),
|
|
553
|
-
// recovery needs whatever response arrived before a failed goto
|
|
624
|
+
// response (feature switch below only gates the waiting), status
|
|
625
|
+
// recovery needs whatever response arrived before a failed goto, and
|
|
626
|
+
// the settle needs the page's open-request bookkeeping from BEFORE
|
|
627
|
+
// navigation — a request that started before the settle begins still
|
|
628
|
+
// counts toward quiet and staleness.
|
|
554
629
|
const tracker = trackMainFrameResponses(page)
|
|
630
|
+
const activity = trackRequestActivity(page)
|
|
555
631
|
// Feature switch: 0 keeps the exact legacy (pre-0.2.5) behavior — the
|
|
556
632
|
// first response decides, no waiting — an escape hatch and the A/B
|
|
557
633
|
// baseline every test proves the bug against.
|
|
@@ -649,8 +725,10 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
649
725
|
}
|
|
650
726
|
|
|
651
727
|
// Best-effort settle for client-rendered content; a timeout just keeps
|
|
652
|
-
// what domcontentloaded already produced.
|
|
653
|
-
|
|
728
|
+
// what domcontentloaded already produced. See {@link settleForContent}
|
|
729
|
+
// for why this is a custom quiet window rather than Playwright's
|
|
730
|
+
// networkidle.
|
|
731
|
+
await this.settleForContent(page, deadline, config, activity)
|
|
654
732
|
|
|
655
733
|
// Deep capture (viewport-virtualized feeds and app-shell documents lose
|
|
656
734
|
// their off-window content at rest — see `src/autoscroll.ts`): a bounded
|
|
@@ -739,6 +817,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
739
817
|
* status-only 204/205 — {@link recoverUnrenderedStatus}), and the response
|
|
740
818
|
* being a download the browser refuses to commit as a document
|
|
741
819
|
* (`Content-Disposition: attachment` — {@link recoverDownloadResponse}).
|
|
820
|
+
* A TRANSIENT transport failure (`net::ERR_NETWORK_CHANGED` and family)
|
|
821
|
+
* is neither: it carries no response and no render decision, only a
|
|
822
|
+
* momentary host-network blip that killed this one navigation — so it is
|
|
823
|
+
* retried on the SAME page, bounded by {@link MAX_TRANSIENT_NAV_RETRIES}
|
|
824
|
+
* attempts, a short backoff, and the remaining waiting budget.
|
|
742
825
|
*
|
|
743
826
|
* @returns the response `goto` resolved with, plus `recovered` when the
|
|
744
827
|
* navigation failure was recovered into a finished result the caller must
|
|
@@ -751,27 +834,43 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
751
834
|
deadline: Deadline,
|
|
752
835
|
denoise: boolean,
|
|
753
836
|
): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult; partial?: boolean }> {
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
837
|
+
for (let attempt = 0; ; attempt++) {
|
|
838
|
+
try {
|
|
839
|
+
const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
|
|
840
|
+
return { response }
|
|
841
|
+
} catch (error) {
|
|
842
|
+
// Our own deadline/abort closing the page mid-navigation is a
|
|
843
|
+
// cancellation (translateError classifies it) — never a status fact.
|
|
844
|
+
if (!deadline.signal.aborted) {
|
|
845
|
+
const recovered = await recoverUnrenderedStatus(error, tracker, url, denoise)
|
|
846
|
+
if (recovered !== undefined) return { response: null, recovered }
|
|
847
|
+
const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
|
|
848
|
+
if (download !== undefined) return { response: null, recovered: download }
|
|
849
|
+
// The waiting budget ran out mid-load: once the document's response
|
|
850
|
+
// arrived the page already holds content, so stop the load and let
|
|
851
|
+
// the caller extract it instead of failing the whole fetch.
|
|
852
|
+
const arrived = tracker.last()
|
|
853
|
+
if (deadline.expired && arrived !== null) {
|
|
854
|
+
await stopLoading(page)
|
|
855
|
+
return { response: arrived, partial: true }
|
|
856
|
+
}
|
|
857
|
+
// The transport blip family: nothing about the PAGE died — the
|
|
858
|
+
// host's network shifted under the browser and Chromium aborted
|
|
859
|
+
// the in-flight navigation. Retry on the same page while the
|
|
860
|
+
// attempt cap, the backoff, and the remaining budget allow; the
|
|
861
|
+
// reserve keeps enough budget for the settle/decode tail.
|
|
862
|
+
if (
|
|
863
|
+
attempt < MAX_TRANSIENT_NAV_RETRIES
|
|
864
|
+
&& isTransientNavigationFailure(error)
|
|
865
|
+
&& deadline.remainingMs() > TRANSIENT_RETRY_RESERVE_MS
|
|
866
|
+
) {
|
|
867
|
+
const backoff = TRANSIENT_RETRY_BACKOFF_MS[Math.min(attempt, TRANSIENT_RETRY_BACKOFF_MS.length - 1)] ?? TRANSIENT_RETRY_BACKOFF_MS.at(-1) ?? 500
|
|
868
|
+
await abortableSleep(backoff, deadline.signal)
|
|
869
|
+
if (!deadline.signal.aborted) continue
|
|
870
|
+
}
|
|
772
871
|
}
|
|
872
|
+
throw error
|
|
773
873
|
}
|
|
774
|
-
throw error
|
|
775
874
|
}
|
|
776
875
|
}
|
|
777
876
|
|
|
@@ -831,6 +930,74 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
831
930
|
return true
|
|
832
931
|
}
|
|
833
932
|
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* Best-effort settle for client-rendered content — a Lighthouse-style
|
|
936
|
+
* bounded network-quiet window replacing Playwright's `networkidle`
|
|
937
|
+
* (officially discouraged): worker-script fetches and cross-origin auth
|
|
938
|
+
* frames hold one request open on the page session FOREVER (their
|
|
939
|
+
* completion events are attributed to other CDP targets), so strict
|
|
940
|
+
* networkidle never arrives on exactly the modern pages the settle exists
|
|
941
|
+
* for, and the legacy wait burned its whole cap on them every fetch.
|
|
942
|
+
*
|
|
943
|
+
* Sequence: a `load` baseline (usually already past — domcontentloaded
|
|
944
|
+
* resolved earlier), then poll until no request has STARTED within
|
|
945
|
+
* {@link settleQuietMs} and every still-open request is older than
|
|
946
|
+
* {@link settleStaleMs} — a request that old is an attribution artifact
|
|
947
|
+
* or a long-poll whose completion no longer gates content. All waits
|
|
948
|
+
* share the {@link effectiveSettleMs} cap and the fetch deadline; a page
|
|
949
|
+
* that exposes no request events degrades to the legacy bounded
|
|
950
|
+
* networkidle wait. Never throws: any exit path keeps what the document
|
|
951
|
+
* already holds.
|
|
952
|
+
*/
|
|
953
|
+
private async settleForContent(
|
|
954
|
+
page: PlaywrightPage,
|
|
955
|
+
deadline: Deadline,
|
|
956
|
+
config: ResolvedConfig,
|
|
957
|
+
activity: RequestActivity | undefined,
|
|
958
|
+
): Promise<void> {
|
|
959
|
+
const beganAt = Date.now()
|
|
960
|
+
const cap = effectiveSettleMs(config)
|
|
961
|
+
if (cap <= 0) {
|
|
962
|
+
this.noteSettleOutcome('off', 0)
|
|
963
|
+
return
|
|
964
|
+
}
|
|
965
|
+
const endsAt = beganAt + Math.min(cap, deadline.remainingMs())
|
|
966
|
+
// The load event is the baseline quiet is measured against; usually it
|
|
967
|
+
// has already fired by the time the challenge loop released the page.
|
|
968
|
+
await page.waitForLoadState('load', { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {})
|
|
969
|
+
if (activity === undefined) {
|
|
970
|
+
// No request events on this surface: the legacy bounded wait.
|
|
971
|
+
await page.waitForLoadState('networkidle', { timeout: Math.max(1, endsAt - Date.now()) }).catch(() => {})
|
|
972
|
+
this.noteSettleOutcome('fallback', Date.now() - beganAt)
|
|
973
|
+
return
|
|
974
|
+
}
|
|
975
|
+
for (;;) {
|
|
976
|
+
if (deadline.signal.aborted) {
|
|
977
|
+
this.noteSettleOutcome('aborted', Date.now() - beganAt)
|
|
978
|
+
return
|
|
979
|
+
}
|
|
980
|
+
if (Date.now() >= endsAt) {
|
|
981
|
+
this.noteSettleOutcome('cap', Date.now() - beganAt)
|
|
982
|
+
return
|
|
983
|
+
}
|
|
984
|
+
if (activity.idle(this.settleQuietMs, this.settleStaleMs)) {
|
|
985
|
+
this.noteSettleOutcome(activity.openCount() > 0 ? 'stale-discounted' : 'quiet', Date.now() - beganAt)
|
|
986
|
+
return
|
|
987
|
+
}
|
|
988
|
+
await sleep(Math.min(this.settlePollMs, Math.max(1, endsAt - Date.now())))
|
|
989
|
+
}
|
|
990
|
+
}
|
|
991
|
+
|
|
992
|
+
/**
|
|
993
|
+
* Settle outcome observation hook — silent in production by design (the
|
|
994
|
+
* settle is best-effort), but subclasses (tests, diagnostics) override it
|
|
995
|
+
* to see which exit path every fetch took: `off` (config), `quiet`
|
|
996
|
+
* (network went quiet), `stale-discounted` (only stale requests remained
|
|
997
|
+
* open), `cap` (window exhausted), `fallback` (no request events — the
|
|
998
|
+
* legacy networkidle wait ran), `aborted` (fetch deadline fired).
|
|
999
|
+
*/
|
|
1000
|
+
protected noteSettleOutcome(outcome: 'off' | 'quiet' | 'stale-discounted' | 'cap' | 'fallback' | 'aborted', elapsedMs: number): void {}
|
|
834
1001
|
}
|
|
835
1002
|
|
|
836
1003
|
/** Resolve after `ms` — the bounded wait's inter-poll nap. */
|
|
@@ -919,6 +1086,62 @@ function trackMainFrameResponses(page: PlaywrightPage): MainFrameTracker {
|
|
|
919
1086
|
}
|
|
920
1087
|
}
|
|
921
1088
|
|
|
1089
|
+
/**
|
|
1090
|
+
* The running record of the page's open requests, for the settle's
|
|
1091
|
+
* network-quiet detection. Attached BEFORE navigation so every request —
|
|
1092
|
+
* including ones that started before the settle begins — counts toward the
|
|
1093
|
+
* quiet window and staleness.
|
|
1094
|
+
*/
|
|
1095
|
+
interface RequestActivity {
|
|
1096
|
+
/**
|
|
1097
|
+
* Whether the page is settle-idle: no request started within `quietMs`,
|
|
1098
|
+
* and every still-open request has been open at least `staleMs`.
|
|
1099
|
+
*/
|
|
1100
|
+
idle(quietMs: number, staleMs: number): boolean
|
|
1101
|
+
/** How many requests are still open (labels the idle exit). */
|
|
1102
|
+
openCount(): number
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* Watch the page for request lifecycle events so the settle can decide
|
|
1107
|
+
* network quietness itself. `undefined` when the page exposes no request
|
|
1108
|
+
* events (minimal fakes): the settle then falls back to the legacy bounded
|
|
1109
|
+
* networkidle wait.
|
|
1110
|
+
*
|
|
1111
|
+
* @param page - the fetch's tab.
|
|
1112
|
+
* @returns the open-request tracker, or undefined on an unsupported surface.
|
|
1113
|
+
*/
|
|
1114
|
+
function trackRequestActivity(page: PlaywrightPage): RequestActivity | undefined {
|
|
1115
|
+
if (typeof page.on !== 'function') return undefined
|
|
1116
|
+
const opened = new Map<PlaywrightWebRequest, number>()
|
|
1117
|
+
let lastStart = Date.now()
|
|
1118
|
+
const onStarted = (request: PlaywrightWebRequest): void => {
|
|
1119
|
+
opened.set(request, Date.now())
|
|
1120
|
+
lastStart = Date.now()
|
|
1121
|
+
}
|
|
1122
|
+
const onSettled = (request: PlaywrightWebRequest): void => {
|
|
1123
|
+
opened.delete(request)
|
|
1124
|
+
}
|
|
1125
|
+
try {
|
|
1126
|
+
page.on('request', onStarted)
|
|
1127
|
+
page.on('requestfinished', onSettled)
|
|
1128
|
+
page.on('requestfailed', onSettled)
|
|
1129
|
+
} catch {
|
|
1130
|
+
return undefined
|
|
1131
|
+
}
|
|
1132
|
+
return {
|
|
1133
|
+
idle: (quietMs, staleMs) => {
|
|
1134
|
+
const now = Date.now()
|
|
1135
|
+
if (now - lastStart < quietMs) return false
|
|
1136
|
+
for (const startedAt of opened.values()) {
|
|
1137
|
+
if (now - startedAt < staleMs) return false
|
|
1138
|
+
}
|
|
1139
|
+
return true
|
|
1140
|
+
},
|
|
1141
|
+
openCount: () => opened.size,
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
|
|
922
1145
|
/**
|
|
923
1146
|
* Whether a `goto` throw is Chromium refusing to RENDER a response it
|
|
924
1147
|
* received — not a transport failure. Two net errors, both documented as
|
|
@@ -943,6 +1166,48 @@ function isUnrenderedStatusFailure(error: unknown, status: number): boolean {
|
|
|
943
1166
|
return message.includes('ERR_ABORTED') && (status === 204 || status === 205)
|
|
944
1167
|
}
|
|
945
1168
|
|
|
1169
|
+
/**
|
|
1170
|
+
* Chromium net errors a plain re-navigation can plausibly fix — the
|
|
1171
|
+
* transport-layer BLIPS, not render decisions (those are recovered above)
|
|
1172
|
+
* and not `ERR_ABORTED` (deliberately excluded: that code is overloaded with
|
|
1173
|
+
* deterministic shapes — SPA cancels, 204/205 aborts, downloads — which the
|
|
1174
|
+
* recoveries already own). `ERR_NETWORK_CHANGED` is the measured badcase:
|
|
1175
|
+
* the host's network configuration shifted under the browser mid-navigation
|
|
1176
|
+
* (route/interface/DNS/proxy change, VPN or DHCP flap, a Docker
|
|
1177
|
+
* embedded-DNS rewrite) and Chromium's NetworkChangeNotifier killed the
|
|
1178
|
+
* in-flight request; the sibling codes are the same story on a different
|
|
1179
|
+
* layer (DNS blips, TCP resets and idle breaks, brief disconnects).
|
|
1180
|
+
* Playwright's own navigation timeouts never match: they reject with a
|
|
1181
|
+
* `TimeoutError` ("Timeout Nms exceeded"), not a `net::ERR_*` code.
|
|
1182
|
+
*/
|
|
1183
|
+
const TRANSIENT_NET_ERROR_RE = /net::ERR_(NETWORK_CHANGED|INTERNET_DISCONNECTED|CONNECTION_RESET|CONNECTION_CLOSED|CONNECTION_TIMED_OUT|TIMED_OUT|EMPTY_RESPONSE|NAME_NOT_RESOLVED|NAME_RESOLUTION_FAILED|ADDRESS_UNREACHABLE)\b/
|
|
1184
|
+
|
|
1185
|
+
/**
|
|
1186
|
+
* Whether a `goto` throw is one of the transient transport failures a
|
|
1187
|
+
* bounded same-page retry can recover (see {@link TRANSIENT_NET_ERROR_RE}).
|
|
1188
|
+
*
|
|
1189
|
+
* @param error - the value `page.goto` rejected with.
|
|
1190
|
+
* @returns true when the navigation died to a momentary network blip.
|
|
1191
|
+
*/
|
|
1192
|
+
function isTransientNavigationFailure(error: unknown): boolean {
|
|
1193
|
+
return error instanceof Error && TRANSIENT_NET_ERROR_RE.test(error.message)
|
|
1194
|
+
}
|
|
1195
|
+
|
|
1196
|
+
/**
|
|
1197
|
+
* Sleep for `ms`, ending EARLY when `signal` aborts: the transient-retry
|
|
1198
|
+
* backoff must never outlive the fetch — a deadline firing mid-backoff
|
|
1199
|
+
* resolves immediately so the caller re-checks the signal and rethrows the
|
|
1200
|
+
* original failure for the timeout/abort classification.
|
|
1201
|
+
*/
|
|
1202
|
+
function abortableSleep(ms: number, signal: AbortSignal): Promise<void> {
|
|
1203
|
+
return new Promise(resolve => {
|
|
1204
|
+
if (signal.aborted) return resolve()
|
|
1205
|
+
const onAbort = () => { clearTimeout(timer); resolve() }
|
|
1206
|
+
const timer = setTimeout(() => { signal.removeEventListener('abort', onAbort); resolve() }, ms)
|
|
1207
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
1208
|
+
})
|
|
1209
|
+
}
|
|
1210
|
+
|
|
946
1211
|
/**
|
|
947
1212
|
* Recover the response behind a status-only navigation failure into the
|
|
948
1213
|
* result the web seam's contract demands — "a successful network fetch of a
|
package/src/types.ts
CHANGED
|
@@ -57,6 +57,15 @@ export interface PlaywrightRequest {
|
|
|
57
57
|
frame?(): unknown
|
|
58
58
|
}
|
|
59
59
|
|
|
60
|
+
/**
|
|
61
|
+
* A network request notification, as `page.on('request')` and its settled
|
|
62
|
+
* siblings deliver it. Identity only — the settle's network-quiet tracker
|
|
63
|
+
* keys its open-request map on the object itself.
|
|
64
|
+
*/
|
|
65
|
+
export interface PlaywrightWebRequest {
|
|
66
|
+
url?(): string
|
|
67
|
+
}
|
|
68
|
+
|
|
60
69
|
/** A page inside a context. */
|
|
61
70
|
export interface PlaywrightPage {
|
|
62
71
|
goto(url: string, options?: { waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit'; timeout?: number }): Promise<PlaywrightResponse | null>
|
|
@@ -77,6 +86,17 @@ export interface PlaywrightPage {
|
|
|
77
86
|
* document). Absent on minimal fakes (content polling covers them).
|
|
78
87
|
*/
|
|
79
88
|
on?(event: 'response', listener: (response: PlaywrightResponse) => void): unknown
|
|
89
|
+
/**
|
|
90
|
+
* Request-start notification — the settle's network-quiet tracker. Absent
|
|
91
|
+
* on minimal fakes (the settle falls back to the legacy bounded
|
|
92
|
+
* networkidle wait).
|
|
93
|
+
*/
|
|
94
|
+
on?(event: 'request', listener: (request: PlaywrightWebRequest) => void): unknown
|
|
95
|
+
/**
|
|
96
|
+
* Request-settled notification (finished or failed) — closes the tracker's
|
|
97
|
+
* open-request bookkeeping. Absent on minimal fakes.
|
|
98
|
+
*/
|
|
99
|
+
on?(event: 'requestfinished' | 'requestfailed', listener: (request: PlaywrightWebRequest) => void): unknown
|
|
80
100
|
/**
|
|
81
101
|
* Download notification — a navigation the browser handed to its download
|
|
82
102
|
* manager instead of committing a document (`Content-Disposition:
|