dsh-web-fetch-playwright 0.2.20 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -15
- package/README.zh-CN.md +15 -15
- package/cordis.patch.yml +0 -4
- package/lib/client.js +21 -71
- package/lib/index.js +23564 -22605
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/candidates.d.ts +7 -0
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +54 -30
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +46 -9
- package/lib/types/types.d.ts +7 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +1 -1
- package/src/candidates.ts +22 -6
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +67 -38
- package/src/index.ts +2 -1
- package/src/provider.ts +198 -37
- package/src/selector-model.ts +1 -1
- package/src/selector.ts +18 -7
- package/src/types.ts +7 -7
package/src/provider.ts
CHANGED
|
@@ -55,9 +55,14 @@
|
|
|
55
55
|
* behavior). The wait tracks the LAST main-frame navigation response (the
|
|
56
56
|
* real page reloads in after the challenge clears) and watches the live DOM
|
|
57
57
|
* so SPA-style clears are caught too. It never clicks, never injects
|
|
58
|
-
* answers, never touches cookies itself; when the budget runs out
|
|
59
|
-
*
|
|
60
|
-
*
|
|
58
|
+
* answers, never touches cookies itself; when the budget runs out, ONE
|
|
59
|
+
* plain-HTTP refetch of the same URL through the page's request context
|
|
60
|
+
* (same cookie jar, the browser's UA, none of Chromium's render decisions
|
|
61
|
+
* — `challengeHttpFallback`, default on) tries to retrieve the real
|
|
62
|
+
* document, gated by the same response-level and content-level challenge
|
|
63
|
+
* classifiers so an interstitial can never pass as content. When that too
|
|
64
|
+
* fails, the fetch fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of
|
|
65
|
+
* returning the interstitial as content.
|
|
61
66
|
*
|
|
62
67
|
* Deep capture (viewport-virtualized feeds and app-shell documents): feed-
|
|
63
68
|
* shaped pages — Discourse topic streams, comment threads, timelines —
|
|
@@ -84,6 +89,17 @@
|
|
|
84
89
|
* degrades to the legacy capture (see `src/autoscroll.ts` for the measured
|
|
85
90
|
* linux.do and Feishu badcases).
|
|
86
91
|
*
|
|
92
|
+
* Deadline (soft, then hard): each fetch has a {@link DEFAULT_TIMEOUT_MS}
|
|
93
|
+
* budget for WAITING — navigation, settle, challenge wait, scroll walk. When
|
|
94
|
+
* it runs out the page is NOT torn down: every wait is bounded by the same
|
|
95
|
+
* budget and simply ends, a navigation still loading is stopped
|
|
96
|
+
* (`window.stop()`) once its main-frame response has arrived, and whatever
|
|
97
|
+
* the page holds by then goes through the normal extraction — a slow or
|
|
98
|
+
* endless page still hands the agent the content it already rendered
|
|
99
|
+
* (flagged `truncated` when the document never finished loading). Only when
|
|
100
|
+
* nothing arrived at all, or extraction itself hangs past
|
|
101
|
+
* {@link EXTRACT_GRACE_MS}, does the fetch fail with `WEB_FETCH_TIMEOUT`.
|
|
102
|
+
*
|
|
87
103
|
* @module dsh-web-fetch-playwright/provider
|
|
88
104
|
*/
|
|
89
105
|
|
|
@@ -91,8 +107,8 @@ import { WebError } from '@deepseek-ai/dsh-web'
|
|
|
91
107
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
|
|
92
108
|
import { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
93
109
|
import type { ChallengeVerdict } from './challenge.ts'
|
|
94
|
-
import { deepCaptureWalk
|
|
95
|
-
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
|
|
110
|
+
import { deepCaptureWalk } from './autoscroll.ts'
|
|
111
|
+
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveMaxConcurrency, effectiveScrollBudgetMs, normalizeCdpEndpoint } from './config.ts'
|
|
96
112
|
import type { ResolvedConfig } from './config.ts'
|
|
97
113
|
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
98
114
|
import { htmlToMarkdown } from './markdown.ts'
|
|
@@ -128,10 +144,23 @@ const MAX_PIPELINE_INPUT_CHARS = 2_000_000
|
|
|
128
144
|
* budget to render after dequeue — and failing fast tells the caller to
|
|
129
145
|
* retry or raise `maxConcurrency` instead of hanging until an abort.
|
|
130
146
|
*/
|
|
131
|
-
const QUEUE_TIMEOUT_MS =
|
|
147
|
+
const QUEUE_TIMEOUT_MS = 10_000
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Per-fetch waiting budget (ms): when it runs out the fetch stops waiting
|
|
151
|
+
* for the page and extracts what it already holds (see the module docs).
|
|
152
|
+
*/
|
|
153
|
+
const DEFAULT_TIMEOUT_MS = 25_000
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Hard backstop (ms) after {@link DEFAULT_TIMEOUT_MS} for reading and
|
|
157
|
+
* decoding what the page holds; only past it is the tab closed and the fetch
|
|
158
|
+
* failed. Budget plus grace stays inside the tool layer's 60s.
|
|
159
|
+
*/
|
|
160
|
+
const EXTRACT_GRACE_MS = 10_000
|
|
132
161
|
|
|
133
|
-
/**
|
|
134
|
-
const
|
|
162
|
+
/** Bound (ms) on the best-effort `window.stop()` of a still-loading page. */
|
|
163
|
+
const STOP_LOADING_GRACE_MS = 1_000
|
|
135
164
|
|
|
136
165
|
/** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
|
|
137
166
|
const SETTLE_MS = 5_000
|
|
@@ -163,9 +192,13 @@ export interface BrowserSession {
|
|
|
163
192
|
}
|
|
164
193
|
|
|
165
194
|
/**
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
195
|
+
* The fetch's two-stage deadline composed with the caller's signal. The SOFT
|
|
196
|
+
* stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
|
|
197
|
+
* it and {@link expired} reports it, so every wait ends there and the fetch
|
|
198
|
+
* extracts what the page holds. The HARD stage (`timeoutMs + graceMs`) aborts
|
|
199
|
+
* {@link signal}, which closes the page. The timeout reason is kept apart
|
|
200
|
+
* from an outer abort so the error translation can pick `WEB_FETCH_TIMEOUT`
|
|
201
|
+
* over `WEB_ABORTED`.
|
|
169
202
|
*/
|
|
170
203
|
class Deadline {
|
|
171
204
|
readonly signal: AbortSignal
|
|
@@ -173,13 +206,13 @@ class Deadline {
|
|
|
173
206
|
private readonly expiresAt: number
|
|
174
207
|
private timedOut = false
|
|
175
208
|
|
|
176
|
-
constructor(outer: AbortSignal | undefined, timeoutMs: number) {
|
|
209
|
+
constructor(outer: AbortSignal | undefined, timeoutMs: number, graceMs = 0) {
|
|
177
210
|
this.signal = this.controller.signal
|
|
178
211
|
this.expiresAt = Date.now() + timeoutMs
|
|
179
212
|
const timer = setTimeout(() => {
|
|
180
213
|
this.timedOut = true
|
|
181
214
|
this.controller.abort(new Error('playwright fetch deadline'))
|
|
182
|
-
}, timeoutMs)
|
|
215
|
+
}, timeoutMs + graceMs)
|
|
183
216
|
if (outer !== undefined) {
|
|
184
217
|
if (outer.aborted) this.controller.abort(outer.reason)
|
|
185
218
|
else outer.addEventListener('abort', () => { this.controller.abort(outer.reason) }, { once: true })
|
|
@@ -187,12 +220,17 @@ class Deadline {
|
|
|
187
220
|
this.signal.addEventListener('abort', () => { clearTimeout(timer) }, { once: true })
|
|
188
221
|
}
|
|
189
222
|
|
|
190
|
-
/** Whether OUR timer fired (vs an outer cancellation). */
|
|
223
|
+
/** Whether OUR hard timer fired (vs an outer cancellation). */
|
|
191
224
|
get isTimeout(): boolean {
|
|
192
225
|
return this.timedOut
|
|
193
226
|
}
|
|
194
227
|
|
|
195
|
-
/**
|
|
228
|
+
/** Whether the soft (waiting) budget has run out. */
|
|
229
|
+
get expired(): boolean {
|
|
230
|
+
return Date.now() >= this.expiresAt
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/** Milliseconds left on the waiting budget, floored at 1 for Playwright options. */
|
|
196
234
|
remainingMs(): number {
|
|
197
235
|
return Math.max(1, this.expiresAt - Date.now())
|
|
198
236
|
}
|
|
@@ -324,6 +362,9 @@ function translateError(error: unknown, deadline: Deadline): WebError {
|
|
|
324
362
|
return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
|
|
325
363
|
}
|
|
326
364
|
if (error instanceof WebError) return error
|
|
365
|
+
// Past the waiting budget with nothing salvageable (no response arrived,
|
|
366
|
+
// or a bounded wait's own timeout surfaced): the budget is the cause.
|
|
367
|
+
if (deadline.expired) return new WebError('playwright web fetch timed out before the page delivered any content', 'WEB_FETCH_TIMEOUT', { cause: error })
|
|
327
368
|
return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
|
|
328
369
|
}
|
|
329
370
|
|
|
@@ -337,6 +378,12 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
337
378
|
|
|
338
379
|
private readonly semaphore = new Semaphore(DEFAULT_MAX_CONCURRENCY_LOCAL)
|
|
339
380
|
|
|
381
|
+
/** Waiting budget per fetch (ms); code-level, overridable by subclasses (tests). */
|
|
382
|
+
protected readonly fetchTimeoutMs: number = DEFAULT_TIMEOUT_MS
|
|
383
|
+
|
|
384
|
+
/** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
|
|
385
|
+
protected readonly extractGraceMs: number = EXTRACT_GRACE_MS
|
|
386
|
+
|
|
340
387
|
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
341
388
|
protected readonly cdpPool: CdpConnectionPool
|
|
342
389
|
|
|
@@ -382,7 +429,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
382
429
|
}
|
|
383
430
|
}
|
|
384
431
|
const url = validateFetchUrl(request.url)
|
|
385
|
-
const deadline = new Deadline(signal,
|
|
432
|
+
const deadline = new Deadline(signal, this.fetchTimeoutMs, this.extractGraceMs)
|
|
386
433
|
|
|
387
434
|
let session: BrowserSession | undefined
|
|
388
435
|
let acquired = false
|
|
@@ -394,6 +441,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
394
441
|
acquired = true
|
|
395
442
|
session = await this.openSession(config, deadline, sessionId)
|
|
396
443
|
if (deadline.signal.aborted) throw new Error('web fetch aborted during session setup')
|
|
444
|
+
if (deadline.expired) throw new Error('the fetch budget ran out during session setup')
|
|
397
445
|
// An aborted deadline must also interrupt Playwright's own waits:
|
|
398
446
|
// closing the page rejects every pending operation on it (and, for
|
|
399
447
|
// fetch-owned contexts, the context close that follows takes the rest).
|
|
@@ -510,6 +558,9 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
510
558
|
const first = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
511
559
|
if (first.recovered !== undefined) return first.recovered
|
|
512
560
|
let response = first.response
|
|
561
|
+
// The document never finished loading inside the budget: extract what
|
|
562
|
+
// arrived and flag the result as incomplete.
|
|
563
|
+
let partial = first.partial === true
|
|
513
564
|
tracker.seed(response)
|
|
514
565
|
|
|
515
566
|
let challengeEntryResponse: PlaywrightResponse | null = null
|
|
@@ -536,7 +587,18 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
536
587
|
try { chained = classifyChallengeHtml(await page.content()) === 'challenge' } catch { chained = false }
|
|
537
588
|
if (!chained) break
|
|
538
589
|
}
|
|
539
|
-
|
|
590
|
+
// No budget left for another window: a re-navigation could only
|
|
591
|
+
// land on the interstitial again.
|
|
592
|
+
if (--attemptsLeft <= 0 || deadline.expired) {
|
|
593
|
+
// The dead-end refetch: many Cloudflare edges serve the real
|
|
594
|
+
// document to plain-HTTP clients (same cookies, no render
|
|
595
|
+
// decision) while withholding the interactive challenge from
|
|
596
|
+
// THIS browser session — measured live on v2ex.com, where the
|
|
597
|
+
// browser drew an empty widget frame and curl drew the article.
|
|
598
|
+
if (effectiveChallengeHttpFallback(config)) {
|
|
599
|
+
const fallback = await this.challengeHttpFallback(page, url, deadline, config.denoise)
|
|
600
|
+
if (fallback !== undefined) return fallback
|
|
601
|
+
}
|
|
540
602
|
const lastStatus = (tracker.last() ?? response)?.status()
|
|
541
603
|
throw new WebError(
|
|
542
604
|
`the site kept serving a Cloudflare challenge (last status ${lastStatus === undefined ? 'unknown' : String(lastStatus)}) for up to ${String(challengeWaitMs)}ms across ${String(effectiveChallengeRetries(config) + 1)} attempt(s); the browser did not clear it naturally — retry later, raise challengeWaitMs, or use a profile whose browser already holds clearance`,
|
|
@@ -549,6 +611,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
549
611
|
const retry = await this.navigate(page, url, tracker, deadline, config.denoise)
|
|
550
612
|
if (retry.recovered !== undefined) return retry.recovered
|
|
551
613
|
response = retry.response
|
|
614
|
+
partial = retry.partial === true
|
|
552
615
|
tracker.seed(response)
|
|
553
616
|
}
|
|
554
617
|
}
|
|
@@ -577,8 +640,12 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
577
640
|
|
|
578
641
|
// Non-HTML decodes straight from the response body; no denoise applies.
|
|
579
642
|
if (kind === 'text') {
|
|
580
|
-
|
|
581
|
-
|
|
643
|
+
// A stopped load has no complete response body to read; the page's
|
|
644
|
+
// rendered text is what arrived.
|
|
645
|
+
const text = partial
|
|
646
|
+
? await renderedText(page)
|
|
647
|
+
: finalResponse !== null ? await finalResponse.text() : await page.content()
|
|
648
|
+
return flagPartial(capResult(finalUrl, statusCode, { kind: 'text', content: text }), partial)
|
|
582
649
|
}
|
|
583
650
|
|
|
584
651
|
// Best-effort settle for client-rendered content; a timeout just keeps
|
|
@@ -589,12 +656,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
589
656
|
// their off-window content at rest — see `src/autoscroll.ts`): a bounded
|
|
590
657
|
// walk stashes what each rendered window shows and, when it restored
|
|
591
658
|
// anything, hands back its own merged, already-stripped serialization.
|
|
592
|
-
// HTML documents only,
|
|
593
|
-
//
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
)
|
|
659
|
+
// HTML documents only, ending at the waiting budget at the latest — the
|
|
660
|
+
// walk then hands back what it stashed so far and extraction runs in
|
|
661
|
+
// the grace window — silently degrading to the legacy capture;
|
|
662
|
+
// `scrollBudgetMs: 0` skips it, as does a budget already spent.
|
|
663
|
+
const scrollBudgetMs = deadline.expired ? 0 : Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs())
|
|
598
664
|
const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : undefined
|
|
599
665
|
|
|
600
666
|
// Serialization safety for the live read: the document is about to be
|
|
@@ -613,14 +679,56 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
613
679
|
if (!config.denoise) {
|
|
614
680
|
// The tool layer's own turndown renders raw HTML; the checkbox only
|
|
615
681
|
// governs the Readability/DOMPurify stage this provider owns.
|
|
616
|
-
return capResult(finalUrl, statusCode, { kind: 'html', content: html })
|
|
682
|
+
return flagPartial(capResult(finalUrl, statusCode, { kind: 'html', content: html }), partial)
|
|
683
|
+
}
|
|
684
|
+
return flagPartial(denoiseResult(finalUrl, statusCode, html), partial)
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* The challenge dead-end's plain-HTTP refetch: one request through the
|
|
689
|
+
* page's own request context (the SAME cookie jar — any clearance the
|
|
690
|
+
* profile holds rides along — but none of Chromium's render decisions),
|
|
691
|
+
* wearing the browser's own User-Agent. Returns the finished result when
|
|
692
|
+
* the answer is a NON-challenge HTML document, undefined otherwise (the
|
|
693
|
+
* caller throws the honest WEB_FETCH_CHALLENGE). A challenge served to
|
|
694
|
+
* this request too, a non-HTML body, or any transport failure keeps the
|
|
695
|
+
* browser-only verdict.
|
|
696
|
+
*/
|
|
697
|
+
private async challengeHttpFallback(page: PlaywrightPage, url: URL, deadline: Deadline, denoise: boolean): Promise<WebFetchResult | undefined> {
|
|
698
|
+
if (page.request === undefined || deadline.expired || deadline.signal.aborted) return undefined
|
|
699
|
+
try {
|
|
700
|
+
// A CDP-attached browser's request context does not inherit the
|
|
701
|
+
// browser's UA — send it explicitly so the edge sees one client.
|
|
702
|
+
let userAgent: string | undefined
|
|
703
|
+
if (typeof page.evaluate === 'function') {
|
|
704
|
+
try {
|
|
705
|
+
const ua = await page.evaluate('navigator.userAgent')
|
|
706
|
+
if (typeof ua === 'string' && ua !== '') userAgent = ua
|
|
707
|
+
} catch {
|
|
708
|
+
userAgent = undefined
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
const response = await page.request.get(url.toString(), {
|
|
712
|
+
headers: {
|
|
713
|
+
...(userAgent === undefined ? {} : { 'user-agent': userAgent }),
|
|
714
|
+
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
715
|
+
'accept-language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
|
716
|
+
},
|
|
717
|
+
timeout: Math.min(deadline.remainingMs(), 15_000),
|
|
718
|
+
signal: deadline.signal,
|
|
719
|
+
})
|
|
720
|
+
const headers = response.headers()
|
|
721
|
+
const status = response.status()
|
|
722
|
+
if (status < 200 || status >= 300) return undefined
|
|
723
|
+
if (classifyChallengeResponse(status, headers) === 'challenge') return undefined
|
|
724
|
+
if (classifyContentType(headers['content-type']) !== 'html') return undefined
|
|
725
|
+
const html = await response.text()
|
|
726
|
+
if (classifyChallengeHtml(html) !== 'none') return undefined
|
|
727
|
+
if (!denoise) return capResult(url.toString(), status, { kind: 'html', content: html })
|
|
728
|
+
return denoiseResult(url.toString(), status, html)
|
|
729
|
+
} catch {
|
|
730
|
+
return undefined
|
|
617
731
|
}
|
|
618
|
-
const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
|
|
619
|
-
const { markdown } = htmlToMarkdown(bounded, finalUrl)
|
|
620
|
-
const result = capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
621
|
-
return bounded !== html
|
|
622
|
-
? { ...result, truncated: true }
|
|
623
|
-
: result
|
|
624
732
|
}
|
|
625
733
|
|
|
626
734
|
/**
|
|
@@ -642,7 +750,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
642
750
|
tracker: MainFrameTracker,
|
|
643
751
|
deadline: Deadline,
|
|
644
752
|
denoise: boolean,
|
|
645
|
-
): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult }> {
|
|
753
|
+
): Promise<{ response: PlaywrightResponse | null; recovered?: WebFetchResult; partial?: boolean }> {
|
|
646
754
|
try {
|
|
647
755
|
const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
|
|
648
756
|
return { response }
|
|
@@ -654,6 +762,14 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
654
762
|
if (recovered !== undefined) return { response: null, recovered }
|
|
655
763
|
const download = await recoverDownloadResponse(error, page, tracker, url, deadline, denoise)
|
|
656
764
|
if (download !== undefined) return { response: null, recovered: download }
|
|
765
|
+
// The waiting budget ran out mid-load: once the document's response
|
|
766
|
+
// arrived the page already holds content, so stop the load and let
|
|
767
|
+
// the caller extract it instead of failing the whole fetch.
|
|
768
|
+
const arrived = tracker.last()
|
|
769
|
+
if (deadline.expired && arrived !== null) {
|
|
770
|
+
await stopLoading(page)
|
|
771
|
+
return { response: arrived, partial: true }
|
|
772
|
+
}
|
|
657
773
|
}
|
|
658
774
|
throw error
|
|
659
775
|
}
|
|
@@ -722,6 +838,39 @@ function sleep(ms: number): Promise<void> {
|
|
|
722
838
|
return new Promise(resolve => { setTimeout(resolve, ms) })
|
|
723
839
|
}
|
|
724
840
|
|
|
841
|
+
/**
|
|
842
|
+
* Best-effort `window.stop()` on a page whose load outran the fetch budget:
|
|
843
|
+
* the parser and pending subresources halt, so the DOM read next is what
|
|
844
|
+
* arrived rather than a moving target. Bounded; failures are ignored.
|
|
845
|
+
*/
|
|
846
|
+
async function stopLoading(page: PlaywrightPage): Promise<void> {
|
|
847
|
+
if (page.evaluate === undefined) return
|
|
848
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
849
|
+
await Promise.race([
|
|
850
|
+
page.evaluate('window.stop()').catch(() => {}),
|
|
851
|
+
new Promise<void>(resolve => { timer = setTimeout(resolve, STOP_LOADING_GRACE_MS) }),
|
|
852
|
+
])
|
|
853
|
+
if (timer !== undefined) clearTimeout(timer)
|
|
854
|
+
}
|
|
855
|
+
|
|
856
|
+
/** The rendered text of a stopped non-HTML document (its body's innerText). */
|
|
857
|
+
async function renderedText(page: PlaywrightPage): Promise<string> {
|
|
858
|
+
if (page.evaluate !== undefined) {
|
|
859
|
+
try {
|
|
860
|
+
const text = await page.evaluate('document.body ? document.body.innerText : ""')
|
|
861
|
+
if (typeof text === 'string') return text
|
|
862
|
+
} catch {
|
|
863
|
+
// fall through to the serialized document
|
|
864
|
+
}
|
|
865
|
+
}
|
|
866
|
+
return await page.content()
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
/** Mark a result built from a document that never finished loading. */
|
|
870
|
+
function flagPartial(result: WebFetchResult, partial: boolean): WebFetchResult {
|
|
871
|
+
return partial ? { ...result, truncated: true } : result
|
|
872
|
+
}
|
|
873
|
+
|
|
725
874
|
/**
|
|
726
875
|
* Whether a response event belongs to the page's main frame as a document
|
|
727
876
|
* navigation — the filter the challenge wait uses to keep the LAST such
|
|
@@ -848,8 +997,7 @@ async function recoverUnrenderedStatus(
|
|
|
848
997
|
return capResult(finalUrl, status, { kind: 'text', content })
|
|
849
998
|
}
|
|
850
999
|
if (!denoise) return capResult(finalUrl, status, { kind: 'html', content })
|
|
851
|
-
|
|
852
|
-
return capResult(finalUrl, status, { kind: 'text', content: markdown })
|
|
1000
|
+
return denoiseResult(finalUrl, status, content)
|
|
853
1001
|
}
|
|
854
1002
|
|
|
855
1003
|
/**
|
|
@@ -961,8 +1109,7 @@ async function recoverDownloadResponse(
|
|
|
961
1109
|
return capResult(finalUrl, statusCode, { kind: 'text', content })
|
|
962
1110
|
}
|
|
963
1111
|
if (!denoise) return capResult(finalUrl, statusCode, { kind: 'html', content })
|
|
964
|
-
|
|
965
|
-
return capResult(finalUrl, statusCode, { kind: 'text', content: markdown })
|
|
1112
|
+
return denoiseResult(finalUrl, statusCode, content)
|
|
966
1113
|
}
|
|
967
1114
|
|
|
968
1115
|
/** Time kept back from the PDF page walk for layout and result assembly. */
|
|
@@ -1077,6 +1224,20 @@ async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
|
|
|
1077
1224
|
}
|
|
1078
1225
|
}
|
|
1079
1226
|
|
|
1227
|
+
/**
|
|
1228
|
+
* Denoise an HTML body into a capped text result. The synchronous pipeline
|
|
1229
|
+
* builds several DOMs per page, so its input is bounded on EVERY path that
|
|
1230
|
+
* reaches it (live render, unrenderable-loader fallback, refetch): an
|
|
1231
|
+
* uncapped 5MB API-reference page needs >4GB of heap. A cut input flags
|
|
1232
|
+
* the result truncated.
|
|
1233
|
+
*/
|
|
1234
|
+
function denoiseResult(url: string, statusCode: number, html: string): WebFetchResult {
|
|
1235
|
+
const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
|
|
1236
|
+
const { markdown } = htmlToMarkdown(bounded, url)
|
|
1237
|
+
const result = capResult(url, statusCode, { kind: 'text', content: markdown })
|
|
1238
|
+
return { ...result, truncated: result.truncated || bounded !== html }
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1080
1241
|
/** Apply the decoded-body cap and flag the cut. */
|
|
1081
1242
|
function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text'; content: string }): WebFetchResult {
|
|
1082
1243
|
const truncated = body.content.length > MAX_BODY_CHARS
|