dsh-web-fetch-playwright 0.2.15 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/lib/client.js +46 -21
- package/lib/index.js +1001 -84
- package/lib/types/autoscroll.d.ts +277 -0
- package/lib/types/challenge.d.ts +23 -9
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -3
- package/lib/types/index.d.ts +3 -1
- package/lib/types/markdown.d.ts +18 -7
- package/lib/types/provider.d.ts +25 -0
- package/lib/types/strip.d.ts +59 -0
- package/package.json +1 -1
- package/src/autoscroll.ts +653 -0
- package/src/challenge.ts +36 -11
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -3
- package/src/index.ts +5 -0
- package/src/markdown.ts +371 -14
- package/src/provider.ts +57 -68
- package/src/strip.ts +72 -0
|
@@ -0,0 +1,653 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deep capture: a bounded in-page scroll walk that defeats viewport
|
|
3
|
+
* virtualization before the document is serialized.
|
|
4
|
+
*
|
|
5
|
+
* WHY. Long "feed-shaped" pages — Discourse topic streams (linux.do,
|
|
6
|
+
* meta.discourse.org), comment threads, timelines — increasingly render only
|
|
7
|
+
* the posts near the current scroll position. Discourse is the measured
|
|
8
|
+
* badcase: at rest the live DOM holds the posts within roughly one viewport
|
|
9
|
+
* of the top, while every post outside that window is a height-only
|
|
10
|
+
* placeholder that keeps the post's identity but none of its content:
|
|
11
|
+
*
|
|
12
|
+
* ```html
|
|
13
|
+
* <div class="post-stream--cloaked" data-post-number="1" id="post_1"
|
|
14
|
+
* style="height:297.9px !important;…">
|
|
15
|
+
* <h2 class="sr-only">由 Diver-QC 于 2 小时前 发布</h2>
|
|
16
|
+
* </div>
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* The placeholder carries the SAME id the rendered `<article id="post_1">`
|
|
20
|
+
* has — every mainstream virtualized list needs some stable identity to
|
|
21
|
+
* recycle its rows. A fetch that renders, never scrolls, and serializes one
|
|
22
|
+
* instant of that DOM returns the skeleton (the screen-reader-only "posted
|
|
23
|
+
* by" headings survive the pipeline as if they were content) while the reply
|
|
24
|
+
* bodies silently vanish. Scrolling to the bottom once does not help either:
|
|
25
|
+
* the window MOVES — measured on the same page, scrolling to the bottom left
|
|
26
|
+
* only posts 31–40 rendered and re-cloaked 1–30. And the denoise pipeline's
|
|
27
|
+
* own coverage gate cannot notice the loss: the page's "visible text" it
|
|
28
|
+
* measures against is the same windowed fragment as the extraction, so the
|
|
29
|
+
* ratio always passes. Only walk depth can see what a resting viewport hides.
|
|
30
|
+
*
|
|
31
|
+
* The SAME pathology, one level deeper, hits app-shell document editors —
|
|
32
|
+
* the second measured badcase is a Feishu wiki page (bocha-ai.feishu.cn).
|
|
33
|
+
* There the body never scrolls at all: html/body are locked at exactly one
|
|
34
|
+
* viewport and the document scrolls inside `div.bear-web-x-container`
|
|
35
|
+
* (measured: documentElement.scrollHeight 720 == innerHeight,
|
|
36
|
+
* window.scrollTo(0, 800) left scrollY at 0, while the container held 4534px
|
|
37
|
+
* against a 656px client). Its blocks are virtualized too, with two twists
|
|
38
|
+
* over Discourse: blocks carry `data-block-id` and NO id, and off-window
|
|
39
|
+
* blocks are REMOVED outright — the whole un-rendered tail is one anonymous
|
|
40
|
+
* `.bear-virtual-renderUnit-placeholder` spacer with an explicit height — so
|
|
41
|
+
* there is neither a window scrollbar to walk nor a degraded-in-place row to
|
|
42
|
+
* swap back. A fetch therefore returns the statically rendered first screen.
|
|
43
|
+
*
|
|
44
|
+
* HOW. Between the settle and the serialization the provider runs one
|
|
45
|
+
* in-page async walk ({@link deepCaptureInPage}, a typed function evaluated
|
|
46
|
+
* by source):
|
|
47
|
+
*
|
|
48
|
+
* 1. Resolve the walk TARGET: the window when the document itself scrolls
|
|
49
|
+
* (the Discourse shape), else the page's dominant inner scroll container
|
|
50
|
+
* (the app-shell shape) — the element with the largest scrollable
|
|
51
|
+
* distance among computed `overflow-y: auto|scroll` elements, gated at
|
|
52
|
+
* half a viewport so dropdown-sized scrollers never trigger a walk. A
|
|
53
|
+
* page with neither is left alone (the legacy capture).
|
|
54
|
+
* 2. STASH every keyed content element — key = id, else a namespaced
|
|
55
|
+
* `data-block-id` — as a detached clone (the RICHEST sighting per key
|
|
56
|
+
* wins), together with its nearest keyed siblings: where it sits in its
|
|
57
|
+
* own list, which is what a removed row is later spliced back next to.
|
|
58
|
+
* 3. WALK down from the top in smooth 0.7-viewport steps (300ms motion +
|
|
59
|
+
* 300ms quiet settle — the rest a feed engine's debounced uncloak needs),
|
|
60
|
+
* collecting after each, until the walk can hold AT THE BOTTOM through a
|
|
61
|
+
* {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
|
|
62
|
+
* (growth restarts the clock — that is what pages load-more sentinels),
|
|
63
|
+
* or the step/budget caps close it.
|
|
64
|
+
*
|
|
65
|
+
* A MutationObserver watches for the walk's whole reason to exist: the
|
|
66
|
+
* page REACTING to scroll by adding or removing elements. Most of the web
|
|
67
|
+
* never does (a long static article only costs latency here), so after
|
|
68
|
+
* {@link SCROLL_INERT_STEPS} careful steps with no reaction the walk
|
|
69
|
+
* probes the rest with one jump to the bottom. Any reaction after that
|
|
70
|
+
* jump — an uncloak, a re-cloak, a load-more append — means the region
|
|
71
|
+
* jumped over may hold content that never rendered, so the walk rewinds
|
|
72
|
+
* to where it jumped from and finishes carefully. Correctness never
|
|
73
|
+
* rests on the warm-up guess; only the cost of a scroll-inert page does.
|
|
74
|
+
* 4. MERGE into a structural clone (`document.cloneNode(true)`: no
|
|
75
|
+
* serialize/re-parse round trip — a runtime-rewritten data script with a
|
|
76
|
+
* literal `</script>` in its text cannot corrupt it — and, having no
|
|
77
|
+
* browsing context, it runs no scripts, upgrades no custom elements and
|
|
78
|
+
* loads no frames):
|
|
79
|
+
* - SPLICE (container walks): stashed keys from inside the scroll
|
|
80
|
+
* container that the clone no longer holds were REMOVED by the engine
|
|
81
|
+
* (the Feishu shape — its placeholders are anonymous) and go back next
|
|
82
|
+
* to their recorded sibling, resolved iteratively so a removed run
|
|
83
|
+
* chains off whichever end survived; a key that rode in inside its
|
|
84
|
+
* spliced parent is already present and skipped. Placement is
|
|
85
|
+
* structural, so it is right whatever order the walk SAW rows in (an
|
|
86
|
+
* anchored open starts mid-document).
|
|
87
|
+
* - SWAP: every keyed element emptier than its snapshot (fewer child
|
|
88
|
+
* elements, or meaningfully less text — the Discourse placeholder) is
|
|
89
|
+
* replaced by that snapshot, except containers (ancestors of another
|
|
90
|
+
* element being swapped), which are assembled from those descendants
|
|
91
|
+
* instead: a container's own snapshot is one window's partial view.
|
|
92
|
+
* The clone is then stripped of scripting elements and serialized; the
|
|
93
|
+
* provider reads that string instead of the live page. The live document
|
|
94
|
+
* is never rewritten, so no late scroll/rAF/timer reaction can touch the
|
|
95
|
+
* capture either.
|
|
96
|
+
*
|
|
97
|
+
* Failure is always silent degradation: a page without `evaluate`, a script
|
|
98
|
+
* error, or an engine that rejects the walk just keeps the legacy
|
|
99
|
+
* single-instant capture — a walk can only add content, never fail a fetch.
|
|
100
|
+
* A walk that restored nothing returns no HTML, and the provider reads the
|
|
101
|
+
* live page exactly as before.
|
|
102
|
+
*
|
|
103
|
+
* The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
|
|
104
|
+
* cloaked placeholders) but never for non-HTML documents. It is bounded by
|
|
105
|
+
* the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
|
|
106
|
+
* and always by the remaining fetch deadline minus a decode reserve — the
|
|
107
|
+
* provider's abort closes the tab, which rejects the pending evaluation.
|
|
108
|
+
*
|
|
109
|
+
* @module dsh-web-fetch-playwright/autoscroll
|
|
110
|
+
*/
|
|
111
|
+
|
|
112
|
+
import { stripScriptingElementsIn } from './strip.ts'
|
|
113
|
+
import type { PlaywrightPage } from './types.ts'
|
|
114
|
+
|
|
115
|
+
/** How far each step scrolls, as a share of the viewport height. */
|
|
116
|
+
export const SCROLL_STEP_RATIO = 0.7
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* How long one step's smooth scroll motion takes (ms). The step ANIMATES
|
|
120
|
+
* down its distance (time-interpolated ticks at ~60fps) instead of
|
|
121
|
+
* teleporting: feed engines' uncloak/re-cloak logic is built around a
|
|
122
|
+
* continuous stream of scroll events, and background-tab timer throttling
|
|
123
|
+
* self-corrects because every tick derives its position from the clock.
|
|
124
|
+
*/
|
|
125
|
+
export const SCROLL_STEP_MOTION_MS = 300
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Per-step settle (ms): the QUIET window after the motion, where a feed
|
|
129
|
+
* engine's debounced uncloak fires (measured: stepping without a rest
|
|
130
|
+
* starves Discourse's cloak manager, which renders nothing new until the
|
|
131
|
+
* scroll rests ~300ms) and the freshly rendered rows become stashable.
|
|
132
|
+
*/
|
|
133
|
+
export const SCROLL_STEP_SETTLE_MS = 300
|
|
134
|
+
|
|
135
|
+
/** How often the walk re-checks the height while holding at the bottom. */
|
|
136
|
+
export const SCROLL_BOTTOM_POLL_MS = 450
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* How long at the bottom, with no height growth, before the walk concludes
|
|
140
|
+
* the page is complete. Real feed engines fetch their next chunk with
|
|
141
|
+
* network latency (measured on linux.do: a pending chunk landed well after
|
|
142
|
+
* two 450ms checks), so this is sized over a slow round trip.
|
|
143
|
+
*/
|
|
144
|
+
export const SCROLL_BOTTOM_PATIENCE_MS = 2_500
|
|
145
|
+
|
|
146
|
+
/**
|
|
147
|
+
* Careful steps without any DOM reaction before the walk treats the page as
|
|
148
|
+
* scroll-inert and probes the rest with a single jump (rewinding if the
|
|
149
|
+
* jump provokes a reaction). Two steps cover ~1.4 viewports — past the
|
|
150
|
+
* render window edge of every measured virtualizer.
|
|
151
|
+
*/
|
|
152
|
+
export const SCROLL_INERT_STEPS = 2
|
|
153
|
+
|
|
154
|
+
/** Hard cap on scroll steps regardless of budget (unbounded feeds). */
|
|
155
|
+
export const SCROLL_MAX_STEPS = 50
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Cap on stashed keys — pathological id counts cannot pin memory. Sized for
|
|
159
|
+
* long topics: a Discourse post carries several ids of its own (post,
|
|
160
|
+
* heading, avatar, …), so a few hundred posts already need a thousand-plus.
|
|
161
|
+
* {@link SCROLL_STASH_MAX_CHARS} is the real memory bound.
|
|
162
|
+
*/
|
|
163
|
+
export const SCROLL_STASH_MAX_ENTRIES = 2_000
|
|
164
|
+
|
|
165
|
+
/** Cap on stashed HTML characters (aligned with the pipeline input cap). */
|
|
166
|
+
export const SCROLL_STASH_MAX_CHARS = 2_000_000
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Cap on ONE stashed element's HTML. A virtualized row is a few KB; an
|
|
170
|
+
* element tens of KB big is a CONTAINER (measured on linux.do: six nested
|
|
171
|
+
* wrappers each embed the whole post stream — main 164KB ⊃
|
|
172
|
+
* main-outlet-wrapper 156KB ⊃ … — plus data-preloaded 300KB and
|
|
173
|
+
* svg-sprites 277KB) or a data island — never a row the merge could
|
|
174
|
+
* rehydrate. Such keys are remembered and never serialized again.
|
|
175
|
+
*/
|
|
176
|
+
export const SCROLL_STASH_MAX_ENTRY_CHARS = 64_000
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Tail of the fetch budget the walk never spends, keeping room for the
|
|
180
|
+
* content read and the synchronous denoise decode.
|
|
181
|
+
*/
|
|
182
|
+
export const SCROLL_FINISH_RESERVE_MS = 4_000
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* A walk budget below this cannot complete even the first window plus one
|
|
186
|
+
* stability check, so the walk is skipped outright.
|
|
187
|
+
*/
|
|
188
|
+
export const SCROLL_MIN_BUDGET_MS = 500
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Floor (px) on the scrollable distance an inner container must offer
|
|
192
|
+
* before the walk targets it, when the document itself does not scroll.
|
|
193
|
+
*/
|
|
194
|
+
export const SCROLL_CONTAINER_MIN_RANGE_PX = 300
|
|
195
|
+
|
|
196
|
+
/** The viewport share an inner container's scrollable distance must reach. */
|
|
197
|
+
export const SCROLL_CONTAINER_MIN_RANGE_RATIO = 0.5
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Text shortfall (chars) versus the snapshot that marks an element as
|
|
201
|
+
* degraded even when its child-element count held — below it, live text
|
|
202
|
+
* (relative timestamps, counters) merely ticked.
|
|
203
|
+
*/
|
|
204
|
+
export const SCROLL_TEXT_DEGRADE_TOLERANCE = 16
|
|
205
|
+
|
|
206
|
+
/** What one completed (or skipped) walk reports back, for tests and logs. */
|
|
207
|
+
export interface DeepCaptureStats {
|
|
208
|
+
/** Scroll steps taken (0 when the walk skipped or had no room). */
|
|
209
|
+
steps: number
|
|
210
|
+
/** Distinct keys stashed across the walk. */
|
|
211
|
+
stashed: number
|
|
212
|
+
/** Degraded elements swapped for their snapshot in the capture. */
|
|
213
|
+
rehydrated: number
|
|
214
|
+
/** Removed keyed elements spliced back (container walks only). */
|
|
215
|
+
spliced?: number
|
|
216
|
+
/**
|
|
217
|
+
* Whether the page reacted to scrolling (added/removed elements). False
|
|
218
|
+
* means the walk took the scroll-inert fast path to the bottom.
|
|
219
|
+
*/
|
|
220
|
+
reactive?: boolean
|
|
221
|
+
/** Walk duration (ms). */
|
|
222
|
+
elapsedMs?: number
|
|
223
|
+
/** Walk-target height (px) — document or inner scroll container. */
|
|
224
|
+
height?: number
|
|
225
|
+
/** Set when the walk skipped without scrolling, and why. */
|
|
226
|
+
skipped?: 'not-scrollable' | 'no-budget'
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/** A finished walk: its stats, plus the merged capture when it restored anything. */
|
|
230
|
+
export interface DeepCapture extends DeepCaptureStats {
|
|
231
|
+
/**
|
|
232
|
+
* The merged, scripting-stripped document serialized like
|
|
233
|
+
* `page.content()` — present only when the walk restored something; the
|
|
234
|
+
* provider then reads it instead of the live page.
|
|
235
|
+
*/
|
|
236
|
+
html?: string
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/** The walk's knobs, handed to {@link deepCaptureInPage} as JSON. */
|
|
240
|
+
export interface DeepCaptureParams {
|
|
241
|
+
budgetMs: number
|
|
242
|
+
stepRatio: number
|
|
243
|
+
stepMotionMs: number
|
|
244
|
+
stepSettleMs: number
|
|
245
|
+
bottomPollMs: number
|
|
246
|
+
bottomPatienceMs: number
|
|
247
|
+
inertSteps: number
|
|
248
|
+
maxSteps: number
|
|
249
|
+
stashMaxEntries: number
|
|
250
|
+
stashMaxChars: number
|
|
251
|
+
stashMaxEntryChars: number
|
|
252
|
+
containerMinRangePx: number
|
|
253
|
+
containerMinRatio: number
|
|
254
|
+
textDegradeTolerance: number
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* The in-page walk. Evaluated BY SOURCE (`fn.toString()` composed into an
|
|
259
|
+
* expression by {@link deepCaptureScript}), so it must stay self-contained:
|
|
260
|
+
* every helper is nested, every knob arrives in `p`, and the one shared
|
|
261
|
+
* routine — the scripting strip — arrives as an argument. Typed here so the
|
|
262
|
+
* compiler checks what used to be an unchecked string.
|
|
263
|
+
*
|
|
264
|
+
* @param p - the walk's knobs.
|
|
265
|
+
* @param strip - {@link stripScriptingElementsIn}, applied to the merged clone.
|
|
266
|
+
* @returns the walk's stats, plus the merged HTML when anything was restored.
|
|
267
|
+
*/
|
|
268
|
+
export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Document) => void): Promise<DeepCapture> {
|
|
269
|
+
const started = Date.now()
|
|
270
|
+
const root = document.documentElement
|
|
271
|
+
const body = document.body
|
|
272
|
+
if (root === null || body === null) return { steps: 0, stashed: 0, rehydrated: 0, skipped: 'not-scrollable' }
|
|
273
|
+
if (p.budgetMs <= 0) return { steps: 0, stashed: 0, rehydrated: 0, skipped: 'no-budget' }
|
|
274
|
+
|
|
275
|
+
const KEYED = '[id], [data-block-id]'
|
|
276
|
+
const SVG_NS = 'http://www.w3.org/2000/svg'
|
|
277
|
+
const sleep = (ms: number): Promise<void> => new Promise(resolve => { setTimeout(resolve, ms) })
|
|
278
|
+
|
|
279
|
+
// ── Target: the window, or the dominant inner scroll container ─────────
|
|
280
|
+
// The app-shell shape (Feishu docs, Notion-class editors) locks html/body
|
|
281
|
+
// at one viewport and scrolls inside one container, where window-only
|
|
282
|
+
// primitives are no-ops. Largest scrollable distance wins; document order
|
|
283
|
+
// keeps the OUTERMOST on ties.
|
|
284
|
+
function findScrollContainer(host: HTMLElement): Element | null {
|
|
285
|
+
const minRange = Math.max(p.containerMinRangePx, window.innerHeight * p.containerMinRatio)
|
|
286
|
+
let best: Element | null = null
|
|
287
|
+
let bestRange = 0
|
|
288
|
+
for (const el of Array.from(host.querySelectorAll('*'))) {
|
|
289
|
+
const range = el.scrollHeight - el.clientHeight
|
|
290
|
+
if (range < minRange || range <= bestRange) continue
|
|
291
|
+
const overflowY = getComputedStyle(el).overflowY
|
|
292
|
+
if (overflowY === 'auto' || overflowY === 'scroll') { best = el; bestRange = range }
|
|
293
|
+
}
|
|
294
|
+
return best
|
|
295
|
+
}
|
|
296
|
+
const documentScrolls = root.scrollHeight > window.innerHeight + 4
|
|
297
|
+
const scroller = documentScrolls ? null : findScrollContainer(body)
|
|
298
|
+
if (!documentScrolls && scroller === null) return { steps: 0, stashed: 0, rehydrated: 0, skipped: 'not-scrollable' }
|
|
299
|
+
const viewportH = (): number => scroller === null ? window.innerHeight : scroller.clientHeight
|
|
300
|
+
const scrollH = (): number => scroller === null ? root.scrollHeight : scroller.scrollHeight
|
|
301
|
+
const curY = (): number => scroller === null ? window.scrollY : scroller.scrollTop
|
|
302
|
+
const atBottom = (): boolean => curY() + viewportH() >= scrollH() - 2
|
|
303
|
+
function scrollToY(y: number): void {
|
|
304
|
+
if (scroller === null) window.scrollTo({ top: y, behavior: 'instant' })
|
|
305
|
+
else scroller.scrollTop = y
|
|
306
|
+
}
|
|
307
|
+
// One step's smooth descent, positioned from the wall clock each tick so
|
|
308
|
+
// background-tab throttling degrades to coarser jumps along the same path.
|
|
309
|
+
function smoothScrollBy(dy: number): Promise<void> {
|
|
310
|
+
return new Promise(resolve => {
|
|
311
|
+
const fromY = curY()
|
|
312
|
+
const t0 = Date.now()
|
|
313
|
+
const tick = (): void => {
|
|
314
|
+
const progress = (Date.now() - t0) / p.stepMotionMs
|
|
315
|
+
if (progress >= 1 || (curY() !== fromY && atBottom())) {
|
|
316
|
+
scrollToY(fromY + dy)
|
|
317
|
+
resolve()
|
|
318
|
+
return
|
|
319
|
+
}
|
|
320
|
+
scrollToY(fromY + Math.round(dy * progress))
|
|
321
|
+
setTimeout(tick, 16)
|
|
322
|
+
}
|
|
323
|
+
tick()
|
|
324
|
+
})
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// ── Identity ────────────────────────────────────────────────────────────
|
|
328
|
+
// id, else data-block-id (namespaced so the two can never collide): app-
|
|
329
|
+
// shell editors key their blocks that way and carry no id at all.
|
|
330
|
+
function keyOf(el: Element): string | null {
|
|
331
|
+
if (el.id !== '') return el.id
|
|
332
|
+
const block = el.getAttribute('data-block-id')
|
|
333
|
+
return block !== null && block !== '' ? `data-block-id:${block}` : null
|
|
334
|
+
}
|
|
335
|
+
// Nearest keyed sibling in one direction: where the element sits in its
|
|
336
|
+
// own list — the splice anchor if the engine removes it later.
|
|
337
|
+
function siblingKey(el: Element, dir: 'previousElementSibling' | 'nextElementSibling'): string | null {
|
|
338
|
+
for (let sib = el[dir]; sib !== null; sib = sib[dir]) {
|
|
339
|
+
const key = keyOf(sib)
|
|
340
|
+
if (key !== null) return key
|
|
341
|
+
}
|
|
342
|
+
return null
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
// ── Reactivity ─────────────────────────────────────────────────────────
|
|
346
|
+
// Element additions/removals are a virtualizer's (or a load-more's)
|
|
347
|
+
// signature; attribute churn (lazy images, sticky-header classes) and
|
|
348
|
+
// text ticks are not, nor are injected scripting/metadata elements.
|
|
349
|
+
const INERT_TAGS = new Set(['SCRIPT', 'STYLE', 'LINK', 'META', 'NOSCRIPT', 'IFRAME', 'TEMPLATE'])
|
|
350
|
+
let reactions = 0
|
|
351
|
+
const counts = (nodes: NodeList): boolean => {
|
|
352
|
+
for (const node of Array.from(nodes)) {
|
|
353
|
+
if (node.nodeType === 1 && !INERT_TAGS.has(node.nodeName)) return true
|
|
354
|
+
}
|
|
355
|
+
return false
|
|
356
|
+
}
|
|
357
|
+
const tally = (records: MutationRecord[]): void => {
|
|
358
|
+
for (const record of records) {
|
|
359
|
+
if (counts(record.addedNodes) || counts(record.removedNodes)) reactions++
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
const observer = new MutationObserver(tally)
|
|
363
|
+
observer.observe(body, { childList: true, subtree: true })
|
|
364
|
+
const reacted = (): boolean => { tally(observer.takeRecords()); return reactions > 0 }
|
|
365
|
+
|
|
366
|
+
// ── Stash ──────────────────────────────────────────────────────────────
|
|
367
|
+
interface Snapshot {
|
|
368
|
+
node: Element // detached clone, owned by stashDoc
|
|
369
|
+
els: number
|
|
370
|
+
text: number
|
|
371
|
+
len: number
|
|
372
|
+
prev: string | null
|
|
373
|
+
next: string | null
|
|
374
|
+
inScroller: boolean
|
|
375
|
+
}
|
|
376
|
+
// Clones live in a document without a browsing context: no scripts run,
|
|
377
|
+
// no custom elements upgrade, nothing loads.
|
|
378
|
+
const stashDoc = document.implementation.createHTMLDocument('')
|
|
379
|
+
const stash = new Map<string, Snapshot>()
|
|
380
|
+
const oversized = new Set<string>()
|
|
381
|
+
let stashChars = 0
|
|
382
|
+
const host: HTMLElement = body
|
|
383
|
+
function collect(): void {
|
|
384
|
+
for (const el of Array.from(host.querySelectorAll(KEYED))) {
|
|
385
|
+
const key = keyOf(el)
|
|
386
|
+
if (key === null || oversized.has(key) || el.namespaceURI === SVG_NS) continue
|
|
387
|
+
const els = el.childElementCount
|
|
388
|
+
const text = (el.textContent ?? '').length
|
|
389
|
+
if (els === 0 && text === 0) continue
|
|
390
|
+
const prev = stash.get(key)
|
|
391
|
+
if (prev !== undefined) {
|
|
392
|
+
// Neighbors refresh on every sighting: any observed sibling is a
|
|
393
|
+
// true one, and later windows see more of the list.
|
|
394
|
+
prev.prev = siblingKey(el, 'previousElementSibling') ?? prev.prev
|
|
395
|
+
prev.next = siblingKey(el, 'nextElementSibling') ?? prev.next
|
|
396
|
+
// Richest sighting wins; a rendered row collapsing back to its
|
|
397
|
+
// placeholder never downgrades the stash.
|
|
398
|
+
if (els <= prev.els && text <= prev.text) continue
|
|
399
|
+
} else if (stash.size >= p.stashMaxEntries) {
|
|
400
|
+
continue
|
|
401
|
+
}
|
|
402
|
+
if (text > p.stashMaxEntryChars) { oversized.add(key); continue }
|
|
403
|
+
const len = el.outerHTML.length
|
|
404
|
+
if (len > p.stashMaxEntryChars) { oversized.add(key); continue }
|
|
405
|
+
const freed = prev === undefined ? 0 : prev.len
|
|
406
|
+
if (stashChars - freed + len > p.stashMaxChars) continue
|
|
407
|
+
stashChars += len - freed
|
|
408
|
+
stash.set(key, {
|
|
409
|
+
node: stashDoc.importNode(el, true),
|
|
410
|
+
els,
|
|
411
|
+
text,
|
|
412
|
+
len,
|
|
413
|
+
prev: prev?.prev ?? siblingKey(el, 'previousElementSibling'),
|
|
414
|
+
next: prev?.next ?? siblingKey(el, 'nextElementSibling'),
|
|
415
|
+
inScroller: scroller !== null && scroller.contains(el),
|
|
416
|
+
})
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
// ── Walk ───────────────────────────────────────────────────────────────
|
|
421
|
+
// Stash whatever the (possibly anchored) start renders, then restart from
|
|
422
|
+
// the top so the walk covers the whole target.
|
|
423
|
+
collect()
|
|
424
|
+
if (curY() > 0) {
|
|
425
|
+
scrollToY(0)
|
|
426
|
+
await sleep(p.stepSettleMs)
|
|
427
|
+
collect()
|
|
428
|
+
}
|
|
429
|
+
// Only reactions to the walk's own scrolling count from here on.
|
|
430
|
+
tally(observer.takeRecords())
|
|
431
|
+
reactions = 0
|
|
432
|
+
|
|
433
|
+
let steps = 0
|
|
434
|
+
let lastHeight = scrollH()
|
|
435
|
+
let lastChangeAt = Date.now()
|
|
436
|
+
let careful = false // the page reacted: never jump again
|
|
437
|
+
let jumpedFrom: number | null = null
|
|
438
|
+
const noteHeight = (): void => {
|
|
439
|
+
const height = scrollH()
|
|
440
|
+
if (height !== lastHeight) { lastHeight = height; lastChangeAt = Date.now() }
|
|
441
|
+
}
|
|
442
|
+
try {
|
|
443
|
+
while (steps < p.maxSteps && Date.now() - started < p.budgetMs) {
|
|
444
|
+
if (jumpedFrom !== null && reacted()) {
|
|
445
|
+
// The jump provoked a reaction: what lies between jumpedFrom and the
|
|
446
|
+
// bottom may never have rendered. Go back and walk it carefully.
|
|
447
|
+
scrollToY(jumpedFrom)
|
|
448
|
+
jumpedFrom = null
|
|
449
|
+
careful = true
|
|
450
|
+
await sleep(p.stepSettleMs)
|
|
451
|
+
collect()
|
|
452
|
+
noteHeight()
|
|
453
|
+
continue
|
|
454
|
+
}
|
|
455
|
+
if (atBottom()) {
|
|
456
|
+
// Bottom patience: growth at any poll restarts the clock (the walk
|
|
457
|
+
// then keeps scrolling into what loaded); completion is only trusted
|
|
458
|
+
// after the full window passes with the height unchanged.
|
|
459
|
+
await sleep(p.bottomPollMs)
|
|
460
|
+
collect()
|
|
461
|
+
const before = lastHeight
|
|
462
|
+
noteHeight()
|
|
463
|
+
if (lastHeight === before && Date.now() - lastChangeAt >= p.bottomPatienceMs) break
|
|
464
|
+
continue
|
|
465
|
+
}
|
|
466
|
+
if (!careful && jumpedFrom === null) {
|
|
467
|
+
if (reacted()) {
|
|
468
|
+
careful = true
|
|
469
|
+
} else if (steps >= p.inertSteps) {
|
|
470
|
+
// Scroll-inert so far: probe the rest in one jump. The jump counts
|
|
471
|
+
// as a change, so a load-more it triggers gets the full patience.
|
|
472
|
+
jumpedFrom = curY()
|
|
473
|
+
steps++
|
|
474
|
+
scrollToY(scrollH())
|
|
475
|
+
lastChangeAt = Date.now()
|
|
476
|
+
await sleep(p.stepSettleMs)
|
|
477
|
+
collect()
|
|
478
|
+
noteHeight()
|
|
479
|
+
continue
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
steps++
|
|
483
|
+
await smoothScrollBy(Math.max(1, Math.round(viewportH() * p.stepRatio)))
|
|
484
|
+
collect()
|
|
485
|
+
await sleep(p.stepSettleMs)
|
|
486
|
+
collect()
|
|
487
|
+
noteHeight()
|
|
488
|
+
}
|
|
489
|
+
} finally {
|
|
490
|
+
observer.disconnect()
|
|
491
|
+
}
|
|
492
|
+
const reactive = reacted()
|
|
493
|
+
const finalHeight = scrollH()
|
|
494
|
+
|
|
495
|
+
// ── Merge into a structural clone (synchronous from here on) ──────────
|
|
496
|
+
const fresh = document.cloneNode(true) as Document
|
|
497
|
+
const present = new Map<string, Element>()
|
|
498
|
+
const index = (node: Element): void => {
|
|
499
|
+
const key = keyOf(node)
|
|
500
|
+
if (key !== null && !present.has(key)) present.set(key, node)
|
|
501
|
+
for (const el of Array.from(node.querySelectorAll(KEYED))) {
|
|
502
|
+
const inner = keyOf(el)
|
|
503
|
+
if (inner !== null && !present.has(inner)) present.set(inner, el)
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
if (fresh.documentElement !== null) index(fresh.documentElement)
|
|
507
|
+
|
|
508
|
+
// SPLICE (container walks): absent keys go back beside a surviving
|
|
509
|
+
// sibling; each pass can anchor the next, so a removed run chains off
|
|
510
|
+
// whichever end survived. Keys with no surviving relative stay out.
|
|
511
|
+
let spliced = 0
|
|
512
|
+
if (scroller !== null) {
|
|
513
|
+
let pending = Array.from(stash.keys()).filter(key => stash.get(key)!.inScroller && !present.has(key))
|
|
514
|
+
let progressed = true
|
|
515
|
+
while (pending.length > 0 && progressed) {
|
|
516
|
+
progressed = false
|
|
517
|
+
const unresolved: string[] = []
|
|
518
|
+
for (const key of pending) {
|
|
519
|
+
if (present.has(key)) { progressed = true; continue } // rode in with its parent
|
|
520
|
+
const snap = stash.get(key)!
|
|
521
|
+
const after = snap.prev === null ? undefined : present.get(snap.prev)
|
|
522
|
+
const before = snap.next === null ? undefined : present.get(snap.next)
|
|
523
|
+
if (after === undefined && before === undefined) { unresolved.push(key); continue }
|
|
524
|
+
const node = fresh.importNode(snap.node, true)
|
|
525
|
+
if (after !== undefined) after.after(node)
|
|
526
|
+
else before!.before(node)
|
|
527
|
+
index(node)
|
|
528
|
+
spliced++
|
|
529
|
+
progressed = true
|
|
530
|
+
}
|
|
531
|
+
pending = unresolved
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
// SWAP degraded leaves for their snapshots; their ancestors are
|
|
536
|
+
// containers, assembled from them rather than swapped wholesale.
|
|
537
|
+
const degraded: Array<{ el: Element; snap: Snapshot }> = []
|
|
538
|
+
for (const el of Array.from(fresh.querySelectorAll(KEYED))) {
|
|
539
|
+
const key = keyOf(el)
|
|
540
|
+
const snap = key === null ? undefined : stash.get(key)
|
|
541
|
+
if (snap === undefined || el.namespaceURI === SVG_NS) continue
|
|
542
|
+
if (el.childElementCount < snap.els || snap.text - (el.textContent ?? '').length > p.textDegradeTolerance) {
|
|
543
|
+
degraded.push({ el, snap })
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
const containers = new Set<Element>()
|
|
547
|
+
for (const { el } of degraded) {
|
|
548
|
+
for (let up = el.parentElement; up !== null && !containers.has(up); up = up.parentElement) containers.add(up)
|
|
549
|
+
}
|
|
550
|
+
let rehydrated = 0
|
|
551
|
+
for (const { el, snap } of degraded) {
|
|
552
|
+
if (containers.has(el)) continue
|
|
553
|
+
el.replaceWith(fresh.importNode(snap.node, true))
|
|
554
|
+
rehydrated++
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
const stats: DeepCapture = {
|
|
558
|
+
steps,
|
|
559
|
+
stashed: stash.size,
|
|
560
|
+
rehydrated,
|
|
561
|
+
spliced,
|
|
562
|
+
reactive,
|
|
563
|
+
elapsedMs: Date.now() - started,
|
|
564
|
+
height: finalHeight,
|
|
565
|
+
}
|
|
566
|
+
if ((rehydrated > 0 || spliced > 0) && fresh.documentElement !== null) {
|
|
567
|
+
strip(fresh)
|
|
568
|
+
// page.content()'s own serialization: doctype, then the root element.
|
|
569
|
+
const doctype = fresh.doctype === null ? '' : new XMLSerializer().serializeToString(fresh.doctype)
|
|
570
|
+
stats.html = doctype + fresh.documentElement.outerHTML
|
|
571
|
+
}
|
|
572
|
+
return stats
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
/**
|
|
576
|
+
* The walk's knobs for one fetch: the module constants plus its budget.
|
|
577
|
+
*
|
|
578
|
+
* @param budgetMs - wall-clock budget the walk may spend inside the page.
|
|
579
|
+
*/
|
|
580
|
+
export function deepCaptureParams(budgetMs: number): DeepCaptureParams {
|
|
581
|
+
return {
|
|
582
|
+
budgetMs: Math.max(0, Math.round(budgetMs)),
|
|
583
|
+
stepRatio: SCROLL_STEP_RATIO,
|
|
584
|
+
stepMotionMs: SCROLL_STEP_MOTION_MS,
|
|
585
|
+
stepSettleMs: SCROLL_STEP_SETTLE_MS,
|
|
586
|
+
bottomPollMs: SCROLL_BOTTOM_POLL_MS,
|
|
587
|
+
bottomPatienceMs: SCROLL_BOTTOM_PATIENCE_MS,
|
|
588
|
+
inertSteps: SCROLL_INERT_STEPS,
|
|
589
|
+
maxSteps: SCROLL_MAX_STEPS,
|
|
590
|
+
stashMaxEntries: SCROLL_STASH_MAX_ENTRIES,
|
|
591
|
+
stashMaxChars: SCROLL_STASH_MAX_CHARS,
|
|
592
|
+
stashMaxEntryChars: SCROLL_STASH_MAX_ENTRY_CHARS,
|
|
593
|
+
containerMinRangePx: SCROLL_CONTAINER_MIN_RANGE_PX,
|
|
594
|
+
containerMinRatio: SCROLL_CONTAINER_MIN_RANGE_RATIO,
|
|
595
|
+
textDegradeTolerance: SCROLL_TEXT_DEGRADE_TOLERANCE,
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Compose the evaluate expression: the in-page walk applied to `params` and
|
|
601
|
+
* the shared scripting strip, both by source.
|
|
602
|
+
*
|
|
603
|
+
* @param params - the walk's knobs (see {@link deepCaptureParams}).
|
|
604
|
+
* @returns the expression string to pass to `page.evaluate`.
|
|
605
|
+
*/
|
|
606
|
+
export function deepCaptureExpression(params: DeepCaptureParams): string {
|
|
607
|
+
return `/* dsh-deep-capture */ (${deepCaptureInPage.toString()})(${JSON.stringify(params)}, ${stripScriptingElementsIn.toString()})`
|
|
608
|
+
}
|
|
609
|
+
|
|
610
|
+
/** {@link deepCaptureExpression} over {@link deepCaptureParams} — one fetch's walk. */
|
|
611
|
+
export function deepCaptureScript(budgetMs: number): string {
|
|
612
|
+
return deepCaptureExpression(deepCaptureParams(budgetMs))
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
/**
|
|
616
|
+
* Run the deep-capture walk in the page, degrading silently to the legacy
|
|
617
|
+
* single-instant capture on any failure: a page without `evaluate`, a budget
|
|
618
|
+
* too small to act, a rejected evaluation (the fetch deadline closes the tab
|
|
619
|
+
* mid-walk), or a result that is not the stats object all just skip it. A
|
|
620
|
+
* walk can only add content, never fail a fetch.
|
|
621
|
+
*
|
|
622
|
+
* @param page - the fetch's tab.
|
|
623
|
+
* @param budgetMs - wall-clock budget for the in-page walk; budgets under
|
|
624
|
+
* {@link SCROLL_MIN_BUDGET_MS} skip it.
|
|
625
|
+
* @returns the walk's stats (with the merged `html` when it restored
|
|
626
|
+
* anything), or undefined when the walk did not run.
|
|
627
|
+
*/
|
|
628
|
+
export async function deepCaptureWalk(page: PlaywrightPage, budgetMs: number): Promise<DeepCapture | undefined> {
|
|
629
|
+
if (typeof page.evaluate !== 'function') return undefined
|
|
630
|
+
if (budgetMs < SCROLL_MIN_BUDGET_MS) return undefined
|
|
631
|
+
try {
|
|
632
|
+
const result = await page.evaluate(deepCaptureScript(budgetMs))
|
|
633
|
+
if (result === null || typeof result !== 'object') return undefined
|
|
634
|
+
const raw = result as Partial<DeepCapture>
|
|
635
|
+
if (typeof raw.steps !== 'number' || typeof raw.stashed !== 'number' || typeof raw.rehydrated !== 'number') {
|
|
636
|
+
return undefined
|
|
637
|
+
}
|
|
638
|
+
return {
|
|
639
|
+
steps: raw.steps,
|
|
640
|
+
stashed: raw.stashed,
|
|
641
|
+
rehydrated: raw.rehydrated,
|
|
642
|
+
...(typeof raw.spliced === 'number' ? { spliced: raw.spliced } : {}),
|
|
643
|
+
...(typeof raw.reactive === 'boolean' ? { reactive: raw.reactive } : {}),
|
|
644
|
+
...(typeof raw.elapsedMs === 'number' ? { elapsedMs: raw.elapsedMs } : {}),
|
|
645
|
+
...(typeof raw.height === 'number' ? { height: raw.height } : {}),
|
|
646
|
+
...(raw.skipped === 'not-scrollable' || raw.skipped === 'no-budget' ? { skipped: raw.skipped } : {}),
|
|
647
|
+
...(typeof raw.html === 'string' ? { html: raw.html } : {}),
|
|
648
|
+
}
|
|
649
|
+
} catch {
|
|
650
|
+
// A failed walk degrades to the legacy capture; the fetch continues.
|
|
651
|
+
return undefined
|
|
652
|
+
}
|
|
653
|
+
}
|