dsh-web-fetch-playwright 0.2.16 → 0.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,653 @@
1
+ /**
2
+ * Deep capture: a bounded in-page scroll walk that defeats viewport
3
+ * virtualization before the document is serialized.
4
+ *
5
+ * WHY. Long "feed-shaped" pages — Discourse topic streams (linux.do,
6
+ * meta.discourse.org), comment threads, timelines — increasingly render only
7
+ * the posts near the current scroll position. Discourse is the measured
8
+ * badcase: at rest the live DOM holds the posts within roughly one viewport
9
+ * of the top, while every post outside that window is a height-only
10
+ * placeholder that keeps the post's identity but none of its content:
11
+ *
12
+ * ```html
13
+ * <div class="post-stream--cloaked" data-post-number="1" id="post_1"
14
+ * style="height:297.9px !important;…">
15
+ * <h2 class="sr-only">由 Diver-QC 于 2 小时前 发布</h2>
16
+ * </div>
17
+ * ```
18
+ *
19
+ * The placeholder carries the SAME id the rendered `<article id="post_1">`
20
+ * has — every mainstream virtualized list needs some stable identity to
21
+ * recycle its rows. A fetch that renders, never scrolls, and serializes one
22
+ * instant of that DOM returns the skeleton (the screen-reader-only "posted
23
+ * by" headings survive the pipeline as if they were content) while the reply
24
+ * bodies silently vanish. Scrolling to the bottom once does not help either:
25
+ * the window MOVES — measured on the same page, scrolling to the bottom left
26
+ * only posts 31–40 rendered and re-cloaked 1–30. And the denoise pipeline's
27
+ * own coverage gate cannot notice the loss: the page's "visible text" it
28
+ * measures against is the same windowed fragment as the extraction, so the
29
+ * ratio always passes. Only walk depth can see what a resting viewport hides.
30
+ *
31
+ * The SAME pathology, one level deeper, hits app-shell document editors —
32
+ * the second measured badcase is a Feishu wiki page (bocha-ai.feishu.cn).
33
+ * There the body never scrolls at all: html/body are locked at exactly one
34
+ * viewport and the document scrolls inside `div.bear-web-x-container`
35
+ * (measured: documentElement.scrollHeight 720 == innerHeight,
36
+ * window.scrollTo(0, 800) left scrollY at 0, while the container held 4534px
37
+ * against a 656px client). Its blocks are virtualized too, with two twists
38
+ * over Discourse: blocks carry `data-block-id` and NO id, and off-window
39
+ * blocks are REMOVED outright — the whole un-rendered tail is one anonymous
40
+ * `.bear-virtual-renderUnit-placeholder` spacer with an explicit height — so
41
+ * there is neither a window scrollbar to walk nor a degraded-in-place row to
42
+ * swap back. A fetch therefore returns the statically rendered first screen.
43
+ *
44
+ * HOW. Between the settle and the serialization the provider runs one
45
+ * in-page async walk ({@link deepCaptureInPage}, a typed function evaluated
46
+ * by source):
47
+ *
48
+ * 1. Resolve the walk TARGET: the window when the document itself scrolls
49
+ * (the Discourse shape), else the page's dominant inner scroll container
50
+ * (the app-shell shape) — the element with the largest scrollable
51
+ * distance among computed `overflow-y: auto|scroll` elements, gated at
52
+ * half a viewport so dropdown-sized scrollers never trigger a walk. A
53
+ * page with neither is left alone (the legacy capture).
54
+ * 2. STASH every keyed content element — key = id, else a namespaced
55
+ * `data-block-id` — as a detached clone (the RICHEST sighting per key
56
+ * wins), together with its nearest keyed siblings: where it sits in its
57
+ * own list, which is what a removed row is later spliced back next to.
58
+ * 3. WALK down from the top in smooth 0.7-viewport steps (300ms motion +
59
+ * 300ms quiet settle — the rest a feed engine's debounced uncloak needs),
60
+ * collecting after each, until the walk can hold AT THE BOTTOM through a
61
+ * {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
62
+ * (growth restarts the clock — that is what pages load-more sentinels),
63
+ * or the step/budget caps close it.
64
+ *
65
+ * A MutationObserver watches for the walk's whole reason to exist: the
66
+ * page REACTING to scroll by adding or removing elements. Most of the web
67
+ * never does (a long static article only costs latency here), so after
68
+ * {@link SCROLL_INERT_STEPS} careful steps with no reaction the walk
69
+ * probes the rest with one jump to the bottom. Any reaction after that
70
+ * jump — an uncloak, a re-cloak, a load-more append — means the region
71
+ * jumped over may hold content that never rendered, so the walk rewinds
72
+ * to where it jumped from and finishes carefully. Correctness never
73
+ * rests on the warm-up guess; only the cost of a scroll-inert page does.
74
+ * 4. MERGE into a structural clone (`document.cloneNode(true)`: no
75
+ * serialize/re-parse round trip — a runtime-rewritten data script with a
76
+ * literal `</script>` in its text cannot corrupt it — and, having no
77
+ * browsing context, it runs no scripts, upgrades no custom elements and
78
+ * loads no frames):
79
+ * - SPLICE (container walks): stashed keys from inside the scroll
80
+ * container that the clone no longer holds were REMOVED by the engine
81
+ * (the Feishu shape — its placeholders are anonymous) and go back next
82
+ * to their recorded sibling, resolved iteratively so a removed run
83
+ * chains off whichever end survived; a key that rode in inside its
84
+ * spliced parent is already present and skipped. Placement is
85
+ * structural, so it is right whatever order the walk SAW rows in (an
86
+ * anchored open starts mid-document).
87
+ * - SWAP: every keyed element emptier than its snapshot (fewer child
88
+ * elements, or meaningfully less text — the Discourse placeholder) is
89
+ * replaced by that snapshot, except containers (ancestors of another
90
+ * element being swapped), which are assembled from those descendants
91
+ * instead: a container's own snapshot is one window's partial view.
92
+ * The clone is then stripped of scripting elements and serialized; the
93
+ * provider reads that string instead of the live page. The live document
94
+ * is never rewritten, so no late scroll/rAF/timer reaction can touch the
95
+ * capture either.
96
+ *
97
+ * Failure is always silent degradation: a page without `evaluate`, a script
98
+ * error, or an engine that rejects the walk just keeps the legacy
99
+ * single-instant capture — a walk can only add content, never fail a fetch.
100
+ * A walk that restored nothing returns no HTML, and the provider reads the
101
+ * live page exactly as before.
102
+ *
103
+ * The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
104
+ * cloaked placeholders) but never for non-HTML documents. It is bounded by
105
+ * the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
106
+ * and always by the remaining fetch deadline minus a decode reserve — the
107
+ * provider's abort closes the tab, which rejects the pending evaluation.
108
+ *
109
+ * @module dsh-web-fetch-playwright/autoscroll
110
+ */
111
+
112
+ import { stripScriptingElementsIn } from './strip.ts'
113
+ import type { PlaywrightPage } from './types.ts'
114
+
115
+ /** How far each step scrolls, as a share of the viewport height. */
116
+ export const SCROLL_STEP_RATIO = 0.7
117
+
118
+ /**
119
+ * How long one step's smooth scroll motion takes (ms). The step ANIMATES
120
+ * down its distance (time-interpolated ticks at ~60fps) instead of
121
+ * teleporting: feed engines' uncloak/re-cloak logic is built around a
122
+ * continuous stream of scroll events, and background-tab timer throttling
123
+ * self-corrects because every tick derives its position from the clock.
124
+ */
125
+ export const SCROLL_STEP_MOTION_MS = 300
126
+
127
+ /**
128
+ * Per-step settle (ms): the QUIET window after the motion, where a feed
129
+ * engine's debounced uncloak fires (measured: stepping without a rest
130
+ * starves Discourse's cloak manager, which renders nothing new until the
131
+ * scroll rests ~300ms) and the freshly rendered rows become stashable.
132
+ */
133
+ export const SCROLL_STEP_SETTLE_MS = 300
134
+
135
+ /** How often the walk re-checks the height while holding at the bottom. */
136
+ export const SCROLL_BOTTOM_POLL_MS = 450
137
+
138
+ /**
139
+ * How long at the bottom, with no height growth, before the walk concludes
140
+ * the page is complete. Real feed engines fetch their next chunk with
141
+ * network latency (measured on linux.do: a pending chunk landed well after
142
+ * two 450ms checks), so this is sized over a slow round trip.
143
+ */
144
+ export const SCROLL_BOTTOM_PATIENCE_MS = 2_500
145
+
146
+ /**
147
+ * Careful steps without any DOM reaction before the walk treats the page as
148
+ * scroll-inert and probes the rest with a single jump (rewinding if the
149
+ * jump provokes a reaction). Two steps cover ~1.4 viewports — past the
150
+ * render window edge of every measured virtualizer.
151
+ */
152
+ export const SCROLL_INERT_STEPS = 2
153
+
154
+ /** Hard cap on scroll steps regardless of budget (unbounded feeds). */
155
+ export const SCROLL_MAX_STEPS = 50
156
+
157
+ /**
158
+ * Cap on stashed keys — pathological id counts cannot pin memory. Sized for
159
+ * long topics: a Discourse post carries several ids of its own (post,
160
+ * heading, avatar, …), so a few hundred posts already need a thousand-plus.
161
+ * {@link SCROLL_STASH_MAX_CHARS} is the real memory bound.
162
+ */
163
+ export const SCROLL_STASH_MAX_ENTRIES = 2_000
164
+
165
+ /** Cap on stashed HTML characters (aligned with the pipeline input cap). */
166
+ export const SCROLL_STASH_MAX_CHARS = 2_000_000
167
+
168
+ /**
169
+ * Cap on ONE stashed element's HTML. A virtualized row is a few KB; an
170
+ * element tens of KB big is a CONTAINER (measured on linux.do: six nested
171
+ * wrappers each embed the whole post stream — main 164KB ⊃
172
+ * main-outlet-wrapper 156KB ⊃ … — plus data-preloaded 300KB and
173
+ * svg-sprites 277KB) or a data island — never a row the merge could
174
+ * rehydrate. Such keys are remembered and never serialized again.
175
+ */
176
+ export const SCROLL_STASH_MAX_ENTRY_CHARS = 64_000
177
+
178
+ /**
179
+ * Tail of the fetch budget the walk never spends, keeping room for the
180
+ * content read and the synchronous denoise decode.
181
+ */
182
+ export const SCROLL_FINISH_RESERVE_MS = 4_000
183
+
184
+ /**
185
+ * A walk budget below this cannot complete even the first window plus one
186
+ * stability check, so the walk is skipped outright.
187
+ */
188
+ export const SCROLL_MIN_BUDGET_MS = 500
189
+
190
+ /**
191
+ * Floor (px) on the scrollable distance an inner container must offer
192
+ * before the walk targets it, when the document itself does not scroll.
193
+ */
194
+ export const SCROLL_CONTAINER_MIN_RANGE_PX = 300
195
+
196
+ /** The viewport share an inner container's scrollable distance must reach. */
197
+ export const SCROLL_CONTAINER_MIN_RANGE_RATIO = 0.5
198
+
199
+ /**
200
+ * Text shortfall (chars) versus the snapshot that marks an element as
201
+ * degraded even when its child-element count held — below it, live text
202
+ * (relative timestamps, counters) merely ticked.
203
+ */
204
+ export const SCROLL_TEXT_DEGRADE_TOLERANCE = 16
205
+
206
+ /** What one completed (or skipped) walk reports back, for tests and logs. */
207
+ export interface DeepCaptureStats {
208
+ /** Scroll steps taken (0 when the walk skipped or had no room). */
209
+ steps: number
210
+ /** Distinct keys stashed across the walk. */
211
+ stashed: number
212
+ /** Degraded elements swapped for their snapshot in the capture. */
213
+ rehydrated: number
214
+ /** Removed keyed elements spliced back (container walks only). */
215
+ spliced?: number
216
+ /**
217
+ * Whether the page reacted to scrolling (added/removed elements). False
218
+ * means the walk took the scroll-inert fast path to the bottom.
219
+ */
220
+ reactive?: boolean
221
+ /** Walk duration (ms). */
222
+ elapsedMs?: number
223
+ /** Walk-target height (px) — document or inner scroll container. */
224
+ height?: number
225
+ /** Set when the walk skipped without scrolling, and why. */
226
+ skipped?: 'not-scrollable' | 'no-budget'
227
+ }
228
+
229
+ /** A finished walk: its stats, plus the merged capture when it restored anything. */
230
+ export interface DeepCapture extends DeepCaptureStats {
231
+ /**
232
+ * The merged, scripting-stripped document serialized like
233
+ * `page.content()` — present only when the walk restored something; the
234
+ * provider then reads it instead of the live page.
235
+ */
236
+ html?: string
237
+ }
238
+
239
+ /** The walk's knobs, handed to {@link deepCaptureInPage} as JSON. */
240
+ export interface DeepCaptureParams {
241
+ budgetMs: number
242
+ stepRatio: number
243
+ stepMotionMs: number
244
+ stepSettleMs: number
245
+ bottomPollMs: number
246
+ bottomPatienceMs: number
247
+ inertSteps: number
248
+ maxSteps: number
249
+ stashMaxEntries: number
250
+ stashMaxChars: number
251
+ stashMaxEntryChars: number
252
+ containerMinRangePx: number
253
+ containerMinRatio: number
254
+ textDegradeTolerance: number
255
+ }
256
+
257
+ /**
258
+ * The in-page walk. Evaluated BY SOURCE (`fn.toString()` composed into an
259
+ * expression by {@link deepCaptureScript}), so it must stay self-contained:
260
+ * every helper is nested, every knob arrives in `p`, and the one shared
261
+ * routine — the scripting strip — arrives as an argument. Typed here so the
262
+ * compiler checks what used to be an unchecked string.
263
+ *
264
+ * @param p - the walk's knobs.
265
+ * @param strip - {@link stripScriptingElementsIn}, applied to the merged clone.
266
+ * @returns the walk's stats, plus the merged HTML when anything was restored.
267
+ */
268
+ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Document) => void): Promise<DeepCapture> {
269
+ const started = Date.now()
270
+ const root = document.documentElement
271
+ const body = document.body
272
+ if (root === null || body === null) return { steps: 0, stashed: 0, rehydrated: 0, skipped: 'not-scrollable' }
273
+ if (p.budgetMs <= 0) return { steps: 0, stashed: 0, rehydrated: 0, skipped: 'no-budget' }
274
+
275
+ const KEYED = '[id], [data-block-id]'
276
+ const SVG_NS = 'http://www.w3.org/2000/svg'
277
+ const sleep = (ms: number): Promise<void> => new Promise(resolve => { setTimeout(resolve, ms) })
278
+
279
+ // ── Target: the window, or the dominant inner scroll container ─────────
280
+ // The app-shell shape (Feishu docs, Notion-class editors) locks html/body
281
+ // at one viewport and scrolls inside one container, where window-only
282
+ // primitives are no-ops. Largest scrollable distance wins; document order
283
+ // keeps the OUTERMOST on ties.
284
+ function findScrollContainer(host: HTMLElement): Element | null {
285
+ const minRange = Math.max(p.containerMinRangePx, window.innerHeight * p.containerMinRatio)
286
+ let best: Element | null = null
287
+ let bestRange = 0
288
+ for (const el of Array.from(host.querySelectorAll('*'))) {
289
+ const range = el.scrollHeight - el.clientHeight
290
+ if (range < minRange || range <= bestRange) continue
291
+ const overflowY = getComputedStyle(el).overflowY
292
+ if (overflowY === 'auto' || overflowY === 'scroll') { best = el; bestRange = range }
293
+ }
294
+ return best
295
+ }
296
+ const documentScrolls = root.scrollHeight > window.innerHeight + 4
297
+ const scroller = documentScrolls ? null : findScrollContainer(body)
298
+ if (!documentScrolls && scroller === null) return { steps: 0, stashed: 0, rehydrated: 0, skipped: 'not-scrollable' }
299
+ const viewportH = (): number => scroller === null ? window.innerHeight : scroller.clientHeight
300
+ const scrollH = (): number => scroller === null ? root.scrollHeight : scroller.scrollHeight
301
+ const curY = (): number => scroller === null ? window.scrollY : scroller.scrollTop
302
+ const atBottom = (): boolean => curY() + viewportH() >= scrollH() - 2
303
+ function scrollToY(y: number): void {
304
+ if (scroller === null) window.scrollTo({ top: y, behavior: 'instant' })
305
+ else scroller.scrollTop = y
306
+ }
307
+ // One step's smooth descent, positioned from the wall clock each tick so
308
+ // background-tab throttling degrades to coarser jumps along the same path.
309
+ function smoothScrollBy(dy: number): Promise<void> {
310
+ return new Promise(resolve => {
311
+ const fromY = curY()
312
+ const t0 = Date.now()
313
+ const tick = (): void => {
314
+ const progress = (Date.now() - t0) / p.stepMotionMs
315
+ if (progress >= 1 || (curY() !== fromY && atBottom())) {
316
+ scrollToY(fromY + dy)
317
+ resolve()
318
+ return
319
+ }
320
+ scrollToY(fromY + Math.round(dy * progress))
321
+ setTimeout(tick, 16)
322
+ }
323
+ tick()
324
+ })
325
+ }
326
+
327
+ // ── Identity ────────────────────────────────────────────────────────────
328
+ // id, else data-block-id (namespaced so the two can never collide): app-
329
+ // shell editors key their blocks that way and carry no id at all.
330
+ function keyOf(el: Element): string | null {
331
+ if (el.id !== '') return el.id
332
+ const block = el.getAttribute('data-block-id')
333
+ return block !== null && block !== '' ? `data-block-id:${block}` : null
334
+ }
335
+ // Nearest keyed sibling in one direction: where the element sits in its
336
+ // own list — the splice anchor if the engine removes it later.
337
+ function siblingKey(el: Element, dir: 'previousElementSibling' | 'nextElementSibling'): string | null {
338
+ for (let sib = el[dir]; sib !== null; sib = sib[dir]) {
339
+ const key = keyOf(sib)
340
+ if (key !== null) return key
341
+ }
342
+ return null
343
+ }
344
+
345
+ // ── Reactivity ─────────────────────────────────────────────────────────
346
+ // Element additions/removals are a virtualizer's (or a load-more's)
347
+ // signature; attribute churn (lazy images, sticky-header classes) and
348
+ // text ticks are not, nor are injected scripting/metadata elements.
349
+ const INERT_TAGS = new Set(['SCRIPT', 'STYLE', 'LINK', 'META', 'NOSCRIPT', 'IFRAME', 'TEMPLATE'])
350
+ let reactions = 0
351
+ const counts = (nodes: NodeList): boolean => {
352
+ for (const node of Array.from(nodes)) {
353
+ if (node.nodeType === 1 && !INERT_TAGS.has(node.nodeName)) return true
354
+ }
355
+ return false
356
+ }
357
+ const tally = (records: MutationRecord[]): void => {
358
+ for (const record of records) {
359
+ if (counts(record.addedNodes) || counts(record.removedNodes)) reactions++
360
+ }
361
+ }
362
+ const observer = new MutationObserver(tally)
363
+ observer.observe(body, { childList: true, subtree: true })
364
+ const reacted = (): boolean => { tally(observer.takeRecords()); return reactions > 0 }
365
+
366
+ // ── Stash ──────────────────────────────────────────────────────────────
367
+ interface Snapshot {
368
+ node: Element // detached clone, owned by stashDoc
369
+ els: number
370
+ text: number
371
+ len: number
372
+ prev: string | null
373
+ next: string | null
374
+ inScroller: boolean
375
+ }
376
+ // Clones live in a document without a browsing context: no scripts run,
377
+ // no custom elements upgrade, nothing loads.
378
+ const stashDoc = document.implementation.createHTMLDocument('')
379
+ const stash = new Map<string, Snapshot>()
380
+ const oversized = new Set<string>()
381
+ let stashChars = 0
382
+ const host: HTMLElement = body
383
+ function collect(): void {
384
+ for (const el of Array.from(host.querySelectorAll(KEYED))) {
385
+ const key = keyOf(el)
386
+ if (key === null || oversized.has(key) || el.namespaceURI === SVG_NS) continue
387
+ const els = el.childElementCount
388
+ const text = (el.textContent ?? '').length
389
+ if (els === 0 && text === 0) continue
390
+ const prev = stash.get(key)
391
+ if (prev !== undefined) {
392
+ // Neighbors refresh on every sighting: any observed sibling is a
393
+ // true one, and later windows see more of the list.
394
+ prev.prev = siblingKey(el, 'previousElementSibling') ?? prev.prev
395
+ prev.next = siblingKey(el, 'nextElementSibling') ?? prev.next
396
+ // Richest sighting wins; a rendered row collapsing back to its
397
+ // placeholder never downgrades the stash.
398
+ if (els <= prev.els && text <= prev.text) continue
399
+ } else if (stash.size >= p.stashMaxEntries) {
400
+ continue
401
+ }
402
+ if (text > p.stashMaxEntryChars) { oversized.add(key); continue }
403
+ const len = el.outerHTML.length
404
+ if (len > p.stashMaxEntryChars) { oversized.add(key); continue }
405
+ const freed = prev === undefined ? 0 : prev.len
406
+ if (stashChars - freed + len > p.stashMaxChars) continue
407
+ stashChars += len - freed
408
+ stash.set(key, {
409
+ node: stashDoc.importNode(el, true),
410
+ els,
411
+ text,
412
+ len,
413
+ prev: prev?.prev ?? siblingKey(el, 'previousElementSibling'),
414
+ next: prev?.next ?? siblingKey(el, 'nextElementSibling'),
415
+ inScroller: scroller !== null && scroller.contains(el),
416
+ })
417
+ }
418
+ }
419
+
420
+ // ── Walk ───────────────────────────────────────────────────────────────
421
+ // Stash whatever the (possibly anchored) start renders, then restart from
422
+ // the top so the walk covers the whole target.
423
+ collect()
424
+ if (curY() > 0) {
425
+ scrollToY(0)
426
+ await sleep(p.stepSettleMs)
427
+ collect()
428
+ }
429
+ // Only reactions to the walk's own scrolling count from here on.
430
+ tally(observer.takeRecords())
431
+ reactions = 0
432
+
433
+ let steps = 0
434
+ let lastHeight = scrollH()
435
+ let lastChangeAt = Date.now()
436
+ let careful = false // the page reacted: never jump again
437
+ let jumpedFrom: number | null = null
438
+ const noteHeight = (): void => {
439
+ const height = scrollH()
440
+ if (height !== lastHeight) { lastHeight = height; lastChangeAt = Date.now() }
441
+ }
442
+ try {
443
+ while (steps < p.maxSteps && Date.now() - started < p.budgetMs) {
444
+ if (jumpedFrom !== null && reacted()) {
445
+ // The jump provoked a reaction: what lies between jumpedFrom and the
446
+ // bottom may never have rendered. Go back and walk it carefully.
447
+ scrollToY(jumpedFrom)
448
+ jumpedFrom = null
449
+ careful = true
450
+ await sleep(p.stepSettleMs)
451
+ collect()
452
+ noteHeight()
453
+ continue
454
+ }
455
+ if (atBottom()) {
456
+ // Bottom patience: growth at any poll restarts the clock (the walk
457
+ // then keeps scrolling into what loaded); completion is only trusted
458
+ // after the full window passes with the height unchanged.
459
+ await sleep(p.bottomPollMs)
460
+ collect()
461
+ const before = lastHeight
462
+ noteHeight()
463
+ if (lastHeight === before && Date.now() - lastChangeAt >= p.bottomPatienceMs) break
464
+ continue
465
+ }
466
+ if (!careful && jumpedFrom === null) {
467
+ if (reacted()) {
468
+ careful = true
469
+ } else if (steps >= p.inertSteps) {
470
+ // Scroll-inert so far: probe the rest in one jump. The jump counts
471
+ // as a change, so a load-more it triggers gets the full patience.
472
+ jumpedFrom = curY()
473
+ steps++
474
+ scrollToY(scrollH())
475
+ lastChangeAt = Date.now()
476
+ await sleep(p.stepSettleMs)
477
+ collect()
478
+ noteHeight()
479
+ continue
480
+ }
481
+ }
482
+ steps++
483
+ await smoothScrollBy(Math.max(1, Math.round(viewportH() * p.stepRatio)))
484
+ collect()
485
+ await sleep(p.stepSettleMs)
486
+ collect()
487
+ noteHeight()
488
+ }
489
+ } finally {
490
+ observer.disconnect()
491
+ }
492
+ const reactive = reacted()
493
+ const finalHeight = scrollH()
494
+
495
+ // ── Merge into a structural clone (synchronous from here on) ──────────
496
+ const fresh = document.cloneNode(true) as Document
497
+ const present = new Map<string, Element>()
498
+ const index = (node: Element): void => {
499
+ const key = keyOf(node)
500
+ if (key !== null && !present.has(key)) present.set(key, node)
501
+ for (const el of Array.from(node.querySelectorAll(KEYED))) {
502
+ const inner = keyOf(el)
503
+ if (inner !== null && !present.has(inner)) present.set(inner, el)
504
+ }
505
+ }
506
+ if (fresh.documentElement !== null) index(fresh.documentElement)
507
+
508
+ // SPLICE (container walks): absent keys go back beside a surviving
509
+ // sibling; each pass can anchor the next, so a removed run chains off
510
+ // whichever end survived. Keys with no surviving relative stay out.
511
+ let spliced = 0
512
+ if (scroller !== null) {
513
+ let pending = Array.from(stash.keys()).filter(key => stash.get(key)!.inScroller && !present.has(key))
514
+ let progressed = true
515
+ while (pending.length > 0 && progressed) {
516
+ progressed = false
517
+ const unresolved: string[] = []
518
+ for (const key of pending) {
519
+ if (present.has(key)) { progressed = true; continue } // rode in with its parent
520
+ const snap = stash.get(key)!
521
+ const after = snap.prev === null ? undefined : present.get(snap.prev)
522
+ const before = snap.next === null ? undefined : present.get(snap.next)
523
+ if (after === undefined && before === undefined) { unresolved.push(key); continue }
524
+ const node = fresh.importNode(snap.node, true)
525
+ if (after !== undefined) after.after(node)
526
+ else before!.before(node)
527
+ index(node)
528
+ spliced++
529
+ progressed = true
530
+ }
531
+ pending = unresolved
532
+ }
533
+ }
534
+
535
+ // SWAP degraded leaves for their snapshots; their ancestors are
536
+ // containers, assembled from them rather than swapped wholesale.
537
+ const degraded: Array<{ el: Element; snap: Snapshot }> = []
538
+ for (const el of Array.from(fresh.querySelectorAll(KEYED))) {
539
+ const key = keyOf(el)
540
+ const snap = key === null ? undefined : stash.get(key)
541
+ if (snap === undefined || el.namespaceURI === SVG_NS) continue
542
+ if (el.childElementCount < snap.els || snap.text - (el.textContent ?? '').length > p.textDegradeTolerance) {
543
+ degraded.push({ el, snap })
544
+ }
545
+ }
546
+ const containers = new Set<Element>()
547
+ for (const { el } of degraded) {
548
+ for (let up = el.parentElement; up !== null && !containers.has(up); up = up.parentElement) containers.add(up)
549
+ }
550
+ let rehydrated = 0
551
+ for (const { el, snap } of degraded) {
552
+ if (containers.has(el)) continue
553
+ el.replaceWith(fresh.importNode(snap.node, true))
554
+ rehydrated++
555
+ }
556
+
557
+ const stats: DeepCapture = {
558
+ steps,
559
+ stashed: stash.size,
560
+ rehydrated,
561
+ spliced,
562
+ reactive,
563
+ elapsedMs: Date.now() - started,
564
+ height: finalHeight,
565
+ }
566
+ if ((rehydrated > 0 || spliced > 0) && fresh.documentElement !== null) {
567
+ strip(fresh)
568
+ // page.content()'s own serialization: doctype, then the root element.
569
+ const doctype = fresh.doctype === null ? '' : new XMLSerializer().serializeToString(fresh.doctype)
570
+ stats.html = doctype + fresh.documentElement.outerHTML
571
+ }
572
+ return stats
573
+ }
574
+
575
+ /**
576
+ * The walk's knobs for one fetch: the module constants plus its budget.
577
+ *
578
+ * @param budgetMs - wall-clock budget the walk may spend inside the page.
579
+ */
580
+ export function deepCaptureParams(budgetMs: number): DeepCaptureParams {
581
+ return {
582
+ budgetMs: Math.max(0, Math.round(budgetMs)),
583
+ stepRatio: SCROLL_STEP_RATIO,
584
+ stepMotionMs: SCROLL_STEP_MOTION_MS,
585
+ stepSettleMs: SCROLL_STEP_SETTLE_MS,
586
+ bottomPollMs: SCROLL_BOTTOM_POLL_MS,
587
+ bottomPatienceMs: SCROLL_BOTTOM_PATIENCE_MS,
588
+ inertSteps: SCROLL_INERT_STEPS,
589
+ maxSteps: SCROLL_MAX_STEPS,
590
+ stashMaxEntries: SCROLL_STASH_MAX_ENTRIES,
591
+ stashMaxChars: SCROLL_STASH_MAX_CHARS,
592
+ stashMaxEntryChars: SCROLL_STASH_MAX_ENTRY_CHARS,
593
+ containerMinRangePx: SCROLL_CONTAINER_MIN_RANGE_PX,
594
+ containerMinRatio: SCROLL_CONTAINER_MIN_RANGE_RATIO,
595
+ textDegradeTolerance: SCROLL_TEXT_DEGRADE_TOLERANCE,
596
+ }
597
+ }
598
+
599
+ /**
600
+ * Compose the evaluate expression: the in-page walk applied to `params` and
601
+ * the shared scripting strip, both by source.
602
+ *
603
+ * @param params - the walk's knobs (see {@link deepCaptureParams}).
604
+ * @returns the expression string to pass to `page.evaluate`.
605
+ */
606
+ export function deepCaptureExpression(params: DeepCaptureParams): string {
607
+ return `/* dsh-deep-capture */ (${deepCaptureInPage.toString()})(${JSON.stringify(params)}, ${stripScriptingElementsIn.toString()})`
608
+ }
609
+
610
+ /** {@link deepCaptureExpression} over {@link deepCaptureParams} — one fetch's walk. */
611
+ export function deepCaptureScript(budgetMs: number): string {
612
+ return deepCaptureExpression(deepCaptureParams(budgetMs))
613
+ }
614
+
615
+ /**
616
+ * Run the deep-capture walk in the page, degrading silently to the legacy
617
+ * single-instant capture on any failure: a page without `evaluate`, a budget
618
+ * too small to act, a rejected evaluation (the fetch deadline closes the tab
619
+ * mid-walk), or a result that is not the stats object all just skip it. A
620
+ * walk can only add content, never fail a fetch.
621
+ *
622
+ * @param page - the fetch's tab.
623
+ * @param budgetMs - wall-clock budget for the in-page walk; budgets under
624
+ * {@link SCROLL_MIN_BUDGET_MS} skip it.
625
+ * @returns the walk's stats (with the merged `html` when it restored
626
+ * anything), or undefined when the walk did not run.
627
+ */
628
+ export async function deepCaptureWalk(page: PlaywrightPage, budgetMs: number): Promise<DeepCapture | undefined> {
629
+ if (typeof page.evaluate !== 'function') return undefined
630
+ if (budgetMs < SCROLL_MIN_BUDGET_MS) return undefined
631
+ try {
632
+ const result = await page.evaluate(deepCaptureScript(budgetMs))
633
+ if (result === null || typeof result !== 'object') return undefined
634
+ const raw = result as Partial<DeepCapture>
635
+ if (typeof raw.steps !== 'number' || typeof raw.stashed !== 'number' || typeof raw.rehydrated !== 'number') {
636
+ return undefined
637
+ }
638
+ return {
639
+ steps: raw.steps,
640
+ stashed: raw.stashed,
641
+ rehydrated: raw.rehydrated,
642
+ ...(typeof raw.spliced === 'number' ? { spliced: raw.spliced } : {}),
643
+ ...(typeof raw.reactive === 'boolean' ? { reactive: raw.reactive } : {}),
644
+ ...(typeof raw.elapsedMs === 'number' ? { elapsedMs: raw.elapsedMs } : {}),
645
+ ...(typeof raw.height === 'number' ? { height: raw.height } : {}),
646
+ ...(raw.skipped === 'not-scrollable' || raw.skipped === 'no-budget' ? { skipped: raw.skipped } : {}),
647
+ ...(typeof raw.html === 'string' ? { html: raw.html } : {}),
648
+ }
649
+ } catch {
650
+ // A failed walk degrades to the legacy capture; the fetch continues.
651
+ return undefined
652
+ }
653
+ }
@@ -190,6 +190,19 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
190
190
  onEdit={(text) => { props.edit('challengeWaitMs', text) }}
191
191
  onReset={() => { props.resetField('challengeWaitMs') }}
192
192
  />
193
+ <ValueField
194
+ id="plugin-config-playwright-scroll-budget"
195
+ label={t('scrollBudgetMs')}
196
+ hint={t('scrollBudgetMsHint')}
197
+ placeholder={t('scrollBudgetMsPlaceholder')}
198
+ overriddenLabel={t('overridden')}
199
+ resetLabel={t('reset')}
200
+ invalidLabel={t('invalidText')}
201
+ disabled={disabled}
202
+ {...state.scrollBudgetMs}
203
+ onEdit={(text) => { props.edit('scrollBudgetMs', text) }}
204
+ onReset={() => { props.resetField('scrollBudgetMs') }}
205
+ />
193
206
  </CardForm>
194
207
  )
195
208
  }