dsh-web-fetch-playwright 0.2.20 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -17
- package/README.zh-CN.md +19 -17
- package/cordis.patch.yml +0 -4
- package/lib/client.js +18 -68
- package/lib/index.js +24049 -22754
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/candidates.d.ts +55 -0
- package/lib/types/challenge.d.ts +2 -2
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +80 -30
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +101 -9
- package/lib/types/types.d.ts +26 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +1 -1
- package/src/candidates.ts +184 -35
- package/src/challenge.ts +2 -2
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +97 -37
- package/src/index.ts +4 -1
- package/src/markdown.ts +13 -2
- package/src/provider.ts +482 -56
- package/src/selector-model.ts +1 -1
- package/src/selector.ts +18 -7
- package/src/types.ts +27 -7
package/src/autoscroll.ts
CHANGED
|
@@ -60,7 +60,8 @@
|
|
|
60
60
|
* collecting after each, until the walk can hold AT THE BOTTOM through a
|
|
61
61
|
* {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
|
|
62
62
|
* (growth restarts the clock — that is what pages load-more sentinels),
|
|
63
|
-
* or the step/budget caps close it.
|
|
63
|
+
* or the step/budget caps close it. A page that never reacted and issued
|
|
64
|
+
* no fetch/XHR holds only {@link SCROLL_INERT_PATIENCE_MS}.
|
|
64
65
|
*
|
|
65
66
|
* A MutationObserver watches for the walk's whole reason to exist: the
|
|
66
67
|
* page REACTING to scroll by adding or removing elements. Most of the web
|
|
@@ -71,6 +72,14 @@
|
|
|
71
72
|
* jumped over may hold content that never rendered, so the walk rewinds
|
|
72
73
|
* to where it jumped from and finishes carefully. Correctness never
|
|
73
74
|
* rests on the warm-up guess; only the cost of a scroll-inert page does.
|
|
75
|
+
*
|
|
76
|
+
* The same observer HARVESTS UNKEYED ROWS — the third shape: react-window
|
|
77
|
+
* lists and X-style timelines give rows no DOM identity at all (the
|
|
78
|
+
* framework key never reaches the DOM), and framework-generated ids
|
|
79
|
+
* (`ember123`, React useId's `:r1:`) change per render so they are not
|
|
80
|
+
* keys either. When the engine removes such a row, the record still
|
|
81
|
+
* holds it whole plus the sibling nodes it sat between; that is the row's
|
|
82
|
+
* stash and its splice anchor, with no layout read at all.
|
|
74
83
|
* 4. MERGE into a structural clone (`document.cloneNode(true)`: no
|
|
75
84
|
* serialize/re-parse round trip — a runtime-rewritten data script with a
|
|
76
85
|
* literal `</script>` in its text cannot corrupt it — and, having no
|
|
@@ -84,11 +93,21 @@
|
|
|
84
93
|
* spliced parent is already present and skipped. Placement is
|
|
85
94
|
* structural, so it is right whatever order the walk SAW rows in (an
|
|
86
95
|
* anchored open starts mid-document).
|
|
96
|
+
* - RESTORE (unkeyed rows): in every list the engine both removed text-
|
|
97
|
+
* bearing rows from and added to during the walk, each removed row the
|
|
98
|
+
* list no longer holds goes back beside the sibling it was removed
|
|
99
|
+
* next to (chains through removed anchors; rows with no surviving
|
|
100
|
+
* anchor at all — a wholesale innerHTML re-render — go above the final
|
|
101
|
+
* window, which is the only place a top-down walk loses them). Rows
|
|
102
|
+
* still present are recognised by a per-list identity attribute
|
|
103
|
+
* (data-index, aria-rowindex, …) or by text; in-place stubs are
|
|
104
|
+
* absorbed.
|
|
87
105
|
* - SWAP: every keyed element emptier than its snapshot (fewer child
|
|
88
106
|
* elements, or meaningfully less text — the Discourse placeholder) is
|
|
89
107
|
* replaced by that snapshot, except containers (ancestors of another
|
|
90
108
|
* element being swapped), which are assembled from those descendants
|
|
91
|
-
* instead: a container's own snapshot is one window's partial view.
|
|
109
|
+
* instead: a container's own snapshot is one window's partial view. A
|
|
110
|
+
* restored list and its ancestors are containers too.
|
|
92
111
|
* The clone is then stripped of scripting elements and serialized; the
|
|
93
112
|
* provider reads that string instead of the live page. The live document
|
|
94
113
|
* is never rewritten, so no late scroll/rAF/timer reaction can touch the
|
|
@@ -98,13 +117,16 @@
|
|
|
98
117
|
* error, or an engine that rejects the walk just keeps the legacy
|
|
99
118
|
* single-instant capture — a walk can only add content, never fail a fetch.
|
|
100
119
|
* A walk that restored nothing returns no HTML, and the provider reads the
|
|
101
|
-
* live page exactly as before
|
|
120
|
+
* live page exactly as before — scrolled back to where it opened when the
|
|
121
|
+
* page reacted, so a virtualizer the merge could not rebuild hands over its
|
|
122
|
+
* opening window rather than the walk's last one.
|
|
102
123
|
*
|
|
103
124
|
* The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
|
|
104
125
|
* cloaked placeholders) but never for non-HTML documents. It is bounded by
|
|
105
126
|
* the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
|
|
106
|
-
* and always by the remaining
|
|
107
|
-
*
|
|
127
|
+
* and always by the fetch's remaining waiting budget: a walk cut short there
|
|
128
|
+
* still merges and returns what it stashed, and the content read plus decode
|
|
129
|
+
* run in the provider's extraction grace window after it.
|
|
108
130
|
*
|
|
109
131
|
* @module dsh-web-fetch-playwright/autoscroll
|
|
110
132
|
*/
|
|
@@ -143,6 +165,17 @@ export const SCROLL_BOTTOM_POLL_MS = 450
|
|
|
143
165
|
*/
|
|
144
166
|
export const SCROLL_BOTTOM_PATIENCE_MS = 2_500
|
|
145
167
|
|
|
168
|
+
/**
|
|
169
|
+
* Bottom hold for a page that has given no sign of loading more: it never
|
|
170
|
+
* added or removed an element in response to the walk's scrolling, and it
|
|
171
|
+
* issued no fetch/XHR since the walk began. The full
|
|
172
|
+
* {@link SCROLL_BOTTOM_PATIENCE_MS} exists for feeds whose next chunk is in
|
|
173
|
+
* flight; holding it on every static article made the walk cost ~4s on
|
|
174
|
+
* pages it could never help. Any reaction or data request (even one that
|
|
175
|
+
* lands during this shorter hold) restores the full patience.
|
|
176
|
+
*/
|
|
177
|
+
export const SCROLL_INERT_PATIENCE_MS = 900
|
|
178
|
+
|
|
146
179
|
/**
|
|
147
180
|
* Careful steps without any DOM reaction before the walk treats the page as
|
|
148
181
|
* scroll-inert and probes the rest with a single jump (rewinding if the
|
|
@@ -175,12 +208,6 @@ export const SCROLL_STASH_MAX_CHARS = 2_000_000
|
|
|
175
208
|
*/
|
|
176
209
|
export const SCROLL_STASH_MAX_ENTRY_CHARS = 64_000
|
|
177
210
|
|
|
178
|
-
/**
|
|
179
|
-
* Tail of the fetch budget the walk never spends, keeping room for the
|
|
180
|
-
* content read and the synchronous denoise decode.
|
|
181
|
-
*/
|
|
182
|
-
export const SCROLL_FINISH_RESERVE_MS = 4_000
|
|
183
|
-
|
|
184
211
|
/**
|
|
185
212
|
* A walk budget below this cannot complete even the first window plus one
|
|
186
213
|
* stability check, so the walk is skipped outright.
|
|
@@ -213,6 +240,14 @@ export interface DeepCaptureStats {
|
|
|
213
240
|
rehydrated: number
|
|
214
241
|
/** Removed keyed elements spliced back (container walks only). */
|
|
215
242
|
spliced?: number
|
|
243
|
+
/** Unkeyed rows a virtualizer removed, restored into their list. */
|
|
244
|
+
restored?: number
|
|
245
|
+
/**
|
|
246
|
+
* True when nothing could be restored on a page that reacted, so the walk
|
|
247
|
+
* scrolled back to where the page opened before handing it to the
|
|
248
|
+
* provider's live read.
|
|
249
|
+
*/
|
|
250
|
+
rewound?: boolean
|
|
216
251
|
/**
|
|
217
252
|
* Whether the page reacted to scrolling (added/removed elements). False
|
|
218
253
|
* means the walk took the scroll-inert fast path to the bottom.
|
|
@@ -244,6 +279,7 @@ export interface DeepCaptureParams {
|
|
|
244
279
|
stepSettleMs: number
|
|
245
280
|
bottomPollMs: number
|
|
246
281
|
bottomPatienceMs: number
|
|
282
|
+
inertPatienceMs: number
|
|
247
283
|
inertSteps: number
|
|
248
284
|
maxSteps: number
|
|
249
285
|
stashMaxEntries: number
|
|
@@ -327,8 +363,13 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
327
363
|
// ── Identity ────────────────────────────────────────────────────────────
|
|
328
364
|
// id, else data-block-id (namespaced so the two can never collide): app-
|
|
329
365
|
// shell editors key their blocks that way and carry no id at all.
|
|
366
|
+
// Framework-GENERATED ids are per-render, not identities: a row re-rendered
|
|
367
|
+
// by Ember (`ember123`) or React's useId (`:r1:`, `«r1»`, `_r_1_`) comes
|
|
368
|
+
// back under a new id, so keying on one only ever stashes dead snapshots.
|
|
369
|
+
// Such elements count as unkeyed and take the unkeyed-row path below.
|
|
370
|
+
const GENERATED_ID = /^ember\d+$|:r[0-9a-z]*:|«r[0-9a-z]*»|^_r_[0-9a-z]+_$/
|
|
330
371
|
function keyOf(el: Element): string | null {
|
|
331
|
-
if (el.id !== '') return el.id
|
|
372
|
+
if (el.id !== '' && !GENERATED_ID.test(el.id)) return el.id
|
|
332
373
|
const block = el.getAttribute('data-block-id')
|
|
333
374
|
return block !== null && block !== '' ? `data-block-id:${block}` : null
|
|
334
375
|
}
|
|
@@ -357,11 +398,36 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
357
398
|
const tally = (records: MutationRecord[]): void => {
|
|
358
399
|
for (const record of records) {
|
|
359
400
|
if (counts(record.addedNodes) || counts(record.removedNodes)) reactions++
|
|
401
|
+
harvest(record)
|
|
360
402
|
}
|
|
361
403
|
}
|
|
362
|
-
|
|
363
|
-
|
|
404
|
+
// (The observer itself starts below, once the unkeyed-row harvest it
|
|
405
|
+
// feeds is declared.)
|
|
364
406
|
const reacted = (): boolean => { tally(observer.takeRecords()); return reactions > 0 }
|
|
407
|
+
// Data requests (fetch/XHR): the other sign of a feed loading more.
|
|
408
|
+
// Images, fonts and beacons are not. Observed rather than read from the
|
|
409
|
+
// resource buffer, which stops growing once full (250 entries in Chromium)
|
|
410
|
+
// on exactly the heavy pages that matter. Resource Timing reports a
|
|
411
|
+
// request only once it COMPLETES: a next chunk still in flight when the
|
|
412
|
+
// short inert hold expires goes unseen, which is the price of that hold.
|
|
413
|
+
let requests = 0
|
|
414
|
+
const tallyRequests = (entries: PerformanceEntryList): void => {
|
|
415
|
+
for (const entry of entries) {
|
|
416
|
+
const type = (entry as PerformanceResourceTiming).initiatorType
|
|
417
|
+
if (type === 'fetch' || type === 'xmlhttprequest') requests++
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
let requestObserver: PerformanceObserver | null = null
|
|
421
|
+
try {
|
|
422
|
+
requestObserver = new PerformanceObserver(list => { tallyRequests(list.getEntries()) })
|
|
423
|
+
requestObserver.observe({ type: 'resource' })
|
|
424
|
+
} catch {
|
|
425
|
+
requestObserver = null // no Resource Timing: reactions alone decide
|
|
426
|
+
}
|
|
427
|
+
const requested = (): boolean => {
|
|
428
|
+
if (requestObserver !== null) tallyRequests(requestObserver.takeRecords())
|
|
429
|
+
return requests > 0
|
|
430
|
+
}
|
|
365
431
|
|
|
366
432
|
// ── Stash ──────────────────────────────────────────────────────────────
|
|
367
433
|
interface Snapshot {
|
|
@@ -379,6 +445,118 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
379
445
|
const stash = new Map<string, Snapshot>()
|
|
380
446
|
const oversized = new Set<string>()
|
|
381
447
|
let stashChars = 0
|
|
448
|
+
|
|
449
|
+
// ── Unkeyed rows ───────────────────────────────────────────────────────
|
|
450
|
+
// Many virtualizers give their rows no DOM identity at all (react-window
|
|
451
|
+
// rows, X-style timeline cells: the framework key never reaches the DOM).
|
|
452
|
+
// Such a row is harvested at the instant the engine REMOVES it: the
|
|
453
|
+
// MutationObserver record hands over the detached row, still whole, plus
|
|
454
|
+
// the sibling nodes it sat between — enough to splice it back into the
|
|
455
|
+
// same list at merge without reading any layout (absolute positioning and
|
|
456
|
+
// transforms do not matter). Nodes are tracked by object identity, so
|
|
457
|
+
// even an anonymous spacer or a row filtered out as noise still anchors
|
|
458
|
+
// its neighbors.
|
|
459
|
+
const ROW_IDENT_ATTRS = [
|
|
460
|
+
'data-index', 'data-item-index', 'data-row-index', 'data-rowindex', 'aria-rowindex', 'aria-posinset',
|
|
461
|
+
'data-key', 'data-row-key', 'data-item-key', 'data-id', 'data-item-id',
|
|
462
|
+
]
|
|
463
|
+
// Transient UI — toasts, live status, spinners, menus — is never content.
|
|
464
|
+
const NOISE_ROLES = new Set(['status', 'alert', 'alertdialog', 'dialog', 'tooltip', 'progressbar', 'menu', 'listbox', 'timer', 'marquee'])
|
|
465
|
+
const squash = (s: string): string => s.replace(/\s+/g, ' ').trim()
|
|
466
|
+
// Identity for de-duplication against the rows still live: a per-LIST
|
|
467
|
+
// attribute when the engine sets one (scoped to the list, so data-index
|
|
468
|
+
// values repeating across lists cannot collide), else the row's text.
|
|
469
|
+
function rowIdent(el: Element): string | null {
|
|
470
|
+
for (const attr of ROW_IDENT_ATTRS) {
|
|
471
|
+
const value = el.getAttribute(attr)
|
|
472
|
+
if (value !== null && value !== '') return `${attr}=${value}`
|
|
473
|
+
}
|
|
474
|
+
return null
|
|
475
|
+
}
|
|
476
|
+
function rowText(el: Element): string | null {
|
|
477
|
+
if (keyOf(el) !== null || INERT_TAGS.has(el.nodeName) || el.namespaceURI === SVG_NS) return null
|
|
478
|
+
if (el.hasAttribute('hidden') || el.getAttribute('aria-hidden') === 'true' || el.getAttribute('aria-busy') === 'true') return null
|
|
479
|
+
const role = el.getAttribute('role')
|
|
480
|
+
if (role !== null && NOISE_ROLES.has(role)) return null
|
|
481
|
+
const text = squash(el.textContent ?? '')
|
|
482
|
+
return text === '' ? null : text
|
|
483
|
+
}
|
|
484
|
+
interface Removal {
|
|
485
|
+
row: { node: Element; text: string; ident: string | null; len: number } | null // null: anchor-only ghost
|
|
486
|
+
/** A ghost repeating an already-stashed row: the node that holds its copy. */
|
|
487
|
+
same?: Node
|
|
488
|
+
parent: Node
|
|
489
|
+
prev: Node | null
|
|
490
|
+
next: Node | null
|
|
491
|
+
seq: number
|
|
492
|
+
}
|
|
493
|
+
const removals = new Map<Node, Removal>()
|
|
494
|
+
// Per list: the one stashed copy of each row identity. An engine that
|
|
495
|
+
// re-renders every frame removes the same row thousands of times; only
|
|
496
|
+
// the first sighting (richest wins) is stashed, the rest only anchor.
|
|
497
|
+
const rowsSeen = new Map<Node, Map<string, Node>>()
|
|
498
|
+
const lists = new Map<Node, { rows: number; added: number }>()
|
|
499
|
+
const addedNodes = new WeakSet<Node>()
|
|
500
|
+
let removalSeq = 0
|
|
501
|
+
let rowsStashed = 0
|
|
502
|
+
// Its OWN character budget: rows churned inside keyed content (a forum
|
|
503
|
+
// engine re-rendering post internals) must never starve the keyed stash,
|
|
504
|
+
// which is what the established feed shapes rely on.
|
|
505
|
+
let rowChars = 0
|
|
506
|
+
// Anchor-only bookkeeping is cheap but holds detached nodes alive.
|
|
507
|
+
const MAX_REMOVALS = 50_000
|
|
508
|
+
function harvest(record: MutationRecord): void {
|
|
509
|
+
const parent = record.target
|
|
510
|
+
if (record.type !== 'childList' || parent.nodeType !== 1 || parent === body || !parent.isConnected) return
|
|
511
|
+
if (scroller !== null ? !scroller.contains(parent) : !body.contains(parent)) return
|
|
512
|
+
let list = lists.get(parent)
|
|
513
|
+
if (list === undefined) { list = { rows: 0, added: 0 }; lists.set(parent, list) }
|
|
514
|
+
for (const node of Array.from(record.addedNodes)) {
|
|
515
|
+
addedNodes.add(node)
|
|
516
|
+
if (node.nodeType === 1 && !INERT_TAGS.has(node.nodeName)) list.added++
|
|
517
|
+
}
|
|
518
|
+
// One record may remove a whole run (innerHTML, replaceChildren): its
|
|
519
|
+
// members anchor on each other, the run's ends on the record's siblings.
|
|
520
|
+
const gone = Array.from(record.removedNodes)
|
|
521
|
+
let seen = rowsSeen.get(parent)
|
|
522
|
+
if (seen === undefined) { seen = new Map(); rowsSeen.set(parent, seen) }
|
|
523
|
+
gone.forEach((node, i) => {
|
|
524
|
+
if (removals.size >= MAX_REMOVALS) return
|
|
525
|
+
const removal: Removal = {
|
|
526
|
+
row: null,
|
|
527
|
+
parent,
|
|
528
|
+
prev: i > 0 ? gone[i - 1]! : record.previousSibling,
|
|
529
|
+
next: i < gone.length - 1 ? gone[i + 1]! : record.nextSibling,
|
|
530
|
+
seq: removalSeq++,
|
|
531
|
+
}
|
|
532
|
+
removals.set(node, removal)
|
|
533
|
+
const text = node.nodeType === 1 ? rowText(node as Element) : null
|
|
534
|
+
if (text === null) return
|
|
535
|
+
const el = node as Element
|
|
536
|
+
const ident = rowIdent(el)
|
|
537
|
+
const identity = ident ?? `${el.nodeName}|${text}`
|
|
538
|
+
const priorNode = seen.get(identity)
|
|
539
|
+
const prior = priorNode === undefined ? undefined : removals.get(priorNode)
|
|
540
|
+
if (priorNode !== undefined) removal.same = priorNode
|
|
541
|
+
if (prior?.row != null && prior.row.text.length >= text.length) return
|
|
542
|
+
if (prior === undefined && rowsStashed >= p.stashMaxEntries) return
|
|
543
|
+
const len = el.outerHTML.length
|
|
544
|
+
const freed = prior?.row?.len ?? 0
|
|
545
|
+
if (len > p.stashMaxEntryChars || rowChars - freed + len > p.stashMaxChars) return
|
|
546
|
+
rowChars += len - freed
|
|
547
|
+
const row = { node: stashDoc.importNode(el, true), text, ident, len }
|
|
548
|
+
if (prior !== undefined) {
|
|
549
|
+
prior.row = row // a richer sighting, kept at the FIRST sighting's place
|
|
550
|
+
} else {
|
|
551
|
+
removal.row = row
|
|
552
|
+
seen.set(identity, node)
|
|
553
|
+
rowsStashed++
|
|
554
|
+
list.rows++
|
|
555
|
+
}
|
|
556
|
+
})
|
|
557
|
+
}
|
|
558
|
+
const observer = new MutationObserver(tally)
|
|
559
|
+
observer.observe(body, { childList: true, subtree: true })
|
|
382
560
|
const host: HTMLElement = body
|
|
383
561
|
function collect(): void {
|
|
384
562
|
for (const el of Array.from(host.querySelectorAll(KEYED))) {
|
|
@@ -420,6 +598,7 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
420
598
|
// ── Walk ───────────────────────────────────────────────────────────────
|
|
421
599
|
// Stash whatever the (possibly anchored) start renders, then restart from
|
|
422
600
|
// the top so the walk covers the whole target.
|
|
601
|
+
const startY = curY()
|
|
423
602
|
collect()
|
|
424
603
|
if (curY() > 0) {
|
|
425
604
|
scrollToY(0)
|
|
@@ -429,6 +608,8 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
429
608
|
// Only reactions to the walk's own scrolling count from here on.
|
|
430
609
|
tally(observer.takeRecords())
|
|
431
610
|
reactions = 0
|
|
611
|
+
requested()
|
|
612
|
+
requests = 0
|
|
432
613
|
|
|
433
614
|
let steps = 0
|
|
434
615
|
let lastHeight = scrollH()
|
|
@@ -460,7 +641,9 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
460
641
|
collect()
|
|
461
642
|
const before = lastHeight
|
|
462
643
|
noteHeight()
|
|
463
|
-
|
|
644
|
+
// `careful` implies reacted(): reactions are never reset mid-walk.
|
|
645
|
+
const patience = reacted() || requested() ? p.bottomPatienceMs : p.inertPatienceMs
|
|
646
|
+
if (lastHeight === before && Date.now() - lastChangeAt >= patience) break
|
|
464
647
|
continue
|
|
465
648
|
}
|
|
466
649
|
if (!careful && jumpedFrom === null) {
|
|
@@ -487,13 +670,64 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
487
670
|
noteHeight()
|
|
488
671
|
}
|
|
489
672
|
} finally {
|
|
673
|
+
// Flush first: disconnect() drops queued records, and the last batch of
|
|
674
|
+
// removals is exactly what the unkeyed-row merge needs.
|
|
675
|
+
tally(observer.takeRecords())
|
|
490
676
|
observer.disconnect()
|
|
677
|
+
requestObserver?.disconnect()
|
|
491
678
|
}
|
|
492
679
|
const reactive = reacted()
|
|
493
680
|
const finalHeight = scrollH()
|
|
494
681
|
|
|
495
682
|
// ── Merge into a structural clone (synchronous from here on) ──────────
|
|
496
683
|
const fresh = document.cloneNode(true) as Document
|
|
684
|
+
|
|
685
|
+
// Unkeyed lists to restore, mapped into the clone NOW — the clone is an
|
|
686
|
+
// exact structural copy only until the merge starts editing it. A list
|
|
687
|
+
// qualifies when the engine both removed text-bearing rows from it and
|
|
688
|
+
// added elements to it during the walk (a virtualizer's churn; a lone
|
|
689
|
+
// toast or a load-more append does not), and it is not a live/status
|
|
690
|
+
// region or carousel whose churn is rotation rather than windowing.
|
|
691
|
+
interface ListPlan {
|
|
692
|
+
live: Node
|
|
693
|
+
fresh: Element
|
|
694
|
+
childOf: Map<Node, Node> // live child → its clone twin
|
|
695
|
+
added: Set<Node> // clone twins of children the engine added during the walk
|
|
696
|
+
}
|
|
697
|
+
const NOISE_REGION = '[aria-live="assertive"], [role="alert"], [role="status"], [role="timer"], [role="marquee"], [aria-roledescription~="carousel"]'
|
|
698
|
+
function freshOf(live: Node): Node | null {
|
|
699
|
+
const path: number[] = []
|
|
700
|
+
for (let at: Node = live; at !== document;) {
|
|
701
|
+
const up = at.parentNode
|
|
702
|
+
if (up === null) return null
|
|
703
|
+
path.push(Array.prototype.indexOf.call(up.childNodes, at) as number)
|
|
704
|
+
at = up
|
|
705
|
+
}
|
|
706
|
+
let at: Node = fresh
|
|
707
|
+
for (let i = path.length - 1; i >= 0; i--) {
|
|
708
|
+
const child = at.childNodes[path[i]!]
|
|
709
|
+
if (child === undefined) return null
|
|
710
|
+
at = child
|
|
711
|
+
}
|
|
712
|
+
return at
|
|
713
|
+
}
|
|
714
|
+
const plans: ListPlan[] = []
|
|
715
|
+
for (const [live, list] of lists) {
|
|
716
|
+
if (list.rows < 2 || list.added < 1 || !live.isConnected || (live as Element).closest(NOISE_REGION) !== null) continue
|
|
717
|
+
const twin = freshOf(live)
|
|
718
|
+
if (twin === null || twin.nodeType !== 1) continue
|
|
719
|
+
const liveKids = Array.from(live.childNodes)
|
|
720
|
+
const freshKids = Array.from(twin.childNodes)
|
|
721
|
+
if (liveKids.length !== freshKids.length) continue
|
|
722
|
+
const childOf = new Map<Node, Node>()
|
|
723
|
+
const added = new Set<Node>()
|
|
724
|
+
liveKids.forEach((child, i) => {
|
|
725
|
+
childOf.set(child, freshKids[i]!)
|
|
726
|
+
if (addedNodes.has(child)) added.add(freshKids[i]!)
|
|
727
|
+
})
|
|
728
|
+
plans.push({ live, fresh: twin as Element, childOf, added })
|
|
729
|
+
}
|
|
730
|
+
|
|
497
731
|
const present = new Map<string, Element>()
|
|
498
732
|
const index = (node: Element): void => {
|
|
499
733
|
const key = keyOf(node)
|
|
@@ -532,8 +766,21 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
532
766
|
}
|
|
533
767
|
}
|
|
534
768
|
|
|
769
|
+
// RESTORE unkeyed rows into their lists, oldest removal first — before the
|
|
770
|
+
// swap, which must see each list already whole.
|
|
771
|
+
// Clone nodes the merge replaced, so a list anchor on one still resolves.
|
|
772
|
+
const replacedBy = new Map<Node, Node>()
|
|
773
|
+
let restored = 0
|
|
774
|
+
const restoredLists: Element[] = []
|
|
775
|
+
for (const plan of plans) {
|
|
776
|
+
const count = restoreList(plan)
|
|
777
|
+
if (count > 0) { restored += count; restoredLists.push(plan.fresh) }
|
|
778
|
+
}
|
|
779
|
+
|
|
535
780
|
// SWAP degraded leaves for their snapshots; their ancestors are
|
|
536
|
-
// containers, assembled from them rather than swapped wholesale.
|
|
781
|
+
// containers, assembled from them rather than swapped wholesale. So is a
|
|
782
|
+
// restored list and everything around it: a keyed wrapper's snapshot is
|
|
783
|
+
// one window's partial view of the rows the restore just made whole.
|
|
537
784
|
const degraded: Array<{ el: Element; snap: Snapshot }> = []
|
|
538
785
|
for (const el of Array.from(fresh.querySelectorAll(KEYED))) {
|
|
539
786
|
const key = keyOf(el)
|
|
@@ -547,6 +794,9 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
547
794
|
for (const { el } of degraded) {
|
|
548
795
|
for (let up = el.parentElement; up !== null && !containers.has(up); up = up.parentElement) containers.add(up)
|
|
549
796
|
}
|
|
797
|
+
for (const list of restoredLists) {
|
|
798
|
+
for (let up: Element | null = list; up !== null && !containers.has(up); up = up.parentElement) containers.add(up)
|
|
799
|
+
}
|
|
550
800
|
let rehydrated = 0
|
|
551
801
|
for (const { el, snap } of degraded) {
|
|
552
802
|
if (containers.has(el)) continue
|
|
@@ -554,22 +804,211 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
|
|
|
554
804
|
rehydrated++
|
|
555
805
|
}
|
|
556
806
|
|
|
807
|
+
function restoreList(plan: ListPlan): number {
|
|
808
|
+
const follow = (node: Node | undefined): Node | null => {
|
|
809
|
+
let at = node
|
|
810
|
+
while (at !== undefined && replacedBy.has(at)) at = replacedBy.get(at)
|
|
811
|
+
return at !== undefined && at.parentNode === plan.fresh ? at : null
|
|
812
|
+
}
|
|
813
|
+
// What the list holds now, for de-duplication: identity attribute and
|
|
814
|
+
// text of every element child (live rows, spliced or swapped ones).
|
|
815
|
+
const byIdent = new Map<string, Element>()
|
|
816
|
+
const byText = new Map<string, Element>()
|
|
817
|
+
const textOf = new Map<Element, string>()
|
|
818
|
+
const note = (el: Element): void => {
|
|
819
|
+
const text = squash(el.textContent ?? '')
|
|
820
|
+
textOf.set(el, text)
|
|
821
|
+
const ident = rowIdent(el)
|
|
822
|
+
if (ident !== null && !byIdent.has(ident)) byIdent.set(ident, el)
|
|
823
|
+
if (text !== '' && !byText.has(`${el.nodeName}|${text}`)) byText.set(`${el.nodeName}|${text}`, el)
|
|
824
|
+
}
|
|
825
|
+
const forget = (el: Element): void => {
|
|
826
|
+
const text = textOf.get(el) ?? ''
|
|
827
|
+
const ident = rowIdent(el)
|
|
828
|
+
if (ident !== null && byIdent.get(ident) === el) byIdent.delete(ident)
|
|
829
|
+
if (byText.get(`${el.nodeName}|${text}`) === el) byText.delete(`${el.nodeName}|${text}`)
|
|
830
|
+
}
|
|
831
|
+
for (const child of Array.from(plan.fresh.children)) note(child)
|
|
832
|
+
|
|
833
|
+
const placed = new Map<Node, Node>() // live removed node → clone node it became
|
|
834
|
+
const rows = Array.from(removals.entries())
|
|
835
|
+
.filter(([node, r]) => r.parent === plan.live && r.row !== null && !node.isConnected)
|
|
836
|
+
.sort((a, b) => a[1].seq - b[1].seq)
|
|
837
|
+
const pending = new Set(rows.map(([node]) => node))
|
|
838
|
+
// Later removals of the same row (ghosts pointing at its stashed copy).
|
|
839
|
+
const repeats = new Map<Node, Removal[]>()
|
|
840
|
+
for (const removal of removals.values()) {
|
|
841
|
+
if (removal.row !== null || removal.same === undefined || removal.parent !== plan.live) continue
|
|
842
|
+
const list = repeats.get(removal.same)
|
|
843
|
+
if (list === undefined) repeats.set(removal.same, [removal])
|
|
844
|
+
else list.push(removal)
|
|
845
|
+
}
|
|
846
|
+
let allowPending = false
|
|
847
|
+
const PENDING = 1
|
|
848
|
+
// Walk an anchor chain to a node of the clone list: a live sibling, a
|
|
849
|
+
// row already restored, or — through rows that will never come back
|
|
850
|
+
// (ghosts, duplicates) — whatever THEY sat next to.
|
|
851
|
+
function resolve(start: Node | null, dir: 'prev' | 'next'): Node | null | typeof PENDING {
|
|
852
|
+
let anchor = start
|
|
853
|
+
for (let hops = 0; anchor !== null && hops <= removals.size; hops++) {
|
|
854
|
+
const done = placed.get(anchor)
|
|
855
|
+
if (done !== undefined) return follow(done)
|
|
856
|
+
if (anchor.parentNode === plan.live) return follow(plan.childOf.get(anchor))
|
|
857
|
+
const removal = removals.get(anchor)
|
|
858
|
+
if (removal === undefined || removal.parent !== plan.live) return null
|
|
859
|
+
if (pending.has(anchor) && !allowPending) return PENDING
|
|
860
|
+
// A repeat of a stashed row stands where that row is restored.
|
|
861
|
+
if (removal.row === null && removal.same !== undefined && removal.same !== anchor) { anchor = removal.same; continue }
|
|
862
|
+
anchor = dir === 'prev' ? removal.prev : removal.next
|
|
863
|
+
}
|
|
864
|
+
return null
|
|
865
|
+
}
|
|
866
|
+
let count = 0
|
|
867
|
+
// A row already in the list is not restored again; one whose live twin
|
|
868
|
+
// (same identity attribute) is a far emptier placeholder replaces it.
|
|
869
|
+
function dedupe(live: Node, row: NonNullable<Removal['row']>): boolean {
|
|
870
|
+
const sameIdent = row.ident === null ? undefined : byIdent.get(row.ident)
|
|
871
|
+
const twin = sameIdent ?? byText.get(`${row.node.nodeName}|${row.text}`)
|
|
872
|
+
if (twin === undefined) return false
|
|
873
|
+
const twinText = textOf.get(twin) ?? ''
|
|
874
|
+
if (sameIdent !== undefined && twinText.length * 4 <= row.text.length && row.text.length - twinText.length > p.textDegradeTolerance) {
|
|
875
|
+
const node = fresh.importNode(row.node, true)
|
|
876
|
+
twin.replaceWith(node)
|
|
877
|
+
forget(twin)
|
|
878
|
+
replacedBy.set(twin, node)
|
|
879
|
+
note(node)
|
|
880
|
+
placed.set(live, node)
|
|
881
|
+
count++
|
|
882
|
+
} else {
|
|
883
|
+
placed.set(live, twin)
|
|
884
|
+
}
|
|
885
|
+
return true
|
|
886
|
+
}
|
|
887
|
+
// Stubs: the placeholder an unkeyed cloak puts in a row's place carries
|
|
888
|
+
// a short piece of the row's own text ("posted by …"). Whichever of the
|
|
889
|
+
// two the merge meets second, only the row survives.
|
|
890
|
+
const restoredHere = new Set<Element>()
|
|
891
|
+
const stubOf = (stub: string, full: string): boolean => stub !== '' && stub.length * 4 <= full.length && full.includes(stub)
|
|
892
|
+
function insert(live: Node, row: NonNullable<Removal['row']>, place: (node: Element) => void): void {
|
|
893
|
+
const node = fresh.importNode(row.node, true)
|
|
894
|
+
place(node)
|
|
895
|
+
const sides = [node.previousElementSibling, node.nextElementSibling]
|
|
896
|
+
// A restored stub beside the row it stood in for adds nothing.
|
|
897
|
+
for (const side of sides) {
|
|
898
|
+
if (side !== null && stubOf(row.text, textOf.get(side) ?? '')) {
|
|
899
|
+
node.remove()
|
|
900
|
+
placed.set(live, side)
|
|
901
|
+
return
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
note(node)
|
|
905
|
+
placed.set(live, node)
|
|
906
|
+
restoredHere.add(node)
|
|
907
|
+
count++
|
|
908
|
+
// A stub beside the restored row goes — one the engine ADDED during the
|
|
909
|
+
// walk, or one restored earlier; never a row the page had all along.
|
|
910
|
+
for (const side of sides) {
|
|
911
|
+
if (side === null || keyOf(side) !== null || !(plan.added.has(side) || restoredHere.has(side))) continue
|
|
912
|
+
if (!stubOf(textOf.get(side) ?? '', row.text)) continue
|
|
913
|
+
forget(side)
|
|
914
|
+
side.remove()
|
|
915
|
+
replacedBy.set(side, node)
|
|
916
|
+
if (restoredHere.delete(side)) count--
|
|
917
|
+
}
|
|
918
|
+
}
|
|
919
|
+
let queue = rows
|
|
920
|
+
while (queue.length > 0) {
|
|
921
|
+
const later: typeof rows = []
|
|
922
|
+
let progressed = false
|
|
923
|
+
for (const entry of queue) {
|
|
924
|
+
const [live, removal] = entry
|
|
925
|
+
const row = removal.row!
|
|
926
|
+
if (dedupe(live, row)) { pending.delete(live); progressed = true; continue }
|
|
927
|
+
// Every sighting of the row knows where it sat at that instant; the
|
|
928
|
+
// first that resolves wins (an anchored open's first window, say,
|
|
929
|
+
// was removed beside rows only a later pass ever pinned down).
|
|
930
|
+
let place: ((node: Element) => void) | null = null
|
|
931
|
+
for (const sighting of [removal, ...(repeats.get(live) ?? [])]) {
|
|
932
|
+
const before = resolve(sighting.next, 'next')
|
|
933
|
+
if (before !== null && before !== PENDING) { place = node => { (before as ChildNode).before(node) }; break }
|
|
934
|
+
const after = resolve(sighting.prev, 'prev')
|
|
935
|
+
if (after !== null && after !== PENDING) { place = node => { (after as ChildNode).after(node) }; break }
|
|
936
|
+
}
|
|
937
|
+
if (place === null) {
|
|
938
|
+
later.push(entry)
|
|
939
|
+
continue
|
|
940
|
+
}
|
|
941
|
+
insert(live, row, place)
|
|
942
|
+
pending.delete(live)
|
|
943
|
+
progressed = true
|
|
944
|
+
}
|
|
945
|
+
queue = later
|
|
946
|
+
if (!progressed) {
|
|
947
|
+
if (allowPending) break
|
|
948
|
+
allowPending = true
|
|
949
|
+
}
|
|
950
|
+
}
|
|
951
|
+
// Rows with no surviving relative (a list re-rendered wholesale leaves no
|
|
952
|
+
// anchor behind): a walk runs top-down and ends at the bottom, so what
|
|
953
|
+
// the engine removed and did not render again lay ABOVE the final
|
|
954
|
+
// window — before the first text-bearing row, in removal order.
|
|
955
|
+
if (queue.length > 0) {
|
|
956
|
+
const first = Array.from(plan.fresh.children).find(el => (textOf.get(el) ?? '') !== '') ?? null
|
|
957
|
+
for (const [live, removal] of queue) {
|
|
958
|
+
const row = removal.row!
|
|
959
|
+
if (dedupe(live, row)) continue
|
|
960
|
+
insert(live, row, node => { plan.fresh.insertBefore(node, first) })
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
return count
|
|
964
|
+
}
|
|
965
|
+
|
|
557
966
|
const stats: DeepCapture = {
|
|
558
967
|
steps,
|
|
559
968
|
stashed: stash.size,
|
|
560
969
|
rehydrated,
|
|
561
970
|
spliced,
|
|
971
|
+
restored,
|
|
562
972
|
reactive,
|
|
563
973
|
elapsedMs: Date.now() - started,
|
|
564
974
|
height: finalHeight,
|
|
565
975
|
}
|
|
566
|
-
if ((rehydrated > 0 || spliced > 0) && fresh.documentElement !== null) {
|
|
976
|
+
if ((rehydrated > 0 || spliced > 0 || restored > 0) && fresh.documentElement !== null) {
|
|
567
977
|
strip(fresh)
|
|
568
978
|
// page.content()'s own serialization: doctype, then the root element.
|
|
569
979
|
const doctype = fresh.doctype === null ? '' : new XMLSerializer().serializeToString(fresh.doctype)
|
|
570
980
|
stats.html = doctype + fresh.documentElement.outerHTML
|
|
981
|
+
} else if (reactive && curY() !== startY) {
|
|
982
|
+
// Nothing restored on a page that reacted: the provider reads the LIVE
|
|
983
|
+
// page, which now shows the walk's last (bottom) window. A virtualizer
|
|
984
|
+
// the merge could not rebuild (rows without text, canvas) would then
|
|
985
|
+
// hand over the tail instead of the opening screen the legacy capture
|
|
986
|
+
// read, so return to where the page opened and let it re-render there.
|
|
987
|
+
scrollToY(startY)
|
|
988
|
+
await settleQuiet()
|
|
989
|
+
stats.rewound = true
|
|
571
990
|
}
|
|
991
|
+
stats.elapsedMs = Date.now() - started
|
|
572
992
|
return stats
|
|
993
|
+
|
|
994
|
+
// Wait until the page stops adding or removing elements (bounded).
|
|
995
|
+
async function settleQuiet(): Promise<void> {
|
|
996
|
+
let churn = 0
|
|
997
|
+
const watch = new MutationObserver(records => {
|
|
998
|
+
for (const record of records) if (counts(record.addedNodes) || counts(record.removedNodes)) churn++
|
|
999
|
+
})
|
|
1000
|
+
watch.observe(body!, { childList: true, subtree: true })
|
|
1001
|
+
try {
|
|
1002
|
+
for (let round = 0; round < 5; round++) {
|
|
1003
|
+
await sleep(p.stepSettleMs)
|
|
1004
|
+
for (const record of watch.takeRecords()) if (counts(record.addedNodes) || counts(record.removedNodes)) churn++
|
|
1005
|
+
if (churn === 0) break
|
|
1006
|
+
churn = 0
|
|
1007
|
+
}
|
|
1008
|
+
} finally {
|
|
1009
|
+
watch.disconnect()
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
573
1012
|
}
|
|
574
1013
|
|
|
575
1014
|
/**
|
|
@@ -585,6 +1024,7 @@ export function deepCaptureParams(budgetMs: number): DeepCaptureParams {
|
|
|
585
1024
|
stepSettleMs: SCROLL_STEP_SETTLE_MS,
|
|
586
1025
|
bottomPollMs: SCROLL_BOTTOM_POLL_MS,
|
|
587
1026
|
bottomPatienceMs: SCROLL_BOTTOM_PATIENCE_MS,
|
|
1027
|
+
inertPatienceMs: SCROLL_INERT_PATIENCE_MS,
|
|
588
1028
|
inertSteps: SCROLL_INERT_STEPS,
|
|
589
1029
|
maxSteps: SCROLL_MAX_STEPS,
|
|
590
1030
|
stashMaxEntries: SCROLL_STASH_MAX_ENTRIES,
|
|
@@ -640,6 +1080,8 @@ export async function deepCaptureWalk(page: PlaywrightPage, budgetMs: number): P
|
|
|
640
1080
|
stashed: raw.stashed,
|
|
641
1081
|
rehydrated: raw.rehydrated,
|
|
642
1082
|
...(typeof raw.spliced === 'number' ? { spliced: raw.spliced } : {}),
|
|
1083
|
+
...(typeof raw.restored === 'number' ? { restored: raw.restored } : {}),
|
|
1084
|
+
...(typeof raw.rewound === 'boolean' ? { rewound: raw.rewound } : {}),
|
|
643
1085
|
...(typeof raw.reactive === 'boolean' ? { reactive: raw.reactive } : {}),
|
|
644
1086
|
...(typeof raw.elapsedMs === 'number' ? { elapsedMs: raw.elapsedMs } : {}),
|
|
645
1087
|
...(typeof raw.height === 'number' ? { height: raw.height } : {}),
|