dsh-web-fetch-playwright 0.2.20 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/autoscroll.ts CHANGED
@@ -60,7 +60,8 @@
60
60
  * collecting after each, until the walk can hold AT THE BOTTOM through a
61
61
  * {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
62
62
  * (growth restarts the clock — that is what pages load-more sentinels),
63
- * or the step/budget caps close it.
63
+ * or the step/budget caps close it. A page that never reacted and issued
64
+ * no fetch/XHR holds only {@link SCROLL_INERT_PATIENCE_MS}.
64
65
  *
65
66
  * A MutationObserver watches for the walk's whole reason to exist: the
66
67
  * page REACTING to scroll by adding or removing elements. Most of the web
@@ -71,6 +72,14 @@
71
72
  * jumped over may hold content that never rendered, so the walk rewinds
72
73
  * to where it jumped from and finishes carefully. Correctness never
73
74
  * rests on the warm-up guess; only the cost of a scroll-inert page does.
75
+ *
76
+ * The same observer HARVESTS UNKEYED ROWS — the third shape: react-window
77
+ * lists and X-style timelines give rows no DOM identity at all (the
78
+ * framework key never reaches the DOM), and framework-generated ids
79
+ * (`ember123`, React useId's `:r1:`) change per render so they are not
80
+ * keys either. When the engine removes such a row, the record still
81
+ * holds it whole plus the sibling nodes it sat between; that is the row's
82
+ * stash and its splice anchor, with no layout read at all.
74
83
  * 4. MERGE into a structural clone (`document.cloneNode(true)`: no
75
84
  * serialize/re-parse round trip — a runtime-rewritten data script with a
76
85
  * literal `</script>` in its text cannot corrupt it — and, having no
@@ -84,11 +93,21 @@
84
93
  * spliced parent is already present and skipped. Placement is
85
94
  * structural, so it is right whatever order the walk SAW rows in (an
86
95
  * anchored open starts mid-document).
96
+ * - RESTORE (unkeyed rows): in every list the engine both removed text-
97
+ * bearing rows from and added to during the walk, each removed row the
98
+ * list no longer holds goes back beside the sibling it was removed
99
+ * next to (chains through removed anchors; rows with no surviving
100
+ * anchor at all — a wholesale innerHTML re-render — go above the final
101
+ * window, which is the only place a top-down walk loses them). Rows
102
+ * still present are recognised by a per-list identity attribute
103
+ * (data-index, aria-rowindex, …) or by text; in-place stubs are
104
+ * absorbed.
87
105
  * - SWAP: every keyed element emptier than its snapshot (fewer child
88
106
  * elements, or meaningfully less text — the Discourse placeholder) is
89
107
  * replaced by that snapshot, except containers (ancestors of another
90
108
  * element being swapped), which are assembled from those descendants
91
- * instead: a container's own snapshot is one window's partial view.
109
+ * instead: a container's own snapshot is one window's partial view. A
110
+ * restored list and its ancestors are containers too.
92
111
  * The clone is then stripped of scripting elements and serialized; the
93
112
  * provider reads that string instead of the live page. The live document
94
113
  * is never rewritten, so no late scroll/rAF/timer reaction can touch the
@@ -98,13 +117,16 @@
98
117
  * error, or an engine that rejects the walk just keeps the legacy
99
118
  * single-instant capture — a walk can only add content, never fail a fetch.
100
119
  * A walk that restored nothing returns no HTML, and the provider reads the
101
- * live page exactly as before.
120
+ * live page exactly as before — scrolled back to where it opened when the
121
+ * page reacted, so a virtualizer the merge could not rebuild hands over its
122
+ * opening window rather than the walk's last one.
102
123
  *
103
124
  * The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
104
125
  * cloaked placeholders) but never for non-HTML documents. It is bounded by
105
126
  * the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
106
- * and always by the remaining fetch deadline minus a decode reserve — the
107
- * provider's abort closes the tab, which rejects the pending evaluation.
127
+ * and always by the fetch's remaining waiting budget: a walk cut short there
128
+ * still merges and returns what it stashed, and the content read plus decode
129
+ * run in the provider's extraction grace window after it.
108
130
  *
109
131
  * @module dsh-web-fetch-playwright/autoscroll
110
132
  */
@@ -143,6 +165,17 @@ export const SCROLL_BOTTOM_POLL_MS = 450
143
165
  */
144
166
  export const SCROLL_BOTTOM_PATIENCE_MS = 2_500
145
167
 
168
+ /**
169
+ * Bottom hold for a page that has given no sign of loading more: it never
170
+ * added or removed an element in response to the walk's scrolling, and it
171
+ * issued no fetch/XHR since the walk began. The full
172
+ * {@link SCROLL_BOTTOM_PATIENCE_MS} exists for feeds whose next chunk is in
173
+ * flight; holding it on every static article made the walk cost ~4s on
174
+ * pages it could never help. Any reaction or data request (even one that
175
+ * lands during this shorter hold) restores the full patience.
176
+ */
177
+ export const SCROLL_INERT_PATIENCE_MS = 900
178
+
146
179
  /**
147
180
  * Careful steps without any DOM reaction before the walk treats the page as
148
181
  * scroll-inert and probes the rest with a single jump (rewinding if the
@@ -175,12 +208,6 @@ export const SCROLL_STASH_MAX_CHARS = 2_000_000
175
208
  */
176
209
  export const SCROLL_STASH_MAX_ENTRY_CHARS = 64_000
177
210
 
178
- /**
179
- * Tail of the fetch budget the walk never spends, keeping room for the
180
- * content read and the synchronous denoise decode.
181
- */
182
- export const SCROLL_FINISH_RESERVE_MS = 4_000
183
-
184
211
  /**
185
212
  * A walk budget below this cannot complete even the first window plus one
186
213
  * stability check, so the walk is skipped outright.
@@ -213,6 +240,14 @@ export interface DeepCaptureStats {
213
240
  rehydrated: number
214
241
  /** Removed keyed elements spliced back (container walks only). */
215
242
  spliced?: number
243
+ /** Unkeyed rows a virtualizer removed, restored into their list. */
244
+ restored?: number
245
+ /**
246
+ * True when nothing could be restored on a page that reacted, so the walk
247
+ * scrolled back to where the page opened before handing it to the
248
+ * provider's live read.
249
+ */
250
+ rewound?: boolean
216
251
  /**
217
252
  * Whether the page reacted to scrolling (added/removed elements). False
218
253
  * means the walk took the scroll-inert fast path to the bottom.
@@ -244,6 +279,7 @@ export interface DeepCaptureParams {
244
279
  stepSettleMs: number
245
280
  bottomPollMs: number
246
281
  bottomPatienceMs: number
282
+ inertPatienceMs: number
247
283
  inertSteps: number
248
284
  maxSteps: number
249
285
  stashMaxEntries: number
@@ -327,8 +363,13 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
327
363
  // ── Identity ────────────────────────────────────────────────────────────
328
364
  // id, else data-block-id (namespaced so the two can never collide): app-
329
365
  // shell editors key their blocks that way and carry no id at all.
366
+ // Framework-GENERATED ids are per-render, not identities: a row re-rendered
367
+ // by Ember (`ember123`) or React's useId (`:r1:`, `«r1»`, `_r_1_`) comes
368
+ // back under a new id, so keying on one only ever stashes dead snapshots.
369
+ // Such elements count as unkeyed and take the unkeyed-row path below.
370
+ const GENERATED_ID = /^ember\d+$|:r[0-9a-z]*:|«r[0-9a-z]*»|^_r_[0-9a-z]+_$/
330
371
  function keyOf(el: Element): string | null {
331
- if (el.id !== '') return el.id
372
+ if (el.id !== '' && !GENERATED_ID.test(el.id)) return el.id
332
373
  const block = el.getAttribute('data-block-id')
333
374
  return block !== null && block !== '' ? `data-block-id:${block}` : null
334
375
  }
@@ -357,11 +398,36 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
357
398
  const tally = (records: MutationRecord[]): void => {
358
399
  for (const record of records) {
359
400
  if (counts(record.addedNodes) || counts(record.removedNodes)) reactions++
401
+ harvest(record)
360
402
  }
361
403
  }
362
- const observer = new MutationObserver(tally)
363
- observer.observe(body, { childList: true, subtree: true })
404
+ // (The observer itself starts below, once the unkeyed-row harvest it
405
+ // feeds is declared.)
364
406
  const reacted = (): boolean => { tally(observer.takeRecords()); return reactions > 0 }
407
+ // Data requests (fetch/XHR): the other sign of a feed loading more.
408
+ // Images, fonts and beacons are not. Observed rather than read from the
409
+ // resource buffer, which stops growing once full (250 entries in Chromium)
410
+ // on exactly the heavy pages that matter. Resource Timing reports a
411
+ // request only once it COMPLETES: a next chunk still in flight when the
412
+ // short inert hold expires goes unseen, which is the price of that hold.
413
+ let requests = 0
414
+ const tallyRequests = (entries: PerformanceEntryList): void => {
415
+ for (const entry of entries) {
416
+ const type = (entry as PerformanceResourceTiming).initiatorType
417
+ if (type === 'fetch' || type === 'xmlhttprequest') requests++
418
+ }
419
+ }
420
+ let requestObserver: PerformanceObserver | null = null
421
+ try {
422
+ requestObserver = new PerformanceObserver(list => { tallyRequests(list.getEntries()) })
423
+ requestObserver.observe({ type: 'resource' })
424
+ } catch {
425
+ requestObserver = null // no Resource Timing: reactions alone decide
426
+ }
427
+ const requested = (): boolean => {
428
+ if (requestObserver !== null) tallyRequests(requestObserver.takeRecords())
429
+ return requests > 0
430
+ }
365
431
 
366
432
  // ── Stash ──────────────────────────────────────────────────────────────
367
433
  interface Snapshot {
@@ -379,6 +445,118 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
379
445
  const stash = new Map<string, Snapshot>()
380
446
  const oversized = new Set<string>()
381
447
  let stashChars = 0
448
+
449
+ // ── Unkeyed rows ───────────────────────────────────────────────────────
450
+ // Many virtualizers give their rows no DOM identity at all (react-window
451
+ // rows, X-style timeline cells: the framework key never reaches the DOM).
452
+ // Such a row is harvested at the instant the engine REMOVES it: the
453
+ // MutationObserver record hands over the detached row, still whole, plus
454
+ // the sibling nodes it sat between — enough to splice it back into the
455
+ // same list at merge without reading any layout (absolute positioning and
456
+ // transforms do not matter). Nodes are tracked by object identity, so
457
+ // even an anonymous spacer or a row filtered out as noise still anchors
458
+ // its neighbors.
459
+ const ROW_IDENT_ATTRS = [
460
+ 'data-index', 'data-item-index', 'data-row-index', 'data-rowindex', 'aria-rowindex', 'aria-posinset',
461
+ 'data-key', 'data-row-key', 'data-item-key', 'data-id', 'data-item-id',
462
+ ]
463
+ // Transient UI — toasts, live status, spinners, menus — is never content.
464
+ const NOISE_ROLES = new Set(['status', 'alert', 'alertdialog', 'dialog', 'tooltip', 'progressbar', 'menu', 'listbox', 'timer', 'marquee'])
465
+ const squash = (s: string): string => s.replace(/\s+/g, ' ').trim()
466
+ // Identity for de-duplication against the rows still live: a per-LIST
467
+ // attribute when the engine sets one (scoped to the list, so data-index
468
+ // values repeating across lists cannot collide), else the row's text.
469
+ function rowIdent(el: Element): string | null {
470
+ for (const attr of ROW_IDENT_ATTRS) {
471
+ const value = el.getAttribute(attr)
472
+ if (value !== null && value !== '') return `${attr}=${value}`
473
+ }
474
+ return null
475
+ }
476
+ function rowText(el: Element): string | null {
477
+ if (keyOf(el) !== null || INERT_TAGS.has(el.nodeName) || el.namespaceURI === SVG_NS) return null
478
+ if (el.hasAttribute('hidden') || el.getAttribute('aria-hidden') === 'true' || el.getAttribute('aria-busy') === 'true') return null
479
+ const role = el.getAttribute('role')
480
+ if (role !== null && NOISE_ROLES.has(role)) return null
481
+ const text = squash(el.textContent ?? '')
482
+ return text === '' ? null : text
483
+ }
484
+ interface Removal {
485
+ row: { node: Element; text: string; ident: string | null; len: number } | null // null: anchor-only ghost
486
+ /** A ghost repeating an already-stashed row: the node that holds its copy. */
487
+ same?: Node
488
+ parent: Node
489
+ prev: Node | null
490
+ next: Node | null
491
+ seq: number
492
+ }
493
+ const removals = new Map<Node, Removal>()
494
+ // Per list: the one stashed copy of each row identity. An engine that
495
+ // re-renders every frame removes the same row thousands of times; only
496
+ // the first sighting (richest wins) is stashed, the rest only anchor.
497
+ const rowsSeen = new Map<Node, Map<string, Node>>()
498
+ const lists = new Map<Node, { rows: number; added: number }>()
499
+ const addedNodes = new WeakSet<Node>()
500
+ let removalSeq = 0
501
+ let rowsStashed = 0
502
+ // Its OWN character budget: rows churned inside keyed content (a forum
503
+ // engine re-rendering post internals) must never starve the keyed stash,
504
+ // which is what the established feed shapes rely on.
505
+ let rowChars = 0
506
+ // Anchor-only bookkeeping is cheap but holds detached nodes alive.
507
+ const MAX_REMOVALS = 50_000
508
+ function harvest(record: MutationRecord): void {
509
+ const parent = record.target
510
+ if (record.type !== 'childList' || parent.nodeType !== 1 || parent === body || !parent.isConnected) return
511
+ if (scroller !== null ? !scroller.contains(parent) : !body.contains(parent)) return
512
+ let list = lists.get(parent)
513
+ if (list === undefined) { list = { rows: 0, added: 0 }; lists.set(parent, list) }
514
+ for (const node of Array.from(record.addedNodes)) {
515
+ addedNodes.add(node)
516
+ if (node.nodeType === 1 && !INERT_TAGS.has(node.nodeName)) list.added++
517
+ }
518
+ // One record may remove a whole run (innerHTML, replaceChildren): its
519
+ // members anchor on each other, the run's ends on the record's siblings.
520
+ const gone = Array.from(record.removedNodes)
521
+ let seen = rowsSeen.get(parent)
522
+ if (seen === undefined) { seen = new Map(); rowsSeen.set(parent, seen) }
523
+ gone.forEach((node, i) => {
524
+ if (removals.size >= MAX_REMOVALS) return
525
+ const removal: Removal = {
526
+ row: null,
527
+ parent,
528
+ prev: i > 0 ? gone[i - 1]! : record.previousSibling,
529
+ next: i < gone.length - 1 ? gone[i + 1]! : record.nextSibling,
530
+ seq: removalSeq++,
531
+ }
532
+ removals.set(node, removal)
533
+ const text = node.nodeType === 1 ? rowText(node as Element) : null
534
+ if (text === null) return
535
+ const el = node as Element
536
+ const ident = rowIdent(el)
537
+ const identity = ident ?? `${el.nodeName}|${text}`
538
+ const priorNode = seen.get(identity)
539
+ const prior = priorNode === undefined ? undefined : removals.get(priorNode)
540
+ if (priorNode !== undefined) removal.same = priorNode
541
+ if (prior?.row != null && prior.row.text.length >= text.length) return
542
+ if (prior === undefined && rowsStashed >= p.stashMaxEntries) return
543
+ const len = el.outerHTML.length
544
+ const freed = prior?.row?.len ?? 0
545
+ if (len > p.stashMaxEntryChars || rowChars - freed + len > p.stashMaxChars) return
546
+ rowChars += len - freed
547
+ const row = { node: stashDoc.importNode(el, true), text, ident, len }
548
+ if (prior !== undefined) {
549
+ prior.row = row // a richer sighting, kept at the FIRST sighting's place
550
+ } else {
551
+ removal.row = row
552
+ seen.set(identity, node)
553
+ rowsStashed++
554
+ list.rows++
555
+ }
556
+ })
557
+ }
558
+ const observer = new MutationObserver(tally)
559
+ observer.observe(body, { childList: true, subtree: true })
382
560
  const host: HTMLElement = body
383
561
  function collect(): void {
384
562
  for (const el of Array.from(host.querySelectorAll(KEYED))) {
@@ -420,6 +598,7 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
420
598
  // ── Walk ───────────────────────────────────────────────────────────────
421
599
  // Stash whatever the (possibly anchored) start renders, then restart from
422
600
  // the top so the walk covers the whole target.
601
+ const startY = curY()
423
602
  collect()
424
603
  if (curY() > 0) {
425
604
  scrollToY(0)
@@ -429,6 +608,8 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
429
608
  // Only reactions to the walk's own scrolling count from here on.
430
609
  tally(observer.takeRecords())
431
610
  reactions = 0
611
+ requested()
612
+ requests = 0
432
613
 
433
614
  let steps = 0
434
615
  let lastHeight = scrollH()
@@ -460,7 +641,9 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
460
641
  collect()
461
642
  const before = lastHeight
462
643
  noteHeight()
463
- if (lastHeight === before && Date.now() - lastChangeAt >= p.bottomPatienceMs) break
644
+ // `careful` implies reacted(): reactions are never reset mid-walk.
645
+ const patience = reacted() || requested() ? p.bottomPatienceMs : p.inertPatienceMs
646
+ if (lastHeight === before && Date.now() - lastChangeAt >= patience) break
464
647
  continue
465
648
  }
466
649
  if (!careful && jumpedFrom === null) {
@@ -487,13 +670,64 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
487
670
  noteHeight()
488
671
  }
489
672
  } finally {
673
+ // Flush first: disconnect() drops queued records, and the last batch of
674
+ // removals is exactly what the unkeyed-row merge needs.
675
+ tally(observer.takeRecords())
490
676
  observer.disconnect()
677
+ requestObserver?.disconnect()
491
678
  }
492
679
  const reactive = reacted()
493
680
  const finalHeight = scrollH()
494
681
 
495
682
  // ── Merge into a structural clone (synchronous from here on) ──────────
496
683
  const fresh = document.cloneNode(true) as Document
684
+
685
+ // Unkeyed lists to restore, mapped into the clone NOW — the clone is an
686
+ // exact structural copy only until the merge starts editing it. A list
687
+ // qualifies when the engine both removed text-bearing rows from it and
688
+ // added elements to it during the walk (a virtualizer's churn; a lone
689
+ // toast or a load-more append does not), and it is not a live/status
690
+ // region or carousel whose churn is rotation rather than windowing.
691
+ interface ListPlan {
692
+ live: Node
693
+ fresh: Element
694
+ childOf: Map<Node, Node> // live child → its clone twin
695
+ added: Set<Node> // clone twins of children the engine added during the walk
696
+ }
697
+ const NOISE_REGION = '[aria-live="assertive"], [role="alert"], [role="status"], [role="timer"], [role="marquee"], [aria-roledescription~="carousel"]'
698
+ function freshOf(live: Node): Node | null {
699
+ const path: number[] = []
700
+ for (let at: Node = live; at !== document;) {
701
+ const up = at.parentNode
702
+ if (up === null) return null
703
+ path.push(Array.prototype.indexOf.call(up.childNodes, at) as number)
704
+ at = up
705
+ }
706
+ let at: Node = fresh
707
+ for (let i = path.length - 1; i >= 0; i--) {
708
+ const child = at.childNodes[path[i]!]
709
+ if (child === undefined) return null
710
+ at = child
711
+ }
712
+ return at
713
+ }
714
+ const plans: ListPlan[] = []
715
+ for (const [live, list] of lists) {
716
+ if (list.rows < 2 || list.added < 1 || !live.isConnected || (live as Element).closest(NOISE_REGION) !== null) continue
717
+ const twin = freshOf(live)
718
+ if (twin === null || twin.nodeType !== 1) continue
719
+ const liveKids = Array.from(live.childNodes)
720
+ const freshKids = Array.from(twin.childNodes)
721
+ if (liveKids.length !== freshKids.length) continue
722
+ const childOf = new Map<Node, Node>()
723
+ const added = new Set<Node>()
724
+ liveKids.forEach((child, i) => {
725
+ childOf.set(child, freshKids[i]!)
726
+ if (addedNodes.has(child)) added.add(freshKids[i]!)
727
+ })
728
+ plans.push({ live, fresh: twin as Element, childOf, added })
729
+ }
730
+
497
731
  const present = new Map<string, Element>()
498
732
  const index = (node: Element): void => {
499
733
  const key = keyOf(node)
@@ -532,8 +766,21 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
532
766
  }
533
767
  }
534
768
 
769
+ // RESTORE unkeyed rows into their lists, oldest removal first — before the
770
+ // swap, which must see each list already whole.
771
+ // Clone nodes the merge replaced, so a list anchor on one still resolves.
772
+ const replacedBy = new Map<Node, Node>()
773
+ let restored = 0
774
+ const restoredLists: Element[] = []
775
+ for (const plan of plans) {
776
+ const count = restoreList(plan)
777
+ if (count > 0) { restored += count; restoredLists.push(plan.fresh) }
778
+ }
779
+
535
780
  // SWAP degraded leaves for their snapshots; their ancestors are
536
- // containers, assembled from them rather than swapped wholesale.
781
+ // containers, assembled from them rather than swapped wholesale. So is a
782
+ // restored list and everything around it: a keyed wrapper's snapshot is
783
+ // one window's partial view of the rows the restore just made whole.
537
784
  const degraded: Array<{ el: Element; snap: Snapshot }> = []
538
785
  for (const el of Array.from(fresh.querySelectorAll(KEYED))) {
539
786
  const key = keyOf(el)
@@ -547,6 +794,9 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
547
794
  for (const { el } of degraded) {
548
795
  for (let up = el.parentElement; up !== null && !containers.has(up); up = up.parentElement) containers.add(up)
549
796
  }
797
+ for (const list of restoredLists) {
798
+ for (let up: Element | null = list; up !== null && !containers.has(up); up = up.parentElement) containers.add(up)
799
+ }
550
800
  let rehydrated = 0
551
801
  for (const { el, snap } of degraded) {
552
802
  if (containers.has(el)) continue
@@ -554,22 +804,211 @@ export async function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Docum
554
804
  rehydrated++
555
805
  }
556
806
 
807
+ function restoreList(plan: ListPlan): number {
808
+ const follow = (node: Node | undefined): Node | null => {
809
+ let at = node
810
+ while (at !== undefined && replacedBy.has(at)) at = replacedBy.get(at)
811
+ return at !== undefined && at.parentNode === plan.fresh ? at : null
812
+ }
813
+ // What the list holds now, for de-duplication: identity attribute and
814
+ // text of every element child (live rows, spliced or swapped ones).
815
+ const byIdent = new Map<string, Element>()
816
+ const byText = new Map<string, Element>()
817
+ const textOf = new Map<Element, string>()
818
+ const note = (el: Element): void => {
819
+ const text = squash(el.textContent ?? '')
820
+ textOf.set(el, text)
821
+ const ident = rowIdent(el)
822
+ if (ident !== null && !byIdent.has(ident)) byIdent.set(ident, el)
823
+ if (text !== '' && !byText.has(`${el.nodeName}|${text}`)) byText.set(`${el.nodeName}|${text}`, el)
824
+ }
825
+ const forget = (el: Element): void => {
826
+ const text = textOf.get(el) ?? ''
827
+ const ident = rowIdent(el)
828
+ if (ident !== null && byIdent.get(ident) === el) byIdent.delete(ident)
829
+ if (byText.get(`${el.nodeName}|${text}`) === el) byText.delete(`${el.nodeName}|${text}`)
830
+ }
831
+ for (const child of Array.from(plan.fresh.children)) note(child)
832
+
833
+ const placed = new Map<Node, Node>() // live removed node → clone node it became
834
+ const rows = Array.from(removals.entries())
835
+ .filter(([node, r]) => r.parent === plan.live && r.row !== null && !node.isConnected)
836
+ .sort((a, b) => a[1].seq - b[1].seq)
837
+ const pending = new Set(rows.map(([node]) => node))
838
+ // Later removals of the same row (ghosts pointing at its stashed copy).
839
+ const repeats = new Map<Node, Removal[]>()
840
+ for (const removal of removals.values()) {
841
+ if (removal.row !== null || removal.same === undefined || removal.parent !== plan.live) continue
842
+ const list = repeats.get(removal.same)
843
+ if (list === undefined) repeats.set(removal.same, [removal])
844
+ else list.push(removal)
845
+ }
846
+ let allowPending = false
847
+ const PENDING = 1
848
+ // Walk an anchor chain to a node of the clone list: a live sibling, a
849
+ // row already restored, or — through rows that will never come back
850
+ // (ghosts, duplicates) — whatever THEY sat next to.
851
+ function resolve(start: Node | null, dir: 'prev' | 'next'): Node | null | typeof PENDING {
852
+ let anchor = start
853
+ for (let hops = 0; anchor !== null && hops <= removals.size; hops++) {
854
+ const done = placed.get(anchor)
855
+ if (done !== undefined) return follow(done)
856
+ if (anchor.parentNode === plan.live) return follow(plan.childOf.get(anchor))
857
+ const removal = removals.get(anchor)
858
+ if (removal === undefined || removal.parent !== plan.live) return null
859
+ if (pending.has(anchor) && !allowPending) return PENDING
860
+ // A repeat of a stashed row stands where that row is restored.
861
+ if (removal.row === null && removal.same !== undefined && removal.same !== anchor) { anchor = removal.same; continue }
862
+ anchor = dir === 'prev' ? removal.prev : removal.next
863
+ }
864
+ return null
865
+ }
866
+ let count = 0
867
+ // A row already in the list is not restored again; one whose live twin
868
+ // (same identity attribute) is a far emptier placeholder replaces it.
869
+ function dedupe(live: Node, row: NonNullable<Removal['row']>): boolean {
870
+ const sameIdent = row.ident === null ? undefined : byIdent.get(row.ident)
871
+ const twin = sameIdent ?? byText.get(`${row.node.nodeName}|${row.text}`)
872
+ if (twin === undefined) return false
873
+ const twinText = textOf.get(twin) ?? ''
874
+ if (sameIdent !== undefined && twinText.length * 4 <= row.text.length && row.text.length - twinText.length > p.textDegradeTolerance) {
875
+ const node = fresh.importNode(row.node, true)
876
+ twin.replaceWith(node)
877
+ forget(twin)
878
+ replacedBy.set(twin, node)
879
+ note(node)
880
+ placed.set(live, node)
881
+ count++
882
+ } else {
883
+ placed.set(live, twin)
884
+ }
885
+ return true
886
+ }
887
+ // Stubs: the placeholder an unkeyed cloak puts in a row's place carries
888
+ // a short piece of the row's own text ("posted by …"). Whichever of the
889
+ // two the merge meets second, only the row survives.
890
+ const restoredHere = new Set<Element>()
891
+ const stubOf = (stub: string, full: string): boolean => stub !== '' && stub.length * 4 <= full.length && full.includes(stub)
892
+ function insert(live: Node, row: NonNullable<Removal['row']>, place: (node: Element) => void): void {
893
+ const node = fresh.importNode(row.node, true)
894
+ place(node)
895
+ const sides = [node.previousElementSibling, node.nextElementSibling]
896
+ // A restored stub beside the row it stood in for adds nothing.
897
+ for (const side of sides) {
898
+ if (side !== null && stubOf(row.text, textOf.get(side) ?? '')) {
899
+ node.remove()
900
+ placed.set(live, side)
901
+ return
902
+ }
903
+ }
904
+ note(node)
905
+ placed.set(live, node)
906
+ restoredHere.add(node)
907
+ count++
908
+ // A stub beside the restored row goes — one the engine ADDED during the
909
+ // walk, or one restored earlier; never a row the page had all along.
910
+ for (const side of sides) {
911
+ if (side === null || keyOf(side) !== null || !(plan.added.has(side) || restoredHere.has(side))) continue
912
+ if (!stubOf(textOf.get(side) ?? '', row.text)) continue
913
+ forget(side)
914
+ side.remove()
915
+ replacedBy.set(side, node)
916
+ if (restoredHere.delete(side)) count--
917
+ }
918
+ }
919
+ let queue = rows
920
+ while (queue.length > 0) {
921
+ const later: typeof rows = []
922
+ let progressed = false
923
+ for (const entry of queue) {
924
+ const [live, removal] = entry
925
+ const row = removal.row!
926
+ if (dedupe(live, row)) { pending.delete(live); progressed = true; continue }
927
+ // Every sighting of the row knows where it sat at that instant; the
928
+ // first that resolves wins (an anchored open's first window, say,
929
+ // was removed beside rows only a later pass ever pinned down).
930
+ let place: ((node: Element) => void) | null = null
931
+ for (const sighting of [removal, ...(repeats.get(live) ?? [])]) {
932
+ const before = resolve(sighting.next, 'next')
933
+ if (before !== null && before !== PENDING) { place = node => { (before as ChildNode).before(node) }; break }
934
+ const after = resolve(sighting.prev, 'prev')
935
+ if (after !== null && after !== PENDING) { place = node => { (after as ChildNode).after(node) }; break }
936
+ }
937
+ if (place === null) {
938
+ later.push(entry)
939
+ continue
940
+ }
941
+ insert(live, row, place)
942
+ pending.delete(live)
943
+ progressed = true
944
+ }
945
+ queue = later
946
+ if (!progressed) {
947
+ if (allowPending) break
948
+ allowPending = true
949
+ }
950
+ }
951
+ // Rows with no surviving relative (a list re-rendered wholesale leaves no
952
+ // anchor behind): a walk runs top-down and ends at the bottom, so what
953
+ // the engine removed and did not render again lay ABOVE the final
954
+ // window — before the first text-bearing row, in removal order.
955
+ if (queue.length > 0) {
956
+ const first = Array.from(plan.fresh.children).find(el => (textOf.get(el) ?? '') !== '') ?? null
957
+ for (const [live, removal] of queue) {
958
+ const row = removal.row!
959
+ if (dedupe(live, row)) continue
960
+ insert(live, row, node => { plan.fresh.insertBefore(node, first) })
961
+ }
962
+ }
963
+ return count
964
+ }
965
+
557
966
  const stats: DeepCapture = {
558
967
  steps,
559
968
  stashed: stash.size,
560
969
  rehydrated,
561
970
  spliced,
971
+ restored,
562
972
  reactive,
563
973
  elapsedMs: Date.now() - started,
564
974
  height: finalHeight,
565
975
  }
566
- if ((rehydrated > 0 || spliced > 0) && fresh.documentElement !== null) {
976
+ if ((rehydrated > 0 || spliced > 0 || restored > 0) && fresh.documentElement !== null) {
567
977
  strip(fresh)
568
978
  // page.content()'s own serialization: doctype, then the root element.
569
979
  const doctype = fresh.doctype === null ? '' : new XMLSerializer().serializeToString(fresh.doctype)
570
980
  stats.html = doctype + fresh.documentElement.outerHTML
981
+ } else if (reactive && curY() !== startY) {
982
+ // Nothing restored on a page that reacted: the provider reads the LIVE
983
+ // page, which now shows the walk's last (bottom) window. A virtualizer
984
+ // the merge could not rebuild (rows without text, canvas) would then
985
+ // hand over the tail instead of the opening screen the legacy capture
986
+ // read, so return to where the page opened and let it re-render there.
987
+ scrollToY(startY)
988
+ await settleQuiet()
989
+ stats.rewound = true
571
990
  }
991
+ stats.elapsedMs = Date.now() - started
572
992
  return stats
993
+
994
+ // Wait until the page stops adding or removing elements (bounded).
995
+ async function settleQuiet(): Promise<void> {
996
+ let churn = 0
997
+ const watch = new MutationObserver(records => {
998
+ for (const record of records) if (counts(record.addedNodes) || counts(record.removedNodes)) churn++
999
+ })
1000
+ watch.observe(body!, { childList: true, subtree: true })
1001
+ try {
1002
+ for (let round = 0; round < 5; round++) {
1003
+ await sleep(p.stepSettleMs)
1004
+ for (const record of watch.takeRecords()) if (counts(record.addedNodes) || counts(record.removedNodes)) churn++
1005
+ if (churn === 0) break
1006
+ churn = 0
1007
+ }
1008
+ } finally {
1009
+ watch.disconnect()
1010
+ }
1011
+ }
573
1012
  }
574
1013
 
575
1014
  /**
@@ -585,6 +1024,7 @@ export function deepCaptureParams(budgetMs: number): DeepCaptureParams {
585
1024
  stepSettleMs: SCROLL_STEP_SETTLE_MS,
586
1025
  bottomPollMs: SCROLL_BOTTOM_POLL_MS,
587
1026
  bottomPatienceMs: SCROLL_BOTTOM_PATIENCE_MS,
1027
+ inertPatienceMs: SCROLL_INERT_PATIENCE_MS,
588
1028
  inertSteps: SCROLL_INERT_STEPS,
589
1029
  maxSteps: SCROLL_MAX_STEPS,
590
1030
  stashMaxEntries: SCROLL_STASH_MAX_ENTRIES,
@@ -640,6 +1080,8 @@ export async function deepCaptureWalk(page: PlaywrightPage, budgetMs: number): P
640
1080
  stashed: raw.stashed,
641
1081
  rehydrated: raw.rehydrated,
642
1082
  ...(typeof raw.spliced === 'number' ? { spliced: raw.spliced } : {}),
1083
+ ...(typeof raw.restored === 'number' ? { restored: raw.restored } : {}),
1084
+ ...(typeof raw.rewound === 'boolean' ? { rewound: raw.rewound } : {}),
643
1085
  ...(typeof raw.reactive === 'boolean' ? { reactive: raw.reactive } : {}),
644
1086
  ...(typeof raw.elapsedMs === 'number' ? { elapsedMs: raw.elapsedMs } : {}),
645
1087
  ...(typeof raw.height === 'number' ? { height: raw.height } : {}),