dsh-web-fetch-playwright 0.2.16 → 0.2.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/index.js CHANGED
@@ -9,6 +9,17 @@ import { fileURLToPath, pathToFileURL } from "node:url";
9
9
  import { accessSync, constants, readFileSync, realpathSync, statSync } from "node:fs";
10
10
  import { delimiter, dirname, join } from "node:path";
11
11
  //#region src/config.ts
12
+ /**
13
+ * Default bounded budget (ms) for the deep-capture scroll walk
14
+ * (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
15
+ * careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
16
+ * the bottom patience window): 20s walks ~25kpx with pagination, covering
17
+ * the measured linux.do badcase (~18kpx, 51 replies) with headroom.
18
+ * Scroll-inert pages take the walk's fast path and spend a few seconds.
19
+ */
20
+ const DEFAULT_SCROLL_BUDGET_MS = 2e4;
21
+ /** Ceiling the schema accepts for `scrollBudgetMs`. */
22
+ const MAX_SCROLL_BUDGET_MS = 3e4;
12
23
  /** Default CDP endpoint when the settings section leaves it blank. */
13
24
  const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
14
25
  /**
@@ -31,7 +42,8 @@ const Config = z.object({
31
42
  denoise: z.boolean().default(true).volatile(),
32
43
  maxConcurrency: z.number().step(1).min(1).max(200).volatile(),
33
44
  challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
34
- challengeRetries: z.number().step(1).min(0).max(3).default(1).volatile()
45
+ challengeRetries: z.number().step(1).min(0).max(3).default(1).volatile(),
46
+ scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile()
35
47
  });
36
48
  /**
37
49
  * Read the live references `apply` holds into one plain input for
@@ -51,7 +63,8 @@ function snapshotsOf(config) {
51
63
  denoise: config.denoise.get(),
52
64
  maxConcurrency: config.maxConcurrency.get(),
53
65
  challengeWaitMs: config.challengeWaitMs.get(),
54
- challengeRetries: config.challengeRetries.get()
66
+ challengeRetries: config.challengeRetries.get(),
67
+ scrollBudgetMs: config.scrollBudgetMs.get()
55
68
  };
56
69
  }
57
70
  /**
@@ -75,7 +88,8 @@ function resolveConfig(config) {
75
88
  denoise: resolved.denoise.get(),
76
89
  maxConcurrency: resolved.maxConcurrency.get(),
77
90
  challengeWaitMs: resolved.challengeWaitMs.get(),
78
- challengeRetries: resolved.challengeRetries.get()
91
+ challengeRetries: resolved.challengeRetries.get(),
92
+ scrollBudgetMs: resolved.scrollBudgetMs.get()
79
93
  };
80
94
  }
81
95
  /**
@@ -113,6 +127,17 @@ function effectiveChallengeRetries(config) {
113
127
  return 1;
114
128
  }
115
129
  /**
130
+ * The scroll-walk budget a fetch actually runs with: an explicit setting
131
+ * wins, else the schema default. `0` means the deep-capture walk is off.
132
+ *
133
+ * @param config - the resolved settings section (or any partial of it).
134
+ * @returns the bounded walk budget in milliseconds.
135
+ */
136
+ function effectiveScrollBudgetMs(config) {
137
+ if (typeof config.scrollBudgetMs === "number") return config.scrollBudgetMs;
138
+ return DEFAULT_SCROLL_BUDGET_MS;
139
+ }
140
+ /**
116
141
  * The context mode a fetch actually runs with: profile (a tab in the remote
117
142
  * browser's real profile) requires the CDP backend — a local launch has no
118
143
  * meaningful shared profile — and the share option (which defaults on, so an
@@ -452,9 +477,597 @@ const CHALLENGE_DOM_PROBE = `(() => {
452
477
  return false
453
478
  })()`;
454
479
  //#endregion
480
+ //#region src/strip.ts
481
+ /**
482
+ * Serialization safety for documents about to be flattened into an HTML
483
+ * string and re-parsed by jsdom: the one in-page cleanup both serialization
484
+ * paths share — the provider's live-document read and the deep-capture
485
+ * walk's merged clone.
486
+ *
487
+ * @module dsh-web-fetch-playwright/strip
488
+ */
489
+ /**
490
+ * In-page function that removes non-content scripting subtrees from a
491
+ * document before it is flattened into an HTML string — the live document
492
+ * before `page.content()`, or the deep-capture walk's merged clone before it
493
+ * serializes it itself (`src/autoscroll.ts`).
494
+ *
495
+ * It is evaluated BY SOURCE (`fn.toString()`), so it must stay
496
+ * self-contained: no imports, no module-level references.
497
+ *
498
+ * WHAT goes, and why each removal is output-neutral versus the old pipeline:
499
+ *
500
+ * - `script` — three consumers ever saw script text before, and none loses
501
+ * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
502
+ * extraction and scoring (so script text never reached the article or
503
+ * influenced the scorer); DOMPurify's default allowlist drops script; the
504
+ * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
505
+ * removal buys is the reason this exists: a page's runtime JS may rewrite
506
+ * a data script's text with a literal `<\/script>` embedded — measured on
507
+ * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
508
+ * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
509
+ * turning the wire's 712 `\u003c` escapes back into raw `<` — and
510
+ * browsers serialize script text verbatim, so the flattened string ends
511
+ * that script at the FIRST embedded end tag and re-parses the remaining
512
+ * ~270KB of JSON as markup. The phantom elements that survive (unknown
513
+ * tags like `<id>`, anchors full of JSON text) are exactly what
514
+ * Readability's scorer reads as article mass, so the extracted "article"
515
+ * is CSS and i18n strings while the real body — still present in the DOM —
516
+ * loses. No string-level patch can undo this after serialization (the
517
+ * string is genuinely ambiguous); only the browser's own parse holds the
518
+ * document's true structure.
519
+ * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
520
+ * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
521
+ * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
522
+ * scripts before removing them, so deleting them would regress the title
523
+ * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
524
+ * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
525
+ * same object) while making the serialization harmless: the tokenizer
526
+ * only terminates on a literal `<\/script`.
527
+ * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
528
+ * real image out of lazy-load `<noscript><img …>` wrappers before
529
+ * `_removeScripts` deletes the noscript; pre-stripping it would silently
530
+ * drop those images from the markdown.
531
+ * - `template` — removed: its children parse into the inert `.content`
532
+ * fragment (invisible to Readability's traversal), DOMPurify's
533
+ * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
534
+ * - `style` stays — its text cannot terminate a re-parse except at its own
535
+ * end tag, and the pipeline already drops it.
536
+ */
537
+ function stripScriptingElementsIn(doc) {
538
+ for (const el of Array.from(doc.querySelectorAll("script, template"))) {
539
+ if (el.tagName === "SCRIPT" && (el.getAttribute("type") ?? "").trim().toLowerCase() === "application/ld+json") {
540
+ el.textContent = (el.textContent ?? "").replace(/<\//g, "<\\/");
541
+ continue;
542
+ }
543
+ el.remove();
544
+ }
545
+ }
546
+ /** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
547
+ const STRIP_SCRIPTING_ELEMENTS = `(${stripScriptingElementsIn.toString()})(document)`;
548
+ //#endregion
549
+ //#region src/autoscroll.ts
550
+ /**
551
+ * Deep capture: a bounded in-page scroll walk that defeats viewport
552
+ * virtualization before the document is serialized.
553
+ *
554
+ * WHY. Long "feed-shaped" pages — Discourse topic streams (linux.do,
555
+ * meta.discourse.org), comment threads, timelines — increasingly render only
556
+ * the posts near the current scroll position. Discourse is the measured
557
+ * badcase: at rest the live DOM holds the posts within roughly one viewport
558
+ * of the top, while every post outside that window is a height-only
559
+ * placeholder that keeps the post's identity but none of its content:
560
+ *
561
+ * ```html
562
+ * <div class="post-stream--cloaked" data-post-number="1" id="post_1"
563
+ * style="height:297.9px !important;…">
564
+ * <h2 class="sr-only">由 Diver-QC 于 2 小时前 发布</h2>
565
+ * </div>
566
+ * ```
567
+ *
568
+ * The placeholder carries the SAME id the rendered `<article id="post_1">`
569
+ * has — every mainstream virtualized list needs some stable identity to
570
+ * recycle its rows. A fetch that renders, never scrolls, and serializes one
571
+ * instant of that DOM returns the skeleton (the screen-reader-only "posted
572
+ * by" headings survive the pipeline as if they were content) while the reply
573
+ * bodies silently vanish. Scrolling to the bottom once does not help either:
574
+ * the window MOVES — measured on the same page, scrolling to the bottom left
575
+ * only posts 31–40 rendered and re-cloaked 1–30. And the denoise pipeline's
576
+ * own coverage gate cannot notice the loss: the page's "visible text" it
577
+ * measures against is the same windowed fragment as the extraction, so the
578
+ * ratio always passes. Only walk depth can see what a resting viewport hides.
579
+ *
580
+ * The SAME pathology, one level deeper, hits app-shell document editors —
581
+ * the second measured badcase is a Feishu wiki page (bocha-ai.feishu.cn).
582
+ * There the body never scrolls at all: html/body are locked at exactly one
583
+ * viewport and the document scrolls inside `div.bear-web-x-container`
584
+ * (measured: documentElement.scrollHeight 720 == innerHeight,
585
+ * window.scrollTo(0, 800) left scrollY at 0, while the container held 4534px
586
+ * against a 656px client). Its blocks are virtualized too, with two twists
587
+ * over Discourse: blocks carry `data-block-id` and NO id, and off-window
588
+ * blocks are REMOVED outright — the whole un-rendered tail is one anonymous
589
+ * `.bear-virtual-renderUnit-placeholder` spacer with an explicit height — so
590
+ * there is neither a window scrollbar to walk nor a degraded-in-place row to
591
+ * swap back. A fetch therefore returns the statically rendered first screen.
592
+ *
593
+ * HOW. Between the settle and the serialization the provider runs one
594
+ * in-page async walk ({@link deepCaptureInPage}, a typed function evaluated
595
+ * by source):
596
+ *
597
+ * 1. Resolve the walk TARGET: the window when the document itself scrolls
598
+ * (the Discourse shape), else the page's dominant inner scroll container
599
+ * (the app-shell shape) — the element with the largest scrollable
600
+ * distance among computed `overflow-y: auto|scroll` elements, gated at
601
+ * half a viewport so dropdown-sized scrollers never trigger a walk. A
602
+ * page with neither is left alone (the legacy capture).
603
+ * 2. STASH every keyed content element — key = id, else a namespaced
604
+ * `data-block-id` — as a detached clone (the RICHEST sighting per key
605
+ * wins), together with its nearest keyed siblings: where it sits in its
606
+ * own list, which is what a removed row is later spliced back next to.
607
+ * 3. WALK down from the top in smooth 0.7-viewport steps (300ms motion +
608
+ * 300ms quiet settle — the rest a feed engine's debounced uncloak needs),
609
+ * collecting after each, until the walk can hold AT THE BOTTOM through a
610
+ * {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
611
+ * (growth restarts the clock — that is what pages load-more sentinels),
612
+ * or the step/budget caps close it.
613
+ *
614
+ * A MutationObserver watches for the walk's whole reason to exist: the
615
+ * page REACTING to scroll by adding or removing elements. Most of the web
616
+ * never does (a long static article only costs latency here), so after
617
+ * {@link SCROLL_INERT_STEPS} careful steps with no reaction the walk
618
+ * probes the rest with one jump to the bottom. Any reaction after that
619
+ * jump — an uncloak, a re-cloak, a load-more append — means the region
620
+ * jumped over may hold content that never rendered, so the walk rewinds
621
+ * to where it jumped from and finishes carefully. Correctness never
622
+ * rests on the warm-up guess; only the cost of a scroll-inert page does.
623
+ * 4. MERGE into a structural clone (`document.cloneNode(true)`: no
624
+ * serialize/re-parse round trip — a runtime-rewritten data script with a
625
+ * literal `<\/script>` in its text cannot corrupt it — and, having no
626
+ * browsing context, it runs no scripts, upgrades no custom elements and
627
+ * loads no frames):
628
+ * - SPLICE (container walks): stashed keys from inside the scroll
629
+ * container that the clone no longer holds were REMOVED by the engine
630
+ * (the Feishu shape — its placeholders are anonymous) and go back next
631
+ * to their recorded sibling, resolved iteratively so a removed run
632
+ * chains off whichever end survived; a key that rode in inside its
633
+ * spliced parent is already present and skipped. Placement is
634
+ * structural, so it is right whatever order the walk SAW rows in (an
635
+ * anchored open starts mid-document).
636
+ * - SWAP: every keyed element emptier than its snapshot (fewer child
637
+ * elements, or meaningfully less text — the Discourse placeholder) is
638
+ * replaced by that snapshot, except containers (ancestors of another
639
+ * element being swapped), which are assembled from those descendants
640
+ * instead: a container's own snapshot is one window's partial view.
641
+ * The clone is then stripped of scripting elements and serialized; the
642
+ * provider reads that string instead of the live page. The live document
643
+ * is never rewritten, so no late scroll/rAF/timer reaction can touch the
644
+ * capture either.
645
+ *
646
+ * Failure is always silent degradation: a page without `evaluate`, a script
647
+ * error, or an engine that rejects the walk just keeps the legacy
648
+ * single-instant capture — a walk can only add content, never fail a fetch.
649
+ * A walk that restored nothing returns no HTML, and the provider reads the
650
+ * live page exactly as before.
651
+ *
652
+ * The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
653
+ * cloaked placeholders) but never for non-HTML documents. It is bounded by
654
+ * the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
655
+ * and always by the remaining fetch deadline minus a decode reserve — the
656
+ * provider's abort closes the tab, which rejects the pending evaluation.
657
+ *
658
+ * @module dsh-web-fetch-playwright/autoscroll
659
+ */
660
+ /** How far each step scrolls, as a share of the viewport height. */
661
+ const SCROLL_STEP_RATIO = .7;
662
+ /**
663
+ * How long at the bottom, with no height growth, before the walk concludes
664
+ * the page is complete. Real feed engines fetch their next chunk with
665
+ * network latency (measured on linux.do: a pending chunk landed well after
666
+ * two 450ms checks), so this is sized over a slow round trip.
667
+ */
668
+ const SCROLL_BOTTOM_PATIENCE_MS = 2500;
669
+ /**
670
+ * Cap on stashed keys — pathological id counts cannot pin memory. Sized for
671
+ * long topics: a Discourse post carries several ids of its own (post,
672
+ * heading, avatar, …), so a few hundred posts already need a thousand-plus.
673
+ * {@link SCROLL_STASH_MAX_CHARS} is the real memory bound.
674
+ */
675
+ const SCROLL_STASH_MAX_ENTRIES = 2e3;
676
+ /** Cap on stashed HTML characters (aligned with the pipeline input cap). */
677
+ const SCROLL_STASH_MAX_CHARS = 2e6;
678
+ /**
679
+ * Cap on ONE stashed element's HTML. A virtualized row is a few KB; an
680
+ * element tens of KB big is a CONTAINER (measured on linux.do: six nested
681
+ * wrappers each embed the whole post stream — main 164KB ⊃
682
+ * main-outlet-wrapper 156KB ⊃ … — plus data-preloaded 300KB and
683
+ * svg-sprites 277KB) or a data island — never a row the merge could
684
+ * rehydrate. Such keys are remembered and never serialized again.
685
+ */
686
+ const SCROLL_STASH_MAX_ENTRY_CHARS = 64e3;
687
+ /**
688
+ * Tail of the fetch budget the walk never spends, keeping room for the
689
+ * content read and the synchronous denoise decode.
690
+ */
691
+ const SCROLL_FINISH_RESERVE_MS = 4e3;
692
+ /** The viewport share an inner container's scrollable distance must reach. */
693
+ const SCROLL_CONTAINER_MIN_RANGE_RATIO = .5;
694
+ /**
695
+ * The in-page walk. Evaluated BY SOURCE (`fn.toString()` composed into an
696
+ * expression by {@link deepCaptureScript}), so it must stay self-contained:
697
+ * every helper is nested, every knob arrives in `p`, and the one shared
698
+ * routine — the scripting strip — arrives as an argument. Typed here so the
699
+ * compiler checks what used to be an unchecked string.
700
+ *
701
+ * @param p - the walk's knobs.
702
+ * @param strip - {@link stripScriptingElementsIn}, applied to the merged clone.
703
+ * @returns the walk's stats, plus the merged HTML when anything was restored.
704
+ */
705
+ async function deepCaptureInPage(p, strip) {
706
+ const started = Date.now();
707
+ const root = document.documentElement;
708
+ const body = document.body;
709
+ if (root === null || body === null) return {
710
+ steps: 0,
711
+ stashed: 0,
712
+ rehydrated: 0,
713
+ skipped: "not-scrollable"
714
+ };
715
+ if (p.budgetMs <= 0) return {
716
+ steps: 0,
717
+ stashed: 0,
718
+ rehydrated: 0,
719
+ skipped: "no-budget"
720
+ };
721
+ const KEYED = "[id], [data-block-id]";
722
+ const SVG_NS = "http://www.w3.org/2000/svg";
723
+ const sleep = (ms) => new Promise((resolve) => {
724
+ setTimeout(resolve, ms);
725
+ });
726
+ function findScrollContainer(host) {
727
+ const minRange = Math.max(p.containerMinRangePx, window.innerHeight * p.containerMinRatio);
728
+ let best = null;
729
+ let bestRange = 0;
730
+ for (const el of Array.from(host.querySelectorAll("*"))) {
731
+ const range = el.scrollHeight - el.clientHeight;
732
+ if (range < minRange || range <= bestRange) continue;
733
+ const overflowY = getComputedStyle(el).overflowY;
734
+ if (overflowY === "auto" || overflowY === "scroll") {
735
+ best = el;
736
+ bestRange = range;
737
+ }
738
+ }
739
+ return best;
740
+ }
741
+ const documentScrolls = root.scrollHeight > window.innerHeight + 4;
742
+ const scroller = documentScrolls ? null : findScrollContainer(body);
743
+ if (!documentScrolls && scroller === null) return {
744
+ steps: 0,
745
+ stashed: 0,
746
+ rehydrated: 0,
747
+ skipped: "not-scrollable"
748
+ };
749
+ const viewportH = () => scroller === null ? window.innerHeight : scroller.clientHeight;
750
+ const scrollH = () => scroller === null ? root.scrollHeight : scroller.scrollHeight;
751
+ const curY = () => scroller === null ? window.scrollY : scroller.scrollTop;
752
+ const atBottom = () => curY() + viewportH() >= scrollH() - 2;
753
+ function scrollToY(y) {
754
+ if (scroller === null) window.scrollTo({
755
+ top: y,
756
+ behavior: "instant"
757
+ });
758
+ else scroller.scrollTop = y;
759
+ }
760
+ function smoothScrollBy(dy) {
761
+ return new Promise((resolve) => {
762
+ const fromY = curY();
763
+ const t0 = Date.now();
764
+ const tick = () => {
765
+ const progress = (Date.now() - t0) / p.stepMotionMs;
766
+ if (progress >= 1 || curY() !== fromY && atBottom()) {
767
+ scrollToY(fromY + dy);
768
+ resolve();
769
+ return;
770
+ }
771
+ scrollToY(fromY + Math.round(dy * progress));
772
+ setTimeout(tick, 16);
773
+ };
774
+ tick();
775
+ });
776
+ }
777
+ function keyOf(el) {
778
+ if (el.id !== "") return el.id;
779
+ const block = el.getAttribute("data-block-id");
780
+ return block !== null && block !== "" ? `data-block-id:${block}` : null;
781
+ }
782
+ function siblingKey(el, dir) {
783
+ for (let sib = el[dir]; sib !== null; sib = sib[dir]) {
784
+ const key = keyOf(sib);
785
+ if (key !== null) return key;
786
+ }
787
+ return null;
788
+ }
789
+ const INERT_TAGS = /* @__PURE__ */ new Set([
790
+ "SCRIPT",
791
+ "STYLE",
792
+ "LINK",
793
+ "META",
794
+ "NOSCRIPT",
795
+ "IFRAME",
796
+ "TEMPLATE"
797
+ ]);
798
+ let reactions = 0;
799
+ const counts = (nodes) => {
800
+ for (const node of Array.from(nodes)) if (node.nodeType === 1 && !INERT_TAGS.has(node.nodeName)) return true;
801
+ return false;
802
+ };
803
+ const tally = (records) => {
804
+ for (const record of records) if (counts(record.addedNodes) || counts(record.removedNodes)) reactions++;
805
+ };
806
+ const observer = new MutationObserver(tally);
807
+ observer.observe(body, {
808
+ childList: true,
809
+ subtree: true
810
+ });
811
+ const reacted = () => {
812
+ tally(observer.takeRecords());
813
+ return reactions > 0;
814
+ };
815
+ const stashDoc = document.implementation.createHTMLDocument("");
816
+ const stash = /* @__PURE__ */ new Map();
817
+ const oversized = /* @__PURE__ */ new Set();
818
+ let stashChars = 0;
819
+ const host = body;
820
+ function collect() {
821
+ for (const el of Array.from(host.querySelectorAll(KEYED))) {
822
+ const key = keyOf(el);
823
+ if (key === null || oversized.has(key) || el.namespaceURI === SVG_NS) continue;
824
+ const els = el.childElementCount;
825
+ const text = (el.textContent ?? "").length;
826
+ if (els === 0 && text === 0) continue;
827
+ const prev = stash.get(key);
828
+ if (prev !== void 0) {
829
+ prev.prev = siblingKey(el, "previousElementSibling") ?? prev.prev;
830
+ prev.next = siblingKey(el, "nextElementSibling") ?? prev.next;
831
+ if (els <= prev.els && text <= prev.text) continue;
832
+ } else if (stash.size >= p.stashMaxEntries) continue;
833
+ if (text > p.stashMaxEntryChars) {
834
+ oversized.add(key);
835
+ continue;
836
+ }
837
+ const len = el.outerHTML.length;
838
+ if (len > p.stashMaxEntryChars) {
839
+ oversized.add(key);
840
+ continue;
841
+ }
842
+ const freed = prev === void 0 ? 0 : prev.len;
843
+ if (stashChars - freed + len > p.stashMaxChars) continue;
844
+ stashChars += len - freed;
845
+ stash.set(key, {
846
+ node: stashDoc.importNode(el, true),
847
+ els,
848
+ text,
849
+ len,
850
+ prev: prev?.prev ?? siblingKey(el, "previousElementSibling"),
851
+ next: prev?.next ?? siblingKey(el, "nextElementSibling"),
852
+ inScroller: scroller !== null && scroller.contains(el)
853
+ });
854
+ }
855
+ }
856
+ collect();
857
+ if (curY() > 0) {
858
+ scrollToY(0);
859
+ await sleep(p.stepSettleMs);
860
+ collect();
861
+ }
862
+ tally(observer.takeRecords());
863
+ reactions = 0;
864
+ let steps = 0;
865
+ let lastHeight = scrollH();
866
+ let lastChangeAt = Date.now();
867
+ let careful = false;
868
+ let jumpedFrom = null;
869
+ const noteHeight = () => {
870
+ const height = scrollH();
871
+ if (height !== lastHeight) {
872
+ lastHeight = height;
873
+ lastChangeAt = Date.now();
874
+ }
875
+ };
876
+ try {
877
+ while (steps < p.maxSteps && Date.now() - started < p.budgetMs) {
878
+ if (jumpedFrom !== null && reacted()) {
879
+ scrollToY(jumpedFrom);
880
+ jumpedFrom = null;
881
+ careful = true;
882
+ await sleep(p.stepSettleMs);
883
+ collect();
884
+ noteHeight();
885
+ continue;
886
+ }
887
+ if (atBottom()) {
888
+ await sleep(p.bottomPollMs);
889
+ collect();
890
+ const before = lastHeight;
891
+ noteHeight();
892
+ if (lastHeight === before && Date.now() - lastChangeAt >= p.bottomPatienceMs) break;
893
+ continue;
894
+ }
895
+ if (!careful && jumpedFrom === null) {
896
+ if (reacted()) careful = true;
897
+ else if (steps >= p.inertSteps) {
898
+ jumpedFrom = curY();
899
+ steps++;
900
+ scrollToY(scrollH());
901
+ lastChangeAt = Date.now();
902
+ await sleep(p.stepSettleMs);
903
+ collect();
904
+ noteHeight();
905
+ continue;
906
+ }
907
+ }
908
+ steps++;
909
+ await smoothScrollBy(Math.max(1, Math.round(viewportH() * p.stepRatio)));
910
+ collect();
911
+ await sleep(p.stepSettleMs);
912
+ collect();
913
+ noteHeight();
914
+ }
915
+ } finally {
916
+ observer.disconnect();
917
+ }
918
+ const reactive = reacted();
919
+ const finalHeight = scrollH();
920
+ const fresh = document.cloneNode(true);
921
+ const present = /* @__PURE__ */ new Map();
922
+ const index = (node) => {
923
+ const key = keyOf(node);
924
+ if (key !== null && !present.has(key)) present.set(key, node);
925
+ for (const el of Array.from(node.querySelectorAll(KEYED))) {
926
+ const inner = keyOf(el);
927
+ if (inner !== null && !present.has(inner)) present.set(inner, el);
928
+ }
929
+ };
930
+ if (fresh.documentElement !== null) index(fresh.documentElement);
931
+ let spliced = 0;
932
+ if (scroller !== null) {
933
+ let pending = Array.from(stash.keys()).filter((key) => stash.get(key).inScroller && !present.has(key));
934
+ let progressed = true;
935
+ while (pending.length > 0 && progressed) {
936
+ progressed = false;
937
+ const unresolved = [];
938
+ for (const key of pending) {
939
+ if (present.has(key)) {
940
+ progressed = true;
941
+ continue;
942
+ }
943
+ const snap = stash.get(key);
944
+ const after = snap.prev === null ? void 0 : present.get(snap.prev);
945
+ const before = snap.next === null ? void 0 : present.get(snap.next);
946
+ if (after === void 0 && before === void 0) {
947
+ unresolved.push(key);
948
+ continue;
949
+ }
950
+ const node = fresh.importNode(snap.node, true);
951
+ if (after !== void 0) after.after(node);
952
+ else before.before(node);
953
+ index(node);
954
+ spliced++;
955
+ progressed = true;
956
+ }
957
+ pending = unresolved;
958
+ }
959
+ }
960
+ const degraded = [];
961
+ for (const el of Array.from(fresh.querySelectorAll(KEYED))) {
962
+ const key = keyOf(el);
963
+ const snap = key === null ? void 0 : stash.get(key);
964
+ if (snap === void 0 || el.namespaceURI === SVG_NS) continue;
965
+ if (el.childElementCount < snap.els || snap.text - (el.textContent ?? "").length > p.textDegradeTolerance) degraded.push({
966
+ el,
967
+ snap
968
+ });
969
+ }
970
+ const containers = /* @__PURE__ */ new Set();
971
+ for (const { el } of degraded) for (let up = el.parentElement; up !== null && !containers.has(up); up = up.parentElement) containers.add(up);
972
+ let rehydrated = 0;
973
+ for (const { el, snap } of degraded) {
974
+ if (containers.has(el)) continue;
975
+ el.replaceWith(fresh.importNode(snap.node, true));
976
+ rehydrated++;
977
+ }
978
+ const stats = {
979
+ steps,
980
+ stashed: stash.size,
981
+ rehydrated,
982
+ spliced,
983
+ reactive,
984
+ elapsedMs: Date.now() - started,
985
+ height: finalHeight
986
+ };
987
+ if ((rehydrated > 0 || spliced > 0) && fresh.documentElement !== null) {
988
+ strip(fresh);
989
+ stats.html = (fresh.doctype === null ? "" : new XMLSerializer().serializeToString(fresh.doctype)) + fresh.documentElement.outerHTML;
990
+ }
991
+ return stats;
992
+ }
993
+ /**
994
+ * The walk's knobs for one fetch: the module constants plus its budget.
995
+ *
996
+ * @param budgetMs - wall-clock budget the walk may spend inside the page.
997
+ */
998
+ function deepCaptureParams(budgetMs) {
999
+ return {
1000
+ budgetMs: Math.max(0, Math.round(budgetMs)),
1001
+ stepRatio: SCROLL_STEP_RATIO,
1002
+ stepMotionMs: 300,
1003
+ stepSettleMs: 300,
1004
+ bottomPollMs: 450,
1005
+ bottomPatienceMs: SCROLL_BOTTOM_PATIENCE_MS,
1006
+ inertSteps: 2,
1007
+ maxSteps: 50,
1008
+ stashMaxEntries: SCROLL_STASH_MAX_ENTRIES,
1009
+ stashMaxChars: SCROLL_STASH_MAX_CHARS,
1010
+ stashMaxEntryChars: SCROLL_STASH_MAX_ENTRY_CHARS,
1011
+ containerMinRangePx: 300,
1012
+ containerMinRatio: SCROLL_CONTAINER_MIN_RANGE_RATIO,
1013
+ textDegradeTolerance: 16
1014
+ };
1015
+ }
1016
+ /**
1017
+ * Compose the evaluate expression: the in-page walk applied to `params` and
1018
+ * the shared scripting strip, both by source.
1019
+ *
1020
+ * @param params - the walk's knobs (see {@link deepCaptureParams}).
1021
+ * @returns the expression string to pass to `page.evaluate`.
1022
+ */
1023
+ function deepCaptureExpression(params) {
1024
+ return `/* dsh-deep-capture */ (${deepCaptureInPage.toString()})(${JSON.stringify(params)}, ${stripScriptingElementsIn.toString()})`;
1025
+ }
1026
+ /** {@link deepCaptureExpression} over {@link deepCaptureParams} — one fetch's walk. */
1027
+ function deepCaptureScript(budgetMs) {
1028
+ return deepCaptureExpression(deepCaptureParams(budgetMs));
1029
+ }
1030
+ /**
1031
+ * Run the deep-capture walk in the page, degrading silently to the legacy
1032
+ * single-instant capture on any failure: a page without `evaluate`, a budget
1033
+ * too small to act, a rejected evaluation (the fetch deadline closes the tab
1034
+ * mid-walk), or a result that is not the stats object all just skip it. A
1035
+ * walk can only add content, never fail a fetch.
1036
+ *
1037
+ * @param page - the fetch's tab.
1038
+ * @param budgetMs - wall-clock budget for the in-page walk; budgets under
1039
+ * {@link SCROLL_MIN_BUDGET_MS} skip it.
1040
+ * @returns the walk's stats (with the merged `html` when it restored
1041
+ * anything), or undefined when the walk did not run.
1042
+ */
1043
+ async function deepCaptureWalk(page, budgetMs) {
1044
+ if (typeof page.evaluate !== "function") return void 0;
1045
+ if (budgetMs < 500) return void 0;
1046
+ try {
1047
+ const result = await page.evaluate(deepCaptureScript(budgetMs));
1048
+ if (result === null || typeof result !== "object") return void 0;
1049
+ const raw = result;
1050
+ if (typeof raw.steps !== "number" || typeof raw.stashed !== "number" || typeof raw.rehydrated !== "number") return;
1051
+ return {
1052
+ steps: raw.steps,
1053
+ stashed: raw.stashed,
1054
+ rehydrated: raw.rehydrated,
1055
+ ...typeof raw.spliced === "number" ? { spliced: raw.spliced } : {},
1056
+ ...typeof raw.reactive === "boolean" ? { reactive: raw.reactive } : {},
1057
+ ...typeof raw.elapsedMs === "number" ? { elapsedMs: raw.elapsedMs } : {},
1058
+ ...typeof raw.height === "number" ? { height: raw.height } : {},
1059
+ ...raw.skipped === "not-scrollable" || raw.skipped === "no-budget" ? { skipped: raw.skipped } : {},
1060
+ ...typeof raw.html === "string" ? { html: raw.html } : {}
1061
+ };
1062
+ } catch {
1063
+ return;
1064
+ }
1065
+ }
1066
+ //#endregion
455
1067
  //#region src/session-window.ts
456
1068
  const ownership = /* @__PURE__ */ new WeakMap();
457
1069
  const unavailable = /* @__PURE__ */ new WeakSet();
1070
+ const identities = /* @__PURE__ */ new WeakMap();
458
1071
  const CLEANUP_GRACE_MS = 100;
459
1072
  function claim(browser, context, keys) {
460
1073
  let owners = ownership.get(browser);
@@ -551,6 +1164,8 @@ async function createSessionPage(context, browser, timeoutMs) {
551
1164
  return session;
552
1165
  };
553
1166
  const identify = async (page) => {
1167
+ const known = identities.get(page);
1168
+ if (known) return known;
554
1169
  const session = await newSession(() => ctx.newCDPSession(page));
555
1170
  let identificationFailed = false;
556
1171
  try {
@@ -600,6 +1215,7 @@ async function createSessionPage(context, browser, timeoutMs) {
600
1215
  targetId = actual.targetId;
601
1216
  if (actual.browserContextId !== identity.browserContextId || await windowOf(actual.targetId) !== expectedWindow) throw new Error("Chromium did not place the session tab in its dedicated window");
602
1217
  claim(browser, context, [`context:${actual.browserContextId}`, `window:${expectedWindow}`]);
1218
+ identities.set(created, actual);
603
1219
  succeeded = true;
604
1220
  return created;
605
1221
  }
@@ -647,6 +1263,10 @@ async function createSessionPage(context, browser, timeoutMs) {
647
1263
  const explicitWindow = await windowOf(explicitTargetId);
648
1264
  if (explicitWindow === bootstrapWindow) throw new Error("Chromium ignored the dedicated session window request");
649
1265
  claim(browser, context, [`context:${identity.browserContextId}`, `window:${explicitWindow}`]);
1266
+ identities.set(created, {
1267
+ targetId: explicitTargetId,
1268
+ browserContextId: identity.browserContextId
1269
+ });
650
1270
  await deadline.run(() => bootstrap.close());
651
1271
  bootstrap = void 0;
652
1272
  succeeded = true;
@@ -1092,11 +1712,14 @@ function installBuiltinSlashResolveCompat() {
1092
1712
  * footers, and ad chrome by scoring link density and text mass) — its result
1093
1713
  * must then pass a coverage gate that distrusts tiny or sliver-thin
1094
1714
  * extractions, which on list-shaped pages (portals, catalogs) is exactly
1095
- * what the link-density penalty leaves behind; DOMPurify sanitizes whatever
1096
- * HTML remains and forbids the layout tags noise lives in; Turndown with
1097
- * the GFM plugin converts to markdown using the same style options and
1098
- * span-safe table rules as the shipped `dsh-tool-web` renderer, so output
1099
- * is consistent with what `web_fetch` produces elsewhere.
1715
+ * what the link-density penalty leaves behind; an accepted extraction is
1716
+ * then pruned of trailing link-dense sections — the post-article
1717
+ * recommendation rails and share bars the scorer's vocabularies cannot name
1718
+ * (they arrive as bare divs with no class to match); DOMPurify sanitizes
1719
+ * whatever HTML remains and forbids the layout tags noise lives in;
1720
+ * Turndown with the GFM plugin converts to markdown using the same style
1721
+ * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
1722
+ * so output is consistent with what `web_fetch` produces elsewhere.
1100
1723
  *
1101
1724
  * Synchronous for its callers and unit-tested against fixture pages: jsdom
1102
1725
  * itself is loaded lazily on the first call — and, inside a dsh profile, only
@@ -1336,6 +1959,209 @@ turndown.addRule("tableRowWithoutSpanExpansion", {
1336
1959
  }
1337
1960
  });
1338
1961
  /**
1962
+ * Count non-whitespace characters under `root` that sit inside an anchor,
1963
+ * skipping the same noise subtrees {@link visibleTextLength} skips so the
1964
+ * numerator and denominator of a link-density ratio stay comparable.
1965
+ *
1966
+ * @param root - the subtree to measure; `null` counts as nothing.
1967
+ * @returns the number of non-whitespace characters inside `<a>` descendants.
1968
+ */
1969
+ function linkTextLength(root) {
1970
+ if (root === null) return 0;
1971
+ let length = 0;
1972
+ const visit = (node, insideAnchor) => {
1973
+ if (node.nodeType === 3) {
1974
+ if (insideAnchor) length += (node.nodeValue ?? "").replace(/\s+/g, "").length;
1975
+ return;
1976
+ }
1977
+ if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return;
1978
+ const anchor = insideAnchor || node.nodeName.toLowerCase() === "a";
1979
+ for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, anchor);
1980
+ };
1981
+ visit(root, false);
1982
+ return length;
1983
+ }
1984
+ /**
1985
+ * Minimum non-whitespace characters a block must carry before its link
1986
+ * density is even evaluated. Blocks below this are spacers, empty wrappers,
1987
+ * or one-line sign-offs — too small to be worth a removal decision, so the
1988
+ * trailing walk skips past them instead of stopping at them.
1989
+ */
1990
+ const TRAILING_MIN_TEXT_CHARS = 40;
1991
+ /**
1992
+ * Link-density share above which a trailing block reads as a link farm:
1993
+ * recommendation feeds, related-article rails, tag clouds, share bars.
1994
+ * Measured on the 163.com dy-article badcase the section's pick carried 84
1995
+ * linked characters inside 183 total (0.46) — recommendation cards dilute
1996
+ * their density with author/date lines, so a 0.5 bar would miss them by
1997
+ * construction; an honest closing paragraph with a citation or two measures
1998
+ * 0.1–0.3, leaving 0.4 in the empty band between.
1999
+ */
2000
+ const TRAILING_LINK_DENSITY = .4;
2001
+ /**
2002
+ * A trailing block qualifies as a link farm only when it is link-heavy in
2003
+ * aggregate too: at least {@link TRAILING_LINK_COUNT} anchors or
2004
+ * {@link TRAILING_LINK_TEXT_CHARS} linked characters. Density alone would
2005
+ * let a single long linked sentence (an inline citation) drag a legitimate
2006
+ * paragraph under the knife.
2007
+ */
2008
+ const TRAILING_LINK_COUNT = 3;
2009
+ const TRAILING_LINK_TEXT_CHARS = 60;
2010
+ /**
2011
+ * A trailing block may never hold more than this share of the whole
2012
+ * article's text. The prune targets *appended* widgets; a block that IS most
2013
+ * of the extraction says the scorer picked the widget, which is the coverage
2014
+ * gate's failure mode to handle, not this walk's.
2015
+ */
2016
+ const TRAILING_MAX_ARTICLE_SHARE = .6;
2017
+ /**
2018
+ * A trailing run may never hold more than this share of its container's
2019
+ * text — the aggregate form of the guard above, and the one that keeps
2020
+ * list-shaped extractions whole. A recommendation rail is a *minority*
2021
+ * appended to real content (the 163 rail is 44% of its container); a
2022
+ * portal or feed whose extraction IS a list of link rows (measured on a
2023
+ * Hacker News homepage capture: ~91 trailing rows, every one individually
2024
+ * link-dense at 0.69–0.94) has a tail run holding ~97% of the container —
2025
+ * there the tail IS the content, the walk declines wholesale, and only a
2026
+ * run that is a minority of its container is removed.
2027
+ */
2028
+ const TRAILING_MAX_RUN_SHARE = .6;
2029
+ /**
2030
+ * Link-density band within which a run and the block the walk stopped at
2031
+ * read as the same material. When the content immediately before the run
2032
+ * is itself as link-dense as the run — measured on a Hacker News homepage
2033
+ * capture, the rows after the run's cut measure 0.94 against the run's
2034
+ * 0.91 — the container is a *uniform list* (feed rows, portal headlines)
2035
+ * and the tail is that list's own content, not a widget appended after it;
2036
+ * a widget hangs after genuinely different material (the 163 rail's 0.44
2037
+ * follows body text at 0.00). Only a run denser than what precedes it is
2038
+ * removed.
2039
+ */
2040
+ const TRAILING_UNIFORM_DENSITY_BAND = .25;
2041
+ /**
2042
+ * Prune trailing link-dense sections from an accepted Readability
2043
+ * extraction — the post-article widget class the scorer itself cannot be
2044
+ * taught to drop.
2045
+ *
2046
+ * Why this exists (badcase: `https://www.163.com/dy/article/…`): Chinese
2047
+ * portal article pages append a recommendation feed inside the very wrapper
2048
+ * Readability scores as the article (`#content` holds the body, the
2049
+ * platform disclaimer, then a rail of image/title/author/date cards). Every
2050
+ * existing noise layer misses it: Readability's `unlikelyCandidates` /
2051
+ * `negative` vocabularies know `related` but not `recommend` (nor the
2052
+ * pinyin families `tuijian`/`xiangguan`), and its own cleaning pass draws
2053
+ * the "mostly links" line at per-block density 0.5 — these cards measure
2054
+ * 0.42–0.48 (the title links over title+author+date text), so every card
2055
+ * clears it; in the extracted HTML the rail is a *bare* div — class and id
2056
+ * are gone — so no class-based rule can ever see it again; DOMPurify's
2057
+ * FORBID_TAGS works on tags, and the rail is plain div/p/h3/a; and the
2058
+ * coverage gate cannot help because the noise rides *inside* the
2059
+ * extraction it measures (the badcase passed at 0.103 — the rail's text
2060
+ * was in both numerator and denominator). Modern sites with CSS-module
2061
+ * class names (`css-1b3x7r`) present the same shape with no vocabulary to
2062
+ * match at all, so the only signal that survives all of this is
2063
+ * structural: a *trailing* run of blocks whose text is mostly links.
2064
+ *
2065
+ * For every container in the extraction the walk gathers the maximal
2066
+ * trailing run of qualifying children — link-farm blocks per the constants
2067
+ * above, plus sub-threshold spacers (empty wrapper divs) — and removes the
2068
+ * run only when BOTH structural guards say "appended widget, not content":
2069
+ * the run is a minority of the container's text
2070
+ * ({@link TRAILING_MAX_RUN_SHARE}), and the run is clearly denser than the
2071
+ * block the walk stopped at ({@link TRAILING_UNIFORM_DENSITY_BAND}). The
2072
+ * guards split the two worlds the per-block view cannot: a rail is
2073
+ * one-or-few appended blocks after genuinely different material (the 163
2074
+ * rail's 0.44 density follows body text at 0.00), while a link-list
2075
+ * extraction — feed rows, portal headlines — is *uniform*: its rows before
2076
+ * any cut measure the same density as the rows after it (0.91 vs 0.94 on
2077
+ * the Hacker News capture), so no tail of it can be mistaken for a widget.
2078
+ *
2079
+ * Deliberately NOT applied to the whole-document fallback: that path
2080
+ * exists for list-shaped pages where the links *are* the content (the
2081
+ * 163.com homepage fix), and pruning link-dense regions there would
2082
+ * re-introduce exactly the recall loss the coverage gate exists to
2083
+ * prevent.
2084
+ *
2085
+ * @param container - parsed `article.content`; mutated in place.
2086
+ */
2087
+ function pruneTrailingLinkDenseSections(container) {
2088
+ const articleText = visibleTextLength(container);
2089
+ if (articleText === 0) return;
2090
+ for (const element of [container, ...Array.from(container.querySelectorAll("*"))]) {
2091
+ if (!container.contains(element) && element !== container) continue;
2092
+ removeTrailingLinkDenseRun(element, articleText);
2093
+ }
2094
+ }
2095
+ /**
2096
+ * Remove one container's maximal trailing run of link-dense children when
2097
+ * that run is a minority of the container's text AND clearly different
2098
+ * material from the content that precedes it.
2099
+ *
2100
+ * @param parent - the container whose element children are examined from
2101
+ * the end.
2102
+ * @param articleText - the whole article's text, for the per-block share
2103
+ * guard.
2104
+ */
2105
+ function removeTrailingLinkDenseRun(parent, articleText) {
2106
+ const children = Array.from(parent.children);
2107
+ if (children.length === 0) return;
2108
+ const containerText = visibleTextLength(parent);
2109
+ if (containerText === 0) return;
2110
+ const run = [];
2111
+ let runText = 0;
2112
+ let runLinked = 0;
2113
+ let stopIndex = -1;
2114
+ for (let index = children.length - 1; index >= 0; index--) {
2115
+ const child = children[index];
2116
+ if (child === void 0) break;
2117
+ const text = visibleTextLength(child);
2118
+ if (text < TRAILING_MIN_TEXT_CHARS) {
2119
+ run.push(child);
2120
+ continue;
2121
+ }
2122
+ const linked = linkTextLength(child);
2123
+ const linkCount = child.querySelectorAll("a").length;
2124
+ if (linked / text >= TRAILING_LINK_DENSITY && (linkCount >= TRAILING_LINK_COUNT || linked >= TRAILING_LINK_TEXT_CHARS) && text <= TRAILING_MAX_ARTICLE_SHARE * articleText) {
2125
+ run.push(child);
2126
+ runText += text;
2127
+ runLinked += linked;
2128
+ continue;
2129
+ }
2130
+ stopIndex = index;
2131
+ break;
2132
+ }
2133
+ if (runText === 0 || stopIndex === -1) return;
2134
+ const stopChild = children[stopIndex];
2135
+ if (stopChild === void 0) return;
2136
+ const stopDensity = linkTextLength(stopChild) / Math.max(1, visibleTextLength(stopChild));
2137
+ if (Math.abs(runLinked / runText - stopDensity) <= TRAILING_UNIFORM_DENSITY_BAND) return;
2138
+ if (runText > TRAILING_MAX_RUN_SHARE * containerText) return;
2139
+ for (const child of run) child.remove();
2140
+ }
2141
+ /**
2142
+ * Run {@link pruneTrailingLinkDenseSections} over one extraction with a
2143
+ * misfire guard: a prune that takes a healthy article below the gate's
2144
+ * absolute floor ({@link GATE_MIN_ARTICLE_TEXT_CHARS}) or empties it
2145
+ * entirely says the walk misread the page, and Readability's original pick
2146
+ * is returned untouched. A *sparse* article (under the floor before the
2147
+ * prune, auto-trusted by the gate) keeps its pruned form — its trailing
2148
+ * links were noise all the same.
2149
+ *
2150
+ * @param articleHtml - the `article.content` HTML Readability produced.
2151
+ * @param document - the live document, for parsing the fragment.
2152
+ * @returns the pruned article HTML, or the original on a misfire.
2153
+ */
2154
+ function prunedArticleContent(articleHtml, document) {
2155
+ const container = document.createElement("div");
2156
+ container.innerHTML = articleHtml;
2157
+ const before = visibleTextLength(container);
2158
+ if (before === 0) return articleHtml;
2159
+ pruneTrailingLinkDenseSections(container);
2160
+ const after = visibleTextLength(container);
2161
+ if (after === 0 || before >= GATE_MIN_ARTICLE_TEXT_CHARS && after < GATE_MIN_ARTICLE_TEXT_CHARS) return articleHtml;
2162
+ return container.innerHTML;
2163
+ }
2164
+ /**
1339
2165
  * Convert one rendered HTML document to denoised markdown.
1340
2166
  *
1341
2167
  * Readability failure (non-article pages, or extractions the coverage gate
@@ -1363,7 +2189,7 @@ function htmlToMarkdown(html, url) {
1363
2189
  const cloned = document.cloneNode(true);
1364
2190
  const article = new Readability(cloned).parse();
1365
2191
  if (article !== null && typeof article.content === "string" && article.content !== "" && articleCoversPage(article.content, document)) {
1366
- source = article.content;
2192
+ source = prunedArticleContent(article.content, document);
1367
2193
  title = article.title ?? void 0;
1368
2194
  }
1369
2195
  } catch {}
@@ -1392,6 +2218,902 @@ function htmlToMarkdown(html, url) {
1392
2218
  };
1393
2219
  }
1394
2220
  //#endregion
2221
+ //#region src/pdf.ts
2222
+ /**
2223
+ * PDF → markdown. A fetched URL that turns out to be a PDF (served inline as
2224
+ * `application/pdf`, as a `Content-Disposition: attachment` download, or as
2225
+ * an anonymous `application/octet-stream` whose bytes start with `%PDF-`) is
2226
+ * routed here instead of failing with `WEB_UNSUPPORTED_CONTENT_TYPE`.
2227
+ *
2228
+ * Two layers, kept apart so the heuristics are testable without pdf.js:
2229
+ *
2230
+ * 1. {@link readPdfDocument} — the pdf.js boundary (through `unpdf`'s
2231
+ * serverless build, loaded lazily so plugin start never pays for it).
2232
+ * Per page it reads the positioned text runs (`getTextContent`) plus the
2233
+ * real font names (`getOperatorList` + `fontExtraProperties`, the only way
2234
+ * pdf.js exposes "NimbusRomNo9L-Medi" instead of a generic family), so the
2235
+ * layout pass can see bold and monospace. Bounded by the fetch deadline,
2236
+ * the caller's abort signal, a page cap, and a collected-character cap.
2237
+ * 2. {@link layoutPdfMarkdown} — pure heuristics over those runs:
2238
+ * - runs → lines (baseline grouping in content-stream order, x-sorted
2239
+ * inside a line; rotated runs such as arXiv's side stamp are dropped);
2240
+ * - running headers/footers (text repeated in the page margins across
2241
+ * pages, digits normalized) and bare page numbers are removed;
2242
+ * - the dominant (body) font size and line pitch are measured;
2243
+ * - headings come from, strongest first: the PDF outline (bookmarks),
2244
+ * section numbering ("3.1 Pre-training", "第二章", "一、"), larger font
2245
+ * sizes (ranked into levels), and standalone bold lines;
2246
+ * - lines are merged into paragraphs (vertical gap, first-line indent,
2247
+ * short terminal line, font change; continuation across columns and
2248
+ * pages when the sentence is unfinished), with dictionary-checked
2249
+ * de-hyphenation and CJK-aware joining;
2250
+ * - bullets / numbered items become markdown lists, monospace blocks
2251
+ * become fenced code (unless the whole document is monospace), lines
2252
+ * with wide column gaps become GFM tables when they line up;
2253
+ * - `<!-- page N -->` markers keep page provenance for citations.
2254
+ *
2255
+ * Scanned PDFs without a text layer, encrypted PDFs, and broken files fail
2256
+ * with dedicated, provider-specific error codes (OCR is out of scope).
2257
+ *
2258
+ * @module dsh-web-fetch-playwright/pdf
2259
+ */
2260
+ /** Largest PDF this provider downloads and parses (bytes). */
2261
+ const MAX_PDF_BYTES = 52428800;
2262
+ /** Most pages read from one document; later pages are reported as truncated. */
2263
+ const MAX_PDF_PAGES = 500;
2264
+ /** No text layer (a scanned / image-only PDF); OCR is not supported. */
2265
+ const WEB_PDF_NO_TEXT_CODE = "WEB_PDF_NO_TEXT";
2266
+ /** Password-protected PDF the parser cannot open. */
2267
+ const WEB_PDF_ENCRYPTED_CODE = "WEB_PDF_ENCRYPTED";
2268
+ /** Bytes claimed (or sniffed) to be a PDF that pdf.js cannot parse. */
2269
+ const WEB_PDF_INVALID_CODE = "WEB_PDF_INVALID";
2270
+ /** PDF larger than {@link MAX_PDF_BYTES}. */
2271
+ const WEB_PDF_TOO_LARGE_CODE = "WEB_PDF_TOO_LARGE";
2272
+ const PDF_MIME_TYPES = /* @__PURE__ */ new Set([
2273
+ "application/pdf",
2274
+ "application/x-pdf",
2275
+ "application/acrobat",
2276
+ "applications/vnd.pdf",
2277
+ "text/pdf",
2278
+ "text/x-pdf"
2279
+ ]);
2280
+ /** Generic binary types a server may use for a PDF; their bytes get sniffed. */
2281
+ const OPAQUE_BINARY_TYPES = /* @__PURE__ */ new Set([
2282
+ "application/octet-stream",
2283
+ "binary/octet-stream",
2284
+ "application/binary",
2285
+ "application/download",
2286
+ "application/x-download",
2287
+ "application/force-download",
2288
+ "application/unknown"
2289
+ ]);
2290
+ /** The bare media type of a `Content-Type` header value. */
2291
+ function mediaType(contentType) {
2292
+ return (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
2293
+ }
2294
+ /** Whether a `Content-Type` names a PDF. */
2295
+ function isPdfContentType(contentType) {
2296
+ return PDF_MIME_TYPES.has(mediaType(contentType));
2297
+ }
2298
+ /**
2299
+ * Whether a response that is not otherwise decodable is worth downloading
2300
+ * to sniff for PDF bytes: a PDF content type, a `.pdf` attachment filename,
2301
+ * a `.pdf` URL path, or an opaque binary type (servers commonly hand PDFs out
2302
+ * as `application/octet-stream`). The magic-byte check then decides.
2303
+ */
2304
+ function mayBePdf(headers, url) {
2305
+ if (isPdfContentType(headers["content-type"])) return true;
2306
+ const disposition = headers["content-disposition"] ?? "";
2307
+ if (/filename\*?\s*=\s*(?:[\w-]+'[^']*')?"?[^";]*\.pdf\b/i.test(disposition)) return true;
2308
+ try {
2309
+ if (/\.pdf$/i.test(new URL(url).pathname)) return true;
2310
+ } catch {}
2311
+ return OPAQUE_BINARY_TYPES.has(mediaType(headers["content-type"]));
2312
+ }
2313
+ /**
2314
+ * Whether the bytes are a PDF: `%PDF-` within the first 1024 bytes (the
2315
+ * spec-tolerated leading-garbage window every mainstream reader accepts).
2316
+ */
2317
+ function isPdfBytes(bytes) {
2318
+ const limit = Math.min(bytes.length - 5, 1024);
2319
+ for (let i = 0; i <= limit; i++) if (bytes[i] === 37 && bytes[i + 1] === 80 && bytes[i + 2] === 68 && bytes[i + 3] === 70 && bytes[i + 4] === 45) return true;
2320
+ return false;
2321
+ }
2322
+ /** Bold-looking PostScript font names (subset prefix already stripped). */
2323
+ const BOLD_FONT_RE = /bold|black|heavy|semibold|demibold|demi\b|[-,]?medi(um)?\b|-B$|,B$|-SB$|CMBX|CMB\d|BX\d|\bHei\b|SimHei|Heiti|HTJ-|HT-|黑体/i;
2324
+ /** Monospace-looking font names. */
2325
+ const MONO_FONT_RE = /mono|courier|consol|menlo|inconsolata|NimbusMonL|CMTT|SFTT|LMMono|SourceCodePro|FiraCode|DejaVuSansMono|LucidaConsole|Code/i;
2326
+ /**
2327
+ * Classify a pdf.js font object by its real (PostScript) name. pdf.js only
2328
+ * exposes the name after the page's operator list loaded the font, and only
2329
+ * with `fontExtraProperties`; anything unreadable classifies as regular.
2330
+ */
2331
+ function classifyFont(page, loadedName, cache) {
2332
+ if (loadedName === void 0) return {
2333
+ bold: false,
2334
+ mono: false
2335
+ };
2336
+ const cached = cache.get(loadedName);
2337
+ if (cached !== void 0) return cached;
2338
+ let name = "";
2339
+ let flagsBold = false;
2340
+ try {
2341
+ const font = page.commonObjs.get(loadedName);
2342
+ name = (font?.name ?? "").replace(/^[A-Z]{6}\+/, "");
2343
+ flagsBold = font?.bold === true || font?.black === true;
2344
+ } catch {}
2345
+ const result = {
2346
+ bold: flagsBold || BOLD_FONT_RE.test(name),
2347
+ mono: MONO_FONT_RE.test(name)
2348
+ };
2349
+ cache.set(loadedName, result);
2350
+ return result;
2351
+ }
2352
+ /** Flatten the outline tree (bounded) into `{ title, depth }` entries. */
2353
+ function flattenOutline(nodes, depth = 0, out = []) {
2354
+ if (nodes === null) return out;
2355
+ for (const node of nodes) {
2356
+ if (out.length >= 2e3) break;
2357
+ const title = (node.title ?? "").replace(/\s+/g, " ").trim();
2358
+ if (title !== "") out.push({
2359
+ title,
2360
+ depth
2361
+ });
2362
+ if (node.items !== void 0 && node.items.length > 0 && depth < 8) flattenOutline(node.items, depth + 1, out);
2363
+ }
2364
+ return out;
2365
+ }
2366
+ /** Map a pdf.js load failure onto the provider's PDF error codes. */
2367
+ function translatePdfError(error) {
2368
+ if (error instanceof WebError) return error;
2369
+ const name = error instanceof Error ? error.name : "";
2370
+ const message = error instanceof Error ? error.message : String(error);
2371
+ if (name === "PasswordException" || /password/i.test(message)) return new WebError("the PDF is password-protected and cannot be read", WEB_PDF_ENCRYPTED_CODE, { cause: error });
2372
+ return new WebError(`the PDF could not be parsed: ${message.split("\n")[0].slice(0, 200)}`, WEB_PDF_INVALID_CODE, { cause: error });
2373
+ }
2374
+ /**
2375
+ * Map Kangxi / CJK radical code points to the unified ideographs they look
2376
+ * like. Some producers (Chromium printing Noto CJK among them) write a
2377
+ * ToUnicode CMap that resolves 文 / 用 / 长 glyphs to U+2F42 ⽂ / U+2F64 ⽤
2378
+ * / U+2EA7 ⻓ — visually identical, but they break search and reading.
2379
+ * NFKC maps exactly these; it is applied to nothing else (fullwidth forms
2380
+ * stay as the document wrote them).
2381
+ */
2382
+ function normalizeRadicals(text) {
2383
+ return text.replace(/[\u2E80-\u2FDF]/g, (ch) => SIMPLIFIED_RADICALS[ch] ?? ch.normalize("NFKC"));
2384
+ }
2385
+ /**
2386
+ * The C-simplified radicals (CJK Radicals Supplement) have no NFKC mapping;
2387
+ * these are the ones that double as everyday simplified characters.
2388
+ */
2389
+ const SIMPLIFIED_RADICALS = {
2390
+ "⻅": "见",
2391
+ "⻆": "角",
2392
+ "⻈": "讠",
2393
+ "⻉": "贝",
2394
+ "⻋": "车",
2395
+ "⻐": "钅",
2396
+ "⻓": "长",
2397
+ "⻔": "门",
2398
+ "⻙": "韦",
2399
+ "⻚": "页",
2400
+ "⻛": "风",
2401
+ "⻜": "飞",
2402
+ "⻠": "饣",
2403
+ "⻢": "马",
2404
+ "⻥": "鱼",
2405
+ "⻦": "鸟",
2406
+ "⻧": "卤",
2407
+ "⻨": "麦",
2408
+ "⻩": "黄",
2409
+ "⻪": "黾",
2410
+ "⻬": "齐",
2411
+ "⻮": "齿",
2412
+ "⻰": "龙"
2413
+ };
2414
+ /** A run is horizontal when the text matrix has (almost) no rotation/skew. */
2415
+ function isHorizontal(t) {
2416
+ const [a = 0, b = 0, c = 0, d = 0] = t;
2417
+ return Math.abs(b) <= Math.abs(a) * .1 + 1e-6 && Math.abs(c) <= Math.abs(d) * .1 + 1e-6 && a > 0 && d > 0;
2418
+ }
2419
+ /**
2420
+ * Read a PDF's positioned text with pdf.js, page by page, inside the given
2421
+ * limits. Text items become {@link PdfTextRun}s (horizontal only — rotated
2422
+ * stamps and vertical margin text are dropped as layout noise).
2423
+ *
2424
+ * @returns the text model, plus whether a limit stopped reading early.
2425
+ */
2426
+ async function readPdfDocument(bytes, options = {}) {
2427
+ const startedAt = Date.now();
2428
+ const budget = options.timeBudgetMs ?? Number.POSITIVE_INFINITY;
2429
+ const maxChars = options.maxChars ?? Number.POSITIVE_INFINITY;
2430
+ const maxPages = options.maxPages ?? 500;
2431
+ const { getDocumentProxy } = await import("unpdf");
2432
+ const data = new Uint8Array(bytes.byteLength);
2433
+ data.set(bytes);
2434
+ let pdf;
2435
+ try {
2436
+ pdf = await getDocumentProxy(data, {
2437
+ fontExtraProperties: true,
2438
+ verbosity: 0,
2439
+ stopAtErrors: false
2440
+ });
2441
+ } catch (error) {
2442
+ throw translatePdfError(error);
2443
+ }
2444
+ try {
2445
+ let title;
2446
+ try {
2447
+ const meta = await pdf.getMetadata();
2448
+ const xmpTitle = meta.metadata?.get("dc:title");
2449
+ const infoTitle = meta.info?.Title;
2450
+ const candidate = typeof xmpTitle === "string" && xmpTitle.trim() !== "" ? xmpTitle : typeof infoTitle === "string" ? infoTitle : "";
2451
+ if (candidate.trim() !== "") title = candidate.replace(/\s+/g, " ").trim();
2452
+ } catch {}
2453
+ let outline = [];
2454
+ try {
2455
+ outline = flattenOutline(await pdf.getOutline());
2456
+ } catch {}
2457
+ const pages = [];
2458
+ const fontCache = /* @__PURE__ */ new Map();
2459
+ let chars = 0;
2460
+ let truncated = false;
2461
+ const pageCount = pdf.numPages;
2462
+ for (let n = 1; n <= pageCount; n++) {
2463
+ if (options.signal?.aborted) throw options.signal.reason instanceof Error ? options.signal.reason : /* @__PURE__ */ new Error("pdf extraction aborted");
2464
+ if (n > maxPages || chars >= maxChars || Date.now() - startedAt >= budget) {
2465
+ truncated = true;
2466
+ break;
2467
+ }
2468
+ const page = await pdf.getPage(n);
2469
+ const content = await page.getTextContent();
2470
+ if (Date.now() - startedAt < budget * .8) try {
2471
+ await page.getOperatorList();
2472
+ } catch {}
2473
+ const [, viewBottom = 0, , viewTop = 0] = page.view;
2474
+ const runs = [];
2475
+ for (const item of content.items) {
2476
+ if (typeof item.str !== "string" || item.str === "" || item.transform === void 0) continue;
2477
+ if (!isHorizontal(item.transform)) continue;
2478
+ const t = item.transform;
2479
+ const size = Math.hypot(t[2] ?? 0, t[3] ?? 0) || item.height || 0;
2480
+ if (!(size > 0)) continue;
2481
+ const font = classifyFont(page, item.fontName, fontCache);
2482
+ runs.push({
2483
+ text: normalizeRadicals(item.str),
2484
+ x: t[4] ?? 0,
2485
+ y: t[5] ?? 0,
2486
+ width: item.width ?? 0,
2487
+ size,
2488
+ bold: font.bold,
2489
+ mono: font.mono
2490
+ });
2491
+ chars += item.str.length;
2492
+ }
2493
+ pages.push({
2494
+ number: n,
2495
+ bottom: Math.min(viewBottom, viewTop),
2496
+ height: Math.abs(viewTop - viewBottom) || 792,
2497
+ runs
2498
+ });
2499
+ page.cleanup();
2500
+ }
2501
+ return {
2502
+ doc: {
2503
+ ...title !== void 0 ? { title } : {},
2504
+ outline,
2505
+ pages,
2506
+ totalPages: pageCount
2507
+ },
2508
+ truncated
2509
+ };
2510
+ } catch (error) {
2511
+ if (options.signal?.aborted) throw error;
2512
+ throw translatePdfError(error);
2513
+ } finally {
2514
+ pdf.loadingTask?.destroy().catch(() => {});
2515
+ }
2516
+ }
2517
+ /**
2518
+ * Extract a PDF as markdown: read it with pdf.js inside the limits, lay it
2519
+ * out, and fail with {@link WEB_PDF_NO_TEXT_CODE} when there is no text layer
2520
+ * to speak of (a scan). Sizes beyond {@link MAX_PDF_BYTES} are refused.
2521
+ */
2522
+ async function extractPdfMarkdown(bytes, options = {}) {
2523
+ if (bytes.length > 52428800) throw new WebError(`the PDF is ${String(Math.round(bytes.length / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE);
2524
+ const { doc, truncated } = await readPdfDocument(bytes, options);
2525
+ const textChars = doc.pages.reduce((sum, page) => sum + page.runs.reduce((s, run) => s + run.text.replace(/\s+/g, "").length, 0), 0);
2526
+ if (doc.pages.length > 0 && textChars < Math.max(1, doc.pages.length * 5)) throw new WebError(`the PDF has no extractable text layer (${String(doc.totalPages)} page(s), likely scanned images); OCR is not supported`, WEB_PDF_NO_TEXT_CODE);
2527
+ let markdown = layoutPdfMarkdown(doc);
2528
+ if (truncated) markdown += `\n\n> [PDF extraction stopped after page ${String(doc.pages.length)} of ${String(doc.totalPages)} (size/time limit).]\n`;
2529
+ return {
2530
+ markdown,
2531
+ totalPages: doc.totalPages,
2532
+ pagesRead: doc.pages.length,
2533
+ truncated
2534
+ };
2535
+ }
2536
+ /** CJK ideographs, kana, hangul, and fullwidth/CJK punctuation. */
2537
+ const CJK_RE = /[⺀-⿿ -〿぀-ヿ㄀-ㇿ㐀-䶿一-鿿가-힯豈-﫿＀-￯]/;
2538
+ function isCjk(ch) {
2539
+ return ch !== void 0 && CJK_RE.test(ch);
2540
+ }
2541
+ /** Letter-spaced CJK ("预 防 和 治 理") reads as one word. */
2542
+ function collapseCjkSpacing(text) {
2543
+ return text.replace(new RegExp(`(?<=${CJK_RE.source}) +(?=${CJK_RE.source})`, "g"), "");
2544
+ }
2545
+ /** Group a page's runs into lines (content-stream order, baseline match). */
2546
+ function buildLines(page) {
2547
+ const groups = [];
2548
+ let current = [];
2549
+ let baseY = 0;
2550
+ let baseSize = 0;
2551
+ for (const run of page.runs) {
2552
+ if (current.length > 0) {
2553
+ const tolerance = Math.max(.6 * Math.max(run.size, baseSize), 1);
2554
+ if (Math.abs(run.y - baseY) <= tolerance) {
2555
+ current.push(run);
2556
+ if (run.text.trim() !== "" && run.size > baseSize) {
2557
+ baseSize = run.size;
2558
+ baseY = run.y;
2559
+ }
2560
+ continue;
2561
+ }
2562
+ groups.push(current);
2563
+ }
2564
+ current = [run];
2565
+ baseY = run.y;
2566
+ baseSize = run.size;
2567
+ }
2568
+ if (current.length > 0) groups.push(current);
2569
+ const lines = [];
2570
+ for (const group of groups) {
2571
+ const line = composeLine(group, page);
2572
+ if (line !== void 0) lines.push(line);
2573
+ }
2574
+ return lines;
2575
+ }
2576
+ /** Join one baseline group's runs (x-sorted) into a {@link Line}. */
2577
+ function composeLine(group, page) {
2578
+ const runs = [...group].sort((a, b) => a.x - b.x);
2579
+ const visible = runs.filter((run) => run.text.trim() !== "");
2580
+ if (visible.length === 0) return void 0;
2581
+ const weight = /* @__PURE__ */ new Map();
2582
+ for (const run of visible) {
2583
+ const key = Math.round(run.size * 10) / 10;
2584
+ weight.set(key, (weight.get(key) ?? 0) + run.text.trim().length);
2585
+ }
2586
+ let size = visible[0].size;
2587
+ let best = -1;
2588
+ for (const [key, w] of weight) if (w > best) {
2589
+ best = w;
2590
+ size = key;
2591
+ }
2592
+ const baseline = visible.find((run) => Math.abs(run.size - size) < .11)?.y ?? visible[0].y;
2593
+ const segments = [];
2594
+ const cells = [""];
2595
+ const cellX = [];
2596
+ let text = "";
2597
+ let prevEnd;
2598
+ let boldChars = 0;
2599
+ let monoChars = 0;
2600
+ let chars = 0;
2601
+ for (const run of runs) {
2602
+ const content = run.text;
2603
+ const trimmed = content.trim();
2604
+ if (trimmed === "") continue;
2605
+ let sep = "";
2606
+ if (prevEnd !== void 0 && text !== "") {
2607
+ const gap = run.x - prevEnd;
2608
+ const last = text[text.length - 1];
2609
+ const first = trimmed[0];
2610
+ if (gap > run.size * 1.5 && !(isCjk(last) && isCjk(first) && gap < run.size * 3)) {
2611
+ cells.push("");
2612
+ cellX.push(run.x);
2613
+ sep = " ";
2614
+ } else if (gap > run.size * .15 || /\s$/.test(content.slice(0, content.length - content.trimStart().length)) || /^\s/.test(content)) sep = isCjk(last) && isCjk(first) ? "" : " ";
2615
+ if (/\s$/.test(text)) sep = "";
2616
+ }
2617
+ const piece = sep + trimmed;
2618
+ if (cellX.length === 0) cellX.push(run.x);
2619
+ text += piece;
2620
+ cells[cells.length - 1] += cells[cells.length - 1] === "" ? trimmed : piece;
2621
+ const prev = segments[segments.length - 1];
2622
+ if (prev !== void 0 && prev.bold === run.bold) prev.text += piece;
2623
+ else segments.push({
2624
+ text: piece,
2625
+ bold: run.bold
2626
+ });
2627
+ prevEnd = Math.max(prevEnd ?? -Infinity, run.x + run.width);
2628
+ const n = trimmed.replace(/\s+/g, "").length;
2629
+ chars += n;
2630
+ if (run.bold) boldChars += n;
2631
+ if (run.mono) monoChars += n;
2632
+ }
2633
+ text = collapseCjkSpacing(text.replace(/\s+/g, " ").trim());
2634
+ if (text === "") return void 0;
2635
+ for (const segment of segments) segment.text = collapseCjkSpacing(segment.text);
2636
+ const x0 = visible.reduce((m, run) => Math.min(m, run.x), Infinity);
2637
+ const x1 = visible.reduce((m, run) => Math.max(m, run.x + run.width), -Infinity);
2638
+ return {
2639
+ page: page.number,
2640
+ text,
2641
+ segments,
2642
+ x0,
2643
+ x1,
2644
+ y: baseline,
2645
+ size,
2646
+ bold: chars > 0 && boldChars / chars >= .8,
2647
+ mono: chars > 0 && monoChars / chars >= .8,
2648
+ cells: cells.map((cell) => collapseCjkSpacing(cell.replace(/\s+/g, " ").trim())),
2649
+ cellX,
2650
+ chars,
2651
+ relY: page.height > 0 ? (baseline - page.bottom) / page.height : .5
2652
+ };
2653
+ }
2654
+ /** Bare page-number shapes found in page margins. */
2655
+ const PAGE_NUMBER_RES = [
2656
+ /^[-–—\s]*(?:page|p\.|pág\.|seite)?\s*\d{1,4}(?:\s*(?:\/|of|von|de)\s*\d{1,4})?[-–—\s]*$/i,
2657
+ /^[-–—\s]*[ivxlcdm]{1,7}[-–—\s]*$/i,
2658
+ /^第\s*\d{1,4}\s*页(?:\s*[,,//]?\s*共\s*\d{1,4}\s*页)?$/,
2659
+ /^[-–—一-―·•]{1,2}\s*\d{1,4}\s*[-–—一-―·•]{1,2}$/
2660
+ ];
2661
+ const isMarginLine = (line) => line.relY > .9 || line.relY < .12;
2662
+ /** Drop running headers/footers and page numbers. */
2663
+ function removeMarginNoise(pages) {
2664
+ const keyOf = (line) => line.text.toLowerCase().replace(/\d+/g, "#").replace(/\s+/g, " ").trim();
2665
+ const pagesByKey = /* @__PURE__ */ new Map();
2666
+ for (const lines of pages) for (const line of lines) {
2667
+ if (!isMarginLine(line)) continue;
2668
+ const key = keyOf(line);
2669
+ let set = pagesByKey.get(key);
2670
+ if (set === void 0) {
2671
+ set = /* @__PURE__ */ new Set();
2672
+ pagesByKey.set(key, set);
2673
+ }
2674
+ set.add(line.page);
2675
+ }
2676
+ const repeatThreshold = Math.max(3, Math.ceil(pages.length * .3));
2677
+ return pages.map((lines) => lines.filter((line) => {
2678
+ if (!isMarginLine(line)) return true;
2679
+ const plain = line.text.normalize("NFKC");
2680
+ if (PAGE_NUMBER_RES.some((re) => re.test(plain))) return false;
2681
+ if (pages.length >= 3 && (pagesByKey.get(keyOf(line))?.size ?? 0) >= repeatThreshold) return false;
2682
+ return true;
2683
+ }));
2684
+ }
2685
+ /** Character-weighted mode of the line font sizes (0.5pt buckets). */
2686
+ function bodySizeOf(lines) {
2687
+ const weight = /* @__PURE__ */ new Map();
2688
+ for (const line of lines) {
2689
+ const key = Math.round(line.size * 2) / 2;
2690
+ weight.set(key, (weight.get(key) ?? 0) + line.chars);
2691
+ }
2692
+ let body = 10;
2693
+ let best = -1;
2694
+ for (const [key, w] of weight) if (w > best) {
2695
+ best = w;
2696
+ body = key;
2697
+ }
2698
+ return body;
2699
+ }
2700
+ /** Median baseline-to-baseline distance between consecutive body lines. */
2701
+ function lineSpacingOf(pages, body) {
2702
+ const deltas = [];
2703
+ for (const lines of pages) for (let i = 1; i < lines.length; i++) {
2704
+ const a = lines[i - 1];
2705
+ const b = lines[i];
2706
+ if (Math.abs(a.size - body) > body * .1 || Math.abs(b.size - body) > body * .1) continue;
2707
+ const dy = a.y - b.y;
2708
+ if (dy > body * .6 && dy < body * 3) deltas.push(dy);
2709
+ }
2710
+ if (deltas.length === 0) return body * 1.2;
2711
+ deltas.sort((p, q) => p - q);
2712
+ return deltas[Math.floor(deltas.length / 2)];
2713
+ }
2714
+ /** Normalization for comparing headings to outline titles. */
2715
+ function headingKey(text) {
2716
+ return text.normalize("NFKC").toLowerCase().replace(/[\s\p{P}\p{S}]/gu, "");
2717
+ }
2718
+ /** Leading section number ("3", "3.1", "A.2") stripped for outline matching. */
2719
+ function stripNumbering(text) {
2720
+ return text.replace(/^(?:\d+(?:\.\d+)*|[A-Z](?:\.\d+)+)\.?\s+/, "").replace(/^(?:第[一二三四五六七八九十百千零〇两\d]+[章节篇部]|[一二三四五六七八九十]+[、..]|[((][一二三四五六七八九十]+[))])\s*/, "");
2721
+ }
2722
+ const NUMBERED_HEADING_RE = /^(\d{1,2}(?:\.\d{1,2}){0,4}|[A-Z](?:\.\d{1,2}){1,4})\.?\s+(?=\S)/;
2723
+ const CN_CHAPTER_RE = /^第[一二三四五六七八九十百千零〇两\d]+[章篇部]/;
2724
+ const CN_SECTION_RE = /^第[一二三四五六七八九十百千零〇两\d]+节/;
2725
+ const CN_ENUM_RE = /^[一二三四五六七八九十]{1,3}[、.]/;
2726
+ const CN_PAREN_ENUM_RE = /^[((][一二三四五六七八九十]{1,3}[))]/;
2727
+ /** A table-of-contents entry: leader dots (or a wide gap) then a page reference. */
2728
+ const TOC_ENTRY_RE = /(?:[.…·.]\s*){4,}[^.…·.]{0,16}?[((]?\s*[\d0-9ivxlc]+\s*[))]?$/i;
2729
+ const EXPLICIT_HEADING_RES = [
2730
+ NUMBERED_HEADING_RE,
2731
+ CN_CHAPTER_RE,
2732
+ CN_SECTION_RE,
2733
+ CN_ENUM_RE,
2734
+ CN_PAREN_ENUM_RE
2735
+ ];
2736
+ const TERMINAL_RE = /[.!?。!?…::;;"”’))\]】]$/;
2737
+ const SENTENCE_END_RE = /[.!?。!?…"”’]$/;
2738
+ const BULLET_RE = /^(?:[•◦▪▫●○■□►▸‣⁃∙·*❖➢✓✔]|[])\s*/;
2739
+ const DASH_BULLET_RE = /^[-–—]\s+(?=\S)/;
2740
+ const ORDERED_RE = /^(\d{1,3})[.)]\s+(?=\S)/;
2741
+ const PAREN_ITEM_RE = /^(?:[((](?:\d{1,3}|[a-zA-Z]|[ivx]{1,4})[))]|[a-z][.)]|[①-⑳])\s*(?=\S)/;
2742
+ /** Heading level placeholder: one below the most recent heading. */
2743
+ const RELATIVE_HEADING = -1;
2744
+ /** Decide a line's heading level, if it is one. */
2745
+ function headingLevel(line, groupLines, stats, standalone, styleRules) {
2746
+ const text = groupLines.map((l) => l.text).join(" ");
2747
+ const chars = groupLines.reduce((s, l) => s + l.chars, 0);
2748
+ if (chars === 0 || chars > 200 || !/\p{L}/u.test(text)) return void 0;
2749
+ const words = text.split(/\s+/).length;
2750
+ const sentenceLike = /[,,;;]$/.test(text) || /[.。]$/.test(text) && !/^\d+(\.\d+)*\.?$/.test(text.split(" ")[0] ?? "");
2751
+ if (stats.titleSize !== void 0 && Math.abs(line.size - stats.titleSize) < .3 && groupLines.length <= 3 && styleRules) return 1;
2752
+ const outlineDepth = stats.outlineKeys.get(headingKey(text)) ?? stats.outlineKeys.get(headingKey(stripNumbering(text)));
2753
+ if (outlineDepth !== void 0 && chars <= 160) return Math.min(6, Math.max(2, outlineDepth + 2 - stats.outlineOffset));
2754
+ const bigger = line.size >= stats.body * 1.12;
2755
+ const emphasized = bigger || line.bold;
2756
+ const styleOk = styleRules && groupLines.every((l) => l.cells.length < 2);
2757
+ const numbered = NUMBERED_HEADING_RE.exec(text);
2758
+ if (numbered !== null && emphasized && !sentenceLike && words <= 16 && chars <= 120) {
2759
+ const depth = numbered[1].split(".").length;
2760
+ return Math.min(6, depth + 1);
2761
+ }
2762
+ if (CN_CHAPTER_RE.test(text) && chars <= 60 && !sentenceLike) return 2;
2763
+ if (CN_SECTION_RE.test(text) && chars <= 60 && !sentenceLike) return 3;
2764
+ if (CN_ENUM_RE.test(text) && chars <= 40 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 2;
2765
+ if (CN_PAREN_ENUM_RE.test(text) && chars <= 30 && !/[。;;]$/.test(text) && (emphasized || standalone)) return 3;
2766
+ if (!styleOk) return void 0;
2767
+ if (bigger && groupLines.length <= 3 && !sentenceLike) {
2768
+ if (stats.titleSize !== void 0 && Math.abs(line.size - stats.titleSize) < .3) return 1;
2769
+ const rank = stats.headingSizes.findIndex((size) => Math.abs(size - Math.round(line.size * 2) / 2) < .3);
2770
+ return Math.min(4, 2 + Math.max(0, rank));
2771
+ }
2772
+ const lastSegment = groupLines[groupLines.length - 1].segments.at(-1);
2773
+ if (line.bold && lastSegment?.bold === true && standalone && groupLines.length <= 2 && chars <= 100 && words <= 14 && !sentenceLike && !TERMINAL_RE.test(text.replace(/[))]$/, ""))) return RELATIVE_HEADING;
2774
+ }
2775
+ /** Group lines into markdown blocks. */
2776
+ function buildBlocks(pages, stats) {
2777
+ const all = pages.flat();
2778
+ const blocks = [];
2779
+ const rightEdge = /* @__PURE__ */ new Map();
2780
+ for (const lines of pages) for (const line of lines) {
2781
+ let right = line.x1;
2782
+ for (const other of lines) if (Math.abs(other.x0 - line.x0) < line.size * 3 && Math.abs(other.size - line.size) < line.size * .15) right = Math.max(right, other.x1);
2783
+ rightEdge.set(line, right);
2784
+ }
2785
+ const pitch = (line) => stats.spacing * (line.size / stats.body);
2786
+ const gapBefore = (i) => {
2787
+ const prev = all[i - 1];
2788
+ const line = all[i];
2789
+ if (prev === void 0 || prev.page !== line.page) return Infinity;
2790
+ return prev.y - line.y;
2791
+ };
2792
+ const isStandalone = (i, span) => {
2793
+ const before = gapBefore(i);
2794
+ const after = i + span < all.length ? gapBefore(i + span) : Infinity;
2795
+ const first = all[i];
2796
+ const sepBefore = before === Infinity || before < 0 || before > pitch(first) * 1.3 || all[i - 1].size !== first.size || all[i - 1].bold !== first.bold;
2797
+ const next = all[i + span];
2798
+ const sepAfter = after === Infinity || after < 0 || after > pitch(first) * 1.3 || next === void 0 || Math.abs(next.size - first.size) > .3;
2799
+ const short = all[i + span - 1].x1 < (rightEdge.get(all[i + span - 1]) ?? Infinity) - first.size * 2;
2800
+ return sepBefore && (sepAfter || short);
2801
+ };
2802
+ const isInlineMono = (index) => {
2803
+ const line = all[index];
2804
+ const prev = all[index - 1];
2805
+ const next = all[index + 1];
2806
+ if (prev === void 0 || prev.mono || next?.mono === true && next.page === line.page && gapBefore(index + 1) > 0 && gapBefore(index + 1) < pitch(line) * 2.2) return false;
2807
+ const dy = gapBefore(index);
2808
+ return dy > 0 && dy < pitch(line) * 1.45 && !TERMINAL_RE.test(prev.text) && line.chars <= 80;
2809
+ };
2810
+ const tableEnd = /* @__PURE__ */ new Map();
2811
+ const inTable = /* @__PURE__ */ new Set();
2812
+ for (let start = 0; start < all.length;) {
2813
+ if (all[start].cells.length < 2 || all[start].mono) {
2814
+ start++;
2815
+ continue;
2816
+ }
2817
+ let end = start + 1;
2818
+ while (end < all.length && all[end].cells.length >= 2 && !all[end].mono && all[end].page === all[start].page && gapBefore(end) > 0 && gapBefore(end) < pitch(all[end]) * 2.5) end++;
2819
+ let first = start;
2820
+ let last = end;
2821
+ while (last - first >= 2 && !rowAligned(all[last - 1], all.slice(first, last - 1))) last--;
2822
+ while (last - first >= 2 && !rowAligned(all[first], all.slice(first + 1, last))) first++;
2823
+ if (last - first >= 2 && columnsAlign(all.slice(first, last))) {
2824
+ tableEnd.set(first, last);
2825
+ for (let k = first; k < last; k++) inTable.add(k);
2826
+ }
2827
+ start = end;
2828
+ }
2829
+ let lastLevel = 1;
2830
+ let i = 0;
2831
+ while (i < all.length) {
2832
+ const line = all[i];
2833
+ let span = 1;
2834
+ while (span < 4 && i + span < all.length && all[i + span].page === line.page && Math.abs(all[i + span].size - line.size) < .3 && all[i + span].bold === line.bold && gapBefore(i + span) > 0 && gapBefore(i + span) < pitch(line) * 1.6 && !EXPLICIT_HEADING_RES.some((re) => re.test(all[i + span].text))) span++;
2835
+ let level;
2836
+ let used = 0;
2837
+ if (span <= 3) {
2838
+ level = headingLevel(line, all.slice(i, i + span), stats, isStandalone(i, span), true);
2839
+ if (level !== void 0) used = span;
2840
+ }
2841
+ if (level === void 0 && span > 1) {
2842
+ level = headingLevel(line, [line], stats, isStandalone(i, 1), false);
2843
+ if (level !== void 0) used = 1;
2844
+ }
2845
+ if (level !== void 0 && line.mono && !stats.monoIsBody) level = void 0;
2846
+ if (level === RELATIVE_HEADING) level = Math.min(6, Math.max(2, lastLevel + 1));
2847
+ else if (level !== void 0) lastLevel = level;
2848
+ if (level !== void 0) {
2849
+ blocks.push({
2850
+ kind: "heading",
2851
+ page: line.page,
2852
+ level,
2853
+ lines: all.slice(i, i + used)
2854
+ });
2855
+ i += used;
2856
+ continue;
2857
+ }
2858
+ const start = i;
2859
+ const kind = tableEnd.has(i) ? "table" : lineKind(line, stats);
2860
+ const block = {
2861
+ kind: kind === "code" && isInlineMono(i) ? "text" : kind,
2862
+ page: line.page,
2863
+ lines: [line]
2864
+ };
2865
+ i++;
2866
+ if (kind === "table") {
2867
+ const end = tableEnd.get(start) ?? start + 1;
2868
+ while (i < end) {
2869
+ block.lines.push(all[i]);
2870
+ i++;
2871
+ }
2872
+ blocks.push(block);
2873
+ continue;
2874
+ }
2875
+ if (kind === "code") {
2876
+ while (i < all.length && all[i].mono && gapBefore(i) > 0 && gapBefore(i) < pitch(all[i]) * 2.2) {
2877
+ block.lines.push(all[i]);
2878
+ i++;
2879
+ }
2880
+ blocks.push(block);
2881
+ continue;
2882
+ }
2883
+ let blockRight = rightEdge.get(line) ?? line.x1;
2884
+ while (i < all.length) {
2885
+ const next = all[i];
2886
+ const prev = block.lines[block.lines.length - 1];
2887
+ if (inTable.has(i)) break;
2888
+ const nextKind = lineKind(next, stats);
2889
+ if (nextKind !== "text" && !(nextKind === "code" && isInlineMono(i))) break;
2890
+ if (Math.abs(next.size - prev.size) > Math.max(next.size, prev.size) * .12) break;
2891
+ const dy = next.page === prev.page ? prev.y - next.y : -Infinity;
2892
+ if (!(dy > 0 && dy < pitch(next) * 1.45)) {
2893
+ if (dy > 0) break;
2894
+ if (TERMINAL_RE.test(prev.text)) break;
2895
+ if (headingStartLike(next)) break;
2896
+ if (next.page !== prev.page && block.kind !== "text") break;
2897
+ } else {
2898
+ if (TOC_ENTRY_RE.test(prev.text)) break;
2899
+ if (next.x0 - prev.x0 > next.size * .8 && block.kind === "text" && next.x1 >= blockRight - next.size * 2 && (TERMINAL_RE.test(prev.text) || prev.x1 < blockRight - prev.size * 2)) break;
2900
+ if (block.kind !== "text" && next.x0 < block.lines[0].x0 + next.size * .3 && TERMINAL_RE.test(prev.text)) break;
2901
+ if (prev.x1 < blockRight - prev.size * 2.5 && SENTENCE_END_RE.test(prev.text)) break;
2902
+ if (next.bold !== prev.bold && next.bold && next.chars < 100 && SENTENCE_END_RE.test(prev.text)) break;
2903
+ }
2904
+ block.lines.push(next);
2905
+ blockRight = Math.max(blockRight, rightEdge.get(next) ?? next.x1);
2906
+ i++;
2907
+ }
2908
+ blocks.push(block);
2909
+ }
2910
+ return blocks;
2911
+ }
2912
+ /**
2913
+ * Whether table-candidate rows share column starts: at least 60% of the rows
2914
+ * have two or more cell left edges within one em of another row's.
2915
+ */
2916
+ function columnsAlign(rows) {
2917
+ const firstCells = rows.map((row) => row.cells.find((cell) => cell !== "") ?? "").map((cell) => cell.length).sort((a, b) => a - b);
2918
+ if (Math.max(...rows.map((row) => row.cells.filter((cell) => cell !== "").length)) <= 2 && (firstCells[Math.floor(firstCells.length / 2)] ?? 0) > 50) return false;
2919
+ return rows.filter((row) => rowAligned(row, rows.filter((other) => other !== row))).length / rows.length >= .6;
2920
+ }
2921
+ /** Whether a row shares at least two column starts with some other row. */
2922
+ function rowAligned(row, others) {
2923
+ return others.some((other) => row.cellX.filter((x) => other.cellX.some((o) => Math.abs(o - x) < row.size)).length >= 2);
2924
+ }
2925
+ /** A line that looks like the start of a new structural element. */
2926
+ function headingStartLike(line) {
2927
+ return BULLET_RE.test(line.text) || ORDERED_RE.test(line.text) || CN_CHAPTER_RE.test(line.text);
2928
+ }
2929
+ /** Classify a non-heading line. */
2930
+ function lineKind(line, stats) {
2931
+ if (line.mono && !stats.monoIsBody) return "code";
2932
+ if (BULLET_RE.test(line.text) && line.text.replace(BULLET_RE, "").trim() !== "") return "bullet";
2933
+ if (DASH_BULLET_RE.test(line.text)) return "bullet";
2934
+ if (ORDERED_RE.test(line.text)) return "ordered";
2935
+ if (PAREN_ITEM_RE.test(line.text)) return "item";
2936
+ return "text";
2937
+ }
2938
+ /** Words seen in the document (lowercased), for de-hyphenation decisions. */
2939
+ function vocabularyOf(lines) {
2940
+ const words = /* @__PURE__ */ new Set();
2941
+ for (const line of lines) for (const match of line.text.matchAll(/[A-Za-z]+(?:-[A-Za-z]+)*/g)) {
2942
+ const word = match[0].toLowerCase();
2943
+ words.add(word);
2944
+ if (word.includes("-")) words.add(word.replace(/-/g, "\0"));
2945
+ }
2946
+ return words;
2947
+ }
2948
+ /** Join two consecutive lines of one paragraph. */
2949
+ function joinText(acc, next, vocabulary) {
2950
+ if (acc === "") return next;
2951
+ const hyphen = /([A-Za-z]+)[-\u2010\u00AD]$/.exec(acc);
2952
+ const lead = /^([a-z]+)/.exec(next);
2953
+ if (hyphen !== null && lead !== null && !/\s[-\u2010\u00AD]$/.test(acc)) {
2954
+ const head = hyphen[1].toLowerCase();
2955
+ const tail = lead[1];
2956
+ if (vocabulary.has(head + tail)) return acc.slice(0, -1) + next;
2957
+ if (vocabulary.has(`${head}\u0000${tail}`)) return `${acc.slice(0, -1)}-${next}`;
2958
+ return acc.slice(0, -1) + next;
2959
+ }
2960
+ const last = acc[acc.length - 1];
2961
+ const first = next[0];
2962
+ if (isCjk(last) || isCjk(first)) return acc + next;
2963
+ if (/[-/]$/.test(acc)) return acc + next;
2964
+ return `${acc} ${next}`;
2965
+ }
2966
+ /** Escape what would otherwise read as markdown structure at a block start. */
2967
+ function escapeLeading(text) {
2968
+ return text.replace(/^(#{1,6}\s|>|[-+*]\s|\d{1,9}[.)]\s)/, "\\$1");
2969
+ }
2970
+ /** Render a line with inline bold (when only part of it is bold). */
2971
+ function renderLine(line, inlineBold) {
2972
+ if (!inlineBold || !line.segments.some((s) => s.bold) || line.segments.every((s) => s.bold)) return line.text;
2973
+ let out = "";
2974
+ for (const segment of line.segments) {
2975
+ const text = segment.text;
2976
+ const core = text.trim();
2977
+ if (segment.bold && core.replace(/[^\p{L}\p{N}]/gu, "").length >= 2 && !/^\*|\*$/.test(core)) {
2978
+ const lead = text.slice(0, text.length - text.trimStart().length);
2979
+ const trail = text.slice(text.trimEnd().length);
2980
+ out += `${lead}**${core}**${trail}`;
2981
+ } else out += text;
2982
+ }
2983
+ return out.replace(/\s+/g, " ").trim();
2984
+ }
2985
+ /**
2986
+ * Merge bold spans split only by whitespace or punctuation set in the
2987
+ * regular face ("**一是…会议**,**推动会议** **精神**" → one span): the
2988
+ * separator moves inside, and a line-break space between CJK spans goes.
2989
+ */
2990
+ function mergeBoldRuns(text) {
2991
+ return text.replace(/\*\*(\s*)([,、,;;::]?)(\s*)\*\*/g, (match, a, punct, b, offset, whole) => {
2992
+ const before = whole[offset - 1];
2993
+ const after = whole[offset + match.length];
2994
+ const space = (a !== "" || b !== "") && !(isCjk(before) || isCjk(after) || isCjk(punct)) ? " " : "";
2995
+ return punct === "" ? space : `${punct}${b !== "" && !isCjk(punct) ? " " : ""}`;
2996
+ });
2997
+ }
2998
+ /** Render a table block as GFM when its rows line up, else as plain rows. */
2999
+ function renderTable(block) {
3000
+ const rows = block.lines.map((line) => line.cells.filter((cell) => cell !== "").map((cell) => cell.replace(/\|/g, "\\|")));
3001
+ const width = Math.max(...rows.map((row) => row.length));
3002
+ const aligned = rows.filter((row) => row.length === width).length;
3003
+ if (rows.length >= 2 && width >= 2 && aligned / rows.length >= .6) {
3004
+ const pad = (row) => `| ${[...row, ...Array(width - row.length).fill("")].slice(0, width).join(" | ")} |`;
3005
+ return [
3006
+ pad(rows[0]),
3007
+ `| ${Array(width).fill("---").join(" | ")} |`,
3008
+ ...rows.slice(1).map(pad)
3009
+ ].join("\n");
3010
+ }
3011
+ return block.lines.map((line) => escapeLeading(line.cells.join(" "))).join(" \n");
3012
+ }
3013
+ /** Whether a metadata title is worth printing as the document heading. */
3014
+ function plausibleTitle(title) {
3015
+ if (title === void 0) return false;
3016
+ const t = title.trim();
3017
+ if (t.length < 3 || t.length > 250) return false;
3018
+ if (/^(?:microsoft\s+(?:word|powerpoint|excel)\b|untitled|slide\s*\d|document\d*$|title$|no title|无标题)/i.test(t)) return false;
3019
+ if (/\.(?:docx?|pdf|pptx?|xlsx?|tex|dvi|indd|qxd|rtf|odt)$/i.test(t)) return false;
3020
+ if (/^[\w-]+$/.test(t) && !/\s/.test(t) && /\d/.test(t)) return false;
3021
+ return true;
3022
+ }
3023
+ /**
3024
+ * Lay a PDF's positioned text out as markdown (see the module docs for the
3025
+ * heuristics). Pure — no pdf.js — so the rules are unit-testable with
3026
+ * synthetic runs.
3027
+ */
3028
+ function layoutPdfMarkdown(doc) {
3029
+ const pages = removeMarginNoise(doc.pages.map(buildLines));
3030
+ const all = pages.flat();
3031
+ if (all.length === 0) return plausibleTitle(doc.title) ? `# ${doc.title}\n` : "";
3032
+ const body = bodySizeOf(all);
3033
+ const totalChars = all.reduce((s, l) => s + l.chars, 0);
3034
+ const monoIsBody = all.reduce((s, l) => s + (l.mono ? l.chars : 0), 0) > totalChars * .5;
3035
+ const sizeChars = /* @__PURE__ */ new Map();
3036
+ for (const line of all) {
3037
+ if (line.size < body * 1.12) continue;
3038
+ const key = Math.round(line.size * 2) / 2;
3039
+ sizeChars.set(key, (sizeChars.get(key) ?? 0) + line.chars);
3040
+ }
3041
+ let headingSizes = [...sizeChars.keys()].filter((size) => (sizeChars.get(size) ?? 0) < Math.max(totalChars * .25, 400)).sort((a, b) => b - a);
3042
+ let titleSize;
3043
+ const firstPage = doc.pages[0]?.number ?? 1;
3044
+ const firstPageLines = all.filter((line) => line.page === firstPage && /\p{L}{2}/u.test(line.text));
3045
+ const firstLargest = firstPageLines.reduce((m, line) => Math.max(m, Math.round(line.size * 2) / 2), 0);
3046
+ if (firstLargest >= body * 1.3) {
3047
+ if (firstPageLines.filter((line) => Math.abs(Math.round(line.size * 2) / 2 - firstLargest) < .3).length <= 3) {
3048
+ titleSize = firstLargest;
3049
+ headingSizes = headingSizes.filter((size) => size < firstLargest - .25);
3050
+ }
3051
+ }
3052
+ headingSizes = headingSizes.slice(0, 3);
3053
+ const outlineKeys = /* @__PURE__ */ new Map();
3054
+ for (const entry of doc.outline) for (const key of [headingKey(entry.title), headingKey(stripNumbering(entry.title))]) if (key.length >= 2 && !outlineKeys.has(key)) outlineKeys.set(key, entry.depth);
3055
+ const roots = doc.outline.filter((entry) => entry.depth === 0);
3056
+ const titleKeys = new Set([doc.title ?? "", ...all.filter((line) => titleSize !== void 0 && Math.abs(line.size - titleSize) < .3).map((line) => line.text)].map(headingKey));
3057
+ const outlineOffset = roots.length === 1 && titleKeys.has(headingKey(roots[0].title)) ? 1 : 0;
3058
+ const blocks = buildBlocks(pages, {
3059
+ body,
3060
+ spacing: lineSpacingOf(pages, body),
3061
+ monoIsBody,
3062
+ headingSizes,
3063
+ ...titleSize !== void 0 ? { titleSize } : {},
3064
+ outlineKeys,
3065
+ outlineOffset
3066
+ });
3067
+ const vocabulary = vocabularyOf(all);
3068
+ const out = [];
3069
+ const metaKey = headingKey(doc.title ?? "");
3070
+ if (!blocks.slice(0, 12).some((block) => block.kind === "heading" && (block.level === 1 || headingKey(block.lines.map((l) => l.text).join("")) === metaKey)) && plausibleTitle(doc.title)) out.push(`# ${doc.title.replace(/\s+/g, " ").trim()}`);
3071
+ let lastPage = 0;
3072
+ for (const block of blocks) {
3073
+ if (block.page !== lastPage) {
3074
+ out.push(`<!-- page ${String(block.page)} -->`);
3075
+ lastPage = block.page;
3076
+ }
3077
+ switch (block.kind) {
3078
+ case "heading": {
3079
+ const text = block.lines.map((l) => l.text).reduce((acc, t) => joinText(acc, t, vocabulary), "");
3080
+ out.push(`${"#".repeat(block.level ?? 2)} ${text}`);
3081
+ break;
3082
+ }
3083
+ case "code": {
3084
+ const left = Math.min(...block.lines.map((l) => l.x0));
3085
+ const advances = block.lines.filter((l) => l.text.length >= 4).map((l) => (l.x1 - l.x0) / l.text.length).sort((p, q) => p - q);
3086
+ const advance = advances[Math.floor(advances.length / 2)] ?? block.lines[0].size * .6;
3087
+ const body = block.lines.map((l) => {
3088
+ const indent = Math.max(0, Math.round((l.x0 - left) / advance));
3089
+ return " ".repeat(Math.min(indent, 40)) + l.text;
3090
+ }).join("\n");
3091
+ out.push(`\`\`\`\n${body.replace(/```/g, "`​``")}\n\`\`\``);
3092
+ break;
3093
+ }
3094
+ case "table":
3095
+ out.push(renderTable(block));
3096
+ break;
3097
+ default: {
3098
+ let text = "";
3099
+ for (const line of block.lines) text = joinText(text, renderLine(line, true), vocabulary);
3100
+ text = mergeBoldRuns(text);
3101
+ if (block.kind === "bullet") {
3102
+ text = text.replace(BULLET_RE, "").replace(DASH_BULLET_RE, "");
3103
+ const prev = out[out.length - 1];
3104
+ if (prev?.startsWith("- ") === true) out[out.length - 1] = `${prev}\n- ${text}`;
3105
+ else out.push(`- ${text}`);
3106
+ } else if (block.kind === "ordered") {
3107
+ const prev = out[out.length - 1];
3108
+ if (prev !== void 0 && /^\d{1,3}\.\s/.test(prev)) out[out.length - 1] = `${prev}\n${text.replace(/^(\d{1,3})\)/, "$1.")}`;
3109
+ else out.push(text.replace(/^(\d{1,3})\)/, "$1."));
3110
+ } else out.push(escapeLeading(text));
3111
+ }
3112
+ }
3113
+ }
3114
+ return `${out.join("\n\n")}\n`;
3115
+ }
3116
+ //#endregion
1395
3117
  //#region src/playwright-resolve.ts
1396
3118
  /**
1397
3119
  * Locates a usable Playwright for the local backend. Resolution order:
@@ -1604,6 +3326,15 @@ function existsAsFile(path) {
1604
3326
  * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
1605
3327
  * a queue and fail fast (rather than hang until abort) when no slot frees.
1606
3328
  *
3329
+ * PDFs: a URL that serves a PDF — committed inline into Chrome's viewer,
3330
+ * handed to the download manager as an attachment, or served as an opaque
3331
+ * `application/octet-stream` whose bytes start with `%PDF-` — is read as
3332
+ * bytes (the navigation body, else a refetch through the page's API request
3333
+ * context) and extracted to markdown by `src/pdf.ts` (pdf.js via `unpdf`
3334
+ * plus layout heuristics: headings, paragraphs, lists, tables, page
3335
+ * markers). Scans without a text layer, encrypted, oversized, and broken
3336
+ * PDFs fail with the `WEB_PDF_*` codes.
3337
+ *
1607
3338
  * Private-network and SSRF protection is not implemented (same stance as the
1608
3339
  * shipped HTTP provider); a page this provider can reach is whatever the
1609
3340
  * browser can reach. Profile mode additionally acts WITH the remote
@@ -1621,6 +3352,31 @@ function existsAsFile(path) {
1621
3352
  * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
1622
3353
  * interstitial as content.
1623
3354
  *
3355
+ * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
3356
+ * shaped pages — Discourse topic streams, comment threads, timelines —
3357
+ * render only the items near the scroll position; everything else is an
3358
+ * empty, height-only placeholder that keeps the item's id but none of its
3359
+ * content, so a single-instant capture hands back the skeleton while the
3360
+ * bodies silently vanish (and the denoise pipeline's coverage gate cannot
3361
+ * notice: the page's own visible text is the same windowed fragment as the
3362
+ * extraction). App-shell document editors (Feishu docs) push the same
3363
+ * pathology one level down: the body never scrolls, blocks are keyed by
3364
+ * data-block-id with no id, and off-window blocks are REMOVED behind an
3365
+ * anonymous height spacer — the fetch would return just the statically
3366
+ * rendered first screen. Before serialization the fetch therefore runs one
3367
+ * bounded in-page scroll walk — over the window or, when the document does
3368
+ * not scroll, the page's dominant inner scroll container — stashing what
3369
+ * each rendered window shows, terminating only AT the bottom with the
3370
+ * height stable (that is what pages load-more sentinels; height stability
3371
+ * elsewhere proves nothing), then rehydrating degraded keyed elements from
3372
+ * their stashed snapshots and splicing removed keys back at their document
3373
+ * position so the final DOM holds the union of every window in document
3374
+ * order. The existing pipeline then runs unchanged. Bounded by
3375
+ * `scrollBudgetMs` (default 20s; 0 restores the legacy single-instant
3376
+ * behavior) and by the remaining fetch deadline; any walk failure silently
3377
+ * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
3378
+ * linux.do and Feishu badcases).
3379
+ *
1624
3380
  * @module dsh-web-fetch-playwright/provider
1625
3381
  */
1626
3382
  /** Stable id this provider registers under (the bundle patch pins it). */
@@ -1777,6 +3533,7 @@ function validateFetchUrl(input) {
1777
3533
  /** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
1778
3534
  function classifyContentType(contentType) {
1779
3535
  const mime = (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
3536
+ if (isPdfContentType(mime)) return "pdf";
1780
3537
  if (mime === "" || mime === "text/html" || mime === "application/xhtml+xml") return "html";
1781
3538
  if (mime.startsWith("text/")) return "text";
1782
3539
  if (mime === "application/json" || mime === "application/xml" || mime.endsWith("+json") || mime.endsWith("+xml")) return "text";
@@ -1970,16 +3727,27 @@ var PlaywrightFetchProvider = class {
1970
3727
  }
1971
3728
  const finalResponse = (challengeWaitMs > 0 ? tracker.last() : void 0) ?? response;
1972
3729
  const kind = classifyContentType(finalResponse?.headers()["content-type"]);
1973
- if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1974
3730
  const finalUrl = page.url();
1975
3731
  const statusCode = finalResponse !== null && !(challengeEntryResponse !== null && finalResponse === challengeEntryResponse) ? finalResponse.status() : 200;
3732
+ if (kind === "pdf" || kind === void 0 && finalResponse !== null && mayBePdf(finalResponse.headers(), finalUrl)) {
3733
+ const download = await downloadPdfBytes(page, finalResponse, finalResponse?.url?.() ?? finalUrl, deadline);
3734
+ if (download.pdf) return await pdfResult(download.bytes, download.url, statusCode, deadline);
3735
+ if (kind === "pdf") throw new WebError("the response claims to be a PDF but its bytes are not one", WEB_PDF_INVALID_CODE);
3736
+ }
3737
+ if (kind === void 0) throw new WebError(`unsupported content type "${finalResponse?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
1976
3738
  if (kind === "text") return capResult(finalUrl, statusCode, {
1977
3739
  kind: "text",
1978
3740
  content: finalResponse !== null ? await finalResponse.text() : await page.content()
1979
3741
  });
1980
3742
  await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
1981
- await stripScriptingElements(page);
1982
- const html = await page.content();
3743
+ const scrollBudgetMs = Math.min(effectiveScrollBudgetMs(config), deadline.remainingMs() - SCROLL_FINISH_RESERVE_MS);
3744
+ const captured = scrollBudgetMs > 0 ? await deepCaptureWalk(page, scrollBudgetMs) : void 0;
3745
+ let html;
3746
+ if (captured?.html !== void 0) html = captured.html;
3747
+ else {
3748
+ await stripScriptingElements(page);
3749
+ html = await page.content();
3750
+ }
1983
3751
  if (!config.denoise) return capResult(finalUrl, statusCode, {
1984
3752
  kind: "html",
1985
3753
  content: html
@@ -2175,7 +3943,7 @@ async function recoverUnrenderedStatus(error, tracker, url, denoise) {
2175
3943
  } catch {
2176
3944
  content = "";
2177
3945
  }
2178
- if (kind === "text" || content === "") return capResult(finalUrl, status, {
3946
+ if (kind !== "html" || content === "") return capResult(finalUrl, status, {
2179
3947
  kind: "text",
2180
3948
  content
2181
3949
  });
@@ -2224,7 +3992,9 @@ function isDownloadFailure(error, response) {
2224
3992
  * download decision — which works identically on the local and CDP
2225
3993
  * backends because it never touches the remote artifact.
2226
3994
  *
2227
- * Undecodable downloads (a PDF, a ZIP) still fail with
3995
+ * A PDF download (declared `application/pdf`, or an opaque binary whose
3996
+ * bytes sniff as `%PDF-`) is extracted to markdown (`src/pdf.ts`);
3997
+ * undecodable downloads (a ZIP) still fail with
2228
3998
  * `WEB_UNSUPPORTED_CONTENT_TYPE`, classified from what the navigation
2229
3999
  * actually served; a playwright build without a request context surfaces a
2230
4000
  * clear "the browser will not render this" error instead of the raw
@@ -2243,7 +4013,13 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
2243
4013
  const tracked = tracker.last();
2244
4014
  if (!isDownloadFailure(error, tracked)) return void 0;
2245
4015
  if (tracked !== null) {
2246
- if (classifyContentType(tracked.headers()["content-type"]) === void 0) throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
4016
+ const trackedKind = classifyContentType(tracked.headers()["content-type"]);
4017
+ if (trackedKind === "pdf" || trackedKind === void 0 && mayBePdf(tracked.headers(), tracked.url?.() ?? url.toString())) {
4018
+ const download = await downloadPdfBytes(page, null, tracked.url?.() ?? url.toString(), deadline, tracked.headers());
4019
+ if (download.pdf) return await pdfResult(download.bytes, download.url, download.status, deadline);
4020
+ throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download; not a PDF)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
4021
+ }
4022
+ if (trackedKind === void 0) throw new WebError(`unsupported content type "${tracked.headers()["content-type"] ?? "unknown"}" (served as a download)`, "WEB_UNSUPPORTED_CONTENT_TYPE");
2247
4023
  }
2248
4024
  if (page.request === void 0) throw new WebError("the response is a file download (Content-Disposition: attachment) the browser will not render, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR", { cause: error });
2249
4025
  const fetchUrl = tracked?.url?.() ?? url.toString();
@@ -2254,6 +4030,12 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
2254
4030
  const headers = refetched.headers();
2255
4031
  const servedType = (headers["content-type"] ?? "").trim() !== "" ? headers["content-type"] : tracked?.headers()["content-type"];
2256
4032
  const kind = classifyContentType(servedType);
4033
+ if (kind === "pdf") {
4034
+ assertPdfSize(headers);
4035
+ const bytes = refetched.body !== void 0 ? await refetched.body() : /* @__PURE__ */ new Uint8Array();
4036
+ if (!isPdfBytes(bytes)) throw new WebError("the response claims to be a PDF but its bytes are not one", WEB_PDF_INVALID_CODE);
4037
+ return await pdfResult(bytes, refetched.url?.() ?? fetchUrl, refetched.status(), deadline);
4038
+ }
2257
4039
  if (kind === void 0) throw new WebError(`unsupported content type "${servedType ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
2258
4040
  const finalUrl = refetched.url?.() ?? fetchUrl;
2259
4041
  const statusCode = refetched.status();
@@ -2272,6 +4054,72 @@ async function recoverDownloadResponse(error, page, tracker, url, deadline, deno
2272
4054
  content: markdown
2273
4055
  });
2274
4056
  }
4057
+ /** Time kept back from the PDF page walk for layout and result assembly. */
4058
+ const PDF_FINISH_RESERVE_MS = 1500;
4059
+ /** Refuse a PDF whose declared length is over the limit — before downloading it. */
4060
+ function assertPdfSize(headers) {
4061
+ const declared = Number.parseInt(headers?.["content-length"] ?? "", 10);
4062
+ if (Number.isFinite(declared) && declared > 52428800) throw new WebError(`the PDF is ${String(Math.round(declared / 1024 / 1024))}MB, above the ${String(MAX_PDF_BYTES / 1024 / 1024)}MB limit`, WEB_PDF_TOO_LARGE_CODE);
4063
+ }
4064
+ /**
4065
+ * Get a (possibly) PDF resource's bytes. The navigation response's own body
4066
+ * is tried first — a PDF Chrome committed into its viewer still has it —
4067
+ * then the page's API request context refetches the URL (same cookie jar,
4068
+ * plain HTTP, no render-or-download decision; the only way to reach a body
4069
+ * Chromium streamed to its download manager). The `%PDF-` magic decides.
4070
+ *
4071
+ * @param page - the fetch's tab (for its request context).
4072
+ * @param response - the navigation response, when the browser committed one.
4073
+ * @param url - the URL to refetch.
4074
+ * @param deadline - the fetch budget.
4075
+ * @param servedHeaders - the navigation's headers (size pre-check) when no
4076
+ * response object is passed.
4077
+ * @returns the bytes, where they came from, and whether they are a PDF.
4078
+ */
4079
+ async function downloadPdfBytes(page, response, url, deadline, servedHeaders) {
4080
+ assertPdfSize(response?.headers() ?? servedHeaders);
4081
+ if (response?.body !== void 0) try {
4082
+ const bytes = await response.body();
4083
+ if (isPdfBytes(bytes)) return {
4084
+ pdf: true,
4085
+ bytes,
4086
+ url: response.url?.() ?? url,
4087
+ status: response.status()
4088
+ };
4089
+ } catch {}
4090
+ if (page.request === void 0) throw new WebError("the response is a PDF the browser will not hand over, and this playwright build exposes no request context to refetch it", "WEB_PROVIDER_ERROR");
4091
+ const refetched = await page.request.get(url, {
4092
+ timeout: deadline.remainingMs(),
4093
+ signal: deadline.signal
4094
+ });
4095
+ assertPdfSize(refetched.headers());
4096
+ const bytes = refetched.body !== void 0 ? await refetched.body() : /* @__PURE__ */ new Uint8Array();
4097
+ return {
4098
+ pdf: isPdfBytes(bytes),
4099
+ bytes,
4100
+ url: refetched.url?.() ?? url,
4101
+ status: refetched.status()
4102
+ };
4103
+ }
4104
+ /**
4105
+ * Extract PDF bytes to a markdown text result inside the fetch budget.
4106
+ * A limit that stopped the page walk early flags the result truncated.
4107
+ */
4108
+ async function pdfResult(bytes, finalUrl, statusCode, deadline) {
4109
+ const extraction = await extractPdfMarkdown(bytes, {
4110
+ signal: deadline.signal,
4111
+ timeBudgetMs: Math.max(1e3, deadline.remainingMs() - PDF_FINISH_RESERVE_MS),
4112
+ maxChars: MAX_BODY_CHARS
4113
+ });
4114
+ const result = capResult(finalUrl, statusCode, {
4115
+ kind: "text",
4116
+ content: extraction.markdown
4117
+ });
4118
+ return extraction.truncated && !result.truncated ? {
4119
+ ...result,
4120
+ truncated: true
4121
+ } : result;
4122
+ }
2275
4123
  /**
2276
4124
  * Probe the live document for challenge markers. Prefers the in-page probe
2277
4125
  * (`page.evaluate`) — the only thing that can see SPA-style clears — and
@@ -2292,66 +4140,12 @@ async function probeStillOnChallenge(page) {
2292
4140
  }
2293
4141
  }
2294
4142
  /**
2295
- * In-page script that removes non-content scripting subtrees from the live
2296
- * document before `page.content()` reads it as an HTML string.
2297
- *
2298
- * WHAT goes, and why each removal is output-neutral versus the old pipeline:
2299
- *
2300
- * - `script` — three consumers ever saw script text before, and none loses
2301
- * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
2302
- * extraction and scoring (so script text never reached the article or
2303
- * influenced the scorer); DOMPurify's default allowlist drops script; the
2304
- * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
2305
- * removal buys is the reason this exists: a page's runtime JS may rewrite
2306
- * a data script's text with a literal `<\/script>` embedded — measured on
2307
- * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
2308
- * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
2309
- * turning the wire's 712 `\u003c` escapes back into raw `<` — and
2310
- * browsers serialize script text verbatim, so the flattened string ends
2311
- * that script at the FIRST embedded end tag and re-parses the remaining
2312
- * ~270KB of JSON as markup. The phantom elements that survive (unknown
2313
- * tags like `<id>`, anchors full of JSON text) are exactly what
2314
- * Readability's scorer reads as article mass, so the extracted "article"
2315
- * is CSS and i18n strings while the real body — still present in the DOM —
2316
- * loses. No string-level patch can undo this after serialization (the
2317
- * string is genuinely ambiguous); only the browser's own parse holds the
2318
- * document's true structure.
2319
- * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
2320
- * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
2321
- * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
2322
- * scripts before removing them, so deleting them would regress the title
2323
- * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
2324
- * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
2325
- * same object) while making the serialization harmless: the tokenizer
2326
- * only terminates on a literal `<\/script`.
2327
- * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
2328
- * real image out of lazy-load `<noscript><img …>` wrappers before
2329
- * `_removeScripts` deletes the noscript; pre-stripping it would silently
2330
- * drop those images from the markdown.
2331
- * - `template` — removed: its children parse into the inert `.content`
2332
- * fragment (invisible to Readability's traversal), DOMPurify's
2333
- * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
2334
- * - `style` stays — its text cannot terminate a re-parse except at its own
2335
- * end tag, and the pipeline already drops it.
2336
- */
2337
- const STRIP_SCRIPTING_ELEMENTS = `(() => {
2338
- for (const el of Array.from(document.querySelectorAll('script, template'))) {
2339
- if (el.tagName === 'SCRIPT' && (el.getAttribute('type') || '').trim().toLowerCase() === 'application/ld+json') {
2340
- // '<\\/' is still valid JSON, so the re-parsed textContent parses to
2341
- // the same object — while the serialized form can no longer close the
2342
- // script element early.
2343
- el.textContent = el.textContent.replace(/<\\//g, '<\\\\/')
2344
- continue
2345
- }
2346
- el.remove()
2347
- }
2348
- })()`;
2349
- /**
2350
4143
  * Remove non-content scripting subtrees from the live document before
2351
- * serialization. Best-effort, mirroring {@link probeStillOnChallenge}: a
2352
- * backend without `evaluate` (fakes, minimal engines) simply keeps the
2353
- * document as-is — the fetch never fails here, and the pipeline degrades to
2354
- * whatever the serialized string itself says.
4144
+ * serialization (see `stripScriptingElementsIn` in `src/strip.ts` for exactly
4145
+ * what goes and why). Best-effort, mirroring {@link probeStillOnChallenge}: a backend
4146
+ * without `evaluate` (fakes, minimal engines) simply keeps the document
4147
+ * as-is — the fetch never fails here, and the pipeline degrades to whatever
4148
+ * the serialized string itself says.
2355
4149
  */
2356
4150
  async function stripScriptingElements(page) {
2357
4151
  if (typeof page.evaluate !== "function") return;
@@ -2488,4 +4282,4 @@ function apply(ctx, config) {
2488
4282
  ctx.web.registerFetchProvider(provider);
2489
4283
  }
2490
4284
  //#endregion
2491
- export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, classifyChallengeHtml, classifyChallengeResponse, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, htmlToMarkdown, inject, isChallengeCompatibleResponse, name, normalizeCdpEndpoint, resolveConfig, snapshotsOf };
4285
+ export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_PDF_BYTES, MAX_PDF_PAGES, MAX_SCROLL_BUDGET_MS, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE, apply, classifyChallengeHtml, classifyChallengeResponse, deepCaptureWalk, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, extractPdfMarkdown, htmlToMarkdown, inject, isChallengeCompatibleResponse, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, name, normalizeCdpEndpoint, readPdfDocument, resolveConfig, snapshotsOf };