dsh-web-fetch-playwright 0.2.16 → 0.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/strip.ts ADDED
@@ -0,0 +1,72 @@
1
+ /**
2
+ * Serialization safety for documents about to be flattened into an HTML
3
+ * string and re-parsed by jsdom: the one in-page cleanup both serialization
4
+ * paths share — the provider's live-document read and the deep-capture
5
+ * walk's merged clone.
6
+ *
7
+ * @module dsh-web-fetch-playwright/strip
8
+ */
9
+
10
+ /**
11
+ * In-page function that removes non-content scripting subtrees from a
12
+ * document before it is flattened into an HTML string — the live document
13
+ * before `page.content()`, or the deep-capture walk's merged clone before it
14
+ * serializes it itself (`src/autoscroll.ts`).
15
+ *
16
+ * It is evaluated BY SOURCE (`fn.toString()`), so it must stay
17
+ * self-contained: no imports, no module-level references.
18
+ *
19
+ * WHAT goes, and why each removal is output-neutral versus the old pipeline:
20
+ *
21
+ * - `script` — three consumers ever saw script text before, and none loses
22
+ * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
23
+ * extraction and scoring (so script text never reached the article or
24
+ * influenced the scorer); DOMPurify's default allowlist drops script; the
25
+ * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
26
+ * removal buys is the reason this exists: a page's runtime JS may rewrite
27
+ * a data script's text with a literal `</script>` embedded — measured on
28
+ * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
29
+ * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
30
+ * turning the wire's 712 `\u003c` escapes back into raw `<` — and
31
+ * browsers serialize script text verbatim, so the flattened string ends
32
+ * that script at the FIRST embedded end tag and re-parses the remaining
33
+ * ~270KB of JSON as markup. The phantom elements that survive (unknown
34
+ * tags like `<id>`, anchors full of JSON text) are exactly what
35
+ * Readability's scorer reads as article mass, so the extracted "article"
36
+ * is CSS and i18n strings while the real body — still present in the DOM —
37
+ * loses. No string-level patch can undo this after serialization (the
38
+ * string is genuinely ambiguous); only the browser's own parse holds the
39
+ * document's true structure.
40
+ * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
41
+ * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
42
+ * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
43
+ * scripts before removing them, so deleting them would regress the title
44
+ * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
45
+ * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
46
+ * same object) while making the serialization harmless: the tokenizer
47
+ * only terminates on a literal `</script`.
48
+ * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
49
+ * real image out of lazy-load `<noscript><img …>` wrappers before
50
+ * `_removeScripts` deletes the noscript; pre-stripping it would silently
51
+ * drop those images from the markdown.
52
+ * - `template` — removed: its children parse into the inert `.content`
53
+ * fragment (invisible to Readability's traversal), DOMPurify's
54
+ * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
55
+ * - `style` stays — its text cannot terminate a re-parse except at its own
56
+ * end tag, and the pipeline already drops it.
57
+ */
58
+ export function stripScriptingElementsIn(doc: Document): void {
59
+ for (const el of Array.from(doc.querySelectorAll('script, template'))) {
60
+ if (el.tagName === 'SCRIPT' && (el.getAttribute('type') ?? '').trim().toLowerCase() === 'application/ld+json') {
61
+ // '<\/' is still valid JSON, so the re-parsed textContent parses to the
62
+ // same object — while the serialized form can no longer close the
63
+ // script element early.
64
+ el.textContent = (el.textContent ?? '').replace(/<\//g, '<\\/')
65
+ continue
66
+ }
67
+ el.remove()
68
+ }
69
+ }
70
+
71
+ /** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
72
+ export const STRIP_SCRIPTING_ELEMENTS = `(${stripScriptingElementsIn.toString()})(document)`