dsh-web-fetch-playwright 0.2.16 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/lib/client.js +42 -17
- package/lib/index.js +866 -71
- package/lib/types/autoscroll.d.ts +277 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -3
- package/lib/types/index.d.ts +3 -1
- package/lib/types/markdown.d.ts +8 -5
- package/lib/types/provider.d.ts +25 -0
- package/lib/types/strip.d.ts +59 -0
- package/package.json +1 -1
- package/src/autoscroll.ts +653 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -3
- package/src/index.ts +5 -0
- package/src/markdown.ts +252 -6
- package/src/provider.ts +57 -68
- package/src/strip.ts +72 -0
package/src/strip.ts
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serialization safety for documents about to be flattened into an HTML
|
|
3
|
+
* string and re-parsed by jsdom: the one in-page cleanup both serialization
|
|
4
|
+
* paths share — the provider's live-document read and the deep-capture
|
|
5
|
+
* walk's merged clone.
|
|
6
|
+
*
|
|
7
|
+
* @module dsh-web-fetch-playwright/strip
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* In-page function that removes non-content scripting subtrees from a
|
|
12
|
+
* document before it is flattened into an HTML string — the live document
|
|
13
|
+
* before `page.content()`, or the deep-capture walk's merged clone before it
|
|
14
|
+
* serializes it itself (`src/autoscroll.ts`).
|
|
15
|
+
*
|
|
16
|
+
* It is evaluated BY SOURCE (`fn.toString()`), so it must stay
|
|
17
|
+
* self-contained: no imports, no module-level references.
|
|
18
|
+
*
|
|
19
|
+
* WHAT goes, and why each removal is output-neutral versus the old pipeline:
|
|
20
|
+
*
|
|
21
|
+
* - `script` — three consumers ever saw script text before, and none loses
|
|
22
|
+
* it here: Readability's `parse()` deletes script/noscript nodes BEFORE
|
|
23
|
+
* extraction and scoring (so script text never reached the article or
|
|
24
|
+
* influenced the scorer); DOMPurify's default allowlist drops script; the
|
|
25
|
+
* tool layer's renderer removes SCRIPT wholesale on the raw path. What the
|
|
26
|
+
* removal buys is the reason this exists: a page's runtime JS may rewrite
|
|
27
|
+
* a data script's text with a literal `</script>` embedded — measured on
|
|
28
|
+
* techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
|
|
29
|
+
* Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
|
|
30
|
+
* turning the wire's 712 `\u003c` escapes back into raw `<` — and
|
|
31
|
+
* browsers serialize script text verbatim, so the flattened string ends
|
|
32
|
+
* that script at the FIRST embedded end tag and re-parses the remaining
|
|
33
|
+
* ~270KB of JSON as markup. The phantom elements that survive (unknown
|
|
34
|
+
* tags like `<id>`, anchors full of JSON text) are exactly what
|
|
35
|
+
* Readability's scorer reads as article mass, so the extracted "article"
|
|
36
|
+
* is CSS and i18n strings while the real body — still present in the DOM —
|
|
37
|
+
* loses. No string-level patch can undo this after serialization (the
|
|
38
|
+
* string is genuinely ambiguous); only the browser's own parse holds the
|
|
39
|
+
* document's true structure.
|
|
40
|
+
* - `script[type="application/ld+json"]` — KEPT, but defused: Readability
|
|
41
|
+
* extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
|
|
42
|
+
* "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
|
|
43
|
+
* scripts before removing them, so deleting them would regress the title
|
|
44
|
+
* on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
|
|
45
|
+
* valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
|
|
46
|
+
* same object) while making the serialization harmless: the tokenizer
|
|
47
|
+
* only terminates on a literal `</script`.
|
|
48
|
+
* - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
|
|
49
|
+
* real image out of lazy-load `<noscript><img …>` wrappers before
|
|
50
|
+
* `_removeScripts` deletes the noscript; pre-stripping it would silently
|
|
51
|
+
* drop those images from the markdown.
|
|
52
|
+
* - `template` — removed: its children parse into the inert `.content`
|
|
53
|
+
* fragment (invisible to Readability's traversal), DOMPurify's
|
|
54
|
+
* FORBID_TAGS drops it, and the tool layer's renderer removes it too.
|
|
55
|
+
* - `style` stays — its text cannot terminate a re-parse except at its own
|
|
56
|
+
* end tag, and the pipeline already drops it.
|
|
57
|
+
*/
|
|
58
|
+
export function stripScriptingElementsIn(doc: Document): void {
|
|
59
|
+
for (const el of Array.from(doc.querySelectorAll('script, template'))) {
|
|
60
|
+
if (el.tagName === 'SCRIPT' && (el.getAttribute('type') ?? '').trim().toLowerCase() === 'application/ld+json') {
|
|
61
|
+
// '<\/' is still valid JSON, so the re-parsed textContent parses to the
|
|
62
|
+
// same object — while the serialized form can no longer close the
|
|
63
|
+
// script element early.
|
|
64
|
+
el.textContent = (el.textContent ?? '').replace(/<\//g, '<\\/')
|
|
65
|
+
continue
|
|
66
|
+
}
|
|
67
|
+
el.remove()
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
|
|
72
|
+
export const STRIP_SCRIPTING_ELEMENTS = `(${stripScriptingElementsIn.toString()})(document)`
|