dsh-web-fetch-playwright 0.2.16 → 0.2.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,59 @@
1
+ /**
2
+ * Serialization safety for documents about to be flattened into an HTML
3
+ * string and re-parsed by jsdom: the one in-page cleanup both serialization
4
+ * paths share — the provider's live-document read and the deep-capture
5
+ * walk's merged clone.
6
+ *
7
+ * @module dsh-web-fetch-playwright/strip
8
+ */
9
+ /**
10
+ * In-page function that removes non-content scripting subtrees from a
11
+ * document before it is flattened into an HTML string — the live document
12
+ * before `page.content()`, or the deep-capture walk's merged clone before it
13
+ * serializes it itself (`src/autoscroll.ts`).
14
+ *
15
+ * It is evaluated BY SOURCE (`fn.toString()`), so it must stay
16
+ * self-contained: no imports, no module-level references.
17
+ *
18
+ * WHAT goes, and why each removal is output-neutral versus the old pipeline:
19
+ *
20
+ * - `script` — three consumers ever saw script text before, and none loses
21
+ * it here: Readability's `parse()` deletes script/noscript nodes BEFORE
22
+ * extraction and scoring (so script text never reached the article or
23
+ * influenced the scorer); DOMPurify's default allowlist drops script; the
24
+ * tool layer's renderer removes SCRIPT wholesale on the raw path. What the
25
+ * removal buys is the reason this exists: a page's runtime JS may rewrite
26
+ * a data script's text with a literal `</script>` embedded — measured on
27
+ * techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
28
+ * Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
29
+ * turning the wire's 712 `\u003c` escapes back into raw `<` — and
30
+ * browsers serialize script text verbatim, so the flattened string ends
31
+ * that script at the FIRST embedded end tag and re-parses the remaining
32
+ * ~270KB of JSON as markup. The phantom elements that survive (unknown
33
+ * tags like `<id>`, anchors full of JSON text) are exactly what
34
+ * Readability's scorer reads as article mass, so the extracted "article"
35
+ * is CSS and i18n strings while the real body — still present in the DOM —
36
+ * loses. No string-level patch can undo this after serialization (the
37
+ * string is genuinely ambiguous); only the browser's own parse holds the
38
+ * document's true structure.
39
+ * - `script[type="application/ld+json"]` — KEPT, but defused: Readability
40
+ * extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
41
+ * "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
42
+ * scripts before removing them, so deleting them would regress the title
43
+ * on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
44
+ * valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
45
+ * same object) while making the serialization harmless: the tokenizer
46
+ * only terminates on a literal `</script`.
47
+ * - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
48
+ * real image out of lazy-load `<noscript><img …>` wrappers before
49
+ * `_removeScripts` deletes the noscript; pre-stripping it would silently
50
+ * drop those images from the markdown.
51
+ * - `template` — removed: its children parse into the inert `.content`
52
+ * fragment (invisible to Readability's traversal), DOMPurify's
53
+ * FORBID_TAGS drops it, and the tool layer's renderer removes it too.
54
+ * - `style` stays — its text cannot terminate a re-parse except at its own
55
+ * end tag, and the pipeline already drops it.
56
+ */
57
+ export declare function stripScriptingElementsIn(doc: Document): void;
58
+ /** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
59
+ export declare const STRIP_SCRIPTING_ELEMENTS: string;
@@ -12,6 +12,11 @@ export interface PlaywrightResponse {
12
12
  status(): number;
13
13
  headers(): Record<string, string>;
14
14
  text(): Promise<string>;
15
+ /**
16
+ * The raw body bytes — the PDF path reads binary bodies through it (a
17
+ * Node `Buffer` is a `Uint8Array`). Absent on minimal fakes.
18
+ */
19
+ body?(): Promise<Uint8Array>;
15
20
  /** This response's URL; absent on minimal fakes. */
16
21
  url?(): string;
17
22
  /**
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.16",
3
+ "version": "0.2.19",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
@@ -93,7 +93,8 @@
93
93
  "dompurify": "^3.2.4",
94
94
  "jsdom": "^26.0.0",
95
95
  "playwright-core": "^1.65.0",
96
- "turndown": "^7.2.0"
96
+ "turndown": "^7.2.0",
97
+ "unpdf": "^2.1.0"
97
98
  },
98
99
  "peerDependencies": {
99
100
  "@deepseek-ai/cordis": ">=4.0.1-rc.1 <5",