dsh-web-fetch-playwright 0.2.3 → 0.2.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -9,7 +9,7 @@ A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (DSH) plug
9
9
  ## Features
10
10
 
11
11
  - **Real browser rendering** — loads the page the way a user sees it, so client-side rendered (SPA) content is captured, not just the raw HTML.
12
- - **Denoise pipeline** — Mozilla Readability extracts the article, DOMPurify removes layout/noise tags (nav, sidebar, footer, ads, forms), and Turndown with the GFM plugin converts to Markdown with the same style options as the shipped `tool-web` renderer.
12
+ - **Denoise pipeline** — Mozilla Readability extracts the article, DOMPurify removes layout/noise tags (nav, sidebar, footer, ads, forms), and Turndown with the GFM plugin converts to Markdown with the same style options as the shipped `tool-web` renderer. Inline `data:` images (build tools like Docusaurus embed screenshots as base64) are elided to size placeholders such as `![alt](data:image/png;base64,...8.9KB)` so they cannot flood the body.
13
13
  - **Two backends** — launch a local Playwright browser, or drive an already-running browser over its DevTools Protocol (CDP) endpoint.
14
14
  - **Browser resolution** — a configured path, a `playwright` CLI on `$PATH`, or the bundled `playwright-core`; CDP needs no local browser at all.
15
15
  - **Isolated or profile sessions (CDP)** — every fetch is scoped to exactly one tab. Local launches close their browser per fetch; the CDP backend keeps **one shared connection** to the remote browser and each fetch opens a tab inside it, closed when done. By default that tab lives in the remote browser's **real profile** (its cookies, localStorage, and persistent logins apply — like `playwright-cli open`); unchecking *Share the browser context* switches to a throwaway isolated context per fetch.
@@ -29,7 +29,7 @@ web_fetch (tool-web)
29
29
  ├─ local: resolve (path → $PATH → bundled playwright-core) → chromium.launch
30
30
  ├─ cdp: connectOverCDP(endpoint)
31
31
  ├─ page.goto → settle (networkidle, best-effort) → page.content()
32
- ├─ denoise: jsdom → Readability → DOMPurify → Turndown(GFM)
32
+ ├─ denoise: jsdom → elide data-URI images → Readability → DOMPurify → Turndown(GFM)
33
33
  └─ Markdown (or raw HTML when denoise is off)
34
34
  ```
35
35
 
package/README.zh-CN.md CHANGED
@@ -9,7 +9,7 @@
9
9
  ## 特性
10
10
 
11
11
  - **真实浏览器渲染** —— 以用户视角加载页面,SPA 客户端渲染内容也能抓到,而非只有原始 HTML。
12
- - **降噪管线** —— Mozilla Readability 提取正文,DOMPurify 移除布局/噪音标签(导航、侧边栏、页脚、广告、表单),Turndown + GFM 插件按与内置 `tool-web` 渲染器一致的风格转成 Markdown。
12
+ - **降噪管线** —— Mozilla Readability 提取正文,DOMPurify 移除布局/噪音标签(导航、侧边栏、页脚、广告、表单),Turndown + GFM 插件按与内置 `tool-web` 渲染器一致的风格转成 Markdown。内联 `data:` 图片(Docusaurus 等构建工具会把截图以 base64 内嵌进 HTML)会被替换为带大小的占位符,如 `![alt](data:image/png;base64,...8.9KB)`,避免 base64 字符流刷屏。
13
13
  - **两种后端** —— 本地启动 Playwright 浏览器,或通过 DevTools 协议(CDP)驱动一个已在运行的浏览器。
14
14
  - **浏览器解析** —— 配置路径 → `$PATH` 上的 `playwright` CLI → 插件自带的 `playwright-core`;CDP 模式完全不需要本地浏览器。
15
15
  - **共享或隔离会话(CDP)** —— 每次抓取严格限定为一个标签页。本地后端每次抓取启动并关闭自己的浏览器;CDP 后端对远端浏览器保持**一条共享连接**,每次抓取只在其里开一个标签页、用完即关。默认该标签页位于远端浏览器的**真实 profile**(沿用其 cookie、localStorage 与已登录会话,效果类似 `playwright-cli open`);取消勾选「共享浏览器上下文」则切换为每次抓取全新隔离 context。
@@ -29,7 +29,7 @@ web_fetch (tool-web)
29
29
  ├─ local: 解析(路径 → $PATH → 内置 playwright-core)→ chromium.launch
30
30
  ├─ cdp: connectOverCDP(endpoint)
31
31
  ├─ page.goto → 等待稳定(networkidle,尽力而为)→ page.content()
32
- ├─ 降噪:jsdom → Readability → DOMPurify → Turndown(GFM)
32
+ ├─ 降噪:jsdom → 内联 data: 图片改占位符 → Readability → DOMPurify → Turndown(GFM)
33
33
  └─ Markdown(关闭降噪时返回原始 HTML)
34
34
  ```
35
35
 
package/dsh.plugin.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.2",
3
+ "version": "0.2.4",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "entry": {
6
6
  "name": "dsh-web-fetch-playwright",
package/lib/client.js CHANGED
@@ -388,21 +388,21 @@ window.__ModuleLoader__.load({
388
388
  document.head.appendChild(tag);
389
389
  }
390
390
  var PluginCard_module_css_default = {
391
- "header": "UM8-Rq_header",
392
- "cardOpen": "UM8-Rq_cardOpen",
393
- "readOnly": "UM8-Rq_readOnly",
394
- "discard": "UM8-Rq_discard",
391
+ "body": "UM8-Rq_body",
395
392
  "headText": "UM8-Rq_headText",
396
- "pending": "UM8-Rq_pending",
397
- "description": "UM8-Rq_description",
398
- "footer": "UM8-Rq_footer",
399
393
  "chevron": "UM8-Rq_chevron",
400
- "failed": "UM8-Rq_failed",
394
+ "readOnly": "UM8-Rq_readOnly",
401
395
  "save": "UM8-Rq_save",
396
+ "cardOpen": "UM8-Rq_cardOpen",
397
+ "failed": "UM8-Rq_failed",
398
+ "header": "UM8-Rq_header",
399
+ "description": "UM8-Rq_description",
400
+ "footer": "UM8-Rq_footer",
401
+ "chevronOpen": "UM8-Rq_chevronOpen",
402
+ "pending": "UM8-Rq_pending",
403
+ "discard": "UM8-Rq_discard",
402
404
  "name": "UM8-Rq_name",
403
- "body": "UM8-Rq_body",
404
- "card": "UM8-Rq_card",
405
- "chevronOpen": "UM8-Rq_chevronOpen"
405
+ "card": "UM8-Rq_card"
406
406
  };
407
407
  //#endregion
408
408
  //#region src/client/PluginCard.tsx
@@ -519,26 +519,26 @@ window.__ModuleLoader__.load({
519
519
  document.head.appendChild(tag);
520
520
  }
521
521
  var fields_module_css_default = {
522
- "radioContent": "CFSaVG_radioContent",
522
+ "radioGroup": "CFSaVG_radioGroup",
523
+ "label": "CFSaVG_label",
523
524
  "field": "CFSaVG_field",
524
- "invalid": "CFSaVG_invalid",
525
+ "radioLabel": "CFSaVG_radioLabel",
526
+ "input": "CFSaVG_input",
527
+ "badge": "CFSaVG_badge",
528
+ "radioText": "CFSaVG_radioText",
525
529
  "hint": "CFSaVG_hint",
526
- "head": "CFSaVG_head",
527
- "hintIndented": "CFSaVG_hintIndented",
528
530
  "fieldEmbedded": "CFSaVG_fieldEmbedded",
529
531
  "inputInvalid": "CFSaVG_inputInvalid",
530
- "radioHint": "CFSaVG_radioHint",
532
+ "badges": "CFSaVG_badges",
531
533
  "checkboxRow": "CFSaVG_checkboxRow",
532
- "radioLabel": "CFSaVG_radioLabel",
533
- "reset": "CFSaVG_reset",
534
- "radioText": "CFSaVG_radioText",
535
534
  "radioOption": "CFSaVG_radioOption",
536
- "badge": "CFSaVG_badge",
537
- "label": "CFSaVG_label",
538
- "badges": "CFSaVG_badges",
539
- "radioGroup": "CFSaVG_radioGroup",
535
+ "head": "CFSaVG_head",
536
+ "radioContent": "CFSaVG_radioContent",
537
+ "reset": "CFSaVG_reset",
538
+ "invalid": "CFSaVG_invalid",
539
+ "hintIndented": "CFSaVG_hintIndented",
540
540
  "radioPick": "CFSaVG_radioPick",
541
- "input": "CFSaVG_input"
541
+ "radioHint": "CFSaVG_radioHint"
542
542
  };
543
543
  //#endregion
544
544
  //#region src/client/fields.tsx
package/lib/index.js CHANGED
@@ -232,6 +232,8 @@ var CdpConnectionPool = class {
232
232
  * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
233
233
  *
234
234
  * The classic stack, in order: jsdom parses the page Playwright rendered;
235
+ * inline `data:` image payloads are elided to size placeholders (build
236
+ * tools inline images as base64, which would otherwise dominate the body);
235
237
  * Mozilla Readability extracts the article (dropping nav bars, sidebars,
236
238
  * footers, and ad chrome by scoring link density and text mass); DOMPurify
237
239
  * sanitizes whatever HTML remains and forbids the layout tags noise lives in;
@@ -273,6 +275,46 @@ const FORBID_ATTR = [
273
275
  "aria-hidden",
274
276
  "role"
275
277
  ];
278
+ /**
279
+ * Elide inline `data:` image payloads to `data:<mime>;base64,...<size>`.
280
+ *
281
+ * Build tools (Docusaurus/webpack, Hugo) inline images above a size cutoff
282
+ * straight into the HTML as data URIs — measured on an onlyoffice.com docs
283
+ * page, 12 inline PNGs were 21.6% of the HTML and, surviving Readability
284
+ * (they are content), DOMPurify (img+data: is on its DATA_URI_TAGS
285
+ * allowlist), and Turndown's default image rule, ended up **65% of the
286
+ * returned markdown body**. Network-level filtering cannot touch them: a
287
+ * data URI is never fetched, so the provider's subrequest abort misses it.
288
+ * The placeholder keeps alt text, MIME type, and the approximate size, so
289
+ * the model still knows an inline image existed. The marker is pure ASCII
290
+ * because Readability re-resolves image srcs through `new URL()`, which
291
+ * would percent-encode anything else.
292
+ */
293
+ function elideDataUriImages(document) {
294
+ for (const img of Array.from(document.querySelectorAll("img"))) {
295
+ const src = img.getAttribute("src");
296
+ if (src === null || !src.startsWith("data:")) continue;
297
+ img.setAttribute("src", dataUriPlaceholder(src));
298
+ }
299
+ }
300
+ /**
301
+ * Shorten one data URI to its header plus a size marker.
302
+ * @param src - the original `data:` URI.
303
+ * @returns e.g. `data:image/png;base64,...8.9KB` (base64 sizes are decoded
304
+ * bytes; non-base64 payloads report their character count).
305
+ */
306
+ function dataUriPlaceholder(src) {
307
+ const commaIndex = src.indexOf(",");
308
+ const header = commaIndex === -1 ? src : src.slice(0, commaIndex + 1);
309
+ const payload = commaIndex === -1 ? "" : src.slice(commaIndex + 1);
310
+ return `${header}...${humanSize(/;base64$/i.test(header.slice(5, -1)) ? Math.max(0, Math.floor(payload.length / 4) * 3 - (payload.match(/=+$/)?.[0].length ?? 0)) : payload.length)}`;
311
+ }
312
+ /** Render a byte count as `123B` / `8.9KB` / `1.2MB`. */
313
+ function humanSize(bytes) {
314
+ if (bytes < 1024) return `${String(bytes)}B`;
315
+ if (bytes < 1048576) return `${(bytes / 1024).toFixed(1)}KB`;
316
+ return `${(bytes / 1048576).toFixed(1)}MB`;
317
+ }
276
318
  /** The shared converter: same style options as `dsh-tool-web`'s renderer. */
277
319
  const turndown = new TurndownService({
278
320
  headingStyle: "atx",
@@ -339,6 +381,7 @@ function htmlToMarkdown(html, url) {
339
381
  });
340
382
  const purify = createDOMPurify(dom.window);
341
383
  const document = dom.window.document;
384
+ elideDataUriImages(document);
342
385
  let source = null;
343
386
  let title;
344
387
  try {
@@ -2,6 +2,8 @@
2
2
  * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
3
3
  *
4
4
  * The classic stack, in order: jsdom parses the page Playwright rendered;
5
+ * inline `data:` image payloads are elided to size placeholders (build
6
+ * tools inline images as base64, which would otherwise dominate the body);
5
7
  * Mozilla Readability extracts the article (dropping nav bars, sidebars,
6
8
  * footers, and ad chrome by scoring link density and text mass); DOMPurify
7
9
  * sanitizes whatever HTML remains and forbids the layout tags noise lives in;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.3",
3
+ "version": "0.2.4",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",
package/src/markdown.ts CHANGED
@@ -2,6 +2,8 @@
2
2
  * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
3
3
  *
4
4
  * The classic stack, in order: jsdom parses the page Playwright rendered;
5
+ * inline `data:` image payloads are elided to size placeholders (build
6
+ * tools inline images as base64, which would otherwise dominate the body);
5
7
  * Mozilla Readability extracts the article (dropping nav bars, sidebars,
6
8
  * footers, and ad chrome by scoring link density and text mass); DOMPurify
7
9
  * sanitizes whatever HTML remains and forbids the layout tags noise lives in;
@@ -30,6 +32,52 @@ const FORBID_TAGS = [
30
32
  /** Attributes stripped from sanitized output (styling survives as noise). */
31
33
  const FORBID_ATTR = ['style', 'class', 'id', 'hidden', 'aria-hidden', 'role']
32
34
 
35
+ /**
36
+ * Elide inline `data:` image payloads to `data:<mime>;base64,...<size>`.
37
+ *
38
+ * Build tools (Docusaurus/webpack, Hugo) inline images above a size cutoff
39
+ * straight into the HTML as data URIs — measured on an onlyoffice.com docs
40
+ * page, 12 inline PNGs were 21.6% of the HTML and, surviving Readability
41
+ * (they are content), DOMPurify (img+data: is on its DATA_URI_TAGS
42
+ * allowlist), and Turndown's default image rule, ended up **65% of the
43
+ * returned markdown body**. Network-level filtering cannot touch them: a
44
+ * data URI is never fetched, so the provider's subrequest abort misses it.
45
+ * The placeholder keeps alt text, MIME type, and the approximate size, so
46
+ * the model still knows an inline image existed. The marker is pure ASCII
47
+ * because Readability re-resolves image srcs through `new URL()`, which
48
+ * would percent-encode anything else.
49
+ */
50
+ function elideDataUriImages(document: Document): void {
51
+ for (const img of Array.from(document.querySelectorAll('img'))) {
52
+ const src = img.getAttribute('src')
53
+ if (src === null || !src.startsWith('data:')) continue
54
+ img.setAttribute('src', dataUriPlaceholder(src))
55
+ }
56
+ }
57
+
58
+ /**
59
+ * Shorten one data URI to its header plus a size marker.
60
+ * @param src - the original `data:` URI.
61
+ * @returns e.g. `data:image/png;base64,...8.9KB` (base64 sizes are decoded
62
+ * bytes; non-base64 payloads report their character count).
63
+ */
64
+ function dataUriPlaceholder(src: string): string {
65
+ const commaIndex = src.indexOf(',')
66
+ const header = commaIndex === -1 ? src : src.slice(0, commaIndex + 1)
67
+ const payload = commaIndex === -1 ? '' : src.slice(commaIndex + 1)
68
+ const size = /;base64$/i.test(header.slice(5, -1))
69
+ ? Math.max(0, Math.floor(payload.length / 4) * 3 - (payload.match(/=+$/)?.[0].length ?? 0))
70
+ : payload.length
71
+ return `${header}...${humanSize(size)}`
72
+ }
73
+
74
+ /** Render a byte count as `123B` / `8.9KB` / `1.2MB`. */
75
+ function humanSize(bytes: number): string {
76
+ if (bytes < 1024) return `${String(bytes)}B`
77
+ if (bytes < 1_048_576) return `${(bytes / 1024).toFixed(1)}KB`
78
+ return `${(bytes / 1_048_576).toFixed(1)}MB`
79
+ }
80
+
33
81
  /** The shared converter: same style options as `dsh-tool-web`'s renderer. */
34
82
  const turndown = new TurndownService({
35
83
  headingStyle: 'atx',
@@ -115,6 +163,13 @@ export function htmlToMarkdown(html: string, url: string): DenoiseResult {
115
163
  const purify = createDOMPurify(dom.window)
116
164
  const document = dom.window.document
117
165
 
166
+ // Before anything downstream reads the DOM: shrink inline data-URI image
167
+ // payloads to size placeholders. Mutating the live document here covers
168
+ // both paths — the Readability clone below, and the document.body fallback
169
+ // (innerHTML reflects the rewritten src) — while Turndown's default image
170
+ // rule keeps handling alt/title escaping on the already-short src.
171
+ elideDataUriImages(document)
172
+
118
173
  let source: string | null = null
119
174
  let title: string | undefined
120
175
  try {