dsh-web-fetch-playwright 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/README.zh-CN.md +2 -2
- package/dsh.plugin.json +1 -1
- package/lib/client.js +24 -24
- package/lib/index.js +43 -0
- package/lib/types/markdown.d.ts +2 -0
- package/package.json +1 -1
- package/src/markdown.ts +55 -0
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (DSH) plug
|
|
|
9
9
|
## Features
|
|
10
10
|
|
|
11
11
|
- **Real browser rendering** — loads the page the way a user sees it, so client-side rendered (SPA) content is captured, not just the raw HTML.
|
|
12
|
-
- **Denoise pipeline** — Mozilla Readability extracts the article, DOMPurify removes layout/noise tags (nav, sidebar, footer, ads, forms), and Turndown with the GFM plugin converts to Markdown with the same style options as the shipped `tool-web` renderer.
|
|
12
|
+
- **Denoise pipeline** — Mozilla Readability extracts the article, DOMPurify removes layout/noise tags (nav, sidebar, footer, ads, forms), and Turndown with the GFM plugin converts to Markdown with the same style options as the shipped `tool-web` renderer. Inline `data:` images (build tools like Docusaurus embed screenshots as base64) are elided to size placeholders such as `` so they cannot flood the body.
|
|
13
13
|
- **Two backends** — launch a local Playwright browser, or drive an already-running browser over its DevTools Protocol (CDP) endpoint.
|
|
14
14
|
- **Browser resolution** — a configured path, a `playwright` CLI on `$PATH`, or the bundled `playwright-core`; CDP needs no local browser at all.
|
|
15
15
|
- **Isolated or profile sessions (CDP)** — every fetch is scoped to exactly one tab. Local launches close their browser per fetch; the CDP backend keeps **one shared connection** to the remote browser and each fetch opens a tab inside it, closed when done. By default that tab lives in the remote browser's **real profile** (its cookies, localStorage, and persistent logins apply — like `playwright-cli open`); unchecking *Share the browser context* switches to a throwaway isolated context per fetch.
|
|
@@ -29,7 +29,7 @@ web_fetch (tool-web)
|
|
|
29
29
|
├─ local: resolve (path → $PATH → bundled playwright-core) → chromium.launch
|
|
30
30
|
├─ cdp: connectOverCDP(endpoint)
|
|
31
31
|
├─ page.goto → settle (networkidle, best-effort) → page.content()
|
|
32
|
-
├─ denoise: jsdom → Readability → DOMPurify → Turndown(GFM)
|
|
32
|
+
├─ denoise: jsdom → elide data-URI images → Readability → DOMPurify → Turndown(GFM)
|
|
33
33
|
└─ Markdown (or raw HTML when denoise is off)
|
|
34
34
|
```
|
|
35
35
|
|
package/README.zh-CN.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
## 特性
|
|
10
10
|
|
|
11
11
|
- **真实浏览器渲染** —— 以用户视角加载页面,SPA 客户端渲染内容也能抓到,而非只有原始 HTML。
|
|
12
|
-
- **降噪管线** —— Mozilla Readability 提取正文,DOMPurify 移除布局/噪音标签(导航、侧边栏、页脚、广告、表单),Turndown + GFM 插件按与内置 `tool-web` 渲染器一致的风格转成 Markdown
|
|
12
|
+
- **降噪管线** —— Mozilla Readability 提取正文,DOMPurify 移除布局/噪音标签(导航、侧边栏、页脚、广告、表单),Turndown + GFM 插件按与内置 `tool-web` 渲染器一致的风格转成 Markdown。内联 `data:` 图片(Docusaurus 等构建工具会把截图以 base64 内嵌进 HTML)会被替换为带大小的占位符,如 ``,避免 base64 字符流刷屏。
|
|
13
13
|
- **两种后端** —— 本地启动 Playwright 浏览器,或通过 DevTools 协议(CDP)驱动一个已在运行的浏览器。
|
|
14
14
|
- **浏览器解析** —— 配置路径 → `$PATH` 上的 `playwright` CLI → 插件自带的 `playwright-core`;CDP 模式完全不需要本地浏览器。
|
|
15
15
|
- **共享或隔离会话(CDP)** —— 每次抓取严格限定为一个标签页。本地后端每次抓取启动并关闭自己的浏览器;CDP 后端对远端浏览器保持**一条共享连接**,每次抓取只在其里开一个标签页、用完即关。默认该标签页位于远端浏览器的**真实 profile**(沿用其 cookie、localStorage 与已登录会话,效果类似 `playwright-cli open`);取消勾选「共享浏览器上下文」则切换为每次抓取全新隔离 context。
|
|
@@ -29,7 +29,7 @@ web_fetch (tool-web)
|
|
|
29
29
|
├─ local: 解析(路径 → $PATH → 内置 playwright-core)→ chromium.launch
|
|
30
30
|
├─ cdp: connectOverCDP(endpoint)
|
|
31
31
|
├─ page.goto → 等待稳定(networkidle,尽力而为)→ page.content()
|
|
32
|
-
├─ 降噪:jsdom → Readability → DOMPurify → Turndown(GFM)
|
|
32
|
+
├─ 降噪:jsdom → 内联 data: 图片改占位符 → Readability → DOMPurify → Turndown(GFM)
|
|
33
33
|
└─ Markdown(关闭降噪时返回原始 HTML)
|
|
34
34
|
```
|
|
35
35
|
|
package/dsh.plugin.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.4",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"entry": {
|
|
6
6
|
"name": "dsh-web-fetch-playwright",
|
package/lib/client.js
CHANGED
|
@@ -388,21 +388,21 @@ window.__ModuleLoader__.load({
|
|
|
388
388
|
document.head.appendChild(tag);
|
|
389
389
|
}
|
|
390
390
|
var PluginCard_module_css_default = {
|
|
391
|
-
"
|
|
392
|
-
"cardOpen": "UM8-Rq_cardOpen",
|
|
393
|
-
"readOnly": "UM8-Rq_readOnly",
|
|
394
|
-
"discard": "UM8-Rq_discard",
|
|
391
|
+
"body": "UM8-Rq_body",
|
|
395
392
|
"headText": "UM8-Rq_headText",
|
|
396
|
-
"pending": "UM8-Rq_pending",
|
|
397
|
-
"description": "UM8-Rq_description",
|
|
398
|
-
"footer": "UM8-Rq_footer",
|
|
399
393
|
"chevron": "UM8-Rq_chevron",
|
|
400
|
-
"
|
|
394
|
+
"readOnly": "UM8-Rq_readOnly",
|
|
401
395
|
"save": "UM8-Rq_save",
|
|
396
|
+
"cardOpen": "UM8-Rq_cardOpen",
|
|
397
|
+
"failed": "UM8-Rq_failed",
|
|
398
|
+
"header": "UM8-Rq_header",
|
|
399
|
+
"description": "UM8-Rq_description",
|
|
400
|
+
"footer": "UM8-Rq_footer",
|
|
401
|
+
"chevronOpen": "UM8-Rq_chevronOpen",
|
|
402
|
+
"pending": "UM8-Rq_pending",
|
|
403
|
+
"discard": "UM8-Rq_discard",
|
|
402
404
|
"name": "UM8-Rq_name",
|
|
403
|
-
"
|
|
404
|
-
"card": "UM8-Rq_card",
|
|
405
|
-
"chevronOpen": "UM8-Rq_chevronOpen"
|
|
405
|
+
"card": "UM8-Rq_card"
|
|
406
406
|
};
|
|
407
407
|
//#endregion
|
|
408
408
|
//#region src/client/PluginCard.tsx
|
|
@@ -519,26 +519,26 @@ window.__ModuleLoader__.load({
|
|
|
519
519
|
document.head.appendChild(tag);
|
|
520
520
|
}
|
|
521
521
|
var fields_module_css_default = {
|
|
522
|
-
"
|
|
522
|
+
"radioGroup": "CFSaVG_radioGroup",
|
|
523
|
+
"label": "CFSaVG_label",
|
|
523
524
|
"field": "CFSaVG_field",
|
|
524
|
-
"
|
|
525
|
+
"radioLabel": "CFSaVG_radioLabel",
|
|
526
|
+
"input": "CFSaVG_input",
|
|
527
|
+
"badge": "CFSaVG_badge",
|
|
528
|
+
"radioText": "CFSaVG_radioText",
|
|
525
529
|
"hint": "CFSaVG_hint",
|
|
526
|
-
"head": "CFSaVG_head",
|
|
527
|
-
"hintIndented": "CFSaVG_hintIndented",
|
|
528
530
|
"fieldEmbedded": "CFSaVG_fieldEmbedded",
|
|
529
531
|
"inputInvalid": "CFSaVG_inputInvalid",
|
|
530
|
-
"
|
|
532
|
+
"badges": "CFSaVG_badges",
|
|
531
533
|
"checkboxRow": "CFSaVG_checkboxRow",
|
|
532
|
-
"radioLabel": "CFSaVG_radioLabel",
|
|
533
|
-
"reset": "CFSaVG_reset",
|
|
534
|
-
"radioText": "CFSaVG_radioText",
|
|
535
534
|
"radioOption": "CFSaVG_radioOption",
|
|
536
|
-
"
|
|
537
|
-
"
|
|
538
|
-
"
|
|
539
|
-
"
|
|
535
|
+
"head": "CFSaVG_head",
|
|
536
|
+
"radioContent": "CFSaVG_radioContent",
|
|
537
|
+
"reset": "CFSaVG_reset",
|
|
538
|
+
"invalid": "CFSaVG_invalid",
|
|
539
|
+
"hintIndented": "CFSaVG_hintIndented",
|
|
540
540
|
"radioPick": "CFSaVG_radioPick",
|
|
541
|
-
"
|
|
541
|
+
"radioHint": "CFSaVG_radioHint"
|
|
542
542
|
};
|
|
543
543
|
//#endregion
|
|
544
544
|
//#region src/client/fields.tsx
|
package/lib/index.js
CHANGED
|
@@ -232,6 +232,8 @@ var CdpConnectionPool = class {
|
|
|
232
232
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
233
233
|
*
|
|
234
234
|
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
235
|
+
* inline `data:` image payloads are elided to size placeholders (build
|
|
236
|
+
* tools inline images as base64, which would otherwise dominate the body);
|
|
235
237
|
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
236
238
|
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
237
239
|
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
|
@@ -273,6 +275,46 @@ const FORBID_ATTR = [
|
|
|
273
275
|
"aria-hidden",
|
|
274
276
|
"role"
|
|
275
277
|
];
|
|
278
|
+
/**
|
|
279
|
+
* Elide inline `data:` image payloads to `data:<mime>;base64,...<size>`.
|
|
280
|
+
*
|
|
281
|
+
* Build tools (Docusaurus/webpack, Hugo) inline images above a size cutoff
|
|
282
|
+
* straight into the HTML as data URIs — measured on an onlyoffice.com docs
|
|
283
|
+
* page, 12 inline PNGs were 21.6% of the HTML and, surviving Readability
|
|
284
|
+
* (they are content), DOMPurify (img+data: is on its DATA_URI_TAGS
|
|
285
|
+
* allowlist), and Turndown's default image rule, ended up **65% of the
|
|
286
|
+
* returned markdown body**. Network-level filtering cannot touch them: a
|
|
287
|
+
* data URI is never fetched, so the provider's subrequest abort misses it.
|
|
288
|
+
* The placeholder keeps alt text, MIME type, and the approximate size, so
|
|
289
|
+
* the model still knows an inline image existed. The marker is pure ASCII
|
|
290
|
+
* because Readability re-resolves image srcs through `new URL()`, which
|
|
291
|
+
* would percent-encode anything else.
|
|
292
|
+
*/
|
|
293
|
+
function elideDataUriImages(document) {
|
|
294
|
+
for (const img of Array.from(document.querySelectorAll("img"))) {
|
|
295
|
+
const src = img.getAttribute("src");
|
|
296
|
+
if (src === null || !src.startsWith("data:")) continue;
|
|
297
|
+
img.setAttribute("src", dataUriPlaceholder(src));
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Shorten one data URI to its header plus a size marker.
|
|
302
|
+
* @param src - the original `data:` URI.
|
|
303
|
+
* @returns e.g. `data:image/png;base64,...8.9KB` (base64 sizes are decoded
|
|
304
|
+
* bytes; non-base64 payloads report their character count).
|
|
305
|
+
*/
|
|
306
|
+
function dataUriPlaceholder(src) {
|
|
307
|
+
const commaIndex = src.indexOf(",");
|
|
308
|
+
const header = commaIndex === -1 ? src : src.slice(0, commaIndex + 1);
|
|
309
|
+
const payload = commaIndex === -1 ? "" : src.slice(commaIndex + 1);
|
|
310
|
+
return `${header}...${humanSize(/;base64$/i.test(header.slice(5, -1)) ? Math.max(0, Math.floor(payload.length / 4) * 3 - (payload.match(/=+$/)?.[0].length ?? 0)) : payload.length)}`;
|
|
311
|
+
}
|
|
312
|
+
/** Render a byte count as `123B` / `8.9KB` / `1.2MB`. */
|
|
313
|
+
function humanSize(bytes) {
|
|
314
|
+
if (bytes < 1024) return `${String(bytes)}B`;
|
|
315
|
+
if (bytes < 1048576) return `${(bytes / 1024).toFixed(1)}KB`;
|
|
316
|
+
return `${(bytes / 1048576).toFixed(1)}MB`;
|
|
317
|
+
}
|
|
276
318
|
/** The shared converter: same style options as `dsh-tool-web`'s renderer. */
|
|
277
319
|
const turndown = new TurndownService({
|
|
278
320
|
headingStyle: "atx",
|
|
@@ -339,6 +381,7 @@ function htmlToMarkdown(html, url) {
|
|
|
339
381
|
});
|
|
340
382
|
const purify = createDOMPurify(dom.window);
|
|
341
383
|
const document = dom.window.document;
|
|
384
|
+
elideDataUriImages(document);
|
|
342
385
|
let source = null;
|
|
343
386
|
let title;
|
|
344
387
|
try {
|
package/lib/types/markdown.d.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
4
|
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
5
|
+
* inline `data:` image payloads are elided to size placeholders (build
|
|
6
|
+
* tools inline images as base64, which would otherwise dominate the body);
|
|
5
7
|
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
6
8
|
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
7
9
|
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.4",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
package/src/markdown.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
3
|
*
|
|
4
4
|
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
5
|
+
* inline `data:` image payloads are elided to size placeholders (build
|
|
6
|
+
* tools inline images as base64, which would otherwise dominate the body);
|
|
5
7
|
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
6
8
|
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
7
9
|
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
|
@@ -30,6 +32,52 @@ const FORBID_TAGS = [
|
|
|
30
32
|
/** Attributes stripped from sanitized output (styling survives as noise). */
|
|
31
33
|
const FORBID_ATTR = ['style', 'class', 'id', 'hidden', 'aria-hidden', 'role']
|
|
32
34
|
|
|
35
|
+
/**
|
|
36
|
+
* Elide inline `data:` image payloads to `data:<mime>;base64,...<size>`.
|
|
37
|
+
*
|
|
38
|
+
* Build tools (Docusaurus/webpack, Hugo) inline images above a size cutoff
|
|
39
|
+
* straight into the HTML as data URIs — measured on an onlyoffice.com docs
|
|
40
|
+
* page, 12 inline PNGs were 21.6% of the HTML and, surviving Readability
|
|
41
|
+
* (they are content), DOMPurify (img+data: is on its DATA_URI_TAGS
|
|
42
|
+
* allowlist), and Turndown's default image rule, ended up **65% of the
|
|
43
|
+
* returned markdown body**. Network-level filtering cannot touch them: a
|
|
44
|
+
* data URI is never fetched, so the provider's subrequest abort misses it.
|
|
45
|
+
* The placeholder keeps alt text, MIME type, and the approximate size, so
|
|
46
|
+
* the model still knows an inline image existed. The marker is pure ASCII
|
|
47
|
+
* because Readability re-resolves image srcs through `new URL()`, which
|
|
48
|
+
* would percent-encode anything else.
|
|
49
|
+
*/
|
|
50
|
+
function elideDataUriImages(document: Document): void {
|
|
51
|
+
for (const img of Array.from(document.querySelectorAll('img'))) {
|
|
52
|
+
const src = img.getAttribute('src')
|
|
53
|
+
if (src === null || !src.startsWith('data:')) continue
|
|
54
|
+
img.setAttribute('src', dataUriPlaceholder(src))
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Shorten one data URI to its header plus a size marker.
|
|
60
|
+
* @param src - the original `data:` URI.
|
|
61
|
+
* @returns e.g. `data:image/png;base64,...8.9KB` (base64 sizes are decoded
|
|
62
|
+
* bytes; non-base64 payloads report their character count).
|
|
63
|
+
*/
|
|
64
|
+
function dataUriPlaceholder(src: string): string {
|
|
65
|
+
const commaIndex = src.indexOf(',')
|
|
66
|
+
const header = commaIndex === -1 ? src : src.slice(0, commaIndex + 1)
|
|
67
|
+
const payload = commaIndex === -1 ? '' : src.slice(commaIndex + 1)
|
|
68
|
+
const size = /;base64$/i.test(header.slice(5, -1))
|
|
69
|
+
? Math.max(0, Math.floor(payload.length / 4) * 3 - (payload.match(/=+$/)?.[0].length ?? 0))
|
|
70
|
+
: payload.length
|
|
71
|
+
return `${header}...${humanSize(size)}`
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Render a byte count as `123B` / `8.9KB` / `1.2MB`. */
|
|
75
|
+
function humanSize(bytes: number): string {
|
|
76
|
+
if (bytes < 1024) return `${String(bytes)}B`
|
|
77
|
+
if (bytes < 1_048_576) return `${(bytes / 1024).toFixed(1)}KB`
|
|
78
|
+
return `${(bytes / 1_048_576).toFixed(1)}MB`
|
|
79
|
+
}
|
|
80
|
+
|
|
33
81
|
/** The shared converter: same style options as `dsh-tool-web`'s renderer. */
|
|
34
82
|
const turndown = new TurndownService({
|
|
35
83
|
headingStyle: 'atx',
|
|
@@ -115,6 +163,13 @@ export function htmlToMarkdown(html: string, url: string): DenoiseResult {
|
|
|
115
163
|
const purify = createDOMPurify(dom.window)
|
|
116
164
|
const document = dom.window.document
|
|
117
165
|
|
|
166
|
+
// Before anything downstream reads the DOM: shrink inline data-URI image
|
|
167
|
+
// payloads to size placeholders. Mutating the live document here covers
|
|
168
|
+
// both paths — the Readability clone below, and the document.body fallback
|
|
169
|
+
// (innerHTML reflects the rewritten src) — while Turndown's default image
|
|
170
|
+
// rule keeps handling alt/title escaping on the already-short src.
|
|
171
|
+
elideDataUriImages(document)
|
|
172
|
+
|
|
118
173
|
let source: string | null = null
|
|
119
174
|
let title: string | undefined
|
|
120
175
|
try {
|