dsh-web-fetch-playwright 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +108 -0
- package/README.zh-CN.md +108 -0
- package/cordis.patch.yml +31 -0
- package/dsh.plugin.json +16 -0
- package/lib/client.js +854 -0
- package/lib/index.js +669 -0
- package/lib/types/client/PluginCard.d.ts +43 -0
- package/lib/types/client/card.d.ts +18 -0
- package/lib/types/client/controller.d.ts +58 -0
- package/lib/types/client/fields.d.ts +69 -0
- package/lib/types/client/form.d.ts +159 -0
- package/lib/types/client/index.d.ts +17 -0
- package/lib/types/client/locales.d.ts +18 -0
- package/lib/types/config.d.ts +42 -0
- package/lib/types/index.d.ts +26 -0
- package/lib/types/markdown.d.ts +37 -0
- package/lib/types/playwright-resolve.d.ts +57 -0
- package/lib/types/provider.d.ts +71 -0
- package/lib/types/types.d.ts +58 -0
- package/package.json +130 -0
- package/src/client/PluginCard.module.css +155 -0
- package/src/client/PluginCard.tsx +110 -0
- package/src/client/card.tsx +117 -0
- package/src/client/controller.ts +89 -0
- package/src/client/fields.module.css +228 -0
- package/src/client/fields.tsx +189 -0
- package/src/client/form.ts +337 -0
- package/src/client/index.ts +45 -0
- package/src/client/locales.ts +81 -0
- package/src/client/styles.d.ts +4 -0
- package/src/config.ts +67 -0
- package/src/gfm.d.ts +9 -0
- package/src/index.ts +52 -0
- package/src/markdown.ts +161 -0
- package/src/playwright-resolve.ts +210 -0
- package/src/provider.ts +329 -0
- package/src/types.ts +51 -0
package/src/markdown.ts
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
3
|
+
*
|
|
4
|
+
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
5
|
+
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
6
|
+
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
7
|
+
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
|
8
|
+
* Turndown with the GFM plugin converts to markdown using the same style
|
|
9
|
+
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
10
|
+
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
11
|
+
*
|
|
12
|
+
* Pure and synchronous — unit-tested against fixture pages.
|
|
13
|
+
*
|
|
14
|
+
* @module dsh-web-fetch-playwright/markdown
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { JSDOM, VirtualConsole } from 'jsdom'
|
|
18
|
+
import createDOMPurify from 'dompurify'
|
|
19
|
+
import { Readability } from '@mozilla/readability'
|
|
20
|
+
import TurndownService from 'turndown'
|
|
21
|
+
import { gfm } from '@joplin/turndown-plugin-gfm'
|
|
22
|
+
|
|
23
|
+
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
24
|
+
const FORBID_TAGS = [
|
|
25
|
+
'nav', 'aside', 'header', 'footer', 'form', 'svg', 'iframe', 'noscript',
|
|
26
|
+
'button', 'select', 'option', 'input', 'textarea', 'dialog', 'canvas',
|
|
27
|
+
'video', 'audio', 'template',
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
/** Attributes stripped from sanitized output (styling survives as noise). */
|
|
31
|
+
const FORBID_ATTR = ['style', 'class', 'id', 'hidden', 'aria-hidden', 'role']
|
|
32
|
+
|
|
33
|
+
/** The shared converter: same style options as `dsh-tool-web`'s renderer. */
|
|
34
|
+
const turndown = new TurndownService({
|
|
35
|
+
headingStyle: 'atx',
|
|
36
|
+
codeBlockStyle: 'fenced',
|
|
37
|
+
bulletListMarker: '-',
|
|
38
|
+
})
|
|
39
|
+
turndown.use(gfm)
|
|
40
|
+
turndown.remove(['script', 'style', 'noscript'])
|
|
41
|
+
|
|
42
|
+
/** Render one GFM table cell without interpreting HTML span counts (tool-web parity). */
|
|
43
|
+
function renderTableCell(content: string, index: number): string {
|
|
44
|
+
const prefix = index === 0 ? '| ' : ' '
|
|
45
|
+
const escaped = content.trim().replace(/\n\r/g, '<br>').replace(/\n/g, '<br>').replace(/\|+/g, '\\|').padEnd(3, ' ')
|
|
46
|
+
return `${prefix}${escaped} |`
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Whether a row is the table's Markdown heading row (tool-web parity). */
|
|
50
|
+
function isTableHeadingRow(row: HTMLTableRowElement): boolean {
|
|
51
|
+
const cells = Array.from(row.cells)
|
|
52
|
+
const section = row.parentElement as HTMLTableSectionElement
|
|
53
|
+
const table = section.parentElement as HTMLTableElement
|
|
54
|
+
return (section.nodeName === 'THEAD' || table.rows[0] === row)
|
|
55
|
+
&& cells.every(cell => cell.nodeName === 'TH')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Map an HTML table-cell alignment to the GFM separator marker (tool-web parity). */
|
|
59
|
+
function tableBorder(cell: HTMLTableCellElement): string {
|
|
60
|
+
const alignment = (cell.getAttribute('align') || cell.style.textAlign || '').toLowerCase()
|
|
61
|
+
if (alignment === 'left') return ':---'
|
|
62
|
+
if (alignment === 'right') return '---:'
|
|
63
|
+
if (alignment === 'center') return ':---:'
|
|
64
|
+
return '---'
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
turndown.addRule('tableCellWithoutSpanExpansion', {
|
|
68
|
+
filter: ['th', 'td'],
|
|
69
|
+
replacement(content, node) {
|
|
70
|
+
const cell = node as HTMLTableCellElement
|
|
71
|
+
const row = cell.parentNode as HTMLTableRowElement
|
|
72
|
+
// GFM cannot represent spanning cells; ignoring colspan keeps conversion
|
|
73
|
+
// work proportional to the source (tool-web's deliberate choice).
|
|
74
|
+
return renderTableCell(content, Array.prototype.indexOf.call(row.childNodes, cell))
|
|
75
|
+
},
|
|
76
|
+
})
|
|
77
|
+
turndown.addRule('tableRowWithoutSpanExpansion', {
|
|
78
|
+
filter: 'tr',
|
|
79
|
+
replacement(content, node) {
|
|
80
|
+
const row = node as HTMLTableRowElement
|
|
81
|
+
const border = isTableHeadingRow(row)
|
|
82
|
+
? Array.from(row.cells, (cell, index) => renderTableCell(tableBorder(cell), index)).join('')
|
|
83
|
+
: ''
|
|
84
|
+
return `\n${content}${border.length > 0 ? `\n${border}` : ''}`
|
|
85
|
+
},
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
/** Which extraction path produced a result. */
|
|
89
|
+
export type DenoiseMode = 'article' | 'document'
|
|
90
|
+
|
|
91
|
+
/** One denoise pipeline outcome. */
|
|
92
|
+
export interface DenoiseResult {
|
|
93
|
+
/** The markdown body (title, when found, already prepended). */
|
|
94
|
+
markdown: string
|
|
95
|
+
/** `article` = Readability extraction; `document` = whole-document fallback. */
|
|
96
|
+
mode: DenoiseMode
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Convert one rendered HTML document to denoised markdown.
|
|
101
|
+
*
|
|
102
|
+
* Readability failure (non-article pages) degrades to converting the
|
|
103
|
+
* sanitized whole document — layout tags are still forbidden, so the
|
|
104
|
+
* fallback stays cleaner than raw turndown, and a degraded page beats an
|
|
105
|
+
* error for a body the browser already rendered.
|
|
106
|
+
*
|
|
107
|
+
* @param html - the rendered page HTML (`page.content()`).
|
|
108
|
+
* @param url - the page URL, used to resolve relative links during parsing.
|
|
109
|
+
* @returns the markdown and the extraction mode used.
|
|
110
|
+
*/
|
|
111
|
+
export function htmlToMarkdown(html: string, url: string): DenoiseResult {
|
|
112
|
+
// jsdom's virtual console defaults to forwarding parse noise; a silent one
|
|
113
|
+
// keeps broken inline CSS on random pages out of host logs.
|
|
114
|
+
const dom = new JSDOM(html, { url, virtualConsole: new VirtualConsole() })
|
|
115
|
+
const purify = createDOMPurify(dom.window)
|
|
116
|
+
const document = dom.window.document
|
|
117
|
+
|
|
118
|
+
let source: string | null = null
|
|
119
|
+
let title: string | undefined
|
|
120
|
+
try {
|
|
121
|
+
// Ungated: isProbablyReaderable is a conservative hint that rejects
|
|
122
|
+
// sparse-but-real articles (measured on a browser-rendered fixture), so
|
|
123
|
+
// the extraction is simply attempted; a null or empty result falls back
|
|
124
|
+
// to the sanitized whole document below. parse() mutates, hence the clone
|
|
125
|
+
// (typed as Document: DOM lib types cloneNode's return as Node).
|
|
126
|
+
const cloned = document.cloneNode(true) as typeof document
|
|
127
|
+
const article = new Readability(cloned).parse()
|
|
128
|
+
if (article !== null && typeof article.content === 'string' && article.content !== '') {
|
|
129
|
+
source = article.content
|
|
130
|
+
title = article.title ?? undefined
|
|
131
|
+
}
|
|
132
|
+
} catch {
|
|
133
|
+
// Readability throws on pathological DOMs; the whole-document path below
|
|
134
|
+
// still returns something usable.
|
|
135
|
+
}
|
|
136
|
+
let mode: DenoiseMode = 'article'
|
|
137
|
+
if (source === null) {
|
|
138
|
+
mode = 'document'
|
|
139
|
+
source = document.body?.innerHTML ?? ''
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// KEEP_CONTENT: false is the load-bearing half of the denoise: DOMPurify
|
|
143
|
+
// un-wraps forbidden tags but keeps their text by default, which would
|
|
144
|
+
// leave nav/footer strings floating in the fallback path. With it, the
|
|
145
|
+
// whole subtree of a forbidden element goes.
|
|
146
|
+
const clean = purify.sanitize(source, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false }) as string
|
|
147
|
+
let markdown: string
|
|
148
|
+
try {
|
|
149
|
+
markdown = turndown.turndown(clean)
|
|
150
|
+
} catch {
|
|
151
|
+
markdown = dom.window.document.createElement('div').textContent ?? ''
|
|
152
|
+
}
|
|
153
|
+
// Collapse the runs of blank lines nested-list removal leaves behind, then
|
|
154
|
+
// prepend the extracted title when the body did not open with one.
|
|
155
|
+
markdown = markdown.replace(/\n{3,}/g, '\n\n').trim()
|
|
156
|
+
const headingTitle = title?.trim()
|
|
157
|
+
if (headingTitle !== undefined && headingTitle !== '' && !markdown.startsWith('# ')) {
|
|
158
|
+
markdown = `# ${headingTitle}\n\n${markdown}`
|
|
159
|
+
}
|
|
160
|
+
return { markdown, mode }
|
|
161
|
+
}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Locates a usable Playwright for the local backend. Resolution order:
|
|
3
|
+
*
|
|
4
|
+
* 1. the configured executable/document path (settings field `playwrightPath`);
|
|
5
|
+
* 2. when blank, a `playwright` executable discovered on `$PATH`;
|
|
6
|
+
* 3. the plugin's own bundled `playwright-core` dependency.
|
|
7
|
+
*
|
|
8
|
+
* A discovered path is classified, not assumed: the Node `playwright` CLI
|
|
9
|
+
* yields its package (whose `chromium` knows that installation's browser
|
|
10
|
+
* registry), while a Chromium-family browser binary yields the bundled
|
|
11
|
+
* `playwright-core` with `executablePath` pinned to it. The CDP backend needs
|
|
12
|
+
* no local browsers at all and always uses the bundled core.
|
|
13
|
+
*
|
|
14
|
+
* Pure probing lives here; the provider composes it with page navigation.
|
|
15
|
+
*
|
|
16
|
+
* @module dsh-web-fetch-playwright/playwright-resolve
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { accessSync, constants, readFileSync, realpathSync, statSync } from 'node:fs'
|
|
20
|
+
import { dirname, join } from 'node:path'
|
|
21
|
+
import { createRequire } from 'node:module'
|
|
22
|
+
import { pathToFileURL } from 'node:url'
|
|
23
|
+
import type { PlaywrightChromium } from './types.ts'
|
|
24
|
+
|
|
25
|
+
/** File magics identifying a native browser executable rather than a script. */
|
|
26
|
+
const EXECUTABLE_MAGICS = [
|
|
27
|
+
Buffer.from([0x7f, 0x45, 0x4c, 0x46]), // ELF (Linux)
|
|
28
|
+
Buffer.from([0xcf, 0xfa, 0xed, 0xfe]), // Mach-O 64 LE (macOS)
|
|
29
|
+
Buffer.from([0xca, 0xfe, 0xba, 0xbe]), // Mach-O fat (macOS)
|
|
30
|
+
Buffer.from([0x4d, 0x5a]), // MZ (Windows)
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
/** A resolved local backend: how to reach `chromium` and what to launch. */
|
|
34
|
+
export interface ResolvedPlaywright {
|
|
35
|
+
/** The `chromium` namespace serving `launch` / `connectOverCDP`. */
|
|
36
|
+
chromium: PlaywrightChromium
|
|
37
|
+
/** Browser binary pinned for `launch({ executablePath })`, when one was configured. */
|
|
38
|
+
executablePath?: string
|
|
39
|
+
/** Provenance for diagnostics. */
|
|
40
|
+
source: string
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Find `name` as an executable file on `$PATH`.
|
|
45
|
+
* @param name - the executable basename.
|
|
46
|
+
* @returns the first matching path, or undefined.
|
|
47
|
+
*/
|
|
48
|
+
export function findOnPath(name: string): string | undefined {
|
|
49
|
+
const pathValue = process.env.PATH ?? ''
|
|
50
|
+
for (const dir of pathValue.split(':')) {
|
|
51
|
+
if (dir === '') continue
|
|
52
|
+
const candidate = join(dir, name)
|
|
53
|
+
try {
|
|
54
|
+
const stat = statSync(candidate)
|
|
55
|
+
if (!stat.isFile()) continue
|
|
56
|
+
accessSync(candidate, constants.X_OK)
|
|
57
|
+
return candidate
|
|
58
|
+
} catch {
|
|
59
|
+
// missing or not executable — keep scanning
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return undefined
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* Whether a file starts with a native-executable magic (ELF / Mach-O / MZ).
|
|
67
|
+
* @param path - the file to sniff.
|
|
68
|
+
* @returns true when the file is a compiled binary.
|
|
69
|
+
*/
|
|
70
|
+
export function isNativeExecutable(path: string): boolean {
|
|
71
|
+
let head: Buffer
|
|
72
|
+
try {
|
|
73
|
+
head = readFileSync(path).subarray(0, 4)
|
|
74
|
+
} catch {
|
|
75
|
+
return false
|
|
76
|
+
}
|
|
77
|
+
return EXECUTABLE_MAGICS.some(magic => head.subarray(0, magic.length).equals(magic))
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/**
|
|
81
|
+
* Walk up from a file (or the file's own directory) to the nearest package
|
|
82
|
+
* root whose manifest names Playwright.
|
|
83
|
+
* @param start - a file path inside or beside the candidate package.
|
|
84
|
+
* @param maxHops - upward directory bound.
|
|
85
|
+
* @returns the package root directory, or undefined.
|
|
86
|
+
*/
|
|
87
|
+
export function findPlaywrightPackageRoot(start: string, maxHops = 6): string | undefined {
|
|
88
|
+
let dir = statSync(start).isDirectory() ? start : dirname(start)
|
|
89
|
+
// Symlinked bins (npm global layout) resolve to the real package files.
|
|
90
|
+
try {
|
|
91
|
+
const real = realpathSync(dir)
|
|
92
|
+
dir = real
|
|
93
|
+
} catch {
|
|
94
|
+
// keep the literal path
|
|
95
|
+
}
|
|
96
|
+
for (let hop = 0; hop <= maxHops; hop++) {
|
|
97
|
+
const manifest = join(dir, 'package.json')
|
|
98
|
+
try {
|
|
99
|
+
const pkg = JSON.parse(readFileSync(manifest, 'utf8')) as { name?: string }
|
|
100
|
+
if (pkg.name === 'playwright' || pkg.name === 'playwright-core' || pkg.name === '@playwright/test') return dir
|
|
101
|
+
} catch {
|
|
102
|
+
// no manifest here — keep climbing
|
|
103
|
+
}
|
|
104
|
+
const parent = dirname(dir)
|
|
105
|
+
if (parent === dir) return undefined
|
|
106
|
+
dir = parent
|
|
107
|
+
}
|
|
108
|
+
return undefined
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Import the Playwright package rooted at `root`, preferring its ESM entry
|
|
113
|
+
* with a CommonJS fallback.
|
|
114
|
+
* @param root - the package root directory.
|
|
115
|
+
* @returns the package's default export (the Playwright module).
|
|
116
|
+
*/
|
|
117
|
+
async function importPlaywrightPackage(root: string): Promise<{ chromium?: unknown }> {
|
|
118
|
+
const require = createRequire(join(root, 'package.json'))
|
|
119
|
+
const entry = require.resolve('.')
|
|
120
|
+
return await import(pathToFileURL(entry).href) as { chromium?: unknown }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** Memoized successful resolutions, keyed by the configured path ("" = auto). */
|
|
124
|
+
const resolvedCache = new Map<string, ResolvedPlaywright>()
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* Resolve the local Playwright backend for one fetch.
|
|
128
|
+
*
|
|
129
|
+
* @param configuredPath - the settings `playwrightPath` (blank = auto-discover).
|
|
130
|
+
* @returns how to launch/connect, with provenance for diagnostics.
|
|
131
|
+
* @throws {Error} with an actionable message when nothing usable exists.
|
|
132
|
+
*/
|
|
133
|
+
export async function resolvePlaywrightBackend(configuredPath: string): Promise<ResolvedPlaywright> {
|
|
134
|
+
const trimmed = configuredPath.trim()
|
|
135
|
+
const cacheKey = trimmed
|
|
136
|
+
const cached = resolvedCache.get(cacheKey)
|
|
137
|
+
if (cached !== undefined) return cached
|
|
138
|
+
|
|
139
|
+
const candidate = trimmed !== '' ? trimmed : findOnPath('playwright')
|
|
140
|
+
if (trimmed !== '' && !existsAsFile(trimmed)) {
|
|
141
|
+
throw new Error(`configured playwright path "${trimmed}" does not exist`)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
if (candidate !== undefined && isNativeExecutable(candidate)) {
|
|
145
|
+
// A pinned browser binary: drive it with the bundled core.
|
|
146
|
+
const core = await importBundledCore()
|
|
147
|
+
const resolved: ResolvedPlaywright = {
|
|
148
|
+
chromium: core,
|
|
149
|
+
executablePath: realpathSync(candidate),
|
|
150
|
+
source: `browser executable ${candidate} (bundled playwright-core)`,
|
|
151
|
+
}
|
|
152
|
+
resolvedCache.set(cacheKey, resolved)
|
|
153
|
+
return resolved
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
if (candidate !== undefined) {
|
|
157
|
+
const root = findPlaywrightPackageRoot(candidate)
|
|
158
|
+
if (root !== undefined) {
|
|
159
|
+
const pkg = await importPlaywrightPackage(root)
|
|
160
|
+
const chromium = pkg.chromium
|
|
161
|
+
if (chromium !== undefined && typeof (chromium as PlaywrightChromium).launch === 'function') {
|
|
162
|
+
const resolved: ResolvedPlaywright = {
|
|
163
|
+
chromium: chromium as PlaywrightChromium,
|
|
164
|
+
source: `playwright package at ${root}${trimmed === '' ? ' (discovered on $PATH)' : ''}`,
|
|
165
|
+
}
|
|
166
|
+
resolvedCache.set(cacheKey, resolved)
|
|
167
|
+
return resolved
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
// A script that is neither a Playwright CLI nor a browser binary.
|
|
171
|
+
if (trimmed !== '') {
|
|
172
|
+
throw new Error(`configured playwright path "${trimmed}" is neither a Playwright CLI nor a browser executable`)
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
const core = await importBundledCore()
|
|
177
|
+
const resolved: ResolvedPlaywright = {
|
|
178
|
+
chromium: core,
|
|
179
|
+
source: 'bundled playwright-core (no playwright found on $PATH; browsers must be discoverable, or set the path)',
|
|
180
|
+
}
|
|
181
|
+
resolvedCache.set(cacheKey, resolved)
|
|
182
|
+
return resolved
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** The CDP backend: the bundled core only — no local browser registry needed. */
|
|
186
|
+
export async function resolveCdpBackend(): Promise<ResolvedPlaywright> {
|
|
187
|
+
const core = await importBundledCore()
|
|
188
|
+
return { chromium: core, source: 'bundled playwright-core over CDP' }
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
let bundledCore: PlaywrightChromium | undefined
|
|
192
|
+
|
|
193
|
+
async function importBundledCore(): Promise<PlaywrightChromium> {
|
|
194
|
+
if (bundledCore !== undefined) return bundledCore
|
|
195
|
+
const pkg = await import('playwright-core') as { chromium?: unknown }
|
|
196
|
+
const chromium = pkg.chromium
|
|
197
|
+
if (chromium === undefined || typeof (chromium as PlaywrightChromium).launch !== 'function') {
|
|
198
|
+
throw new Error('playwright-core dependency did not export a usable chromium namespace')
|
|
199
|
+
}
|
|
200
|
+
bundledCore = chromium as PlaywrightChromium
|
|
201
|
+
return bundledCore
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function existsAsFile(path: string): boolean {
|
|
205
|
+
try {
|
|
206
|
+
return statSync(path).isFile()
|
|
207
|
+
} catch {
|
|
208
|
+
return false
|
|
209
|
+
}
|
|
210
|
+
}
|