dsh-web-fetch-playwright 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,161 @@
1
+ /**
2
+ * The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
3
+ *
4
+ * The classic stack, in order: jsdom parses the page Playwright rendered;
5
+ * Mozilla Readability extracts the article (dropping nav bars, sidebars,
6
+ * footers, and ad chrome by scoring link density and text mass); DOMPurify
7
+ * sanitizes whatever HTML remains and forbids the layout tags noise lives in;
8
+ * Turndown with the GFM plugin converts to markdown using the same style
9
+ * options and span-safe table rules as the shipped `dsh-tool-web` renderer,
10
+ * so output is consistent with what `web_fetch` produces elsewhere.
11
+ *
12
+ * Pure and synchronous — unit-tested against fixture pages.
13
+ *
14
+ * @module dsh-web-fetch-playwright/markdown
15
+ */
16
+
17
+ import { JSDOM, VirtualConsole } from 'jsdom'
18
+ import createDOMPurify from 'dompurify'
19
+ import { Readability } from '@mozilla/readability'
20
+ import TurndownService from 'turndown'
21
+ import { gfm } from '@joplin/turndown-plugin-gfm'
22
+
23
+ /** Layout/noise tags DOMPurify removes outright (with their content). */
24
+ const FORBID_TAGS = [
25
+ 'nav', 'aside', 'header', 'footer', 'form', 'svg', 'iframe', 'noscript',
26
+ 'button', 'select', 'option', 'input', 'textarea', 'dialog', 'canvas',
27
+ 'video', 'audio', 'template',
28
+ ]
29
+
30
+ /** Attributes stripped from sanitized output (styling survives as noise). */
31
+ const FORBID_ATTR = ['style', 'class', 'id', 'hidden', 'aria-hidden', 'role']
32
+
33
+ /** The shared converter: same style options as `dsh-tool-web`'s renderer. */
34
+ const turndown = new TurndownService({
35
+ headingStyle: 'atx',
36
+ codeBlockStyle: 'fenced',
37
+ bulletListMarker: '-',
38
+ })
39
+ turndown.use(gfm)
40
+ turndown.remove(['script', 'style', 'noscript'])
41
+
42
+ /** Render one GFM table cell without interpreting HTML span counts (tool-web parity). */
43
+ function renderTableCell(content: string, index: number): string {
44
+ const prefix = index === 0 ? '| ' : ' '
45
+ const escaped = content.trim().replace(/\n\r/g, '<br>').replace(/\n/g, '<br>').replace(/\|+/g, '\\|').padEnd(3, ' ')
46
+ return `${prefix}${escaped} |`
47
+ }
48
+
49
+ /** Whether a row is the table's Markdown heading row (tool-web parity). */
50
+ function isTableHeadingRow(row: HTMLTableRowElement): boolean {
51
+ const cells = Array.from(row.cells)
52
+ const section = row.parentElement as HTMLTableSectionElement
53
+ const table = section.parentElement as HTMLTableElement
54
+ return (section.nodeName === 'THEAD' || table.rows[0] === row)
55
+ && cells.every(cell => cell.nodeName === 'TH')
56
+ }
57
+
58
+ /** Map an HTML table-cell alignment to the GFM separator marker (tool-web parity). */
59
+ function tableBorder(cell: HTMLTableCellElement): string {
60
+ const alignment = (cell.getAttribute('align') || cell.style.textAlign || '').toLowerCase()
61
+ if (alignment === 'left') return ':---'
62
+ if (alignment === 'right') return '---:'
63
+ if (alignment === 'center') return ':---:'
64
+ return '---'
65
+ }
66
+
67
+ turndown.addRule('tableCellWithoutSpanExpansion', {
68
+ filter: ['th', 'td'],
69
+ replacement(content, node) {
70
+ const cell = node as HTMLTableCellElement
71
+ const row = cell.parentNode as HTMLTableRowElement
72
+ // GFM cannot represent spanning cells; ignoring colspan keeps conversion
73
+ // work proportional to the source (tool-web's deliberate choice).
74
+ return renderTableCell(content, Array.prototype.indexOf.call(row.childNodes, cell))
75
+ },
76
+ })
77
+ turndown.addRule('tableRowWithoutSpanExpansion', {
78
+ filter: 'tr',
79
+ replacement(content, node) {
80
+ const row = node as HTMLTableRowElement
81
+ const border = isTableHeadingRow(row)
82
+ ? Array.from(row.cells, (cell, index) => renderTableCell(tableBorder(cell), index)).join('')
83
+ : ''
84
+ return `\n${content}${border.length > 0 ? `\n${border}` : ''}`
85
+ },
86
+ })
87
+
88
+ /** Which extraction path produced a result. */
89
+ export type DenoiseMode = 'article' | 'document'
90
+
91
+ /** One denoise pipeline outcome. */
92
+ export interface DenoiseResult {
93
+ /** The markdown body (title, when found, already prepended). */
94
+ markdown: string
95
+ /** `article` = Readability extraction; `document` = whole-document fallback. */
96
+ mode: DenoiseMode
97
+ }
98
+
99
+ /**
100
+ * Convert one rendered HTML document to denoised markdown.
101
+ *
102
+ * Readability failure (non-article pages) degrades to converting the
103
+ * sanitized whole document — layout tags are still forbidden, so the
104
+ * fallback stays cleaner than raw turndown, and a degraded page beats an
105
+ * error for a body the browser already rendered.
106
+ *
107
+ * @param html - the rendered page HTML (`page.content()`).
108
+ * @param url - the page URL, used to resolve relative links during parsing.
109
+ * @returns the markdown and the extraction mode used.
110
+ */
111
+ export function htmlToMarkdown(html: string, url: string): DenoiseResult {
112
+ // jsdom's virtual console defaults to forwarding parse noise; a silent one
113
+ // keeps broken inline CSS on random pages out of host logs.
114
+ const dom = new JSDOM(html, { url, virtualConsole: new VirtualConsole() })
115
+ const purify = createDOMPurify(dom.window)
116
+ const document = dom.window.document
117
+
118
+ let source: string | null = null
119
+ let title: string | undefined
120
+ try {
121
+ // Ungated: isProbablyReaderable is a conservative hint that rejects
122
+ // sparse-but-real articles (measured on a browser-rendered fixture), so
123
+ // the extraction is simply attempted; a null or empty result falls back
124
+ // to the sanitized whole document below. parse() mutates, hence the clone
125
+ // (typed as Document: DOM lib types cloneNode's return as Node).
126
+ const cloned = document.cloneNode(true) as typeof document
127
+ const article = new Readability(cloned).parse()
128
+ if (article !== null && typeof article.content === 'string' && article.content !== '') {
129
+ source = article.content
130
+ title = article.title ?? undefined
131
+ }
132
+ } catch {
133
+ // Readability throws on pathological DOMs; the whole-document path below
134
+ // still returns something usable.
135
+ }
136
+ let mode: DenoiseMode = 'article'
137
+ if (source === null) {
138
+ mode = 'document'
139
+ source = document.body?.innerHTML ?? ''
140
+ }
141
+
142
+ // KEEP_CONTENT: false is the load-bearing half of the denoise: DOMPurify
143
+ // un-wraps forbidden tags but keeps their text by default, which would
144
+ // leave nav/footer strings floating in the fallback path. With it, the
145
+ // whole subtree of a forbidden element goes.
146
+ const clean = purify.sanitize(source, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false }) as string
147
+ let markdown: string
148
+ try {
149
+ markdown = turndown.turndown(clean)
150
+ } catch {
151
+ markdown = dom.window.document.createElement('div').textContent ?? ''
152
+ }
153
+ // Collapse the runs of blank lines nested-list removal leaves behind, then
154
+ // prepend the extracted title when the body did not open with one.
155
+ markdown = markdown.replace(/\n{3,}/g, '\n\n').trim()
156
+ const headingTitle = title?.trim()
157
+ if (headingTitle !== undefined && headingTitle !== '' && !markdown.startsWith('# ')) {
158
+ markdown = `# ${headingTitle}\n\n${markdown}`
159
+ }
160
+ return { markdown, mode }
161
+ }
@@ -0,0 +1,210 @@
1
+ /**
2
+ * Locates a usable Playwright for the local backend. Resolution order:
3
+ *
4
+ * 1. the configured executable/document path (settings field `playwrightPath`);
5
+ * 2. when blank, a `playwright` executable discovered on `$PATH`;
6
+ * 3. the plugin's own bundled `playwright-core` dependency.
7
+ *
8
+ * A discovered path is classified, not assumed: the Node `playwright` CLI
9
+ * yields its package (whose `chromium` knows that installation's browser
10
+ * registry), while a Chromium-family browser binary yields the bundled
11
+ * `playwright-core` with `executablePath` pinned to it. The CDP backend needs
12
+ * no local browsers at all and always uses the bundled core.
13
+ *
14
+ * Pure probing lives here; the provider composes it with page navigation.
15
+ *
16
+ * @module dsh-web-fetch-playwright/playwright-resolve
17
+ */
18
+
19
+ import { accessSync, constants, readFileSync, realpathSync, statSync } from 'node:fs'
20
+ import { dirname, join } from 'node:path'
21
+ import { createRequire } from 'node:module'
22
+ import { pathToFileURL } from 'node:url'
23
+ import type { PlaywrightChromium } from './types.ts'
24
+
25
+ /** File magics identifying a native browser executable rather than a script. */
26
+ const EXECUTABLE_MAGICS = [
27
+ Buffer.from([0x7f, 0x45, 0x4c, 0x46]), // ELF (Linux)
28
+ Buffer.from([0xcf, 0xfa, 0xed, 0xfe]), // Mach-O 64 LE (macOS)
29
+ Buffer.from([0xca, 0xfe, 0xba, 0xbe]), // Mach-O fat (macOS)
30
+ Buffer.from([0x4d, 0x5a]), // MZ (Windows)
31
+ ]
32
+
33
+ /** A resolved local backend: how to reach `chromium` and what to launch. */
34
+ export interface ResolvedPlaywright {
35
+ /** The `chromium` namespace serving `launch` / `connectOverCDP`. */
36
+ chromium: PlaywrightChromium
37
+ /** Browser binary pinned for `launch({ executablePath })`, when one was configured. */
38
+ executablePath?: string
39
+ /** Provenance for diagnostics. */
40
+ source: string
41
+ }
42
+
43
+ /**
44
+ * Find `name` as an executable file on `$PATH`.
45
+ * @param name - the executable basename.
46
+ * @returns the first matching path, or undefined.
47
+ */
48
+ export function findOnPath(name: string): string | undefined {
49
+ const pathValue = process.env.PATH ?? ''
50
+ for (const dir of pathValue.split(':')) {
51
+ if (dir === '') continue
52
+ const candidate = join(dir, name)
53
+ try {
54
+ const stat = statSync(candidate)
55
+ if (!stat.isFile()) continue
56
+ accessSync(candidate, constants.X_OK)
57
+ return candidate
58
+ } catch {
59
+ // missing or not executable — keep scanning
60
+ }
61
+ }
62
+ return undefined
63
+ }
64
+
65
+ /**
66
+ * Whether a file starts with a native-executable magic (ELF / Mach-O / MZ).
67
+ * @param path - the file to sniff.
68
+ * @returns true when the file is a compiled binary.
69
+ */
70
+ export function isNativeExecutable(path: string): boolean {
71
+ let head: Buffer
72
+ try {
73
+ head = readFileSync(path).subarray(0, 4)
74
+ } catch {
75
+ return false
76
+ }
77
+ return EXECUTABLE_MAGICS.some(magic => head.subarray(0, magic.length).equals(magic))
78
+ }
79
+
80
+ /**
81
+ * Walk up from a file (or the file's own directory) to the nearest package
82
+ * root whose manifest names Playwright.
83
+ * @param start - a file path inside or beside the candidate package.
84
+ * @param maxHops - upward directory bound.
85
+ * @returns the package root directory, or undefined.
86
+ */
87
+ export function findPlaywrightPackageRoot(start: string, maxHops = 6): string | undefined {
88
+ let dir = statSync(start).isDirectory() ? start : dirname(start)
89
+ // Symlinked bins (npm global layout) resolve to the real package files.
90
+ try {
91
+ const real = realpathSync(dir)
92
+ dir = real
93
+ } catch {
94
+ // keep the literal path
95
+ }
96
+ for (let hop = 0; hop <= maxHops; hop++) {
97
+ const manifest = join(dir, 'package.json')
98
+ try {
99
+ const pkg = JSON.parse(readFileSync(manifest, 'utf8')) as { name?: string }
100
+ if (pkg.name === 'playwright' || pkg.name === 'playwright-core' || pkg.name === '@playwright/test') return dir
101
+ } catch {
102
+ // no manifest here — keep climbing
103
+ }
104
+ const parent = dirname(dir)
105
+ if (parent === dir) return undefined
106
+ dir = parent
107
+ }
108
+ return undefined
109
+ }
110
+
111
+ /**
112
+ * Import the Playwright package rooted at `root`, preferring its ESM entry
113
+ * with a CommonJS fallback.
114
+ * @param root - the package root directory.
115
+ * @returns the package's default export (the Playwright module).
116
+ */
117
+ async function importPlaywrightPackage(root: string): Promise<{ chromium?: unknown }> {
118
+ const require = createRequire(join(root, 'package.json'))
119
+ const entry = require.resolve('.')
120
+ return await import(pathToFileURL(entry).href) as { chromium?: unknown }
121
+ }
122
+
123
+ /** Memoized successful resolutions, keyed by the configured path ("" = auto). */
124
+ const resolvedCache = new Map<string, ResolvedPlaywright>()
125
+
126
+ /**
127
+ * Resolve the local Playwright backend for one fetch.
128
+ *
129
+ * @param configuredPath - the settings `playwrightPath` (blank = auto-discover).
130
+ * @returns how to launch/connect, with provenance for diagnostics.
131
+ * @throws {Error} with an actionable message when nothing usable exists.
132
+ */
133
+ export async function resolvePlaywrightBackend(configuredPath: string): Promise<ResolvedPlaywright> {
134
+ const trimmed = configuredPath.trim()
135
+ const cacheKey = trimmed
136
+ const cached = resolvedCache.get(cacheKey)
137
+ if (cached !== undefined) return cached
138
+
139
+ const candidate = trimmed !== '' ? trimmed : findOnPath('playwright')
140
+ if (trimmed !== '' && !existsAsFile(trimmed)) {
141
+ throw new Error(`configured playwright path "${trimmed}" does not exist`)
142
+ }
143
+
144
+ if (candidate !== undefined && isNativeExecutable(candidate)) {
145
+ // A pinned browser binary: drive it with the bundled core.
146
+ const core = await importBundledCore()
147
+ const resolved: ResolvedPlaywright = {
148
+ chromium: core,
149
+ executablePath: realpathSync(candidate),
150
+ source: `browser executable ${candidate} (bundled playwright-core)`,
151
+ }
152
+ resolvedCache.set(cacheKey, resolved)
153
+ return resolved
154
+ }
155
+
156
+ if (candidate !== undefined) {
157
+ const root = findPlaywrightPackageRoot(candidate)
158
+ if (root !== undefined) {
159
+ const pkg = await importPlaywrightPackage(root)
160
+ const chromium = pkg.chromium
161
+ if (chromium !== undefined && typeof (chromium as PlaywrightChromium).launch === 'function') {
162
+ const resolved: ResolvedPlaywright = {
163
+ chromium: chromium as PlaywrightChromium,
164
+ source: `playwright package at ${root}${trimmed === '' ? ' (discovered on $PATH)' : ''}`,
165
+ }
166
+ resolvedCache.set(cacheKey, resolved)
167
+ return resolved
168
+ }
169
+ }
170
+ // A script that is neither a Playwright CLI nor a browser binary.
171
+ if (trimmed !== '') {
172
+ throw new Error(`configured playwright path "${trimmed}" is neither a Playwright CLI nor a browser executable`)
173
+ }
174
+ }
175
+
176
+ const core = await importBundledCore()
177
+ const resolved: ResolvedPlaywright = {
178
+ chromium: core,
179
+ source: 'bundled playwright-core (no playwright found on $PATH; browsers must be discoverable, or set the path)',
180
+ }
181
+ resolvedCache.set(cacheKey, resolved)
182
+ return resolved
183
+ }
184
+
185
+ /** The CDP backend: the bundled core only — no local browser registry needed. */
186
+ export async function resolveCdpBackend(): Promise<ResolvedPlaywright> {
187
+ const core = await importBundledCore()
188
+ return { chromium: core, source: 'bundled playwright-core over CDP' }
189
+ }
190
+
191
+ let bundledCore: PlaywrightChromium | undefined
192
+
193
+ async function importBundledCore(): Promise<PlaywrightChromium> {
194
+ if (bundledCore !== undefined) return bundledCore
195
+ const pkg = await import('playwright-core') as { chromium?: unknown }
196
+ const chromium = pkg.chromium
197
+ if (chromium === undefined || typeof (chromium as PlaywrightChromium).launch !== 'function') {
198
+ throw new Error('playwright-core dependency did not export a usable chromium namespace')
199
+ }
200
+ bundledCore = chromium as PlaywrightChromium
201
+ return bundledCore
202
+ }
203
+
204
+ function existsAsFile(path: string): boolean {
205
+ try {
206
+ return statSync(path).isFile()
207
+ } catch {
208
+ return false
209
+ }
210
+ }