dsh-web-fetch-playwright 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,329 @@
1
+ /**
2
+ * The Playwright `WebFetchProvider`: renders one URL in a real browser and
3
+ * returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
4
+ * error taxonomy (URL hygiene, abort/timeout translation, content-type
5
+ * classification) so the tool layer sees the same codes from either backend.
6
+ *
7
+ * Lifecycle is per-fetch and fully reversible: local launches close their
8
+ * browser, CDP connections disconnect, and an aborted signal closes both —
9
+ * no browser outlives the call that opened it. Concurrency is capped at
10
+ * {@link MAX_CONCURRENT_FETCHES} local browsers.
11
+ *
12
+ * Private-network and SSRF protection is not implemented (same stance as the
13
+ * shipped HTTP provider); a page this provider can reach is whatever the
14
+ * browser can reach.
15
+ *
16
+ * @module dsh-web-fetch-playwright/provider
17
+ */
18
+
19
+ import { WebError } from '@deepseek-ai/dsh-web'
20
+ import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
21
+ import { normalizeCdpEndpoint } from './config.ts'
22
+ import type { ResolvedConfig } from './config.ts'
23
+ import { htmlToMarkdown } from './markdown.ts'
24
+ import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
25
+ import type { PlaywrightBrowser, PlaywrightContext } from './types.ts'
26
+
27
+ /** Stable id this provider registers under (the bundle patch pins it). */
28
+ export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
29
+
30
+ /** Maximum accepted request URL length (http-provider parity). */
31
+ const MAX_URL_LENGTH = 2048
32
+
33
+ /** Cap on the decoded markdown/HTML body this provider returns. */
34
+ const MAX_BODY_CHARS = 100_000
35
+
36
+ /** Cap on rendered HTML fed into the synchronous denoise pipeline. */
37
+ const MAX_PIPELINE_INPUT_CHARS = 2_000_000
38
+
39
+ /** How many browsers may render at once; further fetches queue. */
40
+ const MAX_CONCURRENT_FETCHES = 2
41
+
42
+ /** Default per-fetch budget (ms), inside the tool layer's 60s. */
43
+ const DEFAULT_TIMEOUT_MS = 45_000
44
+
45
+ /** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
46
+ const SETTLE_MS = 5_000
47
+
48
+ /** One open browser plus its per-fetch context, closed together. */
49
+ export interface BrowserSession {
50
+ browser: PlaywrightBrowser
51
+ context: PlaywrightContext
52
+ }
53
+
54
+ /**
55
+ * A single abort deadline composing the caller's signal with this provider's
56
+ * time budget. The timeout reason is kept apart from an outer abort so the
57
+ * error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
58
+ */
59
+ class Deadline {
60
+ readonly signal: AbortSignal
61
+ private readonly controller = new AbortController()
62
+ private readonly expiresAt: number
63
+ private timedOut = false
64
+
65
+ constructor(outer: AbortSignal | undefined, timeoutMs: number) {
66
+ this.signal = this.controller.signal
67
+ this.expiresAt = Date.now() + timeoutMs
68
+ const timer = setTimeout(() => {
69
+ this.timedOut = true
70
+ this.controller.abort(new Error('playwright fetch deadline'))
71
+ }, timeoutMs)
72
+ if (outer !== undefined) {
73
+ if (outer.aborted) this.controller.abort(outer.reason)
74
+ else outer.addEventListener('abort', () => { this.controller.abort(outer.reason) }, { once: true })
75
+ }
76
+ this.signal.addEventListener('abort', () => { clearTimeout(timer) }, { once: true })
77
+ }
78
+
79
+ /** Whether OUR timer fired (vs an outer cancellation). */
80
+ get isTimeout(): boolean {
81
+ return this.timedOut
82
+ }
83
+
84
+ /** Milliseconds left on the budget, floored at 1 for Playwright options. */
85
+ remainingMs(): number {
86
+ return Math.max(1, this.expiresAt - Date.now())
87
+ }
88
+ }
89
+
90
+ /** A cap-{@link MAX_CONCURRENT_FETCHES} async semaphore with abort support. */
91
+ class Semaphore {
92
+ private active = 0
93
+ private readonly queue: Array<{ start: () => void; abort: () => void }> = []
94
+
95
+ /** Take a slot, or queue until one frees; an aborted signal rejects the wait. */
96
+
97
+ acquire(signal: AbortSignal): Promise<void> {
98
+ if (this.active < MAX_CONCURRENT_FETCHES) {
99
+ this.active++
100
+ return Promise.resolve()
101
+ }
102
+ return new Promise<void>((resolve, reject) => {
103
+ const waiter = {
104
+ start: () => {
105
+ signal.removeEventListener('abort', waiter.abort)
106
+ this.active++
107
+ resolve()
108
+ },
109
+ abort: () => {
110
+ const index = this.queue.indexOf(waiter)
111
+ if (index !== -1) this.queue.splice(index, 1)
112
+ reject(new WebError('web fetch aborted while waiting for a free browser slot', 'WEB_ABORTED'))
113
+ },
114
+ }
115
+ this.queue.push(waiter)
116
+ signal.addEventListener('abort', waiter.abort, { once: true })
117
+ })
118
+ }
119
+
120
+ release(): void {
121
+ const next = this.queue.shift()
122
+ if (next === undefined) {
123
+ this.active = Math.max(0, this.active - 1)
124
+ return
125
+ }
126
+ next.start()
127
+ }
128
+ }
129
+
130
+ /** Validate a request URL: http(s) only, no embedded credentials, bounded. */
131
+ function validateFetchUrl(input: string): URL {
132
+ if (input.length > MAX_URL_LENGTH) {
133
+ throw new WebError(`URL exceeds the maximum length of ${String(MAX_URL_LENGTH)}`, 'WEB_INVALID_URL')
134
+ }
135
+ let url: URL
136
+ try {
137
+ url = new URL(input)
138
+ } catch (error: unknown) {
139
+ throw new WebError(`invalid URL: ${input}`, 'WEB_INVALID_URL', { cause: error })
140
+ }
141
+ if (url.protocol !== 'http:' && url.protocol !== 'https:') {
142
+ throw new WebError(`unsupported URL scheme "${url.protocol}" (only http and https are allowed)`, 'WEB_INVALID_URL')
143
+ }
144
+ if (url.username.length > 0 || url.password.length > 0) {
145
+ throw new WebError('credentials in URLs are not allowed', 'WEB_BLOCKED_URL')
146
+ }
147
+ return url
148
+ }
149
+
150
+ /** The decodable body kinds, mirroring the HTTP provider's classification. */
151
+ type FetchableKind = 'html' | 'text'
152
+
153
+ /** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
154
+ function classifyContentType(contentType: string | undefined): FetchableKind | undefined {
155
+ const mime = (contentType ?? '').replace(/;.*$/s, '').trim().toLowerCase()
156
+ if (mime === '' || mime === 'text/html' || mime === 'application/xhtml+xml') return 'html'
157
+ if (mime.startsWith('text/')) return 'text'
158
+ if (mime === 'application/json' || mime === 'application/xml' || mime.endsWith('+json') || mime.endsWith('+xml')) return 'text'
159
+ return undefined
160
+ }
161
+
162
+ /** Translate a thrown pipeline error into the seam's WebError taxonomy. */
163
+ function translateError(error: unknown, deadline: Deadline): WebError {
164
+ if (deadline.isTimeout) return new WebError('playwright web fetch timed out', 'WEB_FETCH_TIMEOUT', { cause: error })
165
+ if (deadline.signal.aborted) return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
166
+ if (error instanceof WebError) return error
167
+ return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
168
+ }
169
+
170
+ /**
171
+ * The Playwright-backed fetch provider. Configuration is read through a thunk
172
+ * so committed settings-section changes apply to the next fetch with no
173
+ * re-registration.
174
+ */
175
+ export class PlaywrightFetchProvider implements WebFetchProvider {
176
+ readonly id = PLAYWRIGHT_FETCH_PROVIDER_ID
177
+
178
+ private readonly semaphore = new Semaphore()
179
+
180
+ /**
181
+ * @param configSource - thunk returning the currently authoritative config.
182
+ */
183
+ constructor(private readonly configSource: () => ResolvedConfig) {}
184
+
185
+ /** Cheap and side-effect free; backend problems surface per fetch instead. */
186
+ available(): boolean {
187
+ return true
188
+ }
189
+
190
+ async fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult> {
191
+ if (signal?.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED')
192
+ const config = this.configSource()
193
+ const url = validateFetchUrl(request.url)
194
+ const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS)
195
+
196
+ await this.semaphore.acquire(deadline.signal)
197
+ let session: BrowserSession | undefined
198
+ try {
199
+ session = await this.openSession(config, deadline)
200
+ // An aborted deadline must also interrupt Playwright's own waits:
201
+ // closing the context rejects every pending page operation.
202
+ const onAbort = () => {
203
+ void session?.context.close().catch(() => {})
204
+ void session?.browser.close().catch(() => {})
205
+ }
206
+ deadline.signal.addEventListener('abort', onAbort, { once: true })
207
+ try {
208
+ return await this.retrieve(session, url, config, deadline)
209
+ } finally {
210
+ deadline.signal.removeEventListener('abort', onAbort)
211
+ }
212
+ } catch (error: unknown) {
213
+ throw translateError(error, deadline)
214
+ } finally {
215
+ // Local launches exit; CDP connections merely disconnect.
216
+ await session?.context.close().catch(() => {})
217
+ await session?.browser.close().catch(() => {})
218
+ this.semaphore.release()
219
+ }
220
+ }
221
+
222
+ /**
223
+ * Open the configured backend and its per-fetch context. Split out so the
224
+ * test suite can substitute a fake browser.
225
+ * @param config - the resolved settings section.
226
+ * @param deadline - the fetch budget, applied to launch/connect timeouts.
227
+ * @returns the browser session the fetch will use.
228
+ */
229
+ protected async openSession(config: ResolvedConfig, deadline: Deadline): Promise<BrowserSession> {
230
+ const timeout = Math.min(deadline.remainingMs(), 20_000)
231
+ if (config.backend === 'cdp') {
232
+ const endpoint = normalizeCdpEndpoint(config.cdpEndpoint)
233
+ const { chromium, source } = await resolveCdpBackend()
234
+ try {
235
+ const browser = await chromium.connectOverCDP(endpoint, { timeout })
236
+ const context = await browser.newContext()
237
+ await installResourceFilter(context)
238
+ return { browser, context }
239
+ } catch (error: unknown) {
240
+ throw new WebError(
241
+ `cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`,
242
+ 'WEB_PROVIDER_ERROR',
243
+ { cause: error },
244
+ )
245
+ }
246
+ }
247
+ const { chromium, executablePath, source } = await resolvePlaywrightBackend(config.playwrightPath)
248
+ try {
249
+ const browser = await chromium.launch({ headless: true, ...(executablePath !== undefined ? { executablePath } : {}), timeout })
250
+ const context = await browser.newContext()
251
+ await installResourceFilter(context)
252
+ return { browser, context }
253
+ } catch (error: unknown) {
254
+ throw new WebError(
255
+ `cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`,
256
+ 'WEB_PROVIDER_ERROR',
257
+ { cause: error },
258
+ )
259
+ }
260
+ }
261
+
262
+ /** Navigate, settle, and decode one URL inside an open session. */
263
+ private async retrieve(
264
+ session: BrowserSession,
265
+ url: URL,
266
+ config: ResolvedConfig,
267
+ deadline: Deadline,
268
+ ): Promise<WebFetchResult> {
269
+ const page = await session.context.newPage()
270
+ const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
271
+
272
+ const kind = classifyContentType(response?.headers()['content-type'])
273
+ if (kind === undefined) {
274
+ throw new WebError(`unsupported content type "${response?.headers()['content-type'] ?? 'unknown'}"`, 'WEB_UNSUPPORTED_CONTENT_TYPE')
275
+ }
276
+ const finalUrl = page.url()
277
+
278
+ // Non-HTML decodes straight from the response body; no denoise applies.
279
+ if (kind === 'text') {
280
+ const text = response !== null ? await response.text() : await page.content()
281
+ return capResult(finalUrl, response?.status() ?? 200, { kind: 'text', content: text })
282
+ }
283
+
284
+ // Best-effort settle for client-rendered content; a timeout just keeps
285
+ // what domcontentloaded already produced.
286
+ await page.waitForLoadState('networkidle', { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {})
287
+
288
+ const html = await page.content()
289
+ if (!config.denoise) {
290
+ // The tool layer's own turndown renders raw HTML; the checkbox only
291
+ // governs the Readability/DOMPurify stage this provider owns.
292
+ return capResult(finalUrl, response?.status() ?? 200, { kind: 'html', content: html })
293
+ }
294
+ const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html
295
+ const { markdown } = htmlToMarkdown(bounded, finalUrl)
296
+ const result = capResult(finalUrl, response?.status() ?? 200, { kind: 'text', content: markdown })
297
+ return bounded !== html
298
+ ? { ...result, truncated: true }
299
+ : result
300
+ }
301
+ }
302
+
303
+ /** Apply the decoded-body cap and flag the cut. */
304
+ function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text'; content: string }): WebFetchResult {
305
+ const truncated = body.content.length > MAX_BODY_CHARS
306
+ return {
307
+ url,
308
+ statusCode,
309
+ body: { kind: body.kind, content: truncated ? body.content.slice(0, MAX_BODY_CHARS) : body.content },
310
+ truncated,
311
+ }
312
+ }
313
+
314
+ /**
315
+ * Abort image/font/media subrequests: the markdown output keeps their URLs
316
+ * but never renders them, so downloading them only spends the budget.
317
+ * Best-effort — a context that refuses interception still fetches.
318
+ */
319
+ async function installResourceFilter(context: PlaywrightContext): Promise<void> {
320
+ try {
321
+ await context.route('**/*', async (route) => {
322
+ const type = route.request().resourceType()
323
+ if (type === 'image' || type === 'font' || type === 'media') await route.abort()
324
+ else await route.continue()
325
+ })
326
+ } catch {
327
+ // keep going without the filter
328
+ }
329
+ }
package/src/types.ts ADDED
@@ -0,0 +1,51 @@
1
+ /**
2
+ * Structural types over the Playwright surface this plugin uses. Declared
3
+ * locally (not imported from `playwright`) because the runtime module is
4
+ * discovered dynamically — a user's global install, a pinned browser
5
+ * executable, or the bundled `playwright-core` — and only these members are
6
+ * load-bearing.
7
+ *
8
+ * @module dsh-web-fetch-playwright/types
9
+ */
10
+
11
+ /** A navigation response, as `page.goto` returns it. */
12
+ export interface PlaywrightResponse {
13
+ status(): number
14
+ headers(): Record<string, string>
15
+ text(): Promise<string>
16
+ }
17
+
18
+ /** A page inside a context. */
19
+ export interface PlaywrightPage {
20
+ goto(url: string, options?: { waitUntil?: 'load' | 'domcontentloaded' | 'networkidle' | 'commit'; timeout?: number }): Promise<PlaywrightResponse | null>
21
+ waitForLoadState(state?: 'load' | 'domcontentloaded' | 'networkidle', options?: { timeout?: number }): Promise<void>
22
+ url(): string
23
+ content(): Promise<string>
24
+ close(): Promise<void>
25
+ }
26
+
27
+ /** An isolated browser context (the CDP backend gets a fresh one per fetch). */
28
+ export interface PlaywrightContext {
29
+ newPage(): Promise<PlaywrightPage>
30
+ route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
31
+ close(): Promise<void>
32
+ }
33
+
34
+ /** A route interception decision. */
35
+ export interface PlaywrightRoute {
36
+ request(): { resourceType(): string }
37
+ abort(): Promise<void>
38
+ continue(): Promise<void>
39
+ }
40
+
41
+ /** A browser instance (launched locally or connected over CDP). */
42
+ export interface PlaywrightBrowser {
43
+ newContext(): Promise<PlaywrightContext>
44
+ close(): Promise<void>
45
+ }
46
+
47
+ /** The `chromium` namespace of whichever Playwright module serves a fetch. */
48
+ export interface PlaywrightChromium {
49
+ launch(options?: { headless?: boolean; executablePath?: string; timeout?: number }): Promise<PlaywrightBrowser>
50
+ connectOverCDP(endpointURL: string, options?: { timeout?: number }): Promise<PlaywrightBrowser>
51
+ }