dsh-web-fetch-playwright 0.1.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts CHANGED
@@ -4,25 +4,35 @@
4
4
  * error taxonomy (URL hygiene, abort/timeout translation, content-type
5
5
  * classification) so the tool layer sees the same codes from either backend.
6
6
  *
7
- * Lifecycle is per-fetch and fully reversible: local launches close their
8
- * browser, CDP connections disconnect, and an aborted signal closes both —
9
- * no browser outlives the call that opened it. Concurrency is capped at
10
- * {@link MAX_CONCURRENT_FETCHES} local browsers.
7
+ * Lifecycle: the local backend launches a browser per fetch and closes it —
8
+ * nothing outlives the call. The CDP backend keeps ONE shared connection to
9
+ * the remote browser for the provider's lifetime; each fetch only opens a
10
+ * page (tab) inside it and closes that on completion — in a throwaway
11
+ * isolated context, or (the default, `shareBrowserContext`) in the remote
12
+ * browser's default context so its profile, cookies, and persistent logins
13
+ * apply; that default context is never closed. The concurrency cap counts
14
+ * tabs, not browsers. Either way, an aborted signal closes the fetch's page.
15
+ * Concurrency is capped at the `maxConcurrency` setting (explicit, or the
16
+ * backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL} browsers /
17
+ * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
18
+ * a queue and fail fast (rather than hang until abort) when no slot frees.
11
19
  *
12
20
  * Private-network and SSRF protection is not implemented (same stance as the
13
21
  * shipped HTTP provider); a page this provider can reach is whatever the
14
- * browser can reach.
22
+ * browser can reach. Profile mode additionally acts WITH the remote
23
+ * browser's logged-in sessions (see the README's risk notes).
15
24
  *
16
25
  * @module dsh-web-fetch-playwright/provider
17
26
  */
18
27
 
19
28
  import { WebError } from '@deepseek-ai/dsh-web'
20
29
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
21
- import { normalizeCdpEndpoint } from './config.ts'
30
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveContextMode, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
22
31
  import type { ResolvedConfig } from './config.ts'
32
+ import { CdpConnectionPool } from './cdp-pool.ts'
23
33
  import { htmlToMarkdown } from './markdown.ts'
24
34
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
25
- import type { PlaywrightBrowser, PlaywrightContext } from './types.ts'
35
+ import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightRoute } from './types.ts'
26
36
 
27
37
  /** Stable id this provider registers under (the bundle patch pins it). */
28
38
  export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
@@ -36,8 +46,13 @@ const MAX_BODY_CHARS = 100_000
36
46
  /** Cap on rendered HTML fed into the synchronous denoise pipeline. */
37
47
  const MAX_PIPELINE_INPUT_CHARS = 2_000_000
38
48
 
39
- /** How many browsers may render at once; further fetches queue. */
40
- const MAX_CONCURRENT_FETCHES = 2
49
+ /**
50
+ * How long a fetch may sit in the concurrency queue before failing fast.
51
+ * Waiting longer cannot help — the per-fetch deadline leaves too little
52
+ * budget to render after dequeue — and failing fast tells the caller to
53
+ * retry or raise `maxConcurrency` instead of hanging until an abort.
54
+ */
55
+ const QUEUE_TIMEOUT_MS = 20_000
41
56
 
42
57
  /** Default per-fetch budget (ms), inside the tool layer's 60s. */
43
58
  const DEFAULT_TIMEOUT_MS = 45_000
@@ -45,10 +60,28 @@ const DEFAULT_TIMEOUT_MS = 45_000
45
60
  /** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
46
61
  const SETTLE_MS = 5_000
47
62
 
48
- /** One open browser plus its per-fetch context, closed together. */
63
+ /**
64
+ * Grace period (ms) for context/browser closes in the cleanup path. A wedged
65
+ * `close()` must never hold a concurrency slot hostage — a leaked slot would
66
+ * fail every later fetch with the queue error until restart.
67
+ */
68
+ const CLOSE_GRACE_MS = 2_000
69
+
70
+ /**
71
+ * One render session: a browser, the context this fetch works in, and the
72
+ * tab it owns. Local backends own the browser too; CDP sessions ride the
73
+ * shared connection, and in `profile` mode the context is the remote
74
+ * browser's default context — shared, never closed, only the tab is.
75
+ */
49
76
  export interface BrowserSession {
50
77
  browser: PlaywrightBrowser
51
78
  context: PlaywrightContext
79
+ /** The fetch-owned tab this fetch renders in. */
80
+ page: PlaywrightPage
81
+ /** True when `browser` is the CDP pool's shared connection — never closed per fetch. */
82
+ sharedBrowser?: boolean
83
+ /** True when `context` is the remote default context — close only the page. */
84
+ persistent?: boolean
52
85
  }
53
86
 
54
87
  /**
@@ -87,44 +120,88 @@ class Deadline {
87
120
  }
88
121
  }
89
122
 
90
- /** A cap-{@link MAX_CONCURRENT_FETCHES} async semaphore with abort support. */
123
+ /**
124
+ * A bounded async semaphore with abort support. The limit is live-resizable
125
+ * (`resize`) because the config thunk re-reads on every fetch — raising it
126
+ * wakes queued waiters immediately; lowering it lets in-flight holders run
127
+ * out naturally.
128
+ */
91
129
  class Semaphore {
92
130
  private active = 0
93
- private readonly queue: Array<{ start: () => void; abort: () => void }> = []
131
+ private limit: number
132
+ private readonly queue: Array<{ start: () => void; fail: (error: WebError) => void }> = []
94
133
 
95
- /** Take a slot, or queue until one frees; an aborted signal rejects the wait. */
134
+ /** @param limit - how many holders may run at once. */
135
+ constructor(limit: number) {
136
+ this.limit = limit
137
+ }
138
+
139
+ /** Apply a new limit, starting queued waiters for any capacity it opens. */
140
+ resize(limit: number): void {
141
+ this.limit = limit
142
+ this.drain()
143
+ }
96
144
 
97
- acquire(signal: AbortSignal): Promise<void> {
98
- if (this.active < MAX_CONCURRENT_FETCHES) {
145
+ /**
146
+ * Take a slot, or queue until one frees. A queued fetch fails fast on the
147
+ * caller's abort signal or after `queueTimeoutMs` without a slot — the
148
+ * queue wait must not eat the whole fetch budget only to die mid-render.
149
+ */
150
+ acquire(signal: AbortSignal, queueTimeoutMs: number): Promise<void> {
151
+ if (this.active < this.limit) {
99
152
  this.active++
100
153
  return Promise.resolve()
101
154
  }
102
155
  return new Promise<void>((resolve, reject) => {
156
+ const cleanup = () => {
157
+ clearTimeout(timer)
158
+ signal.removeEventListener('abort', onAbort)
159
+ }
103
160
  const waiter = {
104
161
  start: () => {
105
- signal.removeEventListener('abort', waiter.abort)
162
+ cleanup()
106
163
  this.active++
107
164
  resolve()
108
165
  },
109
- abort: () => {
166
+ fail: (error: WebError) => {
110
167
  const index = this.queue.indexOf(waiter)
111
168
  if (index !== -1) this.queue.splice(index, 1)
112
- reject(new WebError('web fetch aborted while waiting for a free browser slot', 'WEB_ABORTED'))
169
+ cleanup()
170
+ reject(error)
113
171
  },
114
172
  }
173
+ const timer = setTimeout(() => {
174
+ waiter.fail(new WebError(
175
+ `all ${String(this.limit)} rendering slots stayed busy for ${String(queueTimeoutMs)}ms; retry shortly, or raise the maxConcurrency setting`,
176
+ 'WEB_FETCH_TIMEOUT',
177
+ ))
178
+ }, queueTimeoutMs)
179
+ const onAbort = () => {
180
+ waiter.fail(new WebError('web fetch aborted while waiting for a free rendering slot', 'WEB_ABORTED'))
181
+ }
115
182
  this.queue.push(waiter)
116
- signal.addEventListener('abort', waiter.abort, { once: true })
183
+ signal.addEventListener('abort', onAbort, { once: true })
117
184
  })
118
185
  }
119
186
 
120
187
  release(): void {
121
- const next = this.queue.shift()
188
+ // Below the limit a freed slot hands straight to the next waiter; above
189
+ // it (the limit was lowered) the slot disappears instead.
190
+ const next = this.active <= this.limit ? this.queue.shift() : undefined
122
191
  if (next === undefined) {
123
192
  this.active = Math.max(0, this.active - 1)
124
193
  return
125
194
  }
126
195
  next.start()
127
196
  }
197
+
198
+ private drain(): void {
199
+ while (this.active < this.limit) {
200
+ const next = this.queue.shift()
201
+ if (next === undefined) return
202
+ next.start()
203
+ }
204
+ }
128
205
  }
129
206
 
130
207
  /** Validate a request URL: http(s) only, no embedded credentials, bounded. */
@@ -162,7 +239,12 @@ function classifyContentType(contentType: string | undefined): FetchableKind | u
162
239
  /** Translate a thrown pipeline error into the seam's WebError taxonomy. */
163
240
  function translateError(error: unknown, deadline: Deadline): WebError {
164
241
  if (deadline.isTimeout) return new WebError('playwright web fetch timed out', 'WEB_FETCH_TIMEOUT', { cause: error })
165
- if (deadline.signal.aborted) return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
242
+ if (deadline.signal.aborted) {
243
+ // An outer cancellation of a queued fetch already carries the precise
244
+ // "waiting for a free rendering slot" message — keep it over the generic abort.
245
+ if (error instanceof WebError) return error
246
+ return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
247
+ }
166
248
  if (error instanceof WebError) return error
167
249
  return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
168
250
  }
@@ -175,34 +257,53 @@ function translateError(error: unknown, deadline: Deadline): WebError {
175
257
  export class PlaywrightFetchProvider implements WebFetchProvider {
176
258
  readonly id = PLAYWRIGHT_FETCH_PROVIDER_ID
177
259
 
178
- private readonly semaphore = new Semaphore()
260
+ private readonly semaphore = new Semaphore(DEFAULT_MAX_CONCURRENCY_LOCAL)
261
+
262
+ /** Shared CDP connection; injectable so the suite can substitute a fake. */
263
+ protected readonly cdpPool: CdpConnectionPool
179
264
 
180
265
  /**
181
266
  * @param configSource - thunk returning the currently authoritative config.
267
+ * @param cdpPool - optional pool over the CDP backend (tests inject fakes).
182
268
  */
183
- constructor(private readonly configSource: () => ResolvedConfig) {}
269
+ constructor(private readonly configSource: () => ResolvedConfig, cdpPool?: CdpConnectionPool) {
270
+ this.cdpPool = cdpPool ?? new CdpConnectionPool(defaultCdpConnect)
271
+ }
184
272
 
185
273
  /** Cheap and side-effect free; backend problems surface per fetch instead. */
186
274
  available(): boolean {
187
275
  return true
188
276
  }
189
277
 
278
+ /**
279
+ * Drop the shared CDP connection (plugin teardown). Local browsers need
280
+ * nothing — they never outlive their fetch. In-flight CDP fetches keep
281
+ * their leases and close them when they finish.
282
+ */
283
+ async dispose(): Promise<void> {
284
+ await this.cdpPool.dispose()
285
+ }
286
+
190
287
  async fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult> {
191
288
  if (signal?.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED')
192
289
  const config = this.configSource()
193
290
  const url = validateFetchUrl(request.url)
194
291
  const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS)
195
292
 
196
- await this.semaphore.acquire(deadline.signal)
197
293
  let session: BrowserSession | undefined
294
+ let acquired = false
198
295
  try {
296
+ // The live config decides the limit per fetch (explicit setting, else
297
+ // the backend default); raising it immediately starts queued waiters.
298
+ this.semaphore.resize(effectiveMaxConcurrency(config))
299
+ await this.semaphore.acquire(deadline.signal, QUEUE_TIMEOUT_MS)
300
+ acquired = true
199
301
  session = await this.openSession(config, deadline)
200
302
  // An aborted deadline must also interrupt Playwright's own waits:
201
- // closing the context rejects every pending page operation.
202
- const onAbort = () => {
203
- void session?.context.close().catch(() => {})
204
- void session?.browser.close().catch(() => {})
205
- }
303
+ // closing the page rejects every pending operation on it (and, for
304
+ // fetch-owned contexts, the context close that follows takes the rest).
305
+ const held = session
306
+ const onAbort = () => { void closeSession(held) }
206
307
  deadline.signal.addEventListener('abort', onAbort, { once: true })
207
308
  try {
208
309
  return await this.retrieve(session, url, config, deadline)
@@ -212,15 +313,17 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
212
313
  } catch (error: unknown) {
213
314
  throw translateError(error, deadline)
214
315
  } finally {
215
- // Local launches exit; CDP connections merely disconnect.
216
- await session?.context.close().catch(() => {})
217
- await session?.browser.close().catch(() => {})
218
- this.semaphore.release()
316
+ // Local launches exit; CDP connections merely disconnect. Both closes
317
+ // are grace-bounded so a wedged browser can never pin a concurrency slot.
318
+ await closeSession(session)
319
+ // Only a slot actually taken is given back — a queued fetch that failed
320
+ // to acquire must not hand a phantom slot to the next waiter.
321
+ if (acquired) this.semaphore.release()
219
322
  }
220
323
  }
221
324
 
222
325
  /**
223
- * Open the configured backend and its per-fetch context. Split out so the
326
+ * Open the configured backend and its per-fetch page. Split out so the
224
327
  * test suite can substitute a fake browser.
225
328
  * @param config - the resolved settings section.
226
329
  * @param deadline - the fetch budget, applied to launch/connect timeouts.
@@ -230,12 +333,21 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
230
333
  const timeout = Math.min(deadline.remainingMs(), 20_000)
231
334
  if (config.backend === 'cdp') {
232
335
  const endpoint = normalizeCdpEndpoint(config.cdpEndpoint)
233
- const { chromium, source } = await resolveCdpBackend()
336
+ const { source } = await resolveCdpBackend()
234
337
  try {
235
- const browser = await chromium.connectOverCDP(endpoint, { timeout })
236
- const context = await browser.newContext()
237
- await installResourceFilter(context)
238
- return { browser, context }
338
+ // One shared connection per provider; this fetch leases a tab — in a
339
+ // throwaway isolated context, or in the remote profile's default
340
+ // context (profile mode), whose persistent logins then apply.
341
+ const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config))
342
+ await installResourceFilter(lease.page)
343
+ guardPopups(lease.page)
344
+ return {
345
+ browser: lease.browser,
346
+ context: lease.context,
347
+ page: lease.page,
348
+ sharedBrowser: true,
349
+ persistent: lease.persistent,
350
+ }
239
351
  } catch (error: unknown) {
240
352
  throw new WebError(
241
353
  `cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`,
@@ -245,12 +357,19 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
245
357
  }
246
358
  }
247
359
  const { chromium, executablePath, source } = await resolvePlaywrightBackend(config.playwrightPath)
360
+ let browser: PlaywrightBrowser | undefined
248
361
  try {
249
- const browser = await chromium.launch({ headless: true, ...(executablePath !== undefined ? { executablePath } : {}), timeout })
362
+ browser = await chromium.launch({ headless: true, ...(executablePath !== undefined ? { executablePath } : {}), timeout })
250
363
  const context = await browser.newContext()
251
- await installResourceFilter(context)
252
- return { browser, context }
364
+ const page = await context.newPage()
365
+ await installResourceFilter(page)
366
+ guardPopups(page)
367
+ return { browser, context, page }
253
368
  } catch (error: unknown) {
369
+ // A partial setup (browser launched, then newContext/newPage failed)
370
+ // must not strand the process — closing the browser takes its
371
+ // contexts and pages with it.
372
+ await browser?.close().catch(() => {})
254
373
  throw new WebError(
255
374
  `cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`,
256
375
  'WEB_PROVIDER_ERROR',
@@ -259,14 +378,14 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
259
378
  }
260
379
  }
261
380
 
262
- /** Navigate, settle, and decode one URL inside an open session. */
381
+ /** Navigate, settle, and decode one URL inside an open session's tab. */
263
382
  private async retrieve(
264
383
  session: BrowserSession,
265
384
  url: URL,
266
385
  config: ResolvedConfig,
267
386
  deadline: Deadline,
268
387
  ): Promise<WebFetchResult> {
269
- const page = await session.context.newPage()
388
+ const page = session.page
270
389
  const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
271
390
 
272
391
  const kind = classifyContentType(response?.headers()['content-type'])
@@ -311,14 +430,54 @@ function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text
311
430
  }
312
431
  }
313
432
 
433
+ /**
434
+ * Close a session's fetch-owned tab, then its context unless the context is
435
+ * the remote default context (profile mode — closing it would tear down the
436
+ * whole shared connection), then its browser when the session launched one
437
+ * (local backend). CDP sessions ride the shared connection, which stays
438
+ * open. Every close is grace-bounded so cleanup always completes and the
439
+ * concurrency slot is released even when Playwright hangs.
440
+ */
441
+ async function closeSession(session: BrowserSession | undefined): Promise<void> {
442
+ if (session === undefined) return
443
+ await closeWithGrace(session.page)
444
+ if (session.persistent !== true) await closeWithGrace(session.context)
445
+ if (session.sharedBrowser !== true) await closeWithGrace(session.browser)
446
+ }
447
+
448
+ /** One best-effort `close()` that resolves within the grace period. */
449
+ async function closeWithGrace(closeable: { close(): Promise<void> }): Promise<void> {
450
+ await new Promise<void>((resolve) => {
451
+ const timer = setTimeout(resolve, CLOSE_GRACE_MS)
452
+ void closeable.close().then(
453
+ () => { clearTimeout(timer); resolve() },
454
+ () => { clearTimeout(timer); resolve() },
455
+ )
456
+ })
457
+ }
458
+
459
+ /**
460
+ * The real CDP connect the pool runs with: the bundled playwright-core's
461
+ * `connectOverCDP`. Kept as a function (not inline) so the pool constructor
462
+ * stays injectable for tests.
463
+ */
464
+ async function defaultCdpConnect(endpoint: string, timeoutMs: number): Promise<PlaywrightBrowser> {
465
+ const { chromium } = await resolveCdpBackend()
466
+ return await chromium.connectOverCDP(endpoint, { timeout: timeoutMs })
467
+ }
468
+
314
469
  /**
315
470
  * Abort image/font/media subrequests: the markdown output keeps their URLs
316
471
  * but never renders them, so downloading them only spends the budget.
317
- * Best-effort — a context that refuses interception still fetches.
472
+ * Installed on the PAGE (never the context): in profile mode a context-level
473
+ * route would intercept the remote browser's other tabs too. Best-effort —
474
+ * a page that refuses interception still fetches.
318
475
  */
319
- async function installResourceFilter(context: PlaywrightContext): Promise<void> {
476
+ async function installResourceFilter(owner: {
477
+ route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
478
+ }): Promise<void> {
320
479
  try {
321
- await context.route('**/*', async (route) => {
480
+ await owner.route('**/*', async (route) => {
322
481
  const type = route.request().resourceType()
323
482
  if (type === 'image' || type === 'font' || type === 'media') await route.abort()
324
483
  else await route.continue()
@@ -327,3 +486,17 @@ async function installResourceFilter(context: PlaywrightContext): Promise<void>
327
486
  // keep going without the filter
328
487
  }
329
488
  }
489
+
490
+ /**
491
+ * Close any popup a fetched page spawns so nothing outlives the fetch's tab
492
+ * — `page.close()` does not auto-close windows the page opened, and in
493
+ * profile mode a stray tab would stay in the user's remote browser.
494
+ * Best-effort: a page that refuses listeners just loses the guard.
495
+ */
496
+ function guardPopups(page: PlaywrightPage): void {
497
+ try {
498
+ page.on?.('popup', popup => { void popup.close().catch(() => {}) })
499
+ } catch {
500
+ // keep going without the guard
501
+ }
502
+ }
package/src/types.ts CHANGED
@@ -22,9 +22,20 @@ export interface PlaywrightPage {
22
22
  url(): string
23
23
  content(): Promise<string>
24
24
  close(): Promise<void>
25
+ /**
26
+ * Resource-filter interception at page level — installed on the page (not
27
+ * its context) so profile mode never intercepts tabs it does not own.
28
+ */
29
+ route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
30
+ /** Popup notification; the fetch closes whatever its page spawns. */
31
+ on?(event: 'popup', listener: (page: PlaywrightPage) => void): unknown
25
32
  }
26
33
 
27
- /** An isolated browser context (the CDP backend gets a fresh one per fetch). */
34
+ /**
35
+ * A browser context: fetch-owned and isolated (local backend, or CDP
36
+ * `isolated` mode), or the remote browser's default context carrying its
37
+ * real profile (CDP `profile` mode — never closed by a fetch).
38
+ */
28
39
  export interface PlaywrightContext {
29
40
  newPage(): Promise<PlaywrightPage>
30
41
  route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
@@ -42,6 +53,16 @@ export interface PlaywrightRoute {
42
53
  export interface PlaywrightBrowser {
43
54
  newContext(): Promise<PlaywrightContext>
44
55
  close(): Promise<void>
56
+ /**
57
+ * Contexts visible to this connection. Over CDP the default context — the
58
+ * remote browser's real profile — is always dispatched first, so `[0]` is
59
+ * it; absent on minimal fakes (isolated mode never calls this).
60
+ */
61
+ contexts?(): PlaywrightContext[]
62
+ /** Liveness probe; absent on minimal fakes (assumed live). */
63
+ isConnected?(): boolean
64
+ /** Optional disconnect notification used to drop a stale shared CDP connection. */
65
+ on?(event: 'disconnected', listener: () => void): unknown
45
66
  }
46
67
 
47
68
  /** The `chromium` namespace of whichever Playwright module serves a fetch. */