dsh-web-fetch-playwright 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/provider.ts CHANGED
@@ -4,10 +4,16 @@
4
4
  * error taxonomy (URL hygiene, abort/timeout translation, content-type
5
5
  * classification) so the tool layer sees the same codes from either backend.
6
6
  *
7
- * Lifecycle is per-fetch and fully reversible: local launches close their
8
- * browser, CDP connections disconnect, and an aborted signal closes both —
9
- * no browser outlives the call that opened it. Concurrency is capped at
10
- * {@link MAX_CONCURRENT_FETCHES} local browsers.
7
+ * Lifecycle: the local backend launches a browser per fetch and closes it —
8
+ * nothing outlives the call. The CDP backend keeps ONE shared connection to
9
+ * the remote browser for the provider's lifetime; each fetch only opens an
10
+ * isolated context (a tab) inside it and closes that on completion, so the
11
+ * concurrency cap counts tabs, not browsers. Either way, an aborted signal
12
+ * closes the fetch's context. Concurrency is capped at the `maxConcurrency`
13
+ * setting (explicit, or the backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL}
14
+ * browsers / {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait
15
+ * briefly in a queue and fail fast (rather than hang until abort) when no
16
+ * slot frees.
11
17
  *
12
18
  * Private-network and SSRF protection is not implemented (same stance as the
13
19
  * shipped HTTP provider); a page this provider can reach is whatever the
@@ -18,8 +24,9 @@
18
24
 
19
25
  import { WebError } from '@deepseek-ai/dsh-web'
20
26
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
21
- import { normalizeCdpEndpoint } from './config.ts'
27
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
22
28
  import type { ResolvedConfig } from './config.ts'
29
+ import { CdpConnectionPool } from './cdp-pool.ts'
23
30
  import { htmlToMarkdown } from './markdown.ts'
24
31
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
25
32
  import type { PlaywrightBrowser, PlaywrightContext } from './types.ts'
@@ -36,8 +43,13 @@ const MAX_BODY_CHARS = 100_000
36
43
  /** Cap on rendered HTML fed into the synchronous denoise pipeline. */
37
44
  const MAX_PIPELINE_INPUT_CHARS = 2_000_000
38
45
 
39
- /** How many browsers may render at once; further fetches queue. */
40
- const MAX_CONCURRENT_FETCHES = 2
46
+ /**
47
+ * How long a fetch may sit in the concurrency queue before failing fast.
48
+ * Waiting longer cannot help — the per-fetch deadline leaves too little
49
+ * budget to render after dequeue — and failing fast tells the caller to
50
+ * retry or raise `maxConcurrency` instead of hanging until an abort.
51
+ */
52
+ const QUEUE_TIMEOUT_MS = 20_000
41
53
 
42
54
  /** Default per-fetch budget (ms), inside the tool layer's 60s. */
43
55
  const DEFAULT_TIMEOUT_MS = 45_000
@@ -45,10 +57,23 @@ const DEFAULT_TIMEOUT_MS = 45_000
45
57
  /** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
46
58
  const SETTLE_MS = 5_000
47
59
 
48
- /** One open browser plus its per-fetch context, closed together. */
60
+ /**
61
+ * Grace period (ms) for context/browser closes in the cleanup path. A wedged
62
+ * `close()` must never hold a concurrency slot hostage — a leaked slot would
63
+ * fail every later fetch with the queue error until restart.
64
+ */
65
+ const CLOSE_GRACE_MS = 2_000
66
+
67
+ /**
68
+ * One render session: a browser and the context this fetch owns. Local
69
+ * backends own the browser too; CDP sessions ride the shared connection and
70
+ * only their context closes at the end.
71
+ */
49
72
  export interface BrowserSession {
50
73
  browser: PlaywrightBrowser
51
74
  context: PlaywrightContext
75
+ /** True when `browser` is the CDP pool's shared connection — never closed per fetch. */
76
+ sharedBrowser?: boolean
52
77
  }
53
78
 
54
79
  /**
@@ -87,44 +112,88 @@ class Deadline {
87
112
  }
88
113
  }
89
114
 
90
- /** A cap-{@link MAX_CONCURRENT_FETCHES} async semaphore with abort support. */
115
+ /**
116
+ * A bounded async semaphore with abort support. The limit is live-resizable
117
+ * (`resize`) because the config thunk re-reads on every fetch — raising it
118
+ * wakes queued waiters immediately; lowering it lets in-flight holders run
119
+ * out naturally.
120
+ */
91
121
  class Semaphore {
92
122
  private active = 0
93
- private readonly queue: Array<{ start: () => void; abort: () => void }> = []
123
+ private limit: number
124
+ private readonly queue: Array<{ start: () => void; fail: (error: WebError) => void }> = []
125
+
126
+ /** @param limit - how many holders may run at once. */
127
+ constructor(limit: number) {
128
+ this.limit = limit
129
+ }
94
130
 
95
- /** Take a slot, or queue until one frees; an aborted signal rejects the wait. */
131
+ /** Apply a new limit, starting queued waiters for any capacity it opens. */
132
+ resize(limit: number): void {
133
+ this.limit = limit
134
+ this.drain()
135
+ }
96
136
 
97
- acquire(signal: AbortSignal): Promise<void> {
98
- if (this.active < MAX_CONCURRENT_FETCHES) {
137
+ /**
138
+ * Take a slot, or queue until one frees. A queued fetch fails fast on the
139
+ * caller's abort signal or after `queueTimeoutMs` without a slot — the
140
+ * queue wait must not eat the whole fetch budget only to die mid-render.
141
+ */
142
+ acquire(signal: AbortSignal, queueTimeoutMs: number): Promise<void> {
143
+ if (this.active < this.limit) {
99
144
  this.active++
100
145
  return Promise.resolve()
101
146
  }
102
147
  return new Promise<void>((resolve, reject) => {
148
+ const cleanup = () => {
149
+ clearTimeout(timer)
150
+ signal.removeEventListener('abort', onAbort)
151
+ }
103
152
  const waiter = {
104
153
  start: () => {
105
- signal.removeEventListener('abort', waiter.abort)
154
+ cleanup()
106
155
  this.active++
107
156
  resolve()
108
157
  },
109
- abort: () => {
158
+ fail: (error: WebError) => {
110
159
  const index = this.queue.indexOf(waiter)
111
160
  if (index !== -1) this.queue.splice(index, 1)
112
- reject(new WebError('web fetch aborted while waiting for a free browser slot', 'WEB_ABORTED'))
161
+ cleanup()
162
+ reject(error)
113
163
  },
114
164
  }
165
+ const timer = setTimeout(() => {
166
+ waiter.fail(new WebError(
167
+ `all ${String(this.limit)} rendering slots stayed busy for ${String(queueTimeoutMs)}ms; retry shortly, or raise the maxConcurrency setting`,
168
+ 'WEB_FETCH_TIMEOUT',
169
+ ))
170
+ }, queueTimeoutMs)
171
+ const onAbort = () => {
172
+ waiter.fail(new WebError('web fetch aborted while waiting for a free rendering slot', 'WEB_ABORTED'))
173
+ }
115
174
  this.queue.push(waiter)
116
- signal.addEventListener('abort', waiter.abort, { once: true })
175
+ signal.addEventListener('abort', onAbort, { once: true })
117
176
  })
118
177
  }
119
178
 
120
179
  release(): void {
121
- const next = this.queue.shift()
180
+ // Below the limit a freed slot hands straight to the next waiter; above
181
+ // it (the limit was lowered) the slot disappears instead.
182
+ const next = this.active <= this.limit ? this.queue.shift() : undefined
122
183
  if (next === undefined) {
123
184
  this.active = Math.max(0, this.active - 1)
124
185
  return
125
186
  }
126
187
  next.start()
127
188
  }
189
+
190
+ private drain(): void {
191
+ while (this.active < this.limit) {
192
+ const next = this.queue.shift()
193
+ if (next === undefined) return
194
+ next.start()
195
+ }
196
+ }
128
197
  }
129
198
 
130
199
  /** Validate a request URL: http(s) only, no embedded credentials, bounded. */
@@ -162,7 +231,12 @@ function classifyContentType(contentType: string | undefined): FetchableKind | u
162
231
  /** Translate a thrown pipeline error into the seam's WebError taxonomy. */
163
232
  function translateError(error: unknown, deadline: Deadline): WebError {
164
233
  if (deadline.isTimeout) return new WebError('playwright web fetch timed out', 'WEB_FETCH_TIMEOUT', { cause: error })
165
- if (deadline.signal.aborted) return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
234
+ if (deadline.signal.aborted) {
235
+ // An outer cancellation of a queued fetch already carries the precise
236
+ // "waiting for a free rendering slot" message — keep it over the generic abort.
237
+ if (error instanceof WebError) return error
238
+ return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
239
+ }
166
240
  if (error instanceof WebError) return error
167
241
  return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
168
242
  }
@@ -175,34 +249,52 @@ function translateError(error: unknown, deadline: Deadline): WebError {
175
249
  export class PlaywrightFetchProvider implements WebFetchProvider {
176
250
  readonly id = PLAYWRIGHT_FETCH_PROVIDER_ID
177
251
 
178
- private readonly semaphore = new Semaphore()
252
+ private readonly semaphore = new Semaphore(DEFAULT_MAX_CONCURRENCY_LOCAL)
253
+
254
+ /** Shared CDP connection; injectable so the suite can substitute a fake. */
255
+ protected readonly cdpPool: CdpConnectionPool
179
256
 
180
257
  /**
181
258
  * @param configSource - thunk returning the currently authoritative config.
259
+ * @param cdpPool - optional pool over the CDP backend (tests inject fakes).
182
260
  */
183
- constructor(private readonly configSource: () => ResolvedConfig) {}
261
+ constructor(private readonly configSource: () => ResolvedConfig, cdpPool?: CdpConnectionPool) {
262
+ this.cdpPool = cdpPool ?? new CdpConnectionPool(defaultCdpConnect)
263
+ }
184
264
 
185
265
  /** Cheap and side-effect free; backend problems surface per fetch instead. */
186
266
  available(): boolean {
187
267
  return true
188
268
  }
189
269
 
270
+ /**
271
+ * Drop the shared CDP connection (plugin teardown). Local browsers need
272
+ * nothing — they never outlive their fetch. In-flight CDP fetches keep
273
+ * their leases and close them when they finish.
274
+ */
275
+ async dispose(): Promise<void> {
276
+ await this.cdpPool.dispose()
277
+ }
278
+
190
279
  async fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult> {
191
280
  if (signal?.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED')
192
281
  const config = this.configSource()
193
282
  const url = validateFetchUrl(request.url)
194
283
  const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS)
195
284
 
196
- await this.semaphore.acquire(deadline.signal)
197
285
  let session: BrowserSession | undefined
286
+ let acquired = false
198
287
  try {
288
+ // The live config decides the limit per fetch (explicit setting, else
289
+ // the backend default); raising it immediately starts queued waiters.
290
+ this.semaphore.resize(effectiveMaxConcurrency(config))
291
+ await this.semaphore.acquire(deadline.signal, QUEUE_TIMEOUT_MS)
292
+ acquired = true
199
293
  session = await this.openSession(config, deadline)
200
294
  // An aborted deadline must also interrupt Playwright's own waits:
201
295
  // closing the context rejects every pending page operation.
202
- const onAbort = () => {
203
- void session?.context.close().catch(() => {})
204
- void session?.browser.close().catch(() => {})
205
- }
296
+ const held = session
297
+ const onAbort = () => { void closeSession(held) }
206
298
  deadline.signal.addEventListener('abort', onAbort, { once: true })
207
299
  try {
208
300
  return await this.retrieve(session, url, config, deadline)
@@ -212,10 +304,12 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
212
304
  } catch (error: unknown) {
213
305
  throw translateError(error, deadline)
214
306
  } finally {
215
- // Local launches exit; CDP connections merely disconnect.
216
- await session?.context.close().catch(() => {})
217
- await session?.browser.close().catch(() => {})
218
- this.semaphore.release()
307
+ // Local launches exit; CDP connections merely disconnect. Both closes
308
+ // are grace-bounded so a wedged browser can never pin a concurrency slot.
309
+ await closeSession(session)
310
+ // Only a slot actually taken is given back — a queued fetch that failed
311
+ // to acquire must not hand a phantom slot to the next waiter.
312
+ if (acquired) this.semaphore.release()
219
313
  }
220
314
  }
221
315
 
@@ -230,12 +324,13 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
230
324
  const timeout = Math.min(deadline.remainingMs(), 20_000)
231
325
  if (config.backend === 'cdp') {
232
326
  const endpoint = normalizeCdpEndpoint(config.cdpEndpoint)
233
- const { chromium, source } = await resolveCdpBackend()
327
+ const { source } = await resolveCdpBackend()
234
328
  try {
235
- const browser = await chromium.connectOverCDP(endpoint, { timeout })
236
- const context = await browser.newContext()
237
- await installResourceFilter(context)
238
- return { browser, context }
329
+ // One shared connection per provider; this fetch leases an isolated
330
+ // context (a tab in the remote browser) and closes only that.
331
+ const lease = await this.cdpPool.acquire(endpoint, timeout)
332
+ await installResourceFilter(lease.context)
333
+ return { browser: lease.browser, context: lease.context, sharedBrowser: true }
239
334
  } catch (error: unknown) {
240
335
  throw new WebError(
241
336
  `cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`,
@@ -311,6 +406,39 @@ function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text
311
406
  }
312
407
  }
313
408
 
409
+ /**
410
+ * Close a session's fetch-owned context, plus its browser when the session
411
+ * launched one (local backend). CDP sessions ride the shared connection,
412
+ * which stays open. Every close is grace-bounded so cleanup always completes
413
+ * and the concurrency slot is released even when Playwright hangs.
414
+ */
415
+ async function closeSession(session: BrowserSession | undefined): Promise<void> {
416
+ if (session === undefined) return
417
+ await closeWithGrace(session.context)
418
+ if (session.sharedBrowser !== true) await closeWithGrace(session.browser)
419
+ }
420
+
421
+ /** One best-effort `close()` that resolves within the grace period. */
422
+ async function closeWithGrace(closeable: { close(): Promise<void> }): Promise<void> {
423
+ await new Promise<void>((resolve) => {
424
+ const timer = setTimeout(resolve, CLOSE_GRACE_MS)
425
+ void closeable.close().then(
426
+ () => { clearTimeout(timer); resolve() },
427
+ () => { clearTimeout(timer); resolve() },
428
+ )
429
+ })
430
+ }
431
+
432
+ /**
433
+ * The real CDP connect the pool runs with: the bundled playwright-core's
434
+ * `connectOverCDP`. Kept as a function (not inline) so the pool constructor
435
+ * stays injectable for tests.
436
+ */
437
+ async function defaultCdpConnect(endpoint: string, timeoutMs: number): Promise<PlaywrightBrowser> {
438
+ const { chromium } = await resolveCdpBackend()
439
+ return await chromium.connectOverCDP(endpoint, { timeout: timeoutMs })
440
+ }
441
+
314
442
  /**
315
443
  * Abort image/font/media subrequests: the markdown output keeps their URLs
316
444
  * but never renders them, so downloading them only spends the budget.
package/src/types.ts CHANGED
@@ -42,6 +42,10 @@ export interface PlaywrightRoute {
42
42
  export interface PlaywrightBrowser {
43
43
  newContext(): Promise<PlaywrightContext>
44
44
  close(): Promise<void>
45
+ /** Liveness probe; absent on minimal fakes (assumed live). */
46
+ isConnected?(): boolean
47
+ /** Optional disconnect notification used to drop a stale shared CDP connection. */
48
+ on?(event: 'disconnected', listener: () => void): unknown
45
49
  }
46
50
 
47
51
  /** The `chromium` namespace of whichever Playwright module serves a fetch. */