dsh-web-fetch-playwright 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/README.zh-CN.md +7 -4
- package/dsh.plugin.json +1 -1
- package/lib/client.js +152 -99
- package/lib/index.js +278 -33
- package/lib/types/cdp-pool.d.ts +74 -0
- package/lib/types/client/controller.d.ts +4 -0
- package/lib/types/client/form.d.ts +10 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -2
- package/lib/types/index.d.ts +2 -0
- package/lib/types/provider.d.ts +28 -6
- package/lib/types/types.d.ts +4 -0
- package/package.json +9 -11
- package/src/cdp-pool.ts +159 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +7 -1
- package/src/client/form.ts +23 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -2
- package/src/index.ts +7 -1
- package/src/provider.ts +163 -35
- package/src/types.ts +4 -0
package/src/provider.ts
CHANGED
|
@@ -4,10 +4,16 @@
|
|
|
4
4
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
5
5
|
* classification) so the tool layer sees the same codes from either backend.
|
|
6
6
|
*
|
|
7
|
-
* Lifecycle
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
8
|
+
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
9
|
+
* the remote browser for the provider's lifetime; each fetch only opens an
|
|
10
|
+
* isolated context (a tab) inside it and closes that on completion, so the
|
|
11
|
+
* concurrency cap counts tabs, not browsers. Either way, an aborted signal
|
|
12
|
+
* closes the fetch's context. Concurrency is capped at the `maxConcurrency`
|
|
13
|
+
* setting (explicit, or the backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL}
|
|
14
|
+
* browsers / {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait
|
|
15
|
+
* briefly in a queue and fail fast (rather than hang until abort) when no
|
|
16
|
+
* slot frees.
|
|
11
17
|
*
|
|
12
18
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
13
19
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
@@ -18,8 +24,9 @@
|
|
|
18
24
|
|
|
19
25
|
import { WebError } from '@deepseek-ai/dsh-web'
|
|
20
26
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
|
|
21
|
-
import { normalizeCdpEndpoint } from './config.ts'
|
|
27
|
+
import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
|
|
22
28
|
import type { ResolvedConfig } from './config.ts'
|
|
29
|
+
import { CdpConnectionPool } from './cdp-pool.ts'
|
|
23
30
|
import { htmlToMarkdown } from './markdown.ts'
|
|
24
31
|
import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
|
|
25
32
|
import type { PlaywrightBrowser, PlaywrightContext } from './types.ts'
|
|
@@ -36,8 +43,13 @@ const MAX_BODY_CHARS = 100_000
|
|
|
36
43
|
/** Cap on rendered HTML fed into the synchronous denoise pipeline. */
|
|
37
44
|
const MAX_PIPELINE_INPUT_CHARS = 2_000_000
|
|
38
45
|
|
|
39
|
-
/**
|
|
40
|
-
|
|
46
|
+
/**
|
|
47
|
+
* How long a fetch may sit in the concurrency queue before failing fast.
|
|
48
|
+
* Waiting longer cannot help — the per-fetch deadline leaves too little
|
|
49
|
+
* budget to render after dequeue — and failing fast tells the caller to
|
|
50
|
+
* retry or raise `maxConcurrency` instead of hanging until an abort.
|
|
51
|
+
*/
|
|
52
|
+
const QUEUE_TIMEOUT_MS = 20_000
|
|
41
53
|
|
|
42
54
|
/** Default per-fetch budget (ms), inside the tool layer's 60s. */
|
|
43
55
|
const DEFAULT_TIMEOUT_MS = 45_000
|
|
@@ -45,10 +57,23 @@ const DEFAULT_TIMEOUT_MS = 45_000
|
|
|
45
57
|
/** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
|
|
46
58
|
const SETTLE_MS = 5_000
|
|
47
59
|
|
|
48
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Grace period (ms) for context/browser closes in the cleanup path. A wedged
|
|
62
|
+
* `close()` must never hold a concurrency slot hostage — a leaked slot would
|
|
63
|
+
* fail every later fetch with the queue error until restart.
|
|
64
|
+
*/
|
|
65
|
+
const CLOSE_GRACE_MS = 2_000
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* One render session: a browser and the context this fetch owns. Local
|
|
69
|
+
* backends own the browser too; CDP sessions ride the shared connection and
|
|
70
|
+
* only their context closes at the end.
|
|
71
|
+
*/
|
|
49
72
|
export interface BrowserSession {
|
|
50
73
|
browser: PlaywrightBrowser
|
|
51
74
|
context: PlaywrightContext
|
|
75
|
+
/** True when `browser` is the CDP pool's shared connection — never closed per fetch. */
|
|
76
|
+
sharedBrowser?: boolean
|
|
52
77
|
}
|
|
53
78
|
|
|
54
79
|
/**
|
|
@@ -87,44 +112,88 @@ class Deadline {
|
|
|
87
112
|
}
|
|
88
113
|
}
|
|
89
114
|
|
|
90
|
-
/**
|
|
115
|
+
/**
|
|
116
|
+
* A bounded async semaphore with abort support. The limit is live-resizable
|
|
117
|
+
* (`resize`) because the config thunk re-reads on every fetch — raising it
|
|
118
|
+
* wakes queued waiters immediately; lowering it lets in-flight holders run
|
|
119
|
+
* out naturally.
|
|
120
|
+
*/
|
|
91
121
|
class Semaphore {
|
|
92
122
|
private active = 0
|
|
93
|
-
private
|
|
123
|
+
private limit: number
|
|
124
|
+
private readonly queue: Array<{ start: () => void; fail: (error: WebError) => void }> = []
|
|
125
|
+
|
|
126
|
+
/** @param limit - how many holders may run at once. */
|
|
127
|
+
constructor(limit: number) {
|
|
128
|
+
this.limit = limit
|
|
129
|
+
}
|
|
94
130
|
|
|
95
|
-
/**
|
|
131
|
+
/** Apply a new limit, starting queued waiters for any capacity it opens. */
|
|
132
|
+
resize(limit: number): void {
|
|
133
|
+
this.limit = limit
|
|
134
|
+
this.drain()
|
|
135
|
+
}
|
|
96
136
|
|
|
97
|
-
|
|
98
|
-
|
|
137
|
+
/**
|
|
138
|
+
* Take a slot, or queue until one frees. A queued fetch fails fast on the
|
|
139
|
+
* caller's abort signal or after `queueTimeoutMs` without a slot — the
|
|
140
|
+
* queue wait must not eat the whole fetch budget only to die mid-render.
|
|
141
|
+
*/
|
|
142
|
+
acquire(signal: AbortSignal, queueTimeoutMs: number): Promise<void> {
|
|
143
|
+
if (this.active < this.limit) {
|
|
99
144
|
this.active++
|
|
100
145
|
return Promise.resolve()
|
|
101
146
|
}
|
|
102
147
|
return new Promise<void>((resolve, reject) => {
|
|
148
|
+
const cleanup = () => {
|
|
149
|
+
clearTimeout(timer)
|
|
150
|
+
signal.removeEventListener('abort', onAbort)
|
|
151
|
+
}
|
|
103
152
|
const waiter = {
|
|
104
153
|
start: () => {
|
|
105
|
-
|
|
154
|
+
cleanup()
|
|
106
155
|
this.active++
|
|
107
156
|
resolve()
|
|
108
157
|
},
|
|
109
|
-
|
|
158
|
+
fail: (error: WebError) => {
|
|
110
159
|
const index = this.queue.indexOf(waiter)
|
|
111
160
|
if (index !== -1) this.queue.splice(index, 1)
|
|
112
|
-
|
|
161
|
+
cleanup()
|
|
162
|
+
reject(error)
|
|
113
163
|
},
|
|
114
164
|
}
|
|
165
|
+
const timer = setTimeout(() => {
|
|
166
|
+
waiter.fail(new WebError(
|
|
167
|
+
`all ${String(this.limit)} rendering slots stayed busy for ${String(queueTimeoutMs)}ms; retry shortly, or raise the maxConcurrency setting`,
|
|
168
|
+
'WEB_FETCH_TIMEOUT',
|
|
169
|
+
))
|
|
170
|
+
}, queueTimeoutMs)
|
|
171
|
+
const onAbort = () => {
|
|
172
|
+
waiter.fail(new WebError('web fetch aborted while waiting for a free rendering slot', 'WEB_ABORTED'))
|
|
173
|
+
}
|
|
115
174
|
this.queue.push(waiter)
|
|
116
|
-
signal.addEventListener('abort',
|
|
175
|
+
signal.addEventListener('abort', onAbort, { once: true })
|
|
117
176
|
})
|
|
118
177
|
}
|
|
119
178
|
|
|
120
179
|
release(): void {
|
|
121
|
-
|
|
180
|
+
// Below the limit a freed slot hands straight to the next waiter; above
|
|
181
|
+
// it (the limit was lowered) the slot disappears instead.
|
|
182
|
+
const next = this.active <= this.limit ? this.queue.shift() : undefined
|
|
122
183
|
if (next === undefined) {
|
|
123
184
|
this.active = Math.max(0, this.active - 1)
|
|
124
185
|
return
|
|
125
186
|
}
|
|
126
187
|
next.start()
|
|
127
188
|
}
|
|
189
|
+
|
|
190
|
+
private drain(): void {
|
|
191
|
+
while (this.active < this.limit) {
|
|
192
|
+
const next = this.queue.shift()
|
|
193
|
+
if (next === undefined) return
|
|
194
|
+
next.start()
|
|
195
|
+
}
|
|
196
|
+
}
|
|
128
197
|
}
|
|
129
198
|
|
|
130
199
|
/** Validate a request URL: http(s) only, no embedded credentials, bounded. */
|
|
@@ -162,7 +231,12 @@ function classifyContentType(contentType: string | undefined): FetchableKind | u
|
|
|
162
231
|
/** Translate a thrown pipeline error into the seam's WebError taxonomy. */
|
|
163
232
|
function translateError(error: unknown, deadline: Deadline): WebError {
|
|
164
233
|
if (deadline.isTimeout) return new WebError('playwright web fetch timed out', 'WEB_FETCH_TIMEOUT', { cause: error })
|
|
165
|
-
if (deadline.signal.aborted)
|
|
234
|
+
if (deadline.signal.aborted) {
|
|
235
|
+
// An outer cancellation of a queued fetch already carries the precise
|
|
236
|
+
// "waiting for a free rendering slot" message — keep it over the generic abort.
|
|
237
|
+
if (error instanceof WebError) return error
|
|
238
|
+
return new WebError('web fetch aborted', 'WEB_ABORTED', { cause: error })
|
|
239
|
+
}
|
|
166
240
|
if (error instanceof WebError) return error
|
|
167
241
|
return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, 'WEB_PROVIDER_ERROR', { cause: error })
|
|
168
242
|
}
|
|
@@ -175,34 +249,52 @@ function translateError(error: unknown, deadline: Deadline): WebError {
|
|
|
175
249
|
export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
176
250
|
readonly id = PLAYWRIGHT_FETCH_PROVIDER_ID
|
|
177
251
|
|
|
178
|
-
private readonly semaphore = new Semaphore()
|
|
252
|
+
private readonly semaphore = new Semaphore(DEFAULT_MAX_CONCURRENCY_LOCAL)
|
|
253
|
+
|
|
254
|
+
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
255
|
+
protected readonly cdpPool: CdpConnectionPool
|
|
179
256
|
|
|
180
257
|
/**
|
|
181
258
|
* @param configSource - thunk returning the currently authoritative config.
|
|
259
|
+
* @param cdpPool - optional pool over the CDP backend (tests inject fakes).
|
|
182
260
|
*/
|
|
183
|
-
constructor(private readonly configSource: () => ResolvedConfig) {
|
|
261
|
+
constructor(private readonly configSource: () => ResolvedConfig, cdpPool?: CdpConnectionPool) {
|
|
262
|
+
this.cdpPool = cdpPool ?? new CdpConnectionPool(defaultCdpConnect)
|
|
263
|
+
}
|
|
184
264
|
|
|
185
265
|
/** Cheap and side-effect free; backend problems surface per fetch instead. */
|
|
186
266
|
available(): boolean {
|
|
187
267
|
return true
|
|
188
268
|
}
|
|
189
269
|
|
|
270
|
+
/**
|
|
271
|
+
* Drop the shared CDP connection (plugin teardown). Local browsers need
|
|
272
|
+
* nothing — they never outlive their fetch. In-flight CDP fetches keep
|
|
273
|
+
* their leases and close them when they finish.
|
|
274
|
+
*/
|
|
275
|
+
async dispose(): Promise<void> {
|
|
276
|
+
await this.cdpPool.dispose()
|
|
277
|
+
}
|
|
278
|
+
|
|
190
279
|
async fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult> {
|
|
191
280
|
if (signal?.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED')
|
|
192
281
|
const config = this.configSource()
|
|
193
282
|
const url = validateFetchUrl(request.url)
|
|
194
283
|
const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS)
|
|
195
284
|
|
|
196
|
-
await this.semaphore.acquire(deadline.signal)
|
|
197
285
|
let session: BrowserSession | undefined
|
|
286
|
+
let acquired = false
|
|
198
287
|
try {
|
|
288
|
+
// The live config decides the limit per fetch (explicit setting, else
|
|
289
|
+
// the backend default); raising it immediately starts queued waiters.
|
|
290
|
+
this.semaphore.resize(effectiveMaxConcurrency(config))
|
|
291
|
+
await this.semaphore.acquire(deadline.signal, QUEUE_TIMEOUT_MS)
|
|
292
|
+
acquired = true
|
|
199
293
|
session = await this.openSession(config, deadline)
|
|
200
294
|
// An aborted deadline must also interrupt Playwright's own waits:
|
|
201
295
|
// closing the context rejects every pending page operation.
|
|
202
|
-
const
|
|
203
|
-
|
|
204
|
-
void session?.browser.close().catch(() => {})
|
|
205
|
-
}
|
|
296
|
+
const held = session
|
|
297
|
+
const onAbort = () => { void closeSession(held) }
|
|
206
298
|
deadline.signal.addEventListener('abort', onAbort, { once: true })
|
|
207
299
|
try {
|
|
208
300
|
return await this.retrieve(session, url, config, deadline)
|
|
@@ -212,10 +304,12 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
212
304
|
} catch (error: unknown) {
|
|
213
305
|
throw translateError(error, deadline)
|
|
214
306
|
} finally {
|
|
215
|
-
// Local launches exit; CDP connections merely disconnect.
|
|
216
|
-
|
|
217
|
-
await session
|
|
218
|
-
|
|
307
|
+
// Local launches exit; CDP connections merely disconnect. Both closes
|
|
308
|
+
// are grace-bounded so a wedged browser can never pin a concurrency slot.
|
|
309
|
+
await closeSession(session)
|
|
310
|
+
// Only a slot actually taken is given back — a queued fetch that failed
|
|
311
|
+
// to acquire must not hand a phantom slot to the next waiter.
|
|
312
|
+
if (acquired) this.semaphore.release()
|
|
219
313
|
}
|
|
220
314
|
}
|
|
221
315
|
|
|
@@ -230,12 +324,13 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
230
324
|
const timeout = Math.min(deadline.remainingMs(), 20_000)
|
|
231
325
|
if (config.backend === 'cdp') {
|
|
232
326
|
const endpoint = normalizeCdpEndpoint(config.cdpEndpoint)
|
|
233
|
-
const {
|
|
327
|
+
const { source } = await resolveCdpBackend()
|
|
234
328
|
try {
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
await
|
|
238
|
-
|
|
329
|
+
// One shared connection per provider; this fetch leases an isolated
|
|
330
|
+
// context (a tab in the remote browser) and closes only that.
|
|
331
|
+
const lease = await this.cdpPool.acquire(endpoint, timeout)
|
|
332
|
+
await installResourceFilter(lease.context)
|
|
333
|
+
return { browser: lease.browser, context: lease.context, sharedBrowser: true }
|
|
239
334
|
} catch (error: unknown) {
|
|
240
335
|
throw new WebError(
|
|
241
336
|
`cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`,
|
|
@@ -311,6 +406,39 @@ function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text
|
|
|
311
406
|
}
|
|
312
407
|
}
|
|
313
408
|
|
|
409
|
+
/**
|
|
410
|
+
* Close a session's fetch-owned context, plus its browser when the session
|
|
411
|
+
* launched one (local backend). CDP sessions ride the shared connection,
|
|
412
|
+
* which stays open. Every close is grace-bounded so cleanup always completes
|
|
413
|
+
* and the concurrency slot is released even when Playwright hangs.
|
|
414
|
+
*/
|
|
415
|
+
async function closeSession(session: BrowserSession | undefined): Promise<void> {
|
|
416
|
+
if (session === undefined) return
|
|
417
|
+
await closeWithGrace(session.context)
|
|
418
|
+
if (session.sharedBrowser !== true) await closeWithGrace(session.browser)
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
/** One best-effort `close()` that resolves within the grace period. */
|
|
422
|
+
async function closeWithGrace(closeable: { close(): Promise<void> }): Promise<void> {
|
|
423
|
+
await new Promise<void>((resolve) => {
|
|
424
|
+
const timer = setTimeout(resolve, CLOSE_GRACE_MS)
|
|
425
|
+
void closeable.close().then(
|
|
426
|
+
() => { clearTimeout(timer); resolve() },
|
|
427
|
+
() => { clearTimeout(timer); resolve() },
|
|
428
|
+
)
|
|
429
|
+
})
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
/**
|
|
433
|
+
* The real CDP connect the pool runs with: the bundled playwright-core's
|
|
434
|
+
* `connectOverCDP`. Kept as a function (not inline) so the pool constructor
|
|
435
|
+
* stays injectable for tests.
|
|
436
|
+
*/
|
|
437
|
+
async function defaultCdpConnect(endpoint: string, timeoutMs: number): Promise<PlaywrightBrowser> {
|
|
438
|
+
const { chromium } = await resolveCdpBackend()
|
|
439
|
+
return await chromium.connectOverCDP(endpoint, { timeout: timeoutMs })
|
|
440
|
+
}
|
|
441
|
+
|
|
314
442
|
/**
|
|
315
443
|
* Abort image/font/media subrequests: the markdown output keeps their URLs
|
|
316
444
|
* but never renders them, so downloading them only spends the budget.
|
package/src/types.ts
CHANGED
|
@@ -42,6 +42,10 @@ export interface PlaywrightRoute {
|
|
|
42
42
|
export interface PlaywrightBrowser {
|
|
43
43
|
newContext(): Promise<PlaywrightContext>
|
|
44
44
|
close(): Promise<void>
|
|
45
|
+
/** Liveness probe; absent on minimal fakes (assumed live). */
|
|
46
|
+
isConnected?(): boolean
|
|
47
|
+
/** Optional disconnect notification used to drop a stale shared CDP connection. */
|
|
48
|
+
on?(event: 'disconnected', listener: () => void): unknown
|
|
45
49
|
}
|
|
46
50
|
|
|
47
51
|
/** The `chromium` namespace of whichever Playwright module serves a fetch. */
|