dsh-web-fetch-playwright 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/README.zh-CN.md +7 -4
- package/dsh.plugin.json +1 -1
- package/lib/client.js +152 -99
- package/lib/index.js +278 -33
- package/lib/types/cdp-pool.d.ts +74 -0
- package/lib/types/client/controller.d.ts +4 -0
- package/lib/types/client/form.d.ts +10 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -2
- package/lib/types/index.d.ts +2 -0
- package/lib/types/provider.d.ts +28 -6
- package/lib/types/types.d.ts +4 -0
- package/package.json +9 -11
- package/src/cdp-pool.ts +159 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +7 -1
- package/src/client/form.ts +23 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -2
- package/src/index.ts +7 -1
- package/src/provider.ts +163 -35
- package/src/types.ts +4 -0
package/lib/types/provider.d.ts
CHANGED
|
@@ -4,10 +4,16 @@
|
|
|
4
4
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
5
5
|
* classification) so the tool layer sees the same codes from either backend.
|
|
6
6
|
*
|
|
7
|
-
* Lifecycle
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
7
|
+
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
8
|
+
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
9
|
+
* the remote browser for the provider's lifetime; each fetch only opens an
|
|
10
|
+
* isolated context (a tab) inside it and closes that on completion, so the
|
|
11
|
+
* concurrency cap counts tabs, not browsers. Either way, an aborted signal
|
|
12
|
+
* closes the fetch's context. Concurrency is capped at the `maxConcurrency`
|
|
13
|
+
* setting (explicit, or the backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL}
|
|
14
|
+
* browsers / {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait
|
|
15
|
+
* briefly in a queue and fail fast (rather than hang until abort) when no
|
|
16
|
+
* slot frees.
|
|
11
17
|
*
|
|
12
18
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
13
19
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
@@ -17,13 +23,20 @@
|
|
|
17
23
|
*/
|
|
18
24
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web';
|
|
19
25
|
import type { ResolvedConfig } from './config.ts';
|
|
26
|
+
import { CdpConnectionPool } from './cdp-pool.ts';
|
|
20
27
|
import type { PlaywrightBrowser, PlaywrightContext } from './types.ts';
|
|
21
28
|
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
22
29
|
export declare const PLAYWRIGHT_FETCH_PROVIDER_ID = "playwright";
|
|
23
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* One render session: a browser and the context this fetch owns. Local
|
|
32
|
+
* backends own the browser too; CDP sessions ride the shared connection and
|
|
33
|
+
* only their context closes at the end.
|
|
34
|
+
*/
|
|
24
35
|
export interface BrowserSession {
|
|
25
36
|
browser: PlaywrightBrowser;
|
|
26
37
|
context: PlaywrightContext;
|
|
38
|
+
/** True when `browser` is the CDP pool's shared connection — never closed per fetch. */
|
|
39
|
+
sharedBrowser?: boolean;
|
|
27
40
|
}
|
|
28
41
|
/**
|
|
29
42
|
* A single abort deadline composing the caller's signal with this provider's
|
|
@@ -50,12 +63,21 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
50
63
|
private readonly configSource;
|
|
51
64
|
readonly id = "playwright";
|
|
52
65
|
private readonly semaphore;
|
|
66
|
+
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
67
|
+
protected readonly cdpPool: CdpConnectionPool;
|
|
53
68
|
/**
|
|
54
69
|
* @param configSource - thunk returning the currently authoritative config.
|
|
70
|
+
* @param cdpPool - optional pool over the CDP backend (tests inject fakes).
|
|
55
71
|
*/
|
|
56
|
-
constructor(configSource: () => ResolvedConfig);
|
|
72
|
+
constructor(configSource: () => ResolvedConfig, cdpPool?: CdpConnectionPool);
|
|
57
73
|
/** Cheap and side-effect free; backend problems surface per fetch instead. */
|
|
58
74
|
available(): boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Drop the shared CDP connection (plugin teardown). Local browsers need
|
|
77
|
+
* nothing — they never outlive their fetch. In-flight CDP fetches keep
|
|
78
|
+
* their leases and close them when they finish.
|
|
79
|
+
*/
|
|
80
|
+
dispose(): Promise<void>;
|
|
59
81
|
fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult>;
|
|
60
82
|
/**
|
|
61
83
|
* Open the configured backend and its per-fetch context. Split out so the
|
package/lib/types/types.d.ts
CHANGED
|
@@ -44,6 +44,10 @@ export interface PlaywrightRoute {
|
|
|
44
44
|
export interface PlaywrightBrowser {
|
|
45
45
|
newContext(): Promise<PlaywrightContext>;
|
|
46
46
|
close(): Promise<void>;
|
|
47
|
+
/** Liveness probe; absent on minimal fakes (assumed live). */
|
|
48
|
+
isConnected?(): boolean;
|
|
49
|
+
/** Optional disconnect notification used to drop a stale shared CDP connection. */
|
|
50
|
+
on?(event: 'disconnected', listener: () => void): unknown;
|
|
47
51
|
}
|
|
48
52
|
/** The `chromium` namespace of whichever Playwright module serves a fetch. */
|
|
49
53
|
export interface PlaywrightChromium {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -31,13 +31,6 @@
|
|
|
31
31
|
"README.zh-CN.md",
|
|
32
32
|
"LICENSE"
|
|
33
33
|
],
|
|
34
|
-
"scripts": {
|
|
35
|
-
"build": "rm -rf lib && tsc -p tsconfig.build.json --emitDeclarationOnly && tsdown",
|
|
36
|
-
"prepare": "pnpm run build",
|
|
37
|
-
"prepublishOnly": "pnpm run build",
|
|
38
|
-
"typecheck": "tsc --noEmit",
|
|
39
|
-
"test": "vitest run"
|
|
40
|
-
},
|
|
41
34
|
"dsh": {
|
|
42
35
|
"bundle": {
|
|
43
36
|
"patch": "./cordis.patch.yml"
|
|
@@ -98,8 +91,8 @@
|
|
|
98
91
|
},
|
|
99
92
|
"peerDependencies": {
|
|
100
93
|
"@deepseek-ai/cordis": ">=4.0.1-rc.1 <5",
|
|
101
|
-
"@deepseek-ai/dsh-settings": ">=0.1.0-rc.6 <1",
|
|
102
|
-
"@deepseek-ai/dsh-web": ">=0.1.0-rc.6 <1"
|
|
94
|
+
"@deepseek-ai/dsh-settings": ">=0.1.0-rc.6 <0.2.0 || >=0.1.1-rc.1 <1",
|
|
95
|
+
"@deepseek-ai/dsh-web": ">=0.1.0-rc.6 <0.2.0 || >=0.1.1-rc.1 <1"
|
|
103
96
|
},
|
|
104
97
|
"peerDependenciesMeta": {
|
|
105
98
|
"@deepseek-ai/cordis": {
|
|
@@ -126,5 +119,10 @@
|
|
|
126
119
|
"tsdown": "^0.22.2",
|
|
127
120
|
"typescript": "^5.8.0",
|
|
128
121
|
"vitest": "^4.1.8"
|
|
122
|
+
},
|
|
123
|
+
"scripts": {
|
|
124
|
+
"build": "rm -rf lib && tsc -p tsconfig.build.json --emitDeclarationOnly && tsdown",
|
|
125
|
+
"typecheck": "tsc --noEmit",
|
|
126
|
+
"test": "vitest run"
|
|
129
127
|
}
|
|
130
|
-
}
|
|
128
|
+
}
|
package/src/cdp-pool.ts
ADDED
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The CDP backend's shared connection: one `connectOverCDP` session reused by
|
|
3
|
+
* every fetch, each fetch leasing an isolated context (a tab in the remote
|
|
4
|
+
* browser) that it closes when done. The connection itself outlives fetches —
|
|
5
|
+
* reconnecting per fetch would spend 100–500ms per call and open one socket
|
|
6
|
+
* per concurrent tab for no benefit.
|
|
7
|
+
*
|
|
8
|
+
* Liveness is handled three ways: `isConnected()` is checked before reuse, a
|
|
9
|
+
* `disconnected` event (when the backend emits it) drops the reference
|
|
10
|
+
* immediately, and a `newContext` failure against a dead connection triggers
|
|
11
|
+
* exactly one reconnect-and-retry before the error surfaces. A changed
|
|
12
|
+
* endpoint (settings edit) replaces the connection.
|
|
13
|
+
*
|
|
14
|
+
* @module dsh-web-fetch-playwright/cdp-pool
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { PlaywrightBrowser, PlaywrightContext } from './types.ts'
|
|
18
|
+
|
|
19
|
+
/** Opens the shared connection; injected so tests can substitute a fake. */
|
|
20
|
+
export type CdpConnect = (endpoint: string, timeoutMs: number) => Promise<PlaywrightBrowser>
|
|
21
|
+
|
|
22
|
+
/** One fetch's lease: the shared browser plus a context it owns. */
|
|
23
|
+
export interface CdpLease {
|
|
24
|
+
/** The shared connection — close only what the lease owns, never this. */
|
|
25
|
+
browser: PlaywrightBrowser
|
|
26
|
+
/** The fetch-owned isolated context; {@link CdpConnectionPool.release} closes it. */
|
|
27
|
+
context: PlaywrightContext
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** A reusable `connectOverCDP` session handing out per-fetch contexts. */
|
|
31
|
+
export class CdpConnectionPool {
|
|
32
|
+
private browser: PlaywrightBrowser | undefined
|
|
33
|
+
private endpoint = ''
|
|
34
|
+
private connecting: Promise<PlaywrightBrowser> | undefined
|
|
35
|
+
private connectingEndpoint = ''
|
|
36
|
+
/** Bumped by dispose/replace so a settling connect knows it was abandoned. */
|
|
37
|
+
private generation = 0
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* @param connect - opens a connection to an endpoint (the provider's real
|
|
41
|
+
* one resolves the bundled playwright-core; tests inject fakes).
|
|
42
|
+
*/
|
|
43
|
+
constructor(private readonly connect: CdpConnect) {}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Lease a fresh isolated context on the shared connection, connecting (or
|
|
47
|
+
* reconnecting) first if needed. Concurrent first fetches share one
|
|
48
|
+
* connect attempt.
|
|
49
|
+
*
|
|
50
|
+
* @param endpoint - normalized CDP endpoint URL.
|
|
51
|
+
* @param timeoutMs - connect timeout budget.
|
|
52
|
+
* @returns the shared browser and a context the caller must release.
|
|
53
|
+
* @throws whatever the connect function throws when no connection can be
|
|
54
|
+
* established (the provider wraps it in a structured WebError).
|
|
55
|
+
*/
|
|
56
|
+
async acquire(endpoint: string, timeoutMs: number): Promise<CdpLease> {
|
|
57
|
+
const browser = await this.ensure(endpoint, timeoutMs)
|
|
58
|
+
try {
|
|
59
|
+
return { browser, context: await browser.newContext() }
|
|
60
|
+
} catch (error: unknown) {
|
|
61
|
+
// The connection may have died between ensure() and newContext(); one
|
|
62
|
+
// fresh connection attempt, then the error propagates as-is.
|
|
63
|
+
if (this.isLive(browser)) throw error
|
|
64
|
+
this.drop(browser)
|
|
65
|
+
const fresh = await this.ensure(endpoint, timeoutMs)
|
|
66
|
+
return { browser: fresh, context: await fresh.newContext() }
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Close a fetch-owned context. The shared connection stays open for the
|
|
72
|
+
* next fetch.
|
|
73
|
+
* @param lease - the lease whose context goes away.
|
|
74
|
+
*/
|
|
75
|
+
async release(lease: CdpLease): Promise<void> {
|
|
76
|
+
await lease.context.close().catch(() => {})
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Drop the shared connection (plugin teardown or endpoint change). Fetches
|
|
81
|
+
* holding leases keep their contexts until they release them. A connect
|
|
82
|
+
* still in flight is abandoned — its own continuation closes the stray
|
|
83
|
+
* browser (nothing here waits on it, so teardown cannot deadlock).
|
|
84
|
+
*/
|
|
85
|
+
async dispose(): Promise<void> {
|
|
86
|
+
const browser = this.browser
|
|
87
|
+
this.browser = undefined
|
|
88
|
+
this.endpoint = ''
|
|
89
|
+
this.connecting = undefined
|
|
90
|
+
this.connectingEndpoint = ''
|
|
91
|
+
this.generation++
|
|
92
|
+
await browser?.close().catch(() => {})
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/** The shared connection for `endpoint`, connecting or reconnecting as needed. */
|
|
96
|
+
private async ensure(endpoint: string, timeoutMs: number): Promise<PlaywrightBrowser> {
|
|
97
|
+
if (this.browser !== undefined && this.endpoint === endpoint && this.isLive(this.browser)) return this.browser
|
|
98
|
+
if (this.connecting !== undefined && this.connectingEndpoint === endpoint) return this.connecting
|
|
99
|
+
// Dead connection or different endpoint: (re)connect. An in-flight connect
|
|
100
|
+
// to another endpoint is abandoned by the generation bump below — its own
|
|
101
|
+
// continuation closes the stray browser and rejects to its waiter.
|
|
102
|
+
return await this.connectFresh(endpoint, timeoutMs)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Start a connection to `endpoint`, superseding whatever was there. */
|
|
106
|
+
private connectFresh(endpoint: string, timeoutMs: number): Promise<PlaywrightBrowser> {
|
|
107
|
+
const stale = this.browser
|
|
108
|
+
this.browser = undefined
|
|
109
|
+
this.endpoint = ''
|
|
110
|
+
const generation = ++this.generation
|
|
111
|
+
const attempt = (async () => {
|
|
112
|
+
const browser = await this.connect(endpoint, timeoutMs)
|
|
113
|
+
// A dispose/replace won the race while we were connecting: this
|
|
114
|
+
// connection is unwanted — close it and surface the abandonment.
|
|
115
|
+
if (generation !== this.generation) {
|
|
116
|
+
await browser.close().catch(() => {})
|
|
117
|
+
throw new Error('cdp connection abandoned before it attached')
|
|
118
|
+
}
|
|
119
|
+
this.browser = browser
|
|
120
|
+
this.endpoint = endpoint
|
|
121
|
+
this.watch(browser)
|
|
122
|
+
return browser
|
|
123
|
+
})()
|
|
124
|
+
this.connecting = attempt
|
|
125
|
+
this.connectingEndpoint = endpoint
|
|
126
|
+
void attempt.then(
|
|
127
|
+
() => { if (this.connecting === attempt) this.connecting = undefined },
|
|
128
|
+
() => { if (this.connecting === attempt) this.connecting = undefined },
|
|
129
|
+
)
|
|
130
|
+
void stale?.close().catch(() => {})
|
|
131
|
+
return attempt
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Clear the reference when this exact connection reports it went away. */
|
|
135
|
+
private watch(browser: PlaywrightBrowser): void {
|
|
136
|
+
try {
|
|
137
|
+
browser.on?.('disconnected', () => {
|
|
138
|
+
if (this.browser === browser) {
|
|
139
|
+
this.browser = undefined
|
|
140
|
+
this.endpoint = ''
|
|
141
|
+
}
|
|
142
|
+
})
|
|
143
|
+
} catch {
|
|
144
|
+
// a backend that refuses listeners just loses proactive detection
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Forget a connection known to be dead (absent probe = assume dead here). */
|
|
149
|
+
private drop(browser: PlaywrightBrowser): void {
|
|
150
|
+
if (this.browser === browser) {
|
|
151
|
+
this.browser = undefined
|
|
152
|
+
this.endpoint = ''
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
private isLive(browser: PlaywrightBrowser): boolean {
|
|
157
|
+
return browser.isConnected?.() !== false
|
|
158
|
+
}
|
|
159
|
+
}
|
package/src/client/card.tsx
CHANGED
|
@@ -112,6 +112,19 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
|
|
|
112
112
|
onEdit={(text) => { props.edit('denoise', text) }}
|
|
113
113
|
onReset={() => { props.resetField('denoise') }}
|
|
114
114
|
/>
|
|
115
|
+
<ValueField
|
|
116
|
+
id="plugin-config-playwright-concurrency"
|
|
117
|
+
label={t('maxConcurrency')}
|
|
118
|
+
hint={t('maxConcurrencyHint')}
|
|
119
|
+
placeholder={t('maxConcurrencyPlaceholder')}
|
|
120
|
+
overriddenLabel={t('overridden')}
|
|
121
|
+
resetLabel={t('reset')}
|
|
122
|
+
invalidLabel={t('invalidText')}
|
|
123
|
+
disabled={disabled}
|
|
124
|
+
{...state.maxConcurrency}
|
|
125
|
+
onEdit={(text) => { props.edit('maxConcurrency', text) }}
|
|
126
|
+
onReset={() => { props.resetField('maxConcurrency') }}
|
|
127
|
+
/>
|
|
115
128
|
</PluginCard>
|
|
116
129
|
)
|
|
117
130
|
}
|
package/src/client/controller.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import type { SettingsScope } from '@deepseek-ai/dsh-client-runtime/client'
|
|
10
|
-
import { CardForm, checkboxField, radioField, textField } from './form.ts'
|
|
10
|
+
import { CardForm, checkboxField, numberField, radioField, textField } from './form.ts'
|
|
11
11
|
import type { CardShell, CardFieldState, CardActions, SnapshotStore } from './form.ts'
|
|
12
12
|
|
|
13
13
|
/**
|
|
@@ -26,6 +26,8 @@ export interface PlaywrightSettings {
|
|
|
26
26
|
cdpEndpoint?: string
|
|
27
27
|
/** Whether the Readability + DOMPurify pipeline runs. */
|
|
28
28
|
denoise?: boolean
|
|
29
|
+
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
30
|
+
maxConcurrency?: number
|
|
29
31
|
}
|
|
30
32
|
|
|
31
33
|
/** What the Playwright card renders. */
|
|
@@ -38,6 +40,8 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
38
40
|
cdpEndpoint: CardFieldState
|
|
39
41
|
/** Denoise checkbox (draft 'true'/'false'). */
|
|
40
42
|
denoise: CardFieldState
|
|
43
|
+
/** Concurrency input (draft decimal integer). */
|
|
44
|
+
maxConcurrency: CardFieldState
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
/** The registration-side face the card's slot entry injects. */
|
|
@@ -64,6 +68,7 @@ export class PlaywrightCardController {
|
|
|
64
68
|
textField('playwrightPath'),
|
|
65
69
|
textField('cdpEndpoint'),
|
|
66
70
|
checkboxField('denoise'),
|
|
71
|
+
numberField('maxConcurrency', 1, 200),
|
|
67
72
|
],
|
|
68
73
|
)
|
|
69
74
|
this.store = this.form.bind(() => this.projection())
|
|
@@ -76,6 +81,7 @@ export class PlaywrightCardController {
|
|
|
76
81
|
playwrightPath: this.form.field('playwrightPath'),
|
|
77
82
|
cdpEndpoint: this.form.field('cdpEndpoint'),
|
|
78
83
|
denoise: this.form.field('denoise'),
|
|
84
|
+
maxConcurrency: this.form.field('maxConcurrency'),
|
|
79
85
|
}
|
|
80
86
|
}
|
|
81
87
|
|
package/src/client/form.ts
CHANGED
|
@@ -140,6 +140,29 @@ export function checkboxField(field: string): CardFieldSpec {
|
|
|
140
140
|
}
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
/**
|
|
144
|
+
* An integer field over an inclusive range. The draft is the decimal string;
|
|
145
|
+
* an empty draft clears the field (re-inherit the schema default), and
|
|
146
|
+
* anything but an integer in range blocks the save as invalid.
|
|
147
|
+
* @param field - field name inside the namespace section.
|
|
148
|
+
* @param min - smallest accepted integer (inclusive).
|
|
149
|
+
* @param max - largest accepted integer (inclusive).
|
|
150
|
+
* @returns the field's conversion spec.
|
|
151
|
+
*/
|
|
152
|
+
export function numberField(field: string, min: number, max: number): CardFieldSpec {
|
|
153
|
+
return {
|
|
154
|
+
field,
|
|
155
|
+
format: value => typeof value === 'number' && Number.isInteger(value) ? String(value) : '',
|
|
156
|
+
parse: (text) => {
|
|
157
|
+
const trimmed = text.trim()
|
|
158
|
+
if (trimmed === '') return { kind: 'clear' }
|
|
159
|
+
if (!/^\d+$/.test(trimmed)) return undefined
|
|
160
|
+
const value = Number(trimmed)
|
|
161
|
+
return value >= min && value <= max ? { kind: 'set', value } : undefined
|
|
162
|
+
},
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
143
166
|
/**
|
|
144
167
|
* Stages the Playwright card's edits over one settings namespace and writes
|
|
145
168
|
* them on save. Publishes through a snapshot store because slot components
|
package/src/client/locales.ts
CHANGED
|
@@ -12,6 +12,7 @@ export type PlaywrightCardLocaleKey =
|
|
|
12
12
|
| 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder'
|
|
13
13
|
| 'cdpEndpoint' | 'cdpEndpointHint'
|
|
14
14
|
| 'denoise' | 'denoiseHint'
|
|
15
|
+
| 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
|
|
15
16
|
| 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse'
|
|
16
17
|
| 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText'
|
|
17
18
|
|
|
@@ -38,6 +39,9 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
38
39
|
cdpEndpointHint: 'host:port or http(s)/ws URL. Leave blank for 127.0.0.1:9222.',
|
|
39
40
|
denoise: 'Enable the denoise algorithm',
|
|
40
41
|
denoiseHint: 'Readability + DOMPurify strip nav bars, sidebars, footers, and ads before converting to markdown.',
|
|
42
|
+
maxConcurrency: 'Max concurrent fetches',
|
|
43
|
+
maxConcurrencyHint: 'How many pages may render at once (1–200). Blank = auto: 4 local browsers, 50 CDP tabs in the remote browser.',
|
|
44
|
+
maxConcurrencyPlaceholder: '(auto: local 4 / CDP 50)',
|
|
41
45
|
overridden: 'Overridden',
|
|
42
46
|
reset: 'Reset to default',
|
|
43
47
|
readOnly: 'This deployment stores settings read-only.',
|
|
@@ -67,6 +71,9 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
67
71
|
cdpEndpointHint: 'host:port 或 http(s)/ws 地址;留空默认 127.0.0.1:9222。',
|
|
68
72
|
denoise: '启用降噪算法',
|
|
69
73
|
denoiseHint: '使用 Readability + DOMPurify 清洗导航栏、侧边栏、页脚与贴片广告后再转为 Markdown。',
|
|
74
|
+
maxConcurrency: '最大并发抓取数',
|
|
75
|
+
maxConcurrencyHint: '同时渲染的页面上限(1–200);留空自动:本地 4 个浏览器,CDP 在远端浏览器里 50 个标签页。',
|
|
76
|
+
maxConcurrencyPlaceholder: '(自动:本地 4 / CDP 50)',
|
|
70
77
|
overridden: '已覆盖',
|
|
71
78
|
reset: '恢复默认',
|
|
72
79
|
readOnly: '本部署的设置为只读。',
|
package/src/config.ts
CHANGED
|
@@ -12,6 +12,24 @@ import z from '@deepseek-ai/schemastery'
|
|
|
12
12
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
13
13
|
export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* Default concurrency for the local backend: each slot launches a whole
|
|
17
|
+
* Chromium, so the default stays frugal; parallel `web_fetch` bursts stop
|
|
18
|
+
* queueing without a browser farm per fetch.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_MAX_CONCURRENCY_LOCAL = 4
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Default concurrency for the CDP backend: the browser already exists, each
|
|
24
|
+
* fetch only opens a tab (isolated context) inside it over one shared
|
|
25
|
+
* connection — so the budget behaves like "max concurrent tabs" and defaults
|
|
26
|
+
* high.
|
|
27
|
+
*/
|
|
28
|
+
export const DEFAULT_MAX_CONCURRENCY_CDP = 50
|
|
29
|
+
|
|
30
|
+
/** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
|
|
31
|
+
export const MAX_CONCURRENCY_CEILING = 200
|
|
32
|
+
|
|
15
33
|
/** Which browser backend serves a fetch. */
|
|
16
34
|
export type PlaywrightBackend = 'local' | 'cdp'
|
|
17
35
|
|
|
@@ -29,6 +47,12 @@ export interface Config {
|
|
|
29
47
|
cdpEndpoint?: string
|
|
30
48
|
/** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
|
|
31
49
|
denoise?: boolean
|
|
50
|
+
/**
|
|
51
|
+
* How many fetches may render at once. Blank = backend default (4 for
|
|
52
|
+
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
53
|
+
* the already-running remote browser).
|
|
54
|
+
*/
|
|
55
|
+
maxConcurrency?: number
|
|
32
56
|
}
|
|
33
57
|
|
|
34
58
|
export const Config: z<Config> = z.object({
|
|
@@ -39,10 +63,30 @@ export const Config: z<Config> = z.object({
|
|
|
39
63
|
playwrightPath: z.string().default(''),
|
|
40
64
|
cdpEndpoint: z.string().default(''),
|
|
41
65
|
denoise: z.boolean().default(true),
|
|
66
|
+
// Optional on purpose: the effective default depends on `backend`, which a
|
|
67
|
+
// static schema default cannot express.
|
|
68
|
+
maxConcurrency: z.number().step(1).min(1).max(MAX_CONCURRENCY_CEILING),
|
|
42
69
|
})
|
|
43
70
|
|
|
44
|
-
/**
|
|
45
|
-
|
|
71
|
+
/**
|
|
72
|
+
* Complete config after schemastery applies the field defaults it owns.
|
|
73
|
+
* `maxConcurrency` stays optional — {@link effectiveMaxConcurrency} resolves
|
|
74
|
+
* it against the backend.
|
|
75
|
+
*/
|
|
76
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & { maxConcurrency?: number }
|
|
77
|
+
|
|
78
|
+
/**
|
|
79
|
+
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
80
|
+
* wins; otherwise the backend default (local launches browsers, CDP only
|
|
81
|
+
* opens tabs, so their budgets differ by an order of magnitude).
|
|
82
|
+
*
|
|
83
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
84
|
+
* @returns the effective limit for the semaphore.
|
|
85
|
+
*/
|
|
86
|
+
export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number {
|
|
87
|
+
if (typeof config.maxConcurrency === 'number') return config.maxConcurrency
|
|
88
|
+
return config.backend === 'cdp' ? DEFAULT_MAX_CONCURRENCY_CDP : DEFAULT_MAX_CONCURRENCY_LOCAL
|
|
89
|
+
}
|
|
46
90
|
|
|
47
91
|
/**
|
|
48
92
|
* Normalize a configured CDP endpoint for `chromium.connectOverCDP`:
|
package/src/index.ts
CHANGED
|
@@ -19,6 +19,8 @@ import { PlaywrightFetchProvider } from './provider.ts'
|
|
|
19
19
|
|
|
20
20
|
export { Config, DEFAULT_CDP_ENDPOINT, normalizeCdpEndpoint } from './config.ts'
|
|
21
21
|
export type { Config as PlaywrightFetchConfig, PlaywrightBackend, ResolvedConfig } from './config.ts'
|
|
22
|
+
export { CdpConnectionPool } from './cdp-pool.ts'
|
|
23
|
+
export type { CdpConnect, CdpLease } from './cdp-pool.ts'
|
|
22
24
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider } from './provider.ts'
|
|
23
25
|
export { htmlToMarkdown } from './markdown.ts'
|
|
24
26
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts'
|
|
@@ -48,5 +50,9 @@ export function apply(ctx: Context, config: Config): void {
|
|
|
48
50
|
// needs no re-registration.
|
|
49
51
|
onChange: () => {},
|
|
50
52
|
})
|
|
51
|
-
|
|
53
|
+
// The CDP backend holds one shared connection for the provider's lifetime;
|
|
54
|
+
// drop it when this plugin unloads so restarts don't strand sockets.
|
|
55
|
+
const provider = new PlaywrightFetchProvider(() => current())
|
|
56
|
+
ctx.effect(() => () => { void provider.dispose() }, 'dsh-web-fetch-playwright: shared CDP connection')
|
|
57
|
+
ctx.web.registerFetchProvider(provider)
|
|
52
58
|
}
|