dsh-web-fetch-playwright 0.1.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -5
- package/README.zh-CN.md +27 -5
- package/dsh.plugin.json +1 -1
- package/lib/client.js +185 -105
- package/lib/index.js +362 -43
- package/lib/types/cdp-pool.d.ts +100 -0
- package/lib/types/client/card.d.ts +4 -3
- package/lib/types/client/controller.d.ts +12 -0
- package/lib/types/client/fields.d.ts +7 -1
- package/lib/types/client/form.d.ts +10 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +62 -2
- package/lib/types/index.d.ts +4 -2
- package/lib/types/provider.d.ts +40 -10
- package/lib/types/types.d.ts +22 -1
- package/package.json +4 -3
- package/src/cdp-pool.ts +211 -0
- package/src/client/card.tsx +46 -17
- package/src/client/controller.ts +17 -1
- package/src/client/fields.module.css +12 -0
- package/src/client/fields.tsx +9 -3
- package/src/client/form.ts +23 -0
- package/src/client/locales.ts +12 -0
- package/src/config.ts +76 -2
- package/src/index.ts +9 -3
- package/src/provider.ts +220 -47
- package/src/types.ts +22 -1
package/src/client/controller.ts
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import type { SettingsScope } from '@deepseek-ai/dsh-client-runtime/client'
|
|
10
|
-
import { CardForm, checkboxField, radioField, textField } from './form.ts'
|
|
10
|
+
import { CardForm, checkboxField, numberField, radioField, textField } from './form.ts'
|
|
11
11
|
import type { CardShell, CardFieldState, CardActions, SnapshotStore } from './form.ts'
|
|
12
12
|
|
|
13
13
|
/**
|
|
@@ -24,8 +24,16 @@ export interface PlaywrightSettings {
|
|
|
24
24
|
playwrightPath?: string
|
|
25
25
|
/** CDP backend: endpoint (host:port / http(s) / ws). */
|
|
26
26
|
cdpEndpoint?: string
|
|
27
|
+
/**
|
|
28
|
+
* CDP backend: whether fetches share the remote browser's context (its
|
|
29
|
+
* real profile — cookies, localStorage, persistent logins) as tabs, or
|
|
30
|
+
* each use a fresh isolated context. Absent = schema default (true).
|
|
31
|
+
*/
|
|
32
|
+
shareBrowserContext?: boolean
|
|
27
33
|
/** Whether the Readability + DOMPurify pipeline runs. */
|
|
28
34
|
denoise?: boolean
|
|
35
|
+
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
36
|
+
maxConcurrency?: number
|
|
29
37
|
}
|
|
30
38
|
|
|
31
39
|
/** What the Playwright card renders. */
|
|
@@ -36,8 +44,12 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
36
44
|
playwrightPath: CardFieldState
|
|
37
45
|
/** CDP endpoint input. */
|
|
38
46
|
cdpEndpoint: CardFieldState
|
|
47
|
+
/** CDP shared-context checkbox (draft 'true'/'false'). */
|
|
48
|
+
shareBrowserContext: CardFieldState
|
|
39
49
|
/** Denoise checkbox (draft 'true'/'false'). */
|
|
40
50
|
denoise: CardFieldState
|
|
51
|
+
/** Concurrency input (draft decimal integer). */
|
|
52
|
+
maxConcurrency: CardFieldState
|
|
41
53
|
}
|
|
42
54
|
|
|
43
55
|
/** The registration-side face the card's slot entry injects. */
|
|
@@ -63,7 +75,9 @@ export class PlaywrightCardController {
|
|
|
63
75
|
radioField('backend', ['local', 'cdp']),
|
|
64
76
|
textField('playwrightPath'),
|
|
65
77
|
textField('cdpEndpoint'),
|
|
78
|
+
checkboxField('shareBrowserContext'),
|
|
66
79
|
checkboxField('denoise'),
|
|
80
|
+
numberField('maxConcurrency', 1, 200),
|
|
67
81
|
],
|
|
68
82
|
)
|
|
69
83
|
this.store = this.form.bind(() => this.projection())
|
|
@@ -75,7 +89,9 @@ export class PlaywrightCardController {
|
|
|
75
89
|
backend: this.form.field('backend'),
|
|
76
90
|
playwrightPath: this.form.field('playwrightPath'),
|
|
77
91
|
cdpEndpoint: this.form.field('cdpEndpoint'),
|
|
92
|
+
shareBrowserContext: this.form.field('shareBrowserContext'),
|
|
78
93
|
denoise: this.form.field('denoise'),
|
|
94
|
+
maxConcurrency: this.form.field('maxConcurrency'),
|
|
79
95
|
}
|
|
80
96
|
}
|
|
81
97
|
|
|
@@ -132,6 +132,12 @@
|
|
|
132
132
|
color: var(--dsw-alias-label-tertiary);
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
+
/* Embedded fields (inside a radio option): align the hint under the label
|
|
136
|
+
text column (checkbox 16px + 10px gap) rather than the box. */
|
|
137
|
+
.hintIndented {
|
|
138
|
+
margin-left: 26px;
|
|
139
|
+
}
|
|
140
|
+
|
|
135
141
|
/* ── radio group ──────────────────────────────────────────────────────── */
|
|
136
142
|
|
|
137
143
|
.radioGroup {
|
|
@@ -202,6 +208,12 @@
|
|
|
202
208
|
margin-left: 24px;
|
|
203
209
|
}
|
|
204
210
|
|
|
211
|
+
/* Stacked nested fields (endpoint input + shared-context checkbox) keep the
|
|
212
|
+
card's field rhythm between them. */
|
|
213
|
+
.radioContent > * + * {
|
|
214
|
+
margin-top: 12px;
|
|
215
|
+
}
|
|
216
|
+
|
|
205
217
|
.radioContent input[type='text']:enabled {
|
|
206
218
|
cursor: text;
|
|
207
219
|
}
|
package/src/client/fields.tsx
CHANGED
|
@@ -143,7 +143,11 @@ export function RadioGroupField(props: {
|
|
|
143
143
|
)
|
|
144
144
|
}
|
|
145
145
|
|
|
146
|
-
/**
|
|
146
|
+
/**
|
|
147
|
+
* A staged checkbox: the denoise toggle and the CDP shared-context toggle.
|
|
148
|
+
* An absent stored value formats as '' — callers render the schema default
|
|
149
|
+
* (denoise on, context shared).
|
|
150
|
+
*/
|
|
147
151
|
export function CheckboxField(props: {
|
|
148
152
|
id: string
|
|
149
153
|
label: string
|
|
@@ -155,9 +159,11 @@ export function CheckboxField(props: {
|
|
|
155
159
|
disabled: boolean
|
|
156
160
|
onEdit: (text: string) => void
|
|
157
161
|
onReset: () => void
|
|
162
|
+
/** Render inside a radio option: no card-level chrome, hint under the label. */
|
|
163
|
+
embedded?: boolean
|
|
158
164
|
}) {
|
|
159
165
|
return (
|
|
160
|
-
<div className={css.field}>
|
|
166
|
+
<div className={props.embedded ? css.fieldEmbedded : css.field}>
|
|
161
167
|
<div className={css.checkboxRow}>
|
|
162
168
|
<input
|
|
163
169
|
id={props.id}
|
|
@@ -183,7 +189,7 @@ export function CheckboxField(props: {
|
|
|
183
189
|
)
|
|
184
190
|
: null}
|
|
185
191
|
</div>
|
|
186
|
-
<p className={css.hint}>{props.hint}</p>
|
|
192
|
+
<p className={props.embedded ? `${css.hint} ${css.hintIndented}` : css.hint}>{props.hint}</p>
|
|
187
193
|
</div>
|
|
188
194
|
)
|
|
189
195
|
}
|
package/src/client/form.ts
CHANGED
|
@@ -140,6 +140,29 @@ export function checkboxField(field: string): CardFieldSpec {
|
|
|
140
140
|
}
|
|
141
141
|
}
|
|
142
142
|
|
|
143
|
+
/**
|
|
144
|
+
* An integer field over an inclusive range. The draft is the decimal string;
|
|
145
|
+
* an empty draft clears the field (re-inherit the schema default), and
|
|
146
|
+
* anything but an integer in range blocks the save as invalid.
|
|
147
|
+
* @param field - field name inside the namespace section.
|
|
148
|
+
* @param min - smallest accepted integer (inclusive).
|
|
149
|
+
* @param max - largest accepted integer (inclusive).
|
|
150
|
+
* @returns the field's conversion spec.
|
|
151
|
+
*/
|
|
152
|
+
export function numberField(field: string, min: number, max: number): CardFieldSpec {
|
|
153
|
+
return {
|
|
154
|
+
field,
|
|
155
|
+
format: value => typeof value === 'number' && Number.isInteger(value) ? String(value) : '',
|
|
156
|
+
parse: (text) => {
|
|
157
|
+
const trimmed = text.trim()
|
|
158
|
+
if (trimmed === '') return { kind: 'clear' }
|
|
159
|
+
if (!/^\d+$/.test(trimmed)) return undefined
|
|
160
|
+
const value = Number(trimmed)
|
|
161
|
+
return value >= min && value <= max ? { kind: 'set', value } : undefined
|
|
162
|
+
},
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
|
|
143
166
|
/**
|
|
144
167
|
* Stages the Playwright card's edits over one settings namespace and writes
|
|
145
168
|
* them on save. Publishes through a snapshot store because slot components
|
package/src/client/locales.ts
CHANGED
|
@@ -11,7 +11,9 @@ export type PlaywrightCardLocaleKey =
|
|
|
11
11
|
| 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint'
|
|
12
12
|
| 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder'
|
|
13
13
|
| 'cdpEndpoint' | 'cdpEndpointHint'
|
|
14
|
+
| 'shareBrowserContext' | 'shareBrowserContextHint'
|
|
14
15
|
| 'denoise' | 'denoiseHint'
|
|
16
|
+
| 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
|
|
15
17
|
| 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse'
|
|
16
18
|
| 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText'
|
|
17
19
|
|
|
@@ -36,8 +38,13 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
36
38
|
playwrightPathPlaceholder: '(auto: playwright from $PATH)',
|
|
37
39
|
cdpEndpoint: 'CDP endpoint',
|
|
38
40
|
cdpEndpointHint: 'host:port or http(s)/ws URL. Leave blank for 127.0.0.1:9222.',
|
|
41
|
+
shareBrowserContext: 'Share the browser context (profile logins)',
|
|
42
|
+
shareBrowserContextHint: 'Each fetch opens a tab in the remote browser\u2019s real profile \u2014 cookies and localStorage are shared and its persistent logins apply. Unchecked: every fetch uses a fresh isolated context.',
|
|
39
43
|
denoise: 'Enable the denoise algorithm',
|
|
40
44
|
denoiseHint: 'Readability + DOMPurify strip nav bars, sidebars, footers, and ads before converting to markdown.',
|
|
45
|
+
maxConcurrency: 'Max concurrent fetches',
|
|
46
|
+
maxConcurrencyHint: 'How many pages may render at once (1–200). Blank = auto: 4 local browsers, 50 CDP tabs in the remote browser.',
|
|
47
|
+
maxConcurrencyPlaceholder: '(auto: local 4 / CDP 50)',
|
|
41
48
|
overridden: 'Overridden',
|
|
42
49
|
reset: 'Reset to default',
|
|
43
50
|
readOnly: 'This deployment stores settings read-only.',
|
|
@@ -65,8 +72,13 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
65
72
|
playwrightPathPlaceholder: '(自动:按 $PATH 查找 playwright)',
|
|
66
73
|
cdpEndpoint: 'CDP 地址',
|
|
67
74
|
cdpEndpointHint: 'host:port 或 http(s)/ws 地址;留空默认 127.0.0.1:9222。',
|
|
75
|
+
shareBrowserContext: '共享浏览器上下文(复用登录态)',
|
|
76
|
+
shareBrowserContextHint: '每次抓取在远端浏览器的真实 profile 里开一个标签页:共享 cookie 与 localStorage、复用已登录会话,页面会看到你的登录身份;取消勾选则每次抓取使用全新隔离上下文。',
|
|
68
77
|
denoise: '启用降噪算法',
|
|
69
78
|
denoiseHint: '使用 Readability + DOMPurify 清洗导航栏、侧边栏、页脚与贴片广告后再转为 Markdown。',
|
|
79
|
+
maxConcurrency: '最大并发抓取数',
|
|
80
|
+
maxConcurrencyHint: '同时渲染的页面上限(1–200);留空自动:本地 4 个浏览器,CDP 在远端浏览器里 50 个标签页。',
|
|
81
|
+
maxConcurrencyPlaceholder: '(自动:本地 4 / CDP 50)',
|
|
70
82
|
overridden: '已覆盖',
|
|
71
83
|
reset: '恢复默认',
|
|
72
84
|
readOnly: '本部署的设置为只读。',
|
package/src/config.ts
CHANGED
|
@@ -12,9 +12,33 @@ import z from '@deepseek-ai/schemastery'
|
|
|
12
12
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
13
13
|
export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
|
|
14
14
|
|
|
15
|
+
/**
|
|
16
|
+
* Default concurrency for the local backend: each slot launches a whole
|
|
17
|
+
* Chromium, so the default stays frugal; parallel `web_fetch` bursts stop
|
|
18
|
+
* queueing without a browser farm per fetch.
|
|
19
|
+
*/
|
|
20
|
+
export const DEFAULT_MAX_CONCURRENCY_LOCAL = 4
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Default concurrency for the CDP backend: the browser already exists, each
|
|
24
|
+
* fetch only opens a tab (isolated context) inside it over one shared
|
|
25
|
+
* connection — so the budget behaves like "max concurrent tabs" and defaults
|
|
26
|
+
* high.
|
|
27
|
+
*/
|
|
28
|
+
export const DEFAULT_MAX_CONCURRENCY_CDP = 50
|
|
29
|
+
|
|
30
|
+
/** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
|
|
31
|
+
export const MAX_CONCURRENCY_CEILING = 200
|
|
32
|
+
|
|
15
33
|
/** Which browser backend serves a fetch. */
|
|
16
34
|
export type PlaywrightBackend = 'local' | 'cdp'
|
|
17
35
|
|
|
36
|
+
/**
|
|
37
|
+
* How the CDP backend scopes a fetch: a throwaway isolated context, or a tab
|
|
38
|
+
* in the remote browser's real profile (default context).
|
|
39
|
+
*/
|
|
40
|
+
export type CdpContextMode = 'isolated' | 'profile'
|
|
41
|
+
|
|
18
42
|
/** Plugin config: everything optional — the schema fills the defaults. */
|
|
19
43
|
export interface Config {
|
|
20
44
|
/** Backend selector: local Playwright launch or a remote CDP endpoint. */
|
|
@@ -27,8 +51,22 @@ export interface Config {
|
|
|
27
51
|
playwrightPath?: string
|
|
28
52
|
/** CDP backend: `host:port`, `http(s)://…`, or `ws(s)://…`. Empty = default. */
|
|
29
53
|
cdpEndpoint?: string
|
|
54
|
+
/**
|
|
55
|
+
* CDP backend only. `true` (default): each fetch is a tab in the remote
|
|
56
|
+
* browser's default context — the real profile — so cookies/localStorage
|
|
57
|
+
* come from (and are written back to) it and its persistent logins apply.
|
|
58
|
+
* `false`: a fresh incognito-like isolated context per fetch, no shared
|
|
59
|
+
* state. Meaningless for the local backend (collapses to isolated).
|
|
60
|
+
*/
|
|
61
|
+
shareBrowserContext?: boolean
|
|
30
62
|
/** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
|
|
31
63
|
denoise?: boolean
|
|
64
|
+
/**
|
|
65
|
+
* How many fetches may render at once. Blank = backend default (4 for
|
|
66
|
+
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
67
|
+
* the already-running remote browser).
|
|
68
|
+
*/
|
|
69
|
+
maxConcurrency?: number
|
|
32
70
|
}
|
|
33
71
|
|
|
34
72
|
export const Config: z<Config> = z.object({
|
|
@@ -38,11 +76,47 @@ export const Config: z<Config> = z.object({
|
|
|
38
76
|
backend: z.union([z.const('local'), z.const('cdp')]).default('local'),
|
|
39
77
|
playwrightPath: z.string().default(''),
|
|
40
78
|
cdpEndpoint: z.string().default(''),
|
|
79
|
+
shareBrowserContext: z.boolean().default(true),
|
|
41
80
|
denoise: z.boolean().default(true),
|
|
81
|
+
// Optional on purpose: the effective default depends on `backend`, which a
|
|
82
|
+
// static schema default cannot express.
|
|
83
|
+
maxConcurrency: z.number().step(1).min(1).max(MAX_CONCURRENCY_CEILING),
|
|
42
84
|
})
|
|
43
85
|
|
|
44
|
-
/**
|
|
45
|
-
|
|
86
|
+
/**
|
|
87
|
+
* Complete config after schemastery applies the field defaults it owns.
|
|
88
|
+
* `maxConcurrency` stays optional — {@link effectiveMaxConcurrency} resolves
|
|
89
|
+
* it against the backend.
|
|
90
|
+
*/
|
|
91
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & { maxConcurrency?: number }
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
95
|
+
* wins; otherwise the backend default (local launches browsers, CDP only
|
|
96
|
+
* opens tabs, so their budgets differ by an order of magnitude).
|
|
97
|
+
*
|
|
98
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
99
|
+
* @returns the effective limit for the semaphore.
|
|
100
|
+
*/
|
|
101
|
+
export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number {
|
|
102
|
+
if (typeof config.maxConcurrency === 'number') return config.maxConcurrency
|
|
103
|
+
return config.backend === 'cdp' ? DEFAULT_MAX_CONCURRENCY_CDP : DEFAULT_MAX_CONCURRENCY_LOCAL
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
108
|
+
* browser's real profile) requires the CDP backend — a local launch has no
|
|
109
|
+
* meaningful shared profile — and the checkbox (which defaults on, so an
|
|
110
|
+
* absent value reads as `true`, mirroring the schema default); everything
|
|
111
|
+
* else collapses to isolated.
|
|
112
|
+
*
|
|
113
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
114
|
+
* @returns the mode the pool acquires its lease with.
|
|
115
|
+
*/
|
|
116
|
+
export function effectiveContextMode(config: Pick<Config, 'backend' | 'shareBrowserContext'>): CdpContextMode {
|
|
117
|
+
if (config.backend === 'cdp' && config.shareBrowserContext !== false) return 'profile'
|
|
118
|
+
return 'isolated'
|
|
119
|
+
}
|
|
46
120
|
|
|
47
121
|
/**
|
|
48
122
|
* Normalize a configured CDP endpoint for `chromium.connectOverCDP`:
|
package/src/index.ts
CHANGED
|
@@ -17,8 +17,10 @@ import { Config } from './config.ts'
|
|
|
17
17
|
import type { ResolvedConfig } from './config.ts'
|
|
18
18
|
import { PlaywrightFetchProvider } from './provider.ts'
|
|
19
19
|
|
|
20
|
-
export { Config, DEFAULT_CDP_ENDPOINT, normalizeCdpEndpoint } from './config.ts'
|
|
21
|
-
export type { Config as PlaywrightFetchConfig, PlaywrightBackend, ResolvedConfig } from './config.ts'
|
|
20
|
+
export { Config, DEFAULT_CDP_ENDPOINT, effectiveContextMode, normalizeCdpEndpoint } from './config.ts'
|
|
21
|
+
export type { Config as PlaywrightFetchConfig, CdpContextMode, PlaywrightBackend, ResolvedConfig } from './config.ts'
|
|
22
|
+
export { CdpConnectionPool } from './cdp-pool.ts'
|
|
23
|
+
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
|
|
22
24
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider } from './provider.ts'
|
|
23
25
|
export { htmlToMarkdown } from './markdown.ts'
|
|
24
26
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts'
|
|
@@ -48,5 +50,9 @@ export function apply(ctx: Context, config: Config): void {
|
|
|
48
50
|
// needs no re-registration.
|
|
49
51
|
onChange: () => {},
|
|
50
52
|
})
|
|
51
|
-
|
|
53
|
+
// The CDP backend holds one shared connection for the provider's lifetime;
|
|
54
|
+
// drop it when this plugin unloads so restarts don't strand sockets.
|
|
55
|
+
const provider = new PlaywrightFetchProvider(() => current())
|
|
56
|
+
ctx.effect(() => () => { void provider.dispose() }, 'dsh-web-fetch-playwright: shared CDP connection')
|
|
57
|
+
ctx.web.registerFetchProvider(provider)
|
|
52
58
|
}
|