dsh-web-fetch-playwright 0.2.0 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -132,6 +132,12 @@
132
132
  color: var(--dsw-alias-label-tertiary);
133
133
  }
134
134
 
135
+ /* Embedded fields (inside a radio option): align the hint under the label
136
+ text column (checkbox 16px + 10px gap) rather than the box. */
137
+ .hintIndented {
138
+ margin-left: 26px;
139
+ }
140
+
135
141
  /* ── radio group ──────────────────────────────────────────────────────── */
136
142
 
137
143
  .radioGroup {
@@ -202,6 +208,12 @@
202
208
  margin-left: 24px;
203
209
  }
204
210
 
211
+ /* Stacked nested fields (endpoint input + shared-context checkbox) keep the
212
+ card's field rhythm between them. */
213
+ .radioContent > * + * {
214
+ margin-top: 12px;
215
+ }
216
+
205
217
  .radioContent input[type='text']:enabled {
206
218
  cursor: text;
207
219
  }
@@ -143,7 +143,11 @@ export function RadioGroupField(props: {
143
143
  )
144
144
  }
145
145
 
146
- /** A staged checkbox: the denoise toggle. An absent stored value defaults to on. */
146
+ /**
147
+ * A staged checkbox: the denoise toggle and the CDP shared-context toggle.
148
+ * An absent stored value formats as '' — callers render the schema default
149
+ * (denoise on, context shared).
150
+ */
147
151
  export function CheckboxField(props: {
148
152
  id: string
149
153
  label: string
@@ -155,9 +159,11 @@ export function CheckboxField(props: {
155
159
  disabled: boolean
156
160
  onEdit: (text: string) => void
157
161
  onReset: () => void
162
+ /** Render inside a radio option: no card-level chrome, hint under the label. */
163
+ embedded?: boolean
158
164
  }) {
159
165
  return (
160
- <div className={css.field}>
166
+ <div className={props.embedded ? css.fieldEmbedded : css.field}>
161
167
  <div className={css.checkboxRow}>
162
168
  <input
163
169
  id={props.id}
@@ -183,7 +189,7 @@ export function CheckboxField(props: {
183
189
  )
184
190
  : null}
185
191
  </div>
186
- <p className={css.hint}>{props.hint}</p>
192
+ <p className={props.embedded ? `${css.hint} ${css.hintIndented}` : css.hint}>{props.hint}</p>
187
193
  </div>
188
194
  )
189
195
  }
@@ -11,6 +11,7 @@ export type PlaywrightCardLocaleKey =
11
11
  | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint'
12
12
  | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder'
13
13
  | 'cdpEndpoint' | 'cdpEndpointHint'
14
+ | 'shareBrowserContext' | 'shareBrowserContextHint'
14
15
  | 'denoise' | 'denoiseHint'
15
16
  | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
16
17
  | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse'
@@ -37,6 +38,8 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
37
38
  playwrightPathPlaceholder: '(auto: playwright from $PATH)',
38
39
  cdpEndpoint: 'CDP endpoint',
39
40
  cdpEndpointHint: 'host:port or http(s)/ws URL. Leave blank for 127.0.0.1:9222.',
41
+ shareBrowserContext: 'Share the browser context (profile logins)',
42
+ shareBrowserContextHint: 'Each fetch opens a tab in the remote browser\u2019s real profile \u2014 cookies and localStorage are shared and its persistent logins apply. Unchecked: every fetch uses a fresh isolated context.',
40
43
  denoise: 'Enable the denoise algorithm',
41
44
  denoiseHint: 'Readability + DOMPurify strip nav bars, sidebars, footers, and ads before converting to markdown.',
42
45
  maxConcurrency: 'Max concurrent fetches',
@@ -69,6 +72,8 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
69
72
  playwrightPathPlaceholder: '(自动:按 $PATH 查找 playwright)',
70
73
  cdpEndpoint: 'CDP 地址',
71
74
  cdpEndpointHint: 'host:port 或 http(s)/ws 地址;留空默认 127.0.0.1:9222。',
75
+ shareBrowserContext: '共享浏览器上下文(复用登录态)',
76
+ shareBrowserContextHint: '每次抓取在远端浏览器的真实 profile 里开一个标签页:共享 cookie 与 localStorage、复用已登录会话,页面会看到你的登录身份;取消勾选则每次抓取使用全新隔离上下文。',
72
77
  denoise: '启用降噪算法',
73
78
  denoiseHint: '使用 Readability + DOMPurify 清洗导航栏、侧边栏、页脚与贴片广告后再转为 Markdown。',
74
79
  maxConcurrency: '最大并发抓取数',
package/src/config.ts CHANGED
@@ -33,6 +33,12 @@ export const MAX_CONCURRENCY_CEILING = 200
33
33
  /** Which browser backend serves a fetch. */
34
34
  export type PlaywrightBackend = 'local' | 'cdp'
35
35
 
36
+ /**
37
+ * How the CDP backend scopes a fetch: a throwaway isolated context, or a tab
38
+ * in the remote browser's real profile (default context).
39
+ */
40
+ export type CdpContextMode = 'isolated' | 'profile'
41
+
36
42
  /** Plugin config: everything optional — the schema fills the defaults. */
37
43
  export interface Config {
38
44
  /** Backend selector: local Playwright launch or a remote CDP endpoint. */
@@ -45,6 +51,14 @@ export interface Config {
45
51
  playwrightPath?: string
46
52
  /** CDP backend: `host:port`, `http(s)://…`, or `ws(s)://…`. Empty = default. */
47
53
  cdpEndpoint?: string
54
+ /**
55
+ * CDP backend only. `true` (default): each fetch is a tab in the remote
56
+ * browser's default context — the real profile — so cookies/localStorage
57
+ * come from (and are written back to) it and its persistent logins apply.
58
+ * `false`: a fresh incognito-like isolated context per fetch, no shared
59
+ * state. Meaningless for the local backend (collapses to isolated).
60
+ */
61
+ shareBrowserContext?: boolean
48
62
  /** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
49
63
  denoise?: boolean
50
64
  /**
@@ -62,6 +76,7 @@ export const Config: z<Config> = z.object({
62
76
  backend: z.union([z.const('local'), z.const('cdp')]).default('local'),
63
77
  playwrightPath: z.string().default(''),
64
78
  cdpEndpoint: z.string().default(''),
79
+ shareBrowserContext: z.boolean().default(true),
65
80
  denoise: z.boolean().default(true),
66
81
  // Optional on purpose: the effective default depends on `backend`, which a
67
82
  // static schema default cannot express.
@@ -88,6 +103,21 @@ export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxCon
88
103
  return config.backend === 'cdp' ? DEFAULT_MAX_CONCURRENCY_CDP : DEFAULT_MAX_CONCURRENCY_LOCAL
89
104
  }
90
105
 
106
+ /**
107
+ * The context mode a fetch actually runs with: profile (a tab in the remote
108
+ * browser's real profile) requires the CDP backend — a local launch has no
109
+ * meaningful shared profile — and the checkbox (which defaults on, so an
110
+ * absent value reads as `true`, mirroring the schema default); everything
111
+ * else collapses to isolated.
112
+ *
113
+ * @param config - the resolved settings section (or any partial of it).
114
+ * @returns the mode the pool acquires its lease with.
115
+ */
116
+ export function effectiveContextMode(config: Pick<Config, 'backend' | 'shareBrowserContext'>): CdpContextMode {
117
+ if (config.backend === 'cdp' && config.shareBrowserContext !== false) return 'profile'
118
+ return 'isolated'
119
+ }
120
+
91
121
  /**
92
122
  * Normalize a configured CDP endpoint for `chromium.connectOverCDP`:
93
123
  * blank becomes the default loopback endpoint; `wss?://` and `https?://`
package/src/index.ts CHANGED
@@ -17,10 +17,10 @@ import { Config } from './config.ts'
17
17
  import type { ResolvedConfig } from './config.ts'
18
18
  import { PlaywrightFetchProvider } from './provider.ts'
19
19
 
20
- export { Config, DEFAULT_CDP_ENDPOINT, normalizeCdpEndpoint } from './config.ts'
21
- export type { Config as PlaywrightFetchConfig, PlaywrightBackend, ResolvedConfig } from './config.ts'
20
+ export { Config, DEFAULT_CDP_ENDPOINT, effectiveContextMode, normalizeCdpEndpoint } from './config.ts'
21
+ export type { Config as PlaywrightFetchConfig, CdpContextMode, PlaywrightBackend, ResolvedConfig } from './config.ts'
22
22
  export { CdpConnectionPool } from './cdp-pool.ts'
23
- export type { CdpConnect, CdpLease } from './cdp-pool.ts'
23
+ export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
24
24
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider } from './provider.ts'
25
25
  export { htmlToMarkdown } from './markdown.ts'
26
26
  export type { DenoiseMode, DenoiseResult } from './markdown.ts'
package/src/provider.ts CHANGED
@@ -6,30 +6,33 @@
6
6
  *
7
7
  * Lifecycle: the local backend launches a browser per fetch and closes it —
8
8
  * nothing outlives the call. The CDP backend keeps ONE shared connection to
9
- * the remote browser for the provider's lifetime; each fetch only opens an
10
- * isolated context (a tab) inside it and closes that on completion, so the
11
- * concurrency cap counts tabs, not browsers. Either way, an aborted signal
12
- * closes the fetch's context. Concurrency is capped at the `maxConcurrency`
13
- * setting (explicit, or the backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL}
14
- * browsers / {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait
15
- * briefly in a queue and fail fast (rather than hang until abort) when no
16
- * slot frees.
9
+ * the remote browser for the provider's lifetime; each fetch only opens a
10
+ * page (tab) inside it and closes that on completion — in a throwaway
11
+ * isolated context, or (the default, `shareBrowserContext`) in the remote
12
+ * browser's default context so its profile, cookies, and persistent logins
13
+ * apply; that default context is never closed. The concurrency cap counts
14
+ * tabs, not browsers. Either way, an aborted signal closes the fetch's page.
15
+ * Concurrency is capped at the `maxConcurrency` setting (explicit, or the
16
+ * backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL} browsers /
17
+ * {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
18
+ * a queue and fail fast (rather than hang until abort) when no slot frees.
17
19
  *
18
20
  * Private-network and SSRF protection is not implemented (same stance as the
19
21
  * shipped HTTP provider); a page this provider can reach is whatever the
20
- * browser can reach.
22
+ * browser can reach. Profile mode additionally acts WITH the remote
23
+ * browser's logged-in sessions (see the README's risk notes).
21
24
  *
22
25
  * @module dsh-web-fetch-playwright/provider
23
26
  */
24
27
 
25
28
  import { WebError } from '@deepseek-ai/dsh-web'
26
29
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web'
27
- import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
30
+ import { DEFAULT_MAX_CONCURRENCY_CDP, DEFAULT_MAX_CONCURRENCY_LOCAL, effectiveContextMode, effectiveMaxConcurrency, normalizeCdpEndpoint } from './config.ts'
28
31
  import type { ResolvedConfig } from './config.ts'
29
32
  import { CdpConnectionPool } from './cdp-pool.ts'
30
33
  import { htmlToMarkdown } from './markdown.ts'
31
34
  import { resolveCdpBackend, resolvePlaywrightBackend } from './playwright-resolve.ts'
32
- import type { PlaywrightBrowser, PlaywrightContext } from './types.ts'
35
+ import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage, PlaywrightRoute } from './types.ts'
33
36
 
34
37
  /** Stable id this provider registers under (the bundle patch pins it). */
35
38
  export const PLAYWRIGHT_FETCH_PROVIDER_ID = 'playwright'
@@ -65,15 +68,20 @@ const SETTLE_MS = 5_000
65
68
  const CLOSE_GRACE_MS = 2_000
66
69
 
67
70
  /**
68
- * One render session: a browser and the context this fetch owns. Local
69
- * backends own the browser too; CDP sessions ride the shared connection and
70
- * only their context closes at the end.
71
+ * One render session: a browser, the context this fetch works in, and the
72
+ * tab it owns. Local backends own the browser too; CDP sessions ride the
73
+ * shared connection, and in `profile` mode the context is the remote
74
+ * browser's default context — shared, never closed, only the tab is.
71
75
  */
72
76
  export interface BrowserSession {
73
77
  browser: PlaywrightBrowser
74
78
  context: PlaywrightContext
79
+ /** The fetch-owned tab this fetch renders in. */
80
+ page: PlaywrightPage
75
81
  /** True when `browser` is the CDP pool's shared connection — never closed per fetch. */
76
82
  sharedBrowser?: boolean
83
+ /** True when `context` is the remote default context — close only the page. */
84
+ persistent?: boolean
77
85
  }
78
86
 
79
87
  /**
@@ -292,7 +300,8 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
292
300
  acquired = true
293
301
  session = await this.openSession(config, deadline)
294
302
  // An aborted deadline must also interrupt Playwright's own waits:
295
- // closing the context rejects every pending page operation.
303
+ // closing the page rejects every pending operation on it (and, for
304
+ // fetch-owned contexts, the context close that follows takes the rest).
296
305
  const held = session
297
306
  const onAbort = () => { void closeSession(held) }
298
307
  deadline.signal.addEventListener('abort', onAbort, { once: true })
@@ -314,7 +323,7 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
314
323
  }
315
324
 
316
325
  /**
317
- * Open the configured backend and its per-fetch context. Split out so the
326
+ * Open the configured backend and its per-fetch page. Split out so the
318
327
  * test suite can substitute a fake browser.
319
328
  * @param config - the resolved settings section.
320
329
  * @param deadline - the fetch budget, applied to launch/connect timeouts.
@@ -326,11 +335,19 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
326
335
  const endpoint = normalizeCdpEndpoint(config.cdpEndpoint)
327
336
  const { source } = await resolveCdpBackend()
328
337
  try {
329
- // One shared connection per provider; this fetch leases an isolated
330
- // context (a tab in the remote browser) and closes only that.
331
- const lease = await this.cdpPool.acquire(endpoint, timeout)
332
- await installResourceFilter(lease.context)
333
- return { browser: lease.browser, context: lease.context, sharedBrowser: true }
338
+ // One shared connection per provider; this fetch leases a tab — in a
339
+ // throwaway isolated context, or in the remote profile's default
340
+ // context (profile mode), whose persistent logins then apply.
341
+ const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config))
342
+ await installResourceFilter(lease.page)
343
+ guardPopups(lease.page)
344
+ return {
345
+ browser: lease.browser,
346
+ context: lease.context,
347
+ page: lease.page,
348
+ sharedBrowser: true,
349
+ persistent: lease.persistent,
350
+ }
334
351
  } catch (error: unknown) {
335
352
  throw new WebError(
336
353
  `cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`,
@@ -340,12 +357,19 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
340
357
  }
341
358
  }
342
359
  const { chromium, executablePath, source } = await resolvePlaywrightBackend(config.playwrightPath)
360
+ let browser: PlaywrightBrowser | undefined
343
361
  try {
344
- const browser = await chromium.launch({ headless: true, ...(executablePath !== undefined ? { executablePath } : {}), timeout })
362
+ browser = await chromium.launch({ headless: true, ...(executablePath !== undefined ? { executablePath } : {}), timeout })
345
363
  const context = await browser.newContext()
346
- await installResourceFilter(context)
347
- return { browser, context }
364
+ const page = await context.newPage()
365
+ await installResourceFilter(page)
366
+ guardPopups(page)
367
+ return { browser, context, page }
348
368
  } catch (error: unknown) {
369
+ // A partial setup (browser launched, then newContext/newPage failed)
370
+ // must not strand the process — closing the browser takes its
371
+ // contexts and pages with it.
372
+ await browser?.close().catch(() => {})
349
373
  throw new WebError(
350
374
  `cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`,
351
375
  'WEB_PROVIDER_ERROR',
@@ -354,14 +378,14 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
354
378
  }
355
379
  }
356
380
 
357
- /** Navigate, settle, and decode one URL inside an open session. */
381
+ /** Navigate, settle, and decode one URL inside an open session's tab. */
358
382
  private async retrieve(
359
383
  session: BrowserSession,
360
384
  url: URL,
361
385
  config: ResolvedConfig,
362
386
  deadline: Deadline,
363
387
  ): Promise<WebFetchResult> {
364
- const page = await session.context.newPage()
388
+ const page = session.page
365
389
  const response = await page.goto(url.toString(), { waitUntil: 'domcontentloaded', timeout: deadline.remainingMs() })
366
390
 
367
391
  const kind = classifyContentType(response?.headers()['content-type'])
@@ -407,14 +431,17 @@ function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text
407
431
  }
408
432
 
409
433
  /**
410
- * Close a session's fetch-owned context, plus its browser when the session
411
- * launched one (local backend). CDP sessions ride the shared connection,
412
- * which stays open. Every close is grace-bounded so cleanup always completes
413
- * and the concurrency slot is released even when Playwright hangs.
434
+ * Close a session's fetch-owned tab, then its context unless the context is
435
+ * the remote default context (profile mode — closing it would tear down the
436
+ * whole shared connection), then its browser when the session launched one
437
+ * (local backend). CDP sessions ride the shared connection, which stays
438
+ * open. Every close is grace-bounded so cleanup always completes and the
439
+ * concurrency slot is released even when Playwright hangs.
414
440
  */
415
441
  async function closeSession(session: BrowserSession | undefined): Promise<void> {
416
442
  if (session === undefined) return
417
- await closeWithGrace(session.context)
443
+ await closeWithGrace(session.page)
444
+ if (session.persistent !== true) await closeWithGrace(session.context)
418
445
  if (session.sharedBrowser !== true) await closeWithGrace(session.browser)
419
446
  }
420
447
 
@@ -442,11 +469,15 @@ async function defaultCdpConnect(endpoint: string, timeoutMs: number): Promise<P
442
469
  /**
443
470
  * Abort image/font/media subrequests: the markdown output keeps their URLs
444
471
  * but never renders them, so downloading them only spends the budget.
445
- * Best-effort — a context that refuses interception still fetches.
472
+ * Installed on the PAGE (never the context): in profile mode a context-level
473
+ * route would intercept the remote browser's other tabs too. Best-effort —
474
+ * a page that refuses interception still fetches.
446
475
  */
447
- async function installResourceFilter(context: PlaywrightContext): Promise<void> {
476
+ async function installResourceFilter(owner: {
477
+ route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
478
+ }): Promise<void> {
448
479
  try {
449
- await context.route('**/*', async (route) => {
480
+ await owner.route('**/*', async (route) => {
450
481
  const type = route.request().resourceType()
451
482
  if (type === 'image' || type === 'font' || type === 'media') await route.abort()
452
483
  else await route.continue()
@@ -455,3 +486,17 @@ async function installResourceFilter(context: PlaywrightContext): Promise<void>
455
486
  // keep going without the filter
456
487
  }
457
488
  }
489
+
490
+ /**
491
+ * Close any popup a fetched page spawns so nothing outlives the fetch's tab
492
+ * — `page.close()` does not auto-close windows the page opened, and in
493
+ * profile mode a stray tab would stay in the user's remote browser.
494
+ * Best-effort: a page that refuses listeners just loses the guard.
495
+ */
496
+ function guardPopups(page: PlaywrightPage): void {
497
+ try {
498
+ page.on?.('popup', popup => { void popup.close().catch(() => {}) })
499
+ } catch {
500
+ // keep going without the guard
501
+ }
502
+ }
package/src/types.ts CHANGED
@@ -22,9 +22,20 @@ export interface PlaywrightPage {
22
22
  url(): string
23
23
  content(): Promise<string>
24
24
  close(): Promise<void>
25
+ /**
26
+ * Resource-filter interception at page level — installed on the page (not
27
+ * its context) so profile mode never intercepts tabs it does not own.
28
+ */
29
+ route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
30
+ /** Popup notification; the fetch closes whatever its page spawns. */
31
+ on?(event: 'popup', listener: (page: PlaywrightPage) => void): unknown
25
32
  }
26
33
 
27
- /** An isolated browser context (the CDP backend gets a fresh one per fetch). */
34
+ /**
35
+ * A browser context: fetch-owned and isolated (local backend, or CDP
36
+ * `isolated` mode), or the remote browser's default context carrying its
37
+ * real profile (CDP `profile` mode — never closed by a fetch).
38
+ */
28
39
  export interface PlaywrightContext {
29
40
  newPage(): Promise<PlaywrightPage>
30
41
  route(glob: string, handler: (route: PlaywrightRoute) => Promise<void>): Promise<void>
@@ -42,6 +53,12 @@ export interface PlaywrightRoute {
42
53
  export interface PlaywrightBrowser {
43
54
  newContext(): Promise<PlaywrightContext>
44
55
  close(): Promise<void>
56
+ /**
57
+ * Contexts visible to this connection. Over CDP the default context — the
58
+ * remote browser's real profile — is always dispatched first, so `[0]` is
59
+ * it; absent on minimal fakes (isolated mode never calls this).
60
+ */
61
+ contexts?(): PlaywrightContext[]
45
62
  /** Liveness probe; absent on minimal fakes (assumed live). */
46
63
  isConnected?(): boolean
47
64
  /** Optional disconnect notification used to drop a stale shared CDP connection. */