dsh-web-fetch-playwright 0.2.13 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -5
- package/README.zh-CN.md +25 -5
- package/lib/client.js +94 -30
- package/lib/index.js +634 -140
- package/lib/types/cdp-pool.d.ts +48 -79
- package/lib/types/client/card.d.ts +1 -1
- package/lib/types/client/controller.d.ts +4 -2
- package/lib/types/client/fields.d.ts +3 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +17 -4
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +12 -7
- package/lib/types/session-window.d.ts +9 -0
- package/lib/types/types.d.ts +8 -3
- package/package.json +6 -5
- package/src/cdp-pool.ts +352 -167
- package/src/client/card.tsx +37 -7
- package/src/client/controller.ts +6 -2
- package/src/client/fields.module.css +10 -1
- package/src/client/fields.tsx +8 -3
- package/src/client/locales.ts +21 -2
- package/src/config.ts +19 -4
- package/src/index.ts +10 -4
- package/src/provider.ts +131 -19
- package/src/session-window.ts +229 -0
- package/src/types.ts +8 -3
package/src/client/controller.ts
CHANGED
|
@@ -33,9 +33,10 @@ export interface PlaywrightSettings {
|
|
|
33
33
|
/**
|
|
34
34
|
* CDP backend: whether fetches share the remote browser's context (its
|
|
35
35
|
* real profile — cookies, localStorage, persistent logins) as tabs, or
|
|
36
|
-
*
|
|
36
|
+
* reuse an isolated context per session or globally. Absent = schema default (true).
|
|
37
37
|
*/
|
|
38
38
|
shareBrowserContext?: boolean
|
|
39
|
+
contextIsolationScope?: 'session' | 'global'
|
|
39
40
|
/** Whether the Readability + DOMPurify pipeline runs. */
|
|
40
41
|
denoise?: boolean
|
|
41
42
|
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
@@ -55,8 +56,9 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
55
56
|
playwrightPath: CardFieldState
|
|
56
57
|
/** CDP endpoint input. */
|
|
57
58
|
cdpEndpoint: CardFieldState
|
|
58
|
-
/** CDP shared-context
|
|
59
|
+
/** CDP shared-context radio (draft 'true'/'false'). */
|
|
59
60
|
shareBrowserContext: CardFieldState
|
|
61
|
+
contextIsolationScope: CardFieldState
|
|
60
62
|
/** Denoise checkbox (draft 'true'/'false'). */
|
|
61
63
|
denoise: CardFieldState
|
|
62
64
|
/** Concurrency input (draft decimal integer). */
|
|
@@ -91,6 +93,7 @@ export class PlaywrightCardController {
|
|
|
91
93
|
textField('playwrightPath'),
|
|
92
94
|
textField('cdpEndpoint'),
|
|
93
95
|
checkboxField('shareBrowserContext'),
|
|
96
|
+
radioField('contextIsolationScope', ['session', 'global']),
|
|
94
97
|
checkboxField('denoise'),
|
|
95
98
|
numberField('maxConcurrency', 1, 200),
|
|
96
99
|
numberField('challengeWaitMs', 0, 60_000),
|
|
@@ -106,6 +109,7 @@ export class PlaywrightCardController {
|
|
|
106
109
|
playwrightPath: this.form.field('playwrightPath'),
|
|
107
110
|
cdpEndpoint: this.form.field('cdpEndpoint'),
|
|
108
111
|
shareBrowserContext: this.form.field('shareBrowserContext'),
|
|
112
|
+
contextIsolationScope: this.form.field('contextIsolationScope'),
|
|
109
113
|
denoise: this.form.field('denoise'),
|
|
110
114
|
maxConcurrency: this.form.field('maxConcurrency'),
|
|
111
115
|
challengeWaitMs: this.form.field('challengeWaitMs'),
|
|
@@ -158,7 +158,16 @@
|
|
|
158
158
|
background: var(--dsw-alias-bg-layer-3);
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
-
.
|
|
161
|
+
.radioOptionPlain {
|
|
162
|
+
display: flex;
|
|
163
|
+
flex-direction: column;
|
|
164
|
+
gap: 8px;
|
|
165
|
+
padding: 4px 0;
|
|
166
|
+
border: 0;
|
|
167
|
+
background: none;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
.radioOption:has(> .radioPick > input[type='radio']:checked) {
|
|
162
171
|
border-color: var(--dsw-alias-brand-primary);
|
|
163
172
|
}
|
|
164
173
|
|
package/src/client/fields.tsx
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
* @module dsh-web-fetch-playwright/client/fields
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
|
+
import { useId } from 'react'
|
|
11
12
|
import type { ReactNode } from 'react'
|
|
12
13
|
import css from './fields.module.css'
|
|
13
14
|
|
|
@@ -86,6 +87,9 @@ export interface RadioOption {
|
|
|
86
87
|
|
|
87
88
|
/** A staged radio group: the backend selector. */
|
|
88
89
|
export function RadioGroupField(props: {
|
|
90
|
+
embedded?: boolean
|
|
91
|
+
/** Render options without card borders or backgrounds. */
|
|
92
|
+
plainOptions?: boolean
|
|
89
93
|
label: string
|
|
90
94
|
options: readonly RadioOption[]
|
|
91
95
|
text: string
|
|
@@ -96,8 +100,9 @@ export function RadioGroupField(props: {
|
|
|
96
100
|
onEdit: (text: string) => void
|
|
97
101
|
onReset: () => void
|
|
98
102
|
}) {
|
|
103
|
+
const name = useId()
|
|
99
104
|
return (
|
|
100
|
-
<fieldset className={css.field}>
|
|
105
|
+
<fieldset className={props.embedded ? css.fieldEmbedded : css.field}>
|
|
101
106
|
<div className={css.head}>
|
|
102
107
|
<span className={css.label}>{props.label}</span>
|
|
103
108
|
{props.overridden
|
|
@@ -118,11 +123,11 @@ export function RadioGroupField(props: {
|
|
|
118
123
|
</div>
|
|
119
124
|
<div className={css.radioGroup} role="radiogroup" aria-label={props.label}>
|
|
120
125
|
{props.options.map(option => (
|
|
121
|
-
<div key={option.value} className={css.radioOption}>
|
|
126
|
+
<div key={option.value} className={props.plainOptions ? css.radioOptionPlain : css.radioOption}>
|
|
122
127
|
<label className={css.radioPick}>
|
|
123
128
|
<input
|
|
124
129
|
type="radio"
|
|
125
|
-
name=
|
|
130
|
+
name={name}
|
|
126
131
|
value={option.value}
|
|
127
132
|
checked={props.text === option.value}
|
|
128
133
|
disabled={props.disabled}
|
package/src/client/locales.ts
CHANGED
|
@@ -12,6 +12,9 @@ export type PlaywrightCardLocaleKey =
|
|
|
12
12
|
| 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder'
|
|
13
13
|
| 'cdpEndpoint' | 'cdpEndpointHint'
|
|
14
14
|
| 'shareBrowserContext' | 'shareBrowserContextHint'
|
|
15
|
+
| 'contextMode' | 'isolateBrowserContext' | 'isolateBrowserContextHint'
|
|
16
|
+
| 'contextIsolationScope' | 'contextScopeSession' | 'contextScopeSessionHint'
|
|
17
|
+
| 'contextScopeGlobal' | 'contextScopeGlobalHint'
|
|
15
18
|
| 'denoise' | 'denoiseHint'
|
|
16
19
|
| 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
|
|
17
20
|
| 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
|
|
@@ -39,7 +42,15 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
39
42
|
cdpEndpoint: 'CDP endpoint',
|
|
40
43
|
cdpEndpointHint: 'host:port or http(s)/ws URL. Leave blank for 127.0.0.1:9222.',
|
|
41
44
|
shareBrowserContext: 'Share the browser context (profile logins)',
|
|
42
|
-
shareBrowserContextHint: 'Each fetch opens a tab in the remote browser\u2019s real profile \u2014 cookies and localStorage are shared and its persistent logins apply.
|
|
45
|
+
shareBrowserContextHint: 'Each fetch opens a tab in the remote browser\u2019s real profile \u2014 cookies and localStorage are shared and its persistent logins apply.',
|
|
46
|
+
contextMode: 'Browser context',
|
|
47
|
+
isolateBrowserContext: 'Isolate from the browser profile',
|
|
48
|
+
isolateBrowserContextHint: 'Use a separate in-memory context with the lifecycle chosen below; no access to the browser profile’s logins.',
|
|
49
|
+
contextIsolationScope: 'Reuse isolated context',
|
|
50
|
+
contextScopeSession: 'Per DSH session',
|
|
51
|
+
contextScopeSessionHint: 'Created on demand per DSH session. Concurrent fetches share the context in separate tabs, each closing its own tab. The context closes when no active or pending fetches and no other open tabs remain; the next fetch starts with fresh cookies. Different sessions keep separate site data.',
|
|
52
|
+
contextScopeGlobal: 'Global reuse across DSH sessions',
|
|
53
|
+
contextScopeGlobalHint: 'All DSH sessions share cookies and localStorage within this plugin instance.',
|
|
43
54
|
denoise: 'Enable the denoise algorithm',
|
|
44
55
|
denoiseHint: 'Readability + DOMPurify strip nav bars, sidebars, footers, and ads before converting to markdown.',
|
|
45
56
|
maxConcurrency: 'Max concurrent fetches',
|
|
@@ -72,7 +83,15 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
72
83
|
cdpEndpoint: 'CDP 地址',
|
|
73
84
|
cdpEndpointHint: 'host:port 或 http(s)/ws 地址;留空默认 127.0.0.1:9222。',
|
|
74
85
|
shareBrowserContext: '共享浏览器上下文(复用登录态)',
|
|
75
|
-
shareBrowserContextHint: '每次抓取在远端浏览器的真实 profile 里开一个标签页:共享 cookie 与 localStorage
|
|
86
|
+
shareBrowserContextHint: '每次抓取在远端浏览器的真实 profile 里开一个标签页:共享 cookie 与 localStorage、复用已登录会话,页面会看到你的登录身份。',
|
|
87
|
+
contextMode: '浏览器上下文',
|
|
88
|
+
isolateBrowserContext: '隔离浏览器上下文',
|
|
89
|
+
isolateBrowserContextHint: '使用独立的内存上下文,生命周期由下方范围决定,不使用远端浏览器 profile 的登录态。',
|
|
90
|
+
contextIsolationScope: '隔离上下文复用范围',
|
|
91
|
+
contextScopeSession: '按 DSH 会话隔离',
|
|
92
|
+
contextScopeSessionHint: '按 DSH 会话按需创建上下文;并发抓取共享上下文,各用独立标签页并各自关闭。没有活动或待处理抓取,且没有其他打开的标签页时关闭上下文;下次抓取使用全新 cookie。不同会话的站点数据互不共享。',
|
|
93
|
+
contextScopeGlobal: '跨 DSH 会话全局复用',
|
|
94
|
+
contextScopeGlobalHint: '本插件实例内,所有 DSH 会话共享 cookie 与 localStorage。',
|
|
76
95
|
denoise: '启用降噪算法',
|
|
77
96
|
denoiseHint: '使用 Readability + DOMPurify 清洗导航栏、侧边栏、页脚与贴片广告后再转为 Markdown。',
|
|
78
97
|
maxConcurrency: '最大并发抓取数',
|
package/src/config.ts
CHANGED
|
@@ -58,11 +58,14 @@ export const MAX_CHALLENGE_RETRIES = 3
|
|
|
58
58
|
export type PlaywrightBackend = 'local' | 'cdp'
|
|
59
59
|
|
|
60
60
|
/**
|
|
61
|
-
* How the CDP backend scopes a fetch: a
|
|
61
|
+
* How the CDP backend scopes a fetch: a reusable isolated context, or a tab
|
|
62
62
|
* in the remote browser's real profile (default context).
|
|
63
63
|
*/
|
|
64
64
|
export type CdpContextMode = 'isolated' | 'profile'
|
|
65
65
|
|
|
66
|
+
/** Isolation key: an on-demand context per DSH session or one retained global context. */
|
|
67
|
+
export type ContextIsolationScope = 'session' | 'global'
|
|
68
|
+
|
|
66
69
|
/** Plugin config: everything optional — the schema fills the defaults. */
|
|
67
70
|
export interface Config {
|
|
68
71
|
/** Backend selector: local Playwright launch or a remote CDP endpoint. */
|
|
@@ -79,10 +82,18 @@ export interface Config {
|
|
|
79
82
|
* CDP backend only. `true` (default): each fetch is a tab in the remote
|
|
80
83
|
* browser's default context — the real profile — so cookies/localStorage
|
|
81
84
|
* come from (and are written back to) it and its persistent logins apply.
|
|
82
|
-
* `false`:
|
|
83
|
-
*
|
|
85
|
+
* `false`: use an on-demand isolated context per session or a retained global context.
|
|
86
|
+
* Meaningless for the local backend (collapses to isolated).
|
|
84
87
|
*/
|
|
85
88
|
shareBrowserContext?: boolean
|
|
89
|
+
/**
|
|
90
|
+
* CDP isolated context scope; defaults to global (retained between fetches).
|
|
91
|
+
* Session contexts are created on demand; concurrent same-session fetches
|
|
92
|
+
* share the context but each opens and closes its own tab. The context closes
|
|
93
|
+
* when no active/pending fetches and no other open tabs remain, so the next
|
|
94
|
+
* fetch starts with fresh cookies/localStorage. Profile mode is unaffected.
|
|
95
|
+
*/
|
|
96
|
+
contextIsolationScope?: ContextIsolationScope
|
|
86
97
|
/** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
|
|
87
98
|
denoise?: boolean
|
|
88
99
|
/**
|
|
@@ -116,6 +127,7 @@ export const Config = z.object({
|
|
|
116
127
|
playwrightPath: z.string().default('').volatile(),
|
|
117
128
|
cdpEndpoint: z.string().default('').volatile(),
|
|
118
129
|
shareBrowserContext: z.boolean().default(true).volatile(),
|
|
130
|
+
contextIsolationScope: z.union([z.const('session'), z.const('global')]).default('global').volatile(),
|
|
119
131
|
denoise: z.boolean().default(true).volatile(),
|
|
120
132
|
// Optional on purpose: the effective default depends on `backend`, which a
|
|
121
133
|
// static schema default cannot express.
|
|
@@ -140,6 +152,7 @@ export interface PlaywrightPluginConfig {
|
|
|
140
152
|
cdpEndpoint: Volatile<string>
|
|
141
153
|
/** Shared-context toggle as a live reference. */
|
|
142
154
|
shareBrowserContext: Volatile<boolean>
|
|
155
|
+
contextIsolationScope: Volatile<ContextIsolationScope>
|
|
143
156
|
/** Denoise toggle as a live reference. */
|
|
144
157
|
denoise: Volatile<boolean>
|
|
145
158
|
/**
|
|
@@ -167,6 +180,7 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
|
|
|
167
180
|
playwrightPath: config.playwrightPath.get(),
|
|
168
181
|
cdpEndpoint: config.cdpEndpoint.get(),
|
|
169
182
|
shareBrowserContext: config.shareBrowserContext.get(),
|
|
183
|
+
contextIsolationScope: config.contextIsolationScope.get(),
|
|
170
184
|
denoise: config.denoise.get(),
|
|
171
185
|
maxConcurrency: config.maxConcurrency.get(),
|
|
172
186
|
challengeWaitMs: config.challengeWaitMs.get(),
|
|
@@ -191,6 +205,7 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
|
|
|
191
205
|
playwrightPath: resolved.playwrightPath.get(),
|
|
192
206
|
cdpEndpoint: resolved.cdpEndpoint.get(),
|
|
193
207
|
shareBrowserContext: resolved.shareBrowserContext.get(),
|
|
208
|
+
contextIsolationScope: resolved.contextIsolationScope.get(),
|
|
194
209
|
denoise: resolved.denoise.get(),
|
|
195
210
|
maxConcurrency: resolved.maxConcurrency.get(),
|
|
196
211
|
challengeWaitMs: resolved.challengeWaitMs.get(),
|
|
@@ -245,7 +260,7 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
|
|
|
245
260
|
/**
|
|
246
261
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
247
262
|
* browser's real profile) requires the CDP backend — a local launch has no
|
|
248
|
-
* meaningful shared profile — and the
|
|
263
|
+
* meaningful shared profile — and the share option (which defaults on, so an
|
|
249
264
|
* absent value reads as `true`, mirroring the schema default); everything
|
|
250
265
|
* else collapses to isolated.
|
|
251
266
|
*
|
package/src/index.ts
CHANGED
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
|
|
17
17
|
import type { Context } from '@deepseek-ai/cordis'
|
|
18
18
|
import type {} from '@deepseek-ai/dsh-web'
|
|
19
|
+
import type {} from '@deepseek-ai/dsh-agent'
|
|
19
20
|
import { resolveConfig, snapshotsOf } from './config.ts'
|
|
20
21
|
import type { PlaywrightPluginConfig } from './config.ts'
|
|
21
22
|
import { PlaywrightFetchProvider } from './provider.ts'
|
|
@@ -35,6 +36,7 @@ export {
|
|
|
35
36
|
export type {
|
|
36
37
|
Config as PlaywrightFetchConfig,
|
|
37
38
|
CdpContextMode,
|
|
39
|
+
ContextIsolationScope,
|
|
38
40
|
PlaywrightBackend,
|
|
39
41
|
PlaywrightPluginConfig,
|
|
40
42
|
ResolvedConfig,
|
|
@@ -75,9 +77,13 @@ export function apply(ctx: Context, config: PlaywrightPluginConfig): void {
|
|
|
75
77
|
// Re-read AND re-resolve per fetch: `snapshotsOf` dereferences the volatile
|
|
76
78
|
// references each time, so a section committed from the settings page
|
|
77
79
|
// between two fetches serves the second one.
|
|
78
|
-
const provider = new PlaywrightFetchProvider(
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
80
|
+
const provider = new PlaywrightFetchProvider(
|
|
81
|
+
() => resolveConfig(snapshotsOf(config)),
|
|
82
|
+
undefined,
|
|
83
|
+
() => ctx.get('agents')?.currentInitiator()?.session.id,
|
|
84
|
+
)
|
|
85
|
+
// Drop all session-dedicated and global/profile-shared CDP connections on
|
|
86
|
+
// unload so restarts cannot strand sockets or owned browser contexts.
|
|
87
|
+
ctx.effect(() => () => { void provider.dispose() }, 'dsh-web-fetch-playwright: CDP connections')
|
|
82
88
|
ctx.web.registerFetchProvider(provider)
|
|
83
89
|
}
|
package/src/provider.ts
CHANGED
|
@@ -19,10 +19,12 @@
|
|
|
19
19
|
* ({@link recoverDownloadResponse}) — instead of a `WEB_PROVIDER_ERROR`.
|
|
20
20
|
*
|
|
21
21
|
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
22
|
-
* nothing outlives the call.
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
22
|
+
* nothing outlives the call. CDP session isolation keeps one connection per
|
|
23
|
+
* DSH session until its context becomes idle; this also isolates gateways that
|
|
24
|
+
* map a connection's first context to a separate browser process. Global/profile
|
|
25
|
+
* modes share a provider-lifetime connection. Each fetch owns only its page;
|
|
26
|
+
* contexts are per-session until idle, global until teardown, or (the default,
|
|
27
|
+
* `shareBrowserContext`) in the remote
|
|
26
28
|
* browser's default context so its profile, cookies, and persistent logins
|
|
27
29
|
* apply; that default context is never closed. The concurrency cap counts
|
|
28
30
|
* tabs, not browsers. Either way, an aborted signal closes the fetch's page.
|
|
@@ -117,8 +119,10 @@ export interface BrowserSession {
|
|
|
117
119
|
page: PlaywrightPage
|
|
118
120
|
/** True when `browser` is the CDP pool's shared connection — never closed per fetch. */
|
|
119
121
|
sharedBrowser?: boolean
|
|
120
|
-
/** True
|
|
122
|
+
/** True for retained contexts (remote profile or pool-owned named context). */
|
|
121
123
|
persistent?: boolean
|
|
124
|
+
/** Pool cleanup: closes this tab and, for session isolation, its idle context. */
|
|
125
|
+
release?: () => Promise<void>
|
|
122
126
|
}
|
|
123
127
|
|
|
124
128
|
/**
|
|
@@ -224,12 +228,8 @@ class Semaphore {
|
|
|
224
228
|
release(): void {
|
|
225
229
|
// Below the limit a freed slot hands straight to the next waiter; above
|
|
226
230
|
// it (the limit was lowered) the slot disappears instead.
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
this.active = Math.max(0, this.active - 1)
|
|
230
|
-
return
|
|
231
|
-
}
|
|
232
|
-
next.start()
|
|
231
|
+
this.active = Math.max(0, this.active - 1)
|
|
232
|
+
this.drain()
|
|
233
233
|
}
|
|
234
234
|
|
|
235
235
|
private drain(): void {
|
|
@@ -303,7 +303,11 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
303
303
|
* @param configSource - thunk returning the currently authoritative config.
|
|
304
304
|
* @param cdpPool - optional pool over the CDP backend (tests inject fakes).
|
|
305
305
|
*/
|
|
306
|
-
constructor(
|
|
306
|
+
constructor(
|
|
307
|
+
private readonly configSource: () => ResolvedConfig,
|
|
308
|
+
cdpPool?: CdpConnectionPool,
|
|
309
|
+
private readonly sessionIdSource: () => string | undefined = () => undefined,
|
|
310
|
+
) {
|
|
307
311
|
this.cdpPool = cdpPool ?? new CdpConnectionPool(defaultCdpConnect)
|
|
308
312
|
}
|
|
309
313
|
|
|
@@ -324,6 +328,18 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
324
328
|
async fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult> {
|
|
325
329
|
if (signal?.aborted) throw new WebError('web fetch aborted', 'WEB_ABORTED')
|
|
326
330
|
const config = this.configSource()
|
|
331
|
+
// Capture identity before queueing: never derive ownership from a shared
|
|
332
|
+
// mutable "current session", nor from process environment variables.
|
|
333
|
+
const sessionScoped = config.backend === 'cdp' && !config.shareBrowserContext && config.contextIsolationScope === 'session'
|
|
334
|
+
let sessionId: string | undefined
|
|
335
|
+
if (sessionScoped) {
|
|
336
|
+
// The identity source may throw (e.g. the agents service was already
|
|
337
|
+
// disposed during unload); surface that as the seam's error, not a raw Error.
|
|
338
|
+
try { sessionId = this.sessionIdSource() } catch { sessionId = undefined }
|
|
339
|
+
if (!sessionId?.trim()) {
|
|
340
|
+
throw new WebError('per-session context isolation requires a DSH session identity', 'WEB_FETCH_SESSION_REQUIRED')
|
|
341
|
+
}
|
|
342
|
+
}
|
|
327
343
|
const url = validateFetchUrl(request.url)
|
|
328
344
|
const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS)
|
|
329
345
|
|
|
@@ -335,7 +351,8 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
335
351
|
this.semaphore.resize(effectiveMaxConcurrency(config))
|
|
336
352
|
await this.semaphore.acquire(deadline.signal, QUEUE_TIMEOUT_MS)
|
|
337
353
|
acquired = true
|
|
338
|
-
session = await this.openSession(config, deadline)
|
|
354
|
+
session = await this.openSession(config, deadline, sessionId)
|
|
355
|
+
if (deadline.signal.aborted) throw new Error('web fetch aborted during session setup')
|
|
339
356
|
// An aborted deadline must also interrupt Playwright's own waits:
|
|
340
357
|
// closing the page rejects every pending operation on it (and, for
|
|
341
358
|
// fetch-owned contexts, the context close that follows takes the rest).
|
|
@@ -366,16 +383,26 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
366
383
|
* @param deadline - the fetch budget, applied to launch/connect timeouts.
|
|
367
384
|
* @returns the browser session the fetch will use.
|
|
368
385
|
*/
|
|
369
|
-
protected async openSession(config: ResolvedConfig, deadline: Deadline): Promise<BrowserSession> {
|
|
386
|
+
protected async openSession(config: ResolvedConfig, deadline: Deadline, sessionId?: string): Promise<BrowserSession> {
|
|
370
387
|
const timeout = Math.min(deadline.remainingMs(), 20_000)
|
|
371
388
|
if (config.backend === 'cdp') {
|
|
372
389
|
const endpoint = normalizeCdpEndpoint(config.cdpEndpoint)
|
|
390
|
+
const profile = effectiveContextMode(config) === 'profile'
|
|
391
|
+
if (!profile && config.contextIsolationScope === 'session' && !sessionId?.trim()) {
|
|
392
|
+
throw new WebError('per-session context isolation requires a DSH session identity', 'WEB_FETCH_SESSION_REQUIRED')
|
|
393
|
+
}
|
|
394
|
+
const mode = profile
|
|
395
|
+
? 'profile' as const
|
|
396
|
+
: { scope: config.contextIsolationScope, name: config.contextIsolationScope === 'session' ? sessionId! : 'global' }
|
|
373
397
|
const { source } = await resolveCdpBackend()
|
|
398
|
+
// A policy contains the scope, NOT the requesting session: switching
|
|
399
|
+
// between sessions must not evict the other session's retained context.
|
|
400
|
+
this.cdpPool.setPolicy(JSON.stringify([endpoint, config.shareBrowserContext,
|
|
401
|
+
config.shareBrowserContext ? '' : config.contextIsolationScope]))
|
|
374
402
|
try {
|
|
375
|
-
//
|
|
376
|
-
//
|
|
377
|
-
|
|
378
|
-
const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config))
|
|
403
|
+
// Lease a tab in a retained named context or the remote default
|
|
404
|
+
// profile. Only the pool owns context/connection cleanup.
|
|
405
|
+
const lease = await this.cdpPool.acquire(endpoint, timeout, mode)
|
|
379
406
|
await installResourceFilter(lease.page)
|
|
380
407
|
guardPopups(lease.page)
|
|
381
408
|
guardDownloads(lease.page)
|
|
@@ -385,15 +412,17 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
385
412
|
page: lease.page,
|
|
386
413
|
sharedBrowser: true,
|
|
387
414
|
persistent: lease.persistent,
|
|
415
|
+
release: () => lease.release(),
|
|
388
416
|
}
|
|
389
417
|
} catch (error: unknown) {
|
|
390
418
|
throw new WebError(
|
|
391
|
-
`cannot
|
|
419
|
+
`cannot open a fetch session via the CDP endpoint ${endpoint} (${source}): ${String(error instanceof Error ? error.message : error)}`,
|
|
392
420
|
'WEB_PROVIDER_ERROR',
|
|
393
421
|
{ cause: error },
|
|
394
422
|
)
|
|
395
423
|
}
|
|
396
424
|
}
|
|
425
|
+
this.cdpPool.setPolicy('local')
|
|
397
426
|
const { chromium, executablePath, source } = await resolvePlaywrightBackend(config.playwrightPath)
|
|
398
427
|
let browser: PlaywrightBrowser | undefined
|
|
399
428
|
try {
|
|
@@ -506,6 +535,13 @@ export class PlaywrightFetchProvider implements WebFetchProvider {
|
|
|
506
535
|
// what domcontentloaded already produced.
|
|
507
536
|
await page.waitForLoadState('networkidle', { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {})
|
|
508
537
|
|
|
538
|
+
// Serialization safety: the document is about to be flattened into an
|
|
539
|
+
// HTML string and re-parsed by jsdom, and scripting elements are the one
|
|
540
|
+
// place that round-trip can break — see {@link STRIP_SCRIPTING_ELEMENTS}
|
|
541
|
+
// for exactly what is removed, what is kept (JSON-LD, noscript), and why
|
|
542
|
+
// each removal cannot change what the pipeline extracts.
|
|
543
|
+
await stripScriptingElements(page)
|
|
544
|
+
|
|
509
545
|
const html = await page.content()
|
|
510
546
|
if (!config.denoise) {
|
|
511
547
|
// The tool layer's own turndown renders raw HTML; the checkbox only
|
|
@@ -871,6 +907,78 @@ async function probeStillOnChallenge(page: PlaywrightPage): Promise<boolean> {
|
|
|
871
907
|
}
|
|
872
908
|
}
|
|
873
909
|
|
|
910
|
+
/**
|
|
911
|
+
* In-page script that removes non-content scripting subtrees from the live
|
|
912
|
+
* document before `page.content()` reads it as an HTML string.
|
|
913
|
+
*
|
|
914
|
+
* WHAT goes, and why each removal is output-neutral versus the old pipeline:
|
|
915
|
+
*
|
|
916
|
+
* - `script` — three consumers ever saw script text before, and none loses
|
|
917
|
+
* it here: Readability's `parse()` deletes script/noscript nodes BEFORE
|
|
918
|
+
* extraction and scoring (so script text never reached the article or
|
|
919
|
+
* influenced the scorer); DOMPurify's default allowlist drops script; the
|
|
920
|
+
* tool layer's renderer removes SCRIPT wholesale on the raw path. What the
|
|
921
|
+
* removal buys is the reason this exists: a page's runtime JS may rewrite
|
|
922
|
+
* a data script's text with a literal `</script>` embedded — measured on
|
|
923
|
+
* techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
|
|
924
|
+
* Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
|
|
925
|
+
* turning the wire's 712 `\u003c` escapes back into raw `<` — and
|
|
926
|
+
* browsers serialize script text verbatim, so the flattened string ends
|
|
927
|
+
* that script at the FIRST embedded end tag and re-parses the remaining
|
|
928
|
+
* ~270KB of JSON as markup. The phantom elements that survive (unknown
|
|
929
|
+
* tags like `<id>`, anchors full of JSON text) are exactly what
|
|
930
|
+
* Readability's scorer reads as article mass, so the extracted "article"
|
|
931
|
+
* is CSS and i18n strings while the real body — still present in the DOM —
|
|
932
|
+
* loses. No string-level patch can undo this after serialization (the
|
|
933
|
+
* string is genuinely ambiguous); only the browser's own parse holds the
|
|
934
|
+
* document's true structure.
|
|
935
|
+
* - `script[type="application/ld+json"]` — KEPT, but defused: Readability
|
|
936
|
+
* extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
|
|
937
|
+
* "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
|
|
938
|
+
* scripts before removing them, so deleting them would regress the title
|
|
939
|
+
* on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
|
|
940
|
+
* valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
|
|
941
|
+
* same object) while making the serialization harmless: the tokenizer
|
|
942
|
+
* only terminates on a literal `</script`.
|
|
943
|
+
* - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
|
|
944
|
+
* real image out of lazy-load `<noscript><img …>` wrappers before
|
|
945
|
+
* `_removeScripts` deletes the noscript; pre-stripping it would silently
|
|
946
|
+
* drop those images from the markdown.
|
|
947
|
+
* - `template` — removed: its children parse into the inert `.content`
|
|
948
|
+
* fragment (invisible to Readability's traversal), DOMPurify's
|
|
949
|
+
* FORBID_TAGS drops it, and the tool layer's renderer removes it too.
|
|
950
|
+
* - `style` stays — its text cannot terminate a re-parse except at its own
|
|
951
|
+
* end tag, and the pipeline already drops it.
|
|
952
|
+
*/
|
|
953
|
+
const STRIP_SCRIPTING_ELEMENTS = `(() => {
|
|
954
|
+
for (const el of Array.from(document.querySelectorAll('script, template'))) {
|
|
955
|
+
if (el.tagName === 'SCRIPT' && (el.getAttribute('type') || '').trim().toLowerCase() === 'application/ld+json') {
|
|
956
|
+
// '<\\/' is still valid JSON, so the re-parsed textContent parses to
|
|
957
|
+
// the same object — while the serialized form can no longer close the
|
|
958
|
+
// script element early.
|
|
959
|
+
el.textContent = el.textContent.replace(/<\\//g, '<\\\\/')
|
|
960
|
+
continue
|
|
961
|
+
}
|
|
962
|
+
el.remove()
|
|
963
|
+
}
|
|
964
|
+
})()`
|
|
965
|
+
|
|
966
|
+
/**
|
|
967
|
+
* Remove non-content scripting subtrees from the live document before
|
|
968
|
+
* serialization. Best-effort, mirroring {@link probeStillOnChallenge}: a
|
|
969
|
+
* backend without `evaluate` (fakes, minimal engines) simply keeps the
|
|
970
|
+
* document as-is — the fetch never fails here, and the pipeline degrades to
|
|
971
|
+
* whatever the serialized string itself says.
|
|
972
|
+
*/
|
|
973
|
+
async function stripScriptingElements(page: PlaywrightPage): Promise<void> {
|
|
974
|
+
if (typeof page.evaluate !== 'function') return
|
|
975
|
+
try {
|
|
976
|
+
await page.evaluate(STRIP_SCRIPTING_ELEMENTS)
|
|
977
|
+
} catch {
|
|
978
|
+
// keep going without the strip
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
|
|
874
982
|
/** Apply the decoded-body cap and flag the cut. */
|
|
875
983
|
function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text'; content: string }): WebFetchResult {
|
|
876
984
|
const truncated = body.content.length > MAX_BODY_CHARS
|
|
@@ -892,6 +1000,10 @@ function capResult(url: string, statusCode: number, body: { kind: 'html' | 'text
|
|
|
892
1000
|
*/
|
|
893
1001
|
async function closeSession(session: BrowserSession | undefined): Promise<void> {
|
|
894
1002
|
if (session === undefined) return
|
|
1003
|
+
if (session.release !== undefined) {
|
|
1004
|
+
await closeWithGrace({ close: session.release })
|
|
1005
|
+
return
|
|
1006
|
+
}
|
|
895
1007
|
await closeWithGrace(session.page)
|
|
896
1008
|
if (session.persistent !== true) await closeWithGrace(session.context)
|
|
897
1009
|
if (session.sharedBrowser !== true) await closeWithGrace(session.browser)
|