dsh-web-fetch-playwright 0.2.20 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -17
- package/README.zh-CN.md +19 -17
- package/cordis.patch.yml +0 -4
- package/lib/client.js +18 -68
- package/lib/index.js +24049 -22754
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/candidates.d.ts +55 -0
- package/lib/types/challenge.d.ts +2 -2
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +80 -30
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +101 -9
- package/lib/types/types.d.ts +26 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +1 -1
- package/src/candidates.ts +184 -35
- package/src/challenge.ts +2 -2
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +97 -37
- package/src/index.ts +4 -1
- package/src/markdown.ts +13 -2
- package/src/provider.ts +482 -56
- package/src/selector-model.ts +1 -1
- package/src/selector.ts +18 -7
- package/src/types.ts +27 -7
package/src/config.ts
CHANGED
|
@@ -17,17 +17,25 @@ import type { Volatile } from '@deepseek-ai/cordis'
|
|
|
17
17
|
import z from '@deepseek-ai/schemastery'
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
|
-
* Default
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
20
|
+
* Default CAP (ms) on the deep-capture scroll walk (`src/autoscroll.ts`).
|
|
21
|
+
* It bounds only pages that keep reacting to scrolling — virtualized feeds
|
|
22
|
+
* and load-more streams — where 20s covers ~25kpx at the careful cadence
|
|
23
|
+
* (`SCROLL_STEP_MOTION_MS` + `SCROLL_STEP_SETTLE_MS` per ~0.7-viewport
|
|
24
|
+
* step). It is not what a typical page pays: a page that never reacts takes
|
|
25
|
+
* `SCROLL_INERT_STEPS` careful steps, one jump to the bottom and a
|
|
26
|
+
* `SCROLL_INERT_PATIENCE_MS` hold, and a page that does not scroll is
|
|
27
|
+
* skipped outright.
|
|
26
28
|
*/
|
|
27
29
|
export const DEFAULT_SCROLL_BUDGET_MS = 20_000
|
|
28
30
|
|
|
29
|
-
/**
|
|
30
|
-
|
|
31
|
+
/**
|
|
32
|
+
* Default cap (ms) on the post-navigation settle — the wait for SPA
|
|
33
|
+
* rendering to finish once the document has loaded (see the settle in
|
|
34
|
+
* `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
|
|
35
|
+
* the page's network goes quiet. Code-level only (tests, demo scripts):
|
|
36
|
+
* not part of the settings schema. `0` disables the settle entirely.
|
|
37
|
+
*/
|
|
38
|
+
export const DEFAULT_SETTLE_MS = 5_000
|
|
31
39
|
|
|
32
40
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
33
41
|
export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
|
|
@@ -53,20 +61,28 @@ export const MAX_CONCURRENCY_CEILING = 200
|
|
|
53
61
|
/**
|
|
54
62
|
* Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
|
|
55
63
|
* the user's real browser passes the verification on its own while the fetch
|
|
56
|
-
* holds the same page and context.
|
|
57
|
-
*
|
|
64
|
+
* holds the same page and context. Bounded further by the 25s per-fetch
|
|
65
|
+
* waiting budget, which keeps a settle/decode tail after the wait.
|
|
58
66
|
*/
|
|
59
67
|
export const DEFAULT_CHALLENGE_WAIT_MS = 15_000
|
|
60
68
|
|
|
61
|
-
/** Ceiling the schema accepts for `challengeWaitMs`. */
|
|
62
|
-
export const MAX_CHALLENGE_WAIT_MS = 60_000
|
|
63
|
-
|
|
64
69
|
/** Default same-page re-navigation attempts after a challenge wait runs out. */
|
|
65
70
|
export const DEFAULT_CHALLENGE_RETRIES = 1
|
|
66
71
|
|
|
67
72
|
/** Ceiling the schema accepts for `challengeRetries`. */
|
|
68
73
|
export const MAX_CHALLENGE_RETRIES = 3
|
|
69
74
|
|
|
75
|
+
/**
|
|
76
|
+
* Default for `challengeHttpFallback`: when the browser cannot clear a
|
|
77
|
+
* challenge inside its window (the interactive checkbox was never offered,
|
|
78
|
+
* or verification refused), refetch the URL once through the page's
|
|
79
|
+
* plain-HTTP request context — the same cookie jar, no render decision —
|
|
80
|
+
* because many Cloudflare edges serve the real document to non-browser
|
|
81
|
+
* clients while withholding the interactive challenge from the browser
|
|
82
|
+
* session (measured live: v2ex.com from a datacenter IP).
|
|
83
|
+
*/
|
|
84
|
+
export const DEFAULT_CHALLENGE_HTTP_FALLBACK = true
|
|
85
|
+
|
|
70
86
|
/** Which browser backend serves a fetch. */
|
|
71
87
|
export type PlaywrightBackend = 'local' | 'cdp'
|
|
72
88
|
|
|
@@ -113,7 +129,9 @@ export interface Config {
|
|
|
113
129
|
* Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
|
|
114
130
|
* the same page/context. `0` (or any config leaving this at 0) restores the
|
|
115
131
|
* legacy behavior: the first response is final, no waiting — the A/B
|
|
116
|
-
* baseline.
|
|
132
|
+
* baseline. Code-level only (tests, demo scripts): not part of the
|
|
133
|
+
* settings schema, so users cannot configure it. Default:
|
|
134
|
+
* {@link DEFAULT_CHALLENGE_WAIT_MS}.
|
|
117
135
|
*/
|
|
118
136
|
challengeWaitMs?: number
|
|
119
137
|
/**
|
|
@@ -122,15 +140,32 @@ export interface Config {
|
|
|
122
140
|
* Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
|
|
123
141
|
*/
|
|
124
142
|
challengeRetries?: number
|
|
143
|
+
/**
|
|
144
|
+
* When the browser cannot clear a challenge inside its window (no
|
|
145
|
+
* checkbox offered, or verification refused), refetch the URL once
|
|
146
|
+
* through the page's plain-HTTP request context (same cookie jar) and
|
|
147
|
+
* return that document when it is not itself a challenge. `false`
|
|
148
|
+
* restores the browser-only posture.
|
|
149
|
+
*/
|
|
150
|
+
challengeHttpFallback?: boolean
|
|
125
151
|
/**
|
|
126
152
|
* Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
|
|
127
153
|
* that defeats viewport virtualization on feed-shaped pages (Discourse
|
|
128
154
|
* topic streams: replies outside the render window are empty height-only
|
|
129
155
|
* placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
|
|
130
|
-
* restores the legacy single-instant capture.
|
|
131
|
-
*
|
|
156
|
+
* restores the legacy single-instant capture. Code-level only (tests,
|
|
157
|
+
* demo scripts): not part of the settings schema, so users cannot
|
|
158
|
+
* configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
|
|
132
159
|
*/
|
|
133
160
|
scrollBudgetMs?: number
|
|
161
|
+
/**
|
|
162
|
+
* Cap (ms) on the post-navigation settle (load baseline + bounded
|
|
163
|
+
* network-quiet window with stale-request discount). Code-level only
|
|
164
|
+
* (tests, demo scripts): not part of the settings schema, so users cannot
|
|
165
|
+
* configure it. `0` disables the settle. Default:
|
|
166
|
+
* {@link DEFAULT_SETTLE_MS}.
|
|
167
|
+
*/
|
|
168
|
+
settleMs?: number
|
|
134
169
|
/**
|
|
135
170
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
136
171
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -154,11 +189,10 @@ export const Config = z.object({
|
|
|
154
189
|
// Optional on purpose: the effective default depends on `backend`, which a
|
|
155
190
|
// static schema default cannot express.
|
|
156
191
|
maxConcurrency: z.number().step(1).min(1).max(MAX_CONCURRENCY_CEILING).volatile(),
|
|
157
|
-
// The
|
|
158
|
-
|
|
192
|
+
// The challenge wait budget and the deep-capture scroll budget are
|
|
193
|
+
// deliberately absent: they are code-level constants, not user settings.
|
|
159
194
|
challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES).volatile(),
|
|
160
|
-
|
|
161
|
-
scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile(),
|
|
195
|
+
challengeHttpFallback: z.boolean().default(DEFAULT_CHALLENGE_HTTP_FALLBACK).volatile(),
|
|
162
196
|
})
|
|
163
197
|
|
|
164
198
|
/**
|
|
@@ -184,12 +218,10 @@ export interface PlaywrightPluginConfig {
|
|
|
184
218
|
* field owns no default — the backend resolves one).
|
|
185
219
|
*/
|
|
186
220
|
maxConcurrency: Volatile<number | undefined>
|
|
187
|
-
/** Challenge wait budget (ms) as a live reference. */
|
|
188
|
-
challengeWaitMs: Volatile<number>
|
|
189
221
|
/** Challenge retry count as a live reference. */
|
|
190
222
|
challengeRetries: Volatile<number>
|
|
191
|
-
/**
|
|
192
|
-
|
|
223
|
+
/** Challenge dead-end plain-HTTP refetch toggle as a live reference. */
|
|
224
|
+
challengeHttpFallback: Volatile<boolean>
|
|
193
225
|
}
|
|
194
226
|
|
|
195
227
|
/**
|
|
@@ -209,9 +241,8 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
|
|
|
209
241
|
contextIsolationScope: config.contextIsolationScope.get(),
|
|
210
242
|
denoise: config.denoise.get(),
|
|
211
243
|
maxConcurrency: config.maxConcurrency.get(),
|
|
212
|
-
challengeWaitMs: config.challengeWaitMs.get(),
|
|
213
244
|
challengeRetries: config.challengeRetries.get(),
|
|
214
|
-
|
|
245
|
+
challengeHttpFallback: config.challengeHttpFallback.get(),
|
|
215
246
|
}
|
|
216
247
|
}
|
|
217
248
|
|
|
@@ -226,7 +257,10 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
|
|
|
226
257
|
* @returns the resolved config.
|
|
227
258
|
*/
|
|
228
259
|
export function resolveConfig(config: Partial<Config>): ResolvedConfig {
|
|
229
|
-
|
|
260
|
+
// The code-level budgets bypass the schema: they pass through untouched
|
|
261
|
+
// (only when explicitly given) and the effective* helpers default them.
|
|
262
|
+
const { challengeWaitMs, scrollBudgetMs, settleMs, ...settings } = config
|
|
263
|
+
const resolved = Config(settings as never) as PlaywrightPluginConfig
|
|
230
264
|
return {
|
|
231
265
|
backend: resolved.backend.get(),
|
|
232
266
|
playwrightPath: resolved.playwrightPath.get(),
|
|
@@ -235,21 +269,23 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
|
|
|
235
269
|
contextIsolationScope: resolved.contextIsolationScope.get(),
|
|
236
270
|
denoise: resolved.denoise.get(),
|
|
237
271
|
maxConcurrency: resolved.maxConcurrency.get(),
|
|
238
|
-
challengeWaitMs: resolved.challengeWaitMs.get(),
|
|
239
272
|
challengeRetries: resolved.challengeRetries.get(),
|
|
240
|
-
|
|
273
|
+
challengeHttpFallback: resolved.challengeHttpFallback.get(),
|
|
274
|
+
...(challengeWaitMs === undefined ? {} : { challengeWaitMs }),
|
|
275
|
+
...(scrollBudgetMs === undefined ? {} : { scrollBudgetMs }),
|
|
276
|
+
...(settleMs === undefined ? {} : { settleMs }),
|
|
241
277
|
}
|
|
242
278
|
}
|
|
243
279
|
|
|
244
280
|
/**
|
|
245
281
|
* Complete config after schemastery applies the field defaults it owns.
|
|
246
|
-
* `maxConcurrency`
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
282
|
+
* `maxConcurrency` stays optional because its effective default depends on
|
|
283
|
+
* `backend`; `challengeWaitMs`, `scrollBudgetMs` and `settleMs` stay
|
|
284
|
+
* optional because they are code-level overrides outside the schema —
|
|
285
|
+
* absent, the effective* helpers supply the built-in defaults.
|
|
250
286
|
*/
|
|
251
|
-
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'>
|
|
252
|
-
& { maxConcurrency?: number; scrollBudgetMs?: number }
|
|
287
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs' | 'settleMs'>
|
|
288
|
+
& { maxConcurrency?: number; challengeWaitMs?: number; scrollBudgetMs?: number; settleMs?: number }
|
|
253
289
|
|
|
254
290
|
/**
|
|
255
291
|
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
@@ -266,7 +302,7 @@ export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxCon
|
|
|
266
302
|
|
|
267
303
|
/**
|
|
268
304
|
* The challenge wait budget a fetch actually runs with: an explicit setting
|
|
269
|
-
* wins, else the
|
|
305
|
+
* wins, else the built-in default. `0` means the challenge path is off.
|
|
270
306
|
*
|
|
271
307
|
* @param config - the resolved settings section (or any partial of it).
|
|
272
308
|
* @returns the bounded wait in milliseconds.
|
|
@@ -288,9 +324,20 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
|
|
|
288
324
|
return DEFAULT_CHALLENGE_RETRIES
|
|
289
325
|
}
|
|
290
326
|
|
|
327
|
+
/**
|
|
328
|
+
* Whether a challenge dead-end may refetch through plain HTTP: an explicit
|
|
329
|
+
* setting wins (absent = on, the default).
|
|
330
|
+
*
|
|
331
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
332
|
+
* @returns true when the plain-HTTP fallback may run.
|
|
333
|
+
*/
|
|
334
|
+
export function effectiveChallengeHttpFallback(config: Pick<Config, 'challengeHttpFallback'>): boolean {
|
|
335
|
+
return config.challengeHttpFallback !== false
|
|
336
|
+
}
|
|
337
|
+
|
|
291
338
|
/**
|
|
292
339
|
* The scroll-walk budget a fetch actually runs with: an explicit setting
|
|
293
|
-
* wins, else the
|
|
340
|
+
* wins, else the built-in default. `0` means the deep-capture walk is off.
|
|
294
341
|
*
|
|
295
342
|
* @param config - the resolved settings section (or any partial of it).
|
|
296
343
|
* @returns the bounded walk budget in milliseconds.
|
|
@@ -300,6 +347,19 @@ export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>):
|
|
|
300
347
|
return DEFAULT_SCROLL_BUDGET_MS
|
|
301
348
|
}
|
|
302
349
|
|
|
350
|
+
/**
|
|
351
|
+
* The settle cap a fetch actually runs with: an explicit value wins, else
|
|
352
|
+
* the built-in default. `0` means the settle is off (domcontentloaded
|
|
353
|
+
* content is read as-is).
|
|
354
|
+
*
|
|
355
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
356
|
+
* @returns the bounded settle cap in milliseconds.
|
|
357
|
+
*/
|
|
358
|
+
export function effectiveSettleMs(config: Pick<Config, 'settleMs'>): number {
|
|
359
|
+
if (typeof config.settleMs === 'number') return config.settleMs
|
|
360
|
+
return DEFAULT_SETTLE_MS
|
|
361
|
+
}
|
|
362
|
+
|
|
303
363
|
/**
|
|
304
364
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
305
365
|
* browser's real profile) requires the CDP backend — a local launch has no
|
package/src/index.ts
CHANGED
|
@@ -24,14 +24,17 @@ import { PlaywrightFetchProvider } from './provider.ts'
|
|
|
24
24
|
export {
|
|
25
25
|
Config,
|
|
26
26
|
DEFAULT_CDP_ENDPOINT,
|
|
27
|
+
DEFAULT_CHALLENGE_HTTP_FALLBACK,
|
|
27
28
|
DEFAULT_CHALLENGE_RETRIES,
|
|
28
29
|
DEFAULT_CHALLENGE_WAIT_MS,
|
|
29
30
|
DEFAULT_SCROLL_BUDGET_MS,
|
|
30
|
-
|
|
31
|
+
DEFAULT_SETTLE_MS,
|
|
32
|
+
effectiveChallengeHttpFallback,
|
|
31
33
|
effectiveChallengeRetries,
|
|
32
34
|
effectiveChallengeWaitMs,
|
|
33
35
|
effectiveContextMode,
|
|
34
36
|
effectiveScrollBudgetMs,
|
|
37
|
+
effectiveSettleMs,
|
|
35
38
|
normalizeCdpEndpoint,
|
|
36
39
|
resolveConfig,
|
|
37
40
|
snapshotsOf,
|
package/src/markdown.ts
CHANGED
|
@@ -227,11 +227,22 @@ function analyze(html: string, url: string): Analysis {
|
|
|
227
227
|
const sanitize = (candidate: RawCandidate): Analysis['sanitized'][number] => {
|
|
228
228
|
// KEEP_CONTENT: false is the load-bearing half of the denoise: DOMPurify
|
|
229
229
|
// un-wraps forbidden tags but keeps their text by default; with it, the
|
|
230
|
-
// whole subtree of a forbidden element goes.
|
|
231
|
-
|
|
230
|
+
// whole subtree of a forbidden element goes. A generator-held NODE is
|
|
231
|
+
// sanitized by import (identical output, measured on the whole-body
|
|
232
|
+
// candidate of the Angular docs badcase); a string is re-parsed.
|
|
233
|
+
const root = purify.sanitize(candidate.node ?? candidate.html, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false, RETURN_DOM: true }) as HTMLElement
|
|
232
234
|
return { kind: candidate.kind, title: candidate.title, legacy: candidate.legacy, root, stats: measureCandidate(candidate.kind, candidate.legacy, root) }
|
|
233
235
|
}
|
|
234
236
|
const sanitized = generated.candidates.map(sanitize)
|
|
237
|
+
// The custom-wrap skip left no `art` candidate — but the selector's
|
|
238
|
+
// features read the art reference (`words_vs_art_log`, the block diffs),
|
|
239
|
+
// and a skipped Readability run would have measured a DEAD-empty one. Seed
|
|
240
|
+
// exactly that, so the surviving candidates' vectors stay what a dead pick
|
|
241
|
+
// yields (verified on the badcase: the sanitized art root holds no text).
|
|
242
|
+
if (generated.facts.readabilitySkipped === true && !sanitized.some(c => c.kind === 'art')) {
|
|
243
|
+
const empty = document.createElement('div')
|
|
244
|
+
sanitized.push({ kind: 'art', title: undefined, legacy: false, root: empty, stats: measureCandidate('art', false, empty) })
|
|
245
|
+
}
|
|
235
246
|
const page = pageFeatures(document, url, generated.facts, sanitized.find(c => c.kind === 'doc')?.stats, visibleTextLength(document.body))
|
|
236
247
|
|
|
237
248
|
let blocks: Block[] = []
|