dsh-web-fetch-playwright 0.2.20 → 0.2.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/config.ts CHANGED
@@ -17,17 +17,25 @@ import type { Volatile } from '@deepseek-ai/cordis'
17
17
  import z from '@deepseek-ai/schemastery'
18
18
 
19
19
  /**
20
- * Default bounded budget (ms) for the deep-capture scroll walk
21
- * (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
22
- * careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
23
- * the bottom patience window): 20s walks ~25kpx with pagination, covering
24
- * the measured linux.do badcase (~18kpx, 51 replies) with headroom.
25
- * Scroll-inert pages take the walk's fast path and spend a few seconds.
20
+ * Default CAP (ms) on the deep-capture scroll walk (`src/autoscroll.ts`).
21
+ * It bounds only pages that keep reacting to scrolling — virtualized feeds
22
+ * and load-more streams — where 20s covers ~25kpx at the careful cadence
23
+ * (`SCROLL_STEP_MOTION_MS` + `SCROLL_STEP_SETTLE_MS` per ~0.7-viewport
24
+ * step). It is not what a typical page pays: a page that never reacts takes
25
+ * `SCROLL_INERT_STEPS` careful steps, one jump to the bottom and a
26
+ * `SCROLL_INERT_PATIENCE_MS` hold, and a page that does not scroll is
27
+ * skipped outright.
26
28
  */
27
29
  export const DEFAULT_SCROLL_BUDGET_MS = 20_000
28
30
 
29
- /** Ceiling the schema accepts for `scrollBudgetMs`. */
30
- export const MAX_SCROLL_BUDGET_MS = 30_000
31
+ /**
32
+ * Default cap (ms) on the post-navigation settle — the wait for SPA
33
+ * rendering to finish once the document has loaded (see the settle in
34
+ * `src/provider.ts`). A CAP, not a fixed cost: the settle ends as soon as
35
+ * the page's network goes quiet. Code-level only (tests, demo scripts):
36
+ * not part of the settings schema. `0` disables the settle entirely.
37
+ */
38
+ export const DEFAULT_SETTLE_MS = 5_000
31
39
 
32
40
  /** Default CDP endpoint when the settings section leaves it blank. */
33
41
  export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
@@ -53,20 +61,28 @@ export const MAX_CONCURRENCY_CEILING = 200
53
61
  /**
54
62
  * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
55
63
  * the user's real browser passes the verification on its own while the fetch
56
- * holds the same page and context. Sized to fit the 45s per-fetch deadline
57
- * with the settle/decode tail (and one retry) still inside it.
64
+ * holds the same page and context. Bounded further by the 25s per-fetch
65
+ * waiting budget, which keeps a settle/decode tail after the wait.
58
66
  */
59
67
  export const DEFAULT_CHALLENGE_WAIT_MS = 15_000
60
68
 
61
- /** Ceiling the schema accepts for `challengeWaitMs`. */
62
- export const MAX_CHALLENGE_WAIT_MS = 60_000
63
-
64
69
  /** Default same-page re-navigation attempts after a challenge wait runs out. */
65
70
  export const DEFAULT_CHALLENGE_RETRIES = 1
66
71
 
67
72
  /** Ceiling the schema accepts for `challengeRetries`. */
68
73
  export const MAX_CHALLENGE_RETRIES = 3
69
74
 
75
+ /**
76
+ * Default for `challengeHttpFallback`: when the browser cannot clear a
77
+ * challenge inside its window (the interactive checkbox was never offered,
78
+ * or verification refused), refetch the URL once through the page's
79
+ * plain-HTTP request context — the same cookie jar, no render decision —
80
+ * because many Cloudflare edges serve the real document to non-browser
81
+ * clients while withholding the interactive challenge from the browser
82
+ * session (measured live: v2ex.com from a datacenter IP).
83
+ */
84
+ export const DEFAULT_CHALLENGE_HTTP_FALLBACK = true
85
+
70
86
  /** Which browser backend serves a fetch. */
71
87
  export type PlaywrightBackend = 'local' | 'cdp'
72
88
 
@@ -113,7 +129,9 @@ export interface Config {
113
129
  * Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
114
130
  * the same page/context. `0` (or any config leaving this at 0) restores the
115
131
  * legacy behavior: the first response is final, no waiting — the A/B
116
- * baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
132
+ * baseline. Code-level only (tests, demo scripts): not part of the
133
+ * settings schema, so users cannot configure it. Default:
134
+ * {@link DEFAULT_CHALLENGE_WAIT_MS}.
117
135
  */
118
136
  challengeWaitMs?: number
119
137
  /**
@@ -122,15 +140,32 @@ export interface Config {
122
140
  * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
123
141
  */
124
142
  challengeRetries?: number
143
+ /**
144
+ * When the browser cannot clear a challenge inside its window (no
145
+ * checkbox offered, or verification refused), refetch the URL once
146
+ * through the page's plain-HTTP request context (same cookie jar) and
147
+ * return that document when it is not itself a challenge. `false`
148
+ * restores the browser-only posture.
149
+ */
150
+ challengeHttpFallback?: boolean
125
151
  /**
126
152
  * Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
127
153
  * that defeats viewport virtualization on feed-shaped pages (Discourse
128
154
  * topic streams: replies outside the render window are empty height-only
129
155
  * placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
130
- * restores the legacy single-instant capture.
131
- * Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
156
+ * restores the legacy single-instant capture. Code-level only (tests,
157
+ * demo scripts): not part of the settings schema, so users cannot
158
+ * configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
132
159
  */
133
160
  scrollBudgetMs?: number
161
+ /**
162
+ * Cap (ms) on the post-navigation settle (load baseline + bounded
163
+ * network-quiet window with stale-request discount). Code-level only
164
+ * (tests, demo scripts): not part of the settings schema, so users cannot
165
+ * configure it. `0` disables the settle. Default:
166
+ * {@link DEFAULT_SETTLE_MS}.
167
+ */
168
+ settleMs?: number
134
169
  /**
135
170
  * How many fetches may render at once. Blank = backend default (4 for
136
171
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -154,11 +189,10 @@ export const Config = z.object({
154
189
  // Optional on purpose: the effective default depends on `backend`, which a
155
190
  // static schema default cannot express.
156
191
  maxConcurrency: z.number().step(1).min(1).max(MAX_CONCURRENCY_CEILING).volatile(),
157
- // The bounded natural-wait knobs; 0 disables the whole challenge path.
158
- challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
192
+ // The challenge wait budget and the deep-capture scroll budget are
193
+ // deliberately absent: they are code-level constants, not user settings.
159
194
  challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES).volatile(),
160
- // The deep-capture scroll walk's bounded budget; 0 disables the walk.
161
- scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile(),
195
+ challengeHttpFallback: z.boolean().default(DEFAULT_CHALLENGE_HTTP_FALLBACK).volatile(),
162
196
  })
163
197
 
164
198
  /**
@@ -184,12 +218,10 @@ export interface PlaywrightPluginConfig {
184
218
  * field owns no default — the backend resolves one).
185
219
  */
186
220
  maxConcurrency: Volatile<number | undefined>
187
- /** Challenge wait budget (ms) as a live reference. */
188
- challengeWaitMs: Volatile<number>
189
221
  /** Challenge retry count as a live reference. */
190
222
  challengeRetries: Volatile<number>
191
- /** Deep-capture scroll-walk budget (ms) as a live reference. */
192
- scrollBudgetMs: Volatile<number>
223
+ /** Challenge dead-end plain-HTTP refetch toggle as a live reference. */
224
+ challengeHttpFallback: Volatile<boolean>
193
225
  }
194
226
 
195
227
  /**
@@ -209,9 +241,8 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
209
241
  contextIsolationScope: config.contextIsolationScope.get(),
210
242
  denoise: config.denoise.get(),
211
243
  maxConcurrency: config.maxConcurrency.get(),
212
- challengeWaitMs: config.challengeWaitMs.get(),
213
244
  challengeRetries: config.challengeRetries.get(),
214
- scrollBudgetMs: config.scrollBudgetMs.get(),
245
+ challengeHttpFallback: config.challengeHttpFallback.get(),
215
246
  }
216
247
  }
217
248
 
@@ -226,7 +257,10 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
226
257
  * @returns the resolved config.
227
258
  */
228
259
  export function resolveConfig(config: Partial<Config>): ResolvedConfig {
229
- const resolved = Config(config as never) as PlaywrightPluginConfig
260
+ // The code-level budgets bypass the schema: they pass through untouched
261
+ // (only when explicitly given) and the effective* helpers default them.
262
+ const { challengeWaitMs, scrollBudgetMs, settleMs, ...settings } = config
263
+ const resolved = Config(settings as never) as PlaywrightPluginConfig
230
264
  return {
231
265
  backend: resolved.backend.get(),
232
266
  playwrightPath: resolved.playwrightPath.get(),
@@ -235,21 +269,23 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
235
269
  contextIsolationScope: resolved.contextIsolationScope.get(),
236
270
  denoise: resolved.denoise.get(),
237
271
  maxConcurrency: resolved.maxConcurrency.get(),
238
- challengeWaitMs: resolved.challengeWaitMs.get(),
239
272
  challengeRetries: resolved.challengeRetries.get(),
240
- scrollBudgetMs: resolved.scrollBudgetMs.get(),
273
+ challengeHttpFallback: resolved.challengeHttpFallback.get(),
274
+ ...(challengeWaitMs === undefined ? {} : { challengeWaitMs }),
275
+ ...(scrollBudgetMs === undefined ? {} : { scrollBudgetMs }),
276
+ ...(settleMs === undefined ? {} : { settleMs }),
241
277
  }
242
278
  }
243
279
 
244
280
  /**
245
281
  * Complete config after schemastery applies the field defaults it owns.
246
- * `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
247
- * its effective default depends on `backend`, the latter so hand-built
248
- * resolved configs (tests) may omit it and let
249
- * {@link effectiveScrollBudgetMs} supply the default the schema would have.
282
+ * `maxConcurrency` stays optional because its effective default depends on
283
+ * `backend`; `challengeWaitMs`, `scrollBudgetMs` and `settleMs` stay
284
+ * optional because they are code-level overrides outside the schema —
285
+ * absent, the effective* helpers supply the built-in defaults.
250
286
  */
251
- export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'>
252
- & { maxConcurrency?: number; scrollBudgetMs?: number }
287
+ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs' | 'settleMs'>
288
+ & { maxConcurrency?: number; challengeWaitMs?: number; scrollBudgetMs?: number; settleMs?: number }
253
289
 
254
290
  /**
255
291
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -266,7 +302,7 @@ export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxCon
266
302
 
267
303
  /**
268
304
  * The challenge wait budget a fetch actually runs with: an explicit setting
269
- * wins, else the schema default. `0` means the challenge path is off.
305
+ * wins, else the built-in default. `0` means the challenge path is off.
270
306
  *
271
307
  * @param config - the resolved settings section (or any partial of it).
272
308
  * @returns the bounded wait in milliseconds.
@@ -288,9 +324,20 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
288
324
  return DEFAULT_CHALLENGE_RETRIES
289
325
  }
290
326
 
327
+ /**
328
+ * Whether a challenge dead-end may refetch through plain HTTP: an explicit
329
+ * setting wins (absent = on, the default).
330
+ *
331
+ * @param config - the resolved settings section (or any partial of it).
332
+ * @returns true when the plain-HTTP fallback may run.
333
+ */
334
+ export function effectiveChallengeHttpFallback(config: Pick<Config, 'challengeHttpFallback'>): boolean {
335
+ return config.challengeHttpFallback !== false
336
+ }
337
+
291
338
  /**
292
339
  * The scroll-walk budget a fetch actually runs with: an explicit setting
293
- * wins, else the schema default. `0` means the deep-capture walk is off.
340
+ * wins, else the built-in default. `0` means the deep-capture walk is off.
294
341
  *
295
342
  * @param config - the resolved settings section (or any partial of it).
296
343
  * @returns the bounded walk budget in milliseconds.
@@ -300,6 +347,19 @@ export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>):
300
347
  return DEFAULT_SCROLL_BUDGET_MS
301
348
  }
302
349
 
350
+ /**
351
+ * The settle cap a fetch actually runs with: an explicit value wins, else
352
+ * the built-in default. `0` means the settle is off (domcontentloaded
353
+ * content is read as-is).
354
+ *
355
+ * @param config - the resolved settings section (or any partial of it).
356
+ * @returns the bounded settle cap in milliseconds.
357
+ */
358
+ export function effectiveSettleMs(config: Pick<Config, 'settleMs'>): number {
359
+ if (typeof config.settleMs === 'number') return config.settleMs
360
+ return DEFAULT_SETTLE_MS
361
+ }
362
+
303
363
  /**
304
364
  * The context mode a fetch actually runs with: profile (a tab in the remote
305
365
  * browser's real profile) requires the CDP backend — a local launch has no
package/src/index.ts CHANGED
@@ -24,14 +24,17 @@ import { PlaywrightFetchProvider } from './provider.ts'
24
24
  export {
25
25
  Config,
26
26
  DEFAULT_CDP_ENDPOINT,
27
+ DEFAULT_CHALLENGE_HTTP_FALLBACK,
27
28
  DEFAULT_CHALLENGE_RETRIES,
28
29
  DEFAULT_CHALLENGE_WAIT_MS,
29
30
  DEFAULT_SCROLL_BUDGET_MS,
30
- MAX_SCROLL_BUDGET_MS,
31
+ DEFAULT_SETTLE_MS,
32
+ effectiveChallengeHttpFallback,
31
33
  effectiveChallengeRetries,
32
34
  effectiveChallengeWaitMs,
33
35
  effectiveContextMode,
34
36
  effectiveScrollBudgetMs,
37
+ effectiveSettleMs,
35
38
  normalizeCdpEndpoint,
36
39
  resolveConfig,
37
40
  snapshotsOf,
package/src/markdown.ts CHANGED
@@ -227,11 +227,22 @@ function analyze(html: string, url: string): Analysis {
227
227
  const sanitize = (candidate: RawCandidate): Analysis['sanitized'][number] => {
228
228
  // KEEP_CONTENT: false is the load-bearing half of the denoise: DOMPurify
229
229
  // un-wraps forbidden tags but keeps their text by default; with it, the
230
- // whole subtree of a forbidden element goes.
231
- const root = purify.sanitize(candidate.html, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false, RETURN_DOM: true }) as HTMLElement
230
+ // whole subtree of a forbidden element goes. A generator-held NODE is
231
+ // sanitized by import (identical output, measured on the whole-body
232
+ // candidate of the Angular docs badcase); a string is re-parsed.
233
+ const root = purify.sanitize(candidate.node ?? candidate.html, { FORBID_TAGS, FORBID_ATTR, KEEP_CONTENT: false, RETURN_DOM: true }) as HTMLElement
232
234
  return { kind: candidate.kind, title: candidate.title, legacy: candidate.legacy, root, stats: measureCandidate(candidate.kind, candidate.legacy, root) }
233
235
  }
234
236
  const sanitized = generated.candidates.map(sanitize)
237
+ // The custom-wrap skip left no `art` candidate — but the selector's
238
+ // features read the art reference (`words_vs_art_log`, the block diffs),
239
+ // and a skipped Readability run would have measured a DEAD-empty one. Seed
240
+ // exactly that, so the surviving candidates' vectors stay what a dead pick
241
+ // yields (verified on the badcase: the sanitized art root holds no text).
242
+ if (generated.facts.readabilitySkipped === true && !sanitized.some(c => c.kind === 'art')) {
243
+ const empty = document.createElement('div')
244
+ sanitized.push({ kind: 'art', title: undefined, legacy: false, root: empty, stats: measureCandidate('art', false, empty) })
245
+ }
235
246
  const page = pageFeatures(document, url, generated.facts, sanitized.find(c => c.kind === 'doc')?.stats, visibleTextLength(document.body))
236
247
 
237
248
  let blocks: Block[] = []