dsh-web-fetch-playwright 0.2.19 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -41,16 +41,6 @@ export interface PlaywrightSettings {
41
41
  denoise?: boolean
42
42
  /** How many fetches may render at once (1–200); blank = backend default. */
43
43
  maxConcurrency?: number
44
- /**
45
- * Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
46
- * same tab (0–60000); 0 = off (return the first response as-is).
47
- */
48
- challengeWaitMs?: number
49
- /**
50
- * Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
51
- * (capture the document as it rests, the legacy single-instant behavior).
52
- */
53
- scrollBudgetMs?: number
54
44
  }
55
45
 
56
46
  /** What the Playwright card renders. */
@@ -68,10 +58,6 @@ export interface PlaywrightCardState extends CardShell {
68
58
  denoise: CardFieldState
69
59
  /** Concurrency input (draft decimal integer). */
70
60
  maxConcurrency: CardFieldState
71
- /** Challenge wait input (draft decimal integer of milliseconds). */
72
- challengeWaitMs: CardFieldState
73
- /** Scroll-walk budget input (draft decimal integer of milliseconds). */
74
- scrollBudgetMs: CardFieldState
75
61
  }
76
62
 
77
63
  /** The registration-side face the card's slot entry injects. */
@@ -103,8 +89,6 @@ export class PlaywrightCardController {
103
89
  radioField('contextIsolationScope', ['session', 'global']),
104
90
  checkboxField('denoise'),
105
91
  numberField('maxConcurrency', 1, 200),
106
- numberField('challengeWaitMs', 0, 60_000),
107
- numberField('scrollBudgetMs', 0, 30_000),
108
92
  ],
109
93
  )
110
94
  this.store = this.form.bind(() => this.projection())
@@ -120,8 +104,6 @@ export class PlaywrightCardController {
120
104
  contextIsolationScope: this.form.field('contextIsolationScope'),
121
105
  denoise: this.form.field('denoise'),
122
106
  maxConcurrency: this.form.field('maxConcurrency'),
123
- challengeWaitMs: this.form.field('challengeWaitMs'),
124
- scrollBudgetMs: this.form.field('scrollBudgetMs'),
125
107
  }
126
108
  }
127
109
 
@@ -17,8 +17,6 @@ export type PlaywrightCardLocaleKey =
17
17
  | 'contextScopeGlobal' | 'contextScopeGlobalHint'
18
18
  | 'denoise' | 'denoiseHint'
19
19
  | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
20
- | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
21
- | 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder'
22
20
  | 'overridden' | 'reset' | 'readOnly'
23
21
  | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText'
24
22
 
@@ -57,12 +55,6 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
57
55
  maxConcurrency: 'Max concurrent fetches',
58
56
  maxConcurrencyHint: 'How many pages may render at once (1–200). Blank = auto: 4 local browsers, 50 CDP tabs in the remote browser.',
59
57
  maxConcurrencyPlaceholder: '(auto: local 4 / CDP 50)',
60
- challengeWaitMs: 'Cloudflare challenge wait (ms)',
61
- challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
62
- challengeWaitMsPlaceholder: '(default: 15000)',
63
- scrollBudgetMs: 'Deep-capture scroll budget (ms)',
64
- scrollBudgetMsHint: 'Bounded in-page scroll walk that captures viewport-virtualized feeds (forum topics, comment threads) whole: posts outside the render window are empty placeholders at rest. 0–30000; 0 = off (capture the page as it rests).',
65
- scrollBudgetMsPlaceholder: '(default: 20000)',
66
58
  overridden: 'Overridden',
67
59
  reset: 'Reset to default',
68
60
  readOnly: 'This deployment stores settings read-only.',
@@ -101,12 +93,6 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
101
93
  maxConcurrency: '最大并发抓取数',
102
94
  maxConcurrencyHint: '同时渲染的页面上限(1–200);留空自动:本地 4 个浏览器,CDP 在远端浏览器里 50 个标签页。',
103
95
  maxConcurrencyPlaceholder: '(自动:本地 4 / CDP 50)',
104
- challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
105
- challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
106
- challengeWaitMsPlaceholder: '(默认:15000)',
107
- scrollBudgetMs: '深度抓取滚动预算(毫秒)',
108
- scrollBudgetMsHint: '有界的页内滚动遍历,完整捕获视口虚拟化的信息流(论坛帖子、评论区):静止时渲染窗口外的帖子只是空占位符。0–30000;0 = 关闭(按页面静止状态抓取)。',
109
- scrollBudgetMsPlaceholder: '(默认:20000)',
110
96
  overridden: '已覆盖',
111
97
  reset: '恢复默认',
112
98
  readOnly: '本部署的设置为只读。',
package/src/config.ts CHANGED
@@ -17,18 +17,17 @@ import type { Volatile } from '@deepseek-ai/cordis'
17
17
  import z from '@deepseek-ai/schemastery'
18
18
 
19
19
  /**
20
- * Default bounded budget (ms) for the deep-capture scroll walk
21
- * (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
22
- * careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
23
- * the bottom patience window): 20s walks ~25kpx with pagination, covering
24
- * the measured linux.do badcase (~18kpx, 51 replies) with headroom.
25
- * Scroll-inert pages take the walk's fast path and spend a few seconds.
20
+ * Default CAP (ms) on the deep-capture scroll walk (`src/autoscroll.ts`).
21
+ * It bounds only pages that keep reacting to scrolling — virtualized feeds
22
+ * and load-more streams — where 20s covers ~25kpx at the careful cadence
23
+ * (`SCROLL_STEP_MOTION_MS` + `SCROLL_STEP_SETTLE_MS` per ~0.7-viewport
24
+ * step). It is not what a typical page pays: a page that never reacts takes
25
+ * `SCROLL_INERT_STEPS` careful steps, one jump to the bottom and a
26
+ * `SCROLL_INERT_PATIENCE_MS` hold, and a page that does not scroll is
27
+ * skipped outright.
26
28
  */
27
29
  export const DEFAULT_SCROLL_BUDGET_MS = 20_000
28
30
 
29
- /** Ceiling the schema accepts for `scrollBudgetMs`. */
30
- export const MAX_SCROLL_BUDGET_MS = 30_000
31
-
32
31
  /** Default CDP endpoint when the settings section leaves it blank. */
33
32
  export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
34
33
 
@@ -53,20 +52,28 @@ export const MAX_CONCURRENCY_CEILING = 200
53
52
  /**
54
53
  * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
55
54
  * the user's real browser passes the verification on its own while the fetch
56
- * holds the same page and context. Sized to fit the 45s per-fetch deadline
57
- * with the settle/decode tail (and one retry) still inside it.
55
+ * holds the same page and context. Bounded further by the 25s per-fetch
56
+ * waiting budget, which keeps a settle/decode tail after the wait.
58
57
  */
59
58
  export const DEFAULT_CHALLENGE_WAIT_MS = 15_000
60
59
 
61
- /** Ceiling the schema accepts for `challengeWaitMs`. */
62
- export const MAX_CHALLENGE_WAIT_MS = 60_000
63
-
64
60
  /** Default same-page re-navigation attempts after a challenge wait runs out. */
65
61
  export const DEFAULT_CHALLENGE_RETRIES = 1
66
62
 
67
63
  /** Ceiling the schema accepts for `challengeRetries`. */
68
64
  export const MAX_CHALLENGE_RETRIES = 3
69
65
 
66
+ /**
67
+ * Default for `challengeHttpFallback`: when the browser cannot clear a
68
+ * challenge inside its window (the interactive checkbox was never offered,
69
+ * or verification refused), refetch the URL once through the page's
70
+ * plain-HTTP request context — the same cookie jar, no render decision —
71
+ * because many Cloudflare edges serve the real document to non-browser
72
+ * clients while withholding the interactive challenge from the browser
73
+ * session (measured live: v2ex.com from a datacenter IP).
74
+ */
75
+ export const DEFAULT_CHALLENGE_HTTP_FALLBACK = true
76
+
70
77
  /** Which browser backend serves a fetch. */
71
78
  export type PlaywrightBackend = 'local' | 'cdp'
72
79
 
@@ -113,7 +120,9 @@ export interface Config {
113
120
  * Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
114
121
  * the same page/context. `0` (or any config leaving this at 0) restores the
115
122
  * legacy behavior: the first response is final, no waiting — the A/B
116
- * baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
123
+ * baseline. Code-level only (tests, demo scripts): not part of the
124
+ * settings schema, so users cannot configure it. Default:
125
+ * {@link DEFAULT_CHALLENGE_WAIT_MS}.
117
126
  */
118
127
  challengeWaitMs?: number
119
128
  /**
@@ -122,13 +131,22 @@ export interface Config {
122
131
  * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
123
132
  */
124
133
  challengeRetries?: number
134
+ /**
135
+ * When the browser cannot clear a challenge inside its window (no
136
+ * checkbox offered, or verification refused), refetch the URL once
137
+ * through the page's plain-HTTP request context (same cookie jar) and
138
+ * return that document when it is not itself a challenge. `false`
139
+ * restores the browser-only posture.
140
+ */
141
+ challengeHttpFallback?: boolean
125
142
  /**
126
143
  * Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
127
144
  * that defeats viewport virtualization on feed-shaped pages (Discourse
128
145
  * topic streams: replies outside the render window are empty height-only
129
146
  * placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
130
- * restores the legacy single-instant capture.
131
- * Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
147
+ * restores the legacy single-instant capture. Code-level only (tests,
148
+ * demo scripts): not part of the settings schema, so users cannot
149
+ * configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
132
150
  */
133
151
  scrollBudgetMs?: number
134
152
  /**
@@ -154,11 +172,10 @@ export const Config = z.object({
154
172
  // Optional on purpose: the effective default depends on `backend`, which a
155
173
  // static schema default cannot express.
156
174
  maxConcurrency: z.number().step(1).min(1).max(MAX_CONCURRENCY_CEILING).volatile(),
157
- // The bounded natural-wait knobs; 0 disables the whole challenge path.
158
- challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
175
+ // The challenge wait budget and the deep-capture scroll budget are
176
+ // deliberately absent: they are code-level constants, not user settings.
159
177
  challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES).volatile(),
160
- // The deep-capture scroll walk's bounded budget; 0 disables the walk.
161
- scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile(),
178
+ challengeHttpFallback: z.boolean().default(DEFAULT_CHALLENGE_HTTP_FALLBACK).volatile(),
162
179
  })
163
180
 
164
181
  /**
@@ -184,12 +201,10 @@ export interface PlaywrightPluginConfig {
184
201
  * field owns no default — the backend resolves one).
185
202
  */
186
203
  maxConcurrency: Volatile<number | undefined>
187
- /** Challenge wait budget (ms) as a live reference. */
188
- challengeWaitMs: Volatile<number>
189
204
  /** Challenge retry count as a live reference. */
190
205
  challengeRetries: Volatile<number>
191
- /** Deep-capture scroll-walk budget (ms) as a live reference. */
192
- scrollBudgetMs: Volatile<number>
206
+ /** Challenge dead-end plain-HTTP refetch toggle as a live reference. */
207
+ challengeHttpFallback: Volatile<boolean>
193
208
  }
194
209
 
195
210
  /**
@@ -209,9 +224,8 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
209
224
  contextIsolationScope: config.contextIsolationScope.get(),
210
225
  denoise: config.denoise.get(),
211
226
  maxConcurrency: config.maxConcurrency.get(),
212
- challengeWaitMs: config.challengeWaitMs.get(),
213
227
  challengeRetries: config.challengeRetries.get(),
214
- scrollBudgetMs: config.scrollBudgetMs.get(),
228
+ challengeHttpFallback: config.challengeHttpFallback.get(),
215
229
  }
216
230
  }
217
231
 
@@ -226,7 +240,10 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
226
240
  * @returns the resolved config.
227
241
  */
228
242
  export function resolveConfig(config: Partial<Config>): ResolvedConfig {
229
- const resolved = Config(config as never) as PlaywrightPluginConfig
243
+ // The code-level budgets bypass the schema: they pass through untouched
244
+ // (only when explicitly given) and the effective* helpers default them.
245
+ const { challengeWaitMs, scrollBudgetMs, ...settings } = config
246
+ const resolved = Config(settings as never) as PlaywrightPluginConfig
230
247
  return {
231
248
  backend: resolved.backend.get(),
232
249
  playwrightPath: resolved.playwrightPath.get(),
@@ -235,21 +252,22 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
235
252
  contextIsolationScope: resolved.contextIsolationScope.get(),
236
253
  denoise: resolved.denoise.get(),
237
254
  maxConcurrency: resolved.maxConcurrency.get(),
238
- challengeWaitMs: resolved.challengeWaitMs.get(),
239
255
  challengeRetries: resolved.challengeRetries.get(),
240
- scrollBudgetMs: resolved.scrollBudgetMs.get(),
256
+ challengeHttpFallback: resolved.challengeHttpFallback.get(),
257
+ ...(challengeWaitMs === undefined ? {} : { challengeWaitMs }),
258
+ ...(scrollBudgetMs === undefined ? {} : { scrollBudgetMs }),
241
259
  }
242
260
  }
243
261
 
244
262
  /**
245
263
  * Complete config after schemastery applies the field defaults it owns.
246
- * `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
247
- * its effective default depends on `backend`, the latter so hand-built
248
- * resolved configs (tests) may omit it and let
249
- * {@link effectiveScrollBudgetMs} supply the default the schema would have.
264
+ * `maxConcurrency` stays optional because its effective default depends on
265
+ * `backend`; `challengeWaitMs` and `scrollBudgetMs` stay optional because
266
+ * they are code-level overrides outside the schema — absent, the
267
+ * effective* helpers supply the built-in defaults.
250
268
  */
251
- export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'>
252
- & { maxConcurrency?: number; scrollBudgetMs?: number }
269
+ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs'>
270
+ & { maxConcurrency?: number; challengeWaitMs?: number; scrollBudgetMs?: number }
253
271
 
254
272
  /**
255
273
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -266,7 +284,7 @@ export function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxCon
266
284
 
267
285
  /**
268
286
  * The challenge wait budget a fetch actually runs with: an explicit setting
269
- * wins, else the schema default. `0` means the challenge path is off.
287
+ * wins, else the built-in default. `0` means the challenge path is off.
270
288
  *
271
289
  * @param config - the resolved settings section (or any partial of it).
272
290
  * @returns the bounded wait in milliseconds.
@@ -288,9 +306,20 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
288
306
  return DEFAULT_CHALLENGE_RETRIES
289
307
  }
290
308
 
309
+ /**
310
+ * Whether a challenge dead-end may refetch through plain HTTP: an explicit
311
+ * setting wins (absent = on, the default).
312
+ *
313
+ * @param config - the resolved settings section (or any partial of it).
314
+ * @returns true when the plain-HTTP fallback may run.
315
+ */
316
+ export function effectiveChallengeHttpFallback(config: Pick<Config, 'challengeHttpFallback'>): boolean {
317
+ return config.challengeHttpFallback !== false
318
+ }
319
+
291
320
  /**
292
321
  * The scroll-walk budget a fetch actually runs with: an explicit setting
293
- * wins, else the schema default. `0` means the deep-capture walk is off.
322
+ * wins, else the built-in default. `0` means the deep-capture walk is off.
294
323
  *
295
324
  * @param config - the resolved settings section (or any partial of it).
296
325
  * @returns the bounded walk budget in milliseconds.
package/src/index.ts CHANGED
@@ -24,10 +24,11 @@ import { PlaywrightFetchProvider } from './provider.ts'
24
24
  export {
25
25
  Config,
26
26
  DEFAULT_CDP_ENDPOINT,
27
+ DEFAULT_CHALLENGE_HTTP_FALLBACK,
27
28
  DEFAULT_CHALLENGE_RETRIES,
28
29
  DEFAULT_CHALLENGE_WAIT_MS,
29
30
  DEFAULT_SCROLL_BUDGET_MS,
30
- MAX_SCROLL_BUDGET_MS,
31
+ effectiveChallengeHttpFallback,
31
32
  effectiveChallengeRetries,
32
33
  effectiveChallengeWaitMs,
33
34
  effectiveContextMode,
@@ -51,10 +52,12 @@ export type { ChallengeVerdict } from './challenge.ts'
51
52
  export { deepCaptureWalk } from './autoscroll.ts'
52
53
  export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
53
54
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
54
- export { htmlToMarkdown } from './markdown.ts'
55
+ export { blockTrainingRows, htmlToMarkdown, renderCandidate, scoreCandidates } from './markdown.ts'
56
+ export { BLOCK_FEATURES, setBlockModel } from './blocks.ts'
55
57
  export { extractPdfMarkdown, isPdfBytes, isPdfContentType, layoutPdfMarkdown, mayBePdf, MAX_PDF_BYTES, MAX_PDF_PAGES, readPdfDocument, WEB_PDF_ENCRYPTED_CODE, WEB_PDF_INVALID_CODE, WEB_PDF_NO_TEXT_CODE, WEB_PDF_TOO_LARGE_CODE } from './pdf.ts'
56
58
  export type { PdfDocumentText, PdfExtraction, PdfExtractOptions, PdfOutlineEntry, PdfPageText, PdfTextRun } from './pdf.ts'
57
- export type { DenoiseMode, DenoiseResult } from './markdown.ts'
59
+ export type { DenoiseMode, DenoiseResult, ScoredCandidate } from './markdown.ts'
60
+ export { FEATURE_NAMES, PAGE_FEATURES } from './selector.ts'
58
61
 
59
62
  /** Cordis plugin name used by loader diagnostics. */
60
63
  export const name = 'dsh-web-fetch-playwright'