dsh-web-fetch-playwright 0.2.15 → 0.2.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/challenge.ts CHANGED
@@ -14,14 +14,22 @@
14
14
  * 2. Content-level — the interstitial's localized `<title>` family
15
15
  * ("Just a moment...", "请稍候…", ...) plus structural markers
16
16
  * (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
17
- * `cf-chl-widget-` Turnstile frames, `window._cf_chl_opt`). Markers are
18
- * deliberately structural: a blog post that merely MENTIONS Cloudflare
19
- * must not classify as a challenge, so the challenge-platform script
20
- * check reads `<script src>` attributes only (never prose or escaped
21
- * code samples) and identifies the runtime POSITIVELY by its
22
- * `/orchestrate/` mount — the JSD/Precursor instrumentation Cloudflare
23
- * injects into every NORMAL page of a protected zone, on several moving
24
- * mounts, is not a challenge signal.
17
+ * `window._cf_chl_opt`). Markers are deliberately structural: a blog
18
+ * post that merely MENTIONS Cloudflare must not classify as a challenge,
19
+ * so the challenge-platform script check reads `<script src>` attributes
20
+ * only (never prose or escaped code samples) and identifies the runtime
21
+ * POSITIVELY by its `/orchestrate/` mount — the JSD/Precursor
22
+ * instrumentation Cloudflare injects into every NORMAL page of a
23
+ * protected zone, on several moving mounts, is not a challenge signal.
24
+ * The Turnstile widget id (`cf-chl-widget-…`) is deliberately NOT a
25
+ * marker: sites embed real Turnstile widgets into normal pages
26
+ * (linux.do — a Discourse forum with the Turnstile plugin mounts one on
27
+ * every topic page), so the widget alone cannot distinguish an
28
+ * interstitial from a normal page behind a Cloudflare edge. Every real
29
+ * interstitial carries at least one independent signal beyond its
30
+ * widget — the localized title, the orchestrate runtime script,
31
+ * `window._cf_chl_opt`, or the `#challenge-*` scaffolding — and those
32
+ * stay sufficient on their own (see {@link CHALLENGE_HTML_MARKERS}).
25
33
  *
26
34
  * The security stance (issue #2): this module only DETECTS and classifies.
27
35
  * Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
@@ -176,9 +184,21 @@ function isChallengeRuntimeScript(src: string): boolean {
176
184
  * The challenge-platform prefix is deliberately absent: script evidence goes
177
185
  * through {@link scriptSrcsOf} + {@link isChallengeRuntimeScript}, which
178
186
  * distinguish a real `src` from prose and runtime from instrumentation.
187
+ *
188
+ * The Turnstile widget id (`id="cf-chl-widget-…"`) is deliberately ABSENT
189
+ * (badcase: linux.do/t/topic/3143588 — a Discourse forum behind Cloudflare
190
+ * whose Turnstile plugin mounts a real widget, id and all, into every normal
191
+ * topic page it serves at HTTP 200). The widget is minted by the same
192
+ * Turnstile runtime whether it renders inside an interstitial or inside a
193
+ * site's own UI, so its presence proves only "this page embeds Turnstile".
194
+ * On a real interstitial the widget always co-occurs with independent proof
195
+ * — the localized title, the orchestrate runtime script,
196
+ * `window._cf_chl_opt`, or the `#challenge-*` scaffolding, each sufficient
197
+ * above — so dropping the widget loses no real interstitial while every
198
+ * widget-embedding normal page (linux.do, login portals, contact forms)
199
+ * stops dying as WEB_FETCH_CHALLENGE after a wait that can never clear.
179
200
  */
180
201
  const CHALLENGE_HTML_MARKERS = [
181
- 'id="cf-chl-widget-',
182
202
  'window._cf_chl_opt',
183
203
  'id="challenge-form"',
184
204
  'id="challenge-running"',
@@ -295,7 +315,13 @@ export function classifyChallengeHtml(html: string): ChallengeVerdict {
295
315
  * check uses, so the two can never drift apart. Script evidence follows the
296
316
  * same rule as {@link classifyChallengeHtml}: only the challenge runtime
297
317
  * counts, and the instrumentation Cloudflare injects into normal pages of a
298
- * protected zone is ignored.
318
+ * protected zone is ignored. The Turnstile widget selector is likewise
319
+ * absent for the same reason {@link CHALLENGE_HTML_MARKERS} omits it: a
320
+ * widget embedded in a NORMAL page (linux.do's Turnstile plugin) would keep
321
+ * the probe reporting "still challenged" forever, so a challenge that HAD
322
+ * cleared could never be confirmed — the wait burned its whole budget and
323
+ * the fetch died as WEB_FETCH_CHALLENGE (last status 200) or the tool
324
+ * layer's timeout.
299
325
  */
300
326
  export const CHALLENGE_DOM_PROBE = `(() => {
301
327
  const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
@@ -303,7 +329,6 @@ export const CHALLENGE_DOM_PROBE = `(() => {
303
329
  const runtime = ${JSON.stringify(CHALLENGE_RUNTIME_SEGMENT)}
304
330
  if (re.test(document.title.trim())) return true
305
331
  if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
306
- if (document.querySelector('[id^="cf-chl-widget-"]')) return true
307
332
  for (const script of Array.from(document.scripts)) {
308
333
  const src = (script.getAttribute('src') || '').toLowerCase()
309
334
  if (src.includes(platform) && src.includes(runtime)) return true
@@ -190,6 +190,19 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
190
190
  onEdit={(text) => { props.edit('challengeWaitMs', text) }}
191
191
  onReset={() => { props.resetField('challengeWaitMs') }}
192
192
  />
193
+ <ValueField
194
+ id="plugin-config-playwright-scroll-budget"
195
+ label={t('scrollBudgetMs')}
196
+ hint={t('scrollBudgetMsHint')}
197
+ placeholder={t('scrollBudgetMsPlaceholder')}
198
+ overriddenLabel={t('overridden')}
199
+ resetLabel={t('reset')}
200
+ invalidLabel={t('invalidText')}
201
+ disabled={disabled}
202
+ {...state.scrollBudgetMs}
203
+ onEdit={(text) => { props.edit('scrollBudgetMs', text) }}
204
+ onReset={() => { props.resetField('scrollBudgetMs') }}
205
+ />
193
206
  </CardForm>
194
207
  )
195
208
  }
@@ -46,6 +46,11 @@ export interface PlaywrightSettings {
46
46
  * same tab (0–60000); 0 = off (return the first response as-is).
47
47
  */
48
48
  challengeWaitMs?: number
49
+ /**
50
+ * Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
51
+ * (capture the document as it rests, the legacy single-instant behavior).
52
+ */
53
+ scrollBudgetMs?: number
49
54
  }
50
55
 
51
56
  /** What the Playwright card renders. */
@@ -65,6 +70,8 @@ export interface PlaywrightCardState extends CardShell {
65
70
  maxConcurrency: CardFieldState
66
71
  /** Challenge wait input (draft decimal integer of milliseconds). */
67
72
  challengeWaitMs: CardFieldState
73
+ /** Scroll-walk budget input (draft decimal integer of milliseconds). */
74
+ scrollBudgetMs: CardFieldState
68
75
  }
69
76
 
70
77
  /** The registration-side face the card's slot entry injects. */
@@ -97,6 +104,7 @@ export class PlaywrightCardController {
97
104
  checkboxField('denoise'),
98
105
  numberField('maxConcurrency', 1, 200),
99
106
  numberField('challengeWaitMs', 0, 60_000),
107
+ numberField('scrollBudgetMs', 0, 30_000),
100
108
  ],
101
109
  )
102
110
  this.store = this.form.bind(() => this.projection())
@@ -113,6 +121,7 @@ export class PlaywrightCardController {
113
121
  denoise: this.form.field('denoise'),
114
122
  maxConcurrency: this.form.field('maxConcurrency'),
115
123
  challengeWaitMs: this.form.field('challengeWaitMs'),
124
+ scrollBudgetMs: this.form.field('scrollBudgetMs'),
116
125
  }
117
126
  }
118
127
 
@@ -18,6 +18,7 @@ export type PlaywrightCardLocaleKey =
18
18
  | 'denoise' | 'denoiseHint'
19
19
  | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
20
20
  | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
21
+ | 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder'
21
22
  | 'overridden' | 'reset' | 'readOnly'
22
23
  | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText'
23
24
 
@@ -59,6 +60,9 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
59
60
  challengeWaitMs: 'Cloudflare challenge wait (ms)',
60
61
  challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
61
62
  challengeWaitMsPlaceholder: '(default: 15000)',
63
+ scrollBudgetMs: 'Deep-capture scroll budget (ms)',
64
+ scrollBudgetMsHint: 'Bounded in-page scroll walk that captures viewport-virtualized feeds (forum topics, comment threads) whole: posts outside the render window are empty placeholders at rest. 0–30000; 0 = off (capture the page as it rests).',
65
+ scrollBudgetMsPlaceholder: '(default: 20000)',
62
66
  overridden: 'Overridden',
63
67
  reset: 'Reset to default',
64
68
  readOnly: 'This deployment stores settings read-only.',
@@ -100,6 +104,9 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
100
104
  challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
101
105
  challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
102
106
  challengeWaitMsPlaceholder: '(默认:15000)',
107
+ scrollBudgetMs: '深度抓取滚动预算(毫秒)',
108
+ scrollBudgetMsHint: '有界的页内滚动遍历,完整捕获视口虚拟化的信息流(论坛帖子、评论区):静止时渲染窗口外的帖子只是空占位符。0–30000;0 = 关闭(按页面静止状态抓取)。',
109
+ scrollBudgetMsPlaceholder: '(默认:20000)',
103
110
  overridden: '已覆盖',
104
111
  reset: '恢复默认',
105
112
  readOnly: '本部署的设置为只读。',
package/src/config.ts CHANGED
@@ -16,6 +16,19 @@
16
16
  import type { Volatile } from '@deepseek-ai/cordis'
17
17
  import z from '@deepseek-ai/schemastery'
18
18
 
19
+ /**
20
+ * Default bounded budget (ms) for the deep-capture scroll walk
21
+ * (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
22
+ * careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
23
+ * the bottom patience window): 20s walks ~25kpx with pagination, covering
24
+ * the measured linux.do badcase (~18kpx, 51 replies) with headroom.
25
+ * Scroll-inert pages take the walk's fast path and spend a few seconds.
26
+ */
27
+ export const DEFAULT_SCROLL_BUDGET_MS = 20_000
28
+
29
+ /** Ceiling the schema accepts for `scrollBudgetMs`. */
30
+ export const MAX_SCROLL_BUDGET_MS = 30_000
31
+
19
32
  /** Default CDP endpoint when the settings section leaves it blank. */
20
33
  export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
21
34
 
@@ -109,6 +122,15 @@ export interface Config {
109
122
  * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
110
123
  */
111
124
  challengeRetries?: number
125
+ /**
126
+ * Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
127
+ * that defeats viewport virtualization on feed-shaped pages (Discourse
128
+ * topic streams: replies outside the render window are empty height-only
129
+ * placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
130
+ * restores the legacy single-instant capture.
131
+ * Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
132
+ */
133
+ scrollBudgetMs?: number
112
134
  /**
113
135
  * How many fetches may render at once. Blank = backend default (4 for
114
136
  * local — each slot launches a browser; 50 for CDP — each slot is a tab in
@@ -135,6 +157,8 @@ export const Config = z.object({
135
157
  // The bounded natural-wait knobs; 0 disables the whole challenge path.
136
158
  challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
137
159
  challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES).volatile(),
160
+ // The deep-capture scroll walk's bounded budget; 0 disables the walk.
161
+ scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile(),
138
162
  })
139
163
 
140
164
  /**
@@ -164,6 +188,8 @@ export interface PlaywrightPluginConfig {
164
188
  challengeWaitMs: Volatile<number>
165
189
  /** Challenge retry count as a live reference. */
166
190
  challengeRetries: Volatile<number>
191
+ /** Deep-capture scroll-walk budget (ms) as a live reference. */
192
+ scrollBudgetMs: Volatile<number>
167
193
  }
168
194
 
169
195
  /**
@@ -185,6 +211,7 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
185
211
  maxConcurrency: config.maxConcurrency.get(),
186
212
  challengeWaitMs: config.challengeWaitMs.get(),
187
213
  challengeRetries: config.challengeRetries.get(),
214
+ scrollBudgetMs: config.scrollBudgetMs.get(),
188
215
  }
189
216
  }
190
217
 
@@ -210,15 +237,19 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
210
237
  maxConcurrency: resolved.maxConcurrency.get(),
211
238
  challengeWaitMs: resolved.challengeWaitMs.get(),
212
239
  challengeRetries: resolved.challengeRetries.get(),
240
+ scrollBudgetMs: resolved.scrollBudgetMs.get(),
213
241
  }
214
242
  }
215
243
 
216
244
  /**
217
245
  * Complete config after schemastery applies the field defaults it owns.
218
- * `maxConcurrency` stays optional — {@link effectiveMaxConcurrency} resolves
219
- * it against the backend.
246
+ * `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
247
+ * its effective default depends on `backend`, the latter so hand-built
248
+ * resolved configs (tests) may omit it and let
249
+ * {@link effectiveScrollBudgetMs} supply the default the schema would have.
220
250
  */
221
- export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & { maxConcurrency?: number }
251
+ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'>
252
+ & { maxConcurrency?: number; scrollBudgetMs?: number }
222
253
 
223
254
  /**
224
255
  * The concurrency limit a fetch actually runs with: an explicit setting
@@ -257,6 +288,18 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
257
288
  return DEFAULT_CHALLENGE_RETRIES
258
289
  }
259
290
 
291
+ /**
292
+ * The scroll-walk budget a fetch actually runs with: an explicit setting
293
+ * wins, else the schema default. `0` means the deep-capture walk is off.
294
+ *
295
+ * @param config - the resolved settings section (or any partial of it).
296
+ * @returns the bounded walk budget in milliseconds.
297
+ */
298
+ export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>): number {
299
+ if (typeof config.scrollBudgetMs === 'number') return config.scrollBudgetMs
300
+ return DEFAULT_SCROLL_BUDGET_MS
301
+ }
302
+
260
303
  /**
261
304
  * The context mode a fetch actually runs with: profile (a tab in the remote
262
305
  * browser's real profile) requires the CDP backend — a local launch has no
package/src/index.ts CHANGED
@@ -26,9 +26,12 @@ export {
26
26
  DEFAULT_CDP_ENDPOINT,
27
27
  DEFAULT_CHALLENGE_RETRIES,
28
28
  DEFAULT_CHALLENGE_WAIT_MS,
29
+ DEFAULT_SCROLL_BUDGET_MS,
30
+ MAX_SCROLL_BUDGET_MS,
29
31
  effectiveChallengeRetries,
30
32
  effectiveChallengeWaitMs,
31
33
  effectiveContextMode,
34
+ effectiveScrollBudgetMs,
32
35
  normalizeCdpEndpoint,
33
36
  resolveConfig,
34
37
  snapshotsOf,
@@ -45,6 +48,8 @@ export { CdpConnectionPool } from './cdp-pool.ts'
45
48
  export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
46
49
  export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
47
50
  export type { ChallengeVerdict } from './challenge.ts'
51
+ export { deepCaptureWalk } from './autoscroll.ts'
52
+ export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
48
53
  export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
49
54
  export { htmlToMarkdown } from './markdown.ts'
50
55
  export type { DenoiseMode, DenoiseResult } from './markdown.ts'