dsh-web-fetch-playwright 0.2.15 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/lib/client.js +46 -21
- package/lib/index.js +1001 -84
- package/lib/types/autoscroll.d.ts +277 -0
- package/lib/types/challenge.d.ts +23 -9
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -3
- package/lib/types/index.d.ts +3 -1
- package/lib/types/markdown.d.ts +18 -7
- package/lib/types/provider.d.ts +25 -0
- package/lib/types/strip.d.ts +59 -0
- package/package.json +1 -1
- package/src/autoscroll.ts +653 -0
- package/src/challenge.ts +36 -11
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -3
- package/src/index.ts +5 -0
- package/src/markdown.ts +371 -14
- package/src/provider.ts +57 -68
- package/src/strip.ts +72 -0
package/src/challenge.ts
CHANGED
|
@@ -14,14 +14,22 @@
|
|
|
14
14
|
* 2. Content-level — the interstitial's localized `<title>` family
|
|
15
15
|
* ("Just a moment...", "请稍候…", ...) plus structural markers
|
|
16
16
|
* (`/cdn-cgi/challenge-platform/` scripts, `#challenge-*` elements,
|
|
17
|
-
* `
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
17
|
+
* `window._cf_chl_opt`). Markers are deliberately structural: a blog
|
|
18
|
+
* post that merely MENTIONS Cloudflare must not classify as a challenge,
|
|
19
|
+
* so the challenge-platform script check reads `<script src>` attributes
|
|
20
|
+
* only (never prose or escaped code samples) and identifies the runtime
|
|
21
|
+
* POSITIVELY by its `/orchestrate/` mount — the JSD/Precursor
|
|
22
|
+
* instrumentation Cloudflare injects into every NORMAL page of a
|
|
23
|
+
* protected zone, on several moving mounts, is not a challenge signal.
|
|
24
|
+
* The Turnstile widget id (`cf-chl-widget-…`) is deliberately NOT a
|
|
25
|
+
* marker: sites embed real Turnstile widgets into normal pages
|
|
26
|
+
* (linux.do — a Discourse forum with the Turnstile plugin mounts one on
|
|
27
|
+
* every topic page), so the widget alone cannot distinguish an
|
|
28
|
+
* interstitial from a normal page behind a Cloudflare edge. Every real
|
|
29
|
+
* interstitial carries at least one independent signal beyond its
|
|
30
|
+
* widget — the localized title, the orchestrate runtime script,
|
|
31
|
+
* `window._cf_chl_opt`, or the `#challenge-*` scaffolding — and those
|
|
32
|
+
* stay sufficient on their own (see {@link CHALLENGE_HTML_MARKERS}).
|
|
25
33
|
*
|
|
26
34
|
* The security stance (issue #2): this module only DETECTS and classifies.
|
|
27
35
|
* Solving, spoofing, CAPTCHA answering, and cookie lifting live outside the
|
|
@@ -176,9 +184,21 @@ function isChallengeRuntimeScript(src: string): boolean {
|
|
|
176
184
|
* The challenge-platform prefix is deliberately absent: script evidence goes
|
|
177
185
|
* through {@link scriptSrcsOf} + {@link isChallengeRuntimeScript}, which
|
|
178
186
|
* distinguish a real `src` from prose and runtime from instrumentation.
|
|
187
|
+
*
|
|
188
|
+
* The Turnstile widget id (`id="cf-chl-widget-…"`) is deliberately ABSENT
|
|
189
|
+
* (badcase: linux.do/t/topic/3143588 — a Discourse forum behind Cloudflare
|
|
190
|
+
* whose Turnstile plugin mounts a real widget, id and all, into every normal
|
|
191
|
+
* topic page it serves at HTTP 200). The widget is minted by the same
|
|
192
|
+
* Turnstile runtime whether it renders inside an interstitial or inside a
|
|
193
|
+
* site's own UI, so its presence proves only "this page embeds Turnstile".
|
|
194
|
+
* On a real interstitial the widget always co-occurs with independent proof
|
|
195
|
+
* — the localized title, the orchestrate runtime script,
|
|
196
|
+
* `window._cf_chl_opt`, or the `#challenge-*` scaffolding, each sufficient
|
|
197
|
+
* above — so dropping the widget loses no real interstitial while every
|
|
198
|
+
* widget-embedding normal page (linux.do, login portals, contact forms)
|
|
199
|
+
* stops dying as WEB_FETCH_CHALLENGE after a wait that can never clear.
|
|
179
200
|
*/
|
|
180
201
|
const CHALLENGE_HTML_MARKERS = [
|
|
181
|
-
'id="cf-chl-widget-',
|
|
182
202
|
'window._cf_chl_opt',
|
|
183
203
|
'id="challenge-form"',
|
|
184
204
|
'id="challenge-running"',
|
|
@@ -295,7 +315,13 @@ export function classifyChallengeHtml(html: string): ChallengeVerdict {
|
|
|
295
315
|
* check uses, so the two can never drift apart. Script evidence follows the
|
|
296
316
|
* same rule as {@link classifyChallengeHtml}: only the challenge runtime
|
|
297
317
|
* counts, and the instrumentation Cloudflare injects into normal pages of a
|
|
298
|
-
* protected zone is ignored.
|
|
318
|
+
* protected zone is ignored. The Turnstile widget selector is likewise
|
|
319
|
+
* absent for the same reason {@link CHALLENGE_HTML_MARKERS} omits it: a
|
|
320
|
+
* widget embedded in a NORMAL page (linux.do's Turnstile plugin) would keep
|
|
321
|
+
* the probe reporting "still challenged" forever, so a challenge that HAD
|
|
322
|
+
* cleared could never be confirmed — the wait burned its whole budget and
|
|
323
|
+
* the fetch died as WEB_FETCH_CHALLENGE (last status 200) or the tool
|
|
324
|
+
* layer's timeout.
|
|
299
325
|
*/
|
|
300
326
|
export const CHALLENGE_DOM_PROBE = `(() => {
|
|
301
327
|
const re = new RegExp(${JSON.stringify(CHALLENGE_TITLE_RE.source)}, ${JSON.stringify(CHALLENGE_TITLE_RE.flags)})
|
|
@@ -303,7 +329,6 @@ export const CHALLENGE_DOM_PROBE = `(() => {
|
|
|
303
329
|
const runtime = ${JSON.stringify(CHALLENGE_RUNTIME_SEGMENT)}
|
|
304
330
|
if (re.test(document.title.trim())) return true
|
|
305
331
|
if (document.querySelector('#challenge-form, #challenge-running, #challenge-stage, #challenge-error-text')) return true
|
|
306
|
-
if (document.querySelector('[id^="cf-chl-widget-"]')) return true
|
|
307
332
|
for (const script of Array.from(document.scripts)) {
|
|
308
333
|
const src = (script.getAttribute('src') || '').toLowerCase()
|
|
309
334
|
if (src.includes(platform) && src.includes(runtime)) return true
|
package/src/client/card.tsx
CHANGED
|
@@ -190,6 +190,19 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
|
|
|
190
190
|
onEdit={(text) => { props.edit('challengeWaitMs', text) }}
|
|
191
191
|
onReset={() => { props.resetField('challengeWaitMs') }}
|
|
192
192
|
/>
|
|
193
|
+
<ValueField
|
|
194
|
+
id="plugin-config-playwright-scroll-budget"
|
|
195
|
+
label={t('scrollBudgetMs')}
|
|
196
|
+
hint={t('scrollBudgetMsHint')}
|
|
197
|
+
placeholder={t('scrollBudgetMsPlaceholder')}
|
|
198
|
+
overriddenLabel={t('overridden')}
|
|
199
|
+
resetLabel={t('reset')}
|
|
200
|
+
invalidLabel={t('invalidText')}
|
|
201
|
+
disabled={disabled}
|
|
202
|
+
{...state.scrollBudgetMs}
|
|
203
|
+
onEdit={(text) => { props.edit('scrollBudgetMs', text) }}
|
|
204
|
+
onReset={() => { props.resetField('scrollBudgetMs') }}
|
|
205
|
+
/>
|
|
193
206
|
</CardForm>
|
|
194
207
|
)
|
|
195
208
|
}
|
package/src/client/controller.ts
CHANGED
|
@@ -46,6 +46,11 @@ export interface PlaywrightSettings {
|
|
|
46
46
|
* same tab (0–60000); 0 = off (return the first response as-is).
|
|
47
47
|
*/
|
|
48
48
|
challengeWaitMs?: number
|
|
49
|
+
/**
|
|
50
|
+
* Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
|
|
51
|
+
* (capture the document as it rests, the legacy single-instant behavior).
|
|
52
|
+
*/
|
|
53
|
+
scrollBudgetMs?: number
|
|
49
54
|
}
|
|
50
55
|
|
|
51
56
|
/** What the Playwright card renders. */
|
|
@@ -65,6 +70,8 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
65
70
|
maxConcurrency: CardFieldState
|
|
66
71
|
/** Challenge wait input (draft decimal integer of milliseconds). */
|
|
67
72
|
challengeWaitMs: CardFieldState
|
|
73
|
+
/** Scroll-walk budget input (draft decimal integer of milliseconds). */
|
|
74
|
+
scrollBudgetMs: CardFieldState
|
|
68
75
|
}
|
|
69
76
|
|
|
70
77
|
/** The registration-side face the card's slot entry injects. */
|
|
@@ -97,6 +104,7 @@ export class PlaywrightCardController {
|
|
|
97
104
|
checkboxField('denoise'),
|
|
98
105
|
numberField('maxConcurrency', 1, 200),
|
|
99
106
|
numberField('challengeWaitMs', 0, 60_000),
|
|
107
|
+
numberField('scrollBudgetMs', 0, 30_000),
|
|
100
108
|
],
|
|
101
109
|
)
|
|
102
110
|
this.store = this.form.bind(() => this.projection())
|
|
@@ -113,6 +121,7 @@ export class PlaywrightCardController {
|
|
|
113
121
|
denoise: this.form.field('denoise'),
|
|
114
122
|
maxConcurrency: this.form.field('maxConcurrency'),
|
|
115
123
|
challengeWaitMs: this.form.field('challengeWaitMs'),
|
|
124
|
+
scrollBudgetMs: this.form.field('scrollBudgetMs'),
|
|
116
125
|
}
|
|
117
126
|
}
|
|
118
127
|
|
package/src/client/locales.ts
CHANGED
|
@@ -18,6 +18,7 @@ export type PlaywrightCardLocaleKey =
|
|
|
18
18
|
| 'denoise' | 'denoiseHint'
|
|
19
19
|
| 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
|
|
20
20
|
| 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
|
|
21
|
+
| 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder'
|
|
21
22
|
| 'overridden' | 'reset' | 'readOnly'
|
|
22
23
|
| 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText'
|
|
23
24
|
|
|
@@ -59,6 +60,9 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
59
60
|
challengeWaitMs: 'Cloudflare challenge wait (ms)',
|
|
60
61
|
challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
|
|
61
62
|
challengeWaitMsPlaceholder: '(default: 15000)',
|
|
63
|
+
scrollBudgetMs: 'Deep-capture scroll budget (ms)',
|
|
64
|
+
scrollBudgetMsHint: 'Bounded in-page scroll walk that captures viewport-virtualized feeds (forum topics, comment threads) whole: posts outside the render window are empty placeholders at rest. 0–30000; 0 = off (capture the page as it rests).',
|
|
65
|
+
scrollBudgetMsPlaceholder: '(default: 20000)',
|
|
62
66
|
overridden: 'Overridden',
|
|
63
67
|
reset: 'Reset to default',
|
|
64
68
|
readOnly: 'This deployment stores settings read-only.',
|
|
@@ -100,6 +104,9 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
100
104
|
challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
|
|
101
105
|
challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
|
|
102
106
|
challengeWaitMsPlaceholder: '(默认:15000)',
|
|
107
|
+
scrollBudgetMs: '深度抓取滚动预算(毫秒)',
|
|
108
|
+
scrollBudgetMsHint: '有界的页内滚动遍历,完整捕获视口虚拟化的信息流(论坛帖子、评论区):静止时渲染窗口外的帖子只是空占位符。0–30000;0 = 关闭(按页面静止状态抓取)。',
|
|
109
|
+
scrollBudgetMsPlaceholder: '(默认:20000)',
|
|
103
110
|
overridden: '已覆盖',
|
|
104
111
|
reset: '恢复默认',
|
|
105
112
|
readOnly: '本部署的设置为只读。',
|
package/src/config.ts
CHANGED
|
@@ -16,6 +16,19 @@
|
|
|
16
16
|
import type { Volatile } from '@deepseek-ai/cordis'
|
|
17
17
|
import z from '@deepseek-ai/schemastery'
|
|
18
18
|
|
|
19
|
+
/**
|
|
20
|
+
* Default bounded budget (ms) for the deep-capture scroll walk
|
|
21
|
+
* (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
|
|
22
|
+
* careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
|
|
23
|
+
* the bottom patience window): 20s walks ~25kpx with pagination, covering
|
|
24
|
+
* the measured linux.do badcase (~18kpx, 51 replies) with headroom.
|
|
25
|
+
* Scroll-inert pages take the walk's fast path and spend a few seconds.
|
|
26
|
+
*/
|
|
27
|
+
export const DEFAULT_SCROLL_BUDGET_MS = 20_000
|
|
28
|
+
|
|
29
|
+
/** Ceiling the schema accepts for `scrollBudgetMs`. */
|
|
30
|
+
export const MAX_SCROLL_BUDGET_MS = 30_000
|
|
31
|
+
|
|
19
32
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
20
33
|
export const DEFAULT_CDP_ENDPOINT = '127.0.0.1:9222'
|
|
21
34
|
|
|
@@ -109,6 +122,15 @@ export interface Config {
|
|
|
109
122
|
* Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
|
|
110
123
|
*/
|
|
111
124
|
challengeRetries?: number
|
|
125
|
+
/**
|
|
126
|
+
* Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
|
|
127
|
+
* that defeats viewport virtualization on feed-shaped pages (Discourse
|
|
128
|
+
* topic streams: replies outside the render window are empty height-only
|
|
129
|
+
* placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
|
|
130
|
+
* restores the legacy single-instant capture.
|
|
131
|
+
* Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
|
|
132
|
+
*/
|
|
133
|
+
scrollBudgetMs?: number
|
|
112
134
|
/**
|
|
113
135
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
114
136
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -135,6 +157,8 @@ export const Config = z.object({
|
|
|
135
157
|
// The bounded natural-wait knobs; 0 disables the whole challenge path.
|
|
136
158
|
challengeWaitMs: z.number().step(100).min(0).max(MAX_CHALLENGE_WAIT_MS).default(DEFAULT_CHALLENGE_WAIT_MS).volatile(),
|
|
137
159
|
challengeRetries: z.number().step(1).min(0).max(MAX_CHALLENGE_RETRIES).default(DEFAULT_CHALLENGE_RETRIES).volatile(),
|
|
160
|
+
// The deep-capture scroll walk's bounded budget; 0 disables the walk.
|
|
161
|
+
scrollBudgetMs: z.number().step(100).min(0).max(MAX_SCROLL_BUDGET_MS).default(DEFAULT_SCROLL_BUDGET_MS).volatile(),
|
|
138
162
|
})
|
|
139
163
|
|
|
140
164
|
/**
|
|
@@ -164,6 +188,8 @@ export interface PlaywrightPluginConfig {
|
|
|
164
188
|
challengeWaitMs: Volatile<number>
|
|
165
189
|
/** Challenge retry count as a live reference. */
|
|
166
190
|
challengeRetries: Volatile<number>
|
|
191
|
+
/** Deep-capture scroll-walk budget (ms) as a live reference. */
|
|
192
|
+
scrollBudgetMs: Volatile<number>
|
|
167
193
|
}
|
|
168
194
|
|
|
169
195
|
/**
|
|
@@ -185,6 +211,7 @@ export function snapshotsOf(config: PlaywrightPluginConfig): Config {
|
|
|
185
211
|
maxConcurrency: config.maxConcurrency.get(),
|
|
186
212
|
challengeWaitMs: config.challengeWaitMs.get(),
|
|
187
213
|
challengeRetries: config.challengeRetries.get(),
|
|
214
|
+
scrollBudgetMs: config.scrollBudgetMs.get(),
|
|
188
215
|
}
|
|
189
216
|
}
|
|
190
217
|
|
|
@@ -210,15 +237,19 @@ export function resolveConfig(config: Partial<Config>): ResolvedConfig {
|
|
|
210
237
|
maxConcurrency: resolved.maxConcurrency.get(),
|
|
211
238
|
challengeWaitMs: resolved.challengeWaitMs.get(),
|
|
212
239
|
challengeRetries: resolved.challengeRetries.get(),
|
|
240
|
+
scrollBudgetMs: resolved.scrollBudgetMs.get(),
|
|
213
241
|
}
|
|
214
242
|
}
|
|
215
243
|
|
|
216
244
|
/**
|
|
217
245
|
* Complete config after schemastery applies the field defaults it owns.
|
|
218
|
-
* `maxConcurrency`
|
|
219
|
-
*
|
|
246
|
+
* `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
|
|
247
|
+
* its effective default depends on `backend`, the latter so hand-built
|
|
248
|
+
* resolved configs (tests) may omit it and let
|
|
249
|
+
* {@link effectiveScrollBudgetMs} supply the default the schema would have.
|
|
220
250
|
*/
|
|
221
|
-
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'
|
|
251
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'>
|
|
252
|
+
& { maxConcurrency?: number; scrollBudgetMs?: number }
|
|
222
253
|
|
|
223
254
|
/**
|
|
224
255
|
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
@@ -257,6 +288,18 @@ export function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries
|
|
|
257
288
|
return DEFAULT_CHALLENGE_RETRIES
|
|
258
289
|
}
|
|
259
290
|
|
|
291
|
+
/**
|
|
292
|
+
* The scroll-walk budget a fetch actually runs with: an explicit setting
|
|
293
|
+
* wins, else the schema default. `0` means the deep-capture walk is off.
|
|
294
|
+
*
|
|
295
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
296
|
+
* @returns the bounded walk budget in milliseconds.
|
|
297
|
+
*/
|
|
298
|
+
export function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>): number {
|
|
299
|
+
if (typeof config.scrollBudgetMs === 'number') return config.scrollBudgetMs
|
|
300
|
+
return DEFAULT_SCROLL_BUDGET_MS
|
|
301
|
+
}
|
|
302
|
+
|
|
260
303
|
/**
|
|
261
304
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
262
305
|
* browser's real profile) requires the CDP backend — a local launch has no
|
package/src/index.ts
CHANGED
|
@@ -26,9 +26,12 @@ export {
|
|
|
26
26
|
DEFAULT_CDP_ENDPOINT,
|
|
27
27
|
DEFAULT_CHALLENGE_RETRIES,
|
|
28
28
|
DEFAULT_CHALLENGE_WAIT_MS,
|
|
29
|
+
DEFAULT_SCROLL_BUDGET_MS,
|
|
30
|
+
MAX_SCROLL_BUDGET_MS,
|
|
29
31
|
effectiveChallengeRetries,
|
|
30
32
|
effectiveChallengeWaitMs,
|
|
31
33
|
effectiveContextMode,
|
|
34
|
+
effectiveScrollBudgetMs,
|
|
32
35
|
normalizeCdpEndpoint,
|
|
33
36
|
resolveConfig,
|
|
34
37
|
snapshotsOf,
|
|
@@ -45,6 +48,8 @@ export { CdpConnectionPool } from './cdp-pool.ts'
|
|
|
45
48
|
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts'
|
|
46
49
|
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts'
|
|
47
50
|
export type { ChallengeVerdict } from './challenge.ts'
|
|
51
|
+
export { deepCaptureWalk } from './autoscroll.ts'
|
|
52
|
+
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts'
|
|
48
53
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts'
|
|
49
54
|
export { htmlToMarkdown } from './markdown.ts'
|
|
50
55
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts'
|