dsh-web-fetch-playwright 0.2.20 → 0.2.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -17
- package/README.zh-CN.md +19 -17
- package/cordis.patch.yml +0 -4
- package/lib/client.js +18 -68
- package/lib/index.js +24049 -22754
- package/lib/types/autoscroll.d.ts +46 -10
- package/lib/types/candidates.d.ts +55 -0
- package/lib/types/challenge.d.ts +2 -2
- package/lib/types/client/controller.d.ts +0 -14
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +80 -30
- package/lib/types/index.d.ts +1 -1
- package/lib/types/provider.d.ts +101 -9
- package/lib/types/types.d.ts +26 -6
- package/package.json +1 -1
- package/src/autoscroll.ts +459 -17
- package/src/block-model.ts +1 -1
- package/src/candidates.ts +184 -35
- package/src/challenge.ts +2 -2
- package/src/client/card.tsx +0 -26
- package/src/client/controller.ts +0 -18
- package/src/client/locales.ts +0 -14
- package/src/config.ts +97 -37
- package/src/index.ts +4 -1
- package/src/markdown.ts +13 -2
- package/src/provider.ts +482 -56
- package/src/selector-model.ts +1 -1
- package/src/selector.ts +18 -7
- package/src/types.ts +27 -7
package/src/candidates.ts
CHANGED
|
@@ -42,6 +42,15 @@ export interface RawCandidate {
|
|
|
42
42
|
kind: CandidateKind
|
|
43
43
|
/** HTML of the extraction; sanitized by the caller. */
|
|
44
44
|
html: string
|
|
45
|
+
/**
|
|
46
|
+
* The extraction as a live DOM node, when the generator already holds one
|
|
47
|
+
* (`doc`/`docc`/`ns_doc` serialize `body`, `main` the container itself).
|
|
48
|
+
* DOMPurify sanitizes a node by importing a copy — the caller's re-parse
|
|
49
|
+
* of `html` (hundreds of ms on large pages, measured on the Angular docs
|
|
50
|
+
* badcase: 555ms for the whole-body candidate alone) is skipped; `html`
|
|
51
|
+
* stays the identity for the dedup below.
|
|
52
|
+
*/
|
|
53
|
+
node?: Element
|
|
45
54
|
/** Title to prepend when the body does not open with a heading. */
|
|
46
55
|
title?: string
|
|
47
56
|
/** Whether the 0.2.18 pipeline would have returned exactly this candidate. */
|
|
@@ -113,8 +122,20 @@ const GATE_MIN_ARTICLE_COVERAGE = 0.1
|
|
|
113
122
|
export function articleCoversPage(articleHtml: string, document: Document): boolean {
|
|
114
123
|
const bodyText = visibleTextLength(document.body)
|
|
115
124
|
if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
|
|
116
|
-
|
|
117
|
-
|
|
125
|
+
return articleCoversParsed(htmlFragment(articleHtml, document), bodyText)
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* {@link articleCoversPage} on an already-parsed article container, so the
|
|
130
|
+
* generator's gate and trailing prune share ONE parse of the article — the
|
|
131
|
+
* extraction is hundreds of KB on heavy pages (measured on the Angular docs
|
|
132
|
+
* badcase: ~400ms per parse of Readability's 485KB pick).
|
|
133
|
+
*
|
|
134
|
+
* @param container - the parsed article (read, never mutated).
|
|
135
|
+
* @param bodyText - the page body's visible text length, computed once.
|
|
136
|
+
*/
|
|
137
|
+
function articleCoversParsed(container: Element, bodyText: number): boolean {
|
|
138
|
+
if (bodyText < GATE_MIN_BODY_TEXT_CHARS) return true
|
|
118
139
|
const articleText = visibleTextLength(container)
|
|
119
140
|
return articleText >= GATE_MIN_ARTICLE_TEXT_CHARS
|
|
120
141
|
&& articleText / bodyText >= GATE_MIN_ARTICLE_COVERAGE
|
|
@@ -190,8 +211,19 @@ function removeTrailingLinkDenseRun(parent: Element, articleText: number): void
|
|
|
190
211
|
|
|
191
212
|
/** Prune with the 0.2.18 misfire guard; returns the original on a misfire. */
|
|
192
213
|
export function prunedArticleContent(articleHtml: string, document: Document): string {
|
|
193
|
-
|
|
194
|
-
|
|
214
|
+
return pruneParsedArticle(htmlFragment(articleHtml, document), articleHtml)
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* {@link prunedArticleContent} on an already-parsed article container (the
|
|
219
|
+
* generator passes the gate's parse: the gate reads it first, then this
|
|
220
|
+
* mutates it — the same order the two-string-parses version produced).
|
|
221
|
+
*
|
|
222
|
+
* @param container - the parsed article; pruned in place.
|
|
223
|
+
* @param articleHtml - the string the container was parsed from (the
|
|
224
|
+
* misfire-guard's verbatim return).
|
|
225
|
+
*/
|
|
226
|
+
function pruneParsedArticle(container: Element, articleHtml: string): string {
|
|
195
227
|
const before = visibleTextLength(container)
|
|
196
228
|
if (before === 0) return articleHtml
|
|
197
229
|
pruneTrailingLinkDenseSections(container)
|
|
@@ -204,21 +236,40 @@ export function prunedArticleContent(articleHtml: string, document: Document): s
|
|
|
204
236
|
|
|
205
237
|
// ------------------------------------------------- recommendation rails ----
|
|
206
238
|
|
|
207
|
-
/**
|
|
239
|
+
/**
|
|
240
|
+
* Link density at or above which a recommendation-named subtree is a rail.
|
|
241
|
+
*/
|
|
208
242
|
const RAIL_LINK_DENSITY = 0.3
|
|
209
243
|
const RAIL_MIN_LINKS = 4
|
|
210
244
|
/** Prose (low-link text) the page must keep outside the rails. */
|
|
211
245
|
const RAIL_MIN_PROSE_CHARS = 80
|
|
212
246
|
const RAIL_PROSE_LINK_DENSITY = 0.2
|
|
213
247
|
|
|
214
|
-
/**
|
|
248
|
+
/**
|
|
249
|
+
* Necessary condition for {@link isRailName} on the raw class+id string: a
|
|
250
|
+
* matching token (`recommended-posts`, `Related`, …) always leaves one of
|
|
251
|
+
* these literals as a substring. Testing it first short-circuits the
|
|
252
|
+
* per-token split on the pages where it matters — {@link markRecommendationRails}
|
|
253
|
+
* walks every element, and on the measured 1.1MB Angular docs badcase the
|
|
254
|
+
* split alone cost ~360ms of the candidate generation.
|
|
255
|
+
*/
|
|
256
|
+
const RAIL_NAME_HINT = /recommend|related|outbrain|taboola/
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* Class/id sub-tokens naming a recommendation / related-content rail: the
|
|
260
|
+
* English words every CMS and theme uses, plus the two syndicated
|
|
261
|
+
* recommendation widgets embedded across thousands of publishers (their
|
|
262
|
+
* containers are named after the vendor, not after what they show). No
|
|
263
|
+
* per-site names: a rail must also pass the link-density and position checks.
|
|
264
|
+
*/
|
|
215
265
|
function isRailName(token: string): boolean {
|
|
216
266
|
return token.includes('recommend') || token.startsWith('related')
|
|
217
|
-
|| token === 'outbrain' || token === 'taboola'
|
|
267
|
+
|| token === 'outbrain' || token === 'taboola'
|
|
218
268
|
}
|
|
219
269
|
|
|
220
270
|
function isRecommendationRail(element: Element): boolean {
|
|
221
271
|
const names = `${element.getAttribute('class') ?? ''} ${element.id}`.toLowerCase()
|
|
272
|
+
if (!RAIL_NAME_HINT.test(names)) return false
|
|
222
273
|
if (!names.split(/[\s_-]+/).some(isRailName)) return false
|
|
223
274
|
if (element.querySelectorAll('a').length < RAIL_MIN_LINKS) return false
|
|
224
275
|
const text = visibleTextLength(element)
|
|
@@ -392,8 +443,14 @@ const CHROME_TOKENS = new Set([
|
|
|
392
443
|
|
|
393
444
|
const CHROME_ROLES = new Set(['navigation', 'dialog', 'alertdialog', 'complementary', 'contentinfo', 'search', 'banner'])
|
|
394
445
|
|
|
395
|
-
/**
|
|
396
|
-
|
|
446
|
+
/**
|
|
447
|
+
* Cookie/consent banner wording (EN/DE/FR/ES/IT/NL/ZH), as a regex source
|
|
448
|
+
* shared with the selector's `consent_hits` feature so both read the same
|
|
449
|
+
* languages. Chinese banners say "Cookie" or 隐私; 同意 ("agree") alone is
|
|
450
|
+
* everyday prose and only counts as the banner's ACTION below.
|
|
451
|
+
*/
|
|
452
|
+
export const CONSENT_TERMS = String.raw`\b(cookies?|consent|datenschutz\w*|einwilligung|zustimm\w*|akzeptier\w*|privacy|gdpr|rgpd)\b|隐私`
|
|
453
|
+
const CONSENT_TEXT = new RegExp(CONSENT_TERMS, 'i')
|
|
397
454
|
const CONSENT_ACTION = /\b(accept|agree|allow|reject|decline|manage|preferences|settings|akzeptieren|zustimmen|einverstanden|ablehnen|accepter|aceptar|accetta|accepteren)\b|接受|同意/i
|
|
398
455
|
|
|
399
456
|
function isChromeElement(element: Element): boolean {
|
|
@@ -496,12 +553,16 @@ export function findRecordRegion(body: Element): RecordRegion | null {
|
|
|
496
553
|
return best
|
|
497
554
|
}
|
|
498
555
|
|
|
499
|
-
/**
|
|
556
|
+
/**
|
|
557
|
+
* A post's timestamp as a locale-neutral calendar date (`2024-09-05`, UTC).
|
|
558
|
+
* Deliberately not any UI's localized rendering: the reader may be any
|
|
559
|
+
* language, and the page itself prints dates in its own locale.
|
|
560
|
+
*/
|
|
500
561
|
function formatForumDate(iso: unknown): string {
|
|
501
562
|
if (typeof iso !== 'string') return ''
|
|
502
563
|
const date = new Date(iso)
|
|
503
564
|
if (Number.isNaN(date.getTime())) return ''
|
|
504
|
-
return date.
|
|
565
|
+
return date.toISOString().slice(0, 10)
|
|
505
566
|
}
|
|
506
567
|
|
|
507
568
|
/**
|
|
@@ -546,11 +607,82 @@ export interface GenerationFacts {
|
|
|
546
607
|
hasPreloaded: boolean
|
|
547
608
|
/** The semantic main container, when found. */
|
|
548
609
|
main: Element | null
|
|
610
|
+
/**
|
|
611
|
+
* True when the Readability family was SKIPPED because nearly all body
|
|
612
|
+
* text is custom-wrapped ({@link customWrappedTextShare}) — every
|
|
613
|
+
* extraction that would carry it sanitizes to nothing. The orchestrator
|
|
614
|
+
* then seeds the selector with the empty-`art` reference stats such a run
|
|
615
|
+
* would have measured, keeping the surviving candidates' features
|
|
616
|
+
* (`words_vs_art_log`, the block diffs) exactly what a dead Readability
|
|
617
|
+
* pick yields.
|
|
618
|
+
*/
|
|
619
|
+
readabilitySkipped?: boolean
|
|
549
620
|
}
|
|
550
621
|
|
|
551
622
|
/** Below this much noscript text the noscript-unwrapped paths are skipped. */
|
|
552
623
|
const NOSCRIPT_MIN_TEXT_CHARS = 200
|
|
553
624
|
|
|
625
|
+
/**
|
|
626
|
+
* At or above this share of custom-wrapped body text the Readability family
|
|
627
|
+
* is skipped (see {@link customWrappedTextShare}).
|
|
628
|
+
*/
|
|
629
|
+
export const CUSTOM_WRAP_SKIP_SHARE = 0.9
|
|
630
|
+
|
|
631
|
+
/**
|
|
632
|
+
* Share of the body's visible text that sits inside custom-element subtrees
|
|
633
|
+
* (tag names containing `-`: `<ms-app>`, `<my-doc>` — the HTML spec reserves
|
|
634
|
+
* them for custom elements; no standard HTML or MathML tag and no tag the
|
|
635
|
+
* denoise pipeline's DOMPurify step keeps contains a dash). DOMPurify's
|
|
636
|
+
* allowlist removes such tags wholesale — `KEEP_CONTENT: false` takes their
|
|
637
|
+
* subtrees with them — so once nearly ALL the page's text is custom-wrapped,
|
|
638
|
+
* every extraction that would carry it sanitizes to a corpse. Measured on
|
|
639
|
+
* the badcase (aistudio.google.com/docs/computer-use, an Angular app whose
|
|
640
|
+
* entire DOM lives under `<ms-app>`): 99% of body text custom-wrapped;
|
|
641
|
+
* Readability's pick sanitized to a root with ZERO words, the whole-document
|
|
642
|
+
* candidate to 39 of ~4.5k words — while the three Readability passes over
|
|
643
|
+
* the 1.1MB document cost ~6.5s of pure latency on candidates that could
|
|
644
|
+
* never win. The DOM-backed fallbacks, the noscript paths and the block
|
|
645
|
+
* classifier — the candidates that survive DOMPurify on this shape, and the
|
|
646
|
+
* ones the selector picks on such pages anyway — carry the extraction.
|
|
647
|
+
*
|
|
648
|
+
* Noise subtrees (script/style/nav/…) count as neither wrapped nor unwrapped;
|
|
649
|
+
* the share is over CONTENT-bearing text only. A partial share (a web
|
|
650
|
+
* component widget inside a normal article) stays below the threshold and
|
|
651
|
+
* keeps the full pipeline: only the shape where the family is provably dead
|
|
652
|
+
* pays for it.
|
|
653
|
+
*/
|
|
654
|
+
export function customWrappedTextShare(body: Element): number {
|
|
655
|
+
let total = 0
|
|
656
|
+
let wrapped = 0
|
|
657
|
+
const visit = (node: Node, insideCustom: boolean): void => {
|
|
658
|
+
if (node.nodeType === 3) {
|
|
659
|
+
const length = (node.nodeValue ?? '').replace(/\s+/g, '').length
|
|
660
|
+
if (length === 0) return
|
|
661
|
+
total += length
|
|
662
|
+
if (insideCustom) wrapped += length
|
|
663
|
+
return
|
|
664
|
+
}
|
|
665
|
+
if (node.nodeType !== 1 || NOISE_SUBTREE_TAGS.has(node.nodeName.toLowerCase())) return
|
|
666
|
+
const custom = insideCustom || node.nodeName.includes('-')
|
|
667
|
+
for (let child = node.firstChild; child !== null; child = child.nextSibling) visit(child, custom)
|
|
668
|
+
}
|
|
669
|
+
visit(body, false)
|
|
670
|
+
return total === 0 ? 0 : wrapped / total
|
|
671
|
+
}
|
|
672
|
+
|
|
673
|
+
/**
|
|
674
|
+
* The title the gated path hands its candidates: Readability's own first
|
|
675
|
+
* source is `og:title`, its second the `<title>` element — both remain
|
|
676
|
+
* readable on custom-element pages, where the DOM wrapping is the framework's
|
|
677
|
+
* but the metadata is the page's.
|
|
678
|
+
*/
|
|
679
|
+
function fallbackTitle(document: Document): string | undefined {
|
|
680
|
+
const og = document.querySelector('meta[property="og:title"]')?.getAttribute('content')?.trim()
|
|
681
|
+
if (og !== undefined && og !== '') return og
|
|
682
|
+
const title = document.querySelector('title')?.textContent?.trim()
|
|
683
|
+
return title === undefined || title === '' ? undefined : title
|
|
684
|
+
}
|
|
685
|
+
|
|
554
686
|
/**
|
|
555
687
|
* Propose every candidate extraction for one parsed page.
|
|
556
688
|
* @param document - the parsed page (data URIs already elided); not mutated.
|
|
@@ -560,38 +692,54 @@ export function generateCandidates(document: Document): { candidates: RawCandida
|
|
|
560
692
|
const body = document.body
|
|
561
693
|
const bodyHtml = body?.innerHTML ?? ''
|
|
562
694
|
|
|
563
|
-
//
|
|
564
|
-
//
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
const gatePass = art !== null && articleCoversPage(art.content, document)
|
|
695
|
+
// The custom-element shape: skip the Readability family outright (see
|
|
696
|
+
// {@link customWrappedTextShare}) — its candidates sanitize to corpses.
|
|
697
|
+
const skipReadability = body !== null && customWrappedTextShare(body) >= CUSTOM_WRAP_SKIP_SHARE
|
|
698
|
+
|
|
699
|
+
let art: ReadabilityPick | null = null
|
|
700
|
+
let lenient: ReadabilityPick | null = null
|
|
701
|
+
let gatePass = false
|
|
571
702
|
let pruneChanged = false
|
|
572
|
-
if (
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
703
|
+
if (!skipReadability) {
|
|
704
|
+
// Rails are marked before Readability (it drops the class names that
|
|
705
|
+
// identify them) and cut from its output; the gate and `artu` see the
|
|
706
|
+
// extraction exactly as unmarked, so the legacy marker and the "keep
|
|
707
|
+
// everything" alternative are unchanged.
|
|
708
|
+
const marked = runReadability(markRecommendationRails(document) ?? document)
|
|
709
|
+
const rails = marked === null ? null : splitRails(marked.content, document)
|
|
710
|
+
art = marked === null || rails === null ? null : { ...marked, content: rails.plain }
|
|
711
|
+
if (art !== null) {
|
|
712
|
+
// One parse of the article serves the gate AND the prune (the common
|
|
713
|
+
// case: no rail was cut, so both read the same string).
|
|
714
|
+
const bodyText = visibleTextLength(body)
|
|
715
|
+
const parsed = htmlFragment(art.content, document)
|
|
716
|
+
gatePass = articleCoversParsed(parsed, bodyText)
|
|
717
|
+
const pruning = rails?.derailed ?? art.content
|
|
718
|
+
const pruned = pruning === art.content
|
|
719
|
+
? pruneParsedArticle(parsed, art.content)
|
|
720
|
+
: prunedArticleContent(pruning, document)
|
|
721
|
+
pruneChanged = pruned !== art.content
|
|
722
|
+
candidates.push({ kind: 'art', html: pruned, title: art.title, legacy: gatePass })
|
|
723
|
+
if (pruneChanged) candidates.push({ kind: 'artu', html: art.content, title: art.title, legacy: false })
|
|
724
|
+
}
|
|
725
|
+
// `linkDensityModifier` is a public Readability option missing from the
|
|
726
|
+
// bundled type declarations; it relaxes the link-density cleaning cutoffs.
|
|
727
|
+
lenient = runReadability(document, { linkDensityModifier: 0.4 } as ConstructorParameters<typeof Readability>[1])
|
|
728
|
+
if (lenient !== null && lenient.content !== art?.content) {
|
|
729
|
+
candidates.push({ kind: 'artx', html: lenient.content, title: lenient.title, legacy: false })
|
|
730
|
+
}
|
|
583
731
|
}
|
|
584
|
-
const title = art?.title ?? lenient?.title
|
|
732
|
+
const title = art?.title ?? lenient?.title ?? (skipReadability ? fallbackTitle(document) : undefined)
|
|
585
733
|
|
|
586
734
|
const main = body === null ? null : findMainContainer(document)
|
|
587
|
-
if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, title, legacy: false })
|
|
735
|
+
if (main !== null) candidates.push({ kind: 'main', html: main.innerHTML, node: main, title, legacy: false })
|
|
588
736
|
|
|
589
|
-
candidates.push({ kind: 'doc', html: bodyHtml, legacy: !gatePass })
|
|
737
|
+
candidates.push({ kind: 'doc', html: bodyHtml, node: body ?? undefined, legacy: !gatePass })
|
|
590
738
|
|
|
591
739
|
if (body !== null) {
|
|
592
740
|
const cleaned = body.cloneNode(true) as Element
|
|
593
741
|
stripPageChrome(cleaned)
|
|
594
|
-
candidates.push({ kind: 'docc', html: cleaned.innerHTML, legacy: false })
|
|
742
|
+
candidates.push({ kind: 'docc', html: cleaned.innerHTML, node: cleaned, legacy: false })
|
|
595
743
|
}
|
|
596
744
|
|
|
597
745
|
const record = body === null ? null : findRecordRegion(body)
|
|
@@ -607,7 +755,7 @@ export function generateCandidates(document: Document): { candidates: RawCandida
|
|
|
607
755
|
unwrapNoscripts(unwrapped)
|
|
608
756
|
const nsArt = runReadability(unwrapped)
|
|
609
757
|
if (nsArt !== null) candidates.push({ kind: 'ns_art', html: prunedArticleContent(nsArt.content, unwrapped), title: nsArt.title, legacy: false })
|
|
610
|
-
candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', legacy: false })
|
|
758
|
+
candidates.push({ kind: 'ns_doc', html: unwrapped.body?.innerHTML ?? '', node: unwrapped.body ?? undefined, legacy: false })
|
|
611
759
|
}
|
|
612
760
|
|
|
613
761
|
const preloaded = discoursePreloadedHtml(document)
|
|
@@ -634,6 +782,7 @@ export function generateCandidates(document: Document): { candidates: RawCandida
|
|
|
634
782
|
hasMain: main !== null,
|
|
635
783
|
hasPreloaded: preloaded !== null,
|
|
636
784
|
main,
|
|
785
|
+
...(skipReadability ? { readabilitySkipped: true } : {}),
|
|
637
786
|
},
|
|
638
787
|
}
|
|
639
788
|
}
|
package/src/challenge.ts
CHANGED
|
@@ -43,8 +43,8 @@ export const CHALLENGE_POLL_INTERVAL_MS = 500
|
|
|
43
43
|
|
|
44
44
|
/**
|
|
45
45
|
* Tail of the fetch budget held back from the challenge wait so a cleared
|
|
46
|
-
* challenge still has time for its settle (
|
|
47
|
-
* content read and the denoise pipeline.
|
|
46
|
+
* challenge still has time for its settle (the `DEFAULT_SETTLE_MS` cap)
|
|
47
|
+
* plus the content read and the denoise pipeline.
|
|
48
48
|
*/
|
|
49
49
|
export const CHALLENGE_FINISH_RESERVE_MS = 8_000
|
|
50
50
|
|
package/src/client/card.tsx
CHANGED
|
@@ -177,32 +177,6 @@ export function PlaywrightCard(props: PlaywrightCardProps) {
|
|
|
177
177
|
onEdit={(text) => { props.edit('maxConcurrency', text) }}
|
|
178
178
|
onReset={() => { props.resetField('maxConcurrency') }}
|
|
179
179
|
/>
|
|
180
|
-
<ValueField
|
|
181
|
-
id="plugin-config-playwright-challenge-wait"
|
|
182
|
-
label={t('challengeWaitMs')}
|
|
183
|
-
hint={t('challengeWaitMsHint')}
|
|
184
|
-
placeholder={t('challengeWaitMsPlaceholder')}
|
|
185
|
-
overriddenLabel={t('overridden')}
|
|
186
|
-
resetLabel={t('reset')}
|
|
187
|
-
invalidLabel={t('invalidText')}
|
|
188
|
-
disabled={disabled}
|
|
189
|
-
{...state.challengeWaitMs}
|
|
190
|
-
onEdit={(text) => { props.edit('challengeWaitMs', text) }}
|
|
191
|
-
onReset={() => { props.resetField('challengeWaitMs') }}
|
|
192
|
-
/>
|
|
193
|
-
<ValueField
|
|
194
|
-
id="plugin-config-playwright-scroll-budget"
|
|
195
|
-
label={t('scrollBudgetMs')}
|
|
196
|
-
hint={t('scrollBudgetMsHint')}
|
|
197
|
-
placeholder={t('scrollBudgetMsPlaceholder')}
|
|
198
|
-
overriddenLabel={t('overridden')}
|
|
199
|
-
resetLabel={t('reset')}
|
|
200
|
-
invalidLabel={t('invalidText')}
|
|
201
|
-
disabled={disabled}
|
|
202
|
-
{...state.scrollBudgetMs}
|
|
203
|
-
onEdit={(text) => { props.edit('scrollBudgetMs', text) }}
|
|
204
|
-
onReset={() => { props.resetField('scrollBudgetMs') }}
|
|
205
|
-
/>
|
|
206
180
|
</CardForm>
|
|
207
181
|
)
|
|
208
182
|
}
|
package/src/client/controller.ts
CHANGED
|
@@ -41,16 +41,6 @@ export interface PlaywrightSettings {
|
|
|
41
41
|
denoise?: boolean
|
|
42
42
|
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
43
43
|
maxConcurrency?: number
|
|
44
|
-
/**
|
|
45
|
-
* Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
|
|
46
|
-
* same tab (0–60000); 0 = off (return the first response as-is).
|
|
47
|
-
*/
|
|
48
|
-
challengeWaitMs?: number
|
|
49
|
-
/**
|
|
50
|
-
* Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
|
|
51
|
-
* (capture the document as it rests, the legacy single-instant behavior).
|
|
52
|
-
*/
|
|
53
|
-
scrollBudgetMs?: number
|
|
54
44
|
}
|
|
55
45
|
|
|
56
46
|
/** What the Playwright card renders. */
|
|
@@ -68,10 +58,6 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
68
58
|
denoise: CardFieldState
|
|
69
59
|
/** Concurrency input (draft decimal integer). */
|
|
70
60
|
maxConcurrency: CardFieldState
|
|
71
|
-
/** Challenge wait input (draft decimal integer of milliseconds). */
|
|
72
|
-
challengeWaitMs: CardFieldState
|
|
73
|
-
/** Scroll-walk budget input (draft decimal integer of milliseconds). */
|
|
74
|
-
scrollBudgetMs: CardFieldState
|
|
75
61
|
}
|
|
76
62
|
|
|
77
63
|
/** The registration-side face the card's slot entry injects. */
|
|
@@ -103,8 +89,6 @@ export class PlaywrightCardController {
|
|
|
103
89
|
radioField('contextIsolationScope', ['session', 'global']),
|
|
104
90
|
checkboxField('denoise'),
|
|
105
91
|
numberField('maxConcurrency', 1, 200),
|
|
106
|
-
numberField('challengeWaitMs', 0, 60_000),
|
|
107
|
-
numberField('scrollBudgetMs', 0, 30_000),
|
|
108
92
|
],
|
|
109
93
|
)
|
|
110
94
|
this.store = this.form.bind(() => this.projection())
|
|
@@ -120,8 +104,6 @@ export class PlaywrightCardController {
|
|
|
120
104
|
contextIsolationScope: this.form.field('contextIsolationScope'),
|
|
121
105
|
denoise: this.form.field('denoise'),
|
|
122
106
|
maxConcurrency: this.form.field('maxConcurrency'),
|
|
123
|
-
challengeWaitMs: this.form.field('challengeWaitMs'),
|
|
124
|
-
scrollBudgetMs: this.form.field('scrollBudgetMs'),
|
|
125
107
|
}
|
|
126
108
|
}
|
|
127
109
|
|
package/src/client/locales.ts
CHANGED
|
@@ -17,8 +17,6 @@ export type PlaywrightCardLocaleKey =
|
|
|
17
17
|
| 'contextScopeGlobal' | 'contextScopeGlobalHint'
|
|
18
18
|
| 'denoise' | 'denoiseHint'
|
|
19
19
|
| 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder'
|
|
20
|
-
| 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder'
|
|
21
|
-
| 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder'
|
|
22
20
|
| 'overridden' | 'reset' | 'readOnly'
|
|
23
21
|
| 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText'
|
|
24
22
|
|
|
@@ -57,12 +55,6 @@ export const en: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
57
55
|
maxConcurrency: 'Max concurrent fetches',
|
|
58
56
|
maxConcurrencyHint: 'How many pages may render at once (1–200). Blank = auto: 4 local browsers, 50 CDP tabs in the remote browser.',
|
|
59
57
|
maxConcurrencyPlaceholder: '(auto: local 4 / CDP 50)',
|
|
60
|
-
challengeWaitMs: 'Cloudflare challenge wait (ms)',
|
|
61
|
-
challengeWaitMsHint: 'Bounded wait for a Cloudflare challenge to clear naturally in the same tab (0–60000; 0 = off). 0 disables: the first response is returned as-is.',
|
|
62
|
-
challengeWaitMsPlaceholder: '(default: 15000)',
|
|
63
|
-
scrollBudgetMs: 'Deep-capture scroll budget (ms)',
|
|
64
|
-
scrollBudgetMsHint: 'Bounded in-page scroll walk that captures viewport-virtualized feeds (forum topics, comment threads) whole: posts outside the render window are empty placeholders at rest. 0–30000; 0 = off (capture the page as it rests).',
|
|
65
|
-
scrollBudgetMsPlaceholder: '(default: 20000)',
|
|
66
58
|
overridden: 'Overridden',
|
|
67
59
|
reset: 'Reset to default',
|
|
68
60
|
readOnly: 'This deployment stores settings read-only.',
|
|
@@ -101,12 +93,6 @@ export const zh: Record<PlaywrightCardLocaleKey, string> = {
|
|
|
101
93
|
maxConcurrency: '最大并发抓取数',
|
|
102
94
|
maxConcurrencyHint: '同时渲染的页面上限(1–200);留空自动:本地 4 个浏览器,CDP 在远端浏览器里 50 个标签页。',
|
|
103
95
|
maxConcurrencyPlaceholder: '(自动:本地 4 / CDP 50)',
|
|
104
|
-
challengeWaitMs: 'Cloudflare 挑战等待上限(毫秒)',
|
|
105
|
-
challengeWaitMsHint: '在同一标签页内有界等待 Cloudflare 验证自然通过(0–60000;0 = 关闭)。关闭时直接返回首次响应——旧版行为。',
|
|
106
|
-
challengeWaitMsPlaceholder: '(默认:15000)',
|
|
107
|
-
scrollBudgetMs: '深度抓取滚动预算(毫秒)',
|
|
108
|
-
scrollBudgetMsHint: '有界的页内滚动遍历,完整捕获视口虚拟化的信息流(论坛帖子、评论区):静止时渲染窗口外的帖子只是空占位符。0–30000;0 = 关闭(按页面静止状态抓取)。',
|
|
109
|
-
scrollBudgetMsPlaceholder: '(默认:20000)',
|
|
110
96
|
overridden: '已覆盖',
|
|
111
97
|
reset: '恢复默认',
|
|
112
98
|
readOnly: '本部署的设置为只读。',
|