dsh-web-fetch-playwright 0.2.16 → 0.2.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -3
- package/README.zh-CN.md +30 -3
- package/lib/client.js +42 -17
- package/lib/index.js +866 -71
- package/lib/types/autoscroll.d.ts +277 -0
- package/lib/types/client/controller.d.ts +7 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -3
- package/lib/types/index.d.ts +3 -1
- package/lib/types/markdown.d.ts +8 -5
- package/lib/types/provider.d.ts +25 -0
- package/lib/types/strip.d.ts +59 -0
- package/package.json +1 -1
- package/src/autoscroll.ts +653 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +9 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -3
- package/src/index.ts +5 -0
- package/src/markdown.ts +252 -6
- package/src/provider.ts +57 -68
- package/src/strip.ts +72 -0
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deep capture: a bounded in-page scroll walk that defeats viewport
|
|
3
|
+
* virtualization before the document is serialized.
|
|
4
|
+
*
|
|
5
|
+
* WHY. Long "feed-shaped" pages — Discourse topic streams (linux.do,
|
|
6
|
+
* meta.discourse.org), comment threads, timelines — increasingly render only
|
|
7
|
+
* the posts near the current scroll position. Discourse is the measured
|
|
8
|
+
* badcase: at rest the live DOM holds the posts within roughly one viewport
|
|
9
|
+
* of the top, while every post outside that window is a height-only
|
|
10
|
+
* placeholder that keeps the post's identity but none of its content:
|
|
11
|
+
*
|
|
12
|
+
* ```html
|
|
13
|
+
* <div class="post-stream--cloaked" data-post-number="1" id="post_1"
|
|
14
|
+
* style="height:297.9px !important;…">
|
|
15
|
+
* <h2 class="sr-only">由 Diver-QC 于 2 小时前 发布</h2>
|
|
16
|
+
* </div>
|
|
17
|
+
* ```
|
|
18
|
+
*
|
|
19
|
+
* The placeholder carries the SAME id the rendered `<article id="post_1">`
|
|
20
|
+
* has — every mainstream virtualized list needs some stable identity to
|
|
21
|
+
* recycle its rows. A fetch that renders, never scrolls, and serializes one
|
|
22
|
+
* instant of that DOM returns the skeleton (the screen-reader-only "posted
|
|
23
|
+
* by" headings survive the pipeline as if they were content) while the reply
|
|
24
|
+
* bodies silently vanish. Scrolling to the bottom once does not help either:
|
|
25
|
+
* the window MOVES — measured on the same page, scrolling to the bottom left
|
|
26
|
+
* only posts 31–40 rendered and re-cloaked 1–30. And the denoise pipeline's
|
|
27
|
+
* own coverage gate cannot notice the loss: the page's "visible text" it
|
|
28
|
+
* measures against is the same windowed fragment as the extraction, so the
|
|
29
|
+
* ratio always passes. Only walk depth can see what a resting viewport hides.
|
|
30
|
+
*
|
|
31
|
+
* The SAME pathology, one level deeper, hits app-shell document editors —
|
|
32
|
+
* the second measured badcase is a Feishu wiki page (bocha-ai.feishu.cn).
|
|
33
|
+
* There the body never scrolls at all: html/body are locked at exactly one
|
|
34
|
+
* viewport and the document scrolls inside `div.bear-web-x-container`
|
|
35
|
+
* (measured: documentElement.scrollHeight 720 == innerHeight,
|
|
36
|
+
* window.scrollTo(0, 800) left scrollY at 0, while the container held 4534px
|
|
37
|
+
* against a 656px client). Its blocks are virtualized too, with two twists
|
|
38
|
+
* over Discourse: blocks carry `data-block-id` and NO id, and off-window
|
|
39
|
+
* blocks are REMOVED outright — the whole un-rendered tail is one anonymous
|
|
40
|
+
* `.bear-virtual-renderUnit-placeholder` spacer with an explicit height — so
|
|
41
|
+
* there is neither a window scrollbar to walk nor a degraded-in-place row to
|
|
42
|
+
* swap back. A fetch therefore returns the statically rendered first screen.
|
|
43
|
+
*
|
|
44
|
+
* HOW. Between the settle and the serialization the provider runs one
|
|
45
|
+
* in-page async walk ({@link deepCaptureInPage}, a typed function evaluated
|
|
46
|
+
* by source):
|
|
47
|
+
*
|
|
48
|
+
* 1. Resolve the walk TARGET: the window when the document itself scrolls
|
|
49
|
+
* (the Discourse shape), else the page's dominant inner scroll container
|
|
50
|
+
* (the app-shell shape) — the element with the largest scrollable
|
|
51
|
+
* distance among computed `overflow-y: auto|scroll` elements, gated at
|
|
52
|
+
* half a viewport so dropdown-sized scrollers never trigger a walk. A
|
|
53
|
+
* page with neither is left alone (the legacy capture).
|
|
54
|
+
* 2. STASH every keyed content element — key = id, else a namespaced
|
|
55
|
+
* `data-block-id` — as a detached clone (the RICHEST sighting per key
|
|
56
|
+
* wins), together with its nearest keyed siblings: where it sits in its
|
|
57
|
+
* own list, which is what a removed row is later spliced back next to.
|
|
58
|
+
* 3. WALK down from the top in smooth 0.7-viewport steps (300ms motion +
|
|
59
|
+
* 300ms quiet settle — the rest a feed engine's debounced uncloak needs),
|
|
60
|
+
* collecting after each, until the walk can hold AT THE BOTTOM through a
|
|
61
|
+
* {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
|
|
62
|
+
* (growth restarts the clock — that is what pages load-more sentinels),
|
|
63
|
+
* or the step/budget caps close it.
|
|
64
|
+
*
|
|
65
|
+
* A MutationObserver watches for the walk's whole reason to exist: the
|
|
66
|
+
* page REACTING to scroll by adding or removing elements. Most of the web
|
|
67
|
+
* never does (a long static article only costs latency here), so after
|
|
68
|
+
* {@link SCROLL_INERT_STEPS} careful steps with no reaction the walk
|
|
69
|
+
* probes the rest with one jump to the bottom. Any reaction after that
|
|
70
|
+
* jump — an uncloak, a re-cloak, a load-more append — means the region
|
|
71
|
+
* jumped over may hold content that never rendered, so the walk rewinds
|
|
72
|
+
* to where it jumped from and finishes carefully. Correctness never
|
|
73
|
+
* rests on the warm-up guess; only the cost of a scroll-inert page does.
|
|
74
|
+
* 4. MERGE into a structural clone (`document.cloneNode(true)`: no
|
|
75
|
+
* serialize/re-parse round trip — a runtime-rewritten data script with a
|
|
76
|
+
* literal `</script>` in its text cannot corrupt it — and, having no
|
|
77
|
+
* browsing context, it runs no scripts, upgrades no custom elements and
|
|
78
|
+
* loads no frames):
|
|
79
|
+
* - SPLICE (container walks): stashed keys from inside the scroll
|
|
80
|
+
* container that the clone no longer holds were REMOVED by the engine
|
|
81
|
+
* (the Feishu shape — its placeholders are anonymous) and go back next
|
|
82
|
+
* to their recorded sibling, resolved iteratively so a removed run
|
|
83
|
+
* chains off whichever end survived; a key that rode in inside its
|
|
84
|
+
* spliced parent is already present and skipped. Placement is
|
|
85
|
+
* structural, so it is right whatever order the walk SAW rows in (an
|
|
86
|
+
* anchored open starts mid-document).
|
|
87
|
+
* - SWAP: every keyed element emptier than its snapshot (fewer child
|
|
88
|
+
* elements, or meaningfully less text — the Discourse placeholder) is
|
|
89
|
+
* replaced by that snapshot, except containers (ancestors of another
|
|
90
|
+
* element being swapped), which are assembled from those descendants
|
|
91
|
+
* instead: a container's own snapshot is one window's partial view.
|
|
92
|
+
* The clone is then stripped of scripting elements and serialized; the
|
|
93
|
+
* provider reads that string instead of the live page. The live document
|
|
94
|
+
* is never rewritten, so no late scroll/rAF/timer reaction can touch the
|
|
95
|
+
* capture either.
|
|
96
|
+
*
|
|
97
|
+
* Failure is always silent degradation: a page without `evaluate`, a script
|
|
98
|
+
* error, or an engine that rejects the walk just keeps the legacy
|
|
99
|
+
* single-instant capture — a walk can only add content, never fail a fetch.
|
|
100
|
+
* A walk that restored nothing returns no HTML, and the provider reads the
|
|
101
|
+
* live page exactly as before.
|
|
102
|
+
*
|
|
103
|
+
* The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
|
|
104
|
+
* cloaked placeholders) but never for non-HTML documents. It is bounded by
|
|
105
|
+
* the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
|
|
106
|
+
* and always by the remaining fetch deadline minus a decode reserve — the
|
|
107
|
+
* provider's abort closes the tab, which rejects the pending evaluation.
|
|
108
|
+
*
|
|
109
|
+
* @module dsh-web-fetch-playwright/autoscroll
|
|
110
|
+
*/
|
|
111
|
+
import type { PlaywrightPage } from './types.ts';
|
|
112
|
+
/** How far each step scrolls, as a share of the viewport height. */
|
|
113
|
+
export declare const SCROLL_STEP_RATIO = 0.7;
|
|
114
|
+
/**
|
|
115
|
+
* How long one step's smooth scroll motion takes (ms). The step ANIMATES
|
|
116
|
+
* down its distance (time-interpolated ticks at ~60fps) instead of
|
|
117
|
+
* teleporting: feed engines' uncloak/re-cloak logic is built around a
|
|
118
|
+
* continuous stream of scroll events, and background-tab timer throttling
|
|
119
|
+
* self-corrects because every tick derives its position from the clock.
|
|
120
|
+
*/
|
|
121
|
+
export declare const SCROLL_STEP_MOTION_MS = 300;
|
|
122
|
+
/**
|
|
123
|
+
* Per-step settle (ms): the QUIET window after the motion, where a feed
|
|
124
|
+
* engine's debounced uncloak fires (measured: stepping without a rest
|
|
125
|
+
* starves Discourse's cloak manager, which renders nothing new until the
|
|
126
|
+
* scroll rests ~300ms) and the freshly rendered rows become stashable.
|
|
127
|
+
*/
|
|
128
|
+
export declare const SCROLL_STEP_SETTLE_MS = 300;
|
|
129
|
+
/** How often the walk re-checks the height while holding at the bottom. */
|
|
130
|
+
export declare const SCROLL_BOTTOM_POLL_MS = 450;
|
|
131
|
+
/**
|
|
132
|
+
* How long at the bottom, with no height growth, before the walk concludes
|
|
133
|
+
* the page is complete. Real feed engines fetch their next chunk with
|
|
134
|
+
* network latency (measured on linux.do: a pending chunk landed well after
|
|
135
|
+
* two 450ms checks), so this is sized over a slow round trip.
|
|
136
|
+
*/
|
|
137
|
+
export declare const SCROLL_BOTTOM_PATIENCE_MS = 2500;
|
|
138
|
+
/**
|
|
139
|
+
* Careful steps without any DOM reaction before the walk treats the page as
|
|
140
|
+
* scroll-inert and probes the rest with a single jump (rewinding if the
|
|
141
|
+
* jump provokes a reaction). Two steps cover ~1.4 viewports — past the
|
|
142
|
+
* render window edge of every measured virtualizer.
|
|
143
|
+
*/
|
|
144
|
+
export declare const SCROLL_INERT_STEPS = 2;
|
|
145
|
+
/** Hard cap on scroll steps regardless of budget (unbounded feeds). */
|
|
146
|
+
export declare const SCROLL_MAX_STEPS = 50;
|
|
147
|
+
/**
|
|
148
|
+
* Cap on stashed keys — pathological id counts cannot pin memory. Sized for
|
|
149
|
+
* long topics: a Discourse post carries several ids of its own (post,
|
|
150
|
+
* heading, avatar, …), so a few hundred posts already need a thousand-plus.
|
|
151
|
+
* {@link SCROLL_STASH_MAX_CHARS} is the real memory bound.
|
|
152
|
+
*/
|
|
153
|
+
export declare const SCROLL_STASH_MAX_ENTRIES = 2000;
|
|
154
|
+
/** Cap on stashed HTML characters (aligned with the pipeline input cap). */
|
|
155
|
+
export declare const SCROLL_STASH_MAX_CHARS = 2000000;
|
|
156
|
+
/**
|
|
157
|
+
* Cap on ONE stashed element's HTML. A virtualized row is a few KB; an
|
|
158
|
+
* element tens of KB big is a CONTAINER (measured on linux.do: six nested
|
|
159
|
+
* wrappers each embed the whole post stream — main 164KB ⊃
|
|
160
|
+
* main-outlet-wrapper 156KB ⊃ … — plus data-preloaded 300KB and
|
|
161
|
+
* svg-sprites 277KB) or a data island — never a row the merge could
|
|
162
|
+
* rehydrate. Such keys are remembered and never serialized again.
|
|
163
|
+
*/
|
|
164
|
+
export declare const SCROLL_STASH_MAX_ENTRY_CHARS = 64000;
|
|
165
|
+
/**
|
|
166
|
+
* Tail of the fetch budget the walk never spends, keeping room for the
|
|
167
|
+
* content read and the synchronous denoise decode.
|
|
168
|
+
*/
|
|
169
|
+
export declare const SCROLL_FINISH_RESERVE_MS = 4000;
|
|
170
|
+
/**
|
|
171
|
+
* A walk budget below this cannot complete even the first window plus one
|
|
172
|
+
* stability check, so the walk is skipped outright.
|
|
173
|
+
*/
|
|
174
|
+
export declare const SCROLL_MIN_BUDGET_MS = 500;
|
|
175
|
+
/**
|
|
176
|
+
* Floor (px) on the scrollable distance an inner container must offer
|
|
177
|
+
* before the walk targets it, when the document itself does not scroll.
|
|
178
|
+
*/
|
|
179
|
+
export declare const SCROLL_CONTAINER_MIN_RANGE_PX = 300;
|
|
180
|
+
/** The viewport share an inner container's scrollable distance must reach. */
|
|
181
|
+
export declare const SCROLL_CONTAINER_MIN_RANGE_RATIO = 0.5;
|
|
182
|
+
/**
|
|
183
|
+
* Text shortfall (chars) versus the snapshot that marks an element as
|
|
184
|
+
* degraded even when its child-element count held — below it, live text
|
|
185
|
+
* (relative timestamps, counters) merely ticked.
|
|
186
|
+
*/
|
|
187
|
+
export declare const SCROLL_TEXT_DEGRADE_TOLERANCE = 16;
|
|
188
|
+
/** What one completed (or skipped) walk reports back, for tests and logs. */
|
|
189
|
+
export interface DeepCaptureStats {
|
|
190
|
+
/** Scroll steps taken (0 when the walk skipped or had no room). */
|
|
191
|
+
steps: number;
|
|
192
|
+
/** Distinct keys stashed across the walk. */
|
|
193
|
+
stashed: number;
|
|
194
|
+
/** Degraded elements swapped for their snapshot in the capture. */
|
|
195
|
+
rehydrated: number;
|
|
196
|
+
/** Removed keyed elements spliced back (container walks only). */
|
|
197
|
+
spliced?: number;
|
|
198
|
+
/**
|
|
199
|
+
* Whether the page reacted to scrolling (added/removed elements). False
|
|
200
|
+
* means the walk took the scroll-inert fast path to the bottom.
|
|
201
|
+
*/
|
|
202
|
+
reactive?: boolean;
|
|
203
|
+
/** Walk duration (ms). */
|
|
204
|
+
elapsedMs?: number;
|
|
205
|
+
/** Walk-target height (px) — document or inner scroll container. */
|
|
206
|
+
height?: number;
|
|
207
|
+
/** Set when the walk skipped without scrolling, and why. */
|
|
208
|
+
skipped?: 'not-scrollable' | 'no-budget';
|
|
209
|
+
}
|
|
210
|
+
/** A finished walk: its stats, plus the merged capture when it restored anything. */
|
|
211
|
+
export interface DeepCapture extends DeepCaptureStats {
|
|
212
|
+
/**
|
|
213
|
+
* The merged, scripting-stripped document serialized like
|
|
214
|
+
* `page.content()` — present only when the walk restored something; the
|
|
215
|
+
* provider then reads it instead of the live page.
|
|
216
|
+
*/
|
|
217
|
+
html?: string;
|
|
218
|
+
}
|
|
219
|
+
/** The walk's knobs, handed to {@link deepCaptureInPage} as JSON. */
|
|
220
|
+
export interface DeepCaptureParams {
|
|
221
|
+
budgetMs: number;
|
|
222
|
+
stepRatio: number;
|
|
223
|
+
stepMotionMs: number;
|
|
224
|
+
stepSettleMs: number;
|
|
225
|
+
bottomPollMs: number;
|
|
226
|
+
bottomPatienceMs: number;
|
|
227
|
+
inertSteps: number;
|
|
228
|
+
maxSteps: number;
|
|
229
|
+
stashMaxEntries: number;
|
|
230
|
+
stashMaxChars: number;
|
|
231
|
+
stashMaxEntryChars: number;
|
|
232
|
+
containerMinRangePx: number;
|
|
233
|
+
containerMinRatio: number;
|
|
234
|
+
textDegradeTolerance: number;
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* The in-page walk. Evaluated BY SOURCE (`fn.toString()` composed into an
|
|
238
|
+
* expression by {@link deepCaptureScript}), so it must stay self-contained:
|
|
239
|
+
* every helper is nested, every knob arrives in `p`, and the one shared
|
|
240
|
+
* routine — the scripting strip — arrives as an argument. Typed here so the
|
|
241
|
+
* compiler checks what used to be an unchecked string.
|
|
242
|
+
*
|
|
243
|
+
* @param p - the walk's knobs.
|
|
244
|
+
* @param strip - {@link stripScriptingElementsIn}, applied to the merged clone.
|
|
245
|
+
* @returns the walk's stats, plus the merged HTML when anything was restored.
|
|
246
|
+
*/
|
|
247
|
+
export declare function deepCaptureInPage(p: DeepCaptureParams, strip: (doc: Document) => void): Promise<DeepCapture>;
|
|
248
|
+
/**
|
|
249
|
+
* The walk's knobs for one fetch: the module constants plus its budget.
|
|
250
|
+
*
|
|
251
|
+
* @param budgetMs - wall-clock budget the walk may spend inside the page.
|
|
252
|
+
*/
|
|
253
|
+
export declare function deepCaptureParams(budgetMs: number): DeepCaptureParams;
|
|
254
|
+
/**
|
|
255
|
+
* Compose the evaluate expression: the in-page walk applied to `params` and
|
|
256
|
+
* the shared scripting strip, both by source.
|
|
257
|
+
*
|
|
258
|
+
* @param params - the walk's knobs (see {@link deepCaptureParams}).
|
|
259
|
+
* @returns the expression string to pass to `page.evaluate`.
|
|
260
|
+
*/
|
|
261
|
+
export declare function deepCaptureExpression(params: DeepCaptureParams): string;
|
|
262
|
+
/** {@link deepCaptureExpression} over {@link deepCaptureParams} — one fetch's walk. */
|
|
263
|
+
export declare function deepCaptureScript(budgetMs: number): string;
|
|
264
|
+
/**
|
|
265
|
+
* Run the deep-capture walk in the page, degrading silently to the legacy
|
|
266
|
+
* single-instant capture on any failure: a page without `evaluate`, a budget
|
|
267
|
+
* too small to act, a rejected evaluation (the fetch deadline closes the tab
|
|
268
|
+
* mid-walk), or a result that is not the stats object all just skip it. A
|
|
269
|
+
* walk can only add content, never fail a fetch.
|
|
270
|
+
*
|
|
271
|
+
* @param page - the fetch's tab.
|
|
272
|
+
* @param budgetMs - wall-clock budget for the in-page walk; budgets under
|
|
273
|
+
* {@link SCROLL_MIN_BUDGET_MS} skip it.
|
|
274
|
+
* @returns the walk's stats (with the merged `html` when it restored
|
|
275
|
+
* anything), or undefined when the walk did not run.
|
|
276
|
+
*/
|
|
277
|
+
export declare function deepCaptureWalk(page: PlaywrightPage, budgetMs: number): Promise<DeepCapture | undefined>;
|
|
@@ -42,6 +42,11 @@ export interface PlaywrightSettings {
|
|
|
42
42
|
* same tab (0–60000); 0 = off (return the first response as-is).
|
|
43
43
|
*/
|
|
44
44
|
challengeWaitMs?: number;
|
|
45
|
+
/**
|
|
46
|
+
* Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
|
|
47
|
+
* (capture the document as it rests, the legacy single-instant behavior).
|
|
48
|
+
*/
|
|
49
|
+
scrollBudgetMs?: number;
|
|
45
50
|
}
|
|
46
51
|
/** What the Playwright card renders. */
|
|
47
52
|
export interface PlaywrightCardState extends CardShell {
|
|
@@ -60,6 +65,8 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
60
65
|
maxConcurrency: CardFieldState;
|
|
61
66
|
/** Challenge wait input (draft decimal integer of milliseconds). */
|
|
62
67
|
challengeWaitMs: CardFieldState;
|
|
68
|
+
/** Scroll-walk budget input (draft decimal integer of milliseconds). */
|
|
69
|
+
scrollBudgetMs: CardFieldState;
|
|
63
70
|
}
|
|
64
71
|
/** The registration-side face the card's slot entry injects. */
|
|
65
72
|
export interface PlaywrightCardFace extends CardActions {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* @module dsh-web-fetch-playwright/client/locales
|
|
6
6
|
*/
|
|
7
7
|
/** Locale keys this card renders. */
|
|
8
|
-
export type PlaywrightCardLocaleKey = 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'contextMode' | 'isolateBrowserContext' | 'isolateBrowserContextHint' | 'contextIsolationScope' | 'contextScopeSession' | 'contextScopeSessionHint' | 'contextScopeGlobal' | 'contextScopeGlobalHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText';
|
|
8
|
+
export type PlaywrightCardLocaleKey = 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'contextMode' | 'isolateBrowserContext' | 'isolateBrowserContextHint' | 'contextIsolationScope' | 'contextScopeSession' | 'contextScopeSessionHint' | 'contextScopeGlobal' | 'contextScopeGlobalHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder' | 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText';
|
|
9
9
|
/** This plugin's dictionary namespace, merged into the locale key map. */
|
|
10
10
|
declare module '@deepseek-ai/dsh-client-ui-slots' {
|
|
11
11
|
interface LocaleNamespaceMap {
|
package/lib/types/config.d.ts
CHANGED
|
@@ -14,6 +14,17 @@
|
|
|
14
14
|
*/
|
|
15
15
|
import type { Volatile } from '@deepseek-ai/cordis';
|
|
16
16
|
import z from '@deepseek-ai/schemastery';
|
|
17
|
+
/**
|
|
18
|
+
* Default bounded budget (ms) for the deep-capture scroll walk
|
|
19
|
+
* (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
|
|
20
|
+
* careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
|
|
21
|
+
* the bottom patience window): 20s walks ~25kpx with pagination, covering
|
|
22
|
+
* the measured linux.do badcase (~18kpx, 51 replies) with headroom.
|
|
23
|
+
* Scroll-inert pages take the walk's fast path and spend a few seconds.
|
|
24
|
+
*/
|
|
25
|
+
export declare const DEFAULT_SCROLL_BUDGET_MS = 20000;
|
|
26
|
+
/** Ceiling the schema accepts for `scrollBudgetMs`. */
|
|
27
|
+
export declare const MAX_SCROLL_BUDGET_MS = 30000;
|
|
17
28
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
18
29
|
export declare const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
19
30
|
/**
|
|
@@ -96,6 +107,15 @@ export interface Config {
|
|
|
96
107
|
* Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
|
|
97
108
|
*/
|
|
98
109
|
challengeRetries?: number;
|
|
110
|
+
/**
|
|
111
|
+
* Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
|
|
112
|
+
* that defeats viewport virtualization on feed-shaped pages (Discourse
|
|
113
|
+
* topic streams: replies outside the render window are empty height-only
|
|
114
|
+
* placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
|
|
115
|
+
* restores the legacy single-instant capture.
|
|
116
|
+
* Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
|
|
117
|
+
*/
|
|
118
|
+
scrollBudgetMs?: number;
|
|
99
119
|
/**
|
|
100
120
|
* How many fetches may render at once. Blank = backend default (4 for
|
|
101
121
|
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
@@ -113,6 +133,7 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
113
133
|
maxConcurrency: z<number, number, "volatile">;
|
|
114
134
|
challengeWaitMs: z<number, number, "volatile-defined">;
|
|
115
135
|
challengeRetries: z<number, number, "volatile-defined">;
|
|
136
|
+
scrollBudgetMs: z<number, number, "volatile-defined">;
|
|
116
137
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
117
138
|
backend: z<"local" | "cdp", "local" | "cdp", "volatile-defined">;
|
|
118
139
|
playwrightPath: z<string, string, "volatile-defined">;
|
|
@@ -123,6 +144,7 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
123
144
|
maxConcurrency: z<number, number, "volatile">;
|
|
124
145
|
challengeWaitMs: z<number, number, "volatile-defined">;
|
|
125
146
|
challengeRetries: z<number, number, "volatile-defined">;
|
|
147
|
+
scrollBudgetMs: z<number, number, "volatile-defined">;
|
|
126
148
|
}>>, "plain">;
|
|
127
149
|
/**
|
|
128
150
|
* The config `apply` receives once the schema resolves: every volatile field
|
|
@@ -151,6 +173,8 @@ export interface PlaywrightPluginConfig {
|
|
|
151
173
|
challengeWaitMs: Volatile<number>;
|
|
152
174
|
/** Challenge retry count as a live reference. */
|
|
153
175
|
challengeRetries: Volatile<number>;
|
|
176
|
+
/** Deep-capture scroll-walk budget (ms) as a live reference. */
|
|
177
|
+
scrollBudgetMs: Volatile<number>;
|
|
154
178
|
}
|
|
155
179
|
/**
|
|
156
180
|
* Read the live references `apply` holds into one plain input for
|
|
@@ -174,11 +198,14 @@ export declare function snapshotsOf(config: PlaywrightPluginConfig): Config;
|
|
|
174
198
|
export declare function resolveConfig(config: Partial<Config>): ResolvedConfig;
|
|
175
199
|
/**
|
|
176
200
|
* Complete config after schemastery applies the field defaults it owns.
|
|
177
|
-
* `maxConcurrency`
|
|
178
|
-
*
|
|
201
|
+
* `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
|
|
202
|
+
* its effective default depends on `backend`, the latter so hand-built
|
|
203
|
+
* resolved configs (tests) may omit it and let
|
|
204
|
+
* {@link effectiveScrollBudgetMs} supply the default the schema would have.
|
|
179
205
|
*/
|
|
180
|
-
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & {
|
|
206
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'> & {
|
|
181
207
|
maxConcurrency?: number;
|
|
208
|
+
scrollBudgetMs?: number;
|
|
182
209
|
};
|
|
183
210
|
/**
|
|
184
211
|
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
@@ -205,6 +232,14 @@ export declare function effectiveChallengeWaitMs(config: Pick<Config, 'challenge
|
|
|
205
232
|
* @returns retries after a wait window runs out.
|
|
206
233
|
*/
|
|
207
234
|
export declare function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number;
|
|
235
|
+
/**
|
|
236
|
+
* The scroll-walk budget a fetch actually runs with: an explicit setting
|
|
237
|
+
* wins, else the schema default. `0` means the deep-capture walk is off.
|
|
238
|
+
*
|
|
239
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
240
|
+
* @returns the bounded walk budget in milliseconds.
|
|
241
|
+
*/
|
|
242
|
+
export declare function effectiveScrollBudgetMs(config: Pick<Config, 'scrollBudgetMs'>): number;
|
|
208
243
|
/**
|
|
209
244
|
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
210
245
|
* browser's real profile) requires the CDP backend — a local launch has no
|
package/lib/types/index.d.ts
CHANGED
|
@@ -15,12 +15,14 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import type { Context } from '@deepseek-ai/cordis';
|
|
17
17
|
import type { PlaywrightPluginConfig } from './config.ts';
|
|
18
|
-
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
|
|
18
|
+
export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_SCROLL_BUDGET_MS, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
|
|
19
19
|
export type { Config as PlaywrightFetchConfig, CdpContextMode, ContextIsolationScope, PlaywrightBackend, PlaywrightPluginConfig, ResolvedConfig, } from './config.ts';
|
|
20
20
|
export { CdpConnectionPool } from './cdp-pool.ts';
|
|
21
21
|
export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts';
|
|
22
22
|
export { CHALLENGE_DOM_PROBE, CHALLENGE_FINISH_RESERVE_MS, CHALLENGE_POLL_INTERVAL_MS, CHALLENGE_TITLE_RE, classifyChallengeHtml, classifyChallengeResponse, isChallengeCompatibleResponse } from './challenge.ts';
|
|
23
23
|
export type { ChallengeVerdict } from './challenge.ts';
|
|
24
|
+
export { deepCaptureWalk } from './autoscroll.ts';
|
|
25
|
+
export type { DeepCapture, DeepCaptureStats } from './autoscroll.ts';
|
|
24
26
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_CHALLENGE_CODE } from './provider.ts';
|
|
25
27
|
export { htmlToMarkdown } from './markdown.ts';
|
|
26
28
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts';
|
package/lib/types/markdown.d.ts
CHANGED
|
@@ -8,11 +8,14 @@
|
|
|
8
8
|
* footers, and ad chrome by scoring link density and text mass) — its result
|
|
9
9
|
* must then pass a coverage gate that distrusts tiny or sliver-thin
|
|
10
10
|
* extractions, which on list-shaped pages (portals, catalogs) is exactly
|
|
11
|
-
* what the link-density penalty leaves behind;
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
11
|
+
* what the link-density penalty leaves behind; an accepted extraction is
|
|
12
|
+
* then pruned of trailing link-dense sections — the post-article
|
|
13
|
+
* recommendation rails and share bars the scorer's vocabularies cannot name
|
|
14
|
+
* (they arrive as bare divs with no class to match); DOMPurify sanitizes
|
|
15
|
+
* whatever HTML remains and forbids the layout tags noise lives in;
|
|
16
|
+
* Turndown with the GFM plugin converts to markdown using the same style
|
|
17
|
+
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
18
|
+
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
16
19
|
*
|
|
17
20
|
* Synchronous for its callers and unit-tested against fixture pages: jsdom
|
|
18
21
|
* itself is loaded lazily on the first call — and, inside a dsh profile, only
|
package/lib/types/provider.d.ts
CHANGED
|
@@ -50,6 +50,31 @@
|
|
|
50
50
|
* fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
|
|
51
51
|
* interstitial as content.
|
|
52
52
|
*
|
|
53
|
+
* Deep capture (viewport-virtualized feeds and app-shell documents): feed-
|
|
54
|
+
* shaped pages — Discourse topic streams, comment threads, timelines —
|
|
55
|
+
* render only the items near the scroll position; everything else is an
|
|
56
|
+
* empty, height-only placeholder that keeps the item's id but none of its
|
|
57
|
+
* content, so a single-instant capture hands back the skeleton while the
|
|
58
|
+
* bodies silently vanish (and the denoise pipeline's coverage gate cannot
|
|
59
|
+
* notice: the page's own visible text is the same windowed fragment as the
|
|
60
|
+
* extraction). App-shell document editors (Feishu docs) push the same
|
|
61
|
+
* pathology one level down: the body never scrolls, blocks are keyed by
|
|
62
|
+
* data-block-id with no id, and off-window blocks are REMOVED behind an
|
|
63
|
+
* anonymous height spacer — the fetch would return just the statically
|
|
64
|
+
* rendered first screen. Before serialization the fetch therefore runs one
|
|
65
|
+
* bounded in-page scroll walk — over the window or, when the document does
|
|
66
|
+
* not scroll, the page's dominant inner scroll container — stashing what
|
|
67
|
+
* each rendered window shows, terminating only AT the bottom with the
|
|
68
|
+
* height stable (that is what pages load-more sentinels; height stability
|
|
69
|
+
* elsewhere proves nothing), then rehydrating degraded keyed elements from
|
|
70
|
+
* their stashed snapshots and splicing removed keys back at their document
|
|
71
|
+
* position so the final DOM holds the union of every window in document
|
|
72
|
+
* order. The existing pipeline then runs unchanged. Bounded by
|
|
73
|
+
* `scrollBudgetMs` (default 20s; 0 restores the legacy single-instant
|
|
74
|
+
* behavior) and by the remaining fetch deadline; any walk failure silently
|
|
75
|
+
* degrades to the legacy capture (see `src/autoscroll.ts` for the measured
|
|
76
|
+
* linux.do and Feishu badcases).
|
|
77
|
+
*
|
|
53
78
|
* @module dsh-web-fetch-playwright/provider
|
|
54
79
|
*/
|
|
55
80
|
import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web';
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serialization safety for documents about to be flattened into an HTML
|
|
3
|
+
* string and re-parsed by jsdom: the one in-page cleanup both serialization
|
|
4
|
+
* paths share — the provider's live-document read and the deep-capture
|
|
5
|
+
* walk's merged clone.
|
|
6
|
+
*
|
|
7
|
+
* @module dsh-web-fetch-playwright/strip
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* In-page function that removes non-content scripting subtrees from a
|
|
11
|
+
* document before it is flattened into an HTML string — the live document
|
|
12
|
+
* before `page.content()`, or the deep-capture walk's merged clone before it
|
|
13
|
+
* serializes it itself (`src/autoscroll.ts`).
|
|
14
|
+
*
|
|
15
|
+
* It is evaluated BY SOURCE (`fn.toString()`), so it must stay
|
|
16
|
+
* self-contained: no imports, no module-level references.
|
|
17
|
+
*
|
|
18
|
+
* WHAT goes, and why each removal is output-neutral versus the old pipeline:
|
|
19
|
+
*
|
|
20
|
+
* - `script` — three consumers ever saw script text before, and none loses
|
|
21
|
+
* it here: Readability's `parse()` deletes script/noscript nodes BEFORE
|
|
22
|
+
* extraction and scoring (so script text never reached the article or
|
|
23
|
+
* influenced the scorer); DOMPurify's default allowlist drops script; the
|
|
24
|
+
* tool layer's renderer removes SCRIPT wholesale on the raw path. What the
|
|
25
|
+
* removal buys is the reason this exists: a page's runtime JS may rewrite
|
|
26
|
+
* a data script's text with a literal `</script>` embedded — measured on
|
|
27
|
+
* techcommunity.microsoft.com (Khoros/Aurora), whose client writes its
|
|
28
|
+
* Apollo cache back into `script#__NEXT_DATA__` via `JSON.stringify`,
|
|
29
|
+
* turning the wire's 712 `\u003c` escapes back into raw `<` — and
|
|
30
|
+
* browsers serialize script text verbatim, so the flattened string ends
|
|
31
|
+
* that script at the FIRST embedded end tag and re-parses the remaining
|
|
32
|
+
* ~270KB of JSON as markup. The phantom elements that survive (unknown
|
|
33
|
+
* tags like `<id>`, anchors full of JSON text) are exactly what
|
|
34
|
+
* Readability's scorer reads as article mass, so the extracted "article"
|
|
35
|
+
* is CSS and i18n strings while the real body — still present in the DOM —
|
|
36
|
+
* loses. No string-level patch can undo this after serialization (the
|
|
37
|
+
* string is genuinely ambiguous); only the browser's own parse holds the
|
|
38
|
+
* document's true structure.
|
|
39
|
+
* - `script[type="application/ld+json"]` — KEPT, but defused: Readability
|
|
40
|
+
* extracts metadata (title/byline/excerpt — e.g. the site-suffix-free
|
|
41
|
+
* "Remote Desktop Protocol" instead of "… - Wikipedia") from JSON-LD
|
|
42
|
+
* scripts before removing them, so deleting them would regress the title
|
|
43
|
+
* on every JSON-LD-carrying page. Rewriting `</` as `<\/` keeps the text
|
|
44
|
+
* valid JSON (`\/` is a legal escape, so the re-parsed text parses to the
|
|
45
|
+
* same object) while making the serialization harmless: the tokenizer
|
|
46
|
+
* only terminates on a literal `</script`.
|
|
47
|
+
* - `noscript` — KEPT: Readability's `_unwrapNoscriptImages` salvages the
|
|
48
|
+
* real image out of lazy-load `<noscript><img …>` wrappers before
|
|
49
|
+
* `_removeScripts` deletes the noscript; pre-stripping it would silently
|
|
50
|
+
* drop those images from the markdown.
|
|
51
|
+
* - `template` — removed: its children parse into the inert `.content`
|
|
52
|
+
* fragment (invisible to Readability's traversal), DOMPurify's
|
|
53
|
+
* FORBID_TAGS drops it, and the tool layer's renderer removes it too.
|
|
54
|
+
* - `style` stays — its text cannot terminate a re-parse except at its own
|
|
55
|
+
* end tag, and the pipeline already drops it.
|
|
56
|
+
*/
|
|
57
|
+
export declare function stripScriptingElementsIn(doc: Document): void;
|
|
58
|
+
/** {@link stripScriptingElementsIn} over the live document, as an evaluate expression. */
|
|
59
|
+
export declare const STRIP_SCRIPTING_ELEMENTS: string;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "dsh-web-fetch-playwright",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.18",
|
|
4
4
|
"description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|