dsh-web-fetch-playwright 0.2.20 → 0.2.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -60,7 +60,8 @@
60
60
  * collecting after each, until the walk can hold AT THE BOTTOM through a
61
61
  * {@link SCROLL_BOTTOM_PATIENCE_MS} window with the height unchanged
62
62
  * (growth restarts the clock — that is what pages load-more sentinels),
63
- * or the step/budget caps close it.
63
+ * or the step/budget caps close it. A page that never reacted and issued
64
+ * no fetch/XHR holds only {@link SCROLL_INERT_PATIENCE_MS}.
64
65
  *
65
66
  * A MutationObserver watches for the walk's whole reason to exist: the
66
67
  * page REACTING to scroll by adding or removing elements. Most of the web
@@ -71,6 +72,14 @@
71
72
  * jumped over may hold content that never rendered, so the walk rewinds
72
73
  * to where it jumped from and finishes carefully. Correctness never
73
74
  * rests on the warm-up guess; only the cost of a scroll-inert page does.
75
+ *
76
+ * The same observer HARVESTS UNKEYED ROWS — the third shape: react-window
77
+ * lists and X-style timelines give rows no DOM identity at all (the
78
+ * framework key never reaches the DOM), and framework-generated ids
79
+ * (`ember123`, React useId's `:r1:`) change per render so they are not
80
+ * keys either. When the engine removes such a row, the record still
81
+ * holds it whole plus the sibling nodes it sat between; that is the row's
82
+ * stash and its splice anchor, with no layout read at all.
74
83
  * 4. MERGE into a structural clone (`document.cloneNode(true)`: no
75
84
  * serialize/re-parse round trip — a runtime-rewritten data script with a
76
85
  * literal `</script>` in its text cannot corrupt it — and, having no
@@ -84,11 +93,21 @@
84
93
  * spliced parent is already present and skipped. Placement is
85
94
  * structural, so it is right whatever order the walk SAW rows in (an
86
95
  * anchored open starts mid-document).
96
+ * - RESTORE (unkeyed rows): in every list the engine both removed text-
97
+ * bearing rows from and added to during the walk, each removed row the
98
+ * list no longer holds goes back beside the sibling it was removed
99
+ * next to (chains through removed anchors; rows with no surviving
100
+ * anchor at all — a wholesale innerHTML re-render — go above the final
101
+ * window, which is the only place a top-down walk loses them). Rows
102
+ * still present are recognised by a per-list identity attribute
103
+ * (data-index, aria-rowindex, …) or by text; in-place stubs are
104
+ * absorbed.
87
105
  * - SWAP: every keyed element emptier than its snapshot (fewer child
88
106
  * elements, or meaningfully less text — the Discourse placeholder) is
89
107
  * replaced by that snapshot, except containers (ancestors of another
90
108
  * element being swapped), which are assembled from those descendants
91
- * instead: a container's own snapshot is one window's partial view.
109
+ * instead: a container's own snapshot is one window's partial view. A
110
+ * restored list and its ancestors are containers too.
92
111
  * The clone is then stripped of scripting elements and serialized; the
93
112
  * provider reads that string instead of the live page. The live document
94
113
  * is never rewritten, so no late scroll/rAF/timer reaction can touch the
@@ -98,13 +117,16 @@
98
117
  * error, or an engine that rejects the walk just keeps the legacy
99
118
  * single-instant capture — a walk can only add content, never fail a fetch.
100
119
  * A walk that restored nothing returns no HTML, and the provider reads the
101
- * live page exactly as before.
120
+ * live page exactly as before — scrolled back to where it opened when the
121
+ * page reacted, so a virtualizer the merge could not rebuild hands over its
122
+ * opening window rather than the walk's last one.
102
123
  *
103
124
  * The walk runs in BOTH denoise modes (the raw-HTML path serializes the same
104
125
  * cloaked placeholders) but never for non-HTML documents. It is bounded by
105
126
  * the `scrollBudgetMs` setting (`0` restores the pre-walk legacy behavior)
106
- * and always by the remaining fetch deadline minus a decode reserve — the
107
- * provider's abort closes the tab, which rejects the pending evaluation.
127
+ * and always by the fetch's remaining waiting budget: a walk cut short there
128
+ * still merges and returns what it stashed, and the content read plus decode
129
+ * run in the provider's extraction grace window after it.
108
130
  *
109
131
  * @module dsh-web-fetch-playwright/autoscroll
110
132
  */
@@ -135,6 +157,16 @@ export declare const SCROLL_BOTTOM_POLL_MS = 450;
135
157
  * two 450ms checks), so this is sized over a slow round trip.
136
158
  */
137
159
  export declare const SCROLL_BOTTOM_PATIENCE_MS = 2500;
160
+ /**
161
+ * Bottom hold for a page that has given no sign of loading more: it never
162
+ * added or removed an element in response to the walk's scrolling, and it
163
+ * issued no fetch/XHR since the walk began. The full
164
+ * {@link SCROLL_BOTTOM_PATIENCE_MS} exists for feeds whose next chunk is in
165
+ * flight; holding it on every static article made the walk cost ~4s on
166
+ * pages it could never help. Any reaction or data request (even one that
167
+ * lands during this shorter hold) restores the full patience.
168
+ */
169
+ export declare const SCROLL_INERT_PATIENCE_MS = 900;
138
170
  /**
139
171
  * Careful steps without any DOM reaction before the walk treats the page as
140
172
  * scroll-inert and probes the rest with a single jump (rewinding if the
@@ -162,11 +194,6 @@ export declare const SCROLL_STASH_MAX_CHARS = 2000000;
162
194
  * rehydrate. Such keys are remembered and never serialized again.
163
195
  */
164
196
  export declare const SCROLL_STASH_MAX_ENTRY_CHARS = 64000;
165
- /**
166
- * Tail of the fetch budget the walk never spends, keeping room for the
167
- * content read and the synchronous denoise decode.
168
- */
169
- export declare const SCROLL_FINISH_RESERVE_MS = 4000;
170
197
  /**
171
198
  * A walk budget below this cannot complete even the first window plus one
172
199
  * stability check, so the walk is skipped outright.
@@ -195,6 +222,14 @@ export interface DeepCaptureStats {
195
222
  rehydrated: number;
196
223
  /** Removed keyed elements spliced back (container walks only). */
197
224
  spliced?: number;
225
+ /** Unkeyed rows a virtualizer removed, restored into their list. */
226
+ restored?: number;
227
+ /**
228
+ * True when nothing could be restored on a page that reacted, so the walk
229
+ * scrolled back to where the page opened before handing it to the
230
+ * provider's live read.
231
+ */
232
+ rewound?: boolean;
198
233
  /**
199
234
  * Whether the page reacted to scrolling (added/removed elements). False
200
235
  * means the walk took the scroll-inert fast path to the bottom.
@@ -224,6 +259,7 @@ export interface DeepCaptureParams {
224
259
  stepSettleMs: number;
225
260
  bottomPollMs: number;
226
261
  bottomPatienceMs: number;
262
+ inertPatienceMs: number;
227
263
  inertSteps: number;
228
264
  maxSteps: number;
229
265
  stashMaxEntries: number;
@@ -109,6 +109,13 @@ export declare function childElements(element: Element): Element[];
109
109
  export declare function unwrapNoscripts(document: Document): number;
110
110
  /** Visible text inside `<noscript>` elements (parsed either way). */
111
111
  export declare function noscriptTextLength(document: Document): number;
112
+ /**
113
+ * Cookie/consent banner wording (EN/DE/FR/ES/IT/NL/ZH), as a regex source
114
+ * shared with the selector's `consent_hits` feature so both read the same
115
+ * languages. Chinese banners say "Cookie" or 隐私; 同意 ("agree") alone is
116
+ * everyday prose and only counts as the banner's ACTION below.
117
+ */
118
+ export declare const CONSENT_TERMS: string;
112
119
  /**
113
120
  * Strip page chrome from a cloned body: chrome-named/roled subtrees, hidden
114
121
  * subtrees, and consent banners recognised by their wording. A subtree is
@@ -37,16 +37,6 @@ export interface PlaywrightSettings {
37
37
  denoise?: boolean;
38
38
  /** How many fetches may render at once (1–200); blank = backend default. */
39
39
  maxConcurrency?: number;
40
- /**
41
- * Bounded wait (ms) for a Cloudflare challenge to clear naturally in the
42
- * same tab (0–60000); 0 = off (return the first response as-is).
43
- */
44
- challengeWaitMs?: number;
45
- /**
46
- * Bounded budget (ms) for the deep-capture scroll walk (0–30000); 0 = off
47
- * (capture the document as it rests, the legacy single-instant behavior).
48
- */
49
- scrollBudgetMs?: number;
50
40
  }
51
41
  /** What the Playwright card renders. */
52
42
  export interface PlaywrightCardState extends CardShell {
@@ -63,10 +53,6 @@ export interface PlaywrightCardState extends CardShell {
63
53
  denoise: CardFieldState;
64
54
  /** Concurrency input (draft decimal integer). */
65
55
  maxConcurrency: CardFieldState;
66
- /** Challenge wait input (draft decimal integer of milliseconds). */
67
- challengeWaitMs: CardFieldState;
68
- /** Scroll-walk budget input (draft decimal integer of milliseconds). */
69
- scrollBudgetMs: CardFieldState;
70
56
  }
71
57
  /** The registration-side face the card's slot entry injects. */
72
58
  export interface PlaywrightCardFace extends CardActions {
@@ -5,7 +5,7 @@
5
5
  * @module dsh-web-fetch-playwright/client/locales
6
6
  */
7
7
  /** Locale keys this card renders. */
8
- export type PlaywrightCardLocaleKey = 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'contextMode' | 'isolateBrowserContext' | 'isolateBrowserContextHint' | 'contextIsolationScope' | 'contextScopeSession' | 'contextScopeSessionHint' | 'contextScopeGlobal' | 'contextScopeGlobalHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'challengeWaitMs' | 'challengeWaitMsHint' | 'challengeWaitMsPlaceholder' | 'scrollBudgetMs' | 'scrollBudgetMsHint' | 'scrollBudgetMsPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText';
8
+ export type PlaywrightCardLocaleKey = 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'shareBrowserContext' | 'shareBrowserContextHint' | 'contextMode' | 'isolateBrowserContext' | 'isolateBrowserContextHint' | 'contextIsolationScope' | 'contextScopeSession' | 'contextScopeSessionHint' | 'contextScopeGlobal' | 'contextScopeGlobalHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'save' | 'saving' | 'discard' | 'saveFailed' | 'invalidText';
9
9
  /** This plugin's dictionary namespace, merged into the locale key map. */
10
10
  declare module '@deepseek-ai/dsh-client-ui-slots' {
11
11
  interface LocaleNamespaceMap {
@@ -15,16 +15,16 @@
15
15
  import type { Volatile } from '@deepseek-ai/cordis';
16
16
  import z from '@deepseek-ai/schemastery';
17
17
  /**
18
- * Default bounded budget (ms) for the deep-capture scroll walk
19
- * (`src/autoscroll.ts`). Sized so long topics stay fully covered at the
20
- * careful cadence (300ms motion + 300ms rest per ~0.7-viewport step, plus
21
- * the bottom patience window): 20s walks ~25kpx with pagination, covering
22
- * the measured linux.do badcase (~18kpx, 51 replies) with headroom.
23
- * Scroll-inert pages take the walk's fast path and spend a few seconds.
18
+ * Default CAP (ms) on the deep-capture scroll walk (`src/autoscroll.ts`).
19
+ * It bounds only pages that keep reacting to scrolling — virtualized feeds
20
+ * and load-more streams — where 20s covers ~25kpx at the careful cadence
21
+ * (`SCROLL_STEP_MOTION_MS` + `SCROLL_STEP_SETTLE_MS` per ~0.7-viewport
22
+ * step). It is not what a typical page pays: a page that never reacts takes
23
+ * `SCROLL_INERT_STEPS` careful steps, one jump to the bottom and a
24
+ * `SCROLL_INERT_PATIENCE_MS` hold, and a page that does not scroll is
25
+ * skipped outright.
24
26
  */
25
27
  export declare const DEFAULT_SCROLL_BUDGET_MS = 20000;
26
- /** Ceiling the schema accepts for `scrollBudgetMs`. */
27
- export declare const MAX_SCROLL_BUDGET_MS = 30000;
28
28
  /** Default CDP endpoint when the settings section leaves it blank. */
29
29
  export declare const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
30
30
  /**
@@ -45,16 +45,24 @@ export declare const MAX_CONCURRENCY_CEILING = 200;
45
45
  /**
46
46
  * Default bounded wait (ms) for a Cloudflare challenge to clear naturally —
47
47
  * the user's real browser passes the verification on its own while the fetch
48
- * holds the same page and context. Sized to fit the 45s per-fetch deadline
49
- * with the settle/decode tail (and one retry) still inside it.
48
+ * holds the same page and context. Bounded further by the 25s per-fetch
49
+ * waiting budget, which keeps a settle/decode tail after the wait.
50
50
  */
51
51
  export declare const DEFAULT_CHALLENGE_WAIT_MS = 15000;
52
- /** Ceiling the schema accepts for `challengeWaitMs`. */
53
- export declare const MAX_CHALLENGE_WAIT_MS = 60000;
54
52
  /** Default same-page re-navigation attempts after a challenge wait runs out. */
55
53
  export declare const DEFAULT_CHALLENGE_RETRIES = 1;
56
54
  /** Ceiling the schema accepts for `challengeRetries`. */
57
55
  export declare const MAX_CHALLENGE_RETRIES = 3;
56
+ /**
57
+ * Default for `challengeHttpFallback`: when the browser cannot clear a
58
+ * challenge inside its window (the interactive checkbox was never offered,
59
+ * or verification refused), refetch the URL once through the page's
60
+ * plain-HTTP request context — the same cookie jar, no render decision —
61
+ * because many Cloudflare edges serve the real document to non-browser
62
+ * clients while withholding the interactive challenge from the browser
63
+ * session (measured live: v2ex.com from a datacenter IP).
64
+ */
65
+ export declare const DEFAULT_CHALLENGE_HTTP_FALLBACK = true;
58
66
  /** Which browser backend serves a fetch. */
59
67
  export type PlaywrightBackend = 'local' | 'cdp';
60
68
  /**
@@ -98,7 +106,9 @@ export interface Config {
98
106
  * Bounded wait (ms) for a Cloudflare challenge to clear naturally inside
99
107
  * the same page/context. `0` (or any config leaving this at 0) restores the
100
108
  * legacy behavior: the first response is final, no waiting — the A/B
101
- * baseline. Schema default: {@link DEFAULT_CHALLENGE_WAIT_MS}.
109
+ * baseline. Code-level only (tests, demo scripts): not part of the
110
+ * settings schema, so users cannot configure it. Default:
111
+ * {@link DEFAULT_CHALLENGE_WAIT_MS}.
102
112
  */
103
113
  challengeWaitMs?: number;
104
114
  /**
@@ -107,13 +117,22 @@ export interface Config {
107
117
  * Schema default: {@link DEFAULT_CHALLENGE_RETRIES}.
108
118
  */
109
119
  challengeRetries?: number;
120
+ /**
121
+ * When the browser cannot clear a challenge inside its window (no
122
+ * checkbox offered, or verification refused), refetch the URL once
123
+ * through the page's plain-HTTP request context (same cookie jar) and
124
+ * return that document when it is not itself a challenge. `false`
125
+ * restores the browser-only posture.
126
+ */
127
+ challengeHttpFallback?: boolean;
110
128
  /**
111
129
  * Bounded budget (ms) for the deep-capture scroll walk — the in-page pass
112
130
  * that defeats viewport virtualization on feed-shaped pages (Discourse
113
131
  * topic streams: replies outside the render window are empty height-only
114
132
  * placeholders carrying the post's id; see `src/autoscroll.ts`). `0`
115
- * restores the legacy single-instant capture.
116
- * Schema default: {@link DEFAULT_SCROLL_BUDGET_MS}.
133
+ * restores the legacy single-instant capture. Code-level only (tests,
134
+ * demo scripts): not part of the settings schema, so users cannot
135
+ * configure it. Default: {@link DEFAULT_SCROLL_BUDGET_MS}.
117
136
  */
118
137
  scrollBudgetMs?: number;
119
138
  /**
@@ -131,9 +150,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
131
150
  contextIsolationScope: z<"session" | "global", "session" | "global", "volatile-defined">;
132
151
  denoise: z<boolean, boolean, "volatile-defined">;
133
152
  maxConcurrency: z<number, number, "volatile">;
134
- challengeWaitMs: z<number, number, "volatile-defined">;
135
153
  challengeRetries: z<number, number, "volatile-defined">;
136
- scrollBudgetMs: z<number, number, "volatile-defined">;
154
+ challengeHttpFallback: z<boolean, boolean, "volatile-defined">;
137
155
  }>>, Schemastery.ObjectT<NoInfer<{
138
156
  backend: z<"local" | "cdp", "local" | "cdp", "volatile-defined">;
139
157
  playwrightPath: z<string, string, "volatile-defined">;
@@ -142,9 +160,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
142
160
  contextIsolationScope: z<"session" | "global", "session" | "global", "volatile-defined">;
143
161
  denoise: z<boolean, boolean, "volatile-defined">;
144
162
  maxConcurrency: z<number, number, "volatile">;
145
- challengeWaitMs: z<number, number, "volatile-defined">;
146
163
  challengeRetries: z<number, number, "volatile-defined">;
147
- scrollBudgetMs: z<number, number, "volatile-defined">;
164
+ challengeHttpFallback: z<boolean, boolean, "volatile-defined">;
148
165
  }>>, "plain">;
149
166
  /**
150
167
  * The config `apply` receives once the schema resolves: every volatile field
@@ -169,12 +186,10 @@ export interface PlaywrightPluginConfig {
169
186
  * field owns no default — the backend resolves one).
170
187
  */
171
188
  maxConcurrency: Volatile<number | undefined>;
172
- /** Challenge wait budget (ms) as a live reference. */
173
- challengeWaitMs: Volatile<number>;
174
189
  /** Challenge retry count as a live reference. */
175
190
  challengeRetries: Volatile<number>;
176
- /** Deep-capture scroll-walk budget (ms) as a live reference. */
177
- scrollBudgetMs: Volatile<number>;
191
+ /** Challenge dead-end plain-HTTP refetch toggle as a live reference. */
192
+ challengeHttpFallback: Volatile<boolean>;
178
193
  }
179
194
  /**
180
195
  * Read the live references `apply` holds into one plain input for
@@ -198,13 +213,14 @@ export declare function snapshotsOf(config: PlaywrightPluginConfig): Config;
198
213
  export declare function resolveConfig(config: Partial<Config>): ResolvedConfig;
199
214
  /**
200
215
  * Complete config after schemastery applies the field defaults it owns.
201
- * `maxConcurrency` and `scrollBudgetMs` stay optional — the former because
202
- * its effective default depends on `backend`, the latter so hand-built
203
- * resolved configs (tests) may omit it and let
204
- * {@link effectiveScrollBudgetMs} supply the default the schema would have.
216
+ * `maxConcurrency` stays optional because its effective default depends on
217
+ * `backend`; `challengeWaitMs` and `scrollBudgetMs` stay optional because
218
+ * they are code-level overrides outside the schema — absent, the
219
+ * effective* helpers supply the built-in defaults.
205
220
  */
206
- export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBudgetMs'> & {
221
+ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'challengeWaitMs' | 'scrollBudgetMs'> & {
207
222
  maxConcurrency?: number;
223
+ challengeWaitMs?: number;
208
224
  scrollBudgetMs?: number;
209
225
  };
210
226
  /**
@@ -218,7 +234,7 @@ export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency' | 'scrollBu
218
234
  export declare function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number;
219
235
  /**
220
236
  * The challenge wait budget a fetch actually runs with: an explicit setting
221
- * wins, else the schema default. `0` means the challenge path is off.
237
+ * wins, else the built-in default. `0` means the challenge path is off.
222
238
  *
223
239
  * @param config - the resolved settings section (or any partial of it).
224
240
  * @returns the bounded wait in milliseconds.
@@ -232,9 +248,17 @@ export declare function effectiveChallengeWaitMs(config: Pick<Config, 'challenge
232
248
  * @returns retries after a wait window runs out.
233
249
  */
234
250
  export declare function effectiveChallengeRetries(config: Pick<Config, 'challengeRetries'>): number;
251
+ /**
252
+ * Whether a challenge dead-end may refetch through plain HTTP: an explicit
253
+ * setting wins (absent = on, the default).
254
+ *
255
+ * @param config - the resolved settings section (or any partial of it).
256
+ * @returns true when the plain-HTTP fallback may run.
257
+ */
258
+ export declare function effectiveChallengeHttpFallback(config: Pick<Config, 'challengeHttpFallback'>): boolean;
235
259
  /**
236
260
  * The scroll-walk budget a fetch actually runs with: an explicit setting
237
- * wins, else the schema default. `0` means the deep-capture walk is off.
261
+ * wins, else the built-in default. `0` means the deep-capture walk is off.
238
262
  *
239
263
  * @param config - the resolved settings section (or any partial of it).
240
264
  * @returns the bounded walk budget in milliseconds.
@@ -15,7 +15,7 @@
15
15
  */
16
16
  import type { Context } from '@deepseek-ai/cordis';
17
17
  import type { PlaywrightPluginConfig } from './config.ts';
18
- export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, MAX_SCROLL_BUDGET_MS, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
18
+ export { Config, DEFAULT_CDP_ENDPOINT, DEFAULT_CHALLENGE_HTTP_FALLBACK, DEFAULT_CHALLENGE_RETRIES, DEFAULT_CHALLENGE_WAIT_MS, DEFAULT_SCROLL_BUDGET_MS, effectiveChallengeHttpFallback, effectiveChallengeRetries, effectiveChallengeWaitMs, effectiveContextMode, effectiveScrollBudgetMs, normalizeCdpEndpoint, resolveConfig, snapshotsOf, } from './config.ts';
19
19
  export type { Config as PlaywrightFetchConfig, CdpContextMode, ContextIsolationScope, PlaywrightBackend, PlaywrightPluginConfig, ResolvedConfig, } from './config.ts';
20
20
  export { CdpConnectionPool } from './cdp-pool.ts';
21
21
  export type { CdpAcquireMode, CdpConnect, CdpLease } from './cdp-pool.ts';
@@ -55,9 +55,14 @@
55
55
  * behavior). The wait tracks the LAST main-frame navigation response (the
56
56
  * real page reloads in after the challenge clears) and watches the live DOM
57
57
  * so SPA-style clears are caught too. It never clicks, never injects
58
- * answers, never touches cookies itself; when the budget runs out the fetch
59
- * fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of returning the
60
- * interstitial as content.
58
+ * answers, never touches cookies itself; when the budget runs out, ONE
59
+ * plain-HTTP refetch of the same URL through the page's request context
60
+ * (same cookie jar, the browser's UA, none of Chromium's render decisions
61
+ * — `challengeHttpFallback`, default on) tries to retrieve the real
62
+ * document, gated by the same response-level and content-level challenge
63
+ * classifiers so an interstitial can never pass as content. When that too
64
+ * fails, the fetch fails with {@link WEB_FETCH_CHALLENGE_CODE} instead of
65
+ * returning the interstitial as content.
61
66
  *
62
67
  * Deep capture (viewport-virtualized feeds and app-shell documents): feed-
63
68
  * shaped pages — Discourse topic streams, comment threads, timelines —
@@ -84,6 +89,17 @@
84
89
  * degrades to the legacy capture (see `src/autoscroll.ts` for the measured
85
90
  * linux.do and Feishu badcases).
86
91
  *
92
+ * Deadline (soft, then hard): each fetch has a {@link DEFAULT_TIMEOUT_MS}
93
+ * budget for WAITING — navigation, settle, challenge wait, scroll walk. When
94
+ * it runs out the page is NOT torn down: every wait is bounded by the same
95
+ * budget and simply ends, a navigation still loading is stopped
96
+ * (`window.stop()`) once its main-frame response has arrived, and whatever
97
+ * the page holds by then goes through the normal extraction — a slow or
98
+ * endless page still hands the agent the content it already rendered
99
+ * (flagged `truncated` when the document never finished loading). Only when
100
+ * nothing arrived at all, or extraction itself hangs past
101
+ * {@link EXTRACT_GRACE_MS}, does the fetch fail with `WEB_FETCH_TIMEOUT`.
102
+ *
87
103
  * @module dsh-web-fetch-playwright/provider
88
104
  */
89
105
  import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@deepseek-ai/dsh-web';
@@ -119,19 +135,25 @@ export interface BrowserSession {
119
135
  release?: () => Promise<void>;
120
136
  }
121
137
  /**
122
- * A single abort deadline composing the caller's signal with this provider's
123
- * time budget. The timeout reason is kept apart from an outer abort so the
124
- * error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
138
+ * The fetch's two-stage deadline composed with the caller's signal. The SOFT
139
+ * stage (`timeoutMs`) only bounds waiting: {@link remainingMs} counts down to
140
+ * it and {@link expired} reports it, so every wait ends there and the fetch
141
+ * extracts what the page holds. The HARD stage (`timeoutMs + graceMs`) aborts
142
+ * {@link signal}, which closes the page. The timeout reason is kept apart
143
+ * from an outer abort so the error translation can pick `WEB_FETCH_TIMEOUT`
144
+ * over `WEB_ABORTED`.
125
145
  */
126
146
  declare class Deadline {
127
147
  readonly signal: AbortSignal;
128
148
  private readonly controller;
129
149
  private readonly expiresAt;
130
150
  private timedOut;
131
- constructor(outer: AbortSignal | undefined, timeoutMs: number);
132
- /** Whether OUR timer fired (vs an outer cancellation). */
151
+ constructor(outer: AbortSignal | undefined, timeoutMs: number, graceMs?: number);
152
+ /** Whether OUR hard timer fired (vs an outer cancellation). */
133
153
  get isTimeout(): boolean;
134
- /** Milliseconds left on the budget, floored at 1 for Playwright options. */
154
+ /** Whether the soft (waiting) budget has run out. */
155
+ get expired(): boolean;
156
+ /** Milliseconds left on the waiting budget, floored at 1 for Playwright options. */
135
157
  remainingMs(): number;
136
158
  }
137
159
  /**
@@ -144,6 +166,10 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
144
166
  private readonly sessionIdSource;
145
167
  readonly id = "playwright";
146
168
  private readonly semaphore;
169
+ /** Waiting budget per fetch (ms); code-level, overridable by subclasses (tests). */
170
+ protected readonly fetchTimeoutMs: number;
171
+ /** Extraction grace after the waiting budget (ms); code-level, overridable by subclasses. */
172
+ protected readonly extractGraceMs: number;
147
173
  /** Shared CDP connection; injectable so the suite can substitute a fake. */
148
174
  protected readonly cdpPool: CdpConnectionPool;
149
175
  /**
@@ -174,6 +200,17 @@ export declare class PlaywrightFetchProvider implements WebFetchProvider {
174
200
  * budget before the final document is read (see the class docs).
175
201
  */
176
202
  private retrieve;
203
+ /**
204
+ * The challenge dead-end's plain-HTTP refetch: one request through the
205
+ * page's own request context (the SAME cookie jar — any clearance the
206
+ * profile holds rides along — but none of Chromium's render decisions),
207
+ * wearing the browser's own User-Agent. Returns the finished result when
208
+ * the answer is a NON-challenge HTML document, undefined otherwise (the
209
+ * caller throws the honest WEB_FETCH_CHALLENGE). A challenge served to
210
+ * this request too, a non-HTML body, or any transport failure keeps the
211
+ * browser-only verdict.
212
+ */
213
+ private challengeHttpFallback;
177
214
  /**
178
215
  * One tracked `goto`: the navigation response on success, or — when
179
216
  * Chromium failed the navigation for a reason that is not a transport
@@ -33,17 +33,18 @@ export interface PlaywrightDownload {
33
33
  cancel(): Promise<void>;
34
34
  }
35
35
  /**
36
- * The page's `APIRequestContext` surface the download recovery refetches
37
- * through — plain HTTP from the Playwright client, the context's cookie jar,
38
- * and none of Chromium's render-or-download semantics. Reuses
39
- * {@link PlaywrightResponse} because that is structurally what an
40
- * `APIResponse` is for the members this plugin reads. Absent on fakes that
41
- * do not exercise the recovery.
36
+ * The page's `APIRequestContext` surface the download recovery and the
37
+ * challenge dead-end fallback refetch through — plain HTTP from the
38
+ * Playwright client, the context's cookie jar, and none of Chromium's
39
+ * render-or-download semantics. Reuses {@link PlaywrightResponse} because
40
+ * that is structurally what an `APIResponse` is for the members this
41
+ * plugin reads. Absent on fakes that do not exercise the recovery.
42
42
  */
43
43
  export interface PlaywrightRequestContext {
44
44
  get(url: string, options?: {
45
45
  timeout?: number;
46
46
  signal?: AbortSignal;
47
+ headers?: Record<string, string>;
47
48
  }): Promise<PlaywrightResponse>;
48
49
  }
49
50
  /** The request side of a response, for main-frame filtering. */
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "dsh-web-fetch-playwright",
3
- "version": "0.2.20",
3
+ "version": "0.2.22",
4
4
  "description": "Playwright/CDP web-fetch provider for DeepSeek Harness: renders pages in a real browser, denoises them (Readability + DOMPurify), and returns markdown.",
5
5
  "type": "module",
6
6
  "license": "MIT",