dsh-web-fetch-playwright 0.1.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +27 -5
- package/README.zh-CN.md +27 -5
- package/dsh.plugin.json +1 -1
- package/lib/client.js +185 -105
- package/lib/index.js +362 -43
- package/lib/types/cdp-pool.d.ts +100 -0
- package/lib/types/client/card.d.ts +4 -3
- package/lib/types/client/controller.d.ts +12 -0
- package/lib/types/client/fields.d.ts +7 -1
- package/lib/types/client/form.d.ts +10 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +62 -2
- package/lib/types/index.d.ts +4 -2
- package/lib/types/provider.d.ts +40 -10
- package/lib/types/types.d.ts +22 -1
- package/package.json +4 -3
- package/src/cdp-pool.ts +211 -0
- package/src/client/card.tsx +46 -17
- package/src/client/controller.ts +17 -1
- package/src/client/fields.module.css +12 -0
- package/src/client/fields.tsx +9 -3
- package/src/client/form.ts +23 -0
- package/src/client/locales.ts +12 -0
- package/src/config.ts +76 -2
- package/src/index.ts +9 -3
- package/src/provider.ts +220 -47
- package/src/types.ts +22 -1
package/lib/index.js
CHANGED
|
@@ -25,9 +25,37 @@ const Config = z.object({
|
|
|
25
25
|
backend: z.union([z.const("local"), z.const("cdp")]).default("local"),
|
|
26
26
|
playwrightPath: z.string().default(""),
|
|
27
27
|
cdpEndpoint: z.string().default(""),
|
|
28
|
-
|
|
28
|
+
shareBrowserContext: z.boolean().default(true),
|
|
29
|
+
denoise: z.boolean().default(true),
|
|
30
|
+
maxConcurrency: z.number().step(1).min(1).max(200)
|
|
29
31
|
});
|
|
30
32
|
/**
|
|
33
|
+
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
34
|
+
* wins; otherwise the backend default (local launches browsers, CDP only
|
|
35
|
+
* opens tabs, so their budgets differ by an order of magnitude).
|
|
36
|
+
*
|
|
37
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
38
|
+
* @returns the effective limit for the semaphore.
|
|
39
|
+
*/
|
|
40
|
+
function effectiveMaxConcurrency(config) {
|
|
41
|
+
if (typeof config.maxConcurrency === "number") return config.maxConcurrency;
|
|
42
|
+
return config.backend === "cdp" ? 50 : 4;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The context mode a fetch actually runs with: profile (a tab in the remote
|
|
46
|
+
* browser's real profile) requires the CDP backend — a local launch has no
|
|
47
|
+
* meaningful shared profile — and the checkbox (which defaults on, so an
|
|
48
|
+
* absent value reads as `true`, mirroring the schema default); everything
|
|
49
|
+
* else collapses to isolated.
|
|
50
|
+
*
|
|
51
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
52
|
+
* @returns the mode the pool acquires its lease with.
|
|
53
|
+
*/
|
|
54
|
+
function effectiveContextMode(config) {
|
|
55
|
+
if (config.backend === "cdp" && config.shareBrowserContext !== false) return "profile";
|
|
56
|
+
return "isolated";
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
31
59
|
* Normalize a configured CDP endpoint for `chromium.connectOverCDP`:
|
|
32
60
|
* blank becomes the default loopback endpoint; `wss?://` and `https?://`
|
|
33
61
|
* pass through; a bare `host:port` gains the `http://` scheme Playwright's
|
|
@@ -45,6 +73,160 @@ function normalizeCdpEndpoint(input) {
|
|
|
45
73
|
return candidate;
|
|
46
74
|
}
|
|
47
75
|
//#endregion
|
|
76
|
+
//#region src/cdp-pool.ts
|
|
77
|
+
/** A reusable `connectOverCDP` session handing out per-fetch pages. */
|
|
78
|
+
var CdpConnectionPool = class {
|
|
79
|
+
connect;
|
|
80
|
+
browser;
|
|
81
|
+
endpoint = "";
|
|
82
|
+
connecting;
|
|
83
|
+
connectingEndpoint = "";
|
|
84
|
+
/** Bumped by dispose/replace so a settling connect knows it was abandoned. */
|
|
85
|
+
generation = 0;
|
|
86
|
+
/**
|
|
87
|
+
* @param connect - opens a connection to an endpoint (the provider's real
|
|
88
|
+
* one resolves the bundled playwright-core; tests inject fakes).
|
|
89
|
+
*/
|
|
90
|
+
constructor(connect) {
|
|
91
|
+
this.connect = connect;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Lease a fetch's page on the shared connection, connecting (or
|
|
95
|
+
* reconnecting) first if needed. Concurrent first fetches share one
|
|
96
|
+
* connect attempt.
|
|
97
|
+
*
|
|
98
|
+
* @param endpoint - normalized CDP endpoint URL.
|
|
99
|
+
* @param timeoutMs - connect timeout budget.
|
|
100
|
+
* @param mode - `isolated` (default): a fresh throwaway context plus a page
|
|
101
|
+
* in it; `profile`: a page in the remote browser's default context — the
|
|
102
|
+
* real profile — which is never closed, only the page is.
|
|
103
|
+
* @returns the shared browser, the page's context, and the page the caller
|
|
104
|
+
* must release.
|
|
105
|
+
* @throws whatever the connect function throws when no connection can be
|
|
106
|
+
* established (the provider wraps it in a structured WebError), or a
|
|
107
|
+
* diagnostic error when `profile` mode finds no default context.
|
|
108
|
+
*/
|
|
109
|
+
async acquire(endpoint, timeoutMs, mode = "isolated") {
|
|
110
|
+
const browser = await this.ensure(endpoint, timeoutMs);
|
|
111
|
+
try {
|
|
112
|
+
return await this.openLease(browser, mode);
|
|
113
|
+
} catch (error) {
|
|
114
|
+
if (this.isLive(browser)) throw error;
|
|
115
|
+
this.drop(browser);
|
|
116
|
+
const fresh = await this.ensure(endpoint, timeoutMs);
|
|
117
|
+
return await this.openLease(fresh, mode);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Open one lease's page on a live connection. The profile-mode handle is
|
|
122
|
+
* taken fresh every call (never cached across connections): a reconnect
|
|
123
|
+
* yields a new Browser object whose `contexts()[0]` must be re-read.
|
|
124
|
+
*/
|
|
125
|
+
async openLease(browser, mode) {
|
|
126
|
+
if (mode === "profile") {
|
|
127
|
+
const context = browser.contexts?.()[0];
|
|
128
|
+
if (context === void 0) throw new Error("the CDP endpoint exposed no default browser context (profile mode requires a real browser profile)");
|
|
129
|
+
return {
|
|
130
|
+
browser,
|
|
131
|
+
context,
|
|
132
|
+
page: await context.newPage(),
|
|
133
|
+
persistent: true
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
const context = await browser.newContext();
|
|
137
|
+
try {
|
|
138
|
+
return {
|
|
139
|
+
browser,
|
|
140
|
+
context,
|
|
141
|
+
page: await context.newPage(),
|
|
142
|
+
persistent: false
|
|
143
|
+
};
|
|
144
|
+
} catch (error) {
|
|
145
|
+
await context.close().catch(() => {});
|
|
146
|
+
throw error;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Close a fetch-owned page, plus its context when the lease owns one. The
|
|
151
|
+
* remote default context (persistent leases) and the shared connection
|
|
152
|
+
* stay for the next fetch.
|
|
153
|
+
* @param lease - the lease whose page (and, when isolated, context) goes away.
|
|
154
|
+
*/
|
|
155
|
+
async release(lease) {
|
|
156
|
+
await lease.page.close().catch(() => {});
|
|
157
|
+
if (!lease.persistent) await lease.context.close().catch(() => {});
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Drop the shared connection (plugin teardown or endpoint change). Fetches
|
|
161
|
+
* holding leases keep their contexts until they release them. A connect
|
|
162
|
+
* still in flight is abandoned — its own continuation closes the stray
|
|
163
|
+
* browser (nothing here waits on it, so teardown cannot deadlock).
|
|
164
|
+
*/
|
|
165
|
+
async dispose() {
|
|
166
|
+
const browser = this.browser;
|
|
167
|
+
this.browser = void 0;
|
|
168
|
+
this.endpoint = "";
|
|
169
|
+
this.connecting = void 0;
|
|
170
|
+
this.connectingEndpoint = "";
|
|
171
|
+
this.generation++;
|
|
172
|
+
await browser?.close().catch(() => {});
|
|
173
|
+
}
|
|
174
|
+
/** The shared connection for `endpoint`, connecting or reconnecting as needed. */
|
|
175
|
+
async ensure(endpoint, timeoutMs) {
|
|
176
|
+
if (this.browser !== void 0 && this.endpoint === endpoint && this.isLive(this.browser)) return this.browser;
|
|
177
|
+
if (this.connecting !== void 0 && this.connectingEndpoint === endpoint) return this.connecting;
|
|
178
|
+
return await this.connectFresh(endpoint, timeoutMs);
|
|
179
|
+
}
|
|
180
|
+
/** Start a connection to `endpoint`, superseding whatever was there. */
|
|
181
|
+
connectFresh(endpoint, timeoutMs) {
|
|
182
|
+
const stale = this.browser;
|
|
183
|
+
this.browser = void 0;
|
|
184
|
+
this.endpoint = "";
|
|
185
|
+
const generation = ++this.generation;
|
|
186
|
+
const attempt = (async () => {
|
|
187
|
+
const browser = await this.connect(endpoint, timeoutMs);
|
|
188
|
+
if (generation !== this.generation) {
|
|
189
|
+
await browser.close().catch(() => {});
|
|
190
|
+
throw new Error("cdp connection abandoned before it attached");
|
|
191
|
+
}
|
|
192
|
+
this.browser = browser;
|
|
193
|
+
this.endpoint = endpoint;
|
|
194
|
+
this.watch(browser);
|
|
195
|
+
return browser;
|
|
196
|
+
})();
|
|
197
|
+
this.connecting = attempt;
|
|
198
|
+
this.connectingEndpoint = endpoint;
|
|
199
|
+
attempt.then(() => {
|
|
200
|
+
if (this.connecting === attempt) this.connecting = void 0;
|
|
201
|
+
}, () => {
|
|
202
|
+
if (this.connecting === attempt) this.connecting = void 0;
|
|
203
|
+
});
|
|
204
|
+
stale?.close().catch(() => {});
|
|
205
|
+
return attempt;
|
|
206
|
+
}
|
|
207
|
+
/** Clear the reference when this exact connection reports it went away. */
|
|
208
|
+
watch(browser) {
|
|
209
|
+
try {
|
|
210
|
+
browser.on?.("disconnected", () => {
|
|
211
|
+
if (this.browser === browser) {
|
|
212
|
+
this.browser = void 0;
|
|
213
|
+
this.endpoint = "";
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
} catch {}
|
|
217
|
+
}
|
|
218
|
+
/** Forget a connection known to be dead (absent probe = assume dead here). */
|
|
219
|
+
drop(browser) {
|
|
220
|
+
if (this.browser === browser) {
|
|
221
|
+
this.browser = void 0;
|
|
222
|
+
this.endpoint = "";
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
isLive(browser) {
|
|
226
|
+
return browser.isConnected?.() !== false;
|
|
227
|
+
}
|
|
228
|
+
};
|
|
229
|
+
//#endregion
|
|
48
230
|
//#region src/markdown.ts
|
|
49
231
|
/**
|
|
50
232
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
@@ -373,14 +555,23 @@ function existsAsFile(path) {
|
|
|
373
555
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
374
556
|
* classification) so the tool layer sees the same codes from either backend.
|
|
375
557
|
*
|
|
376
|
-
* Lifecycle
|
|
377
|
-
*
|
|
378
|
-
*
|
|
379
|
-
*
|
|
558
|
+
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
559
|
+
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
560
|
+
* the remote browser for the provider's lifetime; each fetch only opens a
|
|
561
|
+
* page (tab) inside it and closes that on completion — in a throwaway
|
|
562
|
+
* isolated context, or (the default, `shareBrowserContext`) in the remote
|
|
563
|
+
* browser's default context so its profile, cookies, and persistent logins
|
|
564
|
+
* apply; that default context is never closed. The concurrency cap counts
|
|
565
|
+
* tabs, not browsers. Either way, an aborted signal closes the fetch's page.
|
|
566
|
+
* Concurrency is capped at the `maxConcurrency` setting (explicit, or the
|
|
567
|
+
* backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL} browsers /
|
|
568
|
+
* {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait briefly in
|
|
569
|
+
* a queue and fail fast (rather than hang until abort) when no slot frees.
|
|
380
570
|
*
|
|
381
571
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
382
572
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
383
|
-
* browser can reach.
|
|
573
|
+
* browser can reach. Profile mode additionally acts WITH the remote
|
|
574
|
+
* browser's logged-in sessions (see the README's risk notes).
|
|
384
575
|
*
|
|
385
576
|
* @module dsh-web-fetch-playwright/provider
|
|
386
577
|
*/
|
|
@@ -392,13 +583,24 @@ const MAX_URL_LENGTH = 2048;
|
|
|
392
583
|
const MAX_BODY_CHARS = 1e5;
|
|
393
584
|
/** Cap on rendered HTML fed into the synchronous denoise pipeline. */
|
|
394
585
|
const MAX_PIPELINE_INPUT_CHARS = 2e6;
|
|
395
|
-
/**
|
|
396
|
-
|
|
586
|
+
/**
|
|
587
|
+
* How long a fetch may sit in the concurrency queue before failing fast.
|
|
588
|
+
* Waiting longer cannot help — the per-fetch deadline leaves too little
|
|
589
|
+
* budget to render after dequeue — and failing fast tells the caller to
|
|
590
|
+
* retry or raise `maxConcurrency` instead of hanging until an abort.
|
|
591
|
+
*/
|
|
592
|
+
const QUEUE_TIMEOUT_MS = 2e4;
|
|
397
593
|
/** Default per-fetch budget (ms), inside the tool layer's 60s. */
|
|
398
594
|
const DEFAULT_TIMEOUT_MS = 45e3;
|
|
399
595
|
/** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
|
|
400
596
|
const SETTLE_MS = 5e3;
|
|
401
597
|
/**
|
|
598
|
+
* Grace period (ms) for context/browser closes in the cleanup path. A wedged
|
|
599
|
+
* `close()` must never hold a concurrency slot hostage — a leaked slot would
|
|
600
|
+
* fail every later fetch with the queue error until restart.
|
|
601
|
+
*/
|
|
602
|
+
const CLOSE_GRACE_MS = 2e3;
|
|
603
|
+
/**
|
|
402
604
|
* A single abort deadline composing the caller's signal with this provider's
|
|
403
605
|
* time budget. The timeout reason is kept apart from an outer abort so the
|
|
404
606
|
* error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
|
|
@@ -434,41 +636,78 @@ var Deadline = class {
|
|
|
434
636
|
return Math.max(1, this.expiresAt - Date.now());
|
|
435
637
|
}
|
|
436
638
|
};
|
|
437
|
-
/**
|
|
639
|
+
/**
|
|
640
|
+
* A bounded async semaphore with abort support. The limit is live-resizable
|
|
641
|
+
* (`resize`) because the config thunk re-reads on every fetch — raising it
|
|
642
|
+
* wakes queued waiters immediately; lowering it lets in-flight holders run
|
|
643
|
+
* out naturally.
|
|
644
|
+
*/
|
|
438
645
|
var Semaphore = class {
|
|
439
646
|
active = 0;
|
|
647
|
+
limit;
|
|
440
648
|
queue = [];
|
|
441
|
-
/**
|
|
442
|
-
|
|
443
|
-
|
|
649
|
+
/** @param limit - how many holders may run at once. */
|
|
650
|
+
constructor(limit) {
|
|
651
|
+
this.limit = limit;
|
|
652
|
+
}
|
|
653
|
+
/** Apply a new limit, starting queued waiters for any capacity it opens. */
|
|
654
|
+
resize(limit) {
|
|
655
|
+
this.limit = limit;
|
|
656
|
+
this.drain();
|
|
657
|
+
}
|
|
658
|
+
/**
|
|
659
|
+
* Take a slot, or queue until one frees. A queued fetch fails fast on the
|
|
660
|
+
* caller's abort signal or after `queueTimeoutMs` without a slot — the
|
|
661
|
+
* queue wait must not eat the whole fetch budget only to die mid-render.
|
|
662
|
+
*/
|
|
663
|
+
acquire(signal, queueTimeoutMs) {
|
|
664
|
+
if (this.active < this.limit) {
|
|
444
665
|
this.active++;
|
|
445
666
|
return Promise.resolve();
|
|
446
667
|
}
|
|
447
668
|
return new Promise((resolve, reject) => {
|
|
669
|
+
const cleanup = () => {
|
|
670
|
+
clearTimeout(timer);
|
|
671
|
+
signal.removeEventListener("abort", onAbort);
|
|
672
|
+
};
|
|
448
673
|
const waiter = {
|
|
449
674
|
start: () => {
|
|
450
|
-
|
|
675
|
+
cleanup();
|
|
451
676
|
this.active++;
|
|
452
677
|
resolve();
|
|
453
678
|
},
|
|
454
|
-
|
|
679
|
+
fail: (error) => {
|
|
455
680
|
const index = this.queue.indexOf(waiter);
|
|
456
681
|
if (index !== -1) this.queue.splice(index, 1);
|
|
457
|
-
|
|
682
|
+
cleanup();
|
|
683
|
+
reject(error);
|
|
458
684
|
}
|
|
459
685
|
};
|
|
686
|
+
const timer = setTimeout(() => {
|
|
687
|
+
waiter.fail(new WebError(`all ${String(this.limit)} rendering slots stayed busy for ${String(queueTimeoutMs)}ms; retry shortly, or raise the maxConcurrency setting`, "WEB_FETCH_TIMEOUT"));
|
|
688
|
+
}, queueTimeoutMs);
|
|
689
|
+
const onAbort = () => {
|
|
690
|
+
waiter.fail(new WebError("web fetch aborted while waiting for a free rendering slot", "WEB_ABORTED"));
|
|
691
|
+
};
|
|
460
692
|
this.queue.push(waiter);
|
|
461
|
-
signal.addEventListener("abort",
|
|
693
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
462
694
|
});
|
|
463
695
|
}
|
|
464
696
|
release() {
|
|
465
|
-
const next = this.queue.shift();
|
|
697
|
+
const next = this.active <= this.limit ? this.queue.shift() : void 0;
|
|
466
698
|
if (next === void 0) {
|
|
467
699
|
this.active = Math.max(0, this.active - 1);
|
|
468
700
|
return;
|
|
469
701
|
}
|
|
470
702
|
next.start();
|
|
471
703
|
}
|
|
704
|
+
drain() {
|
|
705
|
+
while (this.active < this.limit) {
|
|
706
|
+
const next = this.queue.shift();
|
|
707
|
+
if (next === void 0) return;
|
|
708
|
+
next.start();
|
|
709
|
+
}
|
|
710
|
+
}
|
|
472
711
|
};
|
|
473
712
|
/** Validate a request URL: http(s) only, no embedded credentials, bounded. */
|
|
474
713
|
function validateFetchUrl(input) {
|
|
@@ -493,7 +732,10 @@ function classifyContentType(contentType) {
|
|
|
493
732
|
/** Translate a thrown pipeline error into the seam's WebError taxonomy. */
|
|
494
733
|
function translateError(error, deadline) {
|
|
495
734
|
if (deadline.isTimeout) return new WebError("playwright web fetch timed out", "WEB_FETCH_TIMEOUT", { cause: error });
|
|
496
|
-
if (deadline.signal.aborted)
|
|
735
|
+
if (deadline.signal.aborted) {
|
|
736
|
+
if (error instanceof WebError) return error;
|
|
737
|
+
return new WebError("web fetch aborted", "WEB_ABORTED", { cause: error });
|
|
738
|
+
}
|
|
497
739
|
if (error instanceof WebError) return error;
|
|
498
740
|
return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
499
741
|
}
|
|
@@ -505,29 +747,44 @@ function translateError(error, deadline) {
|
|
|
505
747
|
var PlaywrightFetchProvider = class {
|
|
506
748
|
configSource;
|
|
507
749
|
id = PLAYWRIGHT_FETCH_PROVIDER_ID;
|
|
508
|
-
semaphore = new Semaphore();
|
|
750
|
+
semaphore = new Semaphore(4);
|
|
751
|
+
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
752
|
+
cdpPool;
|
|
509
753
|
/**
|
|
510
754
|
* @param configSource - thunk returning the currently authoritative config.
|
|
755
|
+
* @param cdpPool - optional pool over the CDP backend (tests inject fakes).
|
|
511
756
|
*/
|
|
512
|
-
constructor(configSource) {
|
|
757
|
+
constructor(configSource, cdpPool) {
|
|
513
758
|
this.configSource = configSource;
|
|
759
|
+
this.cdpPool = cdpPool ?? new CdpConnectionPool(defaultCdpConnect);
|
|
514
760
|
}
|
|
515
761
|
/** Cheap and side-effect free; backend problems surface per fetch instead. */
|
|
516
762
|
available() {
|
|
517
763
|
return true;
|
|
518
764
|
}
|
|
765
|
+
/**
|
|
766
|
+
* Drop the shared CDP connection (plugin teardown). Local browsers need
|
|
767
|
+
* nothing — they never outlive their fetch. In-flight CDP fetches keep
|
|
768
|
+
* their leases and close them when they finish.
|
|
769
|
+
*/
|
|
770
|
+
async dispose() {
|
|
771
|
+
await this.cdpPool.dispose();
|
|
772
|
+
}
|
|
519
773
|
async fetch(request, signal) {
|
|
520
774
|
if (signal?.aborted) throw new WebError("web fetch aborted", "WEB_ABORTED");
|
|
521
775
|
const config = this.configSource();
|
|
522
776
|
const url = validateFetchUrl(request.url);
|
|
523
777
|
const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS);
|
|
524
|
-
await this.semaphore.acquire(deadline.signal);
|
|
525
778
|
let session;
|
|
779
|
+
let acquired = false;
|
|
526
780
|
try {
|
|
781
|
+
this.semaphore.resize(effectiveMaxConcurrency(config));
|
|
782
|
+
await this.semaphore.acquire(deadline.signal, QUEUE_TIMEOUT_MS);
|
|
783
|
+
acquired = true;
|
|
527
784
|
session = await this.openSession(config, deadline);
|
|
785
|
+
const held = session;
|
|
528
786
|
const onAbort = () => {
|
|
529
|
-
|
|
530
|
-
session?.browser.close().catch(() => {});
|
|
787
|
+
closeSession(held);
|
|
531
788
|
};
|
|
532
789
|
deadline.signal.addEventListener("abort", onAbort, { once: true });
|
|
533
790
|
try {
|
|
@@ -538,13 +795,12 @@ var PlaywrightFetchProvider = class {
|
|
|
538
795
|
} catch (error) {
|
|
539
796
|
throw translateError(error, deadline);
|
|
540
797
|
} finally {
|
|
541
|
-
await session
|
|
542
|
-
|
|
543
|
-
this.semaphore.release();
|
|
798
|
+
await closeSession(session);
|
|
799
|
+
if (acquired) this.semaphore.release();
|
|
544
800
|
}
|
|
545
801
|
}
|
|
546
802
|
/**
|
|
547
|
-
* Open the configured backend and its per-fetch
|
|
803
|
+
* Open the configured backend and its per-fetch page. Split out so the
|
|
548
804
|
* test suite can substitute a fake browser.
|
|
549
805
|
* @param config - the resolved settings section.
|
|
550
806
|
* @param deadline - the fetch budget, applied to launch/connect timeouts.
|
|
@@ -554,39 +810,47 @@ var PlaywrightFetchProvider = class {
|
|
|
554
810
|
const timeout = Math.min(deadline.remainingMs(), 2e4);
|
|
555
811
|
if (config.backend === "cdp") {
|
|
556
812
|
const endpoint = normalizeCdpEndpoint(config.cdpEndpoint);
|
|
557
|
-
const {
|
|
813
|
+
const { source } = await resolveCdpBackend();
|
|
558
814
|
try {
|
|
559
|
-
const
|
|
560
|
-
|
|
561
|
-
|
|
815
|
+
const lease = await this.cdpPool.acquire(endpoint, timeout, effectiveContextMode(config));
|
|
816
|
+
await installResourceFilter(lease.page);
|
|
817
|
+
guardPopups(lease.page);
|
|
562
818
|
return {
|
|
563
|
-
browser,
|
|
564
|
-
context
|
|
819
|
+
browser: lease.browser,
|
|
820
|
+
context: lease.context,
|
|
821
|
+
page: lease.page,
|
|
822
|
+
sharedBrowser: true,
|
|
823
|
+
persistent: lease.persistent
|
|
565
824
|
};
|
|
566
825
|
} catch (error) {
|
|
567
826
|
throw new WebError(`cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
568
827
|
}
|
|
569
828
|
}
|
|
570
829
|
const { chromium, executablePath, source } = await resolvePlaywrightBackend(config.playwrightPath);
|
|
830
|
+
let browser;
|
|
571
831
|
try {
|
|
572
|
-
|
|
832
|
+
browser = await chromium.launch({
|
|
573
833
|
headless: true,
|
|
574
834
|
...executablePath !== void 0 ? { executablePath } : {},
|
|
575
835
|
timeout
|
|
576
836
|
});
|
|
577
837
|
const context = await browser.newContext();
|
|
578
|
-
await
|
|
838
|
+
const page = await context.newPage();
|
|
839
|
+
await installResourceFilter(page);
|
|
840
|
+
guardPopups(page);
|
|
579
841
|
return {
|
|
580
842
|
browser,
|
|
581
|
-
context
|
|
843
|
+
context,
|
|
844
|
+
page
|
|
582
845
|
};
|
|
583
846
|
} catch (error) {
|
|
847
|
+
await browser?.close().catch(() => {});
|
|
584
848
|
throw new WebError(`cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
585
849
|
}
|
|
586
850
|
}
|
|
587
|
-
/** Navigate, settle, and decode one URL inside an open session. */
|
|
851
|
+
/** Navigate, settle, and decode one URL inside an open session's tab. */
|
|
588
852
|
async retrieve(session, url, config, deadline) {
|
|
589
|
-
const page =
|
|
853
|
+
const page = session.page;
|
|
590
854
|
const response = await page.goto(url.toString(), {
|
|
591
855
|
waitUntil: "domcontentloaded",
|
|
592
856
|
timeout: deadline.remainingMs()
|
|
@@ -633,19 +897,70 @@ function capResult(url, statusCode, body) {
|
|
|
633
897
|
};
|
|
634
898
|
}
|
|
635
899
|
/**
|
|
900
|
+
* Close a session's fetch-owned tab, then its context unless the context is
|
|
901
|
+
* the remote default context (profile mode — closing it would tear down the
|
|
902
|
+
* whole shared connection), then its browser when the session launched one
|
|
903
|
+
* (local backend). CDP sessions ride the shared connection, which stays
|
|
904
|
+
* open. Every close is grace-bounded so cleanup always completes and the
|
|
905
|
+
* concurrency slot is released even when Playwright hangs.
|
|
906
|
+
*/
|
|
907
|
+
async function closeSession(session) {
|
|
908
|
+
if (session === void 0) return;
|
|
909
|
+
await closeWithGrace(session.page);
|
|
910
|
+
if (session.persistent !== true) await closeWithGrace(session.context);
|
|
911
|
+
if (session.sharedBrowser !== true) await closeWithGrace(session.browser);
|
|
912
|
+
}
|
|
913
|
+
/** One best-effort `close()` that resolves within the grace period. */
|
|
914
|
+
async function closeWithGrace(closeable) {
|
|
915
|
+
await new Promise((resolve) => {
|
|
916
|
+
const timer = setTimeout(resolve, CLOSE_GRACE_MS);
|
|
917
|
+
closeable.close().then(() => {
|
|
918
|
+
clearTimeout(timer);
|
|
919
|
+
resolve();
|
|
920
|
+
}, () => {
|
|
921
|
+
clearTimeout(timer);
|
|
922
|
+
resolve();
|
|
923
|
+
});
|
|
924
|
+
});
|
|
925
|
+
}
|
|
926
|
+
/**
|
|
927
|
+
* The real CDP connect the pool runs with: the bundled playwright-core's
|
|
928
|
+
* `connectOverCDP`. Kept as a function (not inline) so the pool constructor
|
|
929
|
+
* stays injectable for tests.
|
|
930
|
+
*/
|
|
931
|
+
async function defaultCdpConnect(endpoint, timeoutMs) {
|
|
932
|
+
const { chromium } = await resolveCdpBackend();
|
|
933
|
+
return await chromium.connectOverCDP(endpoint, { timeout: timeoutMs });
|
|
934
|
+
}
|
|
935
|
+
/**
|
|
636
936
|
* Abort image/font/media subrequests: the markdown output keeps their URLs
|
|
637
937
|
* but never renders them, so downloading them only spends the budget.
|
|
638
|
-
*
|
|
938
|
+
* Installed on the PAGE (never the context): in profile mode a context-level
|
|
939
|
+
* route would intercept the remote browser's other tabs too. Best-effort —
|
|
940
|
+
* a page that refuses interception still fetches.
|
|
639
941
|
*/
|
|
640
|
-
async function installResourceFilter(
|
|
942
|
+
async function installResourceFilter(owner) {
|
|
641
943
|
try {
|
|
642
|
-
await
|
|
944
|
+
await owner.route("**/*", async (route) => {
|
|
643
945
|
const type = route.request().resourceType();
|
|
644
946
|
if (type === "image" || type === "font" || type === "media") await route.abort();
|
|
645
947
|
else await route.continue();
|
|
646
948
|
});
|
|
647
949
|
} catch {}
|
|
648
950
|
}
|
|
951
|
+
/**
|
|
952
|
+
* Close any popup a fetched page spawns so nothing outlives the fetch's tab
|
|
953
|
+
* — `page.close()` does not auto-close windows the page opened, and in
|
|
954
|
+
* profile mode a stray tab would stay in the user's remote browser.
|
|
955
|
+
* Best-effort: a page that refuses listeners just loses the guard.
|
|
956
|
+
*/
|
|
957
|
+
function guardPopups(page) {
|
|
958
|
+
try {
|
|
959
|
+
page.on?.("popup", (popup) => {
|
|
960
|
+
popup.close().catch(() => {});
|
|
961
|
+
});
|
|
962
|
+
} catch {}
|
|
963
|
+
}
|
|
649
964
|
//#endregion
|
|
650
965
|
//#region src/index.ts
|
|
651
966
|
/** Cordis plugin name used by loader diagnostics. */
|
|
@@ -663,7 +978,11 @@ function apply(ctx, config) {
|
|
|
663
978
|
},
|
|
664
979
|
onChange: () => {}
|
|
665
980
|
});
|
|
666
|
-
|
|
981
|
+
const provider = new PlaywrightFetchProvider(() => current());
|
|
982
|
+
ctx.effect(() => () => {
|
|
983
|
+
provider.dispose();
|
|
984
|
+
}, "dsh-web-fetch-playwright: shared CDP connection");
|
|
985
|
+
ctx.web.registerFetchProvider(provider);
|
|
667
986
|
}
|
|
668
987
|
//#endregion
|
|
669
|
-
export { Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
|
|
988
|
+
export { CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, effectiveContextMode, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The CDP backend's shared connection: one `connectOverCDP` session reused by
|
|
3
|
+
* every fetch, each fetch leasing a page (a tab in the remote browser) that
|
|
4
|
+
* it closes when done — in an isolated throwaway context (`isolated` mode),
|
|
5
|
+
* or in the remote browser's default context (`profile` mode, whose cookies
|
|
6
|
+
* and localStorage are the real profile's, so its persistent logins apply;
|
|
7
|
+
* that context is never closed, only the tab is). The connection itself
|
|
8
|
+
* outlives fetches — reconnecting per fetch would spend 100–500ms per call
|
|
9
|
+
* and open one socket per concurrent tab for no benefit.
|
|
10
|
+
*
|
|
11
|
+
* Liveness is handled three ways: `isConnected()` is checked before reuse, a
|
|
12
|
+
* `disconnected` event (when the backend emits it) drops the reference
|
|
13
|
+
* immediately, and an open-lease failure against a dead connection triggers
|
|
14
|
+
* exactly one reconnect-and-retry before the error surfaces. A changed
|
|
15
|
+
* endpoint (settings edit) replaces the connection.
|
|
16
|
+
*
|
|
17
|
+
* @module dsh-web-fetch-playwright/cdp-pool
|
|
18
|
+
*/
|
|
19
|
+
import type { PlaywrightBrowser, PlaywrightContext, PlaywrightPage } from './types.ts';
|
|
20
|
+
/** Opens the shared connection; injected so tests can substitute a fake. */
|
|
21
|
+
export type CdpConnect = (endpoint: string, timeoutMs: number) => Promise<PlaywrightBrowser>;
|
|
22
|
+
/** How a lease scopes its fetch: throwaway context or the remote profile. */
|
|
23
|
+
export type CdpAcquireMode = 'isolated' | 'profile';
|
|
24
|
+
/** One fetch's lease: the shared browser, a context, and the tab it owns. */
|
|
25
|
+
export interface CdpLease {
|
|
26
|
+
/** The shared connection — close only what the lease owns, never this. */
|
|
27
|
+
browser: PlaywrightBrowser;
|
|
28
|
+
/**
|
|
29
|
+
* The context the page lives in: fetch-owned (`isolated`) or the remote
|
|
30
|
+
* browser's default context (`profile`). NEVER close the latter — closing
|
|
31
|
+
* it tears down the whole shared connection (playwright-core maps a
|
|
32
|
+
* default-context close to "close browser" for this connection).
|
|
33
|
+
*/
|
|
34
|
+
context: PlaywrightContext;
|
|
35
|
+
/** The fetch-owned tab; {@link CdpConnectionPool.release} always closes it. */
|
|
36
|
+
page: PlaywrightPage;
|
|
37
|
+
/** True when `context` is the remote default context: release must not close it. */
|
|
38
|
+
persistent: boolean;
|
|
39
|
+
}
|
|
40
|
+
/** A reusable `connectOverCDP` session handing out per-fetch pages. */
|
|
41
|
+
export declare class CdpConnectionPool {
|
|
42
|
+
private readonly connect;
|
|
43
|
+
private browser;
|
|
44
|
+
private endpoint;
|
|
45
|
+
private connecting;
|
|
46
|
+
private connectingEndpoint;
|
|
47
|
+
/** Bumped by dispose/replace so a settling connect knows it was abandoned. */
|
|
48
|
+
private generation;
|
|
49
|
+
/**
|
|
50
|
+
* @param connect - opens a connection to an endpoint (the provider's real
|
|
51
|
+
* one resolves the bundled playwright-core; tests inject fakes).
|
|
52
|
+
*/
|
|
53
|
+
constructor(connect: CdpConnect);
|
|
54
|
+
/**
|
|
55
|
+
* Lease a fetch's page on the shared connection, connecting (or
|
|
56
|
+
* reconnecting) first if needed. Concurrent first fetches share one
|
|
57
|
+
* connect attempt.
|
|
58
|
+
*
|
|
59
|
+
* @param endpoint - normalized CDP endpoint URL.
|
|
60
|
+
* @param timeoutMs - connect timeout budget.
|
|
61
|
+
* @param mode - `isolated` (default): a fresh throwaway context plus a page
|
|
62
|
+
* in it; `profile`: a page in the remote browser's default context — the
|
|
63
|
+
* real profile — which is never closed, only the page is.
|
|
64
|
+
* @returns the shared browser, the page's context, and the page the caller
|
|
65
|
+
* must release.
|
|
66
|
+
* @throws whatever the connect function throws when no connection can be
|
|
67
|
+
* established (the provider wraps it in a structured WebError), or a
|
|
68
|
+
* diagnostic error when `profile` mode finds no default context.
|
|
69
|
+
*/
|
|
70
|
+
acquire(endpoint: string, timeoutMs: number, mode?: CdpAcquireMode): Promise<CdpLease>;
|
|
71
|
+
/**
|
|
72
|
+
* Open one lease's page on a live connection. The profile-mode handle is
|
|
73
|
+
* taken fresh every call (never cached across connections): a reconnect
|
|
74
|
+
* yields a new Browser object whose `contexts()[0]` must be re-read.
|
|
75
|
+
*/
|
|
76
|
+
private openLease;
|
|
77
|
+
/**
|
|
78
|
+
* Close a fetch-owned page, plus its context when the lease owns one. The
|
|
79
|
+
* remote default context (persistent leases) and the shared connection
|
|
80
|
+
* stay for the next fetch.
|
|
81
|
+
* @param lease - the lease whose page (and, when isolated, context) goes away.
|
|
82
|
+
*/
|
|
83
|
+
release(lease: CdpLease): Promise<void>;
|
|
84
|
+
/**
|
|
85
|
+
* Drop the shared connection (plugin teardown or endpoint change). Fetches
|
|
86
|
+
* holding leases keep their contexts until they release them. A connect
|
|
87
|
+
* still in flight is abandoned — its own continuation closes the stray
|
|
88
|
+
* browser (nothing here waits on it, so teardown cannot deadlock).
|
|
89
|
+
*/
|
|
90
|
+
dispose(): Promise<void>;
|
|
91
|
+
/** The shared connection for `endpoint`, connecting or reconnecting as needed. */
|
|
92
|
+
private ensure;
|
|
93
|
+
/** Start a connection to `endpoint`, superseding whatever was there. */
|
|
94
|
+
private connectFresh;
|
|
95
|
+
/** Clear the reference when this exact connection reports it went away. */
|
|
96
|
+
private watch;
|
|
97
|
+
/** Forget a connection known to be dead (absent probe = assume dead here). */
|
|
98
|
+
private drop;
|
|
99
|
+
private isLive;
|
|
100
|
+
}
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The Playwright plugin-configuration card: backend radio group (each option
|
|
3
|
-
* carrying its backend-specific
|
|
4
|
-
* Playwright, CDP endpoint under Remote CDP) plus
|
|
5
|
-
* staged and saved through the card form like the
|
|
3
|
+
* carrying its backend-specific inputs nested inside — local path under Local
|
|
4
|
+
* Playwright, CDP endpoint and shared-context checkbox under Remote CDP) plus
|
|
5
|
+
* the denoise checkbox, staged and saved through the card form like the
|
|
6
|
+
* built-in plugin cards.
|
|
6
7
|
*
|
|
7
8
|
* @module dsh-web-fetch-playwright/client/card
|
|
8
9
|
*/
|