dsh-web-fetch-playwright 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/README.zh-CN.md +7 -4
- package/dsh.plugin.json +1 -1
- package/lib/client.js +152 -99
- package/lib/index.js +278 -33
- package/lib/types/cdp-pool.d.ts +74 -0
- package/lib/types/client/controller.d.ts +4 -0
- package/lib/types/client/form.d.ts +10 -0
- package/lib/types/client/locales.d.ts +1 -1
- package/lib/types/config.d.ts +38 -2
- package/lib/types/index.d.ts +2 -0
- package/lib/types/provider.d.ts +28 -6
- package/lib/types/types.d.ts +4 -0
- package/package.json +9 -11
- package/src/cdp-pool.ts +159 -0
- package/src/client/card.tsx +13 -0
- package/src/client/controller.ts +7 -1
- package/src/client/form.ts +23 -0
- package/src/client/locales.ts +7 -0
- package/src/config.ts +46 -2
- package/src/index.ts +7 -1
- package/src/provider.ts +163 -35
- package/src/types.ts +4 -0
package/lib/index.js
CHANGED
|
@@ -25,9 +25,22 @@ const Config = z.object({
|
|
|
25
25
|
backend: z.union([z.const("local"), z.const("cdp")]).default("local"),
|
|
26
26
|
playwrightPath: z.string().default(""),
|
|
27
27
|
cdpEndpoint: z.string().default(""),
|
|
28
|
-
denoise: z.boolean().default(true)
|
|
28
|
+
denoise: z.boolean().default(true),
|
|
29
|
+
maxConcurrency: z.number().step(1).min(1).max(200)
|
|
29
30
|
});
|
|
30
31
|
/**
|
|
32
|
+
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
33
|
+
* wins; otherwise the backend default (local launches browsers, CDP only
|
|
34
|
+
* opens tabs, so their budgets differ by an order of magnitude).
|
|
35
|
+
*
|
|
36
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
37
|
+
* @returns the effective limit for the semaphore.
|
|
38
|
+
*/
|
|
39
|
+
function effectiveMaxConcurrency(config) {
|
|
40
|
+
if (typeof config.maxConcurrency === "number") return config.maxConcurrency;
|
|
41
|
+
return config.backend === "cdp" ? 50 : 4;
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
31
44
|
* Normalize a configured CDP endpoint for `chromium.connectOverCDP`:
|
|
32
45
|
* blank becomes the default loopback endpoint; `wss?://` and `https?://`
|
|
33
46
|
* pass through; a bare `host:port` gains the `http://` scheme Playwright's
|
|
@@ -45,6 +58,130 @@ function normalizeCdpEndpoint(input) {
|
|
|
45
58
|
return candidate;
|
|
46
59
|
}
|
|
47
60
|
//#endregion
|
|
61
|
+
//#region src/cdp-pool.ts
|
|
62
|
+
/** A reusable `connectOverCDP` session handing out per-fetch contexts. */
|
|
63
|
+
var CdpConnectionPool = class {
|
|
64
|
+
connect;
|
|
65
|
+
browser;
|
|
66
|
+
endpoint = "";
|
|
67
|
+
connecting;
|
|
68
|
+
connectingEndpoint = "";
|
|
69
|
+
/** Bumped by dispose/replace so a settling connect knows it was abandoned. */
|
|
70
|
+
generation = 0;
|
|
71
|
+
/**
|
|
72
|
+
* @param connect - opens a connection to an endpoint (the provider's real
|
|
73
|
+
* one resolves the bundled playwright-core; tests inject fakes).
|
|
74
|
+
*/
|
|
75
|
+
constructor(connect) {
|
|
76
|
+
this.connect = connect;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Lease a fresh isolated context on the shared connection, connecting (or
|
|
80
|
+
* reconnecting) first if needed. Concurrent first fetches share one
|
|
81
|
+
* connect attempt.
|
|
82
|
+
*
|
|
83
|
+
* @param endpoint - normalized CDP endpoint URL.
|
|
84
|
+
* @param timeoutMs - connect timeout budget.
|
|
85
|
+
* @returns the shared browser and a context the caller must release.
|
|
86
|
+
* @throws whatever the connect function throws when no connection can be
|
|
87
|
+
* established (the provider wraps it in a structured WebError).
|
|
88
|
+
*/
|
|
89
|
+
async acquire(endpoint, timeoutMs) {
|
|
90
|
+
const browser = await this.ensure(endpoint, timeoutMs);
|
|
91
|
+
try {
|
|
92
|
+
return {
|
|
93
|
+
browser,
|
|
94
|
+
context: await browser.newContext()
|
|
95
|
+
};
|
|
96
|
+
} catch (error) {
|
|
97
|
+
if (this.isLive(browser)) throw error;
|
|
98
|
+
this.drop(browser);
|
|
99
|
+
const fresh = await this.ensure(endpoint, timeoutMs);
|
|
100
|
+
return {
|
|
101
|
+
browser: fresh,
|
|
102
|
+
context: await fresh.newContext()
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Close a fetch-owned context. The shared connection stays open for the
|
|
108
|
+
* next fetch.
|
|
109
|
+
* @param lease - the lease whose context goes away.
|
|
110
|
+
*/
|
|
111
|
+
async release(lease) {
|
|
112
|
+
await lease.context.close().catch(() => {});
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Drop the shared connection (plugin teardown or endpoint change). Fetches
|
|
116
|
+
* holding leases keep their contexts until they release them. A connect
|
|
117
|
+
* still in flight is abandoned — its own continuation closes the stray
|
|
118
|
+
* browser (nothing here waits on it, so teardown cannot deadlock).
|
|
119
|
+
*/
|
|
120
|
+
async dispose() {
|
|
121
|
+
const browser = this.browser;
|
|
122
|
+
this.browser = void 0;
|
|
123
|
+
this.endpoint = "";
|
|
124
|
+
this.connecting = void 0;
|
|
125
|
+
this.connectingEndpoint = "";
|
|
126
|
+
this.generation++;
|
|
127
|
+
await browser?.close().catch(() => {});
|
|
128
|
+
}
|
|
129
|
+
/** The shared connection for `endpoint`, connecting or reconnecting as needed. */
|
|
130
|
+
async ensure(endpoint, timeoutMs) {
|
|
131
|
+
if (this.browser !== void 0 && this.endpoint === endpoint && this.isLive(this.browser)) return this.browser;
|
|
132
|
+
if (this.connecting !== void 0 && this.connectingEndpoint === endpoint) return this.connecting;
|
|
133
|
+
return await this.connectFresh(endpoint, timeoutMs);
|
|
134
|
+
}
|
|
135
|
+
/** Start a connection to `endpoint`, superseding whatever was there. */
|
|
136
|
+
connectFresh(endpoint, timeoutMs) {
|
|
137
|
+
const stale = this.browser;
|
|
138
|
+
this.browser = void 0;
|
|
139
|
+
this.endpoint = "";
|
|
140
|
+
const generation = ++this.generation;
|
|
141
|
+
const attempt = (async () => {
|
|
142
|
+
const browser = await this.connect(endpoint, timeoutMs);
|
|
143
|
+
if (generation !== this.generation) {
|
|
144
|
+
await browser.close().catch(() => {});
|
|
145
|
+
throw new Error("cdp connection abandoned before it attached");
|
|
146
|
+
}
|
|
147
|
+
this.browser = browser;
|
|
148
|
+
this.endpoint = endpoint;
|
|
149
|
+
this.watch(browser);
|
|
150
|
+
return browser;
|
|
151
|
+
})();
|
|
152
|
+
this.connecting = attempt;
|
|
153
|
+
this.connectingEndpoint = endpoint;
|
|
154
|
+
attempt.then(() => {
|
|
155
|
+
if (this.connecting === attempt) this.connecting = void 0;
|
|
156
|
+
}, () => {
|
|
157
|
+
if (this.connecting === attempt) this.connecting = void 0;
|
|
158
|
+
});
|
|
159
|
+
stale?.close().catch(() => {});
|
|
160
|
+
return attempt;
|
|
161
|
+
}
|
|
162
|
+
/** Clear the reference when this exact connection reports it went away. */
|
|
163
|
+
watch(browser) {
|
|
164
|
+
try {
|
|
165
|
+
browser.on?.("disconnected", () => {
|
|
166
|
+
if (this.browser === browser) {
|
|
167
|
+
this.browser = void 0;
|
|
168
|
+
this.endpoint = "";
|
|
169
|
+
}
|
|
170
|
+
});
|
|
171
|
+
} catch {}
|
|
172
|
+
}
|
|
173
|
+
/** Forget a connection known to be dead (absent probe = assume dead here). */
|
|
174
|
+
drop(browser) {
|
|
175
|
+
if (this.browser === browser) {
|
|
176
|
+
this.browser = void 0;
|
|
177
|
+
this.endpoint = "";
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
isLive(browser) {
|
|
181
|
+
return browser.isConnected?.() !== false;
|
|
182
|
+
}
|
|
183
|
+
};
|
|
184
|
+
//#endregion
|
|
48
185
|
//#region src/markdown.ts
|
|
49
186
|
/**
|
|
50
187
|
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
@@ -373,10 +510,16 @@ function existsAsFile(path) {
|
|
|
373
510
|
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
374
511
|
* classification) so the tool layer sees the same codes from either backend.
|
|
375
512
|
*
|
|
376
|
-
* Lifecycle
|
|
377
|
-
*
|
|
378
|
-
*
|
|
379
|
-
*
|
|
513
|
+
* Lifecycle: the local backend launches a browser per fetch and closes it —
|
|
514
|
+
* nothing outlives the call. The CDP backend keeps ONE shared connection to
|
|
515
|
+
* the remote browser for the provider's lifetime; each fetch only opens an
|
|
516
|
+
* isolated context (a tab) inside it and closes that on completion, so the
|
|
517
|
+
* concurrency cap counts tabs, not browsers. Either way, an aborted signal
|
|
518
|
+
* closes the fetch's context. Concurrency is capped at the `maxConcurrency`
|
|
519
|
+
* setting (explicit, or the backend default — {@link DEFAULT_MAX_CONCURRENCY_LOCAL}
|
|
520
|
+
* browsers / {@link DEFAULT_MAX_CONCURRENCY_CDP} tabs); further fetches wait
|
|
521
|
+
* briefly in a queue and fail fast (rather than hang until abort) when no
|
|
522
|
+
* slot frees.
|
|
380
523
|
*
|
|
381
524
|
* Private-network and SSRF protection is not implemented (same stance as the
|
|
382
525
|
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
@@ -392,13 +535,24 @@ const MAX_URL_LENGTH = 2048;
|
|
|
392
535
|
const MAX_BODY_CHARS = 1e5;
|
|
393
536
|
/** Cap on rendered HTML fed into the synchronous denoise pipeline. */
|
|
394
537
|
const MAX_PIPELINE_INPUT_CHARS = 2e6;
|
|
395
|
-
/**
|
|
396
|
-
|
|
538
|
+
/**
|
|
539
|
+
* How long a fetch may sit in the concurrency queue before failing fast.
|
|
540
|
+
* Waiting longer cannot help — the per-fetch deadline leaves too little
|
|
541
|
+
* budget to render after dequeue — and failing fast tells the caller to
|
|
542
|
+
* retry or raise `maxConcurrency` instead of hanging until an abort.
|
|
543
|
+
*/
|
|
544
|
+
const QUEUE_TIMEOUT_MS = 2e4;
|
|
397
545
|
/** Default per-fetch budget (ms), inside the tool layer's 60s. */
|
|
398
546
|
const DEFAULT_TIMEOUT_MS = 45e3;
|
|
399
547
|
/** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
|
|
400
548
|
const SETTLE_MS = 5e3;
|
|
401
549
|
/**
|
|
550
|
+
* Grace period (ms) for context/browser closes in the cleanup path. A wedged
|
|
551
|
+
* `close()` must never hold a concurrency slot hostage — a leaked slot would
|
|
552
|
+
* fail every later fetch with the queue error until restart.
|
|
553
|
+
*/
|
|
554
|
+
const CLOSE_GRACE_MS = 2e3;
|
|
555
|
+
/**
|
|
402
556
|
* A single abort deadline composing the caller's signal with this provider's
|
|
403
557
|
* time budget. The timeout reason is kept apart from an outer abort so the
|
|
404
558
|
* error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
|
|
@@ -434,41 +588,78 @@ var Deadline = class {
|
|
|
434
588
|
return Math.max(1, this.expiresAt - Date.now());
|
|
435
589
|
}
|
|
436
590
|
};
|
|
437
|
-
/**
|
|
591
|
+
/**
|
|
592
|
+
* A bounded async semaphore with abort support. The limit is live-resizable
|
|
593
|
+
* (`resize`) because the config thunk re-reads on every fetch — raising it
|
|
594
|
+
* wakes queued waiters immediately; lowering it lets in-flight holders run
|
|
595
|
+
* out naturally.
|
|
596
|
+
*/
|
|
438
597
|
var Semaphore = class {
|
|
439
598
|
active = 0;
|
|
599
|
+
limit;
|
|
440
600
|
queue = [];
|
|
441
|
-
/**
|
|
442
|
-
|
|
443
|
-
|
|
601
|
+
/** @param limit - how many holders may run at once. */
|
|
602
|
+
constructor(limit) {
|
|
603
|
+
this.limit = limit;
|
|
604
|
+
}
|
|
605
|
+
/** Apply a new limit, starting queued waiters for any capacity it opens. */
|
|
606
|
+
resize(limit) {
|
|
607
|
+
this.limit = limit;
|
|
608
|
+
this.drain();
|
|
609
|
+
}
|
|
610
|
+
/**
|
|
611
|
+
* Take a slot, or queue until one frees. A queued fetch fails fast on the
|
|
612
|
+
* caller's abort signal or after `queueTimeoutMs` without a slot — the
|
|
613
|
+
* queue wait must not eat the whole fetch budget only to die mid-render.
|
|
614
|
+
*/
|
|
615
|
+
acquire(signal, queueTimeoutMs) {
|
|
616
|
+
if (this.active < this.limit) {
|
|
444
617
|
this.active++;
|
|
445
618
|
return Promise.resolve();
|
|
446
619
|
}
|
|
447
620
|
return new Promise((resolve, reject) => {
|
|
621
|
+
const cleanup = () => {
|
|
622
|
+
clearTimeout(timer);
|
|
623
|
+
signal.removeEventListener("abort", onAbort);
|
|
624
|
+
};
|
|
448
625
|
const waiter = {
|
|
449
626
|
start: () => {
|
|
450
|
-
|
|
627
|
+
cleanup();
|
|
451
628
|
this.active++;
|
|
452
629
|
resolve();
|
|
453
630
|
},
|
|
454
|
-
|
|
631
|
+
fail: (error) => {
|
|
455
632
|
const index = this.queue.indexOf(waiter);
|
|
456
633
|
if (index !== -1) this.queue.splice(index, 1);
|
|
457
|
-
|
|
634
|
+
cleanup();
|
|
635
|
+
reject(error);
|
|
458
636
|
}
|
|
459
637
|
};
|
|
638
|
+
const timer = setTimeout(() => {
|
|
639
|
+
waiter.fail(new WebError(`all ${String(this.limit)} rendering slots stayed busy for ${String(queueTimeoutMs)}ms; retry shortly, or raise the maxConcurrency setting`, "WEB_FETCH_TIMEOUT"));
|
|
640
|
+
}, queueTimeoutMs);
|
|
641
|
+
const onAbort = () => {
|
|
642
|
+
waiter.fail(new WebError("web fetch aborted while waiting for a free rendering slot", "WEB_ABORTED"));
|
|
643
|
+
};
|
|
460
644
|
this.queue.push(waiter);
|
|
461
|
-
signal.addEventListener("abort",
|
|
645
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
462
646
|
});
|
|
463
647
|
}
|
|
464
648
|
release() {
|
|
465
|
-
const next = this.queue.shift();
|
|
649
|
+
const next = this.active <= this.limit ? this.queue.shift() : void 0;
|
|
466
650
|
if (next === void 0) {
|
|
467
651
|
this.active = Math.max(0, this.active - 1);
|
|
468
652
|
return;
|
|
469
653
|
}
|
|
470
654
|
next.start();
|
|
471
655
|
}
|
|
656
|
+
drain() {
|
|
657
|
+
while (this.active < this.limit) {
|
|
658
|
+
const next = this.queue.shift();
|
|
659
|
+
if (next === void 0) return;
|
|
660
|
+
next.start();
|
|
661
|
+
}
|
|
662
|
+
}
|
|
472
663
|
};
|
|
473
664
|
/** Validate a request URL: http(s) only, no embedded credentials, bounded. */
|
|
474
665
|
function validateFetchUrl(input) {
|
|
@@ -493,7 +684,10 @@ function classifyContentType(contentType) {
|
|
|
493
684
|
/** Translate a thrown pipeline error into the seam's WebError taxonomy. */
|
|
494
685
|
function translateError(error, deadline) {
|
|
495
686
|
if (deadline.isTimeout) return new WebError("playwright web fetch timed out", "WEB_FETCH_TIMEOUT", { cause: error });
|
|
496
|
-
if (deadline.signal.aborted)
|
|
687
|
+
if (deadline.signal.aborted) {
|
|
688
|
+
if (error instanceof WebError) return error;
|
|
689
|
+
return new WebError("web fetch aborted", "WEB_ABORTED", { cause: error });
|
|
690
|
+
}
|
|
497
691
|
if (error instanceof WebError) return error;
|
|
498
692
|
return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
499
693
|
}
|
|
@@ -505,29 +699,44 @@ function translateError(error, deadline) {
|
|
|
505
699
|
var PlaywrightFetchProvider = class {
|
|
506
700
|
configSource;
|
|
507
701
|
id = PLAYWRIGHT_FETCH_PROVIDER_ID;
|
|
508
|
-
semaphore = new Semaphore();
|
|
702
|
+
semaphore = new Semaphore(4);
|
|
703
|
+
/** Shared CDP connection; injectable so the suite can substitute a fake. */
|
|
704
|
+
cdpPool;
|
|
509
705
|
/**
|
|
510
706
|
* @param configSource - thunk returning the currently authoritative config.
|
|
707
|
+
* @param cdpPool - optional pool over the CDP backend (tests inject fakes).
|
|
511
708
|
*/
|
|
512
|
-
constructor(configSource) {
|
|
709
|
+
constructor(configSource, cdpPool) {
|
|
513
710
|
this.configSource = configSource;
|
|
711
|
+
this.cdpPool = cdpPool ?? new CdpConnectionPool(defaultCdpConnect);
|
|
514
712
|
}
|
|
515
713
|
/** Cheap and side-effect free; backend problems surface per fetch instead. */
|
|
516
714
|
available() {
|
|
517
715
|
return true;
|
|
518
716
|
}
|
|
717
|
+
/**
|
|
718
|
+
* Drop the shared CDP connection (plugin teardown). Local browsers need
|
|
719
|
+
* nothing — they never outlive their fetch. In-flight CDP fetches keep
|
|
720
|
+
* their leases and close them when they finish.
|
|
721
|
+
*/
|
|
722
|
+
async dispose() {
|
|
723
|
+
await this.cdpPool.dispose();
|
|
724
|
+
}
|
|
519
725
|
async fetch(request, signal) {
|
|
520
726
|
if (signal?.aborted) throw new WebError("web fetch aborted", "WEB_ABORTED");
|
|
521
727
|
const config = this.configSource();
|
|
522
728
|
const url = validateFetchUrl(request.url);
|
|
523
729
|
const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS);
|
|
524
|
-
await this.semaphore.acquire(deadline.signal);
|
|
525
730
|
let session;
|
|
731
|
+
let acquired = false;
|
|
526
732
|
try {
|
|
733
|
+
this.semaphore.resize(effectiveMaxConcurrency(config));
|
|
734
|
+
await this.semaphore.acquire(deadline.signal, QUEUE_TIMEOUT_MS);
|
|
735
|
+
acquired = true;
|
|
527
736
|
session = await this.openSession(config, deadline);
|
|
737
|
+
const held = session;
|
|
528
738
|
const onAbort = () => {
|
|
529
|
-
|
|
530
|
-
session?.browser.close().catch(() => {});
|
|
739
|
+
closeSession(held);
|
|
531
740
|
};
|
|
532
741
|
deadline.signal.addEventListener("abort", onAbort, { once: true });
|
|
533
742
|
try {
|
|
@@ -538,9 +747,8 @@ var PlaywrightFetchProvider = class {
|
|
|
538
747
|
} catch (error) {
|
|
539
748
|
throw translateError(error, deadline);
|
|
540
749
|
} finally {
|
|
541
|
-
await session
|
|
542
|
-
|
|
543
|
-
this.semaphore.release();
|
|
750
|
+
await closeSession(session);
|
|
751
|
+
if (acquired) this.semaphore.release();
|
|
544
752
|
}
|
|
545
753
|
}
|
|
546
754
|
/**
|
|
@@ -554,14 +762,14 @@ var PlaywrightFetchProvider = class {
|
|
|
554
762
|
const timeout = Math.min(deadline.remainingMs(), 2e4);
|
|
555
763
|
if (config.backend === "cdp") {
|
|
556
764
|
const endpoint = normalizeCdpEndpoint(config.cdpEndpoint);
|
|
557
|
-
const {
|
|
765
|
+
const { source } = await resolveCdpBackend();
|
|
558
766
|
try {
|
|
559
|
-
const
|
|
560
|
-
|
|
561
|
-
await installResourceFilter(context);
|
|
767
|
+
const lease = await this.cdpPool.acquire(endpoint, timeout);
|
|
768
|
+
await installResourceFilter(lease.context);
|
|
562
769
|
return {
|
|
563
|
-
browser,
|
|
564
|
-
context
|
|
770
|
+
browser: lease.browser,
|
|
771
|
+
context: lease.context,
|
|
772
|
+
sharedBrowser: true
|
|
565
773
|
};
|
|
566
774
|
} catch (error) {
|
|
567
775
|
throw new WebError(`cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
@@ -633,6 +841,39 @@ function capResult(url, statusCode, body) {
|
|
|
633
841
|
};
|
|
634
842
|
}
|
|
635
843
|
/**
|
|
844
|
+
* Close a session's fetch-owned context, plus its browser when the session
|
|
845
|
+
* launched one (local backend). CDP sessions ride the shared connection,
|
|
846
|
+
* which stays open. Every close is grace-bounded so cleanup always completes
|
|
847
|
+
* and the concurrency slot is released even when Playwright hangs.
|
|
848
|
+
*/
|
|
849
|
+
async function closeSession(session) {
|
|
850
|
+
if (session === void 0) return;
|
|
851
|
+
await closeWithGrace(session.context);
|
|
852
|
+
if (session.sharedBrowser !== true) await closeWithGrace(session.browser);
|
|
853
|
+
}
|
|
854
|
+
/** One best-effort `close()` that resolves within the grace period. */
|
|
855
|
+
async function closeWithGrace(closeable) {
|
|
856
|
+
await new Promise((resolve) => {
|
|
857
|
+
const timer = setTimeout(resolve, CLOSE_GRACE_MS);
|
|
858
|
+
closeable.close().then(() => {
|
|
859
|
+
clearTimeout(timer);
|
|
860
|
+
resolve();
|
|
861
|
+
}, () => {
|
|
862
|
+
clearTimeout(timer);
|
|
863
|
+
resolve();
|
|
864
|
+
});
|
|
865
|
+
});
|
|
866
|
+
}
|
|
867
|
+
/**
|
|
868
|
+
* The real CDP connect the pool runs with: the bundled playwright-core's
|
|
869
|
+
* `connectOverCDP`. Kept as a function (not inline) so the pool constructor
|
|
870
|
+
* stays injectable for tests.
|
|
871
|
+
*/
|
|
872
|
+
async function defaultCdpConnect(endpoint, timeoutMs) {
|
|
873
|
+
const { chromium } = await resolveCdpBackend();
|
|
874
|
+
return await chromium.connectOverCDP(endpoint, { timeout: timeoutMs });
|
|
875
|
+
}
|
|
876
|
+
/**
|
|
636
877
|
* Abort image/font/media subrequests: the markdown output keeps their URLs
|
|
637
878
|
* but never renders them, so downloading them only spends the budget.
|
|
638
879
|
* Best-effort — a context that refuses interception still fetches.
|
|
@@ -663,7 +904,11 @@ function apply(ctx, config) {
|
|
|
663
904
|
},
|
|
664
905
|
onChange: () => {}
|
|
665
906
|
});
|
|
666
|
-
|
|
907
|
+
const provider = new PlaywrightFetchProvider(() => current());
|
|
908
|
+
ctx.effect(() => () => {
|
|
909
|
+
provider.dispose();
|
|
910
|
+
}, "dsh-web-fetch-playwright: shared CDP connection");
|
|
911
|
+
ctx.web.registerFetchProvider(provider);
|
|
667
912
|
}
|
|
668
913
|
//#endregion
|
|
669
|
-
export { Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
|
|
914
|
+
export { CdpConnectionPool, Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The CDP backend's shared connection: one `connectOverCDP` session reused by
|
|
3
|
+
* every fetch, each fetch leasing an isolated context (a tab in the remote
|
|
4
|
+
* browser) that it closes when done. The connection itself outlives fetches —
|
|
5
|
+
* reconnecting per fetch would spend 100–500ms per call and open one socket
|
|
6
|
+
* per concurrent tab for no benefit.
|
|
7
|
+
*
|
|
8
|
+
* Liveness is handled three ways: `isConnected()` is checked before reuse, a
|
|
9
|
+
* `disconnected` event (when the backend emits it) drops the reference
|
|
10
|
+
* immediately, and a `newContext` failure against a dead connection triggers
|
|
11
|
+
* exactly one reconnect-and-retry before the error surfaces. A changed
|
|
12
|
+
* endpoint (settings edit) replaces the connection.
|
|
13
|
+
*
|
|
14
|
+
* @module dsh-web-fetch-playwright/cdp-pool
|
|
15
|
+
*/
|
|
16
|
+
import type { PlaywrightBrowser, PlaywrightContext } from './types.ts';
|
|
17
|
+
/** Opens the shared connection; injected so tests can substitute a fake. */
|
|
18
|
+
export type CdpConnect = (endpoint: string, timeoutMs: number) => Promise<PlaywrightBrowser>;
|
|
19
|
+
/** One fetch's lease: the shared browser plus a context it owns. */
|
|
20
|
+
export interface CdpLease {
|
|
21
|
+
/** The shared connection — close only what the lease owns, never this. */
|
|
22
|
+
browser: PlaywrightBrowser;
|
|
23
|
+
/** The fetch-owned isolated context; {@link CdpConnectionPool.release} closes it. */
|
|
24
|
+
context: PlaywrightContext;
|
|
25
|
+
}
|
|
26
|
+
/** A reusable `connectOverCDP` session handing out per-fetch contexts. */
|
|
27
|
+
export declare class CdpConnectionPool {
|
|
28
|
+
private readonly connect;
|
|
29
|
+
private browser;
|
|
30
|
+
private endpoint;
|
|
31
|
+
private connecting;
|
|
32
|
+
private connectingEndpoint;
|
|
33
|
+
/** Bumped by dispose/replace so a settling connect knows it was abandoned. */
|
|
34
|
+
private generation;
|
|
35
|
+
/**
|
|
36
|
+
* @param connect - opens a connection to an endpoint (the provider's real
|
|
37
|
+
* one resolves the bundled playwright-core; tests inject fakes).
|
|
38
|
+
*/
|
|
39
|
+
constructor(connect: CdpConnect);
|
|
40
|
+
/**
|
|
41
|
+
* Lease a fresh isolated context on the shared connection, connecting (or
|
|
42
|
+
* reconnecting) first if needed. Concurrent first fetches share one
|
|
43
|
+
* connect attempt.
|
|
44
|
+
*
|
|
45
|
+
* @param endpoint - normalized CDP endpoint URL.
|
|
46
|
+
* @param timeoutMs - connect timeout budget.
|
|
47
|
+
* @returns the shared browser and a context the caller must release.
|
|
48
|
+
* @throws whatever the connect function throws when no connection can be
|
|
49
|
+
* established (the provider wraps it in a structured WebError).
|
|
50
|
+
*/
|
|
51
|
+
acquire(endpoint: string, timeoutMs: number): Promise<CdpLease>;
|
|
52
|
+
/**
|
|
53
|
+
* Close a fetch-owned context. The shared connection stays open for the
|
|
54
|
+
* next fetch.
|
|
55
|
+
* @param lease - the lease whose context goes away.
|
|
56
|
+
*/
|
|
57
|
+
release(lease: CdpLease): Promise<void>;
|
|
58
|
+
/**
|
|
59
|
+
* Drop the shared connection (plugin teardown or endpoint change). Fetches
|
|
60
|
+
* holding leases keep their contexts until they release them. A connect
|
|
61
|
+
* still in flight is abandoned — its own continuation closes the stray
|
|
62
|
+
* browser (nothing here waits on it, so teardown cannot deadlock).
|
|
63
|
+
*/
|
|
64
|
+
dispose(): Promise<void>;
|
|
65
|
+
/** The shared connection for `endpoint`, connecting or reconnecting as needed. */
|
|
66
|
+
private ensure;
|
|
67
|
+
/** Start a connection to `endpoint`, superseding whatever was there. */
|
|
68
|
+
private connectFresh;
|
|
69
|
+
/** Clear the reference when this exact connection reports it went away. */
|
|
70
|
+
private watch;
|
|
71
|
+
/** Forget a connection known to be dead (absent probe = assume dead here). */
|
|
72
|
+
private drop;
|
|
73
|
+
private isLive;
|
|
74
|
+
}
|
|
@@ -22,6 +22,8 @@ export interface PlaywrightSettings {
|
|
|
22
22
|
cdpEndpoint?: string;
|
|
23
23
|
/** Whether the Readability + DOMPurify pipeline runs. */
|
|
24
24
|
denoise?: boolean;
|
|
25
|
+
/** How many fetches may render at once (1–200); blank = backend default. */
|
|
26
|
+
maxConcurrency?: number;
|
|
25
27
|
}
|
|
26
28
|
/** What the Playwright card renders. */
|
|
27
29
|
export interface PlaywrightCardState extends CardShell {
|
|
@@ -33,6 +35,8 @@ export interface PlaywrightCardState extends CardShell {
|
|
|
33
35
|
cdpEndpoint: CardFieldState;
|
|
34
36
|
/** Denoise checkbox (draft 'true'/'false'). */
|
|
35
37
|
denoise: CardFieldState;
|
|
38
|
+
/** Concurrency input (draft decimal integer). */
|
|
39
|
+
maxConcurrency: CardFieldState;
|
|
36
40
|
}
|
|
37
41
|
/** The registration-side face the card's slot entry injects. */
|
|
38
42
|
export interface PlaywrightCardFace extends CardActions {
|
|
@@ -94,6 +94,16 @@ export declare function radioField(field: string, values: readonly string[]): Ca
|
|
|
94
94
|
* @returns the field's conversion spec.
|
|
95
95
|
*/
|
|
96
96
|
export declare function checkboxField(field: string): CardFieldSpec;
|
|
97
|
+
/**
|
|
98
|
+
* An integer field over an inclusive range. The draft is the decimal string;
|
|
99
|
+
* an empty draft clears the field (re-inherit the schema default), and
|
|
100
|
+
* anything but an integer in range blocks the save as invalid.
|
|
101
|
+
* @param field - field name inside the namespace section.
|
|
102
|
+
* @param min - smallest accepted integer (inclusive).
|
|
103
|
+
* @param max - largest accepted integer (inclusive).
|
|
104
|
+
* @returns the field's conversion spec.
|
|
105
|
+
*/
|
|
106
|
+
export declare function numberField(field: string, min: number, max: number): CardFieldSpec;
|
|
97
107
|
/**
|
|
98
108
|
* Stages the Playwright card's edits over one settings namespace and writes
|
|
99
109
|
* them on save. Publishes through a snapshot store because slot components
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
* @module dsh-web-fetch-playwright/client/locales
|
|
6
6
|
*/
|
|
7
7
|
/** Locale keys this card renders. */
|
|
8
|
-
export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'denoise' | 'denoiseHint' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
|
|
8
|
+
export type PlaywrightCardLocaleKey = 'title' | 'description' | 'backendLabel' | 'backendLocal' | 'backendLocalHint' | 'backendCdp' | 'backendCdpHint' | 'playwrightPath' | 'playwrightPathHint' | 'playwrightPathPlaceholder' | 'cdpEndpoint' | 'cdpEndpointHint' | 'denoise' | 'denoiseHint' | 'maxConcurrency' | 'maxConcurrencyHint' | 'maxConcurrencyPlaceholder' | 'overridden' | 'reset' | 'readOnly' | 'expand' | 'collapse' | 'save' | 'saving' | 'discard' | 'unsaved' | 'saveFailed' | 'invalidText';
|
|
9
9
|
/** This plugin's dictionary namespace, merged into the locale key map. */
|
|
10
10
|
declare module '@deepseek-ai/dsh-client-ui-slots' {
|
|
11
11
|
interface LocaleNamespaceMap {
|
package/lib/types/config.d.ts
CHANGED
|
@@ -9,6 +9,21 @@
|
|
|
9
9
|
import z from '@deepseek-ai/schemastery';
|
|
10
10
|
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
11
11
|
export declare const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
12
|
+
/**
|
|
13
|
+
* Default concurrency for the local backend: each slot launches a whole
|
|
14
|
+
* Chromium, so the default stays frugal; parallel `web_fetch` bursts stop
|
|
15
|
+
* queueing without a browser farm per fetch.
|
|
16
|
+
*/
|
|
17
|
+
export declare const DEFAULT_MAX_CONCURRENCY_LOCAL = 4;
|
|
18
|
+
/**
|
|
19
|
+
* Default concurrency for the CDP backend: the browser already exists, each
|
|
20
|
+
* fetch only opens a tab (isolated context) inside it over one shared
|
|
21
|
+
* connection — so the budget behaves like "max concurrent tabs" and defaults
|
|
22
|
+
* high.
|
|
23
|
+
*/
|
|
24
|
+
export declare const DEFAULT_MAX_CONCURRENCY_CDP = 50;
|
|
25
|
+
/** Ceiling the schema accepts for `maxConcurrency` (local slots are browsers). */
|
|
26
|
+
export declare const MAX_CONCURRENCY_CEILING = 200;
|
|
12
27
|
/** Which browser backend serves a fetch. */
|
|
13
28
|
export type PlaywrightBackend = 'local' | 'cdp';
|
|
14
29
|
/** Plugin config: everything optional — the schema fills the defaults. */
|
|
@@ -25,10 +40,31 @@ export interface Config {
|
|
|
25
40
|
cdpEndpoint?: string;
|
|
26
41
|
/** Whether the Readability + DOMPurify denoise pipeline runs before markdown. */
|
|
27
42
|
denoise?: boolean;
|
|
43
|
+
/**
|
|
44
|
+
* How many fetches may render at once. Blank = backend default (4 for
|
|
45
|
+
* local — each slot launches a browser; 50 for CDP — each slot is a tab in
|
|
46
|
+
* the already-running remote browser).
|
|
47
|
+
*/
|
|
48
|
+
maxConcurrency?: number;
|
|
28
49
|
}
|
|
29
50
|
export declare const Config: z<Config>;
|
|
30
|
-
/**
|
|
31
|
-
|
|
51
|
+
/**
|
|
52
|
+
* Complete config after schemastery applies the field defaults it owns.
|
|
53
|
+
* `maxConcurrency` stays optional — {@link effectiveMaxConcurrency} resolves
|
|
54
|
+
* it against the backend.
|
|
55
|
+
*/
|
|
56
|
+
export type ResolvedConfig = Omit<Required<Config>, 'maxConcurrency'> & {
|
|
57
|
+
maxConcurrency?: number;
|
|
58
|
+
};
|
|
59
|
+
/**
|
|
60
|
+
* The concurrency limit a fetch actually runs with: an explicit setting
|
|
61
|
+
* wins; otherwise the backend default (local launches browsers, CDP only
|
|
62
|
+
* opens tabs, so their budgets differ by an order of magnitude).
|
|
63
|
+
*
|
|
64
|
+
* @param config - the resolved settings section (or any partial of it).
|
|
65
|
+
* @returns the effective limit for the semaphore.
|
|
66
|
+
*/
|
|
67
|
+
export declare function effectiveMaxConcurrency(config: Pick<Config, 'backend' | 'maxConcurrency'>): number;
|
|
32
68
|
/**
|
|
33
69
|
* Normalize a configured CDP endpoint for `chromium.connectOverCDP`:
|
|
34
70
|
* blank becomes the default loopback endpoint; `wss?://` and `https?://`
|
package/lib/types/index.d.ts
CHANGED
|
@@ -13,6 +13,8 @@ import type { Context } from '@deepseek-ai/cordis';
|
|
|
13
13
|
import { Config } from './config.ts';
|
|
14
14
|
export { Config, DEFAULT_CDP_ENDPOINT, normalizeCdpEndpoint } from './config.ts';
|
|
15
15
|
export type { Config as PlaywrightFetchConfig, PlaywrightBackend, ResolvedConfig } from './config.ts';
|
|
16
|
+
export { CdpConnectionPool } from './cdp-pool.ts';
|
|
17
|
+
export type { CdpConnect, CdpLease } from './cdp-pool.ts';
|
|
16
18
|
export { PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider } from './provider.ts';
|
|
17
19
|
export { htmlToMarkdown } from './markdown.ts';
|
|
18
20
|
export type { DenoiseMode, DenoiseResult } from './markdown.ts';
|