dsh-web-fetch-playwright 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +108 -0
- package/README.zh-CN.md +108 -0
- package/cordis.patch.yml +31 -0
- package/dsh.plugin.json +16 -0
- package/lib/client.js +854 -0
- package/lib/index.js +669 -0
- package/lib/types/client/PluginCard.d.ts +43 -0
- package/lib/types/client/card.d.ts +18 -0
- package/lib/types/client/controller.d.ts +58 -0
- package/lib/types/client/fields.d.ts +69 -0
- package/lib/types/client/form.d.ts +159 -0
- package/lib/types/client/index.d.ts +17 -0
- package/lib/types/client/locales.d.ts +18 -0
- package/lib/types/config.d.ts +42 -0
- package/lib/types/index.d.ts +26 -0
- package/lib/types/markdown.d.ts +37 -0
- package/lib/types/playwright-resolve.d.ts +57 -0
- package/lib/types/provider.d.ts +71 -0
- package/lib/types/types.d.ts +58 -0
- package/package.json +130 -0
- package/src/client/PluginCard.module.css +155 -0
- package/src/client/PluginCard.tsx +110 -0
- package/src/client/card.tsx +117 -0
- package/src/client/controller.ts +89 -0
- package/src/client/fields.module.css +228 -0
- package/src/client/fields.tsx +189 -0
- package/src/client/form.ts +337 -0
- package/src/client/index.ts +45 -0
- package/src/client/locales.ts +81 -0
- package/src/client/styles.d.ts +4 -0
- package/src/config.ts +67 -0
- package/src/gfm.d.ts +9 -0
- package/src/index.ts +52 -0
- package/src/markdown.ts +161 -0
- package/src/playwright-resolve.ts +210 -0
- package/src/provider.ts +329 -0
- package/src/types.ts +51 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,669 @@
|
|
|
1
|
+
import { createRequire } from "node:module";
|
|
2
|
+
import { installSettingsSection, settingsNamespace } from "@deepseek-ai/dsh-settings";
|
|
3
|
+
import z from "@deepseek-ai/schemastery";
|
|
4
|
+
import { WebError } from "@deepseek-ai/dsh-web";
|
|
5
|
+
import { JSDOM, VirtualConsole } from "jsdom";
|
|
6
|
+
import createDOMPurify from "dompurify";
|
|
7
|
+
import { Readability } from "@mozilla/readability";
|
|
8
|
+
import TurndownService from "turndown";
|
|
9
|
+
import { gfm } from "@joplin/turndown-plugin-gfm";
|
|
10
|
+
import { accessSync, constants, readFileSync, realpathSync, statSync } from "node:fs";
|
|
11
|
+
import { dirname, join } from "node:path";
|
|
12
|
+
import { pathToFileURL } from "node:url";
|
|
13
|
+
//#region src/config.ts
|
|
14
|
+
/**
|
|
15
|
+
* Settings/config surface for the Playwright fetch provider: the schemastery
|
|
16
|
+
* schema the loader validates the row against, the settings namespace's
|
|
17
|
+
* resolved shape, and the small pure normalizers the provider applies per
|
|
18
|
+
* fetch (CDP endpoint shaping) — kept network-free for unit tests.
|
|
19
|
+
*
|
|
20
|
+
* @module dsh-web-fetch-playwright/config
|
|
21
|
+
*/
|
|
22
|
+
/** Default CDP endpoint when the settings section leaves it blank. */
|
|
23
|
+
const DEFAULT_CDP_ENDPOINT = "127.0.0.1:9222";
|
|
24
|
+
const Config = z.object({
|
|
25
|
+
backend: z.union([z.const("local"), z.const("cdp")]).default("local"),
|
|
26
|
+
playwrightPath: z.string().default(""),
|
|
27
|
+
cdpEndpoint: z.string().default(""),
|
|
28
|
+
denoise: z.boolean().default(true)
|
|
29
|
+
});
|
|
30
|
+
/**
|
|
31
|
+
* Normalize a configured CDP endpoint for `chromium.connectOverCDP`:
|
|
32
|
+
* blank becomes the default loopback endpoint; `wss?://` and `https?://`
|
|
33
|
+
* pass through; a bare `host:port` gains the `http://` scheme Playwright's
|
|
34
|
+
* CDP discovery (`GET /json/version`) expects.
|
|
35
|
+
*
|
|
36
|
+
* @param input - the raw configured endpoint.
|
|
37
|
+
* @returns the endpoint string to hand to Playwright.
|
|
38
|
+
* @throws {Error} when the normalized value is not a parseable URL.
|
|
39
|
+
*/
|
|
40
|
+
function normalizeCdpEndpoint(input) {
|
|
41
|
+
const trimmed = input.trim();
|
|
42
|
+
const value = trimmed === "" ? DEFAULT_CDP_ENDPOINT : trimmed;
|
|
43
|
+
const candidate = /^wss?:\/\//i.test(value) || /^https?:\/\//i.test(value) ? value : `http://${value}`;
|
|
44
|
+
new URL(candidate);
|
|
45
|
+
return candidate;
|
|
46
|
+
}
|
|
47
|
+
//#endregion
|
|
48
|
+
//#region src/markdown.ts
|
|
49
|
+
/**
|
|
50
|
+
* The denoise pipeline: rendered HTML → sanitized article HTML → markdown.
|
|
51
|
+
*
|
|
52
|
+
* The classic stack, in order: jsdom parses the page Playwright rendered;
|
|
53
|
+
* Mozilla Readability extracts the article (dropping nav bars, sidebars,
|
|
54
|
+
* footers, and ad chrome by scoring link density and text mass); DOMPurify
|
|
55
|
+
* sanitizes whatever HTML remains and forbids the layout tags noise lives in;
|
|
56
|
+
* Turndown with the GFM plugin converts to markdown using the same style
|
|
57
|
+
* options and span-safe table rules as the shipped `dsh-tool-web` renderer,
|
|
58
|
+
* so output is consistent with what `web_fetch` produces elsewhere.
|
|
59
|
+
*
|
|
60
|
+
* Pure and synchronous — unit-tested against fixture pages.
|
|
61
|
+
*
|
|
62
|
+
* @module dsh-web-fetch-playwright/markdown
|
|
63
|
+
*/
|
|
64
|
+
/** Layout/noise tags DOMPurify removes outright (with their content). */
|
|
65
|
+
const FORBID_TAGS = [
|
|
66
|
+
"nav",
|
|
67
|
+
"aside",
|
|
68
|
+
"header",
|
|
69
|
+
"footer",
|
|
70
|
+
"form",
|
|
71
|
+
"svg",
|
|
72
|
+
"iframe",
|
|
73
|
+
"noscript",
|
|
74
|
+
"button",
|
|
75
|
+
"select",
|
|
76
|
+
"option",
|
|
77
|
+
"input",
|
|
78
|
+
"textarea",
|
|
79
|
+
"dialog",
|
|
80
|
+
"canvas",
|
|
81
|
+
"video",
|
|
82
|
+
"audio",
|
|
83
|
+
"template"
|
|
84
|
+
];
|
|
85
|
+
/** Attributes stripped from sanitized output (styling survives as noise). */
|
|
86
|
+
const FORBID_ATTR = [
|
|
87
|
+
"style",
|
|
88
|
+
"class",
|
|
89
|
+
"id",
|
|
90
|
+
"hidden",
|
|
91
|
+
"aria-hidden",
|
|
92
|
+
"role"
|
|
93
|
+
];
|
|
94
|
+
/** The shared converter: same style options as `dsh-tool-web`'s renderer. */
|
|
95
|
+
const turndown = new TurndownService({
|
|
96
|
+
headingStyle: "atx",
|
|
97
|
+
codeBlockStyle: "fenced",
|
|
98
|
+
bulletListMarker: "-"
|
|
99
|
+
});
|
|
100
|
+
turndown.use(gfm);
|
|
101
|
+
turndown.remove([
|
|
102
|
+
"script",
|
|
103
|
+
"style",
|
|
104
|
+
"noscript"
|
|
105
|
+
]);
|
|
106
|
+
/** Render one GFM table cell without interpreting HTML span counts (tool-web parity). */
|
|
107
|
+
function renderTableCell(content, index) {
|
|
108
|
+
return `${index === 0 ? "| " : " "}${content.trim().replace(/\n\r/g, "<br>").replace(/\n/g, "<br>").replace(/\|+/g, "\\|").padEnd(3, " ")} |`;
|
|
109
|
+
}
|
|
110
|
+
/** Whether a row is the table's Markdown heading row (tool-web parity). */
|
|
111
|
+
function isTableHeadingRow(row) {
|
|
112
|
+
const cells = Array.from(row.cells);
|
|
113
|
+
const section = row.parentElement;
|
|
114
|
+
const table = section.parentElement;
|
|
115
|
+
return (section.nodeName === "THEAD" || table.rows[0] === row) && cells.every((cell) => cell.nodeName === "TH");
|
|
116
|
+
}
|
|
117
|
+
/** Map an HTML table-cell alignment to the GFM separator marker (tool-web parity). */
|
|
118
|
+
function tableBorder(cell) {
|
|
119
|
+
const alignment = (cell.getAttribute("align") || cell.style.textAlign || "").toLowerCase();
|
|
120
|
+
if (alignment === "left") return ":---";
|
|
121
|
+
if (alignment === "right") return "---:";
|
|
122
|
+
if (alignment === "center") return ":---:";
|
|
123
|
+
return "---";
|
|
124
|
+
}
|
|
125
|
+
turndown.addRule("tableCellWithoutSpanExpansion", {
|
|
126
|
+
filter: ["th", "td"],
|
|
127
|
+
replacement(content, node) {
|
|
128
|
+
const cell = node;
|
|
129
|
+
const row = cell.parentNode;
|
|
130
|
+
return renderTableCell(content, Array.prototype.indexOf.call(row.childNodes, cell));
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
turndown.addRule("tableRowWithoutSpanExpansion", {
|
|
134
|
+
filter: "tr",
|
|
135
|
+
replacement(content, node) {
|
|
136
|
+
const row = node;
|
|
137
|
+
const border = isTableHeadingRow(row) ? Array.from(row.cells, (cell, index) => renderTableCell(tableBorder(cell), index)).join("") : "";
|
|
138
|
+
return `\n${content}${border.length > 0 ? `\n${border}` : ""}`;
|
|
139
|
+
}
|
|
140
|
+
});
|
|
141
|
+
/**
|
|
142
|
+
* Convert one rendered HTML document to denoised markdown.
|
|
143
|
+
*
|
|
144
|
+
* Readability failure (non-article pages) degrades to converting the
|
|
145
|
+
* sanitized whole document — layout tags are still forbidden, so the
|
|
146
|
+
* fallback stays cleaner than raw turndown, and a degraded page beats an
|
|
147
|
+
* error for a body the browser already rendered.
|
|
148
|
+
*
|
|
149
|
+
* @param html - the rendered page HTML (`page.content()`).
|
|
150
|
+
* @param url - the page URL, used to resolve relative links during parsing.
|
|
151
|
+
* @returns the markdown and the extraction mode used.
|
|
152
|
+
*/
|
|
153
|
+
function htmlToMarkdown(html, url) {
|
|
154
|
+
const dom = new JSDOM(html, {
|
|
155
|
+
url,
|
|
156
|
+
virtualConsole: new VirtualConsole()
|
|
157
|
+
});
|
|
158
|
+
const purify = createDOMPurify(dom.window);
|
|
159
|
+
const document = dom.window.document;
|
|
160
|
+
let source = null;
|
|
161
|
+
let title;
|
|
162
|
+
try {
|
|
163
|
+
const cloned = document.cloneNode(true);
|
|
164
|
+
const article = new Readability(cloned).parse();
|
|
165
|
+
if (article !== null && typeof article.content === "string" && article.content !== "") {
|
|
166
|
+
source = article.content;
|
|
167
|
+
title = article.title ?? void 0;
|
|
168
|
+
}
|
|
169
|
+
} catch {}
|
|
170
|
+
let mode = "article";
|
|
171
|
+
if (source === null) {
|
|
172
|
+
mode = "document";
|
|
173
|
+
source = document.body?.innerHTML ?? "";
|
|
174
|
+
}
|
|
175
|
+
const clean = purify.sanitize(source, {
|
|
176
|
+
FORBID_TAGS,
|
|
177
|
+
FORBID_ATTR,
|
|
178
|
+
KEEP_CONTENT: false
|
|
179
|
+
});
|
|
180
|
+
let markdown;
|
|
181
|
+
try {
|
|
182
|
+
markdown = turndown.turndown(clean);
|
|
183
|
+
} catch {
|
|
184
|
+
markdown = dom.window.document.createElement("div").textContent ?? "";
|
|
185
|
+
}
|
|
186
|
+
markdown = markdown.replace(/\n{3,}/g, "\n\n").trim();
|
|
187
|
+
const headingTitle = title?.trim();
|
|
188
|
+
if (headingTitle !== void 0 && headingTitle !== "" && !markdown.startsWith("# ")) markdown = `# ${headingTitle}\n\n${markdown}`;
|
|
189
|
+
return {
|
|
190
|
+
markdown,
|
|
191
|
+
mode
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
//#endregion
|
|
195
|
+
//#region src/playwright-resolve.ts
|
|
196
|
+
/**
|
|
197
|
+
* Locates a usable Playwright for the local backend. Resolution order:
|
|
198
|
+
*
|
|
199
|
+
* 1. the configured executable/document path (settings field `playwrightPath`);
|
|
200
|
+
* 2. when blank, a `playwright` executable discovered on `$PATH`;
|
|
201
|
+
* 3. the plugin's own bundled `playwright-core` dependency.
|
|
202
|
+
*
|
|
203
|
+
* A discovered path is classified, not assumed: the Node `playwright` CLI
|
|
204
|
+
* yields its package (whose `chromium` knows that installation's browser
|
|
205
|
+
* registry), while a Chromium-family browser binary yields the bundled
|
|
206
|
+
* `playwright-core` with `executablePath` pinned to it. The CDP backend needs
|
|
207
|
+
* no local browsers at all and always uses the bundled core.
|
|
208
|
+
*
|
|
209
|
+
* Pure probing lives here; the provider composes it with page navigation.
|
|
210
|
+
*
|
|
211
|
+
* @module dsh-web-fetch-playwright/playwright-resolve
|
|
212
|
+
*/
|
|
213
|
+
/** File magics identifying a native browser executable rather than a script. */
|
|
214
|
+
const EXECUTABLE_MAGICS = [
|
|
215
|
+
Buffer.from([
|
|
216
|
+
127,
|
|
217
|
+
69,
|
|
218
|
+
76,
|
|
219
|
+
70
|
|
220
|
+
]),
|
|
221
|
+
Buffer.from([
|
|
222
|
+
207,
|
|
223
|
+
250,
|
|
224
|
+
237,
|
|
225
|
+
254
|
|
226
|
+
]),
|
|
227
|
+
Buffer.from([
|
|
228
|
+
202,
|
|
229
|
+
254,
|
|
230
|
+
186,
|
|
231
|
+
190
|
|
232
|
+
]),
|
|
233
|
+
Buffer.from([77, 90])
|
|
234
|
+
];
|
|
235
|
+
/**
|
|
236
|
+
* Find `name` as an executable file on `$PATH`.
|
|
237
|
+
* @param name - the executable basename.
|
|
238
|
+
* @returns the first matching path, or undefined.
|
|
239
|
+
*/
|
|
240
|
+
function findOnPath(name) {
|
|
241
|
+
const pathValue = process.env.PATH ?? "";
|
|
242
|
+
for (const dir of pathValue.split(":")) {
|
|
243
|
+
if (dir === "") continue;
|
|
244
|
+
const candidate = join(dir, name);
|
|
245
|
+
try {
|
|
246
|
+
if (!statSync(candidate).isFile()) continue;
|
|
247
|
+
accessSync(candidate, constants.X_OK);
|
|
248
|
+
return candidate;
|
|
249
|
+
} catch {}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
/**
|
|
253
|
+
* Whether a file starts with a native-executable magic (ELF / Mach-O / MZ).
|
|
254
|
+
* @param path - the file to sniff.
|
|
255
|
+
* @returns true when the file is a compiled binary.
|
|
256
|
+
*/
|
|
257
|
+
function isNativeExecutable(path) {
|
|
258
|
+
let head;
|
|
259
|
+
try {
|
|
260
|
+
head = readFileSync(path).subarray(0, 4);
|
|
261
|
+
} catch {
|
|
262
|
+
return false;
|
|
263
|
+
}
|
|
264
|
+
return EXECUTABLE_MAGICS.some((magic) => head.subarray(0, magic.length).equals(magic));
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Walk up from a file (or the file's own directory) to the nearest package
|
|
268
|
+
* root whose manifest names Playwright.
|
|
269
|
+
* @param start - a file path inside or beside the candidate package.
|
|
270
|
+
* @param maxHops - upward directory bound.
|
|
271
|
+
* @returns the package root directory, or undefined.
|
|
272
|
+
*/
|
|
273
|
+
function findPlaywrightPackageRoot(start, maxHops = 6) {
|
|
274
|
+
let dir = statSync(start).isDirectory() ? start : dirname(start);
|
|
275
|
+
try {
|
|
276
|
+
dir = realpathSync(dir);
|
|
277
|
+
} catch {}
|
|
278
|
+
for (let hop = 0; hop <= maxHops; hop++) {
|
|
279
|
+
const manifest = join(dir, "package.json");
|
|
280
|
+
try {
|
|
281
|
+
const pkg = JSON.parse(readFileSync(manifest, "utf8"));
|
|
282
|
+
if (pkg.name === "playwright" || pkg.name === "playwright-core" || pkg.name === "@playwright/test") return dir;
|
|
283
|
+
} catch {}
|
|
284
|
+
const parent = dirname(dir);
|
|
285
|
+
if (parent === dir) return void 0;
|
|
286
|
+
dir = parent;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Import the Playwright package rooted at `root`, preferring its ESM entry
|
|
291
|
+
* with a CommonJS fallback.
|
|
292
|
+
* @param root - the package root directory.
|
|
293
|
+
* @returns the package's default export (the Playwright module).
|
|
294
|
+
*/
|
|
295
|
+
async function importPlaywrightPackage(root) {
|
|
296
|
+
const entry = createRequire(join(root, "package.json")).resolve(".");
|
|
297
|
+
return await import(pathToFileURL(entry).href);
|
|
298
|
+
}
|
|
299
|
+
/** Memoized successful resolutions, keyed by the configured path ("" = auto). */
|
|
300
|
+
const resolvedCache = /* @__PURE__ */ new Map();
|
|
301
|
+
/**
|
|
302
|
+
* Resolve the local Playwright backend for one fetch.
|
|
303
|
+
*
|
|
304
|
+
* @param configuredPath - the settings `playwrightPath` (blank = auto-discover).
|
|
305
|
+
* @returns how to launch/connect, with provenance for diagnostics.
|
|
306
|
+
* @throws {Error} with an actionable message when nothing usable exists.
|
|
307
|
+
*/
|
|
308
|
+
async function resolvePlaywrightBackend(configuredPath) {
|
|
309
|
+
const trimmed = configuredPath.trim();
|
|
310
|
+
const cacheKey = trimmed;
|
|
311
|
+
const cached = resolvedCache.get(cacheKey);
|
|
312
|
+
if (cached !== void 0) return cached;
|
|
313
|
+
const candidate = trimmed !== "" ? trimmed : findOnPath("playwright");
|
|
314
|
+
if (trimmed !== "" && !existsAsFile(trimmed)) throw new Error(`configured playwright path "${trimmed}" does not exist`);
|
|
315
|
+
if (candidate !== void 0 && isNativeExecutable(candidate)) {
|
|
316
|
+
const resolved = {
|
|
317
|
+
chromium: await importBundledCore(),
|
|
318
|
+
executablePath: realpathSync(candidate),
|
|
319
|
+
source: `browser executable ${candidate} (bundled playwright-core)`
|
|
320
|
+
};
|
|
321
|
+
resolvedCache.set(cacheKey, resolved);
|
|
322
|
+
return resolved;
|
|
323
|
+
}
|
|
324
|
+
if (candidate !== void 0) {
|
|
325
|
+
const root = findPlaywrightPackageRoot(candidate);
|
|
326
|
+
if (root !== void 0) {
|
|
327
|
+
const chromium = (await importPlaywrightPackage(root)).chromium;
|
|
328
|
+
if (chromium !== void 0 && typeof chromium.launch === "function") {
|
|
329
|
+
const resolved = {
|
|
330
|
+
chromium,
|
|
331
|
+
source: `playwright package at ${root}${trimmed === "" ? " (discovered on $PATH)" : ""}`
|
|
332
|
+
};
|
|
333
|
+
resolvedCache.set(cacheKey, resolved);
|
|
334
|
+
return resolved;
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
if (trimmed !== "") throw new Error(`configured playwright path "${trimmed}" is neither a Playwright CLI nor a browser executable`);
|
|
338
|
+
}
|
|
339
|
+
const resolved = {
|
|
340
|
+
chromium: await importBundledCore(),
|
|
341
|
+
source: "bundled playwright-core (no playwright found on $PATH; browsers must be discoverable, or set the path)"
|
|
342
|
+
};
|
|
343
|
+
resolvedCache.set(cacheKey, resolved);
|
|
344
|
+
return resolved;
|
|
345
|
+
}
|
|
346
|
+
/** The CDP backend: the bundled core only — no local browser registry needed. */
|
|
347
|
+
async function resolveCdpBackend() {
|
|
348
|
+
return {
|
|
349
|
+
chromium: await importBundledCore(),
|
|
350
|
+
source: "bundled playwright-core over CDP"
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
let bundledCore;
|
|
354
|
+
async function importBundledCore() {
|
|
355
|
+
if (bundledCore !== void 0) return bundledCore;
|
|
356
|
+
const chromium = (await import("playwright-core")).chromium;
|
|
357
|
+
if (chromium === void 0 || typeof chromium.launch !== "function") throw new Error("playwright-core dependency did not export a usable chromium namespace");
|
|
358
|
+
bundledCore = chromium;
|
|
359
|
+
return bundledCore;
|
|
360
|
+
}
|
|
361
|
+
function existsAsFile(path) {
|
|
362
|
+
try {
|
|
363
|
+
return statSync(path).isFile();
|
|
364
|
+
} catch {
|
|
365
|
+
return false;
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
//#endregion
|
|
369
|
+
//#region src/provider.ts
|
|
370
|
+
/**
|
|
371
|
+
* The Playwright `WebFetchProvider`: renders one URL in a real browser and
|
|
372
|
+
* returns it as markdown (denoised) or HTML. Mirrors `dsh-web-fetch-http`'s
|
|
373
|
+
* error taxonomy (URL hygiene, abort/timeout translation, content-type
|
|
374
|
+
* classification) so the tool layer sees the same codes from either backend.
|
|
375
|
+
*
|
|
376
|
+
* Lifecycle is per-fetch and fully reversible: local launches close their
|
|
377
|
+
* browser, CDP connections disconnect, and an aborted signal closes both —
|
|
378
|
+
* no browser outlives the call that opened it. Concurrency is capped at
|
|
379
|
+
* {@link MAX_CONCURRENT_FETCHES} local browsers.
|
|
380
|
+
*
|
|
381
|
+
* Private-network and SSRF protection is not implemented (same stance as the
|
|
382
|
+
* shipped HTTP provider); a page this provider can reach is whatever the
|
|
383
|
+
* browser can reach.
|
|
384
|
+
*
|
|
385
|
+
* @module dsh-web-fetch-playwright/provider
|
|
386
|
+
*/
|
|
387
|
+
/** Stable id this provider registers under (the bundle patch pins it). */
|
|
388
|
+
const PLAYWRIGHT_FETCH_PROVIDER_ID = "playwright";
|
|
389
|
+
/** Maximum accepted request URL length (http-provider parity). */
|
|
390
|
+
const MAX_URL_LENGTH = 2048;
|
|
391
|
+
/** Cap on the decoded markdown/HTML body this provider returns. */
|
|
392
|
+
const MAX_BODY_CHARS = 1e5;
|
|
393
|
+
/** Cap on rendered HTML fed into the synchronous denoise pipeline. */
|
|
394
|
+
const MAX_PIPELINE_INPUT_CHARS = 2e6;
|
|
395
|
+
/** How many browsers may render at once; further fetches queue. */
|
|
396
|
+
const MAX_CONCURRENT_FETCHES = 2;
|
|
397
|
+
/** Default per-fetch budget (ms), inside the tool layer's 60s. */
|
|
398
|
+
const DEFAULT_TIMEOUT_MS = 45e3;
|
|
399
|
+
/** Best-effort post-DOM settle wait (ms) so SPA content can finish rendering. */
|
|
400
|
+
const SETTLE_MS = 5e3;
|
|
401
|
+
/**
|
|
402
|
+
* A single abort deadline composing the caller's signal with this provider's
|
|
403
|
+
* time budget. The timeout reason is kept apart from an outer abort so the
|
|
404
|
+
* error translation can pick `WEB_FETCH_TIMEOUT` over `WEB_ABORTED`.
|
|
405
|
+
*/
|
|
406
|
+
var Deadline = class {
|
|
407
|
+
signal;
|
|
408
|
+
controller = new AbortController();
|
|
409
|
+
expiresAt;
|
|
410
|
+
timedOut = false;
|
|
411
|
+
constructor(outer, timeoutMs) {
|
|
412
|
+
this.signal = this.controller.signal;
|
|
413
|
+
this.expiresAt = Date.now() + timeoutMs;
|
|
414
|
+
const timer = setTimeout(() => {
|
|
415
|
+
this.timedOut = true;
|
|
416
|
+
this.controller.abort(/* @__PURE__ */ new Error("playwright fetch deadline"));
|
|
417
|
+
}, timeoutMs);
|
|
418
|
+
if (outer !== void 0) {
|
|
419
|
+
if (outer.aborted) this.controller.abort(outer.reason);
|
|
420
|
+
else outer.addEventListener("abort", () => {
|
|
421
|
+
this.controller.abort(outer.reason);
|
|
422
|
+
}, { once: true });
|
|
423
|
+
}
|
|
424
|
+
this.signal.addEventListener("abort", () => {
|
|
425
|
+
clearTimeout(timer);
|
|
426
|
+
}, { once: true });
|
|
427
|
+
}
|
|
428
|
+
/** Whether OUR timer fired (vs an outer cancellation). */
|
|
429
|
+
get isTimeout() {
|
|
430
|
+
return this.timedOut;
|
|
431
|
+
}
|
|
432
|
+
/** Milliseconds left on the budget, floored at 1 for Playwright options. */
|
|
433
|
+
remainingMs() {
|
|
434
|
+
return Math.max(1, this.expiresAt - Date.now());
|
|
435
|
+
}
|
|
436
|
+
};
|
|
437
|
+
/** A cap-{@link MAX_CONCURRENT_FETCHES} async semaphore with abort support. */
|
|
438
|
+
var Semaphore = class {
|
|
439
|
+
active = 0;
|
|
440
|
+
queue = [];
|
|
441
|
+
/** Take a slot, or queue until one frees; an aborted signal rejects the wait. */
|
|
442
|
+
acquire(signal) {
|
|
443
|
+
if (this.active < MAX_CONCURRENT_FETCHES) {
|
|
444
|
+
this.active++;
|
|
445
|
+
return Promise.resolve();
|
|
446
|
+
}
|
|
447
|
+
return new Promise((resolve, reject) => {
|
|
448
|
+
const waiter = {
|
|
449
|
+
start: () => {
|
|
450
|
+
signal.removeEventListener("abort", waiter.abort);
|
|
451
|
+
this.active++;
|
|
452
|
+
resolve();
|
|
453
|
+
},
|
|
454
|
+
abort: () => {
|
|
455
|
+
const index = this.queue.indexOf(waiter);
|
|
456
|
+
if (index !== -1) this.queue.splice(index, 1);
|
|
457
|
+
reject(new WebError("web fetch aborted while waiting for a free browser slot", "WEB_ABORTED"));
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
this.queue.push(waiter);
|
|
461
|
+
signal.addEventListener("abort", waiter.abort, { once: true });
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
release() {
|
|
465
|
+
const next = this.queue.shift();
|
|
466
|
+
if (next === void 0) {
|
|
467
|
+
this.active = Math.max(0, this.active - 1);
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
next.start();
|
|
471
|
+
}
|
|
472
|
+
};
|
|
473
|
+
/** Validate a request URL: http(s) only, no embedded credentials, bounded. */
|
|
474
|
+
function validateFetchUrl(input) {
|
|
475
|
+
if (input.length > MAX_URL_LENGTH) throw new WebError(`URL exceeds the maximum length of ${String(MAX_URL_LENGTH)}`, "WEB_INVALID_URL");
|
|
476
|
+
let url;
|
|
477
|
+
try {
|
|
478
|
+
url = new URL(input);
|
|
479
|
+
} catch (error) {
|
|
480
|
+
throw new WebError(`invalid URL: ${input}`, "WEB_INVALID_URL", { cause: error });
|
|
481
|
+
}
|
|
482
|
+
if (url.protocol !== "http:" && url.protocol !== "https:") throw new WebError(`unsupported URL scheme "${url.protocol}" (only http and https are allowed)`, "WEB_INVALID_URL");
|
|
483
|
+
if (url.username.length > 0 || url.password.length > 0) throw new WebError("credentials in URLs are not allowed", "WEB_BLOCKED_URL");
|
|
484
|
+
return url;
|
|
485
|
+
}
|
|
486
|
+
/** Classify a `Content-Type` into a decodable kind; undefined = unsupported. */
|
|
487
|
+
function classifyContentType(contentType) {
|
|
488
|
+
const mime = (contentType ?? "").replace(/;.*$/s, "").trim().toLowerCase();
|
|
489
|
+
if (mime === "" || mime === "text/html" || mime === "application/xhtml+xml") return "html";
|
|
490
|
+
if (mime.startsWith("text/")) return "text";
|
|
491
|
+
if (mime === "application/json" || mime === "application/xml" || mime.endsWith("+json") || mime.endsWith("+xml")) return "text";
|
|
492
|
+
}
|
|
493
|
+
/** Translate a thrown pipeline error into the seam's WebError taxonomy. */
|
|
494
|
+
function translateError(error, deadline) {
|
|
495
|
+
if (deadline.isTimeout) return new WebError("playwright web fetch timed out", "WEB_FETCH_TIMEOUT", { cause: error });
|
|
496
|
+
if (deadline.signal.aborted) return new WebError("web fetch aborted", "WEB_ABORTED", { cause: error });
|
|
497
|
+
if (error instanceof WebError) return error;
|
|
498
|
+
return new WebError(`playwright web fetch failed: ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* The Playwright-backed fetch provider. Configuration is read through a thunk
|
|
502
|
+
* so committed settings-section changes apply to the next fetch with no
|
|
503
|
+
* re-registration.
|
|
504
|
+
*/
|
|
505
|
+
var PlaywrightFetchProvider = class {
|
|
506
|
+
configSource;
|
|
507
|
+
id = PLAYWRIGHT_FETCH_PROVIDER_ID;
|
|
508
|
+
semaphore = new Semaphore();
|
|
509
|
+
/**
|
|
510
|
+
* @param configSource - thunk returning the currently authoritative config.
|
|
511
|
+
*/
|
|
512
|
+
constructor(configSource) {
|
|
513
|
+
this.configSource = configSource;
|
|
514
|
+
}
|
|
515
|
+
/** Cheap and side-effect free; backend problems surface per fetch instead. */
|
|
516
|
+
available() {
|
|
517
|
+
return true;
|
|
518
|
+
}
|
|
519
|
+
async fetch(request, signal) {
|
|
520
|
+
if (signal?.aborted) throw new WebError("web fetch aborted", "WEB_ABORTED");
|
|
521
|
+
const config = this.configSource();
|
|
522
|
+
const url = validateFetchUrl(request.url);
|
|
523
|
+
const deadline = new Deadline(signal, DEFAULT_TIMEOUT_MS);
|
|
524
|
+
await this.semaphore.acquire(deadline.signal);
|
|
525
|
+
let session;
|
|
526
|
+
try {
|
|
527
|
+
session = await this.openSession(config, deadline);
|
|
528
|
+
const onAbort = () => {
|
|
529
|
+
session?.context.close().catch(() => {});
|
|
530
|
+
session?.browser.close().catch(() => {});
|
|
531
|
+
};
|
|
532
|
+
deadline.signal.addEventListener("abort", onAbort, { once: true });
|
|
533
|
+
try {
|
|
534
|
+
return await this.retrieve(session, url, config, deadline);
|
|
535
|
+
} finally {
|
|
536
|
+
deadline.signal.removeEventListener("abort", onAbort);
|
|
537
|
+
}
|
|
538
|
+
} catch (error) {
|
|
539
|
+
throw translateError(error, deadline);
|
|
540
|
+
} finally {
|
|
541
|
+
await session?.context.close().catch(() => {});
|
|
542
|
+
await session?.browser.close().catch(() => {});
|
|
543
|
+
this.semaphore.release();
|
|
544
|
+
}
|
|
545
|
+
}
|
|
546
|
+
/**
|
|
547
|
+
* Open the configured backend and its per-fetch context. Split out so the
|
|
548
|
+
* test suite can substitute a fake browser.
|
|
549
|
+
* @param config - the resolved settings section.
|
|
550
|
+
* @param deadline - the fetch budget, applied to launch/connect timeouts.
|
|
551
|
+
* @returns the browser session the fetch will use.
|
|
552
|
+
*/
|
|
553
|
+
async openSession(config, deadline) {
|
|
554
|
+
const timeout = Math.min(deadline.remainingMs(), 2e4);
|
|
555
|
+
if (config.backend === "cdp") {
|
|
556
|
+
const endpoint = normalizeCdpEndpoint(config.cdpEndpoint);
|
|
557
|
+
const { chromium, source } = await resolveCdpBackend();
|
|
558
|
+
try {
|
|
559
|
+
const browser = await chromium.connectOverCDP(endpoint, { timeout });
|
|
560
|
+
const context = await browser.newContext();
|
|
561
|
+
await installResourceFilter(context);
|
|
562
|
+
return {
|
|
563
|
+
browser,
|
|
564
|
+
context
|
|
565
|
+
};
|
|
566
|
+
} catch (error) {
|
|
567
|
+
throw new WebError(`cannot connect to the CDP endpoint ${endpoint} (${source}); is the browser started with --remote-debugging-port? ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
const { chromium, executablePath, source } = await resolvePlaywrightBackend(config.playwrightPath);
|
|
571
|
+
try {
|
|
572
|
+
const browser = await chromium.launch({
|
|
573
|
+
headless: true,
|
|
574
|
+
...executablePath !== void 0 ? { executablePath } : {},
|
|
575
|
+
timeout
|
|
576
|
+
});
|
|
577
|
+
const context = await browser.newContext();
|
|
578
|
+
await installResourceFilter(context);
|
|
579
|
+
return {
|
|
580
|
+
browser,
|
|
581
|
+
context
|
|
582
|
+
};
|
|
583
|
+
} catch (error) {
|
|
584
|
+
throw new WebError(`cannot launch the local browser (${source}); run \`playwright install chromium\` or point the settings path at a playwright/browser executable. ${String(error instanceof Error ? error.message : error)}`, "WEB_PROVIDER_ERROR", { cause: error });
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
/** Navigate, settle, and decode one URL inside an open session. */
|
|
588
|
+
async retrieve(session, url, config, deadline) {
|
|
589
|
+
const page = await session.context.newPage();
|
|
590
|
+
const response = await page.goto(url.toString(), {
|
|
591
|
+
waitUntil: "domcontentloaded",
|
|
592
|
+
timeout: deadline.remainingMs()
|
|
593
|
+
});
|
|
594
|
+
const kind = classifyContentType(response?.headers()["content-type"]);
|
|
595
|
+
if (kind === void 0) throw new WebError(`unsupported content type "${response?.headers()["content-type"] ?? "unknown"}"`, "WEB_UNSUPPORTED_CONTENT_TYPE");
|
|
596
|
+
const finalUrl = page.url();
|
|
597
|
+
if (kind === "text") {
|
|
598
|
+
const text = response !== null ? await response.text() : await page.content();
|
|
599
|
+
return capResult(finalUrl, response?.status() ?? 200, {
|
|
600
|
+
kind: "text",
|
|
601
|
+
content: text
|
|
602
|
+
});
|
|
603
|
+
}
|
|
604
|
+
await page.waitForLoadState("networkidle", { timeout: Math.min(SETTLE_MS, deadline.remainingMs()) }).catch(() => {});
|
|
605
|
+
const html = await page.content();
|
|
606
|
+
if (!config.denoise) return capResult(finalUrl, response?.status() ?? 200, {
|
|
607
|
+
kind: "html",
|
|
608
|
+
content: html
|
|
609
|
+
});
|
|
610
|
+
const bounded = html.length > MAX_PIPELINE_INPUT_CHARS ? html.slice(0, MAX_PIPELINE_INPUT_CHARS) : html;
|
|
611
|
+
const { markdown } = htmlToMarkdown(bounded, finalUrl);
|
|
612
|
+
const result = capResult(finalUrl, response?.status() ?? 200, {
|
|
613
|
+
kind: "text",
|
|
614
|
+
content: markdown
|
|
615
|
+
});
|
|
616
|
+
return bounded !== html ? {
|
|
617
|
+
...result,
|
|
618
|
+
truncated: true
|
|
619
|
+
} : result;
|
|
620
|
+
}
|
|
621
|
+
};
|
|
622
|
+
/** Apply the decoded-body cap and flag the cut. */
|
|
623
|
+
function capResult(url, statusCode, body) {
|
|
624
|
+
const truncated = body.content.length > MAX_BODY_CHARS;
|
|
625
|
+
return {
|
|
626
|
+
url,
|
|
627
|
+
statusCode,
|
|
628
|
+
body: {
|
|
629
|
+
kind: body.kind,
|
|
630
|
+
content: truncated ? body.content.slice(0, MAX_BODY_CHARS) : body.content
|
|
631
|
+
},
|
|
632
|
+
truncated
|
|
633
|
+
};
|
|
634
|
+
}
|
|
635
|
+
/**
|
|
636
|
+
* Abort image/font/media subrequests: the markdown output keeps their URLs
|
|
637
|
+
* but never renders them, so downloading them only spends the budget.
|
|
638
|
+
* Best-effort — a context that refuses interception still fetches.
|
|
639
|
+
*/
|
|
640
|
+
async function installResourceFilter(context) {
|
|
641
|
+
try {
|
|
642
|
+
await context.route("**/*", async (route) => {
|
|
643
|
+
const type = route.request().resourceType();
|
|
644
|
+
if (type === "image" || type === "font" || type === "media") await route.abort();
|
|
645
|
+
else await route.continue();
|
|
646
|
+
});
|
|
647
|
+
} catch {}
|
|
648
|
+
}
|
|
649
|
+
//#endregion
|
|
650
|
+
//#region src/index.ts
|
|
651
|
+
/** Cordis plugin name used by loader diagnostics. */
|
|
652
|
+
const name = "dsh-web-fetch-playwright";
|
|
653
|
+
/** The web seam this provider registers into. */
|
|
654
|
+
const inject = ["web"];
|
|
655
|
+
/** Settings namespace carrying this provider's configuration card. */
|
|
656
|
+
const WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE = settingsNamespace("web-fetch-playwright");
|
|
657
|
+
/** Register the Playwright fetch provider with `ctx.web`. */
|
|
658
|
+
function apply(ctx, config) {
|
|
659
|
+
let current = () => config;
|
|
660
|
+
installSettingsSection(ctx, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, Config, config, {
|
|
661
|
+
setSource: (source) => {
|
|
662
|
+
current = source;
|
|
663
|
+
},
|
|
664
|
+
onChange: () => {}
|
|
665
|
+
});
|
|
666
|
+
ctx.web.registerFetchProvider(new PlaywrightFetchProvider(() => current()));
|
|
667
|
+
}
|
|
668
|
+
//#endregion
|
|
669
|
+
export { Config, DEFAULT_CDP_ENDPOINT, PLAYWRIGHT_FETCH_PROVIDER_ID, PlaywrightFetchProvider, WEB_FETCH_PLAYWRIGHT_SETTINGS_NAMESPACE, apply, htmlToMarkdown, inject, name, normalizeCdpEndpoint };
|