@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +307 -46
- package/README.zh-CN.md +307 -46
- package/client.js +44 -3
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +48 -4
- package/lib/cache.js +134 -9
- package/lib/config.d.ts +25 -0
- package/lib/config.js +42 -0
- package/lib/engines.d.ts +41 -1
- package/lib/engines.js +33 -2
- package/lib/fetch.d.ts +33 -0
- package/lib/fetch.js +218 -6
- package/lib/index.d.ts +26 -0
- package/lib/index.js +113 -27
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/rank.js +4 -1
- package/lib/read-page.d.ts +136 -0
- package/lib/read-page.js +496 -0
- package/lib/render.d.ts +193 -0
- package/lib/render.js +354 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.js +19 -0
- package/lib/types.d.ts +13 -4
- package/package.json +1 -1
package/lib/render.d.ts
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The optional JavaScript-rendering half of `web_fetch`.
|
|
3
|
+
*
|
|
4
|
+
* Some pages answer a bare request with a shell: a framework marker, a mount
|
|
5
|
+
* point and a few hundred characters of navigation, while the article appears
|
|
6
|
+
* only after the page's own scripts run. No amount of proxy tuning fixes that,
|
|
7
|
+
* because the bytes are correct and the browser work is simply missing.
|
|
8
|
+
*
|
|
9
|
+
* A real browser can do that work, but Playwright is a large optional install
|
|
10
|
+
* and a hard dependency would make this plugin fail to load wherever it is
|
|
11
|
+
* absent. So the renderer is discovered at runtime, by a guarded dynamic
|
|
12
|
+
* `import()`, and never named in `package.json`. Nothing here runs at module
|
|
13
|
+
* load: importing this file is always safe.
|
|
14
|
+
*
|
|
15
|
+
* The renderer's surface is declared structurally rather than imported from
|
|
16
|
+
* Playwright's types, so this module compiles and runs with the package
|
|
17
|
+
* missing. Tests inject the same shapes with plain objects.
|
|
18
|
+
*
|
|
19
|
+
* @module @fangchenghao/dsh-web-search/render
|
|
20
|
+
*/
|
|
21
|
+
/** What the user asked for in `renderJs`. */
|
|
22
|
+
export type RenderMode = 'auto' | 'always' | 'off';
|
|
23
|
+
/** What one fetch reports about the optional render rung. */
|
|
24
|
+
export interface RenderReport {
|
|
25
|
+
/** The mode the request resolved to. */
|
|
26
|
+
readonly mode: RenderMode;
|
|
27
|
+
/** Whether a browser launch was tried. */
|
|
28
|
+
readonly attempted: boolean;
|
|
29
|
+
/** Whether the returned body came from the browser. */
|
|
30
|
+
readonly applied: boolean;
|
|
31
|
+
/** Safe, content-free explanation for the decision. */
|
|
32
|
+
readonly reason: string;
|
|
33
|
+
}
|
|
34
|
+
/** Default browser-render deadline, in milliseconds. */
|
|
35
|
+
export declare const DEFAULT_RENDER_TIMEOUT_MS = 15000;
|
|
36
|
+
/** Extra time allowed for `networkidle` after the DOM is ready. */
|
|
37
|
+
export declare const NETWORK_IDLE_EXTRA_MS = 2000;
|
|
38
|
+
/** Below this many readable characters an HTML page is treated as a shell. */
|
|
39
|
+
export declare const SHELL_READABLE_LIMIT = 500;
|
|
40
|
+
/** With a client-render marker, the shell limit rises to this many characters. */
|
|
41
|
+
export declare const SHELL_MARKER_READABLE_LIMIT = 1200;
|
|
42
|
+
/** The parts of a page the shell heuristic looks at. */
|
|
43
|
+
export interface ShellInput {
|
|
44
|
+
readonly statusCode: number;
|
|
45
|
+
readonly body: {
|
|
46
|
+
readonly kind: 'html' | 'text';
|
|
47
|
+
readonly content: string;
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
/** A shell decision plus the numbers and reason behind it. */
|
|
51
|
+
export interface ShellVerdict {
|
|
52
|
+
/** Whether a browser render is worth attempting. */
|
|
53
|
+
readonly shell: boolean;
|
|
54
|
+
/** Characters of visible text, tags and script bodies removed. */
|
|
55
|
+
readonly readableChars: number;
|
|
56
|
+
/** Short, content-free explanation, safe to surface in a report. */
|
|
57
|
+
readonly reason: string;
|
|
58
|
+
}
|
|
59
|
+
/** The page handle this module needs; Playwright's `Page` satisfies it. */
|
|
60
|
+
export interface RendererPage {
|
|
61
|
+
goto(url: string, options: {
|
|
62
|
+
readonly waitUntil: 'domcontentloaded';
|
|
63
|
+
readonly timeout: number;
|
|
64
|
+
}): Promise<{
|
|
65
|
+
status(): number;
|
|
66
|
+
} | null>;
|
|
67
|
+
waitForLoadState(state: 'networkidle', options: {
|
|
68
|
+
readonly timeout: number;
|
|
69
|
+
}): Promise<unknown>;
|
|
70
|
+
content(): Promise<string>;
|
|
71
|
+
url(): string;
|
|
72
|
+
close(): Promise<unknown>;
|
|
73
|
+
}
|
|
74
|
+
/** The browser handle this module needs; Playwright's `Browser` satisfies it. */
|
|
75
|
+
export interface RendererBrowser {
|
|
76
|
+
newPage(): Promise<RendererPage>;
|
|
77
|
+
close(): Promise<unknown>;
|
|
78
|
+
}
|
|
79
|
+
/** The module surface this module needs; `playwright` satisfies it. */
|
|
80
|
+
export interface RendererModule {
|
|
81
|
+
readonly chromium: {
|
|
82
|
+
launch(options: {
|
|
83
|
+
readonly headless: boolean;
|
|
84
|
+
}): Promise<RendererBrowser>;
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
/** Result of the guarded dynamic import. */
|
|
88
|
+
export type RendererLoad = {
|
|
89
|
+
readonly ok: true;
|
|
90
|
+
readonly renderer: RendererModule;
|
|
91
|
+
readonly moduleName: string;
|
|
92
|
+
} | {
|
|
93
|
+
readonly ok: false;
|
|
94
|
+
readonly reason: string;
|
|
95
|
+
};
|
|
96
|
+
/** What one successful render hands back. */
|
|
97
|
+
export interface RenderedPage {
|
|
98
|
+
/** Serialised DOM, after its scripts have run. */
|
|
99
|
+
readonly html: string;
|
|
100
|
+
/** HTTP status of the main navigation, or 200 when there was no response. */
|
|
101
|
+
readonly statusCode: number;
|
|
102
|
+
/** Final URL after redirects. */
|
|
103
|
+
readonly url: string;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Approximate the visible text of an HTML document.
|
|
107
|
+
*
|
|
108
|
+
* The browser is not available at this point (that is the whole question), so
|
|
109
|
+
* this is deliberately a heuristic: drop `head`, script/style/template bodies,
|
|
110
|
+
* comments and tags, then count the characters that remain. It never throws on
|
|
111
|
+
* malformed markup because it only strips.
|
|
112
|
+
*
|
|
113
|
+
* @param html - the document markup.
|
|
114
|
+
* @returns the number of visible characters.
|
|
115
|
+
*/
|
|
116
|
+
export declare function readableTextLength(html: string): number;
|
|
117
|
+
/**
|
|
118
|
+
* Decide whether a fetched page is probably a script-rendered shell.
|
|
119
|
+
*
|
|
120
|
+
* The rules, in order:
|
|
121
|
+
* 1. a body that is not HTML is never a shell (JSON and plain text are final);
|
|
122
|
+
* 2. an HTML body with no readable text is a shell;
|
|
123
|
+
* 3. a 403/429 that still carries readable text is a block page a browser may
|
|
124
|
+
* get past, so it is a shell;
|
|
125
|
+
* 4. fewer than {@link SHELL_READABLE_LIMIT} readable characters is a shell;
|
|
126
|
+
* 5. below {@link SHELL_MARKER_READABLE_LIMIT} with a client-render marker
|
|
127
|
+
* (`__NEXT_DATA__`, `id="root"`, `__NUXT__`, …) is a shell.
|
|
128
|
+
*
|
|
129
|
+
* A non-2xx other than 403/429 is an error page, not a shell: rendering it
|
|
130
|
+
* would only repeat the error more slowly.
|
|
131
|
+
*
|
|
132
|
+
* @param page - status and body of the best outcome so far.
|
|
133
|
+
* @returns the verdict, including the readable length and a safe reason.
|
|
134
|
+
*/
|
|
135
|
+
export declare function detectShellPage(page: ShellInput): ShellVerdict;
|
|
136
|
+
/**
|
|
137
|
+
* Scrub a diagnostic string before it leaves this module.
|
|
138
|
+
*
|
|
139
|
+
* Render errors can carry a target URL (and therefore a token in its query) or
|
|
140
|
+
* a credential-looking assignment. None of that belongs in a tool result, so
|
|
141
|
+
* query strings are blanked, credential keys are redacted and the text is
|
|
142
|
+
* capped. The renderer's own message is never interpolated by callers, which
|
|
143
|
+
* is the stronger guard; this is the second one.
|
|
144
|
+
*
|
|
145
|
+
* @param text - the raw diagnostic.
|
|
146
|
+
* @returns a single-line, redacted, bounded string.
|
|
147
|
+
*/
|
|
148
|
+
export declare function redactRenderReason(text: string): string;
|
|
149
|
+
/**
|
|
150
|
+
* Try `playwright`, then `playwright-core`.
|
|
151
|
+
*
|
|
152
|
+
* Both imports are guarded: a missing package is a supported deployment and
|
|
153
|
+
* yields `{ ok: false }` with a reason, never a thrown error. Nothing is cached
|
|
154
|
+
* here; the fetch provider caches the verdict for its own lifetime.
|
|
155
|
+
*
|
|
156
|
+
* @returns the renderer module, or the reason neither could be loaded.
|
|
157
|
+
*/
|
|
158
|
+
export declare function loadPlaywrightRenderer(): Promise<RendererLoad>;
|
|
159
|
+
/**
|
|
160
|
+
* Render one URL in a headless Chromium and return its serialised DOM.
|
|
161
|
+
*
|
|
162
|
+
* Shape of the attempt: launch, navigate with `domcontentloaded` under the
|
|
163
|
+
* configured budget, give `networkidle` at most {@link NETWORK_IDLE_EXTRA_MS}
|
|
164
|
+
* more (a page that never idles is not an error), then read `content()`.
|
|
165
|
+
*
|
|
166
|
+
* Both the page and the browser are closed in `finally` on every path. An
|
|
167
|
+
* abortion closes them immediately through one listener and rejects with an
|
|
168
|
+
* `AbortError`; a launch that is still in flight when the caller aborts is
|
|
169
|
+
* closed as soon as it resolves, so a cancelled fetch cannot leave a browser
|
|
170
|
+
* behind.
|
|
171
|
+
*
|
|
172
|
+
* @param renderer - a loaded Playwright-shaped module.
|
|
173
|
+
* @param url - the page to open.
|
|
174
|
+
* @param timeoutMs - navigation deadline in milliseconds.
|
|
175
|
+
* @param signal - the caller's cancellation signal, when present.
|
|
176
|
+
* @returns the rendered page.
|
|
177
|
+
*/
|
|
178
|
+
export declare function renderWithPlaywright(renderer: RendererModule, url: string, timeoutMs: number, signal: AbortSignal | undefined): Promise<RenderedPage>;
|
|
179
|
+
/**
|
|
180
|
+
* Classify a render failure without echoing its message.
|
|
181
|
+
*
|
|
182
|
+
* The message can contain the target URL, a selector, or page text, so only
|
|
183
|
+
* the error *name* is used. A timeout is recognised by name or by the word in
|
|
184
|
+
* the message and reported with the configured budget.
|
|
185
|
+
*
|
|
186
|
+
* @param error - the thrown value.
|
|
187
|
+
* @param timeoutMs - the budget the render was given.
|
|
188
|
+
* @returns a safe reason and whether the renderer reported an abort.
|
|
189
|
+
*/
|
|
190
|
+
export declare function describeRenderError(error: unknown, timeoutMs: number): {
|
|
191
|
+
readonly reason: string;
|
|
192
|
+
readonly aborted: boolean;
|
|
193
|
+
};
|
package/lib/render.js
ADDED
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The optional JavaScript-rendering half of `web_fetch`.
|
|
3
|
+
*
|
|
4
|
+
* Some pages answer a bare request with a shell: a framework marker, a mount
|
|
5
|
+
* point and a few hundred characters of navigation, while the article appears
|
|
6
|
+
* only after the page's own scripts run. No amount of proxy tuning fixes that,
|
|
7
|
+
* because the bytes are correct and the browser work is simply missing.
|
|
8
|
+
*
|
|
9
|
+
* A real browser can do that work, but Playwright is a large optional install
|
|
10
|
+
* and a hard dependency would make this plugin fail to load wherever it is
|
|
11
|
+
* absent. So the renderer is discovered at runtime, by a guarded dynamic
|
|
12
|
+
* `import()`, and never named in `package.json`. Nothing here runs at module
|
|
13
|
+
* load: importing this file is always safe.
|
|
14
|
+
*
|
|
15
|
+
* The renderer's surface is declared structurally rather than imported from
|
|
16
|
+
* Playwright's types, so this module compiles and runs with the package
|
|
17
|
+
* missing. Tests inject the same shapes with plain objects.
|
|
18
|
+
*
|
|
19
|
+
* @module @fangchenghao/dsh-web-search/render
|
|
20
|
+
*/
|
|
21
|
+
var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExtension) || function (path, preserveJsx) {
|
|
22
|
+
if (typeof path === "string" && /^\.\.?\//.test(path)) {
|
|
23
|
+
return path.replace(/\.(tsx)$|((?:\.d)?)((?:\.[^./]+?)?)\.([cm]?)ts$/i, function (m, tsx, d, ext, cm) {
|
|
24
|
+
return tsx ? preserveJsx ? ".jsx" : ".js" : d && (!ext || !cm) ? m : (d + ext + "." + cm.toLowerCase() + "js");
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
return path;
|
|
28
|
+
};
|
|
29
|
+
/** Default browser-render deadline, in milliseconds. */
|
|
30
|
+
export const DEFAULT_RENDER_TIMEOUT_MS = 15_000;
|
|
31
|
+
/** Extra time allowed for `networkidle` after the DOM is ready. */
|
|
32
|
+
export const NETWORK_IDLE_EXTRA_MS = 2_000;
|
|
33
|
+
/** Below this many readable characters an HTML page is treated as a shell. */
|
|
34
|
+
export const SHELL_READABLE_LIMIT = 500;
|
|
35
|
+
/** With a client-render marker, the shell limit rises to this many characters. */
|
|
36
|
+
export const SHELL_MARKER_READABLE_LIMIT = 1_200;
|
|
37
|
+
/**
|
|
38
|
+
* Markers that only occur on pages whose content is mounted by JavaScript.
|
|
39
|
+
* They are lower-cased because the haystack is lower-cased once.
|
|
40
|
+
*/
|
|
41
|
+
const SHELL_MARKERS = [
|
|
42
|
+
'__next_data__',
|
|
43
|
+
'id="root"',
|
|
44
|
+
"id='root'",
|
|
45
|
+
'id="app"',
|
|
46
|
+
"id='app'",
|
|
47
|
+
'id="__next"',
|
|
48
|
+
"id='__next'",
|
|
49
|
+
'__nuxt__',
|
|
50
|
+
'ng-version=',
|
|
51
|
+
'data-reactroot',
|
|
52
|
+
];
|
|
53
|
+
/**
|
|
54
|
+
* Approximate the visible text of an HTML document.
|
|
55
|
+
*
|
|
56
|
+
* The browser is not available at this point (that is the whole question), so
|
|
57
|
+
* this is deliberately a heuristic: drop `head`, script/style/template bodies,
|
|
58
|
+
* comments and tags, then count the characters that remain. It never throws on
|
|
59
|
+
* malformed markup because it only strips.
|
|
60
|
+
*
|
|
61
|
+
* @param html - the document markup.
|
|
62
|
+
* @returns the number of visible characters.
|
|
63
|
+
*/
|
|
64
|
+
export function readableTextLength(html) {
|
|
65
|
+
if (html.length === 0)
|
|
66
|
+
return 0;
|
|
67
|
+
return html
|
|
68
|
+
.replace(/<head\b[^>]*>[\s\S]*?<\/head\s*>/gi, ' ')
|
|
69
|
+
.replace(/<(script|style|noscript|template|svg)\b[^>]*>[\s\S]*?<\/\1\s*>/gi, ' ')
|
|
70
|
+
.replace(/<(script|style|noscript|template)\b[^>]*>[\s\S]*$/gi, ' ')
|
|
71
|
+
.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
72
|
+
.replace(/<[^>]*>/g, ' ')
|
|
73
|
+
.replace(/&(?:#\d+|#x[0-9a-f]+|[a-z][a-z0-9]+);/gi, ' ')
|
|
74
|
+
.replace(/\s+/g, ' ')
|
|
75
|
+
.trim().length;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Decide whether a fetched page is probably a script-rendered shell.
|
|
79
|
+
*
|
|
80
|
+
* The rules, in order:
|
|
81
|
+
* 1. a body that is not HTML is never a shell (JSON and plain text are final);
|
|
82
|
+
* 2. an HTML body with no readable text is a shell;
|
|
83
|
+
* 3. a 403/429 that still carries readable text is a block page a browser may
|
|
84
|
+
* get past, so it is a shell;
|
|
85
|
+
* 4. fewer than {@link SHELL_READABLE_LIMIT} readable characters is a shell;
|
|
86
|
+
* 5. below {@link SHELL_MARKER_READABLE_LIMIT} with a client-render marker
|
|
87
|
+
* (`__NEXT_DATA__`, `id="root"`, `__NUXT__`, …) is a shell.
|
|
88
|
+
*
|
|
89
|
+
* A non-2xx other than 403/429 is an error page, not a shell: rendering it
|
|
90
|
+
* would only repeat the error more slowly.
|
|
91
|
+
*
|
|
92
|
+
* @param page - status and body of the best outcome so far.
|
|
93
|
+
* @returns the verdict, including the readable length and a safe reason.
|
|
94
|
+
*/
|
|
95
|
+
export function detectShellPage(page) {
|
|
96
|
+
if (page.body.kind !== 'html') {
|
|
97
|
+
return { shell: false, readableChars: 0, reason: 'the body is not HTML' };
|
|
98
|
+
}
|
|
99
|
+
const readable = readableTextLength(page.body.content);
|
|
100
|
+
const status = page.statusCode;
|
|
101
|
+
const renderableStatus = (status >= 200 && status < 300) || status === 403 || status === 429;
|
|
102
|
+
if (!renderableStatus) {
|
|
103
|
+
return { shell: false, readableChars: readable, reason: `HTTP ${status} is an error response, not a page shell` };
|
|
104
|
+
}
|
|
105
|
+
if (readable === 0) {
|
|
106
|
+
return { shell: true, readableChars: 0, reason: 'the HTML carries no readable text' };
|
|
107
|
+
}
|
|
108
|
+
if (status === 403 || status === 429) {
|
|
109
|
+
return {
|
|
110
|
+
shell: true,
|
|
111
|
+
readableChars: readable,
|
|
112
|
+
reason: `HTTP ${status} still answered with ${readable} readable characters`,
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
if (readable < SHELL_READABLE_LIMIT) {
|
|
116
|
+
return {
|
|
117
|
+
shell: true,
|
|
118
|
+
readableChars: readable,
|
|
119
|
+
reason: `only ${readable} readable characters, below the ${SHELL_READABLE_LIMIT}-character shell limit`,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
if (readable < SHELL_MARKER_READABLE_LIMIT) {
|
|
123
|
+
const haystack = page.body.content.toLowerCase();
|
|
124
|
+
const marker = SHELL_MARKERS.find((candidate) => haystack.includes(candidate));
|
|
125
|
+
if (marker !== undefined) {
|
|
126
|
+
return {
|
|
127
|
+
shell: true,
|
|
128
|
+
readableChars: readable,
|
|
129
|
+
reason: `${readable} readable characters with the client-render marker ${marker}`,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return { shell: false, readableChars: readable, reason: `${readable} readable characters look like real content` };
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Scrub a diagnostic string before it leaves this module.
|
|
137
|
+
*
|
|
138
|
+
* Render errors can carry a target URL (and therefore a token in its query) or
|
|
139
|
+
* a credential-looking assignment. None of that belongs in a tool result, so
|
|
140
|
+
* query strings are blanked, credential keys are redacted and the text is
|
|
141
|
+
* capped. The renderer's own message is never interpolated by callers, which
|
|
142
|
+
* is the stronger guard; this is the second one.
|
|
143
|
+
*
|
|
144
|
+
* @param text - the raw diagnostic.
|
|
145
|
+
* @returns a single-line, redacted, bounded string.
|
|
146
|
+
*/
|
|
147
|
+
export function redactRenderReason(text) {
|
|
148
|
+
const redacted = text
|
|
149
|
+
.replace(/https?:\/\/[^\s]*/gi, (url) => url.replace(/[?#].*$/, '?[redacted]'))
|
|
150
|
+
.replace(/\/\/[^/\s@]+@/g, '//[redacted]@')
|
|
151
|
+
.replace(/\b(token|api[-_]?key|secret|password|passwd|authorization|cookie)\s*=\s*[^\s&;,]+/gi, '$1=[redacted]')
|
|
152
|
+
.replace(/\s+/g, ' ')
|
|
153
|
+
.trim();
|
|
154
|
+
return redacted.length > 240 ? `${redacted.slice(0, 240)}…` : redacted;
|
|
155
|
+
}
|
|
156
|
+
/** The module names tried, in order; `playwright-core` is the smaller install. */
|
|
157
|
+
const PLAYWRIGHT_MODULES = ['playwright', 'playwright-core'];
|
|
158
|
+
/**
|
|
159
|
+
* Read an imported module's `chromium.launch`, accepting a default export.
|
|
160
|
+
* @param loaded - whatever `import()` returned.
|
|
161
|
+
* @returns the module surface, or `undefined` when it is not a renderer.
|
|
162
|
+
*/
|
|
163
|
+
function pickRendererModule(loaded) {
|
|
164
|
+
if (loaded === null || typeof loaded !== 'object')
|
|
165
|
+
return undefined;
|
|
166
|
+
const record = loaded;
|
|
167
|
+
for (const candidate of [record, record.default]) {
|
|
168
|
+
if (candidate === null || typeof candidate !== 'object')
|
|
169
|
+
continue;
|
|
170
|
+
const chromium = candidate.chromium;
|
|
171
|
+
if (chromium !== undefined && typeof chromium.launch === 'function') {
|
|
172
|
+
return candidate;
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Why an `import()` failed, without echoing a path or a message.
|
|
179
|
+
* @param error - the thrown value.
|
|
180
|
+
* @returns `not installed` for a missing package, otherwise the error name.
|
|
181
|
+
*/
|
|
182
|
+
function importFailure(error) {
|
|
183
|
+
if (error !== null
|
|
184
|
+
&& typeof error === 'object'
|
|
185
|
+
&& error.code === 'ERR_MODULE_NOT_FOUND') {
|
|
186
|
+
return 'not installed';
|
|
187
|
+
}
|
|
188
|
+
return error instanceof Error ? error.name : 'import failed';
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Try `playwright`, then `playwright-core`.
|
|
192
|
+
*
|
|
193
|
+
* Both imports are guarded: a missing package is a supported deployment and
|
|
194
|
+
* yields `{ ok: false }` with a reason, never a thrown error. Nothing is cached
|
|
195
|
+
* here; the fetch provider caches the verdict for its own lifetime.
|
|
196
|
+
*
|
|
197
|
+
* @returns the renderer module, or the reason neither could be loaded.
|
|
198
|
+
*/
|
|
199
|
+
export async function loadPlaywrightRenderer() {
|
|
200
|
+
const failures = [];
|
|
201
|
+
// The specifier is a variable on purpose: TypeScript must not try to resolve
|
|
202
|
+
// an optional package that is absent by design, and the import stays dynamic.
|
|
203
|
+
for (const moduleName of PLAYWRIGHT_MODULES) {
|
|
204
|
+
try {
|
|
205
|
+
const loaded = await import(__rewriteRelativeImportExtension(moduleName));
|
|
206
|
+
const renderer = pickRendererModule(loaded);
|
|
207
|
+
if (renderer !== undefined)
|
|
208
|
+
return { ok: true, renderer, moduleName };
|
|
209
|
+
failures.push(`${moduleName}: no chromium.launch export`);
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
failures.push(`${moduleName}: ${importFailure(error)}`);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
return {
|
|
216
|
+
ok: false,
|
|
217
|
+
reason: redactRenderReason(`no JS renderer is installed (${failures.join('; ')})`),
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
/** The abort rejection this module uses, matching `fetch`'s convention. */
|
|
221
|
+
function abortError() {
|
|
222
|
+
return new DOMException('The page render was aborted', 'AbortError');
|
|
223
|
+
}
|
|
224
|
+
/**
|
|
225
|
+
* Read an optional signal without letting control-flow narrowing lie.
|
|
226
|
+
*
|
|
227
|
+
* The first check throws, after which TypeScript narrows `signal.aborted` to
|
|
228
|
+
* `false` for the rest of the function; an await can still change it in reality,
|
|
229
|
+
* so every later check goes through this function.
|
|
230
|
+
*/
|
|
231
|
+
function isAborted(signal) {
|
|
232
|
+
return signal !== undefined && signal.aborted;
|
|
233
|
+
}
|
|
234
|
+
/** Close a handle without letting cleanup mask the render's real outcome. */
|
|
235
|
+
async function closeQuietly(target) {
|
|
236
|
+
if (target === undefined)
|
|
237
|
+
return;
|
|
238
|
+
try {
|
|
239
|
+
await target.close();
|
|
240
|
+
}
|
|
241
|
+
catch {
|
|
242
|
+
// A close that fails is not the interesting failure.
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Render one URL in a headless Chromium and return its serialised DOM.
|
|
247
|
+
*
|
|
248
|
+
* Shape of the attempt: launch, navigate with `domcontentloaded` under the
|
|
249
|
+
* configured budget, give `networkidle` at most {@link NETWORK_IDLE_EXTRA_MS}
|
|
250
|
+
* more (a page that never idles is not an error), then read `content()`.
|
|
251
|
+
*
|
|
252
|
+
* Both the page and the browser are closed in `finally` on every path. An
|
|
253
|
+
* abortion closes them immediately through one listener and rejects with an
|
|
254
|
+
* `AbortError`; a launch that is still in flight when the caller aborts is
|
|
255
|
+
* closed as soon as it resolves, so a cancelled fetch cannot leave a browser
|
|
256
|
+
* behind.
|
|
257
|
+
*
|
|
258
|
+
* @param renderer - a loaded Playwright-shaped module.
|
|
259
|
+
* @param url - the page to open.
|
|
260
|
+
* @param timeoutMs - navigation deadline in milliseconds.
|
|
261
|
+
* @param signal - the caller's cancellation signal, when present.
|
|
262
|
+
* @returns the rendered page.
|
|
263
|
+
*/
|
|
264
|
+
export async function renderWithPlaywright(renderer, url, timeoutMs, signal) {
|
|
265
|
+
if (isAborted(signal))
|
|
266
|
+
throw abortError();
|
|
267
|
+
const budget = Number.isFinite(timeoutMs) && timeoutMs >= 1 ? Math.floor(timeoutMs) : DEFAULT_RENDER_TIMEOUT_MS;
|
|
268
|
+
let browser;
|
|
269
|
+
let page;
|
|
270
|
+
let resolveAbort;
|
|
271
|
+
const aborted = new Promise((resolve) => {
|
|
272
|
+
resolveAbort = resolve;
|
|
273
|
+
});
|
|
274
|
+
const onAbort = () => {
|
|
275
|
+
resolveAbort?.();
|
|
276
|
+
// Requirement, not optimisation: a cancelled render must not keep a
|
|
277
|
+
// browser or a page alive while the caller has already moved on.
|
|
278
|
+
void closeQuietly(page);
|
|
279
|
+
void closeQuietly(browser);
|
|
280
|
+
};
|
|
281
|
+
let listening = false;
|
|
282
|
+
if (signal !== undefined) {
|
|
283
|
+
if (signal.aborted) {
|
|
284
|
+
onAbort();
|
|
285
|
+
}
|
|
286
|
+
else {
|
|
287
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
288
|
+
listening = true;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
try {
|
|
292
|
+
const launching = renderer.chromium.launch({ headless: true });
|
|
293
|
+
const winner = await Promise.race([
|
|
294
|
+
launching.then((value) => ({ browser: value })),
|
|
295
|
+
aborted.then(() => ({ browser: undefined })),
|
|
296
|
+
]);
|
|
297
|
+
if (winner.browser === undefined) {
|
|
298
|
+
// The launch may still complete after the caller is gone; close it then.
|
|
299
|
+
void launching.then((late) => closeQuietly(late)).catch(() => undefined);
|
|
300
|
+
throw abortError();
|
|
301
|
+
}
|
|
302
|
+
browser = winner.browser;
|
|
303
|
+
if (isAborted(signal))
|
|
304
|
+
throw abortError();
|
|
305
|
+
page = await browser.newPage();
|
|
306
|
+
if (isAborted(signal))
|
|
307
|
+
throw abortError();
|
|
308
|
+
const response = await page.goto(url, { waitUntil: 'domcontentloaded', timeout: budget });
|
|
309
|
+
const idleBudget = Math.min(NETWORK_IDLE_EXTRA_MS, budget);
|
|
310
|
+
try {
|
|
311
|
+
await page.waitForLoadState('networkidle', { timeout: idleBudget });
|
|
312
|
+
}
|
|
313
|
+
catch {
|
|
314
|
+
// A page that never goes idle still has a DOM; carry on.
|
|
315
|
+
}
|
|
316
|
+
if (isAborted(signal))
|
|
317
|
+
throw abortError();
|
|
318
|
+
const html = await page.content();
|
|
319
|
+
if (isAborted(signal))
|
|
320
|
+
throw abortError();
|
|
321
|
+
return {
|
|
322
|
+
html,
|
|
323
|
+
statusCode: response === null ? 200 : response.status(),
|
|
324
|
+
url: page.url(),
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
finally {
|
|
328
|
+
if (listening && signal !== undefined)
|
|
329
|
+
signal.removeEventListener('abort', onAbort);
|
|
330
|
+
await closeQuietly(page);
|
|
331
|
+
await closeQuietly(browser);
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Classify a render failure without echoing its message.
|
|
336
|
+
*
|
|
337
|
+
* The message can contain the target URL, a selector, or page text, so only
|
|
338
|
+
* the error *name* is used. A timeout is recognised by name or by the word in
|
|
339
|
+
* the message and reported with the configured budget.
|
|
340
|
+
*
|
|
341
|
+
* @param error - the thrown value.
|
|
342
|
+
* @param timeoutMs - the budget the render was given.
|
|
343
|
+
* @returns a safe reason and whether the renderer reported an abort.
|
|
344
|
+
*/
|
|
345
|
+
export function describeRenderError(error, timeoutMs) {
|
|
346
|
+
const name = error instanceof Error ? error.name : 'unknown error';
|
|
347
|
+
if (name === 'AbortError')
|
|
348
|
+
return { reason: 'the render was aborted', aborted: true };
|
|
349
|
+
const message = error instanceof Error ? error.message : '';
|
|
350
|
+
if (name === 'TimeoutError' || /timeout|timed out/i.test(message)) {
|
|
351
|
+
return { reason: `the render timed out after ${timeoutMs} ms`, aborted: false };
|
|
352
|
+
}
|
|
353
|
+
return { reason: `the render failed (${name})`, aborted: false };
|
|
354
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-facing `search_many` tool: several independent searches in one
|
|
3
|
+
* call, each reusing the same engine layer, cache and cooldown tracker as
|
|
4
|
+
* `web_search`.
|
|
5
|
+
*
|
|
6
|
+
* The batch is deliberately boring: every query is a normal search, failures
|
|
7
|
+
* are recorded per query instead of aborting the batch, and at most three
|
|
8
|
+
* searches run at once so one call cannot spend the whole engine budget. The
|
|
9
|
+
* registration path mirrors `domain-tool.ts` — raw JSON Schema, `output`
|
|
10
|
+
* carrying `{ schema, render }`, the optional `ctx.get('tools')` lookup, a
|
|
11
|
+
* fallback name on collision, and a contained failure path.
|
|
12
|
+
*
|
|
13
|
+
* @module @fangchenghao/dsh-web-search/search-many
|
|
14
|
+
*/
|
|
15
|
+
import type { Source } from './types.ts';
|
|
16
|
+
import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
|
|
17
|
+
/** The name this tool prefers. */
|
|
18
|
+
export declare const SEARCH_MANY_TOOL_NAME = "search_many";
|
|
19
|
+
/** The name used when another plugin already registered `search_many`. */
|
|
20
|
+
export declare const SEARCH_MANY_FALLBACK_NAME = "web_search_many";
|
|
21
|
+
/** How many queries run at the same time; deliberately below the engine count. */
|
|
22
|
+
export declare const SEARCH_MANY_CONCURRENCY = 3;
|
|
23
|
+
/** Queries accepted in one call. */
|
|
24
|
+
export declare const SEARCH_MANY_MIN_QUERIES = 1;
|
|
25
|
+
/** Queries accepted in one call. */
|
|
26
|
+
export declare const SEARCH_MANY_MAX_QUERIES = 5;
|
|
27
|
+
/** Default result budget for a query that names none. */
|
|
28
|
+
export declare const SEARCH_MANY_DEFAULT_LIMIT = 5;
|
|
29
|
+
/** Largest per-query budget the tool accepts. */
|
|
30
|
+
export declare const SEARCH_MANY_MAX_LIMIT = 20;
|
|
31
|
+
/** One source as `search_many` reports it; `site` appears only when asked. */
|
|
32
|
+
export interface SearchManySource extends Source {
|
|
33
|
+
/** Lower-cased hostname of {@link Source.url}, when `withSites` is true. */
|
|
34
|
+
site?: string;
|
|
35
|
+
}
|
|
36
|
+
/** What one query in the batch produced. */
|
|
37
|
+
export interface SearchManyResult {
|
|
38
|
+
/** The query exactly as it was accepted (trimmed). */
|
|
39
|
+
query: string;
|
|
40
|
+
sources: SearchManySource[];
|
|
41
|
+
/** Failures for this query alone; a thrown search becomes one entry here. */
|
|
42
|
+
failures: string[];
|
|
43
|
+
}
|
|
44
|
+
/** What the tool returns to the model. */
|
|
45
|
+
export interface SearchManyValue {
|
|
46
|
+
results: SearchManyResult[];
|
|
47
|
+
/** True when any per-query budget was cut by the deployment's limit. */
|
|
48
|
+
truncated: boolean;
|
|
49
|
+
}
|
|
50
|
+
/** What a single query's runner reports back. */
|
|
51
|
+
export interface SearchManyOutcome {
|
|
52
|
+
sources: Source[];
|
|
53
|
+
failures: string[];
|
|
54
|
+
/** True when the deployment's configured limit cut the requested budget. */
|
|
55
|
+
truncated: boolean;
|
|
56
|
+
}
|
|
57
|
+
/** Run one search; the plugin binds this to the same path as `web_search`. */
|
|
58
|
+
export type SearchManyRunner = (query: string, limit: number, signal?: AbortSignal) => Promise<SearchManyOutcome>;
|
|
59
|
+
/** A tool definition in the host registry's native shape. */
|
|
60
|
+
export interface SearchManyDefinition {
|
|
61
|
+
name: string;
|
|
62
|
+
description: string;
|
|
63
|
+
parameters: ToolJsonSchema;
|
|
64
|
+
output: {
|
|
65
|
+
schema: ToolJsonSchema;
|
|
66
|
+
render: (args: unknown, value: SearchManyValue) => ToolContent[];
|
|
67
|
+
};
|
|
68
|
+
timeoutMs?: number;
|
|
69
|
+
isConcurrencySafe?: (args: unknown) => boolean;
|
|
70
|
+
execute: (args: unknown, exec: {
|
|
71
|
+
signal?: AbortSignal;
|
|
72
|
+
}) => Promise<SearchManyValue>;
|
|
73
|
+
}
|
|
74
|
+
/** The slice of the tool registry this module needs. */
|
|
75
|
+
export interface SearchManyRegistryLike {
|
|
76
|
+
register(definition: SearchManyDefinition): unknown;
|
|
77
|
+
get?(name: string, scope?: unknown): unknown;
|
|
78
|
+
}
|
|
79
|
+
/** Construction seams. */
|
|
80
|
+
export interface SearchManyOptions {
|
|
81
|
+
/** Registration diagnostics. */
|
|
82
|
+
warn?: (message: string) => void;
|
|
83
|
+
}
|
|
84
|
+
/** A successful registration plus the exact way to undo it. */
|
|
85
|
+
export interface SearchManyRegistration {
|
|
86
|
+
name: string;
|
|
87
|
+
dispose: () => void;
|
|
88
|
+
}
|
|
89
|
+
/** Format the model-facing result text. */
|
|
90
|
+
export declare function formatSearchManyValue(value: SearchManyValue): string;
|
|
91
|
+
/**
|
|
92
|
+
* Build the tool definition the host registry accepts.
|
|
93
|
+
* @param name - `search_many`, or the fallback when that name is taken.
|
|
94
|
+
* @param runQuery - executes one search through the plugin's normal path.
|
|
95
|
+
* @returns a registry-ready definition.
|
|
96
|
+
*/
|
|
97
|
+
export declare function createSearchManyDefinition(name: string, runQuery: SearchManyRunner): SearchManyDefinition;
|
|
98
|
+
/**
|
|
99
|
+
* Pick the tool name this deployment can accept.
|
|
100
|
+
* @param tools - the live registry.
|
|
101
|
+
* @returns the preferred or fallback name.
|
|
102
|
+
*/
|
|
103
|
+
export declare function pickSearchManyToolName(tools: SearchManyRegistryLike): string;
|
|
104
|
+
/**
|
|
105
|
+
* Register the tool when a registry is available.
|
|
106
|
+
*
|
|
107
|
+
* Every failure path is contained: a duplicate name falls back, and any
|
|
108
|
+
* registry rejection is reported and swallowed so `web_search` keeps working.
|
|
109
|
+
* The returned registration owns the host disposer, so plugin disposal removes
|
|
110
|
+
* exactly the definition this call added.
|
|
111
|
+
*
|
|
112
|
+
* @param tools - the live registry.
|
|
113
|
+
* @param runQuery - executes one search through the plugin's normal path.
|
|
114
|
+
* @param options - diagnostics.
|
|
115
|
+
* @returns the registered name and its disposer, or `undefined` when skipped.
|
|
116
|
+
*/
|
|
117
|
+
export declare function registerSearchManyTool(tools: SearchManyRegistryLike, runQuery: SearchManyRunner, options?: SearchManyOptions): SearchManyRegistration | undefined;
|