@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +307 -46
- package/README.zh-CN.md +307 -46
- package/client.js +44 -3
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +48 -4
- package/lib/cache.js +134 -9
- package/lib/config.d.ts +25 -0
- package/lib/config.js +42 -0
- package/lib/engines.d.ts +41 -1
- package/lib/engines.js +33 -2
- package/lib/fetch.d.ts +33 -0
- package/lib/fetch.js +218 -6
- package/lib/index.d.ts +26 -0
- package/lib/index.js +113 -27
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/rank.js +4 -1
- package/lib/read-page.d.ts +136 -0
- package/lib/read-page.js +496 -0
- package/lib/render.d.ts +193 -0
- package/lib/render.js +354 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.js +19 -0
- package/lib/types.d.ts +13 -4
- package/package.json +1 -1
package/lib/fetch.js
CHANGED
|
@@ -8,23 +8,34 @@
|
|
|
8
8
|
* answers through the local tunnel in one to three seconds. Searching therefore
|
|
9
9
|
* found pages the agent could never read.
|
|
10
10
|
*
|
|
11
|
-
* This provider closes that loop with a
|
|
11
|
+
* This provider closes that loop with a four-step ladder:
|
|
12
12
|
*
|
|
13
13
|
* 1. direct, which is what the shipped provider already does and stays first so
|
|
14
14
|
* an open network pays no tunnel round trip;
|
|
15
15
|
* 2. the tunnel, using the same proxy resolution the search side uses (explicit
|
|
16
16
|
* option, then the environment, then the macOS system proxy);
|
|
17
|
-
* 3.
|
|
17
|
+
* 3. an optional headless-browser render, for a page whose bytes are a shell
|
|
18
|
+
* and whose text only appears after its scripts run. It runs before the
|
|
19
|
+
* extraction endpoint because it is local (no credential, no third-party
|
|
20
|
+
* request) and because `renderJs: 'always'` must actually mean a browser;
|
|
21
|
+
* 4. AnySearch's extraction endpoint, when a key is configured — it renders the
|
|
18
22
|
* page server-side, so it also rescues a page that answers 403 to a bare
|
|
19
|
-
* request.
|
|
23
|
+
* request and remains the last resort when no browser is installed.
|
|
24
|
+
*
|
|
25
|
+
* Playwright is not a dependency and is never imported at module load: the
|
|
26
|
+
* renderer is discovered by a guarded dynamic `import()` and its absence is
|
|
27
|
+
* recorded in the result, never thrown.
|
|
20
28
|
*
|
|
21
29
|
* A non-2xx response is a result, not a failure, and a failure on one rung is
|
|
22
30
|
* not fatal: the ladder only throws when every rung is unavailable or failed.
|
|
31
|
+
* A render failure specifically keeps the earlier rung's page.
|
|
23
32
|
*
|
|
24
33
|
* @module @fangchenghao/dsh-web-search/fetch
|
|
25
34
|
*/
|
|
26
35
|
import { SearchError } from "./errors.js";
|
|
36
|
+
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
|
|
27
37
|
import { proxyGet, resolveProxy } from "./proxy.js";
|
|
38
|
+
import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
|
|
28
39
|
/** Default user agent; kept in step with the search side. */
|
|
29
40
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
|
|
30
41
|
/** One mebibyte: enough for a documentation page, cheap enough to hold. */
|
|
@@ -63,6 +74,16 @@ function deadline(timeoutMs, signal) {
|
|
|
63
74
|
const timer = AbortSignal.timeout(timeoutMs);
|
|
64
75
|
return signal === undefined ? timer : AbortSignal.any([signal, timer]);
|
|
65
76
|
}
|
|
77
|
+
/**
|
|
78
|
+
* Turn a cancelled request into a rejection instead of a partial result.
|
|
79
|
+
* @param signal - the caller signal that is already aborted.
|
|
80
|
+
*/
|
|
81
|
+
function abortFetch(signal) {
|
|
82
|
+
const reason = signal.reason;
|
|
83
|
+
if (reason instanceof Error)
|
|
84
|
+
throw reason;
|
|
85
|
+
throw new DOMException('The fetch was aborted', 'AbortError');
|
|
86
|
+
}
|
|
66
87
|
/** Direct request used by the first rung. */
|
|
67
88
|
async function directFetch(url, timeoutMs, signal, userAgent, maxBytes) {
|
|
68
89
|
const response = await fetch(url, {
|
|
@@ -120,12 +141,78 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
120
141
|
tunnel: tunnelFetch,
|
|
121
142
|
extract: extractFetch,
|
|
122
143
|
detectProxy: (explicit) => resolveProxy(explicit),
|
|
144
|
+
loadRenderer: loadPlaywrightRenderer,
|
|
145
|
+
render: renderWithPlaywright,
|
|
123
146
|
...overrides,
|
|
124
147
|
};
|
|
125
148
|
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
126
149
|
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
127
150
|
const userAgent = options.userAgent ?? USER_AGENT;
|
|
128
151
|
const apiKey = (options.anysearchApiKey ?? '').trim();
|
|
152
|
+
/**
|
|
153
|
+
* Resolve the two volatile Settings fields per request. Reading them here
|
|
154
|
+
* rather than at construction is what lets a saved edit apply without a
|
|
155
|
+
* remount; caching them made the Settings page lie.
|
|
156
|
+
*/
|
|
157
|
+
const liveFormat = () => typeof options.format === 'function' ? options.format() : (options.format ?? 'markdown');
|
|
158
|
+
const liveMaxLength = () => {
|
|
159
|
+
const raw = typeof options.maxLength === 'function' ? options.maxLength() : options.maxLength;
|
|
160
|
+
return raw ?? DEFAULT_FETCH_MAX_LENGTH;
|
|
161
|
+
};
|
|
162
|
+
/** `renderJs` is volatile too: read it per request, never at construction. */
|
|
163
|
+
const liveRenderMode = () => {
|
|
164
|
+
const raw = typeof options.renderJs === 'function' ? options.renderJs() : options.renderJs;
|
|
165
|
+
return raw === 'always' || raw === 'off' ? raw : 'auto';
|
|
166
|
+
};
|
|
167
|
+
const liveRenderTimeout = () => {
|
|
168
|
+
const raw = typeof options.renderTimeoutMs === 'function' ? options.renderTimeoutMs() : options.renderTimeoutMs;
|
|
169
|
+
return typeof raw === 'number' && Number.isFinite(raw) && raw >= 1
|
|
170
|
+
? Math.floor(raw)
|
|
171
|
+
: DEFAULT_RENDER_TIMEOUT_MS;
|
|
172
|
+
};
|
|
173
|
+
/**
|
|
174
|
+
* Load the renderer at most once per provider. A missing package is a
|
|
175
|
+
* supported deployment, and caching the verdict keeps a shell page from
|
|
176
|
+
* paying a failed module resolution on every fetch; installing Playwright
|
|
177
|
+
* later needs a plugin restart, which is the same rule the other imports
|
|
178
|
+
* follow.
|
|
179
|
+
*/
|
|
180
|
+
let rendererLoad;
|
|
181
|
+
const loadRendererOnce = () => {
|
|
182
|
+
if (rendererLoad === undefined) {
|
|
183
|
+
rendererLoad = Promise.resolve()
|
|
184
|
+
.then(() => deps.loadRenderer())
|
|
185
|
+
.catch((error) => ({
|
|
186
|
+
ok: false,
|
|
187
|
+
reason: redactRenderReason(`the renderer could not be loaded (${error instanceof Error ? error.name : 'unknown error'})`),
|
|
188
|
+
}));
|
|
189
|
+
}
|
|
190
|
+
return rendererLoad;
|
|
191
|
+
};
|
|
192
|
+
// Every rung funnels through here: an HTML body becomes Markdown (or plain
|
|
193
|
+
// text), a text body keeps its content, and both obey the character budget.
|
|
194
|
+
// The Host's own renderer then sees `kind: 'text'`, so it does not convert the
|
|
195
|
+
// same page a second time. The render report rides along as an optional field.
|
|
196
|
+
const finalize = (outcome, render) => {
|
|
197
|
+
const format = liveFormat();
|
|
198
|
+
const maxLength = liveMaxLength();
|
|
199
|
+
if (outcome.body.kind === 'html') {
|
|
200
|
+
const rendered = renderFetchText(outcome.body.content, { format, maxLength, baseUrl: outcome.url });
|
|
201
|
+
return {
|
|
202
|
+
...outcome,
|
|
203
|
+
body: { kind: 'text', content: rendered.content },
|
|
204
|
+
truncated: outcome.truncated || rendered.truncated,
|
|
205
|
+
render,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
const limited = truncateFetchText(outcome.body.content, maxLength);
|
|
209
|
+
return {
|
|
210
|
+
...outcome,
|
|
211
|
+
body: { kind: 'text', content: limited.content },
|
|
212
|
+
truncated: outcome.truncated || limited.truncated,
|
|
213
|
+
render,
|
|
214
|
+
};
|
|
215
|
+
};
|
|
129
216
|
return {
|
|
130
217
|
id: options.id ?? 'http-proxy',
|
|
131
218
|
available: () => true,
|
|
@@ -137,17 +224,112 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
137
224
|
const failures = [];
|
|
138
225
|
/** True once the caller cancelled; a cancellation must not be retried. */
|
|
139
226
|
const cancelled = () => signal !== undefined && signal.aborted;
|
|
227
|
+
/** Reject with the caller's own abort reason when they cancelled. */
|
|
228
|
+
const stopIfCancelled = () => {
|
|
229
|
+
if (signal !== undefined && signal.aborted)
|
|
230
|
+
abortFetch(signal);
|
|
231
|
+
};
|
|
232
|
+
/** Best page from a transport rung, kept if rendering cannot improve it. */
|
|
233
|
+
let fallback;
|
|
234
|
+
/** The last render decision, attached to whatever result wins. */
|
|
235
|
+
let lastReport;
|
|
236
|
+
const makeReport = (mode, attempted, applied, reason) => ({ mode, attempted, applied, reason: redactRenderReason(reason) });
|
|
237
|
+
/**
|
|
238
|
+
* Decide whether to render, and render when warranted.
|
|
239
|
+
*
|
|
240
|
+
* `done: false` means the ladder should continue to extraction while the
|
|
241
|
+
* candidate stays as the fallback. A render never throws its own failure
|
|
242
|
+
* into the ladder: only a caller cancellation, or an AbortError raised by
|
|
243
|
+
* the renderer itself, rejects.
|
|
244
|
+
*/
|
|
245
|
+
const renderStep = async (candidate) => {
|
|
246
|
+
const mode = liveRenderMode();
|
|
247
|
+
if (mode === 'off') {
|
|
248
|
+
return candidate === undefined
|
|
249
|
+
? { done: false, report: makeReport(mode, false, false, 'renderJs is off') }
|
|
250
|
+
: { done: true, outcome: candidate, report: makeReport(mode, false, false, 'renderJs is off') };
|
|
251
|
+
}
|
|
252
|
+
if (candidate === undefined && mode === 'auto') {
|
|
253
|
+
// `auto` needs a body to judge. When every transport failed there is
|
|
254
|
+
// no shell verdict to make, so it must not launch a browser blindly.
|
|
255
|
+
return { done: false, report: makeReport(mode, false, false, 'no page body to judge as a shell') };
|
|
256
|
+
}
|
|
257
|
+
if (candidate !== undefined && mode === 'auto') {
|
|
258
|
+
const verdict = detectShellPage(candidate);
|
|
259
|
+
if (!verdict.shell) {
|
|
260
|
+
return { done: true, outcome: candidate, report: makeReport(mode, false, false, verdict.reason) };
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
stopIfCancelled();
|
|
264
|
+
const loaded = await loadRendererOnce();
|
|
265
|
+
if (!loaded.ok) {
|
|
266
|
+
// Silent skip: no banner, no thrown error, just a recorded reason.
|
|
267
|
+
return { done: false, report: makeReport(mode, false, false, loaded.reason) };
|
|
268
|
+
}
|
|
269
|
+
try {
|
|
270
|
+
const rendered = await deps.render(loaded.renderer, target, liveRenderTimeout(), signal);
|
|
271
|
+
stopIfCancelled();
|
|
272
|
+
if (rendered.html.trim().length === 0) {
|
|
273
|
+
return {
|
|
274
|
+
done: false,
|
|
275
|
+
report: makeReport(mode, true, false, `${loaded.moduleName} returned an empty page`),
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
const { content, truncated } = clamp(rendered.html, maxBytes);
|
|
279
|
+
const outcome = {
|
|
280
|
+
url: rendered.url.length > 0 ? rendered.url : target,
|
|
281
|
+
statusCode: rendered.statusCode,
|
|
282
|
+
body: { kind: 'html', content },
|
|
283
|
+
truncated,
|
|
284
|
+
};
|
|
285
|
+
return {
|
|
286
|
+
done: true,
|
|
287
|
+
outcome,
|
|
288
|
+
report: makeReport(mode, true, true, `${loaded.moduleName} rendered ${readableTextLength(rendered.html)} readable characters`),
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
catch (error) {
|
|
292
|
+
stopIfCancelled();
|
|
293
|
+
const failure = describeRenderError(error, liveRenderTimeout());
|
|
294
|
+
if (failure.aborted) {
|
|
295
|
+
// Even without the caller's signal, the renderer's abort is a
|
|
296
|
+
// cancellation rather than a page to fall back to.
|
|
297
|
+
throw error instanceof Error ? error : new DOMException('The page render was aborted', 'AbortError');
|
|
298
|
+
}
|
|
299
|
+
// Timeout, launch failure, navigation failure: keep the earlier page.
|
|
300
|
+
return { done: false, report: makeReport(mode, true, false, failure.reason) };
|
|
301
|
+
}
|
|
302
|
+
};
|
|
303
|
+
/** Run the render step for a transport result; `undefined` means keep going. */
|
|
304
|
+
const settle = async (candidate) => {
|
|
305
|
+
const step = await renderStep(candidate);
|
|
306
|
+
lastReport = step.report;
|
|
307
|
+
if (step.done)
|
|
308
|
+
return step.outcome ?? candidate;
|
|
309
|
+
fallback = candidate;
|
|
310
|
+
failures.push(`render: ${step.report.reason}`);
|
|
311
|
+
stopIfCancelled();
|
|
312
|
+
return undefined;
|
|
313
|
+
};
|
|
314
|
+
/** Report for a result that never reached the render step. */
|
|
315
|
+
const reportFor = () => lastReport ?? makeReport(liveRenderMode(), false, false, 'the render rung did not run');
|
|
140
316
|
// A blocked host usually fails by timing out, so the direct rung gets a
|
|
141
317
|
// shorter deadline than the tunnel: spending the whole budget on a request
|
|
142
318
|
// that cannot work would leave the rung that can work with nothing.
|
|
319
|
+
let direct;
|
|
143
320
|
try {
|
|
144
|
-
|
|
321
|
+
direct = await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
|
|
145
322
|
}
|
|
146
323
|
catch (error) {
|
|
147
324
|
if (cancelled())
|
|
148
325
|
throw error;
|
|
149
326
|
failures.push(`direct: ${error instanceof Error ? error.message : String(error)}`);
|
|
150
327
|
}
|
|
328
|
+
if (direct !== undefined) {
|
|
329
|
+
const rendered = await settle(direct);
|
|
330
|
+
if (rendered !== undefined)
|
|
331
|
+
return finalize(rendered, reportFor());
|
|
332
|
+
}
|
|
151
333
|
let proxy;
|
|
152
334
|
if (options.useProxy !== false) {
|
|
153
335
|
try {
|
|
@@ -158,18 +340,40 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
158
340
|
}
|
|
159
341
|
}
|
|
160
342
|
if (proxy !== undefined && proxy.length > 0) {
|
|
343
|
+
let tunneled;
|
|
161
344
|
try {
|
|
162
|
-
|
|
345
|
+
tunneled = await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
|
|
163
346
|
}
|
|
164
347
|
catch (error) {
|
|
165
348
|
if (cancelled())
|
|
166
349
|
throw error;
|
|
167
350
|
failures.push(`tunnel: ${error instanceof Error ? error.message : String(error)}`);
|
|
168
351
|
}
|
|
352
|
+
if (tunneled !== undefined) {
|
|
353
|
+
const rendered = await settle(tunneled);
|
|
354
|
+
if (rendered !== undefined)
|
|
355
|
+
return finalize(rendered, reportFor());
|
|
356
|
+
}
|
|
357
|
+
}
|
|
358
|
+
// `always` is also a standalone rescue: a transport that returns 403 or
|
|
359
|
+
// times out may still answer a real browser, and there is no earlier page
|
|
360
|
+
// to fall back to in that branch.
|
|
361
|
+
if (fallback === undefined && liveRenderMode() === 'always') {
|
|
362
|
+
const step = await renderStep(undefined);
|
|
363
|
+
lastReport = step.report;
|
|
364
|
+
if (step.done && step.outcome !== undefined)
|
|
365
|
+
return finalize(step.outcome, step.report);
|
|
366
|
+
if (!step.done)
|
|
367
|
+
failures.push(`render: ${step.report.reason}`);
|
|
169
368
|
}
|
|
170
369
|
if (apiKey.length > 0) {
|
|
171
370
|
try {
|
|
172
|
-
|
|
371
|
+
const extracted = await deps.extract(target, apiKey, timeoutMs, signal);
|
|
372
|
+
// Say where the body came from when the browser was tried first.
|
|
373
|
+
const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
|
|
374
|
+
? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the result came from AnySearch extraction`)
|
|
375
|
+
: reportFor();
|
|
376
|
+
return finalize(extracted, report);
|
|
173
377
|
}
|
|
174
378
|
catch (error) {
|
|
175
379
|
if (cancelled())
|
|
@@ -177,6 +381,14 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
177
381
|
failures.push(`extract: ${error instanceof Error ? error.message : String(error)}`);
|
|
178
382
|
}
|
|
179
383
|
}
|
|
384
|
+
// A failed render is not a failed fetch: hand back the page an earlier
|
|
385
|
+
// rung already produced, with the reason attached.
|
|
386
|
+
if (fallback !== undefined) {
|
|
387
|
+
const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
|
|
388
|
+
? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the earlier rung's result was kept`)
|
|
389
|
+
: reportFor();
|
|
390
|
+
return finalize(fallback, report);
|
|
391
|
+
}
|
|
180
392
|
throw new SearchError(`cannot fetch ${target} (${failures.join('; ')})`, 'SEARCH_NETWORK', true);
|
|
181
393
|
},
|
|
182
394
|
};
|
package/lib/index.d.ts
CHANGED
|
@@ -39,6 +39,13 @@ export interface PluginContext {
|
|
|
39
39
|
get?: (name: string) => unknown;
|
|
40
40
|
/** Fallback for tests and hosts that expose the registry as a property. */
|
|
41
41
|
tools?: unknown;
|
|
42
|
+
/**
|
|
43
|
+
* Cordis lifecycle hook. `effect` is the canonical registration; `on` is the
|
|
44
|
+
* older/test fallback. Both let the tool registration and the cache release
|
|
45
|
+
* their resources when the plugin is disposed.
|
|
46
|
+
*/
|
|
47
|
+
effect?: (execute: () => (() => void) | void) => unknown;
|
|
48
|
+
on?: (event: string, listener: () => void) => unknown;
|
|
42
49
|
}
|
|
43
50
|
/** Configuration as the loader hands it over, already schema-validated. */
|
|
44
51
|
export type ApplyConfig = PluginConfig;
|
|
@@ -65,6 +72,10 @@ declare const _default: typeof apply & {
|
|
|
65
72
|
Config: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<NoInfer<{
|
|
66
73
|
hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
67
74
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
75
|
+
fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
76
|
+
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
77
|
+
renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
78
|
+
renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
68
79
|
wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
69
80
|
proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
70
81
|
limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -76,6 +87,8 @@ declare const _default: typeof apply & {
|
|
|
76
87
|
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
77
88
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
78
89
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
90
|
+
cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
91
|
+
cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
79
92
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
80
93
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
81
94
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -86,6 +99,10 @@ declare const _default: typeof apply & {
|
|
|
86
99
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
87
100
|
hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
88
101
|
timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
102
|
+
fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
103
|
+
fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
104
|
+
renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
105
|
+
renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
89
106
|
wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
90
107
|
proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
91
108
|
limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -97,6 +114,8 @@ declare const _default: typeof apply & {
|
|
|
97
114
|
timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
98
115
|
cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
99
116
|
cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
117
|
+
cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
|
|
118
|
+
cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
100
119
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
101
120
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
102
121
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
@@ -110,6 +129,9 @@ declare const _default: typeof apply & {
|
|
|
110
129
|
export default _default;
|
|
111
130
|
export { HOSTS, search } from './engines.ts';
|
|
112
131
|
export { SearchCache, cacheKey } from './cache.ts';
|
|
132
|
+
export type { CachePersistOptions, SearchCacheOptions } from './cache.ts';
|
|
133
|
+
export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from './cache-persist.ts';
|
|
134
|
+
export type { CacheFileIo, CacheWriteOptions, PersistedEntry, SerializedCache } from './cache-persist.ts';
|
|
113
135
|
export { EngineHealth } from './health.ts';
|
|
114
136
|
export { SearchError, parseRetryAfter } from './errors.ts';
|
|
115
137
|
export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from './parsers.ts';
|
|
@@ -120,4 +142,8 @@ export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, p
|
|
|
120
142
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from './domain-tool.ts';
|
|
121
143
|
export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions } from './verticals.ts';
|
|
122
144
|
export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
|
|
145
|
+
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
|
|
146
|
+
export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
|
|
147
|
+
export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from './read-page.ts';
|
|
148
|
+
export type { ReadPageDefinition, ReadPageOptions, ReadPageRegistration, ReadPageRegistryLike, ReadPageSettings, ReadPageValue, } from './read-page.ts';
|
|
123
149
|
export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
|
package/lib/index.js
CHANGED
|
@@ -15,6 +15,8 @@ import { SearchCache } from "./cache.js";
|
|
|
15
15
|
import { EngineHealth } from "./health.js";
|
|
16
16
|
import { createFetchProvider } from "./fetch.js";
|
|
17
17
|
import { registerDomainTool } from "./domain-tool.js";
|
|
18
|
+
import { registerSearchManyTool } from "./search-many.js";
|
|
19
|
+
import { registerReadPageTool } from "./read-page.js";
|
|
18
20
|
export const name = 'dsh-web-search';
|
|
19
21
|
/** The `ctx.web` service this plugin registers a search provider with. */
|
|
20
22
|
export const inject = ['web'];
|
|
@@ -53,6 +55,26 @@ function warnTool(message) {
|
|
|
53
55
|
// Logging must never be the reason a plugin fails to mount.
|
|
54
56
|
}
|
|
55
57
|
}
|
|
58
|
+
/**
|
|
59
|
+
* Register a cleanup with the plugin lifecycle.
|
|
60
|
+
*
|
|
61
|
+
* `ctx.effect(() => cleanup)` is Cordis's ownership primitive; a host that
|
|
62
|
+
* predates it (or a test double) may only expose a `dispose` event, so that is
|
|
63
|
+
* the fallback. Losing the hook must not break mounting.
|
|
64
|
+
*/
|
|
65
|
+
function onDispose(ctx, cleanup) {
|
|
66
|
+
try {
|
|
67
|
+
if (typeof ctx.effect === 'function') {
|
|
68
|
+
ctx.effect(() => cleanup);
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
if (typeof ctx.on === 'function')
|
|
72
|
+
ctx.on('dispose', cleanup);
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
warnTool(`dispose hook skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
56
78
|
/**
|
|
57
79
|
* Register the search provider.
|
|
58
80
|
* @param ctx - plugin context carrying the `web` service.
|
|
@@ -66,15 +88,63 @@ export function apply(ctx, config = {}) {
|
|
|
66
88
|
// One cache and one health tracker per mounted plugin, so a config change
|
|
67
89
|
// through a restart starts from a clean slate instead of inheriting the
|
|
68
90
|
// cooldowns of a previous instance.
|
|
69
|
-
const cache = new SearchCache({
|
|
91
|
+
const cache = new SearchCache({
|
|
92
|
+
// Read through on every use: a Settings edit must take effect without a
|
|
93
|
+
// remount, which is the whole point of marking these fields volatile.
|
|
94
|
+
ttlMs: () => resolveConfig(section, undefined).cacheTtlMs,
|
|
95
|
+
maxEntries: () => resolveConfig(section, undefined).cacheEntries,
|
|
96
|
+
...(mounted.cachePersist
|
|
97
|
+
? { persist: { ...(mounted.cacheFile === undefined ? {} : { file: mounted.cacheFile }) } }
|
|
98
|
+
: {}),
|
|
99
|
+
});
|
|
70
100
|
const health = new EngineHealth({ baseCooldownMs: mounted.baseCooldownMs, maxCooldownMs: mounted.maxCooldownMs });
|
|
101
|
+
// The disk snapshot is flushed at plugin disposal; the cache also flushes on
|
|
102
|
+
// its own exit hook, so an abrupt process end keeps the newest state.
|
|
103
|
+
onDispose(ctx, () => cache.close());
|
|
104
|
+
/**
|
|
105
|
+
* Run one query through the same engine path as `web_search`.
|
|
106
|
+
*
|
|
107
|
+
* Reading `section` on every call is what makes a Settings commit visible
|
|
108
|
+
* without a restart. The configured limit is the hard ceiling: a caller may
|
|
109
|
+
* ask for fewer results, never for more.
|
|
110
|
+
*/
|
|
111
|
+
const runQuery = async (query, requested, signal) => {
|
|
112
|
+
const resolved = resolveConfig(section, undefined);
|
|
113
|
+
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
114
|
+
throw new Error('web search requires a non-empty query');
|
|
115
|
+
}
|
|
116
|
+
if (signal?.aborted === true)
|
|
117
|
+
throw new Error('web search was cancelled');
|
|
118
|
+
const asked = typeof requested === 'number' && Number.isInteger(requested) && requested > 0
|
|
119
|
+
? requested
|
|
120
|
+
: undefined;
|
|
121
|
+
const limit = Math.min(asked ?? resolved.limit, resolved.limit);
|
|
122
|
+
const { sources, failures } = await search(query, signal, {
|
|
123
|
+
limit,
|
|
124
|
+
wait: resolved.wait,
|
|
125
|
+
timeRange: resolved.timeRange,
|
|
126
|
+
timeoutMs: resolved.timeoutMs,
|
|
127
|
+
...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
|
|
128
|
+
...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
|
|
129
|
+
rank: resolved.rank,
|
|
130
|
+
...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
|
|
131
|
+
...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
|
|
132
|
+
cache,
|
|
133
|
+
health,
|
|
134
|
+
...(signal === undefined ? {} : { signal }),
|
|
135
|
+
});
|
|
136
|
+
return {
|
|
137
|
+
sources,
|
|
138
|
+
failures,
|
|
139
|
+
truncated: asked !== undefined && asked > resolved.limit,
|
|
140
|
+
};
|
|
141
|
+
};
|
|
71
142
|
ctx.web.registerSearchProvider({
|
|
72
143
|
id: PROVIDER_ID,
|
|
73
144
|
available: () => true,
|
|
74
145
|
async search(request, signal) {
|
|
75
146
|
// A Settings commit only mutates the existing references; reading the
|
|
76
|
-
// section
|
|
77
|
-
const resolved = resolveConfig(section, undefined);
|
|
147
|
+
// section inside `runQuery` is what makes the edit apply without a restart.
|
|
78
148
|
const query = request.query;
|
|
79
149
|
if (typeof query !== 'string' || query.trim().length === 0) {
|
|
80
150
|
throw new Error('web search requires a non-empty query');
|
|
@@ -84,27 +154,11 @@ export function apply(ctx, config = {}) {
|
|
|
84
154
|
const asked = typeof request.maxResults === 'number' && Number.isInteger(request.maxResults) && request.maxResults > 0
|
|
85
155
|
? request.maxResults
|
|
86
156
|
: undefined;
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
wait: resolved.wait,
|
|
91
|
-
timeRange: resolved.timeRange,
|
|
92
|
-
timeoutMs: resolved.timeoutMs,
|
|
93
|
-
...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
|
|
94
|
-
...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
|
|
95
|
-
rank: resolved.rank,
|
|
96
|
-
...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
|
|
97
|
-
...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
|
|
98
|
-
cache,
|
|
99
|
-
health,
|
|
100
|
-
...(signal === undefined ? {} : { signal }),
|
|
101
|
-
});
|
|
102
|
-
if (sources.length === 0) {
|
|
103
|
-
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${failures.join('; ')})`);
|
|
157
|
+
const outcome = await runQuery(query, asked, signal);
|
|
158
|
+
if (outcome.sources.length === 0) {
|
|
159
|
+
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
|
|
104
160
|
}
|
|
105
|
-
|
|
106
|
-
return { sources, truncated: false };
|
|
107
|
-
return { sources: sources.slice(0, limit), truncated: true };
|
|
161
|
+
return { sources: outcome.sources, truncated: outcome.truncated };
|
|
108
162
|
},
|
|
109
163
|
});
|
|
110
164
|
// The same tunnel the search side resolves, offered to `web_fetch`: a search
|
|
@@ -112,12 +166,17 @@ export function apply(ctx, config = {}) {
|
|
|
112
166
|
ctx.web.registerFetchProvider(createFetchProvider({
|
|
113
167
|
id: FETCH_PROVIDER_ID,
|
|
114
168
|
timeoutMs: mounted.timeoutMs,
|
|
169
|
+
format: () => resolveConfig(section, undefined).fetchFormat,
|
|
170
|
+
maxLength: () => resolveConfig(section, undefined).fetchMaxLength,
|
|
171
|
+
renderJs: () => resolveConfig(section, undefined).renderJs,
|
|
172
|
+
renderTimeoutMs: () => resolveConfig(section, undefined).renderTimeoutMs,
|
|
115
173
|
...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
|
|
116
174
|
...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
|
|
117
175
|
}));
|
|
118
|
-
// Optional half: `domain_search`. Fail-safe by
|
|
119
|
-
// without `ctx.tools` still loads this plugin and
|
|
120
|
-
// registration path is the host's
|
|
176
|
+
// Optional half: `domain_search`, `search_many` and `read_page`. Fail-safe by
|
|
177
|
+
// construction — a deployment without `ctx.tools` still loads this plugin and
|
|
178
|
+
// keeps `web_search`; the registration path is the host's
|
|
179
|
+
// `ToolRuntime.register(definition)`.
|
|
121
180
|
try {
|
|
122
181
|
const tools = readToolRegistry(ctx);
|
|
123
182
|
if (tools !== undefined) {
|
|
@@ -131,10 +190,34 @@ export function apply(ctx, config = {}) {
|
|
|
131
190
|
};
|
|
132
191
|
return settings;
|
|
133
192
|
}, { warn: warnTool });
|
|
193
|
+
// Batch tool: same optional-registry path, same fail-safe mounting. The
|
|
194
|
+
// runner is the `web_search` path, so both tools share lookups and cache.
|
|
195
|
+
const batch = registerSearchManyTool(tools, (query, limit, signal) => runQuery(query, limit, signal), { warn: warnTool });
|
|
196
|
+
if (batch !== undefined) {
|
|
197
|
+
onDispose(ctx, batch.dispose);
|
|
198
|
+
}
|
|
199
|
+
// Resumable reading: `web_fetch` cannot continue from an offset because
|
|
200
|
+
// the host request carries only `url`, but a plugin-owned tool can. It
|
|
201
|
+
// reuses the same fetch ladder and the same live Settings readers.
|
|
202
|
+
const paged = registerReadPageTool(tools, () => {
|
|
203
|
+
const live = resolveConfig(section, undefined);
|
|
204
|
+
return {
|
|
205
|
+
maxLength: live.fetchMaxLength,
|
|
206
|
+
format: live.fetchFormat,
|
|
207
|
+
timeoutMs: live.timeoutMs,
|
|
208
|
+
renderJs: live.renderJs,
|
|
209
|
+
renderTimeoutMs: live.renderTimeoutMs,
|
|
210
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
211
|
+
...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
|
|
212
|
+
};
|
|
213
|
+
}, { warn: warnTool });
|
|
214
|
+
if (paged !== undefined) {
|
|
215
|
+
onDispose(ctx, paged.dispose);
|
|
216
|
+
}
|
|
134
217
|
}
|
|
135
218
|
}
|
|
136
219
|
catch (error) {
|
|
137
|
-
warnTool(`
|
|
220
|
+
warnTool(`optional tool registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
|
138
221
|
}
|
|
139
222
|
}
|
|
140
223
|
export { Config, resolveConfig } from "./config.js";
|
|
@@ -154,6 +237,7 @@ export { Config, resolveConfig } from "./config.js";
|
|
|
154
237
|
export default Object.assign(apply, { Config, inject });
|
|
155
238
|
export { HOSTS, search } from "./engines.js";
|
|
156
239
|
export { SearchCache, cacheKey } from "./cache.js";
|
|
240
|
+
export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from "./cache-persist.js";
|
|
157
241
|
export { EngineHealth } from "./health.js";
|
|
158
242
|
export { SearchError, parseRetryAfter } from "./errors.js";
|
|
159
243
|
export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from "./parsers.js";
|
|
@@ -162,3 +246,5 @@ export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublis
|
|
|
162
246
|
export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
|
|
163
247
|
export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
|
|
164
248
|
export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
|
|
249
|
+
export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
|
|
250
|
+
export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from "./read-page.js";
|