@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/fetch.js CHANGED
@@ -8,23 +8,34 @@
8
8
  * answers through the local tunnel in one to three seconds. Searching therefore
9
9
  * found pages the agent could never read.
10
10
  *
11
- * This provider closes that loop with a three-step ladder:
11
+ * This provider closes that loop with a four-step ladder:
12
12
  *
13
13
  * 1. direct, which is what the shipped provider already does and stays first so
14
14
  * an open network pays no tunnel round trip;
15
15
  * 2. the tunnel, using the same proxy resolution the search side uses (explicit
16
16
  * option, then the environment, then the macOS system proxy);
17
- * 3. AnySearch's extraction endpoint, when a key is configured — it renders the
17
+ * 3. an optional headless-browser render, for a page whose bytes are a shell
18
+ * and whose text only appears after its scripts run. It runs before the
19
+ * extraction endpoint because it is local (no credential, no third-party
20
+ * request) and because `renderJs: 'always'` must actually mean a browser;
21
+ * 4. AnySearch's extraction endpoint, when a key is configured — it renders the
18
22
  * page server-side, so it also rescues a page that answers 403 to a bare
19
- * request.
23
+ * request and remains the last resort when no browser is installed.
24
+ *
25
+ * Playwright is not a dependency and is never imported at module load: the
26
+ * renderer is discovered by a guarded dynamic `import()` and its absence is
27
+ * recorded in the result, never thrown.
20
28
  *
21
29
  * A non-2xx response is a result, not a failure, and a failure on one rung is
22
30
  * not fatal: the ladder only throws when every rung is unavailable or failed.
31
+ * A render failure specifically keeps the earlier rung's page.
23
32
  *
24
33
  * @module @fangchenghao/dsh-web-search/fetch
25
34
  */
26
35
  import { SearchError } from "./errors.js";
36
+ import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
27
37
  import { proxyGet, resolveProxy } from "./proxy.js";
38
+ import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
28
39
  /** Default user agent; kept in step with the search side. */
29
40
  const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
30
41
  /** One mebibyte: enough for a documentation page, cheap enough to hold. */
@@ -63,6 +74,16 @@ function deadline(timeoutMs, signal) {
63
74
  const timer = AbortSignal.timeout(timeoutMs);
64
75
  return signal === undefined ? timer : AbortSignal.any([signal, timer]);
65
76
  }
77
+ /**
78
+ * Turn a cancelled request into a rejection instead of a partial result.
79
+ * @param signal - the caller signal that is already aborted.
80
+ */
81
+ function abortFetch(signal) {
82
+ const reason = signal.reason;
83
+ if (reason instanceof Error)
84
+ throw reason;
85
+ throw new DOMException('The fetch was aborted', 'AbortError');
86
+ }
66
87
  /** Direct request used by the first rung. */
67
88
  async function directFetch(url, timeoutMs, signal, userAgent, maxBytes) {
68
89
  const response = await fetch(url, {
@@ -120,12 +141,78 @@ export function createFetchProvider(options = {}, overrides = {}) {
120
141
  tunnel: tunnelFetch,
121
142
  extract: extractFetch,
122
143
  detectProxy: (explicit) => resolveProxy(explicit),
144
+ loadRenderer: loadPlaywrightRenderer,
145
+ render: renderWithPlaywright,
123
146
  ...overrides,
124
147
  };
125
148
  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
126
149
  const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
127
150
  const userAgent = options.userAgent ?? USER_AGENT;
128
151
  const apiKey = (options.anysearchApiKey ?? '').trim();
152
+ /**
153
+ * Resolve the two volatile Settings fields per request. Reading them here
154
+ * rather than at construction is what lets a saved edit apply without a
155
+ * remount; caching them made the Settings page lie.
156
+ */
157
+ const liveFormat = () => typeof options.format === 'function' ? options.format() : (options.format ?? 'markdown');
158
+ const liveMaxLength = () => {
159
+ const raw = typeof options.maxLength === 'function' ? options.maxLength() : options.maxLength;
160
+ return raw ?? DEFAULT_FETCH_MAX_LENGTH;
161
+ };
162
+ /** `renderJs` is volatile too: read it per request, never at construction. */
163
+ const liveRenderMode = () => {
164
+ const raw = typeof options.renderJs === 'function' ? options.renderJs() : options.renderJs;
165
+ return raw === 'always' || raw === 'off' ? raw : 'auto';
166
+ };
167
+ const liveRenderTimeout = () => {
168
+ const raw = typeof options.renderTimeoutMs === 'function' ? options.renderTimeoutMs() : options.renderTimeoutMs;
169
+ return typeof raw === 'number' && Number.isFinite(raw) && raw >= 1
170
+ ? Math.floor(raw)
171
+ : DEFAULT_RENDER_TIMEOUT_MS;
172
+ };
173
+ /**
174
+ * Load the renderer at most once per provider. A missing package is a
175
+ * supported deployment, and caching the verdict keeps a shell page from
176
+ * paying a failed module resolution on every fetch; installing Playwright
177
+ * later needs a plugin restart, which is the same rule the other imports
178
+ * follow.
179
+ */
180
+ let rendererLoad;
181
+ const loadRendererOnce = () => {
182
+ if (rendererLoad === undefined) {
183
+ rendererLoad = Promise.resolve()
184
+ .then(() => deps.loadRenderer())
185
+ .catch((error) => ({
186
+ ok: false,
187
+ reason: redactRenderReason(`the renderer could not be loaded (${error instanceof Error ? error.name : 'unknown error'})`),
188
+ }));
189
+ }
190
+ return rendererLoad;
191
+ };
192
+ // Every rung funnels through here: an HTML body becomes Markdown (or plain
193
+ // text), a text body keeps its content, and both obey the character budget.
194
+ // The Host's own renderer then sees `kind: 'text'`, so it does not convert the
195
+ // same page a second time. The render report rides along as an optional field.
196
+ const finalize = (outcome, render) => {
197
+ const format = liveFormat();
198
+ const maxLength = liveMaxLength();
199
+ if (outcome.body.kind === 'html') {
200
+ const rendered = renderFetchText(outcome.body.content, { format, maxLength, baseUrl: outcome.url });
201
+ return {
202
+ ...outcome,
203
+ body: { kind: 'text', content: rendered.content },
204
+ truncated: outcome.truncated || rendered.truncated,
205
+ render,
206
+ };
207
+ }
208
+ const limited = truncateFetchText(outcome.body.content, maxLength);
209
+ return {
210
+ ...outcome,
211
+ body: { kind: 'text', content: limited.content },
212
+ truncated: outcome.truncated || limited.truncated,
213
+ render,
214
+ };
215
+ };
129
216
  return {
130
217
  id: options.id ?? 'http-proxy',
131
218
  available: () => true,
@@ -137,17 +224,112 @@ export function createFetchProvider(options = {}, overrides = {}) {
137
224
  const failures = [];
138
225
  /** True once the caller cancelled; a cancellation must not be retried. */
139
226
  const cancelled = () => signal !== undefined && signal.aborted;
227
+ /** Reject with the caller's own abort reason when they cancelled. */
228
+ const stopIfCancelled = () => {
229
+ if (signal !== undefined && signal.aborted)
230
+ abortFetch(signal);
231
+ };
232
+ /** Best page from a transport rung, kept if rendering cannot improve it. */
233
+ let fallback;
234
+ /** The last render decision, attached to whatever result wins. */
235
+ let lastReport;
236
+ const makeReport = (mode, attempted, applied, reason) => ({ mode, attempted, applied, reason: redactRenderReason(reason) });
237
+ /**
238
+ * Decide whether to render, and render when warranted.
239
+ *
240
+ * `done: false` means the ladder should continue to extraction while the
241
+ * candidate stays as the fallback. A render never throws its own failure
242
+ * into the ladder: only a caller cancellation, or an AbortError raised by
243
+ * the renderer itself, rejects.
244
+ */
245
+ const renderStep = async (candidate) => {
246
+ const mode = liveRenderMode();
247
+ if (mode === 'off') {
248
+ return candidate === undefined
249
+ ? { done: false, report: makeReport(mode, false, false, 'renderJs is off') }
250
+ : { done: true, outcome: candidate, report: makeReport(mode, false, false, 'renderJs is off') };
251
+ }
252
+ if (candidate === undefined && mode === 'auto') {
253
+ // `auto` needs a body to judge. When every transport failed there is
254
+ // no shell verdict to make, so it must not launch a browser blindly.
255
+ return { done: false, report: makeReport(mode, false, false, 'no page body to judge as a shell') };
256
+ }
257
+ if (candidate !== undefined && mode === 'auto') {
258
+ const verdict = detectShellPage(candidate);
259
+ if (!verdict.shell) {
260
+ return { done: true, outcome: candidate, report: makeReport(mode, false, false, verdict.reason) };
261
+ }
262
+ }
263
+ stopIfCancelled();
264
+ const loaded = await loadRendererOnce();
265
+ if (!loaded.ok) {
266
+ // Silent skip: no banner, no thrown error, just a recorded reason.
267
+ return { done: false, report: makeReport(mode, false, false, loaded.reason) };
268
+ }
269
+ try {
270
+ const rendered = await deps.render(loaded.renderer, target, liveRenderTimeout(), signal);
271
+ stopIfCancelled();
272
+ if (rendered.html.trim().length === 0) {
273
+ return {
274
+ done: false,
275
+ report: makeReport(mode, true, false, `${loaded.moduleName} returned an empty page`),
276
+ };
277
+ }
278
+ const { content, truncated } = clamp(rendered.html, maxBytes);
279
+ const outcome = {
280
+ url: rendered.url.length > 0 ? rendered.url : target,
281
+ statusCode: rendered.statusCode,
282
+ body: { kind: 'html', content },
283
+ truncated,
284
+ };
285
+ return {
286
+ done: true,
287
+ outcome,
288
+ report: makeReport(mode, true, true, `${loaded.moduleName} rendered ${readableTextLength(rendered.html)} readable characters`),
289
+ };
290
+ }
291
+ catch (error) {
292
+ stopIfCancelled();
293
+ const failure = describeRenderError(error, liveRenderTimeout());
294
+ if (failure.aborted) {
295
+ // Even without the caller's signal, the renderer's abort is a
296
+ // cancellation rather than a page to fall back to.
297
+ throw error instanceof Error ? error : new DOMException('The page render was aborted', 'AbortError');
298
+ }
299
+ // Timeout, launch failure, navigation failure: keep the earlier page.
300
+ return { done: false, report: makeReport(mode, true, false, failure.reason) };
301
+ }
302
+ };
303
+ /** Run the render step for a transport result; `undefined` means keep going. */
304
+ const settle = async (candidate) => {
305
+ const step = await renderStep(candidate);
306
+ lastReport = step.report;
307
+ if (step.done)
308
+ return step.outcome ?? candidate;
309
+ fallback = candidate;
310
+ failures.push(`render: ${step.report.reason}`);
311
+ stopIfCancelled();
312
+ return undefined;
313
+ };
314
+ /** Report for a result that never reached the render step. */
315
+ const reportFor = () => lastReport ?? makeReport(liveRenderMode(), false, false, 'the render rung did not run');
140
316
  // A blocked host usually fails by timing out, so the direct rung gets a
141
317
  // shorter deadline than the tunnel: spending the whole budget on a request
142
318
  // that cannot work would leave the rung that can work with nothing.
319
+ let direct;
143
320
  try {
144
- return await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
321
+ direct = await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
145
322
  }
146
323
  catch (error) {
147
324
  if (cancelled())
148
325
  throw error;
149
326
  failures.push(`direct: ${error instanceof Error ? error.message : String(error)}`);
150
327
  }
328
+ if (direct !== undefined) {
329
+ const rendered = await settle(direct);
330
+ if (rendered !== undefined)
331
+ return finalize(rendered, reportFor());
332
+ }
151
333
  let proxy;
152
334
  if (options.useProxy !== false) {
153
335
  try {
@@ -158,18 +340,40 @@ export function createFetchProvider(options = {}, overrides = {}) {
158
340
  }
159
341
  }
160
342
  if (proxy !== undefined && proxy.length > 0) {
343
+ let tunneled;
161
344
  try {
162
- return await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
345
+ tunneled = await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
163
346
  }
164
347
  catch (error) {
165
348
  if (cancelled())
166
349
  throw error;
167
350
  failures.push(`tunnel: ${error instanceof Error ? error.message : String(error)}`);
168
351
  }
352
+ if (tunneled !== undefined) {
353
+ const rendered = await settle(tunneled);
354
+ if (rendered !== undefined)
355
+ return finalize(rendered, reportFor());
356
+ }
357
+ }
358
+ // `always` is also a standalone rescue: a transport that returns 403 or
359
+ // times out may still answer a real browser, and there is no earlier page
360
+ // to fall back to in that branch.
361
+ if (fallback === undefined && liveRenderMode() === 'always') {
362
+ const step = await renderStep(undefined);
363
+ lastReport = step.report;
364
+ if (step.done && step.outcome !== undefined)
365
+ return finalize(step.outcome, step.report);
366
+ if (!step.done)
367
+ failures.push(`render: ${step.report.reason}`);
169
368
  }
170
369
  if (apiKey.length > 0) {
171
370
  try {
172
- return await deps.extract(target, apiKey, timeoutMs, signal);
371
+ const extracted = await deps.extract(target, apiKey, timeoutMs, signal);
372
+ // Say where the body came from when the browser was tried first.
373
+ const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
374
+ ? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the result came from AnySearch extraction`)
375
+ : reportFor();
376
+ return finalize(extracted, report);
173
377
  }
174
378
  catch (error) {
175
379
  if (cancelled())
@@ -177,6 +381,14 @@ export function createFetchProvider(options = {}, overrides = {}) {
177
381
  failures.push(`extract: ${error instanceof Error ? error.message : String(error)}`);
178
382
  }
179
383
  }
384
+ // A failed render is not a failed fetch: hand back the page an earlier
385
+ // rung already produced, with the reason attached.
386
+ if (fallback !== undefined) {
387
+ const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
388
+ ? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the earlier rung's result was kept`)
389
+ : reportFor();
390
+ return finalize(fallback, report);
391
+ }
180
392
  throw new SearchError(`cannot fetch ${target} (${failures.join('; ')})`, 'SEARCH_NETWORK', true);
181
393
  },
182
394
  };
package/lib/index.d.ts CHANGED
@@ -39,6 +39,13 @@ export interface PluginContext {
39
39
  get?: (name: string) => unknown;
40
40
  /** Fallback for tests and hosts that expose the registry as a property. */
41
41
  tools?: unknown;
42
+ /**
43
+ * Cordis lifecycle hook. `effect` is the canonical registration; `on` is the
44
+ * older/test fallback. Both let the tool registration and the cache release
45
+ * their resources when the plugin is disposed.
46
+ */
47
+ effect?: (execute: () => (() => void) | void) => unknown;
48
+ on?: (event: string, listener: () => void) => unknown;
42
49
  }
43
50
  /** Configuration as the loader hands it over, already schema-validated. */
44
51
  export type ApplyConfig = PluginConfig;
@@ -65,6 +72,10 @@ declare const _default: typeof apply & {
65
72
  Config: import("@deepseek-ai/schemastery").default<Schemastery.ObjectS<NoInfer<{
66
73
  hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
67
74
  timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
75
+ fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
76
+ fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
77
+ renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
78
+ renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
68
79
  wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
69
80
  proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
70
81
  limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -76,6 +87,8 @@ declare const _default: typeof apply & {
76
87
  timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
77
88
  cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
78
89
  cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
90
+ cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
91
+ cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
79
92
  anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
80
93
  anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
81
94
  baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -86,6 +99,10 @@ declare const _default: typeof apply & {
86
99
  }>>, Schemastery.ObjectT<NoInfer<{
87
100
  hosts: import("@deepseek-ai/schemastery").default<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
88
101
  timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
102
+ fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
103
+ fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
104
+ renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
105
+ renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
89
106
  wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
90
107
  proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
91
108
  limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -97,6 +114,8 @@ declare const _default: typeof apply & {
97
114
  timeRange: import("@deepseek-ai/schemastery").default<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
98
115
  cacheTtlMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
99
116
  cacheEntries: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
117
+ cachePersist: import("@deepseek-ai/schemastery").default<boolean, boolean, "volatile-defined">;
118
+ cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
100
119
  anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
101
120
  anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
102
121
  baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -110,6 +129,9 @@ declare const _default: typeof apply & {
110
129
  export default _default;
111
130
  export { HOSTS, search } from './engines.ts';
112
131
  export { SearchCache, cacheKey } from './cache.ts';
132
+ export type { CachePersistOptions, SearchCacheOptions } from './cache.ts';
133
+ export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from './cache-persist.ts';
134
+ export type { CacheFileIo, CacheWriteOptions, PersistedEntry, SerializedCache } from './cache-persist.ts';
113
135
  export { EngineHealth } from './health.ts';
114
136
  export { SearchError, parseRetryAfter } from './errors.ts';
115
137
  export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from './parsers.ts';
@@ -120,4 +142,8 @@ export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, p
120
142
  export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from './domain-tool.ts';
121
143
  export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions } from './verticals.ts';
122
144
  export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
145
+ export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
146
+ export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
147
+ export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from './read-page.ts';
148
+ export type { ReadPageDefinition, ReadPageOptions, ReadPageRegistration, ReadPageRegistryLike, ReadPageSettings, ReadPageValue, } from './read-page.ts';
123
149
  export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
package/lib/index.js CHANGED
@@ -15,6 +15,8 @@ import { SearchCache } from "./cache.js";
15
15
  import { EngineHealth } from "./health.js";
16
16
  import { createFetchProvider } from "./fetch.js";
17
17
  import { registerDomainTool } from "./domain-tool.js";
18
+ import { registerSearchManyTool } from "./search-many.js";
19
+ import { registerReadPageTool } from "./read-page.js";
18
20
  export const name = 'dsh-web-search';
19
21
  /** The `ctx.web` service this plugin registers a search provider with. */
20
22
  export const inject = ['web'];
@@ -53,6 +55,26 @@ function warnTool(message) {
53
55
  // Logging must never be the reason a plugin fails to mount.
54
56
  }
55
57
  }
58
+ /**
59
+ * Register a cleanup with the plugin lifecycle.
60
+ *
61
+ * `ctx.effect(() => cleanup)` is Cordis's ownership primitive; a host that
62
+ * predates it (or a test double) may only expose a `dispose` event, so that is
63
+ * the fallback. Losing the hook must not break mounting.
64
+ */
65
+ function onDispose(ctx, cleanup) {
66
+ try {
67
+ if (typeof ctx.effect === 'function') {
68
+ ctx.effect(() => cleanup);
69
+ return;
70
+ }
71
+ if (typeof ctx.on === 'function')
72
+ ctx.on('dispose', cleanup);
73
+ }
74
+ catch (error) {
75
+ warnTool(`dispose hook skipped: ${error instanceof Error ? error.message : String(error)}`);
76
+ }
77
+ }
56
78
  /**
57
79
  * Register the search provider.
58
80
  * @param ctx - plugin context carrying the `web` service.
@@ -66,15 +88,63 @@ export function apply(ctx, config = {}) {
66
88
  // One cache and one health tracker per mounted plugin, so a config change
67
89
  // through a restart starts from a clean slate instead of inheriting the
68
90
  // cooldowns of a previous instance.
69
- const cache = new SearchCache({ ttlMs: mounted.cacheTtlMs, maxEntries: mounted.cacheEntries });
91
+ const cache = new SearchCache({
92
+ // Read through on every use: a Settings edit must take effect without a
93
+ // remount, which is the whole point of marking these fields volatile.
94
+ ttlMs: () => resolveConfig(section, undefined).cacheTtlMs,
95
+ maxEntries: () => resolveConfig(section, undefined).cacheEntries,
96
+ ...(mounted.cachePersist
97
+ ? { persist: { ...(mounted.cacheFile === undefined ? {} : { file: mounted.cacheFile }) } }
98
+ : {}),
99
+ });
70
100
  const health = new EngineHealth({ baseCooldownMs: mounted.baseCooldownMs, maxCooldownMs: mounted.maxCooldownMs });
101
+ // The disk snapshot is flushed at plugin disposal; the cache also flushes on
102
+ // its own exit hook, so an abrupt process end keeps the newest state.
103
+ onDispose(ctx, () => cache.close());
104
+ /**
105
+ * Run one query through the same engine path as `web_search`.
106
+ *
107
+ * Reading `section` on every call is what makes a Settings commit visible
108
+ * without a restart. The configured limit is the hard ceiling: a caller may
109
+ * ask for fewer results, never for more.
110
+ */
111
+ const runQuery = async (query, requested, signal) => {
112
+ const resolved = resolveConfig(section, undefined);
113
+ if (typeof query !== 'string' || query.trim().length === 0) {
114
+ throw new Error('web search requires a non-empty query');
115
+ }
116
+ if (signal?.aborted === true)
117
+ throw new Error('web search was cancelled');
118
+ const asked = typeof requested === 'number' && Number.isInteger(requested) && requested > 0
119
+ ? requested
120
+ : undefined;
121
+ const limit = Math.min(asked ?? resolved.limit, resolved.limit);
122
+ const { sources, failures } = await search(query, signal, {
123
+ limit,
124
+ wait: resolved.wait,
125
+ timeRange: resolved.timeRange,
126
+ timeoutMs: resolved.timeoutMs,
127
+ ...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
128
+ ...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
129
+ rank: resolved.rank,
130
+ ...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
131
+ ...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
132
+ cache,
133
+ health,
134
+ ...(signal === undefined ? {} : { signal }),
135
+ });
136
+ return {
137
+ sources,
138
+ failures,
139
+ truncated: asked !== undefined && asked > resolved.limit,
140
+ };
141
+ };
71
142
  ctx.web.registerSearchProvider({
72
143
  id: PROVIDER_ID,
73
144
  available: () => true,
74
145
  async search(request, signal) {
75
146
  // A Settings commit only mutates the existing references; reading the
76
- // section here is what makes the edit apply without a restart.
77
- const resolved = resolveConfig(section, undefined);
147
+ // section inside `runQuery` is what makes the edit apply without a restart.
78
148
  const query = request.query;
79
149
  if (typeof query !== 'string' || query.trim().length === 0) {
80
150
  throw new Error('web search requires a non-empty query');
@@ -84,27 +154,11 @@ export function apply(ctx, config = {}) {
84
154
  const asked = typeof request.maxResults === 'number' && Number.isInteger(request.maxResults) && request.maxResults > 0
85
155
  ? request.maxResults
86
156
  : undefined;
87
- const limit = Math.min(asked ?? resolved.limit, resolved.limit);
88
- const { sources, failures } = await search(query, signal, {
89
- limit,
90
- wait: resolved.wait,
91
- timeRange: resolved.timeRange,
92
- timeoutMs: resolved.timeoutMs,
93
- ...(resolved.proxy === undefined ? {} : { proxy: resolved.proxy }),
94
- ...(resolved.hostIds === undefined ? {} : { hostIds: resolved.hostIds }),
95
- rank: resolved.rank,
96
- ...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
97
- ...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
98
- cache,
99
- health,
100
- ...(signal === undefined ? {} : { signal }),
101
- });
102
- if (sources.length === 0) {
103
- throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${failures.join('; ')})`);
157
+ const outcome = await runQuery(query, asked, signal);
158
+ if (outcome.sources.length === 0) {
159
+ throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
104
160
  }
105
- if (sources.length <= limit)
106
- return { sources, truncated: false };
107
- return { sources: sources.slice(0, limit), truncated: true };
161
+ return { sources: outcome.sources, truncated: outcome.truncated };
108
162
  },
109
163
  });
110
164
  // The same tunnel the search side resolves, offered to `web_fetch`: a search
@@ -112,12 +166,17 @@ export function apply(ctx, config = {}) {
112
166
  ctx.web.registerFetchProvider(createFetchProvider({
113
167
  id: FETCH_PROVIDER_ID,
114
168
  timeoutMs: mounted.timeoutMs,
169
+ format: () => resolveConfig(section, undefined).fetchFormat,
170
+ maxLength: () => resolveConfig(section, undefined).fetchMaxLength,
171
+ renderJs: () => resolveConfig(section, undefined).renderJs,
172
+ renderTimeoutMs: () => resolveConfig(section, undefined).renderTimeoutMs,
115
173
  ...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
116
174
  ...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
117
175
  }));
118
- // Optional half: `domain_search`. Fail-safe by construction — a deployment
119
- // without `ctx.tools` still loads this plugin and keeps `web_search`; the
120
- // registration path is the host's `ToolRuntime.register(definition)`.
176
+ // Optional half: `domain_search`, `search_many` and `read_page`. Fail-safe by
177
+ // construction — a deployment without `ctx.tools` still loads this plugin and
178
+ // keeps `web_search`; the registration path is the host's
179
+ // `ToolRuntime.register(definition)`.
121
180
  try {
122
181
  const tools = readToolRegistry(ctx);
123
182
  if (tools !== undefined) {
@@ -131,10 +190,34 @@ export function apply(ctx, config = {}) {
131
190
  };
132
191
  return settings;
133
192
  }, { warn: warnTool });
193
+ // Batch tool: same optional-registry path, same fail-safe mounting. The
194
+ // runner is the `web_search` path, so both tools share lookups and cache.
195
+ const batch = registerSearchManyTool(tools, (query, limit, signal) => runQuery(query, limit, signal), { warn: warnTool });
196
+ if (batch !== undefined) {
197
+ onDispose(ctx, batch.dispose);
198
+ }
199
+ // Resumable reading: `web_fetch` cannot continue from an offset because
200
+ // the host request carries only `url`, but a plugin-owned tool can. It
201
+ // reuses the same fetch ladder and the same live Settings readers.
202
+ const paged = registerReadPageTool(tools, () => {
203
+ const live = resolveConfig(section, undefined);
204
+ return {
205
+ maxLength: live.fetchMaxLength,
206
+ format: live.fetchFormat,
207
+ timeoutMs: live.timeoutMs,
208
+ renderJs: live.renderJs,
209
+ renderTimeoutMs: live.renderTimeoutMs,
210
+ ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
211
+ ...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
212
+ };
213
+ }, { warn: warnTool });
214
+ if (paged !== undefined) {
215
+ onDispose(ctx, paged.dispose);
216
+ }
134
217
  }
135
218
  }
136
219
  catch (error) {
137
- warnTool(`domain_search skipped: ${error instanceof Error ? error.message : String(error)}`);
220
+ warnTool(`optional tool registration skipped: ${error instanceof Error ? error.message : String(error)}`);
138
221
  }
139
222
  }
140
223
  export { Config, resolveConfig } from "./config.js";
@@ -154,6 +237,7 @@ export { Config, resolveConfig } from "./config.js";
154
237
  export default Object.assign(apply, { Config, inject });
155
238
  export { HOSTS, search } from "./engines.js";
156
239
  export { SearchCache, cacheKey } from "./cache.js";
240
+ export { CACHE_FILE_VERSION, DEFAULT_CACHE_FILE_MAX_BYTES, defaultCacheFile, resolveCacheFile, readCacheFile, serializeCache, writeCacheFile, } from "./cache-persist.js";
157
241
  export { EngineHealth } from "./health.js";
158
242
  export { SearchError, parseRetryAfter } from "./errors.js";
159
243
  export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySearch, echoesQuery, } from "./parsers.js";
@@ -162,3 +246,5 @@ export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublis
162
246
  export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
163
247
  export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
164
248
  export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
249
+ export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
250
+ export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from "./read-page.js";