@fffangchenghao/dsh-web-search 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/engines.d.ts CHANGED
@@ -11,7 +11,7 @@
11
11
  *
12
12
  * @module @fangchenghao/dsh-web-search/engines
13
13
  */
14
- import type { HostSpec, RankOptions, SearchOptions, SearchOutcome } from './types.ts';
14
+ import type { EngineAnswer, HostSpec, RankOptions, Region, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
15
15
  import type { SearchCache } from './cache.ts';
16
16
  import type { EngineHealth } from './health.ts';
17
17
  /**
@@ -34,6 +34,46 @@ export interface EngineRunOptions extends SearchOptions {
34
34
  cache?: SearchCache;
35
35
  /** Cooldown tracker, when one is configured. */
36
36
  health?: EngineHealth;
37
+ /**
38
+ * Read-only observer of the fusion inputs.
39
+ *
40
+ * Benchmark recorders use it to persist each engine's own list and rerun the
41
+ * current ranking offline. It never changes the run: the snapshot is a copy
42
+ * handed over after the fused list is final, and an absent hook costs one
43
+ * `undefined` check per run.
44
+ */
45
+ trace?: (snapshot: SearchTrace) => void;
46
+ }
47
+ /**
48
+ * What one search run handed to the ranking, captured for offline replay.
49
+ *
50
+ * `answers` are the exact per-engine lists that produced `sources`, in engine
51
+ * priority order and already narrowed by an active time window. Only the tier
52
+ * that actually supplied the fused list is present: when the primary tier
53
+ * answered, the fallback engines never ran. A cache hit has no fresh answers,
54
+ * so `answers` is empty and `cached` is `true`.
55
+ */
56
+ export interface SearchTrace {
57
+ /** Caller budget, after the engine layer's own clamp. */
58
+ limit: number;
59
+ /** Region that led this query. */
60
+ lead: Region;
61
+ /** Release policy used for the tier run. */
62
+ wait: WaitPolicy;
63
+ /** Active publication window. */
64
+ timeRange: TimeRange;
65
+ /** Ranking knobs, with the window folded in. */
66
+ rank: RankOptions;
67
+ /** Which path produced `sources`. */
68
+ tier: 'primary' | 'fallback' | 'none' | 'cache';
69
+ /** Per-engine lists that were fused, in engine priority order. */
70
+ answers: EngineAnswer[];
71
+ /** The fused list `search` returned. */
72
+ sources: Source[];
73
+ /** Failure strings accumulated by this run. */
74
+ failures: string[];
75
+ /** Whether the result came from the cache. */
76
+ cached: boolean;
37
77
  }
38
78
  /**
39
79
  * Search the web with every configured engine and fuse the answers.
package/lib/engines.js CHANGED
@@ -66,7 +66,9 @@ export const HOSTS = [
66
66
  },
67
67
  {
68
68
  id: 'so',
69
- tier: 'primary',
69
+ // 360 answers an anti-bot wall from this network and from the tunnel, so it
70
+ // is demoted: tried only when every primary engine came back empty.
71
+ tier: 'fallback',
70
72
  region: 'cn',
71
73
  build: (query) => `https://www.so.com/s?q=${encodeURIComponent(query)}`,
72
74
  parse: parseSo,
@@ -317,14 +319,31 @@ export async function search(query, signal, options) {
317
319
  // next identical query into a cache miss.
318
320
  const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
319
321
  const cached = key === undefined ? undefined : options.cache?.get(key);
320
- if (cached !== undefined)
322
+ if (cached !== undefined) {
323
+ options.trace?.({
324
+ limit,
325
+ lead,
326
+ wait,
327
+ timeRange,
328
+ rank,
329
+ tier: 'cache',
330
+ answers: [],
331
+ sources: cached,
332
+ failures: [...failures],
333
+ cached: true,
334
+ });
321
335
  return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
336
+ }
322
337
  const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
323
338
  const primary = ordered.filter((host) => host.tier === 'primary');
324
339
  const fallback = ordered.filter((host) => host.tier === 'fallback');
340
+ let tier = 'none';
325
341
  let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health));
342
+ if (answers.length > 0)
343
+ tier = 'primary';
326
344
  let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
327
345
  if (sources.length === 0 && fallback.length > 0) {
346
+ tier = 'fallback';
328
347
  answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health));
329
348
  sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
330
349
  }
@@ -332,5 +351,17 @@ export async function search(query, signal, options) {
332
351
  throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
333
352
  if (key !== undefined && sources.length > 0)
334
353
  options.cache?.set(key, sources);
354
+ options.trace?.({
355
+ limit,
356
+ lead,
357
+ wait,
358
+ timeRange,
359
+ rank,
360
+ tier,
361
+ answers,
362
+ sources,
363
+ failures: [...failures],
364
+ cached: false,
365
+ });
335
366
  return { sources, failures, ...(proxy === undefined ? {} : { proxy }) };
336
367
  }
package/lib/fetch.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import type { FetchFormat } from './markdown.ts';
2
+ import type { RenderMode, RenderReport, RenderedPage, RendererLoad, RendererModule } from './render.ts';
2
3
  /** One retrieval outcome, mirroring the Host's `WebFetchResult`. */
3
4
  export interface FetchOutcome {
4
5
  readonly url: string;
@@ -8,6 +9,13 @@ export interface FetchOutcome {
8
9
  readonly content: string;
9
10
  };
10
11
  readonly truncated: boolean;
12
+ /**
13
+ * How the optional render rung decided, when this provider produced the
14
+ * result. The Host ignores the extra field; it is here so a caller (and a
15
+ * test) can see why a shell page was or was not rendered, including that
16
+ * Playwright is absent.
17
+ */
18
+ readonly render?: RenderReport;
11
19
  }
12
20
  /** What `ctx.web.registerFetchProvider` expects. */
13
21
  export interface FetchProvider {
@@ -37,6 +45,14 @@ export interface FetchProviderOptions {
37
45
  /** Character budget for the returned text. */
38
46
  /** Character budget, or a reader for it. */
39
47
  maxLength?: number | (() => number);
48
+ /**
49
+ * Optional browser rendering: `auto` renders only a suspected shell page,
50
+ * `always` renders every fetch, `off` never does. A reader keeps a Settings
51
+ * edit live. Defaults to `auto`; without Playwright it silently does nothing.
52
+ */
53
+ renderJs?: RenderMode | (() => RenderMode);
54
+ /** Browser-render deadline, or a reader for it. Defaults to 15000 ms. */
55
+ renderTimeoutMs?: number | (() => number);
40
56
  /** User agent sent on every rung. */
41
57
  userAgent?: string;
42
58
  }
@@ -50,6 +66,16 @@ export interface FetchDependencies {
50
66
  extract: (url: string, apiKey: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<FetchOutcome>;
51
67
  /** Proxy discovery. */
52
68
  detectProxy: (explicit: string | undefined) => Promise<string | undefined>;
69
+ /**
70
+ * Load the optional renderer. The default dynamically imports Playwright and
71
+ * reports absence instead of throwing; tests inject a fake here.
72
+ */
73
+ loadRenderer: () => Promise<RendererLoad>;
74
+ /**
75
+ * Render one page with a loaded renderer. The default is the real
76
+ * headless-Chromium implementation; tests inject a fake browser.
77
+ */
78
+ render: (renderer: RendererModule, url: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<RenderedPage>;
53
79
  }
54
80
  /**
55
81
  * Build the fetch provider.
package/lib/fetch.js CHANGED
@@ -8,24 +8,34 @@
8
8
  * answers through the local tunnel in one to three seconds. Searching therefore
9
9
  * found pages the agent could never read.
10
10
  *
11
- * This provider closes that loop with a three-step ladder:
11
+ * This provider closes that loop with a four-step ladder:
12
12
  *
13
13
  * 1. direct, which is what the shipped provider already does and stays first so
14
14
  * an open network pays no tunnel round trip;
15
15
  * 2. the tunnel, using the same proxy resolution the search side uses (explicit
16
16
  * option, then the environment, then the macOS system proxy);
17
- * 3. AnySearch's extraction endpoint, when a key is configured — it renders the
17
+ * 3. an optional headless-browser render, for a page whose bytes are a shell
18
+ * and whose text only appears after its scripts run. It runs before the
19
+ * extraction endpoint because it is local (no credential, no third-party
20
+ * request) and because `renderJs: 'always'` must actually mean a browser;
21
+ * 4. AnySearch's extraction endpoint, when a key is configured — it renders the
18
22
  * page server-side, so it also rescues a page that answers 403 to a bare
19
- * request.
23
+ * request and remains the last resort when no browser is installed.
24
+ *
25
+ * Playwright is not a dependency and is never imported at module load: the
26
+ * renderer is discovered by a guarded dynamic `import()` and its absence is
27
+ * recorded in the result, never thrown.
20
28
  *
21
29
  * A non-2xx response is a result, not a failure, and a failure on one rung is
22
30
  * not fatal: the ladder only throws when every rung is unavailable or failed.
31
+ * A render failure specifically keeps the earlier rung's page.
23
32
  *
24
33
  * @module @fangchenghao/dsh-web-search/fetch
25
34
  */
26
35
  import { SearchError } from "./errors.js";
27
36
  import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
28
37
  import { proxyGet, resolveProxy } from "./proxy.js";
38
+ import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
29
39
  /** Default user agent; kept in step with the search side. */
30
40
  const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
31
41
  /** One mebibyte: enough for a documentation page, cheap enough to hold. */
@@ -64,6 +74,16 @@ function deadline(timeoutMs, signal) {
64
74
  const timer = AbortSignal.timeout(timeoutMs);
65
75
  return signal === undefined ? timer : AbortSignal.any([signal, timer]);
66
76
  }
77
+ /**
78
+ * Turn a cancelled request into a rejection instead of a partial result.
79
+ * @param signal - the caller signal that is already aborted.
80
+ */
81
+ function abortFetch(signal) {
82
+ const reason = signal.reason;
83
+ if (reason instanceof Error)
84
+ throw reason;
85
+ throw new DOMException('The fetch was aborted', 'AbortError');
86
+ }
67
87
  /** Direct request used by the first rung. */
68
88
  async function directFetch(url, timeoutMs, signal, userAgent, maxBytes) {
69
89
  const response = await fetch(url, {
@@ -121,6 +141,8 @@ export function createFetchProvider(options = {}, overrides = {}) {
121
141
  tunnel: tunnelFetch,
122
142
  extract: extractFetch,
123
143
  detectProxy: (explicit) => resolveProxy(explicit),
144
+ loadRenderer: loadPlaywrightRenderer,
145
+ render: renderWithPlaywright,
124
146
  ...overrides,
125
147
  };
126
148
  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
@@ -137,11 +159,41 @@ export function createFetchProvider(options = {}, overrides = {}) {
137
159
  const raw = typeof options.maxLength === 'function' ? options.maxLength() : options.maxLength;
138
160
  return raw ?? DEFAULT_FETCH_MAX_LENGTH;
139
161
  };
162
+ /** `renderJs` is volatile too: read it per request, never at construction. */
163
+ const liveRenderMode = () => {
164
+ const raw = typeof options.renderJs === 'function' ? options.renderJs() : options.renderJs;
165
+ return raw === 'always' || raw === 'off' ? raw : 'auto';
166
+ };
167
+ const liveRenderTimeout = () => {
168
+ const raw = typeof options.renderTimeoutMs === 'function' ? options.renderTimeoutMs() : options.renderTimeoutMs;
169
+ return typeof raw === 'number' && Number.isFinite(raw) && raw >= 1
170
+ ? Math.floor(raw)
171
+ : DEFAULT_RENDER_TIMEOUT_MS;
172
+ };
173
+ /**
174
+ * Load the renderer at most once per provider. A missing package is a
175
+ * supported deployment, and caching the verdict keeps a shell page from
176
+ * paying a failed module resolution on every fetch; installing Playwright
177
+ * later needs a plugin restart, which is the same rule the other imports
178
+ * follow.
179
+ */
180
+ let rendererLoad;
181
+ const loadRendererOnce = () => {
182
+ if (rendererLoad === undefined) {
183
+ rendererLoad = Promise.resolve()
184
+ .then(() => deps.loadRenderer())
185
+ .catch((error) => ({
186
+ ok: false,
187
+ reason: redactRenderReason(`the renderer could not be loaded (${error instanceof Error ? error.name : 'unknown error'})`),
188
+ }));
189
+ }
190
+ return rendererLoad;
191
+ };
140
192
  // Every rung funnels through here: an HTML body becomes Markdown (or plain
141
193
  // text), a text body keeps its content, and both obey the character budget.
142
194
  // The Host's own renderer then sees `kind: 'text'`, so it does not convert the
143
- // same page a second time.
144
- const finalize = (outcome) => {
195
+ // same page a second time. The render report rides along as an optional field.
196
+ const finalize = (outcome, render) => {
145
197
  const format = liveFormat();
146
198
  const maxLength = liveMaxLength();
147
199
  if (outcome.body.kind === 'html') {
@@ -150,6 +202,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
150
202
  ...outcome,
151
203
  body: { kind: 'text', content: rendered.content },
152
204
  truncated: outcome.truncated || rendered.truncated,
205
+ render,
153
206
  };
154
207
  }
155
208
  const limited = truncateFetchText(outcome.body.content, maxLength);
@@ -157,6 +210,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
157
210
  ...outcome,
158
211
  body: { kind: 'text', content: limited.content },
159
212
  truncated: outcome.truncated || limited.truncated,
213
+ render,
160
214
  };
161
215
  };
162
216
  return {
@@ -170,17 +224,112 @@ export function createFetchProvider(options = {}, overrides = {}) {
170
224
  const failures = [];
171
225
  /** True once the caller cancelled; a cancellation must not be retried. */
172
226
  const cancelled = () => signal !== undefined && signal.aborted;
227
+ /** Reject with the caller's own abort reason when they cancelled. */
228
+ const stopIfCancelled = () => {
229
+ if (signal !== undefined && signal.aborted)
230
+ abortFetch(signal);
231
+ };
232
+ /** Best page from a transport rung, kept if rendering cannot improve it. */
233
+ let fallback;
234
+ /** The last render decision, attached to whatever result wins. */
235
+ let lastReport;
236
+ const makeReport = (mode, attempted, applied, reason) => ({ mode, attempted, applied, reason: redactRenderReason(reason) });
237
+ /**
238
+ * Decide whether to render, and render when warranted.
239
+ *
240
+ * `done: false` means the ladder should continue to extraction while the
241
+ * candidate stays as the fallback. A render never throws its own failure
242
+ * into the ladder: only a caller cancellation, or an AbortError raised by
243
+ * the renderer itself, rejects.
244
+ */
245
+ const renderStep = async (candidate) => {
246
+ const mode = liveRenderMode();
247
+ if (mode === 'off') {
248
+ return candidate === undefined
249
+ ? { done: false, report: makeReport(mode, false, false, 'renderJs is off') }
250
+ : { done: true, outcome: candidate, report: makeReport(mode, false, false, 'renderJs is off') };
251
+ }
252
+ if (candidate === undefined && mode === 'auto') {
253
+ // `auto` needs a body to judge. When every transport failed there is
254
+ // no shell verdict to make, so it must not launch a browser blindly.
255
+ return { done: false, report: makeReport(mode, false, false, 'no page body to judge as a shell') };
256
+ }
257
+ if (candidate !== undefined && mode === 'auto') {
258
+ const verdict = detectShellPage(candidate);
259
+ if (!verdict.shell) {
260
+ return { done: true, outcome: candidate, report: makeReport(mode, false, false, verdict.reason) };
261
+ }
262
+ }
263
+ stopIfCancelled();
264
+ const loaded = await loadRendererOnce();
265
+ if (!loaded.ok) {
266
+ // Silent skip: no banner, no thrown error, just a recorded reason.
267
+ return { done: false, report: makeReport(mode, false, false, loaded.reason) };
268
+ }
269
+ try {
270
+ const rendered = await deps.render(loaded.renderer, target, liveRenderTimeout(), signal);
271
+ stopIfCancelled();
272
+ if (rendered.html.trim().length === 0) {
273
+ return {
274
+ done: false,
275
+ report: makeReport(mode, true, false, `${loaded.moduleName} returned an empty page`),
276
+ };
277
+ }
278
+ const { content, truncated } = clamp(rendered.html, maxBytes);
279
+ const outcome = {
280
+ url: rendered.url.length > 0 ? rendered.url : target,
281
+ statusCode: rendered.statusCode,
282
+ body: { kind: 'html', content },
283
+ truncated,
284
+ };
285
+ return {
286
+ done: true,
287
+ outcome,
288
+ report: makeReport(mode, true, true, `${loaded.moduleName} rendered ${readableTextLength(rendered.html)} readable characters`),
289
+ };
290
+ }
291
+ catch (error) {
292
+ stopIfCancelled();
293
+ const failure = describeRenderError(error, liveRenderTimeout());
294
+ if (failure.aborted) {
295
+ // Even without the caller's signal, the renderer's abort is a
296
+ // cancellation rather than a page to fall back to.
297
+ throw error instanceof Error ? error : new DOMException('The page render was aborted', 'AbortError');
298
+ }
299
+ // Timeout, launch failure, navigation failure: keep the earlier page.
300
+ return { done: false, report: makeReport(mode, true, false, failure.reason) };
301
+ }
302
+ };
303
+ /** Run the render step for a transport result; `undefined` means keep going. */
304
+ const settle = async (candidate) => {
305
+ const step = await renderStep(candidate);
306
+ lastReport = step.report;
307
+ if (step.done)
308
+ return step.outcome ?? candidate;
309
+ fallback = candidate;
310
+ failures.push(`render: ${step.report.reason}`);
311
+ stopIfCancelled();
312
+ return undefined;
313
+ };
314
+ /** Report for a result that never reached the render step. */
315
+ const reportFor = () => lastReport ?? makeReport(liveRenderMode(), false, false, 'the render rung did not run');
173
316
  // A blocked host usually fails by timing out, so the direct rung gets a
174
317
  // shorter deadline than the tunnel: spending the whole budget on a request
175
318
  // that cannot work would leave the rung that can work with nothing.
319
+ let direct;
176
320
  try {
177
- return finalize(await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes));
321
+ direct = await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
178
322
  }
179
323
  catch (error) {
180
324
  if (cancelled())
181
325
  throw error;
182
326
  failures.push(`direct: ${error instanceof Error ? error.message : String(error)}`);
183
327
  }
328
+ if (direct !== undefined) {
329
+ const rendered = await settle(direct);
330
+ if (rendered !== undefined)
331
+ return finalize(rendered, reportFor());
332
+ }
184
333
  let proxy;
185
334
  if (options.useProxy !== false) {
186
335
  try {
@@ -191,18 +340,40 @@ export function createFetchProvider(options = {}, overrides = {}) {
191
340
  }
192
341
  }
193
342
  if (proxy !== undefined && proxy.length > 0) {
343
+ let tunneled;
194
344
  try {
195
- return finalize(await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes));
345
+ tunneled = await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
196
346
  }
197
347
  catch (error) {
198
348
  if (cancelled())
199
349
  throw error;
200
350
  failures.push(`tunnel: ${error instanceof Error ? error.message : String(error)}`);
201
351
  }
352
+ if (tunneled !== undefined) {
353
+ const rendered = await settle(tunneled);
354
+ if (rendered !== undefined)
355
+ return finalize(rendered, reportFor());
356
+ }
357
+ }
358
+ // `always` is also a standalone rescue: a transport that returns 403 or
359
+ // times out may still answer a real browser, and there is no earlier page
360
+ // to fall back to in that branch.
361
+ if (fallback === undefined && liveRenderMode() === 'always') {
362
+ const step = await renderStep(undefined);
363
+ lastReport = step.report;
364
+ if (step.done && step.outcome !== undefined)
365
+ return finalize(step.outcome, step.report);
366
+ if (!step.done)
367
+ failures.push(`render: ${step.report.reason}`);
202
368
  }
203
369
  if (apiKey.length > 0) {
204
370
  try {
205
- return finalize(await deps.extract(target, apiKey, timeoutMs, signal));
371
+ const extracted = await deps.extract(target, apiKey, timeoutMs, signal);
372
+ // Say where the body came from when the browser was tried first.
373
+ const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
374
+ ? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the result came from AnySearch extraction`)
375
+ : reportFor();
376
+ return finalize(extracted, report);
206
377
  }
207
378
  catch (error) {
208
379
  if (cancelled())
@@ -210,6 +381,14 @@ export function createFetchProvider(options = {}, overrides = {}) {
210
381
  failures.push(`extract: ${error instanceof Error ? error.message : String(error)}`);
211
382
  }
212
383
  }
384
+ // A failed render is not a failed fetch: hand back the page an earlier
385
+ // rung already produced, with the reason attached.
386
+ if (fallback !== undefined) {
387
+ const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
388
+ ? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the earlier rung's result was kept`)
389
+ : reportFor();
390
+ return finalize(fallback, report);
391
+ }
213
392
  throw new SearchError(`cannot fetch ${target} (${failures.join('; ')})`, 'SEARCH_NETWORK', true);
214
393
  },
215
394
  };
package/lib/index.d.ts CHANGED
@@ -74,6 +74,8 @@ declare const _default: typeof apply & {
74
74
  timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
75
75
  fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
76
76
  fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
77
+ renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
78
+ renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
77
79
  wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
78
80
  proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
79
81
  limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -99,6 +101,8 @@ declare const _default: typeof apply & {
99
101
  timeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
100
102
  fetchFormat: import("@deepseek-ai/schemastery").default<"markdown" | "text", "markdown" | "text", "volatile-defined">;
101
103
  fetchMaxLength: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
104
+ renderJs: import("@deepseek-ai/schemastery").default<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
105
+ renderTimeoutMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
102
106
  wait: import("@deepseek-ai/schemastery").default<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
103
107
  proxy: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
104
108
  limit: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
@@ -140,4 +144,6 @@ export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions }
140
144
  export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
141
145
  export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
142
146
  export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
147
+ export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from './read-page.ts';
148
+ export type { ReadPageDefinition, ReadPageOptions, ReadPageRegistration, ReadPageRegistryLike, ReadPageSettings, ReadPageValue, } from './read-page.ts';
143
149
  export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
package/lib/index.js CHANGED
@@ -16,6 +16,7 @@ import { EngineHealth } from "./health.js";
16
16
  import { createFetchProvider } from "./fetch.js";
17
17
  import { registerDomainTool } from "./domain-tool.js";
18
18
  import { registerSearchManyTool } from "./search-many.js";
19
+ import { registerReadPageTool } from "./read-page.js";
19
20
  export const name = 'dsh-web-search';
20
21
  /** The `ctx.web` service this plugin registers a search provider with. */
21
22
  export const inject = ['web'];
@@ -167,10 +168,12 @@ export function apply(ctx, config = {}) {
167
168
  timeoutMs: mounted.timeoutMs,
168
169
  format: () => resolveConfig(section, undefined).fetchFormat,
169
170
  maxLength: () => resolveConfig(section, undefined).fetchMaxLength,
171
+ renderJs: () => resolveConfig(section, undefined).renderJs,
172
+ renderTimeoutMs: () => resolveConfig(section, undefined).renderTimeoutMs,
170
173
  ...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
171
174
  ...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
172
175
  }));
173
- // Optional half: `domain_search` and `search_many`. Fail-safe by
176
+ // Optional half: `domain_search`, `search_many` and `read_page`. Fail-safe by
174
177
  // construction — a deployment without `ctx.tools` still loads this plugin and
175
178
  // keeps `web_search`; the registration path is the host's
176
179
  // `ToolRuntime.register(definition)`.
@@ -193,6 +196,24 @@ export function apply(ctx, config = {}) {
193
196
  if (batch !== undefined) {
194
197
  onDispose(ctx, batch.dispose);
195
198
  }
199
+ // Resumable reading: `web_fetch` cannot continue from an offset because
200
+ // the host request carries only `url`, but a plugin-owned tool can. It
201
+ // reuses the same fetch ladder and the same live Settings readers.
202
+ const paged = registerReadPageTool(tools, () => {
203
+ const live = resolveConfig(section, undefined);
204
+ return {
205
+ maxLength: live.fetchMaxLength,
206
+ format: live.fetchFormat,
207
+ timeoutMs: live.timeoutMs,
208
+ renderJs: live.renderJs,
209
+ renderTimeoutMs: live.renderTimeoutMs,
210
+ ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
211
+ ...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
212
+ };
213
+ }, { warn: warnTool });
214
+ if (paged !== undefined) {
215
+ onDispose(ctx, paged.dispose);
216
+ }
196
217
  }
197
218
  }
198
219
  catch (error) {
@@ -226,3 +247,4 @@ export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
226
247
  export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
227
248
  export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
228
249
  export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
250
+ export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from "./read-page.js";
package/lib/rank.js CHANGED
@@ -195,6 +195,9 @@ function resolveOptions(options) {
195
195
  freshnessDays: positiveOr(options?.freshnessDays, DEFAULT_FRESHNESS_DAYS),
196
196
  timeRange: options?.timeRange,
197
197
  weights: Array.isArray(options?.weights) ? options.weights : undefined,
198
+ now: typeof options?.now === 'number' && Number.isFinite(options.now)
199
+ ? options.now
200
+ : Date.now(),
198
201
  };
199
202
  }
200
203
  /**
@@ -320,7 +323,7 @@ function fuseEntries(entries, limit, options) {
320
323
  if (budget === 0)
321
324
  return [];
322
325
  const config = resolveOptions(options);
323
- const now = Date.now();
326
+ const now = config.now;
324
327
  // Per-engine lists, deduplicated so one engine gets at most one vote per
325
328
  // document and the earliest row keeps its rank.
326
329
  const lists = [];