@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/cache.js CHANGED
@@ -1,7 +1,10 @@
1
+ import { readCacheFile, resolveCacheFile, writeCacheFile } from "./cache-persist.js";
1
2
  /** Default entry lifetime: ten minutes. */
2
3
  const DEFAULT_TTL_MS = 600_000;
3
4
  /** Default number of retained entries. */
4
5
  const DEFAULT_MAX_ENTRIES = 100;
6
+ /** Default minimum gap between disk writes. */
7
+ const DEFAULT_PERSIST_THROTTLE_MS = 2_000;
5
8
  /**
6
9
  * Build the cache key for one search request.
7
10
  *
@@ -53,21 +56,51 @@ function normaliseCount(value, fallback) {
53
56
  */
54
57
  export class SearchCache {
55
58
  /** Entry lifetime in milliseconds; `0` disables the cache completely. */
56
- ttlMs;
59
+ /** Live TTL reader: settings may change without a remount. */
60
+ ttlOf;
57
61
  /** Maximum retained entries; the oldest read is evicted first. */
58
- maxEntries;
62
+ /** Live entry-cap reader: settings may change without a remount. */
63
+ maxOf;
59
64
  /** Injected clock, so tests never wait on real time. */
60
65
  now;
61
66
  /** Insertion order doubles as LRU order: the first key is the coldest. */
62
67
  entries = new Map();
68
+ /** Snapshot destination, or `undefined` for a purely in-memory cache. */
69
+ persistTo;
70
+ /** Minimum gap between writes. */
71
+ throttleMs;
72
+ /** Whether memory changed since the last successful write. */
73
+ dirty = false;
74
+ /** Pending throttled flush. */
75
+ timer;
63
76
  /**
64
- * @param options - TTL and entry cap, both optional.
77
+ * @param options - TTL, entry cap and optional persistence.
65
78
  * @param now - clock override; defaults to `Date.now`.
66
79
  */
67
80
  constructor(options = {}, now) {
68
- this.ttlMs = normaliseDuration(options.ttlMs, DEFAULT_TTL_MS);
69
- this.maxEntries = normaliseCount(options.maxEntries, DEFAULT_MAX_ENTRIES);
81
+ const ttlOption = options.ttlMs;
82
+ const maxOption = options.maxEntries;
83
+ this.ttlOf = () => normaliseDuration(typeof ttlOption === 'function' ? ttlOption() : ttlOption, DEFAULT_TTL_MS);
84
+ this.maxOf = () => normaliseCount(typeof maxOption === 'function' ? maxOption() : maxOption, DEFAULT_MAX_ENTRIES);
70
85
  this.now = now ?? Date.now;
86
+ this.throttleMs = normaliseDuration(options.persist?.throttleMs, DEFAULT_PERSIST_THROTTLE_MS);
87
+ const persist = options.persist;
88
+ if (persist !== undefined && this.ttlOf() > 0 && this.maxOf() > 0) {
89
+ const file = resolveCacheFile(persist.file);
90
+ // Startup restore is synchronous and fully contained: an unreadable or
91
+ // malformed snapshot simply yields an empty cache.
92
+ const loaded = readCacheFile(file, this.now(), this.maxOf(), persist.io);
93
+ for (const entry of loaded) {
94
+ this.entries.set(entry.key, { sources: entry.sources, expiresAt: entry.expiresAt });
95
+ }
96
+ this.persistTo = {
97
+ file,
98
+ ...(persist.maxBytes === undefined ? {} : { maxBytes: persist.maxBytes }),
99
+ ...(persist.io === undefined ? {} : { io: persist.io }),
100
+ };
101
+ liveCaches.add(this);
102
+ installExitHook();
103
+ }
71
104
  }
72
105
  /** Number of retained entries, including ones whose TTL already elapsed. */
73
106
  get size() {
@@ -82,13 +115,14 @@ export class SearchCache {
82
115
  * @returns a copy of the cached sources, or `undefined` on miss/expiry.
83
116
  */
84
117
  get(key) {
85
- if (this.ttlMs === 0)
118
+ if (this.ttlOf() === 0 || this.maxOf() === 0)
86
119
  return undefined;
87
120
  const entry = this.entries.get(key);
88
121
  if (entry === undefined)
89
122
  return undefined;
90
123
  if (this.now() >= entry.expiresAt) {
91
124
  this.entries.delete(key);
125
+ this.touch();
92
126
  return undefined;
93
127
  }
94
128
  // Re-insert to move the entry to the young end of the map.
@@ -104,22 +138,113 @@ export class SearchCache {
104
138
  * @param sources - the merged source list; copied before storage.
105
139
  */
106
140
  set(key, sources) {
107
- if (this.ttlMs === 0 || this.maxEntries === 0)
141
+ const ttlMs = this.ttlOf();
142
+ const maxEntries = this.maxOf();
143
+ if (ttlMs === 0 || maxEntries === 0)
108
144
  return;
109
145
  this.entries.delete(key);
110
146
  this.entries.set(key, {
111
147
  sources: cloneSources(sources),
112
- expiresAt: this.now() + this.ttlMs,
148
+ expiresAt: this.now() + ttlMs,
113
149
  });
114
- while (this.entries.size > this.maxEntries) {
150
+ while (this.entries.size > maxEntries) {
115
151
  const oldest = this.entries.keys().next();
116
152
  if (oldest.done === true)
117
153
  break;
118
154
  this.entries.delete(oldest.value);
119
155
  }
156
+ this.touch();
157
+ }
158
+ /**
159
+ * Write the snapshot now, if there is anything to write.
160
+ *
161
+ * Writes are normally throttled and happen on a timer or at process exit;
162
+ * this is the deterministic seam tests use, and the call a shutdown path can
163
+ * make when it wants the newest state on disk immediately.
164
+ *
165
+ * @returns `true` when the file was replaced, `false` for "nothing to write"
166
+ * or a contained filesystem failure.
167
+ */
168
+ flush() {
169
+ if (this.persistTo === undefined)
170
+ return false;
171
+ if (this.timer !== undefined) {
172
+ clearTimeout(this.timer);
173
+ this.timer = undefined;
174
+ }
175
+ if (!this.dirty)
176
+ return false;
177
+ const entries = [];
178
+ for (const [key, entry] of this.entries) {
179
+ entries.push({ key, expiresAt: entry.expiresAt, sources: cloneSources(entry.sources) });
180
+ }
181
+ const written = writeCacheFile(this.persistTo.file, entries, this.now(), {
182
+ ...(this.persistTo.maxBytes === undefined ? {} : { maxBytes: this.persistTo.maxBytes }),
183
+ ...(this.persistTo.io === undefined ? {} : { io: this.persistTo.io }),
184
+ });
185
+ this.dirty = !written;
186
+ if (!written)
187
+ this.schedule();
188
+ return written;
189
+ }
190
+ /** Stop the exit hook and flush whatever is still pending. */
191
+ close() {
192
+ this.flush();
193
+ if (this.timer !== undefined) {
194
+ clearTimeout(this.timer);
195
+ this.timer = undefined;
196
+ }
197
+ liveCaches.delete(this);
198
+ if (liveCaches.size === 0)
199
+ removeExitHook();
120
200
  }
121
201
  /** Drop every entry. */
122
202
  clear() {
203
+ if (this.entries.size === 0)
204
+ return;
123
205
  this.entries.clear();
206
+ this.touch();
124
207
  }
208
+ /** Mark the snapshot stale and arrange a throttled flush. */
209
+ touch() {
210
+ if (this.persistTo === undefined)
211
+ return;
212
+ this.dirty = true;
213
+ this.schedule();
214
+ }
215
+ /** Arm the single pending timer, if persistence is enabled. */
216
+ schedule() {
217
+ if (this.persistTo === undefined || this.timer !== undefined)
218
+ return;
219
+ this.timer = setTimeout(() => {
220
+ this.timer = undefined;
221
+ this.flush();
222
+ }, this.throttleMs);
223
+ // A pending cache flush must not keep a one-shot CLI process alive; the
224
+ // exit hook still writes it synchronously.
225
+ this.timer.unref();
226
+ }
227
+ }
228
+ /** Caches that still need an exit-time flush. */
229
+ const liveCaches = new Set();
230
+ /** Set once the process-level hook is installed. */
231
+ let exitHookInstalled = false;
232
+ /** Flush every live cache; used only during process exit. */
233
+ function flushLiveCaches() {
234
+ for (const cache of liveCaches)
235
+ cache.flush();
236
+ }
237
+ /** Install one process-wide exit hook, lazily. */
238
+ function installExitHook() {
239
+ if (exitHookInstalled)
240
+ return;
241
+ exitHookInstalled = true;
242
+ process.once('exit', flushLiveCaches);
243
+ }
244
+ /** Remove the hook when no persisted cache remains. */
245
+ function removeExitHook() {
246
+ if (!exitHookInstalled)
247
+ return;
248
+ process.removeListener('exit', flushLiveCaches);
249
+ exitHookInstalled = false;
125
250
  }
package/lib/config.d.ts CHANGED
@@ -20,11 +20,16 @@
20
20
  * @module @fangchenghao/dsh-web-search/config
21
21
  */
22
22
  import z from '@deepseek-ai/schemastery';
23
+ import type { RenderMode } from './render.ts';
23
24
  import type { RankOptions, TimeRange, WaitPolicy } from './types.ts';
24
25
  /** Schema of the plugin's configuration section. */
25
26
  export declare const Config: z<Schemastery.ObjectS<NoInfer<{
26
27
  hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
27
28
  timeoutMs: z<number, number, "volatile-defined">;
29
+ fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
30
+ fetchMaxLength: z<number, number, "volatile-defined">;
31
+ renderJs: z<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
32
+ renderTimeoutMs: z<number, number, "volatile-defined">;
28
33
  wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
29
34
  proxy: z<string, string, "volatile-defined">;
30
35
  limit: z<number, number, "volatile-defined">;
@@ -36,6 +41,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
36
41
  timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
37
42
  cacheTtlMs: z<number, number, "volatile-defined">;
38
43
  cacheEntries: z<number, number, "volatile-defined">;
44
+ cachePersist: z<boolean, boolean, "volatile-defined">;
45
+ cacheFile: z<string, string, "volatile-defined">;
39
46
  anysearchApiKey: z<string, string, "volatile-defined">;
40
47
  anysearchDomain: z<string, string, "volatile-defined">;
41
48
  baseCooldownMs: z<number, number, "volatile-defined">;
@@ -46,6 +53,10 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
46
53
  }>>, Schemastery.ObjectT<NoInfer<{
47
54
  hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
48
55
  timeoutMs: z<number, number, "volatile-defined">;
56
+ fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
57
+ fetchMaxLength: z<number, number, "volatile-defined">;
58
+ renderJs: z<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
59
+ renderTimeoutMs: z<number, number, "volatile-defined">;
49
60
  wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
50
61
  proxy: z<string, string, "volatile-defined">;
51
62
  limit: z<number, number, "volatile-defined">;
@@ -57,6 +68,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
57
68
  timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
58
69
  cacheTtlMs: z<number, number, "volatile-defined">;
59
70
  cacheEntries: z<number, number, "volatile-defined">;
71
+ cachePersist: z<boolean, boolean, "volatile-defined">;
72
+ cacheFile: z<string, string, "volatile-defined">;
60
73
  anysearchApiKey: z<string, string, "volatile-defined">;
61
74
  anysearchDomain: z<string, string, "volatile-defined">;
62
75
  baseCooldownMs: z<number, number, "volatile-defined">;
@@ -92,6 +105,14 @@ export interface ResolvedConfig {
92
105
  hostIds: string[] | undefined;
93
106
  /** Per-engine deadline. */
94
107
  timeoutMs: number;
108
+ /** `web_fetch` output style. */
109
+ fetchFormat: 'markdown' | 'text';
110
+ /** `web_fetch` character budget. */
111
+ fetchMaxLength: number;
112
+ /** Browser rendering policy for `web_fetch`. */
113
+ renderJs: RenderMode;
114
+ /** Browser-render deadline in milliseconds. */
115
+ renderTimeoutMs: number;
95
116
  /** Explicit tunnel URL, or `undefined` to detect one. */
96
117
  proxy: string | undefined;
97
118
  /** Caller budget cap. */
@@ -110,6 +131,10 @@ export interface ResolvedConfig {
110
131
  cacheTtlMs: number;
111
132
  /** Cache capacity in entries. */
112
133
  cacheEntries: number;
134
+ /** Whether the cache snapshot is written to disk. */
135
+ cachePersist: boolean;
136
+ /** Custom snapshot path, or `undefined` for the default one. */
137
+ cacheFile: string | undefined;
113
138
  /** Base cooldown in milliseconds. */
114
139
  baseCooldownMs: number;
115
140
  /** Cooldown ceiling in milliseconds. */
package/lib/config.js CHANGED
@@ -20,6 +20,7 @@
20
20
  * @module @fangchenghao/dsh-web-search/config
21
21
  */
22
22
  import z from '@deepseek-ai/schemastery';
23
+ import { DEFAULT_FETCH_MAX_LENGTH } from "./markdown.js";
23
24
  import { VERTICAL_IDS, isVerticalId } from "./verticals.js";
24
25
  /** Schema of the plugin's configuration section. */
25
26
  export const Config = z.object({
@@ -29,6 +30,30 @@ export const Config = z.object({
29
30
  .description('只启用列出的搜索引擎(如 anysearch、cn.bing),多个用逗号分隔;留空表示全部。不想限制就留空。')
30
31
  .volatile(),
31
32
  timeoutMs: z.natural().min(500).default(15000).description('每次向搜索引擎要结果,最多等它多久(毫秒,1000 = 1 秒)。网络慢可加到 20000;平时保持 15000 就行。').volatile(),
33
+ fetchFormat: z
34
+ .union(['markdown', 'text'])
35
+ .default('markdown')
36
+ .description('抓取网页后返回什么样式:markdown 保留标题、列表、表格和链接,最适合阅读;text 只留纯文字、去掉所有标记。一般保持 markdown。')
37
+ .volatile(),
38
+ fetchMaxLength: z
39
+ .natural()
40
+ .min(1000)
41
+ .max(200000)
42
+ .default(DEFAULT_FETCH_MAX_LENGTH)
43
+ .description('一次抓取最多返回多少个字符(1000-200000);超出会在末尾注明原始长度。想一次多读可调到 50000,日常保持 20000 即可。')
44
+ .volatile(),
45
+ renderJs: z
46
+ .union(['auto', 'always', 'off'])
47
+ .default('auto')
48
+ .description('脚本渲染页面的处理方式:auto 只在正文像空壳网页时才用浏览器渲染(推荐),always 每次抓取都渲染(更慢),off 从不渲染。需要另外安装 Playwright;没装就静默跳过,不影响抓取。')
49
+ .volatile(),
50
+ renderTimeoutMs: z
51
+ .natural()
52
+ .min(1000)
53
+ .max(120000)
54
+ .default(15000)
55
+ .description('用浏览器渲染一个页面最多等多久(毫秒,1000-120000)。页面很重可加到 30000;平时保持 15000 即可。')
56
+ .volatile(),
32
57
  wait: z
33
58
  .union(['auto', 'all', 'fast'])
34
59
  .default('auto')
@@ -54,6 +79,16 @@ export const Config = z.object({
54
79
  .volatile(),
55
80
  cacheTtlMs: z.natural().default(600000).description('同一问题多久内复用上次结果(缓存,即存一份备用),毫秒。想早点看到新内容就调小;保持 600000(10 分钟)即可。').volatile(),
56
81
  cacheEntries: z.natural().default(100).description('最多存多少条旧结果备用(缓存,即存起来省得重复查)。存满后先丢最久没用的;一般保持 100 即可。').volatile(),
82
+ cachePersist: z
83
+ .boolean()
84
+ .default(true)
85
+ .description('是否把缓存保存到硬盘,重启后还能复用上次结果。默认开启(true);不想留痕或磁盘只读时改成 false。')
86
+ .volatile(),
87
+ cacheFile: z
88
+ .string()
89
+ .default('')
90
+ .description('缓存文件放哪;留空用默认 ~/.dsh/cache/dsh-web-search/cache.json。一般不用填,换位置时再写绝对路径。')
91
+ .volatile(),
57
92
  anysearchApiKey: z
58
93
  .string()
59
94
  .default('')
@@ -117,10 +152,15 @@ export function resolveConfig(config, callLimit) {
117
152
  : String(rawDomainSources).split(/[\s,]+/);
118
153
  const domainSources = domainList.map((id) => id.trim()).filter(isVerticalId);
119
154
  const proxy = settings.proxy.trim();
155
+ const cacheFile = settings.cacheFile.trim();
120
156
  const limit = Math.min(callLimit ?? settings.limit, settings.limit);
121
157
  return {
122
158
  hostIds: hosts.length > 0 ? hosts : undefined,
123
159
  timeoutMs: settings.timeoutMs,
160
+ fetchFormat: settings.fetchFormat,
161
+ fetchMaxLength: settings.fetchMaxLength,
162
+ renderJs: settings.renderJs,
163
+ renderTimeoutMs: settings.renderTimeoutMs,
124
164
  proxy: proxy.length > 0 ? proxy : undefined,
125
165
  limit,
126
166
  wait: settings.wait,
@@ -139,6 +179,8 @@ export function resolveConfig(config, callLimit) {
139
179
  },
140
180
  cacheTtlMs: settings.cacheTtlMs,
141
181
  cacheEntries: settings.cacheEntries,
182
+ cachePersist: settings.cachePersist,
183
+ cacheFile: cacheFile.length > 0 ? cacheFile : undefined,
142
184
  baseCooldownMs: settings.baseCooldownMs,
143
185
  maxCooldownMs: settings.maxCooldownMs,
144
186
  };
package/lib/engines.d.ts CHANGED
@@ -11,7 +11,7 @@
11
11
  *
12
12
  * @module @fangchenghao/dsh-web-search/engines
13
13
  */
14
- import type { HostSpec, RankOptions, SearchOptions, SearchOutcome } from './types.ts';
14
+ import type { EngineAnswer, HostSpec, RankOptions, Region, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
15
15
  import type { SearchCache } from './cache.ts';
16
16
  import type { EngineHealth } from './health.ts';
17
17
  /**
@@ -34,6 +34,46 @@ export interface EngineRunOptions extends SearchOptions {
34
34
  cache?: SearchCache;
35
35
  /** Cooldown tracker, when one is configured. */
36
36
  health?: EngineHealth;
37
+ /**
38
+ * Read-only observer of the fusion inputs.
39
+ *
40
+ * Benchmark recorders use it to persist each engine's own list and rerun the
41
+ * current ranking offline. It never changes the run: the snapshot is a copy
42
+ * handed over after the fused list is final, and an absent hook costs one
43
+ * `undefined` check per run.
44
+ */
45
+ trace?: (snapshot: SearchTrace) => void;
46
+ }
47
+ /**
48
+ * What one search run handed to the ranking, captured for offline replay.
49
+ *
50
+ * `answers` are the exact per-engine lists that produced `sources`, in engine
51
+ * priority order and already narrowed by an active time window. Only the tier
52
+ * that actually supplied the fused list is present: when the primary tier
53
+ * answered, the fallback engines never ran. A cache hit has no fresh answers,
54
+ * so `answers` is empty and `cached` is `true`.
55
+ */
56
+ export interface SearchTrace {
57
+ /** Caller budget, after the engine layer's own clamp. */
58
+ limit: number;
59
+ /** Region that led this query. */
60
+ lead: Region;
61
+ /** Release policy used for the tier run. */
62
+ wait: WaitPolicy;
63
+ /** Active publication window. */
64
+ timeRange: TimeRange;
65
+ /** Ranking knobs, with the window folded in. */
66
+ rank: RankOptions;
67
+ /** Which path produced `sources`. */
68
+ tier: 'primary' | 'fallback' | 'none' | 'cache';
69
+ /** Per-engine lists that were fused, in engine priority order. */
70
+ answers: EngineAnswer[];
71
+ /** The fused list `search` returned. */
72
+ sources: Source[];
73
+ /** Failure strings accumulated by this run. */
74
+ failures: string[];
75
+ /** Whether the result came from the cache. */
76
+ cached: boolean;
37
77
  }
38
78
  /**
39
79
  * Search the web with every configured engine and fuse the answers.
package/lib/engines.js CHANGED
@@ -66,7 +66,9 @@ export const HOSTS = [
66
66
  },
67
67
  {
68
68
  id: 'so',
69
- tier: 'primary',
69
+ // 360 answers an anti-bot wall from this network and from the tunnel, so it
70
+ // is demoted: tried only when every primary engine came back empty.
71
+ tier: 'fallback',
70
72
  region: 'cn',
71
73
  build: (query) => `https://www.so.com/s?q=${encodeURIComponent(query)}`,
72
74
  parse: parseSo,
@@ -317,14 +319,31 @@ export async function search(query, signal, options) {
317
319
  // next identical query into a cache miss.
318
320
  const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
319
321
  const cached = key === undefined ? undefined : options.cache?.get(key);
320
- if (cached !== undefined)
322
+ if (cached !== undefined) {
323
+ options.trace?.({
324
+ limit,
325
+ lead,
326
+ wait,
327
+ timeRange,
328
+ rank,
329
+ tier: 'cache',
330
+ answers: [],
331
+ sources: cached,
332
+ failures: [...failures],
333
+ cached: true,
334
+ });
321
335
  return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
336
+ }
322
337
  const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
323
338
  const primary = ordered.filter((host) => host.tier === 'primary');
324
339
  const fallback = ordered.filter((host) => host.tier === 'fallback');
340
+ let tier = 'none';
325
341
  let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health));
342
+ if (answers.length > 0)
343
+ tier = 'primary';
326
344
  let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
327
345
  if (sources.length === 0 && fallback.length > 0) {
346
+ tier = 'fallback';
328
347
  answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health));
329
348
  sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
330
349
  }
@@ -332,5 +351,17 @@ export async function search(query, signal, options) {
332
351
  throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
333
352
  if (key !== undefined && sources.length > 0)
334
353
  options.cache?.set(key, sources);
354
+ options.trace?.({
355
+ limit,
356
+ lead,
357
+ wait,
358
+ timeRange,
359
+ rank,
360
+ tier,
361
+ answers,
362
+ sources,
363
+ failures: [...failures],
364
+ cached: false,
365
+ });
335
366
  return { sources, failures, ...(proxy === undefined ? {} : { proxy }) };
336
367
  }
package/lib/fetch.d.ts CHANGED
@@ -1,3 +1,5 @@
1
+ import type { FetchFormat } from './markdown.ts';
2
+ import type { RenderMode, RenderReport, RenderedPage, RendererLoad, RendererModule } from './render.ts';
1
3
  /** One retrieval outcome, mirroring the Host's `WebFetchResult`. */
2
4
  export interface FetchOutcome {
3
5
  readonly url: string;
@@ -7,6 +9,13 @@ export interface FetchOutcome {
7
9
  readonly content: string;
8
10
  };
9
11
  readonly truncated: boolean;
12
+ /**
13
+ * How the optional render rung decided, when this provider produced the
14
+ * result. The Host ignores the extra field; it is here so a caller (and a
15
+ * test) can see why a shell page was or was not rendered, including that
16
+ * Playwright is absent.
17
+ */
18
+ readonly render?: RenderReport;
10
19
  }
11
20
  /** What `ctx.web.registerFetchProvider` expects. */
12
21
  export interface FetchProvider {
@@ -30,6 +39,20 @@ export interface FetchProviderOptions {
30
39
  anysearchApiKey?: string;
31
40
  /** Bytes kept from a response; the Host applies its own cap too. */
32
41
  maxBytes?: number;
42
+ /** Output style for an HTML body; `markdown` keeps structure. */
43
+ /** Renderer, or a reader for it so a Settings edit applies without a remount. */
44
+ format?: FetchFormat | (() => FetchFormat);
45
+ /** Character budget for the returned text. */
46
+ /** Character budget, or a reader for it. */
47
+ maxLength?: number | (() => number);
48
+ /**
49
+ * Optional browser rendering: `auto` renders only a suspected shell page,
50
+ * `always` renders every fetch, `off` never does. A reader keeps a Settings
51
+ * edit live. Defaults to `auto`; without Playwright it silently does nothing.
52
+ */
53
+ renderJs?: RenderMode | (() => RenderMode);
54
+ /** Browser-render deadline, or a reader for it. Defaults to 15000 ms. */
55
+ renderTimeoutMs?: number | (() => number);
33
56
  /** User agent sent on every rung. */
34
57
  userAgent?: string;
35
58
  }
@@ -43,6 +66,16 @@ export interface FetchDependencies {
43
66
  extract: (url: string, apiKey: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<FetchOutcome>;
44
67
  /** Proxy discovery. */
45
68
  detectProxy: (explicit: string | undefined) => Promise<string | undefined>;
69
+ /**
70
+ * Load the optional renderer. The default dynamically imports Playwright and
71
+ * reports absence instead of throwing; tests inject a fake here.
72
+ */
73
+ loadRenderer: () => Promise<RendererLoad>;
74
+ /**
75
+ * Render one page with a loaded renderer. The default is the real
76
+ * headless-Chromium implementation; tests inject a fake browser.
77
+ */
78
+ render: (renderer: RendererModule, url: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<RenderedPage>;
46
79
  }
47
80
  /**
48
81
  * Build the fetch provider.