@fffangchenghao/dsh-web-search 0.11.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,249 @@
1
+ /**
2
+ * Optional disk persistence for the search cache.
3
+ *
4
+ * The cache itself stays a tiny TTL+LRU map; this module only knows how to
5
+ * turn a snapshot of it into a file and back. Three properties matter more
6
+ * than speed here:
7
+ *
8
+ * - **Never fail the plugin.** A missing, corrupt, foreign-version or
9
+ * unreadable file is treated as an empty cache. Every filesystem call is
10
+ * contained, because a cache is an optimisation and must never be the reason
11
+ * a plugin refuses to mount.
12
+ * - **Never leave a half-written file.** The snapshot is written to a sibling
13
+ * temporary file and then `rename`d over the target, which is atomic on the
14
+ * same filesystem. A failure before the rename leaves the previous file
15
+ * intact and removes the temporary one.
16
+ * - **Never grow without bound.** The serialised snapshot is trimmed in LRU
17
+ * order (oldest first) until it fits the byte budget.
18
+ *
19
+ * Only the search-result summary (`url`, `title`, `snippet`, `publishedAt`) is
20
+ * stored. Page bodies are never touched by this cache, and any extra property a
21
+ * caller attached to a {@link Source} is dropped at the serialisation boundary.
22
+ *
23
+ * @module @fangchenghao/dsh-web-search/cache-persist
24
+ */
25
+ import { readFileSync, renameSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
26
+ import { homedir } from 'node:os';
27
+ import { dirname, join, resolve } from 'node:path';
28
+ /** Version of the on-disk document; a different version is ignored wholesale. */
29
+ export const CACHE_FILE_VERSION = 1;
30
+ /** Default byte budget for the snapshot: 4 MiB. */
31
+ export const DEFAULT_CACHE_FILE_MAX_BYTES = 4 * 1024 * 1024;
32
+ /** The real filesystem, exported so tests can override one operation. */
33
+ export const nodeCacheFileIo = {
34
+ read(path) {
35
+ try {
36
+ return readFileSync(path, 'utf8');
37
+ }
38
+ catch {
39
+ // Missing file, permission error, directory: all mean "empty cache".
40
+ return undefined;
41
+ }
42
+ },
43
+ write(path, data) {
44
+ // The file only holds public search summaries, but 0600 keeps a shared
45
+ // machine from reading another user's queries.
46
+ writeFileSync(path, data, { encoding: 'utf8', mode: 0o600 });
47
+ },
48
+ rename(from, to) {
49
+ renameSync(from, to);
50
+ },
51
+ mkdir(path) {
52
+ mkdirSync(path, { recursive: true });
53
+ },
54
+ remove(path) {
55
+ rmSync(path, { force: true });
56
+ },
57
+ };
58
+ /** Where a snapshot is written when the config names no path. */
59
+ export function defaultCacheFile() {
60
+ return join(homedir(), '.dsh', 'cache', 'dsh-web-search', 'cache.json');
61
+ }
62
+ /**
63
+ * Expand a configured cache path.
64
+ *
65
+ * An empty value selects {@link defaultCacheFile}; a leading `~` expands to the
66
+ * home directory so the documented default can be written verbatim in Settings.
67
+ * Relative paths are resolved against the process working directory.
68
+ *
69
+ * @param value - the `cacheFile` setting, possibly empty.
70
+ * @returns an absolute file path.
71
+ */
72
+ export function resolveCacheFile(value) {
73
+ const trimmed = (value ?? '').trim();
74
+ if (trimmed.length === 0)
75
+ return defaultCacheFile();
76
+ const expanded = trimmed === '~'
77
+ ? homedir()
78
+ : /^~[\\/]/u.test(trimmed)
79
+ ? join(homedir(), trimmed.slice(2))
80
+ : trimmed;
81
+ return resolve(expanded);
82
+ }
83
+ /** True for a value usable as a plain record. */
84
+ function isRecord(value) {
85
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
86
+ }
87
+ /** Read one persisted source, keeping only the four summary fields. */
88
+ function readSource(raw) {
89
+ if (!isRecord(raw))
90
+ return undefined;
91
+ const url = raw['url'];
92
+ if (typeof url !== 'string' || url.trim().length === 0)
93
+ return undefined;
94
+ const source = { url };
95
+ if (typeof raw['title'] === 'string' && raw['title'].length > 0)
96
+ source.title = raw['title'];
97
+ if (typeof raw['snippet'] === 'string' && raw['snippet'].length > 0)
98
+ source.snippet = raw['snippet'];
99
+ if (typeof raw['publishedAt'] === 'string' && raw['publishedAt'].length > 0) {
100
+ source.publishedAt = raw['publishedAt'];
101
+ }
102
+ return source;
103
+ }
104
+ /** Write one source as a summary-only record. */
105
+ function storedSource(source) {
106
+ const stored = { url: source.url };
107
+ if (source.title !== undefined && source.title.length > 0)
108
+ stored['title'] = source.title;
109
+ if (source.snippet !== undefined && source.snippet.length > 0)
110
+ stored['snippet'] = source.snippet;
111
+ if (source.publishedAt !== undefined && source.publishedAt.length > 0) {
112
+ stored['publishedAt'] = source.publishedAt;
113
+ }
114
+ return stored;
115
+ }
116
+ /** Read one persisted entry, rejecting anything malformed or already expired. */
117
+ function readEntry(raw, now) {
118
+ if (!isRecord(raw))
119
+ return undefined;
120
+ const key = raw['key'];
121
+ const expiresAt = raw['expiresAt'];
122
+ const rawSources = raw['sources'];
123
+ if (typeof key !== 'string' || key.length === 0)
124
+ return undefined;
125
+ if (typeof expiresAt !== 'number' || !Number.isFinite(expiresAt) || expiresAt <= now)
126
+ return undefined;
127
+ if (!Array.isArray(rawSources))
128
+ return undefined;
129
+ const sources = [];
130
+ for (const item of rawSources) {
131
+ const source = readSource(item);
132
+ if (source !== undefined)
133
+ sources.push(source);
134
+ }
135
+ // An entry with no usable result is not worth a slot in the file.
136
+ if (sources.length === 0)
137
+ return undefined;
138
+ return { key, expiresAt, sources };
139
+ }
140
+ /** Clamp the byte budget to a usable positive integer. */
141
+ function normaliseMaxBytes(value) {
142
+ return typeof value === 'number' && Number.isFinite(value) && value > 0
143
+ ? Math.floor(value)
144
+ : DEFAULT_CACHE_FILE_MAX_BYTES;
145
+ }
146
+ /**
147
+ * Load the snapshot from disk.
148
+ *
149
+ * Every failure mode collapses to an empty list: absent file, unreadable path,
150
+ * malformed JSON, a different `version`, and individually invalid entries. An
151
+ * expired entry is dropped here rather than retained until its next read.
152
+ *
153
+ * @param file - absolute path to the snapshot.
154
+ * @param now - current time from the cache's injected clock.
155
+ * @param maxEntries - entry cap; the oldest surplus entries are discarded.
156
+ * @param io - filesystem seam, defaulting to the real one.
157
+ * @returns usable entries in stored (oldest to newest) order.
158
+ */
159
+ export function readCacheFile(file, now, maxEntries, io) {
160
+ try {
161
+ const text = (io ?? nodeCacheFileIo).read(file);
162
+ if (text === undefined)
163
+ return [];
164
+ const parsed = JSON.parse(text);
165
+ if (!isRecord(parsed))
166
+ return [];
167
+ if (parsed['version'] !== CACHE_FILE_VERSION)
168
+ return [];
169
+ const rawEntries = parsed['entries'];
170
+ if (!Array.isArray(rawEntries))
171
+ return [];
172
+ const entries = [];
173
+ for (const raw of rawEntries) {
174
+ const entry = readEntry(raw, now);
175
+ if (entry !== undefined)
176
+ entries.push(entry);
177
+ }
178
+ const cap = Number.isFinite(maxEntries) ? Math.max(0, Math.floor(maxEntries)) : entries.length;
179
+ return entries.length > cap ? entries.slice(entries.length - cap) : entries;
180
+ }
181
+ catch {
182
+ // Corrupt JSON, a foreign document, a getter that throws: empty cache.
183
+ return [];
184
+ }
185
+ }
186
+ /**
187
+ * Serialise entries, evicting the oldest until the document fits the budget.
188
+ *
189
+ * @param entries - entries in LRU order (oldest first).
190
+ * @param now - timestamp recorded as `savedAt`.
191
+ * @param maxBytes - byte budget; defaults to 4 MiB.
192
+ * @returns the document and how many entries it carries.
193
+ */
194
+ export function serializeCache(entries, now, maxBytes) {
195
+ const budget = normaliseMaxBytes(maxBytes);
196
+ const kept = entries.map((entry) => ({
197
+ key: entry.key,
198
+ expiresAt: entry.expiresAt,
199
+ sources: entry.sources.map(storedSource),
200
+ }));
201
+ const build = () => JSON.stringify({
202
+ version: CACHE_FILE_VERSION,
203
+ savedAt: now,
204
+ entries: kept,
205
+ });
206
+ let data = build();
207
+ while (kept.length > 0 && Buffer.byteLength(data, 'utf8') > budget) {
208
+ kept.shift();
209
+ data = build();
210
+ }
211
+ return { data, entries: kept.length, bytes: Buffer.byteLength(data, 'utf8') };
212
+ }
213
+ /** Distinguishes temporary files of concurrent writers in this process. */
214
+ let tempCounter = 0;
215
+ /**
216
+ * Atomically replace the snapshot.
217
+ *
218
+ * The temporary file lives beside the target so the `rename` stays on one
219
+ * filesystem. When anything fails (directory not writable, rename rejected),
220
+ * the previous document is untouched, the temporary file is removed, and the
221
+ * call returns `false` instead of throwing.
222
+ *
223
+ * @param file - absolute target path.
224
+ * @param entries - entries in LRU order (oldest first).
225
+ * @param now - timestamp recorded as `savedAt`.
226
+ * @param options - byte budget and filesystem seam.
227
+ * @returns `true` when the target now holds this snapshot.
228
+ */
229
+ export function writeCacheFile(file, entries, now, options = {}) {
230
+ const io = options.io ?? nodeCacheFileIo;
231
+ const { data } = serializeCache(entries, now, options.maxBytes);
232
+ tempCounter += 1;
233
+ const temp = `${file}.${process.pid}.${tempCounter}.tmp`;
234
+ try {
235
+ io.mkdir(dirname(file));
236
+ io.write(temp, data);
237
+ io.rename(temp, file);
238
+ return true;
239
+ }
240
+ catch {
241
+ try {
242
+ io.remove(temp);
243
+ }
244
+ catch {
245
+ // Cleanup is best effort; the previous snapshot is still intact.
246
+ }
247
+ return false;
248
+ }
249
+ }
package/lib/cache.d.ts CHANGED
@@ -13,6 +13,23 @@
13
13
  * @module @fangchenghao/dsh-web-search/cache
14
14
  */
15
15
  import type { CacheOptions, Source, TimeRange } from './types.ts';
16
+ import type { CacheFileIo } from './cache-persist.ts';
17
+ /** Optional disk persistence, layered on top of the in-memory cache. */
18
+ export interface CachePersistOptions {
19
+ /** Custom snapshot path; empty or absent uses `~/.dsh/cache/dsh-web-search/cache.json`. */
20
+ file?: string;
21
+ /** Serialised byte budget; the oldest entries are evicted to fit. */
22
+ maxBytes?: number;
23
+ /** Minimum gap between writes in milliseconds; defaults to two seconds. */
24
+ throttleMs?: number;
25
+ /** Filesystem seam, mainly for tests. */
26
+ io?: CacheFileIo;
27
+ }
28
+ /** Constructor options: {@link CacheOptions} plus optional persistence. */
29
+ export interface SearchCacheOptions extends CacheOptions {
30
+ /** Persist the cache when present; absent keeps the cache purely in memory. */
31
+ persist?: CachePersistOptions;
32
+ }
16
33
  /** The logical inputs that decide whether two searches may share an entry. */
17
34
  export interface CacheKeyParts {
18
35
  /** Raw query text; normalised inside {@link cacheKey}. */
@@ -56,18 +73,28 @@ export declare function cacheKey(parts: CacheKeyParts): string;
56
73
  */
57
74
  export declare class SearchCache {
58
75
  /** Entry lifetime in milliseconds; `0` disables the cache completely. */
59
- private readonly ttlMs;
76
+ /** Live TTL reader: settings may change without a remount. */
77
+ private readonly ttlOf;
60
78
  /** Maximum retained entries; the oldest read is evicted first. */
61
- private readonly maxEntries;
79
+ /** Live entry-cap reader: settings may change without a remount. */
80
+ private readonly maxOf;
62
81
  /** Injected clock, so tests never wait on real time. */
63
82
  private readonly now;
64
83
  /** Insertion order doubles as LRU order: the first key is the coldest. */
65
84
  private readonly entries;
85
+ /** Snapshot destination, or `undefined` for a purely in-memory cache. */
86
+ private readonly persistTo;
87
+ /** Minimum gap between writes. */
88
+ private readonly throttleMs;
89
+ /** Whether memory changed since the last successful write. */
90
+ private dirty;
91
+ /** Pending throttled flush. */
92
+ private timer;
66
93
  /**
67
- * @param options - TTL and entry cap, both optional.
94
+ * @param options - TTL, entry cap and optional persistence.
68
95
  * @param now - clock override; defaults to `Date.now`.
69
96
  */
70
- constructor(options?: CacheOptions, now?: () => number);
97
+ constructor(options?: SearchCacheOptions, now?: () => number);
71
98
  /** Number of retained entries, including ones whose TTL already elapsed. */
72
99
  get size(): number;
73
100
  /**
@@ -87,6 +114,23 @@ export declare class SearchCache {
87
114
  * @param sources - the merged source list; copied before storage.
88
115
  */
89
116
  set(key: string, sources: Source[]): void;
117
+ /**
118
+ * Write the snapshot now, if there is anything to write.
119
+ *
120
+ * Writes are normally throttled and happen on a timer or at process exit;
121
+ * this is the deterministic seam tests use, and the call a shutdown path can
122
+ * make when it wants the newest state on disk immediately.
123
+ *
124
+ * @returns `true` when the file was replaced, `false` for "nothing to write"
125
+ * or a contained filesystem failure.
126
+ */
127
+ flush(): boolean;
128
+ /** Stop the exit hook and flush whatever is still pending. */
129
+ close(): void;
90
130
  /** Drop every entry. */
91
131
  clear(): void;
132
+ /** Mark the snapshot stale and arrange a throttled flush. */
133
+ private touch;
134
+ /** Arm the single pending timer, if persistence is enabled. */
135
+ private schedule;
92
136
  }
package/lib/cache.js CHANGED
@@ -1,7 +1,10 @@
1
+ import { readCacheFile, resolveCacheFile, writeCacheFile } from "./cache-persist.js";
1
2
  /** Default entry lifetime: ten minutes. */
2
3
  const DEFAULT_TTL_MS = 600_000;
3
4
  /** Default number of retained entries. */
4
5
  const DEFAULT_MAX_ENTRIES = 100;
6
+ /** Default minimum gap between disk writes. */
7
+ const DEFAULT_PERSIST_THROTTLE_MS = 2_000;
5
8
  /**
6
9
  * Build the cache key for one search request.
7
10
  *
@@ -53,21 +56,51 @@ function normaliseCount(value, fallback) {
53
56
  */
54
57
  export class SearchCache {
55
58
  /** Entry lifetime in milliseconds; `0` disables the cache completely. */
56
- ttlMs;
59
+ /** Live TTL reader: settings may change without a remount. */
60
+ ttlOf;
57
61
  /** Maximum retained entries; the oldest read is evicted first. */
58
- maxEntries;
62
+ /** Live entry-cap reader: settings may change without a remount. */
63
+ maxOf;
59
64
  /** Injected clock, so tests never wait on real time. */
60
65
  now;
61
66
  /** Insertion order doubles as LRU order: the first key is the coldest. */
62
67
  entries = new Map();
68
+ /** Snapshot destination, or `undefined` for a purely in-memory cache. */
69
+ persistTo;
70
+ /** Minimum gap between writes. */
71
+ throttleMs;
72
+ /** Whether memory changed since the last successful write. */
73
+ dirty = false;
74
+ /** Pending throttled flush. */
75
+ timer;
63
76
  /**
64
- * @param options - TTL and entry cap, both optional.
77
+ * @param options - TTL, entry cap and optional persistence.
65
78
  * @param now - clock override; defaults to `Date.now`.
66
79
  */
67
80
  constructor(options = {}, now) {
68
- this.ttlMs = normaliseDuration(options.ttlMs, DEFAULT_TTL_MS);
69
- this.maxEntries = normaliseCount(options.maxEntries, DEFAULT_MAX_ENTRIES);
81
+ const ttlOption = options.ttlMs;
82
+ const maxOption = options.maxEntries;
83
+ this.ttlOf = () => normaliseDuration(typeof ttlOption === 'function' ? ttlOption() : ttlOption, DEFAULT_TTL_MS);
84
+ this.maxOf = () => normaliseCount(typeof maxOption === 'function' ? maxOption() : maxOption, DEFAULT_MAX_ENTRIES);
70
85
  this.now = now ?? Date.now;
86
+ this.throttleMs = normaliseDuration(options.persist?.throttleMs, DEFAULT_PERSIST_THROTTLE_MS);
87
+ const persist = options.persist;
88
+ if (persist !== undefined && this.ttlOf() > 0 && this.maxOf() > 0) {
89
+ const file = resolveCacheFile(persist.file);
90
+ // Startup restore is synchronous and fully contained: an unreadable or
91
+ // malformed snapshot simply yields an empty cache.
92
+ const loaded = readCacheFile(file, this.now(), this.maxOf(), persist.io);
93
+ for (const entry of loaded) {
94
+ this.entries.set(entry.key, { sources: entry.sources, expiresAt: entry.expiresAt });
95
+ }
96
+ this.persistTo = {
97
+ file,
98
+ ...(persist.maxBytes === undefined ? {} : { maxBytes: persist.maxBytes }),
99
+ ...(persist.io === undefined ? {} : { io: persist.io }),
100
+ };
101
+ liveCaches.add(this);
102
+ installExitHook();
103
+ }
71
104
  }
72
105
  /** Number of retained entries, including ones whose TTL already elapsed. */
73
106
  get size() {
@@ -82,13 +115,14 @@ export class SearchCache {
82
115
  * @returns a copy of the cached sources, or `undefined` on miss/expiry.
83
116
  */
84
117
  get(key) {
85
- if (this.ttlMs === 0)
118
+ if (this.ttlOf() === 0 || this.maxOf() === 0)
86
119
  return undefined;
87
120
  const entry = this.entries.get(key);
88
121
  if (entry === undefined)
89
122
  return undefined;
90
123
  if (this.now() >= entry.expiresAt) {
91
124
  this.entries.delete(key);
125
+ this.touch();
92
126
  return undefined;
93
127
  }
94
128
  // Re-insert to move the entry to the young end of the map.
@@ -104,22 +138,113 @@ export class SearchCache {
104
138
  * @param sources - the merged source list; copied before storage.
105
139
  */
106
140
  set(key, sources) {
107
- if (this.ttlMs === 0 || this.maxEntries === 0)
141
+ const ttlMs = this.ttlOf();
142
+ const maxEntries = this.maxOf();
143
+ if (ttlMs === 0 || maxEntries === 0)
108
144
  return;
109
145
  this.entries.delete(key);
110
146
  this.entries.set(key, {
111
147
  sources: cloneSources(sources),
112
- expiresAt: this.now() + this.ttlMs,
148
+ expiresAt: this.now() + ttlMs,
113
149
  });
114
- while (this.entries.size > this.maxEntries) {
150
+ while (this.entries.size > maxEntries) {
115
151
  const oldest = this.entries.keys().next();
116
152
  if (oldest.done === true)
117
153
  break;
118
154
  this.entries.delete(oldest.value);
119
155
  }
156
+ this.touch();
157
+ }
158
+ /**
159
+ * Write the snapshot now, if there is anything to write.
160
+ *
161
+ * Writes are normally throttled and happen on a timer or at process exit;
162
+ * this is the deterministic seam tests use, and the call a shutdown path can
163
+ * make when it wants the newest state on disk immediately.
164
+ *
165
+ * @returns `true` when the file was replaced, `false` for "nothing to write"
166
+ * or a contained filesystem failure.
167
+ */
168
+ flush() {
169
+ if (this.persistTo === undefined)
170
+ return false;
171
+ if (this.timer !== undefined) {
172
+ clearTimeout(this.timer);
173
+ this.timer = undefined;
174
+ }
175
+ if (!this.dirty)
176
+ return false;
177
+ const entries = [];
178
+ for (const [key, entry] of this.entries) {
179
+ entries.push({ key, expiresAt: entry.expiresAt, sources: cloneSources(entry.sources) });
180
+ }
181
+ const written = writeCacheFile(this.persistTo.file, entries, this.now(), {
182
+ ...(this.persistTo.maxBytes === undefined ? {} : { maxBytes: this.persistTo.maxBytes }),
183
+ ...(this.persistTo.io === undefined ? {} : { io: this.persistTo.io }),
184
+ });
185
+ this.dirty = !written;
186
+ if (!written)
187
+ this.schedule();
188
+ return written;
189
+ }
190
+ /** Stop the exit hook and flush whatever is still pending. */
191
+ close() {
192
+ this.flush();
193
+ if (this.timer !== undefined) {
194
+ clearTimeout(this.timer);
195
+ this.timer = undefined;
196
+ }
197
+ liveCaches.delete(this);
198
+ if (liveCaches.size === 0)
199
+ removeExitHook();
120
200
  }
121
201
  /** Drop every entry. */
122
202
  clear() {
203
+ if (this.entries.size === 0)
204
+ return;
123
205
  this.entries.clear();
206
+ this.touch();
124
207
  }
208
+ /** Mark the snapshot stale and arrange a throttled flush. */
209
+ touch() {
210
+ if (this.persistTo === undefined)
211
+ return;
212
+ this.dirty = true;
213
+ this.schedule();
214
+ }
215
+ /** Arm the single pending timer, if persistence is enabled. */
216
+ schedule() {
217
+ if (this.persistTo === undefined || this.timer !== undefined)
218
+ return;
219
+ this.timer = setTimeout(() => {
220
+ this.timer = undefined;
221
+ this.flush();
222
+ }, this.throttleMs);
223
+ // A pending cache flush must not keep a one-shot CLI process alive; the
224
+ // exit hook still writes it synchronously.
225
+ this.timer.unref();
226
+ }
227
+ }
228
+ /** Caches that still need an exit-time flush. */
229
+ const liveCaches = new Set();
230
+ /** Set once the process-level hook is installed. */
231
+ let exitHookInstalled = false;
232
+ /** Flush every live cache; used only during process exit. */
233
+ function flushLiveCaches() {
234
+ for (const cache of liveCaches)
235
+ cache.flush();
236
+ }
237
+ /** Install one process-wide exit hook, lazily. */
238
+ function installExitHook() {
239
+ if (exitHookInstalled)
240
+ return;
241
+ exitHookInstalled = true;
242
+ process.once('exit', flushLiveCaches);
243
+ }
244
+ /** Remove the hook when no persisted cache remains. */
245
+ function removeExitHook() {
246
+ if (!exitHookInstalled)
247
+ return;
248
+ process.removeListener('exit', flushLiveCaches);
249
+ exitHookInstalled = false;
125
250
  }
package/lib/config.d.ts CHANGED
@@ -25,6 +25,8 @@ import type { RankOptions, TimeRange, WaitPolicy } from './types.ts';
25
25
  export declare const Config: z<Schemastery.ObjectS<NoInfer<{
26
26
  hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
27
27
  timeoutMs: z<number, number, "volatile-defined">;
28
+ fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
29
+ fetchMaxLength: z<number, number, "volatile-defined">;
28
30
  wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
29
31
  proxy: z<string, string, "volatile-defined">;
30
32
  limit: z<number, number, "volatile-defined">;
@@ -36,6 +38,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
36
38
  timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
37
39
  cacheTtlMs: z<number, number, "volatile-defined">;
38
40
  cacheEntries: z<number, number, "volatile-defined">;
41
+ cachePersist: z<boolean, boolean, "volatile-defined">;
42
+ cacheFile: z<string, string, "volatile-defined">;
39
43
  anysearchApiKey: z<string, string, "volatile-defined">;
40
44
  anysearchDomain: z<string, string, "volatile-defined">;
41
45
  baseCooldownMs: z<number, number, "volatile-defined">;
@@ -46,6 +50,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
46
50
  }>>, Schemastery.ObjectT<NoInfer<{
47
51
  hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
48
52
  timeoutMs: z<number, number, "volatile-defined">;
53
+ fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
54
+ fetchMaxLength: z<number, number, "volatile-defined">;
49
55
  wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
50
56
  proxy: z<string, string, "volatile-defined">;
51
57
  limit: z<number, number, "volatile-defined">;
@@ -57,6 +63,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
57
63
  timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
58
64
  cacheTtlMs: z<number, number, "volatile-defined">;
59
65
  cacheEntries: z<number, number, "volatile-defined">;
66
+ cachePersist: z<boolean, boolean, "volatile-defined">;
67
+ cacheFile: z<string, string, "volatile-defined">;
60
68
  anysearchApiKey: z<string, string, "volatile-defined">;
61
69
  anysearchDomain: z<string, string, "volatile-defined">;
62
70
  baseCooldownMs: z<number, number, "volatile-defined">;
@@ -92,6 +100,10 @@ export interface ResolvedConfig {
92
100
  hostIds: string[] | undefined;
93
101
  /** Per-engine deadline. */
94
102
  timeoutMs: number;
103
+ /** `web_fetch` output style. */
104
+ fetchFormat: 'markdown' | 'text';
105
+ /** `web_fetch` character budget. */
106
+ fetchMaxLength: number;
95
107
  /** Explicit tunnel URL, or `undefined` to detect one. */
96
108
  proxy: string | undefined;
97
109
  /** Caller budget cap. */
@@ -110,6 +122,10 @@ export interface ResolvedConfig {
110
122
  cacheTtlMs: number;
111
123
  /** Cache capacity in entries. */
112
124
  cacheEntries: number;
125
+ /** Whether the cache snapshot is written to disk. */
126
+ cachePersist: boolean;
127
+ /** Custom snapshot path, or `undefined` for the default one. */
128
+ cacheFile: string | undefined;
113
129
  /** Base cooldown in milliseconds. */
114
130
  baseCooldownMs: number;
115
131
  /** Cooldown ceiling in milliseconds. */
package/lib/config.js CHANGED
@@ -20,6 +20,7 @@
20
20
  * @module @fangchenghao/dsh-web-search/config
21
21
  */
22
22
  import z from '@deepseek-ai/schemastery';
23
+ import { DEFAULT_FETCH_MAX_LENGTH } from "./markdown.js";
23
24
  import { VERTICAL_IDS, isVerticalId } from "./verticals.js";
24
25
  /** Schema of the plugin's configuration section. */
25
26
  export const Config = z.object({
@@ -29,6 +30,18 @@ export const Config = z.object({
29
30
  .description('只启用列出的搜索引擎(如 anysearch、cn.bing),多个用逗号分隔;留空表示全部。不想限制就留空。')
30
31
  .volatile(),
31
32
  timeoutMs: z.natural().min(500).default(15000).description('每次向搜索引擎要结果,最多等它多久(毫秒,1000 = 1 秒)。网络慢可加到 20000;平时保持 15000 就行。').volatile(),
33
+ fetchFormat: z
34
+ .union(['markdown', 'text'])
35
+ .default('markdown')
36
+ .description('抓取网页后返回什么样式:markdown 保留标题、列表、表格和链接,最适合阅读;text 只留纯文字、去掉所有标记。一般保持 markdown。')
37
+ .volatile(),
38
+ fetchMaxLength: z
39
+ .natural()
40
+ .min(1000)
41
+ .max(200000)
42
+ .default(DEFAULT_FETCH_MAX_LENGTH)
43
+ .description('一次抓取最多返回多少个字符(1000-200000);超出会在末尾注明原始长度。想一次多读可调到 50000,日常保持 20000 即可。')
44
+ .volatile(),
32
45
  wait: z
33
46
  .union(['auto', 'all', 'fast'])
34
47
  .default('auto')
@@ -54,6 +67,16 @@ export const Config = z.object({
54
67
  .volatile(),
55
68
  cacheTtlMs: z.natural().default(600000).description('同一问题多久内复用上次结果(缓存,即存一份备用),毫秒。想早点看到新内容就调小;保持 600000(10 分钟)即可。').volatile(),
56
69
  cacheEntries: z.natural().default(100).description('最多存多少条旧结果备用(缓存,即存起来省得重复查)。存满后先丢最久没用的;一般保持 100 即可。').volatile(),
70
+ cachePersist: z
71
+ .boolean()
72
+ .default(true)
73
+ .description('是否把缓存保存到硬盘,重启后还能复用上次结果。默认开启(true);不想留痕或磁盘只读时改成 false。')
74
+ .volatile(),
75
+ cacheFile: z
76
+ .string()
77
+ .default('')
78
+ .description('缓存文件放哪;留空用默认 ~/.dsh/cache/dsh-web-search/cache.json。一般不用填,换位置时再写绝对路径。')
79
+ .volatile(),
57
80
  anysearchApiKey: z
58
81
  .string()
59
82
  .default('')
@@ -117,10 +140,13 @@ export function resolveConfig(config, callLimit) {
117
140
  : String(rawDomainSources).split(/[\s,]+/);
118
141
  const domainSources = domainList.map((id) => id.trim()).filter(isVerticalId);
119
142
  const proxy = settings.proxy.trim();
143
+ const cacheFile = settings.cacheFile.trim();
120
144
  const limit = Math.min(callLimit ?? settings.limit, settings.limit);
121
145
  return {
122
146
  hostIds: hosts.length > 0 ? hosts : undefined,
123
147
  timeoutMs: settings.timeoutMs,
148
+ fetchFormat: settings.fetchFormat,
149
+ fetchMaxLength: settings.fetchMaxLength,
124
150
  proxy: proxy.length > 0 ? proxy : undefined,
125
151
  limit,
126
152
  wait: settings.wait,
@@ -139,6 +165,8 @@ export function resolveConfig(config, callLimit) {
139
165
  },
140
166
  cacheTtlMs: settings.cacheTtlMs,
141
167
  cacheEntries: settings.cacheEntries,
168
+ cachePersist: settings.cachePersist,
169
+ cacheFile: cacheFile.length > 0 ? cacheFile : undefined,
142
170
  baseCooldownMs: settings.baseCooldownMs,
143
171
  maxCooldownMs: settings.maxCooldownMs,
144
172
  };
package/lib/fetch.d.ts CHANGED
@@ -1,3 +1,4 @@
1
+ import type { FetchFormat } from './markdown.ts';
1
2
  /** One retrieval outcome, mirroring the Host's `WebFetchResult`. */
2
3
  export interface FetchOutcome {
3
4
  readonly url: string;
@@ -30,6 +31,12 @@ export interface FetchProviderOptions {
30
31
  anysearchApiKey?: string;
31
32
  /** Bytes kept from a response; the Host applies its own cap too. */
32
33
  maxBytes?: number;
34
+ /** Output style for an HTML body; `markdown` keeps structure. */
35
+ /** Renderer, or a reader for it so a Settings edit applies without a remount. */
36
+ format?: FetchFormat | (() => FetchFormat);
37
+ /** Character budget for the returned text. */
38
+ /** Character budget, or a reader for it. */
39
+ maxLength?: number | (() => number);
33
40
  /** User agent sent on every rung. */
34
41
  userAgent?: string;
35
42
  }