@fffangchenghao/dsh-web-search 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/client.js CHANGED
@@ -2,7 +2,7 @@
2
2
  * Browser half of `dsh-web-search-bing`.
3
3
  *
4
4
  * Registers one Settings page into the Settings surface's `settings.section`
5
- * slot, so this plugin's 14 configuration keys can be edited in the GUI instead
5
+ * slot, so this plugin's 22 configuration keys can be edited in the GUI instead
6
6
  * of the profile's `cordis.patch.yml`. The page reads and writes the plugin's
7
7
  * Loader entry (`web-search-bing`) through the shared `configForms` service,
8
8
  * which is the same path the shipped settings pages use.
@@ -52,10 +52,11 @@
52
52
  // English copy. Every label the page renders lives here.
53
53
  var en = {
54
54
  title: 'Web search (Bing)',
55
- description: 'Edit the key-free search provider: engine allow-list, wait policy, proxy, result budget, ranking, cache and cooldowns.',
55
+ description: 'Edit the key-free search provider: engine allow-list, wait policy, proxy, result budget, ranking, cache, cooldowns and dedicated sources.',
56
56
  'group.engines': 'Engines and transport',
57
57
  'group.ranking': 'Results and ranking',
58
58
  'group.cache': 'Cache and cooldowns',
59
+ 'group.domain': 'Dedicated sources',
59
60
  'wait.auto': 'Auto (query language)',
60
61
  'wait.all': 'All engines',
61
62
  'wait.fast': 'Fastest answer',
@@ -76,6 +77,10 @@
76
77
  'hint.wait': 'auto decides from the question language, all waits for everyone, fast takes the first answer',
77
78
  'field.proxy': 'Proxy (relay)',
78
79
  'hint.proxy': 'Relay address for blocked sites; empty auto-detects the system proxy',
80
+ 'field.fetchFormat': 'Fetched page style',
81
+ 'hint.fetchFormat': 'markdown keeps headings, lists, tables and links; text keeps words only',
82
+ 'field.fetchMaxLength': 'Fetched characters',
83
+ 'hint.fetchMaxLength': 'Characters returned per fetch (1000-200000); the rest is noted as truncated',
79
84
  'field.limit': 'Results per search',
80
85
  'hint.limit': 'How many results come back, and how they are split across engines',
81
86
  'field.rrfConstant': 'Result blending',
@@ -88,10 +93,23 @@
88
93
  'hint.freshnessBoost': 'How far recent pages move up',
89
94
  'field.freshnessDays': 'Freshness window (days)',
90
95
  'hint.freshnessDays': 'Older pages stop receiving the freshness boost',
96
+ 'field.timeRange': 'Time range',
97
+ 'hint.timeRange': 'Only keep results published inside this window; undated results stay but rank lower',
98
+ 'timeRange.off': 'Any time',
99
+ 'timeRange.day': 'Past 24 hours',
100
+ 'timeRange.week': 'Past 7 days',
101
+ 'timeRange.month': 'Past 30 days',
102
+ 'timeRange.year': 'Past year',
91
103
  'field.cacheTtlMs': 'Cache lifetime',
92
104
  'hint.cacheTtlMs': 'How long an identical question reuses its cached answer (ms)',
93
105
  'field.cacheEntries': 'Cache size',
94
106
  'hint.cacheEntries': 'How many answers are remembered before the least recently used goes',
107
+ 'field.cachePersist': 'Save cache to disk',
108
+ 'hint.cachePersist': 'Keep recent answers across restarts; turn it off to stay memory-only',
109
+ 'cachePersist.true': 'On (save to disk)',
110
+ 'cachePersist.false': 'Off (memory only)',
111
+ 'field.cacheFile': 'Cache file',
112
+ 'hint.cacheFile': 'Empty uses ~/.dsh/cache/dsh-web-search/cache.json',
95
113
  'field.anysearchApiKey': 'AnySearch key',
96
114
  'hint.anysearchApiKey': 'Leave empty to stay anonymous at a lower rate limit; only sent to api.anysearch.com',
97
115
  'field.anysearchDomain': 'Vertical domain',
@@ -100,16 +118,23 @@
100
118
  'hint.baseCooldownMs': 'How long an engine rests after it reports throttling (ms)',
101
119
  'field.maxCooldownMs': 'Max cooldown (ms)',
102
120
  'hint.maxCooldownMs': 'Ceiling for repeated throttling (ms); rarely worth changing',
121
+ 'field.domainSources': 'Dedicated sources',
122
+ 'hint.domainSources': 'github, npm, pypi, arxiv, wikipedia, hn, stackexchange, crates; empty means all eight',
123
+ 'field.domainLimit': 'Results per source',
124
+ 'hint.domainLimit': 'How many results each dedicated source returns (1-20); 5 is a sane default',
125
+ 'field.domainTimeoutMs': 'Source wait limit (ms)',
126
+ 'hint.domainTimeoutMs': 'How long one dedicated source may take; 8000 normally, 15000 on a slow network',
103
127
  invalidNumber: 'Enter a number within the allowed range, or leave blank for the default.'
104
128
  };
105
129
 
106
130
  // Simplified Chinese copy.
107
131
  var zh = {
108
132
  title: '网页搜索(Bing)',
109
- description: '编辑免密钥搜索提供方:引擎白名单、等待策略、代理、结果上限、排序、缓存与冷却。',
133
+ description: '编辑免密钥搜索提供方:引擎白名单、等待策略、代理、结果上限、排序、缓存、冷却与专用检索源。',
110
134
  'group.engines': '引擎与传输',
111
135
  'group.ranking': '结果与排序',
112
136
  'group.cache': '缓存与冷却',
137
+ 'group.domain': '专用检索',
113
138
  'wait.auto': '自动(按提问语言)',
114
139
  'wait.all': '等待全部引擎',
115
140
  'wait.fast': '最快回答',
@@ -130,6 +155,10 @@
130
155
  'hint.wait': 'auto 按提问语言自动决定,all 等全部,fast 先到先用',
131
156
  'field.proxy': '代理(中转站)',
132
157
  'hint.proxy': '上网中转站地址;留空自动识别系统代理',
158
+ 'field.fetchFormat': '抓取内容的样式',
159
+ 'hint.fetchFormat': 'markdown 保留标题、列表、表格和链接;text 只留纯文字',
160
+ 'field.fetchMaxLength': '抓取字符上限',
161
+ 'hint.fetchMaxLength': '一次抓取最多返回多少字符(1000-200000);超出部分会注明已截断',
133
162
  'field.limit': '返回几条',
134
163
  'hint.limit': '一次最多返回几条,也决定各引擎分到几条',
135
164
  'field.rrfConstant': '结果混合力度',
@@ -142,10 +171,23 @@
142
171
  'hint.freshnessBoost': '较新的网页往上提多少;常查新闻可调大',
143
172
  'field.freshnessDays': '多少天内算新',
144
173
  'hint.freshnessDays': '超过这些天的结果不再享受时效加分',
174
+ 'field.timeRange': '时间范围',
175
+ 'hint.timeRange': '只保留这段时间内发布的结果;没有发布日期的结果会保留但排后面',
176
+ 'timeRange.off': '不限时间',
177
+ 'timeRange.day': '最近 24 小时',
178
+ 'timeRange.week': '最近 7 天',
179
+ 'timeRange.month': '最近 30 天',
180
+ 'timeRange.year': '最近 1 年',
145
181
  'field.cacheTtlMs': '缓存保留多久',
146
182
  'hint.cacheTtlMs': '同一问题复用旧结果多久(毫秒);想更快见新内容就调小',
147
183
  'field.cacheEntries': '缓存条数上限',
148
184
  'hint.cacheEntries': '最多记住多少条旧结果,满了丢最久没用的',
185
+ 'field.cachePersist': '缓存保存到磁盘',
186
+ 'hint.cachePersist': '重启后仍复用最近结果;关闭则只在内存中保存',
187
+ 'cachePersist.true': '开启(保存到磁盘)',
188
+ 'cachePersist.false': '关闭(仅内存)',
189
+ 'field.cacheFile': '缓存文件路径',
190
+ 'hint.cacheFile': '留空使用 ~/.dsh/cache/dsh-web-search/cache.json',
149
191
  'field.anysearchApiKey': 'AnySearch 通行证',
150
192
  'hint.anysearchApiKey': '没有就留空,仍可用但速率较低;只发给 api.anysearch.com',
151
193
  'field.anysearchDomain': '专业领域',
@@ -154,6 +196,12 @@
154
196
  'hint.baseCooldownMs': '某个引擎说太频繁后先歇多久(毫秒);默认 5 分钟即可',
155
197
  'field.maxCooldownMs': '歇息上限',
156
198
  'hint.maxCooldownMs': '反复被拒时歇息时间的上限(毫秒);一般不用改',
199
+ 'field.domainSources': '默认查哪些专用源',
200
+ 'hint.domainSources': 'github、npm、pypi、arxiv、wikipedia、hn、stackexchange、crates;留空查全部',
201
+ 'field.domainLimit': '每个源返回几条',
202
+ 'hint.domainLimit': '每个专用源最多返回几条(1-20);默认 5 条即可',
203
+ 'field.domainTimeoutMs': '每个源等待上限',
204
+ 'hint.domainTimeoutMs': '单个专用源最多等多久(毫秒);平时 8000,网络慢可加到 15000',
157
205
  invalidNumber: '请填写允许范围内的数字;留空表示使用默认值。'
158
206
  };
159
207
 
@@ -163,6 +211,8 @@
163
211
  { key: 'wait', group: 'engines', kind: 'select', options: ['auto', 'all', 'fast'] },
164
212
  { key: 'timeoutMs', group: 'engines', kind: 'number', integer: true, min: 500 },
165
213
  { key: 'proxy', group: 'engines', kind: 'text' },
214
+ { key: 'fetchFormat', group: 'engines', kind: 'select', options: ['markdown', 'text'] },
215
+ { key: 'fetchMaxLength', group: 'engines', kind: 'number', integer: true, min: 1000, max: 200000 },
166
216
  { key: 'anysearchApiKey', group: 'engines', kind: 'text' },
167
217
  { key: 'anysearchDomain', group: 'engines', kind: 'text' },
168
218
  { key: 'limit', group: 'ranking', kind: 'number', integer: true, min: 1, max: 20 },
@@ -171,14 +221,20 @@
171
221
  { key: 'authorityBoost', group: 'ranking', kind: 'number', min: 0, max: 10 },
172
222
  { key: 'freshnessBoost', group: 'ranking', kind: 'number', min: 0, max: 10 },
173
223
  { key: 'freshnessDays', group: 'ranking', kind: 'number', integer: true, min: 1 },
224
+ { key: 'timeRange', group: 'ranking', kind: 'select', options: ['off', 'day', 'week', 'month', 'year'] },
174
225
  { key: 'cacheTtlMs', group: 'cache', kind: 'number', integer: true, min: 0 },
175
226
  { key: 'cacheEntries', group: 'cache', kind: 'number', integer: true, min: 0 },
227
+ { key: 'cachePersist', group: 'cache', kind: 'select', options: ['true', 'false'], boolean: true },
228
+ { key: 'cacheFile', group: 'cache', kind: 'text' },
176
229
  { key: 'baseCooldownMs', group: 'cache', kind: 'number', integer: true, min: 0 },
177
- { key: 'maxCooldownMs', group: 'cache', kind: 'number', integer: true, min: 0 }
230
+ { key: 'maxCooldownMs', group: 'cache', kind: 'number', integer: true, min: 0 },
231
+ { key: 'domainSources', group: 'domain', kind: 'list' },
232
+ { key: 'domainLimit', group: 'domain', kind: 'number', integer: true, min: 1, max: 20 },
233
+ { key: 'domainTimeoutMs', group: 'domain', kind: 'number', integer: true, min: 500 }
178
234
  ];
179
235
 
180
236
  /** Groups in render order. */
181
- var GROUPS = ['engines', 'ranking', 'cache'].map(function (group) {
237
+ var GROUPS = ['engines', 'ranking', 'cache', 'domain'].map(function (group) {
182
238
  return { key: group, fields: FIELDS.filter(function (field) { return field.group === group; }) };
183
239
  });
184
240
 
@@ -314,7 +370,7 @@
314
370
  var field = FIELDS[i];
315
371
  var value = values === undefined ? undefined : values[field.key];
316
372
  draft[field.key] = field.kind === 'list'
317
- ? (Array.isArray(value) ? value.join(', ') : '')
373
+ ? (Array.isArray(value) ? value.join(', ') : (typeof value === 'string' ? value : ''))
318
374
  : (value === undefined || value === null ? '' : String(value));
319
375
  }
320
376
  return draft;
@@ -337,8 +393,9 @@
337
393
  return { op: 'set', path: [key], value: list };
338
394
  }
339
395
  if (field.kind === 'select') {
340
- if (raw === current) return null;
341
- return raw === '' ? { op: 'unset', path: [key] } : { op: 'set', path: [key], value: raw };
396
+ var next = field.boolean === true ? raw === 'true' : raw;
397
+ if (next === current) return null;
398
+ return raw === '' ? { op: 'unset', path: [key] } : { op: 'set', path: [key], value: next };
342
399
  }
343
400
  if (field.kind === 'text') {
344
401
  var text = String(raw === undefined || raw === null ? '' : raw);
@@ -486,7 +543,7 @@
486
543
  style: inputStyle,
487
544
  onChange: function (event) { edit(key, event.target.value); }
488
545
  }, field.options.map(function (option) {
489
- return h('option', { key: option, value: option }, t('wait.' + option));
546
+ return h('option', { key: option, value: option }, t(field.key + '.' + option));
490
547
  }));
491
548
  } else {
492
549
  control = h('input', {
@@ -0,0 +1,96 @@
1
+ import type { Source } from './types.ts';
2
+ /** Version of the on-disk document; a different version is ignored wholesale. */
3
+ export declare const CACHE_FILE_VERSION = 1;
4
+ /** Default byte budget for the snapshot: 4 MiB. */
5
+ export declare const DEFAULT_CACHE_FILE_MAX_BYTES: number;
6
+ /** One cache entry as it travels to disk. */
7
+ export interface PersistedEntry {
8
+ /** Deterministic key produced by `cacheKey`. */
9
+ key: string;
10
+ /** Absolute expiry timestamp; expired entries are dropped on load. */
11
+ expiresAt: number;
12
+ /** The search-result summary, never a page body. */
13
+ sources: Source[];
14
+ }
15
+ /** Filesystem seam: every call may throw, and the caller contains it. */
16
+ export interface CacheFileIo {
17
+ /** Read the file as UTF-8, or return `undefined` when it is absent. */
18
+ read: (path: string) => string | undefined;
19
+ /** Write a new file (the temporary one). */
20
+ write: (path: string, data: string) => void;
21
+ /** Atomically replace the target with the temporary file. */
22
+ rename: (from: string, to: string) => void;
23
+ /** Create the containing directory, including parents. */
24
+ mkdir: (path: string) => void;
25
+ /** Best-effort removal of the temporary file. */
26
+ remove: (path: string) => void;
27
+ }
28
+ /** The real filesystem, exported so tests can override one operation. */
29
+ export declare const nodeCacheFileIo: CacheFileIo;
30
+ /** Where a snapshot is written when the config names no path. */
31
+ export declare function defaultCacheFile(): string;
32
+ /**
33
+ * Expand a configured cache path.
34
+ *
35
+ * An empty value selects {@link defaultCacheFile}; a leading `~` expands to the
36
+ * home directory so the documented default can be written verbatim in Settings.
37
+ * Relative paths are resolved against the process working directory.
38
+ *
39
+ * @param value - the `cacheFile` setting, possibly empty.
40
+ * @returns an absolute file path.
41
+ */
42
+ export declare function resolveCacheFile(value?: string): string;
43
+ /**
44
+ * Load the snapshot from disk.
45
+ *
46
+ * Every failure mode collapses to an empty list: absent file, unreadable path,
47
+ * malformed JSON, a different `version`, and individually invalid entries. An
48
+ * expired entry is dropped here rather than retained until its next read.
49
+ *
50
+ * @param file - absolute path to the snapshot.
51
+ * @param now - current time from the cache's injected clock.
52
+ * @param maxEntries - entry cap; the oldest surplus entries are discarded.
53
+ * @param io - filesystem seam, defaulting to the real one.
54
+ * @returns usable entries in stored (oldest to newest) order.
55
+ */
56
+ export declare function readCacheFile(file: string, now: number, maxEntries: number, io?: CacheFileIo): PersistedEntry[];
57
+ /** One serialised snapshot plus what survived the byte cap. */
58
+ export interface SerializedCache {
59
+ /** UTF-8 JSON document, ready for the atomic write. */
60
+ data: string;
61
+ /** Number of entries left after LRU eviction. */
62
+ entries: number;
63
+ /** Size of {@link data} in bytes. */
64
+ bytes: number;
65
+ }
66
+ /**
67
+ * Serialise entries, evicting the oldest until the document fits the budget.
68
+ *
69
+ * @param entries - entries in LRU order (oldest first).
70
+ * @param now - timestamp recorded as `savedAt`.
71
+ * @param maxBytes - byte budget; defaults to 4 MiB.
72
+ * @returns the document and how many entries it carries.
73
+ */
74
+ export declare function serializeCache(entries: readonly PersistedEntry[], now: number, maxBytes?: number): SerializedCache;
75
+ /** Write options, kept separate so the cache owns the policy, not the path. */
76
+ export interface CacheWriteOptions {
77
+ /** Byte budget; defaults to 4 MiB. */
78
+ maxBytes?: number;
79
+ /** Filesystem seam for tests. */
80
+ io?: CacheFileIo;
81
+ }
82
+ /**
83
+ * Atomically replace the snapshot.
84
+ *
85
+ * The temporary file lives beside the target so the `rename` stays on one
86
+ * filesystem. When anything fails (directory not writable, rename rejected),
87
+ * the previous document is untouched, the temporary file is removed, and the
88
+ * call returns `false` instead of throwing.
89
+ *
90
+ * @param file - absolute target path.
91
+ * @param entries - entries in LRU order (oldest first).
92
+ * @param now - timestamp recorded as `savedAt`.
93
+ * @param options - byte budget and filesystem seam.
94
+ * @returns `true` when the target now holds this snapshot.
95
+ */
96
+ export declare function writeCacheFile(file: string, entries: readonly PersistedEntry[], now: number, options?: CacheWriteOptions): boolean;
@@ -0,0 +1,249 @@
1
+ /**
2
+ * Optional disk persistence for the search cache.
3
+ *
4
+ * The cache itself stays a tiny TTL+LRU map; this module only knows how to
5
+ * turn a snapshot of it into a file and back. Three properties matter more
6
+ * than speed here:
7
+ *
8
+ * - **Never fail the plugin.** A missing, corrupt, foreign-version or
9
+ * unreadable file is treated as an empty cache. Every filesystem call is
10
+ * contained, because a cache is an optimisation and must never be the reason
11
+ * a plugin refuses to mount.
12
+ * - **Never leave a half-written file.** The snapshot is written to a sibling
13
+ * temporary file and then `rename`d over the target, which is atomic on the
14
+ * same filesystem. A failure before the rename leaves the previous file
15
+ * intact and removes the temporary one.
16
+ * - **Never grow without bound.** The serialised snapshot is trimmed in LRU
17
+ * order (oldest first) until it fits the byte budget.
18
+ *
19
+ * Only the search-result summary (`url`, `title`, `snippet`, `publishedAt`) is
20
+ * stored. Page bodies are never touched by this cache, and any extra property a
21
+ * caller attached to a {@link Source} is dropped at the serialisation boundary.
22
+ *
23
+ * @module @fangchenghao/dsh-web-search/cache-persist
24
+ */
25
+ import { readFileSync, renameSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
26
+ import { homedir } from 'node:os';
27
+ import { dirname, join, resolve } from 'node:path';
28
+ /** Version of the on-disk document; a different version is ignored wholesale. */
29
+ export const CACHE_FILE_VERSION = 1;
30
+ /** Default byte budget for the snapshot: 4 MiB. */
31
+ export const DEFAULT_CACHE_FILE_MAX_BYTES = 4 * 1024 * 1024;
32
+ /** The real filesystem, exported so tests can override one operation. */
33
+ export const nodeCacheFileIo = {
34
+ read(path) {
35
+ try {
36
+ return readFileSync(path, 'utf8');
37
+ }
38
+ catch {
39
+ // Missing file, permission error, directory: all mean "empty cache".
40
+ return undefined;
41
+ }
42
+ },
43
+ write(path, data) {
44
+ // The file only holds public search summaries, but 0600 keeps a shared
45
+ // machine from reading another user's queries.
46
+ writeFileSync(path, data, { encoding: 'utf8', mode: 0o600 });
47
+ },
48
+ rename(from, to) {
49
+ renameSync(from, to);
50
+ },
51
+ mkdir(path) {
52
+ mkdirSync(path, { recursive: true });
53
+ },
54
+ remove(path) {
55
+ rmSync(path, { force: true });
56
+ },
57
+ };
58
+ /** Where a snapshot is written when the config names no path. */
59
+ export function defaultCacheFile() {
60
+ return join(homedir(), '.dsh', 'cache', 'dsh-web-search', 'cache.json');
61
+ }
62
+ /**
63
+ * Expand a configured cache path.
64
+ *
65
+ * An empty value selects {@link defaultCacheFile}; a leading `~` expands to the
66
+ * home directory so the documented default can be written verbatim in Settings.
67
+ * Relative paths are resolved against the process working directory.
68
+ *
69
+ * @param value - the `cacheFile` setting, possibly empty.
70
+ * @returns an absolute file path.
71
+ */
72
+ export function resolveCacheFile(value) {
73
+ const trimmed = (value ?? '').trim();
74
+ if (trimmed.length === 0)
75
+ return defaultCacheFile();
76
+ const expanded = trimmed === '~'
77
+ ? homedir()
78
+ : /^~[\\/]/u.test(trimmed)
79
+ ? join(homedir(), trimmed.slice(2))
80
+ : trimmed;
81
+ return resolve(expanded);
82
+ }
83
+ /** True for a value usable as a plain record. */
84
+ function isRecord(value) {
85
+ return typeof value === 'object' && value !== null && !Array.isArray(value);
86
+ }
87
+ /** Read one persisted source, keeping only the four summary fields. */
88
+ function readSource(raw) {
89
+ if (!isRecord(raw))
90
+ return undefined;
91
+ const url = raw['url'];
92
+ if (typeof url !== 'string' || url.trim().length === 0)
93
+ return undefined;
94
+ const source = { url };
95
+ if (typeof raw['title'] === 'string' && raw['title'].length > 0)
96
+ source.title = raw['title'];
97
+ if (typeof raw['snippet'] === 'string' && raw['snippet'].length > 0)
98
+ source.snippet = raw['snippet'];
99
+ if (typeof raw['publishedAt'] === 'string' && raw['publishedAt'].length > 0) {
100
+ source.publishedAt = raw['publishedAt'];
101
+ }
102
+ return source;
103
+ }
104
+ /** Write one source as a summary-only record. */
105
+ function storedSource(source) {
106
+ const stored = { url: source.url };
107
+ if (source.title !== undefined && source.title.length > 0)
108
+ stored['title'] = source.title;
109
+ if (source.snippet !== undefined && source.snippet.length > 0)
110
+ stored['snippet'] = source.snippet;
111
+ if (source.publishedAt !== undefined && source.publishedAt.length > 0) {
112
+ stored['publishedAt'] = source.publishedAt;
113
+ }
114
+ return stored;
115
+ }
116
+ /** Read one persisted entry, rejecting anything malformed or already expired. */
117
+ function readEntry(raw, now) {
118
+ if (!isRecord(raw))
119
+ return undefined;
120
+ const key = raw['key'];
121
+ const expiresAt = raw['expiresAt'];
122
+ const rawSources = raw['sources'];
123
+ if (typeof key !== 'string' || key.length === 0)
124
+ return undefined;
125
+ if (typeof expiresAt !== 'number' || !Number.isFinite(expiresAt) || expiresAt <= now)
126
+ return undefined;
127
+ if (!Array.isArray(rawSources))
128
+ return undefined;
129
+ const sources = [];
130
+ for (const item of rawSources) {
131
+ const source = readSource(item);
132
+ if (source !== undefined)
133
+ sources.push(source);
134
+ }
135
+ // An entry with no usable result is not worth a slot in the file.
136
+ if (sources.length === 0)
137
+ return undefined;
138
+ return { key, expiresAt, sources };
139
+ }
140
+ /** Clamp the byte budget to a usable positive integer. */
141
+ function normaliseMaxBytes(value) {
142
+ return typeof value === 'number' && Number.isFinite(value) && value > 0
143
+ ? Math.floor(value)
144
+ : DEFAULT_CACHE_FILE_MAX_BYTES;
145
+ }
146
+ /**
147
+ * Load the snapshot from disk.
148
+ *
149
+ * Every failure mode collapses to an empty list: absent file, unreadable path,
150
+ * malformed JSON, a different `version`, and individually invalid entries. An
151
+ * expired entry is dropped here rather than retained until its next read.
152
+ *
153
+ * @param file - absolute path to the snapshot.
154
+ * @param now - current time from the cache's injected clock.
155
+ * @param maxEntries - entry cap; the oldest surplus entries are discarded.
156
+ * @param io - filesystem seam, defaulting to the real one.
157
+ * @returns usable entries in stored (oldest to newest) order.
158
+ */
159
+ export function readCacheFile(file, now, maxEntries, io) {
160
+ try {
161
+ const text = (io ?? nodeCacheFileIo).read(file);
162
+ if (text === undefined)
163
+ return [];
164
+ const parsed = JSON.parse(text);
165
+ if (!isRecord(parsed))
166
+ return [];
167
+ if (parsed['version'] !== CACHE_FILE_VERSION)
168
+ return [];
169
+ const rawEntries = parsed['entries'];
170
+ if (!Array.isArray(rawEntries))
171
+ return [];
172
+ const entries = [];
173
+ for (const raw of rawEntries) {
174
+ const entry = readEntry(raw, now);
175
+ if (entry !== undefined)
176
+ entries.push(entry);
177
+ }
178
+ const cap = Number.isFinite(maxEntries) ? Math.max(0, Math.floor(maxEntries)) : entries.length;
179
+ return entries.length > cap ? entries.slice(entries.length - cap) : entries;
180
+ }
181
+ catch {
182
+ // Corrupt JSON, a foreign document, a getter that throws: empty cache.
183
+ return [];
184
+ }
185
+ }
186
+ /**
187
+ * Serialise entries, evicting the oldest until the document fits the budget.
188
+ *
189
+ * @param entries - entries in LRU order (oldest first).
190
+ * @param now - timestamp recorded as `savedAt`.
191
+ * @param maxBytes - byte budget; defaults to 4 MiB.
192
+ * @returns the document and how many entries it carries.
193
+ */
194
+ export function serializeCache(entries, now, maxBytes) {
195
+ const budget = normaliseMaxBytes(maxBytes);
196
+ const kept = entries.map((entry) => ({
197
+ key: entry.key,
198
+ expiresAt: entry.expiresAt,
199
+ sources: entry.sources.map(storedSource),
200
+ }));
201
+ const build = () => JSON.stringify({
202
+ version: CACHE_FILE_VERSION,
203
+ savedAt: now,
204
+ entries: kept,
205
+ });
206
+ let data = build();
207
+ while (kept.length > 0 && Buffer.byteLength(data, 'utf8') > budget) {
208
+ kept.shift();
209
+ data = build();
210
+ }
211
+ return { data, entries: kept.length, bytes: Buffer.byteLength(data, 'utf8') };
212
+ }
213
+ /** Distinguishes temporary files of concurrent writers in this process. */
214
+ let tempCounter = 0;
215
+ /**
216
+ * Atomically replace the snapshot.
217
+ *
218
+ * The temporary file lives beside the target so the `rename` stays on one
219
+ * filesystem. When anything fails (directory not writable, rename rejected),
220
+ * the previous document is untouched, the temporary file is removed, and the
221
+ * call returns `false` instead of throwing.
222
+ *
223
+ * @param file - absolute target path.
224
+ * @param entries - entries in LRU order (oldest first).
225
+ * @param now - timestamp recorded as `savedAt`.
226
+ * @param options - byte budget and filesystem seam.
227
+ * @returns `true` when the target now holds this snapshot.
228
+ */
229
+ export function writeCacheFile(file, entries, now, options = {}) {
230
+ const io = options.io ?? nodeCacheFileIo;
231
+ const { data } = serializeCache(entries, now, options.maxBytes);
232
+ tempCounter += 1;
233
+ const temp = `${file}.${process.pid}.${tempCounter}.tmp`;
234
+ try {
235
+ io.mkdir(dirname(file));
236
+ io.write(temp, data);
237
+ io.rename(temp, file);
238
+ return true;
239
+ }
240
+ catch {
241
+ try {
242
+ io.remove(temp);
243
+ }
244
+ catch {
245
+ // Cleanup is best effort; the previous snapshot is still intact.
246
+ }
247
+ return false;
248
+ }
249
+ }
package/lib/cache.d.ts CHANGED
@@ -12,7 +12,24 @@
12
12
  *
13
13
  * @module @fangchenghao/dsh-web-search/cache
14
14
  */
15
- import type { CacheOptions, Source } from './types.ts';
15
+ import type { CacheOptions, Source, TimeRange } from './types.ts';
16
+ import type { CacheFileIo } from './cache-persist.ts';
17
+ /** Optional disk persistence, layered on top of the in-memory cache. */
18
+ export interface CachePersistOptions {
19
+ /** Custom snapshot path; empty or absent uses `~/.dsh/cache/dsh-web-search/cache.json`. */
20
+ file?: string;
21
+ /** Serialised byte budget; the oldest entries are evicted to fit. */
22
+ maxBytes?: number;
23
+ /** Minimum gap between writes in milliseconds; defaults to two seconds. */
24
+ throttleMs?: number;
25
+ /** Filesystem seam, mainly for tests. */
26
+ io?: CacheFileIo;
27
+ }
28
+ /** Constructor options: {@link CacheOptions} plus optional persistence. */
29
+ export interface SearchCacheOptions extends CacheOptions {
30
+ /** Persist the cache when present; absent keeps the cache purely in memory. */
31
+ persist?: CachePersistOptions;
32
+ }
16
33
  /** The logical inputs that decide whether two searches may share an entry. */
17
34
  export interface CacheKeyParts {
18
35
  /** Raw query text; normalised inside {@link cacheKey}. */
@@ -25,6 +42,8 @@ export interface CacheKeyParts {
25
42
  lead: string;
26
43
  /** Engine ids that took part in the run. */
27
44
  engines: string[];
45
+ /** Active publication window; omitted and `off` share a key. */
46
+ timeRange?: TimeRange;
28
47
  }
29
48
  /**
30
49
  * Build the cache key for one search request.
@@ -32,9 +51,11 @@ export interface CacheKeyParts {
32
51
  * The query is trimmed, lowercased and has internal whitespace collapsed, so
33
52
  * `' PostgreSQL Replication '` and `'postgresql replication'` share an
34
53
  * entry. Engine order never matters because the ids are sorted; every other
35
- * field is positional, so differing `limit`, `wait`, `lead` or engine set
36
- * yields a different key. The query is last so a query containing the field
37
- * separator cannot collide with a different request's prefix.
54
+ * field is positional, so differing `limit`, `wait`, `lead`, `timeRange` or
55
+ * engine set yields a different key. `timeRange` is appended only when it is
56
+ * active, which keeps the key for an unfiltered search unchanged. The query is
57
+ * last so a query containing the field separator cannot collide with a
58
+ * different request's prefix.
38
59
  *
39
60
  * @param parts - the logical request.
40
61
  * @returns a deterministic, comparison-safe cache key.
@@ -52,18 +73,28 @@ export declare function cacheKey(parts: CacheKeyParts): string;
52
73
  */
53
74
  export declare class SearchCache {
54
75
  /** Entry lifetime in milliseconds; `0` disables the cache completely. */
55
- private readonly ttlMs;
76
+ /** Live TTL reader: settings may change without a remount. */
77
+ private readonly ttlOf;
56
78
  /** Maximum retained entries; the oldest read is evicted first. */
57
- private readonly maxEntries;
79
+ /** Live entry-cap reader: settings may change without a remount. */
80
+ private readonly maxOf;
58
81
  /** Injected clock, so tests never wait on real time. */
59
82
  private readonly now;
60
83
  /** Insertion order doubles as LRU order: the first key is the coldest. */
61
84
  private readonly entries;
85
+ /** Snapshot destination, or `undefined` for a purely in-memory cache. */
86
+ private readonly persistTo;
87
+ /** Minimum gap between writes. */
88
+ private readonly throttleMs;
89
+ /** Whether memory changed since the last successful write. */
90
+ private dirty;
91
+ /** Pending throttled flush. */
92
+ private timer;
62
93
  /**
63
- * @param options - TTL and entry cap, both optional.
94
+ * @param options - TTL, entry cap and optional persistence.
64
95
  * @param now - clock override; defaults to `Date.now`.
65
96
  */
66
- constructor(options?: CacheOptions, now?: () => number);
97
+ constructor(options?: SearchCacheOptions, now?: () => number);
67
98
  /** Number of retained entries, including ones whose TTL already elapsed. */
68
99
  get size(): number;
69
100
  /**
@@ -83,6 +114,23 @@ export declare class SearchCache {
83
114
  * @param sources - the merged source list; copied before storage.
84
115
  */
85
116
  set(key: string, sources: Source[]): void;
117
+ /**
118
+ * Write the snapshot now, if there is anything to write.
119
+ *
120
+ * Writes are normally throttled and happen on a timer or at process exit;
121
+ * this is the deterministic seam tests use, and the call a shutdown path can
122
+ * make when it wants the newest state on disk immediately.
123
+ *
124
+ * @returns `true` when the file was replaced, `false` for "nothing to write"
125
+ * or a contained filesystem failure.
126
+ */
127
+ flush(): boolean;
128
+ /** Stop the exit hook and flush whatever is still pending. */
129
+ close(): void;
86
130
  /** Drop every entry. */
87
131
  clear(): void;
132
+ /** Mark the snapshot stale and arrange a throttled flush. */
133
+ private touch;
134
+ /** Arm the single pending timer, if persistence is enabled. */
135
+ private schedule;
88
136
  }