@fffangchenghao/dsh-web-search 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +132 -34
- package/README.zh-CN.md +132 -34
- package/client.js +28 -3
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +48 -4
- package/lib/cache.js +134 -9
- package/lib/config.d.ts +16 -0
- package/lib/config.js +28 -0
- package/lib/fetch.d.ts +7 -0
- package/lib/fetch.js +36 -3
- package/lib/index.d.ts +20 -0
- package/lib/index.js +91 -27
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/types.d.ts +7 -4
- package/package.json +1 -1
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional disk persistence for the search cache.
|
|
3
|
+
*
|
|
4
|
+
* The cache itself stays a tiny TTL+LRU map; this module only knows how to
|
|
5
|
+
* turn a snapshot of it into a file and back. Three properties matter more
|
|
6
|
+
* than speed here:
|
|
7
|
+
*
|
|
8
|
+
* - **Never fail the plugin.** A missing, corrupt, foreign-version or
|
|
9
|
+
* unreadable file is treated as an empty cache. Every filesystem call is
|
|
10
|
+
* contained, because a cache is an optimisation and must never be the reason
|
|
11
|
+
* a plugin refuses to mount.
|
|
12
|
+
* - **Never leave a half-written file.** The snapshot is written to a sibling
|
|
13
|
+
* temporary file and then `rename`d over the target, which is atomic on the
|
|
14
|
+
* same filesystem. A failure before the rename leaves the previous file
|
|
15
|
+
* intact and removes the temporary one.
|
|
16
|
+
* - **Never grow without bound.** The serialised snapshot is trimmed in LRU
|
|
17
|
+
* order (oldest first) until it fits the byte budget.
|
|
18
|
+
*
|
|
19
|
+
* Only the search-result summary (`url`, `title`, `snippet`, `publishedAt`) is
|
|
20
|
+
* stored. Page bodies are never touched by this cache, and any extra property a
|
|
21
|
+
* caller attached to a {@link Source} is dropped at the serialisation boundary.
|
|
22
|
+
*
|
|
23
|
+
* @module @fangchenghao/dsh-web-search/cache-persist
|
|
24
|
+
*/
|
|
25
|
+
import { readFileSync, renameSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
|
|
26
|
+
import { homedir } from 'node:os';
|
|
27
|
+
import { dirname, join, resolve } from 'node:path';
|
|
28
|
+
/** Version of the on-disk document; a different version is ignored wholesale. */
|
|
29
|
+
export const CACHE_FILE_VERSION = 1;
|
|
30
|
+
/** Default byte budget for the snapshot: 4 MiB. */
|
|
31
|
+
export const DEFAULT_CACHE_FILE_MAX_BYTES = 4 * 1024 * 1024;
|
|
32
|
+
/** The real filesystem, exported so tests can override one operation. */
|
|
33
|
+
export const nodeCacheFileIo = {
|
|
34
|
+
read(path) {
|
|
35
|
+
try {
|
|
36
|
+
return readFileSync(path, 'utf8');
|
|
37
|
+
}
|
|
38
|
+
catch {
|
|
39
|
+
// Missing file, permission error, directory: all mean "empty cache".
|
|
40
|
+
return undefined;
|
|
41
|
+
}
|
|
42
|
+
},
|
|
43
|
+
write(path, data) {
|
|
44
|
+
// The file only holds public search summaries, but 0600 keeps a shared
|
|
45
|
+
// machine from reading another user's queries.
|
|
46
|
+
writeFileSync(path, data, { encoding: 'utf8', mode: 0o600 });
|
|
47
|
+
},
|
|
48
|
+
rename(from, to) {
|
|
49
|
+
renameSync(from, to);
|
|
50
|
+
},
|
|
51
|
+
mkdir(path) {
|
|
52
|
+
mkdirSync(path, { recursive: true });
|
|
53
|
+
},
|
|
54
|
+
remove(path) {
|
|
55
|
+
rmSync(path, { force: true });
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
/** Where a snapshot is written when the config names no path. */
|
|
59
|
+
export function defaultCacheFile() {
|
|
60
|
+
return join(homedir(), '.dsh', 'cache', 'dsh-web-search', 'cache.json');
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Expand a configured cache path.
|
|
64
|
+
*
|
|
65
|
+
* An empty value selects {@link defaultCacheFile}; a leading `~` expands to the
|
|
66
|
+
* home directory so the documented default can be written verbatim in Settings.
|
|
67
|
+
* Relative paths are resolved against the process working directory.
|
|
68
|
+
*
|
|
69
|
+
* @param value - the `cacheFile` setting, possibly empty.
|
|
70
|
+
* @returns an absolute file path.
|
|
71
|
+
*/
|
|
72
|
+
export function resolveCacheFile(value) {
|
|
73
|
+
const trimmed = (value ?? '').trim();
|
|
74
|
+
if (trimmed.length === 0)
|
|
75
|
+
return defaultCacheFile();
|
|
76
|
+
const expanded = trimmed === '~'
|
|
77
|
+
? homedir()
|
|
78
|
+
: /^~[\\/]/u.test(trimmed)
|
|
79
|
+
? join(homedir(), trimmed.slice(2))
|
|
80
|
+
: trimmed;
|
|
81
|
+
return resolve(expanded);
|
|
82
|
+
}
|
|
83
|
+
/** True for a value usable as a plain record. */
|
|
84
|
+
function isRecord(value) {
|
|
85
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
86
|
+
}
|
|
87
|
+
/** Read one persisted source, keeping only the four summary fields. */
|
|
88
|
+
function readSource(raw) {
|
|
89
|
+
if (!isRecord(raw))
|
|
90
|
+
return undefined;
|
|
91
|
+
const url = raw['url'];
|
|
92
|
+
if (typeof url !== 'string' || url.trim().length === 0)
|
|
93
|
+
return undefined;
|
|
94
|
+
const source = { url };
|
|
95
|
+
if (typeof raw['title'] === 'string' && raw['title'].length > 0)
|
|
96
|
+
source.title = raw['title'];
|
|
97
|
+
if (typeof raw['snippet'] === 'string' && raw['snippet'].length > 0)
|
|
98
|
+
source.snippet = raw['snippet'];
|
|
99
|
+
if (typeof raw['publishedAt'] === 'string' && raw['publishedAt'].length > 0) {
|
|
100
|
+
source.publishedAt = raw['publishedAt'];
|
|
101
|
+
}
|
|
102
|
+
return source;
|
|
103
|
+
}
|
|
104
|
+
/** Write one source as a summary-only record. */
|
|
105
|
+
function storedSource(source) {
|
|
106
|
+
const stored = { url: source.url };
|
|
107
|
+
if (source.title !== undefined && source.title.length > 0)
|
|
108
|
+
stored['title'] = source.title;
|
|
109
|
+
if (source.snippet !== undefined && source.snippet.length > 0)
|
|
110
|
+
stored['snippet'] = source.snippet;
|
|
111
|
+
if (source.publishedAt !== undefined && source.publishedAt.length > 0) {
|
|
112
|
+
stored['publishedAt'] = source.publishedAt;
|
|
113
|
+
}
|
|
114
|
+
return stored;
|
|
115
|
+
}
|
|
116
|
+
/** Read one persisted entry, rejecting anything malformed or already expired. */
|
|
117
|
+
function readEntry(raw, now) {
|
|
118
|
+
if (!isRecord(raw))
|
|
119
|
+
return undefined;
|
|
120
|
+
const key = raw['key'];
|
|
121
|
+
const expiresAt = raw['expiresAt'];
|
|
122
|
+
const rawSources = raw['sources'];
|
|
123
|
+
if (typeof key !== 'string' || key.length === 0)
|
|
124
|
+
return undefined;
|
|
125
|
+
if (typeof expiresAt !== 'number' || !Number.isFinite(expiresAt) || expiresAt <= now)
|
|
126
|
+
return undefined;
|
|
127
|
+
if (!Array.isArray(rawSources))
|
|
128
|
+
return undefined;
|
|
129
|
+
const sources = [];
|
|
130
|
+
for (const item of rawSources) {
|
|
131
|
+
const source = readSource(item);
|
|
132
|
+
if (source !== undefined)
|
|
133
|
+
sources.push(source);
|
|
134
|
+
}
|
|
135
|
+
// An entry with no usable result is not worth a slot in the file.
|
|
136
|
+
if (sources.length === 0)
|
|
137
|
+
return undefined;
|
|
138
|
+
return { key, expiresAt, sources };
|
|
139
|
+
}
|
|
140
|
+
/** Clamp the byte budget to a usable positive integer. */
|
|
141
|
+
function normaliseMaxBytes(value) {
|
|
142
|
+
return typeof value === 'number' && Number.isFinite(value) && value > 0
|
|
143
|
+
? Math.floor(value)
|
|
144
|
+
: DEFAULT_CACHE_FILE_MAX_BYTES;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Load the snapshot from disk.
|
|
148
|
+
*
|
|
149
|
+
* Every failure mode collapses to an empty list: absent file, unreadable path,
|
|
150
|
+
* malformed JSON, a different `version`, and individually invalid entries. An
|
|
151
|
+
* expired entry is dropped here rather than retained until its next read.
|
|
152
|
+
*
|
|
153
|
+
* @param file - absolute path to the snapshot.
|
|
154
|
+
* @param now - current time from the cache's injected clock.
|
|
155
|
+
* @param maxEntries - entry cap; the oldest surplus entries are discarded.
|
|
156
|
+
* @param io - filesystem seam, defaulting to the real one.
|
|
157
|
+
* @returns usable entries in stored (oldest to newest) order.
|
|
158
|
+
*/
|
|
159
|
+
export function readCacheFile(file, now, maxEntries, io) {
|
|
160
|
+
try {
|
|
161
|
+
const text = (io ?? nodeCacheFileIo).read(file);
|
|
162
|
+
if (text === undefined)
|
|
163
|
+
return [];
|
|
164
|
+
const parsed = JSON.parse(text);
|
|
165
|
+
if (!isRecord(parsed))
|
|
166
|
+
return [];
|
|
167
|
+
if (parsed['version'] !== CACHE_FILE_VERSION)
|
|
168
|
+
return [];
|
|
169
|
+
const rawEntries = parsed['entries'];
|
|
170
|
+
if (!Array.isArray(rawEntries))
|
|
171
|
+
return [];
|
|
172
|
+
const entries = [];
|
|
173
|
+
for (const raw of rawEntries) {
|
|
174
|
+
const entry = readEntry(raw, now);
|
|
175
|
+
if (entry !== undefined)
|
|
176
|
+
entries.push(entry);
|
|
177
|
+
}
|
|
178
|
+
const cap = Number.isFinite(maxEntries) ? Math.max(0, Math.floor(maxEntries)) : entries.length;
|
|
179
|
+
return entries.length > cap ? entries.slice(entries.length - cap) : entries;
|
|
180
|
+
}
|
|
181
|
+
catch {
|
|
182
|
+
// Corrupt JSON, a foreign document, a getter that throws: empty cache.
|
|
183
|
+
return [];
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
/**
|
|
187
|
+
* Serialise entries, evicting the oldest until the document fits the budget.
|
|
188
|
+
*
|
|
189
|
+
* @param entries - entries in LRU order (oldest first).
|
|
190
|
+
* @param now - timestamp recorded as `savedAt`.
|
|
191
|
+
* @param maxBytes - byte budget; defaults to 4 MiB.
|
|
192
|
+
* @returns the document and how many entries it carries.
|
|
193
|
+
*/
|
|
194
|
+
export function serializeCache(entries, now, maxBytes) {
|
|
195
|
+
const budget = normaliseMaxBytes(maxBytes);
|
|
196
|
+
const kept = entries.map((entry) => ({
|
|
197
|
+
key: entry.key,
|
|
198
|
+
expiresAt: entry.expiresAt,
|
|
199
|
+
sources: entry.sources.map(storedSource),
|
|
200
|
+
}));
|
|
201
|
+
const build = () => JSON.stringify({
|
|
202
|
+
version: CACHE_FILE_VERSION,
|
|
203
|
+
savedAt: now,
|
|
204
|
+
entries: kept,
|
|
205
|
+
});
|
|
206
|
+
let data = build();
|
|
207
|
+
while (kept.length > 0 && Buffer.byteLength(data, 'utf8') > budget) {
|
|
208
|
+
kept.shift();
|
|
209
|
+
data = build();
|
|
210
|
+
}
|
|
211
|
+
return { data, entries: kept.length, bytes: Buffer.byteLength(data, 'utf8') };
|
|
212
|
+
}
|
|
213
|
+
/** Distinguishes temporary files of concurrent writers in this process. */
|
|
214
|
+
let tempCounter = 0;
|
|
215
|
+
/**
|
|
216
|
+
* Atomically replace the snapshot.
|
|
217
|
+
*
|
|
218
|
+
* The temporary file lives beside the target so the `rename` stays on one
|
|
219
|
+
* filesystem. When anything fails (directory not writable, rename rejected),
|
|
220
|
+
* the previous document is untouched, the temporary file is removed, and the
|
|
221
|
+
* call returns `false` instead of throwing.
|
|
222
|
+
*
|
|
223
|
+
* @param file - absolute target path.
|
|
224
|
+
* @param entries - entries in LRU order (oldest first).
|
|
225
|
+
* @param now - timestamp recorded as `savedAt`.
|
|
226
|
+
* @param options - byte budget and filesystem seam.
|
|
227
|
+
* @returns `true` when the target now holds this snapshot.
|
|
228
|
+
*/
|
|
229
|
+
export function writeCacheFile(file, entries, now, options = {}) {
|
|
230
|
+
const io = options.io ?? nodeCacheFileIo;
|
|
231
|
+
const { data } = serializeCache(entries, now, options.maxBytes);
|
|
232
|
+
tempCounter += 1;
|
|
233
|
+
const temp = `${file}.${process.pid}.${tempCounter}.tmp`;
|
|
234
|
+
try {
|
|
235
|
+
io.mkdir(dirname(file));
|
|
236
|
+
io.write(temp, data);
|
|
237
|
+
io.rename(temp, file);
|
|
238
|
+
return true;
|
|
239
|
+
}
|
|
240
|
+
catch {
|
|
241
|
+
try {
|
|
242
|
+
io.remove(temp);
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
// Cleanup is best effort; the previous snapshot is still intact.
|
|
246
|
+
}
|
|
247
|
+
return false;
|
|
248
|
+
}
|
|
249
|
+
}
|
package/lib/cache.d.ts
CHANGED
|
@@ -13,6 +13,23 @@
|
|
|
13
13
|
* @module @fangchenghao/dsh-web-search/cache
|
|
14
14
|
*/
|
|
15
15
|
import type { CacheOptions, Source, TimeRange } from './types.ts';
|
|
16
|
+
import type { CacheFileIo } from './cache-persist.ts';
|
|
17
|
+
/** Optional disk persistence, layered on top of the in-memory cache. */
|
|
18
|
+
export interface CachePersistOptions {
|
|
19
|
+
/** Custom snapshot path; empty or absent uses `~/.dsh/cache/dsh-web-search/cache.json`. */
|
|
20
|
+
file?: string;
|
|
21
|
+
/** Serialised byte budget; the oldest entries are evicted to fit. */
|
|
22
|
+
maxBytes?: number;
|
|
23
|
+
/** Minimum gap between writes in milliseconds; defaults to two seconds. */
|
|
24
|
+
throttleMs?: number;
|
|
25
|
+
/** Filesystem seam, mainly for tests. */
|
|
26
|
+
io?: CacheFileIo;
|
|
27
|
+
}
|
|
28
|
+
/** Constructor options: {@link CacheOptions} plus optional persistence. */
|
|
29
|
+
export interface SearchCacheOptions extends CacheOptions {
|
|
30
|
+
/** Persist the cache when present; absent keeps the cache purely in memory. */
|
|
31
|
+
persist?: CachePersistOptions;
|
|
32
|
+
}
|
|
16
33
|
/** The logical inputs that decide whether two searches may share an entry. */
|
|
17
34
|
export interface CacheKeyParts {
|
|
18
35
|
/** Raw query text; normalised inside {@link cacheKey}. */
|
|
@@ -56,18 +73,28 @@ export declare function cacheKey(parts: CacheKeyParts): string;
|
|
|
56
73
|
*/
|
|
57
74
|
export declare class SearchCache {
|
|
58
75
|
/** Entry lifetime in milliseconds; `0` disables the cache completely. */
|
|
59
|
-
|
|
76
|
+
/** Live TTL reader: settings may change without a remount. */
|
|
77
|
+
private readonly ttlOf;
|
|
60
78
|
/** Maximum retained entries; the oldest read is evicted first. */
|
|
61
|
-
|
|
79
|
+
/** Live entry-cap reader: settings may change without a remount. */
|
|
80
|
+
private readonly maxOf;
|
|
62
81
|
/** Injected clock, so tests never wait on real time. */
|
|
63
82
|
private readonly now;
|
|
64
83
|
/** Insertion order doubles as LRU order: the first key is the coldest. */
|
|
65
84
|
private readonly entries;
|
|
85
|
+
/** Snapshot destination, or `undefined` for a purely in-memory cache. */
|
|
86
|
+
private readonly persistTo;
|
|
87
|
+
/** Minimum gap between writes. */
|
|
88
|
+
private readonly throttleMs;
|
|
89
|
+
/** Whether memory changed since the last successful write. */
|
|
90
|
+
private dirty;
|
|
91
|
+
/** Pending throttled flush. */
|
|
92
|
+
private timer;
|
|
66
93
|
/**
|
|
67
|
-
* @param options - TTL
|
|
94
|
+
* @param options - TTL, entry cap and optional persistence.
|
|
68
95
|
* @param now - clock override; defaults to `Date.now`.
|
|
69
96
|
*/
|
|
70
|
-
constructor(options?:
|
|
97
|
+
constructor(options?: SearchCacheOptions, now?: () => number);
|
|
71
98
|
/** Number of retained entries, including ones whose TTL already elapsed. */
|
|
72
99
|
get size(): number;
|
|
73
100
|
/**
|
|
@@ -87,6 +114,23 @@ export declare class SearchCache {
|
|
|
87
114
|
* @param sources - the merged source list; copied before storage.
|
|
88
115
|
*/
|
|
89
116
|
set(key: string, sources: Source[]): void;
|
|
117
|
+
/**
|
|
118
|
+
* Write the snapshot now, if there is anything to write.
|
|
119
|
+
*
|
|
120
|
+
* Writes are normally throttled and happen on a timer or at process exit;
|
|
121
|
+
* this is the deterministic seam tests use, and the call a shutdown path can
|
|
122
|
+
* make when it wants the newest state on disk immediately.
|
|
123
|
+
*
|
|
124
|
+
* @returns `true` when the file was replaced, `false` for "nothing to write"
|
|
125
|
+
* or a contained filesystem failure.
|
|
126
|
+
*/
|
|
127
|
+
flush(): boolean;
|
|
128
|
+
/** Stop the exit hook and flush whatever is still pending. */
|
|
129
|
+
close(): void;
|
|
90
130
|
/** Drop every entry. */
|
|
91
131
|
clear(): void;
|
|
132
|
+
/** Mark the snapshot stale and arrange a throttled flush. */
|
|
133
|
+
private touch;
|
|
134
|
+
/** Arm the single pending timer, if persistence is enabled. */
|
|
135
|
+
private schedule;
|
|
92
136
|
}
|
package/lib/cache.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { readCacheFile, resolveCacheFile, writeCacheFile } from "./cache-persist.js";
|
|
1
2
|
/** Default entry lifetime: ten minutes. */
|
|
2
3
|
const DEFAULT_TTL_MS = 600_000;
|
|
3
4
|
/** Default number of retained entries. */
|
|
4
5
|
const DEFAULT_MAX_ENTRIES = 100;
|
|
6
|
+
/** Default minimum gap between disk writes. */
|
|
7
|
+
const DEFAULT_PERSIST_THROTTLE_MS = 2_000;
|
|
5
8
|
/**
|
|
6
9
|
* Build the cache key for one search request.
|
|
7
10
|
*
|
|
@@ -53,21 +56,51 @@ function normaliseCount(value, fallback) {
|
|
|
53
56
|
*/
|
|
54
57
|
export class SearchCache {
|
|
55
58
|
/** Entry lifetime in milliseconds; `0` disables the cache completely. */
|
|
56
|
-
|
|
59
|
+
/** Live TTL reader: settings may change without a remount. */
|
|
60
|
+
ttlOf;
|
|
57
61
|
/** Maximum retained entries; the oldest read is evicted first. */
|
|
58
|
-
|
|
62
|
+
/** Live entry-cap reader: settings may change without a remount. */
|
|
63
|
+
maxOf;
|
|
59
64
|
/** Injected clock, so tests never wait on real time. */
|
|
60
65
|
now;
|
|
61
66
|
/** Insertion order doubles as LRU order: the first key is the coldest. */
|
|
62
67
|
entries = new Map();
|
|
68
|
+
/** Snapshot destination, or `undefined` for a purely in-memory cache. */
|
|
69
|
+
persistTo;
|
|
70
|
+
/** Minimum gap between writes. */
|
|
71
|
+
throttleMs;
|
|
72
|
+
/** Whether memory changed since the last successful write. */
|
|
73
|
+
dirty = false;
|
|
74
|
+
/** Pending throttled flush. */
|
|
75
|
+
timer;
|
|
63
76
|
/**
|
|
64
|
-
* @param options - TTL
|
|
77
|
+
* @param options - TTL, entry cap and optional persistence.
|
|
65
78
|
* @param now - clock override; defaults to `Date.now`.
|
|
66
79
|
*/
|
|
67
80
|
constructor(options = {}, now) {
|
|
68
|
-
|
|
69
|
-
|
|
81
|
+
const ttlOption = options.ttlMs;
|
|
82
|
+
const maxOption = options.maxEntries;
|
|
83
|
+
this.ttlOf = () => normaliseDuration(typeof ttlOption === 'function' ? ttlOption() : ttlOption, DEFAULT_TTL_MS);
|
|
84
|
+
this.maxOf = () => normaliseCount(typeof maxOption === 'function' ? maxOption() : maxOption, DEFAULT_MAX_ENTRIES);
|
|
70
85
|
this.now = now ?? Date.now;
|
|
86
|
+
this.throttleMs = normaliseDuration(options.persist?.throttleMs, DEFAULT_PERSIST_THROTTLE_MS);
|
|
87
|
+
const persist = options.persist;
|
|
88
|
+
if (persist !== undefined && this.ttlOf() > 0 && this.maxOf() > 0) {
|
|
89
|
+
const file = resolveCacheFile(persist.file);
|
|
90
|
+
// Startup restore is synchronous and fully contained: an unreadable or
|
|
91
|
+
// malformed snapshot simply yields an empty cache.
|
|
92
|
+
const loaded = readCacheFile(file, this.now(), this.maxOf(), persist.io);
|
|
93
|
+
for (const entry of loaded) {
|
|
94
|
+
this.entries.set(entry.key, { sources: entry.sources, expiresAt: entry.expiresAt });
|
|
95
|
+
}
|
|
96
|
+
this.persistTo = {
|
|
97
|
+
file,
|
|
98
|
+
...(persist.maxBytes === undefined ? {} : { maxBytes: persist.maxBytes }),
|
|
99
|
+
...(persist.io === undefined ? {} : { io: persist.io }),
|
|
100
|
+
};
|
|
101
|
+
liveCaches.add(this);
|
|
102
|
+
installExitHook();
|
|
103
|
+
}
|
|
71
104
|
}
|
|
72
105
|
/** Number of retained entries, including ones whose TTL already elapsed. */
|
|
73
106
|
get size() {
|
|
@@ -82,13 +115,14 @@ export class SearchCache {
|
|
|
82
115
|
* @returns a copy of the cached sources, or `undefined` on miss/expiry.
|
|
83
116
|
*/
|
|
84
117
|
get(key) {
|
|
85
|
-
if (this.
|
|
118
|
+
if (this.ttlOf() === 0 || this.maxOf() === 0)
|
|
86
119
|
return undefined;
|
|
87
120
|
const entry = this.entries.get(key);
|
|
88
121
|
if (entry === undefined)
|
|
89
122
|
return undefined;
|
|
90
123
|
if (this.now() >= entry.expiresAt) {
|
|
91
124
|
this.entries.delete(key);
|
|
125
|
+
this.touch();
|
|
92
126
|
return undefined;
|
|
93
127
|
}
|
|
94
128
|
// Re-insert to move the entry to the young end of the map.
|
|
@@ -104,22 +138,113 @@ export class SearchCache {
|
|
|
104
138
|
* @param sources - the merged source list; copied before storage.
|
|
105
139
|
*/
|
|
106
140
|
set(key, sources) {
|
|
107
|
-
|
|
141
|
+
const ttlMs = this.ttlOf();
|
|
142
|
+
const maxEntries = this.maxOf();
|
|
143
|
+
if (ttlMs === 0 || maxEntries === 0)
|
|
108
144
|
return;
|
|
109
145
|
this.entries.delete(key);
|
|
110
146
|
this.entries.set(key, {
|
|
111
147
|
sources: cloneSources(sources),
|
|
112
|
-
expiresAt: this.now() +
|
|
148
|
+
expiresAt: this.now() + ttlMs,
|
|
113
149
|
});
|
|
114
|
-
while (this.entries.size >
|
|
150
|
+
while (this.entries.size > maxEntries) {
|
|
115
151
|
const oldest = this.entries.keys().next();
|
|
116
152
|
if (oldest.done === true)
|
|
117
153
|
break;
|
|
118
154
|
this.entries.delete(oldest.value);
|
|
119
155
|
}
|
|
156
|
+
this.touch();
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Write the snapshot now, if there is anything to write.
|
|
160
|
+
*
|
|
161
|
+
* Writes are normally throttled and happen on a timer or at process exit;
|
|
162
|
+
* this is the deterministic seam tests use, and the call a shutdown path can
|
|
163
|
+
* make when it wants the newest state on disk immediately.
|
|
164
|
+
*
|
|
165
|
+
* @returns `true` when the file was replaced, `false` for "nothing to write"
|
|
166
|
+
* or a contained filesystem failure.
|
|
167
|
+
*/
|
|
168
|
+
flush() {
|
|
169
|
+
if (this.persistTo === undefined)
|
|
170
|
+
return false;
|
|
171
|
+
if (this.timer !== undefined) {
|
|
172
|
+
clearTimeout(this.timer);
|
|
173
|
+
this.timer = undefined;
|
|
174
|
+
}
|
|
175
|
+
if (!this.dirty)
|
|
176
|
+
return false;
|
|
177
|
+
const entries = [];
|
|
178
|
+
for (const [key, entry] of this.entries) {
|
|
179
|
+
entries.push({ key, expiresAt: entry.expiresAt, sources: cloneSources(entry.sources) });
|
|
180
|
+
}
|
|
181
|
+
const written = writeCacheFile(this.persistTo.file, entries, this.now(), {
|
|
182
|
+
...(this.persistTo.maxBytes === undefined ? {} : { maxBytes: this.persistTo.maxBytes }),
|
|
183
|
+
...(this.persistTo.io === undefined ? {} : { io: this.persistTo.io }),
|
|
184
|
+
});
|
|
185
|
+
this.dirty = !written;
|
|
186
|
+
if (!written)
|
|
187
|
+
this.schedule();
|
|
188
|
+
return written;
|
|
189
|
+
}
|
|
190
|
+
/** Stop the exit hook and flush whatever is still pending. */
|
|
191
|
+
close() {
|
|
192
|
+
this.flush();
|
|
193
|
+
if (this.timer !== undefined) {
|
|
194
|
+
clearTimeout(this.timer);
|
|
195
|
+
this.timer = undefined;
|
|
196
|
+
}
|
|
197
|
+
liveCaches.delete(this);
|
|
198
|
+
if (liveCaches.size === 0)
|
|
199
|
+
removeExitHook();
|
|
120
200
|
}
|
|
121
201
|
/** Drop every entry. */
|
|
122
202
|
clear() {
|
|
203
|
+
if (this.entries.size === 0)
|
|
204
|
+
return;
|
|
123
205
|
this.entries.clear();
|
|
206
|
+
this.touch();
|
|
124
207
|
}
|
|
208
|
+
/** Mark the snapshot stale and arrange a throttled flush. */
|
|
209
|
+
touch() {
|
|
210
|
+
if (this.persistTo === undefined)
|
|
211
|
+
return;
|
|
212
|
+
this.dirty = true;
|
|
213
|
+
this.schedule();
|
|
214
|
+
}
|
|
215
|
+
/** Arm the single pending timer, if persistence is enabled. */
|
|
216
|
+
schedule() {
|
|
217
|
+
if (this.persistTo === undefined || this.timer !== undefined)
|
|
218
|
+
return;
|
|
219
|
+
this.timer = setTimeout(() => {
|
|
220
|
+
this.timer = undefined;
|
|
221
|
+
this.flush();
|
|
222
|
+
}, this.throttleMs);
|
|
223
|
+
// A pending cache flush must not keep a one-shot CLI process alive; the
|
|
224
|
+
// exit hook still writes it synchronously.
|
|
225
|
+
this.timer.unref();
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
/** Caches that still need an exit-time flush. */
|
|
229
|
+
const liveCaches = new Set();
|
|
230
|
+
/** Set once the process-level hook is installed. */
|
|
231
|
+
let exitHookInstalled = false;
|
|
232
|
+
/** Flush every live cache; used only during process exit. */
|
|
233
|
+
function flushLiveCaches() {
|
|
234
|
+
for (const cache of liveCaches)
|
|
235
|
+
cache.flush();
|
|
236
|
+
}
|
|
237
|
+
/** Install one process-wide exit hook, lazily. */
|
|
238
|
+
function installExitHook() {
|
|
239
|
+
if (exitHookInstalled)
|
|
240
|
+
return;
|
|
241
|
+
exitHookInstalled = true;
|
|
242
|
+
process.once('exit', flushLiveCaches);
|
|
243
|
+
}
|
|
244
|
+
/** Remove the hook when no persisted cache remains. */
|
|
245
|
+
function removeExitHook() {
|
|
246
|
+
if (!exitHookInstalled)
|
|
247
|
+
return;
|
|
248
|
+
process.removeListener('exit', flushLiveCaches);
|
|
249
|
+
exitHookInstalled = false;
|
|
125
250
|
}
|
package/lib/config.d.ts
CHANGED
|
@@ -25,6 +25,8 @@ import type { RankOptions, TimeRange, WaitPolicy } from './types.ts';
|
|
|
25
25
|
export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
26
26
|
hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
27
27
|
timeoutMs: z<number, number, "volatile-defined">;
|
|
28
|
+
fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
29
|
+
fetchMaxLength: z<number, number, "volatile-defined">;
|
|
28
30
|
wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
29
31
|
proxy: z<string, string, "volatile-defined">;
|
|
30
32
|
limit: z<number, number, "volatile-defined">;
|
|
@@ -36,6 +38,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
36
38
|
timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
37
39
|
cacheTtlMs: z<number, number, "volatile-defined">;
|
|
38
40
|
cacheEntries: z<number, number, "volatile-defined">;
|
|
41
|
+
cachePersist: z<boolean, boolean, "volatile-defined">;
|
|
42
|
+
cacheFile: z<string, string, "volatile-defined">;
|
|
39
43
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
40
44
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
41
45
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
@@ -46,6 +50,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
46
50
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
47
51
|
hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
48
52
|
timeoutMs: z<number, number, "volatile-defined">;
|
|
53
|
+
fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
54
|
+
fetchMaxLength: z<number, number, "volatile-defined">;
|
|
49
55
|
wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
50
56
|
proxy: z<string, string, "volatile-defined">;
|
|
51
57
|
limit: z<number, number, "volatile-defined">;
|
|
@@ -57,6 +63,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
57
63
|
timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
58
64
|
cacheTtlMs: z<number, number, "volatile-defined">;
|
|
59
65
|
cacheEntries: z<number, number, "volatile-defined">;
|
|
66
|
+
cachePersist: z<boolean, boolean, "volatile-defined">;
|
|
67
|
+
cacheFile: z<string, string, "volatile-defined">;
|
|
60
68
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
61
69
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
62
70
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
@@ -92,6 +100,10 @@ export interface ResolvedConfig {
|
|
|
92
100
|
hostIds: string[] | undefined;
|
|
93
101
|
/** Per-engine deadline. */
|
|
94
102
|
timeoutMs: number;
|
|
103
|
+
/** `web_fetch` output style. */
|
|
104
|
+
fetchFormat: 'markdown' | 'text';
|
|
105
|
+
/** `web_fetch` character budget. */
|
|
106
|
+
fetchMaxLength: number;
|
|
95
107
|
/** Explicit tunnel URL, or `undefined` to detect one. */
|
|
96
108
|
proxy: string | undefined;
|
|
97
109
|
/** Caller budget cap. */
|
|
@@ -110,6 +122,10 @@ export interface ResolvedConfig {
|
|
|
110
122
|
cacheTtlMs: number;
|
|
111
123
|
/** Cache capacity in entries. */
|
|
112
124
|
cacheEntries: number;
|
|
125
|
+
/** Whether the cache snapshot is written to disk. */
|
|
126
|
+
cachePersist: boolean;
|
|
127
|
+
/** Custom snapshot path, or `undefined` for the default one. */
|
|
128
|
+
cacheFile: string | undefined;
|
|
113
129
|
/** Base cooldown in milliseconds. */
|
|
114
130
|
baseCooldownMs: number;
|
|
115
131
|
/** Cooldown ceiling in milliseconds. */
|
package/lib/config.js
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
* @module @fangchenghao/dsh-web-search/config
|
|
21
21
|
*/
|
|
22
22
|
import z from '@deepseek-ai/schemastery';
|
|
23
|
+
import { DEFAULT_FETCH_MAX_LENGTH } from "./markdown.js";
|
|
23
24
|
import { VERTICAL_IDS, isVerticalId } from "./verticals.js";
|
|
24
25
|
/** Schema of the plugin's configuration section. */
|
|
25
26
|
export const Config = z.object({
|
|
@@ -29,6 +30,18 @@ export const Config = z.object({
|
|
|
29
30
|
.description('只启用列出的搜索引擎(如 anysearch、cn.bing),多个用逗号分隔;留空表示全部。不想限制就留空。')
|
|
30
31
|
.volatile(),
|
|
31
32
|
timeoutMs: z.natural().min(500).default(15000).description('每次向搜索引擎要结果,最多等它多久(毫秒,1000 = 1 秒)。网络慢可加到 20000;平时保持 15000 就行。').volatile(),
|
|
33
|
+
fetchFormat: z
|
|
34
|
+
.union(['markdown', 'text'])
|
|
35
|
+
.default('markdown')
|
|
36
|
+
.description('抓取网页后返回什么样式:markdown 保留标题、列表、表格和链接,最适合阅读;text 只留纯文字、去掉所有标记。一般保持 markdown。')
|
|
37
|
+
.volatile(),
|
|
38
|
+
fetchMaxLength: z
|
|
39
|
+
.natural()
|
|
40
|
+
.min(1000)
|
|
41
|
+
.max(200000)
|
|
42
|
+
.default(DEFAULT_FETCH_MAX_LENGTH)
|
|
43
|
+
.description('一次抓取最多返回多少个字符(1000-200000);超出会在末尾注明原始长度。想一次多读可调到 50000,日常保持 20000 即可。')
|
|
44
|
+
.volatile(),
|
|
32
45
|
wait: z
|
|
33
46
|
.union(['auto', 'all', 'fast'])
|
|
34
47
|
.default('auto')
|
|
@@ -54,6 +67,16 @@ export const Config = z.object({
|
|
|
54
67
|
.volatile(),
|
|
55
68
|
cacheTtlMs: z.natural().default(600000).description('同一问题多久内复用上次结果(缓存,即存一份备用),毫秒。想早点看到新内容就调小;保持 600000(10 分钟)即可。').volatile(),
|
|
56
69
|
cacheEntries: z.natural().default(100).description('最多存多少条旧结果备用(缓存,即存起来省得重复查)。存满后先丢最久没用的;一般保持 100 即可。').volatile(),
|
|
70
|
+
cachePersist: z
|
|
71
|
+
.boolean()
|
|
72
|
+
.default(true)
|
|
73
|
+
.description('是否把缓存保存到硬盘,重启后还能复用上次结果。默认开启(true);不想留痕或磁盘只读时改成 false。')
|
|
74
|
+
.volatile(),
|
|
75
|
+
cacheFile: z
|
|
76
|
+
.string()
|
|
77
|
+
.default('')
|
|
78
|
+
.description('缓存文件放哪;留空用默认 ~/.dsh/cache/dsh-web-search/cache.json。一般不用填,换位置时再写绝对路径。')
|
|
79
|
+
.volatile(),
|
|
57
80
|
anysearchApiKey: z
|
|
58
81
|
.string()
|
|
59
82
|
.default('')
|
|
@@ -117,10 +140,13 @@ export function resolveConfig(config, callLimit) {
|
|
|
117
140
|
: String(rawDomainSources).split(/[\s,]+/);
|
|
118
141
|
const domainSources = domainList.map((id) => id.trim()).filter(isVerticalId);
|
|
119
142
|
const proxy = settings.proxy.trim();
|
|
143
|
+
const cacheFile = settings.cacheFile.trim();
|
|
120
144
|
const limit = Math.min(callLimit ?? settings.limit, settings.limit);
|
|
121
145
|
return {
|
|
122
146
|
hostIds: hosts.length > 0 ? hosts : undefined,
|
|
123
147
|
timeoutMs: settings.timeoutMs,
|
|
148
|
+
fetchFormat: settings.fetchFormat,
|
|
149
|
+
fetchMaxLength: settings.fetchMaxLength,
|
|
124
150
|
proxy: proxy.length > 0 ? proxy : undefined,
|
|
125
151
|
limit,
|
|
126
152
|
wait: settings.wait,
|
|
@@ -139,6 +165,8 @@ export function resolveConfig(config, callLimit) {
|
|
|
139
165
|
},
|
|
140
166
|
cacheTtlMs: settings.cacheTtlMs,
|
|
141
167
|
cacheEntries: settings.cacheEntries,
|
|
168
|
+
cachePersist: settings.cachePersist,
|
|
169
|
+
cacheFile: cacheFile.length > 0 ? cacheFile : undefined,
|
|
142
170
|
baseCooldownMs: settings.baseCooldownMs,
|
|
143
171
|
maxCooldownMs: settings.maxCooldownMs,
|
|
144
172
|
};
|
package/lib/fetch.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { FetchFormat } from './markdown.ts';
|
|
1
2
|
/** One retrieval outcome, mirroring the Host's `WebFetchResult`. */
|
|
2
3
|
export interface FetchOutcome {
|
|
3
4
|
readonly url: string;
|
|
@@ -30,6 +31,12 @@ export interface FetchProviderOptions {
|
|
|
30
31
|
anysearchApiKey?: string;
|
|
31
32
|
/** Bytes kept from a response; the Host applies its own cap too. */
|
|
32
33
|
maxBytes?: number;
|
|
34
|
+
/** Output style for an HTML body; `markdown` keeps structure. */
|
|
35
|
+
/** Renderer, or a reader for it so a Settings edit applies without a remount. */
|
|
36
|
+
format?: FetchFormat | (() => FetchFormat);
|
|
37
|
+
/** Character budget for the returned text. */
|
|
38
|
+
/** Character budget, or a reader for it. */
|
|
39
|
+
maxLength?: number | (() => number);
|
|
33
40
|
/** User agent sent on every rung. */
|
|
34
41
|
userAgent?: string;
|
|
35
42
|
}
|