@fffangchenghao/dsh-web-search 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +247 -41
- package/README.zh-CN.md +247 -41
- package/client.js +66 -9
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +56 -8
- package/lib/cache.js +141 -13
- package/lib/config.d.ts +33 -1
- package/lib/config.js +54 -2
- package/lib/domain-tool.d.ts +119 -0
- package/lib/domain-tool.js +196 -0
- package/lib/engines.js +68 -10
- package/lib/fetch.d.ts +7 -0
- package/lib/fetch.js +36 -3
- package/lib/freshness.d.ts +102 -0
- package/lib/freshness.js +359 -0
- package/lib/index.d.ts +44 -2
- package/lib/index.js +144 -23
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/parsers.js +59 -5
- package/lib/rank.d.ts +14 -1
- package/lib/rank.js +18 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.d.ts +4 -2
- package/lib/transport.js +4 -2
- package/lib/types.d.ts +22 -6
- package/lib/verticals.d.ts +164 -0
- package/lib/verticals.js +539 -0
- package/package.json +4 -4
package/lib/cache.js
CHANGED
|
@@ -1,16 +1,21 @@
|
|
|
1
|
+
import { readCacheFile, resolveCacheFile, writeCacheFile } from "./cache-persist.js";
|
|
1
2
|
/** Default entry lifetime: ten minutes. */
|
|
2
3
|
const DEFAULT_TTL_MS = 600_000;
|
|
3
4
|
/** Default number of retained entries. */
|
|
4
5
|
const DEFAULT_MAX_ENTRIES = 100;
|
|
6
|
+
/** Default minimum gap between disk writes. */
|
|
7
|
+
const DEFAULT_PERSIST_THROTTLE_MS = 2_000;
|
|
5
8
|
/**
|
|
6
9
|
* Build the cache key for one search request.
|
|
7
10
|
*
|
|
8
11
|
* The query is trimmed, lowercased and has internal whitespace collapsed, so
|
|
9
12
|
* `' PostgreSQL Replication '` and `'postgresql replication'` share an
|
|
10
13
|
* entry. Engine order never matters because the ids are sorted; every other
|
|
11
|
-
* field is positional, so differing `limit`, `wait`, `lead` or
|
|
12
|
-
* yields a different key.
|
|
13
|
-
*
|
|
14
|
+
* field is positional, so differing `limit`, `wait`, `lead`, `timeRange` or
|
|
15
|
+
* engine set yields a different key. `timeRange` is appended only when it is
|
|
16
|
+
* active, which keeps the key for an unfiltered search unchanged. The query is
|
|
17
|
+
* last so a query containing the field separator cannot collide with a
|
|
18
|
+
* different request's prefix.
|
|
14
19
|
*
|
|
15
20
|
* @param parts - the logical request.
|
|
16
21
|
* @returns a deterministic, comparison-safe cache key.
|
|
@@ -18,7 +23,8 @@ const DEFAULT_MAX_ENTRIES = 100;
|
|
|
18
23
|
export function cacheKey(parts) {
|
|
19
24
|
const query = parts.query.trim().toLowerCase().replace(/\s+/gu, ' ');
|
|
20
25
|
const engines = [...parts.engines].sort().join('+');
|
|
21
|
-
|
|
26
|
+
const window = parts.timeRange === undefined || parts.timeRange === 'off' ? '' : `${parts.timeRange}|`;
|
|
27
|
+
return `${parts.lead}|${parts.wait}|${parts.limit}|${engines}|${window}${query}`;
|
|
22
28
|
}
|
|
23
29
|
/**
|
|
24
30
|
* Copy the array and each source.
|
|
@@ -50,21 +56,51 @@ function normaliseCount(value, fallback) {
|
|
|
50
56
|
*/
|
|
51
57
|
export class SearchCache {
|
|
52
58
|
/** Entry lifetime in milliseconds; `0` disables the cache completely. */
|
|
53
|
-
|
|
59
|
+
/** Live TTL reader: settings may change without a remount. */
|
|
60
|
+
ttlOf;
|
|
54
61
|
/** Maximum retained entries; the oldest read is evicted first. */
|
|
55
|
-
|
|
62
|
+
/** Live entry-cap reader: settings may change without a remount. */
|
|
63
|
+
maxOf;
|
|
56
64
|
/** Injected clock, so tests never wait on real time. */
|
|
57
65
|
now;
|
|
58
66
|
/** Insertion order doubles as LRU order: the first key is the coldest. */
|
|
59
67
|
entries = new Map();
|
|
68
|
+
/** Snapshot destination, or `undefined` for a purely in-memory cache. */
|
|
69
|
+
persistTo;
|
|
70
|
+
/** Minimum gap between writes. */
|
|
71
|
+
throttleMs;
|
|
72
|
+
/** Whether memory changed since the last successful write. */
|
|
73
|
+
dirty = false;
|
|
74
|
+
/** Pending throttled flush. */
|
|
75
|
+
timer;
|
|
60
76
|
/**
|
|
61
|
-
* @param options - TTL
|
|
77
|
+
* @param options - TTL, entry cap and optional persistence.
|
|
62
78
|
* @param now - clock override; defaults to `Date.now`.
|
|
63
79
|
*/
|
|
64
80
|
constructor(options = {}, now) {
|
|
65
|
-
|
|
66
|
-
|
|
81
|
+
const ttlOption = options.ttlMs;
|
|
82
|
+
const maxOption = options.maxEntries;
|
|
83
|
+
this.ttlOf = () => normaliseDuration(typeof ttlOption === 'function' ? ttlOption() : ttlOption, DEFAULT_TTL_MS);
|
|
84
|
+
this.maxOf = () => normaliseCount(typeof maxOption === 'function' ? maxOption() : maxOption, DEFAULT_MAX_ENTRIES);
|
|
67
85
|
this.now = now ?? Date.now;
|
|
86
|
+
this.throttleMs = normaliseDuration(options.persist?.throttleMs, DEFAULT_PERSIST_THROTTLE_MS);
|
|
87
|
+
const persist = options.persist;
|
|
88
|
+
if (persist !== undefined && this.ttlOf() > 0 && this.maxOf() > 0) {
|
|
89
|
+
const file = resolveCacheFile(persist.file);
|
|
90
|
+
// Startup restore is synchronous and fully contained: an unreadable or
|
|
91
|
+
// malformed snapshot simply yields an empty cache.
|
|
92
|
+
const loaded = readCacheFile(file, this.now(), this.maxOf(), persist.io);
|
|
93
|
+
for (const entry of loaded) {
|
|
94
|
+
this.entries.set(entry.key, { sources: entry.sources, expiresAt: entry.expiresAt });
|
|
95
|
+
}
|
|
96
|
+
this.persistTo = {
|
|
97
|
+
file,
|
|
98
|
+
...(persist.maxBytes === undefined ? {} : { maxBytes: persist.maxBytes }),
|
|
99
|
+
...(persist.io === undefined ? {} : { io: persist.io }),
|
|
100
|
+
};
|
|
101
|
+
liveCaches.add(this);
|
|
102
|
+
installExitHook();
|
|
103
|
+
}
|
|
68
104
|
}
|
|
69
105
|
/** Number of retained entries, including ones whose TTL already elapsed. */
|
|
70
106
|
get size() {
|
|
@@ -79,13 +115,14 @@ export class SearchCache {
|
|
|
79
115
|
* @returns a copy of the cached sources, or `undefined` on miss/expiry.
|
|
80
116
|
*/
|
|
81
117
|
get(key) {
|
|
82
|
-
if (this.
|
|
118
|
+
if (this.ttlOf() === 0 || this.maxOf() === 0)
|
|
83
119
|
return undefined;
|
|
84
120
|
const entry = this.entries.get(key);
|
|
85
121
|
if (entry === undefined)
|
|
86
122
|
return undefined;
|
|
87
123
|
if (this.now() >= entry.expiresAt) {
|
|
88
124
|
this.entries.delete(key);
|
|
125
|
+
this.touch();
|
|
89
126
|
return undefined;
|
|
90
127
|
}
|
|
91
128
|
// Re-insert to move the entry to the young end of the map.
|
|
@@ -101,22 +138,113 @@ export class SearchCache {
|
|
|
101
138
|
* @param sources - the merged source list; copied before storage.
|
|
102
139
|
*/
|
|
103
140
|
set(key, sources) {
|
|
104
|
-
|
|
141
|
+
const ttlMs = this.ttlOf();
|
|
142
|
+
const maxEntries = this.maxOf();
|
|
143
|
+
if (ttlMs === 0 || maxEntries === 0)
|
|
105
144
|
return;
|
|
106
145
|
this.entries.delete(key);
|
|
107
146
|
this.entries.set(key, {
|
|
108
147
|
sources: cloneSources(sources),
|
|
109
|
-
expiresAt: this.now() +
|
|
148
|
+
expiresAt: this.now() + ttlMs,
|
|
110
149
|
});
|
|
111
|
-
while (this.entries.size >
|
|
150
|
+
while (this.entries.size > maxEntries) {
|
|
112
151
|
const oldest = this.entries.keys().next();
|
|
113
152
|
if (oldest.done === true)
|
|
114
153
|
break;
|
|
115
154
|
this.entries.delete(oldest.value);
|
|
116
155
|
}
|
|
156
|
+
this.touch();
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Write the snapshot now, if there is anything to write.
|
|
160
|
+
*
|
|
161
|
+
* Writes are normally throttled and happen on a timer or at process exit;
|
|
162
|
+
* this is the deterministic seam tests use, and the call a shutdown path can
|
|
163
|
+
* make when it wants the newest state on disk immediately.
|
|
164
|
+
*
|
|
165
|
+
* @returns `true` when the file was replaced, `false` for "nothing to write"
|
|
166
|
+
* or a contained filesystem failure.
|
|
167
|
+
*/
|
|
168
|
+
flush() {
|
|
169
|
+
if (this.persistTo === undefined)
|
|
170
|
+
return false;
|
|
171
|
+
if (this.timer !== undefined) {
|
|
172
|
+
clearTimeout(this.timer);
|
|
173
|
+
this.timer = undefined;
|
|
174
|
+
}
|
|
175
|
+
if (!this.dirty)
|
|
176
|
+
return false;
|
|
177
|
+
const entries = [];
|
|
178
|
+
for (const [key, entry] of this.entries) {
|
|
179
|
+
entries.push({ key, expiresAt: entry.expiresAt, sources: cloneSources(entry.sources) });
|
|
180
|
+
}
|
|
181
|
+
const written = writeCacheFile(this.persistTo.file, entries, this.now(), {
|
|
182
|
+
...(this.persistTo.maxBytes === undefined ? {} : { maxBytes: this.persistTo.maxBytes }),
|
|
183
|
+
...(this.persistTo.io === undefined ? {} : { io: this.persistTo.io }),
|
|
184
|
+
});
|
|
185
|
+
this.dirty = !written;
|
|
186
|
+
if (!written)
|
|
187
|
+
this.schedule();
|
|
188
|
+
return written;
|
|
189
|
+
}
|
|
190
|
+
/** Stop the exit hook and flush whatever is still pending. */
|
|
191
|
+
close() {
|
|
192
|
+
this.flush();
|
|
193
|
+
if (this.timer !== undefined) {
|
|
194
|
+
clearTimeout(this.timer);
|
|
195
|
+
this.timer = undefined;
|
|
196
|
+
}
|
|
197
|
+
liveCaches.delete(this);
|
|
198
|
+
if (liveCaches.size === 0)
|
|
199
|
+
removeExitHook();
|
|
117
200
|
}
|
|
118
201
|
/** Drop every entry. */
|
|
119
202
|
clear() {
|
|
203
|
+
if (this.entries.size === 0)
|
|
204
|
+
return;
|
|
120
205
|
this.entries.clear();
|
|
206
|
+
this.touch();
|
|
121
207
|
}
|
|
208
|
+
/** Mark the snapshot stale and arrange a throttled flush. */
|
|
209
|
+
touch() {
|
|
210
|
+
if (this.persistTo === undefined)
|
|
211
|
+
return;
|
|
212
|
+
this.dirty = true;
|
|
213
|
+
this.schedule();
|
|
214
|
+
}
|
|
215
|
+
/** Arm the single pending timer, if persistence is enabled. */
|
|
216
|
+
schedule() {
|
|
217
|
+
if (this.persistTo === undefined || this.timer !== undefined)
|
|
218
|
+
return;
|
|
219
|
+
this.timer = setTimeout(() => {
|
|
220
|
+
this.timer = undefined;
|
|
221
|
+
this.flush();
|
|
222
|
+
}, this.throttleMs);
|
|
223
|
+
// A pending cache flush must not keep a one-shot CLI process alive; the
|
|
224
|
+
// exit hook still writes it synchronously.
|
|
225
|
+
this.timer.unref();
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
/** Caches that still need an exit-time flush. */
|
|
229
|
+
const liveCaches = new Set();
|
|
230
|
+
/** Set once the process-level hook is installed. */
|
|
231
|
+
let exitHookInstalled = false;
|
|
232
|
+
/** Flush every live cache; used only during process exit. */
|
|
233
|
+
function flushLiveCaches() {
|
|
234
|
+
for (const cache of liveCaches)
|
|
235
|
+
cache.flush();
|
|
236
|
+
}
|
|
237
|
+
/** Install one process-wide exit hook, lazily. */
|
|
238
|
+
function installExitHook() {
|
|
239
|
+
if (exitHookInstalled)
|
|
240
|
+
return;
|
|
241
|
+
exitHookInstalled = true;
|
|
242
|
+
process.once('exit', flushLiveCaches);
|
|
243
|
+
}
|
|
244
|
+
/** Remove the hook when no persisted cache remains. */
|
|
245
|
+
function removeExitHook() {
|
|
246
|
+
if (!exitHookInstalled)
|
|
247
|
+
return;
|
|
248
|
+
process.removeListener('exit', flushLiveCaches);
|
|
249
|
+
exitHookInstalled = false;
|
|
122
250
|
}
|
package/lib/config.d.ts
CHANGED
|
@@ -20,11 +20,13 @@
|
|
|
20
20
|
* @module @fangchenghao/dsh-web-search/config
|
|
21
21
|
*/
|
|
22
22
|
import z from '@deepseek-ai/schemastery';
|
|
23
|
-
import type { RankOptions, WaitPolicy } from './types.ts';
|
|
23
|
+
import type { RankOptions, TimeRange, WaitPolicy } from './types.ts';
|
|
24
24
|
/** Schema of the plugin's configuration section. */
|
|
25
25
|
export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
26
26
|
hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
27
27
|
timeoutMs: z<number, number, "volatile-defined">;
|
|
28
|
+
fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
29
|
+
fetchMaxLength: z<number, number, "volatile-defined">;
|
|
28
30
|
wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
29
31
|
proxy: z<string, string, "volatile-defined">;
|
|
30
32
|
limit: z<number, number, "volatile-defined">;
|
|
@@ -33,15 +35,23 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
33
35
|
authorityBoost: z<number, number, "volatile-defined">;
|
|
34
36
|
freshnessBoost: z<number, number, "volatile-defined">;
|
|
35
37
|
freshnessDays: z<number, number, "volatile-defined">;
|
|
38
|
+
timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
36
39
|
cacheTtlMs: z<number, number, "volatile-defined">;
|
|
37
40
|
cacheEntries: z<number, number, "volatile-defined">;
|
|
41
|
+
cachePersist: z<boolean, boolean, "volatile-defined">;
|
|
42
|
+
cacheFile: z<string, string, "volatile-defined">;
|
|
38
43
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
39
44
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
40
45
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
41
46
|
maxCooldownMs: z<number, number, "volatile-defined">;
|
|
47
|
+
domainSources: z<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
48
|
+
domainLimit: z<number, number, "volatile-defined">;
|
|
49
|
+
domainTimeoutMs: z<number, number, "volatile-defined">;
|
|
42
50
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
43
51
|
hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
44
52
|
timeoutMs: z<number, number, "volatile-defined">;
|
|
53
|
+
fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
54
|
+
fetchMaxLength: z<number, number, "volatile-defined">;
|
|
45
55
|
wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
46
56
|
proxy: z<string, string, "volatile-defined">;
|
|
47
57
|
limit: z<number, number, "volatile-defined">;
|
|
@@ -50,12 +60,18 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
50
60
|
authorityBoost: z<number, number, "volatile-defined">;
|
|
51
61
|
freshnessBoost: z<number, number, "volatile-defined">;
|
|
52
62
|
freshnessDays: z<number, number, "volatile-defined">;
|
|
63
|
+
timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
53
64
|
cacheTtlMs: z<number, number, "volatile-defined">;
|
|
54
65
|
cacheEntries: z<number, number, "volatile-defined">;
|
|
66
|
+
cachePersist: z<boolean, boolean, "volatile-defined">;
|
|
67
|
+
cacheFile: z<string, string, "volatile-defined">;
|
|
55
68
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
56
69
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
57
70
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
58
71
|
maxCooldownMs: z<number, number, "volatile-defined">;
|
|
72
|
+
domainSources: z<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
73
|
+
domainLimit: z<number, number, "volatile-defined">;
|
|
74
|
+
domainTimeoutMs: z<number, number, "volatile-defined">;
|
|
59
75
|
}>>, "plain">;
|
|
60
76
|
/** What the schema resolves a field to: a live reference, not the value. */
|
|
61
77
|
export interface LiveRef<T> {
|
|
@@ -84,12 +100,18 @@ export interface ResolvedConfig {
|
|
|
84
100
|
hostIds: string[] | undefined;
|
|
85
101
|
/** Per-engine deadline. */
|
|
86
102
|
timeoutMs: number;
|
|
103
|
+
/** `web_fetch` output style. */
|
|
104
|
+
fetchFormat: 'markdown' | 'text';
|
|
105
|
+
/** `web_fetch` character budget. */
|
|
106
|
+
fetchMaxLength: number;
|
|
87
107
|
/** Explicit tunnel URL, or `undefined` to detect one. */
|
|
88
108
|
proxy: string | undefined;
|
|
89
109
|
/** Caller budget cap. */
|
|
90
110
|
limit: number;
|
|
91
111
|
/** Release policy, deciding how long a tier waits for slow engines. */
|
|
92
112
|
wait: WaitPolicy;
|
|
113
|
+
/** Rolling publication window; `off` disables time filtering. */
|
|
114
|
+
timeRange: TimeRange;
|
|
93
115
|
/** AnySearch credential, or an empty string for anonymous access. */
|
|
94
116
|
anysearchApiKey: string;
|
|
95
117
|
/** AnySearch vertical domain, or an empty string for the open web. */
|
|
@@ -100,10 +122,20 @@ export interface ResolvedConfig {
|
|
|
100
122
|
cacheTtlMs: number;
|
|
101
123
|
/** Cache capacity in entries. */
|
|
102
124
|
cacheEntries: number;
|
|
125
|
+
/** Whether the cache snapshot is written to disk. */
|
|
126
|
+
cachePersist: boolean;
|
|
127
|
+
/** Custom snapshot path, or `undefined` for the default one. */
|
|
128
|
+
cacheFile: string | undefined;
|
|
103
129
|
/** Base cooldown in milliseconds. */
|
|
104
130
|
baseCooldownMs: number;
|
|
105
131
|
/** Cooldown ceiling in milliseconds. */
|
|
106
132
|
maxCooldownMs: number;
|
|
133
|
+
/** Vertical sources `domain_search` queries by default. */
|
|
134
|
+
domainSourceIds: string[];
|
|
135
|
+
/** Default result budget per vertical source. */
|
|
136
|
+
domainLimit: number;
|
|
137
|
+
/** Per-source deadline for vertical requests. */
|
|
138
|
+
domainTimeoutMs: number;
|
|
107
139
|
}
|
|
108
140
|
/**
|
|
109
141
|
* Normalise the validated config into the shape the engine layer consumes.
|
package/lib/config.js
CHANGED
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
* @module @fangchenghao/dsh-web-search/config
|
|
21
21
|
*/
|
|
22
22
|
import z from '@deepseek-ai/schemastery';
|
|
23
|
+
import { DEFAULT_FETCH_MAX_LENGTH } from "./markdown.js";
|
|
24
|
+
import { VERTICAL_IDS, isVerticalId } from "./verticals.js";
|
|
23
25
|
/** Schema of the plugin's configuration section. */
|
|
24
26
|
export const Config = z.object({
|
|
25
27
|
hosts: z
|
|
@@ -28,6 +30,18 @@ export const Config = z.object({
|
|
|
28
30
|
.description('只启用列出的搜索引擎(如 anysearch、cn.bing),多个用逗号分隔;留空表示全部。不想限制就留空。')
|
|
29
31
|
.volatile(),
|
|
30
32
|
timeoutMs: z.natural().min(500).default(15000).description('每次向搜索引擎要结果,最多等它多久(毫秒,1000 = 1 秒)。网络慢可加到 20000;平时保持 15000 就行。').volatile(),
|
|
33
|
+
fetchFormat: z
|
|
34
|
+
.union(['markdown', 'text'])
|
|
35
|
+
.default('markdown')
|
|
36
|
+
.description('抓取网页后返回什么样式:markdown 保留标题、列表、表格和链接,最适合阅读;text 只留纯文字、去掉所有标记。一般保持 markdown。')
|
|
37
|
+
.volatile(),
|
|
38
|
+
fetchMaxLength: z
|
|
39
|
+
.natural()
|
|
40
|
+
.min(1000)
|
|
41
|
+
.max(200000)
|
|
42
|
+
.default(DEFAULT_FETCH_MAX_LENGTH)
|
|
43
|
+
.description('一次抓取最多返回多少个字符(1000-200000);超出会在末尾注明原始长度。想一次多读可调到 50000,日常保持 20000 即可。')
|
|
44
|
+
.volatile(),
|
|
31
45
|
wait: z
|
|
32
46
|
.union(['auto', 'all', 'fast'])
|
|
33
47
|
.default('auto')
|
|
@@ -46,8 +60,23 @@ export const Config = z.object({
|
|
|
46
60
|
authorityBoost: z.number().min(0).max(10).default(0.25).description('上面那份网站名单能让结果往前排多少(0 到 10,0 = 不加强)。觉得好网站排得不够前就调大;不确定保持 0.25。').volatile(),
|
|
47
61
|
freshnessBoost: z.number().min(0).max(10).default(0.2).description('新发布的网页能额外往前排多少(0 到 10,0 = 不看新旧)。常查新闻可调到 0.5;不确定保持 0.2。').volatile(),
|
|
48
62
|
freshnessDays: z.natural().min(1).default(30).description('多新的网页才算新(单位:天)。查新闻可调小到 7,查资料可调大到 365;不确定保持 30。').volatile(),
|
|
63
|
+
timeRange: z
|
|
64
|
+
.union(['off', 'day', 'week', 'month', 'year'])
|
|
65
|
+
.default('off')
|
|
66
|
+
.description('只保留多久以内发布的结果:off 不限时间、day 最近 24 小时、week 最近 7 天、month 最近 30 天、year 最近 1 年。开启后只返回窗口内发布的结果;若一条带日期的都没落进窗口,会退回无日期的结果避免空手;不确定保持 off。')
|
|
67
|
+
.volatile(),
|
|
49
68
|
cacheTtlMs: z.natural().default(600000).description('同一问题多久内复用上次结果(缓存,即存一份备用),毫秒。想早点看到新内容就调小;保持 600000(10 分钟)即可。').volatile(),
|
|
50
69
|
cacheEntries: z.natural().default(100).description('最多存多少条旧结果备用(缓存,即存起来省得重复查)。存满后先丢最久没用的;一般保持 100 即可。').volatile(),
|
|
70
|
+
cachePersist: z
|
|
71
|
+
.boolean()
|
|
72
|
+
.default(true)
|
|
73
|
+
.description('是否把缓存保存到硬盘,重启后还能复用上次结果。默认开启(true);不想留痕或磁盘只读时改成 false。')
|
|
74
|
+
.volatile(),
|
|
75
|
+
cacheFile: z
|
|
76
|
+
.string()
|
|
77
|
+
.default('')
|
|
78
|
+
.description('缓存文件放哪;留空用默认 ~/.dsh/cache/dsh-web-search/cache.json。一般不用填,换位置时再写绝对路径。')
|
|
79
|
+
.volatile(),
|
|
51
80
|
anysearchApiKey: z
|
|
52
81
|
.string()
|
|
53
82
|
.default('')
|
|
@@ -58,6 +87,13 @@ export const Config = z.object({
|
|
|
58
87
|
.description('只搜某一类专业内容(如 finance 财经、academic 学术、code 代码)。要专查某类才填;日常搜索留空。').volatile(),
|
|
59
88
|
baseCooldownMs: z.natural().default(300000).description('搜索引擎说“你查得太频繁了”后,先歇多久(毫秒)。歇太久会少一个引擎,太短易被再拒;保持 300000(5 分钟)即可。').volatile(),
|
|
60
89
|
maxCooldownMs: z.natural().default(1800000).description('它反复被拒时,最长让它歇多久(毫秒),也就是等待一步步加长的封顶值。一般不用改;保持 1800000(30 分钟)即可。').volatile(),
|
|
90
|
+
domainSources: z
|
|
91
|
+
.union([z.array(z.string()), z.string()])
|
|
92
|
+
.default([...VERTICAL_IDS])
|
|
93
|
+
.description('默认查哪些专用源,逗号分隔;留空表示 github、npm 等全部八个源。')
|
|
94
|
+
.volatile(),
|
|
95
|
+
domainLimit: z.natural().min(1).max(20).default(5).description('domain_search 一次最多返回几条(1-20);默认 5 条,想多看可调到 10。').volatile(),
|
|
96
|
+
domainTimeoutMs: z.natural().min(500).default(8000).description('每个专用源最多等多久(毫秒);网络慢可加到 15000,平时 8000 即可。').volatile(),
|
|
61
97
|
});
|
|
62
98
|
/**
|
|
63
99
|
* Read one field, whichever shape it arrived in.
|
|
@@ -93,19 +129,33 @@ export function resolveConfig(config, callLimit) {
|
|
|
93
129
|
const validated = Config(plain);
|
|
94
130
|
const keys = Object.keys(validated);
|
|
95
131
|
const settings = Object.fromEntries(keys.map((key) => [key, read(supplied[key], read(validated[key], undefined))]));
|
|
96
|
-
// A volatile reference hands out a frozen snapshot, so the
|
|
97
|
-
//
|
|
132
|
+
// A volatile reference hands out a frozen snapshot, so the list fields are
|
|
133
|
+
// copied before they leave: a caller may sort or splice what it gets.
|
|
98
134
|
const hosts = [...settings.hosts].filter((id) => id.trim().length > 0);
|
|
135
|
+
// The field accepts an array or the comma separated form the Settings card
|
|
136
|
+
// writes; both normalise to the ids the tool understands.
|
|
137
|
+
const rawDomainSources = settings.domainSources;
|
|
138
|
+
const domainList = Array.isArray(rawDomainSources)
|
|
139
|
+
? rawDomainSources
|
|
140
|
+
: String(rawDomainSources).split(/[\s,]+/);
|
|
141
|
+
const domainSources = domainList.map((id) => id.trim()).filter(isVerticalId);
|
|
99
142
|
const proxy = settings.proxy.trim();
|
|
143
|
+
const cacheFile = settings.cacheFile.trim();
|
|
100
144
|
const limit = Math.min(callLimit ?? settings.limit, settings.limit);
|
|
101
145
|
return {
|
|
102
146
|
hostIds: hosts.length > 0 ? hosts : undefined,
|
|
103
147
|
timeoutMs: settings.timeoutMs,
|
|
148
|
+
fetchFormat: settings.fetchFormat,
|
|
149
|
+
fetchMaxLength: settings.fetchMaxLength,
|
|
104
150
|
proxy: proxy.length > 0 ? proxy : undefined,
|
|
105
151
|
limit,
|
|
106
152
|
wait: settings.wait,
|
|
153
|
+
timeRange: settings.timeRange,
|
|
107
154
|
anysearchApiKey: settings.anysearchApiKey.trim(),
|
|
108
155
|
anysearchDomain: settings.anysearchDomain.trim(),
|
|
156
|
+
domainSourceIds: domainSources.length > 0 ? domainSources : [...VERTICAL_IDS],
|
|
157
|
+
domainLimit: settings.domainLimit,
|
|
158
|
+
domainTimeoutMs: settings.domainTimeoutMs,
|
|
109
159
|
rank: {
|
|
110
160
|
rrfConstant: settings.rrfConstant,
|
|
111
161
|
...(settings.authorityDomains.length > 0 ? { authorityDomains: [...settings.authorityDomains] } : {}),
|
|
@@ -115,6 +165,8 @@ export function resolveConfig(config, callLimit) {
|
|
|
115
165
|
},
|
|
116
166
|
cacheTtlMs: settings.cacheTtlMs,
|
|
117
167
|
cacheEntries: settings.cacheEntries,
|
|
168
|
+
cachePersist: settings.cachePersist,
|
|
169
|
+
cacheFile: cacheFile.length > 0 ? cacheFile : undefined,
|
|
118
170
|
baseCooldownMs: settings.baseCooldownMs,
|
|
119
171
|
maxCooldownMs: settings.maxCooldownMs,
|
|
120
172
|
};
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-facing `domain_search` tool.
|
|
3
|
+
*
|
|
4
|
+
* The host's tool registry is reached through `ctx.get('tools')`, deliberately
|
|
5
|
+
* **not** through `inject`: an inject entry is a hard dependency, so a
|
|
6
|
+
* deployment that loads this plugin without the tool service would refuse to
|
|
7
|
+
* mount the plugin at all and `web_search` would disappear with it. Reading the
|
|
8
|
+
* service at mount time and declining to register when it is absent keeps the
|
|
9
|
+
* plugin fail-safe, and matches how the shipped tool packages read optional
|
|
10
|
+
* services (`ctx.get(...)` without `inject`).
|
|
11
|
+
*
|
|
12
|
+
* The definition is built from the registry's raw JSON-Schema dialect rather
|
|
13
|
+
* than `defineTool`, because this package must not take a runtime dependency on
|
|
14
|
+
* `@deepseek-ai/dsh-tools` (the host owns that package and it is not resolvable
|
|
15
|
+
* from a plugin's own `node_modules`). The shape below is the one
|
|
16
|
+
* `ToolRuntime.register()` accepts: `parameters` is an object-rooted JSON
|
|
17
|
+
* Schema with a `required` **array**, and `output` carries `{ schema, render }`.
|
|
18
|
+
*
|
|
19
|
+
* @module @fangchenghao/dsh-web-search/domain-tool
|
|
20
|
+
*/
|
|
21
|
+
import type { Source } from './types.ts';
|
|
22
|
+
import type { FetchLike } from './verticals.ts';
|
|
23
|
+
/** The name this tool prefers. */
|
|
24
|
+
export declare const DOMAIN_TOOL_NAME = "domain_search";
|
|
25
|
+
/** The name used when another plugin already registered `domain_search`. */
|
|
26
|
+
export declare const DOMAIN_TOOL_FALLBACK_NAME = "web_domain_search";
|
|
27
|
+
/** The subset of JSON Schema the host registry enforces. */
|
|
28
|
+
export interface ToolJsonSchema {
|
|
29
|
+
type?: 'object' | 'array' | 'string' | 'integer' | 'number' | 'boolean' | 'null';
|
|
30
|
+
description?: string;
|
|
31
|
+
properties?: Record<string, ToolJsonSchema>;
|
|
32
|
+
required?: string[];
|
|
33
|
+
additionalProperties?: boolean;
|
|
34
|
+
items?: ToolJsonSchema;
|
|
35
|
+
enum?: string[];
|
|
36
|
+
}
|
|
37
|
+
/** One model-facing content block, as `output.render` returns. */
|
|
38
|
+
export interface ToolContent {
|
|
39
|
+
type: 'text';
|
|
40
|
+
text: string;
|
|
41
|
+
}
|
|
42
|
+
/** A tool definition in the registry's native shape. */
|
|
43
|
+
export interface DomainToolDefinition {
|
|
44
|
+
name: string;
|
|
45
|
+
description: string;
|
|
46
|
+
parameters: ToolJsonSchema;
|
|
47
|
+
output: {
|
|
48
|
+
schema: ToolJsonSchema;
|
|
49
|
+
render: (args: unknown, value: DomainToolValue) => ToolContent[];
|
|
50
|
+
};
|
|
51
|
+
timeoutMs?: number;
|
|
52
|
+
isConcurrencySafe?: (args: unknown) => boolean;
|
|
53
|
+
execute: (args: unknown, exec: {
|
|
54
|
+
signal?: AbortSignal;
|
|
55
|
+
}) => Promise<DomainToolValue>;
|
|
56
|
+
}
|
|
57
|
+
/** The slice of the tool registry this module needs. */
|
|
58
|
+
export interface ToolRegistryLike {
|
|
59
|
+
register(definition: DomainToolDefinition): unknown;
|
|
60
|
+
get?(name: string, scope?: unknown): unknown;
|
|
61
|
+
}
|
|
62
|
+
/** What the tool returns to the model. */
|
|
63
|
+
export interface DomainToolValue {
|
|
64
|
+
sources: Source[];
|
|
65
|
+
failures: string[];
|
|
66
|
+
truncated: boolean;
|
|
67
|
+
}
|
|
68
|
+
/** Settings the tool reads live for every call. */
|
|
69
|
+
export interface DomainToolSettings {
|
|
70
|
+
/** Enabled source ids; empty means every source. */
|
|
71
|
+
sourceIds: readonly string[];
|
|
72
|
+
/** Default result budget per source. */
|
|
73
|
+
limit: number;
|
|
74
|
+
/** Per-source deadline in milliseconds. */
|
|
75
|
+
timeoutMs: number;
|
|
76
|
+
/** Explicit tunnel URL, or `undefined` to auto-detect. */
|
|
77
|
+
proxy?: string;
|
|
78
|
+
}
|
|
79
|
+
/** Construction seams, so the definition is testable without a network. */
|
|
80
|
+
export interface DomainToolOptions {
|
|
81
|
+
/** Replace the platform `fetch` (tests). */
|
|
82
|
+
fetch?: FetchLike;
|
|
83
|
+
/** Replace proxy detection (tests). */
|
|
84
|
+
detectProxy?: () => Promise<string | undefined>;
|
|
85
|
+
/** Registration diagnostics. */
|
|
86
|
+
warn?: (message: string) => void;
|
|
87
|
+
}
|
|
88
|
+
/** Format the model-facing result text. */
|
|
89
|
+
export declare function formatDomainValue(args: unknown, value: DomainToolValue): string;
|
|
90
|
+
/**
|
|
91
|
+
* Build the tool definition the host registry accepts.
|
|
92
|
+
* @param name - `domain_search`, or the fallback when that name is taken.
|
|
93
|
+
* @param readSettings - read the live config for every call (volatile fields).
|
|
94
|
+
* @param options - test seams and diagnostics.
|
|
95
|
+
* @returns a registry-ready definition.
|
|
96
|
+
*/
|
|
97
|
+
export declare function createDomainToolDefinition(name: string, readSettings: () => DomainToolSettings, options?: DomainToolOptions): DomainToolDefinition;
|
|
98
|
+
/**
|
|
99
|
+
* Pick the tool name this deployment can accept.
|
|
100
|
+
*
|
|
101
|
+
* A plugin that already owns `domain_search` must not make this one fail to
|
|
102
|
+
* register, so the fallback name is used instead. A registry without `get` is
|
|
103
|
+
* assumed free.
|
|
104
|
+
* @param tools - the live registry.
|
|
105
|
+
* @returns the preferred or fallback name.
|
|
106
|
+
*/
|
|
107
|
+
export declare function pickDomainToolName(tools: ToolRegistryLike): string;
|
|
108
|
+
/**
|
|
109
|
+
* Register the tool when a registry is available.
|
|
110
|
+
*
|
|
111
|
+
* Every failure path is contained: a missing service is the caller's job to
|
|
112
|
+
* check, a duplicate name falls back, and any registry rejection is reported
|
|
113
|
+
* and swallowed so `web_search` keeps working.
|
|
114
|
+
* @param tools - the live registry.
|
|
115
|
+
* @param readSettings - live config reader.
|
|
116
|
+
* @param options - test seams and diagnostics.
|
|
117
|
+
* @returns the registered name, or `undefined` when registration was skipped.
|
|
118
|
+
*/
|
|
119
|
+
export declare function registerDomainTool(tools: ToolRegistryLike, readSettings: () => DomainToolSettings, options?: DomainToolOptions): string | undefined;
|