@fffangchenghao/dsh-web-search 0.11.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +85 -0
- package/README.md +307 -46
- package/README.zh-CN.md +307 -46
- package/client.js +44 -3
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +48 -4
- package/lib/cache.js +134 -9
- package/lib/config.d.ts +25 -0
- package/lib/config.js +42 -0
- package/lib/engines.d.ts +41 -1
- package/lib/engines.js +33 -2
- package/lib/fetch.d.ts +33 -0
- package/lib/fetch.js +218 -6
- package/lib/index.d.ts +26 -0
- package/lib/index.js +113 -27
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/rank.js +4 -1
- package/lib/read-page.d.ts +136 -0
- package/lib/read-page.js +496 -0
- package/lib/render.d.ts +193 -0
- package/lib/render.js +354 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.js +19 -0
- package/lib/types.d.ts +13 -4
- package/package.json +1 -1
package/lib/cache.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { readCacheFile, resolveCacheFile, writeCacheFile } from "./cache-persist.js";
|
|
1
2
|
/** Default entry lifetime: ten minutes. */
|
|
2
3
|
const DEFAULT_TTL_MS = 600_000;
|
|
3
4
|
/** Default number of retained entries. */
|
|
4
5
|
const DEFAULT_MAX_ENTRIES = 100;
|
|
6
|
+
/** Default minimum gap between disk writes. */
|
|
7
|
+
const DEFAULT_PERSIST_THROTTLE_MS = 2_000;
|
|
5
8
|
/**
|
|
6
9
|
* Build the cache key for one search request.
|
|
7
10
|
*
|
|
@@ -53,21 +56,51 @@ function normaliseCount(value, fallback) {
|
|
|
53
56
|
*/
|
|
54
57
|
export class SearchCache {
|
|
55
58
|
/** Entry lifetime in milliseconds; `0` disables the cache completely. */
|
|
56
|
-
|
|
59
|
+
/** Live TTL reader: settings may change without a remount. */
|
|
60
|
+
ttlOf;
|
|
57
61
|
/** Maximum retained entries; the oldest read is evicted first. */
|
|
58
|
-
|
|
62
|
+
/** Live entry-cap reader: settings may change without a remount. */
|
|
63
|
+
maxOf;
|
|
59
64
|
/** Injected clock, so tests never wait on real time. */
|
|
60
65
|
now;
|
|
61
66
|
/** Insertion order doubles as LRU order: the first key is the coldest. */
|
|
62
67
|
entries = new Map();
|
|
68
|
+
/** Snapshot destination, or `undefined` for a purely in-memory cache. */
|
|
69
|
+
persistTo;
|
|
70
|
+
/** Minimum gap between writes. */
|
|
71
|
+
throttleMs;
|
|
72
|
+
/** Whether memory changed since the last successful write. */
|
|
73
|
+
dirty = false;
|
|
74
|
+
/** Pending throttled flush. */
|
|
75
|
+
timer;
|
|
63
76
|
/**
|
|
64
|
-
* @param options - TTL
|
|
77
|
+
* @param options - TTL, entry cap and optional persistence.
|
|
65
78
|
* @param now - clock override; defaults to `Date.now`.
|
|
66
79
|
*/
|
|
67
80
|
constructor(options = {}, now) {
|
|
68
|
-
|
|
69
|
-
|
|
81
|
+
const ttlOption = options.ttlMs;
|
|
82
|
+
const maxOption = options.maxEntries;
|
|
83
|
+
this.ttlOf = () => normaliseDuration(typeof ttlOption === 'function' ? ttlOption() : ttlOption, DEFAULT_TTL_MS);
|
|
84
|
+
this.maxOf = () => normaliseCount(typeof maxOption === 'function' ? maxOption() : maxOption, DEFAULT_MAX_ENTRIES);
|
|
70
85
|
this.now = now ?? Date.now;
|
|
86
|
+
this.throttleMs = normaliseDuration(options.persist?.throttleMs, DEFAULT_PERSIST_THROTTLE_MS);
|
|
87
|
+
const persist = options.persist;
|
|
88
|
+
if (persist !== undefined && this.ttlOf() > 0 && this.maxOf() > 0) {
|
|
89
|
+
const file = resolveCacheFile(persist.file);
|
|
90
|
+
// Startup restore is synchronous and fully contained: an unreadable or
|
|
91
|
+
// malformed snapshot simply yields an empty cache.
|
|
92
|
+
const loaded = readCacheFile(file, this.now(), this.maxOf(), persist.io);
|
|
93
|
+
for (const entry of loaded) {
|
|
94
|
+
this.entries.set(entry.key, { sources: entry.sources, expiresAt: entry.expiresAt });
|
|
95
|
+
}
|
|
96
|
+
this.persistTo = {
|
|
97
|
+
file,
|
|
98
|
+
...(persist.maxBytes === undefined ? {} : { maxBytes: persist.maxBytes }),
|
|
99
|
+
...(persist.io === undefined ? {} : { io: persist.io }),
|
|
100
|
+
};
|
|
101
|
+
liveCaches.add(this);
|
|
102
|
+
installExitHook();
|
|
103
|
+
}
|
|
71
104
|
}
|
|
72
105
|
/** Number of retained entries, including ones whose TTL already elapsed. */
|
|
73
106
|
get size() {
|
|
@@ -82,13 +115,14 @@ export class SearchCache {
|
|
|
82
115
|
* @returns a copy of the cached sources, or `undefined` on miss/expiry.
|
|
83
116
|
*/
|
|
84
117
|
get(key) {
|
|
85
|
-
if (this.
|
|
118
|
+
if (this.ttlOf() === 0 || this.maxOf() === 0)
|
|
86
119
|
return undefined;
|
|
87
120
|
const entry = this.entries.get(key);
|
|
88
121
|
if (entry === undefined)
|
|
89
122
|
return undefined;
|
|
90
123
|
if (this.now() >= entry.expiresAt) {
|
|
91
124
|
this.entries.delete(key);
|
|
125
|
+
this.touch();
|
|
92
126
|
return undefined;
|
|
93
127
|
}
|
|
94
128
|
// Re-insert to move the entry to the young end of the map.
|
|
@@ -104,22 +138,113 @@ export class SearchCache {
|
|
|
104
138
|
* @param sources - the merged source list; copied before storage.
|
|
105
139
|
*/
|
|
106
140
|
set(key, sources) {
|
|
107
|
-
|
|
141
|
+
const ttlMs = this.ttlOf();
|
|
142
|
+
const maxEntries = this.maxOf();
|
|
143
|
+
if (ttlMs === 0 || maxEntries === 0)
|
|
108
144
|
return;
|
|
109
145
|
this.entries.delete(key);
|
|
110
146
|
this.entries.set(key, {
|
|
111
147
|
sources: cloneSources(sources),
|
|
112
|
-
expiresAt: this.now() +
|
|
148
|
+
expiresAt: this.now() + ttlMs,
|
|
113
149
|
});
|
|
114
|
-
while (this.entries.size >
|
|
150
|
+
while (this.entries.size > maxEntries) {
|
|
115
151
|
const oldest = this.entries.keys().next();
|
|
116
152
|
if (oldest.done === true)
|
|
117
153
|
break;
|
|
118
154
|
this.entries.delete(oldest.value);
|
|
119
155
|
}
|
|
156
|
+
this.touch();
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* Write the snapshot now, if there is anything to write.
|
|
160
|
+
*
|
|
161
|
+
* Writes are normally throttled and happen on a timer or at process exit;
|
|
162
|
+
* this is the deterministic seam tests use, and the call a shutdown path can
|
|
163
|
+
* make when it wants the newest state on disk immediately.
|
|
164
|
+
*
|
|
165
|
+
* @returns `true` when the file was replaced, `false` for "nothing to write"
|
|
166
|
+
* or a contained filesystem failure.
|
|
167
|
+
*/
|
|
168
|
+
flush() {
|
|
169
|
+
if (this.persistTo === undefined)
|
|
170
|
+
return false;
|
|
171
|
+
if (this.timer !== undefined) {
|
|
172
|
+
clearTimeout(this.timer);
|
|
173
|
+
this.timer = undefined;
|
|
174
|
+
}
|
|
175
|
+
if (!this.dirty)
|
|
176
|
+
return false;
|
|
177
|
+
const entries = [];
|
|
178
|
+
for (const [key, entry] of this.entries) {
|
|
179
|
+
entries.push({ key, expiresAt: entry.expiresAt, sources: cloneSources(entry.sources) });
|
|
180
|
+
}
|
|
181
|
+
const written = writeCacheFile(this.persistTo.file, entries, this.now(), {
|
|
182
|
+
...(this.persistTo.maxBytes === undefined ? {} : { maxBytes: this.persistTo.maxBytes }),
|
|
183
|
+
...(this.persistTo.io === undefined ? {} : { io: this.persistTo.io }),
|
|
184
|
+
});
|
|
185
|
+
this.dirty = !written;
|
|
186
|
+
if (!written)
|
|
187
|
+
this.schedule();
|
|
188
|
+
return written;
|
|
189
|
+
}
|
|
190
|
+
/** Stop the exit hook and flush whatever is still pending. */
|
|
191
|
+
close() {
|
|
192
|
+
this.flush();
|
|
193
|
+
if (this.timer !== undefined) {
|
|
194
|
+
clearTimeout(this.timer);
|
|
195
|
+
this.timer = undefined;
|
|
196
|
+
}
|
|
197
|
+
liveCaches.delete(this);
|
|
198
|
+
if (liveCaches.size === 0)
|
|
199
|
+
removeExitHook();
|
|
120
200
|
}
|
|
121
201
|
/** Drop every entry. */
|
|
122
202
|
clear() {
|
|
203
|
+
if (this.entries.size === 0)
|
|
204
|
+
return;
|
|
123
205
|
this.entries.clear();
|
|
206
|
+
this.touch();
|
|
124
207
|
}
|
|
208
|
+
/** Mark the snapshot stale and arrange a throttled flush. */
|
|
209
|
+
touch() {
|
|
210
|
+
if (this.persistTo === undefined)
|
|
211
|
+
return;
|
|
212
|
+
this.dirty = true;
|
|
213
|
+
this.schedule();
|
|
214
|
+
}
|
|
215
|
+
/** Arm the single pending timer, if persistence is enabled. */
|
|
216
|
+
schedule() {
|
|
217
|
+
if (this.persistTo === undefined || this.timer !== undefined)
|
|
218
|
+
return;
|
|
219
|
+
this.timer = setTimeout(() => {
|
|
220
|
+
this.timer = undefined;
|
|
221
|
+
this.flush();
|
|
222
|
+
}, this.throttleMs);
|
|
223
|
+
// A pending cache flush must not keep a one-shot CLI process alive; the
|
|
224
|
+
// exit hook still writes it synchronously.
|
|
225
|
+
this.timer.unref();
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
/** Caches that still need an exit-time flush. */
|
|
229
|
+
const liveCaches = new Set();
|
|
230
|
+
/** Set once the process-level hook is installed. */
|
|
231
|
+
let exitHookInstalled = false;
|
|
232
|
+
/** Flush every live cache; used only during process exit. */
|
|
233
|
+
function flushLiveCaches() {
|
|
234
|
+
for (const cache of liveCaches)
|
|
235
|
+
cache.flush();
|
|
236
|
+
}
|
|
237
|
+
/** Install one process-wide exit hook, lazily. */
|
|
238
|
+
function installExitHook() {
|
|
239
|
+
if (exitHookInstalled)
|
|
240
|
+
return;
|
|
241
|
+
exitHookInstalled = true;
|
|
242
|
+
process.once('exit', flushLiveCaches);
|
|
243
|
+
}
|
|
244
|
+
/** Remove the hook when no persisted cache remains. */
|
|
245
|
+
function removeExitHook() {
|
|
246
|
+
if (!exitHookInstalled)
|
|
247
|
+
return;
|
|
248
|
+
process.removeListener('exit', flushLiveCaches);
|
|
249
|
+
exitHookInstalled = false;
|
|
125
250
|
}
|
package/lib/config.d.ts
CHANGED
|
@@ -20,11 +20,16 @@
|
|
|
20
20
|
* @module @fangchenghao/dsh-web-search/config
|
|
21
21
|
*/
|
|
22
22
|
import z from '@deepseek-ai/schemastery';
|
|
23
|
+
import type { RenderMode } from './render.ts';
|
|
23
24
|
import type { RankOptions, TimeRange, WaitPolicy } from './types.ts';
|
|
24
25
|
/** Schema of the plugin's configuration section. */
|
|
25
26
|
export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
26
27
|
hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
27
28
|
timeoutMs: z<number, number, "volatile-defined">;
|
|
29
|
+
fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
30
|
+
fetchMaxLength: z<number, number, "volatile-defined">;
|
|
31
|
+
renderJs: z<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
32
|
+
renderTimeoutMs: z<number, number, "volatile-defined">;
|
|
28
33
|
wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
29
34
|
proxy: z<string, string, "volatile-defined">;
|
|
30
35
|
limit: z<number, number, "volatile-defined">;
|
|
@@ -36,6 +41,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
36
41
|
timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
37
42
|
cacheTtlMs: z<number, number, "volatile-defined">;
|
|
38
43
|
cacheEntries: z<number, number, "volatile-defined">;
|
|
44
|
+
cachePersist: z<boolean, boolean, "volatile-defined">;
|
|
45
|
+
cacheFile: z<string, string, "volatile-defined">;
|
|
39
46
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
40
47
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
41
48
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
@@ -46,6 +53,10 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
46
53
|
}>>, Schemastery.ObjectT<NoInfer<{
|
|
47
54
|
hosts: z<NoInfer<string[]>, NoInfer<string[]>, "volatile-defined">;
|
|
48
55
|
timeoutMs: z<number, number, "volatile-defined">;
|
|
56
|
+
fetchFormat: z<"markdown" | "text", "markdown" | "text", "volatile-defined">;
|
|
57
|
+
fetchMaxLength: z<number, number, "volatile-defined">;
|
|
58
|
+
renderJs: z<"off" | "auto" | "always", "off" | "auto" | "always", "volatile-defined">;
|
|
59
|
+
renderTimeoutMs: z<number, number, "volatile-defined">;
|
|
49
60
|
wait: z<"auto" | "all" | "fast", "auto" | "all" | "fast", "volatile-defined">;
|
|
50
61
|
proxy: z<string, string, "volatile-defined">;
|
|
51
62
|
limit: z<number, number, "volatile-defined">;
|
|
@@ -57,6 +68,8 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
57
68
|
timeRange: z<"off" | "day" | "week" | "month" | "year", "off" | "day" | "week" | "month" | "year", "volatile-defined">;
|
|
58
69
|
cacheTtlMs: z<number, number, "volatile-defined">;
|
|
59
70
|
cacheEntries: z<number, number, "volatile-defined">;
|
|
71
|
+
cachePersist: z<boolean, boolean, "volatile-defined">;
|
|
72
|
+
cacheFile: z<string, string, "volatile-defined">;
|
|
60
73
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
61
74
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
62
75
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
@@ -92,6 +105,14 @@ export interface ResolvedConfig {
|
|
|
92
105
|
hostIds: string[] | undefined;
|
|
93
106
|
/** Per-engine deadline. */
|
|
94
107
|
timeoutMs: number;
|
|
108
|
+
/** `web_fetch` output style. */
|
|
109
|
+
fetchFormat: 'markdown' | 'text';
|
|
110
|
+
/** `web_fetch` character budget. */
|
|
111
|
+
fetchMaxLength: number;
|
|
112
|
+
/** Browser rendering policy for `web_fetch`. */
|
|
113
|
+
renderJs: RenderMode;
|
|
114
|
+
/** Browser-render deadline in milliseconds. */
|
|
115
|
+
renderTimeoutMs: number;
|
|
95
116
|
/** Explicit tunnel URL, or `undefined` to detect one. */
|
|
96
117
|
proxy: string | undefined;
|
|
97
118
|
/** Caller budget cap. */
|
|
@@ -110,6 +131,10 @@ export interface ResolvedConfig {
|
|
|
110
131
|
cacheTtlMs: number;
|
|
111
132
|
/** Cache capacity in entries. */
|
|
112
133
|
cacheEntries: number;
|
|
134
|
+
/** Whether the cache snapshot is written to disk. */
|
|
135
|
+
cachePersist: boolean;
|
|
136
|
+
/** Custom snapshot path, or `undefined` for the default one. */
|
|
137
|
+
cacheFile: string | undefined;
|
|
113
138
|
/** Base cooldown in milliseconds. */
|
|
114
139
|
baseCooldownMs: number;
|
|
115
140
|
/** Cooldown ceiling in milliseconds. */
|
package/lib/config.js
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
* @module @fangchenghao/dsh-web-search/config
|
|
21
21
|
*/
|
|
22
22
|
import z from '@deepseek-ai/schemastery';
|
|
23
|
+
import { DEFAULT_FETCH_MAX_LENGTH } from "./markdown.js";
|
|
23
24
|
import { VERTICAL_IDS, isVerticalId } from "./verticals.js";
|
|
24
25
|
/** Schema of the plugin's configuration section. */
|
|
25
26
|
export const Config = z.object({
|
|
@@ -29,6 +30,30 @@ export const Config = z.object({
|
|
|
29
30
|
.description('只启用列出的搜索引擎(如 anysearch、cn.bing),多个用逗号分隔;留空表示全部。不想限制就留空。')
|
|
30
31
|
.volatile(),
|
|
31
32
|
timeoutMs: z.natural().min(500).default(15000).description('每次向搜索引擎要结果,最多等它多久(毫秒,1000 = 1 秒)。网络慢可加到 20000;平时保持 15000 就行。').volatile(),
|
|
33
|
+
fetchFormat: z
|
|
34
|
+
.union(['markdown', 'text'])
|
|
35
|
+
.default('markdown')
|
|
36
|
+
.description('抓取网页后返回什么样式:markdown 保留标题、列表、表格和链接,最适合阅读;text 只留纯文字、去掉所有标记。一般保持 markdown。')
|
|
37
|
+
.volatile(),
|
|
38
|
+
fetchMaxLength: z
|
|
39
|
+
.natural()
|
|
40
|
+
.min(1000)
|
|
41
|
+
.max(200000)
|
|
42
|
+
.default(DEFAULT_FETCH_MAX_LENGTH)
|
|
43
|
+
.description('一次抓取最多返回多少个字符(1000-200000);超出会在末尾注明原始长度。想一次多读可调到 50000,日常保持 20000 即可。')
|
|
44
|
+
.volatile(),
|
|
45
|
+
renderJs: z
|
|
46
|
+
.union(['auto', 'always', 'off'])
|
|
47
|
+
.default('auto')
|
|
48
|
+
.description('脚本渲染页面的处理方式:auto 只在正文像空壳网页时才用浏览器渲染(推荐),always 每次抓取都渲染(更慢),off 从不渲染。需要另外安装 Playwright;没装就静默跳过,不影响抓取。')
|
|
49
|
+
.volatile(),
|
|
50
|
+
renderTimeoutMs: z
|
|
51
|
+
.natural()
|
|
52
|
+
.min(1000)
|
|
53
|
+
.max(120000)
|
|
54
|
+
.default(15000)
|
|
55
|
+
.description('用浏览器渲染一个页面最多等多久(毫秒,1000-120000)。页面很重可加到 30000;平时保持 15000 即可。')
|
|
56
|
+
.volatile(),
|
|
32
57
|
wait: z
|
|
33
58
|
.union(['auto', 'all', 'fast'])
|
|
34
59
|
.default('auto')
|
|
@@ -54,6 +79,16 @@ export const Config = z.object({
|
|
|
54
79
|
.volatile(),
|
|
55
80
|
cacheTtlMs: z.natural().default(600000).description('同一问题多久内复用上次结果(缓存,即存一份备用),毫秒。想早点看到新内容就调小;保持 600000(10 分钟)即可。').volatile(),
|
|
56
81
|
cacheEntries: z.natural().default(100).description('最多存多少条旧结果备用(缓存,即存起来省得重复查)。存满后先丢最久没用的;一般保持 100 即可。').volatile(),
|
|
82
|
+
cachePersist: z
|
|
83
|
+
.boolean()
|
|
84
|
+
.default(true)
|
|
85
|
+
.description('是否把缓存保存到硬盘,重启后还能复用上次结果。默认开启(true);不想留痕或磁盘只读时改成 false。')
|
|
86
|
+
.volatile(),
|
|
87
|
+
cacheFile: z
|
|
88
|
+
.string()
|
|
89
|
+
.default('')
|
|
90
|
+
.description('缓存文件放哪;留空用默认 ~/.dsh/cache/dsh-web-search/cache.json。一般不用填,换位置时再写绝对路径。')
|
|
91
|
+
.volatile(),
|
|
57
92
|
anysearchApiKey: z
|
|
58
93
|
.string()
|
|
59
94
|
.default('')
|
|
@@ -117,10 +152,15 @@ export function resolveConfig(config, callLimit) {
|
|
|
117
152
|
: String(rawDomainSources).split(/[\s,]+/);
|
|
118
153
|
const domainSources = domainList.map((id) => id.trim()).filter(isVerticalId);
|
|
119
154
|
const proxy = settings.proxy.trim();
|
|
155
|
+
const cacheFile = settings.cacheFile.trim();
|
|
120
156
|
const limit = Math.min(callLimit ?? settings.limit, settings.limit);
|
|
121
157
|
return {
|
|
122
158
|
hostIds: hosts.length > 0 ? hosts : undefined,
|
|
123
159
|
timeoutMs: settings.timeoutMs,
|
|
160
|
+
fetchFormat: settings.fetchFormat,
|
|
161
|
+
fetchMaxLength: settings.fetchMaxLength,
|
|
162
|
+
renderJs: settings.renderJs,
|
|
163
|
+
renderTimeoutMs: settings.renderTimeoutMs,
|
|
124
164
|
proxy: proxy.length > 0 ? proxy : undefined,
|
|
125
165
|
limit,
|
|
126
166
|
wait: settings.wait,
|
|
@@ -139,6 +179,8 @@ export function resolveConfig(config, callLimit) {
|
|
|
139
179
|
},
|
|
140
180
|
cacheTtlMs: settings.cacheTtlMs,
|
|
141
181
|
cacheEntries: settings.cacheEntries,
|
|
182
|
+
cachePersist: settings.cachePersist,
|
|
183
|
+
cacheFile: cacheFile.length > 0 ? cacheFile : undefined,
|
|
142
184
|
baseCooldownMs: settings.baseCooldownMs,
|
|
143
185
|
maxCooldownMs: settings.maxCooldownMs,
|
|
144
186
|
};
|
package/lib/engines.d.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*
|
|
12
12
|
* @module @fangchenghao/dsh-web-search/engines
|
|
13
13
|
*/
|
|
14
|
-
import type { HostSpec, RankOptions, SearchOptions, SearchOutcome } from './types.ts';
|
|
14
|
+
import type { EngineAnswer, HostSpec, RankOptions, Region, SearchOptions, SearchOutcome, Source, TimeRange, WaitPolicy } from './types.ts';
|
|
15
15
|
import type { SearchCache } from './cache.ts';
|
|
16
16
|
import type { EngineHealth } from './health.ts';
|
|
17
17
|
/**
|
|
@@ -34,6 +34,46 @@ export interface EngineRunOptions extends SearchOptions {
|
|
|
34
34
|
cache?: SearchCache;
|
|
35
35
|
/** Cooldown tracker, when one is configured. */
|
|
36
36
|
health?: EngineHealth;
|
|
37
|
+
/**
|
|
38
|
+
* Read-only observer of the fusion inputs.
|
|
39
|
+
*
|
|
40
|
+
* Benchmark recorders use it to persist each engine's own list and rerun the
|
|
41
|
+
* current ranking offline. It never changes the run: the snapshot is a copy
|
|
42
|
+
* handed over after the fused list is final, and an absent hook costs one
|
|
43
|
+
* `undefined` check per run.
|
|
44
|
+
*/
|
|
45
|
+
trace?: (snapshot: SearchTrace) => void;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* What one search run handed to the ranking, captured for offline replay.
|
|
49
|
+
*
|
|
50
|
+
* `answers` are the exact per-engine lists that produced `sources`, in engine
|
|
51
|
+
* priority order and already narrowed by an active time window. Only the tier
|
|
52
|
+
* that actually supplied the fused list is present: when the primary tier
|
|
53
|
+
* answered, the fallback engines never ran. A cache hit has no fresh answers,
|
|
54
|
+
* so `answers` is empty and `cached` is `true`.
|
|
55
|
+
*/
|
|
56
|
+
export interface SearchTrace {
|
|
57
|
+
/** Caller budget, after the engine layer's own clamp. */
|
|
58
|
+
limit: number;
|
|
59
|
+
/** Region that led this query. */
|
|
60
|
+
lead: Region;
|
|
61
|
+
/** Release policy used for the tier run. */
|
|
62
|
+
wait: WaitPolicy;
|
|
63
|
+
/** Active publication window. */
|
|
64
|
+
timeRange: TimeRange;
|
|
65
|
+
/** Ranking knobs, with the window folded in. */
|
|
66
|
+
rank: RankOptions;
|
|
67
|
+
/** Which path produced `sources`. */
|
|
68
|
+
tier: 'primary' | 'fallback' | 'none' | 'cache';
|
|
69
|
+
/** Per-engine lists that were fused, in engine priority order. */
|
|
70
|
+
answers: EngineAnswer[];
|
|
71
|
+
/** The fused list `search` returned. */
|
|
72
|
+
sources: Source[];
|
|
73
|
+
/** Failure strings accumulated by this run. */
|
|
74
|
+
failures: string[];
|
|
75
|
+
/** Whether the result came from the cache. */
|
|
76
|
+
cached: boolean;
|
|
37
77
|
}
|
|
38
78
|
/**
|
|
39
79
|
* Search the web with every configured engine and fuse the answers.
|
package/lib/engines.js
CHANGED
|
@@ -66,7 +66,9 @@ export const HOSTS = [
|
|
|
66
66
|
},
|
|
67
67
|
{
|
|
68
68
|
id: 'so',
|
|
69
|
-
|
|
69
|
+
// 360 answers an anti-bot wall from this network and from the tunnel, so it
|
|
70
|
+
// is demoted: tried only when every primary engine came back empty.
|
|
71
|
+
tier: 'fallback',
|
|
70
72
|
region: 'cn',
|
|
71
73
|
build: (query) => `https://www.so.com/s?q=${encodeURIComponent(query)}`,
|
|
72
74
|
parse: parseSo,
|
|
@@ -317,14 +319,31 @@ export async function search(query, signal, options) {
|
|
|
317
319
|
// next identical query into a cache miss.
|
|
318
320
|
const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
|
|
319
321
|
const cached = key === undefined ? undefined : options.cache?.get(key);
|
|
320
|
-
if (cached !== undefined)
|
|
322
|
+
if (cached !== undefined) {
|
|
323
|
+
options.trace?.({
|
|
324
|
+
limit,
|
|
325
|
+
lead,
|
|
326
|
+
wait,
|
|
327
|
+
timeRange,
|
|
328
|
+
rank,
|
|
329
|
+
tier: 'cache',
|
|
330
|
+
answers: [],
|
|
331
|
+
sources: cached,
|
|
332
|
+
failures: [...failures],
|
|
333
|
+
cached: true,
|
|
334
|
+
});
|
|
321
335
|
return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
|
|
336
|
+
}
|
|
322
337
|
const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
|
|
323
338
|
const primary = ordered.filter((host) => host.tier === 'primary');
|
|
324
339
|
const fallback = ordered.filter((host) => host.tier === 'fallback');
|
|
340
|
+
let tier = 'none';
|
|
325
341
|
let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
342
|
+
if (answers.length > 0)
|
|
343
|
+
tier = 'primary';
|
|
326
344
|
let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
327
345
|
if (sources.length === 0 && fallback.length > 0) {
|
|
346
|
+
tier = 'fallback';
|
|
328
347
|
answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
329
348
|
sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
330
349
|
}
|
|
@@ -332,5 +351,17 @@ export async function search(query, signal, options) {
|
|
|
332
351
|
throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
|
|
333
352
|
if (key !== undefined && sources.length > 0)
|
|
334
353
|
options.cache?.set(key, sources);
|
|
354
|
+
options.trace?.({
|
|
355
|
+
limit,
|
|
356
|
+
lead,
|
|
357
|
+
wait,
|
|
358
|
+
timeRange,
|
|
359
|
+
rank,
|
|
360
|
+
tier,
|
|
361
|
+
answers,
|
|
362
|
+
sources,
|
|
363
|
+
failures: [...failures],
|
|
364
|
+
cached: false,
|
|
365
|
+
});
|
|
335
366
|
return { sources, failures, ...(proxy === undefined ? {} : { proxy }) };
|
|
336
367
|
}
|
package/lib/fetch.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { FetchFormat } from './markdown.ts';
|
|
2
|
+
import type { RenderMode, RenderReport, RenderedPage, RendererLoad, RendererModule } from './render.ts';
|
|
1
3
|
/** One retrieval outcome, mirroring the Host's `WebFetchResult`. */
|
|
2
4
|
export interface FetchOutcome {
|
|
3
5
|
readonly url: string;
|
|
@@ -7,6 +9,13 @@ export interface FetchOutcome {
|
|
|
7
9
|
readonly content: string;
|
|
8
10
|
};
|
|
9
11
|
readonly truncated: boolean;
|
|
12
|
+
/**
|
|
13
|
+
* How the optional render rung decided, when this provider produced the
|
|
14
|
+
* result. The Host ignores the extra field; it is here so a caller (and a
|
|
15
|
+
* test) can see why a shell page was or was not rendered, including that
|
|
16
|
+
* Playwright is absent.
|
|
17
|
+
*/
|
|
18
|
+
readonly render?: RenderReport;
|
|
10
19
|
}
|
|
11
20
|
/** What `ctx.web.registerFetchProvider` expects. */
|
|
12
21
|
export interface FetchProvider {
|
|
@@ -30,6 +39,20 @@ export interface FetchProviderOptions {
|
|
|
30
39
|
anysearchApiKey?: string;
|
|
31
40
|
/** Bytes kept from a response; the Host applies its own cap too. */
|
|
32
41
|
maxBytes?: number;
|
|
42
|
+
/** Output style for an HTML body; `markdown` keeps structure. */
|
|
43
|
+
/** Renderer, or a reader for it so a Settings edit applies without a remount. */
|
|
44
|
+
format?: FetchFormat | (() => FetchFormat);
|
|
45
|
+
/** Character budget for the returned text. */
|
|
46
|
+
/** Character budget, or a reader for it. */
|
|
47
|
+
maxLength?: number | (() => number);
|
|
48
|
+
/**
|
|
49
|
+
* Optional browser rendering: `auto` renders only a suspected shell page,
|
|
50
|
+
* `always` renders every fetch, `off` never does. A reader keeps a Settings
|
|
51
|
+
* edit live. Defaults to `auto`; without Playwright it silently does nothing.
|
|
52
|
+
*/
|
|
53
|
+
renderJs?: RenderMode | (() => RenderMode);
|
|
54
|
+
/** Browser-render deadline, or a reader for it. Defaults to 15000 ms. */
|
|
55
|
+
renderTimeoutMs?: number | (() => number);
|
|
33
56
|
/** User agent sent on every rung. */
|
|
34
57
|
userAgent?: string;
|
|
35
58
|
}
|
|
@@ -43,6 +66,16 @@ export interface FetchDependencies {
|
|
|
43
66
|
extract: (url: string, apiKey: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<FetchOutcome>;
|
|
44
67
|
/** Proxy discovery. */
|
|
45
68
|
detectProxy: (explicit: string | undefined) => Promise<string | undefined>;
|
|
69
|
+
/**
|
|
70
|
+
* Load the optional renderer. The default dynamically imports Playwright and
|
|
71
|
+
* reports absence instead of throwing; tests inject a fake here.
|
|
72
|
+
*/
|
|
73
|
+
loadRenderer: () => Promise<RendererLoad>;
|
|
74
|
+
/**
|
|
75
|
+
* Render one page with a loaded renderer. The default is the real
|
|
76
|
+
* headless-Chromium implementation; tests inject a fake browser.
|
|
77
|
+
*/
|
|
78
|
+
render: (renderer: RendererModule, url: string, timeoutMs: number, signal: AbortSignal | undefined) => Promise<RenderedPage>;
|
|
46
79
|
}
|
|
47
80
|
/**
|
|
48
81
|
* Build the fetch provider.
|