@fffangchenghao/dsh-web-search 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +247 -41
- package/README.zh-CN.md +247 -41
- package/client.js +66 -9
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +56 -8
- package/lib/cache.js +141 -13
- package/lib/config.d.ts +33 -1
- package/lib/config.js +54 -2
- package/lib/domain-tool.d.ts +119 -0
- package/lib/domain-tool.js +196 -0
- package/lib/engines.js +68 -10
- package/lib/fetch.d.ts +7 -0
- package/lib/fetch.js +36 -3
- package/lib/freshness.d.ts +102 -0
- package/lib/freshness.js +359 -0
- package/lib/index.d.ts +44 -2
- package/lib/index.js +144 -23
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/parsers.js +59 -5
- package/lib/rank.d.ts +14 -1
- package/lib/rank.js +18 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.d.ts +4 -2
- package/lib/transport.js +4 -2
- package/lib/types.d.ts +22 -6
- package/lib/verticals.d.ts +164 -0
- package/lib/verticals.js +539 -0
- package/package.json +4 -4
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { resolveProxy } from "./proxy.js";
|
|
2
|
+
import { searchVerticals, isVerticalId, VERTICAL_IDS } from "./verticals.js";
|
|
3
|
+
/** The name this tool prefers. */
|
|
4
|
+
export const DOMAIN_TOOL_NAME = 'domain_search';
|
|
5
|
+
/** The name used when another plugin already registered `domain_search`. */
|
|
6
|
+
export const DOMAIN_TOOL_FALLBACK_NAME = 'web_domain_search';
|
|
7
|
+
/** True for a value usable as a record. */
|
|
8
|
+
function isRecord(value) {
|
|
9
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
10
|
+
}
|
|
11
|
+
/** Clamp a tool budget into the range the adapters serve. */
|
|
12
|
+
function clampToolLimit(value, fallback) {
|
|
13
|
+
const base = Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
14
|
+
return Math.max(1, Math.min(20, base));
|
|
15
|
+
}
|
|
16
|
+
/** The parameter schema, in the registry's raw JSON-Schema dialect. */
|
|
17
|
+
function parameterSchema() {
|
|
18
|
+
return {
|
|
19
|
+
type: 'object',
|
|
20
|
+
additionalProperties: false,
|
|
21
|
+
properties: {
|
|
22
|
+
query: {
|
|
23
|
+
type: 'string',
|
|
24
|
+
description: 'What to look for: a package name, paper topic, repository, error message, or question.',
|
|
25
|
+
},
|
|
26
|
+
sources: {
|
|
27
|
+
type: 'array',
|
|
28
|
+
items: { type: 'string' },
|
|
29
|
+
description: `Sources to query: ${VERTICAL_IDS.join(', ')}. Omit to use the deployment default.`,
|
|
30
|
+
},
|
|
31
|
+
limit: {
|
|
32
|
+
type: 'integer',
|
|
33
|
+
description: 'Results wanted per source, 1–20. Omit to use the deployment default.',
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
required: ['query'],
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/** The output schema, in the same dialect. */
|
|
40
|
+
function outputSchema() {
|
|
41
|
+
return {
|
|
42
|
+
type: 'object',
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
properties: {
|
|
45
|
+
sources: {
|
|
46
|
+
type: 'array',
|
|
47
|
+
items: {
|
|
48
|
+
type: 'object',
|
|
49
|
+
additionalProperties: false,
|
|
50
|
+
properties: {
|
|
51
|
+
url: { type: 'string' },
|
|
52
|
+
title: { type: 'string' },
|
|
53
|
+
snippet: { type: 'string' },
|
|
54
|
+
publishedAt: { type: 'string' },
|
|
55
|
+
},
|
|
56
|
+
required: ['url'],
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
failures: { type: 'array', items: { type: 'string' } },
|
|
60
|
+
truncated: { type: 'boolean' },
|
|
61
|
+
},
|
|
62
|
+
required: ['sources', 'failures', 'truncated'],
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Parse model arguments into a validated shape, throwing on nonsense. */
|
|
66
|
+
function parseArgs(args, settings) {
|
|
67
|
+
if (!isRecord(args))
|
|
68
|
+
throw new Error('domain_search requires arguments');
|
|
69
|
+
const query = typeof args.query === 'string' ? args.query.trim() : '';
|
|
70
|
+
if (query.length === 0)
|
|
71
|
+
throw new Error('domain_search requires a non-empty query');
|
|
72
|
+
const enabled = settings.sourceIds.filter(isVerticalId);
|
|
73
|
+
let ids = enabled.length > 0 ? enabled : [...VERTICAL_IDS];
|
|
74
|
+
if (Array.isArray(args.sources) && args.sources.length > 0) {
|
|
75
|
+
const asked = args.sources.filter(isVerticalId);
|
|
76
|
+
if (asked.length === 0) {
|
|
77
|
+
throw new Error(`domain_search: no known source in ${JSON.stringify(args.sources)}; known: ${VERTICAL_IDS.join(', ')}`);
|
|
78
|
+
}
|
|
79
|
+
ids = asked;
|
|
80
|
+
}
|
|
81
|
+
const limit = typeof args.limit === 'number'
|
|
82
|
+
? clampToolLimit(args.limit, settings.limit)
|
|
83
|
+
: clampToolLimit(settings.limit, 5);
|
|
84
|
+
return { query, ids, limit };
|
|
85
|
+
}
|
|
86
|
+
/** Format the model-facing result text. */
|
|
87
|
+
export function formatDomainValue(args, value) {
|
|
88
|
+
const query = isRecord(args) && typeof args.query === 'string' ? args.query : 'the query';
|
|
89
|
+
const head = value.sources.length === 0
|
|
90
|
+
? `No dedicated source returned a result for ${JSON.stringify(query)}.`
|
|
91
|
+
: `Dedicated search for ${JSON.stringify(query)}: ${value.sources.length} result(s)${value.truncated ? ' (truncated)' : ''}.`;
|
|
92
|
+
const lines = value.sources.map((source, index) => {
|
|
93
|
+
const title = source.title !== undefined && source.title.length > 0 ? source.title : source.url;
|
|
94
|
+
const body = [`${index + 1}. ${title}`, ` ${source.url}`];
|
|
95
|
+
if (source.snippet !== undefined && source.snippet.length > 0)
|
|
96
|
+
body.push(` ${source.snippet}`);
|
|
97
|
+
if (source.publishedAt !== undefined && source.publishedAt.length > 0)
|
|
98
|
+
body.push(` ${source.publishedAt}`);
|
|
99
|
+
return body.join('\n');
|
|
100
|
+
});
|
|
101
|
+
if (value.failures.length > 0) {
|
|
102
|
+
lines.push('', 'Unavailable sources:', ...value.failures.map((failure) => `- ${failure}`));
|
|
103
|
+
}
|
|
104
|
+
return [head, '', ...lines].join('\n').trimEnd();
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Build the tool definition the host registry accepts.
|
|
108
|
+
* @param name - `domain_search`, or the fallback when that name is taken.
|
|
109
|
+
* @param readSettings - read the live config for every call (volatile fields).
|
|
110
|
+
* @param options - test seams and diagnostics.
|
|
111
|
+
* @returns a registry-ready definition.
|
|
112
|
+
*/
|
|
113
|
+
export function createDomainToolDefinition(name, readSettings, options = {}) {
|
|
114
|
+
const detectProxy = options.detectProxy ?? (() => resolveProxy(undefined));
|
|
115
|
+
return {
|
|
116
|
+
name,
|
|
117
|
+
description: 'Search dedicated, key-free sources instead of the open web: GitHub repositories, npm/PyPI/crates packages, arXiv papers, Wikipedia, Hacker News and Stack Exchange. Use it when the question is about a package, paper, repository or Q&A thread; use web_search for general pages.',
|
|
118
|
+
parameters: parameterSchema(),
|
|
119
|
+
output: {
|
|
120
|
+
schema: outputSchema(),
|
|
121
|
+
render: (args, value) => [{ type: 'text', text: formatDomainValue(args, value) }],
|
|
122
|
+
},
|
|
123
|
+
timeoutMs: 30_000,
|
|
124
|
+
isConcurrencySafe: () => true,
|
|
125
|
+
async execute(args, exec) {
|
|
126
|
+
const settings = readSettings();
|
|
127
|
+
const parsed = parseArgs(args, settings);
|
|
128
|
+
const explicit = typeof settings.proxy === 'string' ? settings.proxy.trim() : '';
|
|
129
|
+
const proxy = explicit.length > 0 ? explicit : await detectProxy();
|
|
130
|
+
const outcome = await searchVerticals(parsed.query, {
|
|
131
|
+
ids: parsed.ids,
|
|
132
|
+
limit: parsed.limit,
|
|
133
|
+
timeoutMs: settings.timeoutMs,
|
|
134
|
+
...(proxy === undefined || proxy.length === 0 ? {} : { proxy }),
|
|
135
|
+
...(exec.signal === undefined ? {} : { signal: exec.signal }),
|
|
136
|
+
...(options.fetch === undefined ? {} : { fetch: options.fetch }),
|
|
137
|
+
});
|
|
138
|
+
return {
|
|
139
|
+
sources: outcome.sources,
|
|
140
|
+
failures: outcome.failures,
|
|
141
|
+
truncated: outcome.truncated,
|
|
142
|
+
};
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Pick the tool name this deployment can accept.
|
|
148
|
+
*
|
|
149
|
+
* A plugin that already owns `domain_search` must not make this one fail to
|
|
150
|
+
* register, so the fallback name is used instead. A registry without `get` is
|
|
151
|
+
* assumed free.
|
|
152
|
+
* @param tools - the live registry.
|
|
153
|
+
* @returns the preferred or fallback name.
|
|
154
|
+
*/
|
|
155
|
+
export function pickDomainToolName(tools) {
|
|
156
|
+
if (typeof tools.get !== 'function')
|
|
157
|
+
return DOMAIN_TOOL_NAME;
|
|
158
|
+
try {
|
|
159
|
+
return tools.get(DOMAIN_TOOL_NAME) === undefined ? DOMAIN_TOOL_NAME : DOMAIN_TOOL_FALLBACK_NAME;
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
return DOMAIN_TOOL_NAME;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/** Report a registration failure without throwing out of the plugin. */
|
|
166
|
+
function report(options, error) {
|
|
167
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
168
|
+
try {
|
|
169
|
+
options.warn?.(`domain_search registration skipped: ${message}`);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
// A diagnostic callback must never break plugin loading.
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Register the tool when a registry is available.
|
|
177
|
+
*
|
|
178
|
+
* Every failure path is contained: a missing service is the caller's job to
|
|
179
|
+
* check, a duplicate name falls back, and any registry rejection is reported
|
|
180
|
+
* and swallowed so `web_search` keeps working.
|
|
181
|
+
* @param tools - the live registry.
|
|
182
|
+
* @param readSettings - live config reader.
|
|
183
|
+
* @param options - test seams and diagnostics.
|
|
184
|
+
* @returns the registered name, or `undefined` when registration was skipped.
|
|
185
|
+
*/
|
|
186
|
+
export function registerDomainTool(tools, readSettings, options = {}) {
|
|
187
|
+
const name = pickDomainToolName(tools);
|
|
188
|
+
try {
|
|
189
|
+
tools.register(createDomainToolDefinition(name, readSettings, options));
|
|
190
|
+
return name;
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
report(options, error);
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
}
|
package/lib/engines.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { SearchError } from "./errors.js";
|
|
2
|
+
import { filterByTimeRange, preferWindow } from "./freshness.js";
|
|
2
3
|
import { resolveProxy } from "./proxy.js";
|
|
3
4
|
import { parseAnySearch, parseBing, parseDdgHtml, parseDdgLite, parseSo } from "./parsers.js";
|
|
4
5
|
import { mergeRegions } from "./rank.js";
|
|
@@ -39,7 +40,7 @@ export const HOSTS = [
|
|
|
39
40
|
tier: 'primary',
|
|
40
41
|
region: 'global',
|
|
41
42
|
viaProxy: true,
|
|
42
|
-
build: (query) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
|
|
43
|
+
build: (query, timeRange) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}${ddgWindow(timeRange)}`,
|
|
43
44
|
parse: parseDdgHtml,
|
|
44
45
|
},
|
|
45
46
|
{
|
|
@@ -51,6 +52,7 @@ export const HOSTS = [
|
|
|
51
52
|
tier: 'fallback',
|
|
52
53
|
region: 'global',
|
|
53
54
|
viaProxy: true,
|
|
55
|
+
// No `filters` parameter: see the measurement note before `HOSTS`.
|
|
54
56
|
build: (query) => `https://www.bing.com/search?q=${encodeURIComponent(query)}&setmkt=en-US&setlang=en&cc=US`,
|
|
55
57
|
parse: parseBing,
|
|
56
58
|
},
|
|
@@ -58,6 +60,7 @@ export const HOSTS = [
|
|
|
58
60
|
id: 'cn.bing',
|
|
59
61
|
tier: 'primary',
|
|
60
62
|
region: 'cn',
|
|
63
|
+
// No `filters` parameter: see the measurement note before `HOSTS`.
|
|
61
64
|
build: (query) => `https://cn.bing.com/search?q=${encodeURIComponent(query)}`,
|
|
62
65
|
parse: parseBing,
|
|
63
66
|
},
|
|
@@ -73,12 +76,48 @@ export const HOSTS = [
|
|
|
73
76
|
tier: 'fallback',
|
|
74
77
|
region: 'global',
|
|
75
78
|
viaProxy: true,
|
|
76
|
-
build: (query) => `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`,
|
|
79
|
+
build: (query, timeRange) => `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${ddgWindow(timeRange)}`,
|
|
77
80
|
parse: parseDdgLite,
|
|
78
81
|
},
|
|
79
82
|
];
|
|
80
83
|
/** Scripts that make a query CJK rather than ASCII. */
|
|
81
84
|
const CJK = /[\u3000-\u303f\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uff00-\uffef]/u;
|
|
85
|
+
/**
|
|
86
|
+
* DuckDuckGo's own date filter, appended to a result-page URL.
|
|
87
|
+
*
|
|
88
|
+
* Measured live on both endpoints: adding `df=d`, `df=w` or `df=m` changed the
|
|
89
|
+
* returned result set, so this is a real pushdown rather than a decorative
|
|
90
|
+
* parameter. `off` (and the absent value) adds nothing.
|
|
91
|
+
*
|
|
92
|
+
* @param timeRange - active window, if any.
|
|
93
|
+
* @returns `&df=x` for a supported window, otherwise an empty string.
|
|
94
|
+
*/
|
|
95
|
+
function ddgWindow(timeRange) {
|
|
96
|
+
switch (timeRange) {
|
|
97
|
+
case 'day':
|
|
98
|
+
return '&df=d';
|
|
99
|
+
case 'week':
|
|
100
|
+
return '&df=w';
|
|
101
|
+
case 'month':
|
|
102
|
+
return '&df=m';
|
|
103
|
+
case 'year':
|
|
104
|
+
return '&df=y';
|
|
105
|
+
case 'off':
|
|
106
|
+
case undefined:
|
|
107
|
+
default:
|
|
108
|
+
return '';
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/*
|
|
112
|
+
* Why Bing has no time parameter.
|
|
113
|
+
*
|
|
114
|
+
* Measured on `cn.bing.com` and `www.bing.com` (which redirects to `cn.bing`):
|
|
115
|
+
* `filters=ex1:"ez1|ez2|ez3|ez5_…"` is accepted — the response marks the
|
|
116
|
+
* matching entry as selected — but the ten returned URLs were byte-for-byte
|
|
117
|
+
* identical to the unfiltered run for both a general and a news query. Bing's
|
|
118
|
+
* filter is therefore not written into the request; the engine layer filters
|
|
119
|
+
* Bing's parsed dates locally instead.
|
|
120
|
+
*/
|
|
82
121
|
/** Soft deadline per wait policy, in milliseconds. */
|
|
83
122
|
const SOFT_DEADLINE = {
|
|
84
123
|
auto: (ascii) => (ascii ? 6000 : 3500),
|
|
@@ -132,7 +171,7 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
|
|
|
132
171
|
releasedEarly = early;
|
|
133
172
|
released();
|
|
134
173
|
};
|
|
135
|
-
const running = hosts.map((host, index) => searchHost(host, query, controller.signal, timeoutMs, proxy).then((outcome) => {
|
|
174
|
+
const running = hosts.map((host, index) => searchHost(host, query, controller.signal, timeoutMs, proxy, limits.timeRange).then((outcome) => {
|
|
136
175
|
if (outcome.ok) {
|
|
137
176
|
outcomes[index] = { host, sources: outcome.sources };
|
|
138
177
|
health?.noteSuccess(host.id);
|
|
@@ -227,6 +266,7 @@ export async function search(query, signal, options) {
|
|
|
227
266
|
const limit = Math.max(1, Math.min(options.limit, 20));
|
|
228
267
|
const timeoutMs = options.timeoutMs ?? 15000;
|
|
229
268
|
const wait = options.wait ?? 'auto';
|
|
269
|
+
const timeRange = options.timeRange ?? 'off';
|
|
230
270
|
const proxy = options.useProxy === false ? undefined : await resolveProxy(options.proxy);
|
|
231
271
|
const allow = options.hostIds !== undefined && options.hostIds.length > 0 ? new Set(options.hostIds) : undefined;
|
|
232
272
|
const hosts = HOSTS.filter((host) => allow === undefined || allow.has(host.id))
|
|
@@ -251,24 +291,42 @@ export async function search(query, signal, options) {
|
|
|
251
291
|
const wanted = usable.filter((host) => leadsQuery(host, lead)).length;
|
|
252
292
|
const softMs = options.softMs ?? SOFT_DEADLINE[wait](ascii);
|
|
253
293
|
const needed = wait === 'all' ? usable.length : wait === 'fast' ? 1 : wanted > 0 ? wanted : Math.min(2, usable.length);
|
|
254
|
-
const limits = { limit, softMs, needed, lead: wait === 'all' ? 'all' : lead };
|
|
255
|
-
|
|
294
|
+
const limits = { limit, softMs, needed, lead: wait === 'all' ? 'all' : lead, timeRange };
|
|
295
|
+
// Ranking does not drop rows; the engine layer does. `timeRange` is passed
|
|
296
|
+
// into the rank options only so undated rows are scored lower while a window
|
|
297
|
+
// is active. Keeping it out of the configured `rank` object preserves the
|
|
298
|
+
// documented shape of `ResolvedConfig.rank`.
|
|
299
|
+
const rank = {
|
|
300
|
+
...(options.rank ?? {}),
|
|
301
|
+
...(timeRange === 'off' ? {} : { timeRange }),
|
|
302
|
+
};
|
|
303
|
+
/**
|
|
304
|
+
* Drop results whose parsed date is outside the window.
|
|
305
|
+
*
|
|
306
|
+
* Undated results survive on purpose: most documentation pages publish no
|
|
307
|
+
* date, and discarding them would empty the list for a library query. They
|
|
308
|
+
* are down-weighted during fusion instead, so a dated in-window page wins a
|
|
309
|
+
* close comparison without erasing the undated long tail.
|
|
310
|
+
*/
|
|
311
|
+
const applyWindow = (answers) => timeRange === 'off'
|
|
312
|
+
? answers
|
|
313
|
+
: answers.map((answer) => ({ host: answer.host, sources: filterByTimeRange(answer.sources, timeRange) }));
|
|
256
314
|
// The key names the engines the run *would* use, not the ones that survived
|
|
257
315
|
// the cooldown filter: an engine entering or leaving a cooldown must not
|
|
258
316
|
// invalidate an entry that is still fresh, or every throttle would turn the
|
|
259
317
|
// next identical query into a cache miss.
|
|
260
|
-
const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id) });
|
|
318
|
+
const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
|
|
261
319
|
const cached = key === undefined ? undefined : options.cache?.get(key);
|
|
262
320
|
if (cached !== undefined)
|
|
263
321
|
return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
|
|
264
322
|
const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
|
|
265
323
|
const primary = ordered.filter((host) => host.tier === 'primary');
|
|
266
324
|
const fallback = ordered.filter((host) => host.tier === 'fallback');
|
|
267
|
-
let answers = await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health);
|
|
268
|
-
let sources = answers.length === 0 ? [] : mergeRegions(answers, lead, limit, rank);
|
|
325
|
+
let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
326
|
+
let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
269
327
|
if (sources.length === 0 && fallback.length > 0) {
|
|
270
|
-
answers = await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health);
|
|
271
|
-
sources = answers.length === 0 ? [] : mergeRegions(answers, lead, limit, rank);
|
|
328
|
+
answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
329
|
+
sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
272
330
|
}
|
|
273
331
|
if (signal?.aborted === true)
|
|
274
332
|
throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
|
package/lib/fetch.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { FetchFormat } from './markdown.ts';
|
|
1
2
|
/** One retrieval outcome, mirroring the Host's `WebFetchResult`. */
|
|
2
3
|
export interface FetchOutcome {
|
|
3
4
|
readonly url: string;
|
|
@@ -30,6 +31,12 @@ export interface FetchProviderOptions {
|
|
|
30
31
|
anysearchApiKey?: string;
|
|
31
32
|
/** Bytes kept from a response; the Host applies its own cap too. */
|
|
32
33
|
maxBytes?: number;
|
|
34
|
+
/** Output style for an HTML body; `markdown` keeps structure. */
|
|
35
|
+
/** Renderer, or a reader for it so a Settings edit applies without a remount. */
|
|
36
|
+
format?: FetchFormat | (() => FetchFormat);
|
|
37
|
+
/** Character budget for the returned text. */
|
|
38
|
+
/** Character budget, or a reader for it. */
|
|
39
|
+
maxLength?: number | (() => number);
|
|
33
40
|
/** User agent sent on every rung. */
|
|
34
41
|
userAgent?: string;
|
|
35
42
|
}
|
package/lib/fetch.js
CHANGED
|
@@ -24,6 +24,7 @@
|
|
|
24
24
|
* @module @fangchenghao/dsh-web-search/fetch
|
|
25
25
|
*/
|
|
26
26
|
import { SearchError } from "./errors.js";
|
|
27
|
+
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
|
|
27
28
|
import { proxyGet, resolveProxy } from "./proxy.js";
|
|
28
29
|
/** Default user agent; kept in step with the search side. */
|
|
29
30
|
const USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36';
|
|
@@ -126,6 +127,38 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
126
127
|
const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
127
128
|
const userAgent = options.userAgent ?? USER_AGENT;
|
|
128
129
|
const apiKey = (options.anysearchApiKey ?? '').trim();
|
|
130
|
+
/**
|
|
131
|
+
* Resolve the two volatile Settings fields per request. Reading them here
|
|
132
|
+
* rather than at construction is what lets a saved edit apply without a
|
|
133
|
+
* remount; caching them made the Settings page lie.
|
|
134
|
+
*/
|
|
135
|
+
const liveFormat = () => typeof options.format === 'function' ? options.format() : (options.format ?? 'markdown');
|
|
136
|
+
const liveMaxLength = () => {
|
|
137
|
+
const raw = typeof options.maxLength === 'function' ? options.maxLength() : options.maxLength;
|
|
138
|
+
return raw ?? DEFAULT_FETCH_MAX_LENGTH;
|
|
139
|
+
};
|
|
140
|
+
// Every rung funnels through here: an HTML body becomes Markdown (or plain
|
|
141
|
+
// text), a text body keeps its content, and both obey the character budget.
|
|
142
|
+
// The Host's own renderer then sees `kind: 'text'`, so it does not convert the
|
|
143
|
+
// same page a second time.
|
|
144
|
+
const finalize = (outcome) => {
|
|
145
|
+
const format = liveFormat();
|
|
146
|
+
const maxLength = liveMaxLength();
|
|
147
|
+
if (outcome.body.kind === 'html') {
|
|
148
|
+
const rendered = renderFetchText(outcome.body.content, { format, maxLength, baseUrl: outcome.url });
|
|
149
|
+
return {
|
|
150
|
+
...outcome,
|
|
151
|
+
body: { kind: 'text', content: rendered.content },
|
|
152
|
+
truncated: outcome.truncated || rendered.truncated,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
const limited = truncateFetchText(outcome.body.content, maxLength);
|
|
156
|
+
return {
|
|
157
|
+
...outcome,
|
|
158
|
+
body: { kind: 'text', content: limited.content },
|
|
159
|
+
truncated: outcome.truncated || limited.truncated,
|
|
160
|
+
};
|
|
161
|
+
};
|
|
129
162
|
return {
|
|
130
163
|
id: options.id ?? 'http-proxy',
|
|
131
164
|
available: () => true,
|
|
@@ -141,7 +174,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
141
174
|
// shorter deadline than the tunnel: spending the whole budget on a request
|
|
142
175
|
// that cannot work would leave the rung that can work with nothing.
|
|
143
176
|
try {
|
|
144
|
-
return await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes);
|
|
177
|
+
return finalize(await deps.direct(target, Math.min(timeoutMs, DIRECT_DEADLINE_MS), signal, userAgent, maxBytes));
|
|
145
178
|
}
|
|
146
179
|
catch (error) {
|
|
147
180
|
if (cancelled())
|
|
@@ -159,7 +192,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
159
192
|
}
|
|
160
193
|
if (proxy !== undefined && proxy.length > 0) {
|
|
161
194
|
try {
|
|
162
|
-
return await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes);
|
|
195
|
+
return finalize(await deps.tunnel(target, proxy, timeoutMs, signal, userAgent, maxBytes));
|
|
163
196
|
}
|
|
164
197
|
catch (error) {
|
|
165
198
|
if (cancelled())
|
|
@@ -169,7 +202,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
169
202
|
}
|
|
170
203
|
if (apiKey.length > 0) {
|
|
171
204
|
try {
|
|
172
|
-
return await deps.extract(target, apiKey, timeoutMs, signal);
|
|
205
|
+
return finalize(await deps.extract(target, apiKey, timeoutMs, signal));
|
|
173
206
|
}
|
|
174
207
|
catch (error) {
|
|
175
208
|
if (cancelled())
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Publication-date parsing and rolling time-window filtering.
|
|
3
|
+
*
|
|
4
|
+
* Search engines disagree about how they spell a date: Bing prints
|
|
5
|
+
* `2025年9月25日` or `2 天之前` at the head of a snippet, 360 prints
|
|
6
|
+
* `发贴时间:2026年1月22日`, DuckDuckGo usually prints nothing, and AnySearch
|
|
7
|
+
* appends `date: 22 hours ago` to its `content` field. All of those become one
|
|
8
|
+
* canonical value, an ISO 8601 UTC string, so ranking and filtering never have
|
|
9
|
+
* to care which engine answered.
|
|
10
|
+
*
|
|
11
|
+
* Two rules keep this safe:
|
|
12
|
+
*
|
|
13
|
+
* - Nothing depends on the machine's timezone. An absolute date is built with
|
|
14
|
+
* `Date.UTC` and anchored at noon UTC, so `2024-05-01` is the same instant in
|
|
15
|
+
* Shanghai, Berlin and Los Angeles and cannot drift into the previous day.
|
|
16
|
+
* - A missing or unparsable date never discards a result. Filtering only drops
|
|
17
|
+
* a result whose date is *known* to be outside the window; unknown dates are
|
|
18
|
+
* kept and merely rank lower (see `rank.ts`), because most documentation pages
|
|
19
|
+
* carry no date at all.
|
|
20
|
+
*
|
|
21
|
+
* @module @fangchenghao/dsh-web-search/freshness
|
|
22
|
+
*/
|
|
23
|
+
import type { Source, TimeRange } from './types.ts';
|
|
24
|
+
/** Milliseconds in one day, shared with the ranking module. */
|
|
25
|
+
export declare const MS_PER_DAY = 86400000;
|
|
26
|
+
/** Rolling window length per range. `off` is handled separately. */
|
|
27
|
+
export declare const TIME_RANGE_MS: Readonly<Record<Exclude<TimeRange, 'off'>, number>>;
|
|
28
|
+
/** Every accepted `timeRange` value, in settings order. */
|
|
29
|
+
export declare const TIME_RANGES: readonly TimeRange[];
|
|
30
|
+
/**
|
|
31
|
+
* Extract a publication date from one fragment of result text.
|
|
32
|
+
*
|
|
33
|
+
* The search order is deliberate: an explicit `date:` marker wins (AnySearch
|
|
34
|
+
* puts the engine's stamp there), then a relative phrase, then an absolute
|
|
35
|
+
* date. `今天` inside a sentence such as `但是我们今天讲的逻辑复制` is not a
|
|
36
|
+
* publication stamp, so relative words only match as whole tokens; a bare date
|
|
37
|
+
* such as `2024-05-01` is unambiguous enough to match anywhere.
|
|
38
|
+
*
|
|
39
|
+
* @param fragment - summary, stamp or result text.
|
|
40
|
+
* @param now - evaluation instant, injected so tests are deterministic.
|
|
41
|
+
* @returns ISO 8601 UTC, or `undefined` when the fragment carries no date.
|
|
42
|
+
*/
|
|
43
|
+
export declare function parsePublishedAt(fragment: string, now?: number): string | undefined;
|
|
44
|
+
/**
|
|
45
|
+
* Parse the first fragment that carries a usable date.
|
|
46
|
+
*
|
|
47
|
+
* Parsers pass the most authoritative fragment first (a dedicated timestamp
|
|
48
|
+
* element, then the snippet), so a date mentioned in prose cannot override the
|
|
49
|
+
* stamp the engine printed.
|
|
50
|
+
*
|
|
51
|
+
* @param fragments - candidate texts, most authoritative first.
|
|
52
|
+
* @param now - evaluation instant, injected for tests.
|
|
53
|
+
* @returns the first parsed ISO 8601 UTC value, or `undefined`.
|
|
54
|
+
*/
|
|
55
|
+
export declare function extractPublishedAt(fragments: readonly (string | undefined)[], now?: number): string | undefined;
|
|
56
|
+
/**
|
|
57
|
+
* Read a publication date as epoch milliseconds.
|
|
58
|
+
* @param publishedAt - canonical ISO string, or `undefined`.
|
|
59
|
+
* @returns epoch milliseconds, or `undefined` when absent or unparsable.
|
|
60
|
+
*/
|
|
61
|
+
export declare function publishedTimestamp(publishedAt: string | undefined): number | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* Whether a result belongs to the rolling window.
|
|
64
|
+
*
|
|
65
|
+
* `off` accepts everything. A missing or unparsable date is accepted on
|
|
66
|
+
* purpose: filtering exists to remove results known to be stale, not to punish
|
|
67
|
+
* pages that publish no metadata. A date in the future is accepted too — it is
|
|
68
|
+
* still inside the window, and ranking clamps its age to zero.
|
|
69
|
+
*
|
|
70
|
+
* @param publishedAt - canonical ISO string, or `undefined`.
|
|
71
|
+
* @param range - active window.
|
|
72
|
+
* @param now - evaluation instant.
|
|
73
|
+
* @returns `true` when the result may stay.
|
|
74
|
+
*/
|
|
75
|
+
export declare function isWithinTimeRange(publishedAt: string | undefined, range: TimeRange, now?: number): boolean;
|
|
76
|
+
/**
|
|
77
|
+
* Keep only results inside the window.
|
|
78
|
+
*
|
|
79
|
+
* A result whose date could not be parsed is not evidence of an old page, so it
|
|
80
|
+
* is held back rather than dropped: when at least one dated result falls inside
|
|
81
|
+
* the window, only those are returned; when none does, the undated results are
|
|
82
|
+
* returned as a fallback so the caller is not left empty-handed. That trades a
|
|
83
|
+
* little precision for never answering "last 24 hours" with nothing at all.
|
|
84
|
+
* @param sources - parsed sources, in engine order.
|
|
85
|
+
* @param range - active window.
|
|
86
|
+
* @param now - evaluation instant.
|
|
87
|
+
* @returns a new list; the input array and its objects are untouched.
|
|
88
|
+
*/
|
|
89
|
+
export declare function filterByTimeRange(sources: readonly Source[], range: TimeRange, now?: number): Source[];
|
|
90
|
+
/**
|
|
91
|
+
* Apply a window to an already merged list.
|
|
92
|
+
*
|
|
93
|
+
* Per-engine filtering can only see one engine's dates, so an engine that dates
|
|
94
|
+
* nothing keeps all of its rows; this pass runs after the merge, where a dated
|
|
95
|
+
* row from any engine can speak for the whole list. When nothing is dated inside
|
|
96
|
+
* the window the merged list is returned unchanged rather than emptied.
|
|
97
|
+
* @param sources - merged sources.
|
|
98
|
+
* @param range - active window.
|
|
99
|
+
* @param now - evaluation instant.
|
|
100
|
+
* @returns the rows to keep.
|
|
101
|
+
*/
|
|
102
|
+
export declare function preferWindow(sources: readonly Source[], range: TimeRange, now?: number): Source[];
|