@fffangchenghao/dsh-web-search 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/README.md +131 -23
- package/README.zh-CN.md +131 -23
- package/client.js +39 -7
- package/lib/cache.d.ts +8 -4
- package/lib/cache.js +7 -4
- package/lib/config.d.ts +17 -1
- package/lib/config.js +26 -2
- package/lib/domain-tool.d.ts +119 -0
- package/lib/domain-tool.js +196 -0
- package/lib/engines.js +68 -10
- package/lib/freshness.d.ts +102 -0
- package/lib/freshness.js +359 -0
- package/lib/index.d.ts +24 -2
- package/lib/index.js +58 -1
- package/lib/parsers.js +59 -5
- package/lib/rank.d.ts +14 -1
- package/lib/rank.js +18 -0
- package/lib/transport.d.ts +4 -2
- package/lib/transport.js +4 -2
- package/lib/types.d.ts +15 -2
- package/lib/verticals.d.ts +164 -0
- package/lib/verticals.js +539 -0
- package/package.json +4 -4
package/lib/config.js
CHANGED
|
@@ -20,6 +20,7 @@
|
|
|
20
20
|
* @module @fangchenghao/dsh-web-search/config
|
|
21
21
|
*/
|
|
22
22
|
import z from '@deepseek-ai/schemastery';
|
|
23
|
+
import { VERTICAL_IDS, isVerticalId } from "./verticals.js";
|
|
23
24
|
/** Schema of the plugin's configuration section. */
|
|
24
25
|
export const Config = z.object({
|
|
25
26
|
hosts: z
|
|
@@ -46,6 +47,11 @@ export const Config = z.object({
|
|
|
46
47
|
authorityBoost: z.number().min(0).max(10).default(0.25).description('上面那份网站名单能让结果往前排多少(0 到 10,0 = 不加强)。觉得好网站排得不够前就调大;不确定保持 0.25。').volatile(),
|
|
47
48
|
freshnessBoost: z.number().min(0).max(10).default(0.2).description('新发布的网页能额外往前排多少(0 到 10,0 = 不看新旧)。常查新闻可调到 0.5;不确定保持 0.2。').volatile(),
|
|
48
49
|
freshnessDays: z.natural().min(1).default(30).description('多新的网页才算新(单位:天)。查新闻可调小到 7,查资料可调大到 365;不确定保持 30。').volatile(),
|
|
50
|
+
timeRange: z
|
|
51
|
+
.union(['off', 'day', 'week', 'month', 'year'])
|
|
52
|
+
.default('off')
|
|
53
|
+
.description('只保留多久以内发布的结果:off 不限时间、day 最近 24 小时、week 最近 7 天、month 最近 30 天、year 最近 1 年。开启后只返回窗口内发布的结果;若一条带日期的都没落进窗口,会退回无日期的结果避免空手;不确定保持 off。')
|
|
54
|
+
.volatile(),
|
|
49
55
|
cacheTtlMs: z.natural().default(600000).description('同一问题多久内复用上次结果(缓存,即存一份备用),毫秒。想早点看到新内容就调小;保持 600000(10 分钟)即可。').volatile(),
|
|
50
56
|
cacheEntries: z.natural().default(100).description('最多存多少条旧结果备用(缓存,即存起来省得重复查)。存满后先丢最久没用的;一般保持 100 即可。').volatile(),
|
|
51
57
|
anysearchApiKey: z
|
|
@@ -58,6 +64,13 @@ export const Config = z.object({
|
|
|
58
64
|
.description('只搜某一类专业内容(如 finance 财经、academic 学术、code 代码)。要专查某类才填;日常搜索留空。').volatile(),
|
|
59
65
|
baseCooldownMs: z.natural().default(300000).description('搜索引擎说“你查得太频繁了”后,先歇多久(毫秒)。歇太久会少一个引擎,太短易被再拒;保持 300000(5 分钟)即可。').volatile(),
|
|
60
66
|
maxCooldownMs: z.natural().default(1800000).description('它反复被拒时,最长让它歇多久(毫秒),也就是等待一步步加长的封顶值。一般不用改;保持 1800000(30 分钟)即可。').volatile(),
|
|
67
|
+
domainSources: z
|
|
68
|
+
.union([z.array(z.string()), z.string()])
|
|
69
|
+
.default([...VERTICAL_IDS])
|
|
70
|
+
.description('默认查哪些专用源,逗号分隔;留空表示 github、npm 等全部八个源。')
|
|
71
|
+
.volatile(),
|
|
72
|
+
domainLimit: z.natural().min(1).max(20).default(5).description('domain_search 一次最多返回几条(1-20);默认 5 条,想多看可调到 10。').volatile(),
|
|
73
|
+
domainTimeoutMs: z.natural().min(500).default(8000).description('每个专用源最多等多久(毫秒);网络慢可加到 15000,平时 8000 即可。').volatile(),
|
|
61
74
|
});
|
|
62
75
|
/**
|
|
63
76
|
* Read one field, whichever shape it arrived in.
|
|
@@ -93,9 +106,16 @@ export function resolveConfig(config, callLimit) {
|
|
|
93
106
|
const validated = Config(plain);
|
|
94
107
|
const keys = Object.keys(validated);
|
|
95
108
|
const settings = Object.fromEntries(keys.map((key) => [key, read(supplied[key], read(validated[key], undefined))]));
|
|
96
|
-
// A volatile reference hands out a frozen snapshot, so the
|
|
97
|
-
//
|
|
109
|
+
// A volatile reference hands out a frozen snapshot, so the list fields are
|
|
110
|
+
// copied before they leave: a caller may sort or splice what it gets.
|
|
98
111
|
const hosts = [...settings.hosts].filter((id) => id.trim().length > 0);
|
|
112
|
+
// The field accepts an array or the comma separated form the Settings card
|
|
113
|
+
// writes; both normalise to the ids the tool understands.
|
|
114
|
+
const rawDomainSources = settings.domainSources;
|
|
115
|
+
const domainList = Array.isArray(rawDomainSources)
|
|
116
|
+
? rawDomainSources
|
|
117
|
+
: String(rawDomainSources).split(/[\s,]+/);
|
|
118
|
+
const domainSources = domainList.map((id) => id.trim()).filter(isVerticalId);
|
|
99
119
|
const proxy = settings.proxy.trim();
|
|
100
120
|
const limit = Math.min(callLimit ?? settings.limit, settings.limit);
|
|
101
121
|
return {
|
|
@@ -104,8 +124,12 @@ export function resolveConfig(config, callLimit) {
|
|
|
104
124
|
proxy: proxy.length > 0 ? proxy : undefined,
|
|
105
125
|
limit,
|
|
106
126
|
wait: settings.wait,
|
|
127
|
+
timeRange: settings.timeRange,
|
|
107
128
|
anysearchApiKey: settings.anysearchApiKey.trim(),
|
|
108
129
|
anysearchDomain: settings.anysearchDomain.trim(),
|
|
130
|
+
domainSourceIds: domainSources.length > 0 ? domainSources : [...VERTICAL_IDS],
|
|
131
|
+
domainLimit: settings.domainLimit,
|
|
132
|
+
domainTimeoutMs: settings.domainTimeoutMs,
|
|
109
133
|
rank: {
|
|
110
134
|
rrfConstant: settings.rrfConstant,
|
|
111
135
|
...(settings.authorityDomains.length > 0 ? { authorityDomains: [...settings.authorityDomains] } : {}),
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-facing `domain_search` tool.
|
|
3
|
+
*
|
|
4
|
+
* The host's tool registry is reached through `ctx.get('tools')`, deliberately
|
|
5
|
+
* **not** through `inject`: an inject entry is a hard dependency, so a
|
|
6
|
+
* deployment that loads this plugin without the tool service would refuse to
|
|
7
|
+
* mount the plugin at all and `web_search` would disappear with it. Reading the
|
|
8
|
+
* service at mount time and declining to register when it is absent keeps the
|
|
9
|
+
* plugin fail-safe, and matches how the shipped tool packages read optional
|
|
10
|
+
* services (`ctx.get(...)` without `inject`).
|
|
11
|
+
*
|
|
12
|
+
* The definition is built from the registry's raw JSON-Schema dialect rather
|
|
13
|
+
* than `defineTool`, because this package must not take a runtime dependency on
|
|
14
|
+
* `@deepseek-ai/dsh-tools` (the host owns that package and it is not resolvable
|
|
15
|
+
* from a plugin's own `node_modules`). The shape below is the one
|
|
16
|
+
* `ToolRuntime.register()` accepts: `parameters` is an object-rooted JSON
|
|
17
|
+
* Schema with a `required` **array**, and `output` carries `{ schema, render }`.
|
|
18
|
+
*
|
|
19
|
+
* @module @fangchenghao/dsh-web-search/domain-tool
|
|
20
|
+
*/
|
|
21
|
+
import type { Source } from './types.ts';
|
|
22
|
+
import type { FetchLike } from './verticals.ts';
|
|
23
|
+
/** The name this tool prefers. */
|
|
24
|
+
export declare const DOMAIN_TOOL_NAME = "domain_search";
|
|
25
|
+
/** The name used when another plugin already registered `domain_search`. */
|
|
26
|
+
export declare const DOMAIN_TOOL_FALLBACK_NAME = "web_domain_search";
|
|
27
|
+
/** The subset of JSON Schema the host registry enforces. */
|
|
28
|
+
export interface ToolJsonSchema {
|
|
29
|
+
type?: 'object' | 'array' | 'string' | 'integer' | 'number' | 'boolean' | 'null';
|
|
30
|
+
description?: string;
|
|
31
|
+
properties?: Record<string, ToolJsonSchema>;
|
|
32
|
+
required?: string[];
|
|
33
|
+
additionalProperties?: boolean;
|
|
34
|
+
items?: ToolJsonSchema;
|
|
35
|
+
enum?: string[];
|
|
36
|
+
}
|
|
37
|
+
/** One model-facing content block, as `output.render` returns. */
|
|
38
|
+
export interface ToolContent {
|
|
39
|
+
type: 'text';
|
|
40
|
+
text: string;
|
|
41
|
+
}
|
|
42
|
+
/** A tool definition in the registry's native shape. */
|
|
43
|
+
export interface DomainToolDefinition {
|
|
44
|
+
name: string;
|
|
45
|
+
description: string;
|
|
46
|
+
parameters: ToolJsonSchema;
|
|
47
|
+
output: {
|
|
48
|
+
schema: ToolJsonSchema;
|
|
49
|
+
render: (args: unknown, value: DomainToolValue) => ToolContent[];
|
|
50
|
+
};
|
|
51
|
+
timeoutMs?: number;
|
|
52
|
+
isConcurrencySafe?: (args: unknown) => boolean;
|
|
53
|
+
execute: (args: unknown, exec: {
|
|
54
|
+
signal?: AbortSignal;
|
|
55
|
+
}) => Promise<DomainToolValue>;
|
|
56
|
+
}
|
|
57
|
+
/** The slice of the tool registry this module needs. */
|
|
58
|
+
export interface ToolRegistryLike {
|
|
59
|
+
register(definition: DomainToolDefinition): unknown;
|
|
60
|
+
get?(name: string, scope?: unknown): unknown;
|
|
61
|
+
}
|
|
62
|
+
/** What the tool returns to the model. */
|
|
63
|
+
export interface DomainToolValue {
|
|
64
|
+
sources: Source[];
|
|
65
|
+
failures: string[];
|
|
66
|
+
truncated: boolean;
|
|
67
|
+
}
|
|
68
|
+
/** Settings the tool reads live for every call. */
|
|
69
|
+
export interface DomainToolSettings {
|
|
70
|
+
/** Enabled source ids; empty means every source. */
|
|
71
|
+
sourceIds: readonly string[];
|
|
72
|
+
/** Default result budget per source. */
|
|
73
|
+
limit: number;
|
|
74
|
+
/** Per-source deadline in milliseconds. */
|
|
75
|
+
timeoutMs: number;
|
|
76
|
+
/** Explicit tunnel URL, or `undefined` to auto-detect. */
|
|
77
|
+
proxy?: string;
|
|
78
|
+
}
|
|
79
|
+
/** Construction seams, so the definition is testable without a network. */
|
|
80
|
+
export interface DomainToolOptions {
|
|
81
|
+
/** Replace the platform `fetch` (tests). */
|
|
82
|
+
fetch?: FetchLike;
|
|
83
|
+
/** Replace proxy detection (tests). */
|
|
84
|
+
detectProxy?: () => Promise<string | undefined>;
|
|
85
|
+
/** Registration diagnostics. */
|
|
86
|
+
warn?: (message: string) => void;
|
|
87
|
+
}
|
|
88
|
+
/** Format the model-facing result text. */
|
|
89
|
+
export declare function formatDomainValue(args: unknown, value: DomainToolValue): string;
|
|
90
|
+
/**
|
|
91
|
+
* Build the tool definition the host registry accepts.
|
|
92
|
+
* @param name - `domain_search`, or the fallback when that name is taken.
|
|
93
|
+
* @param readSettings - read the live config for every call (volatile fields).
|
|
94
|
+
* @param options - test seams and diagnostics.
|
|
95
|
+
* @returns a registry-ready definition.
|
|
96
|
+
*/
|
|
97
|
+
export declare function createDomainToolDefinition(name: string, readSettings: () => DomainToolSettings, options?: DomainToolOptions): DomainToolDefinition;
|
|
98
|
+
/**
|
|
99
|
+
* Pick the tool name this deployment can accept.
|
|
100
|
+
*
|
|
101
|
+
* A plugin that already owns `domain_search` must not make this one fail to
|
|
102
|
+
* register, so the fallback name is used instead. A registry without `get` is
|
|
103
|
+
* assumed free.
|
|
104
|
+
* @param tools - the live registry.
|
|
105
|
+
* @returns the preferred or fallback name.
|
|
106
|
+
*/
|
|
107
|
+
export declare function pickDomainToolName(tools: ToolRegistryLike): string;
|
|
108
|
+
/**
|
|
109
|
+
* Register the tool when a registry is available.
|
|
110
|
+
*
|
|
111
|
+
* Every failure path is contained: a missing service is the caller's job to
|
|
112
|
+
* check, a duplicate name falls back, and any registry rejection is reported
|
|
113
|
+
* and swallowed so `web_search` keeps working.
|
|
114
|
+
* @param tools - the live registry.
|
|
115
|
+
* @param readSettings - live config reader.
|
|
116
|
+
* @param options - test seams and diagnostics.
|
|
117
|
+
* @returns the registered name, or `undefined` when registration was skipped.
|
|
118
|
+
*/
|
|
119
|
+
export declare function registerDomainTool(tools: ToolRegistryLike, readSettings: () => DomainToolSettings, options?: DomainToolOptions): string | undefined;
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
import { resolveProxy } from "./proxy.js";
|
|
2
|
+
import { searchVerticals, isVerticalId, VERTICAL_IDS } from "./verticals.js";
|
|
3
|
+
/** The name this tool prefers. */
|
|
4
|
+
export const DOMAIN_TOOL_NAME = 'domain_search';
|
|
5
|
+
/** The name used when another plugin already registered `domain_search`. */
|
|
6
|
+
export const DOMAIN_TOOL_FALLBACK_NAME = 'web_domain_search';
|
|
7
|
+
/** True for a value usable as a record. */
|
|
8
|
+
function isRecord(value) {
|
|
9
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
10
|
+
}
|
|
11
|
+
/** Clamp a tool budget into the range the adapters serve. */
|
|
12
|
+
function clampToolLimit(value, fallback) {
|
|
13
|
+
const base = Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
14
|
+
return Math.max(1, Math.min(20, base));
|
|
15
|
+
}
|
|
16
|
+
/** The parameter schema, in the registry's raw JSON-Schema dialect. */
|
|
17
|
+
function parameterSchema() {
|
|
18
|
+
return {
|
|
19
|
+
type: 'object',
|
|
20
|
+
additionalProperties: false,
|
|
21
|
+
properties: {
|
|
22
|
+
query: {
|
|
23
|
+
type: 'string',
|
|
24
|
+
description: 'What to look for: a package name, paper topic, repository, error message, or question.',
|
|
25
|
+
},
|
|
26
|
+
sources: {
|
|
27
|
+
type: 'array',
|
|
28
|
+
items: { type: 'string' },
|
|
29
|
+
description: `Sources to query: ${VERTICAL_IDS.join(', ')}. Omit to use the deployment default.`,
|
|
30
|
+
},
|
|
31
|
+
limit: {
|
|
32
|
+
type: 'integer',
|
|
33
|
+
description: 'Results wanted per source, 1–20. Omit to use the deployment default.',
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
required: ['query'],
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/** The output schema, in the same dialect. */
|
|
40
|
+
function outputSchema() {
|
|
41
|
+
return {
|
|
42
|
+
type: 'object',
|
|
43
|
+
additionalProperties: false,
|
|
44
|
+
properties: {
|
|
45
|
+
sources: {
|
|
46
|
+
type: 'array',
|
|
47
|
+
items: {
|
|
48
|
+
type: 'object',
|
|
49
|
+
additionalProperties: false,
|
|
50
|
+
properties: {
|
|
51
|
+
url: { type: 'string' },
|
|
52
|
+
title: { type: 'string' },
|
|
53
|
+
snippet: { type: 'string' },
|
|
54
|
+
publishedAt: { type: 'string' },
|
|
55
|
+
},
|
|
56
|
+
required: ['url'],
|
|
57
|
+
},
|
|
58
|
+
},
|
|
59
|
+
failures: { type: 'array', items: { type: 'string' } },
|
|
60
|
+
truncated: { type: 'boolean' },
|
|
61
|
+
},
|
|
62
|
+
required: ['sources', 'failures', 'truncated'],
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Parse model arguments into a validated shape, throwing on nonsense. */
|
|
66
|
+
function parseArgs(args, settings) {
|
|
67
|
+
if (!isRecord(args))
|
|
68
|
+
throw new Error('domain_search requires arguments');
|
|
69
|
+
const query = typeof args.query === 'string' ? args.query.trim() : '';
|
|
70
|
+
if (query.length === 0)
|
|
71
|
+
throw new Error('domain_search requires a non-empty query');
|
|
72
|
+
const enabled = settings.sourceIds.filter(isVerticalId);
|
|
73
|
+
let ids = enabled.length > 0 ? enabled : [...VERTICAL_IDS];
|
|
74
|
+
if (Array.isArray(args.sources) && args.sources.length > 0) {
|
|
75
|
+
const asked = args.sources.filter(isVerticalId);
|
|
76
|
+
if (asked.length === 0) {
|
|
77
|
+
throw new Error(`domain_search: no known source in ${JSON.stringify(args.sources)}; known: ${VERTICAL_IDS.join(', ')}`);
|
|
78
|
+
}
|
|
79
|
+
ids = asked;
|
|
80
|
+
}
|
|
81
|
+
const limit = typeof args.limit === 'number'
|
|
82
|
+
? clampToolLimit(args.limit, settings.limit)
|
|
83
|
+
: clampToolLimit(settings.limit, 5);
|
|
84
|
+
return { query, ids, limit };
|
|
85
|
+
}
|
|
86
|
+
/** Format the model-facing result text. */
|
|
87
|
+
export function formatDomainValue(args, value) {
|
|
88
|
+
const query = isRecord(args) && typeof args.query === 'string' ? args.query : 'the query';
|
|
89
|
+
const head = value.sources.length === 0
|
|
90
|
+
? `No dedicated source returned a result for ${JSON.stringify(query)}.`
|
|
91
|
+
: `Dedicated search for ${JSON.stringify(query)}: ${value.sources.length} result(s)${value.truncated ? ' (truncated)' : ''}.`;
|
|
92
|
+
const lines = value.sources.map((source, index) => {
|
|
93
|
+
const title = source.title !== undefined && source.title.length > 0 ? source.title : source.url;
|
|
94
|
+
const body = [`${index + 1}. ${title}`, ` ${source.url}`];
|
|
95
|
+
if (source.snippet !== undefined && source.snippet.length > 0)
|
|
96
|
+
body.push(` ${source.snippet}`);
|
|
97
|
+
if (source.publishedAt !== undefined && source.publishedAt.length > 0)
|
|
98
|
+
body.push(` ${source.publishedAt}`);
|
|
99
|
+
return body.join('\n');
|
|
100
|
+
});
|
|
101
|
+
if (value.failures.length > 0) {
|
|
102
|
+
lines.push('', 'Unavailable sources:', ...value.failures.map((failure) => `- ${failure}`));
|
|
103
|
+
}
|
|
104
|
+
return [head, '', ...lines].join('\n').trimEnd();
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Build the tool definition the host registry accepts.
|
|
108
|
+
* @param name - `domain_search`, or the fallback when that name is taken.
|
|
109
|
+
* @param readSettings - read the live config for every call (volatile fields).
|
|
110
|
+
* @param options - test seams and diagnostics.
|
|
111
|
+
* @returns a registry-ready definition.
|
|
112
|
+
*/
|
|
113
|
+
export function createDomainToolDefinition(name, readSettings, options = {}) {
|
|
114
|
+
const detectProxy = options.detectProxy ?? (() => resolveProxy(undefined));
|
|
115
|
+
return {
|
|
116
|
+
name,
|
|
117
|
+
description: 'Search dedicated, key-free sources instead of the open web: GitHub repositories, npm/PyPI/crates packages, arXiv papers, Wikipedia, Hacker News and Stack Exchange. Use it when the question is about a package, paper, repository or Q&A thread; use web_search for general pages.',
|
|
118
|
+
parameters: parameterSchema(),
|
|
119
|
+
output: {
|
|
120
|
+
schema: outputSchema(),
|
|
121
|
+
render: (args, value) => [{ type: 'text', text: formatDomainValue(args, value) }],
|
|
122
|
+
},
|
|
123
|
+
timeoutMs: 30_000,
|
|
124
|
+
isConcurrencySafe: () => true,
|
|
125
|
+
async execute(args, exec) {
|
|
126
|
+
const settings = readSettings();
|
|
127
|
+
const parsed = parseArgs(args, settings);
|
|
128
|
+
const explicit = typeof settings.proxy === 'string' ? settings.proxy.trim() : '';
|
|
129
|
+
const proxy = explicit.length > 0 ? explicit : await detectProxy();
|
|
130
|
+
const outcome = await searchVerticals(parsed.query, {
|
|
131
|
+
ids: parsed.ids,
|
|
132
|
+
limit: parsed.limit,
|
|
133
|
+
timeoutMs: settings.timeoutMs,
|
|
134
|
+
...(proxy === undefined || proxy.length === 0 ? {} : { proxy }),
|
|
135
|
+
...(exec.signal === undefined ? {} : { signal: exec.signal }),
|
|
136
|
+
...(options.fetch === undefined ? {} : { fetch: options.fetch }),
|
|
137
|
+
});
|
|
138
|
+
return {
|
|
139
|
+
sources: outcome.sources,
|
|
140
|
+
failures: outcome.failures,
|
|
141
|
+
truncated: outcome.truncated,
|
|
142
|
+
};
|
|
143
|
+
},
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Pick the tool name this deployment can accept.
|
|
148
|
+
*
|
|
149
|
+
* A plugin that already owns `domain_search` must not make this one fail to
|
|
150
|
+
* register, so the fallback name is used instead. A registry without `get` is
|
|
151
|
+
* assumed free.
|
|
152
|
+
* @param tools - the live registry.
|
|
153
|
+
* @returns the preferred or fallback name.
|
|
154
|
+
*/
|
|
155
|
+
export function pickDomainToolName(tools) {
|
|
156
|
+
if (typeof tools.get !== 'function')
|
|
157
|
+
return DOMAIN_TOOL_NAME;
|
|
158
|
+
try {
|
|
159
|
+
return tools.get(DOMAIN_TOOL_NAME) === undefined ? DOMAIN_TOOL_NAME : DOMAIN_TOOL_FALLBACK_NAME;
|
|
160
|
+
}
|
|
161
|
+
catch {
|
|
162
|
+
return DOMAIN_TOOL_NAME;
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
/** Report a registration failure without throwing out of the plugin. */
|
|
166
|
+
function report(options, error) {
|
|
167
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
168
|
+
try {
|
|
169
|
+
options.warn?.(`domain_search registration skipped: ${message}`);
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
// A diagnostic callback must never break plugin loading.
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* Register the tool when a registry is available.
|
|
177
|
+
*
|
|
178
|
+
* Every failure path is contained: a missing service is the caller's job to
|
|
179
|
+
* check, a duplicate name falls back, and any registry rejection is reported
|
|
180
|
+
* and swallowed so `web_search` keeps working.
|
|
181
|
+
* @param tools - the live registry.
|
|
182
|
+
* @param readSettings - live config reader.
|
|
183
|
+
* @param options - test seams and diagnostics.
|
|
184
|
+
* @returns the registered name, or `undefined` when registration was skipped.
|
|
185
|
+
*/
|
|
186
|
+
export function registerDomainTool(tools, readSettings, options = {}) {
|
|
187
|
+
const name = pickDomainToolName(tools);
|
|
188
|
+
try {
|
|
189
|
+
tools.register(createDomainToolDefinition(name, readSettings, options));
|
|
190
|
+
return name;
|
|
191
|
+
}
|
|
192
|
+
catch (error) {
|
|
193
|
+
report(options, error);
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
}
|
package/lib/engines.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { SearchError } from "./errors.js";
|
|
2
|
+
import { filterByTimeRange, preferWindow } from "./freshness.js";
|
|
2
3
|
import { resolveProxy } from "./proxy.js";
|
|
3
4
|
import { parseAnySearch, parseBing, parseDdgHtml, parseDdgLite, parseSo } from "./parsers.js";
|
|
4
5
|
import { mergeRegions } from "./rank.js";
|
|
@@ -39,7 +40,7 @@ export const HOSTS = [
|
|
|
39
40
|
tier: 'primary',
|
|
40
41
|
region: 'global',
|
|
41
42
|
viaProxy: true,
|
|
42
|
-
build: (query) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`,
|
|
43
|
+
build: (query, timeRange) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}${ddgWindow(timeRange)}`,
|
|
43
44
|
parse: parseDdgHtml,
|
|
44
45
|
},
|
|
45
46
|
{
|
|
@@ -51,6 +52,7 @@ export const HOSTS = [
|
|
|
51
52
|
tier: 'fallback',
|
|
52
53
|
region: 'global',
|
|
53
54
|
viaProxy: true,
|
|
55
|
+
// No `filters` parameter: see the measurement note before `HOSTS`.
|
|
54
56
|
build: (query) => `https://www.bing.com/search?q=${encodeURIComponent(query)}&setmkt=en-US&setlang=en&cc=US`,
|
|
55
57
|
parse: parseBing,
|
|
56
58
|
},
|
|
@@ -58,6 +60,7 @@ export const HOSTS = [
|
|
|
58
60
|
id: 'cn.bing',
|
|
59
61
|
tier: 'primary',
|
|
60
62
|
region: 'cn',
|
|
63
|
+
// No `filters` parameter: see the measurement note before `HOSTS`.
|
|
61
64
|
build: (query) => `https://cn.bing.com/search?q=${encodeURIComponent(query)}`,
|
|
62
65
|
parse: parseBing,
|
|
63
66
|
},
|
|
@@ -73,12 +76,48 @@ export const HOSTS = [
|
|
|
73
76
|
tier: 'fallback',
|
|
74
77
|
region: 'global',
|
|
75
78
|
viaProxy: true,
|
|
76
|
-
build: (query) => `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}`,
|
|
79
|
+
build: (query, timeRange) => `https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(query)}${ddgWindow(timeRange)}`,
|
|
77
80
|
parse: parseDdgLite,
|
|
78
81
|
},
|
|
79
82
|
];
|
|
80
83
|
/** Scripts that make a query CJK rather than ASCII. */
|
|
81
84
|
const CJK = /[\u3000-\u303f\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uff00-\uffef]/u;
|
|
85
|
+
/**
|
|
86
|
+
* DuckDuckGo's own date filter, appended to a result-page URL.
|
|
87
|
+
*
|
|
88
|
+
* Measured live on both endpoints: adding `df=d`, `df=w` or `df=m` changed the
|
|
89
|
+
* returned result set, so this is a real pushdown rather than a decorative
|
|
90
|
+
* parameter. `off` (and the absent value) adds nothing.
|
|
91
|
+
*
|
|
92
|
+
* @param timeRange - active window, if any.
|
|
93
|
+
* @returns `&df=x` for a supported window, otherwise an empty string.
|
|
94
|
+
*/
|
|
95
|
+
function ddgWindow(timeRange) {
|
|
96
|
+
switch (timeRange) {
|
|
97
|
+
case 'day':
|
|
98
|
+
return '&df=d';
|
|
99
|
+
case 'week':
|
|
100
|
+
return '&df=w';
|
|
101
|
+
case 'month':
|
|
102
|
+
return '&df=m';
|
|
103
|
+
case 'year':
|
|
104
|
+
return '&df=y';
|
|
105
|
+
case 'off':
|
|
106
|
+
case undefined:
|
|
107
|
+
default:
|
|
108
|
+
return '';
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
/*
|
|
112
|
+
* Why Bing has no time parameter.
|
|
113
|
+
*
|
|
114
|
+
* Measured on `cn.bing.com` and `www.bing.com` (which redirects to `cn.bing`):
|
|
115
|
+
* `filters=ex1:"ez1|ez2|ez3|ez5_…"` is accepted — the response marks the
|
|
116
|
+
* matching entry as selected — but the ten returned URLs were byte-for-byte
|
|
117
|
+
* identical to the unfiltered run for both a general and a news query. Bing's
|
|
118
|
+
* filter is therefore not written into the request; the engine layer filters
|
|
119
|
+
* Bing's parsed dates locally instead.
|
|
120
|
+
*/
|
|
82
121
|
/** Soft deadline per wait policy, in milliseconds. */
|
|
83
122
|
const SOFT_DEADLINE = {
|
|
84
123
|
auto: (ascii) => (ascii ? 6000 : 3500),
|
|
@@ -132,7 +171,7 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
|
|
|
132
171
|
releasedEarly = early;
|
|
133
172
|
released();
|
|
134
173
|
};
|
|
135
|
-
const running = hosts.map((host, index) => searchHost(host, query, controller.signal, timeoutMs, proxy).then((outcome) => {
|
|
174
|
+
const running = hosts.map((host, index) => searchHost(host, query, controller.signal, timeoutMs, proxy, limits.timeRange).then((outcome) => {
|
|
136
175
|
if (outcome.ok) {
|
|
137
176
|
outcomes[index] = { host, sources: outcome.sources };
|
|
138
177
|
health?.noteSuccess(host.id);
|
|
@@ -227,6 +266,7 @@ export async function search(query, signal, options) {
|
|
|
227
266
|
const limit = Math.max(1, Math.min(options.limit, 20));
|
|
228
267
|
const timeoutMs = options.timeoutMs ?? 15000;
|
|
229
268
|
const wait = options.wait ?? 'auto';
|
|
269
|
+
const timeRange = options.timeRange ?? 'off';
|
|
230
270
|
const proxy = options.useProxy === false ? undefined : await resolveProxy(options.proxy);
|
|
231
271
|
const allow = options.hostIds !== undefined && options.hostIds.length > 0 ? new Set(options.hostIds) : undefined;
|
|
232
272
|
const hosts = HOSTS.filter((host) => allow === undefined || allow.has(host.id))
|
|
@@ -251,24 +291,42 @@ export async function search(query, signal, options) {
|
|
|
251
291
|
const wanted = usable.filter((host) => leadsQuery(host, lead)).length;
|
|
252
292
|
const softMs = options.softMs ?? SOFT_DEADLINE[wait](ascii);
|
|
253
293
|
const needed = wait === 'all' ? usable.length : wait === 'fast' ? 1 : wanted > 0 ? wanted : Math.min(2, usable.length);
|
|
254
|
-
const limits = { limit, softMs, needed, lead: wait === 'all' ? 'all' : lead };
|
|
255
|
-
|
|
294
|
+
const limits = { limit, softMs, needed, lead: wait === 'all' ? 'all' : lead, timeRange };
|
|
295
|
+
// Ranking does not drop rows; the engine layer does. `timeRange` is passed
|
|
296
|
+
// into the rank options only so undated rows are scored lower while a window
|
|
297
|
+
// is active. Keeping it out of the configured `rank` object preserves the
|
|
298
|
+
// documented shape of `ResolvedConfig.rank`.
|
|
299
|
+
const rank = {
|
|
300
|
+
...(options.rank ?? {}),
|
|
301
|
+
...(timeRange === 'off' ? {} : { timeRange }),
|
|
302
|
+
};
|
|
303
|
+
/**
|
|
304
|
+
* Drop results whose parsed date is outside the window.
|
|
305
|
+
*
|
|
306
|
+
* Undated results survive on purpose: most documentation pages publish no
|
|
307
|
+
* date, and discarding them would empty the list for a library query. They
|
|
308
|
+
* are down-weighted during fusion instead, so a dated in-window page wins a
|
|
309
|
+
* close comparison without erasing the undated long tail.
|
|
310
|
+
*/
|
|
311
|
+
const applyWindow = (answers) => timeRange === 'off'
|
|
312
|
+
? answers
|
|
313
|
+
: answers.map((answer) => ({ host: answer.host, sources: filterByTimeRange(answer.sources, timeRange) }));
|
|
256
314
|
// The key names the engines the run *would* use, not the ones that survived
|
|
257
315
|
// the cooldown filter: an engine entering or leaving a cooldown must not
|
|
258
316
|
// invalidate an entry that is still fresh, or every throttle would turn the
|
|
259
317
|
// next identical query into a cache miss.
|
|
260
|
-
const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id) });
|
|
318
|
+
const key = options.cache === undefined ? undefined : cacheKey({ query, limit, wait, lead, engines: hosts.map((host) => host.id), timeRange });
|
|
261
319
|
const cached = key === undefined ? undefined : options.cache?.get(key);
|
|
262
320
|
if (cached !== undefined)
|
|
263
321
|
return { sources: cached, failures, ...(proxy === undefined ? {} : { proxy }) };
|
|
264
322
|
const ordered = [...usable.filter((host) => leadsQuery(host, lead)), ...usable.filter((host) => !leadsQuery(host, lead))];
|
|
265
323
|
const primary = ordered.filter((host) => host.tier === 'primary');
|
|
266
324
|
const fallback = ordered.filter((host) => host.tier === 'fallback');
|
|
267
|
-
let answers = await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health);
|
|
268
|
-
let sources = answers.length === 0 ? [] : mergeRegions(answers, lead, limit, rank);
|
|
325
|
+
let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
326
|
+
let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
269
327
|
if (sources.length === 0 && fallback.length > 0) {
|
|
270
|
-
answers = await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health);
|
|
271
|
-
sources = answers.length === 0 ? [] : mergeRegions(answers, lead, limit, rank);
|
|
328
|
+
answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health));
|
|
329
|
+
sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
|
|
272
330
|
}
|
|
273
331
|
if (signal?.aborted === true)
|
|
274
332
|
throw new SearchError('web search was cancelled', 'SEARCH_ABORTED', false);
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Publication-date parsing and rolling time-window filtering.
|
|
3
|
+
*
|
|
4
|
+
* Search engines disagree about how they spell a date: Bing prints
|
|
5
|
+
* `2025年9月25日` or `2 天之前` at the head of a snippet, 360 prints
|
|
6
|
+
* `发贴时间:2026年1月22日`, DuckDuckGo usually prints nothing, and AnySearch
|
|
7
|
+
* appends `date: 22 hours ago` to its `content` field. All of those become one
|
|
8
|
+
* canonical value, an ISO 8601 UTC string, so ranking and filtering never have
|
|
9
|
+
* to care which engine answered.
|
|
10
|
+
*
|
|
11
|
+
* Two rules keep this safe:
|
|
12
|
+
*
|
|
13
|
+
* - Nothing depends on the machine's timezone. An absolute date is built with
|
|
14
|
+
* `Date.UTC` and anchored at noon UTC, so `2024-05-01` is the same instant in
|
|
15
|
+
* Shanghai, Berlin and Los Angeles and cannot drift into the previous day.
|
|
16
|
+
* - A missing or unparsable date never discards a result. Filtering only drops
|
|
17
|
+
* a result whose date is *known* to be outside the window; unknown dates are
|
|
18
|
+
* kept and merely rank lower (see `rank.ts`), because most documentation pages
|
|
19
|
+
* carry no date at all.
|
|
20
|
+
*
|
|
21
|
+
* @module @fangchenghao/dsh-web-search/freshness
|
|
22
|
+
*/
|
|
23
|
+
import type { Source, TimeRange } from './types.ts';
|
|
24
|
+
/** Milliseconds in one day, shared with the ranking module. */
|
|
25
|
+
export declare const MS_PER_DAY = 86400000;
|
|
26
|
+
/** Rolling window length per range. `off` is handled separately. */
|
|
27
|
+
export declare const TIME_RANGE_MS: Readonly<Record<Exclude<TimeRange, 'off'>, number>>;
|
|
28
|
+
/** Every accepted `timeRange` value, in settings order. */
|
|
29
|
+
export declare const TIME_RANGES: readonly TimeRange[];
|
|
30
|
+
/**
|
|
31
|
+
* Extract a publication date from one fragment of result text.
|
|
32
|
+
*
|
|
33
|
+
* The search order is deliberate: an explicit `date:` marker wins (AnySearch
|
|
34
|
+
* puts the engine's stamp there), then a relative phrase, then an absolute
|
|
35
|
+
* date. `今天` inside a sentence such as `但是我们今天讲的逻辑复制` is not a
|
|
36
|
+
* publication stamp, so relative words only match as whole tokens; a bare date
|
|
37
|
+
* such as `2024-05-01` is unambiguous enough to match anywhere.
|
|
38
|
+
*
|
|
39
|
+
* @param fragment - summary, stamp or result text.
|
|
40
|
+
* @param now - evaluation instant, injected so tests are deterministic.
|
|
41
|
+
* @returns ISO 8601 UTC, or `undefined` when the fragment carries no date.
|
|
42
|
+
*/
|
|
43
|
+
export declare function parsePublishedAt(fragment: string, now?: number): string | undefined;
|
|
44
|
+
/**
|
|
45
|
+
* Parse the first fragment that carries a usable date.
|
|
46
|
+
*
|
|
47
|
+
* Parsers pass the most authoritative fragment first (a dedicated timestamp
|
|
48
|
+
* element, then the snippet), so a date mentioned in prose cannot override the
|
|
49
|
+
* stamp the engine printed.
|
|
50
|
+
*
|
|
51
|
+
* @param fragments - candidate texts, most authoritative first.
|
|
52
|
+
* @param now - evaluation instant, injected for tests.
|
|
53
|
+
* @returns the first parsed ISO 8601 UTC value, or `undefined`.
|
|
54
|
+
*/
|
|
55
|
+
export declare function extractPublishedAt(fragments: readonly (string | undefined)[], now?: number): string | undefined;
|
|
56
|
+
/**
|
|
57
|
+
* Read a publication date as epoch milliseconds.
|
|
58
|
+
* @param publishedAt - canonical ISO string, or `undefined`.
|
|
59
|
+
* @returns epoch milliseconds, or `undefined` when absent or unparsable.
|
|
60
|
+
*/
|
|
61
|
+
export declare function publishedTimestamp(publishedAt: string | undefined): number | undefined;
|
|
62
|
+
/**
|
|
63
|
+
* Whether a result belongs to the rolling window.
|
|
64
|
+
*
|
|
65
|
+
* `off` accepts everything. A missing or unparsable date is accepted on
|
|
66
|
+
* purpose: filtering exists to remove results known to be stale, not to punish
|
|
67
|
+
* pages that publish no metadata. A date in the future is accepted too — it is
|
|
68
|
+
* still inside the window, and ranking clamps its age to zero.
|
|
69
|
+
*
|
|
70
|
+
* @param publishedAt - canonical ISO string, or `undefined`.
|
|
71
|
+
* @param range - active window.
|
|
72
|
+
* @param now - evaluation instant.
|
|
73
|
+
* @returns `true` when the result may stay.
|
|
74
|
+
*/
|
|
75
|
+
export declare function isWithinTimeRange(publishedAt: string | undefined, range: TimeRange, now?: number): boolean;
|
|
76
|
+
/**
|
|
77
|
+
* Keep only results inside the window.
|
|
78
|
+
*
|
|
79
|
+
* A result whose date could not be parsed is not evidence of an old page, so it
|
|
80
|
+
* is held back rather than dropped: when at least one dated result falls inside
|
|
81
|
+
* the window, only those are returned; when none does, the undated results are
|
|
82
|
+
* returned as a fallback so the caller is not left empty-handed. That trades a
|
|
83
|
+
* little precision for never answering "last 24 hours" with nothing at all.
|
|
84
|
+
* @param sources - parsed sources, in engine order.
|
|
85
|
+
* @param range - active window.
|
|
86
|
+
* @param now - evaluation instant.
|
|
87
|
+
* @returns a new list; the input array and its objects are untouched.
|
|
88
|
+
*/
|
|
89
|
+
export declare function filterByTimeRange(sources: readonly Source[], range: TimeRange, now?: number): Source[];
|
|
90
|
+
/**
|
|
91
|
+
* Apply a window to an already merged list.
|
|
92
|
+
*
|
|
93
|
+
* Per-engine filtering can only see one engine's dates, so an engine that dates
|
|
94
|
+
* nothing keeps all of its rows; this pass runs after the merge, where a dated
|
|
95
|
+
* row from any engine can speak for the whole list. When nothing is dated inside
|
|
96
|
+
* the window the merged list is returned unchanged rather than emptied.
|
|
97
|
+
* @param sources - merged sources.
|
|
98
|
+
* @param range - active window.
|
|
99
|
+
* @param now - evaluation instant.
|
|
100
|
+
* @returns the rows to keep.
|
|
101
|
+
*/
|
|
102
|
+
export declare function preferWindow(sources: readonly Source[], range: TimeRange, now?: number): Source[];
|