@fffangchenghao/dsh-web-search 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +82 -0
- package/README.md +247 -41
- package/README.zh-CN.md +247 -41
- package/client.js +66 -9
- package/lib/cache-persist.d.ts +96 -0
- package/lib/cache-persist.js +249 -0
- package/lib/cache.d.ts +56 -8
- package/lib/cache.js +141 -13
- package/lib/config.d.ts +33 -1
- package/lib/config.js +54 -2
- package/lib/domain-tool.d.ts +119 -0
- package/lib/domain-tool.js +196 -0
- package/lib/engines.js +68 -10
- package/lib/fetch.d.ts +7 -0
- package/lib/fetch.js +36 -3
- package/lib/freshness.d.ts +102 -0
- package/lib/freshness.js +359 -0
- package/lib/index.d.ts +44 -2
- package/lib/index.js +144 -23
- package/lib/markdown.d.ts +84 -0
- package/lib/markdown.js +847 -0
- package/lib/parsers.js +59 -5
- package/lib/rank.d.ts +14 -1
- package/lib/rank.js +18 -0
- package/lib/search-many.d.ts +117 -0
- package/lib/search-many.js +286 -0
- package/lib/transport.d.ts +4 -2
- package/lib/transport.js +4 -2
- package/lib/types.d.ts +22 -6
- package/lib/verticals.d.ts +164 -0
- package/lib/verticals.js +539 -0
- package/package.json +4 -4
package/lib/parsers.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { extractPublishedAt, parsePublishedAt } from "./freshness.js";
|
|
1
2
|
/** Named HTML entities that actually show up in result markup. */
|
|
2
3
|
const ENTITIES = {
|
|
3
4
|
amp: '&',
|
|
@@ -146,7 +147,17 @@ export function parseSo(payload, baseUrl) {
|
|
|
146
147
|
const title = text(anchor[2]);
|
|
147
148
|
const summary = /<span[^>]*class="res-list-summary[^"]*"[^>]*>([\s\S]*?)<\/span>/.exec(item);
|
|
148
149
|
const snippet = summary ? text(summary[1]) : '';
|
|
149
|
-
|
|
150
|
+
// 360 prints the stamp in a separate span (`发贴时间:2026年1月22日`), not in
|
|
151
|
+
// the summary. Reading only that span avoids treating prose such as
|
|
152
|
+
// `但是我们今天讲的逻辑复制` as a publication date.
|
|
153
|
+
const stamp = /(?:发[贴布]|发布)时间[::]\s*([^<]{0,60})/.exec(item);
|
|
154
|
+
const publishedAt = stamp ? parsePublishedAt(text(stamp[1])) : undefined;
|
|
155
|
+
sources.push({
|
|
156
|
+
url,
|
|
157
|
+
...(title ? { title } : {}),
|
|
158
|
+
...(snippet ? { snippet } : {}),
|
|
159
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
160
|
+
});
|
|
150
161
|
}
|
|
151
162
|
return sources;
|
|
152
163
|
}
|
|
@@ -171,7 +182,16 @@ export function parseBing(payload, baseUrl) {
|
|
|
171
182
|
const title = text(anchor[2]);
|
|
172
183
|
const paragraph = /<p[^>]*>([\s\S]*?)<\/p>/.exec(item);
|
|
173
184
|
const snippet = paragraph ? text(paragraph[1]) : '';
|
|
174
|
-
|
|
185
|
+
// Bing puts the date at the head of the caption (`2025年9月25日 …`,
|
|
186
|
+
// `2 天之前 …`) or in a `news_dt` span; some captions end with an ISO date.
|
|
187
|
+
const newsDate = /<span[^>]*class="news_dt"[^>]*>([\s\S]*?)<\/span>/.exec(item);
|
|
188
|
+
const publishedAt = extractPublishedAt([newsDate ? text(newsDate[1]) : undefined, snippet]);
|
|
189
|
+
sources.push({
|
|
190
|
+
url,
|
|
191
|
+
...(title ? { title } : {}),
|
|
192
|
+
...(snippet ? { snippet } : {}),
|
|
193
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
194
|
+
});
|
|
175
195
|
}
|
|
176
196
|
return sources;
|
|
177
197
|
}
|
|
@@ -200,7 +220,17 @@ export function parseDdgLite(payload, baseUrl) {
|
|
|
200
220
|
continue;
|
|
201
221
|
const title = text(match[1]);
|
|
202
222
|
const snippet = text(/<td[^>]*class=['"]result-snippet['"][^>]*>([\s\S]*?)<\/td>/i.exec(payload.slice(start, end))?.[1] ?? '');
|
|
203
|
-
|
|
223
|
+
// The lite page has no timestamp in today's markup; if one ever appears it
|
|
224
|
+
// is the authoritative stamp, otherwise a date mentioned in the snippet is
|
|
225
|
+
// the only evidence available.
|
|
226
|
+
const stamp = /<[a-z]+[^>]*class=['"]result-timestamp['"][^>]*>([\s\S]*?)<\/[a-z]+>/i.exec(payload.slice(start, end));
|
|
227
|
+
const publishedAt = extractPublishedAt([stamp?.[1], snippet]);
|
|
228
|
+
sources.push({
|
|
229
|
+
url,
|
|
230
|
+
...(title ? { title } : {}),
|
|
231
|
+
...(snippet ? { snippet } : {}),
|
|
232
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
233
|
+
});
|
|
204
234
|
}
|
|
205
235
|
return sources;
|
|
206
236
|
}
|
|
@@ -230,7 +260,16 @@ export function parseDdgHtml(payload, baseUrl) {
|
|
|
230
260
|
continue;
|
|
231
261
|
const title = text(match[1]);
|
|
232
262
|
const snippet = text(/<[a-z]+[^>]*class=['"]result__snippet['"][^>]*>([\s\S]*?)<\/[a-z]+>/i.exec(payload.slice(start, end))?.[1] ?? '');
|
|
233
|
-
|
|
263
|
+
// DuckDuckGo's HTML endpoint usually omits the date; when a timestamp span
|
|
264
|
+
// is present it is preferred over a date that only appears in the snippet.
|
|
265
|
+
const stamp = /<[a-z]+[^>]*class=['"]result__timestamp['"][^>]*>([\s\S]*?)<\/[a-z]+>/i.exec(payload.slice(start, end));
|
|
266
|
+
const publishedAt = extractPublishedAt([stamp?.[1], snippet]);
|
|
267
|
+
sources.push({
|
|
268
|
+
url,
|
|
269
|
+
...(title ? { title } : {}),
|
|
270
|
+
...(snippet ? { snippet } : {}),
|
|
271
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
272
|
+
});
|
|
234
273
|
}
|
|
235
274
|
return sources;
|
|
236
275
|
}
|
|
@@ -275,7 +314,22 @@ export function parseAnySearch(payload, baseUrl) {
|
|
|
275
314
|
continue;
|
|
276
315
|
const title = typeof row.title === 'string' ? row.title.trim() : '';
|
|
277
316
|
const snippet = typeof row.snippet === 'string' ? row.snippet.trim() : '';
|
|
278
|
-
|
|
317
|
+
// Measured against the live endpoint: rows carry title/url/snippet/content,
|
|
318
|
+
// and the publication stamp is appended to `content` as `date: 22 hours
|
|
319
|
+
// ago` rather than exposed as a top-level date field. A top-level `date`
|
|
320
|
+
// is accepted too, so a future API change does not need a parser rewrite.
|
|
321
|
+
// Only the marked value is parsed: the rest of `content` is article prose
|
|
322
|
+
// and may mention unrelated dates (`Retrieved June 2, 2024`).
|
|
323
|
+
const content = typeof row.content === 'string' ? row.content : '';
|
|
324
|
+
const marker = /(?:^|[\s|])date\s*[::]\s*([^|\n]+?)\s*$/i.exec(content);
|
|
325
|
+
const topLevel = typeof row.date === 'string' ? row.date : undefined;
|
|
326
|
+
const publishedAt = extractPublishedAt([topLevel, marker?.[1]]);
|
|
327
|
+
sources.push({
|
|
328
|
+
url,
|
|
329
|
+
...(title ? { title } : {}),
|
|
330
|
+
...(snippet ? { snippet } : {}),
|
|
331
|
+
...(publishedAt === undefined ? {} : { publishedAt }),
|
|
332
|
+
});
|
|
279
333
|
}
|
|
280
334
|
return sources;
|
|
281
335
|
}
|
package/lib/rank.d.ts
CHANGED
|
@@ -31,7 +31,11 @@
|
|
|
31
31
|
* happens to sit at the same rank;
|
|
32
32
|
* - freshness: a result published a few days ago is more useful than the same
|
|
33
33
|
* result published years ago, with the boost fading linearly to zero at
|
|
34
|
-
* `freshnessDays
|
|
34
|
+
* `freshnessDays`;
|
|
35
|
+
* - an active time filter: a dated in-window result beats a result with no
|
|
36
|
+
* date at the same rank. The penalty is small and applied only while
|
|
37
|
+
* filtering is on, so an undated page can still win on relevance and the
|
|
38
|
+
* default (`timeRange: off`) is byte-for-byte the old behaviour.
|
|
35
39
|
*
|
|
36
40
|
* The bilingual region policy is kept intact: engines whose region matches the
|
|
37
41
|
* query language's lead (or are marked `both`) are fused first and fill the
|
|
@@ -55,6 +59,15 @@ export declare const DEFAULT_AUTHORITY_BOOST = 0.25;
|
|
|
55
59
|
export declare const DEFAULT_FRESHNESS_BOOST = 0.2;
|
|
56
60
|
/** Age in days after which the freshness boost has faded to zero. */
|
|
57
61
|
export declare const DEFAULT_FRESHNESS_DAYS = 30;
|
|
62
|
+
/**
|
|
63
|
+
* Score factor for a result with no usable date while a time window is active.
|
|
64
|
+
*
|
|
65
|
+
* Ten percent is roughly six fusion positions at `rrfConstant = 60`: enough for
|
|
66
|
+
* a dated result to win a close comparison, small enough that an undated result
|
|
67
|
+
* still outranks a dated one that is clearly less relevant. With every result
|
|
68
|
+
* undated the factor is uniform, so the order is unchanged.
|
|
69
|
+
*/
|
|
70
|
+
export declare const UNDATED_WHEN_FILTERING_FACTOR = 0.9;
|
|
58
71
|
/**
|
|
59
72
|
* Deduplicate by URL, preserving first-seen order.
|
|
60
73
|
*
|
package/lib/rank.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { publishedTimestamp } from "./freshness.js";
|
|
1
2
|
/**
|
|
2
3
|
* Reciprocal-rank-fusion constant over the interleaved slot order; a larger
|
|
3
4
|
* value flattens the curve so authority and freshness reorder more freely.
|
|
@@ -33,6 +34,15 @@ export const DEFAULT_AUTHORITY_BOOST = 0.25;
|
|
|
33
34
|
export const DEFAULT_FRESHNESS_BOOST = 0.2;
|
|
34
35
|
/** Age in days after which the freshness boost has faded to zero. */
|
|
35
36
|
export const DEFAULT_FRESHNESS_DAYS = 30;
|
|
37
|
+
/**
|
|
38
|
+
* Score factor for a result with no usable date while a time window is active.
|
|
39
|
+
*
|
|
40
|
+
* Ten percent is roughly six fusion positions at `rrfConstant = 60`: enough for
|
|
41
|
+
* a dated result to win a close comparison, small enough that an undated result
|
|
42
|
+
* still outranks a dated one that is clearly less relevant. With every result
|
|
43
|
+
* undated the factor is uniform, so the order is unchanged.
|
|
44
|
+
*/
|
|
45
|
+
export const UNDATED_WHEN_FILTERING_FACTOR = 0.9;
|
|
36
46
|
/**
|
|
37
47
|
* Scores closer together than this count as equal, so the deterministic
|
|
38
48
|
* tie-break decides instead of floating-point summation order.
|
|
@@ -183,6 +193,7 @@ function resolveOptions(options) {
|
|
|
183
193
|
authorityBoost: nonNegativeOr(options?.authorityBoost, DEFAULT_AUTHORITY_BOOST),
|
|
184
194
|
freshnessBoost: nonNegativeOr(options?.freshnessBoost, DEFAULT_FRESHNESS_BOOST),
|
|
185
195
|
freshnessDays: positiveOr(options?.freshnessDays, DEFAULT_FRESHNESS_DAYS),
|
|
196
|
+
timeRange: options?.timeRange,
|
|
186
197
|
weights: Array.isArray(options?.weights) ? options.weights : undefined,
|
|
187
198
|
};
|
|
188
199
|
}
|
|
@@ -373,12 +384,19 @@ function fuseEntries(entries, limit, options) {
|
|
|
373
384
|
if (candidates.size === 0)
|
|
374
385
|
return [];
|
|
375
386
|
const ranked = [...candidates.values()];
|
|
387
|
+
const filtering = config.timeRange !== undefined && config.timeRange !== 'off';
|
|
376
388
|
for (const candidate of ranked) {
|
|
377
389
|
const voteCount = votes.get(candidate.key) ?? 1;
|
|
378
390
|
let score = voteCount / (config.rrfConstant + candidate.position);
|
|
379
391
|
if (isAuthoritative(candidate.source.url, config.authorityDomains))
|
|
380
392
|
score *= 1 + config.authorityBoost;
|
|
381
393
|
score *= freshnessFactor(candidate.source.publishedAt, config.freshnessDays, config.freshnessBoost, now);
|
|
394
|
+
// A missing (or unparsable) date is not evidence of staleness, so the row
|
|
395
|
+
// is kept. While a window is active it is discounted instead of being
|
|
396
|
+
// dropped, which keeps the long tail of undated documentation visible.
|
|
397
|
+
if (filtering && publishedTimestamp(candidate.source.publishedAt) === undefined) {
|
|
398
|
+
score *= UNDATED_WHEN_FILTERING_FACTOR;
|
|
399
|
+
}
|
|
382
400
|
candidate.score = score;
|
|
383
401
|
}
|
|
384
402
|
ranked.sort(compareRanked);
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The model-facing `search_many` tool: several independent searches in one
|
|
3
|
+
* call, each reusing the same engine layer, cache and cooldown tracker as
|
|
4
|
+
* `web_search`.
|
|
5
|
+
*
|
|
6
|
+
* The batch is deliberately boring: every query is a normal search, failures
|
|
7
|
+
* are recorded per query instead of aborting the batch, and at most three
|
|
8
|
+
* searches run at once so one call cannot spend the whole engine budget. The
|
|
9
|
+
* registration path mirrors `domain-tool.ts` — raw JSON Schema, `output`
|
|
10
|
+
* carrying `{ schema, render }`, the optional `ctx.get('tools')` lookup, a
|
|
11
|
+
* fallback name on collision, and a contained failure path.
|
|
12
|
+
*
|
|
13
|
+
* @module @fangchenghao/dsh-web-search/search-many
|
|
14
|
+
*/
|
|
15
|
+
import type { Source } from './types.ts';
|
|
16
|
+
import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
|
|
17
|
+
/** The name this tool prefers. */
|
|
18
|
+
export declare const SEARCH_MANY_TOOL_NAME = "search_many";
|
|
19
|
+
/** The name used when another plugin already registered `search_many`. */
|
|
20
|
+
export declare const SEARCH_MANY_FALLBACK_NAME = "web_search_many";
|
|
21
|
+
/** How many queries run at the same time; deliberately below the engine count. */
|
|
22
|
+
export declare const SEARCH_MANY_CONCURRENCY = 3;
|
|
23
|
+
/** Queries accepted in one call. */
|
|
24
|
+
export declare const SEARCH_MANY_MIN_QUERIES = 1;
|
|
25
|
+
/** Queries accepted in one call. */
|
|
26
|
+
export declare const SEARCH_MANY_MAX_QUERIES = 5;
|
|
27
|
+
/** Default result budget for a query that names none. */
|
|
28
|
+
export declare const SEARCH_MANY_DEFAULT_LIMIT = 5;
|
|
29
|
+
/** Largest per-query budget the tool accepts. */
|
|
30
|
+
export declare const SEARCH_MANY_MAX_LIMIT = 20;
|
|
31
|
+
/** One source as `search_many` reports it; `site` appears only when asked. */
|
|
32
|
+
export interface SearchManySource extends Source {
|
|
33
|
+
/** Lower-cased hostname of {@link Source.url}, when `withSites` is true. */
|
|
34
|
+
site?: string;
|
|
35
|
+
}
|
|
36
|
+
/** What one query in the batch produced. */
|
|
37
|
+
export interface SearchManyResult {
|
|
38
|
+
/** The query exactly as it was accepted (trimmed). */
|
|
39
|
+
query: string;
|
|
40
|
+
sources: SearchManySource[];
|
|
41
|
+
/** Failures for this query alone; a thrown search becomes one entry here. */
|
|
42
|
+
failures: string[];
|
|
43
|
+
}
|
|
44
|
+
/** What the tool returns to the model. */
|
|
45
|
+
export interface SearchManyValue {
|
|
46
|
+
results: SearchManyResult[];
|
|
47
|
+
/** True when any per-query budget was cut by the deployment's limit. */
|
|
48
|
+
truncated: boolean;
|
|
49
|
+
}
|
|
50
|
+
/** What a single query's runner reports back. */
|
|
51
|
+
export interface SearchManyOutcome {
|
|
52
|
+
sources: Source[];
|
|
53
|
+
failures: string[];
|
|
54
|
+
/** True when the deployment's configured limit cut the requested budget. */
|
|
55
|
+
truncated: boolean;
|
|
56
|
+
}
|
|
57
|
+
/** Run one search; the plugin binds this to the same path as `web_search`. */
|
|
58
|
+
export type SearchManyRunner = (query: string, limit: number, signal?: AbortSignal) => Promise<SearchManyOutcome>;
|
|
59
|
+
/** A tool definition in the host registry's native shape. */
|
|
60
|
+
export interface SearchManyDefinition {
|
|
61
|
+
name: string;
|
|
62
|
+
description: string;
|
|
63
|
+
parameters: ToolJsonSchema;
|
|
64
|
+
output: {
|
|
65
|
+
schema: ToolJsonSchema;
|
|
66
|
+
render: (args: unknown, value: SearchManyValue) => ToolContent[];
|
|
67
|
+
};
|
|
68
|
+
timeoutMs?: number;
|
|
69
|
+
isConcurrencySafe?: (args: unknown) => boolean;
|
|
70
|
+
execute: (args: unknown, exec: {
|
|
71
|
+
signal?: AbortSignal;
|
|
72
|
+
}) => Promise<SearchManyValue>;
|
|
73
|
+
}
|
|
74
|
+
/** The slice of the tool registry this module needs. */
|
|
75
|
+
export interface SearchManyRegistryLike {
|
|
76
|
+
register(definition: SearchManyDefinition): unknown;
|
|
77
|
+
get?(name: string, scope?: unknown): unknown;
|
|
78
|
+
}
|
|
79
|
+
/** Construction seams. */
|
|
80
|
+
export interface SearchManyOptions {
|
|
81
|
+
/** Registration diagnostics. */
|
|
82
|
+
warn?: (message: string) => void;
|
|
83
|
+
}
|
|
84
|
+
/** A successful registration plus the exact way to undo it. */
|
|
85
|
+
export interface SearchManyRegistration {
|
|
86
|
+
name: string;
|
|
87
|
+
dispose: () => void;
|
|
88
|
+
}
|
|
89
|
+
/** Format the model-facing result text. */
|
|
90
|
+
export declare function formatSearchManyValue(value: SearchManyValue): string;
|
|
91
|
+
/**
|
|
92
|
+
* Build the tool definition the host registry accepts.
|
|
93
|
+
* @param name - `search_many`, or the fallback when that name is taken.
|
|
94
|
+
* @param runQuery - executes one search through the plugin's normal path.
|
|
95
|
+
* @returns a registry-ready definition.
|
|
96
|
+
*/
|
|
97
|
+
export declare function createSearchManyDefinition(name: string, runQuery: SearchManyRunner): SearchManyDefinition;
|
|
98
|
+
/**
|
|
99
|
+
* Pick the tool name this deployment can accept.
|
|
100
|
+
* @param tools - the live registry.
|
|
101
|
+
* @returns the preferred or fallback name.
|
|
102
|
+
*/
|
|
103
|
+
export declare function pickSearchManyToolName(tools: SearchManyRegistryLike): string;
|
|
104
|
+
/**
|
|
105
|
+
* Register the tool when a registry is available.
|
|
106
|
+
*
|
|
107
|
+
* Every failure path is contained: a duplicate name falls back, and any
|
|
108
|
+
* registry rejection is reported and swallowed so `web_search` keeps working.
|
|
109
|
+
* The returned registration owns the host disposer, so plugin disposal removes
|
|
110
|
+
* exactly the definition this call added.
|
|
111
|
+
*
|
|
112
|
+
* @param tools - the live registry.
|
|
113
|
+
* @param runQuery - executes one search through the plugin's normal path.
|
|
114
|
+
* @param options - diagnostics.
|
|
115
|
+
* @returns the registered name and its disposer, or `undefined` when skipped.
|
|
116
|
+
*/
|
|
117
|
+
export declare function registerSearchManyTool(tools: SearchManyRegistryLike, runQuery: SearchManyRunner, options?: SearchManyOptions): SearchManyRegistration | undefined;
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
/** The name this tool prefers. */
|
|
2
|
+
export const SEARCH_MANY_TOOL_NAME = 'search_many';
|
|
3
|
+
/** The name used when another plugin already registered `search_many`. */
|
|
4
|
+
export const SEARCH_MANY_FALLBACK_NAME = 'web_search_many';
|
|
5
|
+
/** How many queries run at the same time; deliberately below the engine count. */
|
|
6
|
+
export const SEARCH_MANY_CONCURRENCY = 3;
|
|
7
|
+
/** Queries accepted in one call. */
|
|
8
|
+
export const SEARCH_MANY_MIN_QUERIES = 1;
|
|
9
|
+
/** Queries accepted in one call. */
|
|
10
|
+
export const SEARCH_MANY_MAX_QUERIES = 5;
|
|
11
|
+
/** Default result budget for a query that names none. */
|
|
12
|
+
export const SEARCH_MANY_DEFAULT_LIMIT = 5;
|
|
13
|
+
/** Largest per-query budget the tool accepts. */
|
|
14
|
+
export const SEARCH_MANY_MAX_LIMIT = 20;
|
|
15
|
+
/** True for a value usable as a record. */
|
|
16
|
+
function isRecord(value) {
|
|
17
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
18
|
+
}
|
|
19
|
+
/** The parameter schema, in the registry's raw JSON-Schema dialect. */
|
|
20
|
+
function parameterSchema() {
|
|
21
|
+
return {
|
|
22
|
+
type: 'object',
|
|
23
|
+
additionalProperties: false,
|
|
24
|
+
properties: {
|
|
25
|
+
queries: {
|
|
26
|
+
type: 'array',
|
|
27
|
+
items: { type: 'string' },
|
|
28
|
+
description: `The searches to run, ${SEARCH_MANY_MIN_QUERIES}–${SEARCH_MANY_MAX_QUERIES} independent queries. Each runs as its own \`web_search\`.`,
|
|
29
|
+
},
|
|
30
|
+
limit: {
|
|
31
|
+
type: 'integer',
|
|
32
|
+
description: `Results wanted per query, 1–${SEARCH_MANY_MAX_LIMIT}; defaults to ${SEARCH_MANY_DEFAULT_LIMIT}.`,
|
|
33
|
+
},
|
|
34
|
+
withSites: {
|
|
35
|
+
type: 'boolean',
|
|
36
|
+
description: 'Attach the result hostname to every source. Defaults to false.',
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
required: ['queries'],
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
/** The output schema, in the same dialect. */
|
|
43
|
+
function outputSchema() {
|
|
44
|
+
const sourceSchema = {
|
|
45
|
+
type: 'object',
|
|
46
|
+
additionalProperties: false,
|
|
47
|
+
properties: {
|
|
48
|
+
url: { type: 'string' },
|
|
49
|
+
title: { type: 'string' },
|
|
50
|
+
snippet: { type: 'string' },
|
|
51
|
+
publishedAt: { type: 'string' },
|
|
52
|
+
site: { type: 'string' },
|
|
53
|
+
},
|
|
54
|
+
required: ['url'],
|
|
55
|
+
};
|
|
56
|
+
return {
|
|
57
|
+
type: 'object',
|
|
58
|
+
additionalProperties: false,
|
|
59
|
+
properties: {
|
|
60
|
+
results: {
|
|
61
|
+
type: 'array',
|
|
62
|
+
items: {
|
|
63
|
+
type: 'object',
|
|
64
|
+
additionalProperties: false,
|
|
65
|
+
properties: {
|
|
66
|
+
query: { type: 'string' },
|
|
67
|
+
sources: { type: 'array', items: sourceSchema },
|
|
68
|
+
failures: { type: 'array', items: { type: 'string' } },
|
|
69
|
+
},
|
|
70
|
+
required: ['query', 'sources', 'failures'],
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
truncated: { type: 'boolean' },
|
|
74
|
+
},
|
|
75
|
+
required: ['results', 'truncated'],
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
/** Parse model arguments into a validated shape, throwing on nonsense. */
|
|
79
|
+
function parseArgs(args) {
|
|
80
|
+
if (!isRecord(args))
|
|
81
|
+
throw new Error('search_many requires arguments');
|
|
82
|
+
const rawQueries = args['queries'];
|
|
83
|
+
if (!Array.isArray(rawQueries)) {
|
|
84
|
+
throw new Error(`search_many requires a queries array with ${SEARCH_MANY_MIN_QUERIES}–${SEARCH_MANY_MAX_QUERIES} entries`);
|
|
85
|
+
}
|
|
86
|
+
if (rawQueries.length < SEARCH_MANY_MIN_QUERIES || rawQueries.length > SEARCH_MANY_MAX_QUERIES) {
|
|
87
|
+
throw new Error(`search_many accepts ${SEARCH_MANY_MIN_QUERIES}–${SEARCH_MANY_MAX_QUERIES} queries, got ${rawQueries.length}`);
|
|
88
|
+
}
|
|
89
|
+
const queries = [];
|
|
90
|
+
for (const raw of rawQueries) {
|
|
91
|
+
if (typeof raw !== 'string' || raw.trim().length === 0) {
|
|
92
|
+
throw new Error('search_many requires every query to be a non-empty string');
|
|
93
|
+
}
|
|
94
|
+
queries.push(raw.trim());
|
|
95
|
+
}
|
|
96
|
+
const rawLimit = args['limit'];
|
|
97
|
+
let limit = SEARCH_MANY_DEFAULT_LIMIT;
|
|
98
|
+
if (rawLimit !== undefined) {
|
|
99
|
+
if (typeof rawLimit !== 'number' || !Number.isInteger(rawLimit)
|
|
100
|
+
|| rawLimit < 1 || rawLimit > SEARCH_MANY_MAX_LIMIT) {
|
|
101
|
+
throw new Error(`search_many limit must be an integer between 1 and ${SEARCH_MANY_MAX_LIMIT}`);
|
|
102
|
+
}
|
|
103
|
+
limit = rawLimit;
|
|
104
|
+
}
|
|
105
|
+
const rawSites = args['withSites'];
|
|
106
|
+
if (rawSites !== undefined && typeof rawSites !== 'boolean') {
|
|
107
|
+
throw new Error('search_many withSites must be a boolean');
|
|
108
|
+
}
|
|
109
|
+
return { queries, limit, withSites: rawSites === true };
|
|
110
|
+
}
|
|
111
|
+
/** Return the hostname a source is served from, or `undefined` for a bad URL. */
|
|
112
|
+
function siteOf(url) {
|
|
113
|
+
try {
|
|
114
|
+
const host = new URL(url).hostname.trim().toLowerCase();
|
|
115
|
+
return host.length === 0 ? undefined : host;
|
|
116
|
+
}
|
|
117
|
+
catch {
|
|
118
|
+
return undefined;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/** Attach the hostname without mutating the cached source object. */
|
|
122
|
+
function withSite(source) {
|
|
123
|
+
const site = siteOf(source.url);
|
|
124
|
+
return site === undefined ? { ...source } : { ...source, site };
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Run `worker` over items with at most `limit` in flight, preserving order.
|
|
128
|
+
* A dedicated helper keeps the concurrency contract visible instead of relying
|
|
129
|
+
* on an unstated promise-pool behaviour.
|
|
130
|
+
*/
|
|
131
|
+
async function mapLimit(items, limit, worker) {
|
|
132
|
+
const results = new Array(items.length);
|
|
133
|
+
let next = 0;
|
|
134
|
+
const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
|
|
135
|
+
for (;;) {
|
|
136
|
+
const index = next;
|
|
137
|
+
next += 1;
|
|
138
|
+
if (index >= items.length)
|
|
139
|
+
return;
|
|
140
|
+
results[index] = await worker(items[index], index);
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
await Promise.all(runners);
|
|
144
|
+
return results;
|
|
145
|
+
}
|
|
146
|
+
/** Format the model-facing result text. */
|
|
147
|
+
export function formatSearchManyValue(value) {
|
|
148
|
+
const total = value.results.reduce((sum, result) => sum + result.sources.length, 0);
|
|
149
|
+
const failed = value.results.reduce((sum, result) => sum + result.failures.length, 0);
|
|
150
|
+
const head = `Batch search: ${value.results.length} quer${value.results.length === 1 ? 'y' : 'ies'}, `
|
|
151
|
+
+ `${total} result(s)${failed === 0 ? '' : `, ${failed} failure(s)`}${value.truncated ? ' (truncated)' : ''}.`;
|
|
152
|
+
const sections = value.results.map((result, index) => {
|
|
153
|
+
const lines = [`${index + 1}. ${JSON.stringify(result.query)} — ${result.sources.length} result(s)`];
|
|
154
|
+
for (const [position, source] of result.sources.entries()) {
|
|
155
|
+
const title = source.title !== undefined && source.title.length > 0 ? source.title : source.url;
|
|
156
|
+
lines.push(` ${position + 1}. ${title}`);
|
|
157
|
+
lines.push(` ${source.url}${source.site === undefined ? '' : ` [${source.site}]`}`);
|
|
158
|
+
if (source.snippet !== undefined && source.snippet.length > 0)
|
|
159
|
+
lines.push(` ${source.snippet}`);
|
|
160
|
+
if (source.publishedAt !== undefined && source.publishedAt.length > 0) {
|
|
161
|
+
lines.push(` ${source.publishedAt}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
if (result.failures.length > 0)
|
|
165
|
+
lines.push(` failures: ${result.failures.join('; ')}`);
|
|
166
|
+
return lines.join('\n');
|
|
167
|
+
});
|
|
168
|
+
return [head, '', ...sections].join('\n').trimEnd();
|
|
169
|
+
}
|
|
170
|
+
/**
|
|
171
|
+
* Build the tool definition the host registry accepts.
|
|
172
|
+
* @param name - `search_many`, or the fallback when that name is taken.
|
|
173
|
+
* @param runQuery - executes one search through the plugin's normal path.
|
|
174
|
+
* @returns a registry-ready definition.
|
|
175
|
+
*/
|
|
176
|
+
export function createSearchManyDefinition(name, runQuery) {
|
|
177
|
+
return {
|
|
178
|
+
name,
|
|
179
|
+
description: 'Run 1–5 independent web searches in one call and return each query\'s results separately. Use it when one question needs several distinct queries; a failed query does not stop the others. For a single query use web_search.',
|
|
180
|
+
parameters: parameterSchema(),
|
|
181
|
+
output: {
|
|
182
|
+
schema: outputSchema(),
|
|
183
|
+
render: (_args, value) => [{ type: 'text', text: formatSearchManyValue(value) }],
|
|
184
|
+
},
|
|
185
|
+
timeoutMs: 30_000,
|
|
186
|
+
isConcurrencySafe: () => true,
|
|
187
|
+
async execute(args, exec) {
|
|
188
|
+
const parsed = parseArgs(args);
|
|
189
|
+
const outcomes = await mapLimit(parsed.queries, SEARCH_MANY_CONCURRENCY, async (query) => {
|
|
190
|
+
try {
|
|
191
|
+
const outcome = await runQuery(query, parsed.limit, exec.signal);
|
|
192
|
+
const mapped = parsed.withSites
|
|
193
|
+
? outcome.sources.map(withSite)
|
|
194
|
+
: outcome.sources.map((source) => ({ ...source }));
|
|
195
|
+
const truncated = outcome.truncated || mapped.length > parsed.limit;
|
|
196
|
+
return {
|
|
197
|
+
result: {
|
|
198
|
+
query,
|
|
199
|
+
sources: mapped.slice(0, parsed.limit),
|
|
200
|
+
failures: [...outcome.failures],
|
|
201
|
+
},
|
|
202
|
+
truncated,
|
|
203
|
+
};
|
|
204
|
+
}
|
|
205
|
+
catch (error) {
|
|
206
|
+
return {
|
|
207
|
+
result: {
|
|
208
|
+
query,
|
|
209
|
+
sources: [],
|
|
210
|
+
failures: [error instanceof Error ? error.message : String(error)],
|
|
211
|
+
},
|
|
212
|
+
truncated: false,
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
});
|
|
216
|
+
return {
|
|
217
|
+
results: outcomes.map((entry) => entry.result),
|
|
218
|
+
truncated: outcomes.some((entry) => entry.truncated),
|
|
219
|
+
};
|
|
220
|
+
},
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Pick the tool name this deployment can accept.
|
|
225
|
+
* @param tools - the live registry.
|
|
226
|
+
* @returns the preferred or fallback name.
|
|
227
|
+
*/
|
|
228
|
+
export function pickSearchManyToolName(tools) {
|
|
229
|
+
if (typeof tools.get !== 'function')
|
|
230
|
+
return SEARCH_MANY_TOOL_NAME;
|
|
231
|
+
try {
|
|
232
|
+
return tools.get(SEARCH_MANY_TOOL_NAME) === undefined ? SEARCH_MANY_TOOL_NAME : SEARCH_MANY_FALLBACK_NAME;
|
|
233
|
+
}
|
|
234
|
+
catch {
|
|
235
|
+
return SEARCH_MANY_TOOL_NAME;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/** Report a registration failure without throwing out of the plugin. */
|
|
239
|
+
function report(options, error) {
|
|
240
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
241
|
+
try {
|
|
242
|
+
options.warn?.(`search_many registration skipped: ${message}`);
|
|
243
|
+
}
|
|
244
|
+
catch {
|
|
245
|
+
// A diagnostic callback must never break plugin loading.
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
/**
|
|
249
|
+
* Register the tool when a registry is available.
|
|
250
|
+
*
|
|
251
|
+
* Every failure path is contained: a duplicate name falls back, and any
|
|
252
|
+
* registry rejection is reported and swallowed so `web_search` keeps working.
|
|
253
|
+
* The returned registration owns the host disposer, so plugin disposal removes
|
|
254
|
+
* exactly the definition this call added.
|
|
255
|
+
*
|
|
256
|
+
* @param tools - the live registry.
|
|
257
|
+
* @param runQuery - executes one search through the plugin's normal path.
|
|
258
|
+
* @param options - diagnostics.
|
|
259
|
+
* @returns the registered name and its disposer, or `undefined` when skipped.
|
|
260
|
+
*/
|
|
261
|
+
export function registerSearchManyTool(tools, runQuery, options = {}) {
|
|
262
|
+
const name = pickSearchManyToolName(tools);
|
|
263
|
+
try {
|
|
264
|
+
const registered = tools.register(createSearchManyDefinition(name, runQuery));
|
|
265
|
+
const disposeHost = typeof registered === 'function' ? registered : undefined;
|
|
266
|
+
let disposed = false;
|
|
267
|
+
return {
|
|
268
|
+
name,
|
|
269
|
+
dispose: () => {
|
|
270
|
+
if (disposed)
|
|
271
|
+
return;
|
|
272
|
+
disposed = true;
|
|
273
|
+
try {
|
|
274
|
+
disposeHost?.();
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
// Disposal must stay contained; the plugin is already going away.
|
|
278
|
+
}
|
|
279
|
+
},
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
catch (error) {
|
|
283
|
+
report(options, error);
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
}
|
package/lib/transport.d.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
*
|
|
9
9
|
* @module @fangchenghao/dsh-web-search/transport
|
|
10
10
|
*/
|
|
11
|
-
import type { HostSpec, Source } from './types.ts';
|
|
11
|
+
import type { HostSpec, Source, TimeRange } from './types.ts';
|
|
12
12
|
import { SearchError } from './errors.ts';
|
|
13
13
|
/** Outcome of asking one engine once. */
|
|
14
14
|
export type AttemptOutcome = {
|
|
@@ -31,6 +31,8 @@ export type AttemptOutcome = {
|
|
|
31
31
|
* @param signal - cancellation.
|
|
32
32
|
* @param timeoutMs - per-attempt deadline.
|
|
33
33
|
* @param proxy - tunnel URL, or `undefined` when none is available.
|
|
34
|
+
* @param timeRange - active publication window, forwarded to engines that can
|
|
35
|
+
* push it into their query (currently DuckDuckGo's `df`).
|
|
34
36
|
* @returns the outcome of the route that got furthest; cancellation is rethrown.
|
|
35
37
|
*/
|
|
36
|
-
export declare function searchHost(host: HostSpec, query: string, signal: AbortSignal | undefined, timeoutMs?: number, proxy?: string): Promise<AttemptOutcome>;
|
|
38
|
+
export declare function searchHost(host: HostSpec, query: string, signal: AbortSignal | undefined, timeoutMs?: number, proxy?: string, timeRange?: TimeRange): Promise<AttemptOutcome>;
|
package/lib/transport.js
CHANGED
|
@@ -123,10 +123,12 @@ async function attempt(host, url, signal, timeoutMs, proxy, query) {
|
|
|
123
123
|
* @param signal - cancellation.
|
|
124
124
|
* @param timeoutMs - per-attempt deadline.
|
|
125
125
|
* @param proxy - tunnel URL, or `undefined` when none is available.
|
|
126
|
+
* @param timeRange - active publication window, forwarded to engines that can
|
|
127
|
+
* push it into their query (currently DuckDuckGo's `df`).
|
|
126
128
|
* @returns the outcome of the route that got furthest; cancellation is rethrown.
|
|
127
129
|
*/
|
|
128
|
-
export async function searchHost(host, query, signal, timeoutMs = 15000, proxy) {
|
|
129
|
-
const url = host.build(query);
|
|
130
|
+
export async function searchHost(host, query, signal, timeoutMs = 15000, proxy, timeRange) {
|
|
131
|
+
const url = host.build(query, timeRange);
|
|
130
132
|
const route = host.viaProxy === true ? proxy : undefined;
|
|
131
133
|
const first = await attempt(host, url, signal, timeoutMs, route, query);
|
|
132
134
|
if (first.ok || route !== undefined || proxy === undefined || signal?.aborted === true)
|