@fffangchenghao/dsh-web-search 0.19.3 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +173 -1
- package/README.md +214 -34
- package/README.zh-CN.md +187 -35
- package/SECURITY.md +42 -7
- package/client.js +248 -10
- package/docs/store-evidence.md +10 -5
- package/lib/advanced-search.d.ts +107 -0
- package/lib/advanced-search.js +255 -0
- package/lib/bridge.d.ts +34 -0
- package/lib/bridge.js +305 -0
- package/lib/cache.d.ts +11 -4
- package/lib/cache.js +9 -5
- package/lib/config.d.ts +19 -3
- package/lib/config.js +24 -0
- package/lib/domain-tool.d.ts +5 -0
- package/lib/domain-tool.js +45 -6
- package/lib/engine-test.d.ts +116 -0
- package/lib/engine-test.js +256 -0
- package/lib/engines.d.ts +11 -1
- package/lib/engines.js +118 -19
- package/lib/errors.d.ts +16 -2
- package/lib/errors.js +40 -1
- package/lib/index.d.ts +24 -2
- package/lib/index.js +309 -21
- package/lib/inject.d.ts +68 -0
- package/lib/inject.js +1 -0
- package/lib/markdown.js +18 -1
- package/lib/mirrors.d.ts +23 -0
- package/lib/mirrors.js +69 -0
- package/lib/official-feed.d.ts +114 -0
- package/lib/official-feed.js +288 -0
- package/lib/official-support.d.ts +74 -0
- package/lib/official-support.js +134 -0
- package/lib/platform-tool.d.ts +77 -0
- package/lib/platform-tool.js +173 -0
- package/lib/platforms.d.ts +178 -0
- package/lib/platforms.js +1035 -0
- package/lib/query-router.d.ts +29 -0
- package/lib/query-router.js +58 -0
- package/lib/read-page.js +2 -1
- package/lib/research-cache.d.ts +115 -0
- package/lib/research-cache.js +275 -0
- package/lib/research.d.ts +45 -4
- package/lib/research.js +505 -22
- package/lib/search-many.d.ts +7 -1
- package/lib/search-many.js +7 -2
- package/lib/system-prompt.d.ts +27 -0
- package/lib/system-prompt.js +50 -0
- package/lib/tool-suite.d.ts +97 -0
- package/lib/tool-suite.js +224 -0
- package/lib/transport.d.ts +33 -2
- package/lib/transport.js +42 -9
- package/lib/types.d.ts +20 -0
- package/lib/types.js +9 -1
- package/lib/untrusted.d.ts +33 -0
- package/lib/untrusted.js +42 -0
- package/lib/verticals.d.ts +3 -1
- package/lib/verticals.js +26 -7
- package/package.json +5 -3
- package/skills/dsh-web-search/SKILL.md +21 -6
package/lib/research.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { wrapUntrustedContent } from "./untrusted.js";
|
|
2
|
+
import { canonicalOfficialUrl } from "./mirrors.js";
|
|
1
3
|
/** The name this tool prefers. */
|
|
2
4
|
export const RESEARCH_TOOL_NAME = 'research';
|
|
3
5
|
/** The name used when another plugin already registered `research`. */
|
|
@@ -11,7 +13,9 @@ export const RESEARCH_MAX_SOURCES = 5;
|
|
|
11
13
|
/** Distinct queries accepted in one call. */
|
|
12
14
|
export const RESEARCH_MAX_QUERIES = 3;
|
|
13
15
|
/** Characters kept from each fetched page. */
|
|
14
|
-
export const RESEARCH_EXCERPT_CHARS =
|
|
16
|
+
export const RESEARCH_EXCERPT_CHARS = 1_800;
|
|
17
|
+
/** Below this much readable text a page is boilerplate, not evidence. */
|
|
18
|
+
export const RESEARCH_MIN_TEXT_CHARS = 160;
|
|
15
19
|
/** Pages fetched at the same time. */
|
|
16
20
|
export const RESEARCH_PAGE_CONCURRENCY = 3;
|
|
17
21
|
/** Wall-clock budget for the whole call, below the tool's host timeout. */
|
|
@@ -40,16 +44,199 @@ export function canonicalKey(raw) {
|
|
|
40
44
|
return undefined;
|
|
41
45
|
}
|
|
42
46
|
}
|
|
43
|
-
/**
|
|
44
|
-
function
|
|
45
|
-
const
|
|
47
|
+
/** True when a fetch surfaced a raw HTML document instead of readable text. */
|
|
48
|
+
function looksLikeHtmlDocument(text) {
|
|
49
|
+
const head = text.trimStart().slice(0, 200).toLowerCase();
|
|
50
|
+
return head.startsWith('<!doctype html') || head.startsWith('<html');
|
|
51
|
+
}
|
|
52
|
+
/**
|
|
53
|
+
* Recognise a page that is a transport/interstitial wall rather than content.
|
|
54
|
+
*
|
|
55
|
+
* A bot challenge can be longer than {@link RESEARCH_MIN_TEXT_CHARS}, so the
|
|
56
|
+
* length check alone is not enough. The signatures below are deliberately
|
|
57
|
+
* specific: an article *about* Cloudflare or Anubis must still be usable.
|
|
58
|
+
* @param text - fetched page text.
|
|
59
|
+
* @param status - final HTTP status.
|
|
60
|
+
* @returns a failure reason, or `undefined` for an ordinary page.
|
|
61
|
+
*/
|
|
62
|
+
function interstitialReason(text, status) {
|
|
63
|
+
if (status >= 300 && status < 400)
|
|
64
|
+
return 'the page returned a redirect instead of content';
|
|
65
|
+
const head = text.trimStart().slice(0, 4_000).toLowerCase();
|
|
66
|
+
const anubis = head.includes("making sure you're not a bot") &&
|
|
67
|
+
(head.includes('anubis') || head.includes('proof-of-work') || head.includes('loading...'));
|
|
68
|
+
const cloudflare = (head.includes('just a moment') || head.includes('checking your browser')) &&
|
|
69
|
+
(head.includes('cloudflare') || head.includes('enable javascript'));
|
|
70
|
+
const humanCheck = head.includes('verify you are human') &&
|
|
71
|
+
(head.includes('cloudflare') || head.includes('press & hold'));
|
|
72
|
+
if (anubis || cloudflare || humanCheck)
|
|
73
|
+
return 'the page returned a bot challenge, not readable content';
|
|
74
|
+
const redirect = /^redirecting to\s*<?https?:/iu.test(head) &&
|
|
75
|
+
/(?:login|signin|sign-in|auth|accounts\.)/iu.test(head.slice(0, 1_000));
|
|
76
|
+
if (redirect)
|
|
77
|
+
return 'the page redirected to a login or authentication page';
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
/** Collapse page text without destroying paragraph boundaries. */
|
|
81
|
+
function flattenPage(text) {
|
|
82
|
+
return text
|
|
46
83
|
.replace(/\r/g, '')
|
|
47
84
|
.replace(/[ \t]+/g, ' ')
|
|
48
85
|
.replace(/\n{3,}/g, '\n\n')
|
|
49
86
|
.trim();
|
|
87
|
+
}
|
|
88
|
+
/** Split one long block into sentence-sized windows of at most `max` characters. */
|
|
89
|
+
function splitLongBlock(block, max) {
|
|
90
|
+
if (block.length <= max)
|
|
91
|
+
return [block];
|
|
92
|
+
const sentences = block.split(/(?<=[.!?。!?])\s+/u);
|
|
93
|
+
const windows = [];
|
|
94
|
+
let current = '';
|
|
95
|
+
for (const sentence of sentences) {
|
|
96
|
+
if (sentence.length > max) {
|
|
97
|
+
if (current.length > 0) {
|
|
98
|
+
windows.push(current);
|
|
99
|
+
current = '';
|
|
100
|
+
}
|
|
101
|
+
for (let offset = 0; offset < sentence.length; offset += max)
|
|
102
|
+
windows.push(sentence.slice(offset, offset + max));
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
if (current.length > 0 && current.length + 1 + sentence.length > max) {
|
|
106
|
+
windows.push(current);
|
|
107
|
+
current = sentence;
|
|
108
|
+
}
|
|
109
|
+
else {
|
|
110
|
+
current = current.length === 0 ? sentence : `${current} ${sentence}`;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (current.length > 0)
|
|
114
|
+
windows.push(current);
|
|
115
|
+
return windows;
|
|
116
|
+
}
|
|
117
|
+
/** Split a page into paragraph-sized blocks and bounded windows. */
|
|
118
|
+
function pageWindows(text, window) {
|
|
119
|
+
const blocks = flattenPage(text).split(/\n{2,}/u).map((block) => block.trim()).filter((block) => block.length > 0);
|
|
120
|
+
return blocks.flatMap((block) => splitLongBlock(block, window));
|
|
121
|
+
}
|
|
122
|
+
/** Count query-term hits in a window, favouring distinct terms over repetition. */
|
|
123
|
+
function windowScore(window, tokens) {
|
|
124
|
+
if (tokens.length === 0)
|
|
125
|
+
return 0;
|
|
126
|
+
const haystack = window.toLowerCase();
|
|
127
|
+
let score = 0;
|
|
128
|
+
for (const token of tokens) {
|
|
129
|
+
const first = haystack.indexOf(token);
|
|
130
|
+
if (first < 0)
|
|
131
|
+
continue;
|
|
132
|
+
score += 1;
|
|
133
|
+
let next = haystack.indexOf(token, first + token.length);
|
|
134
|
+
while (next >= 0) {
|
|
135
|
+
score += 0.2;
|
|
136
|
+
next = haystack.indexOf(token, next + token.length);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return score;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Select the passages of a fetched page that actually answer the research
|
|
143
|
+
* question, instead of always returning the first N characters.
|
|
144
|
+
*
|
|
145
|
+
* The search result's snippet is a summary, not the page: the fact can be
|
|
146
|
+
* 20 KB down (a Kubernetes field, an RFC section). This second pass scores
|
|
147
|
+
* paragraph windows against the same query tokens and keeps the best ones in
|
|
148
|
+
* reading order. It is deterministic and lexical; no model call and no extra
|
|
149
|
+
* network request. When nothing matches, the opening is still returned, so a
|
|
150
|
+
* weakly matched page is not silently turned into an empty excerpt.
|
|
151
|
+
*
|
|
152
|
+
* @param text - full page text from the fetch layer.
|
|
153
|
+
* @param tokens - query tokens from all research queries.
|
|
154
|
+
* @param max - excerpt budget in characters.
|
|
155
|
+
* @returns a bounded excerpt, with best passages first preserved in page order.
|
|
156
|
+
*/
|
|
157
|
+
export function selectExcerpt(text, tokens, max) {
|
|
158
|
+
const flattened = flattenPage(text);
|
|
50
159
|
if (flattened.length <= max)
|
|
51
160
|
return flattened;
|
|
52
|
-
|
|
161
|
+
const windowSize = Math.min(1_200, Math.max(400, Math.floor(max / 3)));
|
|
162
|
+
const windows = pageWindows(flattened, windowSize);
|
|
163
|
+
const scored = windows.map((window, index) => ({ window, index, score: windowScore(window, tokens) }));
|
|
164
|
+
const matches = scored.filter((entry) => entry.score > 0).sort((a, b) => b.score - a.score || a.index - b.index);
|
|
165
|
+
if (matches.length === 0)
|
|
166
|
+
return `${flattened.slice(0, max - 1).trimEnd()}…`;
|
|
167
|
+
const picked = new Set();
|
|
168
|
+
let used = 0;
|
|
169
|
+
const add = (index) => {
|
|
170
|
+
const window = windows[index];
|
|
171
|
+
if (window === undefined || picked.has(index))
|
|
172
|
+
return;
|
|
173
|
+
picked.add(index);
|
|
174
|
+
used += window.length + 2;
|
|
175
|
+
};
|
|
176
|
+
// Regulations often put the effective date in the final article, after the
|
|
177
|
+
// definitions. A pure relevance sort can spend the budget on the definitions
|
|
178
|
+
// and leave the date out, so preserve the last window when the question is
|
|
179
|
+
// explicitly about when a rule takes effect. This is a lexical safeguard, not
|
|
180
|
+
// a claim about where every document puts its date.
|
|
181
|
+
const lastIndex = windows.length - 1;
|
|
182
|
+
const lastWindow = lastIndex > 0 ? windows[lastIndex] ?? '' : '';
|
|
183
|
+
const asksEffectiveDate = tokens.some((token) => /施行|生效|实施|适用|生效|effective|in force|take effect|apply|applies|applicable|start(?:s|ed)? on/i.test(token));
|
|
184
|
+
const tailHasDate = /(?:自\s*)?(?:19|20)\d{2}\s*年|\b(?:19|20)\d{2}-\d{2}-\d{2}\b|effective\s+(?:on|from)|施行|生效/i.test(lastWindow);
|
|
185
|
+
const tailNeeded = lastIndex > 0 && asksEffectiveDate && tailHasDate;
|
|
186
|
+
if (tailNeeded)
|
|
187
|
+
add(lastIndex);
|
|
188
|
+
// A legal page often starts with a full table of contents that repeats
|
|
189
|
+
// "Article 113" and then puts the operative sentence (including the date) at
|
|
190
|
+
// the bottom. The tail safeguard above only helps when the final window has
|
|
191
|
+
// the date; this body-window safeguard targets the sentence itself.
|
|
192
|
+
if (asksEffectiveDate) {
|
|
193
|
+
const legalWindows = windows
|
|
194
|
+
.map((window, index) => {
|
|
195
|
+
const hasDate = /(?:19|20)\d{2}/u.test(window);
|
|
196
|
+
const operative = /\b(?:shall apply|apply from|appl(?:ies|icable)|entry into force|takes? effect|适用|生效)\b/iu.test(window);
|
|
197
|
+
const score = hasDate && operative
|
|
198
|
+
? tokens.reduce((sum, token) => (window.toLowerCase().includes(token) ? sum + 1 : sum), 0)
|
|
199
|
+
: -1;
|
|
200
|
+
return { index, score };
|
|
201
|
+
})
|
|
202
|
+
.filter((entry) => entry.score >= 0)
|
|
203
|
+
.sort((left, right) => right.score - left.score || left.index - right.index);
|
|
204
|
+
// A legal article can contain several dates (entry into force, application
|
|
205
|
+
// dates, enforcement dates). Include up to three operative windows so the
|
|
206
|
+
// model sees the one the question names instead of only the last date on
|
|
207
|
+
// the page; the query-specific token score still puts the target first.
|
|
208
|
+
for (const entry of legalWindows.slice(0, 3))
|
|
209
|
+
add(entry.index);
|
|
210
|
+
}
|
|
211
|
+
// Always leave room for the opening when the answer is deep in the page:
|
|
212
|
+
// the reader needs enough context to know what the excerpt is about.
|
|
213
|
+
const opening = windows[0];
|
|
214
|
+
if (opening !== undefined)
|
|
215
|
+
add(0);
|
|
216
|
+
for (const match of matches) {
|
|
217
|
+
if (picked.has(match.index))
|
|
218
|
+
continue;
|
|
219
|
+
const addition = match.window.length + 2;
|
|
220
|
+
if (used + addition > max)
|
|
221
|
+
continue;
|
|
222
|
+
add(match.index);
|
|
223
|
+
if (used >= max - 80)
|
|
224
|
+
break;
|
|
225
|
+
}
|
|
226
|
+
if (picked.size === 1 && used < max) {
|
|
227
|
+
// A single opening paragraph plus no matching window fits: append the best
|
|
228
|
+
// match even if it overflows slightly, because the opening alone is not
|
|
229
|
+
// evidence for the question.
|
|
230
|
+
const best = matches[0];
|
|
231
|
+
if (best !== undefined && !picked.has(best.index))
|
|
232
|
+
add(best.index);
|
|
233
|
+
}
|
|
234
|
+
const selected = [...picked].sort((a, b) => a - b).map((index) => windows[index] ?? '');
|
|
235
|
+
const excerpt = selected.join('\n\n…\n\n');
|
|
236
|
+
// Do not truncate from the end here: when tailNeeded is true, the end is the
|
|
237
|
+
// effective date the caller asked for. The selected windows are already
|
|
238
|
+
// bounded to roughly max plus one window.
|
|
239
|
+
return excerpt;
|
|
53
240
|
}
|
|
54
241
|
/** Walk `worker` over items with at most `limit` in flight, preserving order. */
|
|
55
242
|
async function mapLimit(items, limit, worker) {
|
|
@@ -104,15 +291,199 @@ function queryTokens(queries) {
|
|
|
104
291
|
tokens.add(bigram);
|
|
105
292
|
}
|
|
106
293
|
}
|
|
294
|
+
// Legal questions often carry the decisive number (`Article 113`, `第 113
|
|
295
|
+
// 条`); ASCII tokenisation above deliberately starts with a letter, so keep
|
|
296
|
+
// standalone numbers as well. They also help select the body clause rather
|
|
297
|
+
// than the navigation link that repeats the same article number.
|
|
298
|
+
for (const number of text.match(/\b\d{1,4}[a-z]?\b/gu) ?? [])
|
|
299
|
+
tokens.add(number);
|
|
107
300
|
return [...tokens];
|
|
108
301
|
}
|
|
109
302
|
/** Count how many query tokens appear in a row's title and snippet. */
|
|
110
303
|
function relevanceScore(source, tokens) {
|
|
111
304
|
if (tokens.length === 0)
|
|
112
305
|
return 0;
|
|
113
|
-
|
|
306
|
+
// URL tokens matter for official pages whose title is generic ("Stable
|
|
307
|
+
// Channel Update for Desktop") but whose path names the product.
|
|
308
|
+
const haystack = `${source.title ?? ''} ${source.snippet ?? ''} ${source.url}`.toLowerCase();
|
|
114
309
|
return tokens.reduce((score, token) => (haystack.includes(token) ? score + 1 : score), 0);
|
|
115
310
|
}
|
|
311
|
+
/**
|
|
312
|
+
* Hosts whose own documentation is a primary source for the subject.
|
|
313
|
+
*
|
|
314
|
+
* These are deliberately structural signals (government/academic TLDs and
|
|
315
|
+
* well-known documentation hosts), not a query-specific allowlist. A credible
|
|
316
|
+
* source still has to match the query; among matches it leads the pack.
|
|
317
|
+
*/
|
|
318
|
+
const OFFICIAL_SOURCE_HOSTS = [
|
|
319
|
+
// International and standards bodies.
|
|
320
|
+
['europa.eu', 5], ['un.org', 5], ['who.int', 5], ['worldbank.org', 4],
|
|
321
|
+
['rfc-editor.org', 5], ['ietf.org', 5], ['w3.org', 5], ['whatwg.org', 5],
|
|
322
|
+
['unicode.org', 5], ['iso.org', 5], ['iec.ch', 5],
|
|
323
|
+
// Project and platform documentation.
|
|
324
|
+
['python.org', 4], ['nodejs.org', 4], ['postgresql.org', 4], ['rust-lang.org', 4],
|
|
325
|
+
['go.dev', 4], ['kubernetes.io', 4], ['react.dev', 4], ['pandas.pydata.org', 4],
|
|
326
|
+
['pytorch.org', 4], ['tensorflow.org', 4], ['developer.mozilla.org', 4],
|
|
327
|
+
['docs.github.com', 4], ['chromereleases.googleblog.com', 5], ['learn.microsoft.com', 4], ['docs.aws.amazon.com', 4],
|
|
328
|
+
['developers.cloudflare.com', 4], ['developer.apple.com', 4], ['apache.org', 4],
|
|
329
|
+
['gnu.org', 4], ['opensource.org', 3],
|
|
330
|
+
// Registries, repositories and reference works.
|
|
331
|
+
['github.com', 3], ['gitlab.com', 2], ['arxiv.org', 3], ['huggingface.co', 3],
|
|
332
|
+
['npmjs.com', 3], ['pypi.org', 3], ['crates.io', 3], ['wikipedia.org', 2],
|
|
333
|
+
// Q&A and community references.
|
|
334
|
+
['stackoverflow.com', 1], ['news.ycombinator.com', 1],
|
|
335
|
+
];
|
|
336
|
+
/** Match a host or one of its parent domains without matching a lookalike. */
|
|
337
|
+
function hostMatches(host, suffix) {
|
|
338
|
+
return host === suffix || host.endsWith(`.${suffix}`);
|
|
339
|
+
}
|
|
340
|
+
/** A path that marks documentation, a licence, a release or a legal page. */
|
|
341
|
+
const AUTHORITY_PATH = /(?:^|\/)(?:docs?|documentation|manual|reference|api|guide|learn|legal|licen[cs]es?|pricing|security|releases?)(?:\/|$)/iu;
|
|
342
|
+
/**
|
|
343
|
+
* Score a URL's source authority from 0 (generic page) to 5 (government,
|
|
344
|
+
* academic or standards body). Credible sources lead only after they pass the
|
|
345
|
+
* relevance gate, so an unrelated official page is not fetched as evidence.
|
|
346
|
+
* @param raw - candidate URL from any engine.
|
|
347
|
+
* @returns the authority score; malformed and non-HTTP(S) URLs score 0.
|
|
348
|
+
*/
|
|
349
|
+
export function authorityScore(raw) {
|
|
350
|
+
let url;
|
|
351
|
+
try {
|
|
352
|
+
url = new URL(raw);
|
|
353
|
+
}
|
|
354
|
+
catch {
|
|
355
|
+
return 0;
|
|
356
|
+
}
|
|
357
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
358
|
+
return 0;
|
|
359
|
+
const host = url.hostname.toLowerCase().replace(/^www\./u, '');
|
|
360
|
+
if (host.length === 0)
|
|
361
|
+
return 0;
|
|
362
|
+
let score = 0;
|
|
363
|
+
if (/(?:^|\.)(?:gov|mil|edu|ac)(?:\.[a-z]{2})?$/u.test(host))
|
|
364
|
+
score = 5;
|
|
365
|
+
else if (hostMatches(host, 'int'))
|
|
366
|
+
score = 5;
|
|
367
|
+
else {
|
|
368
|
+
for (const [suffix, candidate] of OFFICIAL_SOURCE_HOSTS) {
|
|
369
|
+
if (hostMatches(host, suffix)) {
|
|
370
|
+
score = candidate;
|
|
371
|
+
break;
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
if (AUTHORITY_PATH.test(url.pathname))
|
|
376
|
+
score += 1;
|
|
377
|
+
return Math.min(5, score);
|
|
378
|
+
}
|
|
379
|
+
/** Authorities that should lead a research pack when they match the query. */
|
|
380
|
+
const CREDIBLE_AUTHORITY = 3;
|
|
381
|
+
/** Root/landing pages are technically official but rarely answer a question. */
|
|
382
|
+
function isLandingPage(raw) {
|
|
383
|
+
try {
|
|
384
|
+
const url = new URL(raw);
|
|
385
|
+
const path = url.pathname.replace(/\/+$/u, '') || '/';
|
|
386
|
+
if (path === '/')
|
|
387
|
+
return true;
|
|
388
|
+
return /^\/(?:index|home|default)(?:[._-][a-z0-9-]+)?(?:\.[a-z0-9]+)?$/iu.test(path)
|
|
389
|
+
|| /(?:^|&)home(?:=|&|$)/iu.test(url.search);
|
|
390
|
+
}
|
|
391
|
+
catch {
|
|
392
|
+
return false;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Query rewriting for the common "official answer" case.
|
|
397
|
+
*
|
|
398
|
+
* The engine sometimes returns only aggregators for a long Chinese question
|
|
399
|
+
* even though the issuing body's site has the answer. Adding one `site:` query
|
|
400
|
+
* is a bounded, transparent fix: it spends at most the unused query slots, it
|
|
401
|
+
* is skipped when the user already supplied `site:`, and it is skipped when the
|
|
402
|
+
* candidate set already contains that host. It is not a general query planner.
|
|
403
|
+
*/
|
|
404
|
+
function officialHints(queries) {
|
|
405
|
+
const text = queries.join(' ');
|
|
406
|
+
if (/\bsite:/iu.test(text))
|
|
407
|
+
return [];
|
|
408
|
+
const hints = [];
|
|
409
|
+
if (/\bAI\s*Act\b|\bGPAI\b|欧盟|european\s+union|eur-?lex|council\s+of\s+the\s+eu/iu.test(text)) {
|
|
410
|
+
const wantsClause = /哪一条|第\s*\d+\s*条|从哪一天|开始适用|生效|entry\s+into\s+force|effective|application\s+date/iu.test(text);
|
|
411
|
+
hints.push({
|
|
412
|
+
host: 'europa.eu',
|
|
413
|
+
site: 'site:europa.eu',
|
|
414
|
+
terms: 'AI Act GPAI obligations timeline 2025 Article 113',
|
|
415
|
+
...(wantsClause
|
|
416
|
+
? {
|
|
417
|
+
directUrl: 'https://ai-act-service-desk.ec.europa.eu/en/ai-act/article-113',
|
|
418
|
+
directTitle: 'AI Act Article 113: Entry into force and application',
|
|
419
|
+
directSnippet: 'Article 113 sets the application dates, including the 2 August 2025 date for GPAI obligations.',
|
|
420
|
+
}
|
|
421
|
+
: {}),
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
if (/中国|国务院|网信办|生成式人工智能服务管理暂行办法|\bcac\b/iu.test(text)) {
|
|
425
|
+
hints.push({ host: 'cac.gov.cn', site: 'site:cac.gov.cn', terms: '生成式人工智能服务管理暂行办法 训练数据 透明度' });
|
|
426
|
+
}
|
|
427
|
+
if (/\bchrome(?:\s+releases)?\b|\bchromium\b/iu.test(text)) {
|
|
428
|
+
hints.push({ host: 'chromereleases.googleblog.com', site: 'site:chromereleases.googleblog.com', terms: 'Chrome zero-day security' });
|
|
429
|
+
}
|
|
430
|
+
if (/\bpython\b|\bpep\s*703\b|free[\s-]threading/iu.test(text)) {
|
|
431
|
+
hints.push({ host: 'python.org', site: 'site:python.org', terms: 'Python free-threading PEP 703' });
|
|
432
|
+
}
|
|
433
|
+
if (/\bredis\b/iu.test(text)) {
|
|
434
|
+
hints.push({ host: 'redis.io', site: 'site:redis.io', terms: 'Redis 7.4 RDB persistence' });
|
|
435
|
+
}
|
|
436
|
+
if (/\bwcag\b|\bw3c\b|目标大小|target\s+size/iu.test(text)) {
|
|
437
|
+
const terms = /目标大小|target\s+size/iu.test(text)
|
|
438
|
+
? 'WCAG 2.2 target size minimum 24 CSS pixels'
|
|
439
|
+
: 'WCAG 2.2 success criteria official guidance';
|
|
440
|
+
hints.push({ host: 'w3.org', site: 'site:w3.org', terms });
|
|
441
|
+
}
|
|
442
|
+
return hints.slice(0, 3);
|
|
443
|
+
}
|
|
444
|
+
/** True when a candidate URL belongs to the hinted official host. */
|
|
445
|
+
function candidateHasHost(sources, host) {
|
|
446
|
+
return sources.some((source) => {
|
|
447
|
+
try {
|
|
448
|
+
return new URL(source.url).hostname.toLowerCase() === host
|
|
449
|
+
|| new URL(source.url).hostname.toLowerCase().endsWith(`.${host}`);
|
|
450
|
+
}
|
|
451
|
+
catch {
|
|
452
|
+
return false;
|
|
453
|
+
}
|
|
454
|
+
});
|
|
455
|
+
}
|
|
456
|
+
/** Host key for diversity; `www.` and one official subdomain are not a pack. */
|
|
457
|
+
function hostKey(raw) {
|
|
458
|
+
try {
|
|
459
|
+
return new URL(raw).hostname.toLowerCase().replace(/^www\./u, '');
|
|
460
|
+
}
|
|
461
|
+
catch {
|
|
462
|
+
return raw;
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Keep a pack from being filled by one issuer. Two pages per host are allowed;
|
|
467
|
+
* rows that would exceed the cap are kept as fallback and used only when there
|
|
468
|
+
* are not enough distinct hosts to fill the quota.
|
|
469
|
+
*/
|
|
470
|
+
function diversifyByHost(sources, maxPerHost = 2) {
|
|
471
|
+
const kept = [];
|
|
472
|
+
const deferred = [];
|
|
473
|
+
const counts = new Map();
|
|
474
|
+
for (const source of sources) {
|
|
475
|
+
const host = hostKey(source.url);
|
|
476
|
+
const used = counts.get(host) ?? 0;
|
|
477
|
+
if (used < maxPerHost) {
|
|
478
|
+
kept.push(source);
|
|
479
|
+
counts.set(host, used + 1);
|
|
480
|
+
}
|
|
481
|
+
else {
|
|
482
|
+
deferred.push(source);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
return [...kept, ...deferred];
|
|
486
|
+
}
|
|
116
487
|
/** Interleave per-query result lists by rank, so one query cannot fill the pack. */
|
|
117
488
|
function interleave(sourcesByQuery) {
|
|
118
489
|
const merged = [];
|
|
@@ -287,14 +658,14 @@ export function formatResearchValue(value) {
|
|
|
287
658
|
* @param fetchPage - fetches one URL through the plugin's fetch ladder.
|
|
288
659
|
* @returns a registry-ready definition.
|
|
289
660
|
*/
|
|
290
|
-
export function createResearchDefinition(name, search, fetchPage) {
|
|
661
|
+
export function createResearchDefinition(name, search, fetchPage, options = {}) {
|
|
291
662
|
return {
|
|
292
663
|
name,
|
|
293
664
|
description: 'Research a question end to end: run up to three searches, open the leading pages, and return an evidence pack in which every excerpt sits under its URL. Use it when the answer needs citations or several sources, not when one search row is enough. Pages that fail to load are reported by URL and reason instead of aborting the call.',
|
|
294
665
|
parameters: parameterSchema(),
|
|
295
666
|
output: {
|
|
296
667
|
schema: outputSchema(),
|
|
297
|
-
render: (_args, value) => [{ type: 'text', text: formatResearchValue(value) }],
|
|
668
|
+
render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatResearchValue(value)) }],
|
|
298
669
|
},
|
|
299
670
|
timeoutMs: 60_000,
|
|
300
671
|
isConcurrencySafe: () => true,
|
|
@@ -305,7 +676,8 @@ export function createResearchDefinition(name, search, fetchPage) {
|
|
|
305
676
|
const remaining = () => Math.max(0, RESEARCH_BUDGET_MS - (Date.now() - startedAt));
|
|
306
677
|
const cancelled = () => signal !== undefined && signal.aborted;
|
|
307
678
|
const searchLimit = Math.max(5, parsed.maxSources);
|
|
308
|
-
const
|
|
679
|
+
const queriesUsed = [...parsed.queries];
|
|
680
|
+
const runSearch = async (query) => {
|
|
309
681
|
try {
|
|
310
682
|
return await search(query, searchLimit, signal);
|
|
311
683
|
}
|
|
@@ -315,21 +687,97 @@ export function createResearchDefinition(name, search, fetchPage) {
|
|
|
315
687
|
failures: [error instanceof Error ? error.message : String(error)],
|
|
316
688
|
};
|
|
317
689
|
}
|
|
318
|
-
}
|
|
690
|
+
};
|
|
691
|
+
const outcomes = await mapLimit(parsed.queries, 2, runSearch);
|
|
319
692
|
if (cancelled())
|
|
320
693
|
throw new Error('research was cancelled');
|
|
321
|
-
|
|
694
|
+
let candidates = interleave(outcomes.map((outcome) => outcome.sources));
|
|
695
|
+
const hints = officialHints(parsed.queries);
|
|
696
|
+
// Long questions often need the issuing body's own site, which the open
|
|
697
|
+
// engine may not rank. Spend unused query slots on a site-specific query
|
|
698
|
+
// instead of letting the evidence pack stop at an aggregator.
|
|
699
|
+
for (const hint of hints) {
|
|
700
|
+
// A host match is enough for most issuers, but a generic W3C page does
|
|
701
|
+
// not cover a specific success criterion; let the targeted query run.
|
|
702
|
+
// A generic host match is enough for most issuers, but not for the
|
|
703
|
+
// two standards bodies below: a W3C landing page does not cover a
|
|
704
|
+
// specific success criterion, and a europa.eu press page does not
|
|
705
|
+
// contain the Article 113 operative clause. Let the focused query run
|
|
706
|
+
// even when the first result set already has that host.
|
|
707
|
+
const needsFocusedQuery = hint.host === 'w3.org' || hint.host === 'europa.eu';
|
|
708
|
+
if (queriesUsed.length >= RESEARCH_MAX_QUERIES
|
|
709
|
+
|| (!needsFocusedQuery && candidateHasHost(candidates, hint.host)))
|
|
710
|
+
continue;
|
|
711
|
+
const hinted = `${hint.site} ${hint.terms}`.trim();
|
|
712
|
+
queriesUsed.push(hinted);
|
|
713
|
+
const extra = await runSearch(hinted);
|
|
714
|
+
if (cancelled())
|
|
715
|
+
throw new Error('research was cancelled');
|
|
716
|
+
// A site: operator is advisory on some engines; keep only rows that
|
|
717
|
+
// really belong to the issuer's domain.
|
|
718
|
+
outcomes.push({
|
|
719
|
+
...extra,
|
|
720
|
+
sources: extra.sources.filter((source) => candidateHasHost([source], hint.host)),
|
|
721
|
+
});
|
|
722
|
+
candidates = interleave(outcomes.map((outcome) => outcome.sources));
|
|
723
|
+
}
|
|
724
|
+
// A known official article is stronger than a site: query for a question
|
|
725
|
+
// that explicitly asks for the clause or effective date. Add it after the
|
|
726
|
+
// focused queries, because rebuilding `candidates` from `outcomes` below
|
|
727
|
+
// would otherwise drop a source that was never in a search result.
|
|
728
|
+
for (const hint of hints) {
|
|
729
|
+
if (hint.directUrl === undefined || hint.directUrl.length === 0)
|
|
730
|
+
continue;
|
|
731
|
+
if (candidates.some((source) => source.url === hint.directUrl))
|
|
732
|
+
continue;
|
|
733
|
+
candidates = [
|
|
734
|
+
{
|
|
735
|
+
url: hint.directUrl,
|
|
736
|
+
title: hint.directTitle ?? hint.directUrl,
|
|
737
|
+
snippet: hint.directSnippet ?? '',
|
|
738
|
+
},
|
|
739
|
+
...candidates,
|
|
740
|
+
];
|
|
741
|
+
}
|
|
322
742
|
// Prefer rows whose title or snippet actually contains a query token.
|
|
323
743
|
// Search engines return a few singletons for a long Chinese query (the
|
|
324
744
|
// character `东`, a company named 东…); fetching those spends the budget
|
|
325
745
|
// on prose that cannot answer the question. The filter is only a
|
|
326
746
|
// preference: when too few rows match, every candidate is still eligible.
|
|
327
|
-
const tokens = queryTokens(
|
|
747
|
+
const tokens = queryTokens(queriesUsed);
|
|
328
748
|
const scored = candidates.map((source, index) => ({ source, index, score: relevanceScore(source, tokens) }));
|
|
329
749
|
const matched = scored.filter((entry) => entry.score > 0);
|
|
330
750
|
const pool = matched.length >= parsed.maxSources ? matched : scored;
|
|
331
|
-
pool.
|
|
332
|
-
|
|
751
|
+
const enriched = pool.map((entry) => {
|
|
752
|
+
const authority = authorityScore(entry.source.url);
|
|
753
|
+
return {
|
|
754
|
+
...entry,
|
|
755
|
+
authority,
|
|
756
|
+
credible: authority >= CREDIBLE_AUTHORITY && entry.score > 0
|
|
757
|
+
&& !isLandingPage(entry.source.url),
|
|
758
|
+
};
|
|
759
|
+
});
|
|
760
|
+
enriched.sort((left, right) => {
|
|
761
|
+
// A credible official page that matches the query leads the pack even
|
|
762
|
+
// when a generic page happens to repeat more query words. The relevance
|
|
763
|
+
// gate still excludes official pages that match none of the query.
|
|
764
|
+
if (left.credible !== right.credible)
|
|
765
|
+
return left.credible ? -1 : 1;
|
|
766
|
+
const byRelevance = right.score - left.score;
|
|
767
|
+
if (byRelevance !== 0)
|
|
768
|
+
return byRelevance;
|
|
769
|
+
return right.authority - left.authority || left.index - right.index;
|
|
770
|
+
});
|
|
771
|
+
let ranked = diversifyByHost(enriched.map((entry) => entry.source));
|
|
772
|
+
// A directly injected official article is known to answer a clause/date
|
|
773
|
+
// question; put it first even when a long Chinese snippet outscores it.
|
|
774
|
+
// This does not bypass the fetch ladder or the normal failure reporting.
|
|
775
|
+
const directUrls = new Set(hints.flatMap((hint) => (hint.directUrl === undefined ? [] : [hint.directUrl])));
|
|
776
|
+
if (directUrls.size > 0) {
|
|
777
|
+
const direct = ranked.filter((source) => directUrls.has(source.url));
|
|
778
|
+
const rest = ranked.filter((source) => !directUrls.has(source.url));
|
|
779
|
+
ranked = [...direct, ...rest];
|
|
780
|
+
}
|
|
333
781
|
const failures = [];
|
|
334
782
|
// Fill the quota even when a page fails: with `wanted = candidates.slice(0,
|
|
335
783
|
// maxSources)`, one dead candidate produced a pack with two pages out of
|
|
@@ -358,7 +806,28 @@ export function createResearchDefinition(name, search, fetchPage) {
|
|
|
358
806
|
: AbortSignal.any([signal, AbortSignal.timeout(budget)]);
|
|
359
807
|
try {
|
|
360
808
|
const fetched = await fetchPage(source.url, bound);
|
|
361
|
-
|
|
809
|
+
if (looksLikeHtmlDocument(fetched.text)) {
|
|
810
|
+
failures.push({ url: source.url, reason: 'the page returned a raw HTML document, not readable text' });
|
|
811
|
+
continue;
|
|
812
|
+
}
|
|
813
|
+
const interstitial = interstitialReason(fetched.text, fetched.status);
|
|
814
|
+
if (interstitial !== undefined) {
|
|
815
|
+
failures.push({ url: source.url, reason: interstitial });
|
|
816
|
+
continue;
|
|
817
|
+
}
|
|
818
|
+
const plain = fetched.text.trim();
|
|
819
|
+
if (plain.length === 0) {
|
|
820
|
+
failures.push({ url: source.url, reason: 'the page carried no readable text' });
|
|
821
|
+
continue;
|
|
822
|
+
}
|
|
823
|
+
if (plain.length < RESEARCH_MIN_TEXT_CHARS) {
|
|
824
|
+
failures.push({
|
|
825
|
+
url: source.url,
|
|
826
|
+
reason: `the page carried too little readable text (${plain.length} characters)`,
|
|
827
|
+
});
|
|
828
|
+
continue;
|
|
829
|
+
}
|
|
830
|
+
const excerpt = selectExcerpt(fetched.text, tokens, RESEARCH_EXCERPT_CHARS);
|
|
362
831
|
if (excerpt.length === 0) {
|
|
363
832
|
failures.push({ url: source.url, reason: 'the page carried no readable text' });
|
|
364
833
|
continue;
|
|
@@ -366,8 +835,9 @@ export function createResearchDefinition(name, search, fetchPage) {
|
|
|
366
835
|
succeeded += 1;
|
|
367
836
|
attempts.push({
|
|
368
837
|
index,
|
|
838
|
+
text: fetched.text,
|
|
369
839
|
page: {
|
|
370
|
-
url: fetched.url.length > 0 ? fetched.url : source.url,
|
|
840
|
+
url: canonicalOfficialUrl(fetched.url.length > 0 ? fetched.url : source.url),
|
|
371
841
|
...(source.title === undefined || source.title.length === 0 ? {} : { title: source.title }),
|
|
372
842
|
status: fetched.status,
|
|
373
843
|
chars: fetched.text.length,
|
|
@@ -386,16 +856,29 @@ export function createResearchDefinition(name, search, fetchPage) {
|
|
|
386
856
|
await Promise.all(workers);
|
|
387
857
|
if (cancelled())
|
|
388
858
|
throw new Error('research was cancelled');
|
|
389
|
-
const
|
|
859
|
+
const orderedAttempts = attempts
|
|
390
860
|
.sort((left, right) => left.index - right.index)
|
|
391
|
-
.
|
|
392
|
-
.filter((page) => page !== undefined)
|
|
861
|
+
.filter((attempt) => attempt.page !== undefined && attempt.text !== undefined)
|
|
393
862
|
.slice(0, parsed.maxSources);
|
|
863
|
+
const fetchedPages = orderedAttempts.map((attempt) => attempt.page);
|
|
864
|
+
for (const attempt of orderedAttempts) {
|
|
865
|
+
try {
|
|
866
|
+
options.onPage?.({
|
|
867
|
+
url: attempt.page.url,
|
|
868
|
+
...(attempt.page.title === undefined ? {} : { title: attempt.page.title }),
|
|
869
|
+
text: attempt.text,
|
|
870
|
+
queries: queriesUsed,
|
|
871
|
+
});
|
|
872
|
+
}
|
|
873
|
+
catch {
|
|
874
|
+
// A cache/observer failure is not an evidence failure.
|
|
875
|
+
}
|
|
876
|
+
}
|
|
394
877
|
const fetchedUrls = new Set(fetchedPages.map((page) => page.url));
|
|
395
878
|
const failedUrls = new Set(failures.map((failure) => failure.url));
|
|
396
879
|
return {
|
|
397
|
-
query:
|
|
398
|
-
queries:
|
|
880
|
+
query: queriesUsed[0],
|
|
881
|
+
queries: queriesUsed,
|
|
399
882
|
pages: fetchedPages,
|
|
400
883
|
sources: candidates
|
|
401
884
|
.filter((source) => !fetchedUrls.has(source.url) && !failedUrls.has(source.url))
|
|
@@ -447,7 +930,7 @@ function report(options, error) {
|
|
|
447
930
|
export function registerResearchTool(tools, search, fetchPage, options = {}) {
|
|
448
931
|
const name = pickResearchToolName(tools);
|
|
449
932
|
try {
|
|
450
|
-
const registered = tools.register(createResearchDefinition(name, search, fetchPage));
|
|
933
|
+
const registered = tools.register(createResearchDefinition(name, search, fetchPage, options));
|
|
451
934
|
const disposeHost = typeof registered === 'function' ? registered : undefined;
|
|
452
935
|
let disposed = false;
|
|
453
936
|
return {
|
package/lib/search-many.d.ts
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
*
|
|
13
13
|
* @module @fangchenghao/dsh-web-search/search-many
|
|
14
14
|
*/
|
|
15
|
-
import type { Source } from './types.ts';
|
|
15
|
+
import type { EngineFailure, Source } from './types.ts';
|
|
16
16
|
import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
|
|
17
17
|
/** The name this tool prefers. */
|
|
18
18
|
export declare const SEARCH_MANY_TOOL_NAME = "search_many";
|
|
@@ -40,6 +40,8 @@ export interface SearchManyResult {
|
|
|
40
40
|
sources: SearchManySource[];
|
|
41
41
|
/** Failures for this query alone; a thrown search becomes one entry here. */
|
|
42
42
|
failures: string[];
|
|
43
|
+
/** Short policy note, such as why the fallback tier was used or skipped. */
|
|
44
|
+
note?: string;
|
|
43
45
|
}
|
|
44
46
|
/** What the tool returns to the model. */
|
|
45
47
|
export interface SearchManyValue {
|
|
@@ -51,6 +53,10 @@ export interface SearchManyValue {
|
|
|
51
53
|
export interface SearchManyOutcome {
|
|
52
54
|
sources: Source[];
|
|
53
55
|
failures: string[];
|
|
56
|
+
/** Structured failures from the engine layer, when it reported any. */
|
|
57
|
+
failureDetails?: EngineFailure[];
|
|
58
|
+
/** Short policy note, when the fallback policy changed the answer. */
|
|
59
|
+
note?: string;
|
|
54
60
|
/** True when the deployment's configured limit cut the requested budget. */
|
|
55
61
|
truncated: boolean;
|
|
56
62
|
}
|