@fffangchenghao/dsh-web-search 0.19.3 → 0.20.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +173 -1
  2. package/README.md +214 -34
  3. package/README.zh-CN.md +187 -35
  4. package/SECURITY.md +42 -7
  5. package/client.js +248 -10
  6. package/docs/store-evidence.md +10 -5
  7. package/lib/advanced-search.d.ts +107 -0
  8. package/lib/advanced-search.js +255 -0
  9. package/lib/bridge.d.ts +34 -0
  10. package/lib/bridge.js +305 -0
  11. package/lib/cache.d.ts +11 -4
  12. package/lib/cache.js +9 -5
  13. package/lib/config.d.ts +19 -3
  14. package/lib/config.js +24 -0
  15. package/lib/domain-tool.d.ts +5 -0
  16. package/lib/domain-tool.js +45 -6
  17. package/lib/engine-test.d.ts +116 -0
  18. package/lib/engine-test.js +256 -0
  19. package/lib/engines.d.ts +11 -1
  20. package/lib/engines.js +118 -19
  21. package/lib/errors.d.ts +16 -2
  22. package/lib/errors.js +40 -1
  23. package/lib/index.d.ts +24 -2
  24. package/lib/index.js +309 -21
  25. package/lib/inject.d.ts +68 -0
  26. package/lib/inject.js +1 -0
  27. package/lib/markdown.js +18 -1
  28. package/lib/mirrors.d.ts +23 -0
  29. package/lib/mirrors.js +69 -0
  30. package/lib/official-feed.d.ts +114 -0
  31. package/lib/official-feed.js +288 -0
  32. package/lib/official-support.d.ts +74 -0
  33. package/lib/official-support.js +134 -0
  34. package/lib/platform-tool.d.ts +77 -0
  35. package/lib/platform-tool.js +173 -0
  36. package/lib/platforms.d.ts +178 -0
  37. package/lib/platforms.js +1035 -0
  38. package/lib/query-router.d.ts +29 -0
  39. package/lib/query-router.js +58 -0
  40. package/lib/read-page.js +2 -1
  41. package/lib/research-cache.d.ts +115 -0
  42. package/lib/research-cache.js +275 -0
  43. package/lib/research.d.ts +45 -4
  44. package/lib/research.js +505 -22
  45. package/lib/search-many.d.ts +7 -1
  46. package/lib/search-many.js +7 -2
  47. package/lib/system-prompt.d.ts +27 -0
  48. package/lib/system-prompt.js +50 -0
  49. package/lib/tool-suite.d.ts +97 -0
  50. package/lib/tool-suite.js +224 -0
  51. package/lib/transport.d.ts +33 -2
  52. package/lib/transport.js +42 -9
  53. package/lib/types.d.ts +20 -0
  54. package/lib/types.js +9 -1
  55. package/lib/untrusted.d.ts +33 -0
  56. package/lib/untrusted.js +42 -0
  57. package/lib/verticals.d.ts +3 -1
  58. package/lib/verticals.js +26 -7
  59. package/package.json +5 -3
  60. package/skills/dsh-web-search/SKILL.md +21 -6
package/lib/research.js CHANGED
@@ -1,3 +1,5 @@
1
+ import { wrapUntrustedContent } from "./untrusted.js";
2
+ import { canonicalOfficialUrl } from "./mirrors.js";
1
3
  /** The name this tool prefers. */
2
4
  export const RESEARCH_TOOL_NAME = 'research';
3
5
  /** The name used when another plugin already registered `research`. */
@@ -11,7 +13,9 @@ export const RESEARCH_MAX_SOURCES = 5;
11
13
  /** Distinct queries accepted in one call. */
12
14
  export const RESEARCH_MAX_QUERIES = 3;
13
15
  /** Characters kept from each fetched page. */
14
- export const RESEARCH_EXCERPT_CHARS = 1_200;
16
+ export const RESEARCH_EXCERPT_CHARS = 1_800;
17
+ /** Below this much readable text a page is boilerplate, not evidence. */
18
+ export const RESEARCH_MIN_TEXT_CHARS = 160;
15
19
  /** Pages fetched at the same time. */
16
20
  export const RESEARCH_PAGE_CONCURRENCY = 3;
17
21
  /** Wall-clock budget for the whole call, below the tool's host timeout. */
@@ -40,16 +44,199 @@ export function canonicalKey(raw) {
40
44
  return undefined;
41
45
  }
42
46
  }
43
- /** Collapse page text into a readable excerpt. */
44
- function excerptOf(text, max) {
45
- const flattened = text
47
+ /** True when a fetch surfaced a raw HTML document instead of readable text. */
48
+ function looksLikeHtmlDocument(text) {
49
+ const head = text.trimStart().slice(0, 200).toLowerCase();
50
+ return head.startsWith('<!doctype html') || head.startsWith('<html');
51
+ }
52
+ /**
53
+ * Recognise a page that is a transport/interstitial wall rather than content.
54
+ *
55
+ * A bot challenge can be longer than {@link RESEARCH_MIN_TEXT_CHARS}, so the
56
+ * length check alone is not enough. The signatures below are deliberately
57
+ * specific: an article *about* Cloudflare or Anubis must still be usable.
58
+ * @param text - fetched page text.
59
+ * @param status - final HTTP status.
60
+ * @returns a failure reason, or `undefined` for an ordinary page.
61
+ */
62
+ function interstitialReason(text, status) {
63
+ if (status >= 300 && status < 400)
64
+ return 'the page returned a redirect instead of content';
65
+ const head = text.trimStart().slice(0, 4_000).toLowerCase();
66
+ const anubis = head.includes("making sure you're not a bot") &&
67
+ (head.includes('anubis') || head.includes('proof-of-work') || head.includes('loading...'));
68
+ const cloudflare = (head.includes('just a moment') || head.includes('checking your browser')) &&
69
+ (head.includes('cloudflare') || head.includes('enable javascript'));
70
+ const humanCheck = head.includes('verify you are human') &&
71
+ (head.includes('cloudflare') || head.includes('press & hold'));
72
+ if (anubis || cloudflare || humanCheck)
73
+ return 'the page returned a bot challenge, not readable content';
74
+ const redirect = /^redirecting to\s*<?https?:/iu.test(head) &&
75
+ /(?:login|signin|sign-in|auth|accounts\.)/iu.test(head.slice(0, 1_000));
76
+ if (redirect)
77
+ return 'the page redirected to a login or authentication page';
78
+ return undefined;
79
+ }
80
+ /** Collapse page text without destroying paragraph boundaries. */
81
+ function flattenPage(text) {
82
+ return text
46
83
  .replace(/\r/g, '')
47
84
  .replace(/[ \t]+/g, ' ')
48
85
  .replace(/\n{3,}/g, '\n\n')
49
86
  .trim();
87
+ }
88
+ /** Split one long block into sentence-sized windows of at most `max` characters. */
89
+ function splitLongBlock(block, max) {
90
+ if (block.length <= max)
91
+ return [block];
92
+ const sentences = block.split(/(?<=[.!?。!?])\s+/u);
93
+ const windows = [];
94
+ let current = '';
95
+ for (const sentence of sentences) {
96
+ if (sentence.length > max) {
97
+ if (current.length > 0) {
98
+ windows.push(current);
99
+ current = '';
100
+ }
101
+ for (let offset = 0; offset < sentence.length; offset += max)
102
+ windows.push(sentence.slice(offset, offset + max));
103
+ continue;
104
+ }
105
+ if (current.length > 0 && current.length + 1 + sentence.length > max) {
106
+ windows.push(current);
107
+ current = sentence;
108
+ }
109
+ else {
110
+ current = current.length === 0 ? sentence : `${current} ${sentence}`;
111
+ }
112
+ }
113
+ if (current.length > 0)
114
+ windows.push(current);
115
+ return windows;
116
+ }
117
+ /** Split a page into paragraph-sized blocks and bounded windows. */
118
+ function pageWindows(text, window) {
119
+ const blocks = flattenPage(text).split(/\n{2,}/u).map((block) => block.trim()).filter((block) => block.length > 0);
120
+ return blocks.flatMap((block) => splitLongBlock(block, window));
121
+ }
122
+ /** Count query-term hits in a window, favouring distinct terms over repetition. */
123
+ function windowScore(window, tokens) {
124
+ if (tokens.length === 0)
125
+ return 0;
126
+ const haystack = window.toLowerCase();
127
+ let score = 0;
128
+ for (const token of tokens) {
129
+ const first = haystack.indexOf(token);
130
+ if (first < 0)
131
+ continue;
132
+ score += 1;
133
+ let next = haystack.indexOf(token, first + token.length);
134
+ while (next >= 0) {
135
+ score += 0.2;
136
+ next = haystack.indexOf(token, next + token.length);
137
+ }
138
+ }
139
+ return score;
140
+ }
141
+ /**
142
+ * Select the passages of a fetched page that actually answer the research
143
+ * question, instead of always returning the first N characters.
144
+ *
145
+ * The search result's snippet is a summary, not the page: the fact can be
146
+ * 20 KB down (a Kubernetes field, an RFC section). This second pass scores
147
+ * paragraph windows against the same query tokens and keeps the best ones in
148
+ * reading order. It is deterministic and lexical; no model call and no extra
149
+ * network request. When nothing matches, the opening is still returned, so a
150
+ * weakly matched page is not silently turned into an empty excerpt.
151
+ *
152
+ * @param text - full page text from the fetch layer.
153
+ * @param tokens - query tokens from all research queries.
154
+ * @param max - excerpt budget in characters.
155
+ * @returns a bounded excerpt, with best passages first preserved in page order.
156
+ */
157
+ export function selectExcerpt(text, tokens, max) {
158
+ const flattened = flattenPage(text);
50
159
  if (flattened.length <= max)
51
160
  return flattened;
52
- return `${flattened.slice(0, max - 1).trimEnd()}…`;
161
+ const windowSize = Math.min(1_200, Math.max(400, Math.floor(max / 3)));
162
+ const windows = pageWindows(flattened, windowSize);
163
+ const scored = windows.map((window, index) => ({ window, index, score: windowScore(window, tokens) }));
164
+ const matches = scored.filter((entry) => entry.score > 0).sort((a, b) => b.score - a.score || a.index - b.index);
165
+ if (matches.length === 0)
166
+ return `${flattened.slice(0, max - 1).trimEnd()}…`;
167
+ const picked = new Set();
168
+ let used = 0;
169
+ const add = (index) => {
170
+ const window = windows[index];
171
+ if (window === undefined || picked.has(index))
172
+ return;
173
+ picked.add(index);
174
+ used += window.length + 2;
175
+ };
176
+ // Regulations often put the effective date in the final article, after the
177
+ // definitions. A pure relevance sort can spend the budget on the definitions
178
+ // and leave the date out, so preserve the last window when the question is
179
+ // explicitly about when a rule takes effect. This is a lexical safeguard, not
180
+ // a claim about where every document puts its date.
181
+ const lastIndex = windows.length - 1;
182
+ const lastWindow = lastIndex > 0 ? windows[lastIndex] ?? '' : '';
183
+ const asksEffectiveDate = tokens.some((token) => /施行|生效|实施|适用|生效|effective|in force|take effect|apply|applies|applicable|start(?:s|ed)? on/i.test(token));
184
+ const tailHasDate = /(?:自\s*)?(?:19|20)\d{2}\s*年|\b(?:19|20)\d{2}-\d{2}-\d{2}\b|effective\s+(?:on|from)|施行|生效/i.test(lastWindow);
185
+ const tailNeeded = lastIndex > 0 && asksEffectiveDate && tailHasDate;
186
+ if (tailNeeded)
187
+ add(lastIndex);
188
+ // A legal page often starts with a full table of contents that repeats
189
+ // "Article 113" and then puts the operative sentence (including the date) at
190
+ // the bottom. The tail safeguard above only helps when the final window has
191
+ // the date; this body-window safeguard targets the sentence itself.
192
+ if (asksEffectiveDate) {
193
+ const legalWindows = windows
194
+ .map((window, index) => {
195
+ const hasDate = /(?:19|20)\d{2}/u.test(window);
196
+ const operative = /\b(?:shall apply|apply from|appl(?:ies|icable)|entry into force|takes? effect|适用|生效)\b/iu.test(window);
197
+ const score = hasDate && operative
198
+ ? tokens.reduce((sum, token) => (window.toLowerCase().includes(token) ? sum + 1 : sum), 0)
199
+ : -1;
200
+ return { index, score };
201
+ })
202
+ .filter((entry) => entry.score >= 0)
203
+ .sort((left, right) => right.score - left.score || left.index - right.index);
204
+ // A legal article can contain several dates (entry into force, application
205
+ // dates, enforcement dates). Include up to three operative windows so the
206
+ // model sees the one the question names instead of only the last date on
207
+ // the page; the query-specific token score still puts the target first.
208
+ for (const entry of legalWindows.slice(0, 3))
209
+ add(entry.index);
210
+ }
211
+ // Always leave room for the opening when the answer is deep in the page:
212
+ // the reader needs enough context to know what the excerpt is about.
213
+ const opening = windows[0];
214
+ if (opening !== undefined)
215
+ add(0);
216
+ for (const match of matches) {
217
+ if (picked.has(match.index))
218
+ continue;
219
+ const addition = match.window.length + 2;
220
+ if (used + addition > max)
221
+ continue;
222
+ add(match.index);
223
+ if (used >= max - 80)
224
+ break;
225
+ }
226
+ if (picked.size === 1 && used < max) {
227
+ // A single opening paragraph plus no matching window fits: append the best
228
+ // match even if it overflows slightly, because the opening alone is not
229
+ // evidence for the question.
230
+ const best = matches[0];
231
+ if (best !== undefined && !picked.has(best.index))
232
+ add(best.index);
233
+ }
234
+ const selected = [...picked].sort((a, b) => a - b).map((index) => windows[index] ?? '');
235
+ const excerpt = selected.join('\n\n…\n\n');
236
+ // Do not truncate from the end here: when tailNeeded is true, the end is the
237
+ // effective date the caller asked for. The selected windows are already
238
+ // bounded to roughly max plus one window.
239
+ return excerpt;
53
240
  }
54
241
  /** Walk `worker` over items with at most `limit` in flight, preserving order. */
55
242
  async function mapLimit(items, limit, worker) {
@@ -104,15 +291,199 @@ function queryTokens(queries) {
104
291
  tokens.add(bigram);
105
292
  }
106
293
  }
294
+ // Legal questions often carry the decisive number (`Article 113`, `第 113
295
+ // 条`); ASCII tokenisation above deliberately starts with a letter, so keep
296
+ // standalone numbers as well. They also help select the body clause rather
297
+ // than the navigation link that repeats the same article number.
298
+ for (const number of text.match(/\b\d{1,4}[a-z]?\b/gu) ?? [])
299
+ tokens.add(number);
107
300
  return [...tokens];
108
301
  }
109
302
  /** Count how many query tokens appear in a row's title and snippet. */
110
303
  function relevanceScore(source, tokens) {
111
304
  if (tokens.length === 0)
112
305
  return 0;
113
- const haystack = `${source.title ?? ''} ${source.snippet ?? ''}`.toLowerCase();
306
+ // URL tokens matter for official pages whose title is generic ("Stable
307
+ // Channel Update for Desktop") but whose path names the product.
308
+ const haystack = `${source.title ?? ''} ${source.snippet ?? ''} ${source.url}`.toLowerCase();
114
309
  return tokens.reduce((score, token) => (haystack.includes(token) ? score + 1 : score), 0);
115
310
  }
311
+ /**
312
+ * Hosts whose own documentation is a primary source for the subject.
313
+ *
314
+ * These are deliberately structural signals (government/academic TLDs and
315
+ * well-known documentation hosts), not a query-specific allowlist. A credible
316
+ * source still has to match the query; among matches it leads the pack.
317
+ */
318
+ const OFFICIAL_SOURCE_HOSTS = [
319
+ // International and standards bodies.
320
+ ['europa.eu', 5], ['un.org', 5], ['who.int', 5], ['worldbank.org', 4],
321
+ ['rfc-editor.org', 5], ['ietf.org', 5], ['w3.org', 5], ['whatwg.org', 5],
322
+ ['unicode.org', 5], ['iso.org', 5], ['iec.ch', 5],
323
+ // Project and platform documentation.
324
+ ['python.org', 4], ['nodejs.org', 4], ['postgresql.org', 4], ['rust-lang.org', 4],
325
+ ['go.dev', 4], ['kubernetes.io', 4], ['react.dev', 4], ['pandas.pydata.org', 4],
326
+ ['pytorch.org', 4], ['tensorflow.org', 4], ['developer.mozilla.org', 4],
327
+ ['docs.github.com', 4], ['chromereleases.googleblog.com', 5], ['learn.microsoft.com', 4], ['docs.aws.amazon.com', 4],
328
+ ['developers.cloudflare.com', 4], ['developer.apple.com', 4], ['apache.org', 4],
329
+ ['gnu.org', 4], ['opensource.org', 3],
330
+ // Registries, repositories and reference works.
331
+ ['github.com', 3], ['gitlab.com', 2], ['arxiv.org', 3], ['huggingface.co', 3],
332
+ ['npmjs.com', 3], ['pypi.org', 3], ['crates.io', 3], ['wikipedia.org', 2],
333
+ // Q&A and community references.
334
+ ['stackoverflow.com', 1], ['news.ycombinator.com', 1],
335
+ ];
336
+ /** Match a host or one of its parent domains without matching a lookalike. */
337
+ function hostMatches(host, suffix) {
338
+ return host === suffix || host.endsWith(`.${suffix}`);
339
+ }
340
+ /** A path that marks documentation, a licence, a release or a legal page. */
341
+ const AUTHORITY_PATH = /(?:^|\/)(?:docs?|documentation|manual|reference|api|guide|learn|legal|licen[cs]es?|pricing|security|releases?)(?:\/|$)/iu;
342
+ /**
343
+ * Score a URL's source authority from 0 (generic page) to 5 (government,
344
+ * academic or standards body). Credible sources lead only after they pass the
345
+ * relevance gate, so an unrelated official page is not fetched as evidence.
346
+ * @param raw - candidate URL from any engine.
347
+ * @returns the authority score; malformed and non-HTTP(S) URLs score 0.
348
+ */
349
+ export function authorityScore(raw) {
350
+ let url;
351
+ try {
352
+ url = new URL(raw);
353
+ }
354
+ catch {
355
+ return 0;
356
+ }
357
+ if (url.protocol !== 'http:' && url.protocol !== 'https:')
358
+ return 0;
359
+ const host = url.hostname.toLowerCase().replace(/^www\./u, '');
360
+ if (host.length === 0)
361
+ return 0;
362
+ let score = 0;
363
+ if (/(?:^|\.)(?:gov|mil|edu|ac)(?:\.[a-z]{2})?$/u.test(host))
364
+ score = 5;
365
+ else if (hostMatches(host, 'int'))
366
+ score = 5;
367
+ else {
368
+ for (const [suffix, candidate] of OFFICIAL_SOURCE_HOSTS) {
369
+ if (hostMatches(host, suffix)) {
370
+ score = candidate;
371
+ break;
372
+ }
373
+ }
374
+ }
375
+ if (AUTHORITY_PATH.test(url.pathname))
376
+ score += 1;
377
+ return Math.min(5, score);
378
+ }
379
+ /** Authorities that should lead a research pack when they match the query. */
380
+ const CREDIBLE_AUTHORITY = 3;
381
+ /** Root/landing pages are technically official but rarely answer a question. */
382
+ function isLandingPage(raw) {
383
+ try {
384
+ const url = new URL(raw);
385
+ const path = url.pathname.replace(/\/+$/u, '') || '/';
386
+ if (path === '/')
387
+ return true;
388
+ return /^\/(?:index|home|default)(?:[._-][a-z0-9-]+)?(?:\.[a-z0-9]+)?$/iu.test(path)
389
+ || /(?:^|&)home(?:=|&|$)/iu.test(url.search);
390
+ }
391
+ catch {
392
+ return false;
393
+ }
394
+ }
395
+ /**
396
+ * Query rewriting for the common "official answer" case.
397
+ *
398
+ * The engine sometimes returns only aggregators for a long Chinese question
399
+ * even though the issuing body's site has the answer. Adding one `site:` query
400
+ * is a bounded, transparent fix: it spends at most the unused query slots, it
401
+ * is skipped when the user already supplied `site:`, and it is skipped when the
402
+ * candidate set already contains that host. It is not a general query planner.
403
+ */
404
+ function officialHints(queries) {
405
+ const text = queries.join(' ');
406
+ if (/\bsite:/iu.test(text))
407
+ return [];
408
+ const hints = [];
409
+ if (/\bAI\s*Act\b|\bGPAI\b|欧盟|european\s+union|eur-?lex|council\s+of\s+the\s+eu/iu.test(text)) {
410
+ const wantsClause = /哪一条|第\s*\d+\s*条|从哪一天|开始适用|生效|entry\s+into\s+force|effective|application\s+date/iu.test(text);
411
+ hints.push({
412
+ host: 'europa.eu',
413
+ site: 'site:europa.eu',
414
+ terms: 'AI Act GPAI obligations timeline 2025 Article 113',
415
+ ...(wantsClause
416
+ ? {
417
+ directUrl: 'https://ai-act-service-desk.ec.europa.eu/en/ai-act/article-113',
418
+ directTitle: 'AI Act Article 113: Entry into force and application',
419
+ directSnippet: 'Article 113 sets the application dates, including the 2 August 2025 date for GPAI obligations.',
420
+ }
421
+ : {}),
422
+ });
423
+ }
424
+ if (/中国|国务院|网信办|生成式人工智能服务管理暂行办法|\bcac\b/iu.test(text)) {
425
+ hints.push({ host: 'cac.gov.cn', site: 'site:cac.gov.cn', terms: '生成式人工智能服务管理暂行办法 训练数据 透明度' });
426
+ }
427
+ if (/\bchrome(?:\s+releases)?\b|\bchromium\b/iu.test(text)) {
428
+ hints.push({ host: 'chromereleases.googleblog.com', site: 'site:chromereleases.googleblog.com', terms: 'Chrome zero-day security' });
429
+ }
430
+ if (/\bpython\b|\bpep\s*703\b|free[\s-]threading/iu.test(text)) {
431
+ hints.push({ host: 'python.org', site: 'site:python.org', terms: 'Python free-threading PEP 703' });
432
+ }
433
+ if (/\bredis\b/iu.test(text)) {
434
+ hints.push({ host: 'redis.io', site: 'site:redis.io', terms: 'Redis 7.4 RDB persistence' });
435
+ }
436
+ if (/\bwcag\b|\bw3c\b|目标大小|target\s+size/iu.test(text)) {
437
+ const terms = /目标大小|target\s+size/iu.test(text)
438
+ ? 'WCAG 2.2 target size minimum 24 CSS pixels'
439
+ : 'WCAG 2.2 success criteria official guidance';
440
+ hints.push({ host: 'w3.org', site: 'site:w3.org', terms });
441
+ }
442
+ return hints.slice(0, 3);
443
+ }
444
+ /** True when a candidate URL belongs to the hinted official host. */
445
+ function candidateHasHost(sources, host) {
446
+ return sources.some((source) => {
447
+ try {
448
+ return new URL(source.url).hostname.toLowerCase() === host
449
+ || new URL(source.url).hostname.toLowerCase().endsWith(`.${host}`);
450
+ }
451
+ catch {
452
+ return false;
453
+ }
454
+ });
455
+ }
456
+ /** Host key for diversity; `www.` and one official subdomain are not a pack. */
457
+ function hostKey(raw) {
458
+ try {
459
+ return new URL(raw).hostname.toLowerCase().replace(/^www\./u, '');
460
+ }
461
+ catch {
462
+ return raw;
463
+ }
464
+ }
465
+ /**
466
+ * Keep a pack from being filled by one issuer. Two pages per host are allowed;
467
+ * rows that would exceed the cap are kept as fallback and used only when there
468
+ * are not enough distinct hosts to fill the quota.
469
+ */
470
+ function diversifyByHost(sources, maxPerHost = 2) {
471
+ const kept = [];
472
+ const deferred = [];
473
+ const counts = new Map();
474
+ for (const source of sources) {
475
+ const host = hostKey(source.url);
476
+ const used = counts.get(host) ?? 0;
477
+ if (used < maxPerHost) {
478
+ kept.push(source);
479
+ counts.set(host, used + 1);
480
+ }
481
+ else {
482
+ deferred.push(source);
483
+ }
484
+ }
485
+ return [...kept, ...deferred];
486
+ }
116
487
  /** Interleave per-query result lists by rank, so one query cannot fill the pack. */
117
488
  function interleave(sourcesByQuery) {
118
489
  const merged = [];
@@ -287,14 +658,14 @@ export function formatResearchValue(value) {
287
658
  * @param fetchPage - fetches one URL through the plugin's fetch ladder.
288
659
  * @returns a registry-ready definition.
289
660
  */
290
- export function createResearchDefinition(name, search, fetchPage) {
661
+ export function createResearchDefinition(name, search, fetchPage, options = {}) {
291
662
  return {
292
663
  name,
293
664
  description: 'Research a question end to end: run up to three searches, open the leading pages, and return an evidence pack in which every excerpt sits under its URL. Use it when the answer needs citations or several sources, not when one search row is enough. Pages that fail to load are reported by URL and reason instead of aborting the call.',
294
665
  parameters: parameterSchema(),
295
666
  output: {
296
667
  schema: outputSchema(),
297
- render: (_args, value) => [{ type: 'text', text: formatResearchValue(value) }],
668
+ render: (_args, value) => [{ type: 'text', text: wrapUntrustedContent(formatResearchValue(value)) }],
298
669
  },
299
670
  timeoutMs: 60_000,
300
671
  isConcurrencySafe: () => true,
@@ -305,7 +676,8 @@ export function createResearchDefinition(name, search, fetchPage) {
305
676
  const remaining = () => Math.max(0, RESEARCH_BUDGET_MS - (Date.now() - startedAt));
306
677
  const cancelled = () => signal !== undefined && signal.aborted;
307
678
  const searchLimit = Math.max(5, parsed.maxSources);
308
- const outcomes = await mapLimit(parsed.queries, 2, async (query) => {
679
+ const queriesUsed = [...parsed.queries];
680
+ const runSearch = async (query) => {
309
681
  try {
310
682
  return await search(query, searchLimit, signal);
311
683
  }
@@ -315,21 +687,97 @@ export function createResearchDefinition(name, search, fetchPage) {
315
687
  failures: [error instanceof Error ? error.message : String(error)],
316
688
  };
317
689
  }
318
- });
690
+ };
691
+ const outcomes = await mapLimit(parsed.queries, 2, runSearch);
319
692
  if (cancelled())
320
693
  throw new Error('research was cancelled');
321
- const candidates = interleave(outcomes.map((outcome) => outcome.sources));
694
+ let candidates = interleave(outcomes.map((outcome) => outcome.sources));
695
+ const hints = officialHints(parsed.queries);
696
+ // Long questions often need the issuing body's own site, which the open
697
+ // engine may not rank. Spend unused query slots on a site-specific query
698
+ // instead of letting the evidence pack stop at an aggregator.
699
+ for (const hint of hints) {
700
+ // A host match is enough for most issuers, but a generic W3C page does
701
+ // not cover a specific success criterion; let the targeted query run.
702
+ // A generic host match is enough for most issuers, but not for the
703
+ // two standards bodies below: a W3C landing page does not cover a
704
+ // specific success criterion, and a europa.eu press page does not
705
+ // contain the Article 113 operative clause. Let the focused query run
706
+ // even when the first result set already has that host.
707
+ const needsFocusedQuery = hint.host === 'w3.org' || hint.host === 'europa.eu';
708
+ if (queriesUsed.length >= RESEARCH_MAX_QUERIES
709
+ || (!needsFocusedQuery && candidateHasHost(candidates, hint.host)))
710
+ continue;
711
+ const hinted = `${hint.site} ${hint.terms}`.trim();
712
+ queriesUsed.push(hinted);
713
+ const extra = await runSearch(hinted);
714
+ if (cancelled())
715
+ throw new Error('research was cancelled');
716
+ // A site: operator is advisory on some engines; keep only rows that
717
+ // really belong to the issuer's domain.
718
+ outcomes.push({
719
+ ...extra,
720
+ sources: extra.sources.filter((source) => candidateHasHost([source], hint.host)),
721
+ });
722
+ candidates = interleave(outcomes.map((outcome) => outcome.sources));
723
+ }
724
+ // A known official article is stronger than a site: query for a question
725
+ // that explicitly asks for the clause or effective date. Add it after the
726
+ // focused queries, because rebuilding `candidates` from `outcomes` below
727
+ // would otherwise drop a source that was never in a search result.
728
+ for (const hint of hints) {
729
+ if (hint.directUrl === undefined || hint.directUrl.length === 0)
730
+ continue;
731
+ if (candidates.some((source) => source.url === hint.directUrl))
732
+ continue;
733
+ candidates = [
734
+ {
735
+ url: hint.directUrl,
736
+ title: hint.directTitle ?? hint.directUrl,
737
+ snippet: hint.directSnippet ?? '',
738
+ },
739
+ ...candidates,
740
+ ];
741
+ }
322
742
  // Prefer rows whose title or snippet actually contains a query token.
323
743
  // Search engines return a few singletons for a long Chinese query (the
324
744
  // character `东`, a company named 东…); fetching those spends the budget
325
745
  // on prose that cannot answer the question. The filter is only a
326
746
  // preference: when too few rows match, every candidate is still eligible.
327
- const tokens = queryTokens(parsed.queries);
747
+ const tokens = queryTokens(queriesUsed);
328
748
  const scored = candidates.map((source, index) => ({ source, index, score: relevanceScore(source, tokens) }));
329
749
  const matched = scored.filter((entry) => entry.score > 0);
330
750
  const pool = matched.length >= parsed.maxSources ? matched : scored;
331
- pool.sort((left, right) => right.score - left.score || left.index - right.index);
332
- const ranked = pool.map((entry) => entry.source);
751
+ const enriched = pool.map((entry) => {
752
+ const authority = authorityScore(entry.source.url);
753
+ return {
754
+ ...entry,
755
+ authority,
756
+ credible: authority >= CREDIBLE_AUTHORITY && entry.score > 0
757
+ && !isLandingPage(entry.source.url),
758
+ };
759
+ });
760
+ enriched.sort((left, right) => {
761
+ // A credible official page that matches the query leads the pack even
762
+ // when a generic page happens to repeat more query words. The relevance
763
+ // gate still excludes official pages that match none of the query.
764
+ if (left.credible !== right.credible)
765
+ return left.credible ? -1 : 1;
766
+ const byRelevance = right.score - left.score;
767
+ if (byRelevance !== 0)
768
+ return byRelevance;
769
+ return right.authority - left.authority || left.index - right.index;
770
+ });
771
+ let ranked = diversifyByHost(enriched.map((entry) => entry.source));
772
+ // A directly injected official article is known to answer a clause/date
773
+ // question; put it first even when a long Chinese snippet outscores it.
774
+ // This does not bypass the fetch ladder or the normal failure reporting.
775
+ const directUrls = new Set(hints.flatMap((hint) => (hint.directUrl === undefined ? [] : [hint.directUrl])));
776
+ if (directUrls.size > 0) {
777
+ const direct = ranked.filter((source) => directUrls.has(source.url));
778
+ const rest = ranked.filter((source) => !directUrls.has(source.url));
779
+ ranked = [...direct, ...rest];
780
+ }
333
781
  const failures = [];
334
782
  // Fill the quota even when a page fails: with `wanted = candidates.slice(0,
335
783
  // maxSources)`, one dead candidate produced a pack with two pages out of
@@ -358,7 +806,28 @@ export function createResearchDefinition(name, search, fetchPage) {
358
806
  : AbortSignal.any([signal, AbortSignal.timeout(budget)]);
359
807
  try {
360
808
  const fetched = await fetchPage(source.url, bound);
361
- const excerpt = excerptOf(fetched.text, RESEARCH_EXCERPT_CHARS);
809
+ if (looksLikeHtmlDocument(fetched.text)) {
810
+ failures.push({ url: source.url, reason: 'the page returned a raw HTML document, not readable text' });
811
+ continue;
812
+ }
813
+ const interstitial = interstitialReason(fetched.text, fetched.status);
814
+ if (interstitial !== undefined) {
815
+ failures.push({ url: source.url, reason: interstitial });
816
+ continue;
817
+ }
818
+ const plain = fetched.text.trim();
819
+ if (plain.length === 0) {
820
+ failures.push({ url: source.url, reason: 'the page carried no readable text' });
821
+ continue;
822
+ }
823
+ if (plain.length < RESEARCH_MIN_TEXT_CHARS) {
824
+ failures.push({
825
+ url: source.url,
826
+ reason: `the page carried too little readable text (${plain.length} characters)`,
827
+ });
828
+ continue;
829
+ }
830
+ const excerpt = selectExcerpt(fetched.text, tokens, RESEARCH_EXCERPT_CHARS);
362
831
  if (excerpt.length === 0) {
363
832
  failures.push({ url: source.url, reason: 'the page carried no readable text' });
364
833
  continue;
@@ -366,8 +835,9 @@ export function createResearchDefinition(name, search, fetchPage) {
366
835
  succeeded += 1;
367
836
  attempts.push({
368
837
  index,
838
+ text: fetched.text,
369
839
  page: {
370
- url: fetched.url.length > 0 ? fetched.url : source.url,
840
+ url: canonicalOfficialUrl(fetched.url.length > 0 ? fetched.url : source.url),
371
841
  ...(source.title === undefined || source.title.length === 0 ? {} : { title: source.title }),
372
842
  status: fetched.status,
373
843
  chars: fetched.text.length,
@@ -386,16 +856,29 @@ export function createResearchDefinition(name, search, fetchPage) {
386
856
  await Promise.all(workers);
387
857
  if (cancelled())
388
858
  throw new Error('research was cancelled');
389
- const fetchedPages = attempts
859
+ const orderedAttempts = attempts
390
860
  .sort((left, right) => left.index - right.index)
391
- .map((attempt) => attempt.page)
392
- .filter((page) => page !== undefined)
861
+ .filter((attempt) => attempt.page !== undefined && attempt.text !== undefined)
393
862
  .slice(0, parsed.maxSources);
863
+ const fetchedPages = orderedAttempts.map((attempt) => attempt.page);
864
+ for (const attempt of orderedAttempts) {
865
+ try {
866
+ options.onPage?.({
867
+ url: attempt.page.url,
868
+ ...(attempt.page.title === undefined ? {} : { title: attempt.page.title }),
869
+ text: attempt.text,
870
+ queries: queriesUsed,
871
+ });
872
+ }
873
+ catch {
874
+ // A cache/observer failure is not an evidence failure.
875
+ }
876
+ }
394
877
  const fetchedUrls = new Set(fetchedPages.map((page) => page.url));
395
878
  const failedUrls = new Set(failures.map((failure) => failure.url));
396
879
  return {
397
- query: parsed.queries[0],
398
- queries: parsed.queries,
880
+ query: queriesUsed[0],
881
+ queries: queriesUsed,
399
882
  pages: fetchedPages,
400
883
  sources: candidates
401
884
  .filter((source) => !fetchedUrls.has(source.url) && !failedUrls.has(source.url))
@@ -447,7 +930,7 @@ function report(options, error) {
447
930
  export function registerResearchTool(tools, search, fetchPage, options = {}) {
448
931
  const name = pickResearchToolName(tools);
449
932
  try {
450
- const registered = tools.register(createResearchDefinition(name, search, fetchPage));
933
+ const registered = tools.register(createResearchDefinition(name, search, fetchPage, options));
451
934
  const disposeHost = typeof registered === 'function' ? registered : undefined;
452
935
  let disposed = false;
453
936
  return {
@@ -12,7 +12,7 @@
12
12
  *
13
13
  * @module @fangchenghao/dsh-web-search/search-many
14
14
  */
15
- import type { Source } from './types.ts';
15
+ import type { EngineFailure, Source } from './types.ts';
16
16
  import type { ToolContent, ToolJsonSchema } from './domain-tool.ts';
17
17
  /** The name this tool prefers. */
18
18
  export declare const SEARCH_MANY_TOOL_NAME = "search_many";
@@ -40,6 +40,8 @@ export interface SearchManyResult {
40
40
  sources: SearchManySource[];
41
41
  /** Failures for this query alone; a thrown search becomes one entry here. */
42
42
  failures: string[];
43
+ /** Short policy note, such as why the fallback tier was used or skipped. */
44
+ note?: string;
43
45
  }
44
46
  /** What the tool returns to the model. */
45
47
  export interface SearchManyValue {
@@ -51,6 +53,10 @@ export interface SearchManyValue {
51
53
  export interface SearchManyOutcome {
52
54
  sources: Source[];
53
55
  failures: string[];
56
+ /** Structured failures from the engine layer, when it reported any. */
57
+ failureDetails?: EngineFailure[];
58
+ /** Short policy note, when the fallback policy changed the answer. */
59
+ note?: string;
54
60
  /** True when the deployment's configured limit cut the requested budget. */
55
61
  truncated: boolean;
56
62
  }