@fffangchenghao/dsh-web-search 0.14.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/client.js CHANGED
@@ -46,6 +46,8 @@
46
46
  /** Loader entry id the Host projects this plugin's Config under. */
47
47
  var ENTRY = 'web-search-bing';
48
48
  var PACKAGE_NAME = 'dsh-web-search-bing';
49
+ /** Keep in step with package.json; compared with the npm registry. */
50
+ var VERSION = '0.16.0';
49
51
  /** Snapshot used while no configuration service is reachable. */
50
52
  var EMPTY_STATE = { status: 'unavailable', value: undefined, base: undefined, user: undefined, revision: undefined, writable: false };
51
53
 
@@ -53,6 +55,10 @@
53
55
  var en = {
54
56
  title: 'Web search (Bing)',
55
57
  description: 'Edit the key-free search provider: engine allow-list, wait policy, proxy, result budget, ranking, cache, cooldowns and dedicated sources.',
58
+ jumpTo: 'Jump to',
59
+ latest: 'Latest on npm',
60
+ upToDate: 'up to date',
61
+ updateAvailable: 'update available',
56
62
  'group.engines': 'Engines and transport',
57
63
  'group.ranking': 'Results and ranking',
58
64
  'group.cache': 'Cache and cooldowns',
@@ -118,7 +124,7 @@
118
124
  'field.cacheFile': 'Cache file',
119
125
  'hint.cacheFile': 'Empty uses ~/.dsh/cache/dsh-web-search/cache.json',
120
126
  'field.anysearchApiKey': 'AnySearch key',
121
- 'hint.anysearchApiKey': 'Leave empty to stay anonymous at a lower rate limit; only sent to api.anysearch.com',
127
+ 'hint.anysearchApiKey': 'Leave empty to stay anonymous at a lower rate limit; only sent to api.anysearch.com. Separate several keys with commas and the plugin rotates them.',
122
128
  'field.anysearchDomain': 'Vertical domain',
123
129
  'hint.anysearchDomain': 'Restrict to one vertical (finance, academic, code…); leave empty normally',
124
130
  'field.baseCooldownMs': 'Cooldown (ms)',
@@ -126,7 +132,7 @@
126
132
  'field.maxCooldownMs': 'Max cooldown (ms)',
127
133
  'hint.maxCooldownMs': 'Ceiling for repeated throttling (ms); rarely worth changing',
128
134
  'field.domainSources': 'Dedicated sources',
129
- 'hint.domainSources': 'github, npm, pypi, arxiv, wikipedia, hn, stackexchange, crates; empty means all eight',
135
+ 'hint.domainSources': 'Eight are on by default (github, npm, pypi, arxiv, wikipedia, hn, stackexchange, crates); 26 more exist. Empty means the default eight.',
130
136
  'field.domainLimit': 'Results per source',
131
137
  'hint.domainLimit': 'How many results each dedicated source returns (1-20); 5 is a sane default',
132
138
  'field.domainTimeoutMs': 'Source wait limit (ms)',
@@ -137,6 +143,10 @@
137
143
  // Simplified Chinese copy.
138
144
  var zh = {
139
145
  title: '网页搜索(Bing)',
146
+ jumpTo: '跳到',
147
+ latest: 'npm 最新版',
148
+ upToDate: '已是最新',
149
+ updateAvailable: '有新版本',
140
150
  description: '编辑免密钥搜索提供方:引擎白名单、等待策略、代理、结果上限、排序、缓存、冷却与专用检索源。',
141
151
  'group.engines': '引擎与传输',
142
152
  'group.ranking': '结果与排序',
@@ -203,7 +213,7 @@
203
213
  'field.cacheFile': '缓存文件路径',
204
214
  'hint.cacheFile': '留空使用 ~/.dsh/cache/dsh-web-search/cache.json',
205
215
  'field.anysearchApiKey': 'AnySearch 通行证',
206
- 'hint.anysearchApiKey': '没有就留空,仍可用但速率较低;只发给 api.anysearch.com',
216
+ 'hint.anysearchApiKey': '没有就留空,仍可用但速率较低;只发给 api.anysearch.com。有多个 key 用逗号分隔,插件会自动轮换。',
207
217
  'field.anysearchDomain': '专业领域',
208
218
  'hint.anysearchDomain': '只搜某类专业内容(finance/academic/code 等);日常留空',
209
219
  'field.baseCooldownMs': '被拒后歇多久',
@@ -211,7 +221,7 @@
211
221
  'field.maxCooldownMs': '歇息上限',
212
222
  'hint.maxCooldownMs': '反复被拒时歇息时间的上限(毫秒);一般不用改',
213
223
  'field.domainSources': '默认查哪些专用源',
214
- 'hint.domainSources': 'github、npm、pypi、arxiv、wikipedia、hn、stackexchange、crates;留空查全部',
224
+ 'hint.domainSources': '默认开八个(github、npm、pypi、arxiv、wikipedia、hn、stackexchange、crates),另有 26 个可用;留空即默认八个。',
215
225
  'field.domainLimit': '每个源返回几条',
216
226
  'hint.domainLimit': '每个专用源最多返回几条(1-20);默认 5 条即可',
217
227
  'field.domainTimeoutMs': '每个源等待上限',
@@ -466,6 +476,27 @@
466
476
  var errorPair = React.useState({});
467
477
  var errors = errorPair[0];
468
478
  var setErrors = errorPair[1];
479
+ var latestPair = React.useState(null);
480
+ var latest = latestPair[0];
481
+ var setLatest = latestPair[1];
482
+
483
+ // Ask the npm registry once per mount. A failure (offline, proxy
484
+ // down, registry error) hides the line instead of showing an
485
+ // invented version; the page must never lie about an update.
486
+ React.useEffect(function () {
487
+ var alive = true;
488
+ try {
489
+ fetch('https://registry.npmjs.org/@fffangchenghao%2Fdsh-web-search/latest', { headers: { accept: 'application/json' } })
490
+ .then(function (response) { return response.ok ? response.json() : null; })
491
+ .then(function (body) {
492
+ if (alive && body !== null && typeof body.version === 'string') setLatest(body.version);
493
+ })
494
+ .catch(function () {});
495
+ } catch (ignored) {
496
+ /* fetch unavailable: the page simply omits the line */
497
+ }
498
+ return function () { alive = false; };
499
+ }, []);
469
500
 
470
501
  React.useEffect(function () {
471
502
  setDraft(null);
@@ -526,6 +557,14 @@
526
557
  h('h3', { key: 'title', style: { margin: 0, fontSize: '16px', lineHeight: 1.4, color: 'var(--dsw-alias-label-primary)' } }, t('title')),
527
558
  h('p', { key: 'description', style: { margin: 0, fontSize: '13px', lineHeight: 1.5, color: 'var(--dsw-alias-label-tertiary)' } }, t('description'))
528
559
  ];
560
+ if (latest !== null) {
561
+ var newer = latest !== VERSION;
562
+ children.push(h('p', {
563
+ key: 'latest',
564
+ style: { margin: 0, fontSize: '12px', color: newer ? 'var(--dsw-alias-state-warn-primary)' : 'var(--dsw-alias-label-tertiary)' },
565
+ role: 'status'
566
+ }, t('latest') + ': ' + latest + (newer ? ' — ' + t('updateAvailable') : ' — ' + t('upToDate'))));
567
+ }
529
568
 
530
569
  if (!ready) {
531
570
  if (!surface.hasService()) {
@@ -545,8 +584,36 @@
545
584
  children.push(h('p', { key: 'readonly', style: { margin: 0, fontSize: '12px', color: 'var(--dsw-alias-label-tertiary)' } }, t('readOnly')));
546
585
  }
547
586
 
587
+ var jump = GROUPS.map(function (group) {
588
+ return h('a', {
589
+ key: group.key,
590
+ href: '#dsh-ws-group-' + group.key,
591
+ onClick: function (event) {
592
+ event.preventDefault();
593
+ try {
594
+ var target = document.getElementById('dsh-ws-group-' + group.key);
595
+ if (target !== null) target.scrollIntoView({ behavior: 'smooth', block: 'start' });
596
+ } catch (ignored) {
597
+ /* older engines still jump through the href */
598
+ }
599
+ },
600
+ style: { fontSize: '12px', color: 'var(--dsw-alias-label-secondary)', textDecoration: 'none', whiteSpace: 'nowrap' }
601
+ }, t('group.' + group.key));
602
+ });
603
+ children.push(h('nav', {
604
+ key: 'toc',
605
+ 'aria-label': t('jumpTo'),
606
+ style: {
607
+ position: 'sticky', top: 0, zIndex: 1,
608
+ display: 'flex', alignItems: 'center', gap: '10px', flexWrap: 'wrap',
609
+ padding: '6px 0',
610
+ background: 'var(--dsw-alias-bg-base)',
611
+ borderBottom: '1px solid var(--dsw-alias-border-l1)'
612
+ }
613
+ }, [h('span', { key: 'label', style: { fontSize: '11px', color: 'var(--dsw-alias-label-tertiary)' } }, t('jumpTo'))].concat(jump)));
614
+
548
615
  children.push(h('div', { key: 'groups', style: { display: 'flex', flexDirection: 'column', gap: '12px' } }, GROUPS.map(function (group) {
549
- return h('section', { key: group.key, style: groupStyle }, [
616
+ return h('section', { key: group.key, id: 'dsh-ws-group-' + group.key, style: groupStyle }, [
550
617
  h('h4', { key: 'title', style: { margin: 0, fontSize: '13px', lineHeight: 1.4, color: 'var(--dsw-alias-label-primary)' } }, t('group.' + group.key)),
551
618
  h('div', { key: 'grid', style: gridStyle }, group.fields.map(function (field) {
552
619
  var key = field.key;
@@ -14,7 +14,9 @@ measurement. This page is both.
14
14
 
15
15
  1. **Fan out.** Every enabled engine is queried concurrently with its own
16
16
  deadline (`timeoutMs`, default 15 s) and its own cooldown state. An engine
17
- that just failed is skipped instead of being retried on every query.
17
+ that just failed is skipped instead of being retried on every query. The
18
+ primary tier answers first; the fallback tier only runs when every primary
19
+ came back empty.
18
20
  2. **Normalise.** Each parser returns `{title, url, snippet, position, votes}`
19
21
  with the engine's own result order preserved. URLs are canonicalised before
20
22
  de-duplication so `?utm_source=…` cannot smuggle the same page in twice.
@@ -75,16 +77,18 @@ comparable across commits; `--record` is the only mode that touches the network.
75
77
 
76
78
  ![Baseline and sweep](images/bench.png)
77
79
 
78
- Baseline (v0.13.0, 20 queries, k=8):
80
+ Baseline (v0.15.0, 20 queries, k=8, tier-aware replay):
79
81
 
80
82
  | metric | overall | Chinese | English |
81
83
  |---|---:|---:|---:|
82
- | P@8 | 0.3625 | 0.3250 | 0.4000 |
83
- | R@8 | 0.7417 | 0.7333 | 0.7500 |
84
- | MRR@8 | 0.7600 | 0.6700 | 0.8500 |
84
+ | P@8 | 0.4375 | 0.4000 | 0.4750 |
85
+ | R@8 | 0.7917 | 0.8167 | 0.7667 |
86
+ | MRR@8 | 0.8071 | 0.7976 | 0.8167 |
85
87
 
86
- One query (`zh-linux-612`) returns zero expected domains; that is visible in
87
- the output on purpose rather than averaged away.
88
+ Replay is tier-aware: it applies the same primary-then-fallback gate the live
89
+ `search()` applies, instead of ranking every recorded engine as if it had
90
+ answered. That changed the protocol in 0.15.0, so numbers from earlier releases
91
+ (0.7600 MRR) are not directly comparable to these.
88
92
 
89
93
  ## Parameter sweep
90
94
 
@@ -97,24 +101,35 @@ node scripts/sweep.mjs --quick # 27-config subset
97
101
  node scripts/sweep.mjs --json # includes the per-config CI data
98
102
  ```
99
103
 
100
- Marginal MRR@8 (all other knobs at defaults):
104
+ Marginal means over the whole grid (every other knob varies with it):
101
105
 
102
106
  | `rrfConstant` | MRR@8 | `authorityBoost` | MRR@8 | `freshnessBoost` | MRR@8 |
103
107
  |---:|---:|---:|---:|---:|---:|
104
- | 40 | 0.7712 | 0.1 | 0.7733 | 0.1 | 0.7681 |
105
- | **60** | **0.7604** | **0.25** | **0.7645** | **0.2** | **0.7598** |
106
- | 80 | 0.7456 | 0.4 | 0.7395 | 0.35 | 0.7494 |
107
-
108
- The best grid point (`rrf=40, authority=0.1, freshness=0.1`) measures MRR 0.8042
109
- against the default 0.7600 — a gain of +0.0442 — but its 95 % paired-bootstrap
110
- interval is `[-0.0038, +0.1229]`, which straddles zero. The entire gain comes
111
- from two of twenty queries. That is a promising direction, not a proven win, so
112
- the defaults did not change. The sweep is kept as a script precisely so this
113
- question can be re-asked after the query set grows.
114
-
115
- The same discipline applied to engine weights: dropping `cn.bing` to weight 0.25
116
- and raising `ddg-html` to 2 measured MRR 0.7708 with a lower bound of 0.0000 —
117
- inside noise, so it was not adopted.
108
+ | 40 | 0.8339 | 0.1 | 0.8275 | 0.1 | 0.8230 |
109
+ | **60** | **0.8278** | **0.25** | **0.8262** | **0.2** | **0.8245** |
110
+ | 80 | 0.8195 | 0.4 | 0.8275 | 0.35 | 0.8336 |
111
+
112
+ The best grid point (`rrf=40, authority=0.25, freshness=0.35`) measures MRR
113
+ 0.8583 against the default 0.8071 — a gain of +0.0512 — but its 95 %
114
+ paired-bootstrap interval `[0.0000, +0.1250]` touches zero, and the marginals
115
+ above differ from the default by less than the interval width. On a 20-query
116
+ corpus that is a direction, not a proven win, so the defaults did not change.
117
+ The sweep is kept as a script precisely so this question can be re-asked after
118
+ the query set grows.
119
+
120
+ The same discipline decided the new engines' tiers. On one recorded fixture
121
+ set, with the tier-aware replay:
122
+
123
+ | configuration | MRR@8 |
124
+ |---|---:|
125
+ | previous primaries only | 0.7738 |
126
+ | **previous primaries + Exa** | **0.8071** |
127
+ | previous primaries + Firecrawl | 0.7208 |
128
+ | previous primaries + both | 0.7542 |
129
+
130
+ Exa therefore leads; Firecrawl is parsed and tested but ships as the fallback,
131
+ available through `hostIds: ['firecrawl']` and used whenever every primary
132
+ returns nothing.
118
133
 
119
134
  ## Known limits
120
135
 
@@ -122,6 +137,9 @@ inside noise, so it was not adopted.
122
137
  constraint in the words themselves. The freshness boost can only act on dates
123
138
  it can parse out of snippets; when an engine's snippet omits the date, the
124
139
  ranker cannot recover it. This is documented rather than papered over.
140
+ - **Replay, not live search.** The bench replays recorded rows, so it measures
141
+ the ranker and the fusion, not today's engine availability. `--record` is the
142
+ only mode that touches the network.
125
143
  - **Snippet quality bounds ranking.** If no engine returns the expected domain
126
144
  in its top 20, fusion cannot invent it; `bench/` makes that visible per query.
127
145
  - **The fixture set is 20 queries.** Small enough to run on every change, too
package/lib/config.js CHANGED
@@ -21,7 +21,7 @@
21
21
  */
22
22
  import z from '@deepseek-ai/schemastery';
23
23
  import { DEFAULT_FETCH_MAX_LENGTH } from "./markdown.js";
24
- import { VERTICAL_IDS, isVerticalId } from "./verticals.js";
24
+ import { DEFAULT_VERTICAL_IDS, VERTICAL_IDS, isVerticalId } from "./verticals.js";
25
25
  /** Schema of the plugin's configuration section. */
26
26
  export const Config = z.object({
27
27
  hosts: z
@@ -92,7 +92,7 @@ export const Config = z.object({
92
92
  anysearchApiKey: z
93
93
  .string()
94
94
  .default('')
95
- .description('AnySearch 通行证(API Key):没有就留空,能用但慢;填了更稳且只发 api.anysearch.com。').volatile(),
95
+ .description('AnySearch 通行证(API Key):留空则匿名(慢一些);填了更稳且只发 api.anysearch.com。有多个 key 用逗号分隔,插件会轮流使用,某个被限流就自动换下一个。').volatile(),
96
96
  anysearchDomain: z
97
97
  .string()
98
98
  .default('')
@@ -101,8 +101,8 @@ export const Config = z.object({
101
101
  maxCooldownMs: z.natural().default(1800000).description('它反复被拒时,最长让它歇多久(毫秒),也就是等待一步步加长的封顶值。一般不用改;保持 1800000(30 分钟)即可。').volatile(),
102
102
  domainSources: z
103
103
  .union([z.array(z.string()), z.string()])
104
- .default([...VERTICAL_IDS])
105
- .description('默认查哪些专用源,逗号分隔;留空表示 github、npm 等全部八个源。')
104
+ .default([...DEFAULT_VERTICAL_IDS])
105
+ .description('默认查哪些专用源,逗号分隔;留空表示默认的 8 个(github、npm、pypi、arxiv、wikipedia、hn、stackexchange、crates)。其余源可用 sources 参数按需指定。')
106
106
  .volatile(),
107
107
  domainLimit: z.natural().min(1).max(20).default(5).description('domain_search 一次最多返回几条(1-20);默认 5 条,想多看可调到 10。').volatile(),
108
108
  domainTimeoutMs: z.natural().min(500).default(8000).description('每个专用源最多等多久(毫秒);网络慢可加到 15000,平时 8000 即可。').volatile(),
@@ -167,7 +167,7 @@ export function resolveConfig(config, callLimit) {
167
167
  timeRange: settings.timeRange,
168
168
  anysearchApiKey: settings.anysearchApiKey.trim(),
169
169
  anysearchDomain: settings.anysearchDomain.trim(),
170
- domainSourceIds: domainSources.length > 0 ? domainSources : [...VERTICAL_IDS],
170
+ domainSourceIds: domainSources.length > 0 ? domainSources : [...DEFAULT_VERTICAL_IDS],
171
171
  domainLimit: settings.domainLimit,
172
172
  domainTimeoutMs: settings.domainTimeoutMs,
173
173
  rank: {
@@ -1,5 +1,5 @@
1
1
  import { resolveProxy } from "./proxy.js";
2
- import { searchVerticals, isVerticalId, VERTICAL_IDS } from "./verticals.js";
2
+ import { DEFAULT_VERTICAL_IDS, searchVerticals, isVerticalId, VERTICAL_IDS } from "./verticals.js";
3
3
  /** The name this tool prefers. */
4
4
  export const DOMAIN_TOOL_NAME = 'domain_search';
5
5
  /** The name used when another plugin already registered `domain_search`. */
@@ -70,7 +70,7 @@ function parseArgs(args, settings) {
70
70
  if (query.length === 0)
71
71
  throw new Error('domain_search requires a non-empty query');
72
72
  const enabled = settings.sourceIds.filter(isVerticalId);
73
- let ids = enabled.length > 0 ? enabled : [...VERTICAL_IDS];
73
+ let ids = enabled.length > 0 ? enabled : [...DEFAULT_VERTICAL_IDS];
74
74
  if (Array.isArray(args.sources) && args.sources.length > 0) {
75
75
  const asked = args.sources.filter(isVerticalId);
76
76
  if (asked.length === 0) {
package/lib/engines.js CHANGED
@@ -1,10 +1,11 @@
1
1
  import { SearchError } from "./errors.js";
2
2
  import { filterByTimeRange, preferWindow } from "./freshness.js";
3
3
  import { resolveProxy } from "./proxy.js";
4
- import { parseAnySearch, parseBing, parseDdgHtml, parseDdgLite, parseSo } from "./parsers.js";
4
+ import { parseAnySearch, parseBing, parseDdgHtml, parseDdgLite, parseExaMcp, parseFirecrawl, parseSo } from "./parsers.js";
5
5
  import { mergeRegions } from "./rank.js";
6
6
  import { cacheKey } from "./cache.js";
7
7
  import { searchHost } from "./transport.js";
8
+ import { keyFailureKind, keyPoolFor } from "./keys.js";
8
9
  /** How many rows to ask AnySearch for; the merge picks from them. */
9
10
  // AnySearch caps max_results at 10: asking for more returns 10 with HTTP
10
11
  // 200, so the constant matches the real ceiling instead of over-asking.
@@ -35,6 +36,48 @@ export const HOSTS = [
35
36
  body: (query) => JSON.stringify({ query, max_results: ANYSEARCH_MAX_RESULTS }),
36
37
  parse: parseAnySearch,
37
38
  },
39
+ {
40
+ id: 'exa',
41
+ // A semantic (embedding) index rather than a keyword page: it answers with
42
+ // the page itself plus highlights. The MCP endpoint accepts anonymous
43
+ // calls, so it costs no key. Measured on the same recorded fixtures with the
44
+ // tier-aware replay, MRR@8 0.8071 with Exa in the primary tier versus
45
+ // 0.7738 without it, and it removes the one zero-hit query.
46
+ tier: 'primary',
47
+ region: 'global',
48
+ weight: 1,
49
+ viaProxy: true,
50
+ method: 'POST',
51
+ skipEcho: true,
52
+ headers: { accept: 'application/json, text/event-stream' },
53
+ build: () => 'https://mcp.exa.ai/mcp',
54
+ body: (query) => JSON.stringify({
55
+ jsonrpc: '2.0',
56
+ id: 1,
57
+ method: 'tools/call',
58
+ params: { name: 'web_search_exa', arguments: { query, numResults: 10 } },
59
+ }),
60
+ parse: parseExaMcp,
61
+ },
62
+ {
63
+ id: 'firecrawl',
64
+ // The keyless tier of the Firecrawl search API. It returns extracted page
65
+ // Markdown rather than snippets, so only the opening prose is kept as the
66
+ // row's snippet; it handled a Chinese query with the same shape. It ships as
67
+ // a fallback because it diluted the fusion when it led: MRR@8 0.7208 with
68
+ // Firecrawl in the primary tier versus 0.8071 with Exa there instead, on the
69
+ // same fixtures (standalone 0.4625). `hostIds: ['firecrawl']` still forces
70
+ // it, and it answers whenever every primary came back empty.
71
+ tier: 'fallback',
72
+ region: 'both',
73
+ weight: 1,
74
+ viaProxy: true,
75
+ method: 'POST',
76
+ skipEcho: true,
77
+ build: () => 'https://api.firecrawl.dev/v1/search',
78
+ body: (query) => JSON.stringify({ query, limit: 10 }),
79
+ parse: parseFirecrawl,
80
+ },
38
81
  {
39
82
  id: 'ddg-html',
40
83
  tier: 'primary',
@@ -130,6 +173,51 @@ const SOFT_DEADLINE = {
130
173
  function leadsQuery(host, lead) {
131
174
  return host.region === 'both' || host.region === lead;
132
175
  }
176
+ /**
177
+ * Ask one engine, rotating to the next key when the failure blames the key.
178
+ *
179
+ * A 401 or 429 from the one credentialed engine is the only failure a second
180
+ * attempt can fix; an engine that timed out or lost the network will fail the
181
+ * same way with a different token. At most two keys are tried per engine per
182
+ * tier, which bounds the added latency to one extra request.
183
+ * @param host - the engine definition.
184
+ * @param query - the user query.
185
+ * @param signal - caller cancellation.
186
+ * @param timeoutMs - per-engine deadline.
187
+ * @param proxy - tunnel URL, or `undefined`.
188
+ * @param timeRange - active publication window.
189
+ * @param pool - configured keys, possibly empty.
190
+ * @param domain - AnySearch vertical domain, if configured.
191
+ * @returns the last attempt's outcome.
192
+ */
193
+ async function searchHostWithKeys(host, query, signal, timeoutMs, proxy, timeRange, pool, domain) {
194
+ if (host.id !== 'anysearch' || pool.size === 0) {
195
+ return searchHost(host, query, signal, timeoutMs, proxy, timeRange);
196
+ }
197
+ let key = pool.pick();
198
+ let last;
199
+ for (let attempt = 0; attempt < 2 && key !== undefined; attempt += 1) {
200
+ const configured = resolveHost(host, {
201
+ anysearchApiKey: key,
202
+ ...(domain === undefined ? {} : { anysearchDomain: domain }),
203
+ });
204
+ const outcome = await searchHost(configured, query, signal, timeoutMs, proxy, timeRange);
205
+ if (outcome.ok) {
206
+ pool.reward(key);
207
+ return outcome;
208
+ }
209
+ last = outcome;
210
+ const kind = keyFailureKind(outcome.error);
211
+ if (kind === undefined || signal?.aborted === true)
212
+ return outcome;
213
+ pool.penalize(key, kind);
214
+ const next = pool.pick();
215
+ if (next === undefined || next === key)
216
+ return outcome;
217
+ key = next;
218
+ }
219
+ return last;
220
+ }
133
221
  /**
134
222
  * Run one tier concurrently and return the answers that arrived in time.
135
223
  * @param hosts - engines of this tier, in priority order.
@@ -142,7 +230,7 @@ function leadsQuery(host, lead) {
142
230
  * @param health - cooldown tracker to update, when one is configured.
143
231
  * @returns one answer per engine that produced sources.
144
232
  */
145
- async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits, health) {
233
+ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits, health, pool, anysearchDomain) {
146
234
  const counters = limits.lead === 'all' || !hosts.some((host) => leadsQuery(host, limits.lead))
147
235
  ? () => true
148
236
  : (host) => leadsQuery(host, limits.lead);
@@ -173,7 +261,7 @@ async function runTier(hosts, query, signal, timeoutMs, failures, proxy, limits,
173
261
  releasedEarly = early;
174
262
  released();
175
263
  };
176
- const running = hosts.map((host, index) => searchHost(host, query, controller.signal, timeoutMs, proxy, limits.timeRange).then((outcome) => {
264
+ const running = hosts.map((host, index) => searchHostWithKeys(host, query, controller.signal, timeoutMs, proxy, limits.timeRange, pool, anysearchDomain).then((outcome) => {
177
265
  if (outcome.ok) {
178
266
  outcomes[index] = { host, sources: outcome.sources };
179
267
  health?.noteSuccess(host.id);
@@ -275,10 +363,12 @@ export async function search(query, signal, options) {
275
363
  const wait = options.wait ?? 'auto';
276
364
  const timeRange = options.timeRange ?? 'off';
277
365
  const proxy = options.useProxy === false ? undefined : await resolveProxy(options.proxy);
366
+ const pool = keyPoolFor(options.anysearchApiKey);
367
+ const anysearchDomain = (options.anysearchDomain ?? '').trim();
278
368
  const allow = options.hostIds !== undefined && options.hostIds.length > 0 ? new Set(options.hostIds) : undefined;
279
369
  const hosts = HOSTS.filter((host) => allow === undefined || allow.has(host.id))
280
370
  .filter((host) => host.viaProxy !== true || proxy !== undefined)
281
- .map((host) => resolveHost(host, options));
371
+ .map((host) => resolveHost(host, options.anysearchDomain === undefined ? {} : { anysearchDomain: options.anysearchDomain }));
282
372
  // Skip engines that are cooling down, but never end up with no engine at all:
283
373
  // a throttled engine is still better than no search.
284
374
  const now = Date.now();
@@ -343,13 +433,13 @@ export async function search(query, signal, options) {
343
433
  const primary = ordered.filter((host) => host.tier === 'primary');
344
434
  const fallback = ordered.filter((host) => host.tier === 'fallback');
345
435
  let tier = 'none';
346
- let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health));
436
+ let answers = applyWindow(await runTier(primary, query, signal, timeoutMs, failures, proxy, limits, options.health, pool, anysearchDomain));
347
437
  if (answers.length > 0)
348
438
  tier = 'primary';
349
439
  let sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
350
440
  if (sources.length === 0 && fallback.length > 0) {
351
441
  tier = 'fallback';
352
- answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health));
442
+ answers = applyWindow(await runTier(fallback, query, signal, timeoutMs, failures, proxy, limits, options.health, pool, anysearchDomain));
353
443
  sources = answers.length === 0 ? [] : preferWindow(mergeRegions(answers, lead, limit, rank), timeRange);
354
444
  }
355
445
  if (signal?.aborted === true)
package/lib/fetch.js CHANGED
@@ -33,6 +33,7 @@
33
33
  * @module @fangchenghao/dsh-web-search/fetch
34
34
  */
35
35
  import { SearchError } from "./errors.js";
36
+ import { keyFailureKind, keyPoolFor } from "./keys.js";
36
37
  import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
37
38
  import { proxyGet, resolveProxy } from "./proxy.js";
38
39
  import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
@@ -155,6 +156,45 @@ async function extractFetch(url, apiKey, timeoutMs, signal) {
155
156
  const title = typeof payload.data.title === 'string' && payload.data.title.length > 0 ? `# ${payload.data.title}\n\n` : '';
156
157
  return { url, statusCode: 200, body: { kind: 'text', content: `${title}${payload.data.content}` }, truncated: false };
157
158
  }
159
+ /**
160
+ * Run AnySearch extraction, rotating to the next key when the failure blames
161
+ * the key rather than the page.
162
+ *
163
+ * A 401 means this key will never work again; a 429 means its quota needs a
164
+ * rest. Both are key-level facts the next fetch must remember, which is why the
165
+ * pool outlives this call instead of living in the provider closure.
166
+ * @param extract - transport hook.
167
+ * @param url - target page.
168
+ * @param pool - configured keys.
169
+ * @param budgetMs - remaining overall budget for this rung.
170
+ * @param signal - caller cancellation.
171
+ * @returns the extraction outcome.
172
+ */
173
+ async function extractWithKeys(extract, url, pool, budgetMs, signal) {
174
+ let key = pool.pick();
175
+ if (key === undefined)
176
+ throw new SearchError('every configured AnySearch key is resting', 'SEARCH_THROTTLED', true);
177
+ let last;
178
+ for (let attempt = 0; attempt < 2 && key !== undefined; attempt += 1) {
179
+ try {
180
+ const outcome = await extract(url, key, budgetMs, signal);
181
+ pool.reward(key);
182
+ return outcome;
183
+ }
184
+ catch (error) {
185
+ last = error;
186
+ const kind = keyFailureKind(error instanceof SearchError ? error : { message: error instanceof Error ? error.message : String(error) });
187
+ if (kind === undefined || signal?.aborted === true)
188
+ throw error;
189
+ pool.penalize(key, kind);
190
+ const next = pool.pick();
191
+ if (next === undefined || next === key)
192
+ throw error;
193
+ key = next;
194
+ }
195
+ }
196
+ throw last;
197
+ }
158
198
  /**
159
199
  * Build the fetch provider.
160
200
  * @param options - timeouts, tunnel and extraction settings.
@@ -174,7 +214,7 @@ export function createFetchProvider(options = {}, overrides = {}) {
174
214
  const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
175
215
  const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
176
216
  const userAgent = options.userAgent ?? USER_AGENT;
177
- const apiKey = (options.anysearchApiKey ?? '').trim();
217
+ const keyPool = keyPoolFor(options.anysearchApiKey);
178
218
  /**
179
219
  * Resolve the two volatile Settings fields per request. Reading them here
180
220
  * rather than at construction is what lets a saved edit apply without a
@@ -430,12 +470,12 @@ export function createFetchProvider(options = {}, overrides = {}) {
430
470
  if (!step.done)
431
471
  failures.push(`render: ${step.report.reason}`);
432
472
  }
433
- if (apiKey.length > 0) {
473
+ if (keyPool.size > 0) {
434
474
  try {
435
475
  const extractBudget = Math.min(timeoutMs, remaining());
436
476
  if (extractBudget < MIN_RUNG_MS)
437
477
  throw new Error('no time left in the fetch budget');
438
- const extracted = await deps.extract(target, apiKey, extractBudget, signal);
478
+ const extracted = await extractWithKeys(deps.extract, target, keyPool, extractBudget, signal);
439
479
  // Say where the body came from when the browser was tried first.
440
480
  const report = lastReport !== undefined && lastReport.attempted && !lastReport.applied
441
481
  ? makeReport(lastReport.mode, true, false, `${lastReport.reason}; the result came from AnySearch extraction`)
package/lib/index.d.ts CHANGED
@@ -138,12 +138,14 @@ export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySe
138
138
  export { fuse, interleave, dedupe, mergeRegions } from './rank.ts';
139
139
  export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublishedAt, publishedTimestamp, isWithinTimeRange, filterByTimeRange, } from './freshness.ts';
140
140
  export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from './proxy.ts';
141
- export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from './verticals.ts';
141
+ export { DEFAULT_VERTICAL_IDS, VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from './verticals.ts';
142
142
  export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from './domain-tool.ts';
143
143
  export type { VerticalId, VerticalSpec, VerticalOutcome, VerticalSearchOptions } from './verticals.ts';
144
144
  export type { DomainToolSettings, DomainToolValue, DomainToolDefinition, ToolRegistryLike } from './domain-tool.ts';
145
145
  export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from './search-many.ts';
146
146
  export type { SearchManyDefinition, SearchManyOutcome, SearchManyRegistration, SearchManyRegistryLike, SearchManyResult, SearchManyRunner, SearchManySource, SearchManyValue, } from './search-many.ts';
147
+ export { RESEARCH_TOOL_NAME, RESEARCH_TOOL_FALLBACK_NAME, RESEARCH_DEFAULT_SOURCES, RESEARCH_MAX_SOURCES, RESEARCH_MAX_QUERIES, RESEARCH_EXCERPT_CHARS, RESEARCH_BUDGET_MS, createResearchDefinition, registerResearchTool, pickResearchToolName, formatResearchValue, } from './research.ts';
148
+ export type { ResearchDefinition, ResearchFetcher, ResearchFetchedPage, ResearchFetchFailure, ResearchPage, ResearchRegistration, ResearchRegistryLike, ResearchSearcher, ResearchSearchOutcome, ResearchValue, } from './research.ts';
147
149
  export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from './read-page.ts';
148
150
  export type { ReadPageDefinition, ReadPageOptions, ReadPageRegistration, ReadPageRegistryLike, ReadPageSettings, ReadPageValue, } from './read-page.ts';
149
151
  export type { Source, SearchOutcome, RankOptions, SearchOptions, HostSpec, Region, TimeRange, WaitPolicy } from './types.ts';
package/lib/index.js CHANGED
@@ -17,6 +17,7 @@ import { createFetchProvider } from "./fetch.js";
17
17
  import { registerDomainTool } from "./domain-tool.js";
18
18
  import { registerSearchManyTool } from "./search-many.js";
19
19
  import { registerReadPageTool } from "./read-page.js";
20
+ import { registerResearchTool } from "./research.js";
20
21
  export const name = 'dsh-web-search';
21
22
  /** The `ctx.web` service this plugin registers a search provider with. */
22
23
  export const inject = ['web'];
@@ -214,6 +215,33 @@ export function apply(ctx, config = {}) {
214
215
  if (paged !== undefined) {
215
216
  onDispose(ctx, paged.dispose);
216
217
  }
218
+ // Research: search -> fetch -> cite in one call. The fetch side reuses the
219
+ // same ladder as `web_fetch`, but through its own provider: the renderer
220
+ // load is cached per provider, and a research pack must not replace the
221
+ // page the host already fetched for the user.
222
+ const researchProvider = createFetchProvider({
223
+ id: 'http-proxy-research',
224
+ timeoutMs: mounted.timeoutMs,
225
+ format: 'markdown',
226
+ maxLength: 6_000,
227
+ renderJs: () => resolveConfig(section, undefined).renderJs,
228
+ renderTimeoutMs: () => resolveConfig(section, undefined).renderTimeoutMs,
229
+ ...(mounted.proxy === undefined ? {} : { proxy: mounted.proxy }),
230
+ ...(mounted.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: mounted.anysearchApiKey }),
231
+ });
232
+ const researchFetch = async (url, signal) => {
233
+ const outcome = await researchProvider.fetch({ url }, signal);
234
+ return {
235
+ url: outcome.url,
236
+ status: outcome.statusCode,
237
+ text: outcome.body.kind === 'text' ? outcome.body.content : '',
238
+ truncated: outcome.truncated,
239
+ };
240
+ };
241
+ const research = registerResearchTool(tools, (query, limit, signal) => runQuery(query, limit, signal), researchFetch, { warn: warnTool });
242
+ if (research !== undefined) {
243
+ onDispose(ctx, research.dispose);
244
+ }
217
245
  }
218
246
  }
219
247
  catch (error) {
@@ -244,7 +272,8 @@ export { resolveHref, parseDdgHtml, parseDdgLite, parseBing, parseSo, parseAnySe
244
272
  export { fuse, interleave, dedupe, mergeRegions } from "./rank.js";
245
273
  export { MS_PER_DAY, TIME_RANGE_MS, TIME_RANGES, parsePublishedAt, extractPublishedAt, publishedTimestamp, isWithinTimeRange, filterByTimeRange, } from "./freshness.js";
246
274
  export { parseProxy, proxyFromEnv, resolveProxy, proxyGet } from "./proxy.js";
247
- export { VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
275
+ export { DEFAULT_VERTICAL_IDS, VERTICAL_IDS, VERTICALS, searchVerticals, verticalById, isVerticalId, parseGithub, parseNpm, parsePypi, parseArxiv, parseWikipedia, parseHn, parseStackExchange, parseCrates, } from "./verticals.js";
248
276
  export { DOMAIN_TOOL_NAME, DOMAIN_TOOL_FALLBACK_NAME, createDomainToolDefinition, registerDomainTool, formatDomainValue, } from "./domain-tool.js";
249
277
  export { SEARCH_MANY_TOOL_NAME, SEARCH_MANY_FALLBACK_NAME, SEARCH_MANY_CONCURRENCY, SEARCH_MANY_DEFAULT_LIMIT, createSearchManyDefinition, registerSearchManyTool, pickSearchManyToolName, formatSearchManyValue, } from "./search-many.js";
278
+ export { RESEARCH_TOOL_NAME, RESEARCH_TOOL_FALLBACK_NAME, RESEARCH_DEFAULT_SOURCES, RESEARCH_MAX_SOURCES, RESEARCH_MAX_QUERIES, RESEARCH_EXCERPT_CHARS, RESEARCH_BUDGET_MS, createResearchDefinition, registerResearchTool, pickResearchToolName, formatResearchValue, } from "./research.js";
250
279
  export { READ_PAGE_TOOL_NAME, READ_PAGE_TOOL_FALLBACK_NAME, READ_PAGE_MIN_LENGTH, READ_PAGE_MAX_LENGTH, READ_PAGE_DEFAULT_LENGTH, READ_PAGE_INTERNAL_MAX_CHARS, READ_PAGE_CACHE_ENTRIES, READ_PAGE_PROVIDER_ID, createReadPageDefinition, registerReadPageTool, pickReadPageToolName, formatReadPageValue, } from "./read-page.js";