@fffangchenghao/dsh-web-search 0.21.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,177 @@
1
+ import { proxyGet } from "./proxy.js";
2
+ import { text } from "./parsers.js";
3
+ /** Public GitHub release endpoint for the Rust repository. */
4
+ export const RUST_RELEASE_API = 'https://api.github.com/repos/rust-lang/rust/releases/tags';
5
+ /** Official Rust announcement host. */
6
+ export const RUST_BLOG_HOST = 'blog.rust-lang.org';
7
+ const DEFAULT_TIMEOUT_MS = 6_000;
8
+ /** Extract a Rust version from a release query. */
9
+ export function rustReleaseVersion(query) {
10
+ if (typeof query !== 'string' || !/\brust\b/iu.test(query))
11
+ return undefined;
12
+ const match = /\brust(?:\s+(?:release|version|v))?\s*([0-9]{1,2}\.[0-9]{1,2}(?:\.[0-9]{1,2})?)\b/iu.exec(query);
13
+ if (match?.[1] === undefined)
14
+ return undefined;
15
+ const raw = match[1];
16
+ const version = raw.split('.').length === 2 ? `${raw}.0` : raw;
17
+ return { version, tag: version };
18
+ }
19
+ /** True for a query asking for a specific Rust release's notes. */
20
+ export function isRustReleaseQuery(query) {
21
+ if (typeof query !== 'string')
22
+ return false;
23
+ if (rustReleaseVersion(query) === undefined)
24
+ return false;
25
+ return /(?:release|releases|changelog|notes?|announcement|发布|发行|版本|更新|说明|日志|公告)/iu.test(query);
26
+ }
27
+ /** Trim release Markdown/HTML into a bounded snippet. */
28
+ function releaseSnippet(body, version) {
29
+ const plain = text(typeof body === 'string' ? body : '')
30
+ .replace(/\[([^\]]+)\]\([^)]*\)/gu, '$1')
31
+ .replace(/https?:\/\/\S+/gu, ' ')
32
+ .replace(/\s+/gu, ' ')
33
+ .trim();
34
+ const prefix = `Official Rust ${version} release notes. `;
35
+ if (plain.length === 0)
36
+ return prefix.trim();
37
+ const budget = Math.max(80, 420 - prefix.length);
38
+ return `${prefix}${plain.length > budget ? `${plain.slice(0, budget).trimEnd()}…` : plain}`;
39
+ }
40
+ /**
41
+ * Build the canonical announcement URL from a GitHub release timestamp.
42
+ *
43
+ * Rust publishes the post on the UTC date in the release metadata. Using UTC
44
+ * avoids shifting the path when the machine is in a negative-offset timezone.
45
+ */
46
+ export function rustBlogUrl(version, publishedAt) {
47
+ const date = new Date(publishedAt);
48
+ if (Number.isNaN(date.getTime()))
49
+ return undefined;
50
+ const yyyy = String(date.getUTCFullYear()).padStart(4, '0');
51
+ const mm = String(date.getUTCMonth() + 1).padStart(2, '0');
52
+ const dd = String(date.getUTCDate()).padStart(2, '0');
53
+ return `https://${RUST_BLOG_HOST}/${yyyy}/${mm}/${dd}/Rust-${version}/`;
54
+ }
55
+ /**
56
+ * Parse a GitHub release response into one official-blog source.
57
+ *
58
+ * A malformed response, a missing publication date or a tag that does not
59
+ * match the requested version returns `undefined`; the caller then keeps the
60
+ * ordinary search rows.
61
+ *
62
+ * @param payload - GitHub JSON response body.
63
+ * @param requested - normalized version from the query.
64
+ * @returns the official source, or `undefined`.
65
+ */
66
+ export function parseRustRelease(payload, requested) {
67
+ if (typeof payload !== 'string' || payload.trim().length === 0)
68
+ return undefined;
69
+ let body;
70
+ try {
71
+ body = JSON.parse(payload);
72
+ }
73
+ catch {
74
+ return undefined;
75
+ }
76
+ if (typeof body !== 'object' || body === null)
77
+ return undefined;
78
+ const record = body;
79
+ const publishedAt = typeof record['published_at'] === 'string' ? record['published_at'] : '';
80
+ const url = rustBlogUrl(requested.version, publishedAt);
81
+ if (url === undefined)
82
+ return undefined;
83
+ const tag = typeof record['tag_name'] === 'string' ? record['tag_name'] : '';
84
+ if (tag.length > 0 && tag !== requested.tag && tag !== requested.version)
85
+ return undefined;
86
+ const title = `Announcing Rust ${requested.version} - Rust Blog`;
87
+ const snippet = releaseSnippet(record['body'], requested.version);
88
+ return {
89
+ url,
90
+ title,
91
+ snippet,
92
+ ...(Number.isNaN(new Date(publishedAt).getTime()) ? {} : { publishedAt: new Date(publishedAt).toISOString() }),
93
+ };
94
+ }
95
+ /** One direct GET with its own abort budget. */
96
+ async function directJson(url, headers, options, timeoutMs) {
97
+ const controller = new AbortController();
98
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
99
+ const abort = () => controller.abort();
100
+ options.signal?.addEventListener('abort', abort, { once: true });
101
+ try {
102
+ const send = options.fetch ?? fetch;
103
+ const response = await send(url, { headers, signal: controller.signal });
104
+ if (!response.ok)
105
+ throw new Error(`HTTP ${response.status}`);
106
+ return await response.text();
107
+ }
108
+ finally {
109
+ clearTimeout(timer);
110
+ options.signal?.removeEventListener('abort', abort);
111
+ }
112
+ }
113
+ /**
114
+ * Read the GitHub release endpoint.
115
+ *
116
+ * Measurement showed this API is often fast directly and slow through the
117
+ * configured tunnel (a successful proxy request took ~4 s while the direct
118
+ * request took <1 s). Try direct first with a short exploratory budget, then
119
+ * spend the rest of the caller's deadline on the tunnel when direct fails.
120
+ * A test seam bypasses the tunnel so unit tests stay deterministic.
121
+ */
122
+ async function fetchJson(url, options, timeoutMs) {
123
+ const headers = {
124
+ accept: 'application/vnd.github+json',
125
+ 'x-github-api-version': '2022-11-28',
126
+ 'user-agent': 'Mozilla/5.0 (compatible; dsh-web-search; +https://github.com/fangchenghao/dsh-web-search)',
127
+ };
128
+ const started = Date.now();
129
+ if (options.fetch !== undefined)
130
+ return directJson(url, headers, options, timeoutMs);
131
+ try {
132
+ return await directJson(url, headers, options, Math.min(timeoutMs, 1_500));
133
+ }
134
+ catch (error) {
135
+ if (options.proxy === undefined || options.proxy.length === 0)
136
+ throw error;
137
+ const remaining = timeoutMs - (Date.now() - started);
138
+ if (remaining < 250)
139
+ throw error;
140
+ const response = await proxyGet(url, {
141
+ proxy: options.proxy,
142
+ timeoutMs: remaining,
143
+ headers,
144
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
145
+ });
146
+ if (response.status < 200 || response.status >= 300)
147
+ throw new Error(`HTTP ${response.status}`);
148
+ return response.text;
149
+ }
150
+ }
151
+ /**
152
+ * Fetch the official Rust release page for a query.
153
+ *
154
+ * Failure is contained: an unavailable API, rate limit or malformed body
155
+ * returns `undefined`, and the caller keeps its ordinary search rows.
156
+ *
157
+ * @param query - the original user query.
158
+ * @param options - proxy, deadline, cancellation and fetch seam.
159
+ * @returns the official blog source, or `undefined`.
160
+ */
161
+ export async function fetchRustRelease(query, options = {}) {
162
+ if (!isRustReleaseQuery(query))
163
+ return undefined;
164
+ const requested = rustReleaseVersion(query);
165
+ if (requested === undefined)
166
+ return undefined;
167
+ const timeoutMs = typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0
168
+ ? Math.min(options.timeoutMs, DEFAULT_TIMEOUT_MS)
169
+ : DEFAULT_TIMEOUT_MS;
170
+ try {
171
+ const payload = await fetchJson(`${RUST_RELEASE_API}/${requested.tag}`, options, timeoutMs);
172
+ return parseRustRelease(payload, requested);
173
+ }
174
+ catch {
175
+ return undefined;
176
+ }
177
+ }
@@ -0,0 +1,40 @@
1
+ /**
2
+ * Route a versioned query to the narrow official lookup that can answer it.
3
+ *
4
+ * Keep this as a separate seam rather than putting network calls inside
5
+ * `engines.search()`: the engine layer stays a pure keyless fan-out, while the
6
+ * provider, MCP server and public benchmark can all enrich the same result with
7
+ * the project's own release facts.
8
+ *
9
+ * @module @fangchenghao/dsh-web-search/official-sources
10
+ */
11
+ import type { Source } from './types.ts';
12
+ /** Options shared by the official lookup implementations. */
13
+ export interface OfficialLookupOptions {
14
+ /** Tunnel URL from the plugin's `proxy` setting, when configured. */
15
+ proxy?: string;
16
+ /** Per-request deadline in milliseconds. */
17
+ timeoutMs?: number;
18
+ /** Caller cancellation. */
19
+ signal?: AbortSignal;
20
+ /** Test seam; defaults to the platform `fetch`. */
21
+ fetch?: typeof fetch;
22
+ }
23
+ /** Scoped npm package name in an explicitly npm-scoped query. */
24
+ export declare function scopedNpmPackage(query: string): string | undefined;
25
+ /**
26
+ * Start the applicable official lookup, or return `undefined` when the query
27
+ * has no official resolver.
28
+ *
29
+ * Returning the promise instead of awaiting it lets a caller overlap the lookup
30
+ * with the ordinary search fan-out.
31
+ */
32
+ export declare function officialLookup(query: string, options?: OfficialLookupOptions): Promise<Source | undefined> | undefined;
33
+ /**
34
+ * Put official rows first and remove duplicate URLs.
35
+ *
36
+ * The order of `official` is preserved; ordinary rows keep their relative
37
+ * order. A source with no usable URL is discarded rather than allowed to
38
+ * displace a real result.
39
+ */
40
+ export declare function mergeOfficial(sources: readonly Source[], official: Source | undefined): Source[];
@@ -0,0 +1,68 @@
1
+ import { fetchPostgresSupport, isPostgresVersionQuery } from "./official-support.js";
2
+ import { fetchRustRelease, isRustReleaseQuery } from "./official-release.js";
3
+ import { searchVerticals } from "./verticals.js";
4
+ /** Scoped npm package name in an explicitly npm-scoped query. */
5
+ export function scopedNpmPackage(query) {
6
+ if (typeof query !== 'string' || !/\bnpm(?:js)?\b/iu.test(query))
7
+ return undefined;
8
+ const match = /(?:^|\s)(@[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._-]*)(?=\s|$)/iu.exec(query.trim());
9
+ return match?.[1];
10
+ }
11
+ /** Resolve one scoped npm package through the existing npm vertical. */
12
+ async function fetchNpmPackage(name, options) {
13
+ try {
14
+ const result = await searchVerticals(name, {
15
+ ids: ['npm'],
16
+ limit: 1,
17
+ timeoutMs: typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0
18
+ ? Math.min(options.timeoutMs, 8_000)
19
+ : 8_000,
20
+ ...(options.proxy === undefined ? {} : { proxy: options.proxy }),
21
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
22
+ });
23
+ return result.sources[0];
24
+ }
25
+ catch {
26
+ return undefined;
27
+ }
28
+ }
29
+ /**
30
+ * Start the applicable official lookup, or return `undefined` when the query
31
+ * has no official resolver.
32
+ *
33
+ * Returning the promise instead of awaiting it lets a caller overlap the lookup
34
+ * with the ordinary search fan-out.
35
+ */
36
+ export function officialLookup(query, options = {}) {
37
+ if (isPostgresVersionQuery(query)) {
38
+ return fetchPostgresSupport(options);
39
+ }
40
+ if (isRustReleaseQuery(query)) {
41
+ return fetchRustRelease(query, options);
42
+ }
43
+ const npm = scopedNpmPackage(query);
44
+ if (npm !== undefined) {
45
+ return fetchNpmPackage(npm, options);
46
+ }
47
+ return undefined;
48
+ }
49
+ /**
50
+ * Put official rows first and remove duplicate URLs.
51
+ *
52
+ * The order of `official` is preserved; ordinary rows keep their relative
53
+ * order. A source with no usable URL is discarded rather than allowed to
54
+ * displace a real result.
55
+ */
56
+ export function mergeOfficial(sources, official) {
57
+ if (official === undefined || official.url.length === 0)
58
+ return [...sources];
59
+ const seen = new Set([official.url]);
60
+ const merged = [official];
61
+ for (const source of sources) {
62
+ if (source.url.length === 0 || seen.has(source.url))
63
+ continue;
64
+ seen.add(source.url);
65
+ merged.push(source);
66
+ }
67
+ return merged;
68
+ }
@@ -123,6 +123,19 @@ export declare function parseNpm(payload: string, _url: string, limit?: number):
123
123
  * @returns the package as one source, or `[]`.
124
124
  */
125
125
  export declare function parsePypi(payload: string, _url: string, limit?: number): Source[];
126
+ /**
127
+ * Turn a human query into an arXiv `search_query`.
128
+ *
129
+ * `all:retrieval augmented generation survey` is interpreted loosely and can
130
+ * rank an image-generation paper above actual surveys. arXiv's API honours
131
+ * explicit boolean operators, so a survey query becomes a quoted topic phrase
132
+ * plus `all:survey`; other multi-word queries become ANDed terms. A query that
133
+ * already contains field syntax or booleans is passed through unchanged.
134
+ *
135
+ * @param query - the human query.
136
+ * @returns a value for the `search_query` parameter.
137
+ */
138
+ export declare function arxivSearchQuery(query: string): string;
126
139
  /**
127
140
  * Parse the arXiv Atom feed.
128
141
  * @param payload - `export.arxiv.org/api/query` XML.
package/lib/verticals.js CHANGED
@@ -276,6 +276,41 @@ export function parsePypi(payload, _url, limit = MAX_VERTICAL_LIMIT) {
276
276
  const source = sourceOf(str(info.package_url) || `https://pypi.org/project/${encodeURIComponent(name)}/`, version.length > 0 ? `${name} ${version}` : name, snippet, str(first?.upload_time_iso_8601) || str(first?.upload_time));
277
277
  return source === undefined ? [] : [source];
278
278
  }
279
+ /** Paper-filler words that add noise to arXiv's `all:` search. */
280
+ const ARXIV_FILLER = new Set([
281
+ 'a', 'about', 'an', 'and', 'are', 'find', 'for', 'from', 'in', 'is', 'latest',
282
+ 'literature', 'newest', 'of', 'on', 'or', 'paper', 'papers', 'preprint',
283
+ 'preprints', 'recent', 'research', 'review', 'reviews', 'search', 'the', 'to',
284
+ 'with',
285
+ ]);
286
+ /**
287
+ * Turn a human query into an arXiv `search_query`.
288
+ *
289
+ * `all:retrieval augmented generation survey` is interpreted loosely and can
290
+ * rank an image-generation paper above actual surveys. arXiv's API honours
291
+ * explicit boolean operators, so a survey query becomes a quoted topic phrase
292
+ * plus `all:survey`; other multi-word queries become ANDed terms. A query that
293
+ * already contains field syntax or booleans is passed through unchanged.
294
+ *
295
+ * @param query - the human query.
296
+ * @returns a value for the `search_query` parameter.
297
+ */
298
+ export function arxivSearchQuery(query) {
299
+ const trimmed = query.trim();
300
+ if (trimmed.length === 0)
301
+ return 'all:';
302
+ if (/^(?:all|abs|au|cat|co|id|jr|rn|ti):/iu.test(trimmed) || /\b(?:AND|OR|ANDNOT)\b/u.test(trimmed) || trimmed.includes('"')) {
303
+ return trimmed;
304
+ }
305
+ const words = trimmed.split(/\s+/u).filter((word) => word.length > 0);
306
+ const survey = words.some((word) => /^(?:survey|surveys|综述)$/iu.test(word));
307
+ const terms = words.filter((word) => !ARXIV_FILLER.has(word.toLowerCase()) && !/^(?:survey|surveys|综述)$/iu.test(word));
308
+ if (survey && terms.length > 0) {
309
+ return `all:"${terms.join(' ')}" AND all:survey`;
310
+ }
311
+ const usable = terms.length > 0 ? terms : words;
312
+ return usable.map((term) => `all:${term}`).join(' AND ');
313
+ }
279
314
  /**
280
315
  * Parse the arXiv Atom feed.
281
316
  * @param payload - `export.arxiv.org/api/query` XML.
@@ -1228,7 +1263,7 @@ export const VERTICALS = [
1228
1263
  label: 'arXiv',
1229
1264
  accept: 'application/atom+xml',
1230
1265
  defaultOn: true,
1231
- build: (query, limit) => `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${limit}`,
1266
+ build: (query, limit) => `https://export.arxiv.org/api/query?search_query=${encodeURIComponent(arxivSearchQuery(query))}&start=0&max_results=${limit}`,
1232
1267
  parse: parseArxiv,
1233
1268
  },
1234
1269
  {
@@ -1620,7 +1655,7 @@ export async function searchVerticals(query, options = {}) {
1620
1655
  try {
1621
1656
  const built = spec.build(trimmed, limit);
1622
1657
  const url = spec.id === 'arxiv' && options.timeRange !== undefined && options.timeRange !== 'off'
1623
- ? `${built.replace(/search_query=all:([^&]*)/u, 'search_query=all:%22$1%22')}&sortBy=submittedDate&sortOrder=descending`
1658
+ ? `${built.replace(/search_query=[^&]*/u, `search_query=${encodeURIComponent(`all:"${trimmed}"`)}`)}&sortBy=submittedDate&sortOrder=descending`
1624
1659
  : built;
1625
1660
  const payload = await fetchPayload(spec, url, options, timeoutMs);
1626
1661
  const sources = spec.parse(payload, url, limit);
package/mcp/server.mjs CHANGED
@@ -17,6 +17,7 @@
17
17
  * Environment:
18
18
  * DSH_WEB_SEARCH_PROXY / HTTPS_PROXY / HTTP_PROXY optional tunnel
19
19
  * ANYSEARCH_API_KEY optional AnySearch key
20
+ * FIRECRAWL_API_KEY optional Firecrawl key
20
21
  * DSH_WEB_SEARCH_MCP_LIMIT default result limit (8)
21
22
  *
22
23
  * The server writes protocol messages to stdout and diagnostics to stderr.
@@ -31,6 +32,8 @@ import { search } from '../lib/engines.js';
31
32
  import { createFetchProvider } from '../lib/fetch.js';
32
33
  import { searchVerticals } from '../lib/verticals.js';
33
34
  import { createResearchDefinition, formatResearchValue } from '../lib/research.js';
35
+ import { mergeOfficial, officialLookup } from '../lib/official-sources.js';
36
+ import { arxivLookup, mergeArxiv } from '../lib/arxiv.js';
34
37
  import { wrapUntrustedContent } from '../lib/untrusted.js';
35
38
 
36
39
  const PACKAGE = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
@@ -44,6 +47,7 @@ const MAX_TEXT = 80_000;
44
47
  const env = process.env;
45
48
  const proxy = env.DSH_WEB_SEARCH_PROXY ?? env.HTTPS_PROXY ?? env.HTTP_PROXY ?? undefined;
46
49
  const anysearchApiKey = env.ANYSEARCH_API_KEY ?? env.DSH_WEB_SEARCH_ANYSEARCH_KEY ?? '';
50
+ const firecrawlApiKey = env.FIRECRAWL_API_KEY ?? env.DSH_WEB_SEARCH_FIRECRAWL_KEY ?? '';
47
51
  const defaultLimit = (() => {
48
52
  const raw = Number(env.DSH_WEB_SEARCH_MCP_LIMIT);
49
53
  return Number.isInteger(raw) && raw > 0 ? Math.min(raw, MAX_LIMIT) : DEFAULT_LIMIT;
@@ -85,6 +89,7 @@ function searchOptions(limit, timeoutMs) {
85
89
  wait: 'auto',
86
90
  ...(proxy === undefined ? {} : { proxy }),
87
91
  ...(anysearchApiKey.length === 0 ? {} : { anysearchApiKey }),
92
+ ...(firecrawlApiKey.length === 0 ? {} : { firecrawlApiKey }),
88
93
  };
89
94
  }
90
95
 
@@ -171,8 +176,18 @@ async function callTool(name, args) {
171
176
  if (query.length === 0) throw new Error('search requires a non-empty query');
172
177
  const limit = intArg(args.limit, defaultLimit, 1, MAX_LIMIT);
173
178
  const timeoutMs = intArg(args.timeoutMs, 12_000, 1_000, 30_000);
179
+ const official = officialLookup(query, {
180
+ timeoutMs: Math.min(timeoutMs, 2_500),
181
+ ...(proxy === undefined ? {} : { proxy }),
182
+ });
183
+ const arxiv = arxivLookup(query, {
184
+ limit: 4,
185
+ timeoutMs: Math.min(timeoutMs, 3_500),
186
+ ...(proxy === undefined ? {} : { proxy }),
187
+ });
174
188
  const outcome = await search(query, undefined, searchOptions(limit, timeoutMs));
175
- return formatSources(query, outcome.sources, { failures: outcome.failures });
189
+ const sources = mergeArxiv(mergeOfficial(outcome.sources, await official), await arxiv);
190
+ return formatSources(query, sources.slice(0, limit), { failures: outcome.failures });
176
191
  }
177
192
  if (name === 'fetch') {
178
193
  const url = typeof args.url === 'string' ? args.url.trim() : '';
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@fffangchenghao/dsh-web-search",
3
- "version": "0.21.1",
4
- "description": "DSH bundle: key-free web search, page fetch and one web_tools action tool for verticals, platforms and cited research; eight engines fused with RRF, 34 vertical sources, official-source checks, a replayable benchmark and a dependency-free MCP server.",
3
+ "version": "0.23.0",
4
+ "description": "DSH bundle: key-free web search, page fetch and one web_tools action tool for verticals, platforms and cited research; eight engines fused with RRF, 34 vertical sources, arXiv research lookups and readable papers, official release facts, a replayable benchmark, an optional Firecrawl key and a dependency-free MCP server.",
5
5
  "type": "module",
6
6
  "main": "lib/index.js",
7
7
  "types": "lib/index.d.ts",
@@ -32,7 +32,8 @@
32
32
  "docs/quickstart.md",
33
33
  "docs/cross-agent.md",
34
34
  "docs/comparison.md",
35
- "docs/images/web-tools-demo.svg"
35
+ "docs/images/web-tools-demo.svg",
36
+ "docs/latency-and-sources.md"
36
37
  ],
37
38
  "scripts": {
38
39
  "build": "tsc -p tsconfig.json",