@fffangchenghao/dsh-web-search 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +16 -11
- package/README.zh-CN.md +15 -10
- package/client.js +1 -1
- package/docs/comparison.md +23 -4
- package/docs/cross-agent.md +1 -0
- package/docs/images/web-tools-demo.svg +1 -1
- package/docs/latency-and-sources.md +199 -0
- package/docs/store-evidence.md +1 -1
- package/lib/arxiv-url.d.ts +40 -0
- package/lib/arxiv-url.js +62 -0
- package/lib/arxiv.d.ts +46 -0
- package/lib/arxiv.js +70 -0
- package/lib/config.d.ts +4 -0
- package/lib/config.js +5 -0
- package/lib/engines.d.ts +18 -6
- package/lib/engines.js +46 -11
- package/lib/fetch.js +7 -3
- package/lib/index.d.ts +2 -0
- package/lib/index.js +48 -16
- package/lib/markdown.js +17 -1
- package/lib/official-release.d.ts +73 -0
- package/lib/official-release.js +177 -0
- package/lib/official-sources.d.ts +40 -0
- package/lib/official-sources.js +68 -0
- package/lib/verticals.d.ts +13 -0
- package/lib/verticals.js +37 -2
- package/mcp/server.mjs +16 -1
- package/package.json +4 -3
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { proxyGet } from "./proxy.js";
|
|
2
|
+
import { text } from "./parsers.js";
|
|
3
|
+
/** Public GitHub release endpoint for the Rust repository. */
|
|
4
|
+
export const RUST_RELEASE_API = 'https://api.github.com/repos/rust-lang/rust/releases/tags';
|
|
5
|
+
/** Official Rust announcement host. */
|
|
6
|
+
export const RUST_BLOG_HOST = 'blog.rust-lang.org';
|
|
7
|
+
const DEFAULT_TIMEOUT_MS = 6_000;
|
|
8
|
+
/** Extract a Rust version from a release query. */
|
|
9
|
+
export function rustReleaseVersion(query) {
|
|
10
|
+
if (typeof query !== 'string' || !/\brust\b/iu.test(query))
|
|
11
|
+
return undefined;
|
|
12
|
+
const match = /\brust(?:\s+(?:release|version|v))?\s*([0-9]{1,2}\.[0-9]{1,2}(?:\.[0-9]{1,2})?)\b/iu.exec(query);
|
|
13
|
+
if (match?.[1] === undefined)
|
|
14
|
+
return undefined;
|
|
15
|
+
const raw = match[1];
|
|
16
|
+
const version = raw.split('.').length === 2 ? `${raw}.0` : raw;
|
|
17
|
+
return { version, tag: version };
|
|
18
|
+
}
|
|
19
|
+
/** True for a query asking for a specific Rust release's notes. */
|
|
20
|
+
export function isRustReleaseQuery(query) {
|
|
21
|
+
if (typeof query !== 'string')
|
|
22
|
+
return false;
|
|
23
|
+
if (rustReleaseVersion(query) === undefined)
|
|
24
|
+
return false;
|
|
25
|
+
return /(?:release|releases|changelog|notes?|announcement|发布|发行|版本|更新|说明|日志|公告)/iu.test(query);
|
|
26
|
+
}
|
|
27
|
+
/** Trim release Markdown/HTML into a bounded snippet. */
|
|
28
|
+
function releaseSnippet(body, version) {
|
|
29
|
+
const plain = text(typeof body === 'string' ? body : '')
|
|
30
|
+
.replace(/\[([^\]]+)\]\([^)]*\)/gu, '$1')
|
|
31
|
+
.replace(/https?:\/\/\S+/gu, ' ')
|
|
32
|
+
.replace(/\s+/gu, ' ')
|
|
33
|
+
.trim();
|
|
34
|
+
const prefix = `Official Rust ${version} release notes. `;
|
|
35
|
+
if (plain.length === 0)
|
|
36
|
+
return prefix.trim();
|
|
37
|
+
const budget = Math.max(80, 420 - prefix.length);
|
|
38
|
+
return `${prefix}${plain.length > budget ? `${plain.slice(0, budget).trimEnd()}…` : plain}`;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Build the canonical announcement URL from a GitHub release timestamp.
|
|
42
|
+
*
|
|
43
|
+
* Rust publishes the post on the UTC date in the release metadata. Using UTC
|
|
44
|
+
* avoids shifting the path when the machine is in a negative-offset timezone.
|
|
45
|
+
*/
|
|
46
|
+
export function rustBlogUrl(version, publishedAt) {
|
|
47
|
+
const date = new Date(publishedAt);
|
|
48
|
+
if (Number.isNaN(date.getTime()))
|
|
49
|
+
return undefined;
|
|
50
|
+
const yyyy = String(date.getUTCFullYear()).padStart(4, '0');
|
|
51
|
+
const mm = String(date.getUTCMonth() + 1).padStart(2, '0');
|
|
52
|
+
const dd = String(date.getUTCDate()).padStart(2, '0');
|
|
53
|
+
return `https://${RUST_BLOG_HOST}/${yyyy}/${mm}/${dd}/Rust-${version}/`;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Parse a GitHub release response into one official-blog source.
|
|
57
|
+
*
|
|
58
|
+
* A malformed response, a missing publication date or a tag that does not
|
|
59
|
+
* match the requested version returns `undefined`; the caller then keeps the
|
|
60
|
+
* ordinary search rows.
|
|
61
|
+
*
|
|
62
|
+
* @param payload - GitHub JSON response body.
|
|
63
|
+
* @param requested - normalized version from the query.
|
|
64
|
+
* @returns the official source, or `undefined`.
|
|
65
|
+
*/
|
|
66
|
+
export function parseRustRelease(payload, requested) {
|
|
67
|
+
if (typeof payload !== 'string' || payload.trim().length === 0)
|
|
68
|
+
return undefined;
|
|
69
|
+
let body;
|
|
70
|
+
try {
|
|
71
|
+
body = JSON.parse(payload);
|
|
72
|
+
}
|
|
73
|
+
catch {
|
|
74
|
+
return undefined;
|
|
75
|
+
}
|
|
76
|
+
if (typeof body !== 'object' || body === null)
|
|
77
|
+
return undefined;
|
|
78
|
+
const record = body;
|
|
79
|
+
const publishedAt = typeof record['published_at'] === 'string' ? record['published_at'] : '';
|
|
80
|
+
const url = rustBlogUrl(requested.version, publishedAt);
|
|
81
|
+
if (url === undefined)
|
|
82
|
+
return undefined;
|
|
83
|
+
const tag = typeof record['tag_name'] === 'string' ? record['tag_name'] : '';
|
|
84
|
+
if (tag.length > 0 && tag !== requested.tag && tag !== requested.version)
|
|
85
|
+
return undefined;
|
|
86
|
+
const title = `Announcing Rust ${requested.version} - Rust Blog`;
|
|
87
|
+
const snippet = releaseSnippet(record['body'], requested.version);
|
|
88
|
+
return {
|
|
89
|
+
url,
|
|
90
|
+
title,
|
|
91
|
+
snippet,
|
|
92
|
+
...(Number.isNaN(new Date(publishedAt).getTime()) ? {} : { publishedAt: new Date(publishedAt).toISOString() }),
|
|
93
|
+
};
|
|
94
|
+
}
|
|
95
|
+
/** One direct GET with its own abort budget. */
|
|
96
|
+
async function directJson(url, headers, options, timeoutMs) {
|
|
97
|
+
const controller = new AbortController();
|
|
98
|
+
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
99
|
+
const abort = () => controller.abort();
|
|
100
|
+
options.signal?.addEventListener('abort', abort, { once: true });
|
|
101
|
+
try {
|
|
102
|
+
const send = options.fetch ?? fetch;
|
|
103
|
+
const response = await send(url, { headers, signal: controller.signal });
|
|
104
|
+
if (!response.ok)
|
|
105
|
+
throw new Error(`HTTP ${response.status}`);
|
|
106
|
+
return await response.text();
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
clearTimeout(timer);
|
|
110
|
+
options.signal?.removeEventListener('abort', abort);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Read the GitHub release endpoint.
|
|
115
|
+
*
|
|
116
|
+
* Measurement showed this API is often fast directly and slow through the
|
|
117
|
+
* configured tunnel (a successful proxy request took ~4 s while the direct
|
|
118
|
+
* request took <1 s). Try direct first with a short exploratory budget, then
|
|
119
|
+
* spend the rest of the caller's deadline on the tunnel when direct fails.
|
|
120
|
+
* A test seam bypasses the tunnel so unit tests stay deterministic.
|
|
121
|
+
*/
|
|
122
|
+
async function fetchJson(url, options, timeoutMs) {
|
|
123
|
+
const headers = {
|
|
124
|
+
accept: 'application/vnd.github+json',
|
|
125
|
+
'x-github-api-version': '2022-11-28',
|
|
126
|
+
'user-agent': 'Mozilla/5.0 (compatible; dsh-web-search; +https://github.com/fangchenghao/dsh-web-search)',
|
|
127
|
+
};
|
|
128
|
+
const started = Date.now();
|
|
129
|
+
if (options.fetch !== undefined)
|
|
130
|
+
return directJson(url, headers, options, timeoutMs);
|
|
131
|
+
try {
|
|
132
|
+
return await directJson(url, headers, options, Math.min(timeoutMs, 1_500));
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
if (options.proxy === undefined || options.proxy.length === 0)
|
|
136
|
+
throw error;
|
|
137
|
+
const remaining = timeoutMs - (Date.now() - started);
|
|
138
|
+
if (remaining < 250)
|
|
139
|
+
throw error;
|
|
140
|
+
const response = await proxyGet(url, {
|
|
141
|
+
proxy: options.proxy,
|
|
142
|
+
timeoutMs: remaining,
|
|
143
|
+
headers,
|
|
144
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
145
|
+
});
|
|
146
|
+
if (response.status < 200 || response.status >= 300)
|
|
147
|
+
throw new Error(`HTTP ${response.status}`);
|
|
148
|
+
return response.text;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Fetch the official Rust release page for a query.
|
|
153
|
+
*
|
|
154
|
+
* Failure is contained: an unavailable API, rate limit or malformed body
|
|
155
|
+
* returns `undefined`, and the caller keeps its ordinary search rows.
|
|
156
|
+
*
|
|
157
|
+
* @param query - the original user query.
|
|
158
|
+
* @param options - proxy, deadline, cancellation and fetch seam.
|
|
159
|
+
* @returns the official blog source, or `undefined`.
|
|
160
|
+
*/
|
|
161
|
+
export async function fetchRustRelease(query, options = {}) {
|
|
162
|
+
if (!isRustReleaseQuery(query))
|
|
163
|
+
return undefined;
|
|
164
|
+
const requested = rustReleaseVersion(query);
|
|
165
|
+
if (requested === undefined)
|
|
166
|
+
return undefined;
|
|
167
|
+
const timeoutMs = typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0
|
|
168
|
+
? Math.min(options.timeoutMs, DEFAULT_TIMEOUT_MS)
|
|
169
|
+
: DEFAULT_TIMEOUT_MS;
|
|
170
|
+
try {
|
|
171
|
+
const payload = await fetchJson(`${RUST_RELEASE_API}/${requested.tag}`, options, timeoutMs);
|
|
172
|
+
return parseRustRelease(payload, requested);
|
|
173
|
+
}
|
|
174
|
+
catch {
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Route a versioned query to the narrow official lookup that can answer it.
|
|
3
|
+
*
|
|
4
|
+
* Keep this as a separate seam rather than putting network calls inside
|
|
5
|
+
* `engines.search()`: the engine layer stays a pure keyless fan-out, while the
|
|
6
|
+
* provider, MCP server and public benchmark can all enrich the same result with
|
|
7
|
+
* the project's own release facts.
|
|
8
|
+
*
|
|
9
|
+
* @module @fangchenghao/dsh-web-search/official-sources
|
|
10
|
+
*/
|
|
11
|
+
import type { Source } from './types.ts';
|
|
12
|
+
/** Options shared by the official lookup implementations. */
|
|
13
|
+
export interface OfficialLookupOptions {
|
|
14
|
+
/** Tunnel URL from the plugin's `proxy` setting, when configured. */
|
|
15
|
+
proxy?: string;
|
|
16
|
+
/** Per-request deadline in milliseconds. */
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
/** Caller cancellation. */
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
/** Test seam; defaults to the platform `fetch`. */
|
|
21
|
+
fetch?: typeof fetch;
|
|
22
|
+
}
|
|
23
|
+
/** Scoped npm package name in an explicitly npm-scoped query. */
|
|
24
|
+
export declare function scopedNpmPackage(query: string): string | undefined;
|
|
25
|
+
/**
|
|
26
|
+
* Start the applicable official lookup, or return `undefined` when the query
|
|
27
|
+
* has no official resolver.
|
|
28
|
+
*
|
|
29
|
+
* Returning the promise instead of awaiting it lets a caller overlap the lookup
|
|
30
|
+
* with the ordinary search fan-out.
|
|
31
|
+
*/
|
|
32
|
+
export declare function officialLookup(query: string, options?: OfficialLookupOptions): Promise<Source | undefined> | undefined;
|
|
33
|
+
/**
|
|
34
|
+
* Put official rows first and remove duplicate URLs.
|
|
35
|
+
*
|
|
36
|
+
* The order of `official` is preserved; ordinary rows keep their relative
|
|
37
|
+
* order. A source with no usable URL is discarded rather than allowed to
|
|
38
|
+
* displace a real result.
|
|
39
|
+
*/
|
|
40
|
+
export declare function mergeOfficial(sources: readonly Source[], official: Source | undefined): Source[];
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { fetchPostgresSupport, isPostgresVersionQuery } from "./official-support.js";
|
|
2
|
+
import { fetchRustRelease, isRustReleaseQuery } from "./official-release.js";
|
|
3
|
+
import { searchVerticals } from "./verticals.js";
|
|
4
|
+
/** Scoped npm package name in an explicitly npm-scoped query. */
|
|
5
|
+
export function scopedNpmPackage(query) {
|
|
6
|
+
if (typeof query !== 'string' || !/\bnpm(?:js)?\b/iu.test(query))
|
|
7
|
+
return undefined;
|
|
8
|
+
const match = /(?:^|\s)(@[a-z0-9][a-z0-9._-]*\/[a-z0-9][a-z0-9._-]*)(?=\s|$)/iu.exec(query.trim());
|
|
9
|
+
return match?.[1];
|
|
10
|
+
}
|
|
11
|
+
/** Resolve one scoped npm package through the existing npm vertical. */
|
|
12
|
+
async function fetchNpmPackage(name, options) {
|
|
13
|
+
try {
|
|
14
|
+
const result = await searchVerticals(name, {
|
|
15
|
+
ids: ['npm'],
|
|
16
|
+
limit: 1,
|
|
17
|
+
timeoutMs: typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0
|
|
18
|
+
? Math.min(options.timeoutMs, 8_000)
|
|
19
|
+
: 8_000,
|
|
20
|
+
...(options.proxy === undefined ? {} : { proxy: options.proxy }),
|
|
21
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
22
|
+
});
|
|
23
|
+
return result.sources[0];
|
|
24
|
+
}
|
|
25
|
+
catch {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Start the applicable official lookup, or return `undefined` when the query
|
|
31
|
+
* has no official resolver.
|
|
32
|
+
*
|
|
33
|
+
* Returning the promise instead of awaiting it lets a caller overlap the lookup
|
|
34
|
+
* with the ordinary search fan-out.
|
|
35
|
+
*/
|
|
36
|
+
export function officialLookup(query, options = {}) {
|
|
37
|
+
if (isPostgresVersionQuery(query)) {
|
|
38
|
+
return fetchPostgresSupport(options);
|
|
39
|
+
}
|
|
40
|
+
if (isRustReleaseQuery(query)) {
|
|
41
|
+
return fetchRustRelease(query, options);
|
|
42
|
+
}
|
|
43
|
+
const npm = scopedNpmPackage(query);
|
|
44
|
+
if (npm !== undefined) {
|
|
45
|
+
return fetchNpmPackage(npm, options);
|
|
46
|
+
}
|
|
47
|
+
return undefined;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Put official rows first and remove duplicate URLs.
|
|
51
|
+
*
|
|
52
|
+
* The order of `official` is preserved; ordinary rows keep their relative
|
|
53
|
+
* order. A source with no usable URL is discarded rather than allowed to
|
|
54
|
+
* displace a real result.
|
|
55
|
+
*/
|
|
56
|
+
export function mergeOfficial(sources, official) {
|
|
57
|
+
if (official === undefined || official.url.length === 0)
|
|
58
|
+
return [...sources];
|
|
59
|
+
const seen = new Set([official.url]);
|
|
60
|
+
const merged = [official];
|
|
61
|
+
for (const source of sources) {
|
|
62
|
+
if (source.url.length === 0 || seen.has(source.url))
|
|
63
|
+
continue;
|
|
64
|
+
seen.add(source.url);
|
|
65
|
+
merged.push(source);
|
|
66
|
+
}
|
|
67
|
+
return merged;
|
|
68
|
+
}
|
package/lib/verticals.d.ts
CHANGED
|
@@ -123,6 +123,19 @@ export declare function parseNpm(payload: string, _url: string, limit?: number):
|
|
|
123
123
|
* @returns the package as one source, or `[]`.
|
|
124
124
|
*/
|
|
125
125
|
export declare function parsePypi(payload: string, _url: string, limit?: number): Source[];
|
|
126
|
+
/**
|
|
127
|
+
* Turn a human query into an arXiv `search_query`.
|
|
128
|
+
*
|
|
129
|
+
* `all:retrieval augmented generation survey` is interpreted loosely and can
|
|
130
|
+
* rank an image-generation paper above actual surveys. arXiv's API honours
|
|
131
|
+
* explicit boolean operators, so a survey query becomes a quoted topic phrase
|
|
132
|
+
* plus `all:survey`; other multi-word queries become ANDed terms. A query that
|
|
133
|
+
* already contains field syntax or booleans is passed through unchanged.
|
|
134
|
+
*
|
|
135
|
+
* @param query - the human query.
|
|
136
|
+
* @returns a value for the `search_query` parameter.
|
|
137
|
+
*/
|
|
138
|
+
export declare function arxivSearchQuery(query: string): string;
|
|
126
139
|
/**
|
|
127
140
|
* Parse the arXiv Atom feed.
|
|
128
141
|
* @param payload - `export.arxiv.org/api/query` XML.
|
package/lib/verticals.js
CHANGED
|
@@ -276,6 +276,41 @@ export function parsePypi(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
|
276
276
|
const source = sourceOf(str(info.package_url) || `https://pypi.org/project/${encodeURIComponent(name)}/`, version.length > 0 ? `${name} ${version}` : name, snippet, str(first?.upload_time_iso_8601) || str(first?.upload_time));
|
|
277
277
|
return source === undefined ? [] : [source];
|
|
278
278
|
}
|
|
279
|
+
/** Paper-filler words that add noise to arXiv's `all:` search. */
|
|
280
|
+
const ARXIV_FILLER = new Set([
|
|
281
|
+
'a', 'about', 'an', 'and', 'are', 'find', 'for', 'from', 'in', 'is', 'latest',
|
|
282
|
+
'literature', 'newest', 'of', 'on', 'or', 'paper', 'papers', 'preprint',
|
|
283
|
+
'preprints', 'recent', 'research', 'review', 'reviews', 'search', 'the', 'to',
|
|
284
|
+
'with',
|
|
285
|
+
]);
|
|
286
|
+
/**
|
|
287
|
+
* Turn a human query into an arXiv `search_query`.
|
|
288
|
+
*
|
|
289
|
+
* `all:retrieval augmented generation survey` is interpreted loosely and can
|
|
290
|
+
* rank an image-generation paper above actual surveys. arXiv's API honours
|
|
291
|
+
* explicit boolean operators, so a survey query becomes a quoted topic phrase
|
|
292
|
+
* plus `all:survey`; other multi-word queries become ANDed terms. A query that
|
|
293
|
+
* already contains field syntax or booleans is passed through unchanged.
|
|
294
|
+
*
|
|
295
|
+
* @param query - the human query.
|
|
296
|
+
* @returns a value for the `search_query` parameter.
|
|
297
|
+
*/
|
|
298
|
+
export function arxivSearchQuery(query) {
|
|
299
|
+
const trimmed = query.trim();
|
|
300
|
+
if (trimmed.length === 0)
|
|
301
|
+
return 'all:';
|
|
302
|
+
if (/^(?:all|abs|au|cat|co|id|jr|rn|ti):/iu.test(trimmed) || /\b(?:AND|OR|ANDNOT)\b/u.test(trimmed) || trimmed.includes('"')) {
|
|
303
|
+
return trimmed;
|
|
304
|
+
}
|
|
305
|
+
const words = trimmed.split(/\s+/u).filter((word) => word.length > 0);
|
|
306
|
+
const survey = words.some((word) => /^(?:survey|surveys|综述)$/iu.test(word));
|
|
307
|
+
const terms = words.filter((word) => !ARXIV_FILLER.has(word.toLowerCase()) && !/^(?:survey|surveys|综述)$/iu.test(word));
|
|
308
|
+
if (survey && terms.length > 0) {
|
|
309
|
+
return `all:"${terms.join(' ')}" AND all:survey`;
|
|
310
|
+
}
|
|
311
|
+
const usable = terms.length > 0 ? terms : words;
|
|
312
|
+
return usable.map((term) => `all:${term}`).join(' AND ');
|
|
313
|
+
}
|
|
279
314
|
/**
|
|
280
315
|
* Parse the arXiv Atom feed.
|
|
281
316
|
* @param payload - `export.arxiv.org/api/query` XML.
|
|
@@ -1228,7 +1263,7 @@ export const VERTICALS = [
|
|
|
1228
1263
|
label: 'arXiv',
|
|
1229
1264
|
accept: 'application/atom+xml',
|
|
1230
1265
|
defaultOn: true,
|
|
1231
|
-
build: (query, limit) => `https://export.arxiv.org/api/query?search_query
|
|
1266
|
+
build: (query, limit) => `https://export.arxiv.org/api/query?search_query=${encodeURIComponent(arxivSearchQuery(query))}&start=0&max_results=${limit}`,
|
|
1232
1267
|
parse: parseArxiv,
|
|
1233
1268
|
},
|
|
1234
1269
|
{
|
|
@@ -1620,7 +1655,7 @@ export async function searchVerticals(query, options = {}) {
|
|
|
1620
1655
|
try {
|
|
1621
1656
|
const built = spec.build(trimmed, limit);
|
|
1622
1657
|
const url = spec.id === 'arxiv' && options.timeRange !== undefined && options.timeRange !== 'off'
|
|
1623
|
-
? `${built.replace(/search_query=
|
|
1658
|
+
? `${built.replace(/search_query=[^&]*/u, `search_query=${encodeURIComponent(`all:"${trimmed}"`)}`)}&sortBy=submittedDate&sortOrder=descending`
|
|
1624
1659
|
: built;
|
|
1625
1660
|
const payload = await fetchPayload(spec, url, options, timeoutMs);
|
|
1626
1661
|
const sources = spec.parse(payload, url, limit);
|
package/mcp/server.mjs
CHANGED
|
@@ -17,6 +17,7 @@
|
|
|
17
17
|
* Environment:
|
|
18
18
|
* DSH_WEB_SEARCH_PROXY / HTTPS_PROXY / HTTP_PROXY optional tunnel
|
|
19
19
|
* ANYSEARCH_API_KEY optional AnySearch key
|
|
20
|
+
* FIRECRAWL_API_KEY optional Firecrawl key
|
|
20
21
|
* DSH_WEB_SEARCH_MCP_LIMIT default result limit (8)
|
|
21
22
|
*
|
|
22
23
|
* The server writes protocol messages to stdout and diagnostics to stderr.
|
|
@@ -31,6 +32,8 @@ import { search } from '../lib/engines.js';
|
|
|
31
32
|
import { createFetchProvider } from '../lib/fetch.js';
|
|
32
33
|
import { searchVerticals } from '../lib/verticals.js';
|
|
33
34
|
import { createResearchDefinition, formatResearchValue } from '../lib/research.js';
|
|
35
|
+
import { mergeOfficial, officialLookup } from '../lib/official-sources.js';
|
|
36
|
+
import { arxivLookup, mergeArxiv } from '../lib/arxiv.js';
|
|
34
37
|
import { wrapUntrustedContent } from '../lib/untrusted.js';
|
|
35
38
|
|
|
36
39
|
const PACKAGE = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
|
@@ -44,6 +47,7 @@ const MAX_TEXT = 80_000;
|
|
|
44
47
|
const env = process.env;
|
|
45
48
|
const proxy = env.DSH_WEB_SEARCH_PROXY ?? env.HTTPS_PROXY ?? env.HTTP_PROXY ?? undefined;
|
|
46
49
|
const anysearchApiKey = env.ANYSEARCH_API_KEY ?? env.DSH_WEB_SEARCH_ANYSEARCH_KEY ?? '';
|
|
50
|
+
const firecrawlApiKey = env.FIRECRAWL_API_KEY ?? env.DSH_WEB_SEARCH_FIRECRAWL_KEY ?? '';
|
|
47
51
|
const defaultLimit = (() => {
|
|
48
52
|
const raw = Number(env.DSH_WEB_SEARCH_MCP_LIMIT);
|
|
49
53
|
return Number.isInteger(raw) && raw > 0 ? Math.min(raw, MAX_LIMIT) : DEFAULT_LIMIT;
|
|
@@ -85,6 +89,7 @@ function searchOptions(limit, timeoutMs) {
|
|
|
85
89
|
wait: 'auto',
|
|
86
90
|
...(proxy === undefined ? {} : { proxy }),
|
|
87
91
|
...(anysearchApiKey.length === 0 ? {} : { anysearchApiKey }),
|
|
92
|
+
...(firecrawlApiKey.length === 0 ? {} : { firecrawlApiKey }),
|
|
88
93
|
};
|
|
89
94
|
}
|
|
90
95
|
|
|
@@ -171,8 +176,18 @@ async function callTool(name, args) {
|
|
|
171
176
|
if (query.length === 0) throw new Error('search requires a non-empty query');
|
|
172
177
|
const limit = intArg(args.limit, defaultLimit, 1, MAX_LIMIT);
|
|
173
178
|
const timeoutMs = intArg(args.timeoutMs, 12_000, 1_000, 30_000);
|
|
179
|
+
const official = officialLookup(query, {
|
|
180
|
+
timeoutMs: Math.min(timeoutMs, 2_500),
|
|
181
|
+
...(proxy === undefined ? {} : { proxy }),
|
|
182
|
+
});
|
|
183
|
+
const arxiv = arxivLookup(query, {
|
|
184
|
+
limit: 4,
|
|
185
|
+
timeoutMs: Math.min(timeoutMs, 3_500),
|
|
186
|
+
...(proxy === undefined ? {} : { proxy }),
|
|
187
|
+
});
|
|
174
188
|
const outcome = await search(query, undefined, searchOptions(limit, timeoutMs));
|
|
175
|
-
|
|
189
|
+
const sources = mergeArxiv(mergeOfficial(outcome.sources, await official), await arxiv);
|
|
190
|
+
return formatSources(query, sources.slice(0, limit), { failures: outcome.failures });
|
|
176
191
|
}
|
|
177
192
|
if (name === 'fetch') {
|
|
178
193
|
const url = typeof args.url === 'string' ? args.url.trim() : '';
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fffangchenghao/dsh-web-search",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "DSH bundle: key-free web search, page fetch and one web_tools action tool for verticals, platforms and cited research; eight engines fused with RRF, 34 vertical sources, official
|
|
3
|
+
"version": "0.23.0",
|
|
4
|
+
"description": "DSH bundle: key-free web search, page fetch and one web_tools action tool for verticals, platforms and cited research; eight engines fused with RRF, 34 vertical sources, arXiv research lookups and readable papers, official release facts, a replayable benchmark, an optional Firecrawl key and a dependency-free MCP server.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"types": "lib/index.d.ts",
|
|
@@ -32,7 +32,8 @@
|
|
|
32
32
|
"docs/quickstart.md",
|
|
33
33
|
"docs/cross-agent.md",
|
|
34
34
|
"docs/comparison.md",
|
|
35
|
-
"docs/images/web-tools-demo.svg"
|
|
35
|
+
"docs/images/web-tools-demo.svg",
|
|
36
|
+
"docs/latency-and-sources.md"
|
|
36
37
|
],
|
|
37
38
|
"scripts": {
|
|
38
39
|
"build": "tsc -p tsconfig.json",
|