@fffangchenghao/dsh-web-search 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +67 -0
- package/README.md +16 -11
- package/README.zh-CN.md +15 -10
- package/client.js +1 -1
- package/docs/comparison.md +23 -4
- package/docs/cross-agent.md +1 -0
- package/docs/images/web-tools-demo.svg +1 -1
- package/docs/latency-and-sources.md +199 -0
- package/docs/store-evidence.md +1 -1
- package/lib/arxiv-url.d.ts +40 -0
- package/lib/arxiv-url.js +62 -0
- package/lib/arxiv.d.ts +46 -0
- package/lib/arxiv.js +70 -0
- package/lib/config.d.ts +4 -0
- package/lib/config.js +5 -0
- package/lib/engines.d.ts +18 -6
- package/lib/engines.js +46 -11
- package/lib/fetch.js +7 -3
- package/lib/index.d.ts +2 -0
- package/lib/index.js +48 -16
- package/lib/markdown.js +17 -1
- package/lib/official-release.d.ts +73 -0
- package/lib/official-release.js +177 -0
- package/lib/official-sources.d.ts +40 -0
- package/lib/official-sources.js +68 -0
- package/lib/verticals.d.ts +13 -0
- package/lib/verticals.js +37 -2
- package/mcp/server.mjs +16 -1
- package/package.json +4 -3
package/lib/arxiv-url.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recognize arXiv page URLs and map PDF requests to the HTML full-text view.
|
|
3
|
+
*
|
|
4
|
+
* arXiv serves the same paper as `/abs/<id>` (metadata and abstract),
|
|
5
|
+
* `/html/<id>` (full text) and `/pdf/<id>` (binary PDF). This plugin converts
|
|
6
|
+
* HTML and text to Markdown but deliberately does not pretend binary PDF bytes
|
|
7
|
+
* are prose, so a PDF link is read through the HTML view instead.
|
|
8
|
+
*
|
|
9
|
+
* @module @fangchenghao/dsh-web-search/arxiv-url
|
|
10
|
+
*/
|
|
11
|
+
const HOST = /^(?:www\.|export\.)?arxiv\.org$/iu;
|
|
12
|
+
/**
|
|
13
|
+
* Parse an arXiv abs/html/pdf URL.
|
|
14
|
+
*
|
|
15
|
+
* @param url - any URL string.
|
|
16
|
+
* @returns the identifier and kind, or `undefined` for other hosts.
|
|
17
|
+
*/
|
|
18
|
+
export function parseArxivUrl(url) {
|
|
19
|
+
let parsed;
|
|
20
|
+
try {
|
|
21
|
+
parsed = new URL(url);
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
if (!HOST.test(parsed.hostname))
|
|
27
|
+
return undefined;
|
|
28
|
+
const match = /^\/(abs|html|pdf)\/(.+?)\/?$/iu.exec(parsed.pathname);
|
|
29
|
+
if (match === null)
|
|
30
|
+
return undefined;
|
|
31
|
+
const kind = match[1]?.toLowerCase();
|
|
32
|
+
let raw = match[2] ?? '';
|
|
33
|
+
raw = raw.replace(/\.pdf$/iu, '');
|
|
34
|
+
const versionMatch = /v(\d+)$/iu.exec(raw);
|
|
35
|
+
const version = versionMatch?.[1];
|
|
36
|
+
const id = version === undefined ? raw : raw.slice(0, -versionMatch[0].length);
|
|
37
|
+
if (id.length === 0 || !/[a-z0-9]/iu.test(id))
|
|
38
|
+
return undefined;
|
|
39
|
+
return { id, kind, ...(version === undefined ? {} : { version }) };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The HTML full-text URL for a paper.
|
|
43
|
+
*
|
|
44
|
+
* @param reference - parsed arXiv URL.
|
|
45
|
+
* @returns an `https://arxiv.org/html/...` URL.
|
|
46
|
+
*/
|
|
47
|
+
export function arxivHtmlUrl(reference) {
|
|
48
|
+
const suffix = reference.version === undefined ? '' : `v${reference.version}`;
|
|
49
|
+
return `https://arxiv.org/html/${reference.id}${suffix}`;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Map a PDF URL to the HTML view; leave other URLs alone.
|
|
53
|
+
*
|
|
54
|
+
* @param url - the caller's URL.
|
|
55
|
+
* @returns the URL the fetch ladder should read.
|
|
56
|
+
*/
|
|
57
|
+
export function arxivReadableUrl(url) {
|
|
58
|
+
const reference = parseArxivUrl(url);
|
|
59
|
+
if (reference === undefined || reference.kind !== 'pdf')
|
|
60
|
+
return url;
|
|
61
|
+
return arxivHtmlUrl(reference);
|
|
62
|
+
}
|
package/lib/arxiv.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Explicit `arXiv` research queries, answered by the existing arXiv vertical.
|
|
3
|
+
*
|
|
4
|
+
* A general web query that merely contains the word `arxiv` should not be
|
|
5
|
+
* hijacked, but a query that explicitly asks arXiv for a survey or paper is a
|
|
6
|
+
* clear registry lookup. The lookup runs in parallel with the ordinary
|
|
7
|
+
* fan-out, so it cannot delay a query that is not about arXiv.
|
|
8
|
+
*
|
|
9
|
+
* @module @fangchenghao/dsh-web-search/arxiv
|
|
10
|
+
*/
|
|
11
|
+
import type { Source } from './types.ts';
|
|
12
|
+
/** Options for the narrow arXiv lookup. */
|
|
13
|
+
export interface ArxivLookupOptions {
|
|
14
|
+
/** Result budget; defaults to four. */
|
|
15
|
+
limit?: number;
|
|
16
|
+
/** Deadline in milliseconds; defaults to 3.5 s. */
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
/** Configured tunnel. */
|
|
19
|
+
proxy?: string;
|
|
20
|
+
/** Caller cancellation. */
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The query text with the registry name removed, or `undefined` when the
|
|
25
|
+
* query is not explicitly arXiv-scoped.
|
|
26
|
+
*
|
|
27
|
+
* @param query - raw user query.
|
|
28
|
+
* @returns cleaned query, or `undefined`.
|
|
29
|
+
*/
|
|
30
|
+
export declare function arxivQueryText(query: string): string | undefined;
|
|
31
|
+
/**
|
|
32
|
+
* Query the arXiv vertical for an explicitly scoped query.
|
|
33
|
+
*
|
|
34
|
+
* @param query - raw user query.
|
|
35
|
+
* @param options - deadline, budget, proxy and cancellation.
|
|
36
|
+
* @returns matching sources, or an empty array when the query is not scoped.
|
|
37
|
+
*/
|
|
38
|
+
export declare function arxivLookup(query: string, options?: ArxivLookupOptions): Promise<Source[]>;
|
|
39
|
+
/**
|
|
40
|
+
* Put arXiv rows first and remove duplicate URLs.
|
|
41
|
+
*
|
|
42
|
+
* @param sources - ordinary fused rows.
|
|
43
|
+
* @param arxiv - rows from the arXiv vertical.
|
|
44
|
+
* @returns merged rows.
|
|
45
|
+
*/
|
|
46
|
+
export declare function mergeArxiv(sources: readonly Source[], arxiv: readonly Source[]): Source[];
|
package/lib/arxiv.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { searchVerticals } from "./verticals.js";
|
|
2
|
+
const ARXIV_QUERY = /\barxiv(?:\.org)?\b/iu;
|
|
3
|
+
const ARXIV_ID = /\barxiv\s*:\s*(\d{4}\.\d{4,5})/iu;
|
|
4
|
+
/** A paper intent keeps "how to use arXiv" from becoming a paper search. */
|
|
5
|
+
const RESEARCH_INTENT = /\b(?:paper|papers|preprint|preprints|survey|surveys|study|studies|research|literature|review|reviews)\b|论文|文献|预印本|综述|研究/iu;
|
|
6
|
+
/**
|
|
7
|
+
* The query text with the registry name removed, or `undefined` when the
|
|
8
|
+
* query is not explicitly arXiv-scoped.
|
|
9
|
+
*
|
|
10
|
+
* @param query - raw user query.
|
|
11
|
+
* @returns cleaned query, or `undefined`.
|
|
12
|
+
*/
|
|
13
|
+
export function arxivQueryText(query) {
|
|
14
|
+
if (typeof query !== 'string')
|
|
15
|
+
return undefined;
|
|
16
|
+
const trimmed = query.trim();
|
|
17
|
+
if (trimmed.length === 0)
|
|
18
|
+
return undefined;
|
|
19
|
+
const id = ARXIV_ID.exec(trimmed);
|
|
20
|
+
if (id !== null)
|
|
21
|
+
return id[1];
|
|
22
|
+
if (!ARXIV_QUERY.test(trimmed) || !RESEARCH_INTENT.test(trimmed))
|
|
23
|
+
return undefined;
|
|
24
|
+
const cleaned = trimmed.replace(ARXIV_QUERY, ' ').replace(/\s+/gu, ' ').trim();
|
|
25
|
+
return cleaned.length === 0 ? undefined : cleaned;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Query the arXiv vertical for an explicitly scoped query.
|
|
29
|
+
*
|
|
30
|
+
* @param query - raw user query.
|
|
31
|
+
* @param options - deadline, budget, proxy and cancellation.
|
|
32
|
+
* @returns matching sources, or an empty array when the query is not scoped.
|
|
33
|
+
*/
|
|
34
|
+
export async function arxivLookup(query, options = {}) {
|
|
35
|
+
const text = arxivQueryText(query);
|
|
36
|
+
if (text === undefined)
|
|
37
|
+
return [];
|
|
38
|
+
const limit = options.limit ?? 4;
|
|
39
|
+
try {
|
|
40
|
+
const result = await searchVerticals(text, {
|
|
41
|
+
ids: ['arxiv'],
|
|
42
|
+
limit,
|
|
43
|
+
timeoutMs: options.timeoutMs ?? 3_500,
|
|
44
|
+
...(options.proxy === undefined ? {} : { proxy: options.proxy }),
|
|
45
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
46
|
+
});
|
|
47
|
+
return result.sources;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return [];
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Put arXiv rows first and remove duplicate URLs.
|
|
55
|
+
*
|
|
56
|
+
* @param sources - ordinary fused rows.
|
|
57
|
+
* @param arxiv - rows from the arXiv vertical.
|
|
58
|
+
* @returns merged rows.
|
|
59
|
+
*/
|
|
60
|
+
export function mergeArxiv(sources, arxiv) {
|
|
61
|
+
const seen = new Set();
|
|
62
|
+
const merged = [];
|
|
63
|
+
for (const source of [...arxiv, ...sources]) {
|
|
64
|
+
if (source.url.length === 0 || seen.has(source.url))
|
|
65
|
+
continue;
|
|
66
|
+
seen.add(source.url);
|
|
67
|
+
merged.push(source);
|
|
68
|
+
}
|
|
69
|
+
return merged;
|
|
70
|
+
}
|
package/lib/config.d.ts
CHANGED
|
@@ -48,6 +48,7 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
48
48
|
cacheFile: z<string, string, "volatile-defined">;
|
|
49
49
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
50
50
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
51
|
+
firecrawlApiKey: z<string, string, "volatile-defined">;
|
|
51
52
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
52
53
|
maxCooldownMs: z<number, number, "volatile-defined">;
|
|
53
54
|
domainSources: z<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
@@ -79,6 +80,7 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
|
|
|
79
80
|
cacheFile: z<string, string, "volatile-defined">;
|
|
80
81
|
anysearchApiKey: z<string, string, "volatile-defined">;
|
|
81
82
|
anysearchDomain: z<string, string, "volatile-defined">;
|
|
83
|
+
firecrawlApiKey: z<string, string, "volatile-defined">;
|
|
82
84
|
baseCooldownMs: z<number, number, "volatile-defined">;
|
|
83
85
|
maxCooldownMs: z<number, number, "volatile-defined">;
|
|
84
86
|
domainSources: z<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
@@ -139,6 +141,8 @@ export interface ResolvedConfig {
|
|
|
139
141
|
anysearchApiKey: string;
|
|
140
142
|
/** AnySearch vertical domain, or an empty string for the open web. */
|
|
141
143
|
anysearchDomain: string;
|
|
144
|
+
/** Optional Firecrawl credential for the quota-limited fallback. */
|
|
145
|
+
firecrawlApiKey: string;
|
|
142
146
|
/** Ranking knobs. */
|
|
143
147
|
rank: RankOptions;
|
|
144
148
|
/** Cache lifetime in milliseconds. */
|
package/lib/config.js
CHANGED
|
@@ -112,6 +112,10 @@ export const Config = z.object({
|
|
|
112
112
|
.string()
|
|
113
113
|
.default('')
|
|
114
114
|
.description('只搜某一类专业内容(如 finance 财经、academic 学术、code 代码)。要专查某类才填;日常搜索留空。').volatile(),
|
|
115
|
+
firecrawlApiKey: z
|
|
116
|
+
.string()
|
|
117
|
+
.default('')
|
|
118
|
+
.description('Firecrawl 通行证(可选):留空仍可匿名调用兜底源,但匿名配额很紧,连续查询会返回 429;填自己的 key 后 Firecrawl 兜底更稳定。不填不影响主搜索引擎。').volatile(),
|
|
115
119
|
baseCooldownMs: z.natural().default(300000).description('搜索引擎说“你查得太频繁了”后,先歇多久(毫秒)。歇太久会少一个引擎,太短易被再拒;保持 300000(5 分钟)即可。').volatile(),
|
|
116
120
|
maxCooldownMs: z.natural().default(1800000).description('它反复被拒时,最长让它歇多久(毫秒),也就是等待一步步加长的封顶值。一般不用改;保持 1800000(30 分钟)即可。').volatile(),
|
|
117
121
|
domainSources: z
|
|
@@ -190,6 +194,7 @@ export function resolveConfig(config, callLimit) {
|
|
|
190
194
|
fallbackOn: [...settings.fallbackOn],
|
|
191
195
|
anysearchApiKey: settings.anysearchApiKey.trim(),
|
|
192
196
|
anysearchDomain: settings.anysearchDomain.trim(),
|
|
197
|
+
firecrawlApiKey: settings.firecrawlApiKey.trim(),
|
|
193
198
|
domainSourceIds: domainSources.length > 0 ? domainSources : [...DEFAULT_VERTICAL_IDS],
|
|
194
199
|
domainLimit: settings.domainLimit,
|
|
195
200
|
domainTimeoutMs: settings.domainTimeoutMs,
|
package/lib/engines.d.ts
CHANGED
|
@@ -22,12 +22,24 @@ import type { EngineHealth } from './health.ts';
|
|
|
22
22
|
* runs as a first choice or only when no primary answered.
|
|
23
23
|
*/
|
|
24
24
|
export declare const HOSTS: readonly HostSpec[];
|
|
25
|
+
/**
|
|
26
|
+
* Whether a tier may release before its soft deadline.
|
|
27
|
+
*
|
|
28
|
+
* `limit` and `needed` are intentionally separate: eight rows from one engine
|
|
29
|
+
* are still a single-engine answer, while eight rows from two engines are a
|
|
30
|
+
* genuine cross-engine result. Counting every region here is what prevents a
|
|
31
|
+
* slow lead engine from holding a query open after another region supplied
|
|
32
|
+
* enough rows.
|
|
33
|
+
*/
|
|
34
|
+
export declare function tierReleaseReady(available: number, contributors: number, limit: number, needed: number): boolean;
|
|
25
35
|
/** Options for {@link search}: the seam's budget plus the plugin's runtime. */
|
|
26
36
|
export interface EngineRunOptions extends SearchOptions {
|
|
27
37
|
/** AnySearch Bearer token; empty or absent means anonymous access. */
|
|
28
38
|
anysearchApiKey?: string;
|
|
29
39
|
/** AnySearch vertical domain, such as `finance` or `finance.us_stock`. */
|
|
30
40
|
anysearchDomain?: string;
|
|
41
|
+
/** Optional Firecrawl Bearer token; absent keeps the keyless fallback. */
|
|
42
|
+
firecrawlApiKey?: string;
|
|
31
43
|
/** Ranking knobs, falling back to the documented defaults. */
|
|
32
44
|
rank?: RankOptions;
|
|
33
45
|
/** Failure classes that may trigger the fallback tier; absent means all. */
|
|
@@ -95,13 +107,13 @@ export interface SearchTrace {
|
|
|
95
107
|
/**
|
|
96
108
|
* Apply per-run configuration to an engine definition.
|
|
97
109
|
*
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
* exported `HOSTS` catalogue static, so a stale credential can never leak
|
|
101
|
-
* the next run through a mutated descriptor.
|
|
110
|
+
* AnySearch can be pointed at one of its vertical domains; both AnySearch and
|
|
111
|
+
* Firecrawl authenticate with an optional Bearer token. Returning a copy keeps
|
|
112
|
+
* the exported `HOSTS` catalogue static, so a stale credential can never leak
|
|
113
|
+
* into the next run through a mutated descriptor.
|
|
102
114
|
* @param host - the engine definition from `HOSTS`.
|
|
103
|
-
* @param options - the run options carrying
|
|
115
|
+
* @param options - the run options carrying credentials and the domain.
|
|
104
116
|
* @returns the engine definition to use for this run.
|
|
105
117
|
*/
|
|
106
|
-
export declare function resolveHost(host: HostSpec, options: Pick<EngineRunOptions, 'anysearchApiKey' | 'anysearchDomain'>): HostSpec;
|
|
118
|
+
export declare function resolveHost(host: HostSpec, options: Pick<EngineRunOptions, 'anysearchApiKey' | 'anysearchDomain' | 'firecrawlApiKey'>): HostSpec;
|
|
107
119
|
export declare function search(query: string, signal: AbortSignal | undefined, options: EngineRunOptions): Promise<SearchOutcome>;
|
package/lib/engines.js
CHANGED
|
@@ -179,7 +179,10 @@ function ddgWindow(timeRange) {
|
|
|
179
179
|
*/
|
|
180
180
|
/** Soft deadline per wait policy, in milliseconds. */
|
|
181
181
|
const SOFT_DEADLINE = {
|
|
182
|
-
|
|
182
|
+
// Auto is the interactive path: cap a slow fan-out leg instead of letting
|
|
183
|
+
// the caller absorb the slowest engine. `all` keeps the completeness escape
|
|
184
|
+
// hatch for callers that explicitly want every engine.
|
|
185
|
+
auto: (ascii) => (ascii ? 5000 : 3000),
|
|
183
186
|
all: () => 10000,
|
|
184
187
|
fast: () => 3500,
|
|
185
188
|
};
|
|
@@ -250,6 +253,18 @@ function allowsFallback(details, fallbackOn) {
|
|
|
250
253
|
const allowed = new Set(fallbackOn);
|
|
251
254
|
return details.every((detail) => allowed.has(detail.kind));
|
|
252
255
|
}
|
|
256
|
+
/**
|
|
257
|
+
* Whether a tier may release before its soft deadline.
|
|
258
|
+
*
|
|
259
|
+
* `limit` and `needed` are intentionally separate: eight rows from one engine
|
|
260
|
+
* are still a single-engine answer, while eight rows from two engines are a
|
|
261
|
+
* genuine cross-engine result. Counting every region here is what prevents a
|
|
262
|
+
* slow lead engine from holding a query open after another region supplied
|
|
263
|
+
* enough rows.
|
|
264
|
+
*/
|
|
265
|
+
export function tierReleaseReady(available, contributors, limit, needed) {
|
|
266
|
+
return available >= limit && contributors >= needed;
|
|
267
|
+
}
|
|
253
268
|
/**
|
|
254
269
|
* Ask one engine, rotating to the next key when the failure blames the key.
|
|
255
270
|
*
|
|
@@ -319,14 +334,22 @@ async function runTier(hosts, query, signal, timeoutMs, failures, failureDetails
|
|
|
319
334
|
const rawCounts = new Array(hosts.length).fill(0);
|
|
320
335
|
const answered = () => outcomes.filter((outcome) => outcome instanceof SearchError || outcome !== undefined);
|
|
321
336
|
const available = () => outcomes.reduce((total, outcome) => total + (outcome instanceof SearchError || outcome === undefined ? 0 : outcome.sources.length), 0);
|
|
322
|
-
|
|
337
|
+
// A contributor is any engine that returned rows, not only an engine in the
|
|
338
|
+
// leading region. The lead still decides ordering and the `leadSettled`
|
|
339
|
+
// shortcut; it must not also veto an early release when a healthy global
|
|
340
|
+
// engine has already supplied useful rows. This is the fan-out tail issue:
|
|
341
|
+
// a slow AnySearch request should not hold the whole Chinese query open
|
|
342
|
+
// after CN Bing and a global engine have enough rows between them.
|
|
343
|
+
const contributors = () => outcomes.filter((outcome) => !(outcome instanceof SearchError) && outcome !== undefined && outcome.sources.length > 0).length;
|
|
323
344
|
const leadTotal = hosts.filter(counters).length || hosts.length;
|
|
324
345
|
const needed = Math.min(limits.needed, leadTotal);
|
|
325
346
|
// Nothing more can arrive from the leading region once every one of its
|
|
326
347
|
// engines settled, so a failed engine there must not hold the tier open until
|
|
327
348
|
// the soft deadline.
|
|
328
349
|
const leadSettled = () => outcomes.filter((outcome, index) => counters(hosts[index]) && outcome !== undefined).length >= leadTotal;
|
|
329
|
-
const done = () => answered().length === hosts.length ||
|
|
350
|
+
const done = () => answered().length === hosts.length ||
|
|
351
|
+
leadSettled() ||
|
|
352
|
+
tierReleaseReady(available(), contributors(), limits.limit, needed);
|
|
330
353
|
let releasedEarly = false;
|
|
331
354
|
let closed = false;
|
|
332
355
|
let released = () => { };
|
|
@@ -405,15 +428,19 @@ async function runTier(hosts, query, signal, timeoutMs, failures, failureDetails
|
|
|
405
428
|
/**
|
|
406
429
|
* Apply per-run configuration to an engine definition.
|
|
407
430
|
*
|
|
408
|
-
*
|
|
409
|
-
*
|
|
410
|
-
* exported `HOSTS` catalogue static, so a stale credential can never leak
|
|
411
|
-
* the next run through a mutated descriptor.
|
|
431
|
+
* AnySearch can be pointed at one of its vertical domains; both AnySearch and
|
|
432
|
+
* Firecrawl authenticate with an optional Bearer token. Returning a copy keeps
|
|
433
|
+
* the exported `HOSTS` catalogue static, so a stale credential can never leak
|
|
434
|
+
* into the next run through a mutated descriptor.
|
|
412
435
|
* @param host - the engine definition from `HOSTS`.
|
|
413
|
-
* @param options - the run options carrying
|
|
436
|
+
* @param options - the run options carrying credentials and the domain.
|
|
414
437
|
* @returns the engine definition to use for this run.
|
|
415
438
|
*/
|
|
416
439
|
export function resolveHost(host, options) {
|
|
440
|
+
if (host.id === 'firecrawl') {
|
|
441
|
+
const firecrawlKey = (options.firecrawlApiKey ?? '').trim();
|
|
442
|
+
return firecrawlKey.length === 0 ? host : { ...host, headers: { authorization: `Bearer ${firecrawlKey}` } };
|
|
443
|
+
}
|
|
417
444
|
if (host.id !== 'anysearch')
|
|
418
445
|
return host;
|
|
419
446
|
const key = (options.anysearchApiKey ?? '').trim();
|
|
@@ -453,7 +480,11 @@ export async function search(query, signal, options) {
|
|
|
453
480
|
const allow = options.hostIds !== undefined && options.hostIds.length > 0 ? new Set(options.hostIds) : undefined;
|
|
454
481
|
const hosts = HOSTS.filter((host) => allow === undefined || allow.has(host.id))
|
|
455
482
|
.filter((host) => host.viaProxy !== true || proxy !== undefined)
|
|
456
|
-
.map((host) => resolveHost(host,
|
|
483
|
+
.map((host) => resolveHost(host, {
|
|
484
|
+
...(options.anysearchApiKey === undefined ? {} : { anysearchApiKey: options.anysearchApiKey }),
|
|
485
|
+
...(options.anysearchDomain === undefined ? {} : { anysearchDomain: options.anysearchDomain }),
|
|
486
|
+
...(options.firecrawlApiKey === undefined ? {} : { firecrawlApiKey: options.firecrawlApiKey }),
|
|
487
|
+
}));
|
|
457
488
|
// Skip engines that are cooling down, but never end up with no engine at all:
|
|
458
489
|
// a throttled engine is still better than no search.
|
|
459
490
|
const now = Date.now();
|
|
@@ -475,9 +506,13 @@ export async function search(query, signal, options) {
|
|
|
475
506
|
const fallbackOn = options.fallbackOn;
|
|
476
507
|
const ascii = !CJK.test(query);
|
|
477
508
|
const lead = ascii ? 'global' : 'cn';
|
|
478
|
-
const wanted = usable.filter((host) => leadsQuery(host, lead)).length;
|
|
479
509
|
const softMs = options.softMs ?? SOFT_DEADLINE[wait](ascii);
|
|
480
|
-
|
|
510
|
+
// `auto` is a latency policy, not a completeness policy: two independent
|
|
511
|
+
// contributors are enough to make fusion meaningful. Waiting for every lead
|
|
512
|
+
// engine made one slow AnySearch leg hold an otherwise complete answer open
|
|
513
|
+
// for the full soft deadline. `all` still asks for every engine, and `fast`
|
|
514
|
+
// keeps its first-answer behavior.
|
|
515
|
+
const needed = wait === 'all' ? usable.length : wait === 'fast' ? 1 : Math.min(2, usable.length);
|
|
481
516
|
const limits = { limit, softMs, needed, lead: wait === 'all' ? 'all' : lead, timeRange };
|
|
482
517
|
// Ranking does not drop rows; the engine layer does. `timeRange` is passed
|
|
483
518
|
// into the rank options only so undated rows are scored lower while a window
|
package/lib/fetch.js
CHANGED
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
import { SearchError } from "./errors.js";
|
|
36
36
|
import { keyFailureKind, keyPoolFor } from "./keys.js";
|
|
37
37
|
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
|
|
38
|
+
import { arxivReadableUrl } from "./arxiv-url.js";
|
|
38
39
|
import { proxyGet, resolveProxy } from "./proxy.js";
|
|
39
40
|
import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
|
|
40
41
|
/** Default user agent; kept in step with the search side. */
|
|
@@ -388,10 +389,13 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
388
389
|
id: options.id ?? 'http-proxy',
|
|
389
390
|
available: () => true,
|
|
390
391
|
async fetch(request, signal) {
|
|
391
|
-
const
|
|
392
|
-
if (typeof
|
|
393
|
-
throw new SearchError(`not an http(s) URL: ${String(
|
|
392
|
+
const requested = request.url;
|
|
393
|
+
if (typeof requested !== 'string' || !/^https?:\/\//i.test(requested)) {
|
|
394
|
+
throw new SearchError(`not an http(s) URL: ${String(requested)}`, 'SEARCH_PARSE', false);
|
|
394
395
|
}
|
|
396
|
+
// arXiv PDFs are binary and would otherwise be rejected as unsupported.
|
|
397
|
+
// Read the HTML full-text view instead; ordinary HTML pages pass through.
|
|
398
|
+
const target = arxivReadableUrl(requested);
|
|
395
399
|
// The ladder budget belongs to THIS request. Starting it in the factory
|
|
396
400
|
// made it start at plugin mount, so every fetch more than 25 s after a
|
|
397
401
|
// restart failed with "no time left in the fetch budget" before touching
|
package/lib/index.d.ts
CHANGED
|
@@ -97,6 +97,7 @@ declare const _default: typeof apply & {
|
|
|
97
97
|
cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
98
98
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
99
99
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
100
|
+
firecrawlApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
100
101
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
101
102
|
maxCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
102
103
|
domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
|
@@ -128,6 +129,7 @@ declare const _default: typeof apply & {
|
|
|
128
129
|
cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
129
130
|
anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
130
131
|
anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
132
|
+
firecrawlApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
|
|
131
133
|
baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
132
134
|
maxCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
|
|
133
135
|
domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
|
package/lib/index.js
CHANGED
|
@@ -27,7 +27,8 @@ import { registerEngineTestBridge, registerUpdateBridge } from "./bridge.js";
|
|
|
27
27
|
import { registerAdvancedSearchTool, undatedWindowNote } from "./advanced-search.js";
|
|
28
28
|
import { searchVerticals } from "./verticals.js";
|
|
29
29
|
import { routePackageQuery } from "./query-router.js";
|
|
30
|
-
import {
|
|
30
|
+
import { mergeOfficial, officialLookup } from "./official-sources.js";
|
|
31
|
+
import { arxivLookup, arxivQueryText, mergeArxiv } from "./arxiv.js";
|
|
31
32
|
import { chromeFeedNote, chromeWindow, fetchChromeReleases, filterChromeChannel, hasNonStableChromePosts, isChromeSecurityQuery, } from "./official-feed.js";
|
|
32
33
|
import { registerSearchPrompt } from "./system-prompt.js";
|
|
33
34
|
import { registerUnifiedTool } from "./tool-suite.js";
|
|
@@ -147,6 +148,7 @@ export function apply(ctx, config = {}) {
|
|
|
147
148
|
rank: resolved.rank,
|
|
148
149
|
...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
|
|
149
150
|
...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
|
|
151
|
+
...(resolved.firecrawlApiKey.length === 0 ? {} : { firecrawlApiKey: resolved.firecrawlApiKey }),
|
|
150
152
|
cache,
|
|
151
153
|
health,
|
|
152
154
|
...(signal === undefined ? {} : { signal }),
|
|
@@ -236,6 +238,7 @@ export function apply(ctx, config = {}) {
|
|
|
236
238
|
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
237
239
|
...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
|
|
238
240
|
...(live.anysearchDomain.length === 0 ? {} : { anysearchDomain: live.anysearchDomain }),
|
|
241
|
+
...(live.firecrawlApiKey.length === 0 ? {} : { firecrawlApiKey: live.firecrawlApiKey }),
|
|
239
242
|
hostIds: [engine],
|
|
240
243
|
});
|
|
241
244
|
};
|
|
@@ -254,27 +257,36 @@ export function apply(ctx, config = {}) {
|
|
|
254
257
|
const asked = typeof request.maxResults === 'number' && Number.isInteger(request.maxResults) && request.maxResults > 0
|
|
255
258
|
? request.maxResults
|
|
256
259
|
: undefined;
|
|
260
|
+
const live = resolveConfig(section, undefined);
|
|
261
|
+
// Start an official lookup before the fan-out and await it afterwards.
|
|
262
|
+
// It then costs max(search, lookup), not search + lookup, which matters
|
|
263
|
+
// for a version query where the first result should be authoritative but
|
|
264
|
+
// the ordinary engines still need to fill the remaining rows.
|
|
265
|
+
const official = officialLookup(query, {
|
|
266
|
+
timeoutMs: Math.min(live.timeoutMs, 2_500),
|
|
267
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
268
|
+
...(signal === undefined ? {} : { signal }),
|
|
269
|
+
});
|
|
270
|
+
// An explicit `arXiv` query gets the preprint index in parallel with the
|
|
271
|
+
// ordinary fan-out. This is separate from the official-release lookup:
|
|
272
|
+
// a survey query needs arXiv's own search, not a release announcement.
|
|
273
|
+
const arxiv = arxivLookup(query, {
|
|
274
|
+
limit: 4,
|
|
275
|
+
timeoutMs: Math.min(live.timeoutMs, 3_500),
|
|
276
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
277
|
+
...(signal === undefined ? {} : { signal }),
|
|
278
|
+
});
|
|
257
279
|
const outcome = await runQuery(query, asked, signal);
|
|
258
|
-
// A version/EOL question about PostgreSQL is better answered by
|
|
259
|
-
// project's own
|
|
280
|
+
// A version/EOL question about PostgreSQL or Rust is better answered by
|
|
281
|
+
// the project's own release facts than by whatever an engine ranks first.
|
|
260
282
|
// The lookup is query-scoped and failure-tolerant; it never replaces the
|
|
261
283
|
// ordinary rows when the page is unavailable.
|
|
262
|
-
let sources = outcome.sources;
|
|
263
|
-
|
|
264
|
-
const live = resolveConfig(section, undefined);
|
|
265
|
-
const official = await fetchPostgresSupport({
|
|
266
|
-
timeoutMs: Math.min(live.timeoutMs, 8_000),
|
|
267
|
-
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
268
|
-
...(signal === undefined ? {} : { signal }),
|
|
269
|
-
});
|
|
270
|
-
if (official !== undefined) {
|
|
271
|
-
sources = [official, ...sources.filter((source) => source.url !== official.url)];
|
|
272
|
-
}
|
|
273
|
-
}
|
|
284
|
+
let sources = mergeOfficial(outcome.sources, await official);
|
|
285
|
+
sources = mergeArxiv(sources, await arxiv);
|
|
274
286
|
if (sources.length === 0) {
|
|
275
287
|
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
|
|
276
288
|
}
|
|
277
|
-
const cap = Math.min(asked ??
|
|
289
|
+
const cap = Math.min(asked ?? live.limit, live.limit);
|
|
278
290
|
return { sources: sources.slice(0, cap), truncated: outcome.truncated || sources.length > cap };
|
|
279
291
|
},
|
|
280
292
|
});
|
|
@@ -319,6 +331,26 @@ export function apply(ctx, config = {}) {
|
|
|
319
331
|
* returns nothing, so a registry outage cannot empty the batch.
|
|
320
332
|
*/
|
|
321
333
|
const runBatchQuery = async (query, limit, signal) => {
|
|
334
|
+
// An explicit arXiv query is a paper lookup, not a general web query.
|
|
335
|
+
// Keep the ordinary path as the fallback when the index returns no rows.
|
|
336
|
+
const arxivText = arxivQueryText(query);
|
|
337
|
+
if (arxivText !== undefined) {
|
|
338
|
+
const live = resolveConfig(section, undefined);
|
|
339
|
+
const papers = await arxivLookup(query, {
|
|
340
|
+
limit,
|
|
341
|
+
timeoutMs: live.domainTimeoutMs,
|
|
342
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
343
|
+
...(signal === undefined ? {} : { signal }),
|
|
344
|
+
});
|
|
345
|
+
if (papers.length > 0) {
|
|
346
|
+
return {
|
|
347
|
+
sources: papers.slice(0, limit),
|
|
348
|
+
failures: [],
|
|
349
|
+
note: `arXiv query routed to the arXiv index for ${JSON.stringify(arxivText)}; rows are preprints with stable abs URLs.`,
|
|
350
|
+
truncated: papers.length > limit,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
}
|
|
322
354
|
const route = routePackageQuery(query);
|
|
323
355
|
if (route !== undefined) {
|
|
324
356
|
const live = resolveConfig(section, undefined);
|
package/lib/markdown.js
CHANGED
|
@@ -374,13 +374,29 @@ function prune(node) {
|
|
|
374
374
|
kept.push(child);
|
|
375
375
|
continue;
|
|
376
376
|
}
|
|
377
|
-
if (DROP_TAGS.has(child.tag)
|
|
377
|
+
if (DROP_TAGS.has(child.tag))
|
|
378
|
+
continue;
|
|
379
|
+
// A wrapper whose class says "footer" may still contain the page's main
|
|
380
|
+
// article. arXiv currently wraps its whole document in
|
|
381
|
+
// `div.flex-wrap-footer`; dropping that wrapper removed every readable
|
|
382
|
+
// character from the abstract page. Chrome inside the wrapper is still
|
|
383
|
+
// pruned by its own tag/attributes.
|
|
384
|
+
if (!NOISE_EXEMPT_TAGS.has(child.tag) && isNoise(child) && !containsLandmark(child))
|
|
378
385
|
continue;
|
|
379
386
|
prune(child);
|
|
380
387
|
kept.push(child);
|
|
381
388
|
}
|
|
382
389
|
node.children = kept;
|
|
383
390
|
}
|
|
391
|
+
/** Whether a subtree contains a real content landmark. */
|
|
392
|
+
function containsLandmark(node) {
|
|
393
|
+
let found = false;
|
|
394
|
+
walkElements(node, (child) => {
|
|
395
|
+
if (child.tag === 'main' || child.tag === 'article')
|
|
396
|
+
found = true;
|
|
397
|
+
});
|
|
398
|
+
return found;
|
|
399
|
+
}
|
|
384
400
|
/** Whether an element is hidden or named like site furniture. */
|
|
385
401
|
function isNoise(node) {
|
|
386
402
|
const role = (node.attrs['role'] ?? '').toLowerCase();
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Official Rust release notes for an explicitly versioned release query.
|
|
3
|
+
*
|
|
4
|
+
* Search engines often return the Rust home page or a third-party changelog for
|
|
5
|
+
* a query such as `Rust 1.90 发布说明`, even when the official announcement is
|
|
6
|
+
* indexed. The release announcement has a deterministic shape once its date is
|
|
7
|
+
* known: `https://blog.rust-lang.org/YYYY/MM/DD/Rust-X.Y.Z/`. GitHub's public
|
|
8
|
+
* release endpoint is key-free and exposes that date, so this module can add
|
|
9
|
+
* the official page as a high-priority row without hard-coding release dates.
|
|
10
|
+
*
|
|
11
|
+
* The lookup is deliberately narrow. Only a Rust query that names a numeric
|
|
12
|
+
* version *and* asks for release notes is enriched; a generic Rust tutorial
|
|
13
|
+
* keeps the normal engine ranking.
|
|
14
|
+
*
|
|
15
|
+
* @module @fangchenghao/dsh-web-search/official-release
|
|
16
|
+
*/
|
|
17
|
+
import type { Source } from './types.ts';
|
|
18
|
+
/** Public GitHub release endpoint for the Rust repository. */
|
|
19
|
+
export declare const RUST_RELEASE_API = "https://api.github.com/repos/rust-lang/rust/releases/tags";
|
|
20
|
+
/** Official Rust announcement host. */
|
|
21
|
+
export declare const RUST_BLOG_HOST = "blog.rust-lang.org";
|
|
22
|
+
/** Inputs accepted by {@link fetchRustRelease}. */
|
|
23
|
+
export interface RustReleaseOptions {
|
|
24
|
+
/** Tunnel URL from the plugin's `proxy` setting, when configured. */
|
|
25
|
+
proxy?: string;
|
|
26
|
+
/** Per-request deadline in milliseconds. */
|
|
27
|
+
timeoutMs?: number;
|
|
28
|
+
/** Caller cancellation. */
|
|
29
|
+
signal?: AbortSignal;
|
|
30
|
+
/** Test seam; defaults to the platform `fetch`. */
|
|
31
|
+
fetch?: typeof fetch;
|
|
32
|
+
}
|
|
33
|
+
/** A version such as `1.90`, normalized to a release tag like `1.90.0`. */
|
|
34
|
+
export interface RustVersion {
|
|
35
|
+
/** User-facing version, e.g. `1.90.0`. */
|
|
36
|
+
readonly version: string;
|
|
37
|
+
/** GitHub tag, normally identical to {@link version}. */
|
|
38
|
+
readonly tag: string;
|
|
39
|
+
}
|
|
40
|
+
/** Extract a Rust version from a release query. */
|
|
41
|
+
export declare function rustReleaseVersion(query: string): RustVersion | undefined;
|
|
42
|
+
/** True for a query asking for a specific Rust release's notes. */
|
|
43
|
+
export declare function isRustReleaseQuery(query: string): boolean;
|
|
44
|
+
/**
|
|
45
|
+
* Build the canonical announcement URL from a GitHub release timestamp.
|
|
46
|
+
*
|
|
47
|
+
* Rust publishes the post on the UTC date in the release metadata. Using UTC
|
|
48
|
+
* avoids shifting the path when the machine is in a negative-offset timezone.
|
|
49
|
+
*/
|
|
50
|
+
export declare function rustBlogUrl(version: string, publishedAt: string): string | undefined;
|
|
51
|
+
/**
|
|
52
|
+
* Parse a GitHub release response into one official-blog source.
|
|
53
|
+
*
|
|
54
|
+
* A malformed response, a missing publication date or a tag that does not
|
|
55
|
+
* match the requested version returns `undefined`; the caller then keeps the
|
|
56
|
+
* ordinary search rows.
|
|
57
|
+
*
|
|
58
|
+
* @param payload - GitHub JSON response body.
|
|
59
|
+
* @param requested - normalized version from the query.
|
|
60
|
+
* @returns the official source, or `undefined`.
|
|
61
|
+
*/
|
|
62
|
+
export declare function parseRustRelease(payload: string, requested: RustVersion): Source | undefined;
|
|
63
|
+
/**
|
|
64
|
+
* Fetch the official Rust release page for a query.
|
|
65
|
+
*
|
|
66
|
+
* Failure is contained: an unavailable API, rate limit or malformed body
|
|
67
|
+
* returns `undefined`, and the caller keeps its ordinary search rows.
|
|
68
|
+
*
|
|
69
|
+
* @param query - the original user query.
|
|
70
|
+
* @param options - proxy, deadline, cancellation and fetch seam.
|
|
71
|
+
* @returns the official blog source, or `undefined`.
|
|
72
|
+
*/
|
|
73
|
+
export declare function fetchRustRelease(query: string, options?: RustReleaseOptions): Promise<Source | undefined>;
|