@fffangchenghao/dsh-web-search 0.21.1 → 0.23.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,62 @@
1
+ /**
2
+ * Recognize arXiv page URLs and map PDF requests to the HTML full-text view.
3
+ *
4
+ * arXiv serves the same paper as `/abs/<id>` (metadata and abstract),
5
+ * `/html/<id>` (full text) and `/pdf/<id>` (binary PDF). This plugin converts
6
+ * HTML and text to Markdown but deliberately does not pretend binary PDF bytes
7
+ * are prose, so a PDF link is read through the HTML view instead.
8
+ *
9
+ * @module @fangchenghao/dsh-web-search/arxiv-url
10
+ */
11
+ const HOST = /^(?:www\.|export\.)?arxiv\.org$/iu;
12
+ /**
13
+ * Parse an arXiv abs/html/pdf URL.
14
+ *
15
+ * @param url - any URL string.
16
+ * @returns the identifier and kind, or `undefined` for other hosts.
17
+ */
18
+ export function parseArxivUrl(url) {
19
+ let parsed;
20
+ try {
21
+ parsed = new URL(url);
22
+ }
23
+ catch {
24
+ return undefined;
25
+ }
26
+ if (!HOST.test(parsed.hostname))
27
+ return undefined;
28
+ const match = /^\/(abs|html|pdf)\/(.+?)\/?$/iu.exec(parsed.pathname);
29
+ if (match === null)
30
+ return undefined;
31
+ const kind = match[1]?.toLowerCase();
32
+ let raw = match[2] ?? '';
33
+ raw = raw.replace(/\.pdf$/iu, '');
34
+ const versionMatch = /v(\d+)$/iu.exec(raw);
35
+ const version = versionMatch?.[1];
36
+ const id = version === undefined ? raw : raw.slice(0, -versionMatch[0].length);
37
+ if (id.length === 0 || !/[a-z0-9]/iu.test(id))
38
+ return undefined;
39
+ return { id, kind, ...(version === undefined ? {} : { version }) };
40
+ }
41
+ /**
42
+ * The HTML full-text URL for a paper.
43
+ *
44
+ * @param reference - parsed arXiv URL.
45
+ * @returns an `https://arxiv.org/html/...` URL.
46
+ */
47
+ export function arxivHtmlUrl(reference) {
48
+ const suffix = reference.version === undefined ? '' : `v${reference.version}`;
49
+ return `https://arxiv.org/html/${reference.id}${suffix}`;
50
+ }
51
+ /**
52
+ * Map a PDF URL to the HTML view; leave other URLs alone.
53
+ *
54
+ * @param url - the caller's URL.
55
+ * @returns the URL the fetch ladder should read.
56
+ */
57
+ export function arxivReadableUrl(url) {
58
+ const reference = parseArxivUrl(url);
59
+ if (reference === undefined || reference.kind !== 'pdf')
60
+ return url;
61
+ return arxivHtmlUrl(reference);
62
+ }
package/lib/arxiv.d.ts ADDED
@@ -0,0 +1,46 @@
1
+ /**
2
+ * Explicit `arXiv` research queries, answered by the existing arXiv vertical.
3
+ *
4
+ * A general web query that merely contains the word `arxiv` should not be
5
+ * hijacked, but a query that explicitly asks arXiv for a survey or paper is a
6
+ * clear registry lookup. The lookup runs in parallel with the ordinary
7
+ * fan-out, so it cannot delay a query that is not about arXiv.
8
+ *
9
+ * @module @fangchenghao/dsh-web-search/arxiv
10
+ */
11
+ import type { Source } from './types.ts';
12
+ /** Options for the narrow arXiv lookup. */
13
+ export interface ArxivLookupOptions {
14
+ /** Result budget; defaults to four. */
15
+ limit?: number;
16
+ /** Deadline in milliseconds; defaults to 3.5 s. */
17
+ timeoutMs?: number;
18
+ /** Configured tunnel. */
19
+ proxy?: string;
20
+ /** Caller cancellation. */
21
+ signal?: AbortSignal;
22
+ }
23
+ /**
24
+ * The query text with the registry name removed, or `undefined` when the
25
+ * query is not explicitly arXiv-scoped.
26
+ *
27
+ * @param query - raw user query.
28
+ * @returns cleaned query, or `undefined`.
29
+ */
30
+ export declare function arxivQueryText(query: string): string | undefined;
31
+ /**
32
+ * Query the arXiv vertical for an explicitly scoped query.
33
+ *
34
+ * @param query - raw user query.
35
+ * @param options - deadline, budget, proxy and cancellation.
36
+ * @returns matching sources, or an empty array when the query is not scoped.
37
+ */
38
+ export declare function arxivLookup(query: string, options?: ArxivLookupOptions): Promise<Source[]>;
39
+ /**
40
+ * Put arXiv rows first and remove duplicate URLs.
41
+ *
42
+ * @param sources - ordinary fused rows.
43
+ * @param arxiv - rows from the arXiv vertical.
44
+ * @returns merged rows.
45
+ */
46
+ export declare function mergeArxiv(sources: readonly Source[], arxiv: readonly Source[]): Source[];
package/lib/arxiv.js ADDED
@@ -0,0 +1,70 @@
1
+ import { searchVerticals } from "./verticals.js";
2
+ const ARXIV_QUERY = /\barxiv(?:\.org)?\b/iu;
3
+ const ARXIV_ID = /\barxiv\s*:\s*(\d{4}\.\d{4,5})/iu;
4
+ /** A paper intent keeps "how to use arXiv" from becoming a paper search. */
5
+ const RESEARCH_INTENT = /\b(?:paper|papers|preprint|preprints|survey|surveys|study|studies|research|literature|review|reviews)\b|论文|文献|预印本|综述|研究/iu;
6
+ /**
7
+ * The query text with the registry name removed, or `undefined` when the
8
+ * query is not explicitly arXiv-scoped.
9
+ *
10
+ * @param query - raw user query.
11
+ * @returns cleaned query, or `undefined`.
12
+ */
13
+ export function arxivQueryText(query) {
14
+ if (typeof query !== 'string')
15
+ return undefined;
16
+ const trimmed = query.trim();
17
+ if (trimmed.length === 0)
18
+ return undefined;
19
+ const id = ARXIV_ID.exec(trimmed);
20
+ if (id !== null)
21
+ return id[1];
22
+ if (!ARXIV_QUERY.test(trimmed) || !RESEARCH_INTENT.test(trimmed))
23
+ return undefined;
24
+ const cleaned = trimmed.replace(ARXIV_QUERY, ' ').replace(/\s+/gu, ' ').trim();
25
+ return cleaned.length === 0 ? undefined : cleaned;
26
+ }
27
+ /**
28
+ * Query the arXiv vertical for an explicitly scoped query.
29
+ *
30
+ * @param query - raw user query.
31
+ * @param options - deadline, budget, proxy and cancellation.
32
+ * @returns matching sources, or an empty array when the query is not scoped.
33
+ */
34
+ export async function arxivLookup(query, options = {}) {
35
+ const text = arxivQueryText(query);
36
+ if (text === undefined)
37
+ return [];
38
+ const limit = options.limit ?? 4;
39
+ try {
40
+ const result = await searchVerticals(text, {
41
+ ids: ['arxiv'],
42
+ limit,
43
+ timeoutMs: options.timeoutMs ?? 3_500,
44
+ ...(options.proxy === undefined ? {} : { proxy: options.proxy }),
45
+ ...(options.signal === undefined ? {} : { signal: options.signal }),
46
+ });
47
+ return result.sources;
48
+ }
49
+ catch {
50
+ return [];
51
+ }
52
+ }
53
+ /**
54
+ * Put arXiv rows first and remove duplicate URLs.
55
+ *
56
+ * @param sources - ordinary fused rows.
57
+ * @param arxiv - rows from the arXiv vertical.
58
+ * @returns merged rows.
59
+ */
60
+ export function mergeArxiv(sources, arxiv) {
61
+ const seen = new Set();
62
+ const merged = [];
63
+ for (const source of [...arxiv, ...sources]) {
64
+ if (source.url.length === 0 || seen.has(source.url))
65
+ continue;
66
+ seen.add(source.url);
67
+ merged.push(source);
68
+ }
69
+ return merged;
70
+ }
package/lib/config.d.ts CHANGED
@@ -48,6 +48,7 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
48
48
  cacheFile: z<string, string, "volatile-defined">;
49
49
  anysearchApiKey: z<string, string, "volatile-defined">;
50
50
  anysearchDomain: z<string, string, "volatile-defined">;
51
+ firecrawlApiKey: z<string, string, "volatile-defined">;
51
52
  baseCooldownMs: z<number, number, "volatile-defined">;
52
53
  maxCooldownMs: z<number, number, "volatile-defined">;
53
54
  domainSources: z<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
@@ -79,6 +80,7 @@ export declare const Config: z<Schemastery.ObjectS<NoInfer<{
79
80
  cacheFile: z<string, string, "volatile-defined">;
80
81
  anysearchApiKey: z<string, string, "volatile-defined">;
81
82
  anysearchDomain: z<string, string, "volatile-defined">;
83
+ firecrawlApiKey: z<string, string, "volatile-defined">;
82
84
  baseCooldownMs: z<number, number, "volatile-defined">;
83
85
  maxCooldownMs: z<number, number, "volatile-defined">;
84
86
  domainSources: z<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
@@ -139,6 +141,8 @@ export interface ResolvedConfig {
139
141
  anysearchApiKey: string;
140
142
  /** AnySearch vertical domain, or an empty string for the open web. */
141
143
  anysearchDomain: string;
144
+ /** Optional Firecrawl credential for the quota-limited fallback. */
145
+ firecrawlApiKey: string;
142
146
  /** Ranking knobs. */
143
147
  rank: RankOptions;
144
148
  /** Cache lifetime in milliseconds. */
package/lib/config.js CHANGED
@@ -112,6 +112,10 @@ export const Config = z.object({
112
112
  .string()
113
113
  .default('')
114
114
  .description('只搜某一类专业内容(如 finance 财经、academic 学术、code 代码)。要专查某类才填;日常搜索留空。').volatile(),
115
+ firecrawlApiKey: z
116
+ .string()
117
+ .default('')
118
+ .description('Firecrawl 通行证(可选):留空仍可匿名调用兜底源,但匿名配额很紧,连续查询会返回 429;填自己的 key 后 Firecrawl 兜底更稳定。不填不影响主搜索引擎。').volatile(),
115
119
  baseCooldownMs: z.natural().default(300000).description('搜索引擎说“你查得太频繁了”后,先歇多久(毫秒)。歇太久会少一个引擎,太短易被再拒;保持 300000(5 分钟)即可。').volatile(),
116
120
  maxCooldownMs: z.natural().default(1800000).description('它反复被拒时,最长让它歇多久(毫秒),也就是等待一步步加长的封顶值。一般不用改;保持 1800000(30 分钟)即可。').volatile(),
117
121
  domainSources: z
@@ -190,6 +194,7 @@ export function resolveConfig(config, callLimit) {
190
194
  fallbackOn: [...settings.fallbackOn],
191
195
  anysearchApiKey: settings.anysearchApiKey.trim(),
192
196
  anysearchDomain: settings.anysearchDomain.trim(),
197
+ firecrawlApiKey: settings.firecrawlApiKey.trim(),
193
198
  domainSourceIds: domainSources.length > 0 ? domainSources : [...DEFAULT_VERTICAL_IDS],
194
199
  domainLimit: settings.domainLimit,
195
200
  domainTimeoutMs: settings.domainTimeoutMs,
package/lib/engines.d.ts CHANGED
@@ -22,12 +22,24 @@ import type { EngineHealth } from './health.ts';
22
22
  * runs as a first choice or only when no primary answered.
23
23
  */
24
24
  export declare const HOSTS: readonly HostSpec[];
25
+ /**
26
+ * Whether a tier may release before its soft deadline.
27
+ *
28
+ * `limit` and `needed` are intentionally separate: eight rows from one engine
29
+ * are still a single-engine answer, while eight rows from two engines are a
30
+ * genuine cross-engine result. Counting every region here is what prevents a
31
+ * slow lead engine from holding a query open after another region supplied
32
+ * enough rows.
33
+ */
34
+ export declare function tierReleaseReady(available: number, contributors: number, limit: number, needed: number): boolean;
25
35
  /** Options for {@link search}: the seam's budget plus the plugin's runtime. */
26
36
  export interface EngineRunOptions extends SearchOptions {
27
37
  /** AnySearch Bearer token; empty or absent means anonymous access. */
28
38
  anysearchApiKey?: string;
29
39
  /** AnySearch vertical domain, such as `finance` or `finance.us_stock`. */
30
40
  anysearchDomain?: string;
41
+ /** Optional Firecrawl Bearer token; absent keeps the keyless fallback. */
42
+ firecrawlApiKey?: string;
31
43
  /** Ranking knobs, falling back to the documented defaults. */
32
44
  rank?: RankOptions;
33
45
  /** Failure classes that may trigger the fallback tier; absent means all. */
@@ -95,13 +107,13 @@ export interface SearchTrace {
95
107
  /**
96
108
  * Apply per-run configuration to an engine definition.
97
109
  *
98
- * Only AnySearch is configurable today: it authenticates with a Bearer token and
99
- * can be pointed at one of its vertical domains. Returning a copy keeps the
100
- * exported `HOSTS` catalogue static, so a stale credential can never leak into
101
- * the next run through a mutated descriptor.
110
+ * AnySearch can be pointed at one of its vertical domains; both AnySearch and
111
+ * Firecrawl authenticate with an optional Bearer token. Returning a copy keeps
112
+ * the exported `HOSTS` catalogue static, so a stale credential can never leak
113
+ * into the next run through a mutated descriptor.
102
114
  * @param host - the engine definition from `HOSTS`.
103
- * @param options - the run options carrying the credential and domain.
115
+ * @param options - the run options carrying credentials and the domain.
104
116
  * @returns the engine definition to use for this run.
105
117
  */
106
- export declare function resolveHost(host: HostSpec, options: Pick<EngineRunOptions, 'anysearchApiKey' | 'anysearchDomain'>): HostSpec;
118
+ export declare function resolveHost(host: HostSpec, options: Pick<EngineRunOptions, 'anysearchApiKey' | 'anysearchDomain' | 'firecrawlApiKey'>): HostSpec;
107
119
  export declare function search(query: string, signal: AbortSignal | undefined, options: EngineRunOptions): Promise<SearchOutcome>;
package/lib/engines.js CHANGED
@@ -179,7 +179,10 @@ function ddgWindow(timeRange) {
179
179
  */
180
180
  /** Soft deadline per wait policy, in milliseconds. */
181
181
  const SOFT_DEADLINE = {
182
- auto: (ascii) => (ascii ? 6000 : 3500),
182
+ // Auto is the interactive path: cap a slow fan-out leg instead of letting
183
+ // the caller absorb the slowest engine. `all` keeps the completeness escape
184
+ // hatch for callers that explicitly want every engine.
185
+ auto: (ascii) => (ascii ? 5000 : 3000),
183
186
  all: () => 10000,
184
187
  fast: () => 3500,
185
188
  };
@@ -250,6 +253,18 @@ function allowsFallback(details, fallbackOn) {
250
253
  const allowed = new Set(fallbackOn);
251
254
  return details.every((detail) => allowed.has(detail.kind));
252
255
  }
256
+ /**
257
+ * Whether a tier may release before its soft deadline.
258
+ *
259
+ * `limit` and `needed` are intentionally separate: eight rows from one engine
260
+ * are still a single-engine answer, while eight rows from two engines are a
261
+ * genuine cross-engine result. Counting every region here is what prevents a
262
+ * slow lead engine from holding a query open after another region supplied
263
+ * enough rows.
264
+ */
265
+ export function tierReleaseReady(available, contributors, limit, needed) {
266
+ return available >= limit && contributors >= needed;
267
+ }
253
268
  /**
254
269
  * Ask one engine, rotating to the next key when the failure blames the key.
255
270
  *
@@ -319,14 +334,22 @@ async function runTier(hosts, query, signal, timeoutMs, failures, failureDetails
319
334
  const rawCounts = new Array(hosts.length).fill(0);
320
335
  const answered = () => outcomes.filter((outcome) => outcome instanceof SearchError || outcome !== undefined);
321
336
  const available = () => outcomes.reduce((total, outcome) => total + (outcome instanceof SearchError || outcome === undefined ? 0 : outcome.sources.length), 0);
322
- const contributors = () => outcomes.filter((outcome, index) => counters(hosts[index]) && !(outcome instanceof SearchError) && outcome !== undefined && outcome.sources.length > 0).length;
337
+ // A contributor is any engine that returned rows, not only an engine in the
338
+ // leading region. The lead still decides ordering and the `leadSettled`
339
+ // shortcut; it must not also veto an early release when a healthy global
340
+ // engine has already supplied useful rows. This is the fan-out tail issue:
341
+ // a slow AnySearch request should not hold the whole Chinese query open
342
+ // after CN Bing and a global engine have enough rows between them.
343
+ const contributors = () => outcomes.filter((outcome) => !(outcome instanceof SearchError) && outcome !== undefined && outcome.sources.length > 0).length;
323
344
  const leadTotal = hosts.filter(counters).length || hosts.length;
324
345
  const needed = Math.min(limits.needed, leadTotal);
325
346
  // Nothing more can arrive from the leading region once every one of its
326
347
  // engines settled, so a failed engine there must not hold the tier open until
327
348
  // the soft deadline.
328
349
  const leadSettled = () => outcomes.filter((outcome, index) => counters(hosts[index]) && outcome !== undefined).length >= leadTotal;
329
- const done = () => answered().length === hosts.length || leadSettled() || (available() >= limits.limit && contributors() >= needed);
350
+ const done = () => answered().length === hosts.length ||
351
+ leadSettled() ||
352
+ tierReleaseReady(available(), contributors(), limits.limit, needed);
330
353
  let releasedEarly = false;
331
354
  let closed = false;
332
355
  let released = () => { };
@@ -405,15 +428,19 @@ async function runTier(hosts, query, signal, timeoutMs, failures, failureDetails
405
428
  /**
406
429
  * Apply per-run configuration to an engine definition.
407
430
  *
408
- * Only AnySearch is configurable today: it authenticates with a Bearer token and
409
- * can be pointed at one of its vertical domains. Returning a copy keeps the
410
- * exported `HOSTS` catalogue static, so a stale credential can never leak into
411
- * the next run through a mutated descriptor.
431
+ * AnySearch can be pointed at one of its vertical domains; both AnySearch and
432
+ * Firecrawl authenticate with an optional Bearer token. Returning a copy keeps
433
+ * the exported `HOSTS` catalogue static, so a stale credential can never leak
434
+ * into the next run through a mutated descriptor.
412
435
  * @param host - the engine definition from `HOSTS`.
413
- * @param options - the run options carrying the credential and domain.
436
+ * @param options - the run options carrying credentials and the domain.
414
437
  * @returns the engine definition to use for this run.
415
438
  */
416
439
  export function resolveHost(host, options) {
440
+ if (host.id === 'firecrawl') {
441
+ const firecrawlKey = (options.firecrawlApiKey ?? '').trim();
442
+ return firecrawlKey.length === 0 ? host : { ...host, headers: { authorization: `Bearer ${firecrawlKey}` } };
443
+ }
417
444
  if (host.id !== 'anysearch')
418
445
  return host;
419
446
  const key = (options.anysearchApiKey ?? '').trim();
@@ -453,7 +480,11 @@ export async function search(query, signal, options) {
453
480
  const allow = options.hostIds !== undefined && options.hostIds.length > 0 ? new Set(options.hostIds) : undefined;
454
481
  const hosts = HOSTS.filter((host) => allow === undefined || allow.has(host.id))
455
482
  .filter((host) => host.viaProxy !== true || proxy !== undefined)
456
- .map((host) => resolveHost(host, options.anysearchDomain === undefined ? {} : { anysearchDomain: options.anysearchDomain }));
483
+ .map((host) => resolveHost(host, {
484
+ ...(options.anysearchApiKey === undefined ? {} : { anysearchApiKey: options.anysearchApiKey }),
485
+ ...(options.anysearchDomain === undefined ? {} : { anysearchDomain: options.anysearchDomain }),
486
+ ...(options.firecrawlApiKey === undefined ? {} : { firecrawlApiKey: options.firecrawlApiKey }),
487
+ }));
457
488
  // Skip engines that are cooling down, but never end up with no engine at all:
458
489
  // a throttled engine is still better than no search.
459
490
  const now = Date.now();
@@ -475,9 +506,13 @@ export async function search(query, signal, options) {
475
506
  const fallbackOn = options.fallbackOn;
476
507
  const ascii = !CJK.test(query);
477
508
  const lead = ascii ? 'global' : 'cn';
478
- const wanted = usable.filter((host) => leadsQuery(host, lead)).length;
479
509
  const softMs = options.softMs ?? SOFT_DEADLINE[wait](ascii);
480
- const needed = wait === 'all' ? usable.length : wait === 'fast' ? 1 : wanted > 0 ? wanted : Math.min(2, usable.length);
510
+ // `auto` is a latency policy, not a completeness policy: two independent
511
+ // contributors are enough to make fusion meaningful. Waiting for every lead
512
+ // engine made one slow AnySearch leg hold an otherwise complete answer open
513
+ // for the full soft deadline. `all` still asks for every engine, and `fast`
514
+ // keeps its first-answer behavior.
515
+ const needed = wait === 'all' ? usable.length : wait === 'fast' ? 1 : Math.min(2, usable.length);
481
516
  const limits = { limit, softMs, needed, lead: wait === 'all' ? 'all' : lead, timeRange };
482
517
  // Ranking does not drop rows; the engine layer does. `timeRange` is passed
483
518
  // into the rank options only so undated rows are scored lower while a window
package/lib/fetch.js CHANGED
@@ -35,6 +35,7 @@
35
35
  import { SearchError } from "./errors.js";
36
36
  import { keyFailureKind, keyPoolFor } from "./keys.js";
37
37
  import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
38
+ import { arxivReadableUrl } from "./arxiv-url.js";
38
39
  import { proxyGet, resolveProxy } from "./proxy.js";
39
40
  import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
40
41
  /** Default user agent; kept in step with the search side. */
@@ -388,10 +389,13 @@ export function createFetchProvider(options = {}, overrides = {}) {
388
389
  id: options.id ?? 'http-proxy',
389
390
  available: () => true,
390
391
  async fetch(request, signal) {
391
- const target = request.url;
392
- if (typeof target !== 'string' || !/^https?:\/\//i.test(target)) {
393
- throw new SearchError(`not an http(s) URL: ${String(target)}`, 'SEARCH_PARSE', false);
392
+ const requested = request.url;
393
+ if (typeof requested !== 'string' || !/^https?:\/\//i.test(requested)) {
394
+ throw new SearchError(`not an http(s) URL: ${String(requested)}`, 'SEARCH_PARSE', false);
394
395
  }
396
+ // arXiv PDFs are binary and would otherwise be rejected as unsupported.
397
+ // Read the HTML full-text view instead; ordinary HTML pages pass through.
398
+ const target = arxivReadableUrl(requested);
395
399
  // The ladder budget belongs to THIS request. Starting it in the factory
396
400
  // made it start at plugin mount, so every fetch more than 25 s after a
397
401
  // restart failed with "no time left in the fetch budget" before touching
package/lib/index.d.ts CHANGED
@@ -97,6 +97,7 @@ declare const _default: typeof apply & {
97
97
  cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
98
98
  anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
99
99
  anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
100
+ firecrawlApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
100
101
  baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
101
102
  maxCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
102
103
  domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
@@ -128,6 +129,7 @@ declare const _default: typeof apply & {
128
129
  cacheFile: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
129
130
  anysearchApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
130
131
  anysearchDomain: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
132
+ firecrawlApiKey: import("@deepseek-ai/schemastery").default<string, string, "volatile-defined">;
131
133
  baseCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
132
134
  maxCooldownMs: import("@deepseek-ai/schemastery").default<number, number, "volatile-defined">;
133
135
  domainSources: import("@deepseek-ai/schemastery").default<NoInfer<string | string[]>, NoInfer<string | string[]>, "volatile-defined">;
package/lib/index.js CHANGED
@@ -27,7 +27,8 @@ import { registerEngineTestBridge, registerUpdateBridge } from "./bridge.js";
27
27
  import { registerAdvancedSearchTool, undatedWindowNote } from "./advanced-search.js";
28
28
  import { searchVerticals } from "./verticals.js";
29
29
  import { routePackageQuery } from "./query-router.js";
30
- import { fetchPostgresSupport, isPostgresVersionQuery } from "./official-support.js";
30
+ import { mergeOfficial, officialLookup } from "./official-sources.js";
31
+ import { arxivLookup, arxivQueryText, mergeArxiv } from "./arxiv.js";
31
32
  import { chromeFeedNote, chromeWindow, fetchChromeReleases, filterChromeChannel, hasNonStableChromePosts, isChromeSecurityQuery, } from "./official-feed.js";
32
33
  import { registerSearchPrompt } from "./system-prompt.js";
33
34
  import { registerUnifiedTool } from "./tool-suite.js";
@@ -147,6 +148,7 @@ export function apply(ctx, config = {}) {
147
148
  rank: resolved.rank,
148
149
  ...(resolved.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: resolved.anysearchApiKey }),
149
150
  ...(resolved.anysearchDomain.length === 0 ? {} : { anysearchDomain: resolved.anysearchDomain }),
151
+ ...(resolved.firecrawlApiKey.length === 0 ? {} : { firecrawlApiKey: resolved.firecrawlApiKey }),
150
152
  cache,
151
153
  health,
152
154
  ...(signal === undefined ? {} : { signal }),
@@ -236,6 +238,7 @@ export function apply(ctx, config = {}) {
236
238
  ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
237
239
  ...(live.anysearchApiKey.length === 0 ? {} : { anysearchApiKey: live.anysearchApiKey }),
238
240
  ...(live.anysearchDomain.length === 0 ? {} : { anysearchDomain: live.anysearchDomain }),
241
+ ...(live.firecrawlApiKey.length === 0 ? {} : { firecrawlApiKey: live.firecrawlApiKey }),
239
242
  hostIds: [engine],
240
243
  });
241
244
  };
@@ -254,27 +257,36 @@ export function apply(ctx, config = {}) {
254
257
  const asked = typeof request.maxResults === 'number' && Number.isInteger(request.maxResults) && request.maxResults > 0
255
258
  ? request.maxResults
256
259
  : undefined;
260
+ const live = resolveConfig(section, undefined);
261
+ // Start an official lookup before the fan-out and await it afterwards.
262
+ // It then costs max(search, lookup), not search + lookup, which matters
263
+ // for a version query where the first result should be authoritative but
264
+ // the ordinary engines still need to fill the remaining rows.
265
+ const official = officialLookup(query, {
266
+ timeoutMs: Math.min(live.timeoutMs, 2_500),
267
+ ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
268
+ ...(signal === undefined ? {} : { signal }),
269
+ });
270
+ // An explicit `arXiv` query gets the preprint index in parallel with the
271
+ // ordinary fan-out. This is separate from the official-release lookup:
272
+ // a survey query needs arXiv's own search, not a release announcement.
273
+ const arxiv = arxivLookup(query, {
274
+ limit: 4,
275
+ timeoutMs: Math.min(live.timeoutMs, 3_500),
276
+ ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
277
+ ...(signal === undefined ? {} : { signal }),
278
+ });
257
279
  const outcome = await runQuery(query, asked, signal);
258
- // A version/EOL question about PostgreSQL is better answered by the
259
- // project's own support table than by whatever an engine ranks first.
280
+ // A version/EOL question about PostgreSQL or Rust is better answered by
281
+ // the project's own release facts than by whatever an engine ranks first.
260
282
  // The lookup is query-scoped and failure-tolerant; it never replaces the
261
283
  // ordinary rows when the page is unavailable.
262
- let sources = outcome.sources;
263
- if (isPostgresVersionQuery(query)) {
264
- const live = resolveConfig(section, undefined);
265
- const official = await fetchPostgresSupport({
266
- timeoutMs: Math.min(live.timeoutMs, 8_000),
267
- ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
268
- ...(signal === undefined ? {} : { signal }),
269
- });
270
- if (official !== undefined) {
271
- sources = [official, ...sources.filter((source) => source.url !== official.url)];
272
- }
273
- }
284
+ let sources = mergeOfficial(outcome.sources, await official);
285
+ sources = mergeArxiv(sources, await arxiv);
274
286
  if (sources.length === 0) {
275
287
  throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
276
288
  }
277
- const cap = Math.min(asked ?? resolveConfig(section, undefined).limit, resolveConfig(section, undefined).limit);
289
+ const cap = Math.min(asked ?? live.limit, live.limit);
278
290
  return { sources: sources.slice(0, cap), truncated: outcome.truncated || sources.length > cap };
279
291
  },
280
292
  });
@@ -319,6 +331,26 @@ export function apply(ctx, config = {}) {
319
331
  * returns nothing, so a registry outage cannot empty the batch.
320
332
  */
321
333
  const runBatchQuery = async (query, limit, signal) => {
334
+ // An explicit arXiv query is a paper lookup, not a general web query.
335
+ // Keep the ordinary path as the fallback when the index returns no rows.
336
+ const arxivText = arxivQueryText(query);
337
+ if (arxivText !== undefined) {
338
+ const live = resolveConfig(section, undefined);
339
+ const papers = await arxivLookup(query, {
340
+ limit,
341
+ timeoutMs: live.domainTimeoutMs,
342
+ ...(live.proxy === undefined ? {} : { proxy: live.proxy }),
343
+ ...(signal === undefined ? {} : { signal }),
344
+ });
345
+ if (papers.length > 0) {
346
+ return {
347
+ sources: papers.slice(0, limit),
348
+ failures: [],
349
+ note: `arXiv query routed to the arXiv index for ${JSON.stringify(arxivText)}; rows are preprints with stable abs URLs.`,
350
+ truncated: papers.length > limit,
351
+ };
352
+ }
353
+ }
322
354
  const route = routePackageQuery(query);
323
355
  if (route !== undefined) {
324
356
  const live = resolveConfig(section, undefined);
package/lib/markdown.js CHANGED
@@ -374,13 +374,29 @@ function prune(node) {
374
374
  kept.push(child);
375
375
  continue;
376
376
  }
377
- if (DROP_TAGS.has(child.tag) || (!NOISE_EXEMPT_TAGS.has(child.tag) && isNoise(child)))
377
+ if (DROP_TAGS.has(child.tag))
378
+ continue;
379
+ // A wrapper whose class says "footer" may still contain the page's main
380
+ // article. arXiv currently wraps its whole document in
381
+ // `div.flex-wrap-footer`; dropping that wrapper removed every readable
382
+ // character from the abstract page. Chrome inside the wrapper is still
383
+ // pruned by its own tag/attributes.
384
+ if (!NOISE_EXEMPT_TAGS.has(child.tag) && isNoise(child) && !containsLandmark(child))
378
385
  continue;
379
386
  prune(child);
380
387
  kept.push(child);
381
388
  }
382
389
  node.children = kept;
383
390
  }
391
+ /** Whether a subtree contains a real content landmark. */
392
+ function containsLandmark(node) {
393
+ let found = false;
394
+ walkElements(node, (child) => {
395
+ if (child.tag === 'main' || child.tag === 'article')
396
+ found = true;
397
+ });
398
+ return found;
399
+ }
384
400
  /** Whether an element is hidden or named like site furniture. */
385
401
  function isNoise(node) {
386
402
  const role = (node.attrs['role'] ?? '').toLowerCase();
@@ -0,0 +1,73 @@
1
+ /**
2
+ * Official Rust release notes for an explicitly versioned release query.
3
+ *
4
+ * Search engines often return the Rust home page or a third-party changelog for
5
+ * a query such as `Rust 1.90 发布说明`, even when the official announcement is
6
+ * indexed. The release announcement has a deterministic shape once its date is
7
+ * known: `https://blog.rust-lang.org/YYYY/MM/DD/Rust-X.Y.Z/`. GitHub's public
8
+ * release endpoint is key-free and exposes that date, so this module can add
9
+ * the official page as a high-priority row without hard-coding release dates.
10
+ *
11
+ * The lookup is deliberately narrow. Only a Rust query that names a numeric
12
+ * version *and* asks for release notes is enriched; a generic Rust tutorial
13
+ * keeps the normal engine ranking.
14
+ *
15
+ * @module @fangchenghao/dsh-web-search/official-release
16
+ */
17
+ import type { Source } from './types.ts';
18
+ /** Public GitHub release endpoint for the Rust repository. */
19
+ export declare const RUST_RELEASE_API = "https://api.github.com/repos/rust-lang/rust/releases/tags";
20
+ /** Official Rust announcement host. */
21
+ export declare const RUST_BLOG_HOST = "blog.rust-lang.org";
22
+ /** Inputs accepted by {@link fetchRustRelease}. */
23
+ export interface RustReleaseOptions {
24
+ /** Tunnel URL from the plugin's `proxy` setting, when configured. */
25
+ proxy?: string;
26
+ /** Per-request deadline in milliseconds. */
27
+ timeoutMs?: number;
28
+ /** Caller cancellation. */
29
+ signal?: AbortSignal;
30
+ /** Test seam; defaults to the platform `fetch`. */
31
+ fetch?: typeof fetch;
32
+ }
33
+ /** A version such as `1.90`, normalized to a release tag like `1.90.0`. */
34
+ export interface RustVersion {
35
+ /** User-facing version, e.g. `1.90.0`. */
36
+ readonly version: string;
37
+ /** GitHub tag, normally identical to {@link version}. */
38
+ readonly tag: string;
39
+ }
40
+ /** Extract a Rust version from a release query. */
41
+ export declare function rustReleaseVersion(query: string): RustVersion | undefined;
42
+ /** True for a query asking for a specific Rust release's notes. */
43
+ export declare function isRustReleaseQuery(query: string): boolean;
44
+ /**
45
+ * Build the canonical announcement URL from a GitHub release timestamp.
46
+ *
47
+ * Rust publishes the post on the UTC date in the release metadata. Using UTC
48
+ * avoids shifting the path when the machine is in a negative-offset timezone.
49
+ */
50
+ export declare function rustBlogUrl(version: string, publishedAt: string): string | undefined;
51
+ /**
52
+ * Parse a GitHub release response into one official-blog source.
53
+ *
54
+ * A malformed response, a missing publication date or a tag that does not
55
+ * match the requested version returns `undefined`; the caller then keeps the
56
+ * ordinary search rows.
57
+ *
58
+ * @param payload - GitHub JSON response body.
59
+ * @param requested - normalized version from the query.
60
+ * @returns the official source, or `undefined`.
61
+ */
62
+ export declare function parseRustRelease(payload: string, requested: RustVersion): Source | undefined;
63
+ /**
64
+ * Fetch the official Rust release page for a query.
65
+ *
66
+ * Failure is contained: an unavailable API, rate limit or malformed body
67
+ * returns `undefined`, and the caller keeps its ordinary search rows.
68
+ *
69
+ * @param query - the original user query.
70
+ * @param options - proxy, deadline, cancellation and fetch seam.
71
+ * @returns the official blog source, or `undefined`.
72
+ */
73
+ export declare function fetchRustRelease(query: string, options?: RustReleaseOptions): Promise<Source | undefined>;