@fffangchenghao/dsh-web-search 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +8 -7
- package/README.zh-CN.md +8 -7
- package/client.js +1 -1
- package/docs/comparison.md +14 -11
- package/docs/images/web-tools-demo.svg +1 -1
- package/docs/latency-and-sources.md +38 -20
- package/docs/store-evidence.md +1 -1
- package/lib/arxiv-url.d.ts +40 -0
- package/lib/arxiv-url.js +62 -0
- package/lib/arxiv.d.ts +46 -0
- package/lib/arxiv.js +70 -0
- package/lib/fetch.js +7 -3
- package/lib/index.js +32 -1
- package/lib/markdown.js +17 -1
- package/lib/verticals.d.ts +13 -0
- package/lib/verticals.js +37 -2
- package/mcp/server.mjs +7 -1
- package/package.json +2 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,39 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.23.0
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- **Explicit arXiv research queries.** `web_search` and `search_many` route a
|
|
7
|
+
query that names arXiv with paper/survey intent to the arXiv Atom API, in
|
|
8
|
+
parallel with the ordinary fan-out. The lookup is capped at 3.5 s, never
|
|
9
|
+
replaces ordinary rows when it fails, and returns stable `arxiv.org/abs/`
|
|
10
|
+
URLs. `arXiv:1706.03762` resolves the paper by id.
|
|
11
|
+
- **Boolean arXiv queries.** A survey query becomes a quoted topic phrase plus
|
|
12
|
+
`all:survey`; other multi-word queries become ANDed terms. This fixes the
|
|
13
|
+
image-generation paper that used to rank above actual RAG surveys for
|
|
14
|
+
`retrieval augmented generation survey arXiv`. Explicit field syntax and
|
|
15
|
+
boolean expressions are passed through unchanged.
|
|
16
|
+
- **arXiv PDFs are readable.** A `/pdf/<id>` URL is transparently read through
|
|
17
|
+
arXiv's HTML full-text view instead of failing as binary, including versioned
|
|
18
|
+
URLs and old-style archive identifiers. Plain arXiv abstract pages now parse
|
|
19
|
+
correctly when the whole document is wrapped in a container whose class is
|
|
20
|
+
named `footer` (`div.flex-wrap-footer` on the live site).
|
|
21
|
+
|
|
22
|
+
### Fixed
|
|
23
|
+
- **Empty arXiv abstract pages.** The pruning pass no longer drops a wrapper
|
|
24
|
+
that contains a `<main>` or `<article>` merely because its class mentions
|
|
25
|
+
`footer`; site chrome inside the wrapper is still removed by its own tag and
|
|
26
|
+
attributes.
|
|
27
|
+
|
|
28
|
+
### Documentation
|
|
29
|
+
- **Research path documented.** The README and Chinese README explain the arXiv
|
|
30
|
+
routing/PDF behavior and update the test count to 687 across 56 files. The
|
|
31
|
+
public 30-query follow-up now records 30/30 official hits, mean 2263 ms and
|
|
32
|
+
p95 2997 ms; the older 0.22.0 follow-up remains checked in as historical
|
|
33
|
+
evidence.
|
|
34
|
+
|
|
35
|
+
**Test suite: 687 tests across 56 files** (686 pass, 1 skipped).
|
|
36
|
+
|
|
3
37
|
## 0.22.0
|
|
4
38
|
|
|
5
39
|
### Changed
|
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
<p align="center">
|
|
12
12
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm version"></a>
|
|
13
|
-
<img src="https://img.shields.io/badge/tests-
|
|
13
|
+
<img src="https://img.shields.io/badge/tests-687%20tests-brightgreen" alt="tests">
|
|
14
14
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="engines">
|
|
15
15
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
16
16
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -21,12 +21,13 @@
|
|
|
21
21
|
|
|
22
22
|
- **No API key, eight engines.** AnySearch, CN Bing, DuckDuckGo HTML, www Bing, DuckDuckGo Lite and 360 Search are scraped directly; Exa joins the primary tier through its keyless MCP endpoint, and Firecrawl's search API is the measured fallback. The plugin works out of the box.
|
|
23
23
|
- **Official release facts, not generic ranking.** Explicit PostgreSQL, Rust and scoped npm queries start a narrow official lookup in parallel; `Rust 1.90 发布说明` returns the canonical Rust Blog announcement at rank 1. The lookup is capped at 2.5 s and failure leaves the ordinary rows intact.
|
|
24
|
+
- **arXiv is a first-class research source.** An explicit arXiv paper or survey query is routed to arXiv's own API in parallel, with boolean terms instead of a loose keyword string, so a survey query returns surveys rather than incidental mentions. arXiv PDF links are read through the HTML full-text view instead of being rejected as binary, and abstract pages parse even when a site-wide wrapper is named `footer`.
|
|
24
25
|
- **A tail budget and an honest quota story.** Interactive searches release after two independent contributors and cap at 5 s for Latin-script queries / 3 s for CJK queries. Firecrawl stays a fallback; `firecrawlApiKey` is optional for users who need repeatable quota. Keyless Firecrawl cannot be made dependable from the client side.
|
|
25
26
|
- **The ranking is measured, not asserted.** Cross-engine reciprocal-rank fusion with authority and freshness boosts, a 20-query offline benchmark, and an 81-configuration parameter sweep — one of the few search plugins here that publishes its metric, per-query output and confidence intervals. See [How the ranking works](docs/rrf-ranking.md).
|
|
26
27
|
- **One package, two runtimes.** DSH gets the native plugin and one `web_tools` entry; Claude Code, Codex and other MCP clients get the same keyless search through a dependency-free stdio server with `search`, `fetch`, `research` and `domain_search`. See [Cross-agent MCP](docs/cross-agent.md).
|
|
27
28
|
- **Measured in public, including the losses.** A 30-query bilingual comparison records availability, latency, unique domains, official-domain presence and every failure against keyless Firecrawl and anonymous Exa. The follow-up documents the Exa architecture difference, the 3 s CJK budget and the Rust official-source lookup. Exa is faster here; this plugin returns more distinct domains. Both test arms were keyless. The method and raw results are in [docs/comparison.md](docs/comparison.md) and a [public gist](https://gist.github.com/fangchenghao/3db9838fa98928435c1c2df38202c373).
|
|
28
|
-
- **Fetching admits failure.** Binary documents and empty 200 responses are rejected by name (`SEARCH_UNSUPPORTED`, `SEARCH_EMPTY`) instead of being returned as noise; the direct → tunnel → render → extract ladder shares one 25 s budget and stops when there is no time left. The budget is per request — there is a regression test for exactly that.
|
|
29
|
-
- **
|
|
29
|
+
- **Fetching admits failure.** Binary documents and empty 200 responses are rejected by name (`SEARCH_UNSUPPORTED`, `SEARCH_EMPTY`) instead of being returned as noise; arXiv PDF links are the deliberate exception and are read through the HTML full-text view; the direct → tunnel → render → extract ladder shares one 25 s budget and stops when there is no time left. The budget is per request — there is a regression test for exactly that.
|
|
30
|
+
- **687 tests across 56 files**, a committed `lib/`, declared `engines.dsh` / `engines.node`, and a [SECURITY.md](SECURITY.md) that lists every process, host and file the plugin touches.
|
|
30
31
|
- **Bilingual documentation.** This README and [中文说明](README.zh-CN.md) are maintained side by side.
|
|
31
32
|
|
|
32
33
|

|
|
@@ -43,7 +44,7 @@ The web-search corner of the DSH plugin ecosystem is crowded. A fresh snapshot
|
|
|
43
44
|
| Key-free path | 8 engines, all key-free | ~30-engine free/paid chain | 5 defaults (seam, Exa, DDG, Bing, Jina) | Firecrawl + Keenable keyless defaults | 1 anonymous AnySearch API |
|
|
44
45
|
| Dedicated vertical sources | 34 (8 on by default) | 41 | platform registry, not a vertical catalog | web + X, no vertical catalog | vertical API behind AnySearch |
|
|
45
46
|
| Model-facing tools | 1 (`web_tools`, 8 actions) + 4 MCP tools | ~10 | 2 (`web_index` / `web_call`, 20 actions) | native search/fetch + skill | 5 |
|
|
46
|
-
| Tests |
|
|
47
|
+
| Tests | 56 files / 687 cases, locally verified | no test script in `package.json` | 774 unit + 94 bench, self-reported | Vitest configured; count not published | Vitest configured; count not published |
|
|
47
48
|
| GitHub stars | 0 | 620 | 77 | 836 | 663 |
|
|
48
49
|
| npm downloads, last month | 1,087 | 52,436 | 6,420 | 43,151 | 9,251 |
|
|
49
50
|
| npm unpacked size | 1.2 MB | 0.70 MB | 3.05 MB | 0.51 MB | 0.27 MB |
|
|
@@ -54,7 +55,7 @@ read is:
|
|
|
54
55
|
- **Feature depth:** upper-middle. `dsh-free-search` has more engines and
|
|
55
56
|
verticals; `dsh-web-search-pro` has a deeper evidence/ranking pipeline and far
|
|
56
57
|
more platform backends; `modsearch` is the stronger cross-host bridge.
|
|
57
|
-
- **Engineering and verification:** near the top.
|
|
58
|
+
- **Engineering and verification:** near the top. 687 local tests, a replayable
|
|
58
59
|
ranking benchmark, declared security boundaries and one runtime dependency
|
|
59
60
|
put it alongside the best-maintained entries, while remaining much smaller
|
|
60
61
|
than the Playwright-heavy option.
|
|
@@ -62,7 +63,7 @@ read is:
|
|
|
62
63
|
Chrome, PostgreSQL and AI Act paths and the 34-source vertical layer are not
|
|
63
64
|
common in the other plugins.
|
|
64
65
|
- **Adoption and maturity:** bottom third of this sample. The public npm release
|
|
65
|
-
is 0.
|
|
66
|
+
is 0.23.0, matching this tree, but the GitHub repository is four days old
|
|
66
67
|
with no stars and little independent adoption.
|
|
67
68
|
- **Tool-surface economy:** now competitive. 0.20.x exposes one `web_tools`
|
|
68
69
|
entry with eight actions, close to `dsh-web-search-pro`'s two tools with 20
|
|
@@ -731,7 +732,7 @@ Four usual causes:
|
|
|
731
732
|
2. **The content is rendered by JavaScript**: what comes back is an empty shell. With Playwright installed it is rendered automatically; without it, you see the shell. **If stripping the shell leaves no readable text at all, the fetch fails with `SEARCH_EMPTY` and says why — including "no renderer installed" — instead of returning an empty page as a success.**
|
|
732
733
|
3. **It needs a login or a subscription**: fetching carries no session of yours.
|
|
733
734
|
4. **Direct requests time out**: 5 seconds direct, then the tunnel; if neither works and no key is configured you get an error, and with a key there is one more rung, server-side extraction. The whole ladder shares a 25-second budget, so later rungs get only what is left. The message names each rung's failure.
|
|
734
|
-
5. **Binary documents**: PDFs, images and archives are refused explicitly (`SEARCH_UNSUPPORTED`) with the content type, rather than handing you a byte stream as prose.
|
|
735
|
+
5. **Binary documents**: PDFs, images and archives are refused explicitly (`SEARCH_UNSUPPORTED`) with the content type, rather than handing you a byte stream as prose. The exception is an arXiv `/pdf/<id>` link: the fetch ladder rewrites it to arXiv's HTML full-text view, so research papers can be read without a separate PDF extractor. For other PDFs, extract the text with another tool first.
|
|
735
736
|
|
|
736
737
|
360's captcha belongs to the first cause, but it used to be especially quiet: from some networks the request 302s to the qcaptcha "unexpected access" page while the HTTP status is still 200. The plugin now records that as `SEARCH_BLOCKED`; querying that engine alone shows `so: bot verification page` instead of zero results, and 360 has been demoted to the fallback tier.
|
|
737
738
|
|
package/README.zh-CN.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
<p align="center">
|
|
11
11
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm 版本"></a>
|
|
12
|
-
<img src="https://img.shields.io/badge/tests-
|
|
12
|
+
<img src="https://img.shields.io/badge/tests-687%20tests-brightgreen" alt="测试">
|
|
13
13
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="引擎">
|
|
14
14
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
15
15
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -20,12 +20,13 @@
|
|
|
20
20
|
|
|
21
21
|
- **不要 API key,八个引擎。** AnySearch、必应中国、DuckDuckGo HTML、必应国际、DuckDuckGo Lite、360 搜索直接抓取;Exa 走它的免密钥 MCP 通道加入主引擎,Firecrawl 的搜索 API 作为实测后的兜底。装好就能用。
|
|
22
22
|
- **官方发布事实优先。** PostgreSQL、Rust 和显式 npm 包查询会并行做一次窄查询;`Rust 1.90 发布说明` 的第 1 条是 Rust Blog 官方公告。查询上限 2.5 秒,失败就保留普通结果。
|
|
23
|
+
- **arXiv 是一等科研源。** 显式包含 arXiv 的论文/综述查询会并行走 arXiv 自己的 API,用布尔词而不是松散关键词,所以「RAG survey」返回的是综述而不是偶然提及。arXiv 的 `/pdf/` 链接会改读 HTML 全文,不再被当成二进制拒绝;摘要页即使被命名为 `footer` 的站点外层包住,也能正常提取。
|
|
23
24
|
- **尾部有预算,配额不粉饰。** 交互搜索在两个独立引擎返回后提前释放,拉丁文查询 5 秒、中文查询 3 秒封顶。Firecrawl 仍是兜底;需要稳定配额可以配 `firecrawlApiKey`,但免 key 的服务端配额不是客户端能修好的。
|
|
24
25
|
- **排序是量出来的,不是写出来的。** 跨引擎 RRF 融合,叠加权威度与时效加权;20 条离线基准 + 81 组参数扫描,是这类插件里少数同时公开指标、逐条输出和置信区间的。方法见 [排序是怎么做的](docs/rrf-ranking.md)。
|
|
25
26
|
- **一个包,两种运行时。** DSH 里是原生插件和唯一的 `web_tools` 入口;Claude Code、Codex 等 MCP 客户端通过零依赖的 stdio server 使用同一套免密钥搜索,提供 `search`、`fetch`、`research`、`domain_search`。见 [跨 Agent MCP](docs/cross-agent.md)。
|
|
26
27
|
- **公开测量,也公开输的地方。** 30 条中英查询对比记录可用性、延迟、独立域名、官方域名命中与每一条失败;对比对象是免 key Firecrawl 和匿名 Exa。这里 Exa 更快,本插件返回的独立域名更多;两条对比路径都未使用 key。方法和原始结果在 [docs/comparison.md](docs/comparison.md);Exa 架构差异、中文 3 秒预算和 Rust 官方源见 [延迟与官方源](docs/latency-and-sources.md)。
|
|
27
|
-
- **抓取会承认失败。** 二进制文档和「200 但正文为空」会被明确拒绝(`SEARCH_UNSUPPORTED`、`SEARCH_EMPTY`),而不是当噪音返回;direct → tunnel → render → extract 四级阶梯共享 25 秒预算,预算不足就不再起新的一级。预算是「每次请求」而不是「插件启动时」,这点有回归测试守着。
|
|
28
|
-
- **
|
|
28
|
+
- **抓取会承认失败。** 二进制文档和「200 但正文为空」会被明确拒绝(`SEARCH_UNSUPPORTED`、`SEARCH_EMPTY`),而不是当噪音返回;arXiv 的 PDF 链接是有意保留的例外,会改读 HTML 全文;direct → tunnel → render → extract 四级阶梯共享 25 秒预算,预算不足就不再起新的一级。预算是「每次请求」而不是「插件启动时」,这点有回归测试守着。
|
|
29
|
+
- **56 个测试文件、687 个用例**,`lib/` 产物入库,声明了 `engines.dsh` / `engines.node`,[SECURITY.md](SECURITY.md) 逐项列出插件会碰的进程、主机和文件。
|
|
29
30
|
- **双语文档。** 本文件和 [English README](README.md) 同步维护。
|
|
30
31
|
|
|
31
32
|

|
|
@@ -42,7 +43,7 @@ DSH 插件生态里的「联网搜索」已经很挤。下面是一张 2026-10-2
|
|
|
42
43
|
| 免 key 路径 | 8 个引擎,全部免 key | ~30 个免费/付费引擎链 | 默认 5 个(seam、Exa、DDG、Bing、Jina) | Firecrawl + Keenable 免 key 默认 | 1 个 AnySearch 匿名 API |
|
|
43
44
|
| 专门数据源 | 34 个(默认开 8 个) | 41 个 | 平台注册表,不是垂直源目录 | 网页 + X,无垂直源目录 | AnySearch 的垂直 API |
|
|
44
45
|
| 面向模型的工具 | 1 个(`web_tools`,8 个 action)+ 4 个 MCP 工具 | ~10 个 | 2 个(`web_index` / `web_call`,20 个动作) | 原生搜索/抓取 + skill | 5 个 |
|
|
45
|
-
| 测试 |
|
|
46
|
+
| 测试 | 56 文件 / 687 用例,本地实测 | `package.json` 无测试脚本 | 774 单元 + 94 bench,自报 | 配了 Vitest,未公布数量 | 配了 Vitest,未公布数量 |
|
|
46
47
|
| GitHub stars | 0 | 620 | 77 | 836 | 663 |
|
|
47
48
|
| npm 近一月下载 | 1,087 | 52,436 | 6,420 | 43,151 | 9,251 |
|
|
48
49
|
| npm 解包大小 | 1.2 MB | 0.70 MB | 3.05 MB | 0.51 MB | 0.27 MB |
|
|
@@ -52,11 +53,11 @@ DSH 插件生态里的「联网搜索」已经很挤。下面是一张 2026-10-2
|
|
|
52
53
|
- **功能深度:中上。** `dsh-free-search` 引擎和垂直源更多;
|
|
53
54
|
`dsh-web-search-pro` 的证据/排序管线和平台后端更深;`modsearch` 的跨宿主
|
|
54
55
|
能力更强。
|
|
55
|
-
- **工程质量与可验证性:接近第一梯队。**
|
|
56
|
+
- **工程质量与可验证性:接近第一梯队。** 687 个本地测试、可重放的排序基准、
|
|
56
57
|
明确的安全边界、只有 1 个运行时依赖,同时体积远小于 Playwright 重型方案。
|
|
57
58
|
- **免 key + 官方源纠错:最强卖点。** Chrome、PostgreSQL、AI Act 的官方路径
|
|
58
59
|
和 34 个垂直源,不是其他插件普遍具备的。
|
|
59
|
-
- **采用度与成熟度:这个样本里的后段。** 公开 npm 已是 0.
|
|
60
|
+
- **采用度与成熟度:这个样本里的后段。** 公开 npm 已是 0.23.0,与这棵树一致;但 GitHub 仓库创建 4 天、0 stars,
|
|
60
61
|
还缺少独立采用。
|
|
61
62
|
- **工具面的经济性:现在有竞争力。** 0.20.x 只暴露一个 `web_tools` + 8 个
|
|
62
63
|
action,接近 `dsh-web-search-pro` 的 2 工具 + 20 动作;旧的八个工具名保留为
|
|
@@ -673,7 +674,7 @@ pnpm install
|
|
|
673
674
|
2. **正文靠 JavaScript 渲染**:抓回来的是空壳 HTML,看不到内容。装了 Playwright 后会自动渲染;没装就只能看到壳。**如果去壳之后一个字都不剩,插件会报错(`SEARCH_EMPTY`)并写明「没有可用渲染器」之类的判定,而不是把空壳当成成功结果返回**。
|
|
674
675
|
3. **要登录或付费**:抓取没有你的登录态。
|
|
675
676
|
4. **直连超时**:会先在 5 秒内试直连,失败转隧道;两个都不通、又没配 key 时报错,配了 key 还会再试服务端抽取。整条梯队总共只有 25 秒预算,后面的档位只能花剩下的时间。报错里会写清楚每一级失败的原因。
|
|
676
|
-
5. **二进制文档**:PDF、图片、压缩包会明确拒绝(`SEARCH_UNSUPPORTED
|
|
677
|
+
5. **二进制文档**:PDF、图片、压缩包会明确拒绝(`SEARCH_UNSUPPORTED`)并告出内容类型,不会把字节流当成正文塞给你。例外是 arXiv 的 `/pdf/<id>` 链接:抓取阶梯会把它改写成 arXiv 的 HTML 全文,因此无需另装 PDF 提取器就能读论文。其他 PDF 仍建议先用别的工具取文本。
|
|
677
678
|
|
|
678
679
|
360 的验证码页属于第 1 类,但它以前特别隐蔽:从部分网络访问会 302 到 qcaptcha 的「访问异常页面」,HTTP 状态却是 200。插件现在会把它记成 `SEARCH_BLOCKED`,单独查该引擎会看到 `so: bot verification page`,而不是 0 条;360 也因此降为兜底档。
|
|
679
680
|
|
package/client.js
CHANGED
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
var ENTRY = 'web-search-bing';
|
|
75
75
|
var PACKAGE_NAME = 'dsh-web-search-bing';
|
|
76
76
|
/** Keep in step with package.json; compared with the npm registry. */
|
|
77
|
-
var VERSION = '0.
|
|
77
|
+
var VERSION = '0.23.0';
|
|
78
78
|
/** Snapshot used while no configuration service is reachable. */
|
|
79
79
|
var EMPTY_STATE = { status: 'unavailable', value: undefined, base: undefined, user: undefined, revision: undefined, writable: false };
|
|
80
80
|
|
package/docs/comparison.md
CHANGED
|
@@ -43,7 +43,8 @@ error: Claude Code documentation moved from `docs.anthropic.com` to
|
|
|
43
43
|
`code.claude.com`; all three providers returned the new host. With that expected
|
|
44
44
|
host corrected, the counts would be **29/30, 30/30 and 30/30**. The remaining
|
|
45
45
|
real plugin miss is the Chinese Rust 1.90 query, which returned the Rust home
|
|
46
|
-
page rather than the release blog.
|
|
46
|
+
page rather than the release blog. Both plugin misses are fixed in 0.23.0:
|
|
47
|
+
the Rust official lookup and the explicit arXiv research route.
|
|
47
48
|
|
|
48
49
|
By category (first sweep; answered / official / mean ms):
|
|
49
50
|
|
|
@@ -121,21 +122,23 @@ The script writes a JSON file next to a Markdown report. The three raw files are
|
|
|
121
122
|
immediately after sweep 1 to test repeatability, which is why Firecrawl's quota
|
|
122
123
|
failed. A single sweep is a snapshot, not a permanent ranking.
|
|
123
124
|
|
|
124
|
-
## Follow-
|
|
125
|
+
## Follow-ups after the 0.21.1 baseline
|
|
125
126
|
|
|
126
|
-
The original clean sweep above is kept unchanged as the baseline.
|
|
127
|
-
plugin-only
|
|
128
|
-
|
|
127
|
+
The original clean sweep above is kept unchanged as the baseline. Later
|
|
128
|
+
plugin-only runs after the tail-tolerant early release, a 3 s CJK deadline, the
|
|
129
|
+
Rust official-release lookup and the arXiv research path are also checked in:
|
|
129
130
|
|
|
130
|
-
- [plugin-only follow-up](
|
|
131
|
+
- [0.22.0 plugin-only follow-up](https://github.com/fangchenghao/dsh-web-search/blob/main/bench/compare30/results-2026-10-26-plugin-after.json)
|
|
132
|
+
- [0.23.0 plugin-only follow-up](https://github.com/fangchenghao/dsh-web-search/blob/main/bench/compare30/results-2026-10-26-plugin-arxiv.json)
|
|
131
133
|
- [latency and sources note](latency-and-sources.md)
|
|
132
134
|
|
|
133
135
|
The five reported Chinese release queries went from 3502–3506 ms each to
|
|
134
|
-
2565–3012 ms, and all five now hit their expected official domain. The
|
|
135
|
-
30-query
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
136
|
+
2565–3012 ms, and all five now hit their expected official domain. The 0.22.0
|
|
137
|
+
30-query run still had one official miss (arXiv) and its mean was 3768 ms under
|
|
138
|
+
a slower network window. The 0.23.0 run answers 30/30, hits the official domain
|
|
139
|
+
30/30, and measures mean 2263 ms / p95 2997 ms; that faster window is not
|
|
140
|
+
presented as a pure code win. The arXiv query now returns real surveys at rank
|
|
141
|
+
1 instead of dictionary pages. Exa remains faster.
|
|
139
142
|
|
|
140
143
|
## Bottom line
|
|
141
144
|
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
<rect x="70" y="620" width="330" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
53
53
|
<text x="92" y="651" class="m" font-size="13">Engines</text><text x="92" y="677" class="t" font-size="20" font-weight="700">8 key-free paths</text>
|
|
54
54
|
<rect x="420" y="620" width="330" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
55
|
-
<text x="442" y="651" class="m" font-size="13">Tests</text><text x="442" y="677" class="t" font-size="20" font-weight="700">
|
|
55
|
+
<text x="442" y="651" class="m" font-size="13">Tests</text><text x="442" y="677" class="t" font-size="20" font-weight="700">687 local cases</text>
|
|
56
56
|
<rect x="770" y="620" width="360" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
57
57
|
<text x="792" y="651" class="m" font-size="13">Cross-agent</text><text x="792" y="677" class="t" font-size="20" font-weight="700">MCP stdio server</text>
|
|
58
58
|
</g>
|
|
@@ -122,28 +122,46 @@ All five now hit the expected official host. AnySearch still appears as a
|
|
|
122
122
|
deadline failure on a few runs; that is the `wait` budget doing its job, not a
|
|
123
123
|
blocking wait. The user-visible tail is capped at about 3 s for these queries.
|
|
124
124
|
|
|
125
|
-
### Full 30-query follow-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
| Metric | Original clean sweep |
|
|
130
|
-
| --- | ---: | ---: |
|
|
131
|
-
| Answered | 30/30 | 30/30 |
|
|
132
|
-
| Official-domain hit | 28/30 | 29/30 |
|
|
133
|
-
| Mean unique domains | 5.83 | 5.67 |
|
|
134
|
-
| Mean latency | 3008 ms | 3768 ms |
|
|
135
|
-
| p50 | 2791 ms | 4000 ms |
|
|
136
|
-
| p95 | 5188 ms | 5003 ms |
|
|
137
|
-
|
|
138
|
-
The follow-up's
|
|
139
|
-
variable upstream latency, and the new official lookup adds work
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
125
|
+
### Full 30-query follow-ups
|
|
126
|
+
|
|
127
|
+
Plugin-only runs with the official lookups enabled produced:
|
|
128
|
+
|
|
129
|
+
| Metric | Original clean sweep | 0.22.0 follow-up | 0.23.0 follow-up |
|
|
130
|
+
| --- | ---: | ---: | ---: |
|
|
131
|
+
| Answered | 30/30 | 30/30 | 30/30 |
|
|
132
|
+
| Official-domain hit | 28/30 | 29/30 | 30/30 |
|
|
133
|
+
| Mean unique domains | 5.83 | 5.67 | 5.47 |
|
|
134
|
+
| Mean latency | 3008 ms | 3768 ms | 2263 ms |
|
|
135
|
+
| p50 | 2791 ms | 4000 ms | 2112 ms |
|
|
136
|
+
| p95 | 5188 ms | 5003 ms | 2997 ms |
|
|
137
|
+
|
|
138
|
+
The 0.22.0 follow-up's mean got **worse** than the original clean sweep: it
|
|
139
|
+
ran under more variable upstream latency, and the new official lookup adds work
|
|
140
|
+
to release queries. The 0.23.0 run is faster, but part of that is a better
|
|
141
|
+
network window, not only code; the honest claim is that it fixes the arXiv
|
|
142
|
+
official miss and the survey ranking, not that every query is now faster.
|
|
143
|
+
|
|
144
|
+
The 0.23.0 run's mean unique domains is slightly lower because the explicitly
|
|
145
|
+
routed arXiv query now returns eight arXiv rows instead of eight unrelated
|
|
146
|
+
domains. That is the intended trade for a research query. Raw data:
|
|
147
|
+
[`results-2026-10-26-plugin-after.json`](https://github.com/fangchenghao/dsh-web-search/blob/main/bench/compare30/results-2026-10-26-plugin-after.json)
|
|
148
|
+
(0.22.0) and
|
|
149
|
+
[`results-2026-10-26-plugin-arxiv.json`](https://github.com/fangchenghao/dsh-web-search/blob/main/bench/compare30/results-2026-10-26-plugin-arxiv.json)
|
|
150
|
+
(0.23.0).
|
|
151
|
+
|
|
152
|
+
### arXiv follow-up
|
|
153
|
+
|
|
154
|
+
The arXiv benchmark query used to fail not because the arXiv API was down but
|
|
155
|
+
because the query was never routed there: the ordinary fan-out returned
|
|
156
|
+
dictionary and encyclopedia pages for the word "retrieval". The 0.23.0 run
|
|
157
|
+
routes explicit arXiv paper/survey queries to the arXiv Atom API, uses boolean
|
|
158
|
+
terms (`all:"retrieval augmented generation" AND all:survey`), and ranks the
|
|
159
|
+
first survey at position 1. Fetching was separately broken: the abstract page
|
|
160
|
+
was erased by a `div.flex-wrap-footer` wrapper, and `/pdf/` links were rejected
|
|
161
|
+
as binary. Both are fixed; PDF links now read arXiv's HTML full-text view.
|
|
144
162
|
|
|
145
163
|
The five-query table is the targeted result for the reported Chinese tail; the
|
|
146
|
-
30-query table is the honest full-run result
|
|
164
|
+
30-query table is the honest full-run result.
|
|
147
165
|
|
|
148
166
|
## 4. Do we need a vector database?
|
|
149
167
|
|
package/docs/store-evidence.md
CHANGED
|
@@ -23,7 +23,7 @@ Only `0.2.1-rc.2` is declared, because that is the version actually exercised.
|
|
|
23
23
|
## Environment
|
|
24
24
|
|
|
25
25
|
- macOS (arm64), DeepSeek Harness **0.2.1-rc.2** (`dsh --version`), Node 22.18+.
|
|
26
|
-
- Runtime artifacts: `lib/` is committed to this repository (
|
|
26
|
+
- Runtime artifacts: `lib/` is committed to this repository (86 files, 952 KiB),
|
|
27
27
|
so the fixed commit's tree contains `lib/index.js` and `lib/index.d.ts`.
|
|
28
28
|
- Disposable profile for the destructive steps: a fresh `DSH_HOME` created with
|
|
29
29
|
`mktemp -d`; the real `~/.dsh` was never written to. Bundled pnpm 11.28.5.
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recognize arXiv page URLs and map PDF requests to the HTML full-text view.
|
|
3
|
+
*
|
|
4
|
+
* arXiv serves the same paper as `/abs/<id>` (metadata and abstract),
|
|
5
|
+
* `/html/<id>` (full text) and `/pdf/<id>` (binary PDF). This plugin converts
|
|
6
|
+
* HTML and text to Markdown but deliberately does not pretend binary PDF bytes
|
|
7
|
+
* are prose, so a PDF link is read through the HTML view instead.
|
|
8
|
+
*
|
|
9
|
+
* @module @fangchenghao/dsh-web-search/arxiv-url
|
|
10
|
+
*/
|
|
11
|
+
/** One recognized arXiv URL. */
|
|
12
|
+
export interface ArxivUrl {
|
|
13
|
+
/** Canonical identifier, including an old-style archive prefix. */
|
|
14
|
+
id: string;
|
|
15
|
+
/** Page kind in the original URL. */
|
|
16
|
+
kind: 'abs' | 'html' | 'pdf';
|
|
17
|
+
/** Version suffix, when the URL named one. */
|
|
18
|
+
version?: string;
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Parse an arXiv abs/html/pdf URL.
|
|
22
|
+
*
|
|
23
|
+
* @param url - any URL string.
|
|
24
|
+
* @returns the identifier and kind, or `undefined` for other hosts.
|
|
25
|
+
*/
|
|
26
|
+
export declare function parseArxivUrl(url: string): ArxivUrl | undefined;
|
|
27
|
+
/**
|
|
28
|
+
* The HTML full-text URL for a paper.
|
|
29
|
+
*
|
|
30
|
+
* @param reference - parsed arXiv URL.
|
|
31
|
+
* @returns an `https://arxiv.org/html/...` URL.
|
|
32
|
+
*/
|
|
33
|
+
export declare function arxivHtmlUrl(reference: ArxivUrl): string;
|
|
34
|
+
/**
|
|
35
|
+
* Map a PDF URL to the HTML view; leave other URLs alone.
|
|
36
|
+
*
|
|
37
|
+
* @param url - the caller's URL.
|
|
38
|
+
* @returns the URL the fetch ladder should read.
|
|
39
|
+
*/
|
|
40
|
+
export declare function arxivReadableUrl(url: string): string;
|
package/lib/arxiv-url.js
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Recognize arXiv page URLs and map PDF requests to the HTML full-text view.
|
|
3
|
+
*
|
|
4
|
+
* arXiv serves the same paper as `/abs/<id>` (metadata and abstract),
|
|
5
|
+
* `/html/<id>` (full text) and `/pdf/<id>` (binary PDF). This plugin converts
|
|
6
|
+
* HTML and text to Markdown but deliberately does not pretend binary PDF bytes
|
|
7
|
+
* are prose, so a PDF link is read through the HTML view instead.
|
|
8
|
+
*
|
|
9
|
+
* @module @fangchenghao/dsh-web-search/arxiv-url
|
|
10
|
+
*/
|
|
11
|
+
const HOST = /^(?:www\.|export\.)?arxiv\.org$/iu;
|
|
12
|
+
/**
|
|
13
|
+
* Parse an arXiv abs/html/pdf URL.
|
|
14
|
+
*
|
|
15
|
+
* @param url - any URL string.
|
|
16
|
+
* @returns the identifier and kind, or `undefined` for other hosts.
|
|
17
|
+
*/
|
|
18
|
+
export function parseArxivUrl(url) {
|
|
19
|
+
let parsed;
|
|
20
|
+
try {
|
|
21
|
+
parsed = new URL(url);
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return undefined;
|
|
25
|
+
}
|
|
26
|
+
if (!HOST.test(parsed.hostname))
|
|
27
|
+
return undefined;
|
|
28
|
+
const match = /^\/(abs|html|pdf)\/(.+?)\/?$/iu.exec(parsed.pathname);
|
|
29
|
+
if (match === null)
|
|
30
|
+
return undefined;
|
|
31
|
+
const kind = match[1]?.toLowerCase();
|
|
32
|
+
let raw = match[2] ?? '';
|
|
33
|
+
raw = raw.replace(/\.pdf$/iu, '');
|
|
34
|
+
const versionMatch = /v(\d+)$/iu.exec(raw);
|
|
35
|
+
const version = versionMatch?.[1];
|
|
36
|
+
const id = version === undefined ? raw : raw.slice(0, -versionMatch[0].length);
|
|
37
|
+
if (id.length === 0 || !/[a-z0-9]/iu.test(id))
|
|
38
|
+
return undefined;
|
|
39
|
+
return { id, kind, ...(version === undefined ? {} : { version }) };
|
|
40
|
+
}
|
|
41
|
+
/**
|
|
42
|
+
* The HTML full-text URL for a paper.
|
|
43
|
+
*
|
|
44
|
+
* @param reference - parsed arXiv URL.
|
|
45
|
+
* @returns an `https://arxiv.org/html/...` URL.
|
|
46
|
+
*/
|
|
47
|
+
export function arxivHtmlUrl(reference) {
|
|
48
|
+
const suffix = reference.version === undefined ? '' : `v${reference.version}`;
|
|
49
|
+
return `https://arxiv.org/html/${reference.id}${suffix}`;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Map a PDF URL to the HTML view; leave other URLs alone.
|
|
53
|
+
*
|
|
54
|
+
* @param url - the caller's URL.
|
|
55
|
+
* @returns the URL the fetch ladder should read.
|
|
56
|
+
*/
|
|
57
|
+
export function arxivReadableUrl(url) {
|
|
58
|
+
const reference = parseArxivUrl(url);
|
|
59
|
+
if (reference === undefined || reference.kind !== 'pdf')
|
|
60
|
+
return url;
|
|
61
|
+
return arxivHtmlUrl(reference);
|
|
62
|
+
}
|
package/lib/arxiv.d.ts
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Explicit `arXiv` research queries, answered by the existing arXiv vertical.
|
|
3
|
+
*
|
|
4
|
+
* A general web query that merely contains the word `arxiv` should not be
|
|
5
|
+
* hijacked, but a query that explicitly asks arXiv for a survey or paper is a
|
|
6
|
+
* clear registry lookup. The lookup runs in parallel with the ordinary
|
|
7
|
+
* fan-out, so it cannot delay a query that is not about arXiv.
|
|
8
|
+
*
|
|
9
|
+
* @module @fangchenghao/dsh-web-search/arxiv
|
|
10
|
+
*/
|
|
11
|
+
import type { Source } from './types.ts';
|
|
12
|
+
/** Options for the narrow arXiv lookup. */
|
|
13
|
+
export interface ArxivLookupOptions {
|
|
14
|
+
/** Result budget; defaults to four. */
|
|
15
|
+
limit?: number;
|
|
16
|
+
/** Deadline in milliseconds; defaults to 3.5 s. */
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
/** Configured tunnel. */
|
|
19
|
+
proxy?: string;
|
|
20
|
+
/** Caller cancellation. */
|
|
21
|
+
signal?: AbortSignal;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* The query text with the registry name removed, or `undefined` when the
|
|
25
|
+
* query is not explicitly arXiv-scoped.
|
|
26
|
+
*
|
|
27
|
+
* @param query - raw user query.
|
|
28
|
+
* @returns cleaned query, or `undefined`.
|
|
29
|
+
*/
|
|
30
|
+
export declare function arxivQueryText(query: string): string | undefined;
|
|
31
|
+
/**
|
|
32
|
+
* Query the arXiv vertical for an explicitly scoped query.
|
|
33
|
+
*
|
|
34
|
+
* @param query - raw user query.
|
|
35
|
+
* @param options - deadline, budget, proxy and cancellation.
|
|
36
|
+
* @returns matching sources, or an empty array when the query is not scoped.
|
|
37
|
+
*/
|
|
38
|
+
export declare function arxivLookup(query: string, options?: ArxivLookupOptions): Promise<Source[]>;
|
|
39
|
+
/**
|
|
40
|
+
* Put arXiv rows first and remove duplicate URLs.
|
|
41
|
+
*
|
|
42
|
+
* @param sources - ordinary fused rows.
|
|
43
|
+
* @param arxiv - rows from the arXiv vertical.
|
|
44
|
+
* @returns merged rows.
|
|
45
|
+
*/
|
|
46
|
+
export declare function mergeArxiv(sources: readonly Source[], arxiv: readonly Source[]): Source[];
|
package/lib/arxiv.js
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { searchVerticals } from "./verticals.js";
|
|
2
|
+
const ARXIV_QUERY = /\barxiv(?:\.org)?\b/iu;
|
|
3
|
+
const ARXIV_ID = /\barxiv\s*:\s*(\d{4}\.\d{4,5})/iu;
|
|
4
|
+
/** A paper intent keeps "how to use arXiv" from becoming a paper search. */
|
|
5
|
+
const RESEARCH_INTENT = /\b(?:paper|papers|preprint|preprints|survey|surveys|study|studies|research|literature|review|reviews)\b|论文|文献|预印本|综述|研究/iu;
|
|
6
|
+
/**
|
|
7
|
+
* The query text with the registry name removed, or `undefined` when the
|
|
8
|
+
* query is not explicitly arXiv-scoped.
|
|
9
|
+
*
|
|
10
|
+
* @param query - raw user query.
|
|
11
|
+
* @returns cleaned query, or `undefined`.
|
|
12
|
+
*/
|
|
13
|
+
export function arxivQueryText(query) {
|
|
14
|
+
if (typeof query !== 'string')
|
|
15
|
+
return undefined;
|
|
16
|
+
const trimmed = query.trim();
|
|
17
|
+
if (trimmed.length === 0)
|
|
18
|
+
return undefined;
|
|
19
|
+
const id = ARXIV_ID.exec(trimmed);
|
|
20
|
+
if (id !== null)
|
|
21
|
+
return id[1];
|
|
22
|
+
if (!ARXIV_QUERY.test(trimmed) || !RESEARCH_INTENT.test(trimmed))
|
|
23
|
+
return undefined;
|
|
24
|
+
const cleaned = trimmed.replace(ARXIV_QUERY, ' ').replace(/\s+/gu, ' ').trim();
|
|
25
|
+
return cleaned.length === 0 ? undefined : cleaned;
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Query the arXiv vertical for an explicitly scoped query.
|
|
29
|
+
*
|
|
30
|
+
* @param query - raw user query.
|
|
31
|
+
* @param options - deadline, budget, proxy and cancellation.
|
|
32
|
+
* @returns matching sources, or an empty array when the query is not scoped.
|
|
33
|
+
*/
|
|
34
|
+
export async function arxivLookup(query, options = {}) {
|
|
35
|
+
const text = arxivQueryText(query);
|
|
36
|
+
if (text === undefined)
|
|
37
|
+
return [];
|
|
38
|
+
const limit = options.limit ?? 4;
|
|
39
|
+
try {
|
|
40
|
+
const result = await searchVerticals(text, {
|
|
41
|
+
ids: ['arxiv'],
|
|
42
|
+
limit,
|
|
43
|
+
timeoutMs: options.timeoutMs ?? 3_500,
|
|
44
|
+
...(options.proxy === undefined ? {} : { proxy: options.proxy }),
|
|
45
|
+
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
46
|
+
});
|
|
47
|
+
return result.sources;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return [];
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Put arXiv rows first and remove duplicate URLs.
|
|
55
|
+
*
|
|
56
|
+
* @param sources - ordinary fused rows.
|
|
57
|
+
* @param arxiv - rows from the arXiv vertical.
|
|
58
|
+
* @returns merged rows.
|
|
59
|
+
*/
|
|
60
|
+
export function mergeArxiv(sources, arxiv) {
|
|
61
|
+
const seen = new Set();
|
|
62
|
+
const merged = [];
|
|
63
|
+
for (const source of [...arxiv, ...sources]) {
|
|
64
|
+
if (source.url.length === 0 || seen.has(source.url))
|
|
65
|
+
continue;
|
|
66
|
+
seen.add(source.url);
|
|
67
|
+
merged.push(source);
|
|
68
|
+
}
|
|
69
|
+
return merged;
|
|
70
|
+
}
|
package/lib/fetch.js
CHANGED
|
@@ -35,6 +35,7 @@
|
|
|
35
35
|
import { SearchError } from "./errors.js";
|
|
36
36
|
import { keyFailureKind, keyPoolFor } from "./keys.js";
|
|
37
37
|
import { DEFAULT_FETCH_MAX_LENGTH, renderFetchText, truncateFetchText } from "./markdown.js";
|
|
38
|
+
import { arxivReadableUrl } from "./arxiv-url.js";
|
|
38
39
|
import { proxyGet, resolveProxy } from "./proxy.js";
|
|
39
40
|
import { DEFAULT_RENDER_TIMEOUT_MS, describeRenderError, detectShellPage, loadPlaywrightRenderer, readableTextLength, redactRenderReason, renderWithPlaywright, } from "./render.js";
|
|
40
41
|
/** Default user agent; kept in step with the search side. */
|
|
@@ -388,10 +389,13 @@ export function createFetchProvider(options = {}, overrides = {}) {
|
|
|
388
389
|
id: options.id ?? 'http-proxy',
|
|
389
390
|
available: () => true,
|
|
390
391
|
async fetch(request, signal) {
|
|
391
|
-
const
|
|
392
|
-
if (typeof
|
|
393
|
-
throw new SearchError(`not an http(s) URL: ${String(
|
|
392
|
+
const requested = request.url;
|
|
393
|
+
if (typeof requested !== 'string' || !/^https?:\/\//i.test(requested)) {
|
|
394
|
+
throw new SearchError(`not an http(s) URL: ${String(requested)}`, 'SEARCH_PARSE', false);
|
|
394
395
|
}
|
|
396
|
+
// arXiv PDFs are binary and would otherwise be rejected as unsupported.
|
|
397
|
+
// Read the HTML full-text view instead; ordinary HTML pages pass through.
|
|
398
|
+
const target = arxivReadableUrl(requested);
|
|
395
399
|
// The ladder budget belongs to THIS request. Starting it in the factory
|
|
396
400
|
// made it start at plugin mount, so every fetch more than 25 s after a
|
|
397
401
|
// restart failed with "no time left in the fetch budget" before touching
|
package/lib/index.js
CHANGED
|
@@ -28,6 +28,7 @@ import { registerAdvancedSearchTool, undatedWindowNote } from "./advanced-search
|
|
|
28
28
|
import { searchVerticals } from "./verticals.js";
|
|
29
29
|
import { routePackageQuery } from "./query-router.js";
|
|
30
30
|
import { mergeOfficial, officialLookup } from "./official-sources.js";
|
|
31
|
+
import { arxivLookup, arxivQueryText, mergeArxiv } from "./arxiv.js";
|
|
31
32
|
import { chromeFeedNote, chromeWindow, fetchChromeReleases, filterChromeChannel, hasNonStableChromePosts, isChromeSecurityQuery, } from "./official-feed.js";
|
|
32
33
|
import { registerSearchPrompt } from "./system-prompt.js";
|
|
33
34
|
import { registerUnifiedTool } from "./tool-suite.js";
|
|
@@ -266,12 +267,22 @@ export function apply(ctx, config = {}) {
|
|
|
266
267
|
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
267
268
|
...(signal === undefined ? {} : { signal }),
|
|
268
269
|
});
|
|
270
|
+
// An explicit `arXiv` query gets the preprint index in parallel with the
|
|
271
|
+
// ordinary fan-out. This is separate from the official-release lookup:
|
|
272
|
+
// a survey query needs arXiv's own search, not a release announcement.
|
|
273
|
+
const arxiv = arxivLookup(query, {
|
|
274
|
+
limit: 4,
|
|
275
|
+
timeoutMs: Math.min(live.timeoutMs, 3_500),
|
|
276
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
277
|
+
...(signal === undefined ? {} : { signal }),
|
|
278
|
+
});
|
|
269
279
|
const outcome = await runQuery(query, asked, signal);
|
|
270
280
|
// A version/EOL question about PostgreSQL or Rust is better answered by
|
|
271
281
|
// the project's own release facts than by whatever an engine ranks first.
|
|
272
282
|
// The lookup is query-scoped and failure-tolerant; it never replaces the
|
|
273
283
|
// ordinary rows when the page is unavailable.
|
|
274
|
-
|
|
284
|
+
let sources = mergeOfficial(outcome.sources, await official);
|
|
285
|
+
sources = mergeArxiv(sources, await arxiv);
|
|
275
286
|
if (sources.length === 0) {
|
|
276
287
|
throw new Error(`no search engine returned results for ${JSON.stringify(query)} (${outcome.failures.join('; ')})`);
|
|
277
288
|
}
|
|
@@ -320,6 +331,26 @@ export function apply(ctx, config = {}) {
|
|
|
320
331
|
* returns nothing, so a registry outage cannot empty the batch.
|
|
321
332
|
*/
|
|
322
333
|
const runBatchQuery = async (query, limit, signal) => {
|
|
334
|
+
// An explicit arXiv query is a paper lookup, not a general web query.
|
|
335
|
+
// Keep the ordinary path as the fallback when the index returns no rows.
|
|
336
|
+
const arxivText = arxivQueryText(query);
|
|
337
|
+
if (arxivText !== undefined) {
|
|
338
|
+
const live = resolveConfig(section, undefined);
|
|
339
|
+
const papers = await arxivLookup(query, {
|
|
340
|
+
limit,
|
|
341
|
+
timeoutMs: live.domainTimeoutMs,
|
|
342
|
+
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
343
|
+
...(signal === undefined ? {} : { signal }),
|
|
344
|
+
});
|
|
345
|
+
if (papers.length > 0) {
|
|
346
|
+
return {
|
|
347
|
+
sources: papers.slice(0, limit),
|
|
348
|
+
failures: [],
|
|
349
|
+
note: `arXiv query routed to the arXiv index for ${JSON.stringify(arxivText)}; rows are preprints with stable abs URLs.`,
|
|
350
|
+
truncated: papers.length > limit,
|
|
351
|
+
};
|
|
352
|
+
}
|
|
353
|
+
}
|
|
323
354
|
const route = routePackageQuery(query);
|
|
324
355
|
if (route !== undefined) {
|
|
325
356
|
const live = resolveConfig(section, undefined);
|
package/lib/markdown.js
CHANGED
|
@@ -374,13 +374,29 @@ function prune(node) {
|
|
|
374
374
|
kept.push(child);
|
|
375
375
|
continue;
|
|
376
376
|
}
|
|
377
|
-
if (DROP_TAGS.has(child.tag)
|
|
377
|
+
if (DROP_TAGS.has(child.tag))
|
|
378
|
+
continue;
|
|
379
|
+
// A wrapper whose class says "footer" may still contain the page's main
|
|
380
|
+
// article. arXiv currently wraps its whole document in
|
|
381
|
+
// `div.flex-wrap-footer`; dropping that wrapper removed every readable
|
|
382
|
+
// character from the abstract page. Chrome inside the wrapper is still
|
|
383
|
+
// pruned by its own tag/attributes.
|
|
384
|
+
if (!NOISE_EXEMPT_TAGS.has(child.tag) && isNoise(child) && !containsLandmark(child))
|
|
378
385
|
continue;
|
|
379
386
|
prune(child);
|
|
380
387
|
kept.push(child);
|
|
381
388
|
}
|
|
382
389
|
node.children = kept;
|
|
383
390
|
}
|
|
391
|
+
/** Whether a subtree contains a real content landmark. */
|
|
392
|
+
function containsLandmark(node) {
|
|
393
|
+
let found = false;
|
|
394
|
+
walkElements(node, (child) => {
|
|
395
|
+
if (child.tag === 'main' || child.tag === 'article')
|
|
396
|
+
found = true;
|
|
397
|
+
});
|
|
398
|
+
return found;
|
|
399
|
+
}
|
|
384
400
|
/** Whether an element is hidden or named like site furniture. */
|
|
385
401
|
function isNoise(node) {
|
|
386
402
|
const role = (node.attrs['role'] ?? '').toLowerCase();
|
package/lib/verticals.d.ts
CHANGED
|
@@ -123,6 +123,19 @@ export declare function parseNpm(payload: string, _url: string, limit?: number):
|
|
|
123
123
|
* @returns the package as one source, or `[]`.
|
|
124
124
|
*/
|
|
125
125
|
export declare function parsePypi(payload: string, _url: string, limit?: number): Source[];
|
|
126
|
+
/**
|
|
127
|
+
* Turn a human query into an arXiv `search_query`.
|
|
128
|
+
*
|
|
129
|
+
* `all:retrieval augmented generation survey` is interpreted loosely and can
|
|
130
|
+
* rank an image-generation paper above actual surveys. arXiv's API honours
|
|
131
|
+
* explicit boolean operators, so a survey query becomes a quoted topic phrase
|
|
132
|
+
* plus `all:survey`; other multi-word queries become ANDed terms. A query that
|
|
133
|
+
* already contains field syntax or booleans is passed through unchanged.
|
|
134
|
+
*
|
|
135
|
+
* @param query - the human query.
|
|
136
|
+
* @returns a value for the `search_query` parameter.
|
|
137
|
+
*/
|
|
138
|
+
export declare function arxivSearchQuery(query: string): string;
|
|
126
139
|
/**
|
|
127
140
|
* Parse the arXiv Atom feed.
|
|
128
141
|
* @param payload - `export.arxiv.org/api/query` XML.
|
package/lib/verticals.js
CHANGED
|
@@ -276,6 +276,41 @@ export function parsePypi(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
|
276
276
|
const source = sourceOf(str(info.package_url) || `https://pypi.org/project/${encodeURIComponent(name)}/`, version.length > 0 ? `${name} ${version}` : name, snippet, str(first?.upload_time_iso_8601) || str(first?.upload_time));
|
|
277
277
|
return source === undefined ? [] : [source];
|
|
278
278
|
}
|
|
279
|
+
/** Paper-filler words that add noise to arXiv's `all:` search. */
|
|
280
|
+
const ARXIV_FILLER = new Set([
|
|
281
|
+
'a', 'about', 'an', 'and', 'are', 'find', 'for', 'from', 'in', 'is', 'latest',
|
|
282
|
+
'literature', 'newest', 'of', 'on', 'or', 'paper', 'papers', 'preprint',
|
|
283
|
+
'preprints', 'recent', 'research', 'review', 'reviews', 'search', 'the', 'to',
|
|
284
|
+
'with',
|
|
285
|
+
]);
|
|
286
|
+
/**
|
|
287
|
+
* Turn a human query into an arXiv `search_query`.
|
|
288
|
+
*
|
|
289
|
+
* `all:retrieval augmented generation survey` is interpreted loosely and can
|
|
290
|
+
* rank an image-generation paper above actual surveys. arXiv's API honours
|
|
291
|
+
* explicit boolean operators, so a survey query becomes a quoted topic phrase
|
|
292
|
+
* plus `all:survey`; other multi-word queries become ANDed terms. A query that
|
|
293
|
+
* already contains field syntax or booleans is passed through unchanged.
|
|
294
|
+
*
|
|
295
|
+
* @param query - the human query.
|
|
296
|
+
* @returns a value for the `search_query` parameter.
|
|
297
|
+
*/
|
|
298
|
+
export function arxivSearchQuery(query) {
|
|
299
|
+
const trimmed = query.trim();
|
|
300
|
+
if (trimmed.length === 0)
|
|
301
|
+
return 'all:';
|
|
302
|
+
if (/^(?:all|abs|au|cat|co|id|jr|rn|ti):/iu.test(trimmed) || /\b(?:AND|OR|ANDNOT)\b/u.test(trimmed) || trimmed.includes('"')) {
|
|
303
|
+
return trimmed;
|
|
304
|
+
}
|
|
305
|
+
const words = trimmed.split(/\s+/u).filter((word) => word.length > 0);
|
|
306
|
+
const survey = words.some((word) => /^(?:survey|surveys|综述)$/iu.test(word));
|
|
307
|
+
const terms = words.filter((word) => !ARXIV_FILLER.has(word.toLowerCase()) && !/^(?:survey|surveys|综述)$/iu.test(word));
|
|
308
|
+
if (survey && terms.length > 0) {
|
|
309
|
+
return `all:"${terms.join(' ')}" AND all:survey`;
|
|
310
|
+
}
|
|
311
|
+
const usable = terms.length > 0 ? terms : words;
|
|
312
|
+
return usable.map((term) => `all:${term}`).join(' AND ');
|
|
313
|
+
}
|
|
279
314
|
/**
|
|
280
315
|
* Parse the arXiv Atom feed.
|
|
281
316
|
* @param payload - `export.arxiv.org/api/query` XML.
|
|
@@ -1228,7 +1263,7 @@ export const VERTICALS = [
|
|
|
1228
1263
|
label: 'arXiv',
|
|
1229
1264
|
accept: 'application/atom+xml',
|
|
1230
1265
|
defaultOn: true,
|
|
1231
|
-
build: (query, limit) => `https://export.arxiv.org/api/query?search_query
|
|
1266
|
+
build: (query, limit) => `https://export.arxiv.org/api/query?search_query=${encodeURIComponent(arxivSearchQuery(query))}&start=0&max_results=${limit}`,
|
|
1232
1267
|
parse: parseArxiv,
|
|
1233
1268
|
},
|
|
1234
1269
|
{
|
|
@@ -1620,7 +1655,7 @@ export async function searchVerticals(query, options = {}) {
|
|
|
1620
1655
|
try {
|
|
1621
1656
|
const built = spec.build(trimmed, limit);
|
|
1622
1657
|
const url = spec.id === 'arxiv' && options.timeRange !== undefined && options.timeRange !== 'off'
|
|
1623
|
-
? `${built.replace(/search_query=
|
|
1658
|
+
? `${built.replace(/search_query=[^&]*/u, `search_query=${encodeURIComponent(`all:"${trimmed}"`)}`)}&sortBy=submittedDate&sortOrder=descending`
|
|
1624
1659
|
: built;
|
|
1625
1660
|
const payload = await fetchPayload(spec, url, options, timeoutMs);
|
|
1626
1661
|
const sources = spec.parse(payload, url, limit);
|
package/mcp/server.mjs
CHANGED
|
@@ -33,6 +33,7 @@ import { createFetchProvider } from '../lib/fetch.js';
|
|
|
33
33
|
import { searchVerticals } from '../lib/verticals.js';
|
|
34
34
|
import { createResearchDefinition, formatResearchValue } from '../lib/research.js';
|
|
35
35
|
import { mergeOfficial, officialLookup } from '../lib/official-sources.js';
|
|
36
|
+
import { arxivLookup, mergeArxiv } from '../lib/arxiv.js';
|
|
36
37
|
import { wrapUntrustedContent } from '../lib/untrusted.js';
|
|
37
38
|
|
|
38
39
|
const PACKAGE = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
|
@@ -179,8 +180,13 @@ async function callTool(name, args) {
|
|
|
179
180
|
timeoutMs: Math.min(timeoutMs, 2_500),
|
|
180
181
|
...(proxy === undefined ? {} : { proxy }),
|
|
181
182
|
});
|
|
183
|
+
const arxiv = arxivLookup(query, {
|
|
184
|
+
limit: 4,
|
|
185
|
+
timeoutMs: Math.min(timeoutMs, 3_500),
|
|
186
|
+
...(proxy === undefined ? {} : { proxy }),
|
|
187
|
+
});
|
|
182
188
|
const outcome = await search(query, undefined, searchOptions(limit, timeoutMs));
|
|
183
|
-
const sources = mergeOfficial(outcome.sources, await official);
|
|
189
|
+
const sources = mergeArxiv(mergeOfficial(outcome.sources, await official), await arxiv);
|
|
184
190
|
return formatSources(query, sources.slice(0, limit), { failures: outcome.failures });
|
|
185
191
|
}
|
|
186
192
|
if (name === 'fetch') {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fffangchenghao/dsh-web-search",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "DSH bundle: key-free web search, page fetch and one web_tools action tool for verticals, platforms and cited research; eight engines fused with RRF, 34 vertical sources, official release facts, a replayable benchmark, an optional Firecrawl key and a dependency-free MCP server.",
|
|
3
|
+
"version": "0.23.0",
|
|
4
|
+
"description": "DSH bundle: key-free web search, page fetch and one web_tools action tool for verticals, platforms and cited research; eight engines fused with RRF, 34 vertical sources, arXiv research lookups and readable papers, official release facts, a replayable benchmark, an optional Firecrawl key and a dependency-free MCP server.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
7
7
|
"types": "lib/index.d.ts",
|