@fffangchenghao/dsh-web-search 0.24.1 → 0.24.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +8 -8
- package/README.zh-CN.md +8 -8
- package/SECURITY.md +13 -7
- package/client.js +1 -1
- package/docs/comparison.md +1 -1
- package/docs/images/bench.png +0 -0
- package/docs/images/bench.svg +84 -0
- package/docs/images/web-tools-demo.svg +1 -1
- package/docs/latency-and-sources.md +1 -1
- package/docs/rrf-ranking.md +1 -1
- package/docs/store-evidence.md +1 -1
- package/lib/bridge.js +53 -2
- package/lib/fetch.js +59 -7
- package/lib/index.js +2 -2
- package/lib/markdown.js +35 -5
- package/lib/proxy.d.ts +9 -6
- package/lib/proxy.js +62 -23
- package/lib/rank.js +2 -2
- package/package.json +4 -2
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,60 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.24.3
|
|
4
|
+
|
|
5
|
+
Audit fixes from an independent Claude Fable 5.1 review (reasoning effort
|
|
6
|
+
`xhigh`). Search endpoints and ranking inputs are unchanged, except where noted.
|
|
7
|
+
|
|
8
|
+
### Fixed
|
|
9
|
+
- **A fractional engine weight silently dropped rows.** `fuse()` used the raw
|
|
10
|
+
weight as an array index, so a weight such as `2.5` produced non-integer row
|
|
11
|
+
positions and skipped every odd slot. It now uses the same floored weight as
|
|
12
|
+
`interleave()`.
|
|
13
|
+
- **Windows `ProxyEnable` was compared as a string.** `0x00000000` looked
|
|
14
|
+
enabled, so a disabled system proxy could be selected. The DWORD is now parsed
|
|
15
|
+
as a hexadecimal number.
|
|
16
|
+
- **IPv6 proxy targets used bracketed hostnames for SNI and TCP probes.** They
|
|
17
|
+
are now unbracketed before `net.connect`/`tls.connect`; `CONNECT` still writes
|
|
18
|
+
the bracketed authority. `https://` proxy URLs are rejected explicitly instead
|
|
19
|
+
of being handed to a plaintext CONNECT tunnel that cannot speak TLS.
|
|
20
|
+
- **Fetch bodies were unbounded before `maxBytes` was applied.** The direct path
|
|
21
|
+
now reads at most the byte budget from the response stream and cancels the
|
|
22
|
+
rest; the tunnel path caps the raw response and decompressed output. A
|
|
23
|
+
decompression bomb can no longer expand without a ceiling.
|
|
24
|
+
- **The Settings bridge accepted any Host on a loopback socket.** It now requires
|
|
25
|
+
a loopback Host name (DNS-rebinding defense), a JSON content type, and catches
|
|
26
|
+
body-read errors instead of leaving an async handler rejected.
|
|
27
|
+
- **Markdown rendering had three denial-of-service/forgery edges.** Raw-text
|
|
28
|
+
close tags reuse one lowercase copy instead of re-lowercasing the page per
|
|
29
|
+
`<script>`; element nesting is capped at 512 so recursive pruning/rendering
|
|
30
|
+
cannot overflow the stack; and stash placeholders carry a per-render nonce so
|
|
31
|
+
page text cannot forge a code placeholder.
|
|
32
|
+
- **The truncation notice counted outside `fetchMaxLength`.** The notice is now
|
|
33
|
+
part of the character budget, so `fetchMaxLength` is a real upper bound.
|
|
34
|
+
`clamp()` now describes its byte-level cut accurately.
|
|
35
|
+
|
|
36
|
+
**Test suite: 698 tests across 56 files** (697 pass, 1 skipped).
|
|
37
|
+
|
|
38
|
+
## 0.24.2
|
|
39
|
+
|
|
40
|
+
Documentation and image patch; no runtime behavior changed.
|
|
41
|
+
|
|
42
|
+
### Fixed
|
|
43
|
+
- **The benchmark chart no longer claims to be v0.15.0.** `docs/images/bench.png`
|
|
44
|
+
was regenerated from the current offline replay and labelled v0.24.x. The
|
|
45
|
+
numbers are unchanged (P@8 0.4375, R@8 0.7917, MRR@8 0.8071) because the
|
|
46
|
+
ranking path did not change; the chart now says so without presenting an old
|
|
47
|
+
release as the current one.
|
|
48
|
+
- **Baseline labels now say "current" everywhere.** README, Chinese README and
|
|
49
|
+
`docs/rrf-ranking.md` still explain that the tier-aware replay protocol was
|
|
50
|
+
introduced in 0.15.0, but no longer call the current baseline a "0.15.0
|
|
51
|
+
baseline".
|
|
52
|
+
- **The chart has an editable source.** `docs/images/bench.svg` is checked in
|
|
53
|
+
next to the PNG. Both files are shipped in the npm package so the relative
|
|
54
|
+
image link in `docs/rrf-ranking.md` resolves there too.
|
|
55
|
+
|
|
56
|
+
**Test suite: 690 tests across 56 files** (689 pass, 1 skipped).
|
|
57
|
+
|
|
3
58
|
## 0.24.1
|
|
4
59
|
|
|
5
60
|
Documentation and metadata patch; no search or fetch behavior changed.
|
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
<p align="center">
|
|
12
12
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm version"></a>
|
|
13
|
-
<img src="https://img.shields.io/badge/tests-
|
|
13
|
+
<img src="https://img.shields.io/badge/tests-698%20tests-brightgreen" alt="tests">
|
|
14
14
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="engines">
|
|
15
15
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
16
16
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
- **Measured in public, including the losses.** A 30-query bilingual comparison records availability, latency, unique domains, official-domain presence and every failure against keyless Firecrawl and anonymous Exa. The follow-up documents the Exa architecture difference, the 3 s CJK budget and the Rust official-source lookup. Exa is faster here; this plugin returns more distinct domains. Both test arms were keyless. The method and raw results are in [docs/comparison.md](docs/comparison.md) and a [public gist](https://gist.github.com/fangchenghao/3db9838fa98928435c1c2df38202c373).
|
|
29
29
|
- **Fetching admits failure.** Binary documents and empty 200 responses are rejected by name (`SEARCH_UNSUPPORTED`, `SEARCH_EMPTY`) instead of being returned as noise; arXiv PDF links are the deliberate exception and are read through the HTML full-text view; the direct → tunnel → render → extract ladder shares one 25 s budget and stops when there is no time left. The budget is per request — there is a regression test for exactly that.
|
|
30
30
|
- **Windows is a first-class host.** Pure ESM, no native addon or install script; the system proxy is discovered from Windows Internet Settings, and the full suite runs on a `windows-latest` CI runner. See [Install on Windows](#windows).
|
|
31
|
-
- **
|
|
31
|
+
- **698 tests across 56 files**, a committed `lib/`, declared `engines.dsh` / `engines.node`, and a [SECURITY.md](SECURITY.md) that lists every process, host and file the plugin touches.
|
|
32
32
|
- **Bilingual documentation.** This README and [中文说明](README.zh-CN.md) are maintained side by side.
|
|
33
33
|
|
|
34
34
|

|
|
@@ -45,7 +45,7 @@ The web-search corner of the DSH plugin ecosystem is crowded. A dated snapshot
|
|
|
45
45
|
| Key-free path | 8 engines, all key-free | ~30-engine free/paid chain | 5 defaults (seam, Exa, DDG, Bing, Jina) | Firecrawl + Keenable keyless defaults | 1 anonymous AnySearch API |
|
|
46
46
|
| Dedicated vertical sources | 34 (8 on by default) | 41 | platform registry, not a vertical catalog | web + X, no vertical catalog | vertical API behind AnySearch |
|
|
47
47
|
| Model-facing tools | 1 (`web_tools`, 8 actions) + 4 MCP tools | ~10 | 2 (`web_index` / `web_call`, 20 actions) | native search/fetch + skill | 5 |
|
|
48
|
-
| Tests | 56 files /
|
|
48
|
+
| Tests | 56 files / 698 cases, locally verified | no test script in `package.json` | 774 unit + 94 bench, self-reported | Vitest configured; count not published | Vitest configured; count not published |
|
|
49
49
|
| GitHub stars | 0 | 620 | 77 | 836 | 663 |
|
|
50
50
|
| npm downloads, last month | 1,087 | 52,436 | 6,420 | 43,151 | 9,251 |
|
|
51
51
|
| npm unpacked size | 1.2 MB | 0.70 MB | 3.05 MB | 0.51 MB | 0.27 MB |
|
|
@@ -56,7 +56,7 @@ read is:
|
|
|
56
56
|
- **Feature depth:** upper-middle. `dsh-free-search` has more engines and
|
|
57
57
|
verticals; `dsh-web-search-pro` has a deeper evidence/ranking pipeline and far
|
|
58
58
|
more platform backends; `modsearch` is the stronger cross-host bridge.
|
|
59
|
-
- **Engineering and verification:** near the top.
|
|
59
|
+
- **Engineering and verification:** near the top. 698 local tests, a replayable
|
|
60
60
|
ranking benchmark, declared security boundaries and one runtime dependency
|
|
61
61
|
put it alongside the best-maintained entries, while remaining much smaller
|
|
62
62
|
than the Playwright-heavy option.
|
|
@@ -65,7 +65,7 @@ read is:
|
|
|
65
65
|
common in the other plugins.
|
|
66
66
|
- **Adoption and maturity:** bottom third of this sample. At the 2026-10-25
|
|
67
67
|
snapshot the repository was four days old, had no stars and little
|
|
68
|
-
independent adoption; the current npm release is 0.24.
|
|
68
|
+
independent adoption; the current npm release is 0.24.3.
|
|
69
69
|
- **Tool-surface economy:** now competitive. 0.20.x exposes one `web_tools`
|
|
70
70
|
entry with eight actions, close to `dsh-web-search-pro`'s two tools with 20
|
|
71
71
|
actions; the legacy eight-name surface remains opt-in for old prompts.
|
|
@@ -340,7 +340,7 @@ Each line is the plain version first, the technical note second. The technical n
|
|
|
340
340
|
|
|
341
341
|
21. **Search quality has a repeatable benchmark.**
|
|
342
342
|
Ranking changes are no longer judged by feel: a fixed set of 20 queries produces the same scores every time.
|
|
343
|
-
*Technical: `bench/` holds 20 queries (10 Chinese, 10 English), five in each of docs / release / package / news, each annotated with 1–3 expected domains. `node scripts/bench.mjs` replays recorded fixtures offline, never touches the network, and prints precision@8, recall@8, MRR@8 plus each engine's contribution; `--record` re-records live (1.5 s between queries) to refresh the fixtures; `--json` gives machine-readable output for version comparisons, and past results live in `bench/results/`.
|
|
343
|
+
*Technical: `bench/` holds 20 queries (10 Chinese, 10 English), five in each of docs / release / package / news, each annotated with 1–3 expected domains. `node scripts/bench.mjs` replays recorded fixtures offline, never touches the network, and prints precision@8, recall@8, MRR@8 plus each engine's contribution; `--record` re-records live (1.5 s between queries) to refresh the fixtures; `--json` gives machine-readable output for version comparisons, and past results live in `bench/results/`. Current baseline (re-run on the 0.24.x line; tier-aware replay, so the recorded fallback engines only count when production would run them): P@8 `0.4375`, R@8 `0.7917`, MRR@8 `0.8071`; Chinese `0.4000 / 0.8167 / 0.7976`, English `0.4750 / 0.7667 / 0.8167`. A separate five-query long-Chinese diagnostic (`bench/queries-long.json`, recorded with `--queries ... --fixtures ...`) scores `0.4500 / 1.0000 / 0.7667`, with every query hitting an expected domain; it is kept separate so the 20-query baseline stays comparable. The controlled experiments and known gaps are in the appendix.*
|
|
344
344
|
|
|
345
345
|
22. **Research, in one call.**
|
|
346
346
|
For "what do the official docs and two other sources actually say about X", `research` runs up to three searches, opens the leading pages through the same fetch ladder, and returns one evidence pack in which every excerpt sits under its URL. A page that will not load is reported by URL and reason instead of aborting the pack; bot challenges and login redirects are rejected rather than quoted as evidence.
|
|
@@ -792,7 +792,7 @@ No. `package.json` does not list Playwright, and the plugin never downloads or i
|
|
|
792
792
|
|
|
793
793
|
### How do you know search quality improved?
|
|
794
794
|
|
|
795
|
-
From the fixed query set in `bench/`, not from intuition. There are 20 queries (10 Chinese, 10 English), five in each of docs / release / package / news, each annotated with 1–3 expected domains. `node scripts/bench.mjs` replays recorded fixtures offline and prints P@8 / R@8 / MRR@8. The
|
|
795
|
+
From the fixed query set in `bench/`, not from intuition. There are 20 queries (10 Chinese, 10 English), five in each of docs / release / package / news, each annotated with 1–3 expected domains. `node scripts/bench.mjs` replays recorded fixtures offline and prints P@8 / R@8 / MRR@8. The current 20-query baseline (tier-aware replay, protocol introduced in 0.15.0) is **0.4375 / 0.7917 / 0.8071**; Chinese 0.4000 / 0.8167 / 0.7976, English 0.4750 / 0.7667 / 0.8167. Run it before and after a ranking change and the direction is a number. Twenty queries are not statistical significance; news recall is only 0.467 and the per-query table still prints every miss — the appendix says so.
|
|
796
796
|
|
|
797
797
|
### Why do the results differ from what I get searching by hand?
|
|
798
798
|
|
|
@@ -1040,7 +1040,7 @@ Measured on `https://docs.python.org/3/library/asyncio-task.html`: starting at `
|
|
|
1040
1040
|
|
|
1041
1041
|
Metrics: P@8 = matched rows in the top 8 ÷ 8 (fixed denominator; a short list is not excused); R@8 = distinct expected domains found in the top 8 ÷ all expected domains; MRR@8 = the mean reciprocal rank of the first match over 20 queries, with a miss scoring 0. Domain matching is one-way from subdomain to parent: `docs.python.org` matches expected `python.org`, but the reverse does not.
|
|
1042
1042
|
|
|
1043
|
-
**
|
|
1043
|
+
**Current baseline** (v0.24.x replay; tier-aware protocol introduced in 0.15.0):
|
|
1044
1044
|
|
|
1045
1045
|
| Scope | P@8 | R@8 | MRR@8 |
|
|
1046
1046
|
| --- | --- | --- | --- |
|
package/README.zh-CN.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
<p align="center">
|
|
11
11
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm 版本"></a>
|
|
12
|
-
<img src="https://img.shields.io/badge/tests-
|
|
12
|
+
<img src="https://img.shields.io/badge/tests-698%20tests-brightgreen" alt="测试">
|
|
13
13
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="引擎">
|
|
14
14
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
15
15
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
- **公开测量,也公开输的地方。** 30 条中英查询对比记录可用性、延迟、独立域名、官方域名命中与每一条失败;对比对象是免 key Firecrawl 和匿名 Exa。这里 Exa 更快,本插件返回的独立域名更多;两条对比路径都未使用 key。方法和原始结果在 [docs/comparison.md](docs/comparison.md);Exa 架构差异、中文 3 秒预算和 Rust 官方源见 [延迟与官方源](docs/latency-and-sources.md)。
|
|
28
28
|
- **抓取会承认失败。** 二进制文档和「200 但正文为空」会被明确拒绝(`SEARCH_UNSUPPORTED`、`SEARCH_EMPTY`),而不是当噪音返回;arXiv 的 PDF 链接是有意保留的例外,会改读 HTML 全文;direct → tunnel → render → extract 四级阶梯共享 25 秒预算,预算不足就不再起新的一级。预算是「每次请求」而不是「插件启动时」,这点有回归测试守着。
|
|
29
29
|
- **Windows 是一等宿主。** 纯 ESM,没有原生扩展或安装脚本;会自动读取 Windows 的 Internet 设置代理,完整测试套件也在 `windows-latest` CI 上跑通。安装见[Windows 小节](#windows)。
|
|
30
|
-
- **56 个测试文件、
|
|
30
|
+
- **56 个测试文件、698 个用例**,`lib/` 产物入库,声明了 `engines.dsh` / `engines.node`,[SECURITY.md](SECURITY.md) 逐项列出插件会碰的进程、主机和文件。
|
|
31
31
|
- **双语文档。** 本文件和 [English README](README.md) 同步维护。
|
|
32
32
|
|
|
33
33
|

|
|
@@ -44,7 +44,7 @@ DSH 插件生态里的「联网搜索」已经很挤。下面是一张 2026-10-2
|
|
|
44
44
|
| 免 key 路径 | 8 个引擎,全部免 key | ~30 个免费/付费引擎链 | 默认 5 个(seam、Exa、DDG、Bing、Jina) | Firecrawl + Keenable 免 key 默认 | 1 个 AnySearch 匿名 API |
|
|
45
45
|
| 专门数据源 | 34 个(默认开 8 个) | 41 个 | 平台注册表,不是垂直源目录 | 网页 + X,无垂直源目录 | AnySearch 的垂直 API |
|
|
46
46
|
| 面向模型的工具 | 1 个(`web_tools`,8 个 action)+ 4 个 MCP 工具 | ~10 个 | 2 个(`web_index` / `web_call`,20 个动作) | 原生搜索/抓取 + skill | 5 个 |
|
|
47
|
-
| 测试 | 56 文件 /
|
|
47
|
+
| 测试 | 56 文件 / 698 用例,本地实测 | `package.json` 无测试脚本 | 774 单元 + 94 bench,自报 | 配了 Vitest,未公布数量 | 配了 Vitest,未公布数量 |
|
|
48
48
|
| GitHub stars | 0 | 620 | 77 | 836 | 663 |
|
|
49
49
|
| npm 近一月下载 | 1,087 | 52,436 | 6,420 | 43,151 | 9,251 |
|
|
50
50
|
| npm 解包大小 | 1.2 MB | 0.70 MB | 3.05 MB | 0.51 MB | 0.27 MB |
|
|
@@ -54,11 +54,11 @@ DSH 插件生态里的「联网搜索」已经很挤。下面是一张 2026-10-2
|
|
|
54
54
|
- **功能深度:中上。** `dsh-free-search` 引擎和垂直源更多;
|
|
55
55
|
`dsh-web-search-pro` 的证据/排序管线和平台后端更深;`modsearch` 的跨宿主
|
|
56
56
|
能力更强。
|
|
57
|
-
- **工程质量与可验证性:接近第一梯队。**
|
|
57
|
+
- **工程质量与可验证性:接近第一梯队。** 698 个本地测试、可重放的排序基准、
|
|
58
58
|
明确的安全边界、只有 1 个运行时依赖,同时体积远小于 Playwright 重型方案。
|
|
59
59
|
- **免 key + 官方源纠错:最强卖点。** Chrome、PostgreSQL、AI Act 的官方路径
|
|
60
60
|
和 34 个垂直源,不是其他插件普遍具备的。
|
|
61
|
-
- **采用度与成熟度:这个样本里的后段。** 2026-10-25 快照时 GitHub 仓库刚创建 4 天、0 stars,还缺少独立采用;当前 npm 版本是 0.24.
|
|
61
|
+
- **采用度与成熟度:这个样本里的后段。** 2026-10-25 快照时 GitHub 仓库刚创建 4 天、0 stars,还缺少独立采用;当前 npm 版本是 0.24.3。
|
|
62
62
|
- **工具面的经济性:现在有竞争力。** 0.20.x 只暴露一个 `web_tools` + 8 个
|
|
63
63
|
action,接近 `dsh-web-search-pro` 的 2 工具 + 20 动作;旧的八个工具名保留为
|
|
64
64
|
可选兼容模式。
|
|
@@ -331,7 +331,7 @@ Harness 里有个叫 `web` 的**服务**,它自己不会搜索,只负责「
|
|
|
331
331
|
|
|
332
332
|
21. **搜索质量有了可重复的基准。**
|
|
333
333
|
改排序不再靠感觉:仓库里有一组固定的 20 条查询,每次改完都能跑出同一套分数。
|
|
334
|
-
*技术补充:`bench/` 下 20 条查询(中文 10、英文 10),四类各 5 条(文档 / 版本发布 / 包与库 / 新闻),每条标注 1–3 个期望域名。`node scripts/bench.mjs` 离线读夹具重放,不联网,输出 precision@8、recall@8、MRR@8 与每个引擎的贡献;`--record` 联网重新录制(每条限速 1.5 秒),用于更新夹具;`--json` 输出机器可读结果,便于版本间对比,历史结果存在 `bench/results/`。当前基线(0.15.0
|
|
334
|
+
*技术补充:`bench/` 下 20 条查询(中文 10、英文 10),四类各 5 条(文档 / 版本发布 / 包与库 / 新闻),每条标注 1–3 个期望域名。`node scripts/bench.mjs` 离线读夹具重放,不联网,输出 precision@8、recall@8、MRR@8 与每个引擎的贡献;`--record` 联网重新录制(每条限速 1.5 秒),用于更新夹具;`--json` 输出机器可读结果,便于版本间对比,历史结果存在 `bench/results/`。当前基线(0.24.x 重放,沿用 0.15.0 的 tier 回放协议):P@8 `0.4375`、R@8 `0.7917`、MRR@8 `0.8071`;中文 `0.4000 / 0.8167 / 0.7976`,英文 `0.4750 / 0.7667 / 0.8167`。另有五条中文长查询的独立诊断集(`bench/queries-long.json`,用 `--queries ... --fixtures ...` 录制):`0.4500 / 1.0000 / 0.7667`,五条都命中期望域名;它与 20 条基线分开保存,避免改变基线的可比性。对照实验和已知短板见附录。*
|
|
335
335
|
|
|
336
336
|
22. **一次调用完成小调研。**
|
|
337
337
|
问「官方文档和另外两个来源到底怎么说」时,`research` 先跑最多三条查询,再用与 `web_fetch` 相同的抓取阶梯打开靠前的页面,最后返回一份证据包:每段摘录下面就是它的 URL。打不开的页面按 URL 和原因列出,不会让整份证据包失败;人机验证页和登录跳转页会被拒绝,不会被当作证据引用。
|
|
@@ -730,7 +730,7 @@ pnpm install
|
|
|
730
730
|
|
|
731
731
|
### 你们怎么知道搜索质量变好了?
|
|
732
732
|
|
|
733
|
-
靠 `bench/` 里的固定查询集,而不是拍脑袋。20 条查询(中文 10、英文 10),文档 / 版本发布 / 包与库 / 新闻四类各 5 条,每条标注 1–3 个期望域名。`node scripts/bench.mjs` 离线重放录好的夹具,输出 P@8 / R@8 / MRR@8
|
|
733
|
+
靠 `bench/` 里的固定查询集,而不是拍脑袋。20 条查询(中文 10、英文 10),文档 / 版本发布 / 包与库 / 新闻四类各 5 条,每条标注 1–3 个期望域名。`node scripts/bench.mjs` 离线重放录好的夹具,输出 P@8 / R@8 / MRR@8。当前基线(沿用 0.15.0 的 tier 回放协议)是 **0.4375 / 0.7917 / 0.8071**;中文 0.4000 / 0.8167 / 0.7976,英文 0.4750 / 0.7667 / 0.8167。改排序前后各跑一次,数字是变好还是变坏一目了然。样本只有 20 条,不能当统计显著性;新闻类 recall 只有 0.467,逐条输出会把每次没命中都打出来——这些短板都写在附录里。
|
|
734
734
|
|
|
735
735
|
### 为什么搜索结果和我手动搜的不一样?
|
|
736
736
|
|
|
@@ -978,7 +978,7 @@ read_page 调用
|
|
|
978
978
|
|
|
979
979
|
指标:P@8 = 前 8 条里命中期望域名的行数 ÷ 8(固定分母,返回不足 8 条也不放宽);R@8 = 前 8 条覆盖的期望域名 ÷ 全部期望域名;MRR@8 = 第一条命中的排名倒数,20 条取平均,没命中记 0。域名匹配允许「子域命中父域」单向成立:`docs.python.org` 命中期望的 `python.org`,反过来不算。
|
|
980
980
|
|
|
981
|
-
|
|
981
|
+
**当前基线**(0.24.x 重放;沿用 0.15.0 的 tier 回放协议):
|
|
982
982
|
|
|
983
983
|
| 范围 | P@8 | R@8 | MRR@8 |
|
|
984
984
|
| --- | --- | --- | --- |
|
package/SECURITY.md
CHANGED
|
@@ -43,16 +43,22 @@ Outbound TCP/HTTPS only, plus one local system query described below.
|
|
|
43
43
|
On Windows the system proxy is read locally from `HKCU\Software\Microsoft\Windows\CurrentVersion\Internet Settings`; that registry read is not network traffic.
|
|
44
44
|
|
|
45
45
|
`web_fetch` and `read_page` reach **any** HTTP(S) URL the model asks for; that is
|
|
46
|
-
their purpose.
|
|
46
|
+
their purpose. There is no private-address block: DSH's local agent already has
|
|
47
|
+
the host's permissions, and blocking `localhost` would break reading a local
|
|
48
|
+
development server. A deployment that treats the model as untrusted should put
|
|
49
|
+
the network boundary in the Host, not in this plugin. `engine_test` sends its probe query to the engines it tests, just
|
|
47
50
|
like a normal search; `advanced_search` is an ordinary search with a per-call
|
|
48
51
|
date window, except a windowed Chrome security/release query may read the
|
|
49
|
-
official Chrome Releases feed above. The Settings page's engine test
|
|
50
|
-
|
|
51
|
-
/api/dsh-web-search/
|
|
52
|
-
itself
|
|
52
|
+
official Chrome Releases feed above. The Settings page's engine test and **Install update** button use **inbound**
|
|
53
|
+
routes on the host's own `webServer` (`POST /api/dsh-web-search/engine-test`
|
|
54
|
+
and `POST /api/dsh-web-search/update`); the plugin never opens a listening
|
|
55
|
+
socket itself. Both routes accept POST only, require a loopback socket and a
|
|
56
|
+
loopback `Host` (so DNS rebinding cannot reach them), require
|
|
57
|
+
`application/json`, and reject cross-origin callers.
|
|
53
58
|
|
|
54
59
|
With `useProxy` enabled they speak to the configured proxy
|
|
55
|
-
(default `127.0.0.1:7890`, auto-detected) using HTTP `CONNECT`;
|
|
60
|
+
(default `127.0.0.1:7890`, auto-detected) using HTTP `CONNECT`; `https://`
|
|
61
|
+
proxy URLs are rejected because the tunnel is plaintext to the proxy. A 400 ms TCP
|
|
56
62
|
connect first checks that the proxy port is listening so a stale setting cannot
|
|
57
63
|
stall a search. The plugin never contacts an analytics, telemetry, or update
|
|
58
64
|
endpoint.
|
|
@@ -129,7 +135,7 @@ no cookies. Fetched research bodies exist only in process memory for
|
|
|
129
135
|
| one `platform_search` | 40 s tool budget; 8 s per direct platform attempt |
|
|
130
136
|
| cache | TTL and entry count are bounded and both configurable, `0` disables |
|
|
131
137
|
| system proxy probe | 2 s, then the child is killed |
|
|
132
|
-
|
|
|
138
|
+
| settings bridge request | loopback socket and Host, POST + JSON only, 16 KiB body |
|
|
133
139
|
| one engine test | 8 s per engine, three probes in flight, no cache or cooldown writes |
|
|
134
140
|
| one advanced_search | 30 s tool budget, same engine deadlines as web_search |
|
|
135
141
|
|
package/client.js
CHANGED
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
var ENTRY = 'web-search-bing';
|
|
75
75
|
var PACKAGE_NAME = 'dsh-web-search-bing';
|
|
76
76
|
/** Keep in step with package.json; compared with the npm registry. */
|
|
77
|
-
var VERSION = '0.24.
|
|
77
|
+
var VERSION = '0.24.3';
|
|
78
78
|
/** Snapshot used while no configuration service is reachable. */
|
|
79
79
|
var EMPTY_STATE = { status: 'unavailable', value: undefined, base: undefined, user: undefined, revision: undefined, writable: false };
|
|
80
80
|
|
package/docs/comparison.md
CHANGED
|
@@ -6,7 +6,7 @@ whether a provider answers at all, how long it takes, how many distinct domains
|
|
|
6
6
|
it returns, and whether a page on a known official domain appears in the top 8.
|
|
7
7
|
The failures are part of the result.
|
|
8
8
|
|
|
9
|
-
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1
|
|
9
|
+
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1 and 0.24.2 are documentation/metadata patches; 0.24.3 fixes fetch/proxy/markdown reliability without changing the measured search path. These patches do not change the search, ranking or source-adapter path measured below, so the 0.23.0 numbers remain the latest comparison.
|
|
10
10
|
|
|
11
11
|
## Method
|
|
12
12
|
|
|
Binary file
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="760" viewBox="0 0 1400 760" role="img" aria-labelledby="title desc">
|
|
2
|
+
<title id="title">Ranking quality, measured offline: baseline and parameter sweep</title>
|
|
3
|
+
<desc id="desc">Baseline precision, recall and MRR at 8 for the current 0.24.x replay over 20 fixed queries, plus the 81-configuration marginal sweep.</desc>
|
|
4
|
+
<defs>
|
|
5
|
+
<linearGradient id="bg" x1="0" y1="0" x2="1" y2="1">
|
|
6
|
+
<stop offset="0" stop-color="#0a1120"/><stop offset="0.55" stop-color="#0d1626"/><stop offset="1" stop-color="#0a1322"/>
|
|
7
|
+
</linearGradient>
|
|
8
|
+
<linearGradient id="card" x1="0" y1="0" x2="0" y2="1">
|
|
9
|
+
<stop offset="0" stop-color="#122138"/><stop offset="1" stop-color="#0d1829"/>
|
|
10
|
+
</linearGradient>
|
|
11
|
+
<filter id="glow" x="-40%" y="-40%" width="180%" height="180%">
|
|
12
|
+
<feGaussianBlur stdDeviation="8" result="blur"/><feMerge><feMergeNode in="blur"/><feMergeNode in="SourceGraphic"/></feMerge>
|
|
13
|
+
</filter>
|
|
14
|
+
<style>
|
|
15
|
+
.sans { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif; }
|
|
16
|
+
.title { fill: #f4f7fb; font-size: 38px; font-weight: 700; letter-spacing: -0.6px; }
|
|
17
|
+
.sub { fill: #8ea3bf; font-size: 17px; }
|
|
18
|
+
.h2 { fill: #f4f7fb; font-size: 24px; font-weight: 700; }
|
|
19
|
+
.h3 { fill: #9db1cc; font-size: 15px; font-weight: 600; letter-spacing: 0.2px; }
|
|
20
|
+
.muted { fill: #8195b0; font-size: 14px; }
|
|
21
|
+
.tiny { fill: #6f839d; font-size: 12.5px; }
|
|
22
|
+
.val { fill: #dbe8f8; font-size: 17px; font-weight: 600; }
|
|
23
|
+
.axis { fill: #8ba0bb; font-size: 15px; font-weight: 600; }
|
|
24
|
+
.blue { fill: #11243d; stroke: #5b8def; }
|
|
25
|
+
.green { fill: #112a2a; stroke: #37c996; }
|
|
26
|
+
.orange { fill: #2b2418; stroke: #f0a43a; }
|
|
27
|
+
</style>
|
|
28
|
+
</defs>
|
|
29
|
+
<rect width="1400" height="760" fill="url(#bg)"/>
|
|
30
|
+
<text x="60" y="62" class="sans title">Ranking quality, measured offline</text>
|
|
31
|
+
<text x="60" y="98" class="sans sub">20 fixed queries, tier-aware replay of recorded fixtures; no network. v0.24.x.</text>
|
|
32
|
+
<rect x="60" y="135" width="600" height="565" rx="16" fill="url(#card)" stroke="#294563" stroke-width="1.5"/>
|
|
33
|
+
<rect x="700" y="135" width="640" height="565" rx="16" fill="url(#card)" stroke="#294563" stroke-width="1.5"/>
|
|
34
|
+
<text x="90" y="184" class="sans h2">Baseline (v0.24.x replay)</text>
|
|
35
|
+
<text x="730" y="184" class="sans h2">Parameter sweep: marginal MRR@8</text>
|
|
36
|
+
<text x="730" y="211" class="sans muted">81 configurations, paired bootstrap vs defaults</text>
|
|
37
|
+
<rect x="190" y="465.6" width="82" height="144.4" rx="5" fill="none" stroke="#5b8def" stroke-width="2.5" class="blue" opacity="0.95"/>
|
|
38
|
+
<text x="231" y="495.6" text-anchor="middle" class="sans val" fill="#5b8def">0.4375</text>
|
|
39
|
+
<text x="231" y="652" text-anchor="middle" class="sans axis">P@8</text>
|
|
40
|
+
<rect x="395" y="348.8" width="82" height="261.2" rx="5" fill="none" stroke="#37c996" stroke-width="2.5" class="green" opacity="0.95"/>
|
|
41
|
+
<text x="436" y="378.8" text-anchor="middle" class="sans val" fill="#37c996">0.7917</text>
|
|
42
|
+
<text x="436" y="652" text-anchor="middle" class="sans axis">R@8</text>
|
|
43
|
+
<rect x="515" y="343.6" width="82" height="266.4" rx="5" fill="none" stroke="#f0a43a" stroke-width="2.5" class="orange" opacity="0.95"/>
|
|
44
|
+
<text x="556" y="373.6" text-anchor="middle" class="sans val" fill="#f0a43a">0.8071</text>
|
|
45
|
+
<text x="556" y="652" text-anchor="middle" class="sans axis">MRR@8</text>
|
|
46
|
+
<line x1="88" y1="610" x2="632" y2="610" stroke="#39536f" stroke-width="1"/>
|
|
47
|
+
<text x="90" y="681" class="sans tiny">n=20, k=8; 8 engines (7 primary + 1 fallback), zero-hit queries: 0</text>
|
|
48
|
+
<text x="745" y="258" class="sans h3">rrfConstant</text>
|
|
49
|
+
<rect x="745" y="442.7" width="38" height="147.3" rx="4" fill="#0f2036" stroke="#37c996" stroke-width="2" opacity="0.97"/>
|
|
50
|
+
<text x="764" y="432.7" text-anchor="middle" class="sans tiny" fill="#37c996">0.834</text>
|
|
51
|
+
<text x="764" y="615" text-anchor="middle" class="sans tiny">40</text>
|
|
52
|
+
<rect x="792" y="465.8" width="38" height="124.2" rx="4" fill="#0f2036" stroke="#5b8def" stroke-width="2" opacity="0.97"/>
|
|
53
|
+
<text x="811" y="455.8" text-anchor="middle" class="sans tiny" fill="#5b8def">0.828</text>
|
|
54
|
+
<text x="811" y="615" text-anchor="middle" class="sans tiny">60</text>
|
|
55
|
+
<rect x="839" y="496.9" width="38" height="93.1" rx="4" fill="#0f2036" stroke="#37c996" stroke-width="2" opacity="0.97"/>
|
|
56
|
+
<text x="858" y="486.9" text-anchor="middle" class="sans tiny" fill="#37c996">0.819</text>
|
|
57
|
+
<text x="858" y="615" text-anchor="middle" class="sans tiny">80</text>
|
|
58
|
+
<text x="935" y="258" class="sans h3">authorityBoost</text>
|
|
59
|
+
<rect x="935" y="466.9" width="38" height="123.1" rx="4" fill="#0f2036" stroke="#37c996" stroke-width="2" opacity="0.97"/>
|
|
60
|
+
<text x="954" y="456.9" text-anchor="middle" class="sans tiny" fill="#37c996">0.827</text>
|
|
61
|
+
<text x="954" y="615" text-anchor="middle" class="sans tiny">0.1</text>
|
|
62
|
+
<rect x="982" y="471.6" width="38" height="118.4" rx="4" fill="#0f2036" stroke="#5b8def" stroke-width="2" opacity="0.97"/>
|
|
63
|
+
<text x="1001" y="461.6" text-anchor="middle" class="sans tiny" fill="#5b8def">0.826</text>
|
|
64
|
+
<text x="1001" y="615" text-anchor="middle" class="sans tiny">0.25</text>
|
|
65
|
+
<rect x="1029" y="466.9" width="38" height="123.1" rx="4" fill="#0f2036" stroke="#37c996" stroke-width="2" opacity="0.97"/>
|
|
66
|
+
<text x="1048" y="456.9" text-anchor="middle" class="sans tiny" fill="#37c996">0.827</text>
|
|
67
|
+
<text x="1048" y="615" text-anchor="middle" class="sans tiny">0.4</text>
|
|
68
|
+
<text x="1125" y="258" class="sans h3">freshnessBoost</text>
|
|
69
|
+
<rect x="1125" y="483.6" width="38" height="106.4" rx="4" fill="#0f2036" stroke="#37c996" stroke-width="2" opacity="0.97"/>
|
|
70
|
+
<text x="1144" y="473.6" text-anchor="middle" class="sans tiny" fill="#37c996">0.823</text>
|
|
71
|
+
<text x="1144" y="615" text-anchor="middle" class="sans tiny">0.1</text>
|
|
72
|
+
<rect x="1172" y="478.0" width="38" height="112.0" rx="4" fill="#0f2036" stroke="#5b8def" stroke-width="2" opacity="0.97"/>
|
|
73
|
+
<text x="1191" y="468.0" text-anchor="middle" class="sans tiny" fill="#5b8def">0.825</text>
|
|
74
|
+
<text x="1191" y="615" text-anchor="middle" class="sans tiny">0.2</text>
|
|
75
|
+
<rect x="1219" y="443.8" width="38" height="146.2" rx="4" fill="#0f2036" stroke="#37c996" stroke-width="2" opacity="0.97"/>
|
|
76
|
+
<text x="1238" y="433.8" text-anchor="middle" class="sans tiny" fill="#37c996">0.834</text>
|
|
77
|
+
<text x="1238" y="615" text-anchor="middle" class="sans tiny">0.35</text>
|
|
78
|
+
<line x1="728" y1="590" x2="1310" y2="590" stroke="#39536f" stroke-width="1"/>
|
|
79
|
+
<rect x="730" y="632" width="580" height="62" rx="8" fill="#0b1728" stroke="#2f4b6a" stroke-width="1"/>
|
|
80
|
+
<text x="746" y="651" class="sans tiny" fill="#dbe8f8" font-weight="700">Best grid point: rrf=40, authority=0.25, freshness=0.35</text>
|
|
81
|
+
<text x="746" y="668" class="sans tiny" fill="#37c996">MRR 0.8583 vs 0.8071 default (+0.0512)</text>
|
|
82
|
+
<text x="746" y="685" class="sans tiny" fill="#f0a43a">95% CI [0.0000, +0.1250] touches zero: a direction, not a proven win</text>
|
|
83
|
+
<text x="730" y="725" class="sans tiny" fill="#6f839d">Blue = shipped default. Marginals average every configuration in the grid.</text>
|
|
84
|
+
</svg>
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
<rect x="70" y="620" width="330" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
53
53
|
<text x="92" y="651" class="m" font-size="13">Engines</text><text x="92" y="677" class="t" font-size="20" font-weight="700">8 key-free paths</text>
|
|
54
54
|
<rect x="420" y="620" width="330" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
55
|
-
<text x="442" y="651" class="m" font-size="13">Tests</text><text x="442" y="677" class="t" font-size="20" font-weight="700">
|
|
55
|
+
<text x="442" y="651" class="m" font-size="13">Tests</text><text x="442" y="677" class="t" font-size="20" font-weight="700">698 local cases</text>
|
|
56
56
|
<rect x="770" y="620" width="360" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
57
57
|
<text x="792" y="651" class="m" font-size="13">Cross-agent</text><text x="792" y="677" class="t" font-size="20" font-weight="700">MCP stdio server</text>
|
|
58
58
|
</g>
|
|
@@ -6,7 +6,7 @@ This note answers three questions from the public 30-query comparison:
|
|
|
6
6
|
2. Can the AnySearch tail and the Chinese Rust miss be fixed?
|
|
7
7
|
3. Does the fix require a vector database or a different retrieval model?
|
|
8
8
|
|
|
9
|
-
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1
|
|
9
|
+
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1 and 0.24.2 are documentation/metadata patches; 0.24.3 fixes fetch/proxy/markdown reliability without changing the measured search path. The search path is unchanged, so these 0.23.0 measurements still describe 0.24.x.
|
|
10
10
|
|
|
11
11
|
The short answer: Exa is a different architecture, not a faster RRF loop. This
|
|
12
12
|
plugin can borrow tail-tolerance and source-specific facts, but copying Exa's
|
package/docs/rrf-ranking.md
CHANGED
|
@@ -77,7 +77,7 @@ comparable across commits; `--record` is the only mode that touches the network.
|
|
|
77
77
|
|
|
78
78
|

|
|
79
79
|
|
|
80
|
-
Baseline (v0.
|
|
80
|
+
Baseline (current v0.24.x replay, 20 queries, k=8; tier-aware protocol introduced in 0.15.0):
|
|
81
81
|
|
|
82
82
|
| metric | overall | Chinese | English |
|
|
83
83
|
|---|---:|---:|---:|
|
package/docs/store-evidence.md
CHANGED
|
@@ -29,7 +29,7 @@ Only `0.2.1-rc.2` is declared, because that is the version actually exercised.
|
|
|
29
29
|
`mktemp -d`; the real `~/.dsh` was never written to. Bundled pnpm 11.28.5.
|
|
30
30
|
- **Windows CI:** GitHub Actions `windows-latest`, Node 24. Install, typecheck,
|
|
31
31
|
the full test suite, build and the entry-point import all pass. Run:
|
|
32
|
-
https://github.com/fangchenghao/dsh-web-search/actions/runs/
|
|
32
|
+
https://github.com/fangchenghao/dsh-web-search/actions/runs/44131883819
|
|
33
33
|
(the matrix also runs on macOS and Ubuntu). This is a CI environment, not a
|
|
34
34
|
claim about every Windows terminal or proxy client.
|
|
35
35
|
|
package/lib/bridge.js
CHANGED
|
@@ -23,6 +23,30 @@ function hostOf(value) {
|
|
|
23
23
|
return undefined;
|
|
24
24
|
}
|
|
25
25
|
}
|
|
26
|
+
/** Whether a Host hostname is a loopback literal. */
|
|
27
|
+
function isLoopbackHostname(hostname) {
|
|
28
|
+
const value = hostname.replace(/\.$/u, '');
|
|
29
|
+
return value === 'localhost'
|
|
30
|
+
|| value === '::1'
|
|
31
|
+
|| value === '[::1]'
|
|
32
|
+
|| /^127(?:\.\d{1,3}){3}$/u.test(value);
|
|
33
|
+
}
|
|
34
|
+
/** Host header hostname, without a port; undefined for a malformed header. */
|
|
35
|
+
function hostnameOf(value) {
|
|
36
|
+
if (typeof value !== 'string' || value.length === 0)
|
|
37
|
+
return undefined;
|
|
38
|
+
try {
|
|
39
|
+
return new URL(`http://${value}`).hostname.toLowerCase();
|
|
40
|
+
}
|
|
41
|
+
catch {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
/** Whether a POST declares the JSON body this bridge parses. */
|
|
46
|
+
function isJsonRequest(request) {
|
|
47
|
+
const value = request.headers['content-type'];
|
|
48
|
+
return typeof value === 'string' && /^application\/(?:[a-z0-9.+-]+\+)?json\s*(?:;|$)/iu.test(value);
|
|
49
|
+
}
|
|
26
50
|
/**
|
|
27
51
|
* Refuse cross-site and non-loopback callers.
|
|
28
52
|
*
|
|
@@ -33,6 +57,11 @@ function hostOf(value) {
|
|
|
33
57
|
function isAllowed(request) {
|
|
34
58
|
if (!isLoopback(request.socket.remoteAddress))
|
|
35
59
|
return false;
|
|
60
|
+
// A loopback socket is not enough: DNS rebinding makes a remote page
|
|
61
|
+
// same-origin while its Host/Origin still name the attacker's domain.
|
|
62
|
+
const hostname = hostnameOf(request.headers.host);
|
|
63
|
+
if (hostname === undefined || !isLoopbackHostname(hostname))
|
|
64
|
+
return false;
|
|
36
65
|
if (request.headers['sec-fetch-site'] === 'cross-site')
|
|
37
66
|
return false;
|
|
38
67
|
const origin = request.headers.origin;
|
|
@@ -161,7 +190,18 @@ export function registerUpdateBridge(ctx) {
|
|
|
161
190
|
writeJson(response, 403, { ok: false, code: 'forbidden', message: 'loopback settings only' });
|
|
162
191
|
return;
|
|
163
192
|
}
|
|
164
|
-
|
|
193
|
+
if (!isJsonRequest(request)) {
|
|
194
|
+
writeJson(response, 415, { ok: false, code: 'unsupported-media-type', message: 'application/json required' });
|
|
195
|
+
return;
|
|
196
|
+
}
|
|
197
|
+
let body;
|
|
198
|
+
try {
|
|
199
|
+
body = await readJson(request);
|
|
200
|
+
}
|
|
201
|
+
catch {
|
|
202
|
+
writeJson(response, 400, { ok: false, code: 'invalid-json', message: 'could not read the JSON body' });
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
165
205
|
if (body === undefined || typeof body !== 'object' || body === null || Array.isArray(body)) {
|
|
166
206
|
writeJson(response, 400, { ok: false, code: 'invalid-json', message: 'body must be a JSON object' });
|
|
167
207
|
return;
|
|
@@ -267,7 +307,18 @@ export function registerEngineTestBridge(ctx, runTest) {
|
|
|
267
307
|
writeJson(response, 403, { ok: false, code: 'forbidden', message: 'loopback settings only' });
|
|
268
308
|
return;
|
|
269
309
|
}
|
|
270
|
-
|
|
310
|
+
if (!isJsonRequest(request)) {
|
|
311
|
+
writeJson(response, 415, { ok: false, code: 'unsupported-media-type', message: 'application/json required' });
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
let body;
|
|
315
|
+
try {
|
|
316
|
+
body = await readJson(request);
|
|
317
|
+
}
|
|
318
|
+
catch {
|
|
319
|
+
writeJson(response, 400, { ok: false, code: 'invalid-json', message: 'could not read the JSON body' });
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
271
322
|
if (body === undefined) {
|
|
272
323
|
writeJson(response, 400, { ok: false, code: 'invalid-json', message: 'malformed or oversized JSON body' });
|
|
273
324
|
return;
|
package/lib/fetch.js
CHANGED
|
@@ -74,15 +74,67 @@ function mimeOf(contentType) {
|
|
|
74
74
|
return (contentType ?? '').split(';')[0]?.trim() ?? '';
|
|
75
75
|
}
|
|
76
76
|
/**
|
|
77
|
-
* Cut a body at the byte budget
|
|
77
|
+
* Cut a decoded body at the byte budget.
|
|
78
|
+
*
|
|
79
|
+
* The cut can land inside a multi-byte code point; `TextDecoder` replaces that
|
|
80
|
+
* final partial sequence rather than leaving a lone surrogate. The budget is a
|
|
81
|
+
* byte budget, not a character budget, so the result may be shorter.
|
|
82
|
+
*
|
|
78
83
|
* @param text - the decoded body.
|
|
79
84
|
* @param maxBytes - the byte budget.
|
|
80
85
|
* @returns the body and whether it was cut.
|
|
81
86
|
*/
|
|
82
87
|
function clamp(text, maxBytes) {
|
|
83
|
-
|
|
88
|
+
const bytes = Buffer.from(text, 'utf8');
|
|
89
|
+
if (bytes.length <= maxBytes)
|
|
84
90
|
return { content: text, truncated: false };
|
|
85
|
-
|
|
91
|
+
const content = new TextDecoder('utf-8').decode(bytes.subarray(0, Math.max(0, maxBytes))).replace(/\uFFFD+$/u, '');
|
|
92
|
+
return { content, truncated: true };
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* Read a fetch body without buffering more than the byte budget.
|
|
96
|
+
*
|
|
97
|
+
* `response.text()` materialises the whole body first, so a 1 MiB clamp after
|
|
98
|
+
* it has already downloaded 500 MiB. This reader stops at the budget and
|
|
99
|
+
* cancels the stream.
|
|
100
|
+
*
|
|
101
|
+
* @param response - the fetch response.
|
|
102
|
+
* @param maxBytes - maximum UTF-8 bytes to retain.
|
|
103
|
+
* @returns decoded text and whether the body was cut.
|
|
104
|
+
*/
|
|
105
|
+
async function readTextBounded(response, maxBytes) {
|
|
106
|
+
if (response.body === null)
|
|
107
|
+
return { content: '', truncated: false };
|
|
108
|
+
const reader = response.body.getReader();
|
|
109
|
+
const chunks = [];
|
|
110
|
+
let total = 0;
|
|
111
|
+
let truncated = false;
|
|
112
|
+
try {
|
|
113
|
+
for (;;) {
|
|
114
|
+
const { done, value } = await reader.read();
|
|
115
|
+
if (done)
|
|
116
|
+
break;
|
|
117
|
+
if (value === undefined)
|
|
118
|
+
continue;
|
|
119
|
+
const left = maxBytes - total;
|
|
120
|
+
if (value.byteLength > left) {
|
|
121
|
+
if (left > 0)
|
|
122
|
+
chunks.push(value.subarray(0, left));
|
|
123
|
+
total += Math.max(0, left);
|
|
124
|
+
truncated = true;
|
|
125
|
+
await reader.cancel();
|
|
126
|
+
break;
|
|
127
|
+
}
|
|
128
|
+
chunks.push(value);
|
|
129
|
+
total += value.byteLength;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
finally {
|
|
133
|
+
reader.releaseLock();
|
|
134
|
+
}
|
|
135
|
+
const bytes = Buffer.concat(chunks.map((chunk) => Buffer.from(chunk)));
|
|
136
|
+
const content = new TextDecoder('utf-8').decode(bytes).replace(/\uFFFD+$/u, '');
|
|
137
|
+
return { content, truncated };
|
|
86
138
|
}
|
|
87
139
|
/**
|
|
88
140
|
* Combine a caller signal with the per-attempt deadline.
|
|
@@ -165,15 +217,14 @@ export async function directFetch(url, timeoutMs, signal, userAgent, maxBytes) {
|
|
|
165
217
|
let target = url;
|
|
166
218
|
let last;
|
|
167
219
|
for (let hop = 0; hop <= REDIRECT_HOPS; hop += 1) {
|
|
168
|
-
const left = Math.max(
|
|
220
|
+
const left = Math.max(1, timeoutMs - (Date.now() - startedAt));
|
|
169
221
|
const response = await fetch(target, {
|
|
170
222
|
signal: deadline(left, signal),
|
|
171
223
|
redirect: 'follow',
|
|
172
224
|
headers: { 'user-agent': userAgent, accept: 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8' },
|
|
173
225
|
});
|
|
174
226
|
const contentType = response.headers.get('content-type');
|
|
175
|
-
const
|
|
176
|
-
const { content, truncated } = clamp(text, maxBytes);
|
|
227
|
+
const { content, truncated } = await readTextBounded(response, maxBytes);
|
|
177
228
|
last = {
|
|
178
229
|
url: response.url === '' ? target : response.url,
|
|
179
230
|
statusCode: response.status,
|
|
@@ -196,11 +247,12 @@ async function tunnelFetch(url, proxy, timeoutMs, signal, userAgent, maxBytes) {
|
|
|
196
247
|
let target = url;
|
|
197
248
|
let last;
|
|
198
249
|
for (let hop = 0; hop <= REDIRECT_HOPS; hop += 1) {
|
|
199
|
-
const left = Math.max(
|
|
250
|
+
const left = Math.max(1, timeoutMs - (Date.now() - startedAt));
|
|
200
251
|
const response = await proxyGet(target, {
|
|
201
252
|
proxy,
|
|
202
253
|
headers: { 'user-agent': userAgent, accept: 'text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8' },
|
|
203
254
|
timeoutMs: left,
|
|
255
|
+
maxBytes: Math.max(1, maxBytes),
|
|
204
256
|
...(signal === undefined ? {} : { signal }),
|
|
205
257
|
});
|
|
206
258
|
const { content, truncated } = clamp(response.text, maxBytes);
|
package/lib/index.js
CHANGED
|
@@ -266,7 +266,7 @@ export function apply(ctx, config = {}) {
|
|
|
266
266
|
timeoutMs: Math.min(live.timeoutMs, 2_500),
|
|
267
267
|
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
268
268
|
...(signal === undefined ? {} : { signal }),
|
|
269
|
-
});
|
|
269
|
+
})?.catch(() => undefined);
|
|
270
270
|
// An explicit `arXiv` query gets the preprint index in parallel with the
|
|
271
271
|
// ordinary fan-out. This is separate from the official-release lookup:
|
|
272
272
|
// a survey query needs arXiv's own search, not a release announcement.
|
|
@@ -275,7 +275,7 @@ export function apply(ctx, config = {}) {
|
|
|
275
275
|
timeoutMs: Math.min(live.timeoutMs, 3_500),
|
|
276
276
|
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
277
277
|
...(signal === undefined ? {} : { signal }),
|
|
278
|
-
});
|
|
278
|
+
}).catch(() => []);
|
|
279
279
|
const outcome = await runQuery(query, asked, signal);
|
|
280
280
|
// A version/EOL question about PostgreSQL or Rust is better answered by
|
|
281
281
|
// the project's own release facts than by whatever an engine ranks first.
|
package/lib/markdown.js
CHANGED
|
@@ -20,6 +20,14 @@
|
|
|
20
20
|
*/
|
|
21
21
|
/** Default `fetchMaxLength`: about 20k characters of prose. */
|
|
22
22
|
export const DEFAULT_FETCH_MAX_LENGTH = 20_000;
|
|
23
|
+
/**
|
|
24
|
+
* Maximum element nesting kept by the parser.
|
|
25
|
+
*
|
|
26
|
+
* The tree builder itself is iterative, but pruning and rendering recurse. A
|
|
27
|
+
* hostile page can nest thousands of `<div>`s; refusing to grow past this depth
|
|
28
|
+
* keeps every later pass within the JavaScript stack.
|
|
29
|
+
*/
|
|
30
|
+
const MAX_DOM_DEPTH = 512;
|
|
23
31
|
/** Named entities that appear in real prose; unknown names are left alone. */
|
|
24
32
|
const ENTITIES = {
|
|
25
33
|
amp: '&',
|
|
@@ -188,6 +196,12 @@ function parseAttributes(raw) {
|
|
|
188
196
|
function parseHtml(html) {
|
|
189
197
|
const root = element('#root', {});
|
|
190
198
|
const stack = [root];
|
|
199
|
+
/** Opening tags dropped at MAX_DOM_DEPTH, so their closing tags can be ignored too. */
|
|
200
|
+
let flattened = 0;
|
|
201
|
+
// `toLowerCase()` would change indices for a few non-ASCII strings. Keep a
|
|
202
|
+
// reusable lowercase copy only when its length still maps 1:1 to the input.
|
|
203
|
+
const lowered = html.toLowerCase();
|
|
204
|
+
const lower = lowered.length === html.length ? lowered : undefined;
|
|
191
205
|
const pushText = (value) => {
|
|
192
206
|
if (value.length === 0)
|
|
193
207
|
return;
|
|
@@ -214,6 +228,10 @@ function parseHtml(html) {
|
|
|
214
228
|
}
|
|
215
229
|
};
|
|
216
230
|
const closeTag = (name) => {
|
|
231
|
+
if (flattened > 0) {
|
|
232
|
+
flattened -= 1;
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
217
235
|
for (let index = stack.length - 1; index >= 1; index -= 1) {
|
|
218
236
|
const open = stack[index];
|
|
219
237
|
if (open === undefined)
|
|
@@ -230,6 +248,13 @@ function parseHtml(html) {
|
|
|
230
248
|
const top = stack[stack.length - 1];
|
|
231
249
|
if (top === undefined)
|
|
232
250
|
return;
|
|
251
|
+
// Deeper markup is flattened into the current element. Count the dropped
|
|
252
|
+
// opener so its closing tag does not pop one of the real ancestors.
|
|
253
|
+
if (stack.length >= MAX_DOM_DEPTH) {
|
|
254
|
+
if (!selfClosing && !VOID_TAGS.has(name))
|
|
255
|
+
flattened += 1;
|
|
256
|
+
return;
|
|
257
|
+
}
|
|
233
258
|
if (top.tag === 'p' && BLOCK_TAGS.has(name))
|
|
234
259
|
stack.pop();
|
|
235
260
|
if (name === 'li')
|
|
@@ -306,7 +331,7 @@ function parseHtml(html) {
|
|
|
306
331
|
const name = (match[0] ?? '').toLowerCase();
|
|
307
332
|
index = end + 1;
|
|
308
333
|
if (RAW_TEXT_TAGS.has(name)) {
|
|
309
|
-
const closeAt = html.toLowerCase().indexOf(`</${name}`, index);
|
|
334
|
+
const closeAt = (lower ?? html.toLowerCase()).indexOf(`</${name}`, index);
|
|
310
335
|
if (closeAt === -1) {
|
|
311
336
|
index = html.length;
|
|
312
337
|
continue;
|
|
@@ -515,13 +540,13 @@ function pickMain(root) {
|
|
|
515
540
|
/** Replace a code fragment with a placeholder so clean-up cannot touch it. */
|
|
516
541
|
function stashCode(ctx, text) {
|
|
517
542
|
ctx.stash.push(text);
|
|
518
|
-
return
|
|
543
|
+
return `${ctx.stashToken}${ctx.stash.length - 1}\uE001`;
|
|
519
544
|
}
|
|
520
545
|
/** Put stashed code fragments back after whitespace clean-up. */
|
|
521
546
|
function restoreCode(text, ctx) {
|
|
522
547
|
let out = text;
|
|
523
548
|
for (let index = 0; index < ctx.stash.length; index += 1) {
|
|
524
|
-
out = out.split(
|
|
549
|
+
out = out.split(`${ctx.stashToken}${index}\uE001`).join(ctx.stash[index] ?? '');
|
|
525
550
|
}
|
|
526
551
|
return out;
|
|
527
552
|
}
|
|
@@ -877,6 +902,7 @@ export function htmlToMarkdown(html, options = {}) {
|
|
|
877
902
|
format: options.format ?? 'markdown',
|
|
878
903
|
baseUrl: options.baseUrl,
|
|
879
904
|
stash: [],
|
|
905
|
+
stashToken: `\uE000${Math.random().toString(36).slice(2, 10)}:`,
|
|
880
906
|
};
|
|
881
907
|
return restoreCode(cleanup(renderElement(node, ctx)), ctx).trim();
|
|
882
908
|
};
|
|
@@ -905,13 +931,17 @@ export function truncateFetchText(content, maxLength) {
|
|
|
905
931
|
const limit = Number.isFinite(maxLength) && maxLength >= 1 ? Math.floor(maxLength) : DEFAULT_FETCH_MAX_LENGTH;
|
|
906
932
|
if (content.length <= limit)
|
|
907
933
|
return { content, truncated: false };
|
|
908
|
-
|
|
934
|
+
const fullNotice = `\n\n[truncated at ${limit} chars; original length ${content.length} chars; the rest is not shown]`;
|
|
935
|
+
// The notice is part of the configured budget, not an allowance on top of
|
|
936
|
+
// it. For a tiny budget, shorten the notice rather than exceeding the limit.
|
|
937
|
+
const notice = fullNotice.length >= limit ? fullNotice.slice(0, Math.max(0, limit - 1)) : fullNotice;
|
|
938
|
+
const budget = Math.max(1, limit - notice.length);
|
|
939
|
+
let cut = content.slice(0, budget);
|
|
909
940
|
const last = cut.charCodeAt(cut.length - 1);
|
|
910
941
|
// Never leave half of a surrogate pair at the cut point.
|
|
911
942
|
if (last >= 0xd800 && last <= 0xdbff)
|
|
912
943
|
cut = cut.slice(0, -1);
|
|
913
944
|
cut = cut.replace(/\s+$/, '');
|
|
914
|
-
const notice = `\n\n[truncated at ${limit} chars; original length ${content.length} chars; the rest is not shown]`;
|
|
915
945
|
return { content: `${cut}${notice}`, truncated: true };
|
|
916
946
|
}
|
|
917
947
|
/**
|
package/lib/proxy.d.ts
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
export interface ProxyTarget {
|
|
3
3
|
/** Proxy host name or IP literal, without brackets or port. */
|
|
4
4
|
host: string;
|
|
5
|
-
/** Proxy port; already defaulted to 80
|
|
5
|
+
/** Proxy port; already defaulted to 80 by {@link parseProxy}. */
|
|
6
6
|
port: number;
|
|
7
7
|
/** Ready-to-send `Proxy-Authorization` header value, when credentials exist. */
|
|
8
8
|
authorization?: string;
|
|
@@ -34,6 +34,8 @@ export interface ProxyRequestOptions {
|
|
|
34
34
|
method?: 'GET' | 'POST';
|
|
35
35
|
/** Request body, sent verbatim for `POST`. */
|
|
36
36
|
body?: string;
|
|
37
|
+
/** Maximum response bytes to retain; defaults to 8 MiB. */
|
|
38
|
+
maxBytes?: number;
|
|
37
39
|
}
|
|
38
40
|
/** A raw response split into its parts, before it is mapped to `{ url }`. */
|
|
39
41
|
export interface ParsedResponse {
|
|
@@ -42,11 +44,12 @@ export interface ParsedResponse {
|
|
|
42
44
|
text: string;
|
|
43
45
|
}
|
|
44
46
|
/**
|
|
45
|
-
* Parse a proxy URL
|
|
47
|
+
* Parse a proxy URL.
|
|
46
48
|
*
|
|
47
|
-
* `
|
|
48
|
-
*
|
|
49
|
-
*
|
|
49
|
+
* `CONNECT` is written over a plain TCP connection to the proxy. An `https:`
|
|
50
|
+
* proxy would require a TLS handshake to the proxy before that CONNECT; this
|
|
51
|
+
* transport does not implement it, so the scheme is rejected instead of being
|
|
52
|
+
* half-supported (which used to fail with an opaque `socket hang up`).
|
|
50
53
|
*
|
|
51
54
|
* @param raw - candidate `http://[user:pass@]host[:port]` string.
|
|
52
55
|
* @returns the proxy host, port and optional `Proxy-Authorization` value.
|
|
@@ -119,7 +122,7 @@ export declare function resolveProxy(explicit?: string): Promise<string | undefi
|
|
|
119
122
|
* @param buffer - the bytes received from the socket.
|
|
120
123
|
* @returns the parsed response, or `undefined` when the head is incomplete.
|
|
121
124
|
*/
|
|
122
|
-
export declare function readResponse(buffer: Buffer): ParsedResponse | undefined;
|
|
125
|
+
export declare function readResponse(buffer: Buffer, maxBytes?: number): ParsedResponse | undefined;
|
|
123
126
|
/**
|
|
124
127
|
* Perform a request through the proxy, following redirects.
|
|
125
128
|
*
|
package/lib/proxy.js
CHANGED
|
@@ -30,6 +30,8 @@ const CACHE_MS = 60_000;
|
|
|
30
30
|
const DEFAULT_TIMEOUT_MS = 15_000;
|
|
31
31
|
/** Default number of redirect hops to follow. */
|
|
32
32
|
const DEFAULT_REDIRECTS = 4;
|
|
33
|
+
/** Hard ceiling for one tunnel response, before decompression. */
|
|
34
|
+
const DEFAULT_MAX_BYTES = 8 * 1024 * 1024;
|
|
33
35
|
/**
|
|
34
36
|
* Environment variables read by {@link proxyFromEnv}, highest precedence first.
|
|
35
37
|
* Both casings are listed because shells disagree about which one they export.
|
|
@@ -55,11 +57,12 @@ function decodeCredential(value) {
|
|
|
55
57
|
}
|
|
56
58
|
}
|
|
57
59
|
/**
|
|
58
|
-
* Parse a proxy URL
|
|
60
|
+
* Parse a proxy URL.
|
|
59
61
|
*
|
|
60
|
-
* `
|
|
61
|
-
*
|
|
62
|
-
*
|
|
62
|
+
* `CONNECT` is written over a plain TCP connection to the proxy. An `https:`
|
|
63
|
+
* proxy would require a TLS handshake to the proxy before that CONNECT; this
|
|
64
|
+
* transport does not implement it, so the scheme is rejected instead of being
|
|
65
|
+
* half-supported (which used to fail with an opaque `socket hang up`).
|
|
63
66
|
*
|
|
64
67
|
* @param raw - candidate `http://[user:pass@]host[:port]` string.
|
|
65
68
|
* @returns the proxy host, port and optional `Proxy-Authorization` value.
|
|
@@ -76,19 +79,19 @@ export function parseProxy(raw) {
|
|
|
76
79
|
catch {
|
|
77
80
|
throw new Error(`unparsable proxy URL: ${JSON.stringify(raw)}`);
|
|
78
81
|
}
|
|
79
|
-
if (url.protocol !== 'http:'
|
|
80
|
-
throw new Error(`unsupported proxy scheme ${JSON.stringify(url.protocol)} in ${JSON.stringify(raw)}: only http://
|
|
82
|
+
if (url.protocol !== 'http:') {
|
|
83
|
+
throw new Error(`unsupported proxy scheme ${JSON.stringify(url.protocol)} in ${JSON.stringify(raw)}: only http:// proxies are supported`);
|
|
81
84
|
}
|
|
82
85
|
if (url.hostname.length === 0)
|
|
83
86
|
throw new Error(`proxy URL has no host: ${JSON.stringify(raw)}`);
|
|
84
|
-
const port = url.port === '' ?
|
|
87
|
+
const port = url.port === '' ? 80 : Number(url.port);
|
|
85
88
|
if (!Number.isInteger(port) || port <= 0 || port > 65_535) {
|
|
86
89
|
throw new Error(`proxy URL has an invalid port: ${JSON.stringify(raw)}`);
|
|
87
90
|
}
|
|
88
91
|
const username = decodeCredential(url.username);
|
|
89
92
|
const password = decodeCredential(url.password);
|
|
90
93
|
const authorization = username.length === 0 && password.length === 0 ? undefined : `Basic ${Buffer.from(`${username}:${password}`).toString('base64')}`;
|
|
91
|
-
return { host: url.hostname, port, ...(authorization === undefined ? {} : { authorization }) };
|
|
94
|
+
return { host: url.hostname.replace(/^\[|\]$/g, ''), port, ...(authorization === undefined ? {} : { authorization }) };
|
|
92
95
|
}
|
|
93
96
|
/**
|
|
94
97
|
* Parse a proxy string for the detection paths, which treat an unusable value
|
|
@@ -215,7 +218,8 @@ async function captureProbe(command, args, timeoutMs = 2_000) {
|
|
|
215
218
|
* @returns an `http://` proxy URL, or `undefined` when disabled/unusable.
|
|
216
219
|
*/
|
|
217
220
|
export function proxyFromWindowsRegistry(output) {
|
|
218
|
-
|
|
221
|
+
const enabled = Number.parseInt(/ProxyEnable\s+REG_DWORD\s+0x([0-9a-f]+)/iu.exec(output)?.[1] ?? '0', 16);
|
|
222
|
+
if (!Number.isFinite(enabled) || enabled === 0)
|
|
219
223
|
return undefined;
|
|
220
224
|
const raw = /ProxyServer\s+REG_SZ\s+(.+)/iu.exec(output)?.[1]?.trim();
|
|
221
225
|
if (raw === undefined || raw.length === 0)
|
|
@@ -272,7 +276,7 @@ export async function proxyFromSystem() {
|
|
|
272
276
|
/** Check that something is listening, so a stale setting cannot stall a search. */
|
|
273
277
|
async function reachable(proxy, timeoutMs = 400) {
|
|
274
278
|
return await new Promise((resolve) => {
|
|
275
|
-
const socket = net.connect({ host: proxy.hostname, port: Number(proxy.port || 80) });
|
|
279
|
+
const socket = net.connect({ host: proxy.hostname.replace(/^\[|\]$/g, ''), port: Number(proxy.port || 80) });
|
|
276
280
|
const done = (value) => {
|
|
277
281
|
socket.destroy();
|
|
278
282
|
resolve(value);
|
|
@@ -314,13 +318,15 @@ export async function resolveProxy(explicit) {
|
|
|
314
318
|
}
|
|
315
319
|
/** Open a `CONNECT` tunnel to `host:port` through the proxy. */
|
|
316
320
|
async function openTunnel(proxy, host, port, timeoutMs, signal) {
|
|
321
|
+
const bare = host.replace(/^\[|\]$/g, '');
|
|
322
|
+
const authority = net.isIP(bare) === 6 ? `[${bare}]:${port}` : `${bare}:${port}`;
|
|
317
323
|
const request = http.request({
|
|
318
324
|
host: proxy.host,
|
|
319
325
|
port: proxy.port,
|
|
320
326
|
method: 'CONNECT',
|
|
321
|
-
path:
|
|
327
|
+
path: authority,
|
|
322
328
|
headers: {
|
|
323
|
-
host:
|
|
329
|
+
host: authority,
|
|
324
330
|
...(proxy.authorization === undefined ? {} : { 'proxy-authorization': proxy.authorization }),
|
|
325
331
|
},
|
|
326
332
|
});
|
|
@@ -344,9 +350,10 @@ async function openTunnel(proxy, host, port, timeoutMs, signal) {
|
|
|
344
350
|
}
|
|
345
351
|
}
|
|
346
352
|
/** Write one request over a connected socket and collect the raw response. */
|
|
347
|
-
function exchange(socket, head, timeoutMs, signal) {
|
|
353
|
+
function exchange(socket, head, timeoutMs, signal, maxBytes = DEFAULT_MAX_BYTES) {
|
|
348
354
|
return new Promise((resolve, reject) => {
|
|
349
355
|
const chunks = [];
|
|
356
|
+
let size = 0;
|
|
350
357
|
let settled = false;
|
|
351
358
|
const done = (settle, value) => {
|
|
352
359
|
if (settled)
|
|
@@ -369,7 +376,23 @@ function exchange(socket, head, timeoutMs, signal) {
|
|
|
369
376
|
const timer = setTimeout(() => fail(new Error('request timed out')), timeoutMs);
|
|
370
377
|
const abort = () => fail(new Error('aborted'));
|
|
371
378
|
signal?.addEventListener('abort', abort, { once: true });
|
|
372
|
-
socket.on('data', (chunk) =>
|
|
379
|
+
socket.on('data', (chunk) => {
|
|
380
|
+
if (settled)
|
|
381
|
+
return;
|
|
382
|
+
const left = maxBytes - size;
|
|
383
|
+
if (left <= 0) {
|
|
384
|
+
done(resolve, Buffer.concat(chunks));
|
|
385
|
+
return;
|
|
386
|
+
}
|
|
387
|
+
if (chunk.length > left) {
|
|
388
|
+
chunks.push(chunk.subarray(0, left));
|
|
389
|
+
size += left;
|
|
390
|
+
done(resolve, Buffer.concat(chunks));
|
|
391
|
+
return;
|
|
392
|
+
}
|
|
393
|
+
chunks.push(chunk);
|
|
394
|
+
size += chunk.length;
|
|
395
|
+
});
|
|
373
396
|
socket.on('end', () => done(resolve, Buffer.concat(chunks)));
|
|
374
397
|
socket.on('error', (error) => fail(error));
|
|
375
398
|
socket.write(head);
|
|
@@ -402,7 +425,7 @@ function dechunk(buffer) {
|
|
|
402
425
|
* @param buffer - the bytes received from the socket.
|
|
403
426
|
* @returns the parsed response, or `undefined` when the head is incomplete.
|
|
404
427
|
*/
|
|
405
|
-
export function readResponse(buffer) {
|
|
428
|
+
export function readResponse(buffer, maxBytes = DEFAULT_MAX_BYTES) {
|
|
406
429
|
const separator = buffer.indexOf('\r\n\r\n');
|
|
407
430
|
if (separator === -1)
|
|
408
431
|
return undefined;
|
|
@@ -424,15 +447,24 @@ export function readResponse(buffer) {
|
|
|
424
447
|
const encoding = (headers['content-encoding'] ?? '').toLowerCase();
|
|
425
448
|
try {
|
|
426
449
|
if (encoding.includes('gzip'))
|
|
427
|
-
body = zlib.gunzipSync(body);
|
|
450
|
+
body = zlib.gunzipSync(body, { maxOutputLength: maxBytes });
|
|
428
451
|
else if (encoding.includes('deflate'))
|
|
429
|
-
body = zlib.inflateSync(body);
|
|
452
|
+
body = zlib.inflateSync(body, { maxOutputLength: maxBytes });
|
|
430
453
|
else if (encoding.includes('br'))
|
|
431
|
-
body = zlib.brotliDecompressSync(body);
|
|
454
|
+
body = zlib.brotliDecompressSync(body, { maxOutputLength: maxBytes });
|
|
432
455
|
}
|
|
433
|
-
catch {
|
|
434
|
-
//
|
|
456
|
+
catch (error) {
|
|
457
|
+
// If a decompression bomb hit the output ceiling, never fall back to the
|
|
458
|
+
// compressed bytes: they are not text. Keep a short explicit marker so the
|
|
459
|
+
// caller can see that the page was too large. A genuinely malformed or
|
|
460
|
+
// mislabelled body stays as-is, as before.
|
|
461
|
+
const code = error.code;
|
|
462
|
+
if (code === 'ERR_BUFFER_TOO_LARGE' || code === 'Z_BUF_ERROR') {
|
|
463
|
+
body = Buffer.from('[response body exceeded the decompression limit]');
|
|
464
|
+
}
|
|
435
465
|
}
|
|
466
|
+
if (body.length > maxBytes)
|
|
467
|
+
body = body.subarray(0, maxBytes);
|
|
436
468
|
return { status, headers, text: body.toString('utf8') };
|
|
437
469
|
}
|
|
438
470
|
/**
|
|
@@ -457,16 +489,23 @@ export async function proxyGet(url, options) {
|
|
|
457
489
|
const proxy = parseProxy(options.proxy);
|
|
458
490
|
const timeoutMs = options.timeoutMs !== undefined && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? options.timeoutMs : DEFAULT_TIMEOUT_MS;
|
|
459
491
|
const redirects = options.redirects !== undefined && Number.isInteger(options.redirects) ? options.redirects : DEFAULT_REDIRECTS;
|
|
492
|
+
const maxBytes = options.maxBytes !== undefined && Number.isFinite(options.maxBytes) && options.maxBytes > 0
|
|
493
|
+
? Math.floor(options.maxBytes)
|
|
494
|
+
: DEFAULT_MAX_BYTES;
|
|
460
495
|
let current = new URL(url);
|
|
461
496
|
let method = options.method === 'POST' ? 'POST' : 'GET';
|
|
462
497
|
let payload = method === 'POST' && typeof options.body === 'string' ? options.body : undefined;
|
|
463
498
|
for (let hop = 0; hop <= redirects; hop += 1) {
|
|
464
499
|
const secure = current.protocol === 'https:';
|
|
465
500
|
const port = Number(current.port || (secure ? 443 : 80));
|
|
466
|
-
const
|
|
501
|
+
const targetHost = current.hostname.replace(/^\[|\]$/g, '');
|
|
502
|
+
const raw = await openTunnel(proxy, targetHost, port, timeoutMs, options.signal);
|
|
467
503
|
let socket = raw;
|
|
468
504
|
if (secure) {
|
|
469
|
-
const
|
|
505
|
+
const servername = net.isIP(targetHost) === 0 ? targetHost : undefined;
|
|
506
|
+
// `host` is the identity Node checks the certificate against. Omitting it
|
|
507
|
+
// makes the check fall back to the proxy host for IP targets.
|
|
508
|
+
const upgraded = tls.connect({ socket: raw, host: targetHost, ...(servername === undefined ? {} : { servername }) });
|
|
470
509
|
// `once` also rejects on 'error', so a refused TLS handshake surfaces
|
|
471
510
|
// instead of becoming an unhandled socket event.
|
|
472
511
|
await once(upgraded, 'secureConnect');
|
|
@@ -485,7 +524,7 @@ export async function proxyGet(url, options) {
|
|
|
485
524
|
'\r\n' +
|
|
486
525
|
(payload ?? '');
|
|
487
526
|
// Follow one redirect hop at a time, re-opening the tunnel per hop.
|
|
488
|
-
const response = readResponse(await exchange(socket, head, timeoutMs, options.signal));
|
|
527
|
+
const response = readResponse(await exchange(socket, head, timeoutMs, options.signal, maxBytes), maxBytes);
|
|
489
528
|
const location = response?.headers.location;
|
|
490
529
|
if (response !== undefined && response.status >= 300 && response.status < 400 && location !== undefined && hop < redirects) {
|
|
491
530
|
// 307/308 preserve the method and body; the older statuses downgrade to GET.
|
package/lib/rank.js
CHANGED
|
@@ -360,7 +360,7 @@ function fuseEntries(entries, limit, options) {
|
|
|
360
360
|
weights.push(engineWeight(config.weights, entry.engineIndex, entry.host));
|
|
361
361
|
}
|
|
362
362
|
const rounds = lists.reduce((deepest, rows, index) => {
|
|
363
|
-
const weight = weights
|
|
363
|
+
const weight = slotWeight(weights, index);
|
|
364
364
|
if (weight === 0 || rows.length === 0)
|
|
365
365
|
return deepest;
|
|
366
366
|
return Math.max(deepest, Math.ceil(rows.length / weight));
|
|
@@ -377,7 +377,7 @@ function fuseEntries(entries, limit, options) {
|
|
|
377
377
|
const rows = lists[index];
|
|
378
378
|
if (rows === undefined)
|
|
379
379
|
continue;
|
|
380
|
-
const weight = weights
|
|
380
|
+
const weight = slotWeight(weights, index);
|
|
381
381
|
for (let offset = 0; offset < weight; offset += 1) {
|
|
382
382
|
const row = rows[round * weight + offset];
|
|
383
383
|
if (row === undefined)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fffangchenghao/dsh-web-search",
|
|
3
|
-
"version": "0.24.
|
|
3
|
+
"version": "0.24.3",
|
|
4
4
|
"description": "DSH bundle: key-free web search, page fetch and research for DeepSeek Harness; eight engines fused with RRF, 34 vertical sources, arXiv, Windows/macOS system-proxy support and a dependency-free MCP server.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -37,7 +37,9 @@
|
|
|
37
37
|
"docs/plugin-settings-form.md",
|
|
38
38
|
"docs/troubleshooting.md",
|
|
39
39
|
"docs/architecture.md",
|
|
40
|
-
"docs/engines.md"
|
|
40
|
+
"docs/engines.md",
|
|
41
|
+
"docs/images/bench.png",
|
|
42
|
+
"docs/images/bench.svg"
|
|
41
43
|
],
|
|
42
44
|
"scripts": {
|
|
43
45
|
"build": "tsc -p tsconfig.json",
|