@fffangchenghao/dsh-web-search 0.19.0 → 0.19.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +31 -0
- package/README.md +4 -4
- package/README.zh-CN.md +4 -4
- package/client.js +1 -1
- package/docs/rrf-ranking.md +10 -0
- package/lib/markdown.js +25 -3
- package/lib/transport.js +7 -0
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,36 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.19.2
|
|
4
|
+
|
|
5
|
+
### Fixed
|
|
6
|
+
- **`web_fetch` no longer returns only a page's footer.** The reading-root picker
|
|
7
|
+
preferred the densest container, and a short comment form can be far denser
|
|
8
|
+
than an article wrapped in `<div>` markup. On PostgreSQL's documentation that
|
|
9
|
+
returned 307 characters (`Submit correction…`) instead of the chapter. The
|
|
10
|
+
picker now falls back to the largest container that holds a real share of the
|
|
11
|
+
page text, and then to the body, before accepting a fragment. The same URL now
|
|
12
|
+
returns the chapter introduction and body (6,078 characters in the text
|
|
13
|
+
format); GitHub and PyTorch pages keep their content. One regression test.
|
|
14
|
+
|
|
15
|
+
## 0.19.1
|
|
16
|
+
|
|
17
|
+
### Changed
|
|
18
|
+
- **The core-word retry now runs only when the first answer is unusable.**
|
|
19
|
+
`transport` checks the original rows against the relevance gate before
|
|
20
|
+
spending a second request. Queries whose `cn.bing` answer already has a
|
|
21
|
+
relevant row (`pip 安装 requests 库`, `React useEffect 用法`,
|
|
22
|
+
`PostgreSQL 逻辑复制 配置`) keep their original latency; the dictionary-page
|
|
23
|
+
failures (`东数西算 八个国家枢纽节点 定位`, `如何配置 postgres 逻辑复制`) still
|
|
24
|
+
retry and recover. Live comparison for the three working queries: rows and
|
|
25
|
+
expected-domain hits unchanged, p50 latency back to the pre-variant value
|
|
26
|
+
(React 2.8 s → 0.6 s in the sample).
|
|
27
|
+
- **Regression evidence for 0.19.0/0.19.1.** Replaying the 20-query corpus
|
|
28
|
+
against the 0.15.0 result shows delta 0 on P@8, R@8, MRR@8, zero-hit and every
|
|
29
|
+
category/language slice; the five-query long corpus is unchanged at
|
|
30
|
+
`0.4500 / 1.0000 / 0.7667`. Only 3 of the 20 baseline queries trigger a
|
|
31
|
+
variant at all (`zh-postgres-logical`, `zh-pip-requests`,
|
|
32
|
+
`zh-react-useeffect`), and after the guard none of them does.
|
|
33
|
+
|
|
3
34
|
## 0.19.0
|
|
4
35
|
|
|
5
36
|
### Added
|
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
<p align="center">
|
|
11
11
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm version"></a>
|
|
12
|
-
<img src="https://img.shields.io/badge/tests-
|
|
12
|
+
<img src="https://img.shields.io/badge/tests-499%20passing-brightgreen" alt="tests">
|
|
13
13
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="engines">
|
|
14
14
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
15
15
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
- **No API key, eight engines.** AnySearch, CN Bing, DuckDuckGo HTML, www Bing, DuckDuckGo Lite and 360 Search are scraped directly; Exa joins the primary tier through its keyless MCP endpoint, and Firecrawl's search API is the measured fallback. The plugin works out of the box.
|
|
22
22
|
- **The ranking is measured, not asserted.** Cross-engine reciprocal-rank fusion with authority and freshness boosts, a 20-query offline benchmark, and an 81-configuration parameter sweep — the only search plugin here that publishes both its metric and its confidence intervals. See [How the ranking works](docs/rrf-ranking.md).
|
|
23
23
|
- **Fetching admits failure.** Binary documents and empty 200 responses are rejected by name (`SEARCH_UNSUPPORTED`, `SEARCH_EMPTY`) instead of being returned as noise; the direct → tunnel → render → extract ladder shares one 25 s budget and stops when there is no time left. The budget is per request — there is a regression test for exactly that.
|
|
24
|
-
- **
|
|
24
|
+
- **500 tests across 34 files**, a committed `lib/`, declared `engines.dsh` / `engines.node`, and a [SECURITY.md](SECURITY.md) that lists every process, host and file the plugin touches.
|
|
25
25
|
- **Bilingual documentation.** This README and [中文说明](README.zh-CN.md) are maintained side by side.
|
|
26
26
|
|
|
27
27
|

|
|
@@ -37,7 +37,7 @@ The web-search corner of the DSH plugin ecosystem is crowded, and most rivals ar
|
|
|
37
37
|
| Key-free engines | 8 | ~30 | 7 channels (+X) | 1 official API |
|
|
38
38
|
| Dedicated vertical sources | 34 (8 on by default) | 41 | — | — |
|
|
39
39
|
| Model-facing tools | 6 | ~10 | 3 | 5 |
|
|
40
|
-
| Test suite | 34 files /
|
|
40
|
+
| Test suite | 34 files / 500 cases | none | 29 files | 6 files |
|
|
41
41
|
| Ranking benchmark | 20 queries + 81-config sweep | — | — | — |
|
|
42
42
|
| Store status | source contract satisfied, awaiting review | unlisted | no entry | blocked (review-required) |
|
|
43
43
|
|
|
@@ -242,7 +242,7 @@ Each line is the plain version first, the technical note second. The technical n
|
|
|
242
242
|
|
|
243
243
|
5. **Chinese and English are handled separately, and Chinese rows must talk about the question.**
|
|
244
244
|
English queries lead with global sources, Chinese queries with Chinese sources, so "how to …" is not answered with a dictionary page for the word `how`. A scraped Chinese engine can still fall back to single-character dictionary and company pages for a long query; those rows are dropped before fusion unless they share at least two words with the question, and `.gov.cn` / `.edu.cn` official pages get the authority boost.
|
|
245
|
-
*Technical: `lead = cn` when the query contains CJK characters, `lead = global` for plain ASCII; the other region only appends when the leading one comes up short. `src/relevance.ts` splits a CJK query into character bigrams (an ASCII word counts double), requires two weight units of overlap with title, snippet or URL, and returns English queries untouched. A whole-phrase engine that misses a long query is asked once more with its core words (`src/variants.ts`): `东数西算 八个国家枢纽节点 定位` also asks `东数西算`, `如何配置 postgres 逻辑复制` also asks `postgres 逻辑复制`. Measured on the first query, `cn.bing` answered with ten dictionary/company rows before this; it now contributes the NDRC and gov.cn articles, and the fused list has eight relevant rows.*
|
|
245
|
+
*Technical: `lead = cn` when the query contains CJK characters, `lead = global` for plain ASCII; the other region only appends when the leading one comes up short. `src/relevance.ts` splits a CJK query into character bigrams (an ASCII word counts double), requires two weight units of overlap with title, snippet or URL, and returns English queries untouched. A whole-phrase engine that misses a long query is asked once more with its core words (`src/variants.ts`) — but only when its first answer contains no row that survives the relevance gate, so a working short query pays neither the request nor the latency: `东数西算 八个国家枢纽节点 定位` also asks `东数西算`, `如何配置 postgres 逻辑复制` also asks `postgres 逻辑复制`. Measured on the first query, `cn.bing` answered with ten dictionary/company rows before this; it now contributes the NDRC and gov.cn articles, and the fused list has eight relevant rows.*
|
|
246
246
|
|
|
247
247
|
6. **You choose speed or completeness.**
|
|
248
248
|
`auto` waits up to 6 s for English and 3.5 s for Chinese; `fast` takes whoever answers first; `all` waits for everyone, up to 10 s.
|
package/README.zh-CN.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
<p align="center">
|
|
11
11
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm 版本"></a>
|
|
12
|
-
<img src="https://img.shields.io/badge/tests-
|
|
12
|
+
<img src="https://img.shields.io/badge/tests-499%20passing-brightgreen" alt="测试">
|
|
13
13
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="引擎">
|
|
14
14
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
15
15
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -21,7 +21,7 @@
|
|
|
21
21
|
- **不要 API key,八个引擎。** AnySearch、必应中国、DuckDuckGo HTML、必应国际、DuckDuckGo Lite、360 搜索直接抓取;Exa 走它的免密钥 MCP 通道加入主引擎,Firecrawl 的搜索 API 作为实测后的兜底。装好就能用。
|
|
22
22
|
- **排序是量出来的,不是写出来的。** 跨引擎 RRF 融合,叠加权威度与时效加权;20 条离线基准 + 81 组参数扫描,是这类插件里唯一同时公开指标和置信区间的。方法见 [排序是怎么做的](docs/rrf-ranking.md)。
|
|
23
23
|
- **抓取会承认失败。** 二进制文档和「200 但正文为空」会被明确拒绝(`SEARCH_UNSUPPORTED`、`SEARCH_EMPTY`),而不是当噪音返回;direct → tunnel → render → extract 四级阶梯共享 25 秒预算,预算不足就不再起新的一级。预算是「每次请求」而不是「插件启动时」,这点有回归测试守着。
|
|
24
|
-
- **34 个测试文件、
|
|
24
|
+
- **34 个测试文件、500 个用例**,`lib/` 产物入库,声明了 `engines.dsh` / `engines.node`,[SECURITY.md](SECURITY.md) 逐项列出插件会碰的进程、主机和文件。
|
|
25
25
|
- **双语文档。** 本文件和 [English README](README.md) 同步维护。
|
|
26
26
|
|
|
27
27
|

|
|
@@ -37,7 +37,7 @@ DSH 插件生态里的「联网搜索」这一块已经很挤,大多数竞品
|
|
|
37
37
|
| 免 key 引擎 | 8 | ~30 | 7 条通道(含 X) | 1 个官方 API |
|
|
38
38
|
| 专门数据源 | 34(默认开 8 个)| 41 | — | — |
|
|
39
39
|
| 面向模型的工具 | 6 | ~10 | 3 | 5 |
|
|
40
|
-
| 测试 | 34 文件 /
|
|
40
|
+
| 测试 | 34 文件 / 500 用例 | 无 | 29 文件 | 6 文件 |
|
|
41
41
|
| 排序基准 | 20 查询 + 81 组扫描 | — | — | — |
|
|
42
42
|
| 商店状态 | 源码契约已满足,待复核 | unlisted | 无条目 | blocked(review-required) |
|
|
43
43
|
|
|
@@ -242,7 +242,7 @@ Harness 里有个叫 `web` 的**服务**,它自己不会搜索,只负责「
|
|
|
242
242
|
|
|
243
243
|
5. **中英文分开处理,而且中文结果必须真的在讲这个问题。**
|
|
244
244
|
英文问题先看国外来源,中文问题先看中文来源,避免「how to …」被某个中文引擎按单词 `how` 匹配成词典页。中文长查询仍可能被引擎退化成单字词典页、公司页;这些行在融合前会被丢掉,除非标题/摘要/URL 与问题至少重合两个词;`.gov.cn` / `.edu.cn` 官方页面另有权威度加权。
|
|
245
|
-
*技术补充:查询含 CJK 字符时 `lead = cn`,纯 ASCII 时 `lead = global`;非主导区只在主导区凑不够条数时才追加。`src/relevance.ts` 把中文查询切成二字 bigram(ASCII 单词权重翻倍),要求与标题、摘要或 URL 至少两个权重单位的重合;英文查询原样放行。对整串匹配型引擎还会用核心词补问一次(`src/variants.ts
|
|
245
|
+
*技术补充:查询含 CJK 字符时 `lead = cn`,纯 ASCII 时 `lead = global`;非主导区只在主导区凑不够条数时才追加。`src/relevance.ts` 把中文查询切成二字 bigram(ASCII 单词权重翻倍),要求与标题、摘要或 URL 至少两个权重单位的重合;英文查询原样放行。对整串匹配型引擎还会用核心词补问一次(`src/variants.ts`),但只在原答案没有任何一条能通过相关性门槛时才补问——短查询正常工作时不付这次请求和延迟:「东数西算 八个国家枢纽节点 定位」会补问「东数西算」,「如何配置 postgres 逻辑复制」会补问「postgres 逻辑复制」。实测第一条查询:`cn.bing` 原来返回十条词典/公司页,现在能贡献发改委和 gov.cn 文章,融合结果八条全部相关。*
|
|
246
246
|
|
|
247
247
|
6. **快慢可以选。**
|
|
248
248
|
默认 `auto`:英文最多等 6 秒、中文最多等 3.5 秒;`fast` 谁先回就用谁;`all` 等所有引擎,最多 10 秒。
|
package/client.js
CHANGED
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
var ENTRY = 'web-search-bing';
|
|
48
48
|
var PACKAGE_NAME = 'dsh-web-search-bing';
|
|
49
49
|
/** Keep in step with package.json; compared with the npm registry. */
|
|
50
|
-
var VERSION = '0.19.
|
|
50
|
+
var VERSION = '0.19.2';
|
|
51
51
|
/** Snapshot used while no configuration service is reachable. */
|
|
52
52
|
var EMPTY_STATE = { status: 'unavailable', value: undefined, base: undefined, user: undefined, revision: undefined, writable: false };
|
|
53
53
|
|
package/docs/rrf-ranking.md
CHANGED
|
@@ -168,6 +168,16 @@ fused rows after the variant retry, against zero usable rows before it for the
|
|
|
168
168
|
policy query. The corpus is deliberately separate so the 20-query baseline in
|
|
169
169
|
`bench/queries.json` stays comparable.
|
|
170
170
|
|
|
171
|
+
## Retry cost guard
|
|
172
|
+
|
|
173
|
+
The core-word retry is not free: it is one more request to the same engine.
|
|
174
|
+
`src/transport.ts` therefore checks the first answer with the same relevance
|
|
175
|
+
gate before retrying. Only an answer with zero surviving rows triggers the
|
|
176
|
+
variant; `pip 安装 requests 库` and `React useEffect 用法` keep their original
|
|
177
|
+
latency (measured live: 2.7 s → 3.5 s and 1.5 s → 2.8 s without the guard,
|
|
178
|
+
1.5 s and 0.6 s with it), while the dictionary-page failures still retry and
|
|
179
|
+
recover. The 20-query replay compare is delta 0 on P@8, R@8 and MRR@8.
|
|
180
|
+
|
|
171
181
|
## Known limits
|
|
172
182
|
|
|
173
183
|
- **Relative Chinese time words.** A query like "上周发布的那版" carries its time
|
package/lib/markdown.js
CHANGED
|
@@ -423,6 +423,21 @@ function densestContainer(body) {
|
|
|
423
423
|
});
|
|
424
424
|
return best;
|
|
425
425
|
}
|
|
426
|
+
/** Largest container by visible text, used when density picks a stub. */
|
|
427
|
+
function largestContainer(body) {
|
|
428
|
+
let best = body;
|
|
429
|
+
let bestCharacters = characterCount(body);
|
|
430
|
+
walkElements(body, (node) => {
|
|
431
|
+
if (!CONTAINER_TAGS.has(node.tag))
|
|
432
|
+
return;
|
|
433
|
+
const characters = characterCount(node);
|
|
434
|
+
if (characters > bestCharacters) {
|
|
435
|
+
best = node;
|
|
436
|
+
bestCharacters = characters;
|
|
437
|
+
}
|
|
438
|
+
});
|
|
439
|
+
return best;
|
|
440
|
+
}
|
|
426
441
|
/** Pick the reading root: article, main, then the densest container. */
|
|
427
442
|
function pickMain(root) {
|
|
428
443
|
const body = findElement(root, (node) => node.tag === 'body') ?? root;
|
|
@@ -452,10 +467,17 @@ function pickMain(root) {
|
|
|
452
467
|
}
|
|
453
468
|
if (characterCount(densest) >= floor)
|
|
454
469
|
return densest;
|
|
455
|
-
//
|
|
456
|
-
|
|
470
|
+
// Density alone can crown a short, dense stub — a comment form or a footer —
|
|
471
|
+
// while the prose sits in a container with more markup and fewer characters
|
|
472
|
+
// per tag. Prefer a container that still holds a real share of the page text,
|
|
473
|
+
// then fall back to the body when every candidate is a fragment.
|
|
474
|
+
const largest = largestContainer(body);
|
|
475
|
+
if (characterCount(largest) >= floor)
|
|
476
|
+
return largest;
|
|
477
|
+
const ranked = [article, main, densest, largest].filter((node) => node !== undefined);
|
|
457
478
|
ranked.sort((left, right) => characterCount(right) - characterCount(left));
|
|
458
|
-
|
|
479
|
+
const best = ranked[0] ?? body;
|
|
480
|
+
return characterCount(best) * 2 < characterCount(body) ? body : best;
|
|
459
481
|
}
|
|
460
482
|
/** Replace a code fragment with a placeholder so clean-up cannot touch it. */
|
|
461
483
|
function stashCode(ctx, text) {
|
package/lib/transport.js
CHANGED
|
@@ -2,6 +2,7 @@ import { SearchError, parseRetryAfter } from "./errors.js";
|
|
|
2
2
|
import { proxyGet } from "./proxy.js";
|
|
3
3
|
import { echoesQuery } from "./parsers.js";
|
|
4
4
|
import { gateFor } from "./semaphore.js";
|
|
5
|
+
import { isRelevant, queryProfile } from "./relevance.js";
|
|
5
6
|
/** Markers of an interstitial anti-bot page, narrow enough not to hit prose. */
|
|
6
7
|
const BOT_WALL_MARKERS = ['qcaptcha', '访问异常', 'unusual traffic', 'verify you are human', '安全验证'];
|
|
7
8
|
/**
|
|
@@ -169,6 +170,12 @@ export async function searchHost(host, query, signal, timeoutMs = 15000, proxy,
|
|
|
169
170
|
const variants = host.variants?.(query) ?? [];
|
|
170
171
|
if (variants.length === 0 || signal?.aborted === true)
|
|
171
172
|
return result;
|
|
173
|
+
// Only pay for the extra request when the engine's first answer is unusable.
|
|
174
|
+
// A working short query (`pip 安装 requests 库`, `React useEffect 用法`) has
|
|
175
|
+
// relevant rows already, and asking again doubled its latency for nothing.
|
|
176
|
+
const profile = queryProfile(query);
|
|
177
|
+
if (result.ok && result.sources.some((source) => isRelevant(source, profile)))
|
|
178
|
+
return result;
|
|
172
179
|
const merged = result.ok ? [...result.sources] : [];
|
|
173
180
|
const seen = new Set(merged.map((source) => source.url));
|
|
174
181
|
for (const variant of variants) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fffangchenghao/dsh-web-search",
|
|
3
|
-
"version": "0.19.
|
|
3
|
+
"version": "0.19.2",
|
|
4
4
|
"description": "DSH bundle: a key-free web search provider. Leads with the anonymous AnySearch API and falls back to public result pages (DuckDuckGo, Bing, 360), merged by reciprocal-rank fusion with a bilingual region policy, an engine cooldown and a TTL cache.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|