@fffangchenghao/dsh-web-search 0.24.3 → 0.24.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +5 -5
- package/README.zh-CN.md +4 -4
- package/client.js +1 -1
- package/docs/comparison.md +1 -1
- package/docs/images/web-tools-demo.svg +1 -1
- package/docs/latency-and-sources.md +1 -1
- package/docs/store-evidence.md +1 -1
- package/lib/cache-persist.js +9 -1
- package/lib/cache.js +15 -6
- package/lib/domain-tool.d.ts +2 -0
- package/lib/domain-tool.js +7 -1
- package/lib/engine-test.js +16 -2
- package/lib/index.js +9 -4
- package/lib/official-release.js +6 -0
- package/lib/official-sources.js +4 -2
- package/lib/official-support.js +4 -0
- package/lib/parsers.js +38 -21
- package/lib/platform-tool.d.ts +1 -0
- package/lib/platform-tool.js +7 -1
- package/lib/platforms.js +65 -2
- package/lib/proxy.js +4 -0
- package/lib/read-page.js +30 -11
- package/lib/render.js +117 -22
- package/lib/research.d.ts +5 -0
- package/lib/research.js +76 -12
- package/lib/search-many.js +16 -2
- package/lib/tool-suite.d.ts +2 -0
- package/lib/tool-suite.js +14 -1
- package/lib/transport.js +3 -2
- package/lib/verticals.js +11 -5
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,68 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.24.5
|
|
4
|
+
|
|
5
|
+
Completed the delayed GPT-6-astra `xhigh` audit batch: tool registration,
|
|
6
|
+
rendering, advanced/search_many/engine_test, domain and platform tools.
|
|
7
|
+
|
|
8
|
+
### Fixed
|
|
9
|
+
- **`web_tools` ignored `limit` for `advanced_search`.** The unified dispatcher
|
|
10
|
+
now maps `limit` to `maxResults` when `maxResults` is absent; an explicit
|
|
11
|
+
`maxResults` still wins. Its own parser still validates the range.
|
|
12
|
+
- **Tool disposers were dropped by three registrars.** `registerUnifiedTool`,
|
|
13
|
+
`registerDomainTool` and `registerPlatformTool` now forward the host's
|
|
14
|
+
`tools.register()` disposer through an optional `onRegister` callback, and
|
|
15
|
+
legacy fallback registrations are wired into `onDispose` too.
|
|
16
|
+
- **`readableTextLength` was quadratic on repeated unclosed raw-text tags.**
|
|
17
|
+
The regex chain is replaced with a single forward scanner; repeated unclosed
|
|
18
|
+
`<head>`/`<script>` now stops at the end instead of rescanning the suffix.
|
|
19
|
+
- **Render cancellation covered only browser launch.** `renderWithPlaywright`
|
|
20
|
+
now has one total deadline covering launch, `newPage`, navigation, idle wait
|
|
21
|
+
and `content`; every phase races an abort signal, late handles are closed,
|
|
22
|
+
and page/browser cleanup runs concurrently with a short cap.
|
|
23
|
+
- **Batch tools swallowed cancellation.** `search_many` and `engine_test` now
|
|
24
|
+
reject on a caller abort, stop pulling queued work, and never start a
|
|
25
|
+
pre-aborted batch.
|
|
26
|
+
|
|
27
|
+
**Test suite: 717 tests across 56 files** (716 pass, 1 skipped).
|
|
28
|
+
|
|
29
|
+
## 0.24.4
|
|
30
|
+
|
|
31
|
+
Second independent audit, this time with `Vendor2/GPT-6-astra` at reasoning
|
|
32
|
+
effort `xhigh`. Search and ranking inputs are unchanged except where stated.
|
|
33
|
+
|
|
34
|
+
### Fixed
|
|
35
|
+
- **A failed proxy-first route no longer removes the direct fallback.** After a
|
|
36
|
+
tunnel error, `searchHost` now retries the same engine directly instead of
|
|
37
|
+
treating `undefined` as "no alternate route".
|
|
38
|
+
- **Platform searches have one deadline for direct, alternate and fallback.**
|
|
39
|
+
A fallback that ignores its signal can no longer hang the whole `Promise.all`;
|
|
40
|
+
caller cancellation still propagates as `SEARCH_ABORTED`.
|
|
41
|
+
- **Already-cancelled requests do not start network work.** The official
|
|
42
|
+
release/support readers and the proxy transport check the signal before the
|
|
43
|
+
first request; the research tool also refuses to start a search after cancel.
|
|
44
|
+
- **Scoped npm lookups verify the package identity.** The official resolver asks
|
|
45
|
+
for several search rows and accepts only an exact name or `name@version`
|
|
46
|
+
title instead of trusting the first fuzzy hit.
|
|
47
|
+
- **OpenAlex abstracts keep repeated words.** Every inverted-index position is
|
|
48
|
+
rebuilt, not only the first.
|
|
49
|
+
- **arXiv date windows preserve explicit field queries.** `ti:`, `cat:`, boolean
|
|
50
|
+
expressions and quotes are passed through; only the sort parameters are added.
|
|
51
|
+
- **Tracker unwrapping is host/path scoped.** A normal URL containing `uddg=` or
|
|
52
|
+
`u=a1...` is no longer rewritten as if it came from Bing or DuckDuckGo.
|
|
53
|
+
- **Exa same-line `Highlights:` text is kept.** The inline value now becomes the
|
|
54
|
+
snippet instead of being discarded.
|
|
55
|
+
- **The search-cache key is an unambiguous JSON tuple.** Optional field values
|
|
56
|
+
and the query can no longer collide through delimiter prefixes.
|
|
57
|
+
- **Research enforces its 45 s budget across search and fetch**, reports search
|
|
58
|
+
failures in the evidence pack, and hard-caps the selected excerpt.
|
|
59
|
+
- **`read_page` chunk index/count follows surrogate-safe boundaries**, so a
|
|
60
|
+
cut emoji cannot make pagination repeat or skip a chunk.
|
|
61
|
+
- **A cache snapshot that cannot fit an empty envelope is not written**, leaving
|
|
62
|
+
the previous file intact rather than replacing it with over-budget data.
|
|
63
|
+
|
|
64
|
+
**Test suite: 711 tests across 56 files** (710 pass, 1 skipped).
|
|
65
|
+
|
|
3
66
|
## 0.24.3
|
|
4
67
|
|
|
5
68
|
Audit fixes from an independent Claude Fable 5.1 review (reasoning effort
|
package/README.md
CHANGED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
|
|
11
11
|
<p align="center">
|
|
12
12
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm version"></a>
|
|
13
|
-
<img src="https://img.shields.io/badge/tests-
|
|
13
|
+
<img src="https://img.shields.io/badge/tests-717%20tests-brightgreen" alt="tests">
|
|
14
14
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="engines">
|
|
15
15
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
16
16
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -28,7 +28,7 @@
|
|
|
28
28
|
- **Measured in public, including the losses.** A 30-query bilingual comparison records availability, latency, unique domains, official-domain presence and every failure against keyless Firecrawl and anonymous Exa. The follow-up documents the Exa architecture difference, the 3 s CJK budget and the Rust official-source lookup. Exa is faster here; this plugin returns more distinct domains. Both test arms were keyless. The method and raw results are in [docs/comparison.md](docs/comparison.md) and a [public gist](https://gist.github.com/fangchenghao/3db9838fa98928435c1c2df38202c373).
|
|
29
29
|
- **Fetching admits failure.** Binary documents and empty 200 responses are rejected by name (`SEARCH_UNSUPPORTED`, `SEARCH_EMPTY`) instead of being returned as noise; arXiv PDF links are the deliberate exception and are read through the HTML full-text view; the direct → tunnel → render → extract ladder shares one 25 s budget and stops when there is no time left. The budget is per request — there is a regression test for exactly that.
|
|
30
30
|
- **Windows is a first-class host.** Pure ESM, no native addon or install script; the system proxy is discovered from Windows Internet Settings, and the full suite runs on a `windows-latest` CI runner. See [Install on Windows](#windows).
|
|
31
|
-
- **
|
|
31
|
+
- **717 tests across 56 files**, a committed `lib/`, declared `engines.dsh` / `engines.node`, and a [SECURITY.md](SECURITY.md) that lists every process, host and file the plugin touches.
|
|
32
32
|
- **Bilingual documentation.** This README and [中文说明](README.zh-CN.md) are maintained side by side.
|
|
33
33
|
|
|
34
34
|

|
|
@@ -45,7 +45,7 @@ The web-search corner of the DSH plugin ecosystem is crowded. A dated snapshot
|
|
|
45
45
|
| Key-free path | 8 engines, all key-free | ~30-engine free/paid chain | 5 defaults (seam, Exa, DDG, Bing, Jina) | Firecrawl + Keenable keyless defaults | 1 anonymous AnySearch API |
|
|
46
46
|
| Dedicated vertical sources | 34 (8 on by default) | 41 | platform registry, not a vertical catalog | web + X, no vertical catalog | vertical API behind AnySearch |
|
|
47
47
|
| Model-facing tools | 1 (`web_tools`, 8 actions) + 4 MCP tools | ~10 | 2 (`web_index` / `web_call`, 20 actions) | native search/fetch + skill | 5 |
|
|
48
|
-
| Tests | 56 files /
|
|
48
|
+
| Tests | 56 files / 717 cases, locally verified | no test script in `package.json` | 774 unit + 94 bench, self-reported | Vitest configured; count not published | Vitest configured; count not published |
|
|
49
49
|
| GitHub stars | 0 | 620 | 77 | 836 | 663 |
|
|
50
50
|
| npm downloads, last month | 1,087 | 52,436 | 6,420 | 43,151 | 9,251 |
|
|
51
51
|
| npm unpacked size | 1.2 MB | 0.70 MB | 3.05 MB | 0.51 MB | 0.27 MB |
|
|
@@ -56,7 +56,7 @@ read is:
|
|
|
56
56
|
- **Feature depth:** upper-middle. `dsh-free-search` has more engines and
|
|
57
57
|
verticals; `dsh-web-search-pro` has a deeper evidence/ranking pipeline and far
|
|
58
58
|
more platform backends; `modsearch` is the stronger cross-host bridge.
|
|
59
|
-
- **Engineering and verification:** near the top.
|
|
59
|
+
- **Engineering and verification:** near the top. 717 local tests, a replayable
|
|
60
60
|
ranking benchmark, declared security boundaries and one runtime dependency
|
|
61
61
|
put it alongside the best-maintained entries, while remaining much smaller
|
|
62
62
|
than the Playwright-heavy option.
|
|
@@ -65,7 +65,7 @@ read is:
|
|
|
65
65
|
common in the other plugins.
|
|
66
66
|
- **Adoption and maturity:** bottom third of this sample. At the 2026-10-25
|
|
67
67
|
snapshot the repository was four days old, had no stars and little
|
|
68
|
-
independent adoption; the current npm release is 0.24.
|
|
68
|
+
independent adoption; the current npm release is 0.24.5.
|
|
69
69
|
- **Tool-surface economy:** now competitive. 0.20.x exposes one `web_tools`
|
|
70
70
|
entry with eight actions, close to `dsh-web-search-pro`'s two tools with 20
|
|
71
71
|
actions; the legacy eight-name surface remains opt-in for old prompts.
|
package/README.zh-CN.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
<p align="center">
|
|
11
11
|
<a href="https://www.npmjs.com/package/@fffangchenghao/dsh-web-search"><img src="https://img.shields.io/npm/v/@fffangchenghao/dsh-web-search.svg" alt="npm 版本"></a>
|
|
12
|
-
<img src="https://img.shields.io/badge/tests-
|
|
12
|
+
<img src="https://img.shields.io/badge/tests-717%20tests-brightgreen" alt="测试">
|
|
13
13
|
<img src="https://img.shields.io/badge/engines-8%2C%20zero%20API%20key-brightgreen" alt="引擎">
|
|
14
14
|
<img src="https://img.shields.io/badge/node-%3E%3D22.18-blue" alt="node">
|
|
15
15
|
<img src="https://img.shields.io/badge/dsh-%3E%3D0.2.1--rc.2%20%3C0.3.0-blueviolet" alt="dsh">
|
|
@@ -27,7 +27,7 @@
|
|
|
27
27
|
- **公开测量,也公开输的地方。** 30 条中英查询对比记录可用性、延迟、独立域名、官方域名命中与每一条失败;对比对象是免 key Firecrawl 和匿名 Exa。这里 Exa 更快,本插件返回的独立域名更多;两条对比路径都未使用 key。方法和原始结果在 [docs/comparison.md](docs/comparison.md);Exa 架构差异、中文 3 秒预算和 Rust 官方源见 [延迟与官方源](docs/latency-and-sources.md)。
|
|
28
28
|
- **抓取会承认失败。** 二进制文档和「200 但正文为空」会被明确拒绝(`SEARCH_UNSUPPORTED`、`SEARCH_EMPTY`),而不是当噪音返回;arXiv 的 PDF 链接是有意保留的例外,会改读 HTML 全文;direct → tunnel → render → extract 四级阶梯共享 25 秒预算,预算不足就不再起新的一级。预算是「每次请求」而不是「插件启动时」,这点有回归测试守着。
|
|
29
29
|
- **Windows 是一等宿主。** 纯 ESM,没有原生扩展或安装脚本;会自动读取 Windows 的 Internet 设置代理,完整测试套件也在 `windows-latest` CI 上跑通。安装见[Windows 小节](#windows)。
|
|
30
|
-
- **56 个测试文件、
|
|
30
|
+
- **56 个测试文件、717 个用例**,`lib/` 产物入库,声明了 `engines.dsh` / `engines.node`,[SECURITY.md](SECURITY.md) 逐项列出插件会碰的进程、主机和文件。
|
|
31
31
|
- **双语文档。** 本文件和 [English README](README.md) 同步维护。
|
|
32
32
|
|
|
33
33
|

|
|
@@ -54,11 +54,11 @@ DSH 插件生态里的「联网搜索」已经很挤。下面是一张 2026-10-2
|
|
|
54
54
|
- **功能深度:中上。** `dsh-free-search` 引擎和垂直源更多;
|
|
55
55
|
`dsh-web-search-pro` 的证据/排序管线和平台后端更深;`modsearch` 的跨宿主
|
|
56
56
|
能力更强。
|
|
57
|
-
- **工程质量与可验证性:接近第一梯队。**
|
|
57
|
+
- **工程质量与可验证性:接近第一梯队。** 717 个本地测试、可重放的排序基准、
|
|
58
58
|
明确的安全边界、只有 1 个运行时依赖,同时体积远小于 Playwright 重型方案。
|
|
59
59
|
- **免 key + 官方源纠错:最强卖点。** Chrome、PostgreSQL、AI Act 的官方路径
|
|
60
60
|
和 34 个垂直源,不是其他插件普遍具备的。
|
|
61
|
-
- **采用度与成熟度:这个样本里的后段。** 2026-10-25 快照时 GitHub 仓库刚创建 4 天、0 stars,还缺少独立采用;当前 npm 版本是 0.24.
|
|
61
|
+
- **采用度与成熟度:这个样本里的后段。** 2026-10-25 快照时 GitHub 仓库刚创建 4 天、0 stars,还缺少独立采用;当前 npm 版本是 0.24.5。
|
|
62
62
|
- **工具面的经济性:现在有竞争力。** 0.20.x 只暴露一个 `web_tools` + 8 个
|
|
63
63
|
action,接近 `dsh-web-search-pro` 的 2 工具 + 20 动作;旧的八个工具名保留为
|
|
64
64
|
可选兼容模式。
|
package/client.js
CHANGED
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
var ENTRY = 'web-search-bing';
|
|
75
75
|
var PACKAGE_NAME = 'dsh-web-search-bing';
|
|
76
76
|
/** Keep in step with package.json; compared with the npm registry. */
|
|
77
|
-
var VERSION = '0.24.
|
|
77
|
+
var VERSION = '0.24.5';
|
|
78
78
|
/** Snapshot used while no configuration service is reachable. */
|
|
79
79
|
var EMPTY_STATE = { status: 'unavailable', value: undefined, base: undefined, user: undefined, revision: undefined, writable: false };
|
|
80
80
|
|
package/docs/comparison.md
CHANGED
|
@@ -6,7 +6,7 @@ whether a provider answers at all, how long it takes, how many distinct domains
|
|
|
6
6
|
it returns, and whether a page on a known official domain appears in the top 8.
|
|
7
7
|
The failures are part of the result.
|
|
8
8
|
|
|
9
|
-
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1 and 0.24.2 are documentation/metadata patches; 0.24.3
|
|
9
|
+
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1 and 0.24.2 are documentation/metadata patches; 0.24.3–0.24.5 fix fetch/proxy/markdown/cache/tool reliability without changing the measured search path. These patches do not change the search, ranking or source-adapter path measured below, so the 0.23.0 numbers remain the latest comparison.
|
|
10
10
|
|
|
11
11
|
## Method
|
|
12
12
|
|
|
@@ -52,7 +52,7 @@
|
|
|
52
52
|
<rect x="70" y="620" width="330" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
53
53
|
<text x="92" y="651" class="m" font-size="13">Engines</text><text x="92" y="677" class="t" font-size="20" font-weight="700">8 key-free paths</text>
|
|
54
54
|
<rect x="420" y="620" width="330" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
55
|
-
<text x="442" y="651" class="m" font-size="13">Tests</text><text x="442" y="677" class="t" font-size="20" font-weight="700">
|
|
55
|
+
<text x="442" y="651" class="m" font-size="13">Tests</text><text x="442" y="677" class="t" font-size="20" font-weight="700">717 local cases</text>
|
|
56
56
|
<rect x="770" y="620" width="360" height="76" rx="14" fill="#101d31" stroke="#26364f"/>
|
|
57
57
|
<text x="792" y="651" class="m" font-size="13">Cross-agent</text><text x="792" y="677" class="t" font-size="20" font-weight="700">MCP stdio server</text>
|
|
58
58
|
</g>
|
|
@@ -6,7 +6,7 @@ This note answers three questions from the public 30-query comparison:
|
|
|
6
6
|
2. Can the AnySearch tail and the Chinese Rust miss be fixed?
|
|
7
7
|
3. Does the fix require a vector database or a different retrieval model?
|
|
8
8
|
|
|
9
|
-
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1 and 0.24.2 are documentation/metadata patches; 0.24.3
|
|
9
|
+
> **Version note (2026-11-02):** 0.24.0 added Windows system-proxy discovery, and 0.24.1 and 0.24.2 are documentation/metadata patches; 0.24.3–0.24.5 fix fetch/proxy/markdown/cache/tool reliability without changing the measured search path. The search path is unchanged, so these 0.23.0 measurements still describe 0.24.x.
|
|
10
10
|
|
|
11
11
|
The short answer: Exa is a different architecture, not a faster RRF loop. This
|
|
12
12
|
plugin can borrow tail-tolerance and source-specific facts, but copying Exa's
|
package/docs/store-evidence.md
CHANGED
|
@@ -29,7 +29,7 @@ Only `0.2.1-rc.2` is declared, because that is the version actually exercised.
|
|
|
29
29
|
`mktemp -d`; the real `~/.dsh` was never written to. Bundled pnpm 11.28.5.
|
|
30
30
|
- **Windows CI:** GitHub Actions `windows-latest`, Node 24. Install, typecheck,
|
|
31
31
|
the full test suite, build and the entry-point import all pass. Run:
|
|
32
|
-
https://github.com/fangchenghao/dsh-web-search/actions/runs/
|
|
32
|
+
https://github.com/fangchenghao/dsh-web-search/actions/runs/44234913344
|
|
33
33
|
(the matrix also runs on macOS and Ubuntu). This is a CI environment, not a
|
|
34
34
|
claim about every Windows terminal or proxy client.
|
|
35
35
|
|
package/lib/cache-persist.js
CHANGED
|
@@ -228,7 +228,15 @@ let tempCounter = 0;
|
|
|
228
228
|
*/
|
|
229
229
|
export function writeCacheFile(file, entries, now, options = {}) {
|
|
230
230
|
const io = options.io ?? nodeCacheFileIo;
|
|
231
|
-
const
|
|
231
|
+
const snapshot = serializeCache(entries, now, options.maxBytes);
|
|
232
|
+
const budget = typeof options.maxBytes === 'number' && Number.isFinite(options.maxBytes) && options.maxBytes > 0
|
|
233
|
+
? Math.floor(options.maxBytes)
|
|
234
|
+
: DEFAULT_CACHE_FILE_MAX_BYTES;
|
|
235
|
+
// The empty envelope itself can exceed a tiny custom budget. Refuse the
|
|
236
|
+
// write rather than replacing a valid old snapshot with an over-budget one.
|
|
237
|
+
if (snapshot.bytes > budget)
|
|
238
|
+
return false;
|
|
239
|
+
const { data } = snapshot;
|
|
232
240
|
tempCounter += 1;
|
|
233
241
|
const temp = `${file}.${process.pid}.${tempCounter}.tmp`;
|
|
234
242
|
try {
|
package/lib/cache.js
CHANGED
|
@@ -23,12 +23,21 @@ const DEFAULT_PERSIST_THROTTLE_MS = 2_000;
|
|
|
23
23
|
*/
|
|
24
24
|
export function cacheKey(parts) {
|
|
25
25
|
const query = parts.query.trim().toLowerCase().replace(/\s+/gu, ' ');
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
26
|
+
// A JSON tuple is unambiguous and keeps every optional value in its own
|
|
27
|
+
// slot. The old delimiter join let `safeSearch: 'off'` + a query containing
|
|
28
|
+
// `safe=strict|` collide with `safeSearch: 'strict'` + a shorter query.
|
|
29
|
+
return JSON.stringify([
|
|
30
|
+
'v2',
|
|
31
|
+
parts.lead,
|
|
32
|
+
parts.wait,
|
|
33
|
+
parts.limit,
|
|
34
|
+
[...parts.engines].sort(),
|
|
35
|
+
parts.timeRange === undefined || parts.timeRange === 'off' ? null : parts.timeRange,
|
|
36
|
+
parts.safeSearch === undefined || parts.safeSearch === 'off' ? null : parts.safeSearch,
|
|
37
|
+
parts.fallbackOn === undefined ? null : [...parts.fallbackOn].sort(),
|
|
38
|
+
parts.blockedDomains === undefined || parts.blockedDomains.length === 0 ? null : [...parts.blockedDomains].sort(),
|
|
39
|
+
query,
|
|
40
|
+
]);
|
|
32
41
|
}
|
|
33
42
|
/**
|
|
34
43
|
* Copy the array and each source.
|
package/lib/domain-tool.d.ts
CHANGED
|
@@ -89,6 +89,8 @@ export interface DomainToolOptions {
|
|
|
89
89
|
detectProxy?: () => Promise<string | undefined>;
|
|
90
90
|
/** Registration diagnostics. */
|
|
91
91
|
warn?: (message: string) => void;
|
|
92
|
+
/** Receives the host's disposer, when registration returns one. */
|
|
93
|
+
onRegister?: (dispose: (() => void) | undefined) => void;
|
|
92
94
|
}
|
|
93
95
|
/** Format the model-facing result text. */
|
|
94
96
|
export declare function formatDomainValue(args: unknown, value: DomainToolValue): string;
|
package/lib/domain-tool.js
CHANGED
|
@@ -225,7 +225,13 @@ function report(options, error) {
|
|
|
225
225
|
export function registerDomainTool(tools, readSettings, options = {}) {
|
|
226
226
|
const name = pickDomainToolName(tools);
|
|
227
227
|
try {
|
|
228
|
-
tools.register(createDomainToolDefinition(name, readSettings, options));
|
|
228
|
+
const registered = tools.register(createDomainToolDefinition(name, readSettings, options));
|
|
229
|
+
try {
|
|
230
|
+
options.onRegister?.(typeof registered === 'function' ? registered : undefined);
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
// A registration observer must not break mounting.
|
|
234
|
+
}
|
|
229
235
|
return name;
|
|
230
236
|
}
|
|
231
237
|
catch (error) {
|
package/lib/engine-test.js
CHANGED
|
@@ -17,11 +17,15 @@ function failureReason(outcome) {
|
|
|
17
17
|
return outcome.failureDetails[0]?.reason ?? outcome.failures[0];
|
|
18
18
|
}
|
|
19
19
|
/** Run `worker` over items with at most `limit` in flight, preserving order. */
|
|
20
|
-
async function mapLimit(items, limit, worker) {
|
|
20
|
+
async function mapLimit(items, limit, worker, signal) {
|
|
21
21
|
const results = new Array(items.length);
|
|
22
22
|
let next = 0;
|
|
23
23
|
const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
|
|
24
24
|
for (;;) {
|
|
25
|
+
if (signal?.aborted === true) {
|
|
26
|
+
const reason = signal.reason;
|
|
27
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
28
|
+
}
|
|
25
29
|
const index = next;
|
|
26
30
|
next += 1;
|
|
27
31
|
if (index >= items.length)
|
|
@@ -133,10 +137,18 @@ function parseArgs(args) {
|
|
|
133
137
|
* @returns per-engine results plus counts.
|
|
134
138
|
*/
|
|
135
139
|
export async function runEngineTestValue(runTest, engines, query, signal) {
|
|
140
|
+
if (signal?.aborted === true) {
|
|
141
|
+
const reason = signal.reason;
|
|
142
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
143
|
+
}
|
|
136
144
|
const results = await mapLimit(engines, ENGINE_TEST_CONCURRENCY, async (engine) => {
|
|
137
145
|
const started = Date.now();
|
|
138
146
|
try {
|
|
139
147
|
const outcome = await runTest(engine, query, signal);
|
|
148
|
+
if (signal?.aborted === true) {
|
|
149
|
+
const reason = signal.reason;
|
|
150
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
151
|
+
}
|
|
140
152
|
const latencyMs = Date.now() - started;
|
|
141
153
|
if (outcome.sources.length > 0) {
|
|
142
154
|
const first = outcome.sources[0];
|
|
@@ -162,6 +174,8 @@ export async function runEngineTestValue(runTest, engines, query, signal) {
|
|
|
162
174
|
};
|
|
163
175
|
}
|
|
164
176
|
catch (error) {
|
|
177
|
+
if (signal?.aborted === true)
|
|
178
|
+
throw error;
|
|
165
179
|
return {
|
|
166
180
|
engine,
|
|
167
181
|
ok: false,
|
|
@@ -171,7 +185,7 @@ export async function runEngineTestValue(runTest, engines, query, signal) {
|
|
|
171
185
|
reason: error instanceof Error ? error.message : String(error),
|
|
172
186
|
};
|
|
173
187
|
}
|
|
174
|
-
});
|
|
188
|
+
}, signal);
|
|
175
189
|
const ok = results.filter((result) => result.ok).length;
|
|
176
190
|
return { query, results, ok, failed: results.length - ok };
|
|
177
191
|
}
|
package/lib/index.js
CHANGED
|
@@ -404,7 +404,8 @@ export function apply(ctx, config = {}) {
|
|
|
404
404
|
...(live.proxy === undefined ? {} : { proxy: live.proxy }),
|
|
405
405
|
};
|
|
406
406
|
return settings;
|
|
407
|
-
}, { warn: warnTool
|
|
407
|
+
}, { warn: warnTool, onRegister: (dispose) => { if (dispose !== undefined)
|
|
408
|
+
onDispose(ctx, dispose); } });
|
|
408
409
|
// Batch tool: same optional-registry path, same fail-safe mounting. Most
|
|
409
410
|
// queries use the `web_search` path; an explicit package-registry query
|
|
410
411
|
// can use the existing domain source instead.
|
|
@@ -520,7 +521,8 @@ export function apply(ctx, config = {}) {
|
|
|
520
521
|
truncated: outcome.truncated,
|
|
521
522
|
};
|
|
522
523
|
};
|
|
523
|
-
const platform = registerPlatformTool(toolTarget, platformRunner, { warn: warnTool
|
|
524
|
+
const platform = registerPlatformTool(toolTarget, platformRunner, { warn: warnTool, onRegister: (dispose) => { if (dispose !== undefined)
|
|
525
|
+
onDispose(ctx, dispose); } });
|
|
524
526
|
// Diagnostics: one tiny query per engine, no cache and no cooldown
|
|
525
527
|
// mutation. It is registered last because it is the least essential tool.
|
|
526
528
|
const engineTest = registerEngineTestTool(toolTarget, runEngineTest, { warn: warnTool });
|
|
@@ -531,12 +533,15 @@ export function apply(ctx, config = {}) {
|
|
|
531
533
|
// One entry point, eight actions. If the host rejects both candidate
|
|
532
534
|
// names, fall back to the legacy definitions rather than losing every
|
|
533
535
|
// optional capability.
|
|
534
|
-
const unified = registerUnifiedTool(tools, captured, { warn: warnTool
|
|
536
|
+
const unified = registerUnifiedTool(tools, captured, { warn: warnTool, onRegister: (dispose) => { if (dispose !== undefined)
|
|
537
|
+
onDispose(ctx, dispose); } });
|
|
535
538
|
if (unified === undefined && captured.length > 0) {
|
|
536
539
|
warnTool('web_tools unavailable; registering legacy tool names as a fallback');
|
|
537
540
|
for (const definition of captured) {
|
|
538
541
|
try {
|
|
539
|
-
tools.register(definition);
|
|
542
|
+
const registered = tools.register(definition);
|
|
543
|
+
if (typeof registered === 'function')
|
|
544
|
+
onDispose(ctx, registered);
|
|
540
545
|
}
|
|
541
546
|
catch (error) {
|
|
542
547
|
warnTool(`legacy fallback registration skipped: ${error instanceof Error ? error.message : String(error)}`);
|
package/lib/official-release.js
CHANGED
|
@@ -94,6 +94,10 @@ export function parseRustRelease(payload, requested) {
|
|
|
94
94
|
}
|
|
95
95
|
/** One direct GET with its own abort budget. */
|
|
96
96
|
async function directJson(url, headers, options, timeoutMs) {
|
|
97
|
+
if (options.signal?.aborted === true) {
|
|
98
|
+
const reason = options.signal.reason;
|
|
99
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
100
|
+
}
|
|
97
101
|
const controller = new AbortController();
|
|
98
102
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
99
103
|
const abort = () => controller.abort();
|
|
@@ -132,6 +136,8 @@ async function fetchJson(url, options, timeoutMs) {
|
|
|
132
136
|
return await directJson(url, headers, options, Math.min(timeoutMs, 1_500));
|
|
133
137
|
}
|
|
134
138
|
catch (error) {
|
|
139
|
+
if (options.signal?.aborted === true)
|
|
140
|
+
throw error;
|
|
135
141
|
if (options.proxy === undefined || options.proxy.length === 0)
|
|
136
142
|
throw error;
|
|
137
143
|
const remaining = timeoutMs - (Date.now() - started);
|
package/lib/official-sources.js
CHANGED
|
@@ -13,14 +13,16 @@ async function fetchNpmPackage(name, options) {
|
|
|
13
13
|
try {
|
|
14
14
|
const result = await searchVerticals(name, {
|
|
15
15
|
ids: ['npm'],
|
|
16
|
-
|
|
16
|
+
// Search, not a registry exact lookup: ask for a few rows and verify the
|
|
17
|
+
// package identity instead of trusting the first fuzzy result.
|
|
18
|
+
limit: 10,
|
|
17
19
|
timeoutMs: typeof options.timeoutMs === 'number' && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0
|
|
18
20
|
? Math.min(options.timeoutMs, 8_000)
|
|
19
21
|
: 8_000,
|
|
20
22
|
...(options.proxy === undefined ? {} : { proxy: options.proxy }),
|
|
21
23
|
...(options.signal === undefined ? {} : { signal: options.signal }),
|
|
22
24
|
});
|
|
23
|
-
return result.sources
|
|
25
|
+
return result.sources.find((source) => source.title === name || source.title?.startsWith(`${name}@`) === true);
|
|
24
26
|
}
|
|
25
27
|
catch {
|
|
26
28
|
return undefined;
|
package/lib/official-support.js
CHANGED
|
@@ -75,6 +75,10 @@ export function postgresSupportSource(releases) {
|
|
|
75
75
|
}
|
|
76
76
|
/** Read the page over the configured tunnel, or directly when none is set. */
|
|
77
77
|
async function fetchText(url, options, timeoutMs) {
|
|
78
|
+
if (options.signal?.aborted === true) {
|
|
79
|
+
const reason = options.signal.reason;
|
|
80
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
81
|
+
}
|
|
78
82
|
if (options.proxy !== undefined && options.proxy.length > 0) {
|
|
79
83
|
const response = await proxyGet(url, {
|
|
80
84
|
proxy: options.proxy,
|
package/lib/parsers.js
CHANGED
|
@@ -82,38 +82,52 @@ export function resolveHref(raw, baseUrl) {
|
|
|
82
82
|
if (typeof raw !== 'string' || raw.length === 0)
|
|
83
83
|
return undefined;
|
|
84
84
|
// Attribute values are HTML-escaped, so a Bing click tracker arrives as
|
|
85
|
-
// `...&u=a1<base64url>`; decode before
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
85
|
+
// `...&u=a1<base64url>`; decode before parsing.
|
|
86
|
+
const candidate = raw.replace(/&/gi, '&').replace(/�*38;/g, '&');
|
|
87
|
+
let parsed;
|
|
88
|
+
try {
|
|
89
|
+
parsed = new URL(candidate, baseUrl);
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return undefined;
|
|
93
|
+
}
|
|
94
|
+
// Only the search engine's own wrapper is unwrapped. A normal page whose
|
|
95
|
+
// query string happens to contain `uddg` or `u=a1...` must keep its URL.
|
|
96
|
+
const host = parsed.hostname.toLowerCase();
|
|
97
|
+
let target;
|
|
98
|
+
if ((host === 'bing.com' || host === 'www.bing.com') && parsed.pathname === '/ck/a') {
|
|
99
|
+
const payload = parsed.searchParams.get('u');
|
|
100
|
+
const match = payload === null ? null : /^a1([A-Za-z0-9_-]+={0,2})$/u.exec(payload);
|
|
101
|
+
if (match !== null) {
|
|
102
|
+
try {
|
|
103
|
+
target = Buffer.from(match[1].replace(/=+$/u, '').replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8');
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
94
108
|
}
|
|
95
109
|
}
|
|
96
|
-
else {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
if (uddg) {
|
|
110
|
+
else if ((host === 'duckduckgo.com' || host.endsWith('.duckduckgo.com')) && parsed.pathname === '/l/') {
|
|
111
|
+
const wrapped = parsed.searchParams.get('uddg');
|
|
112
|
+
if (wrapped !== null && wrapped.length > 0) {
|
|
100
113
|
try {
|
|
101
|
-
|
|
114
|
+
target = decodeURIComponent(wrapped);
|
|
102
115
|
}
|
|
103
116
|
catch {
|
|
104
117
|
return undefined;
|
|
105
118
|
}
|
|
106
119
|
}
|
|
107
120
|
}
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
121
|
+
if (target !== undefined) {
|
|
122
|
+
try {
|
|
123
|
+
const unwrapped = new URL(target);
|
|
124
|
+
return unwrapped.protocol === 'http:' || unwrapped.protocol === 'https:' ? unwrapped.toString() : undefined;
|
|
125
|
+
}
|
|
126
|
+
catch {
|
|
111
127
|
return undefined;
|
|
112
|
-
|
|
113
|
-
}
|
|
114
|
-
catch {
|
|
115
|
-
return undefined;
|
|
128
|
+
}
|
|
116
129
|
}
|
|
130
|
+
return parsed.protocol === 'http:' || parsed.protocol === 'https:' ? parsed.toString() : undefined;
|
|
117
131
|
}
|
|
118
132
|
/**
|
|
119
133
|
* Parse a 360 (so.com) results page. The real URL lives in `data-mdurl`;
|
|
@@ -457,6 +471,9 @@ export function parseExaMcp(payload, baseUrl) {
|
|
|
457
471
|
}
|
|
458
472
|
else if (/^highlights\s*:/i.test(line)) {
|
|
459
473
|
inHighlights = true;
|
|
474
|
+
const inline = line.replace(/^highlights\s*:/i, '').trim();
|
|
475
|
+
if (inline.length > 0)
|
|
476
|
+
highlights.push(inline);
|
|
460
477
|
}
|
|
461
478
|
else if (/^(published|author)\s*:/i.test(line)) {
|
|
462
479
|
inHighlights = false;
|
package/lib/platform-tool.d.ts
CHANGED
|
@@ -73,5 +73,6 @@ export declare function pickPlatformToolName(tools: PlatformToolRegistryLike): s
|
|
|
73
73
|
/** Register the tool; any registry failure is contained. */
|
|
74
74
|
export declare function registerPlatformTool(tools: PlatformToolRegistryLike, run: PlatformRunner, options?: {
|
|
75
75
|
warn?: (message: string) => void;
|
|
76
|
+
onRegister?: (dispose: (() => void) | undefined) => void;
|
|
76
77
|
}): string | undefined;
|
|
77
78
|
export { MAX_PLATFORM_LIMIT };
|
package/lib/platform-tool.js
CHANGED
|
@@ -157,7 +157,13 @@ export function pickPlatformToolName(tools) {
|
|
|
157
157
|
export function registerPlatformTool(tools, run, options = {}) {
|
|
158
158
|
const name = pickPlatformToolName(tools);
|
|
159
159
|
try {
|
|
160
|
-
tools.register(createPlatformToolDefinition(name, run));
|
|
160
|
+
const registered = tools.register(createPlatformToolDefinition(name, run));
|
|
161
|
+
try {
|
|
162
|
+
options.onRegister?.(typeof registered === 'function' ? registered : undefined);
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
// A registration observer must not break mounting.
|
|
166
|
+
}
|
|
161
167
|
return name;
|
|
162
168
|
}
|
|
163
169
|
catch (error) {
|
package/lib/platforms.js
CHANGED
|
@@ -267,6 +267,26 @@ function deadline(timeoutMs, signal) {
|
|
|
267
267
|
const timer = AbortSignal.timeout(timeoutMs);
|
|
268
268
|
return signal === undefined ? timer : AbortSignal.any([signal, timer]);
|
|
269
269
|
}
|
|
270
|
+
/** Race a promise against an abort signal, observing both outcomes. */
|
|
271
|
+
function withAbort(promise, signal) {
|
|
272
|
+
if (signal.aborted) {
|
|
273
|
+
return Promise.reject(signal.reason instanceof Error ? signal.reason : new Error('platform search was aborted'));
|
|
274
|
+
}
|
|
275
|
+
return new Promise((resolve, reject) => {
|
|
276
|
+
const onAbort = () => {
|
|
277
|
+
signal.removeEventListener('abort', onAbort);
|
|
278
|
+
reject(signal.reason instanceof Error ? signal.reason : new Error('platform search was aborted'));
|
|
279
|
+
};
|
|
280
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
281
|
+
promise.then((value) => {
|
|
282
|
+
signal.removeEventListener('abort', onAbort);
|
|
283
|
+
resolve(value);
|
|
284
|
+
}, (error) => {
|
|
285
|
+
signal.removeEventListener('abort', onAbort);
|
|
286
|
+
reject(error);
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
}
|
|
270
290
|
/** Turn a non-2xx status into a reason. */
|
|
271
291
|
function statusError(status) {
|
|
272
292
|
return new Error(status === 0 ? 'no response' : `HTTP ${status}`);
|
|
@@ -828,7 +848,11 @@ async function fallbackAnswer(spec, query, limit, options, directReason) {
|
|
|
828
848
|
}
|
|
829
849
|
const fallbackQuery = `site:${spec.site} ${query}`;
|
|
830
850
|
try {
|
|
831
|
-
|
|
851
|
+
if (options.signal?.aborted === true) {
|
|
852
|
+
const reason = options.signal.reason;
|
|
853
|
+
throw reason instanceof Error ? reason : new Error('platform search was aborted');
|
|
854
|
+
}
|
|
855
|
+
const result = await withAbort(options.fallbackSearch(fallbackQuery, limit, options.signal), options.signal ?? new AbortController().signal);
|
|
832
856
|
// A search engine may ignore the site: operator and return generic pages.
|
|
833
857
|
// A platform result must at least live on that platform's domain.
|
|
834
858
|
const allowed = normaliseSources(result.sources, limit * 2)
|
|
@@ -875,7 +899,7 @@ async function fallbackAnswer(spec, query, limit, options, directReason) {
|
|
|
875
899
|
* @param timeoutMs - per-platform deadline.
|
|
876
900
|
* @returns the platform answer.
|
|
877
901
|
*/
|
|
878
|
-
async function
|
|
902
|
+
async function searchPlatformInner(spec, query, limit, options, timeoutMs) {
|
|
879
903
|
if (spec.fallbackOnly === true || spec.build === undefined || spec.parse === undefined) {
|
|
880
904
|
if (options.fallbackSearch === undefined) {
|
|
881
905
|
return {
|
|
@@ -927,6 +951,45 @@ async function searchPlatform(spec, query, limit, options, timeoutMs) {
|
|
|
927
951
|
}
|
|
928
952
|
return fallbackAnswer(spec, query, limit, options, directReason);
|
|
929
953
|
}
|
|
954
|
+
/**
|
|
955
|
+
* Run one platform under a single deadline that also covers the fallback.
|
|
956
|
+
*
|
|
957
|
+
* `requestText` has its own per-request timeout, but an alternate endpoint or
|
|
958
|
+
* an injected fallback search can otherwise get a fresh budget, and a fallback
|
|
959
|
+
* that ignores its signal can hang the whole `Promise.all` forever.
|
|
960
|
+
*
|
|
961
|
+
* @param spec - platform being served.
|
|
962
|
+
* @param query - original query, already trimmed.
|
|
963
|
+
* @param limit - per-platform budget.
|
|
964
|
+
* @param options - caller options.
|
|
965
|
+
* @param timeoutMs - total deadline for this platform.
|
|
966
|
+
* @returns the platform answer; timeout becomes a platform failure unless the
|
|
967
|
+
* caller itself cancelled.
|
|
968
|
+
*/
|
|
969
|
+
async function searchPlatform(spec, query, limit, options, timeoutMs) {
|
|
970
|
+
const controller = new AbortController();
|
|
971
|
+
const timer = setTimeout(() => controller.abort(new Error(`platform ${spec.id} timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
972
|
+
const combined = options.signal === undefined
|
|
973
|
+
? controller.signal
|
|
974
|
+
: AbortSignal.any([options.signal, controller.signal]);
|
|
975
|
+
const scoped = { ...options, signal: combined };
|
|
976
|
+
try {
|
|
977
|
+
return await withAbort(searchPlatformInner(spec, query, limit, scoped, timeoutMs), combined);
|
|
978
|
+
}
|
|
979
|
+
catch (error) {
|
|
980
|
+
if (options.signal?.aborted === true)
|
|
981
|
+
throw error;
|
|
982
|
+
return {
|
|
983
|
+
id: spec.id,
|
|
984
|
+
sources: [],
|
|
985
|
+
via: spec.fallbackOnly === true ? 'fallback' : 'api',
|
|
986
|
+
failure: failureText(spec.id, messageOf(error)),
|
|
987
|
+
};
|
|
988
|
+
}
|
|
989
|
+
finally {
|
|
990
|
+
clearTimeout(timer);
|
|
991
|
+
}
|
|
992
|
+
}
|
|
930
993
|
/**
|
|
931
994
|
* Search the requested platforms and merge their answers.
|
|
932
995
|
*
|
package/lib/proxy.js
CHANGED
|
@@ -486,6 +486,10 @@ export function readResponse(buffer, maxBytes = DEFAULT_MAX_BYTES) {
|
|
|
486
486
|
* @returns `{ status, text, url, headers }` for the final hop.
|
|
487
487
|
*/
|
|
488
488
|
export async function proxyGet(url, options) {
|
|
489
|
+
if (options.signal?.aborted === true) {
|
|
490
|
+
const reason = options.signal.reason;
|
|
491
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
492
|
+
}
|
|
489
493
|
const proxy = parseProxy(options.proxy);
|
|
490
494
|
const timeoutMs = options.timeoutMs !== undefined && Number.isFinite(options.timeoutMs) && options.timeoutMs > 0 ? options.timeoutMs : DEFAULT_TIMEOUT_MS;
|
|
491
495
|
const redirects = options.redirects !== undefined && Number.isInteger(options.redirects) ? options.redirects : DEFAULT_REDIRECTS;
|
package/lib/read-page.js
CHANGED
|
@@ -401,20 +401,39 @@ export function createReadPageDefinition(name, readSettings, options = {}) {
|
|
|
401
401
|
if (parsed.offset > totalLength) {
|
|
402
402
|
throw new Error(`read_page: offset ${parsed.offset} is past the end of ${parsed.url} (totalLength ${totalLength}); use offset 0 to start over or ${totalLength} to read the tail`);
|
|
403
403
|
}
|
|
404
|
-
//
|
|
405
|
-
//
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
if (
|
|
410
|
-
end
|
|
404
|
+
// Walk the real page boundaries. A surrogate pair at a cut point moves
|
|
405
|
+
// that boundary one code unit back, so the chunk count and index cannot
|
|
406
|
+
// be derived from `ceil(totalLength / maxLength)` without drifting.
|
|
407
|
+
const pageEnd = (start) => {
|
|
408
|
+
let end = Math.min(totalLength, start + parsed.maxLength);
|
|
409
|
+
if (end > start && end < totalLength) {
|
|
410
|
+
const last = page.content.charCodeAt(end - 1);
|
|
411
|
+
if (last >= 0xd800 && last <= 0xdbff)
|
|
412
|
+
end -= 1;
|
|
413
|
+
}
|
|
414
|
+
// A one-character budget on a surrogate pair must still advance.
|
|
415
|
+
return end > start ? end : Math.min(totalLength, start + 1);
|
|
416
|
+
};
|
|
417
|
+
let count = 0;
|
|
418
|
+
let index = 0;
|
|
419
|
+
let cursor = 0;
|
|
420
|
+
while (cursor < totalLength) {
|
|
421
|
+
const boundary = pageEnd(cursor);
|
|
422
|
+
count += 1;
|
|
423
|
+
if (parsed.offset >= cursor && parsed.offset < boundary)
|
|
424
|
+
index = count;
|
|
425
|
+
cursor = boundary;
|
|
426
|
+
}
|
|
427
|
+
if (totalLength === 0) {
|
|
428
|
+
count = 1;
|
|
429
|
+
index = 1;
|
|
430
|
+
}
|
|
431
|
+
else if (parsed.offset === totalLength) {
|
|
432
|
+
index = count;
|
|
411
433
|
}
|
|
434
|
+
const end = pageEnd(parsed.offset);
|
|
412
435
|
const content = page.content.slice(parsed.offset, end);
|
|
413
436
|
const nextOffset = end < totalLength ? end : null;
|
|
414
|
-
const count = totalLength === 0 ? 1 : Math.ceil(totalLength / parsed.maxLength);
|
|
415
|
-
const index = totalLength === 0
|
|
416
|
-
? 1
|
|
417
|
-
: Math.min(count, Math.floor(parsed.offset / parsed.maxLength) + 1);
|
|
418
437
|
const value = {
|
|
419
438
|
url: parsed.url,
|
|
420
439
|
content,
|
package/lib/render.js
CHANGED
|
@@ -64,15 +64,67 @@ const SHELL_MARKERS = [
|
|
|
64
64
|
export function readableTextLength(html) {
|
|
65
65
|
if (html.length === 0)
|
|
66
66
|
return 0;
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
.
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
.
|
|
67
|
+
const lower = html.toLowerCase();
|
|
68
|
+
let count = 0;
|
|
69
|
+
let pendingSpace = false;
|
|
70
|
+
let index = 0;
|
|
71
|
+
const advancePast = (needle, from) => {
|
|
72
|
+
const at = lower.indexOf(needle, from);
|
|
73
|
+
if (at < 0)
|
|
74
|
+
return html.length;
|
|
75
|
+
const end = html.indexOf('>', at + needle.length);
|
|
76
|
+
return end < 0 ? html.length : end + 1;
|
|
77
|
+
};
|
|
78
|
+
while (index < html.length) {
|
|
79
|
+
if (html.startsWith('<!--', index)) {
|
|
80
|
+
const end = html.indexOf('-->', index + 4);
|
|
81
|
+
index = end < 0 ? html.length : end + 3;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
if (html[index] === '<') {
|
|
85
|
+
const end = html.indexOf('>', index + 1);
|
|
86
|
+
if (end < 0) {
|
|
87
|
+
// A lone '<' is text, not markup.
|
|
88
|
+
index += 1;
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
const raw = html.slice(index + 1, end).trim();
|
|
92
|
+
if (raw.startsWith('!') || raw.startsWith('?')) {
|
|
93
|
+
index = end + 1;
|
|
94
|
+
continue;
|
|
95
|
+
}
|
|
96
|
+
const name = /^\/?\s*([a-zA-Z][a-zA-Z0-9:_-]*)/u.exec(raw)?.[1]?.toLowerCase();
|
|
97
|
+
if (name === 'head') {
|
|
98
|
+
index = advancePast('</head', end + 1);
|
|
99
|
+
continue;
|
|
100
|
+
}
|
|
101
|
+
if (name !== undefined && ['script', 'style', 'noscript', 'template', 'svg'].includes(name)) {
|
|
102
|
+
index = advancePast(`</${name}`, end + 1);
|
|
103
|
+
continue;
|
|
104
|
+
}
|
|
105
|
+
index = end + 1;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (html[index] === '&') {
|
|
109
|
+
const entity = /^&(?:#\d+|#x[0-9a-f]+|[a-z][a-z0-9]+);/iu.exec(html.slice(index));
|
|
110
|
+
if (entity !== null) {
|
|
111
|
+
pendingSpace = true;
|
|
112
|
+
index += entity[0].length;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (/\s/u.test(html[index])) {
|
|
117
|
+
pendingSpace = true;
|
|
118
|
+
}
|
|
119
|
+
else {
|
|
120
|
+
if (pendingSpace && count > 0)
|
|
121
|
+
count += 1;
|
|
122
|
+
pendingSpace = false;
|
|
123
|
+
count += 1;
|
|
124
|
+
}
|
|
125
|
+
index += 1;
|
|
126
|
+
}
|
|
127
|
+
return count;
|
|
76
128
|
}
|
|
77
129
|
/**
|
|
78
130
|
* Decide whether a fetched page is probably a script-rendered shell.
|
|
@@ -235,12 +287,22 @@ function isAborted(signal) {
|
|
|
235
287
|
async function closeQuietly(target) {
|
|
236
288
|
if (target === undefined)
|
|
237
289
|
return;
|
|
290
|
+
let timer;
|
|
238
291
|
try {
|
|
239
|
-
await
|
|
292
|
+
await Promise.race([
|
|
293
|
+
target.close().catch(() => undefined),
|
|
294
|
+
new Promise((resolve) => {
|
|
295
|
+
timer = setTimeout(resolve, 1_000);
|
|
296
|
+
}),
|
|
297
|
+
]);
|
|
240
298
|
}
|
|
241
299
|
catch {
|
|
242
300
|
// A close that fails is not the interesting failure.
|
|
243
301
|
}
|
|
302
|
+
finally {
|
|
303
|
+
if (timer !== undefined)
|
|
304
|
+
clearTimeout(timer);
|
|
305
|
+
}
|
|
244
306
|
}
|
|
245
307
|
/**
|
|
246
308
|
* Render one URL in a headless Chromium and return its serialised DOM.
|
|
@@ -268,26 +330,34 @@ export async function renderWithPlaywright(renderer, url, timeoutMs, signal) {
|
|
|
268
330
|
let browser;
|
|
269
331
|
let page;
|
|
270
332
|
let resolveAbort;
|
|
333
|
+
let stopped = false;
|
|
271
334
|
const aborted = new Promise((resolve) => {
|
|
272
335
|
resolveAbort = resolve;
|
|
273
336
|
});
|
|
274
|
-
const
|
|
337
|
+
const stop = () => {
|
|
338
|
+
if (stopped)
|
|
339
|
+
return;
|
|
340
|
+
stopped = true;
|
|
275
341
|
resolveAbort?.();
|
|
276
342
|
// Requirement, not optimisation: a cancelled render must not keep a
|
|
277
343
|
// browser or a page alive while the caller has already moved on.
|
|
278
344
|
void closeQuietly(page);
|
|
279
345
|
void closeQuietly(browser);
|
|
280
346
|
};
|
|
347
|
+
const onAbort = () => stop();
|
|
281
348
|
let listening = false;
|
|
282
349
|
if (signal !== undefined) {
|
|
283
350
|
if (signal.aborted) {
|
|
284
|
-
|
|
351
|
+
stop();
|
|
285
352
|
}
|
|
286
353
|
else {
|
|
287
354
|
signal.addEventListener('abort', onAbort, { once: true });
|
|
288
355
|
listening = true;
|
|
289
356
|
}
|
|
290
357
|
}
|
|
358
|
+
// The navigation timeout does not bound launch, newPage, content or a close
|
|
359
|
+
// that never resolves. One total deadline does.
|
|
360
|
+
const timer = setTimeout(stop, Math.max(1, budget + Math.min(NETWORK_IDLE_EXTRA_MS, budget)));
|
|
291
361
|
try {
|
|
292
362
|
const launching = renderer.chromium.launch({ headless: true });
|
|
293
363
|
const winner = await Promise.race([
|
|
@@ -300,23 +370,41 @@ export async function renderWithPlaywright(renderer, url, timeoutMs, signal) {
|
|
|
300
370
|
throw abortError();
|
|
301
371
|
}
|
|
302
372
|
browser = winner.browser;
|
|
303
|
-
if (isAborted(signal))
|
|
373
|
+
if (stopped || isAborted(signal))
|
|
374
|
+
throw abortError();
|
|
375
|
+
const pageAttempt = browser.newPage();
|
|
376
|
+
const pageWinner = await Promise.race([
|
|
377
|
+
pageAttempt.then((value) => ({ page: value })),
|
|
378
|
+
aborted.then(() => ({ page: undefined })),
|
|
379
|
+
]);
|
|
380
|
+
if (pageWinner.page === undefined) {
|
|
381
|
+
void pageAttempt.then((late) => closeQuietly(late)).catch(() => undefined);
|
|
304
382
|
throw abortError();
|
|
305
|
-
|
|
306
|
-
|
|
383
|
+
}
|
|
384
|
+
page = pageWinner.page;
|
|
385
|
+
if (stopped || isAborted(signal))
|
|
307
386
|
throw abortError();
|
|
308
|
-
const response = await
|
|
387
|
+
const response = await Promise.race([
|
|
388
|
+
page.goto(url, { waitUntil: 'domcontentloaded', timeout: budget }),
|
|
389
|
+
aborted.then(() => { throw abortError(); }),
|
|
390
|
+
]);
|
|
309
391
|
const idleBudget = Math.min(NETWORK_IDLE_EXTRA_MS, budget);
|
|
310
392
|
try {
|
|
311
|
-
await
|
|
393
|
+
await Promise.race([
|
|
394
|
+
page.waitForLoadState('networkidle', { timeout: idleBudget }),
|
|
395
|
+
aborted,
|
|
396
|
+
]);
|
|
312
397
|
}
|
|
313
398
|
catch {
|
|
314
399
|
// A page that never goes idle still has a DOM; carry on.
|
|
315
400
|
}
|
|
316
|
-
if (isAborted(signal))
|
|
401
|
+
if (stopped || isAborted(signal))
|
|
317
402
|
throw abortError();
|
|
318
|
-
const html = await
|
|
319
|
-
|
|
403
|
+
const html = await Promise.race([
|
|
404
|
+
page.content(),
|
|
405
|
+
aborted.then(() => { throw abortError(); }),
|
|
406
|
+
]);
|
|
407
|
+
if (stopped || isAborted(signal))
|
|
320
408
|
throw abortError();
|
|
321
409
|
return {
|
|
322
410
|
html,
|
|
@@ -324,11 +412,18 @@ export async function renderWithPlaywright(renderer, url, timeoutMs, signal) {
|
|
|
324
412
|
url: page.url(),
|
|
325
413
|
};
|
|
326
414
|
}
|
|
415
|
+
catch (error) {
|
|
416
|
+
if (stopped || isAborted(signal))
|
|
417
|
+
throw abortError();
|
|
418
|
+
throw error;
|
|
419
|
+
}
|
|
327
420
|
finally {
|
|
421
|
+
clearTimeout(timer);
|
|
328
422
|
if (listening && signal !== undefined)
|
|
329
423
|
signal.removeEventListener('abort', onAbort);
|
|
330
|
-
|
|
331
|
-
|
|
424
|
+
// Start both closes together; a page close that hangs must not prevent the
|
|
425
|
+
// browser close from running.
|
|
426
|
+
await Promise.all([closeQuietly(page), closeQuietly(browser)]);
|
|
332
427
|
}
|
|
333
428
|
}
|
|
334
429
|
/**
|
package/lib/research.d.ts
CHANGED
|
@@ -87,6 +87,11 @@ export interface ResearchValue {
|
|
|
87
87
|
}>;
|
|
88
88
|
/** Fetch failures, one entry per URL. */
|
|
89
89
|
failures: ResearchFetchFailure[];
|
|
90
|
+
/** Search errors, one entry per failed query; omitted when none failed. */
|
|
91
|
+
searchFailures?: Array<{
|
|
92
|
+
query: string;
|
|
93
|
+
reason: string;
|
|
94
|
+
}>;
|
|
90
95
|
/** Number of search rows seen before the fetch budget cut them. */
|
|
91
96
|
found: number;
|
|
92
97
|
/** True when the search row list was cut by the fetch budget. */
|
package/lib/research.js
CHANGED
|
@@ -169,9 +169,13 @@ export function selectExcerpt(text, tokens, max) {
|
|
|
169
169
|
const add = (index) => {
|
|
170
170
|
const window = windows[index];
|
|
171
171
|
if (window === undefined || picked.has(index))
|
|
172
|
-
return;
|
|
172
|
+
return false;
|
|
173
|
+
const addition = window.length + 2;
|
|
174
|
+
if (used + addition > max)
|
|
175
|
+
return false;
|
|
173
176
|
picked.add(index);
|
|
174
|
-
used +=
|
|
177
|
+
used += addition;
|
|
178
|
+
return true;
|
|
175
179
|
};
|
|
176
180
|
// Regulations often put the effective date in the final article, after the
|
|
177
181
|
// definitions. A pure relevance sort can spend the budget on the definitions
|
|
@@ -233,10 +237,12 @@ export function selectExcerpt(text, tokens, max) {
|
|
|
233
237
|
}
|
|
234
238
|
const selected = [...picked].sort((a, b) => a - b).map((index) => windows[index] ?? '');
|
|
235
239
|
const excerpt = selected.join('\n\n…\n\n');
|
|
236
|
-
//
|
|
237
|
-
//
|
|
238
|
-
//
|
|
239
|
-
|
|
240
|
+
// The tail/legal/opening safeguards must not bypass the caller's excerpt
|
|
241
|
+
// budget. If a single window still overshoots, cut it explicitly; losing a
|
|
242
|
+
// suffix is better than returning a multi-thousand-character excerpt.
|
|
243
|
+
if (excerpt.length <= max)
|
|
244
|
+
return excerpt;
|
|
245
|
+
return max > 0 ? `${excerpt.slice(0, Math.max(0, max - 1)).trimEnd()}…` : '';
|
|
240
246
|
}
|
|
241
247
|
/** Walk `worker` over items with at most `limit` in flight, preserving order. */
|
|
242
248
|
async function mapLimit(items, limit, worker) {
|
|
@@ -254,6 +260,26 @@ async function mapLimit(items, limit, worker) {
|
|
|
254
260
|
await Promise.all(runners);
|
|
255
261
|
return results;
|
|
256
262
|
}
|
|
263
|
+
/** Race a promise against a signal while observing both outcomes. */
|
|
264
|
+
function withAbort(promise, signal) {
|
|
265
|
+
if (signal.aborted) {
|
|
266
|
+
return Promise.reject(signal.reason instanceof Error ? signal.reason : new Error('research was aborted'));
|
|
267
|
+
}
|
|
268
|
+
return new Promise((resolve, reject) => {
|
|
269
|
+
const onAbort = () => {
|
|
270
|
+
signal.removeEventListener('abort', onAbort);
|
|
271
|
+
reject(signal.reason instanceof Error ? signal.reason : new Error('research was aborted'));
|
|
272
|
+
};
|
|
273
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
274
|
+
promise.then((value) => {
|
|
275
|
+
signal.removeEventListener('abort', onAbort);
|
|
276
|
+
resolve(value);
|
|
277
|
+
}, (error) => {
|
|
278
|
+
signal.removeEventListener('abort', onAbort);
|
|
279
|
+
reject(error);
|
|
280
|
+
});
|
|
281
|
+
});
|
|
282
|
+
}
|
|
257
283
|
/** Words that carry no retrieval signal in a research query. */
|
|
258
284
|
const QUERY_STOPWORDS = new Set([
|
|
259
285
|
'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'on', 'for', 'with', 'from',
|
|
@@ -607,6 +633,15 @@ function outputSchema() {
|
|
|
607
633
|
required: ['url', 'reason'],
|
|
608
634
|
},
|
|
609
635
|
},
|
|
636
|
+
searchFailures: {
|
|
637
|
+
type: 'array',
|
|
638
|
+
items: {
|
|
639
|
+
type: 'object',
|
|
640
|
+
additionalProperties: false,
|
|
641
|
+
properties: { query: { type: 'string' }, reason: { type: 'string' } },
|
|
642
|
+
required: ['query', 'reason'],
|
|
643
|
+
},
|
|
644
|
+
},
|
|
610
645
|
found: { type: 'integer' },
|
|
611
646
|
truncated: { type: 'boolean' },
|
|
612
647
|
},
|
|
@@ -644,10 +679,20 @@ export function formatResearchValue(value) {
|
|
|
644
679
|
'## Found but not fetched',
|
|
645
680
|
...value.sources.map((source) => `- ${source.title !== undefined && source.title.length > 0 ? `${source.title} — ` : ''}${source.url}`),
|
|
646
681
|
];
|
|
682
|
+
const searchFailures = value.searchFailures === undefined || value.searchFailures.length === 0
|
|
683
|
+
? []
|
|
684
|
+
: ['## Search failures', ...value.searchFailures.map((failure) => `- ${JSON.stringify(failure.query)}: ${failure.reason}`)];
|
|
647
685
|
const failures = value.failures.length === 0
|
|
648
686
|
? []
|
|
649
687
|
: ['## Fetch failures', ...value.failures.map((failure) => `- ${failure.url}: ${failure.reason}`)];
|
|
650
|
-
return [
|
|
688
|
+
return [
|
|
689
|
+
...head,
|
|
690
|
+
'',
|
|
691
|
+
...sections,
|
|
692
|
+
...(notFetched.length === 0 ? [] : ['', ...notFetched]),
|
|
693
|
+
...(searchFailures.length === 0 ? [] : ['', ...searchFailures]),
|
|
694
|
+
...(failures.length === 0 ? [] : ['', ...failures]),
|
|
695
|
+
]
|
|
651
696
|
.join('\n')
|
|
652
697
|
.trimEnd();
|
|
653
698
|
}
|
|
@@ -673,15 +718,26 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
673
718
|
const parsed = parseArgs(args);
|
|
674
719
|
const signal = exec.signal;
|
|
675
720
|
const startedAt = Date.now();
|
|
721
|
+
// The tool's declared 45 s budget must cover the searches too. Passing
|
|
722
|
+
// only the host signal let a slow search consume the whole 60 s host
|
|
723
|
+
// timeout before the page ladder ever saw `remaining()`.
|
|
724
|
+
const deadline = AbortSignal.timeout(RESEARCH_BUDGET_MS);
|
|
725
|
+
const workSignal = signal === undefined ? deadline : AbortSignal.any([signal, deadline]);
|
|
676
726
|
const remaining = () => Math.max(0, RESEARCH_BUDGET_MS - (Date.now() - startedAt));
|
|
677
727
|
const cancelled = () => signal !== undefined && signal.aborted;
|
|
678
728
|
const searchLimit = Math.max(5, parsed.maxSources);
|
|
679
729
|
const queriesUsed = [...parsed.queries];
|
|
680
730
|
const runSearch = async (query) => {
|
|
681
731
|
try {
|
|
682
|
-
|
|
732
|
+
if (workSignal.aborted === true) {
|
|
733
|
+
const reason = workSignal.reason;
|
|
734
|
+
throw reason instanceof Error ? reason : new Error('research was aborted');
|
|
735
|
+
}
|
|
736
|
+
return await withAbort(search(query, searchLimit, workSignal), workSignal);
|
|
683
737
|
}
|
|
684
738
|
catch (error) {
|
|
739
|
+
if (signal?.aborted === true)
|
|
740
|
+
throw error;
|
|
685
741
|
return {
|
|
686
742
|
sources: [],
|
|
687
743
|
failures: [error instanceof Error ? error.message : String(error)],
|
|
@@ -691,12 +747,15 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
691
747
|
const outcomes = await mapLimit(parsed.queries, 2, runSearch);
|
|
692
748
|
if (cancelled())
|
|
693
749
|
throw new Error('research was cancelled');
|
|
750
|
+
const searchFailures = parsed.queries.flatMap((query, index) => (outcomes[index]?.failures ?? []).map((reason) => ({ query, reason })));
|
|
694
751
|
let candidates = interleave(outcomes.map((outcome) => outcome.sources));
|
|
695
752
|
const hints = officialHints(parsed.queries);
|
|
696
753
|
// Long questions often need the issuing body's own site, which the open
|
|
697
754
|
// engine may not rank. Spend unused query slots on a site-specific query
|
|
698
755
|
// instead of letting the evidence pack stop at an aggregator.
|
|
699
756
|
for (const hint of hints) {
|
|
757
|
+
if (remaining() === 0)
|
|
758
|
+
break;
|
|
700
759
|
// A host match is enough for most issuers, but a generic W3C page does
|
|
701
760
|
// not cover a specific success criterion; let the targeted query run.
|
|
702
761
|
// A generic host match is enough for most issuers, but not for the
|
|
@@ -713,6 +772,8 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
713
772
|
const extra = await runSearch(hinted);
|
|
714
773
|
if (cancelled())
|
|
715
774
|
throw new Error('research was cancelled');
|
|
775
|
+
for (const reason of extra.failures)
|
|
776
|
+
searchFailures.push({ query: hinted, reason });
|
|
716
777
|
// A site: operator is advisory on some engines; keep only rows that
|
|
717
778
|
// really belong to the issuer's domain.
|
|
718
779
|
outcomes.push({
|
|
@@ -801,11 +862,13 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
801
862
|
failures.push({ url: source.url, reason: 'no time left in the research budget' });
|
|
802
863
|
return;
|
|
803
864
|
}
|
|
804
|
-
const bound =
|
|
805
|
-
? AbortSignal.timeout(budget)
|
|
806
|
-
: AbortSignal.any([signal, AbortSignal.timeout(budget)]);
|
|
865
|
+
const bound = AbortSignal.any([workSignal, AbortSignal.timeout(budget)]);
|
|
807
866
|
try {
|
|
808
|
-
|
|
867
|
+
if (bound.aborted === true) {
|
|
868
|
+
const reason = bound.reason;
|
|
869
|
+
throw reason instanceof Error ? reason : new Error('research budget exhausted');
|
|
870
|
+
}
|
|
871
|
+
const fetched = await withAbort(fetchPage(source.url, bound), bound);
|
|
809
872
|
if (looksLikeHtmlDocument(fetched.text)) {
|
|
810
873
|
failures.push({ url: source.url, reason: 'the page returned a raw HTML document, not readable text' });
|
|
811
874
|
continue;
|
|
@@ -888,6 +951,7 @@ export function createResearchDefinition(name, search, fetchPage, options = {})
|
|
|
888
951
|
...(source.title === undefined || source.title.length === 0 ? {} : { title: source.title }),
|
|
889
952
|
})),
|
|
890
953
|
failures,
|
|
954
|
+
...(searchFailures.length === 0 ? {} : { searchFailures }),
|
|
891
955
|
found: candidates.length,
|
|
892
956
|
truncated: candidates.length > parsed.maxSources,
|
|
893
957
|
};
|
package/lib/search-many.js
CHANGED
|
@@ -130,11 +130,15 @@ function withSite(source) {
|
|
|
130
130
|
* A dedicated helper keeps the concurrency contract visible instead of relying
|
|
131
131
|
* on an unstated promise-pool behaviour.
|
|
132
132
|
*/
|
|
133
|
-
async function mapLimit(items, limit, worker) {
|
|
133
|
+
async function mapLimit(items, limit, worker, signal) {
|
|
134
134
|
const results = new Array(items.length);
|
|
135
135
|
let next = 0;
|
|
136
136
|
const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
|
|
137
137
|
for (;;) {
|
|
138
|
+
if (signal?.aborted === true) {
|
|
139
|
+
const reason = signal.reason;
|
|
140
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
141
|
+
}
|
|
138
142
|
const index = next;
|
|
139
143
|
next += 1;
|
|
140
144
|
if (index >= items.length)
|
|
@@ -190,9 +194,17 @@ export function createSearchManyDefinition(name, runQuery) {
|
|
|
190
194
|
isConcurrencySafe: () => true,
|
|
191
195
|
async execute(args, exec) {
|
|
192
196
|
const parsed = parseArgs(args);
|
|
197
|
+
if (exec.signal?.aborted === true) {
|
|
198
|
+
const reason = exec.signal.reason;
|
|
199
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
200
|
+
}
|
|
193
201
|
const outcomes = await mapLimit(parsed.queries, SEARCH_MANY_CONCURRENCY, async (query) => {
|
|
194
202
|
try {
|
|
195
203
|
const outcome = await runQuery(query, parsed.limit, exec.signal);
|
|
204
|
+
if (exec.signal?.aborted === true) {
|
|
205
|
+
const reason = exec.signal.reason;
|
|
206
|
+
throw reason instanceof Error ? reason : new DOMException('The operation was aborted', 'AbortError');
|
|
207
|
+
}
|
|
196
208
|
const mapped = parsed.withSites
|
|
197
209
|
? outcome.sources.map(withSite)
|
|
198
210
|
: outcome.sources.map((source) => ({ ...source }));
|
|
@@ -208,6 +220,8 @@ export function createSearchManyDefinition(name, runQuery) {
|
|
|
208
220
|
};
|
|
209
221
|
}
|
|
210
222
|
catch (error) {
|
|
223
|
+
if (exec.signal?.aborted === true)
|
|
224
|
+
throw error;
|
|
211
225
|
return {
|
|
212
226
|
result: {
|
|
213
227
|
query,
|
|
@@ -217,7 +231,7 @@ export function createSearchManyDefinition(name, runQuery) {
|
|
|
217
231
|
truncated: false,
|
|
218
232
|
};
|
|
219
233
|
}
|
|
220
|
-
});
|
|
234
|
+
}, exec.signal);
|
|
221
235
|
return {
|
|
222
236
|
results: outcomes.map((entry) => entry.result),
|
|
223
237
|
truncated: outcomes.some((entry) => entry.truncated),
|
package/lib/tool-suite.d.ts
CHANGED
|
@@ -79,6 +79,8 @@ export interface UnifiedToolRegistryLike {
|
|
|
79
79
|
export interface UnifiedToolOptions {
|
|
80
80
|
/** Diagnostics for a rejected registration. */
|
|
81
81
|
warn?: (message: string) => void;
|
|
82
|
+
/** Receives the host's disposer, when registration returns one. */
|
|
83
|
+
onRegister?: (dispose: (() => void) | undefined) => void;
|
|
82
84
|
}
|
|
83
85
|
/**
|
|
84
86
|
* Build the single unified definition.
|
package/lib/tool-suite.js
CHANGED
|
@@ -174,6 +174,13 @@ export function createUnifiedToolDefinition(name, definitions) {
|
|
|
174
174
|
},
|
|
175
175
|
async execute(args, exec) {
|
|
176
176
|
const { action, rest } = parseEnvelope(args);
|
|
177
|
+
// The unified schema exposes `limit` for several actions. advanced_search's
|
|
178
|
+
// native parameter is `maxResults`; translate only when no explicit
|
|
179
|
+
// maxResults was supplied, then let its own parser validate the range.
|
|
180
|
+
if (action === 'advanced_search' && rest['maxResults'] === undefined && rest['limit'] !== undefined) {
|
|
181
|
+
rest['maxResults'] = rest['limit'];
|
|
182
|
+
delete rest['limit'];
|
|
183
|
+
}
|
|
177
184
|
const definition = byAction.get(action);
|
|
178
185
|
if (definition === undefined) {
|
|
179
186
|
throw new Error(`web_tools action ${action} is not registered in this deployment`);
|
|
@@ -208,7 +215,13 @@ export function pickUnifiedToolName(tools) {
|
|
|
208
215
|
export function registerUnifiedTool(tools, definitions, options = {}) {
|
|
209
216
|
const name = pickUnifiedToolName(tools);
|
|
210
217
|
try {
|
|
211
|
-
tools.register(createUnifiedToolDefinition(name, definitions));
|
|
218
|
+
const registered = tools.register(createUnifiedToolDefinition(name, definitions));
|
|
219
|
+
try {
|
|
220
|
+
options.onRegister?.(typeof registered === 'function' ? registered : undefined);
|
|
221
|
+
}
|
|
222
|
+
catch {
|
|
223
|
+
// A registration observer must not break mounting.
|
|
224
|
+
}
|
|
212
225
|
return name;
|
|
213
226
|
}
|
|
214
227
|
catch (error) {
|
package/lib/transport.js
CHANGED
|
@@ -199,9 +199,10 @@ async function attemptWithRoute(host, query, signal, timeoutMs, proxy, timeRange
|
|
|
199
199
|
}
|
|
200
200
|
if (signal?.aborted === true || proxy === undefined)
|
|
201
201
|
return first;
|
|
202
|
+
// When the failed first route was the proxy, the alternate is `undefined`,
|
|
203
|
+
// which means direct — not "no alternate". The old guard here removed the
|
|
204
|
+
// direct fallback exactly when a proxy-first route failed.
|
|
202
205
|
const alternateRoute = firstRoute === proxy ? undefined : proxy;
|
|
203
|
-
if (alternateRoute === undefined)
|
|
204
|
-
return first;
|
|
205
206
|
const retry = await attempt(host, url, signal, timeoutMs, alternateRoute, query);
|
|
206
207
|
if (retry.ok && retry.route !== undefined)
|
|
207
208
|
health?.noteRouteSuccess(host.id, retry.route);
|
package/lib/verticals.js
CHANGED
|
@@ -152,10 +152,13 @@ function openAlexAbstract(value) {
|
|
|
152
152
|
for (const [word, positions] of Object.entries(value)) {
|
|
153
153
|
if (!Array.isArray(positions) || word.trim().length === 0)
|
|
154
154
|
continue;
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
155
|
+
// The inverted index maps one word to every occurrence. Keeping only the
|
|
156
|
+
// first position silently deleted repeated words from the abstract.
|
|
157
|
+
for (const item of positions) {
|
|
158
|
+
if (typeof item === 'number' && Number.isSafeInteger(item) && item >= 0) {
|
|
159
|
+
words.push({ word: word.trim(), position: item });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
159
162
|
}
|
|
160
163
|
words.sort((a, b) => a.position - b.position);
|
|
161
164
|
return words.map((entry) => entry.word).join(' ');
|
|
@@ -1654,8 +1657,11 @@ export async function searchVerticals(query, options = {}) {
|
|
|
1654
1657
|
const answers = await Promise.all(specs.map(async (spec) => {
|
|
1655
1658
|
try {
|
|
1656
1659
|
const built = spec.build(trimmed, limit);
|
|
1660
|
+
// arXiv's builder already encodes fields/booleans correctly. A date
|
|
1661
|
+
// window only asks arXiv to sort newest first; replacing its
|
|
1662
|
+
// `search_query` mangled `ti:`/`cat:` queries into a quoted phrase.
|
|
1657
1663
|
const url = spec.id === 'arxiv' && options.timeRange !== undefined && options.timeRange !== 'off'
|
|
1658
|
-
? `${built
|
|
1664
|
+
? `${built}&sortBy=submittedDate&sortOrder=descending`
|
|
1659
1665
|
: built;
|
|
1660
1666
|
const payload = await fetchPayload(spec, url, options, timeoutMs);
|
|
1661
1667
|
const sources = spec.parse(payload, url, limit);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fffangchenghao/dsh-web-search",
|
|
3
|
-
"version": "0.24.
|
|
3
|
+
"version": "0.24.5",
|
|
4
4
|
"description": "DSH bundle: key-free web search, page fetch and research for DeepSeek Harness; eight engines fused with RRF, 34 vertical sources, arXiv, Windows/macOS system-proxy support and a dependency-free MCP server.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|