@nite07/search-core 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +84 -56
- package/README.zh.md +111 -0
- package/dist/adapters.d.ts +6 -1
- package/dist/adapters.js +65 -15
- package/dist/browser-backends.js +1 -0
- package/dist/browser-limits.d.ts +11 -0
- package/dist/browser-limits.js +11 -0
- package/dist/browser.d.ts +142 -11
- package/dist/browser.js +856 -65
- package/dist/catalog.js +1 -1
- package/dist/config.js +27 -17
- package/dist/index.d.ts +5 -2
- package/dist/index.js +3 -1
- package/dist/static.d.ts +1 -0
- package/dist/static.js +1 -0
- package/dist/types.d.ts +5 -0
- package/dist/xhs/index.d.ts +13 -3
- package/dist/xhs/index.js +77 -44
- package/dist/xhs/render.d.ts +33 -0
- package/dist/xhs/render.js +52 -0
- package/package.json +5 -4
package/README.md
CHANGED
|
@@ -1,19 +1,11 @@
|
|
|
1
1
|
# @nite07/search-core
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
the per-backend adapters, the HTTP transport and a single `SearchEngine`, with no dependency on any
|
|
5
|
-
host framework — the DeepSeek Harness plugin and the MCP server are both thin shells over this.
|
|
3
|
+
English | [中文](README.zh.md)
|
|
6
4
|
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
- **Bounded retries.** Only the configured HTTP statuses are retried, `Retry-After` is honoured, and
|
|
12
|
-
a retry never changes the provider, the mode or the credential.
|
|
13
|
-
- **Honest failures.** A partial failure returns what worked and names what failed; a total failure
|
|
14
|
-
says the search failed instead of returning an empty result. Failures are never cached.
|
|
15
|
-
- **Cost visibility.** Every access mode carries a cost tier and a dated quota note, so a caller can
|
|
16
|
-
show a user the price before enabling anything.
|
|
5
|
+
A provider-agnostic multi-backend web search engine in TypeScript. It ships the backend catalog, the
|
|
6
|
+
per-backend adapters, the HTTP transport and a single `SearchEngine`, with no dependency on any host
|
|
7
|
+
framework — the [DeepSeek Harness plugin](../dsh-search/README.md) and the [MCP server](../websearch-mcp/README.md)
|
|
8
|
+
are both thin shells over this.
|
|
17
9
|
|
|
18
10
|
## Install
|
|
19
11
|
|
|
@@ -21,13 +13,13 @@ host framework — the DeepSeek Harness plugin and the MCP server are both thin
|
|
|
21
13
|
npm install @nite07/search-core
|
|
22
14
|
```
|
|
23
15
|
|
|
24
|
-
##
|
|
16
|
+
## Quick start
|
|
25
17
|
|
|
26
18
|
```ts
|
|
27
|
-
import { SearchEngine,
|
|
19
|
+
import { SearchEngine, defaultConfig } from '@nite07/search-core';
|
|
28
20
|
|
|
29
21
|
const config = defaultConfig();
|
|
30
|
-
config.backends.exa!.enabled = true;
|
|
22
|
+
config.backends.exa!.enabled = true; // anonymous mode: no credential needed
|
|
31
23
|
config.backends.keenable!.enabled = true;
|
|
32
24
|
|
|
33
25
|
const engine = new SearchEngine(
|
|
@@ -36,65 +28,101 @@ const engine = new SearchEngine(
|
|
|
36
28
|
);
|
|
37
29
|
|
|
38
30
|
const result = await engine.search({ query: 'cordis plugin loader', maxResults: 8 });
|
|
39
|
-
console.log(result.sources); // citeable sources
|
|
40
|
-
console.log(result.content); //
|
|
31
|
+
console.log(result.sources); // citeable sources: url, title, snippet, publishedAt
|
|
32
|
+
console.log(result.content); // human-readable report, including any failure
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
Every backend starts disabled, so the two lines above are what turns them on.
|
|
36
|
+
|
|
37
|
+
## Choosing backends
|
|
38
|
+
|
|
39
|
+
`catalog` lists all 24 entries with each backend's access modes, cost tier, quota note and options.
|
|
40
|
+
Most have a free or keyless path; a few need a key, OAuth, a session cookie, or a browser you host.
|
|
41
|
+
|
|
42
|
+
```ts
|
|
43
|
+
import { catalog } from '@nite07/search-core';
|
|
44
|
+
|
|
45
|
+
for (const backend of catalog) {
|
|
46
|
+
console.log(backend.id, backend.modes.map(mode => `${mode.mode}:${mode.tier}`).join(' '));
|
|
47
|
+
}
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
Enable one in the mode you want, then set its options if it has any:
|
|
51
|
+
|
|
52
|
+
```ts
|
|
53
|
+
config.backends.searxng!.enabled = true; // self-hosted, no credential
|
|
54
|
+
config.backends.searxng!.mode = 'anonymous';
|
|
55
|
+
config.backends.searxng!.endpoint = 'http://127.0.0.1:8385/search';
|
|
41
56
|
```
|
|
42
57
|
|
|
43
|
-
|
|
44
|
-
`backendCredentialRef(id)` (`DSH_SEARCH_<BACKEND_ID>_KEY`, with hyphens replaced by underscores and
|
|
45
|
-
the ID uppercased); browser modes resolve only `browserCredentialRef()`
|
|
46
|
-
(`DSH_SEARCH_BROWSER_TOKEN`). Both helpers are exported for hosts that store credentials. Anonymous
|
|
47
|
-
modes never resolve a credential. `parseConfig` drops saved `credentialRef` fields rather than using
|
|
48
|
-
them to redirect a lookup.
|
|
58
|
+
The full per-backend quota and price table: [BACKENDS.md](../dsh-search/docs/BACKENDS.md).
|
|
49
59
|
|
|
50
|
-
|
|
51
|
-
`mode`, `backends`, `exclude`, `backend`, `compact`, `maxResults` — without touching the saved
|
|
52
|
-
configuration.
|
|
60
|
+
## Credentials
|
|
53
61
|
|
|
54
|
-
|
|
62
|
+
Credential names are fixed in code, not configuration fields, and both helpers are exported for hosts
|
|
63
|
+
that store secrets:
|
|
55
64
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
65
|
+
- `backendCredentialRef(id)` — `DSH_SEARCH_<BACKEND_ID>_KEY`, with hyphens replaced by underscores and
|
|
66
|
+
the id uppercased.
|
|
67
|
+
- `browserCredentialRef()` — `DSH_SEARCH_BROWSER_TOKEN`, the single token every browser-backed reader
|
|
68
|
+
uses.
|
|
60
69
|
|
|
61
|
-
|
|
70
|
+
Anonymous modes never resolve a credential, and `parseConfig` drops saved `credentialRef` fields rather
|
|
71
|
+
than letting a file redirect a lookup.
|
|
62
72
|
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
73
|
+
## Results and failures
|
|
74
|
+
|
|
75
|
+
`search()` always resolves with a report; it does not throw for a backend that failed:
|
|
76
|
+
|
|
77
|
+
```ts
|
|
78
|
+
const result = await engine.search({ query: 'x', maxResults: 8 });
|
|
79
|
+
result.sources; // the sources that worked, after dedup and fusion
|
|
80
|
+
result.content; // a readable report naming every failed backend and why
|
|
81
|
+
result.outcomes; // per-backend detail: ok, error kind, duration, attempts
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
A partial failure keeps what worked. A total failure says the search failed instead of returning an
|
|
85
|
+
empty list, and failures are never cached.
|
|
86
|
+
|
|
87
|
+
`search(input, signal?, onlyBackend?, overrides?)` accepts per-call strategy overrides — `mode`,
|
|
88
|
+
`backends`, `exclude`, `backend`, `compact`, `maxResults` — without touching the saved configuration.
|
|
89
|
+
|
|
90
|
+
## Modes of access
|
|
91
|
+
|
|
92
|
+
A backend is used in exactly the mode you selected: `anonymous`, `key`, `oauth`, `session` or
|
|
93
|
+
`browser`. There is no fallback between modes, keys or endpoints, and retries stay on the same provider
|
|
94
|
+
and mode. Only the configured HTTP statuses are retried, `Retry-After` is honoured, and a retry never
|
|
95
|
+
changes the provider.
|
|
96
|
+
|
|
97
|
+
## Browser-backed backends
|
|
98
|
+
|
|
99
|
+
Sites that answer only their own page runtime (currently Xiaohongshu) read through a browser you host.
|
|
100
|
+
Run the container from [`image/`](./image/README.md), sign in once over VNC, then:
|
|
67
101
|
|
|
68
102
|
```ts
|
|
69
103
|
config.browser.endpoint = 'ws://127.0.0.1:3000/ws';
|
|
104
|
+
config.browser.statePath = '/path/to/search-browser/data/state.json';
|
|
105
|
+
config.browser.maxConcurrentReads = 3; // optional per-site cap; default 2, range 1–8
|
|
70
106
|
config.backends.xiaohongshu!.enabled = true;
|
|
71
107
|
config.backends.xiaohongshu!.mode = 'browser';
|
|
72
108
|
```
|
|
73
109
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
missing token does not block attachment to a browser endpoint that accepts unauthenticated clients.
|
|
110
|
+
The endpoint token is resolved from `DSH_SEARCH_BROWSER_TOKEN`. A missing token does not block
|
|
111
|
+
attachment to an endpoint that accepts unauthenticated clients. The library only attaches and only
|
|
112
|
+
reads: signing in and keeping the profile stay with whoever owns that browser.
|
|
78
113
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
profile management stay with whoever owns that browser — this library only attaches, and only reads.
|
|
114
|
+
`maxConcurrentReads` bounds how many reads run on one site at once. It is a local safety bound, not a
|
|
115
|
+
site quota; an out-of-range value is a configuration error rather than a silent clamp.
|
|
82
116
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
the same version; Playwright's protocol refuses a mismatch with `428 Precondition Required`. Reads
|
|
86
|
-
available: `searchXhs`, `fetchXhsNote`, `fetchXhsComments`.
|
|
117
|
+
Direct readers are exported for hosts that want one call without the engine: `searchXhs`,
|
|
118
|
+
`fetchXhsNote`, `fetchXhsComments`, `searchX`, `fetchXThread` and `fetchRedditThread`.
|
|
87
119
|
|
|
88
|
-
|
|
89
|
-
per-post conversation) are mined from that document, then called directly with the cookies. No
|
|
90
|
-
transaction-id signing, no bundle crawling, and a stale operation id is rediscovered once.
|
|
120
|
+
How sessions, generations and recycling work internally: [docs/BROWSER-SESSIONS.md](../../docs/BROWSER-SESSIONS.md).
|
|
91
121
|
|
|
92
|
-
##
|
|
122
|
+
## Field mapping
|
|
93
123
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
npm run check # typecheck, tests, build
|
|
97
|
-
```
|
|
124
|
+
Which response key each adapter reads, verified against the vendor docs, is recorded in
|
|
125
|
+
[docs/FIELD-MAPPING.md](docs/FIELD-MAPPING.md) along with the gaps that are not bugs.
|
|
98
126
|
|
|
99
127
|
## License
|
|
100
128
|
|
package/README.zh.md
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# @nite07/search-core
|
|
2
|
+
|
|
3
|
+
[English](README.md) | 中文
|
|
4
|
+
|
|
5
|
+
一个不绑定宿主的 TypeScript 多后端网页搜索引擎。后端目录、各家的适配层、HTTP 传输和一个 `SearchEngine` 都在这里,不依赖任何宿主框架——[DeepSeek Harness 插件](../dsh-search/README.zh.md) 和 [MCP 服务](../websearch-mcp/README.zh.md) 都只是它的外壳。
|
|
6
|
+
|
|
7
|
+
## 安装
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
npm install @nite07/search-core
|
|
11
|
+
```
|
|
12
|
+
|
|
13
|
+
## 最小示例
|
|
14
|
+
|
|
15
|
+
```ts
|
|
16
|
+
import { SearchEngine, defaultConfig } from '@nite07/search-core';
|
|
17
|
+
|
|
18
|
+
const config = defaultConfig();
|
|
19
|
+
config.backends.exa!.enabled = true; // 匿名模式,不需要凭证
|
|
20
|
+
config.backends.keenable!.enabled = true;
|
|
21
|
+
|
|
22
|
+
const engine = new SearchEngine(
|
|
23
|
+
config,
|
|
24
|
+
async reference => process.env[reference], // 凭证从哪来由宿主决定
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
const result = await engine.search({ query: 'cordis plugin loader', maxResults: 8 });
|
|
28
|
+
console.log(result.sources); // 可引用的来源:url、title、snippet、publishedAt
|
|
29
|
+
console.log(result.content); // 可读的报告,失败原因也在里面
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
所有后端初始都是关闭的,上面那两行就是打开它们的地方。
|
|
33
|
+
|
|
34
|
+
## 选后端
|
|
35
|
+
|
|
36
|
+
`catalog` 列出全部 24 个条目,包含每个后端的接入模式、费用档、额度说明和可选项。大部分有免费或免 Key 的用法,少数需要 Key、OAuth、会话 cookie,或者一个你自己托管的浏览器。
|
|
37
|
+
|
|
38
|
+
```ts
|
|
39
|
+
import { catalog } from '@nite07/search-core';
|
|
40
|
+
|
|
41
|
+
for (const backend of catalog) {
|
|
42
|
+
console.log(backend.id, backend.modes.map(mode => `${mode.mode}:${mode.tier}`).join(' '));
|
|
43
|
+
}
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
按你想要的模式启用,需要的话再设选项:
|
|
47
|
+
|
|
48
|
+
```ts
|
|
49
|
+
config.backends.searxng!.enabled = true; // 自托管,不需要凭证
|
|
50
|
+
config.backends.searxng!.mode = 'anonymous';
|
|
51
|
+
config.backends.searxng!.endpoint = 'http://127.0.0.1:8385/search';
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
各家的额度与价格表见 [BACKENDS.md](../dsh-search/docs/BACKENDS.md)。
|
|
55
|
+
|
|
56
|
+
## 凭证
|
|
57
|
+
|
|
58
|
+
凭证名字固定在代码里,不是配置项;两个辅助函数都导出了,宿主想怎么存密钥都行:
|
|
59
|
+
|
|
60
|
+
- `backendCredentialRef(id)` — `DSH_SEARCH_<后端ID>_KEY`,连字符换成下划线、id 转大写。
|
|
61
|
+
- `browserCredentialRef()` — `DSH_SEARCH_BROWSER_TOKEN`,所有走浏览器的读取共用这一个令牌。
|
|
62
|
+
|
|
63
|
+
匿名模式从不解析凭证,`parseConfig` 会丢弃保存的 `credentialRef` 字段,不让配置文件把查询指到别处。
|
|
64
|
+
|
|
65
|
+
## 结果与失败
|
|
66
|
+
|
|
67
|
+
`search()` 总会给出报告,后端失败不会抛异常:
|
|
68
|
+
|
|
69
|
+
```ts
|
|
70
|
+
const result = await engine.search({ query: 'x', maxResults: 8 });
|
|
71
|
+
result.sources; // 成功的那部分,已去重并融合排序
|
|
72
|
+
result.content; // 可读报告,点名每个失败的后端和原因
|
|
73
|
+
result.outcomes; // 每个后端的细节:ok、错误类型、耗时、尝试次数
|
|
74
|
+
```
|
|
75
|
+
|
|
76
|
+
部分失败保留成功的结果;全部失败时直说搜索失败,而不是回一个空列表。失败的调用不进缓存。
|
|
77
|
+
|
|
78
|
+
`search(input, signal?, onlyBackend?, overrides?)` 支持单次调用的策略覆盖:`mode`、`backends`、`exclude`、`backend`、`compact`、`maxResults`,不改动已保存的配置。
|
|
79
|
+
|
|
80
|
+
## 接入模式
|
|
81
|
+
|
|
82
|
+
后端只按你选定的模式工作:`anonymous`、`key`、`oauth`、`session` 或 `browser`。模式、密钥、端点之间都不会自动切换,重试也留在同一家、同一模式。只重试配置里列出的 HTTP 状态码,`Retry-After` 会被遵守,重试不会换服务商。
|
|
83
|
+
|
|
84
|
+
## 需要浏览器的后端
|
|
85
|
+
|
|
86
|
+
只认自己页面运行时的站点(目前是小红书)走你自己托管的浏览器。按 [`image/`](./image/README.zh.md) 起容器、在 VNC 里登录一次,然后:
|
|
87
|
+
|
|
88
|
+
```ts
|
|
89
|
+
config.browser.endpoint = 'ws://127.0.0.1:3000/ws';
|
|
90
|
+
config.browser.statePath = '/path/to/search-browser/data/state.json';
|
|
91
|
+
config.browser.maxConcurrentReads = 3; // 可选,同站点并发上限;默认 2,范围 1–8
|
|
92
|
+
config.backends.xiaohongshu!.enabled = true;
|
|
93
|
+
config.backends.xiaohongshu!.mode = 'browser';
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
端点令牌从 `DSH_SEARCH_BROWSER_TOKEN` 解析。端点本身允许匿名连接时,缺令牌也不影响连上去。这个库只负责连上去和读取,登录和 profile 始终归那个浏览器的主人。
|
|
97
|
+
|
|
98
|
+
`maxConcurrentReads` 限制同一个站点上同时跑几个读取。它是本地的安全上限,不是站点额度;超出范围会当作配置错误,而不是悄悄截断。
|
|
99
|
+
|
|
100
|
+
不想经过引擎、只想发一次请求的宿主,可以直接用导出的读取函数:`searchXhs`、`fetchXhsNote`、`fetchXhsComments`、`searchX`、`fetchXThread`、`fetchRedditThread`。
|
|
101
|
+
|
|
102
|
+
会话、代际和回收的内部机制见 [docs/BROWSER-SESSIONS.md](../../docs/BROWSER-SESSIONS.md)。
|
|
103
|
+
|
|
104
|
+
## 字段映射
|
|
105
|
+
|
|
106
|
+
每个适配器读哪个响应字段、对着官方文档核实过的结果,以及哪些缺口不算 bug,都记在
|
|
107
|
+
[docs/FIELD-MAPPING.md](docs/FIELD-MAPPING.md)。
|
|
108
|
+
|
|
109
|
+
## 许可
|
|
110
|
+
|
|
111
|
+
MIT。
|
package/dist/adapters.d.ts
CHANGED
|
@@ -5,7 +5,11 @@ import { type TransportOptions } from './transport.js';
|
|
|
5
5
|
* file and the timeout from the shared browser configuration, and needs no other wiring.
|
|
6
6
|
*/
|
|
7
7
|
export interface BrowserRead {
|
|
8
|
-
/**
|
|
8
|
+
/**
|
|
9
|
+
* Browser reads get the endpoint, the optional session file, the per-site read cap and the
|
|
10
|
+
* caller's signal. The cap comes from the shared browser configuration and is part of the session
|
|
11
|
+
* identity, so a reader that drops it would silently join a differently bounded session.
|
|
12
|
+
*/
|
|
9
13
|
search(input: {
|
|
10
14
|
query: string;
|
|
11
15
|
maxResults: number;
|
|
@@ -15,6 +19,7 @@ export interface BrowserRead {
|
|
|
15
19
|
wsEndpoint: string;
|
|
16
20
|
statePath?: string;
|
|
17
21
|
timeoutMs: number;
|
|
22
|
+
maxConcurrentReads: number;
|
|
18
23
|
signal: AbortSignal;
|
|
19
24
|
}): Promise<Source[]>;
|
|
20
25
|
}
|
package/dist/adapters.js
CHANGED
|
@@ -1,17 +1,41 @@
|
|
|
1
1
|
import { requestJson, requestText, SearchError } from './transport.js';
|
|
2
2
|
import { callMcp } from './mcp.js';
|
|
3
3
|
import { languageFor, regionFor } from './locale-map.js';
|
|
4
|
+
import { DEFAULT_MAX_CONCURRENT_READS } from './browser-limits.js';
|
|
4
5
|
import { searchReddit } from './reddit/index.js';
|
|
5
6
|
import { parseXCredentials, searchX } from './x/index.js';
|
|
6
7
|
const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : {};
|
|
7
8
|
const array = (v) => Array.isArray(v) ? v : [];
|
|
8
9
|
const string = (v) => typeof v === 'string' ? v : undefined;
|
|
9
|
-
|
|
10
|
+
/** First populated value among the candidate keys; a provider rename only costs one entry here. */
|
|
11
|
+
function pick(row, spec) {
|
|
12
|
+
for (const key of typeof spec === 'string' ? [spec] : spec) {
|
|
13
|
+
const value = row[key];
|
|
14
|
+
if (value !== undefined && value !== null && value !== '')
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
return undefined;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Dates arrive as strings, and as unix seconds or milliseconds from providers that only expose a
|
|
21
|
+
* numeric timestamp. Anything else is dropped rather than guessed at.
|
|
22
|
+
*/
|
|
23
|
+
function dateString(value) {
|
|
24
|
+
const text = string(value);
|
|
25
|
+
if (text)
|
|
26
|
+
return text;
|
|
27
|
+
if (typeof value !== 'number' || !Number.isFinite(value) || value <= 0)
|
|
28
|
+
return undefined;
|
|
29
|
+
const date = new Date(value < 1e12 ? value * 1000 : value);
|
|
30
|
+
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
31
|
+
}
|
|
32
|
+
function sources(rows, overrides = {}) {
|
|
33
|
+
const keys = { title: 'title', url: 'url', snippet: 'snippet', date: 'published_at', ...overrides };
|
|
10
34
|
if (!Array.isArray(rows))
|
|
11
35
|
throw new SearchError('protocol', 'Expected a results array from provider.');
|
|
12
36
|
return rows.flatMap(v => {
|
|
13
37
|
const r = obj(v);
|
|
14
|
-
const url = string(r
|
|
38
|
+
const url = string(pick(r, keys.url));
|
|
15
39
|
if (!url)
|
|
16
40
|
return [];
|
|
17
41
|
try {
|
|
@@ -24,12 +48,37 @@ function sources(rows, keys = { title: 'title', url: 'url', snippet: 'snippet',
|
|
|
24
48
|
}
|
|
25
49
|
// Every field is optional, and an explicit undefined is not valid JSON: the tool layer refuses
|
|
26
50
|
// such a value outright, so absent fields are simply left out.
|
|
27
|
-
const title = string(r
|
|
28
|
-
const snippet = string(r
|
|
29
|
-
const date =
|
|
51
|
+
const title = string(pick(r, keys.title));
|
|
52
|
+
const snippet = string(pick(r, keys.snippet));
|
|
53
|
+
const date = dateString(pick(r, keys.date));
|
|
30
54
|
return [{ url, ...(title ? { title } : {}), ...(snippet ? { snippet } : {}), ...(date ? { publishedAt: date } : {}) }];
|
|
31
55
|
});
|
|
32
56
|
}
|
|
57
|
+
/** MCP servers name the result body differently; these are the keys seen in practice, most specific first. */
|
|
58
|
+
const MCP_TEXT_KEYS = ['snippet', 'snippets', 'text', 'description', 'summary', 'excerpt', 'excerpts', 'highlights', 'content', 'body'];
|
|
59
|
+
const MCP_DATE_KEYS = ['published_at', 'publishedAt', 'publishedDate', 'published', 'page_age', 'date'];
|
|
60
|
+
/** The first populated candidate as text: a string as-is, an array of strings or of {text} joined. */
|
|
61
|
+
function pickText(row, keys) {
|
|
62
|
+
for (const key of keys) {
|
|
63
|
+
const value = row[key];
|
|
64
|
+
const joined = typeof value === 'string' ? value : Array.isArray(value) ? value.map(v => typeof v === 'string' ? v : string(obj(v).text) || '').filter(Boolean).join('\n') : undefined;
|
|
65
|
+
if (joined?.trim())
|
|
66
|
+
return joined.trim().slice(0, 2000);
|
|
67
|
+
}
|
|
68
|
+
return undefined;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Normalise one MCP result row. The original row is kept so the provider keeps its own fields; the
|
|
72
|
+
* prefixed keys are what `sources()` reads. Trusting a single key would silently drop every snippet
|
|
73
|
+
* the moment a server renames its body field.
|
|
74
|
+
*/
|
|
75
|
+
function mcpRow(value) {
|
|
76
|
+
const row = obj(value);
|
|
77
|
+
const snippet = pickText(row, MCP_TEXT_KEYS);
|
|
78
|
+
const title = typeof row.title === 'string' && row.title.trim() ? row.title : string(row.name);
|
|
79
|
+
const publishedAt = pickText(row, MCP_DATE_KEYS);
|
|
80
|
+
return { ...row, ...(title ? { title } : {}), ...(snippet ? { snippet } : {}), ...(publishedAt ? { published_at: publishedAt } : {}) };
|
|
81
|
+
}
|
|
33
82
|
export const browserBackends = {};
|
|
34
83
|
import { registerBrowserBackends } from './browser-backends.js';
|
|
35
84
|
registerBrowserBackends(browserBackends);
|
|
@@ -47,7 +96,8 @@ export async function runAdapter(id, i, t) {
|
|
|
47
96
|
throw new SearchError('configuration', `${id} reads through a browser, but no endpoint is configured. Set the shared browser endpoint, or this backend's own.`);
|
|
48
97
|
const statePath = String(o.statePath || c.browser?.statePath || '');
|
|
49
98
|
const timeoutMs = Number(c.browser?.timeoutMs || b.timeoutMs);
|
|
50
|
-
const
|
|
99
|
+
const maxConcurrentReads = Number(c.browser?.maxConcurrentReads ?? DEFAULT_MAX_CONCURRENT_READS);
|
|
100
|
+
const sources = await browser.search({ query: i.query, maxResults: i.maxResults, options: o, key: i.key }, { wsEndpoint: endpoint, ...(statePath ? { statePath } : {}), timeoutMs, maxConcurrentReads, signal: t.signal });
|
|
51
101
|
return { sources };
|
|
52
102
|
}
|
|
53
103
|
if (id === 'x') {
|
|
@@ -99,7 +149,7 @@ export async function runAdapter(id, i, t) {
|
|
|
99
149
|
const r = obj(await json({ query: i.query, max_results: Math.min(i.maxResults, 10), zone: o.zone, format: 'json', ...(languageFor('anysearch', c.language) ? { language: languageFor('anysearch', c.language) } : {}) }));
|
|
100
150
|
if (r.code !== undefined && r.code !== 0 && r.code !== 200)
|
|
101
151
|
throw new SearchError('protocol', 'AnySearch reported an application-level error; inspect the account console.');
|
|
102
|
-
return { sources: sources(obj(r.data).results) };
|
|
152
|
+
return { sources: sources(obj(r.data).results, { title: 'title', url: 'url', snippet: ['snippet', 'content'] }) };
|
|
103
153
|
}
|
|
104
154
|
if (id === 'linkup') {
|
|
105
155
|
bearer();
|
|
@@ -145,7 +195,7 @@ export async function runAdapter(id, i, t) {
|
|
|
145
195
|
const result = await callMcp(endpoint.href, tool, args, t);
|
|
146
196
|
const structured = obj(result.structuredContent);
|
|
147
197
|
if (Array.isArray(structured.results))
|
|
148
|
-
return { sources: sources(structured.results) };
|
|
198
|
+
return { sources: sources(array(structured.results).map(mcpRow)) };
|
|
149
199
|
const text = array(result.content).map(v => string(obj(v).text) || '').join('\n');
|
|
150
200
|
if (!text.trim())
|
|
151
201
|
throw new SearchError('protocol', 'MCP search response has no recognized result payload.');
|
|
@@ -153,7 +203,7 @@ export async function runAdapter(id, i, t) {
|
|
|
153
203
|
const parsed = obj(JSON.parse(text));
|
|
154
204
|
const list = [parsed.results, obj(parsed.data).results, obj(parsed.results).web].find(Array.isArray);
|
|
155
205
|
if (Array.isArray(list))
|
|
156
|
-
return { sources: sources(list) };
|
|
206
|
+
return { sources: sources(list.map(mcpRow)) };
|
|
157
207
|
}
|
|
158
208
|
catch (e) {
|
|
159
209
|
if (e instanceof SearchError)
|
|
@@ -186,14 +236,14 @@ export async function runAdapter(id, i, t) {
|
|
|
186
236
|
if (id === 'baidu') {
|
|
187
237
|
bearer();
|
|
188
238
|
const r = obj(await json({ messages: [{ role: 'user', content: i.query }], search_source: 'baidu_search_v2', edition: o.edition, resource_type_filter: [{ type: 'web', top_k: i.maxResults }] }));
|
|
189
|
-
return { sources: sources(r.references, { title: 'title', url: 'url', snippet: 'snippet', date: 'date' }) };
|
|
239
|
+
return { sources: sources(r.references, { title: 'title', url: 'url', snippet: ['snippet', 'content'], date: 'date' }) };
|
|
190
240
|
}
|
|
191
241
|
if (id === 'doubao') {
|
|
192
242
|
bearer();
|
|
193
243
|
const r = obj(await json({ Query: i.query, SearchType: 'web', Count: i.maxResults, Filter: { NeedUrl: true }, QueryControl: { QueryRewrite: o.queryRewrite } }));
|
|
194
244
|
if (obj(r.ResponseMetadata).Error)
|
|
195
245
|
throw new SearchError('protocol', 'Doubao returned an application-level error.');
|
|
196
|
-
return { sources: sources(obj(r.Result).WebResults, { title: 'Title', url: 'Url', snippet: 'Snippet', date: 'PublishTime' }) };
|
|
246
|
+
return { sources: sources(obj(r.Result).WebResults, { title: 'Title', url: 'Url', snippet: ['Summary', 'Snippet'], date: 'PublishTime' }) };
|
|
197
247
|
}
|
|
198
248
|
if (id === 'zhihu-web' || id === 'zhihu-site') {
|
|
199
249
|
bearer();
|
|
@@ -201,18 +251,18 @@ export async function runAdapter(id, i, t) {
|
|
|
201
251
|
const r = obj(await get({ Query: i.query, Count: Math.min(i.maxResults, id === 'zhihu-web' ? 20 : 10), ...(id === 'zhihu-web' ? { SearchDB: o.SearchDB, Filter: o.Filter } : {}) }));
|
|
202
252
|
if (r.Code !== 0)
|
|
203
253
|
throw new SearchError('protocol', 'Zhihu reported an application-level failure; check Access Secret and account permissions.');
|
|
204
|
-
return { sources: sources(obj(r.Data).Items, { title: 'Title', url: 'Url', snippet: 'ContentText', date: 'PublishTime' }) };
|
|
254
|
+
return { sources: sources(obj(r.Data).Items, { title: 'Title', url: 'Url', snippet: 'ContentText', date: ['PublishTime', 'EditTime', 'CreateTime'] }) };
|
|
205
255
|
}
|
|
206
256
|
if (id === 'open-websearch') {
|
|
207
257
|
const r = obj(await json({ query: i.query, limit: i.maxResults, engines: [o.engine], searchMode: o.searchMode }));
|
|
208
258
|
if (r.status !== 'ok')
|
|
209
259
|
throw new SearchError('protocol', 'Self-hosted search daemon reported failure.');
|
|
210
|
-
return { sources: sources(r.data, { title: 'title', url: 'url', snippet: 'description'
|
|
260
|
+
return { sources: sources(obj(r.data).results, { title: 'title', url: 'url', snippet: 'description' }) };
|
|
211
261
|
}
|
|
212
262
|
if (id === 'serpex') {
|
|
213
263
|
bearer();
|
|
214
264
|
const r = obj(await json({ q: i.query }));
|
|
215
|
-
return { sources: sources(r.results), warnings: ['Serpex controls its own internal upstream routing; the plugin did not switch providers.'] };
|
|
265
|
+
return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'snippet' }), warnings: ['Serpex controls its own internal upstream routing; the plugin did not switch providers.'] };
|
|
216
266
|
}
|
|
217
267
|
if (id === 'mojeek') {
|
|
218
268
|
const r = obj(await get({ q: i.query, api_key: key, fmt: 'json' }));
|
|
@@ -240,7 +290,7 @@ export async function runAdapter(id, i, t) {
|
|
|
240
290
|
for (const x of array(v.content)) {
|
|
241
291
|
const s = obj(x);
|
|
242
292
|
if (s.type === 'web_search_result')
|
|
243
|
-
result.push(...sources([s], { title: 'title', url: 'url',
|
|
293
|
+
result.push(...sources([s], { title: 'title', url: 'url', date: 'page_age' }));
|
|
244
294
|
}
|
|
245
295
|
}
|
|
246
296
|
}
|
package/dist/browser-backends.js
CHANGED
|
@@ -17,6 +17,7 @@ export function registerBrowserBackends(registry) {
|
|
|
17
17
|
...(input.key ? { token: input.key } : {}),
|
|
18
18
|
...(ctx.statePath ? { statePath: ctx.statePath } : {}),
|
|
19
19
|
timeoutMs: ctx.timeoutMs,
|
|
20
|
+
maxConcurrentReads: ctx.maxConcurrentReads,
|
|
20
21
|
signal: ctx.signal,
|
|
21
22
|
});
|
|
22
23
|
},
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local admission bound for browser reads, kept in a browser-safe module.
|
|
3
|
+
*
|
|
4
|
+
* The configuration schema, the settings UI and `BrowserSession` all share these numbers. They live
|
|
5
|
+
* outside `browser.ts` on purpose: `browser.ts` imports patchright, and the `/static` entry that a
|
|
6
|
+
* settings client bundles must never pull a browser driver in. This file has no imports at all.
|
|
7
|
+
*/
|
|
8
|
+
/** Conservative local starting point; not a Xiaohongshu quota and not verified against the site. */
|
|
9
|
+
export declare const DEFAULT_MAX_CONCURRENT_READS = 2;
|
|
10
|
+
export declare const MIN_MAX_CONCURRENT_READS = 1;
|
|
11
|
+
export declare const MAX_CONCURRENT_READS = 8;
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The local admission bound for browser reads, kept in a browser-safe module.
|
|
3
|
+
*
|
|
4
|
+
* The configuration schema, the settings UI and `BrowserSession` all share these numbers. They live
|
|
5
|
+
* outside `browser.ts` on purpose: `browser.ts` imports patchright, and the `/static` entry that a
|
|
6
|
+
* settings client bundles must never pull a browser driver in. This file has no imports at all.
|
|
7
|
+
*/
|
|
8
|
+
/** Conservative local starting point; not a Xiaohongshu quota and not verified against the site. */
|
|
9
|
+
export const DEFAULT_MAX_CONCURRENT_READS = 2;
|
|
10
|
+
export const MIN_MAX_CONCURRENT_READS = 1;
|
|
11
|
+
export const MAX_CONCURRENT_READS = 8;
|