@240xu/dsh-websearch 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/lib/backends/anthropic-like.js +3 -13
- package/lib/backends/brave.js +4 -2
- package/lib/backends/ddg.js +1 -1
- package/lib/backends/exa.js +1 -1
- package/lib/backends/mojeek.js +4 -2
- package/lib/backends/openai.js +2 -2
- package/lib/backends/parallel.js +22 -8
- package/lib/backends/searxng.js +7 -2
- package/lib/backends/serper.js +4 -2
- package/lib/backends/tavily.js +4 -2
- package/lib/client.js +47 -15
- package/lib/provider.js +28 -11
- package/lib/util/log.js +1 -1
- package/lib/util/mcp-client.js +15 -14
- package/lib/util/rank.js +1 -1
- package/package.json +1 -1
- package/lib/backends/index.js +0 -121
- package/lib/util/rpc.js +0 -262
package/README.md
CHANGED
|
@@ -29,7 +29,7 @@ DSH([DeepSeek Harness](https://github.com/deepseek-ai/dsh))原生插件:
|
|
|
29
29
|
| `anthropic` | `ANTHROPIC_API_KEY` | `https://api.anthropic.com/v1/messages` | Anthropic Messages + `web_search_20250305`, model `claude-sonnet-4-6` | Claude Code 同机制 |
|
|
30
30
|
| `openai` | `OPENAI_API_KEY` | `https://api.openai.com/v1/responses` | Responses API native `web_search` tool, parse `url_citation` annotations | Codex 同机制 |
|
|
31
31
|
|
|
32
|
-
|
|
32
|
+
> 可用性说明:available() 只做本地配置检查(不联网探测可达性);密钥在每次搜索时按操作解析,缺失时该后端以明确错误码软失败(不阻塞其他后端)。全部缺 Key 时错误会指向设置页。
|
|
33
33
|
|
|
34
34
|
### SearXNG 实例 | Public instances
|
|
35
35
|
|
|
@@ -125,7 +125,7 @@ pnpm install
|
|
|
125
125
|
- **Honest timeouts**: each backend runs under an internal abort controller merged into the caller signal; when only the internal timer fires, the failure is reclassified as WEB_PROVIDER_ERROR ("backend <id> timed out") instead of being masked as a user cancellation.
|
|
126
126
|
- **Concurrency & Timeout Control**: `concurrency` (default 6) limits simultaneous calls; `backendTimeoutMs` (default 30s) caps each backend.
|
|
127
127
|
- **Streamable-http MCP, custom handshake**: `lib/util/mcp-client.js` implements `initialize → notifications/initialized → tools/call` with `Mcp-Session-Id` header caching against Exa and Parallel — no dependency on the full MCP SDK.
|
|
128
|
-
- **
|
|
128
|
+
- **Host-logger diagnostics**: each backend routes request/outcome through lib/util/log.js to the host logger (ctx.logger), best-effort and never thrown - deliberately NOT session events: the session event vocabulary is a closed set third-party plugins must not extend (an unknown envelope type makes the whole session log unreadable).
|
|
129
129
|
|
|
130
130
|
## Architecture | 架构
|
|
131
131
|
|
|
@@ -157,7 +157,7 @@ tests/
|
|
|
157
157
|
## Test | 测试
|
|
158
158
|
|
|
159
159
|
```bash
|
|
160
|
-
node --test tests/
|
|
160
|
+
node --test tests/
|
|
161
161
|
```
|
|
162
162
|
|
|
163
163
|
33/33 pass.
|
|
@@ -19,7 +19,7 @@ import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort
|
|
|
19
19
|
import { WebError } from "@deepseek-ai/dsh-web";
|
|
20
20
|
|
|
21
21
|
/** Attribution header; bumped with the package version. */
|
|
22
|
-
const UA = "dsh-websearch/2.
|
|
22
|
+
const UA = "dsh-websearch/2.3";
|
|
23
23
|
|
|
24
24
|
/**
|
|
25
25
|
* Factory: create an Anthropic-like Messages backend.
|
|
@@ -30,7 +30,6 @@ const UA = "dsh-websearch/2.1";
|
|
|
30
30
|
* @param {string} spec.defaultModel
|
|
31
31
|
* @param {string} spec.defaultApiVersion e.g. "2023-06-01"
|
|
32
32
|
* @param {string} spec.toolName "web_search_20250305"
|
|
33
|
-
* @param {string} [spec.futureToolName] optional newer tool for dual fallback
|
|
34
33
|
* @returns {object} backend object { id, requiresCredential, available, search }
|
|
35
34
|
*/
|
|
36
35
|
export function createAnthropicLikeBackend(spec) {
|
|
@@ -40,7 +39,6 @@ export function createAnthropicLikeBackend(spec) {
|
|
|
40
39
|
defaultModel,
|
|
41
40
|
defaultApiVersion,
|
|
42
41
|
toolName,
|
|
43
|
-
futureToolName,
|
|
44
42
|
} = spec;
|
|
45
43
|
|
|
46
44
|
return {
|
|
@@ -154,7 +152,7 @@ export function createAnthropicLikeBackend(spec) {
|
|
|
154
152
|
}
|
|
155
153
|
|
|
156
154
|
try {
|
|
157
|
-
const result = mapAnthropicResponse(await response.json(), id
|
|
155
|
+
const result = mapAnthropicResponse(await response.json(), id);
|
|
158
156
|
hooks.recordOutcome?.({ status: "ok", sources: result.sources.length });
|
|
159
157
|
return { sources: result.sources, content: undefined, truncated: false };
|
|
160
158
|
} catch (err) {
|
|
@@ -174,15 +172,7 @@ export function createAnthropicLikeBackend(spec) {
|
|
|
174
172
|
};
|
|
175
173
|
}
|
|
176
174
|
|
|
177
|
-
|
|
178
|
-
* Optional dual-tool fallback: try the newer tool first; if that tool name is
|
|
179
|
-
* rejected by the API (HTTP error mapping is upstream of us), the provider
|
|
180
|
-
* retries with the established fallback. Here we just expose both — the
|
|
181
|
-
* provider loops through candidates and uses whichever result block appears.
|
|
182
|
-
*/
|
|
183
|
-
export function tryDualToolResponse(response, id, primaryTool, fallbackTool) {
|
|
184
|
-
return mapAnthropicResponse(response, id, primaryTool);
|
|
185
|
-
}
|
|
175
|
+
|
|
186
176
|
|
|
187
177
|
/**
|
|
188
178
|
* Walk web_search_tool_result blocks for citeable web_search_result items,
|
package/lib/backends/brave.js
CHANGED
|
@@ -51,8 +51,10 @@ export const braveBackend = {
|
|
|
51
51
|
headers: {
|
|
52
52
|
"X-Subscription-Token": apiKey,
|
|
53
53
|
"Accept": "application/json",
|
|
54
|
-
"User-Agent": "dsh-websearch/2.
|
|
54
|
+
"User-Agent": "dsh-websearch/2.3",
|
|
55
55
|
},
|
|
56
|
+
redirect: "error",
|
|
57
|
+
|
|
56
58
|
signal,
|
|
57
59
|
});
|
|
58
60
|
} catch (error) {
|
|
@@ -61,7 +63,7 @@ export const braveBackend = {
|
|
|
61
63
|
}
|
|
62
64
|
|
|
63
65
|
if (!response.ok) {
|
|
64
|
-
const text = await response.text().catch(() => "");
|
|
66
|
+
const text = await response.text().catch(() => "").then((tx) => tx.slice(0, 300));
|
|
65
67
|
hooks?.recordOutcome?.({ status: "error", message: `HTTP ${response.status}: ${text}` });
|
|
66
68
|
throw new WebError(`Brave Search API error (HTTP ${response.status}): ${text}`, "WEB_PROVIDER_ERROR");
|
|
67
69
|
}
|
package/lib/backends/ddg.js
CHANGED
|
@@ -40,7 +40,7 @@ export const ddgBackend = {
|
|
|
40
40
|
headers: {
|
|
41
41
|
"Content-Type": "application/x-www-form-urlencoded",
|
|
42
42
|
"Accept": "text/html",
|
|
43
|
-
"User-Agent": "dsh-websearch/2.
|
|
43
|
+
"User-Agent": "dsh-websearch/2.3",
|
|
44
44
|
},
|
|
45
45
|
body,
|
|
46
46
|
...(signal !== undefined ? { signal } : {}),
|
package/lib/backends/exa.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// .../Highlights: .../ ..." separated by "\n\n---\n\n"; we split into source objects.
|
|
8
8
|
// Exa is keyless — its public MCP endpoint requires no API key.
|
|
9
9
|
|
|
10
|
-
import { mcpCall
|
|
10
|
+
import { mcpCall } from "../util/mcp-client.js";
|
|
11
11
|
import { WebError } from "@deepseek-ai/dsh-web";
|
|
12
12
|
|
|
13
13
|
const EXA_ENDPOINT = "https://mcp.exa.ai/mcp";
|
package/lib/backends/mojeek.js
CHANGED
|
@@ -41,7 +41,7 @@ export const mojeekBackend = {
|
|
|
41
41
|
const headers = {
|
|
42
42
|
"Accept": "application/json",
|
|
43
43
|
"Authorization": "Bearer " + apiKey,
|
|
44
|
-
"User-Agent": "dsh-websearch/2.
|
|
44
|
+
"User-Agent": "dsh-websearch/2.3",
|
|
45
45
|
};
|
|
46
46
|
|
|
47
47
|
hooks?.recordRequest?.({
|
|
@@ -55,6 +55,8 @@ export const mojeekBackend = {
|
|
|
55
55
|
response = await fetch(url, {
|
|
56
56
|
method: "GET",
|
|
57
57
|
headers,
|
|
58
|
+
redirect: "error",
|
|
59
|
+
|
|
58
60
|
signal,
|
|
59
61
|
});
|
|
60
62
|
} catch (error) {
|
|
@@ -63,7 +65,7 @@ export const mojeekBackend = {
|
|
|
63
65
|
}
|
|
64
66
|
|
|
65
67
|
if (!response.ok) {
|
|
66
|
-
const text = await response.text().catch(() => "");
|
|
68
|
+
const text = await response.text().catch(() => "").then((tx) => tx.slice(0, 300));
|
|
67
69
|
hooks?.recordOutcome?.({ status: "error", message: "HTTP " + response.status + ": " + text });
|
|
68
70
|
throw new WebError("Mojeek API error (HTTP " + response.status + "): " + text, "WEB_PROVIDER_ERROR");
|
|
69
71
|
}
|
package/lib/backends/openai.js
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
import { throwIfSearchAborted, searchAborted, isAbortError } from "../util/abort.js";
|
|
14
14
|
import { WebError } from "@deepseek-ai/dsh-web";
|
|
15
15
|
|
|
16
|
-
const UA = "dsh-websearch/2.
|
|
16
|
+
const UA = "dsh-websearch/2.3";
|
|
17
17
|
|
|
18
18
|
export const openaiBackend = {
|
|
19
19
|
id: "openai",
|
|
@@ -47,7 +47,7 @@ export const openaiBackend = {
|
|
|
47
47
|
);
|
|
48
48
|
}
|
|
49
49
|
const baseURL = (options.baseURL || "https://api.openai.com/v1").replace(/\/$/, "");
|
|
50
|
-
const model = options.model || "gpt-
|
|
50
|
+
const model = options.model || "gpt-4o";
|
|
51
51
|
const endpoint = `${baseURL}/responses`;
|
|
52
52
|
|
|
53
53
|
const body = {
|
package/lib/backends/parallel.js
CHANGED
|
@@ -13,18 +13,15 @@
|
|
|
13
13
|
// JSON-string with { search_id, results: [{ url, title, publish_date, excerpts:[] }] }.
|
|
14
14
|
// Parallel is keyless — its public MCP endpoint requires no API key.
|
|
15
15
|
|
|
16
|
-
import {
|
|
16
|
+
import { randomUUID } from "node:crypto";
|
|
17
|
+
import { mcpCall } from "../util/mcp-client.js";
|
|
17
18
|
import { WebError } from "@deepseek-ai/dsh-web";
|
|
18
19
|
|
|
19
20
|
const PARALLEL_ENDPOINT = "https://search.parallel.ai/mcp";
|
|
20
21
|
const PARALLEL_TOOL = "web_search";
|
|
21
22
|
|
|
22
|
-
/** Per-process session id
|
|
23
|
-
|
|
24
|
-
const SESSION_ID =
|
|
25
|
-
"dsh-websearch-" +
|
|
26
|
-
process.pid?.toString(36) + "-" +
|
|
27
|
-
Math.random().toString(36).slice(2, 10);
|
|
23
|
+
/** Per-process session id - stable so Parallel free-tier rate limiting correlates calls. */
|
|
24
|
+
const SESSION_ID = "dsh-websearch-" + randomUUID().slice(0, 8);
|
|
28
25
|
|
|
29
26
|
export const parallelBackend = {
|
|
30
27
|
id: "parallel",
|
|
@@ -60,9 +57,26 @@ export const parallelBackend = {
|
|
|
60
57
|
);
|
|
61
58
|
}
|
|
62
59
|
|
|
63
|
-
|
|
60
|
+
if (result?.isError === true) {
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
const message = String(result.content?.[0]?.text ?? "unknown");
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
hooks.recordOutcome?.({ status: "error", message });
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
throw new WebError(`parallel returned an MCP error: ${message}`, "WEB_PROVIDER_ERROR");
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
|
|
64
75
|
const sources = parseParallelResult(result, request.maxResults ?? 8);
|
|
65
76
|
|
|
77
|
+
|
|
78
|
+
hooks.recordOutcome?.({ status: "ok", count: sources.length });
|
|
79
|
+
|
|
66
80
|
if (result?.isError === true) {
|
|
67
81
|
throw new WebError(
|
|
68
82
|
`parallel returned an MCP error: ${String(result.content?.[0]?.text ?? "unknown")}`,
|
package/lib/backends/searxng.js
CHANGED
|
@@ -44,7 +44,7 @@ export const searxngBackend = {
|
|
|
44
44
|
|
|
45
45
|
const headers = {
|
|
46
46
|
"Accept": "application/json",
|
|
47
|
-
"User-Agent": "dsh-websearch/2.
|
|
47
|
+
"User-Agent": "dsh-websearch/2.3",
|
|
48
48
|
};
|
|
49
49
|
|
|
50
50
|
if (apiKey) {
|
|
@@ -62,6 +62,8 @@ export const searxngBackend = {
|
|
|
62
62
|
response = await fetch(url, {
|
|
63
63
|
method: "GET",
|
|
64
64
|
headers,
|
|
65
|
+
redirect: "error",
|
|
66
|
+
|
|
65
67
|
signal,
|
|
66
68
|
});
|
|
67
69
|
} catch (error) {
|
|
@@ -71,8 +73,11 @@ export const searxngBackend = {
|
|
|
71
73
|
}
|
|
72
74
|
|
|
73
75
|
if (!response.ok) {
|
|
74
|
-
const text = await response.text().catch(() => "");
|
|
76
|
+
const text = await response.text().catch(() => "").then((tx) => tx.slice(0, 300));
|
|
75
77
|
hooks?.recordOutcome?.({ status: "error", message: "HTTP " + response.status + ": " + text });
|
|
78
|
+
if (response.status === 429 || response.status >= 500) {
|
|
79
|
+
throw new WebError("SearXNG API error (HTTP " + response.status + "): " + text + " (instance struggling? point searxngBaseURL at another public instance)", "WEB_PROVIDER_ERROR");
|
|
80
|
+
}
|
|
76
81
|
throw new WebError("SearXNG API error (HTTP " + response.status + "): " + text, "WEB_PROVIDER_ERROR");
|
|
77
82
|
}
|
|
78
83
|
|
package/lib/backends/serper.js
CHANGED
|
@@ -52,9 +52,11 @@ export const serperBackend = {
|
|
|
52
52
|
"X-API-KEY": apiKey,
|
|
53
53
|
"Content-Type": "application/json",
|
|
54
54
|
"Accept": "application/json",
|
|
55
|
-
"User-Agent": "dsh-websearch/2.
|
|
55
|
+
"User-Agent": "dsh-websearch/2.3",
|
|
56
56
|
},
|
|
57
57
|
body: JSON.stringify(body),
|
|
58
|
+
redirect: "error",
|
|
59
|
+
|
|
58
60
|
signal,
|
|
59
61
|
});
|
|
60
62
|
} catch (error) {
|
|
@@ -63,7 +65,7 @@ export const serperBackend = {
|
|
|
63
65
|
}
|
|
64
66
|
|
|
65
67
|
if (!response.ok) {
|
|
66
|
-
const text = await response.text().catch(() => "");
|
|
68
|
+
const text = await response.text().catch(() => "").then((tx) => tx.slice(0, 300));
|
|
67
69
|
hooks?.recordOutcome?.({ status: "error", message: "HTTP " + response.status + ": " + text });
|
|
68
70
|
throw new WebError("Serper API error (HTTP " + response.status + "): " + text, "WEB_PROVIDER_ERROR");
|
|
69
71
|
}
|
package/lib/backends/tavily.js
CHANGED
|
@@ -56,9 +56,11 @@ export const tavilyBackend = {
|
|
|
56
56
|
"Authorization": `Bearer ${apiKey}`,
|
|
57
57
|
"Content-Type": "application/json",
|
|
58
58
|
"Accept": "application/json",
|
|
59
|
-
"User-Agent": "dsh-websearch/2.
|
|
59
|
+
"User-Agent": "dsh-websearch/2.3",
|
|
60
60
|
},
|
|
61
61
|
body: JSON.stringify(body),
|
|
62
|
+
redirect: "error",
|
|
63
|
+
|
|
62
64
|
signal,
|
|
63
65
|
});
|
|
64
66
|
} catch (error) {
|
|
@@ -67,7 +69,7 @@ export const tavilyBackend = {
|
|
|
67
69
|
}
|
|
68
70
|
|
|
69
71
|
if (!response.ok) {
|
|
70
|
-
const text = await response.text().catch(() => "");
|
|
72
|
+
const text = await response.text().catch(() => "").then((tx) => tx.slice(0, 300));
|
|
71
73
|
hooks?.recordOutcome?.({ status: "error", message: `HTTP ${response.status}: ${text}` });
|
|
72
74
|
throw new WebError(`Tavily API error (HTTP ${response.status}): ${text}`, "WEB_PROVIDER_ERROR");
|
|
73
75
|
}
|
package/lib/client.js
CHANGED
|
@@ -15,7 +15,7 @@ window.__ModuleLoader__.load({
|
|
|
15
15
|
{ id: "exa", keyEnv: null, label: "Exa (MCP, 免费)" },
|
|
16
16
|
{ id: "parallel", keyEnv: null, label: "Parallel (MCP, 免费)" },
|
|
17
17
|
{ id: "ddg", keyEnv: null, label: "DuckDuckGo (HTML)" },
|
|
18
|
-
{ id: "searxng", keyEnv: "SEARXNG_API_KEY", label: "SearXNG (元搜索)" },
|
|
18
|
+
{ id: "searxng", keyEnv: "SEARXNG_API_KEY", optionalKey: true, label: "SearXNG (元搜索)" },
|
|
19
19
|
{ id: "brave", keyEnv: "BRAVE_API_KEY", label: "Brave Search" },
|
|
20
20
|
{ id: "tavily", keyEnv: "TAVILY_API_KEY", label: "Tavily (AI)" },
|
|
21
21
|
{ id: "serper", keyEnv: "SERPER_API_KEY", label: "Serper.dev" },
|
|
@@ -32,8 +32,9 @@ window.__ModuleLoader__.load({
|
|
|
32
32
|
};
|
|
33
33
|
/** Boolean settings staged as "true"/"false" strings by the selects. */
|
|
34
34
|
const BOOLEAN_FIELDS = ["rerank", "safeSearch", "resultTelemetry"];
|
|
35
|
-
const KEYLESS = BACKENDS.filter((b) => !b.keyEnv);
|
|
36
|
-
const KEYED = BACKENDS.filter((b) => b.keyEnv);
|
|
35
|
+
const KEYLESS = BACKENDS.filter((b) => !b.keyEnv || b.optionalKey);
|
|
36
|
+
const KEYED = BACKENDS.filter((b) => b.keyEnv && !b.optionalKey);
|
|
37
|
+
const OPTIONAL_KEYS = BACKENDS.filter((b) => b.keyEnv && b.optionalKey);
|
|
37
38
|
|
|
38
39
|
/** Where the user goes to get/rotate each key. */
|
|
39
40
|
const KEY_URLS = {
|
|
@@ -90,6 +91,10 @@ window.__ModuleLoader__.load({
|
|
|
90
91
|
langPlaceholder: "如 en、zh-CN,留空为默认",
|
|
91
92
|
envProvided: "环境变量提供", readOnly: "只读,无需填写", conflict: "设置已在其他端被修改,请重试", keyFailed: "该 Key 保存失败",
|
|
92
93
|
telemetryLabel: "结果健康遥测",
|
|
94
|
+
fallbackHint: "全部后端已关闭:搜索将回退到默认免 Key 后端",
|
|
95
|
+
invalidFields: "以下字段未通过校验:",
|
|
96
|
+
readOnlyScope: "当前配置只读(可能由宿主锁定)",
|
|
97
|
+
optionalTitle: "私有实例可选 Key",
|
|
93
98
|
};
|
|
94
99
|
const en = {
|
|
95
100
|
nav: "Web Search",
|
|
@@ -138,6 +143,10 @@ window.__ModuleLoader__.load({
|
|
|
138
143
|
conflict: "Changed elsewhere - please retry",
|
|
139
144
|
keyFailed: "failed to save this key",
|
|
140
145
|
telemetryLabel: "Result health telemetry",
|
|
146
|
+
fallbackHint: "All backends off: search falls back to the default keyless four.",
|
|
147
|
+
invalidFields: "Fields failed validation:",
|
|
148
|
+
readOnlyScope: "Configuration is read-only (locked by host?)",
|
|
149
|
+
optionalTitle: "Private instance optional keys",
|
|
141
150
|
};
|
|
142
151
|
|
|
143
152
|
//#region styles
|
|
@@ -177,7 +186,8 @@ window.__ModuleLoader__.load({
|
|
|
177
186
|
this.saving = false;
|
|
178
187
|
this.failed = false;
|
|
179
188
|
this.conflict = false;
|
|
180
|
-
this.failedKeys = new
|
|
189
|
+
this.failedKeys = new Map(); // envRef -> failure reason
|
|
190
|
+
this.failedFields = new Set();
|
|
181
191
|
this.savedAt = 0;
|
|
182
192
|
this.store = createStore(this.projection());
|
|
183
193
|
this.scope.subscribe(() => this.republish());
|
|
@@ -209,7 +219,8 @@ window.__ModuleLoader__.load({
|
|
|
209
219
|
return s ? (s.clear ? "" : s.text) : undefined;
|
|
210
220
|
},
|
|
211
221
|
conflict: this.conflict,
|
|
212
|
-
failedKeys: [...this.failedKeys],
|
|
222
|
+
failedKeys: [...this.failedKeys].map(([ref, reason]) => ({ ref, reason })),
|
|
223
|
+
failedFields: [...this.failedFields],
|
|
213
224
|
saving: this.saving,
|
|
214
225
|
failed: this.failed,
|
|
215
226
|
version: (this.staged.size + ":" + [...this.staged.keys()].join(",")),
|
|
@@ -224,7 +235,12 @@ window.__ModuleLoader__.load({
|
|
|
224
235
|
|
|
225
236
|
/** Toggles write immediately (no staging); globals/keys stage + save. */
|
|
226
237
|
async setToggle(id, checked) {
|
|
227
|
-
|
|
238
|
+
try {
|
|
239
|
+
await this.scope.set("enable" + id.charAt(0).toUpperCase() + id.slice(1), checked);
|
|
240
|
+
} catch (err) {
|
|
241
|
+
this.failed = true;
|
|
242
|
+
this.conflict = String(err).includes("SETTINGS_CONFLICT");
|
|
243
|
+
}
|
|
228
244
|
|
|
229
245
|
this.republish();
|
|
230
246
|
}
|
|
@@ -245,7 +261,8 @@ window.__ModuleLoader__.load({
|
|
|
245
261
|
|
|
246
262
|
async save() {
|
|
247
263
|
if (this.saving) return;
|
|
248
|
-
this.saving = true; this.failed = false;
|
|
264
|
+
this.saving = true; this.failed = false;
|
|
265
|
+
this.conflict = false; this.failedFields.clear(); this.republish();
|
|
249
266
|
const doneFields = [];
|
|
250
267
|
const doneKeys = [];
|
|
251
268
|
try {
|
|
@@ -253,13 +270,13 @@ window.__ModuleLoader__.load({
|
|
|
253
270
|
// Empty draft clears the override so the schema default applies again.
|
|
254
271
|
if (staged.clear || staged.text === "") { await this.scope.unset(field); doneFields.push(field); continue; }
|
|
255
272
|
if (SELECT_FIELDS[field]) {
|
|
256
|
-
if (!SELECT_FIELDS[field].includes(staged.text)) { this.failed = true; continue; }
|
|
273
|
+
if (!SELECT_FIELDS[field].includes(staged.text)) { this.failed = true; this.failedFields.add(field); continue; }
|
|
257
274
|
await this.scope.set(field, staged.text);
|
|
258
275
|
doneFields.push(field);
|
|
259
276
|
continue;
|
|
260
277
|
}
|
|
261
278
|
if (BOOLEAN_FIELDS.includes(field)) {
|
|
262
|
-
if (staged.text !== "true" && staged.text !== "false") { this.failed = true; continue; }
|
|
279
|
+
if (staged.text !== "true" && staged.text !== "false") { this.failed = true; this.failedFields.add(field); continue; }
|
|
263
280
|
await this.scope.set(field, staged.text === "true");
|
|
264
281
|
doneFields.push(field);
|
|
265
282
|
continue;
|
|
@@ -274,7 +291,7 @@ window.__ModuleLoader__.load({
|
|
|
274
291
|
backendTimeoutMs: [1000, 120000],
|
|
275
292
|
};
|
|
276
293
|
const [lo, hi] = BOUNDS[field] ?? [-Infinity, Infinity];
|
|
277
|
-
|
|
294
|
+
{ this.failed = true; this.failedFields.add(field); continue; }
|
|
278
295
|
await this.scope.set(field, num);
|
|
279
296
|
doneFields.push(field);
|
|
280
297
|
} else {
|
|
@@ -293,7 +310,7 @@ window.__ModuleLoader__.load({
|
|
|
293
310
|
} catch (keyErr) {
|
|
294
311
|
// env-shadowed reference (fail-loud doctrine): isolate so other keys still land.
|
|
295
312
|
this.failed = true;
|
|
296
|
-
this.failedKeys.
|
|
313
|
+
this.failedKeys.set(ref, String(keyErr && keyErr.message ? keyErr.message : keyErr).slice(0, 140));
|
|
297
314
|
void keyErr;
|
|
298
315
|
}
|
|
299
316
|
}
|
|
@@ -331,7 +348,7 @@ window.__ModuleLoader__.load({
|
|
|
331
348
|
// no client-side default list to drift against the server.
|
|
332
349
|
const checked = props.value === true;
|
|
333
350
|
// Status dot: green keyless-or-keyed, amber enabled-but-key-missing.
|
|
334
|
-
const needsKey = !!b.keyEnv && !props.keyConfigured;
|
|
351
|
+
const needsKey = !!b.keyEnv && !b.optionalKey && !props.keyConfigured;
|
|
335
352
|
const dotColor = needsKey ? (checked ? "#e6a700" : "#888") : "#2ea043";
|
|
336
353
|
return react_jsx_runtime.jsxs("label", { className: "wsc-toggle", style: { display: "flex", alignItems: "center", gap: 8, cursor: "pointer" }, children: [
|
|
337
354
|
react_jsx_runtime.jsx("input", {
|
|
@@ -373,7 +390,7 @@ window.__ModuleLoader__.load({
|
|
|
373
390
|
|
|
374
391
|
// Status summary: enabled backends + configured keys at a glance.
|
|
375
392
|
const enabledCount = BACKENDS.filter((b) => state.value[toggleField(b.id)] === true).length;
|
|
376
|
-
const keyedBackends = BACKENDS.filter((b) => b.keyEnv);
|
|
393
|
+
const keyedBackends = BACKENDS.filter((b) => b.keyEnv && !b.optionalKey);
|
|
377
394
|
const keysReady = keyedBackends.filter((b) => state.keyState[b.keyEnv]?.configured).length;
|
|
378
395
|
|
|
379
396
|
const recencyOptions = [
|
|
@@ -399,6 +416,8 @@ window.__ModuleLoader__.load({
|
|
|
399
416
|
react_jsx_runtime.jsxs("span", { children: [props.t("statusKeys"), ": ", react_jsx_runtime.jsx("b", { children: keysReady + "/" + keyedBackends.length })] }),
|
|
400
417
|
] }),
|
|
401
418
|
react_jsx_runtime.jsx("p", { style: { margin: 0, fontSize: 12, opacity: .65 }, children: props.t("description") }),
|
|
419
|
+
(enabledCount === 0) && react_jsx_runtime.jsx("p", { style: { margin: 0, fontSize: 12, color: "#e6a700" }, children: props.t("fallbackHint") }),
|
|
420
|
+
(state.writable === false) && react_jsx_runtime.jsx("p", { style: { margin: 0, fontSize: 12, color: "#e6a700" }, children: props.t("readOnlyScope") }),
|
|
402
421
|
react_jsx_runtime.jsx("h5", { className: "wsc-sectionTitle", children: props.t("groupFree") }),
|
|
403
422
|
react_jsx_runtime.jsx("div", { className: "wsc-toggles", children: KEYLESS.map((b) =>
|
|
404
423
|
react_jsx_runtime.jsx(ToggleRow, {
|
|
@@ -475,7 +494,7 @@ window.__ModuleLoader__.load({
|
|
|
475
494
|
}),
|
|
476
495
|
] }),
|
|
477
496
|
react_jsx_runtime.jsx("h4", { className: "wsc-sectionTitle", children: props.t("keysTitle") }),
|
|
478
|
-
react_jsx_runtime.jsx("div", { className: "wsc-grid", children: BACKENDS.filter((b) => b.keyEnv).map((b) =>
|
|
497
|
+
react_jsx_runtime.jsx("div", { className: "wsc-grid", children: BACKENDS.filter((b) => b.keyEnv && !b.optionalKey).map((b) =>
|
|
479
498
|
react_jsx_runtime.jsxs("div", { children: [
|
|
480
499
|
react_jsx_runtime.jsxs("div", { className: "wsc-fieldHead", children: [
|
|
481
500
|
react_jsx_runtime.jsx("span", { className: "wsc-label", children: props.t("keyLabel", { id: b.id }) }),
|
|
@@ -500,11 +519,24 @@ window.__ModuleLoader__.load({
|
|
|
500
519
|
state.failedKeys.includes(b.keyEnv) && react_jsx_runtime.jsx("p", { className: "wsc-hint", style: { color: "#e5534b" }, children: props.t("keyFailed") }),
|
|
501
520
|
] }, b.id)
|
|
502
521
|
) }),
|
|
522
|
+
react_jsx_runtime.jsx("h5", { className: "wsc-sectionTitle", children: props.t("optionalTitle") }),
|
|
523
|
+
react_jsx_runtime.jsx("div", { className: "wsc-grid", children: OPTIONAL_KEYS.map((b) =>
|
|
524
|
+
react_jsx_runtime.jsxs("div", { children: [
|
|
525
|
+
react_jsx_runtime.jsx("div", { className: "wsc-fieldHead", children: react_jsx_runtime.jsx("span", { className: "wsc-label", children: props.t("keyLabel", { id: b.id }) }) }),
|
|
526
|
+
react_jsx_runtime.jsx("input", {
|
|
527
|
+
className: "wsc-keyInput", type: "password", autoComplete: "off",
|
|
528
|
+
placeholder: b.keyEnv,
|
|
529
|
+
value: state.stagedKeys[b.keyEnv] ?? "",
|
|
530
|
+
onChange: (e) => card.editKey(b.keyEnv, e.target.value),
|
|
531
|
+
}),
|
|
532
|
+
] }, b.id),
|
|
533
|
+
) }),
|
|
503
534
|
state.stagedCount > 0 && react_jsx_runtime.jsx("p", { style: { margin: 0, fontSize: 12, opacity: .6 }, children: props.t("unsavedHint") }),
|
|
504
535
|
react_jsx_runtime.jsxs("div", { className: "wsc-actions", children: [
|
|
505
|
-
react_jsx_runtime.jsx("button", { className: "wsc-btn", disabled: state.saving || state.stagedCount === 0, onClick: () => card.save(), children: state.saving ? props.t("saving") : props.t("save") }),
|
|
536
|
+
react_jsx_runtime.jsx("button", { className: "wsc-btn", disabled: state.saving || state.stagedCount === 0 || state.writable === false, onClick: () => card.save(), children: state.saving ? props.t("saving") : props.t("save") }),
|
|
506
537
|
(state.stagedCount > 0) && react_jsx_runtime.jsx("button", { className: "wsc-btn wsc-btnGhost", onClick: () => card.discard(), children: props.t("discard") }),
|
|
507
538
|
state.failed && react_jsx_runtime.jsx("span", { className: "wsc-status", style: { color: "#e5534b" }, children: props.t("failed") }),
|
|
539
|
+
(state.failedFields.length > 0) && react_jsx_runtime.jsx("span", { className: "wsc-status", style: { color: "#e5534b", fontSize: 11 }, children: props.t("invalidFields") + ": " + state.failedFields.join(", ") }),
|
|
508
540
|
state.conflict && react_jsx_runtime.jsx("span", { className: "wsc-status", style: { color: "#e6a700" }, children: props.t("conflict") }),
|
|
509
541
|
!state.failed && !state.saving && state.savedAt > 0 && react_jsx_runtime.jsx("span", { className: "wsc-status", style: { color: "#2ea043" }, children: props.t("savedOk") }),
|
|
510
542
|
] }),
|
package/lib/provider.js
CHANGED
|
@@ -87,7 +87,13 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
|
|
|
87
87
|
async search(request, signal) {
|
|
88
88
|
throwIfSearchAborted(signal);
|
|
89
89
|
const opts = resolveOptions();
|
|
90
|
-
const
|
|
90
|
+
const filters = normalizeFilters({
|
|
91
|
+
...(opts.filters ?? {}),
|
|
92
|
+
...(request?.filters ?? {}),
|
|
93
|
+
});
|
|
94
|
+
let effReq = { ...request, maxResults: Math.max(1, Number(request?.maxResults) || opts.numResults || 8) };
|
|
95
|
+
if (Object.keys(filters).length > 0) effReq = { ...effReq, filters };
|
|
96
|
+
const maxResults = effReq.maxResults;
|
|
91
97
|
const concurrency = opts.concurrency ?? 6;
|
|
92
98
|
const backendTimeoutMs = opts.backendTimeoutMs ?? 30000;
|
|
93
99
|
|
|
@@ -112,12 +118,6 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
|
|
|
112
118
|
// Settings-level default filters, overridden by per-request filters
|
|
113
119
|
// (normalizeFilters drops unknown/invalid values so typos degrade to
|
|
114
120
|
// "no filter" instead of breaking the search).
|
|
115
|
-
const filters = normalizeFilters({
|
|
116
|
-
...(opts.filters ?? {}),
|
|
117
|
-
...(request?.filters ?? {}),
|
|
118
|
-
});
|
|
119
|
-
const effectiveRequest =
|
|
120
|
-
Object.keys(filters).length > 0 ? { ...request, filters } : request;
|
|
121
121
|
|
|
122
122
|
const telemetry = [];
|
|
123
123
|
// Concurrency limiter
|
|
@@ -130,13 +130,25 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
|
|
|
130
130
|
// failures instead of being silently demoted like user cancellations.
|
|
131
131
|
const runBackend = ({ id, backend, beOpts, hooks }) => {
|
|
132
132
|
const startedAtMs = Date.now();
|
|
133
|
+
// Trust boundary: baseURL is operator-configurable and API keys are sent to it.
|
|
134
|
+
// Enforce http(s); plain http only for private hosts (self-hosted SearXNG).
|
|
135
|
+
// Reject embedded credentials. Failure is a per-backend soft error.
|
|
136
|
+
if (beOpts.baseURL !== undefined) {
|
|
137
|
+
let bu = null;
|
|
138
|
+
try { bu = new URL(beOpts.baseURL); } catch { bu = null; }
|
|
139
|
+
const privHost = bu ? /^(localhost|127(\.\d{1,3}){3}|10(\.\d{1,3}){3}|192\.168(\.\d{1,3}){3}|172\.(1[6-9]|2\d|3[01])(\.\d{1,3}){3})$/.test(bu.hostname) : false;
|
|
140
|
+
const schemeOk = bu && (bu.protocol === "https:" || (bu.protocol === "http:" && privHost));
|
|
141
|
+
if (!schemeOk || bu.username || bu.password) {
|
|
142
|
+
throw new WebError(`backend "${id}" has an invalid baseURL: https required (plain http only for private hosts), credentials-in-URL rejected`, "WEB_PROVIDER_ERROR");
|
|
143
|
+
}
|
|
144
|
+
}
|
|
133
145
|
const controller = new AbortController();
|
|
134
146
|
const mergedSignal = signal
|
|
135
147
|
? AbortSignal.any([signal, controller.signal])
|
|
136
148
|
: controller.signal;
|
|
137
149
|
const timeoutId = setTimeout(() => controller.abort(), backendTimeoutMs);
|
|
138
150
|
return backend
|
|
139
|
-
.search(
|
|
151
|
+
.search(effReq, mergedSignal, beOpts, hooks)
|
|
140
152
|
.then(
|
|
141
153
|
(value) => {
|
|
142
154
|
telemetry.push({ id, ok: true, ms: Date.now() - startedAtMs, count: value?.sources?.length ?? 0 });
|
|
@@ -190,7 +202,12 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
|
|
|
190
202
|
if (mergedContent === undefined && typeof settled.value?.content === "string" && settled.value.content.length > 0) {
|
|
191
203
|
mergedContent = settled.value.content;
|
|
192
204
|
}
|
|
193
|
-
for (const
|
|
205
|
+
for (const src of settled.value?.sources ?? []) {
|
|
206
|
+
// Scheme whitelist: parsed URLs reach model/GUI consumers; drop
|
|
207
|
+
// javascript:/data: payloads from compromised or poisoned backends.
|
|
208
|
+
if (src && typeof src.url === "string" && !/^https?:\/\//i.test(src.url)) continue;
|
|
209
|
+
collected.push(src);
|
|
210
|
+
}
|
|
194
211
|
} else {
|
|
195
212
|
// settled.status === "rejected"
|
|
196
213
|
const err = settled.reason;
|
|
@@ -226,7 +243,7 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
|
|
|
226
243
|
let outContent = mergedContent;
|
|
227
244
|
if (opts.resultTelemetry !== false && telemetry.length > 0) {
|
|
228
245
|
const line = "[websearch backends] " + telemetry
|
|
229
|
-
.map((t) => t.id + (t.ok ? " ✓" : " ✗") + t.ms + "ms" + (t.ok ? "/" + t.count : ""))
|
|
246
|
+
.map((t) => t.id + (t.ok ? " ✓" : " ✗") + t.ms + "ms" + (t.ok ? "/" + t.count : (t.error ? " (" + String(t.error).slice(0, 40) + ")" : "")))
|
|
230
247
|
.join(" · ");
|
|
231
248
|
outContent = outContent === undefined ? line : outContent + "\n\n" + line;
|
|
232
249
|
}
|
|
@@ -244,7 +261,7 @@ export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
|
|
|
244
261
|
.join("; ");
|
|
245
262
|
const allKeyMissing =
|
|
246
263
|
failures.length > 0 &&
|
|
247
|
-
failures.every((f) => f.err instanceof WebError && f.err.code === "WEB_PROVIDER_UNAVAILABLE");
|
|
264
|
+
failures.every((f) => f.err instanceof WebError && (f.err.code === "WEB_PROVIDER_UNAVAILABLE" || f.err.code === "WEB_PROVIDER_CREDENTIAL_MISSING"));
|
|
248
265
|
const hint = allKeyMissing
|
|
249
266
|
? " (every enabled backend is missing its API key - add keys in Settings > Web Search, or enable a keyless backend)"
|
|
250
267
|
: "";
|
package/lib/util/log.js
CHANGED
|
@@ -56,7 +56,7 @@ export function recordBackendOutcome(ctx, backendId, outcome) {
|
|
|
56
56
|
const logger = ctx && ctx.logger;
|
|
57
57
|
if (!logger) return;
|
|
58
58
|
const status = outcome && typeof outcome.status === "string" ? outcome.status : "";
|
|
59
|
-
const line = "[websearch:" + backendId + "] outcome " + JSON.stringify(outcome ?? {});
|
|
59
|
+
const line = "[websearch:" + backendId + "] outcome " + JSON.stringify(outcome ?? {}).slice(0, 500);
|
|
60
60
|
if (status === "error" && typeof logger.warn === "function") logger.warn(line);
|
|
61
61
|
else if (typeof logger.debug === "function") logger.debug(line);
|
|
62
62
|
} catch {
|
package/lib/util/mcp-client.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
import { WebError } from "@deepseek-ai/dsh-web";
|
|
9
9
|
|
|
10
|
-
const UA = "dsh-websearch/2.
|
|
10
|
+
const UA = "dsh-websearch/2.3";
|
|
11
11
|
const PROTOCOL_VERSION = "2025-06-18";
|
|
12
12
|
const SESSION_TTL_MS = 10 * 60 * 1000; // 10 minutes
|
|
13
13
|
|
|
@@ -58,7 +58,7 @@ async function ensureSession(url, signal) {
|
|
|
58
58
|
}
|
|
59
59
|
|
|
60
60
|
if (!initResponse.ok) {
|
|
61
|
-
const text = await initResponse.text().catch(() => "");
|
|
61
|
+
const text = await initResponse.text().catch(() => "").then((tx) => tx.slice(0, 300));
|
|
62
62
|
throw new WebError(`MCP initialize HTTP ${initResponse.status}: ${text}`, "WEB_PROVIDER_ERROR");
|
|
63
63
|
}
|
|
64
64
|
|
|
@@ -75,7 +75,7 @@ async function ensureSession(url, signal) {
|
|
|
75
75
|
};
|
|
76
76
|
|
|
77
77
|
try {
|
|
78
|
-
await fetch(url, {
|
|
78
|
+
const notifyResponse = await fetch(url, {
|
|
79
79
|
method: "POST",
|
|
80
80
|
headers: {
|
|
81
81
|
"Content-Type": "application/json",
|
|
@@ -86,6 +86,8 @@ async function ensureSession(url, signal) {
|
|
|
86
86
|
signal,
|
|
87
87
|
redirect: "error",
|
|
88
88
|
});
|
|
89
|
+
// Drain the discarded body so undici returns the socket to the pool.
|
|
90
|
+
notifyResponse.body?.cancel?.().catch(() => {});
|
|
89
91
|
} catch (error) {
|
|
90
92
|
// Best-effort: Exa requires this, Parallel tolerates absence
|
|
91
93
|
if (signal?.aborted) throw error;
|
|
@@ -104,17 +106,16 @@ async function ensureSession(url, signal) {
|
|
|
104
106
|
function parseEnvelope(text, expectedId) {
|
|
105
107
|
// Try SSE format first: "event: message\ndata: {...}\n\n"
|
|
106
108
|
if (text.includes("data:")) {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
if (dataLine) {
|
|
109
|
+
// SSE spec: one JSON payload may span multiple data: lines - join them.
|
|
110
|
+
const joined = text
|
|
111
|
+
.trim()
|
|
112
|
+
.split("\n")
|
|
113
|
+
.filter((l) => l.startsWith("data:"))
|
|
114
|
+
.map((l) => l.slice(5).trim())
|
|
115
|
+
.join("");
|
|
116
|
+
if (joined) {
|
|
116
117
|
try {
|
|
117
|
-
return JSON.parse(
|
|
118
|
+
return JSON.parse(joined);
|
|
118
119
|
} catch {}
|
|
119
120
|
}
|
|
120
121
|
}
|
|
@@ -205,7 +206,7 @@ export async function mcpCall(url, toolName, args, extraHeaders = {}, signal) {
|
|
|
205
206
|
}
|
|
206
207
|
|
|
207
208
|
if (!response.ok) {
|
|
208
|
-
const text = await response.text().catch(() => "");
|
|
209
|
+
const text = await response.text().catch(() => "").then((tx) => tx.slice(0, 300));
|
|
209
210
|
throw new WebError(`MCP tools/call HTTP ${response.status}: ${text}`, "WEB_PROVIDER_ERROR");
|
|
210
211
|
}
|
|
211
212
|
|
package/lib/util/rank.js
CHANGED
|
@@ -97,7 +97,7 @@ function scoreSource(source, queryTokens) {
|
|
|
97
97
|
let score = 0;
|
|
98
98
|
const titleTokens = tokenize(source.title ?? "");
|
|
99
99
|
const snippetTokens = tokenize(source.snippet ?? "");
|
|
100
|
-
const urlTokens = tokenize(String(source.url ?? "")
|
|
100
|
+
const urlTokens = tokenize(String(source.url ?? ""));
|
|
101
101
|
for (const q of queryTokens) {
|
|
102
102
|
if (titleTokens.has(q)) score += 3;
|
|
103
103
|
if (snippetTokens.has(q)) score += 2;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@240xu/dsh-websearch",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.4.0",
|
|
4
4
|
"description": "Unified web search provider for DeepSeek Harness (ctx.web): 11 backends \u2014 Exa + Parallel + DDG + SearXNG (keyless), DeepSeek + Anthropic + OpenAI + Brave + Tavily + Serper + Mojeek (key-gated via Settings credentials). Registers provider id \"unified\" for zero-config web_search tool.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
package/lib/backends/index.js
DELETED
|
@@ -1,121 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Unified backend registry for dsh-websearch.
|
|
3
|
-
* Auto-discovers and exports all backend implementations.
|
|
4
|
-
*/
|
|
5
|
-
|
|
6
|
-
import { exaBackend } from "./exa.js";
|
|
7
|
-
import { parallelBackend } from "./parallel.js";
|
|
8
|
-
import { ddgBackend } from "./ddg.js";
|
|
9
|
-
import { deepseekBackend, anthropicBackend } from "./anthropic-like.js";
|
|
10
|
-
import { openaiBackend } from "./openai.js";
|
|
11
|
-
import { braveBackend } from "./brave.js";
|
|
12
|
-
import { tavilyBackend } from "./tavily.js";
|
|
13
|
-
import { serperBackend } from "./serper.js";
|
|
14
|
-
import { searxngBackend } from "./searxng.js";
|
|
15
|
-
import { mojeekBackend } from "./mojeek.js";
|
|
16
|
-
|
|
17
|
-
// Individual backend exports for direct import
|
|
18
|
-
export { exaBackend } from "./exa.js";
|
|
19
|
-
export { parallelBackend } from "./parallel.js";
|
|
20
|
-
export { ddgBackend } from "./ddg.js";
|
|
21
|
-
export { deepseekBackend, anthropicBackend } from "./anthropic-like.js";
|
|
22
|
-
export { openaiBackend } from "./openai.js";
|
|
23
|
-
export { braveBackend } from "./brave.js";
|
|
24
|
-
export { tavilyBackend } from "./tavily.js";
|
|
25
|
-
export { serperBackend } from "./serper.js";
|
|
26
|
-
export { searxngBackend } from "./searxng.js";
|
|
27
|
-
export { mojeekBackend } from "./mojeek.js";
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
/** All registered backends. Order = default fan-out priority. */
|
|
31
|
-
export const BACKENDS = {
|
|
32
|
-
exa: exaBackend,
|
|
33
|
-
parallel: parallelBackend,
|
|
34
|
-
ddg: ddgBackend,
|
|
35
|
-
searxng: searxngBackend, // keyless, new
|
|
36
|
-
brave: braveBackend, // key-gated, new
|
|
37
|
-
tavily: tavilyBackend, // key-gated, new
|
|
38
|
-
serper: serperBackend, // key-gated, new
|
|
39
|
-
mojeek: mojeekBackend, // key-gated, new
|
|
40
|
-
deepseek: deepseekBackend,
|
|
41
|
-
anthropic: anthropicBackend,
|
|
42
|
-
openai: openaiBackend,
|
|
43
|
-
};
|
|
44
|
-
|
|
45
|
-
/** All backend IDs in registry order. */
|
|
46
|
-
export const ALL_BACKEND_IDS = Object.keys(BACKENDS);
|
|
47
|
-
|
|
48
|
-
/** Default enabled backends (keyless only). */
|
|
49
|
-
export const DEFAULT_ENABLED = ["exa", "parallel", "ddg", "searxng"];
|
|
50
|
-
|
|
51
|
-
/** Environment variable names for key-gated backends. */
|
|
52
|
-
export const DEFAULT_KEY_ENV = {
|
|
53
|
-
deepseek: "DEEPSEEK_API_KEY",
|
|
54
|
-
anthropic: "ANTHROPIC_API_KEY",
|
|
55
|
-
openai: "OPENAI_API_KEY",
|
|
56
|
-
brave: "BRAVE_API_KEY",
|
|
57
|
-
tavily: "TAVILY_API_KEY",
|
|
58
|
-
serper: "SERPER_API_KEY",
|
|
59
|
-
mojeek: "MOJEEK_API_KEY",
|
|
60
|
-
};
|
|
61
|
-
|
|
62
|
-
/** Default base URLs for each backend. */
|
|
63
|
-
export const DEFAULT_BASE_URLS = {
|
|
64
|
-
exa: "https://mcp.exa.ai/mcp",
|
|
65
|
-
parallel: "https://search.parallel.ai/mcp",
|
|
66
|
-
ddg: "https://html.duckduckgo.com/html/",
|
|
67
|
-
searxng: "https://searx.be",
|
|
68
|
-
brave: "https://api.search.brave.com/res/v1/web/search",
|
|
69
|
-
tavily: "https://api.tavily.com/search",
|
|
70
|
-
serper: "https://google.serper.dev/search",
|
|
71
|
-
mojeek: "https://api.mojeek.com/v1/search",
|
|
72
|
-
deepseek: "https://api.deepseek.com/anthropic/v1",
|
|
73
|
-
anthropic: "https://api.anthropic.com/v1",
|
|
74
|
-
openai: "https://api.openai.com/v1",
|
|
75
|
-
};
|
|
76
|
-
|
|
77
|
-
/** Backend type classification for GUI. */
|
|
78
|
-
export const BACKEND_TYPES = {
|
|
79
|
-
exa: "mcp",
|
|
80
|
-
parallel: "mcp",
|
|
81
|
-
ddg: "keyless",
|
|
82
|
-
searxng: "keyless",
|
|
83
|
-
brave: "credential",
|
|
84
|
-
tavily: "credential",
|
|
85
|
-
serper: "credential",
|
|
86
|
-
mojeek: "credential",
|
|
87
|
-
deepseek: "credential",
|
|
88
|
-
anthropic: "credential",
|
|
89
|
-
openai: "credential",
|
|
90
|
-
};
|
|
91
|
-
|
|
92
|
-
/** Human-readable labels for GUI. */
|
|
93
|
-
export const BACKEND_LABELS = {
|
|
94
|
-
exa: "Exa (MCP, 免费无 Key)",
|
|
95
|
-
parallel: "Parallel (MCP, 免费无 Key)",
|
|
96
|
-
ddg: "DuckDuckGo (HTML 抓取, 兜底)",
|
|
97
|
-
searxng: "SearXNG (元搜索, 免费无 Key)",
|
|
98
|
-
brave: "Brave Search (独立索引, 2000/月免费)",
|
|
99
|
-
tavily: "Tavily (AI 专用, 含答案摘要)",
|
|
100
|
-
serper: "Serper.dev (Google 抓取, 2500/月免费)",
|
|
101
|
-
mojeek: "Mojeek (独立索引, 1000/天免费)",
|
|
102
|
-
deepseek: "DeepSeek (原生 web_search, 需 Key)",
|
|
103
|
-
anthropic: "Anthropic (Claude 原生 web_search, 需 Key)",
|
|
104
|
-
openai: "OpenAI (Responses API web_search, 需 Key)",
|
|
105
|
-
};
|
|
106
|
-
|
|
107
|
-
/** Backend descriptions for GUI tooltips. */
|
|
108
|
-
export const BACKEND_DESCRIPTIONS = {
|
|
109
|
-
exa: "流式 HTTP MCP 协议,原生 web_search_exa 工具,结果含标题/URL/高亮摘要",
|
|
110
|
-
parallel: "流式 HTTP MCP 协议,原生 web_search 工具,返回结构化 JSON 含摘要数组",
|
|
111
|
-
ddg: "HTML 页面抓取解析,通过 uddg= 参数还原真实 URL,零依赖兜底",
|
|
112
|
-
searxng: "开源元搜索引擎聚合,可指向公共实例或自托管,隐私友好",
|
|
113
|
-
brave: "Brave 独立搜索索引,REST API,免费 2000 次/月,结果质量高",
|
|
114
|
-
tavily: "AI 专用搜索,可返回 AI 生成答案 + 结构化结果,支持深度搜索模式",
|
|
115
|
-
serper: "抓取 Google SERP,结构化 organic[] 结果,免费 2500 次/月,速度极快",
|
|
116
|
-
mojeek: "英国独立搜索引擎,无追踪,免费 1000 次/天,英文为主",
|
|
117
|
-
deepseek: "DeepSeek Anthropic 兼容 API + 原生 web_search_20250305 工具",
|
|
118
|
-
anthropic: "Claude 官方 Messages API + web_search_20250305 原生工具",
|
|
119
|
-
openai: "OpenAI Responses API + 原生 web_search 工具,解析 url_citation 注释",
|
|
120
|
-
};
|
|
121
|
-
|
package/lib/util/rpc.js
DELETED
|
@@ -1,262 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @module dsh-websearch/util/rpc
|
|
3
|
-
*
|
|
4
|
-
* Minimal MCP JSON-RPC 2.0 client for the streamable-http transport used by
|
|
5
|
-
* Exa (https://mcp.exa.ai/mcp) and Parallel (https://search.parallel.ai/mcp).
|
|
6
|
-
*
|
|
7
|
-
* Streamable-HTTP MCP servers require a 2-step handshake per session:
|
|
8
|
-
* 1. POST initialize → server replies with its capabilities and (for Exa)
|
|
9
|
-
* a Mcp-Session-Id header. Parallel also returns this header.
|
|
10
|
-
* 2. POST notifications/initialized → notify the server the client is ready.
|
|
11
|
-
* (Exa rejects tools/call without this; Parallel tolerates its absence
|
|
12
|
-
* but sending it is harmless.)
|
|
13
|
-
* 3. POST tools/call with the Mcp-Session-Id header → returns either a plain
|
|
14
|
-
* JSON body OR a text/event-stream of "data: {…}\n\n" frames.
|
|
15
|
-
*
|
|
16
|
-
* We cache the Mcp-Session-Id per server URL inside the process so subsequent
|
|
17
|
-
* searches skip the handshake (and one cache entry survives across calls).
|
|
18
|
-
* Sessions are NOT shared across processes (each dsh-web worker owns its own).
|
|
19
|
-
*
|
|
20
|
-
* Failures are zh-folded into WebError so the provider fan-out can demote them
|
|
21
|
-
* to soft nulls when only one backend fails.
|
|
22
|
-
*/
|
|
23
|
-
import { WebError } from "@deepseek-ai/dsh-web";
|
|
24
|
-
import { throwIfSearchAborted, searchAborted, isAbortError } from "./abort.js";
|
|
25
|
-
|
|
26
|
-
/** User-Agent header; bumped with the package version. */
|
|
27
|
-
export const UA = "dsh-websearch/2.1";
|
|
28
|
-
|
|
29
|
-
/** MCP protocol version we negotiate. */
|
|
30
|
-
const PROTOCOL_VERSION = "2025-06-18";
|
|
31
|
-
|
|
32
|
-
/** Per-URL session cache: url → { sessionId, initializedAt }. */
|
|
33
|
-
const sessionCache = new Map();
|
|
34
|
-
const SESSION_TTL_MS = 10 * 60 * 1000; // Exa sessions live indefinitely; this is a courtesy cap.
|
|
35
|
-
|
|
36
|
-
/** Monotonic request id so concurrent inits don't collide. */
|
|
37
|
-
let nextReqId = 1;
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Initialize (or reuse) a streamable-http MCP session.
|
|
41
|
-
* Sends initialize + notifications/initialized. Returns the Mcp-Session-Id,
|
|
42
|
-
* caching it so callers don't pay the handshake more than once per TTL.
|
|
43
|
-
*
|
|
44
|
-
* @param {string} url
|
|
45
|
-
* @param {AbortSignal} [signal]
|
|
46
|
-
* @returns {Promise<string|null>} Mcp-Session-Id (or null if the server omits it).
|
|
47
|
-
* @throws {WebError} on transport error or abort.
|
|
48
|
-
*/
|
|
49
|
-
async function ensureSession(url, signal) {
|
|
50
|
-
throwIfSearchAborted(signal);
|
|
51
|
-
const cached = sessionCache.get(url);
|
|
52
|
-
if (cached && Date.now() - cached.initializedAt < SESSION_TTL_MS && cached.sessionId) {
|
|
53
|
-
return cached.sessionId;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
const initId = nextReqId++;
|
|
57
|
-
const initBody = JSON.stringify({
|
|
58
|
-
jsonrpc: "2.0",
|
|
59
|
-
id: initId,
|
|
60
|
-
method: "initialize",
|
|
61
|
-
params: {
|
|
62
|
-
protocolVersion: PROTOCOL_VERSION,
|
|
63
|
-
capabilities: {},
|
|
64
|
-
clientInfo: { name: "dsh-websearch", version: "1.0" },
|
|
65
|
-
},
|
|
66
|
-
});
|
|
67
|
-
|
|
68
|
-
let res;
|
|
69
|
-
try {
|
|
70
|
-
res = await fetch(url, {
|
|
71
|
-
method: "POST",
|
|
72
|
-
redirect: "error",
|
|
73
|
-
headers: {
|
|
74
|
-
"Content-Type": "application/json",
|
|
75
|
-
"Accept": "application/json, text/event-stream",
|
|
76
|
-
"User-Agent": UA,
|
|
77
|
-
},
|
|
78
|
-
body: initBody,
|
|
79
|
-
...(signal !== undefined ? { signal } : {}),
|
|
80
|
-
});
|
|
81
|
-
} catch (error) {
|
|
82
|
-
if (signal?.aborted === true || isAbortError(error)) throw searchAborted(signal, error);
|
|
83
|
-
throw new WebError(
|
|
84
|
-
`mcp initialize fetch failed for ${url}: ${String(error?.message ?? error)}`,
|
|
85
|
-
"WEB_PROVIDER_ERROR",
|
|
86
|
-
{ cause: error },
|
|
87
|
-
);
|
|
88
|
-
}
|
|
89
|
-
throwIfSearchAborted(signal);
|
|
90
|
-
if (!res.ok) {
|
|
91
|
-
let detail = "";
|
|
92
|
-
try { detail = (await res.text()).slice(0, 300); } catch {}
|
|
93
|
-
throw new WebError(
|
|
94
|
-
`mcp initialize HTTP ${res.status} for ${url}${detail ? ": " + detail : ""}`,
|
|
95
|
-
"WEB_PROVIDER_ERROR",
|
|
96
|
-
);
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
// Drain the initialize response body (we don't need its content).
|
|
100
|
-
await res.text();
|
|
101
|
-
|
|
102
|
-
const sessionId = res.headers.get("Mcp-Session-Id") || null;
|
|
103
|
-
|
|
104
|
-
// Send notifications/initialized (some servers reject tools/call without it).
|
|
105
|
-
if (sessionId) {
|
|
106
|
-
try {
|
|
107
|
-
await fetch(url, {
|
|
108
|
-
method: "POST",
|
|
109
|
-
redirect: "error",
|
|
110
|
-
headers: {
|
|
111
|
-
"Content-Type": "application/json",
|
|
112
|
-
"Accept": "application/json, text/event-stream",
|
|
113
|
-
"User-Agent": UA,
|
|
114
|
-
"Mcp-Session-Id": sessionId,
|
|
115
|
-
},
|
|
116
|
-
body: JSON.stringify({ jsonrpc: "2.0", method: "notifications/initialized" }),
|
|
117
|
-
...(signal !== undefined ? { signal } : {}),
|
|
118
|
-
});
|
|
119
|
-
} catch (error) {
|
|
120
|
-
// notifications/initialized is best-effort; ignore transport errors here.
|
|
121
|
-
// (If the session is gone we'll discover that via tools/call.) Re-raise aborts.
|
|
122
|
-
if (signal?.aborted === true || isAbortError(error)) throw searchAborted(signal, error);
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
|
|
126
|
-
sessionCache.set(url, { sessionId, initializedAt: Date.now() });
|
|
127
|
-
return sessionId;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
/**
|
|
131
|
-
* Parse a streamable-HTTP MCP response body into the first JSON-RPC envelope.
|
|
132
|
-
*
|
|
133
|
-
* Servers respond with EITHER:
|
|
134
|
-
* - a single JSON object (Parallel: {"jsonrpc":"2.0","id":N,"result":{…}} or {...,"error":{…}}), OR
|
|
135
|
-
* - a text/event-stream of "event: <type>\n data: {…}\n\n" frames (Exa).
|
|
136
|
-
*
|
|
137
|
-
* We scan line-by-line; SSE data: frames take precedence, otherwise we fall back
|
|
138
|
-
* to scanning whole-line JSON.
|
|
139
|
-
*
|
|
140
|
-
* @param {string} text
|
|
141
|
-
* @param {number} reqId the request id we expect to be echoed back
|
|
142
|
-
* @returns {{result?:object, error?:object}}
|
|
143
|
-
* @throws if no parseable envelope is found.
|
|
144
|
-
*/
|
|
145
|
-
function parseEnvelope(text, reqId) {
|
|
146
|
-
const lines = String(text).split("\n");
|
|
147
|
-
let lastJson = null;
|
|
148
|
-
for (const raw of lines) {
|
|
149
|
-
const line = raw.trim();
|
|
150
|
-
if (!line) continue;
|
|
151
|
-
let json;
|
|
152
|
-
if (line.startsWith("data: ")) {
|
|
153
|
-
json = line.slice(6).trim();
|
|
154
|
-
} else if (line.startsWith("{")) {
|
|
155
|
-
json = line;
|
|
156
|
-
} else {
|
|
157
|
-
continue;
|
|
158
|
-
}
|
|
159
|
-
if (!json.startsWith("{")) continue;
|
|
160
|
-
let obj;
|
|
161
|
-
try { obj = JSON.parse(json); } catch { continue; }
|
|
162
|
-
// Prefer the envelope matching our request id; otherwise keep the last seen.
|
|
163
|
-
if (typeof obj.id === "number" && obj.id === reqId) return obj;
|
|
164
|
-
lastJson = obj;
|
|
165
|
-
}
|
|
166
|
-
return lastJson;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
/**
|
|
170
|
-
* JSON-RPC 2.0 tools/call against a streamable-http MCP server (after the
|
|
171
|
-
* initialize + notifications/initialized handshake is established or reused).
|
|
172
|
-
*
|
|
173
|
-
* @param {string} url the MCP server endpoint
|
|
174
|
-
* @param {string} toolName the tool to invoke (e.g. "web_search_exa", "web_search")
|
|
175
|
-
* @param {object} args the tool's arguments (per its input schema)
|
|
176
|
-
* @param {object} [extraHeaders]
|
|
177
|
-
* @param {AbortSignal} [signal]
|
|
178
|
-
* @returns {Promise<object>} the RPC `result` payload (the `content` array etc.)
|
|
179
|
-
* @throws {WebError} on HTTP failure, RPC error, abort, or unparseable response.
|
|
180
|
-
*/
|
|
181
|
-
export async function rpcCall(url, toolName, args, extraHeaders, signal) {
|
|
182
|
-
const sessionId = await ensureSession(url, signal);
|
|
183
|
-
const reqId = nextReqId++;
|
|
184
|
-
|
|
185
|
-
const headers = {
|
|
186
|
-
"Content-Type": "application/json",
|
|
187
|
-
"Accept": "application/json, text/event-stream",
|
|
188
|
-
"User-Agent": UA,
|
|
189
|
-
...(sessionId ? { "Mcp-Session-Id": sessionId } : {}),
|
|
190
|
-
...(extraHeaders || {}),
|
|
191
|
-
};
|
|
192
|
-
const body = JSON.stringify({
|
|
193
|
-
jsonrpc: "2.0",
|
|
194
|
-
id: reqId,
|
|
195
|
-
method: "tools/call",
|
|
196
|
-
params: { name: toolName, arguments: args },
|
|
197
|
-
});
|
|
198
|
-
|
|
199
|
-
let res;
|
|
200
|
-
try {
|
|
201
|
-
res = await fetch(url, {
|
|
202
|
-
method: "POST",
|
|
203
|
-
redirect: "error",
|
|
204
|
-
headers,
|
|
205
|
-
body,
|
|
206
|
-
...(signal !== undefined ? { signal } : {}),
|
|
207
|
-
});
|
|
208
|
-
} catch (error) {
|
|
209
|
-
if (signal?.aborted === true || isAbortError(error)) throw searchAborted(signal, error);
|
|
210
|
-
throw new WebError(
|
|
211
|
-
`mcp rpc ${toolName} fetch failed: ${String(error?.message ?? error)}`,
|
|
212
|
-
"WEB_PROVIDER_ERROR",
|
|
213
|
-
{ cause: error },
|
|
214
|
-
);
|
|
215
|
-
}
|
|
216
|
-
throwIfSearchAborted(signal);
|
|
217
|
-
|
|
218
|
-
if (!res.ok) {
|
|
219
|
-
let detail = "";
|
|
220
|
-
try { detail = (await res.text()).slice(0, 300); } catch {}
|
|
221
|
-
// If the server rejected the request id/session, nuke the cached session
|
|
222
|
-
// so the next call re-initializes. 404/401/403 typically mean session death.
|
|
223
|
-
if (res.status === 401 || res.status === 403 || res.status === 404) {
|
|
224
|
-
sessionCache.delete(url);
|
|
225
|
-
}
|
|
226
|
-
throw new WebError(
|
|
227
|
-
`mcp rpc ${toolName} HTTP ${res.status}${detail ? ": " + detail : ""}`,
|
|
228
|
-
"WEB_PROVIDER_ERROR",
|
|
229
|
-
);
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
const text = await res.text();
|
|
233
|
-
throwIfSearchAborted(signal);
|
|
234
|
-
|
|
235
|
-
const env = parseEnvelope(text, reqId);
|
|
236
|
-
if (!env) {
|
|
237
|
-
throw new WebError(
|
|
238
|
-
`mcp rpc ${toolName} returned an unparseable response`,
|
|
239
|
-
"WEB_PROVIDER_ERROR",
|
|
240
|
-
);
|
|
241
|
-
}
|
|
242
|
-
if (env.error) {
|
|
243
|
-
throw new WebError(
|
|
244
|
-
`mcp rpc ${toolName} error: ${env.error.message || JSON.stringify(env.error)}`,
|
|
245
|
-
"WEB_PROVIDER_ERROR",
|
|
246
|
-
);
|
|
247
|
-
}
|
|
248
|
-
if (!env.result) {
|
|
249
|
-
throw new WebError(
|
|
250
|
-
`mcp rpc ${toolName} returned a response with no result`,
|
|
251
|
-
"WEB_PROVIDER_ERROR",
|
|
252
|
-
);
|
|
253
|
-
}
|
|
254
|
-
return env.result;
|
|
255
|
-
}
|
|
256
|
-
|
|
257
|
-
/** Drop the cached session for a url (used by backends after a fatal error). */
|
|
258
|
-
export function invalidateSession(url) {
|
|
259
|
-
sessionCache.delete(url);
|
|
260
|
-
}
|
|
261
|
-
|
|
262
|
-
export { throwIfSearchAborted, searchAborted, isAbortError };
|