@240xu/dsh-websearch 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +136 -0
- package/cordis.patch.yml +3 -0
- package/examples/cordis.patch.yml +17 -0
- package/lib/backends/anthropic-like.js +262 -0
- package/lib/backends/brave.js +98 -0
- package/lib/backends/ddg.js +136 -0
- package/lib/backends/exa.js +97 -0
- package/lib/backends/index.js +121 -0
- package/lib/backends/mojeek.js +103 -0
- package/lib/backends/openai.js +171 -0
- package/lib/backends/parallel.js +105 -0
- package/lib/backends/searxng.js +107 -0
- package/lib/backends/serper.js +115 -0
- package/lib/backends/tavily.js +112 -0
- package/lib/client.js +317 -0
- package/lib/index.js +277 -0
- package/lib/provider.js +225 -0
- package/lib/util/abort.js +82 -0
- package/lib/util/log.js +51 -0
- package/lib/util/mcp-client.js +233 -0
- package/lib/util/rpc.js +262 -0
- package/package.json +70 -0
package/lib/index.js
ADDED
|
@@ -0,0 +1,277 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* dsh-websearch: unified web search provider for the DSH web seam (ctx.web).
|
|
3
|
+
* Registers ONE provider under id "unified" that fans out to multiple backends
|
|
4
|
+
* concurrently and merges (URL-deduped) results. Keyless backends (Exa, Parallel,
|
|
5
|
+
* DuckDuckGo, SearXNG) give zero-config search; key-gated backends (DeepSeek/
|
|
6
|
+
* Anthropic/OpenAI/Brave/Tavily/Serper/Mojeek) activate when their API key is
|
|
7
|
+
* stored via the credentials service or the launching environment.
|
|
8
|
+
*
|
|
9
|
+
* @module dsh-websearch
|
|
10
|
+
*/
|
|
11
|
+
import z from "@deepseek-ai/schemastery";
|
|
12
|
+
import { credentialRef } from "@deepseek-ai/dsh-credentials";
|
|
13
|
+
import {
|
|
14
|
+
installSettingsSection,
|
|
15
|
+
settingsNamespace,
|
|
16
|
+
} from "@deepseek-ai/dsh-settings";
|
|
17
|
+
import { launchEnvironmentOf } from "@deepseek-ai/dsh-launch-environment";
|
|
18
|
+
import { WebError } from "@deepseek-ai/dsh-web";
|
|
19
|
+
|
|
20
|
+
// Import all backends
|
|
21
|
+
import { exaBackend } from "./backends/exa.js";
|
|
22
|
+
import { parallelBackend } from "./backends/parallel.js";
|
|
23
|
+
import { ddgBackend } from "./backends/ddg.js";
|
|
24
|
+
import {
|
|
25
|
+
deepseekBackend,
|
|
26
|
+
anthropicBackend,
|
|
27
|
+
} from "./backends/anthropic-like.js";
|
|
28
|
+
import { openaiBackend } from "./backends/openai.js";
|
|
29
|
+
import { braveBackend } from "./backends/brave.js";
|
|
30
|
+
import { tavilyBackend } from "./backends/tavily.js";
|
|
31
|
+
import { serperBackend } from "./backends/serper.js";
|
|
32
|
+
import { searxngBackend } from "./backends/searxng.js";
|
|
33
|
+
import { mojeekBackend } from "./backends/mojeek.js";
|
|
34
|
+
import { createUnifiedSearchProvider } from "./provider.js";
|
|
35
|
+
|
|
36
|
+
/** Cordis plugin name used by loader diagnostics. */
|
|
37
|
+
export const name = "unified-search";
|
|
38
|
+
/** The web seam this provider registers into. */
|
|
39
|
+
export const inject = ["web"];
|
|
40
|
+
|
|
41
|
+
/** Settings namespace key. */
|
|
42
|
+
const NAMESPACE = settingsNamespace("unified-search");
|
|
43
|
+
|
|
44
|
+
/** Full backend registry. Order = default fan-out priority. */
|
|
45
|
+
const BACKENDS = {
|
|
46
|
+
exa: exaBackend,
|
|
47
|
+
parallel: parallelBackend,
|
|
48
|
+
ddg: ddgBackend,
|
|
49
|
+
searxng: searxngBackend,
|
|
50
|
+
brave: braveBackend,
|
|
51
|
+
tavily: tavilyBackend,
|
|
52
|
+
serper: serperBackend,
|
|
53
|
+
mojeek: mojeekBackend,
|
|
54
|
+
deepseek: deepseekBackend,
|
|
55
|
+
anthropic: anthropicBackend,
|
|
56
|
+
openai: openaiBackend,
|
|
57
|
+
};
|
|
58
|
+
|
|
59
|
+
/** All backend ids in registry order. */
|
|
60
|
+
const ALL_BACKENDS = Object.keys(BACKENDS);
|
|
61
|
+
|
|
62
|
+
/** Backends enabled by default (keyless only). */
|
|
63
|
+
const DEFAULT_ENABLED = ["exa", "parallel", "ddg", "searxng"];
|
|
64
|
+
|
|
65
|
+
/** Environment-variable names each key-gated backend reads. */
|
|
66
|
+
const DEFAULT_KEY_ENV = {
|
|
67
|
+
deepseek: "DEEPSEEK_API_KEY",
|
|
68
|
+
anthropic: "ANTHROPIC_API_KEY",
|
|
69
|
+
openai: "OPENAI_API_KEY",
|
|
70
|
+
brave: "BRAVE_API_KEY",
|
|
71
|
+
tavily: "TAVILY_API_KEY",
|
|
72
|
+
serper: "SERPER_API_KEY",
|
|
73
|
+
mojeek: "MOJEEK_API_KEY",
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
/** Default base URLs. */
|
|
77
|
+
const DEFAULT_BASE_URLS = {
|
|
78
|
+
exa: "https://mcp.exa.ai/mcp",
|
|
79
|
+
parallel: "https://search.parallel.ai/mcp",
|
|
80
|
+
ddg: "https://html.duckduckgo.com/html/",
|
|
81
|
+
searxng: "https://searx.be",
|
|
82
|
+
brave: "https://api.search.brave.com/res/v1/web/search",
|
|
83
|
+
tavily: "https://api.tavily.com/search",
|
|
84
|
+
serper: "https://google.serper.dev/search",
|
|
85
|
+
mojeek: "https://api.mojeek.com/v1/search",
|
|
86
|
+
deepseek: "https://api.deepseek.com/anthropic/v1",
|
|
87
|
+
anthropic: "https://api.anthropic.com/v1",
|
|
88
|
+
openai: "https://api.openai.com/v1",
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
/** Env var that overrides the default backend sweep order / enabled set. */
|
|
92
|
+
const ENABLED_ENV = "DSH_UNIFIED_SEARCH_BACKENDS";
|
|
93
|
+
|
|
94
|
+
/** Default models for model-carrying backends. */
|
|
95
|
+
const DEFAULT_MODELS = {
|
|
96
|
+
deepseek: "deepseek-v4-flash",
|
|
97
|
+
anthropic: "claude-sonnet-4-6",
|
|
98
|
+
openai: "gpt-4o",
|
|
99
|
+
};
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* GUI-friendly labels for each backend (shown in the Settings panel).
|
|
103
|
+
*/
|
|
104
|
+
const DEFAULT_LABELS = {
|
|
105
|
+
exa: "Exa (MCP, 免费无 Key)",
|
|
106
|
+
parallel: "Parallel (MCP, 免费无 Key)",
|
|
107
|
+
ddg: "DuckDuckGo (HTML 抓取, 兜底)",
|
|
108
|
+
searxng: "SearXNG (元搜索, 免费无 Key)",
|
|
109
|
+
brave: "Brave Search (独立索引, 2000/月免费)",
|
|
110
|
+
tavily: "Tavily (AI 专用, 含答案摘要)",
|
|
111
|
+
serper: "Serper.dev (Google 抓取, 2500/月免费)",
|
|
112
|
+
mojeek: "Mojeek (独立索引, 1000/天免费)",
|
|
113
|
+
deepseek: "DeepSeek (原生 web_search, 需 Key)",
|
|
114
|
+
anthropic: "Anthropic (Claude 原生 web_search, 需 Key)",
|
|
115
|
+
openai: "OpenAI (Responses API web_search, 需 Key)",
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Settings schema — flat, GUI-friendly. Every object property is optional by
|
|
120
|
+
* default in schemastery; defaults are supplied via .default(). API keys are
|
|
121
|
+
* credential-refs (env-var names), which the Settings UI renders as a
|
|
122
|
+
* credential selector / secret field.
|
|
123
|
+
*/
|
|
124
|
+
export const Config = z.object({
|
|
125
|
+
// Global
|
|
126
|
+
numResults: z.number().step(1).min(1).max(50).default(8)
|
|
127
|
+
.description("每次搜索返回的最大结果数"),
|
|
128
|
+
concurrency: z.number().step(1).min(1).max(10).default(6)
|
|
129
|
+
.description("同时并发的后端数量"),
|
|
130
|
+
backendTimeoutMs: z.number().step(1000).min(1000).max(120000).default(30000)
|
|
131
|
+
.description("单后端最大等待时间(毫秒)"),
|
|
132
|
+
|
|
133
|
+
// Enable toggles (one per backend)
|
|
134
|
+
enableExa: z.boolean().default(true).description(DEFAULT_LABELS.exa),
|
|
135
|
+
enableParallel: z.boolean().default(true).description(DEFAULT_LABELS.parallel),
|
|
136
|
+
enableDdg: z.boolean().default(true).description(DEFAULT_LABELS.ddg),
|
|
137
|
+
enableSearxng: z.boolean().default(true).description(DEFAULT_LABELS.searxng),
|
|
138
|
+
enableBrave: z.boolean().default(false).description(DEFAULT_LABELS.brave),
|
|
139
|
+
enableTavily: z.boolean().default(false).description(DEFAULT_LABELS.tavily),
|
|
140
|
+
enableSerper: z.boolean().default(false).description(DEFAULT_LABELS.serper),
|
|
141
|
+
enableMojeek: z.boolean().default(false).description(DEFAULT_LABELS.mojeek),
|
|
142
|
+
enableDeepseek: z.boolean().default(false).description(DEFAULT_LABELS.deepseek),
|
|
143
|
+
enableAnthropic: z.boolean().default(false).description(DEFAULT_LABELS.anthropic),
|
|
144
|
+
enableOpenai: z.boolean().default(false).description(DEFAULT_LABELS.openai),
|
|
145
|
+
|
|
146
|
+
// Credential refs (env-var names; secrets live in the credentials service)
|
|
147
|
+
braveApiKeyEnv: z.string().role("credential-ref").default(DEFAULT_KEY_ENV.brave),
|
|
148
|
+
tavilyApiKeyEnv: z.string().role("credential-ref").default(DEFAULT_KEY_ENV.tavily),
|
|
149
|
+
serperApiKeyEnv: z.string().role("credential-ref").default(DEFAULT_KEY_ENV.serper),
|
|
150
|
+
mojeekApiKeyEnv: z.string().role("credential-ref").default(DEFAULT_KEY_ENV.mojeek),
|
|
151
|
+
deepseekApiKeyEnv: z.string().role("credential-ref").default(DEFAULT_KEY_ENV.deepseek),
|
|
152
|
+
anthropicApiKeyEnv: z.string().role("credential-ref").default(DEFAULT_KEY_ENV.anthropic),
|
|
153
|
+
openaiApiKeyEnv: z.string().role("credential-ref").default(DEFAULT_KEY_ENV.openai),
|
|
154
|
+
searxngApiKeyEnv: z.string().role("credential-ref").default("SEARXNG_API_KEY")
|
|
155
|
+
.description("SearXNG 私有实例的 API Key(公共实例留空)"),
|
|
156
|
+
|
|
157
|
+
// Base URLs (defaults from built-in registry)
|
|
158
|
+
braveBaseURL: z.string().default(DEFAULT_BASE_URLS.brave),
|
|
159
|
+
tavilyBaseURL: z.string().default(DEFAULT_BASE_URLS.tavily),
|
|
160
|
+
serperBaseURL: z.string().default(DEFAULT_BASE_URLS.serper),
|
|
161
|
+
mojeekBaseURL: z.string().default(DEFAULT_BASE_URLS.mojeek),
|
|
162
|
+
searxngBaseURL: z.string().default(DEFAULT_BASE_URLS.searxng),
|
|
163
|
+
deepseekBaseURL: z.string().default(DEFAULT_BASE_URLS.deepseek),
|
|
164
|
+
anthropicBaseURL: z.string().default(DEFAULT_BASE_URLS.anthropic),
|
|
165
|
+
openaiBaseURL: z.string().default(DEFAULT_BASE_URLS.openai),
|
|
166
|
+
|
|
167
|
+
// Models
|
|
168
|
+
deepseekModel: z.string().default(DEFAULT_MODELS.deepseek),
|
|
169
|
+
anthropicModel: z.string().default(DEFAULT_MODELS.anthropic),
|
|
170
|
+
openaiModel: z.string().default(DEFAULT_MODELS.openai),
|
|
171
|
+
tavilySearchDepth: z.union(["basic", "advanced"]).default("basic")
|
|
172
|
+
.description("Tavily 搜索深度"),
|
|
173
|
+
|
|
174
|
+
// Legacy compatibility: explicit enabled-backend list overrides toggles
|
|
175
|
+
enabledBackends: z.array(z.string())
|
|
176
|
+
.description("显式启用后端列表(优先于上方开关;留空则用开关)"),
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Project one resolved settings section into the options snapshot the provider
|
|
181
|
+
* serves its NEXT search with. Key resolution: credentialRef -> credentials
|
|
182
|
+
* service -> env.
|
|
183
|
+
*/
|
|
184
|
+
export function resolveOptions(ctx, config) {
|
|
185
|
+
// 1. Determine enabled backends: explicit list > toggles > env > defaults
|
|
186
|
+
let enabledBackends = [];
|
|
187
|
+
|
|
188
|
+
if (config?.enabledBackends && config.enabledBackends.length > 0) {
|
|
189
|
+
enabledBackends = config.enabledBackends.filter((id) => ALL_BACKENDS.includes(id));
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (enabledBackends.length === 0) {
|
|
193
|
+
const toggleMap = {
|
|
194
|
+
exa: config?.enableExa,
|
|
195
|
+
parallel: config?.enableParallel,
|
|
196
|
+
ddg: config?.enableDdg,
|
|
197
|
+
searxng: config?.enableSearxng,
|
|
198
|
+
brave: config?.enableBrave,
|
|
199
|
+
tavily: config?.enableTavily,
|
|
200
|
+
serper: config?.enableSerper,
|
|
201
|
+
mojeek: config?.enableMojeek,
|
|
202
|
+
deepseek: config?.enableDeepseek,
|
|
203
|
+
anthropic: config?.enableAnthropic,
|
|
204
|
+
openai: config?.enableOpenai,
|
|
205
|
+
};
|
|
206
|
+
enabledBackends = ALL_BACKENDS.filter((id) => toggleMap[id] === true);
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
if (enabledBackends.length === 0) {
|
|
210
|
+
const envList = launchEnvironmentOf(ctx).get(ENABLED_ENV)?.value;
|
|
211
|
+
if (envList && envList.length > 0) {
|
|
212
|
+
enabledBackends = envList
|
|
213
|
+
.split(/[,\s]+/)
|
|
214
|
+
.map((s) => s.trim())
|
|
215
|
+
.filter((s) => ALL_BACKENDS.includes(s));
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
if (enabledBackends.length === 0) {
|
|
220
|
+
enabledBackends = [...DEFAULT_ENABLED];
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// 2. Resolve each backend option
|
|
224
|
+
const backends = {};
|
|
225
|
+
for (const id of ALL_BACKENDS) {
|
|
226
|
+
const keyEnvName = config?.[`${id}ApiKeyEnv`] ?? DEFAULT_KEY_ENV[id];
|
|
227
|
+
const ref = credentialRef(keyEnvName);
|
|
228
|
+
|
|
229
|
+
backends[id] = {
|
|
230
|
+
apiKeyEnv: keyEnvName,
|
|
231
|
+
baseURL: config?.[`${id}BaseURL`] ?? DEFAULT_BASE_URLS[id],
|
|
232
|
+
model: config?.[`${id}Model`] ?? DEFAULT_MODELS[id],
|
|
233
|
+
searchDepth: config?.tavilySearchDepth,
|
|
234
|
+
label: DEFAULT_LABELS[id] ?? id,
|
|
235
|
+
requiresKey: DEFAULT_KEY_ENV[id] !== undefined,
|
|
236
|
+
resolveApiKey: async () => {
|
|
237
|
+
const credentials = ctx.get("credentials");
|
|
238
|
+
if (credentials !== undefined) return (await credentials.resolve(ref))?.value;
|
|
239
|
+
const ambient = launchEnvironmentOf(ctx).get(ref);
|
|
240
|
+
return ambient !== undefined && ambient.value.length > 0 ? ambient.value : undefined;
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
return {
|
|
246
|
+
enabledBackends,
|
|
247
|
+
numResults: config?.numResults ?? 8,
|
|
248
|
+
concurrency: config?.concurrency ?? 6,
|
|
249
|
+
backendTimeoutMs: config?.backendTimeoutMs ?? 30000,
|
|
250
|
+
backends,
|
|
251
|
+
ctx,
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Register the unified search provider with ctx.web. Installs a settings
|
|
257
|
+
* section so users can configure enabled backends / numResults / base URLs /
|
|
258
|
+
* model names / credential references in the DSH Settings panel.
|
|
259
|
+
*/
|
|
260
|
+
export function apply(ctx, config) {
|
|
261
|
+
let current = () => config;
|
|
262
|
+
|
|
263
|
+
installSettingsSection(ctx, NAMESPACE, Config, config, {
|
|
264
|
+
setSource: (source) => {
|
|
265
|
+
current = source;
|
|
266
|
+
},
|
|
267
|
+
onChange: () => {},
|
|
268
|
+
});
|
|
269
|
+
|
|
270
|
+
const provider = createUnifiedSearchProvider({
|
|
271
|
+
ctx,
|
|
272
|
+
resolveOptions: () => resolveOptions(ctx, current()),
|
|
273
|
+
backends: BACKENDS,
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
ctx.web.registerSearchProvider(provider);
|
|
277
|
+
}
|
package/lib/provider.js
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
// provider.js — UnifiedSearchProvider: one ctx.web search provider (id "unified")
|
|
2
|
+
// that fans out to multiple backends concurrently and merges results. This
|
|
3
|
+
// keeps the seam's selection rule happy (only one provider is registered, so
|
|
4
|
+
// no WEB_PROVIDER_AMBIGUOUS error) while giving the user richer results than
|
|
5
|
+
// any single backend alone.
|
|
6
|
+
//
|
|
7
|
+
// Per-backend abort demotion: a single backend throwing WEB_ABORTED becomes a
|
|
8
|
+
// soft null (so other backends still contribute) — UNLESS the caller's own
|
|
9
|
+
// AbortSignal fired (then we rethrow WEB_ABORTED up to the seam). Non-abort
|
|
10
|
+
// failures are also soft: the provider stays usable as long as ≥1 backend
|
|
11
|
+
// returns sources. Only when every enabled backend fails do we throw
|
|
12
|
+
// WEB_PROVIDER_ERROR.
|
|
13
|
+
//
|
|
14
|
+
// Concurrency control: limits simultaneous backend calls to opts.concurrency (default 6).
|
|
15
|
+
// Per-backend timeout: each backend call wrapped with opts.backendTimeoutMs (default 30s).
|
|
16
|
+
|
|
17
|
+
import { WebError } from "@deepseek-ai/dsh-web";
|
|
18
|
+
import { maybeAbortError, searchAborted, throwIfSearchAborted, isAbortedWebError } from "./util/abort.js";
|
|
19
|
+
import {
|
|
20
|
+
recordBackendRequest,
|
|
21
|
+
recordBackendOutcome,
|
|
22
|
+
} from "./util/log.js";
|
|
23
|
+
|
|
24
|
+
const PROVIDER_ID = "unified";
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Simple concurrency limiter using a semaphore pattern.
|
|
28
|
+
* No external dependency needed.
|
|
29
|
+
*/
|
|
30
|
+
function createLimiter(concurrency) {
|
|
31
|
+
const queue = [];
|
|
32
|
+
let running = 0;
|
|
33
|
+
|
|
34
|
+
return function limit(fn) {
|
|
35
|
+
return new Promise((resolve, reject) => {
|
|
36
|
+
queue.push({ fn, resolve, reject });
|
|
37
|
+
process();
|
|
38
|
+
|
|
39
|
+
function process() {
|
|
40
|
+
while (running < concurrency && queue.length > 0) {
|
|
41
|
+
const { fn, resolve, reject } = queue.shift();
|
|
42
|
+
running++;
|
|
43
|
+
Promise.resolve(fn()).then(resolve).catch(reject).finally(() => {
|
|
44
|
+
running--;
|
|
45
|
+
process();
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Build the UnifiedSearchProvider.
|
|
55
|
+
*
|
|
56
|
+
* @param {object} deps
|
|
57
|
+
* @param {object} deps.ctx the Cordis plugin context
|
|
58
|
+
* @param {Function} deps.resolveOptions () => resolved options snapshot
|
|
59
|
+
* @param {object} deps.backends keyed backend registry
|
|
60
|
+
*/
|
|
61
|
+
export function createUnifiedSearchProvider({ ctx, resolveOptions, backends }) {
|
|
62
|
+
return {
|
|
63
|
+
id: PROVIDER_ID,
|
|
64
|
+
|
|
65
|
+
available() {
|
|
66
|
+
const opts = resolveOptions();
|
|
67
|
+
// Provider is usable iff at least one enabled backend is available.
|
|
68
|
+
for (const id of opts.enabledBackends) {
|
|
69
|
+
const be = backends[id];
|
|
70
|
+
if (!be) continue;
|
|
71
|
+
const beOpts = opts.backends?.[id] ?? {};
|
|
72
|
+
if (be.available(beOpts)) return true;
|
|
73
|
+
}
|
|
74
|
+
return false;
|
|
75
|
+
},
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Fan out to all enabled & available backends concurrently, merge results
|
|
79
|
+
* (URL-dedup, keep first-seen snippet/title), enforce maxResults.
|
|
80
|
+
*
|
|
81
|
+
* @param {WebSearchRequest} request
|
|
82
|
+
* @param {AbortSignal} signal
|
|
83
|
+
* @returns {Promise<WebSearchResult>}
|
|
84
|
+
*/
|
|
85
|
+
async search(request, signal) {
|
|
86
|
+
throwIfSearchAborted(signal);
|
|
87
|
+
const opts = resolveOptions();
|
|
88
|
+
const maxResults = request.maxResults ?? opts.numResults ?? 8;
|
|
89
|
+
const concurrency = opts.concurrency ?? 6;
|
|
90
|
+
const backendTimeoutMs = opts.backendTimeoutMs ?? 30000;
|
|
91
|
+
|
|
92
|
+
// Pick enabled & available backends.
|
|
93
|
+
const eligible = [];
|
|
94
|
+
for (const id of opts.enabledBackends) {
|
|
95
|
+
const be = backends[id];
|
|
96
|
+
if (!be) continue;
|
|
97
|
+
const beOpts = opts.backends?.[id] ?? {};
|
|
98
|
+
const avail = be.available(beOpts);
|
|
99
|
+
if (!avail) continue;
|
|
100
|
+
eligible.push({ id, backend: be, beOpts });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (eligible.length === 0) {
|
|
104
|
+
throw new WebError(
|
|
105
|
+
"unified search: no enabled backend is available (configure a backend or set an API key)",
|
|
106
|
+
"WEB_PROVIDER_UNAVAILABLE",
|
|
107
|
+
);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Concurrency limiter
|
|
111
|
+
const limit = createLimiter(opts.concurrency ?? 6);
|
|
112
|
+
|
|
113
|
+
// Wrap each backend call with timeout
|
|
114
|
+
const runWithTimeout = (backend, req, sig, beOpts, hooks) => {
|
|
115
|
+
const timeoutMs = backendTimeoutMs;
|
|
116
|
+
const controller = new AbortController();
|
|
117
|
+
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
|
118
|
+
|
|
119
|
+
// Merge signals
|
|
120
|
+
const mergedSignal = sig
|
|
121
|
+
? AbortSignal.any([sig, controller.signal])
|
|
122
|
+
: controller.signal;
|
|
123
|
+
|
|
124
|
+
return Promise.race([
|
|
125
|
+
backend.search(req, mergedSignal, beOpts, hooks),
|
|
126
|
+
new Promise((_, reject) => {
|
|
127
|
+
controller.signal.addEventListener('abort', () => {
|
|
128
|
+
if (controller.signal.reason === 'timeout' || !sig?.aborted) {
|
|
129
|
+
reject(new WebError(`Backend timed out after ${timeoutMs}ms`, "WEB_PROVIDER_ERROR"));
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
})
|
|
133
|
+
]).finally(() => clearTimeout(timeoutId));
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
const results = await Promise.allSettled(
|
|
137
|
+
eligible.map(({ id, backend, beOpts }) => {
|
|
138
|
+
const recordRequest = (req) => recordBackendRequest(ctx, id, req);
|
|
139
|
+
const recordOutcome = (out) => recordBackendOutcome(ctx, id, out);
|
|
140
|
+
const hooks = { recordRequest, recordOutcome };
|
|
141
|
+
|
|
142
|
+
return limit(async () => {
|
|
143
|
+
return runWithTimeout(backend, request, signal, beOpts, hooks);
|
|
144
|
+
});
|
|
145
|
+
}),
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
throwIfSearchAborted(signal);
|
|
149
|
+
|
|
150
|
+
// Merge: dedup by url (case-insensitive normalized), keep first-seen
|
|
151
|
+
// non-empty title/snippet/publishedAt. Tolerate soft-null settled values.
|
|
152
|
+
const seen = new Set();
|
|
153
|
+
const merged = [];
|
|
154
|
+
let mergedContent;
|
|
155
|
+
let hardAborted = false;
|
|
156
|
+
let hardError = null;
|
|
157
|
+
const failures = [];
|
|
158
|
+
|
|
159
|
+
for (const settled of results) {
|
|
160
|
+
if (settled.status === "fulfilled") {
|
|
161
|
+
if (mergedContent === undefined && typeof settled.value?.content === "string" && settled.value.content.length > 0) {
|
|
162
|
+
mergedContent = settled.value.content;
|
|
163
|
+
}
|
|
164
|
+
const srcs = settled.value?.sources ?? [];
|
|
165
|
+
for (const s of srcs) {
|
|
166
|
+
if (!s || typeof s.url !== "string" || s.url.length === 0) continue;
|
|
167
|
+
const key = s.url.toLowerCase();
|
|
168
|
+
if (seen.has(key)) {
|
|
169
|
+
// Merge fields into the existing entry (fill-in missing ones).
|
|
170
|
+
const existing = merged.find((m) => m.url.toLowerCase() === key);
|
|
171
|
+
if (existing) {
|
|
172
|
+
if (!existing.title && s.title) existing.title = s.title;
|
|
173
|
+
if (!existing.snippet && s.snippet) existing.snippet = s.snippet;
|
|
174
|
+
if (!existing.publishedAt && s.publishedAt) existing.publishedAt = s.publishedAt;
|
|
175
|
+
}
|
|
176
|
+
continue;
|
|
177
|
+
}
|
|
178
|
+
seen.add(key);
|
|
179
|
+
merged.push({ ...s });
|
|
180
|
+
if (merged.length >= maxResults) break;
|
|
181
|
+
}
|
|
182
|
+
if (merged.length >= maxResults) break;
|
|
183
|
+
} else {
|
|
184
|
+
// settled.status === "rejected"
|
|
185
|
+
const err = settled.reason;
|
|
186
|
+
if (err instanceof WebError && err.code === "WEB_ABORTED") {
|
|
187
|
+
// Per-backend abort → soft unless caller signal fired.
|
|
188
|
+
if (signal?.aborted === true) hardAborted = true;
|
|
189
|
+
else failures.push({ id: "abort", err });
|
|
190
|
+
} else {
|
|
191
|
+
failures.push({ id: "error", err });
|
|
192
|
+
if (hardError === null) hardError = err;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
// Caller-cancellation check (hard abort fires before we yield results).
|
|
198
|
+
if (signal?.aborted === true || hardAborted) {
|
|
199
|
+
throw searchAborted(signal);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// If we got ANY sources, return them even if some backends failed.
|
|
203
|
+
if (merged.length > 0) {
|
|
204
|
+
const sources = merged.slice(0, maxResults);
|
|
205
|
+
const wasCapped =
|
|
206
|
+
merged.length > maxResults ||
|
|
207
|
+
results.some((r) => r.status === "fulfilled" && (r.value?.sources?.length ?? 0) > maxResults);
|
|
208
|
+
return { sources, ...(mergedContent !== undefined ? { content: mergedContent } : {}), truncated: sources.length >= maxResults && wasCapped };
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
// All enabled backends failed — synthesize a meaningful WEB_PROVIDER_ERROR.
|
|
212
|
+
const firstDetail =
|
|
213
|
+
hardError instanceof Error
|
|
214
|
+
? hardError.message
|
|
215
|
+
: String(hardError ?? "unknown");
|
|
216
|
+
throw new WebError(
|
|
217
|
+
`unified search: all enabled backends failed (${failures.length}). First: ${firstDetail}`,
|
|
218
|
+
"WEB_PROVIDER_ERROR",
|
|
219
|
+
{ cause: hardError ?? undefined },
|
|
220
|
+
);
|
|
221
|
+
},
|
|
222
|
+
};
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export default { createUnifiedSearchProvider, PROVIDER_ID };
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module dsh-websearch/util/abort
|
|
3
|
+
*
|
|
4
|
+
* Abort/cancellation plumbing for the unified search provider. The provider
|
|
5
|
+
* fan-outs to multiple backends concurrently; every backend's fetch must
|
|
6
|
+
* observe the caller's signal so a cancelled search releases network quota.
|
|
7
|
+
*
|
|
8
|
+
* Two WebError codes the seam recognizes:
|
|
9
|
+
* - "WEB_ABORTED": user/tool cancelled; no further retries attempted.
|
|
10
|
+
* - "WEB_PROVIDER_ERROR": a backend failed (network/http/parse/key-missing).
|
|
11
|
+
*
|
|
12
|
+
* A single backend aborting is demoted to a soft null by the provider so other
|
|
13
|
+
* backends still contribute. Only when the caller-cancellable AbortSignal fires
|
|
14
|
+
* does the provider throw the hard WEB_ABORTED up to the seam.
|
|
15
|
+
*/
|
|
16
|
+
import { WebError } from "@deepseek-ai/dsh-web";
|
|
17
|
+
|
|
18
|
+
/** DOMException#AbortError covers both signal.abort() and AbortSignal.timeout(ms). */
|
|
19
|
+
export function isAbortError(error) {
|
|
20
|
+
return error instanceof DOMException && error.name === "AbortError";
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** True for any WebError carrying the WEB_ABORTED code. */
|
|
24
|
+
export function isAbortedWebError(error) {
|
|
25
|
+
return error instanceof WebError && error.code === "WEB_ABORTED";
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Build the provider's stable cancellation error, retaining the caller's reason. */
|
|
29
|
+
export function searchAborted(signal, fallback) {
|
|
30
|
+
return new WebError("unified search aborted", "WEB_ABORTED", {
|
|
31
|
+
cause: signal?.aborted === true ? signal.reason : fallback,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Convenience: throw WEB_ABORTED if the caller already cancelled. */
|
|
36
|
+
export function throwIfSearchAborted(signal) {
|
|
37
|
+
if (signal?.aborted === true) throw searchAborted(signal);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Race an async operation against caller cancellation. The underlying operation
|
|
42
|
+
* keeps running after abort (we cannot cancel an in-flight credential resolve);
|
|
43
|
+
* we just reject the wrapper promise. Backend fetch paths forward signal directly.
|
|
44
|
+
*/
|
|
45
|
+
export function abortable(operation, signal) {
|
|
46
|
+
if (signal === undefined) return operation;
|
|
47
|
+
if (signal.aborted) return Promise.reject(searchAborted(signal));
|
|
48
|
+
return new Promise((resolve, reject) => {
|
|
49
|
+
const onAbort = () => reject(searchAborted(signal));
|
|
50
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
51
|
+
operation.then(
|
|
52
|
+
(value) => {
|
|
53
|
+
signal.removeEventListener("abort", onAbort);
|
|
54
|
+
resolve(value);
|
|
55
|
+
},
|
|
56
|
+
(error) => {
|
|
57
|
+
signal.removeEventListener("abort", onAbort);
|
|
58
|
+
reject(error instanceof WebError ? error : new Error(String(error?.message ?? error), { cause: error }));
|
|
59
|
+
},
|
|
60
|
+
);
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* True when a thrown error reflects caller cancellation (not a backend failure).
|
|
66
|
+
* Used by backends to decide: demote-to-null vs rethrow. Checks signal.aborted,
|
|
67
|
+
* DOMException AbortError, and WebError WEB_ABORTED in that order.
|
|
68
|
+
*/
|
|
69
|
+
export function wasAborted(error, signal) {
|
|
70
|
+
return (
|
|
71
|
+
signal?.aborted === true ||
|
|
72
|
+
isAbortError(error) ||
|
|
73
|
+
isAbortedWebError(error)
|
|
74
|
+
);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Alias for wasAborted — the backend-facing helper used to decide whether a
|
|
79
|
+
* caught error should be converted to a soft null (caller cancelled) or
|
|
80
|
+
* rethrown as WEB_PROVIDER_ERROR (backend failed).
|
|
81
|
+
*/
|
|
82
|
+
export const maybeAbortError = wasAborted;
|
package/lib/util/log.js
ADDED
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module dsh-websearch/util/log
|
|
3
|
+
*
|
|
4
|
+
* Emit a structured "search/backend-request" event onto the current session,
|
|
5
|
+
* so debugging / token-cost attribution is possible from the dsh session log.
|
|
6
|
+
* Mirrors the official dsh-web-search-deepseek `recordRequest` pattern but
|
|
7
|
+
* is per-backend (each backend records its own endpoint + body so it is
|
|
8
|
+
* auditable which backend fired which request).
|
|
9
|
+
*
|
|
10
|
+
* Resilient to missing agents plane — if the session ledger is unavailable,
|
|
11
|
+
* the call silently no-ops (we MUST NOT break search just because we can't
|
|
12
|
+
* record an audit row).
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Record one backend request to the current session's event ledger.
|
|
17
|
+
*
|
|
18
|
+
* @param {object} ctx - the Cordis plugin context (`ctx.get("agents")`).
|
|
19
|
+
* @param {string} backendId - e.g. "exa" / "parallel" / "ddg" / "deepseek" / "anthropic" / "openai".
|
|
20
|
+
* @param {object} request - { endpoint, toolName, body, headers? } — headers are
|
|
21
|
+
* filtered by the caller (do not pass Authorization/x-api-key here).
|
|
22
|
+
*/
|
|
23
|
+
export function recordBackendRequest(ctx, backendId, request) {
|
|
24
|
+
try {
|
|
25
|
+
const agents = ctx.get?.("agents");
|
|
26
|
+
const initiator = agents?.currentInitiator?.();
|
|
27
|
+
const session = initiator?.session;
|
|
28
|
+
if (session?.append) {
|
|
29
|
+
session.append("web/unified-search-backend-request", { backendId, ...request, ignorable: true });
|
|
30
|
+
}
|
|
31
|
+
} catch {
|
|
32
|
+
// audit logging is best-effort; never fatal
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Record a backend's final outcome (number of sources returned / error message).
|
|
38
|
+
* Pairs with recordBackendRequest; useful for token-cost attribution.
|
|
39
|
+
*/
|
|
40
|
+
export function recordBackendOutcome(ctx, backendId, outcome) {
|
|
41
|
+
try {
|
|
42
|
+
const agents = ctx.get?.("agents");
|
|
43
|
+
const initiator = agents?.currentInitiator?.();
|
|
44
|
+
const session = initiator?.session;
|
|
45
|
+
if (session?.append) {
|
|
46
|
+
session.append("web/unified-search-backend-outcome", { backendId, ...outcome, ignorable: true });
|
|
47
|
+
}
|
|
48
|
+
} catch {
|
|
49
|
+
// best-effort
|
|
50
|
+
}
|
|
51
|
+
}
|