@magic6/dsh-web-fallback-chain 0.0.0-stage → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +57 -0
- package/LICENSE +21 -0
- package/README.md +245 -2
- package/README.zh-CN.md +213 -0
- package/cordis.patch.yml +24 -0
- package/icon.svg +15 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +81 -4
- package/src/chains/fetch-plugin.js +213 -0
- package/src/chains/fetch.js +224 -0
- package/src/chains/search-plugin.js +108 -0
- package/src/chains/search.js +159 -0
- package/src/client.js +1655 -0
- package/src/config.js +245 -0
- package/src/index.js +137 -0
- package/src/policy/index.js +37 -0
- package/src/policy/modes.js +179 -0
- package/src/policy/providers.js +191 -0
- package/src/prompt.js +104 -0
- package/src/providers/http.js +213 -0
- package/src/providers/index.js +65 -0
- package/src/providers/vendors.js +309 -0
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The three vendors, each reduced to what differs between them.
|
|
3
|
+
*
|
|
4
|
+
* Every leg here is a pair of functions — `search` and/or `fetch` — plus the two
|
|
5
|
+
* things the seam needs from a provider: an `id` and an `available()` that never
|
|
6
|
+
* touches the network. Retry, timeout, key lookup and error coding live in
|
|
7
|
+
* `http.js` so a vendor file only describes its own envelope and its own quirks.
|
|
8
|
+
*
|
|
9
|
+
* ## The quirks that matter, all measured
|
|
10
|
+
*
|
|
11
|
+
* - **AnySearch `/v1/search` caps at 10 results** regardless of a larger
|
|
12
|
+
* `max_results`; asking for 20 returns 10. The cap is enforced here so callers
|
|
13
|
+
* see the same number they would from any other leg.
|
|
14
|
+
* - **AnySearch returns `snippet` (short, query-focused) and `content` (long
|
|
15
|
+
* body) separately.** The benchmark found the short field is what a `brief`
|
|
16
|
+
* caller wants and the long one is what a `deep` caller wants, so both are
|
|
17
|
+
* carried and the chain's budget decides which survives.
|
|
18
|
+
* - **DeepSeek returns `summary` (600–800 chars) as well as `snippet`
|
|
19
|
+
* (100–200).** Mapping only `snippet` throws away ~86% of the text, so
|
|
20
|
+
* `summary` is preferred when present.
|
|
21
|
+
* - **Tavily has no date field** on results; DeepSeek does. This is why the chain
|
|
22
|
+
* cannot synthesise recency for a free answer.
|
|
23
|
+
* - **Tavily's free tier is monthly**, so it sits below AnySearch in the shipped
|
|
24
|
+
* ranking even though their measured quality is similar.
|
|
25
|
+
*
|
|
26
|
+
* @module dsh-web-fallback-chain/providers/vendors
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { callJson, bearer, credentialSource, keyFrom, providerError } from './http.js'
|
|
30
|
+
|
|
31
|
+
const DS_BASE = 'https://api.deepseek.com/v1/tools'
|
|
32
|
+
const AS_BASE = 'https://api.anysearch.com/v1'
|
|
33
|
+
const TV_BASE = 'https://api.tavily.com'
|
|
34
|
+
|
|
35
|
+
/** An ISO-ish date, normalised to `YYYY-MM-DD` when it parses. */
|
|
36
|
+
function asDate(value) {
|
|
37
|
+
if (typeof value !== 'string' || value.length === 0) return undefined
|
|
38
|
+
const match = /^(\d{4})-(\d{2})-(\d{2})/.exec(value)
|
|
39
|
+
return match ? `${match[1]}-${match[2]}-${match[3]}` : value
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// ─── DeepSeek ───────────────────────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* DeepSeek search/fetch. Billed per successful request; the only leg with index
|
|
46
|
+
* dates and the only one whose search quality led every free alternative.
|
|
47
|
+
*
|
|
48
|
+
* @param options - `{env, timeoutMs, needContent}`.
|
|
49
|
+
*/
|
|
50
|
+
export function deepseekSearchProvider({ env, credentials, timeoutMs, needContent = false }) {
|
|
51
|
+
const key = credentialSource({ credentials, env, name: 'DEEPSEEK_API_KEY' })
|
|
52
|
+
return {
|
|
53
|
+
id: 'deepseek-search-rich',
|
|
54
|
+
available: () => key.peek() !== undefined,
|
|
55
|
+
async search(request, signal) {
|
|
56
|
+
const apiKey = await key.resolve()
|
|
57
|
+
if (apiKey === undefined) throw providerError('deepseek-search-rich: DEEPSEEK_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
|
|
58
|
+
const body = await callJson({
|
|
59
|
+
url: `${DS_BASE}/search`,
|
|
60
|
+
method: 'POST',
|
|
61
|
+
headers: bearer(apiKey),
|
|
62
|
+
body: {
|
|
63
|
+
query: String(request?.query ?? ''),
|
|
64
|
+
count: Math.min(Math.max(Number(request?.maxResults) || 10, 1), 50),
|
|
65
|
+
need_content: needContent === true,
|
|
66
|
+
},
|
|
67
|
+
timeoutMs,
|
|
68
|
+
signal,
|
|
69
|
+
provider: 'deepseek-search-rich',
|
|
70
|
+
})
|
|
71
|
+
const results = Array.isArray(body?.results) ? body.results : []
|
|
72
|
+
return {
|
|
73
|
+
// `truncated` is required by WebSearchResult: `dsh-tool-web` writes it
|
|
74
|
+
// straight into the tool output, so omitting it puts an `undefined`
|
|
75
|
+
// into a value that must be lossless JSON and fails validation with
|
|
76
|
+
// `value is not lossless JSON`.
|
|
77
|
+
truncated: false,
|
|
78
|
+
sources: results.map((r) => ({
|
|
79
|
+
title: String(r?.title ?? ''),
|
|
80
|
+
url: String(r?.url ?? ''),
|
|
81
|
+
// `summary` carries the substance; `snippet` is a 100–200 char teaser.
|
|
82
|
+
snippet: String(r?.summary || r?.snippet || r?.content || ''),
|
|
83
|
+
...(asDate(r?.date) === undefined ? {} : { publishedAt: asDate(r?.date) }),
|
|
84
|
+
...(r?.site_name === undefined ? {} : { siteName: String(r.site_name) }),
|
|
85
|
+
})),
|
|
86
|
+
}
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* DeepSeek fetch. Returns markdown; knowingly degrades to a site footer with
|
|
93
|
+
* HTTP 200 on some sites, which is why the fetch chain takes a length floor.
|
|
94
|
+
*/
|
|
95
|
+
export function deepseekFetchProvider({ env, credentials, timeoutMs }) {
|
|
96
|
+
const key = credentialSource({ credentials, env, name: 'DEEPSEEK_API_KEY' })
|
|
97
|
+
return {
|
|
98
|
+
id: 'deepseek-fetch-rich',
|
|
99
|
+
available: () => key.peek() !== undefined,
|
|
100
|
+
async fetch(request, signal) {
|
|
101
|
+
const apiKey = await key.resolve()
|
|
102
|
+
if (apiKey === undefined) throw providerError('deepseek-fetch-rich: DEEPSEEK_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
|
|
103
|
+
const body = await callJson({
|
|
104
|
+
url: `${DS_BASE}/fetch`,
|
|
105
|
+
method: 'POST',
|
|
106
|
+
headers: bearer(apiKey),
|
|
107
|
+
body: { url: String(request?.url ?? '') },
|
|
108
|
+
timeoutMs,
|
|
109
|
+
signal,
|
|
110
|
+
provider: 'deepseek-fetch-rich',
|
|
111
|
+
})
|
|
112
|
+
return {
|
|
113
|
+
url: String(body?.url ?? request?.url ?? ''),
|
|
114
|
+
statusCode: 200,
|
|
115
|
+
body: { kind: 'text', content: String(body?.content ?? '') },
|
|
116
|
+
truncated: false,
|
|
117
|
+
}
|
|
118
|
+
},
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
// ─── AnySearch ──────────────────────────────────────────────────────────────
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* AnySearch search/extract, with optional vertical-domain routing.
|
|
126
|
+
*
|
|
127
|
+
* `tag` + `params` change the query's meaning, not just its ranking: the same
|
|
128
|
+
* text under `code.doc` returns source documentation where general search
|
|
129
|
+
* returns an unrelated namesake. They are therefore an explicit opt-in per
|
|
130
|
+
* plugin config, never inferred from the query.
|
|
131
|
+
*
|
|
132
|
+
* @param options - `{env, timeoutMs, tag, params}`.
|
|
133
|
+
*/
|
|
134
|
+
export function anysearchProvider({ env, credentials, timeoutMs, tag = '', params = '' }) {
|
|
135
|
+
const key = credentialSource({ credentials, env, name: 'ANYSEARCH_API_KEY' })
|
|
136
|
+
const parseParams = () => {
|
|
137
|
+
const out = {}
|
|
138
|
+
for (const pair of String(params ?? '').split(',')) {
|
|
139
|
+
const [k, ...rest] = pair.split('=')
|
|
140
|
+
if (k === undefined || k.trim() === '') continue
|
|
141
|
+
out[k.trim()] = rest.join('=').trim()
|
|
142
|
+
}
|
|
143
|
+
return out
|
|
144
|
+
}
|
|
145
|
+
return {
|
|
146
|
+
id: 'anysearch',
|
|
147
|
+
available: () => key.peek() !== undefined,
|
|
148
|
+
async search(request, signal) {
|
|
149
|
+
const apiKey = await key.resolve()
|
|
150
|
+
if (apiKey === undefined) throw providerError('anysearch: ANYSEARCH_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
|
|
151
|
+
// The API caps at 10 per call; clamping here keeps the reported count honest.
|
|
152
|
+
const max = Math.min(Math.max(Number(request?.maxResults) || 10, 1), 10)
|
|
153
|
+
const body = await callJson({
|
|
154
|
+
url: `${AS_BASE}/search`,
|
|
155
|
+
method: 'POST',
|
|
156
|
+
headers: bearer(apiKey),
|
|
157
|
+
body: {
|
|
158
|
+
query: String(request?.query ?? ''),
|
|
159
|
+
max_results: max,
|
|
160
|
+
...(String(tag ?? '').length > 0 ? { tag } : {}),
|
|
161
|
+
...(Object.keys(parseParams()).length > 0 ? { params: parseParams() } : {}),
|
|
162
|
+
},
|
|
163
|
+
timeoutMs,
|
|
164
|
+
signal,
|
|
165
|
+
provider: 'anysearch',
|
|
166
|
+
})
|
|
167
|
+
if (body?.code !== undefined && body.code !== 0) {
|
|
168
|
+
throw providerError(`anysearch: ${body?.message ?? 'error'}`, 'WEB_PROVIDER_ERROR')
|
|
169
|
+
}
|
|
170
|
+
const results = Array.isArray(body?.data?.results) ? body.data.results : []
|
|
171
|
+
return {
|
|
172
|
+
// See the note on the DeepSeek leg: required field, must not be `undefined`.
|
|
173
|
+
truncated: false,
|
|
174
|
+
sources: results.slice(0, max).map((r) => {
|
|
175
|
+
// `snippet` is the query-focused extract; `content` is the fetched body,
|
|
176
|
+
// which usually opens with site chrome ("# Moddable Blog | # IoT Dev…").
|
|
177
|
+
// Measured on 30 results: capping content-first yields a 300-char window
|
|
178
|
+
// that is 40% navigation junk, snippet-first 0%. So `snippet` leads, and
|
|
179
|
+
// `content` is kept only as a fallback for results that lack it.
|
|
180
|
+
const snippet = String(r?.snippet ?? '')
|
|
181
|
+
const content = String(r?.content ?? '')
|
|
182
|
+
const primary = snippet.length > 0 ? snippet : content
|
|
183
|
+
return {
|
|
184
|
+
title: String(r?.title ?? ''),
|
|
185
|
+
url: String(r?.url ?? ''),
|
|
186
|
+
snippet: primary,
|
|
187
|
+
// Keep the longer body addressable without paying for it up front.
|
|
188
|
+
...(content.length > primary.length ? { summary: content } : {}),
|
|
189
|
+
}
|
|
190
|
+
}),
|
|
191
|
+
}
|
|
192
|
+
},
|
|
193
|
+
async fetch(request, signal) {
|
|
194
|
+
const apiKey = await key.resolve()
|
|
195
|
+
if (apiKey === undefined) throw providerError('anysearch: ANYSEARCH_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
|
|
196
|
+
const body = await callJson({
|
|
197
|
+
url: `${AS_BASE}/extract`,
|
|
198
|
+
method: 'POST',
|
|
199
|
+
headers: bearer(apiKey),
|
|
200
|
+
body: { url: String(request?.url ?? '') },
|
|
201
|
+
timeoutMs,
|
|
202
|
+
signal,
|
|
203
|
+
provider: 'anysearch',
|
|
204
|
+
})
|
|
205
|
+
if (body?.code !== undefined && body.code !== 0) {
|
|
206
|
+
throw providerError(`anysearch: ${body?.message ?? 'extract failed'}`, 'WEB_PROVIDER_UNAVAILABLE')
|
|
207
|
+
}
|
|
208
|
+
const content = String(body?.data?.content ?? '')
|
|
209
|
+
if (content.length === 0) {
|
|
210
|
+
throw providerError('anysearch: extract returned no content', 'WEB_PROVIDER_UNAVAILABLE')
|
|
211
|
+
}
|
|
212
|
+
return {
|
|
213
|
+
url: String(request?.url ?? ''),
|
|
214
|
+
statusCode: 200,
|
|
215
|
+
body: { kind: 'text', content },
|
|
216
|
+
truncated: false,
|
|
217
|
+
}
|
|
218
|
+
},
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ─── Tavily ─────────────────────────────────────────────────────────────────
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Tavily search/extract. The only leg here that extracts PDFs, and the only one
|
|
226
|
+
* with `map`/`crawl`; its free tier is monthly (~1500), so it is ranked below
|
|
227
|
+
* AnySearch despite comparable measured quality.
|
|
228
|
+
*
|
|
229
|
+
* @param options - `{env, timeoutMs}`.
|
|
230
|
+
*/
|
|
231
|
+
export function tavilyProvider({ env, credentials, timeoutMs }) {
|
|
232
|
+
const key = credentialSource({ credentials, env, name: 'TAVILY_API_KEY' })
|
|
233
|
+
return {
|
|
234
|
+
id: 'tavily',
|
|
235
|
+
available: () => key.peek() !== undefined,
|
|
236
|
+
async search(request, signal) {
|
|
237
|
+
const apiKey = await key.resolve()
|
|
238
|
+
if (apiKey === undefined) throw providerError('tavily: TAVILY_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
|
|
239
|
+
const body = await callJson({
|
|
240
|
+
url: `${TV_BASE}/search`,
|
|
241
|
+
method: 'POST',
|
|
242
|
+
headers: bearer(apiKey),
|
|
243
|
+
body: {
|
|
244
|
+
query: String(request?.query ?? ''),
|
|
245
|
+
max_results: Math.min(Math.max(Number(request?.maxResults) || 5, 1), 20),
|
|
246
|
+
},
|
|
247
|
+
timeoutMs,
|
|
248
|
+
signal,
|
|
249
|
+
provider: 'tavily',
|
|
250
|
+
})
|
|
251
|
+
const results = Array.isArray(body?.results) ? body.results : []
|
|
252
|
+
return {
|
|
253
|
+
// See the note on the DeepSeek leg: required field, must not be `undefined`.
|
|
254
|
+
truncated: false,
|
|
255
|
+
sources: results.map((r) => ({
|
|
256
|
+
title: String(r?.title ?? ''),
|
|
257
|
+
url: String(r?.url ?? ''),
|
|
258
|
+
snippet: String(r?.content ?? ''),
|
|
259
|
+
...(asDate(r?.published_date) === undefined ? {} : { publishedAt: asDate(r?.published_date) }),
|
|
260
|
+
})),
|
|
261
|
+
}
|
|
262
|
+
},
|
|
263
|
+
async fetch(request, signal) {
|
|
264
|
+
const apiKey = await key.resolve()
|
|
265
|
+
if (apiKey === undefined) throw providerError('tavily: TAVILY_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
|
|
266
|
+
const body = await callJson({
|
|
267
|
+
url: `${TV_BASE}/extract`,
|
|
268
|
+
method: 'POST',
|
|
269
|
+
headers: bearer(apiKey),
|
|
270
|
+
body: { urls: [String(request?.url ?? '')] },
|
|
271
|
+
timeoutMs,
|
|
272
|
+
signal,
|
|
273
|
+
provider: 'tavily',
|
|
274
|
+
})
|
|
275
|
+
const first = Array.isArray(body?.results) ? body.results[0] : undefined
|
|
276
|
+
const content = String(first?.raw_content ?? '')
|
|
277
|
+
if (content.length === 0) {
|
|
278
|
+
const reason = Array.isArray(body?.failed_results) && body.failed_results[0]?.error
|
|
279
|
+
? String(body.failed_results[0].error)
|
|
280
|
+
: 'no content'
|
|
281
|
+
throw providerError(`tavily: ${reason}`, 'WEB_PROVIDER_UNAVAILABLE')
|
|
282
|
+
}
|
|
283
|
+
return {
|
|
284
|
+
url: String(first?.url ?? request?.url ?? ''),
|
|
285
|
+
statusCode: 200,
|
|
286
|
+
body: { kind: 'text', content },
|
|
287
|
+
truncated: false,
|
|
288
|
+
}
|
|
289
|
+
},
|
|
290
|
+
/**
|
|
291
|
+
* Discover URLs on a site. Not part of the seam; exported for callers that
|
|
292
|
+
* want it explicitly (a crawl needs a map first).
|
|
293
|
+
*/
|
|
294
|
+
async map(request, signal) {
|
|
295
|
+
const apiKey = await key.resolve()
|
|
296
|
+
if (apiKey === undefined) throw providerError('tavily: TAVILY_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
|
|
297
|
+
const body = await callJson({
|
|
298
|
+
url: `${TV_BASE}/map`,
|
|
299
|
+
method: 'POST',
|
|
300
|
+
headers: bearer(apiKey),
|
|
301
|
+
body: { url: String(request?.url ?? ''), limit: Number(request?.limit) || 20 },
|
|
302
|
+
timeoutMs,
|
|
303
|
+
signal,
|
|
304
|
+
provider: 'tavily',
|
|
305
|
+
})
|
|
306
|
+
return { urls: Array.isArray(body?.results) ? body.results.map(String) : [] }
|
|
307
|
+
},
|
|
308
|
+
}
|
|
309
|
+
}
|