@magic6/dsh-web-fallback-chain 0.0.0-stage → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,309 @@
1
+ /**
2
+ * The three vendors, each reduced to what differs between them.
3
+ *
4
+ * Every leg here is a pair of functions — `search` and/or `fetch` — plus the two
5
+ * things the seam needs from a provider: an `id` and an `available()` that never
6
+ * touches the network. Retry, timeout, key lookup and error coding live in
7
+ * `http.js` so a vendor file only describes its own envelope and its own quirks.
8
+ *
9
+ * ## The quirks that matter, all measured
10
+ *
11
+ * - **AnySearch `/v1/search` caps at 10 results** regardless of a larger
12
+ * `max_results`; asking for 20 returns 10. The cap is enforced here so callers
13
+ * see the same number they would from any other leg.
14
+ * - **AnySearch returns `snippet` (short, query-focused) and `content` (long
15
+ * body) separately.** The benchmark found the short field is what a `brief`
16
+ * caller wants and the long one is what a `deep` caller wants, so both are
17
+ * carried and the chain's budget decides which survives.
18
+ * - **DeepSeek returns `summary` (600–800 chars) as well as `snippet`
19
+ * (100–200).** Mapping only `snippet` throws away ~86% of the text, so
20
+ * `summary` is preferred when present.
21
+ * - **Tavily has no date field** on results; DeepSeek does. This is why the chain
22
+ * cannot synthesise recency for a free answer.
23
+ * - **Tavily's free tier is monthly**, so it sits below AnySearch in the shipped
24
+ * ranking even though their measured quality is similar.
25
+ *
26
+ * @module dsh-web-fallback-chain/providers/vendors
27
+ */
28
+
29
+ import { callJson, bearer, credentialSource, keyFrom, providerError } from './http.js'
30
+
31
+ const DS_BASE = 'https://api.deepseek.com/v1/tools'
32
+ const AS_BASE = 'https://api.anysearch.com/v1'
33
+ const TV_BASE = 'https://api.tavily.com'
34
+
35
+ /** An ISO-ish date, normalised to `YYYY-MM-DD` when it parses. */
36
+ function asDate(value) {
37
+ if (typeof value !== 'string' || value.length === 0) return undefined
38
+ const match = /^(\d{4})-(\d{2})-(\d{2})/.exec(value)
39
+ return match ? `${match[1]}-${match[2]}-${match[3]}` : value
40
+ }
41
+
42
+ // ─── DeepSeek ───────────────────────────────────────────────────────────────
43
+
44
+ /**
45
+ * DeepSeek search/fetch. Billed per successful request; the only leg with index
46
+ * dates and the only one whose search quality led every free alternative.
47
+ *
48
+ * @param options - `{env, timeoutMs, needContent}`.
49
+ */
50
+ export function deepseekSearchProvider({ env, credentials, timeoutMs, needContent = false }) {
51
+ const key = credentialSource({ credentials, env, name: 'DEEPSEEK_API_KEY' })
52
+ return {
53
+ id: 'deepseek-search-rich',
54
+ available: () => key.peek() !== undefined,
55
+ async search(request, signal) {
56
+ const apiKey = await key.resolve()
57
+ if (apiKey === undefined) throw providerError('deepseek-search-rich: DEEPSEEK_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
58
+ const body = await callJson({
59
+ url: `${DS_BASE}/search`,
60
+ method: 'POST',
61
+ headers: bearer(apiKey),
62
+ body: {
63
+ query: String(request?.query ?? ''),
64
+ count: Math.min(Math.max(Number(request?.maxResults) || 10, 1), 50),
65
+ need_content: needContent === true,
66
+ },
67
+ timeoutMs,
68
+ signal,
69
+ provider: 'deepseek-search-rich',
70
+ })
71
+ const results = Array.isArray(body?.results) ? body.results : []
72
+ return {
73
+ // `truncated` is required by WebSearchResult: `dsh-tool-web` writes it
74
+ // straight into the tool output, so omitting it puts an `undefined`
75
+ // into a value that must be lossless JSON and fails validation with
76
+ // `value is not lossless JSON`.
77
+ truncated: false,
78
+ sources: results.map((r) => ({
79
+ title: String(r?.title ?? ''),
80
+ url: String(r?.url ?? ''),
81
+ // `summary` carries the substance; `snippet` is a 100–200 char teaser.
82
+ snippet: String(r?.summary || r?.snippet || r?.content || ''),
83
+ ...(asDate(r?.date) === undefined ? {} : { publishedAt: asDate(r?.date) }),
84
+ ...(r?.site_name === undefined ? {} : { siteName: String(r.site_name) }),
85
+ })),
86
+ }
87
+ },
88
+ }
89
+ }
90
+
91
+ /**
92
+ * DeepSeek fetch. Returns markdown; knowingly degrades to a site footer with
93
+ * HTTP 200 on some sites, which is why the fetch chain takes a length floor.
94
+ */
95
+ export function deepseekFetchProvider({ env, credentials, timeoutMs }) {
96
+ const key = credentialSource({ credentials, env, name: 'DEEPSEEK_API_KEY' })
97
+ return {
98
+ id: 'deepseek-fetch-rich',
99
+ available: () => key.peek() !== undefined,
100
+ async fetch(request, signal) {
101
+ const apiKey = await key.resolve()
102
+ if (apiKey === undefined) throw providerError('deepseek-fetch-rich: DEEPSEEK_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
103
+ const body = await callJson({
104
+ url: `${DS_BASE}/fetch`,
105
+ method: 'POST',
106
+ headers: bearer(apiKey),
107
+ body: { url: String(request?.url ?? '') },
108
+ timeoutMs,
109
+ signal,
110
+ provider: 'deepseek-fetch-rich',
111
+ })
112
+ return {
113
+ url: String(body?.url ?? request?.url ?? ''),
114
+ statusCode: 200,
115
+ body: { kind: 'text', content: String(body?.content ?? '') },
116
+ truncated: false,
117
+ }
118
+ },
119
+ }
120
+ }
121
+
122
+ // ─── AnySearch ──────────────────────────────────────────────────────────────
123
+
124
+ /**
125
+ * AnySearch search/extract, with optional vertical-domain routing.
126
+ *
127
+ * `tag` + `params` change the query's meaning, not just its ranking: the same
128
+ * text under `code.doc` returns source documentation where general search
129
+ * returns an unrelated namesake. They are therefore an explicit opt-in per
130
+ * plugin config, never inferred from the query.
131
+ *
132
+ * @param options - `{env, timeoutMs, tag, params}`.
133
+ */
134
+ export function anysearchProvider({ env, credentials, timeoutMs, tag = '', params = '' }) {
135
+ const key = credentialSource({ credentials, env, name: 'ANYSEARCH_API_KEY' })
136
+ const parseParams = () => {
137
+ const out = {}
138
+ for (const pair of String(params ?? '').split(',')) {
139
+ const [k, ...rest] = pair.split('=')
140
+ if (k === undefined || k.trim() === '') continue
141
+ out[k.trim()] = rest.join('=').trim()
142
+ }
143
+ return out
144
+ }
145
+ return {
146
+ id: 'anysearch',
147
+ available: () => key.peek() !== undefined,
148
+ async search(request, signal) {
149
+ const apiKey = await key.resolve()
150
+ if (apiKey === undefined) throw providerError('anysearch: ANYSEARCH_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
151
+ // The API caps at 10 per call; clamping here keeps the reported count honest.
152
+ const max = Math.min(Math.max(Number(request?.maxResults) || 10, 1), 10)
153
+ const body = await callJson({
154
+ url: `${AS_BASE}/search`,
155
+ method: 'POST',
156
+ headers: bearer(apiKey),
157
+ body: {
158
+ query: String(request?.query ?? ''),
159
+ max_results: max,
160
+ ...(String(tag ?? '').length > 0 ? { tag } : {}),
161
+ ...(Object.keys(parseParams()).length > 0 ? { params: parseParams() } : {}),
162
+ },
163
+ timeoutMs,
164
+ signal,
165
+ provider: 'anysearch',
166
+ })
167
+ if (body?.code !== undefined && body.code !== 0) {
168
+ throw providerError(`anysearch: ${body?.message ?? 'error'}`, 'WEB_PROVIDER_ERROR')
169
+ }
170
+ const results = Array.isArray(body?.data?.results) ? body.data.results : []
171
+ return {
172
+ // See the note on the DeepSeek leg: required field, must not be `undefined`.
173
+ truncated: false,
174
+ sources: results.slice(0, max).map((r) => {
175
+ // `snippet` is the query-focused extract; `content` is the fetched body,
176
+ // which usually opens with site chrome ("# Moddable Blog | # IoT Dev…").
177
+ // Measured on 30 results: capping content-first yields a 300-char window
178
+ // that is 40% navigation junk, snippet-first 0%. So `snippet` leads, and
179
+ // `content` is kept only as a fallback for results that lack it.
180
+ const snippet = String(r?.snippet ?? '')
181
+ const content = String(r?.content ?? '')
182
+ const primary = snippet.length > 0 ? snippet : content
183
+ return {
184
+ title: String(r?.title ?? ''),
185
+ url: String(r?.url ?? ''),
186
+ snippet: primary,
187
+ // Keep the longer body addressable without paying for it up front.
188
+ ...(content.length > primary.length ? { summary: content } : {}),
189
+ }
190
+ }),
191
+ }
192
+ },
193
+ async fetch(request, signal) {
194
+ const apiKey = await key.resolve()
195
+ if (apiKey === undefined) throw providerError('anysearch: ANYSEARCH_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
196
+ const body = await callJson({
197
+ url: `${AS_BASE}/extract`,
198
+ method: 'POST',
199
+ headers: bearer(apiKey),
200
+ body: { url: String(request?.url ?? '') },
201
+ timeoutMs,
202
+ signal,
203
+ provider: 'anysearch',
204
+ })
205
+ if (body?.code !== undefined && body.code !== 0) {
206
+ throw providerError(`anysearch: ${body?.message ?? 'extract failed'}`, 'WEB_PROVIDER_UNAVAILABLE')
207
+ }
208
+ const content = String(body?.data?.content ?? '')
209
+ if (content.length === 0) {
210
+ throw providerError('anysearch: extract returned no content', 'WEB_PROVIDER_UNAVAILABLE')
211
+ }
212
+ return {
213
+ url: String(request?.url ?? ''),
214
+ statusCode: 200,
215
+ body: { kind: 'text', content },
216
+ truncated: false,
217
+ }
218
+ },
219
+ }
220
+ }
221
+
222
+ // ─── Tavily ─────────────────────────────────────────────────────────────────
223
+
224
+ /**
225
+ * Tavily search/extract. The only leg here that extracts PDFs, and the only one
226
+ * with `map`/`crawl`; its free tier is monthly (~1500), so it is ranked below
227
+ * AnySearch despite comparable measured quality.
228
+ *
229
+ * @param options - `{env, timeoutMs}`.
230
+ */
231
+ export function tavilyProvider({ env, credentials, timeoutMs }) {
232
+ const key = credentialSource({ credentials, env, name: 'TAVILY_API_KEY' })
233
+ return {
234
+ id: 'tavily',
235
+ available: () => key.peek() !== undefined,
236
+ async search(request, signal) {
237
+ const apiKey = await key.resolve()
238
+ if (apiKey === undefined) throw providerError('tavily: TAVILY_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
239
+ const body = await callJson({
240
+ url: `${TV_BASE}/search`,
241
+ method: 'POST',
242
+ headers: bearer(apiKey),
243
+ body: {
244
+ query: String(request?.query ?? ''),
245
+ max_results: Math.min(Math.max(Number(request?.maxResults) || 5, 1), 20),
246
+ },
247
+ timeoutMs,
248
+ signal,
249
+ provider: 'tavily',
250
+ })
251
+ const results = Array.isArray(body?.results) ? body.results : []
252
+ return {
253
+ // See the note on the DeepSeek leg: required field, must not be `undefined`.
254
+ truncated: false,
255
+ sources: results.map((r) => ({
256
+ title: String(r?.title ?? ''),
257
+ url: String(r?.url ?? ''),
258
+ snippet: String(r?.content ?? ''),
259
+ ...(asDate(r?.published_date) === undefined ? {} : { publishedAt: asDate(r?.published_date) }),
260
+ })),
261
+ }
262
+ },
263
+ async fetch(request, signal) {
264
+ const apiKey = await key.resolve()
265
+ if (apiKey === undefined) throw providerError('tavily: TAVILY_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
266
+ const body = await callJson({
267
+ url: `${TV_BASE}/extract`,
268
+ method: 'POST',
269
+ headers: bearer(apiKey),
270
+ body: { urls: [String(request?.url ?? '')] },
271
+ timeoutMs,
272
+ signal,
273
+ provider: 'tavily',
274
+ })
275
+ const first = Array.isArray(body?.results) ? body.results[0] : undefined
276
+ const content = String(first?.raw_content ?? '')
277
+ if (content.length === 0) {
278
+ const reason = Array.isArray(body?.failed_results) && body.failed_results[0]?.error
279
+ ? String(body.failed_results[0].error)
280
+ : 'no content'
281
+ throw providerError(`tavily: ${reason}`, 'WEB_PROVIDER_UNAVAILABLE')
282
+ }
283
+ return {
284
+ url: String(first?.url ?? request?.url ?? ''),
285
+ statusCode: 200,
286
+ body: { kind: 'text', content },
287
+ truncated: false,
288
+ }
289
+ },
290
+ /**
291
+ * Discover URLs on a site. Not part of the seam; exported for callers that
292
+ * want it explicitly (a crawl needs a map first).
293
+ */
294
+ async map(request, signal) {
295
+ const apiKey = await key.resolve()
296
+ if (apiKey === undefined) throw providerError('tavily: TAVILY_API_KEY is not set', 'WEB_PROVIDER_UNAUTHORIZED')
297
+ const body = await callJson({
298
+ url: `${TV_BASE}/map`,
299
+ method: 'POST',
300
+ headers: bearer(apiKey),
301
+ body: { url: String(request?.url ?? ''), limit: Number(request?.limit) || 20 },
302
+ timeoutMs,
303
+ signal,
304
+ provider: 'tavily',
305
+ })
306
+ return { urls: Array.isArray(body?.results) ? body.results.map(String) : [] }
307
+ },
308
+ }
309
+ }