@moguiyu/dsh-tavily 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -21
- package/lib/client.js +30 -95
- package/package.json +6 -13
- package/src/backend.js +297 -0
- package/src/client.js +30 -95
- package/src/index.js +30 -157
- package/src/lib.js +65 -0
- package/src/tools.js +632 -0
package/src/tools.js
ADDED
|
@@ -0,0 +1,632 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tool half of `@moguiyu/dsh-tavily`: the advanced `tavily_search` model tool
|
|
3
|
+
* plus the extra Tavily direct tools (`tavily_extract`, `tavily_map`,
|
|
4
|
+
* `tavily_crawl`). Keys resolve from the `TAVILY_API_KEYS` credential per
|
|
5
|
+
* call; rotation is round-robin with failover on 401/429.
|
|
6
|
+
*
|
|
7
|
+
* The tools are an OPTION for search, never a replacement: they are
|
|
8
|
+
* independent of the built-in `web_search` provider, and registering them
|
|
9
|
+
* touches `web.searchProvider` not at all.
|
|
10
|
+
*
|
|
11
|
+
* There is deliberately NO tool-level on/off switch. The plugin is either
|
|
12
|
+
* composed or not, and the Plugins page's own toggle is the only on/off.
|
|
13
|
+
* A tool-level switch needs the tool half to be disposable and re-installable
|
|
14
|
+
* at runtime across two packages, and that contract is what broke twice
|
|
15
|
+
* (0.1.6 row restart, then the 0.2.x dependency-range skew). Removing the
|
|
16
|
+
* capability removes the failure mode; see `docs/agents/verification.md`.
|
|
17
|
+
*/
|
|
18
|
+
import { defineTool } from '@deepseek-ai/dsh-tools'
|
|
19
|
+
|
|
20
|
+
const TAVILY_BASE_URL = 'https://api.tavily.com'
|
|
21
|
+
|
|
22
|
+
/** Clamp a number into [min, max]; `fallback` when not finite. */
|
|
23
|
+
export function clampInt(value, min, max, fallback) {
|
|
24
|
+
const n = Number(value)
|
|
25
|
+
if (!Number.isFinite(n)) return fallback
|
|
26
|
+
return Math.min(max, Math.max(min, Math.round(n)))
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/** Normalize and validate model arguments (exported for tests). */
|
|
30
|
+
export function normalizeArgs(args) {
|
|
31
|
+
const query = typeof args.query === 'string' ? args.query.trim() : ''
|
|
32
|
+
if (query.length === 0) throw new Error('tavily_search: query must be a non-empty string')
|
|
33
|
+
const maxResults = clampInt(args.max_results, 1, 20, 5)
|
|
34
|
+
const searchDepth = args.search_depth === 'advanced' ? 'advanced' : 'basic'
|
|
35
|
+
const topic = args.topic === 'news' ? 'news' : 'general'
|
|
36
|
+
const days = args.days === undefined ? undefined : clampInt(args.days, 1, 30, undefined)
|
|
37
|
+
const includeAnswer = args.include_answer === true
|
|
38
|
+
const includeRawContent = args.include_raw_content === true
|
|
39
|
+
const includeDomains = Array.isArray(args.include_domains)
|
|
40
|
+
? args.include_domains.filter((d) => typeof d === 'string' && d.length > 0)
|
|
41
|
+
: []
|
|
42
|
+
const excludeDomains = Array.isArray(args.exclude_domains)
|
|
43
|
+
? args.exclude_domains.filter((d) => typeof d === 'string' && d.length > 0)
|
|
44
|
+
: []
|
|
45
|
+
return { query, maxResults, searchDepth, topic, days, includeAnswer, includeRawContent, includeDomains, excludeDomains }
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function hostnameOf(url) {
|
|
49
|
+
try {
|
|
50
|
+
return new URL(url).hostname
|
|
51
|
+
} catch {
|
|
52
|
+
return url
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** Combine a caller signal with a hard timeout; call dispose() after the request. */
|
|
57
|
+
function combineSignals(signal, ms) {
|
|
58
|
+
const controller = new AbortController()
|
|
59
|
+
const timer = setTimeout(() => controller.abort(), ms)
|
|
60
|
+
const onAbort = () => controller.abort()
|
|
61
|
+
if (signal !== undefined && signal !== null) {
|
|
62
|
+
if (signal.aborted) controller.abort()
|
|
63
|
+
else signal.addEventListener('abort', onAbort, { once: true })
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
signal: controller.signal,
|
|
67
|
+
dispose() {
|
|
68
|
+
clearTimeout(timer)
|
|
69
|
+
if (signal !== undefined && signal !== null) signal.removeEventListener('abort', onAbort)
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function buildBody(input) {
|
|
75
|
+
const body = {
|
|
76
|
+
query: input.query,
|
|
77
|
+
max_results: input.maxResults,
|
|
78
|
+
search_depth: input.searchDepth,
|
|
79
|
+
}
|
|
80
|
+
if (input.topic === 'news') {
|
|
81
|
+
body.topic = 'news'
|
|
82
|
+
if (input.days !== undefined) body.days = input.days
|
|
83
|
+
}
|
|
84
|
+
if (input.includeAnswer) body.include_answer = true
|
|
85
|
+
if (input.includeRawContent) body.include_raw_content = true
|
|
86
|
+
if (input.includeDomains.length > 0) body.include_domains = input.includeDomains
|
|
87
|
+
if (input.excludeDomains.length > 0) body.exclude_domains = input.excludeDomains
|
|
88
|
+
return body
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** POST to `https://api.tavily.com/<operation>`; returns a normalized outcome. */
|
|
92
|
+
async function callOperation(operation, body, key, exec, timeoutMs) {
|
|
93
|
+
const combined = combineSignals(exec.signal, timeoutMs)
|
|
94
|
+
try {
|
|
95
|
+
const response = await fetch(`${TAVILY_BASE_URL}/${operation}`, {
|
|
96
|
+
method: 'POST',
|
|
97
|
+
headers: {
|
|
98
|
+
'content-type': 'application/json',
|
|
99
|
+
accept: 'application/json',
|
|
100
|
+
authorization: 'Bearer ' + key,
|
|
101
|
+
'user-agent': 'dsh-tool-tavily-search/0.2.0',
|
|
102
|
+
},
|
|
103
|
+
body: JSON.stringify(body),
|
|
104
|
+
signal: combined.signal,
|
|
105
|
+
})
|
|
106
|
+
if (response.status === 401 || response.status === 429) {
|
|
107
|
+
return { kind: 'retryable', statusCode: response.status }
|
|
108
|
+
}
|
|
109
|
+
if (!response.ok) {
|
|
110
|
+
let message = 'HTTP ' + response.status
|
|
111
|
+
try {
|
|
112
|
+
const parsed = await response.json()
|
|
113
|
+
if (parsed !== null && typeof parsed === 'object') {
|
|
114
|
+
if (typeof parsed.error === 'string') message = parsed.error
|
|
115
|
+
else if (parsed.error !== null && typeof parsed.error === 'object' && typeof parsed.error.message === 'string') message = parsed.error.message
|
|
116
|
+
}
|
|
117
|
+
} catch {
|
|
118
|
+
/* keep the status message */
|
|
119
|
+
}
|
|
120
|
+
throw new Error(`tavily_${operation}: ${message}`)
|
|
121
|
+
}
|
|
122
|
+
return { kind: 'ok', statusCode: response.status, json: await response.json() }
|
|
123
|
+
} catch (error) {
|
|
124
|
+
if (error instanceof Error && error.name === 'AbortError') {
|
|
125
|
+
throw new Error(`tavily_${operation}: request timed out or was aborted`)
|
|
126
|
+
}
|
|
127
|
+
throw error
|
|
128
|
+
} finally {
|
|
129
|
+
combined.dispose()
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function projectResults(json) {
|
|
134
|
+
const results = Array.isArray(json.results) ? json.results : []
|
|
135
|
+
const mapped = results.map((r) => {
|
|
136
|
+
const item = { url: String(r.url !== undefined && r.url !== null ? r.url : '') }
|
|
137
|
+
if (typeof r.title === 'string' && r.title.length > 0) item.title = r.title
|
|
138
|
+
if (typeof r.content === 'string' && r.content.length > 0) item.content = r.content
|
|
139
|
+
if (typeof r.score === 'number') item.score = r.score
|
|
140
|
+
if (typeof r.published_date === 'string' && r.published_date.length > 0) item.publishedAt = r.published_date
|
|
141
|
+
if (typeof r.raw_content === 'string' && r.raw_content.length > 0) {
|
|
142
|
+
item.rawContent = r.raw_content.length > 4000 ? r.raw_content.slice(0, 4000) + '…' : r.raw_content
|
|
143
|
+
}
|
|
144
|
+
return item
|
|
145
|
+
})
|
|
146
|
+
const value = { results: mapped }
|
|
147
|
+
if (typeof json.answer === 'string' && json.answer.length > 0) value.answer = json.answer
|
|
148
|
+
return value
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
function formatOutput(value) {
|
|
152
|
+
const parts = []
|
|
153
|
+
if (typeof value.answer === 'string' && value.answer.length > 0) parts.push(value.answer)
|
|
154
|
+
if (value.results.length > 0) {
|
|
155
|
+
const lines = value.results.map((r) => {
|
|
156
|
+
const label = typeof r.title === 'string' && r.title.length > 0 ? r.title : hostnameOf(r.url)
|
|
157
|
+
const meta = []
|
|
158
|
+
if (typeof r.content === 'string' && r.content.length > 0) meta.push(r.content)
|
|
159
|
+
if (typeof r.publishedAt === 'string' && r.publishedAt.length > 0) meta.push('(' + r.publishedAt + ')')
|
|
160
|
+
if (typeof r.score === 'number') meta.push('(relevance ' + r.score.toFixed(2) + ')')
|
|
161
|
+
const suffix = meta.length > 0 ? ' — ' + meta.join(' ') : ''
|
|
162
|
+
return '- [' + label + '](' + r.url + ')' + suffix
|
|
163
|
+
})
|
|
164
|
+
parts.push('Sources:\n' + lines.join('\n'))
|
|
165
|
+
} else {
|
|
166
|
+
parts.push('No results found.')
|
|
167
|
+
}
|
|
168
|
+
parts.push('Cite the relevant URLs above as markdown links in your answer.')
|
|
169
|
+
return parts.join('\n\n')
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/* ---------------------------------------------------------------------------
|
|
173
|
+
* tavily_extract / tavily_map / tavily_crawl argument normalization, body
|
|
174
|
+
* builders, and response projections. Exported (pure) for tests so they run
|
|
175
|
+
* without a live Tavily harness.
|
|
176
|
+
* ------------------------------------------------------------------------- */
|
|
177
|
+
|
|
178
|
+
function requiredUrlValue(input, name) {
|
|
179
|
+
const value = typeof input === 'string' ? input.trim() : ''
|
|
180
|
+
if (!URL.canParse(value)) throw new Error(`${name} must be an absolute URL`)
|
|
181
|
+
const protocol = new URL(value).protocol
|
|
182
|
+
if (protocol !== 'http:' && protocol !== 'https:') throw new Error(`${name} must use HTTP or HTTPS`)
|
|
183
|
+
return value
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function requiredUrl(args, name) {
|
|
187
|
+
return requiredUrlValue(args[name], name)
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function requiredUrls(args) {
|
|
191
|
+
if (!Array.isArray(args.urls) || args.urls.length === 0) throw new Error('urls must contain at least one URL')
|
|
192
|
+
return args.urls.map((value, index) => requiredUrlValue(value, 'urls[' + index + ']'))
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
function optionalString(args, name) {
|
|
196
|
+
if (args[name] === undefined) return undefined
|
|
197
|
+
if (typeof args[name] !== 'string' || args[name].trim().length === 0) throw new Error(`${name} must be a non-empty string`)
|
|
198
|
+
return args[name].trim()
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function optionalStringList(args, name) {
|
|
202
|
+
if (args[name] === undefined) return undefined
|
|
203
|
+
if (!Array.isArray(args[name])) throw new Error(`${name} must be an array of non-empty strings`)
|
|
204
|
+
return args[name].map((value, index) => {
|
|
205
|
+
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
206
|
+
throw new Error(`${name}[${index}] must be a non-empty string`)
|
|
207
|
+
}
|
|
208
|
+
return value.trim()
|
|
209
|
+
})
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function optionalPositiveInt(args, name) {
|
|
213
|
+
if (args[name] === undefined) return undefined
|
|
214
|
+
if (!Number.isInteger(args[name]) || args[name] < 1) throw new Error(`${name} must be a positive integer`)
|
|
215
|
+
return args[name]
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
function optionalBoolean(args, name) {
|
|
219
|
+
if (args[name] === undefined) return undefined
|
|
220
|
+
if (typeof args[name] !== 'boolean') throw new Error(`${name} must be a boolean`)
|
|
221
|
+
return args[name]
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function optionalEnum(args, name, allowed) {
|
|
225
|
+
if (args[name] === undefined) return undefined
|
|
226
|
+
if (typeof args[name] !== 'string' || !allowed.includes(args[name])) {
|
|
227
|
+
throw new Error(`${name} must be one of ${allowed.join(', ')}`)
|
|
228
|
+
}
|
|
229
|
+
return args[name]
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
function include(body, key, value) {
|
|
233
|
+
if (value !== undefined) body[key] = value
|
|
234
|
+
return body
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
/** Normalize `tavily_extract` arguments. */
|
|
238
|
+
export function normalizeExtractArgs(args) {
|
|
239
|
+
return {
|
|
240
|
+
urls: requiredUrls(args),
|
|
241
|
+
extractDepth: optionalEnum(args, 'extract_depth', ['basic', 'advanced']),
|
|
242
|
+
includeImages: optionalBoolean(args, 'include_images'),
|
|
243
|
+
format: optionalEnum(args, 'format', ['markdown', 'text']),
|
|
244
|
+
includeFavicon: optionalBoolean(args, 'include_favicon'),
|
|
245
|
+
query: optionalString(args, 'query'),
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/** Normalize `tavily_map` / `tavily_crawl` arguments. */
|
|
250
|
+
export function normalizeNavArgs(args, { crawl }) {
|
|
251
|
+
const out = {
|
|
252
|
+
crawl,
|
|
253
|
+
url: requiredUrl(args, 'url'),
|
|
254
|
+
instructions: optionalString(args, 'instructions'),
|
|
255
|
+
maxDepth: optionalPositiveInt(args, 'max_depth'),
|
|
256
|
+
maxBreadth: optionalPositiveInt(args, 'max_breadth'),
|
|
257
|
+
limit: optionalPositiveInt(args, 'limit'),
|
|
258
|
+
selectPaths: optionalStringList(args, 'select_paths'),
|
|
259
|
+
selectDomains: optionalStringList(args, 'select_domains'),
|
|
260
|
+
allowExternal: optionalBoolean(args, 'allow_external'),
|
|
261
|
+
}
|
|
262
|
+
if (crawl) {
|
|
263
|
+
out.extractDepth = optionalEnum(args, 'extract_depth', ['basic', 'advanced'])
|
|
264
|
+
out.format = optionalEnum(args, 'format', ['markdown', 'text'])
|
|
265
|
+
out.includeFavicon = optionalBoolean(args, 'include_favicon')
|
|
266
|
+
out.chunksPerSource = optionalPositiveInt(args, 'chunks_per_source')
|
|
267
|
+
}
|
|
268
|
+
return out
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
export function buildExtractBody(input) {
|
|
272
|
+
const body = { urls: input.urls }
|
|
273
|
+
include(body, 'extract_depth', input.extractDepth)
|
|
274
|
+
include(body, 'include_images', input.includeImages)
|
|
275
|
+
include(body, 'format', input.format)
|
|
276
|
+
include(body, 'include_favicon', input.includeFavicon)
|
|
277
|
+
include(body, 'query', input.query)
|
|
278
|
+
return body
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
export function buildNavBody(input) {
|
|
282
|
+
const body = { url: input.url }
|
|
283
|
+
include(body, 'instructions', input.instructions)
|
|
284
|
+
include(body, 'max_depth', input.maxDepth)
|
|
285
|
+
include(body, 'max_breadth', input.maxBreadth)
|
|
286
|
+
include(body, 'limit', input.limit)
|
|
287
|
+
include(body, 'select_paths', input.selectPaths)
|
|
288
|
+
include(body, 'select_domains', input.selectDomains)
|
|
289
|
+
include(body, 'allow_external', input.allowExternal)
|
|
290
|
+
if (input.crawl) {
|
|
291
|
+
include(body, 'extract_depth', input.extractDepth)
|
|
292
|
+
include(body, 'format', input.format)
|
|
293
|
+
include(body, 'include_favicon', input.includeFavicon)
|
|
294
|
+
include(body, 'chunks_per_source', input.chunksPerSource)
|
|
295
|
+
}
|
|
296
|
+
return body
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
function projectPage(result) {
|
|
300
|
+
if (result === null || typeof result !== 'object' || typeof result.url !== 'string' || result.url.length === 0) return undefined
|
|
301
|
+
return {
|
|
302
|
+
url: result.url,
|
|
303
|
+
...(typeof result.raw_content === 'string' ? { rawContent: result.raw_content } : {}),
|
|
304
|
+
...(Array.isArray(result.images) ? { images: result.images.filter((image) => typeof image === 'string') } : {}),
|
|
305
|
+
...(typeof result.favicon === 'string' ? { favicon: result.favicon } : {}),
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function projectPages(response) {
|
|
310
|
+
return Array.isArray(response?.results) ? response.results.flatMap((result) => {
|
|
311
|
+
const page = projectPage(result)
|
|
312
|
+
return page === undefined ? [] : [page]
|
|
313
|
+
}) : []
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function projectFailures(response) {
|
|
317
|
+
return Array.isArray(response?.failed_results) ? response.failed_results.flatMap((result) => {
|
|
318
|
+
if (result === null || typeof result !== 'object') return []
|
|
319
|
+
const value = {
|
|
320
|
+
...(typeof result.url === 'string' ? { url: result.url } : {}),
|
|
321
|
+
...(typeof result.error === 'string' ? { error: result.error } : {}),
|
|
322
|
+
}
|
|
323
|
+
return Object.keys(value).length === 0 ? [] : [value]
|
|
324
|
+
}) : []
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
function projectUrls(response) {
|
|
328
|
+
return Array.isArray(response?.results)
|
|
329
|
+
? response.results.flatMap((result) => typeof result === 'string' ? [result] : (typeof result?.url === 'string' ? [result.url] : []))
|
|
330
|
+
: []
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
function responseTime(response) {
|
|
334
|
+
return typeof response?.response_time === 'number' ? response.response_time : undefined
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/** Project a `tavily_extract` response. */
|
|
338
|
+
export function projectExtract(response) {
|
|
339
|
+
const out = { results: projectPages(response), failedResults: projectFailures(response) }
|
|
340
|
+
const rt = responseTime(response)
|
|
341
|
+
if (rt !== undefined) out.responseTime = rt
|
|
342
|
+
return out
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** Project a `tavily_map` response. */
|
|
346
|
+
export function projectMap(response) {
|
|
347
|
+
const out = { urls: projectUrls(response) }
|
|
348
|
+
if (typeof response?.base_url === 'string') out.baseUrl = response.base_url
|
|
349
|
+
const rt = responseTime(response)
|
|
350
|
+
if (rt !== undefined) out.responseTime = rt
|
|
351
|
+
return out
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/** Project a `tavily_crawl` response. */
|
|
355
|
+
export function projectCrawl(response) {
|
|
356
|
+
const out = { results: projectPages(response) }
|
|
357
|
+
if (typeof response?.base_url === 'string') out.baseUrl = response.base_url
|
|
358
|
+
const rt = responseTime(response)
|
|
359
|
+
if (rt !== undefined) out.responseTime = rt
|
|
360
|
+
return out
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
function renderJson(_args, value) {
|
|
364
|
+
return [{ type: 'text', text: JSON.stringify(value, null, 2) }]
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
const PAGE_SCHEMA = {
|
|
368
|
+
type: 'object',
|
|
369
|
+
additionalProperties: false,
|
|
370
|
+
properties: {
|
|
371
|
+
url: { type: 'string', required: true },
|
|
372
|
+
rawContent: { type: 'string' },
|
|
373
|
+
images: { type: 'array', items: { type: 'string' } },
|
|
374
|
+
favicon: { type: 'string' },
|
|
375
|
+
},
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
const EXTRACT_OUTPUT_SCHEMA = {
|
|
379
|
+
type: 'object',
|
|
380
|
+
additionalProperties: false,
|
|
381
|
+
properties: {
|
|
382
|
+
results: { type: 'array', required: true, items: PAGE_SCHEMA },
|
|
383
|
+
failedResults: {
|
|
384
|
+
type: 'array',
|
|
385
|
+
required: true,
|
|
386
|
+
items: { type: 'object', additionalProperties: false, properties: { url: { type: 'string' }, error: { type: 'string' } } },
|
|
387
|
+
},
|
|
388
|
+
responseTime: { type: 'number' },
|
|
389
|
+
},
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const MAP_OUTPUT_SCHEMA = {
|
|
393
|
+
type: 'object',
|
|
394
|
+
additionalProperties: false,
|
|
395
|
+
properties: {
|
|
396
|
+
baseUrl: { type: 'string' },
|
|
397
|
+
urls: { type: 'array', required: true, items: { type: 'string' } },
|
|
398
|
+
responseTime: { type: 'number' },
|
|
399
|
+
},
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
const CRAWL_OUTPUT_SCHEMA = {
|
|
403
|
+
type: 'object',
|
|
404
|
+
additionalProperties: false,
|
|
405
|
+
properties: {
|
|
406
|
+
baseUrl: { type: 'string' },
|
|
407
|
+
results: { type: 'array', required: true, items: PAGE_SCHEMA },
|
|
408
|
+
responseTime: { type: 'number' },
|
|
409
|
+
},
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const NAVIGATION_PARAMETERS = {
|
|
413
|
+
url: { type: 'string', required: true, description: 'The HTTP(S) URL to map or crawl.' },
|
|
414
|
+
instructions: { type: 'string', description: 'Natural-language guidance for which pages to include.' },
|
|
415
|
+
max_depth: { type: 'number', description: 'Maximum link depth to explore.' },
|
|
416
|
+
max_breadth: { type: 'number', description: 'Maximum links to follow from one page.' },
|
|
417
|
+
limit: { type: 'number', description: 'Maximum pages to process.' },
|
|
418
|
+
select_paths: { type: 'array', items: { type: 'string' }, description: 'URL path regular expressions to include.' },
|
|
419
|
+
select_domains: { type: 'array', items: { type: 'string' }, description: 'Domain regular expressions to include.' },
|
|
420
|
+
allow_external: { type: 'boolean', description: 'Whether to include external links.' },
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/**
|
|
424
|
+
* Register the `tavily_search`, `tavily_extract`, `tavily_map`, and
|
|
425
|
+
* `tavily_crawl` model tools on `ctx`, and return one disposer that
|
|
426
|
+
* unregisters all of them plus the two prompt sections.
|
|
427
|
+
*
|
|
428
|
+
* No web search provider is registered here: `web_search` keeps its native
|
|
429
|
+
* provider and `ctx.web` is never touched.
|
|
430
|
+
*/
|
|
431
|
+
export function installTavilyTool(ctx) {
|
|
432
|
+
let rotation = 0
|
|
433
|
+
|
|
434
|
+
// Every registration is collected into one disposer so the fiber tears the
|
|
435
|
+
// half down cleanly: on row unload, and on the Plugins page's runtime unload
|
|
436
|
+
// of the bundle (0.1.6-alpha.2 unloads without restarting anything).
|
|
437
|
+
const disposers = []
|
|
438
|
+
const tools = {
|
|
439
|
+
register: (definition) => { disposers.push(ctx.tools.register(definition)) },
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
async function resolveKeys() {
|
|
443
|
+
const credentials = ctx.get('credentials')
|
|
444
|
+
if (credentials !== undefined) {
|
|
445
|
+
try {
|
|
446
|
+
const resolved = await credentials.resolve('TAVILY_API_KEYS')
|
|
447
|
+
if (resolved !== undefined && typeof resolved.value === 'string' && resolved.value.length > 0) {
|
|
448
|
+
const list = resolved.value.split(',').map((s) => s.trim()).filter((s) => s.length > 0)
|
|
449
|
+
if (list.length > 0) return { keys: list, source: resolved.source }
|
|
450
|
+
}
|
|
451
|
+
} catch (error) {
|
|
452
|
+
ctx.logger.warn('tavily_search: credential resolution failed: %s', error instanceof Error ? error.message : String(error))
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
return { keys: [], source: 'unconfigured' }
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
/** Run `body` through the rotation/failover loop; returns the response JSON. */
|
|
459
|
+
async function requestWithRotation(operation, body, timeoutMs, exec, detail) {
|
|
460
|
+
const { keys, source } = await resolveKeys()
|
|
461
|
+
if (keys.length === 0) {
|
|
462
|
+
throw new Error(`tavily_${operation}: TAVILY_API_KEYS is not configured — add it via the Tavily Search settings card or ~/.dsh/.credentials.yaml (comma-separated)`)
|
|
463
|
+
}
|
|
464
|
+
const attempts = keys.length
|
|
465
|
+
let lastRetryable = 0
|
|
466
|
+
for (let attempt = 0; attempt < attempts; attempt++) {
|
|
467
|
+
const index = (rotation + attempt) % keys.length
|
|
468
|
+
const key = keys[index]
|
|
469
|
+
const started = Date.now()
|
|
470
|
+
const outcome = await callOperation(operation, body, key, exec, timeoutMs)
|
|
471
|
+
if (outcome.kind === 'ok') {
|
|
472
|
+
rotation = (index + 1) % keys.length
|
|
473
|
+
const extra = typeof detail === 'function' ? detail(outcome.json) : {}
|
|
474
|
+
ctx.logger.info(`tavily_${operation}: ok %j`, { ...extra, status: outcome.statusCode, ms: Date.now() - started, keys: keys.length, source })
|
|
475
|
+
return outcome.json
|
|
476
|
+
}
|
|
477
|
+
lastRetryable = outcome.statusCode
|
|
478
|
+
rotation = (index + 1) % keys.length
|
|
479
|
+
ctx.logger.info(`tavily_${operation}: key %d/%d returned HTTP %d, rotating`, index + 1, keys.length, outcome.statusCode)
|
|
480
|
+
}
|
|
481
|
+
throw new Error(`tavily_${operation}: all ${attempts} configured key(s) failed with HTTP ${lastRetryable} (invalid key or rate limit)`)
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
tools.register(defineTool({
|
|
485
|
+
name: 'tavily_search',
|
|
486
|
+
description: 'Search the web through the Tavily API. Returns ranked results with titles, URLs, snippets, relevance scores, and an optional generated answer. Supports basic/advanced depth, news topic with a freshness window, and domain allow/deny filters.',
|
|
487
|
+
parameters: {
|
|
488
|
+
query: { type: 'string', required: true, description: 'The search query.' },
|
|
489
|
+
max_results: { type: 'number', description: 'Number of results to return, 1-20 (default 5).' },
|
|
490
|
+
search_depth: { type: 'string', enum: ['basic', 'advanced'], description: 'basic is faster and cheaper; advanced returns deeper analysis (default basic).' },
|
|
491
|
+
topic: { type: 'string', enum: ['general', 'news'], description: 'Search scope: general web or news (default general).' },
|
|
492
|
+
days: { type: 'number', description: 'News freshness window in days, e.g. 7 for the last week; only applies when topic is news.' },
|
|
493
|
+
include_answer: { type: 'boolean', description: 'Ask Tavily to generate a concise answer to the query (default false).' },
|
|
494
|
+
include_raw_content: { type: 'boolean', description: 'Include raw page content for each result (default false; capped per result).' },
|
|
495
|
+
include_domains: { type: 'array', items: { type: 'string' }, description: 'Only search within these domains.' },
|
|
496
|
+
exclude_domains: { type: 'array', items: { type: 'string' }, description: 'Exclude these domains from results.' },
|
|
497
|
+
},
|
|
498
|
+
output: {
|
|
499
|
+
schema: {
|
|
500
|
+
type: 'object',
|
|
501
|
+
additionalProperties: false,
|
|
502
|
+
properties: {
|
|
503
|
+
answer: { type: 'string' },
|
|
504
|
+
results: {
|
|
505
|
+
type: 'array',
|
|
506
|
+
required: true,
|
|
507
|
+
items: {
|
|
508
|
+
type: 'object',
|
|
509
|
+
additionalProperties: false,
|
|
510
|
+
properties: {
|
|
511
|
+
url: { type: 'string', required: true },
|
|
512
|
+
title: { type: 'string' },
|
|
513
|
+
content: { type: 'string' },
|
|
514
|
+
score: { type: 'number' },
|
|
515
|
+
publishedAt: { type: 'string' },
|
|
516
|
+
rawContent: { type: 'string' },
|
|
517
|
+
},
|
|
518
|
+
},
|
|
519
|
+
},
|
|
520
|
+
},
|
|
521
|
+
},
|
|
522
|
+
render: (_args, value) => [{ type: 'text', text: formatOutput(value) }],
|
|
523
|
+
},
|
|
524
|
+
timeoutMs: 30000,
|
|
525
|
+
isConcurrencySafe: () => true,
|
|
526
|
+
async execute(args, exec) {
|
|
527
|
+
const input = normalizeArgs(args)
|
|
528
|
+
const json = await requestWithRotation('search', buildBody(input), 30000, exec, (j) => ({
|
|
529
|
+
query: input.query,
|
|
530
|
+
results: Array.isArray(j.results) ? j.results.length : 0,
|
|
531
|
+
}))
|
|
532
|
+
return projectResults(json)
|
|
533
|
+
},
|
|
534
|
+
}))
|
|
535
|
+
|
|
536
|
+
tools.register(defineTool({
|
|
537
|
+
name: 'tavily_extract',
|
|
538
|
+
description: 'Extract complete content from one or more HTTP(S) URLs through Tavily. Use it to retrieve pages directly when a URL is known.',
|
|
539
|
+
parameters: {
|
|
540
|
+
urls: { type: 'array', required: true, items: { type: 'string' }, description: 'HTTP(S) URLs to extract.' },
|
|
541
|
+
extract_depth: { type: 'string', enum: ['basic', 'advanced'], description: 'Extraction depth. Advanced retrieves more page structure.' },
|
|
542
|
+
include_images: { type: 'boolean', description: 'Include extracted image URLs.' },
|
|
543
|
+
format: { type: 'string', enum: ['markdown', 'text'], description: 'Content format.' },
|
|
544
|
+
include_favicon: { type: 'boolean', description: 'Include favicon URLs when Tavily provides them.' },
|
|
545
|
+
query: { type: 'string', description: 'Optional intent used to rank extracted chunks by relevance.' },
|
|
546
|
+
},
|
|
547
|
+
output: { schema: EXTRACT_OUTPUT_SCHEMA, render: renderJson },
|
|
548
|
+
timeoutMs: 60000,
|
|
549
|
+
isConcurrencySafe: () => true,
|
|
550
|
+
async execute(args, exec) {
|
|
551
|
+
const json = await requestWithRotation('extract', buildExtractBody(normalizeExtractArgs(args)), 60000, exec)
|
|
552
|
+
return projectExtract(json)
|
|
553
|
+
},
|
|
554
|
+
}))
|
|
555
|
+
|
|
556
|
+
tools.register(defineTool({
|
|
557
|
+
name: 'tavily_map',
|
|
558
|
+
description: 'Discover URLs in a website through Tavily without extracting their page content. Use it to inspect site structure before selecting pages to extract or crawl.',
|
|
559
|
+
parameters: NAVIGATION_PARAMETERS,
|
|
560
|
+
output: { schema: MAP_OUTPUT_SCHEMA, render: renderJson },
|
|
561
|
+
timeoutMs: 60000,
|
|
562
|
+
isConcurrencySafe: () => true,
|
|
563
|
+
async execute(args, exec) {
|
|
564
|
+
const json = await requestWithRotation('map', buildNavBody(normalizeNavArgs(args, { crawl: false })), 60000, exec)
|
|
565
|
+
return projectMap(json)
|
|
566
|
+
},
|
|
567
|
+
}))
|
|
568
|
+
|
|
569
|
+
tools.register(defineTool({
|
|
570
|
+
name: 'tavily_crawl',
|
|
571
|
+
description: 'Crawl a website through Tavily and return the complete extracted content of discovered pages.',
|
|
572
|
+
parameters: {
|
|
573
|
+
...NAVIGATION_PARAMETERS,
|
|
574
|
+
extract_depth: { type: 'string', enum: ['basic', 'advanced'], description: 'Extraction depth for each crawled page.' },
|
|
575
|
+
format: { type: 'string', enum: ['markdown', 'text'], description: 'Content format for each crawled page.' },
|
|
576
|
+
include_favicon: { type: 'boolean', description: 'Include favicon URLs when Tavily provides them.' },
|
|
577
|
+
chunks_per_source: { type: 'number', description: 'Maximum extracted chunks returned per crawled source.' },
|
|
578
|
+
},
|
|
579
|
+
output: { schema: CRAWL_OUTPUT_SCHEMA, render: renderJson },
|
|
580
|
+
timeoutMs: 120000,
|
|
581
|
+
isConcurrencySafe: () => true,
|
|
582
|
+
async execute(args, exec) {
|
|
583
|
+
const json = await requestWithRotation('crawl', buildNavBody(normalizeNavArgs(args, { crawl: true })), 120000, exec)
|
|
584
|
+
return projectCrawl(json)
|
|
585
|
+
},
|
|
586
|
+
}))
|
|
587
|
+
|
|
588
|
+
const systemPrompt = ctx.get('systemPrompt')
|
|
589
|
+
const section = (value) => {
|
|
590
|
+
if (systemPrompt !== undefined) disposers.push(systemPrompt.section(value))
|
|
591
|
+
}
|
|
592
|
+
if (systemPrompt !== undefined) {
|
|
593
|
+
section({
|
|
594
|
+
name: 'tool:tavily_search',
|
|
595
|
+
order: 111,
|
|
596
|
+
text: 'Use the tavily_search tool for web search powered by the Tavily API. It supports result count, search depth, news topic with a freshness window, domain allow/deny filters, and an optional generated answer (include_answer). Cite the returned URLs as markdown links in your answer.',
|
|
597
|
+
})
|
|
598
|
+
section({
|
|
599
|
+
name: 'tool:tavily_direct',
|
|
600
|
+
order: 112,
|
|
601
|
+
text: 'For pages already known by URL, use tavily_extract to pull their full content, tavily_map to discover a site\'s links without fetching content, and tavily_crawl to capture the extracted content of an entire site. These are optional extras on top of the built-in web_search tool, which is never replaced.',
|
|
602
|
+
})
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
const credentials = ctx.get('credentials')
|
|
606
|
+
if (credentials !== undefined) {
|
|
607
|
+
credentials.describe('TAVILY_API_KEYS').then((info) => {
|
|
608
|
+
if (!info.configured) {
|
|
609
|
+
ctx.logger.warn('tavily_search: TAVILY_API_KEYS credential is not configured; add it via the Tavily Search settings card or ~/.dsh/.credentials.yaml')
|
|
610
|
+
}
|
|
611
|
+
}).catch((error) => {
|
|
612
|
+
ctx.logger.warn('tavily_search: credential describe failed: %s', error instanceof Error ? error.message : String(error))
|
|
613
|
+
})
|
|
614
|
+
}
|
|
615
|
+
|
|
616
|
+
/**
|
|
617
|
+
* Unregister everything this install registered. Idempotent: every disposer
|
|
618
|
+
* is popped as it runs, so a second call is a no-op.
|
|
619
|
+
*/
|
|
620
|
+
return function disposeTavilyTool() {
|
|
621
|
+
while (disposers.length > 0) {
|
|
622
|
+
const dispose = disposers.pop()
|
|
623
|
+
try {
|
|
624
|
+
dispose()
|
|
625
|
+
} catch (error) {
|
|
626
|
+
ctx.logger.warn('tavily_search: tool teardown failed: %s', error instanceof Error ? error.message : String(error))
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
|
|
632
|
+
|