@magic6/dsh-web-fallback-chain 0.0.0-stage → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +49 -0
- package/LICENSE +21 -0
- package/README.md +245 -2
- package/README.zh-CN.md +213 -0
- package/cordis.patch.yml +24 -0
- package/icon.svg +15 -0
- package/locale/en.json +6 -0
- package/locale/zh.json +6 -0
- package/package.json +81 -4
- package/src/chains/fetch-plugin.js +213 -0
- package/src/chains/fetch.js +224 -0
- package/src/chains/search-plugin.js +108 -0
- package/src/chains/search.js +159 -0
- package/src/client.js +1655 -0
- package/src/config.js +245 -0
- package/src/index.js +137 -0
- package/src/policy/index.js +37 -0
- package/src/policy/modes.js +179 -0
- package/src/policy/providers.js +191 -0
- package/src/prompt.js +104 -0
- package/src/providers/http.js +213 -0
- package/src/providers/index.js +65 -0
- package/src/providers/vendors.js +309 -0
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Provider metadata: the one place a leg's cost, quality and shape are declared.
|
|
3
|
+
*
|
|
4
|
+
* ## Why metadata lives here and not in the provider
|
|
5
|
+
*
|
|
6
|
+
* The web seam exposes a provider's `id` and nothing else — no price, no quality
|
|
7
|
+
* score, no output shape. A chain that orders legs by anything richer than "the
|
|
8
|
+
* order you typed" therefore needs that knowledge to come from somewhere. It
|
|
9
|
+
* lives in this registry so the assumption is visible, diffable and overridable
|
|
10
|
+
* instead of being buried in a sort function.
|
|
11
|
+
*
|
|
12
|
+
* ## User-extensible by design
|
|
13
|
+
*
|
|
14
|
+
* `defaults` covers the legs this deployment measured. A user adding a leg we
|
|
15
|
+
* never tested (a paid vendor, a self-hosted SearXNG, a corporate gateway) must
|
|
16
|
+
* not have to fork the plugin: they add one entry to `providers` in the profile
|
|
17
|
+
* patch and it participates in ordering, free-mode filtering and budget caps
|
|
18
|
+
* exactly like a built-in one. An entry that redeclares a built-in id overrides
|
|
19
|
+
* it field-by-field rather than replacing it.
|
|
20
|
+
*
|
|
21
|
+
* ## Fields
|
|
22
|
+
*
|
|
23
|
+
* - `cost` — `'free'` or `'paid'`. The only field free mode reads. `'paid'` does
|
|
24
|
+
* NOT mean "requires a key": AnySearch needs a key and is still free.
|
|
25
|
+
* - `quality` — 0–10, higher is better. Ordering only; never a promise. The
|
|
26
|
+
* defaults come from this repository's benchmark (see `research/`), rounded,
|
|
27
|
+
* and are meant to be replaced when a deployment measures its own.
|
|
28
|
+
* - `maxChars` — the per-result budget this leg should be truncated to. Search
|
|
29
|
+
* legs and list-shaped pages want a small number; a page fetched on request
|
|
30
|
+
* wants the whole thing. `null` means "no cap of our own".
|
|
31
|
+
* - `requiresKey` — env var name that must be present for the leg to be usable.
|
|
32
|
+
* Advisory: providers already report `available()`; this is for preflight
|
|
33
|
+
* messages and documentation.
|
|
34
|
+
* - `kinds` — which capabilities this leg can serve (`'search'`, `'fetch'`).
|
|
35
|
+
* A leg may serve both (AnySearch does) and then appears in both chains.
|
|
36
|
+
* - `note` — one line for humans, shown by the preflight command.
|
|
37
|
+
*
|
|
38
|
+
* @module dsh-web-chain-core/providers
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
/** Capability names the chain understands. */
|
|
42
|
+
export const KINDS = ['search', 'fetch']
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Built-in metadata, ordered by measured quality per thousand tokens.
|
|
46
|
+
*
|
|
47
|
+
* Sources (this repository, `research/web-provider-bench-2026-11-02/`):
|
|
48
|
+
* - search: blind relevance@5 40/50 paid vs 28–29/50 free; strict answer-hit at
|
|
49
|
+
* 300 chars/result 6/10 paid vs 5/10 free.
|
|
50
|
+
* - fetch: blind extraction 15 pages; AnySearch won 9/15, DeepSeek 4/8 (only 8
|
|
51
|
+
* of 15 pages returned anything), local 2/14.
|
|
52
|
+
* - `maxChars.searched` = 300 is the measured knee: 300 → 1200 chars per result
|
|
53
|
+
* is 3.8× the tokens for no change in strict answer-hit.
|
|
54
|
+
*/
|
|
55
|
+
export const DEFAULTS = {
|
|
56
|
+
'deepseek-search-rich': {
|
|
57
|
+
cost: 'paid', quality: 9.6, requiresKey: 'DEEPSEEK_API_KEY', kinds: ['search'],
|
|
58
|
+
maxChars: { brief: 300, deep: 1200, list: 300, page: null },
|
|
59
|
+
note: 'DeepSeek /v1/tools/search via dsh-web-providers: maps summary (600-800c), not just snippet, and keeps site_name',
|
|
60
|
+
},
|
|
61
|
+
'deepseek-search-api': {
|
|
62
|
+
cost: 'paid', quality: 9.0, requiresKey: 'DEEPSEEK_API_KEY', kinds: ['search'],
|
|
63
|
+
maxChars: { brief: 300, deep: 1200, list: 300, page: null },
|
|
64
|
+
note: 'DeepSeek /v1/tools/search via the official plugin: maps snippet only (100-200c) — measured to drop ~86% of the text the API returns',
|
|
65
|
+
},
|
|
66
|
+
'deepseek-fetch-rich': {
|
|
67
|
+
cost: 'paid', quality: 5.4, requiresKey: 'DEEPSEEK_API_KEY', kinds: ['fetch'],
|
|
68
|
+
maxChars: { brief: 300, deep: 1200, list: 300, page: null },
|
|
69
|
+
note: 'DeepSeek /v1/tools/fetch via dsh-web-providers',
|
|
70
|
+
},
|
|
71
|
+
'deepseek-fetch-api': {
|
|
72
|
+
cost: 'paid', quality: 5.4, requiresKey: 'DEEPSEEK_API_KEY', kinds: ['fetch'],
|
|
73
|
+
maxChars: { brief: 300, deep: 1200, list: 300, page: null },
|
|
74
|
+
note: 'DeepSeek /v1/tools/fetch via the official plugin — returned nothing on 7/15 pages and degrades silently (HTTP 200 + footer)',
|
|
75
|
+
},
|
|
76
|
+
'anysearch': {
|
|
77
|
+
cost: 'free', quality: 7.3, requiresKey: 'ANYSEARCH_API_KEY', kinds: ['search', 'fetch'],
|
|
78
|
+
maxChars: { brief: 300, deep: 1200, list: 300, page: null },
|
|
79
|
+
note: 'AnySearch /v1/search + /v1/extract — only leg that won on bot-walled sites (oschina, Reddit); 1000 calls/day, max 10 results',
|
|
80
|
+
},
|
|
81
|
+
'tavily': {
|
|
82
|
+
cost: 'free', quality: 7.3, requiresKey: 'TAVILY_API_KEY', kinds: ['search', 'fetch'],
|
|
83
|
+
maxChars: { brief: 300, deep: 1200, list: 300, page: null },
|
|
84
|
+
note: 'Tavily — good content length, only free leg that extracts PDFs; ~1500 calls/month so keep it off the hot path',
|
|
85
|
+
},
|
|
86
|
+
'http': {
|
|
87
|
+
cost: 'free', quality: 4.0, requiresKey: null, kinds: ['fetch'],
|
|
88
|
+
maxChars: { brief: 300, deep: 1200, list: 1500, page: null },
|
|
89
|
+
note: 'local HTTP fetch — best structural fidelity, but mojibake on some GBK pages and needs maxBodyChars raised for big pages',
|
|
90
|
+
},
|
|
91
|
+
'trafilatura': {
|
|
92
|
+
cost: 'free', quality: 5.6, requiresKey: null, kinds: ['fetch'],
|
|
93
|
+
maxChars: { brief: 300, deep: 1200, list: 1500, page: null },
|
|
94
|
+
note: 'NOT IMPLEMENTED — local trafilatura extraction scored highest on the pages it appeared in the benchmark, but it needs a python sidecar and this deployment cannot install one (PEP 668). Kept in the metadata table so a future leg can claim the id; deliberately absent from the shipped orders so no fetch reports it as an unregistered attempt.',
|
|
95
|
+
},
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Fields a metadata entry may carry, with the coercion each one gets. */
|
|
99
|
+
const SCHEMA = {
|
|
100
|
+
cost: (v) => {
|
|
101
|
+
if (v !== 'free' && v !== 'paid') throw new Error(`provider metadata: cost must be "free" or "paid", got ${JSON.stringify(v)}`)
|
|
102
|
+
return v
|
|
103
|
+
},
|
|
104
|
+
quality: (v) => {
|
|
105
|
+
const n = Number(v)
|
|
106
|
+
if (!Number.isFinite(n) || n < 0 || n > 10) throw new Error(`provider metadata: quality must be 0–10, got ${JSON.stringify(v)}`)
|
|
107
|
+
return n
|
|
108
|
+
},
|
|
109
|
+
maxChars: (v) => {
|
|
110
|
+
if (v === null) return { brief: null, deep: null, list: null, page: null }
|
|
111
|
+
if (typeof v !== 'object') throw new Error(`provider metadata: maxChars must be an object or null, got ${JSON.stringify(v)}`)
|
|
112
|
+
const out = {}
|
|
113
|
+
for (const [mode, cap] of Object.entries(v)) {
|
|
114
|
+
if (cap !== null && (!Number.isInteger(cap) || cap <= 0)) {
|
|
115
|
+
throw new Error(`provider metadata: maxChars.${mode} must be a positive integer or null, got ${JSON.stringify(cap)}`)
|
|
116
|
+
}
|
|
117
|
+
out[mode] = cap
|
|
118
|
+
}
|
|
119
|
+
return out
|
|
120
|
+
},
|
|
121
|
+
requiresKey: (v) => (v === null || v === undefined ? null : String(v)),
|
|
122
|
+
kinds: (v) => {
|
|
123
|
+
const list = Array.isArray(v) ? v : [v]
|
|
124
|
+
for (const k of list) {
|
|
125
|
+
if (!KINDS.includes(k)) throw new Error(`provider metadata: unknown kind ${JSON.stringify(k)}; expected one of ${KINDS.join(', ')}`)
|
|
126
|
+
}
|
|
127
|
+
return [...new Set(list)]
|
|
128
|
+
},
|
|
129
|
+
note: (v) => String(v ?? ''),
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Merge user-supplied entries over the built-ins.
|
|
134
|
+
*
|
|
135
|
+
* Overriding is field-by-field so a user who only wants to correct a price can
|
|
136
|
+
* write `{ anysearch: { cost: 'paid' } }` without restating quality or caps.
|
|
137
|
+
*
|
|
138
|
+
* @param overrides - `{ [id]: partialMetadata }` from plugin config.
|
|
139
|
+
* @returns a new metadata map; the built-in object is never mutated.
|
|
140
|
+
* @throws when an override field is malformed, naming the id and field.
|
|
141
|
+
*/
|
|
142
|
+
export function mergeProviders(overrides = {}) {
|
|
143
|
+
const merged = {}
|
|
144
|
+
for (const [id, meta] of Object.entries(DEFAULTS)) merged[id] = { ...meta, id }
|
|
145
|
+
|
|
146
|
+
for (const [id, patch] of Object.entries(overrides ?? {})) {
|
|
147
|
+
if (!patch || typeof patch !== 'object') throw new Error(`provider metadata: entry "${id}" must be an object`)
|
|
148
|
+
const base = merged[id] ?? {
|
|
149
|
+
id, cost: 'free', quality: 5, requiresKey: null, kinds: [...KINDS], note: 'user-defined provider',
|
|
150
|
+
}
|
|
151
|
+
const next = { ...base, id }
|
|
152
|
+
for (const [field, value] of Object.entries(patch)) {
|
|
153
|
+
if (field === 'id') continue
|
|
154
|
+
const coerce = SCHEMA[field]
|
|
155
|
+
if (coerce === undefined) {
|
|
156
|
+
throw new Error(`provider metadata: ${id}.${field} is not a known field (known: ${Object.keys(SCHEMA).join(', ')})`)
|
|
157
|
+
}
|
|
158
|
+
try {
|
|
159
|
+
next[field] = coerce(value)
|
|
160
|
+
} catch (error) {
|
|
161
|
+
throw new Error(`${id}: ${error.message}`)
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
// A partial maxChars override merges onto the built-in caps.
|
|
165
|
+
if (patch.maxChars !== undefined && base.maxChars && patch.maxChars !== null) {
|
|
166
|
+
next.maxChars = { ...base.maxChars, ...SCHEMA.maxChars(patch.maxChars) }
|
|
167
|
+
}
|
|
168
|
+
merged[id] = next
|
|
169
|
+
}
|
|
170
|
+
return merged
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/** Split ids into those that bill and those that do not, preserving order. */
|
|
174
|
+
export function partitionByCost(ids, providers) {
|
|
175
|
+
const free = []
|
|
176
|
+
const paid = []
|
|
177
|
+
for (const id of ids) {
|
|
178
|
+
// An unknown id is treated as free: an unlisted leg is more likely a local
|
|
179
|
+
// or user-added free source than a silent charge.
|
|
180
|
+
const meta = providers?.[id]
|
|
181
|
+
;(meta?.cost === 'paid' ? paid : free).push(id)
|
|
182
|
+
}
|
|
183
|
+
return { free, paid }
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** True when the id's `requiresKey` env var is absent, given an env lookup. */
|
|
187
|
+
export function missingKey(id, providers, env = process.env) {
|
|
188
|
+
const key = providers?.[id]?.requiresKey
|
|
189
|
+
if (!key) return null
|
|
190
|
+
return env?.[key] ? null : key
|
|
191
|
+
}
|
package/src/prompt.js
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The one system-prompt section this plugin contributes.
|
|
3
|
+
*
|
|
4
|
+
* ## Why this exists
|
|
5
|
+
*
|
|
6
|
+
* `dsh-tool-web` already injects a paragraph per tool, but both paragraphs are
|
|
7
|
+
* about **trust and citation**: "results are external, untrusted data" and "cite
|
|
8
|
+
* the URL". Neither says *when to reach for the tool*, and nothing in the static
|
|
9
|
+
* tool description does either. The observed failure mode is not calling the tool
|
|
10
|
+
* wrongly — it is not calling it at all on questions whose answer lives in
|
|
11
|
+
* documentation or someone else's repository, because the local checkout is
|
|
12
|
+
* right there and looks like the authoritative source.
|
|
13
|
+
*
|
|
14
|
+
* That is a routing problem, so it belongs in a prompt section rather than in a
|
|
15
|
+
* tool description: the tool description is generated by `dsh-tool-web` and is
|
|
16
|
+
* not ours to change, while `ctx.systemPrompt.section()` is a supported seam that
|
|
17
|
+
* composes additively with the shipped sections.
|
|
18
|
+
*
|
|
19
|
+
* ## Why it is worded as a discriminator
|
|
20
|
+
*
|
|
21
|
+
* "Use the web more" is not actionable and does not survive contact with a
|
|
22
|
+
* tempting local source. The section instead draws one falsifiable line — 用法
|
|
23
|
+
* (how does this API/framework/library work) versus 调试 (why does my own code
|
|
24
|
+
* fail) — because that is the distinction whose absence caused the miss. A rule
|
|
25
|
+
* with a checkable criterion is worth more than an exhortation.
|
|
26
|
+
*
|
|
27
|
+
* ## Placement
|
|
28
|
+
*
|
|
29
|
+
* Sits between the fetch section and LSP: after both shipped web sections, so it
|
|
30
|
+
* reads as a qualification of them rather than a competing preamble. Registered
|
|
31
|
+
* under the plugin's own name so a composition can shadow it.
|
|
32
|
+
*
|
|
33
|
+
* @module dsh-web-fallback-chain/prompt
|
|
34
|
+
*/
|
|
35
|
+
|
|
36
|
+
/** Stable section name; a composition can shadow or replace it by this name. */
|
|
37
|
+
export const PROMPT_SECTION = 'tool:web_routing'
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Fallback order used when the system-prompt service does not publish
|
|
41
|
+
* `TOOL_WEB_FETCH`. Mirrors `SECTION_ORDERS.TOOL_WEB_FETCH` (2100) so the section
|
|
42
|
+
* lands in the same place on a deployment that names its orders differently.
|
|
43
|
+
*/
|
|
44
|
+
const FALLBACK_ORDER = 2100
|
|
45
|
+
|
|
46
|
+
/** The section text. Kept as one string so it can be asserted in tests. */
|
|
47
|
+
export const PROMPT_TEXT = [
|
|
48
|
+
'Search the web before reading local source when the question is how something is *meant* to be used.',
|
|
49
|
+
'',
|
|
50
|
+
'- **Usage questions** — "how do I write X with framework/API Y", "what is the recommended way", "is there an existing implementation of Z" — start with `web_search`, then `web_fetch` the best source. The answer lives in docs or someone else\'s repository, not in the checkout you happen to have open. Prefer official documentation, then real source on GitHub, then community write-ups.',
|
|
51
|
+
'- **Debugging questions** — "why does *my* code fail", "why is this config not taking effect" — start from the local evidence (logs, the code you just wrote, the resolved config). That is first-hand data no search can replace.',
|
|
52
|
+
'- When both apply, search first to establish what correct looks like, then read locally to find where this case departs from it.',
|
|
53
|
+
'- After two failed attempts at the same problem, stop editing and go look for a working implementation instead of producing a third variation.',
|
|
54
|
+
].join('\n')
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Register the routing section.
|
|
58
|
+
*
|
|
59
|
+
* ## Why `ctx.get()` and not `ctx.systemPrompt`
|
|
60
|
+
*
|
|
61
|
+
* Cordis's context proxy throws `cannot get property "X" without inject` for any
|
|
62
|
+
* property that is not a declared dependency — and it throws at *property read*,
|
|
63
|
+
* so a guard like `if (ctx.systemPrompt === undefined)` never gets a chance to be
|
|
64
|
+
* false; the read itself is the failure. Because `applyPromptSection` runs inside
|
|
65
|
+
* `apply()`, that throw takes the whole plugin down with it: neither chain
|
|
66
|
+
* registers, and the seam reports the far less actionable
|
|
67
|
+
* `configured web provider "fetch-chain" is not registered`.
|
|
68
|
+
*
|
|
69
|
+
* `ctx.get(name)` is the documented way to read a service **without** the inject
|
|
70
|
+
* requirement: it returns `undefined` for an absent service instead of throwing.
|
|
71
|
+
* That keeps `systemPrompt` and `tools` optional collaborators without listing
|
|
72
|
+
* them in `inject` — listing them would turn their absence into a hard dependency
|
|
73
|
+
* that blocks the providers, which is the worse trade.
|
|
74
|
+
*
|
|
75
|
+
* Late-bound on purpose: the section text is a thunk, so it is re-read whenever
|
|
76
|
+
* the prompt is assembled and the section disappears if the tool it talks about
|
|
77
|
+
* is not registered in this scope.
|
|
78
|
+
*
|
|
79
|
+
* @param ctx - context whose `systemPrompt` service receives the section.
|
|
80
|
+
*/
|
|
81
|
+
export function applyPromptSection(ctx) {
|
|
82
|
+
// `ctx.get` is a mixin method present on every context; the optional call keeps
|
|
83
|
+
// a bare test double (a plain object) usable here.
|
|
84
|
+
const softGet = (serviceName) => (typeof ctx?.get === 'function' ? ctx.get(serviceName) : undefined)
|
|
85
|
+
const prompt = softGet('systemPrompt')
|
|
86
|
+
// Optional service: a deployment may compose without a prompt builder (a bare
|
|
87
|
+
// test host, for instance). Missing here is not an error worth failing over.
|
|
88
|
+
if (prompt === undefined || prompt === null || typeof prompt.section !== 'function') return
|
|
89
|
+
const tools = softGet('tools')
|
|
90
|
+
|
|
91
|
+
const order = typeof prompt.getSectionOrder === 'function'
|
|
92
|
+
? prompt.getSectionOrder('TOOL_WEB_FETCH')
|
|
93
|
+
: undefined
|
|
94
|
+
|
|
95
|
+
prompt.section({
|
|
96
|
+
name: PROMPT_SECTION,
|
|
97
|
+
order: Number.isFinite(order) ? order + 1 : FALLBACK_ORDER,
|
|
98
|
+
text: ({ scope }) => {
|
|
99
|
+
if (tools === undefined || tools === null || typeof tools.get !== 'function') return PROMPT_TEXT
|
|
100
|
+
// Only speak when there is something to speak about.
|
|
101
|
+
return tools.get('web_search', scope) === undefined ? '' : PROMPT_TEXT
|
|
102
|
+
},
|
|
103
|
+
})
|
|
104
|
+
}
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `dsh-web-providers` — the search and fetch legs this deployment actually uses.
|
|
3
|
+
*
|
|
4
|
+
* ## Why one package instead of one per vendor
|
|
5
|
+
*
|
|
6
|
+
* Every leg here is the same shape: an HTTP call with a bearer key, a response
|
|
7
|
+
* envelope to unwrap, and a handful of failure codes worth translating into the
|
|
8
|
+
* seam's vocabulary. Splitting that into four packages would duplicate the key
|
|
9
|
+
* loading, the timeout handling and the error mapping four times, and the copies
|
|
10
|
+
* would drift. The vendor-specific part of each leg is genuinely small, so it
|
|
11
|
+
* lives in one file per vendor while the shared plumbing lives in `http.js`.
|
|
12
|
+
*
|
|
13
|
+
* ## What each leg declares
|
|
14
|
+
*
|
|
15
|
+
* - **AnySearch** — `/v1/search` and `/v1/extract`. The only leg that won on
|
|
16
|
+
* bot-walled sites (oschina, Reddit) in the benchmark behind these plugins.
|
|
17
|
+
* Also exposes 17 vertical domains with their own sub-domains and required
|
|
18
|
+
* params, which is a *different* capability from general web search: the same
|
|
19
|
+
* query `cordis registerService` returns the EU research database from general
|
|
20
|
+
* search and the actual source docs from `code.doc`. Vertical routing is opt-in
|
|
21
|
+
* per call, never guessed.
|
|
22
|
+
* - **Tavily** — `/search` and `/extract`, plus `/crawl` and `/map`, which no
|
|
23
|
+
* other leg here offers. Its free tier is monthly (~1500 calls), not daily, so
|
|
24
|
+
* it is ranked below AnySearch for the hot path.
|
|
25
|
+
* - **DeepSeek** — `/v1/tools/search` and `/v1/tools/fetch`. Best relevance and
|
|
26
|
+
* the only leg with real index dates; billed per successful request. Its fetch
|
|
27
|
+
* degrades silently (HTTP 200 + site footer), which is why the fetch chain's
|
|
28
|
+
* `minBodyChars` exists.
|
|
29
|
+
*
|
|
30
|
+
* ## Credentials
|
|
31
|
+
*
|
|
32
|
+
* Read from the environment at call time, never cached at mount, because the
|
|
33
|
+
* harness snapshots `.env` into `process.env` at boot and a key rotated in the
|
|
34
|
+
* file is invisible until restart — reporting `available() === false` on every
|
|
35
|
+
* call keeps that visible instead of failing once and looking healthy after.
|
|
36
|
+
*
|
|
37
|
+
* @module dsh-web-fallback-chain/providers/http
|
|
38
|
+
*/
|
|
39
|
+
|
|
40
|
+
import z from '@deepseek-ai/schemastery'
|
|
41
|
+
|
|
42
|
+
/** Cordis plugin name used by loader diagnostics. */
|
|
43
|
+
export const name = 'web-providers'
|
|
44
|
+
|
|
45
|
+
/** The web seam these providers register into. */
|
|
46
|
+
export const inject = ['web']
|
|
47
|
+
|
|
48
|
+
/** Provider ids this package can register. */
|
|
49
|
+
export const SEARCH_IDS = ['deepseek-search-rich', 'anysearch', 'tavily']
|
|
50
|
+
export const FETCH_IDS = ['deepseek-fetch-rich', 'anysearch', 'tavily']
|
|
51
|
+
|
|
52
|
+
export const Config = z.object({
|
|
53
|
+
/** Which legs to mount. Empty means all of them; a leg without a key stays registered but unavailable. */
|
|
54
|
+
enable: z.array(z.string()).default([]).volatile(),
|
|
55
|
+
/** Per-request timeout in milliseconds. */
|
|
56
|
+
timeoutMs: z.number().min(1000).default(30000).volatile(),
|
|
57
|
+
/** Redirect hops a leg may follow (only the legs that fetch raw HTML care). */
|
|
58
|
+
maxRedirects: z.number().min(0).max(10).default(5).volatile(),
|
|
59
|
+
/** Send `need_content: true` to DeepSeek search (bigger payload, same price). */
|
|
60
|
+
deepseekNeedContent: z.boolean().default(false).volatile(),
|
|
61
|
+
/** AnySearch vertical domain routing: off by default, because it is a different query, not a refinement. */
|
|
62
|
+
anysearchTag: z.string().default('').volatile(),
|
|
63
|
+
/** Extra AnySearch vertical params, as `key=value,key2=value2`. */
|
|
64
|
+
anysearchParams: z.string().default('').volatile(),
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
export { Config as default }
|
|
68
|
+
|
|
69
|
+
/** Options that would leak a key into a log line. */
|
|
70
|
+
const SECRET_HEADER = /authorization/i
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* The machine-routable code for an HTTP status, in the seam's vocabulary.
|
|
74
|
+
*
|
|
75
|
+
* The seam distinguishes "the provider is unusable" from "this request failed";
|
|
76
|
+
* a 401/403 is the former (a rotated or missing key) and must not be retried by
|
|
77
|
+
* a chain, while a 5xx is the latter and should fall through.
|
|
78
|
+
*
|
|
79
|
+
* @param status - HTTP status.
|
|
80
|
+
* @returns the code string a chain can branch on.
|
|
81
|
+
*/
|
|
82
|
+
export function codeForStatus(status) {
|
|
83
|
+
if (status === 401 || status === 403) return 'WEB_PROVIDER_UNAUTHORIZED'
|
|
84
|
+
if (status === 429) return 'WEB_PROVIDER_RATE_LIMITED'
|
|
85
|
+
if (status >= 500) return 'WEB_PROVIDER_UNAVAILABLE'
|
|
86
|
+
return 'WEB_PROVIDER_ERROR'
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** A WebError-shaped error: a `code` the seam and the chains understand. */
|
|
90
|
+
export function providerError(message, code) {
|
|
91
|
+
return Object.assign(new Error(message), { code })
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* One HTTP call, with the parts every leg needs.
|
|
96
|
+
*
|
|
97
|
+
* Returns the parsed JSON body; throws a coded error otherwise. Deliberately no
|
|
98
|
+
* retry: a chain already retries by moving to the next leg, and a hidden retry
|
|
99
|
+
* inside a leg would make the chain's per-leg timing meaningless.
|
|
100
|
+
*
|
|
101
|
+
* @param options - `{url, method, headers, body, timeoutMs, signal, provider}`.
|
|
102
|
+
* @returns the parsed response body.
|
|
103
|
+
*/
|
|
104
|
+
export async function callJson({ url, method = 'GET', headers = {}, body, timeoutMs = 30000, signal, provider }) {
|
|
105
|
+
const controller = new AbortController()
|
|
106
|
+
const onAbort = () => controller.abort(signal?.reason)
|
|
107
|
+
if (signal?.aborted) throw providerError(`${provider}: cancelled before the request`, 'WEB_PROVIDER_UNAVAILABLE')
|
|
108
|
+
signal?.addEventListener?.('abort', onAbort, { once: true })
|
|
109
|
+
const timer = setTimeout(() => controller.abort(new Error('timeout')), timeoutMs)
|
|
110
|
+
let response
|
|
111
|
+
try {
|
|
112
|
+
response = await fetch(url, {
|
|
113
|
+
method,
|
|
114
|
+
headers: { 'Content-Type': 'application/json', ...headers },
|
|
115
|
+
...(body === undefined ? {} : { body: JSON.stringify(body) }),
|
|
116
|
+
signal: controller.signal,
|
|
117
|
+
})
|
|
118
|
+
} catch (error) {
|
|
119
|
+
const timedOut = error?.name === 'AbortError'
|
|
120
|
+
throw providerError(
|
|
121
|
+
`${provider}: ${timedOut ? `no answer within ${timeoutMs}ms` : String(error?.message ?? error)}`,
|
|
122
|
+
timedOut ? 'WEB_FETCH_TIMEOUT' : 'WEB_PROVIDER_UNAVAILABLE',
|
|
123
|
+
)
|
|
124
|
+
} finally {
|
|
125
|
+
clearTimeout(timer)
|
|
126
|
+
signal?.removeEventListener?.('abort', onAbort)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const text = await response.text()
|
|
130
|
+
if (!response.ok) {
|
|
131
|
+
throw providerError(
|
|
132
|
+
`${provider}: HTTP ${response.status} ${text.slice(0, 200)}`,
|
|
133
|
+
codeForStatus(response.status),
|
|
134
|
+
)
|
|
135
|
+
}
|
|
136
|
+
try {
|
|
137
|
+
return JSON.parse(text)
|
|
138
|
+
} catch {
|
|
139
|
+
throw providerError(`${provider}: response was not JSON (${text.slice(0, 120)})`, 'WEB_PROVIDER_ERROR')
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/** Build an `Authorization` header for a bearer key, or nothing when absent. */
|
|
144
|
+
export function bearer(key) {
|
|
145
|
+
return key ? { Authorization: `Bearer ${key}` } : {}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** Read an env var at call time (see the module note about the boot snapshot). */
|
|
149
|
+
export function keyFrom(env, name) {
|
|
150
|
+
const value = env?.[name]
|
|
151
|
+
return typeof value === 'string' && value.length > 0 ? value : undefined
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Read a credential the way the product does: the credential service first, then
|
|
156
|
+
* the ambient environment.
|
|
157
|
+
*
|
|
158
|
+
* ## Why this exists
|
|
159
|
+
*
|
|
160
|
+
* A key typed into a settings page is written to the credential store, and the
|
|
161
|
+
* store is consulted **before** `process.env` — that precedence is the documented
|
|
162
|
+
* behaviour of `CredentialRef` ("the provider-managed store first, then ambient
|
|
163
|
+
* fallbacks such as the process environment and `.env` files"). Reading only
|
|
164
|
+
* `process.env` therefore made a key entered in the UI invisible to the providers:
|
|
165
|
+
* the page would say "set" while every call said "401".
|
|
166
|
+
*
|
|
167
|
+
* The seam contract forces two shapes:
|
|
168
|
+
*
|
|
169
|
+
* - `available()` is synchronous, so it answers from the ambient snapshot. A key
|
|
170
|
+
* that exists only in the credential store can still report unavailable until the
|
|
171
|
+
* first call — acceptable, and the alternative (making `available` async) does not
|
|
172
|
+
* typecheck against the seam.
|
|
173
|
+
* - `search()` / `fetch()` are asynchronous, so they resolve through the service and
|
|
174
|
+
* see a UI-entered key immediately.
|
|
175
|
+
*
|
|
176
|
+
* @param options - `{ credentials, env, name }`; `credentials` may be undefined on a
|
|
177
|
+
* deployment without the service, in which case this degrades to the env read.
|
|
178
|
+
* @returns an object with a synchronous `peek()` and an async `resolve()`.
|
|
179
|
+
*/
|
|
180
|
+
export function credentialSource({ credentials, env, name }) {
|
|
181
|
+
const ambient = () => keyFrom(env, name)
|
|
182
|
+
return {
|
|
183
|
+
/** Ambient value only — for the synchronous availability probe. */
|
|
184
|
+
peek: ambient,
|
|
185
|
+
/**
|
|
186
|
+
* The value to authenticate with: the store first, ambient second.
|
|
187
|
+
*
|
|
188
|
+
* A rejected lookup must not fail the call: the whole point of a fallback chain
|
|
189
|
+
* is that one leg being misconfigured produces the next leg, not an exception.
|
|
190
|
+
*/
|
|
191
|
+
async resolve() {
|
|
192
|
+
if (credentials !== undefined && typeof credentials.resolve === 'function') {
|
|
193
|
+
try {
|
|
194
|
+
// `credentialRef` is re-exported by the service module; the import is lazy so
|
|
195
|
+
// this file stays loadable in a plain node test without the product's graph.
|
|
196
|
+
const { credentialRef } = await import('@deepseek-ai/dsh-credentials')
|
|
197
|
+
const resolved = await credentials.resolve(credentialRef(name))
|
|
198
|
+
if (typeof resolved?.value === 'string' && resolved.value.length > 0) return resolved.value
|
|
199
|
+
} catch {
|
|
200
|
+
// Fall through to the ambient value.
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
return ambient()
|
|
204
|
+
},
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Collect the headers that are safe to log. */
|
|
209
|
+
export function redactHeaders(headers) {
|
|
210
|
+
const out = {}
|
|
211
|
+
for (const [k, v] of Object.entries(headers ?? {})) out[k] = SECRET_HEADER.test(k) ? '<redacted>' : v
|
|
212
|
+
return out
|
|
213
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `dsh-web-providers` — mount the search and fetch legs this deployment uses.
|
|
3
|
+
*
|
|
4
|
+
* Registers each enabled leg into `ctx.web` under the ids the chains expect
|
|
5
|
+
* (`deepseek-search`, `deepseek-fetch`, `anysearch`, `tavily`). A leg whose key
|
|
6
|
+
* is missing is still registered but reports itself unavailable, so a chain can
|
|
7
|
+
* name it and the degradation stays visible instead of looking like the leg was
|
|
8
|
+
* never configured.
|
|
9
|
+
*
|
|
10
|
+
* ## Why the ids are shared between search and fetch
|
|
11
|
+
*
|
|
12
|
+
* AnySearch and Tavily serve both capabilities under one vendor name. Registering
|
|
13
|
+
* them as the same id in both registries keeps the provider metadata (cost,
|
|
14
|
+
* quality, budget) in one place — a leg that is `free` for search is also free
|
|
15
|
+
* for fetch, and the chains read one entry.
|
|
16
|
+
*
|
|
17
|
+
* @module dsh-web-fallback-chain/providers
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import { readField } from '../config.js'
|
|
21
|
+
import { deepseekSearchProvider, deepseekFetchProvider, anysearchProvider, tavilyProvider } from './vendors.js'
|
|
22
|
+
|
|
23
|
+
export { codeForStatus, providerError, callJson, bearer, keyFrom, redactHeaders } from './http.js'
|
|
24
|
+
export { deepseekSearchProvider, deepseekFetchProvider, anysearchProvider, tavilyProvider } from './vendors.js'
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Build every enabled leg.
|
|
28
|
+
*
|
|
29
|
+
* Exported so a test (or a preflight command) can inspect the legs without
|
|
30
|
+
* mounting the plugin.
|
|
31
|
+
*
|
|
32
|
+
* @param config - plugin config, plain or volatile.
|
|
33
|
+
* @param env - environment to read keys from; defaults to `process.env`.
|
|
34
|
+
* @param credentials - the product's credential service, when the deployment has
|
|
35
|
+
* one. Passed through so a key entered in the settings page is used; without it
|
|
36
|
+
* the providers read only the ambient environment.
|
|
37
|
+
* @returns `{search: Map<id, provider>, fetch: Map<id, provider>}`.
|
|
38
|
+
*/
|
|
39
|
+
export function buildProviders(config, env = process.env, credentials = undefined) {
|
|
40
|
+
const timeoutMs = readField(config, 'timeoutMs', 30000)
|
|
41
|
+
const needContent = readField(config, 'deepseekNeedContent', false)
|
|
42
|
+
const tag = readField(config, 'anysearchTag', '')
|
|
43
|
+
const params = readField(config, 'anysearchParams', '')
|
|
44
|
+
const enabled = readField(config, 'enable', [])
|
|
45
|
+
const wanted = (id) => !Array.isArray(enabled) || enabled.length === 0 || enabled.includes(id)
|
|
46
|
+
|
|
47
|
+
const search = new Map()
|
|
48
|
+
const fetch = new Map()
|
|
49
|
+
|
|
50
|
+
if (wanted('deepseek-search-rich')) search.set('deepseek-search-rich', deepseekSearchProvider({ env, credentials, timeoutMs, needContent }))
|
|
51
|
+
if (wanted('deepseek-fetch-rich')) fetch.set('deepseek-fetch-rich', deepseekFetchProvider({ env, credentials, timeoutMs }))
|
|
52
|
+
if (wanted('anysearch')) {
|
|
53
|
+
const leg = anysearchProvider({ env, credentials, timeoutMs, tag, params })
|
|
54
|
+
search.set('anysearch', leg)
|
|
55
|
+
fetch.set('anysearch', leg)
|
|
56
|
+
}
|
|
57
|
+
if (wanted('tavily')) {
|
|
58
|
+
const leg = tavilyProvider({ env, credentials, timeoutMs })
|
|
59
|
+
search.set('tavily', leg)
|
|
60
|
+
fetch.set('tavily', leg)
|
|
61
|
+
}
|
|
62
|
+
return { search, fetch }
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
|