@nite07/search-core 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +66 -2
- package/dist/adapters.d.ts +3 -0
- package/dist/adapters.js +269 -0
- package/dist/catalog.d.ts +3 -0
- package/dist/catalog.js +30 -0
- package/dist/config.d.ts +4 -0
- package/dist/config.js +133 -0
- package/dist/engine.d.ts +27 -0
- package/dist/engine.js +232 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +17 -0
- package/dist/locale-map.d.ts +18 -0
- package/dist/locale-map.js +38 -0
- package/dist/mcp.d.ts +4 -0
- package/dist/mcp.js +57 -0
- package/dist/reddit/auth.d.ts +77 -0
- package/dist/reddit/auth.js +257 -0
- package/dist/reddit/device.d.ts +16 -0
- package/dist/reddit/device.js +201 -0
- package/dist/reddit/gql.d.ts +77 -0
- package/dist/reddit/gql.js +162 -0
- package/dist/reddit/index.d.ts +42 -0
- package/dist/reddit/index.js +248 -0
- package/dist/reddit/shape.d.ts +94 -0
- package/dist/reddit/shape.js +267 -0
- package/dist/static.d.ts +9 -0
- package/dist/static.js +8 -0
- package/dist/tencent.d.ts +3 -0
- package/dist/tencent.js +111 -0
- package/dist/transport.d.ts +20 -0
- package/dist/transport.js +97 -0
- package/dist/types.d.ts +152 -0
- package/dist/types.js +1 -0
- package/dist/ua.d.ts +6 -0
- package/dist/ua.js +34 -0
- package/dist/version.d.ts +3 -0
- package/dist/version.js +13 -0
- package/dist/x/auth.d.ts +28 -0
- package/dist/x/auth.js +68 -0
- package/dist/x/discovery.d.ts +47 -0
- package/dist/x/discovery.js +180 -0
- package/dist/x/index.d.ts +32 -0
- package/dist/x/index.js +45 -0
- package/dist/x/official.d.ts +23 -0
- package/dist/x/official.js +77 -0
- package/dist/x/parse.d.ts +49 -0
- package/dist/x/parse.js +196 -0
- package/dist/x/session.d.ts +25 -0
- package/dist/x/session.js +103 -0
- package/dist/x/thread.d.ts +30 -0
- package/dist/x/thread.js +88 -0
- package/package.json +52 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 nite
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,67 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @nite07/search-core
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
A provider-agnostic multi-backend web search engine in TypeScript. It ships the backend catalog,
|
|
4
|
+
the per-backend adapters, the HTTP transport and a single `SearchEngine`, with no dependency on any
|
|
5
|
+
host framework — the DeepSeek Harness plugin and the MCP server are both thin shells over this.
|
|
6
|
+
|
|
7
|
+
## What it guarantees
|
|
8
|
+
|
|
9
|
+
- **Explicit access mode per backend.** A backend is used in exactly the mode the caller selected:
|
|
10
|
+
anonymous, key, OAuth or a session cookie. There is no fallback between modes, keys or endpoints.
|
|
11
|
+
- **Bounded retries.** Only the configured HTTP statuses are retried, `Retry-After` is honoured, and
|
|
12
|
+
a retry never changes the provider, the mode or the credential.
|
|
13
|
+
- **Honest failures.** A partial failure returns what worked and names what failed; a total failure
|
|
14
|
+
says the search failed instead of returning an empty result. Failures are never cached.
|
|
15
|
+
- **Cost visibility.** Every access mode carries a cost tier and a dated quota note, so a caller can
|
|
16
|
+
show a user the price before enabling anything.
|
|
17
|
+
|
|
18
|
+
## Install
|
|
19
|
+
|
|
20
|
+
```sh
|
|
21
|
+
npm install @nite07/search-core
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
## Use
|
|
25
|
+
|
|
26
|
+
```ts
|
|
27
|
+
import { SearchEngine, catalog, defaultConfig } from '@nite07/search-core';
|
|
28
|
+
|
|
29
|
+
const config = defaultConfig();
|
|
30
|
+
config.backends.exa!.enabled = true; // anonymous mode works without a credential
|
|
31
|
+
config.backends.keenable!.enabled = true;
|
|
32
|
+
|
|
33
|
+
const engine = new SearchEngine(
|
|
34
|
+
config,
|
|
35
|
+
async reference => process.env[reference], // resolve credentials however your host stores them
|
|
36
|
+
);
|
|
37
|
+
|
|
38
|
+
const result = await engine.search({ query: 'cordis plugin loader', maxResults: 8 });
|
|
39
|
+
console.log(result.sources); // citeable sources
|
|
40
|
+
console.log(result.content); // failure report, when anything failed
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
`search(input, signal?, onlyBackend?, overrides?)` accepts per-call strategy overrides —
|
|
44
|
+
`mode`, `backends`, `exclude`, `backend`, `compact`, `maxResults` — without touching the saved
|
|
45
|
+
configuration.
|
|
46
|
+
|
|
47
|
+
## Backends
|
|
48
|
+
|
|
49
|
+
23 entries across four access modes: anonymous tiers, free allowances, trial credits, paid APIs, and
|
|
50
|
+
two session-cookie backends (X, Reddit) that use the account you already have. `catalog` carries the
|
|
51
|
+
mode list, cost tier, quota note and options for each one. Reddit threads are available through
|
|
52
|
+
`fetchRedditThread`, X searches through `searchX`, and X conversations through `fetchXThread`.
|
|
53
|
+
|
|
54
|
+
X is read the way the site itself reads it: the operations the web app embeds in a page (search and
|
|
55
|
+
per-post conversation) are mined from that document, then called directly with the cookies. No
|
|
56
|
+
transaction-id signing, no bundle crawling, and a stale operation id is rediscovered once.
|
|
57
|
+
|
|
58
|
+
## Development
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
npm install
|
|
62
|
+
npm run check # typecheck, tests, build
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## License
|
|
66
|
+
|
|
67
|
+
MIT.
|
package/dist/adapters.js
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
import { requestJson, requestText, SearchError } from './transport.js';
|
|
2
|
+
import { callMcp } from './mcp.js';
|
|
3
|
+
import { languageFor, regionFor } from './locale-map.js';
|
|
4
|
+
import { searchTencent } from './tencent.js';
|
|
5
|
+
import { searchReddit } from './reddit/index.js';
|
|
6
|
+
import { parseXCredentials, searchX } from './x/index.js';
|
|
7
|
+
const obj = (v) => v && typeof v === 'object' && !Array.isArray(v) ? v : {};
|
|
8
|
+
const array = (v) => Array.isArray(v) ? v : [];
|
|
9
|
+
const string = (v) => typeof v === 'string' ? v : undefined;
|
|
10
|
+
function sources(rows, keys = { title: 'title', url: 'url', snippet: 'snippet', date: 'published_at' }) {
|
|
11
|
+
if (!Array.isArray(rows))
|
|
12
|
+
throw new SearchError('protocol', 'Expected a results array from provider.');
|
|
13
|
+
return rows.flatMap(v => {
|
|
14
|
+
const r = obj(v);
|
|
15
|
+
const url = string(r[keys.url]);
|
|
16
|
+
if (!url)
|
|
17
|
+
return [];
|
|
18
|
+
try {
|
|
19
|
+
const u = new URL(url);
|
|
20
|
+
if (!['https:', 'http:'].includes(u.protocol) || u.username || u.password)
|
|
21
|
+
return [];
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return [];
|
|
25
|
+
}
|
|
26
|
+
return [{ url, title: string(r[keys.title]), snippet: string(r[keys.snippet]), publishedAt: string(r[keys.date]) }];
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
export async function runAdapter(id, i, t) {
|
|
30
|
+
const b = i.backend;
|
|
31
|
+
const o = b.options;
|
|
32
|
+
const c = i.config;
|
|
33
|
+
const headers = { 'Content-Type': 'application/json' };
|
|
34
|
+
if (id === 'tencent-wsa')
|
|
35
|
+
return searchTencent(i, t);
|
|
36
|
+
if (id === 'x') {
|
|
37
|
+
const credentials = parseXCredentials(i.key);
|
|
38
|
+
if (b.mode !== 'key' && b.mode !== 'session')
|
|
39
|
+
throw new SearchError('configuration', 'X supports only the key and session access modes.');
|
|
40
|
+
// The X adapters own their HTTP calls, so reserve the attempt budget here.
|
|
41
|
+
await t.beforeAttempt();
|
|
42
|
+
t.onAttempt();
|
|
43
|
+
const postsPerSearch = Number(o.postsPerSearch ?? 0);
|
|
44
|
+
return { sources: await searchX({ query: i.query, maxResults: i.maxResults }, { mode: b.mode === 'key' ? 'key' : 'session', credentials, signal: t.signal, timeoutMs: b.timeoutMs, postsPerSearch: postsPerSearch > 0 ? postsPerSearch : undefined, fetch: t.fetch }) };
|
|
45
|
+
}
|
|
46
|
+
if (id === 'reddit') {
|
|
47
|
+
let credentials = {};
|
|
48
|
+
if (b.mode === 'oauth') {
|
|
49
|
+
try {
|
|
50
|
+
const parsed = JSON.parse(i.key ?? '');
|
|
51
|
+
if (!parsed || typeof parsed !== 'object')
|
|
52
|
+
throw 0;
|
|
53
|
+
credentials = parsed;
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
throw new SearchError('configuration', 'Reddit OAuth credentials must be JSON with clientId and clientSecret.');
|
|
57
|
+
}
|
|
58
|
+
if (!credentials.clientId || !credentials.clientSecret)
|
|
59
|
+
throw new SearchError('configuration', 'Reddit OAuth credentials require clientId and clientSecret.');
|
|
60
|
+
}
|
|
61
|
+
// This adapter owns its HTTP calls (token flow + GraphQL), so reserve the attempt budget here.
|
|
62
|
+
await t.beforeAttempt();
|
|
63
|
+
t.onAttempt();
|
|
64
|
+
return { sources: await searchReddit({ query: i.query, maxResults: i.maxResults, sort: String(o.sort || 'relevance'), time: String(o.time || 'all'), subreddit: String(o.subreddit || '') || undefined }, { mode: b.mode === 'oauth' ? 'oauth' : 'anonymous', credentials, signal: t.signal, timeoutMs: b.timeoutMs, fetch: t.fetch }) };
|
|
65
|
+
}
|
|
66
|
+
const key = b.mode === 'key' ? i.key : undefined;
|
|
67
|
+
const json = (body, h = headers) => requestJson(b.endpoint, { method: 'POST', headers: h, body: JSON.stringify(body) }, t);
|
|
68
|
+
const get = (params, h = headers) => { const url = new URL(b.endpoint); for (const [k, v] of Object.entries(params))
|
|
69
|
+
if (v !== undefined && v !== '')
|
|
70
|
+
url.searchParams.set(k, String(v)); return requestJson(url.href, { headers: h }, t); };
|
|
71
|
+
const bearer = () => { if (key)
|
|
72
|
+
headers.Authorization = `Bearer ${key}`; };
|
|
73
|
+
if (id === 'keenable') {
|
|
74
|
+
headers['X-Keenable-Title'] = String(o.clientTitle);
|
|
75
|
+
if (key)
|
|
76
|
+
headers['X-API-Key'] = key;
|
|
77
|
+
const r = obj(await json({ query: i.query, max_results: i.maxResults, snippet_max_length: o.snippet_max_length, ...(c.includeDomains.length ? { site: c.includeDomains.join(',') } : {}) }));
|
|
78
|
+
return { sources: sources(r.results) };
|
|
79
|
+
}
|
|
80
|
+
if (id === 'anysearch') {
|
|
81
|
+
bearer();
|
|
82
|
+
const r = obj(await json({ query: i.query, max_results: Math.min(i.maxResults, 10), zone: o.zone, format: 'json', ...(languageFor('anysearch', c.language) ? { language: languageFor('anysearch', c.language) } : {}) }));
|
|
83
|
+
if (r.code !== undefined && r.code !== 0 && r.code !== 200)
|
|
84
|
+
throw new SearchError('protocol', 'AnySearch reported an application-level error; inspect the account console.');
|
|
85
|
+
return { sources: sources(obj(r.data).results) };
|
|
86
|
+
}
|
|
87
|
+
if (id === 'linkup') {
|
|
88
|
+
bearer();
|
|
89
|
+
const r = obj(await json({ q: i.query, depth: o.depth, outputType: 'searchResults', maxResults: i.maxResults }));
|
|
90
|
+
return { sources: sources(r.results, { title: 'name', url: 'url', snippet: 'content', date: 'publishedAt' }) };
|
|
91
|
+
}
|
|
92
|
+
if (id === 'tavily') {
|
|
93
|
+
bearer();
|
|
94
|
+
const r = obj(await json({ query: i.query, max_results: Math.min(i.maxResults, 20), search_depth: o.search_depth, auto_parameters: false, topic: o.topic, include_answer: o.include_answer, include_domains: c.includeDomains, exclude_domains: c.excludeDomains, ...(c.recency !== 'any' ? { time_range: c.recency } : {}) }));
|
|
95
|
+
return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'content', date: 'published_date' }), content: string(r.answer) };
|
|
96
|
+
}
|
|
97
|
+
if (id === 'brave') {
|
|
98
|
+
if (key)
|
|
99
|
+
headers['X-Subscription-Token'] = key;
|
|
100
|
+
const r = obj(await get({ q: i.query, count: Math.min(i.maxResults, 20), search_lang: languageFor('brave', c.language), country: regionFor('brave', c.region), safesearch: c.safeSearch ? 'strict' : 'off', freshness: { any: undefined, day: 'pd', week: 'pw', month: 'pm', year: 'py' }[c.recency] }));
|
|
101
|
+
return { sources: sources(obj(r.web).results, { title: 'title', url: 'url', snippet: 'description', date: 'page_age' }) };
|
|
102
|
+
}
|
|
103
|
+
if (id === 'serper') {
|
|
104
|
+
if (key)
|
|
105
|
+
headers['X-API-KEY'] = key;
|
|
106
|
+
const r = obj(await json({ q: i.query, num: i.maxResults, hl: languageFor('serper', c.language), gl: regionFor('serper', c.region), tbs: { any: undefined, day: 'qdr:d', week: 'qdr:w', month: 'qdr:m', year: 'qdr:y' }[c.recency] }));
|
|
107
|
+
return { sources: sources(r.organic, { title: 'title', url: 'link', snippet: 'snippet', date: 'date' }) };
|
|
108
|
+
}
|
|
109
|
+
if (id === 'searxng') {
|
|
110
|
+
bearer();
|
|
111
|
+
const r = obj(await get({ q: i.query, format: 'json', language: languageFor('searxng', c.language) || 'auto', time_range: c.recency === 'any' ? undefined : c.recency, safesearch: c.safeSearch ? 2 : 0, engines: o.engines, categories: o.categories }));
|
|
112
|
+
return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'content', date: 'publishedDate' }) };
|
|
113
|
+
}
|
|
114
|
+
if (id === 'exa' && b.mode === 'key') {
|
|
115
|
+
headers['x-api-key'] = key;
|
|
116
|
+
const date = c.recency === 'any' ? undefined : new Date(Date.now() - ({ day: 1, week: 7, month: 30, year: 365 }[c.recency]) * 86400000).toISOString();
|
|
117
|
+
const r = obj(await json({ query: i.query, numResults: i.maxResults, type: o.type, includeDomains: c.includeDomains, excludeDomains: c.excludeDomains, startPublishedDate: date, contents: { text: { maxCharacters: o.textMaxCharacters } } }));
|
|
118
|
+
return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'text', date: 'publishedDate' }) };
|
|
119
|
+
}
|
|
120
|
+
if ((id === 'exa' || id === 'parallel' || id === 'you') && b.mode === 'anonymous') {
|
|
121
|
+
const tool = id === 'exa' ? 'web_search_exa' : id === 'you' ? 'you-search' : 'web_search';
|
|
122
|
+
const args = id === 'exa' ? { query: i.query, objective: i.query, numResults: i.maxResults } : id === 'you' ? { query: i.query, count: i.maxResults } : { objective: i.query, search_queries: [i.query] };
|
|
123
|
+
const endpoint = new URL(b.endpoint);
|
|
124
|
+
if (id === 'you')
|
|
125
|
+
endpoint.searchParams.set('profile', 'free');
|
|
126
|
+
if (id === 'parallel')
|
|
127
|
+
endpoint.searchParams.set('tools', 'web_search');
|
|
128
|
+
const result = await callMcp(endpoint.href, tool, args, t);
|
|
129
|
+
const structured = obj(result.structuredContent);
|
|
130
|
+
if (Array.isArray(structured.results))
|
|
131
|
+
return { sources: sources(structured.results) };
|
|
132
|
+
const text = array(result.content).map(v => string(obj(v).text) || '').join('\n');
|
|
133
|
+
if (!text.trim())
|
|
134
|
+
throw new SearchError('protocol', 'MCP search response has no recognized result payload.');
|
|
135
|
+
try {
|
|
136
|
+
const parsed = obj(JSON.parse(text));
|
|
137
|
+
const list = [parsed.results, obj(parsed.data).results, obj(parsed.results).web].find(Array.isArray);
|
|
138
|
+
if (Array.isArray(list))
|
|
139
|
+
return { sources: sources(list) };
|
|
140
|
+
}
|
|
141
|
+
catch (e) {
|
|
142
|
+
if (e instanceof SearchError)
|
|
143
|
+
throw e;
|
|
144
|
+
}
|
|
145
|
+
const found = [];
|
|
146
|
+
const blocks = text.split(/(?=Title:)/);
|
|
147
|
+
for (const block of blocks) {
|
|
148
|
+
const url = block.match(/URL:\s*(https?:\/\/\S+)/)?.[1];
|
|
149
|
+
if (url)
|
|
150
|
+
found.push({ url, title: block.match(/Title:\s*(.+)/)?.[1], snippet: block.split(/(?:Text|Content):\s*/).slice(1).join(' ').slice(0, 4000) });
|
|
151
|
+
}
|
|
152
|
+
if (!found.length && text.trim())
|
|
153
|
+
throw new SearchError('protocol', 'MCP result format contains no verifiable source URLs.');
|
|
154
|
+
return { sources: found };
|
|
155
|
+
}
|
|
156
|
+
if (id === 'parallel') {
|
|
157
|
+
if (key)
|
|
158
|
+
headers['x-api-key'] = key;
|
|
159
|
+
const r = obj(await json({ search_queries: [i.query], mode: o.mode, advanced_settings: { max_results: i.maxResults } }));
|
|
160
|
+
const rows = array(r.results).map(v => { const row = obj(v); return { ...row, excerpt: array(row.excerpts).filter(x => typeof x === 'string').join('\n').slice(0, Number(o.maxCharactersPerResult)) }; });
|
|
161
|
+
return { sources: sources(rows, { title: 'title', url: 'url', snippet: 'excerpt', date: 'publish_date' }) };
|
|
162
|
+
}
|
|
163
|
+
if (id === 'you') {
|
|
164
|
+
if (key)
|
|
165
|
+
headers['X-API-Key'] = key;
|
|
166
|
+
const r = obj(await json({ query: i.query, count: i.maxResults, strict_operators: true, ...(languageFor('you', c.language) ? { language: languageFor('you', c.language) } : {}), ...(regionFor('you', c.region) ? { country: regionFor('you', c.region) } : {}) }));
|
|
167
|
+
return { sources: sources(obj(r.results).web, { title: 'title', url: 'url', snippet: 'description', date: 'page_age' }) };
|
|
168
|
+
}
|
|
169
|
+
if (id === 'baidu') {
|
|
170
|
+
bearer();
|
|
171
|
+
const r = obj(await json({ messages: [{ role: 'user', content: i.query }], search_source: 'baidu_search_v2', edition: o.edition, resource_type_filter: [{ type: 'web', top_k: i.maxResults }] }));
|
|
172
|
+
return { sources: sources(r.references, { title: 'title', url: 'url', snippet: 'snippet', date: 'date' }) };
|
|
173
|
+
}
|
|
174
|
+
if (id === 'doubao') {
|
|
175
|
+
bearer();
|
|
176
|
+
const r = obj(await json({ Query: i.query, SearchType: 'web', Count: i.maxResults, Filter: { NeedUrl: true }, QueryControl: { QueryRewrite: o.queryRewrite } }));
|
|
177
|
+
if (obj(r.ResponseMetadata).Error)
|
|
178
|
+
throw new SearchError('protocol', 'Doubao returned an application-level error.');
|
|
179
|
+
return { sources: sources(obj(r.Result).WebResults, { title: 'Title', url: 'Url', snippet: 'Snippet', date: 'PublishTime' }) };
|
|
180
|
+
}
|
|
181
|
+
if (id === 'zhihu-web' || id === 'zhihu-site') {
|
|
182
|
+
bearer();
|
|
183
|
+
headers['X-Request-Timestamp'] = String(Math.floor(Date.now() / 1000));
|
|
184
|
+
const r = obj(await get({ Query: i.query, Count: Math.min(i.maxResults, id === 'zhihu-web' ? 20 : 10), ...(id === 'zhihu-web' ? { SearchDB: o.SearchDB, Filter: o.Filter } : {}) }));
|
|
185
|
+
if (r.Code !== 0)
|
|
186
|
+
throw new SearchError('protocol', 'Zhihu reported an application-level failure; check Access Secret and account permissions.');
|
|
187
|
+
return { sources: sources(obj(r.Data).Items, { title: 'Title', url: 'Url', snippet: 'ContentText', date: 'PublishTime' }) };
|
|
188
|
+
}
|
|
189
|
+
if (id === 'open-websearch') {
|
|
190
|
+
const r = obj(await json({ query: i.query, limit: i.maxResults, engines: [o.engine], searchMode: o.searchMode }));
|
|
191
|
+
if (r.status !== 'ok')
|
|
192
|
+
throw new SearchError('protocol', 'Self-hosted search daemon reported failure.');
|
|
193
|
+
return { sources: sources(r.data, { title: 'title', url: 'url', snippet: 'description', date: 'date' }) };
|
|
194
|
+
}
|
|
195
|
+
if (id === 'serpex') {
|
|
196
|
+
bearer();
|
|
197
|
+
const r = obj(await json({ q: i.query }));
|
|
198
|
+
return { sources: sources(r.results), warnings: ['Serpex controls its own internal upstream routing; the plugin did not switch providers.'] };
|
|
199
|
+
}
|
|
200
|
+
if (id === 'mojeek') {
|
|
201
|
+
const r = obj(await get({ q: i.query, api_key: key, fmt: 'json' }));
|
|
202
|
+
const response = obj(r.response);
|
|
203
|
+
if (response.status !== 'OK' && response.status !== 200 && response.status !== '200')
|
|
204
|
+
throw new SearchError('protocol', 'Mojeek reported an application-level error.');
|
|
205
|
+
return { sources: sources(response.results, { title: 'title', url: 'url', snippet: 'desc', date: 'date' }) };
|
|
206
|
+
}
|
|
207
|
+
if (id === 'deepseek') {
|
|
208
|
+
bearer();
|
|
209
|
+
const r = obj(await json({ query: i.query, count: i.maxResults, need_content: o.need_content }));
|
|
210
|
+
return { sources: sources(r.results, { title: 'title', url: 'url', snippet: 'snippet', date: 'date' }) };
|
|
211
|
+
}
|
|
212
|
+
if (id === 'anthropic') {
|
|
213
|
+
if (key)
|
|
214
|
+
headers['x-api-key'] = key;
|
|
215
|
+
headers['anthropic-version'] = '2023-06-01';
|
|
216
|
+
const r = obj(await json({ model: o.model, max_tokens: o.maxTokens, tools: [{ type: 'web_search_20250305', name: 'web_search', max_uses: o.maxUses, ...(c.includeDomains.length ? { allowed_domains: c.includeDomains } : {}), ...(c.excludeDomains.length ? { blocked_domains: c.excludeDomains } : {}) }], messages: [{ role: 'user', content: i.query }] }));
|
|
217
|
+
const result = [];
|
|
218
|
+
for (const block of array(r.content)) {
|
|
219
|
+
const v = obj(block);
|
|
220
|
+
if (v.type === 'web_search_tool_result') {
|
|
221
|
+
if (obj(v.content).type === 'web_search_tool_result_error')
|
|
222
|
+
throw new SearchError('protocol', 'Anthropic search tool reported failure.');
|
|
223
|
+
for (const x of array(v.content)) {
|
|
224
|
+
const s = obj(x);
|
|
225
|
+
if (s.type === 'web_search_result')
|
|
226
|
+
result.push(...sources([s], { title: 'title', url: 'url', snippet: 'page_content', date: 'page_age' }));
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return { sources: result };
|
|
231
|
+
}
|
|
232
|
+
if (id === 'openai') {
|
|
233
|
+
bearer();
|
|
234
|
+
const r = obj(await json({ model: o.model, input: i.query, tools: [{ type: 'web_search' }], tool_choice: 'required', include: ['web_search_call.action.sources'], max_output_tokens: o.maxOutputTokens }));
|
|
235
|
+
if (r.error || r.status === 'failed' || r.status === 'incomplete')
|
|
236
|
+
throw new SearchError('protocol', 'OpenAI response did not complete.');
|
|
237
|
+
const result = [];
|
|
238
|
+
for (const output of array(r.output))
|
|
239
|
+
for (const block of array(obj(output).content))
|
|
240
|
+
for (const a of array(obj(block).annotations)) {
|
|
241
|
+
const citation = obj(a);
|
|
242
|
+
if (citation.type === 'url_citation')
|
|
243
|
+
result.push(...sources([citation]));
|
|
244
|
+
}
|
|
245
|
+
return { sources: result };
|
|
246
|
+
}
|
|
247
|
+
if (id === 'ddg') {
|
|
248
|
+
const raw = await requestText(b.endpoint, { method: 'POST', headers: { 'Content-Type': 'application/x-www-form-urlencoded' }, body: new URLSearchParams({ q: i.query }).toString() }, t);
|
|
249
|
+
if (/anomaly|challenge-form|bots use DuckDuckGo/i.test(raw))
|
|
250
|
+
throw new SearchError('rate-limit', 'DuckDuckGo returned a challenge; no bypass was attempted.');
|
|
251
|
+
const decode = (s) => s.replace(/<[^>]*>/g, ' ').replace(/&/g, '&').replace(/"/g, '"').replace(/'/g, "'").replace(/</g, '<').replace(/>/g, '>').replace(/\s+/g, ' ').trim();
|
|
252
|
+
const result = [];
|
|
253
|
+
for (const match of raw.matchAll(/<a\b[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g)) {
|
|
254
|
+
let url = decode(match[1]);
|
|
255
|
+
try {
|
|
256
|
+
const u = new URL(url, 'https://duckduckgo.com');
|
|
257
|
+
url = u.searchParams.get('uddg') || u.href;
|
|
258
|
+
}
|
|
259
|
+
catch {
|
|
260
|
+
continue;
|
|
261
|
+
}
|
|
262
|
+
result.push(...sources([{ url, title: decode(match[2]) }]));
|
|
263
|
+
}
|
|
264
|
+
if (!result.length && !/No results found|result--no-result/.test(raw))
|
|
265
|
+
throw new SearchError('protocol', 'DuckDuckGo HTML layout was not recognized.');
|
|
266
|
+
return { sources: result };
|
|
267
|
+
}
|
|
268
|
+
throw new SearchError('unverified', 'Backend adapter is not verified and cannot be called.');
|
|
269
|
+
}
|
package/dist/catalog.js
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
const text = (en, zh) => ({ en, zh });
|
|
2
|
+
const option = (key, label, zh, value, choices) => ({ key, type: choices ? 'select' : typeof value, label: text(label, zh), help: text(`Native ${key} parameter. See the backend documentation.`, `原生 ${key} 参数,详见后端文档。`), default: value, ...(choices ? { choices } : {}) });
|
|
3
|
+
const mode = (access, endpoint, tier, en, zh, pricingUrl, shape) => ({ mode: access, endpoint, tier, limits: text(en, zh), pricingUrl, ...(shape ? { credentialShape: text(shape.en, shape.zh) } : {}) });
|
|
4
|
+
const backend = (id, title, docs, modes, options = [], capabilities = [], status = 'documented', evidence = 'documented') => ({ id, title, docs, modes, options, capabilities, status, evidence, verifiedAt: '2026-11-17' });
|
|
5
|
+
export const catalog = [
|
|
6
|
+
backend('keenable', 'Keenable', 'https://docs.keenable.ai/api-reference/search.md', [mode('anonymous', 'https://api.keenable.ai/v1/search/public', 'free', '1,000 requests/hour and 10/second per IP/endpoint; public title header required.', '每 IP/端点 1000 次/小时、10 次/秒;需要公开调用标识。', 'https://docs.keenable.ai/pricing.md'), mode('key', 'https://api.keenable.ai/v1/search', 'allowance', '100,000 searches/month free; then $4/1,000. Confirm current account terms.', '每月 10 万次免费,其后 $4/千次;以账户当前条款为准。', 'https://docs.keenable.ai/pricing.md')], [option('clientTitle', 'Client title', '调用方名称', 'dsh-search'), option('snippet_max_length', 'Snippet maximum length', '摘要最大长度', 2000)], ['includeDomains'], 'documented', 'live'),
|
|
7
|
+
backend('anysearch', 'AnySearch', 'https://anysearch.com/docs/api-endpoints/v1-search.md', [mode('anonymous', 'https://api.anysearch.com/v1/search', 'free', 'Anonymous access needs no key, metered per IP with lower rate limits and a daily free reset (UTC). No account or credential is created automatically.', '匿名访问无需密钥,按 IP 计量、限流更低、每日(UTC)重置;不会自动创建账户或密钥。', 'https://anysearch.com/pricing.md'), mode('key', 'https://api.anysearch.com/v1/search', 'allowance', 'Free plan: 1,000/day, 10 QPS; Pro calls have separate pricing.', '免费档 1000 次/天、10 QPS;Pro 调用另行计费。', 'https://anysearch.com/pricing.md')], [option('zone', 'Zone', '搜索区域', 'intl', ['intl', 'cn'])], ['language'], 'documented', 'live'),
|
|
8
|
+
backend('linkup', 'Linkup', 'https://docs.linkup.so/pages/documentation/endpoints/search/reference.md', [mode('key', 'https://api.linkup.so/v1/search', 'allowance', 'Eligible professional email accounts may receive $20/month. Raw search: $0.005/call; deep $0.05/call.', '符合资格的专业邮箱账户可能享有每月 $20;普通搜索 $0.005/次,deep $0.05/次。', 'https://docs.linkup.so/pages/documentation/platform/pricing.md')], [option('depth', 'Search depth', '搜索深度', 'standard', ['flash', 'fast', 'standard', 'deep'])]),
|
|
9
|
+
backend('tavily', 'Tavily', 'https://docs.tavily.com/documentation/api-reference/endpoint/search', [mode('key', 'https://api.tavily.com/search', 'allowance', '1,000 free credits/month; basic 1 credit, advanced 2. PAYG $0.008/credit.', '每月免费 1000 credits;basic 1 credit,advanced 2;按量 $0.008/credit。', 'https://www.tavily.com/pricing')], [option('search_depth', 'Search depth', '搜索深度', 'basic', ['basic', 'advanced']), option('topic', 'Topic', '主题', 'general', ['general', 'news', 'finance']), option('include_answer', 'Include answer', '包含答案', false)], ['includeDomains', 'excludeDomains', 'recency']),
|
|
10
|
+
backend('exa', 'Exa', 'https://docs.exa.ai/reference/search', [mode('anonymous', 'https://mcp.exa.ai/mcp', 'free', 'Official free keyless MCP: no API key or sign-in, search and fetch at a free rate limit (no published number). Keyless needs a direct IPv4 connection; proxied or IPv6 paths get 403. Result quality and latency were the best of the free tiers in local testing.', '官方免费无 Key MCP:无需密钥或登录,搜索与抓取按免费限流(未公布数值)。无 Key 需要直连 IPv4,代理或 IPv6 路径会返回 403。本机实测中其质量与延迟在免费档里最好。', 'https://exa.ai/docs/get-started/exa-mcp'), mode('key', 'https://api.exa.ai/search', 'allowance', '$10 balance reset monthly; auto search $7/1,000 up to 10 results, extra results cost more.', '每月余额重置为 $10;auto 搜索 $7/千次(最多 10 结果),额外结果另计费。', 'https://exa.ai/pricing')], [option('type', 'Search type', '搜索类型', 'auto', ['auto', 'neural', 'fast']), option('textMaxCharacters', 'Text maximum characters', '正文最大字符数', 2000)], ['includeDomains', 'excludeDomains', 'recency'], 'documented', 'live'),
|
|
11
|
+
backend('parallel', 'Parallel', 'https://docs.parallel.ai/search/search-quickstart', [mode('anonymous', 'https://mcp.parallel.ai/mcp', 'free', 'Free anonymous MCP with lower rate limits; numeric quota is not published.', '免费匿名 MCP,限流更低;数值额度未公布。', 'https://parallel.ai/pricing'), mode('key', 'https://api.parallel.ai/v1/search', 'paid', 'API usage is billed under account pricing; credits are not guaranteed.', 'API 按账户价格计费,不保证赠送额度。', 'https://parallel.ai/pricing')], [option('mode', 'Search mode', '搜索模式', 'fast', ['fast', 'turbo', 'basic', 'advanced']), option('maxCharactersPerResult', 'Maximum characters per result', '每条结果最大字符数', 2000)], [], 'documented', 'live'),
|
|
12
|
+
backend('brave', 'Brave Search', 'https://api-dashboard.search.brave.com/app/documentation/web-search/get-started', [mode('key', 'https://api.search.brave.com/res/v1/web/search', 'allowance', '$5 monthly credits; $5/1,000 searches, 50 QPS. Check card/activation requirements.', '每月 $5 credits;$5/千次搜索、50 QPS;需核对绑卡与激活要求。', 'https://brave.com/search/api/')], [], ['language', 'region', 'recency', 'safeSearch']),
|
|
13
|
+
backend('searxng', 'SearXNG', 'https://docs.searxng.org/dev/search_api.html', [mode('anonymous', 'http://127.0.0.1:8385/search', 'self-hosted', 'No vendor per-call fee; infrastructure costs and instance rate limits apply. JSON output must be enabled.', '无厂商调用费,有自建运行成本及实例限流;需开启 JSON 输出。', 'https://docs.searxng.org/'), mode('key', 'http://127.0.0.1:8385/search', 'self-hosted', 'For an instance explicitly protected by a Bearer-auth gateway; not a native SearXNG API-key feature.', '仅适用于自行配置 Bearer 鉴权网关的实例;不是 SearXNG 原生 Key 功能。', 'https://docs.searxng.org/')], [option('engines', 'Engines', '引擎列表', ''), option('categories', 'Categories', '分类', 'general')], ['language', 'recency', 'safeSearch'], 'documented', 'live'),
|
|
14
|
+
backend('ddg', 'DuckDuckGo', 'https://duckduckgo.com/duckduckgo-help-pages/results/syndication/', [mode('anonymous', 'https://html.duckduckgo.com/html/', 'free', 'HTML discovery, no SLA or guaranteed quota; blocking/challenges may occur.', 'HTML 搜索,无 SLA 或保证额度;可能限流或出现验证码。', 'https://duckduckgo.com/')]),
|
|
15
|
+
backend('mojeek', 'Mojeek', 'https://www.mojeek.com/services/search/web-search-api/', [mode('key', 'https://api.mojeek.com/search', 'paid', 'API plan pricing applies. Do not assume a perpetual free quota.', '按 API 套餐计费,不承诺长期免费额度。', 'https://www.mojeek.com/services/search/web-search-api/')], [], []),
|
|
16
|
+
backend('anthropic', 'Anthropic', 'https://platform.claude.com/docs/en/agents-and-tools/tool-use/web-search-tool', [mode('key', 'https://api.anthropic.com/v1/messages', 'paid', 'Web search fees plus model input/output tokens.', '搜索工具费用加模型输入/输出 token 费用。', 'https://platform.claude.com/docs/en/about-claude/pricing')], [option('model', 'Model', '模型', 'claude-sonnet-4-6'), option('maxTokens', 'Maximum output tokens', '最大输出 tokens', 2048), option('maxUses', 'Maximum search uses', '最多搜索次数', 1)], ['includeDomains', 'excludeDomains']),
|
|
17
|
+
backend('openai', 'OpenAI', 'https://platform.openai.com/docs/guides/tools-web-search', [mode('key', 'https://api.openai.com/v1/responses', 'paid', 'Web search/tool charges plus model tokens; account terms apply.', '搜索工具与模型 token 分别计费,以账户条款为准。', 'https://openai.com/api/pricing/')], [option('model', 'Model', '模型', 'gpt-4.1'), option('maxOutputTokens', 'Maximum output tokens', '最大输出 tokens', 2048)]),
|
|
18
|
+
backend('zhihu-web', 'Zhihu global search', 'https://developer.zhihu.com/docs?key=global_search', [mode('key', 'https://developer.zhihu.com/api/v1/content/global_search', 'allowance', 'Limited-time daily free allowance shown in account console; then ¥5/1,000. Resets at 00:00 Asia/Shanghai.', '限时每日额度以控制台为准,其后 ¥5/千次;北京时间零点重置。', 'https://developer.zhihu.com/docs?key=billing_price')], [option('SearchDB', 'Search database', '搜索数据库', 'all', ['all', 'realtime', 'static']), option('Filter', 'Native filter expression', '原生筛选表达式', '')]),
|
|
19
|
+
backend('zhihu-site', 'Zhihu site search', 'https://developer.zhihu.com/docs?key=zhihu_search', [mode('key', 'https://developer.zhihu.com/api/v1/content/zhihu_search', 'allowance', 'Limited-time daily allowance shown in account console; then ¥20/1,000. Shared across account secrets.', '限时每日额度以控制台为准,其后 ¥20/千次;同账户密钥共享额度。', 'https://developer.zhihu.com/docs?key=billing_price')]),
|
|
20
|
+
backend('deepseek', 'DeepSeek', 'https://api-docs.deepseek.com/guides/search/', [mode('key', 'https://api.deepseek.com/v1/tools/search', 'paid', '$0.75/1,000 successful nonempty requests; 100 QPS/account.', '$0.75/千次成功且非空请求;每账户 100 QPS。', 'https://api-docs.deepseek.com/guides/search/')], [option('need_content', 'Retrieve content', '获取正文', false)]),
|
|
21
|
+
backend('you', 'You.com', 'https://you.com/docs/api-reference/search/v1-search.md', [mode('anonymous', 'https://api.you.com/mcp', 'free', 'Free MCP profile: 100 queries/day; shared rate limits may apply.', '免费 MCP profile:100 次/天,可能存在共享限流。', 'https://you.com/docs/agents/mcp-server.md'), mode('key', 'https://ydc-index.io/v1/search', 'paid', '$5/1,000 searches; optional full-page retrieval costs extra.', '$5/千次搜索;可选全文获取另外计费。', 'https://you.com/docs/administration/billing.md')], [], ['language', 'region', 'recency'], 'documented', 'live'),
|
|
22
|
+
backend('baidu', 'Baidu Qianfan', 'https://cloud.baidu.cn/doc/qianfan/s/nmweljpt3', [mode('key', 'https://qianfan.baidubce.com/v2/ai_search/web_search', 'allowance', '1,500/month free, default 3 QPS; domestic standard ¥0.036/request, higher editions cost more.', '每月 1500 次免费,默认 3 QPS;国内 standard ¥0.036/次,更高版本价格更高。', 'https://cloud.baidu.cn/doc/qianfan/s/nmweljpt3')], [option('edition', 'Edition', '版本', 'standard', ['standard', 'enhanced', 'flagship'])]),
|
|
23
|
+
backend('doubao', 'Volcengine Doubao', 'https://docs.volcengine.com/docs/networked-search/networked-search?lang=zh', [mode('key', 'https://open.feedcoopapi.com/search_api/web_search', 'allowance', '500/month shared free allowance, 10 QPS; usage-based web ¥0.020/request. Subscription keys have independent limits.', '每月共享 500 次免费,10 QPS;按量 web ¥0.020/次;订阅 Key 有独立限制。', 'https://docs.volcengine.com/docs/networked-search/online-search-product-billing?lang=zh')], [option('queryRewrite', 'Provider query rewrite', '供应商改写查询', false)]),
|
|
24
|
+
backend('open-websearch', 'open-websearch', 'https://github.com/Aas-ee/open-webSearch', [mode('anonymous', 'http://127.0.0.1:3210/search', 'self-hosted', 'Self-hosted daemon; no guaranteed quota, engine blocking and infrastructure costs apply. Project personal-use restrictions apply.', '自建 daemon,无保证额度;有引擎限流及运行成本,注意项目个人使用限制。', 'https://github.com/Aas-ee/open-webSearch')], [option('engine', 'Fixed upstream engine', '固定上游引擎', 'duckduckgo'), option('searchMode', 'Request strategy', '请求方式', 'request', ['request', 'playwright'])]),
|
|
25
|
+
backend('serpex', 'Serpex', 'https://serpex.dev/docs/api/search', [mode('key', 'https://api.serpex.dev/api/search', 'trial', '200 one-time credits; paid top-ups start around $0.80/1,000. Vendor routes internally; no fixed upstream engine.', '一次性 200 credits;充值约 $0.80/千次起;供应商内部路由,不能固定其上游引擎。', 'https://serpex.dev/docs/getting-started/pricing')]),
|
|
26
|
+
backend('x', 'X (Twitter)', 'https://docs.x.com/x-api/posts/search/quickstart/recent-search', [mode('session', 'https://x.com/i/api/graphql', 'account', 'Uses the browser session cookies (auth_token, ct0) of your own X account. No fee, but X changes its request signing and query ids regularly, so this mode can break until the adapter is updated.', '使用你自己 X 账号的网页 cookie(auth_token、ct0)。不产生费用,但 X 会不定期改动前端签名与 queryId,失效后需要维护者更新适配器。', 'https://github.com/bestnite/x-mcp', { en: '{"authToken":"...","ct0":"..."}, or paste auth_token=...; ct0=... straight from DevTools', zh: '{"authToken":"...","ct0":"..."},也可以直接粘贴浏览器里的 auth_token=...; ct0=...' }), mode('key', 'https://api.x.com/2/tweets/search/recent', 'paid', 'Official API with an app-only bearer token as the key. Pay per post read: about $0.005 each, so a 10-result search costs roughly $0.05. Recent search covers the last 7 days; reads are capped by the posts-per-search option.', '官方 API,密钥填 app-only bearer token。按读取帖数计费:约 $0.005/条,取 10 条约 $0.05。近期搜索只覆盖最近 7 天,读取条数受「每次搜索帖数」参数限制。', 'https://docs.x.com/x-api/getting-started/pricing', { en: '{"bearerToken":"<app-only bearer token>"}', zh: '{"bearerToken":"<app-only bearer 令牌>"}' })], [option('postsPerSearch', 'Posts read per search', '每次搜索读取帖数', 10)], ['recency']),
|
|
27
|
+
backend('reddit', 'Reddit', 'https://github.com/reddit-archive/reddit/wiki/API', [mode('anonymous', 'https://gql-fed.reddit.com/', 'free', 'No credentials. Uses the Reddit Android app endpoint, because the public REST search and comments endpoints answer 403 to anonymous tokens since 2026-10. No published quota; shared rate limits.', '无需凭证。走 Reddit 安卓客户端端点——自 2026-10 起公开 REST 的搜索与评论对匿名 token 一律返回 403。额度未公布,属共享限流。', 'https://github.com/bestnite/reddit-readonly-mcp'), mode('oauth', 'https://oauth.reddit.com/search', 'allowance', 'Official OAuth API: about 100 queries/minute per client for non-commercial use. client_credentials reads public data; add a username and password for a script grant.', '官方 OAuth API:非商业用途约每客户端 100 次/分钟。client_credentials 即可读公开数据;补用户名密码则用 script 授权。', 'https://support.reddithelp.com/hc/en-us/articles/16160319875092-Reddit-Data-API-Wiki', { en: '{"clientId":"...","clientSecret":"..."}; add "username" and "password" for a script grant', zh: '{"clientId":"...","clientSecret":"..."};需要脚本授权就再加 "username" 与 "password"' })], [option('sort', 'Sort', '排序', 'relevance', ['relevance', 'hot', 'top', 'new', 'comments']), option('time', 'Time window', '时间范围', 'all', ['all', 'hour', 'day', 'week', 'month', 'year']), option('subreddit', 'Limit to subreddit', '限定版块', '')], ['recency']),
|
|
28
|
+
backend('tencent-wsa', 'Tencent Cloud WSA', 'https://cloud.tencent.com/document/product/1806/121811', [mode('key', 'https://wsa.tencentcloudapi.com/', 'paid', '¥14/¥20/¥26 per 1,000 by tier. Store a JSON credential {secretId,secretKey}; no free allowance verified.', '分档 ¥14/¥20/¥26 每千次。凭证填写 JSON {secretId,secretKey};免费额度未核实。', 'https://cloud.tencent.com/product/wsa', { en: '{"secretId":"...","secretKey":"..."}', zh: '{"secretId":"...","secretKey":"..."}' })], [{ ...option('Mode', 'Search mode', '搜索模式', 0), min: 0, max: 2 }]),
|
|
29
|
+
];
|
|
30
|
+
export const backendById = (id) => catalog.find(item => item.id === id);
|
package/dist/config.d.ts
ADDED
package/dist/config.js
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { catalog } from './catalog.js';
|
|
2
|
+
import { LANGUAGES, REGIONS } from './locale-map.js';
|
|
3
|
+
export function defaultConfig() {
|
|
4
|
+
return {
|
|
5
|
+
mode: 'parallel', singleBackend: 'keenable', maxResults: 8, concurrency: 4, overallTimeoutMs: 30000,
|
|
6
|
+
language: '', region: '', recency: 'any', safeSearch: true, includeDomains: [], excludeDomains: [], dedup: 'url', fusion: 'rrf',
|
|
7
|
+
retry: { enabled: true, statuses: [408, 429, 500, 502, 503, 504], maxRetries: 2, initialDelayMs: 500, maxDelayMs: 5000, networkErrors: false, timeouts: false },
|
|
8
|
+
cache: { enabled: true, ttlSeconds: 900, maxEntries: 100 }, breaker: { enabled: false, threshold: 3, cooldownMs: 60000 }, diagnostics: { enabled: true, maxEntries: 50 }, tools: { webSearchPlus: true, compactByDefault: false },
|
|
9
|
+
backends: Object.fromEntries(catalog.map(d => [d.id, { enabled: false, mode: d.modes[0].mode, endpoint: d.modes[0].endpoint, credentialRef: `DSH_SEARCH_${d.id.replaceAll('-', '_').toUpperCase()}_KEY`, timeoutMs: 15000, rateLimitPerMinute: 0, dailyAttemptLimit: 0, options: Object.fromEntries(d.options.map(o => [o.key, o.default])) }]))
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
export function validateConfig(value) {
|
|
13
|
+
const issues = [];
|
|
14
|
+
const fail = (path, message) => issues.push({ path, message });
|
|
15
|
+
const object = (v) => !!v && typeof v === 'object' && !Array.isArray(v);
|
|
16
|
+
if (!object(value))
|
|
17
|
+
return [{ path: 'config', message: 'Expected a configuration object.' }];
|
|
18
|
+
function shape(v, exemplar, path) {
|
|
19
|
+
if (Array.isArray(exemplar)) {
|
|
20
|
+
if (!Array.isArray(v)) {
|
|
21
|
+
fail(path, 'Expected an array.');
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
if (v.length > 200)
|
|
25
|
+
fail(path, 'Too many entries.');
|
|
26
|
+
if (path === 'retry.statuses') {
|
|
27
|
+
if (v.some(n => !Number.isInteger(n) || n < 400 || n > 599))
|
|
28
|
+
fail(path, 'Expected HTTP error status codes (400–599).');
|
|
29
|
+
}
|
|
30
|
+
else if (v.some(x => typeof x !== 'string' || x.length > 253 || !/^([a-zA-Z0-9-]+\.)*[a-zA-Z0-9-]+$/.test(x)))
|
|
31
|
+
fail(path, 'Expected domain names without scheme, path, or wildcards.');
|
|
32
|
+
}
|
|
33
|
+
else if (object(exemplar)) {
|
|
34
|
+
if (!object(v)) {
|
|
35
|
+
fail(path, 'Expected an object.');
|
|
36
|
+
return;
|
|
37
|
+
}
|
|
38
|
+
// Retired backends may still sit in a saved row; they are pruned on load, not rejected.
|
|
39
|
+
for (const k of Object.keys(v))
|
|
40
|
+
if (!(k in exemplar) && path !== 'backends')
|
|
41
|
+
fail(path ? `${path}.${k}` : k, 'Unknown configuration field.');
|
|
42
|
+
for (const [k, e] of Object.entries(exemplar))
|
|
43
|
+
shape(v[k], e, path ? `${path}.${k}` : k);
|
|
44
|
+
}
|
|
45
|
+
else if (typeof v !== typeof exemplar || (typeof v === 'number' && !Number.isFinite(v)))
|
|
46
|
+
fail(path, `Expected ${typeof exemplar}.`);
|
|
47
|
+
else if (typeof v === 'string' && v.length > 4096)
|
|
48
|
+
fail(path, 'Value is too long.');
|
|
49
|
+
}
|
|
50
|
+
shape(value, defaultConfig(), '');
|
|
51
|
+
if (issues.length)
|
|
52
|
+
return issues;
|
|
53
|
+
const c = value;
|
|
54
|
+
const choice = (v, allowed, p) => { if (!allowed.includes(v))
|
|
55
|
+
fail(p, 'Unsupported selection.'); };
|
|
56
|
+
const num = (v, min, max, p) => { if (!Number.isInteger(v) || v < min || v > max)
|
|
57
|
+
fail(p, `Expected an integer between ${min} and ${max}.`); };
|
|
58
|
+
if (c.language && !LANGUAGES.includes(c.language))
|
|
59
|
+
fail('language', 'Unsupported language code.');
|
|
60
|
+
if (c.region && !REGIONS.includes(c.region))
|
|
61
|
+
fail('region', 'Unsupported region code.');
|
|
62
|
+
choice(c.mode, ['single', 'parallel'], 'mode');
|
|
63
|
+
choice(c.singleBackend, catalog.map(x => x.id), 'singleBackend');
|
|
64
|
+
choice(c.recency, ['any', 'day', 'week', 'month', 'year'], 'recency');
|
|
65
|
+
choice(c.dedup, ['url', 'url-title'], 'dedup');
|
|
66
|
+
choice(c.fusion, ['rrf', 'ordered'], 'fusion');
|
|
67
|
+
num(c.maxResults, 1, 50, 'maxResults');
|
|
68
|
+
num(c.concurrency, 1, 20, 'concurrency');
|
|
69
|
+
num(c.overallTimeoutMs, 1000, 180000, 'overallTimeoutMs');
|
|
70
|
+
num(c.retry.maxRetries, 0, 5, 'retry.maxRetries');
|
|
71
|
+
num(c.retry.initialDelayMs, 0, 60000, 'retry.initialDelayMs');
|
|
72
|
+
num(c.retry.maxDelayMs, 0, 120000, 'retry.maxDelayMs');
|
|
73
|
+
if (c.retry.initialDelayMs > c.retry.maxDelayMs)
|
|
74
|
+
fail('retry.maxDelayMs', 'Maximum delay must not be shorter than initial delay.');
|
|
75
|
+
num(c.cache.ttlSeconds, 1, 86400, 'cache.ttlSeconds');
|
|
76
|
+
num(c.cache.maxEntries, 1, 1000, 'cache.maxEntries');
|
|
77
|
+
num(c.breaker.threshold, 1, 100, 'breaker.threshold');
|
|
78
|
+
num(c.breaker.cooldownMs, 1000, 3600000, 'breaker.cooldownMs');
|
|
79
|
+
num(c.diagnostics.maxEntries, 1, 500, 'diagnostics.maxEntries');
|
|
80
|
+
for (const d of catalog) {
|
|
81
|
+
const b = c.backends[d.id];
|
|
82
|
+
const p = `backends.${d.id}`;
|
|
83
|
+
choice(b.mode, d.modes.map(m => m.mode), `${p}.mode`);
|
|
84
|
+
if (b.enabled && d.status === 'unverified')
|
|
85
|
+
fail(`${p}.enabled`, 'Backend protocol is not verified.');
|
|
86
|
+
if (b.endpoint) {
|
|
87
|
+
try {
|
|
88
|
+
const url = new URL(b.endpoint);
|
|
89
|
+
if (!['http:', 'https:'].includes(url.protocol) || url.username || url.password || url.hash || url.search)
|
|
90
|
+
throw 0;
|
|
91
|
+
const local = url.hostname === 'localhost' || url.hostname === '127.0.0.1' || url.hostname === '[::1]';
|
|
92
|
+
if (url.protocol === 'http:' && !local)
|
|
93
|
+
throw 0;
|
|
94
|
+
}
|
|
95
|
+
catch {
|
|
96
|
+
fail(`${p}.endpoint`, 'Use an HTTPS endpoint without credentials, query, or fragment; HTTP is allowed only on loopback.');
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
else if (b.enabled)
|
|
100
|
+
fail(`${p}.endpoint`, 'An endpoint is required.');
|
|
101
|
+
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(b.credentialRef))
|
|
102
|
+
fail(`${p}.credentialRef`, 'Expected an environment-style credential reference.');
|
|
103
|
+
num(b.timeoutMs, 1000, 120000, `${p}.timeoutMs`);
|
|
104
|
+
num(b.rateLimitPerMinute, 0, 100000, `${p}.rateLimitPerMinute`);
|
|
105
|
+
num(b.dailyAttemptLimit, 0, 1000000, `${p}.dailyAttemptLimit`);
|
|
106
|
+
for (const o of d.options) {
|
|
107
|
+
const v = b.options[o.key];
|
|
108
|
+
if (o.choices && !o.choices.includes(String(v)))
|
|
109
|
+
fail(`${p}.options.${o.key}`, 'Unsupported option.');
|
|
110
|
+
if (typeof v === 'number')
|
|
111
|
+
num(v, o.min ?? 1, o.max ?? 100000, `${p}.options.${o.key}`);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
return issues;
|
|
115
|
+
}
|
|
116
|
+
export function parseConfig(value) {
|
|
117
|
+
const defaults = defaultConfig();
|
|
118
|
+
// Migrate the backend map before validation: drop entries this release no longer ships, add ones it now ships.
|
|
119
|
+
if (value && typeof value === 'object' && !Array.isArray(value) && value.backends && typeof value.backends === 'object') {
|
|
120
|
+
const raw = structuredClone(value);
|
|
121
|
+
for (const id of Object.keys(raw.backends))
|
|
122
|
+
if (!(id in defaults.backends))
|
|
123
|
+
delete raw.backends[id];
|
|
124
|
+
for (const [id, fallback] of Object.entries(defaults.backends))
|
|
125
|
+
if (!raw.backends[id])
|
|
126
|
+
raw.backends[id] = fallback;
|
|
127
|
+
value = raw;
|
|
128
|
+
}
|
|
129
|
+
const issues = validateConfig(value);
|
|
130
|
+
if (issues.length)
|
|
131
|
+
throw new Error(issues.map(x => `${x.path}: ${x.message}`).join('\n'));
|
|
132
|
+
return structuredClone(value);
|
|
133
|
+
}
|
package/dist/engine.d.ts
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { BackendOutcome, SearchConfig, SearchInput, SearchOverrides, SearchResult } from './types.js';
|
|
2
|
+
export declare function revision(config: SearchConfig): string;
|
|
3
|
+
export declare class SearchEngine {
|
|
4
|
+
private config;
|
|
5
|
+
private resolveKey;
|
|
6
|
+
private fetcher;
|
|
7
|
+
private reserveAttempt?;
|
|
8
|
+
private cache;
|
|
9
|
+
private history;
|
|
10
|
+
private usage;
|
|
11
|
+
private failures;
|
|
12
|
+
constructor(config: SearchConfig, resolveKey: (ref: string) => Promise<string | undefined>, fetcher?: typeof fetch, reserveAttempt?: ((id: string, mode: string, perMinute: number, daily: number) => Promise<{
|
|
13
|
+
attempts: number;
|
|
14
|
+
day: string;
|
|
15
|
+
}>) | undefined);
|
|
16
|
+
clearCache(): void;
|
|
17
|
+
clearDiagnostics(): void;
|
|
18
|
+
diagnostics(): BackendOutcome[];
|
|
19
|
+
counters(): {
|
|
20
|
+
[k: string]: {
|
|
21
|
+
attempts: number;
|
|
22
|
+
day: string;
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
search(input: SearchInput, signal?: AbortSignal, onlyBackend?: string, overrides?: SearchOverrides): Promise<SearchResult>;
|
|
26
|
+
private call;
|
|
27
|
+
}
|