@fffangchenghao/dsh-web-search 0.13.11 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +67 -25
- package/README.zh-CN.md +65 -23
- package/SECURITY.md +4 -0
- package/client.js +2 -2
- package/docs/rrf-ranking.md +147 -0
- package/lib/config.js +4 -4
- package/lib/domain-tool.js +2 -2
- package/lib/engines.js +43 -1
- package/lib/index.d.ts +3 -1
- package/lib/index.js +30 -1
- package/lib/parsers.d.ts +25 -0
- package/lib/parsers.js +139 -0
- package/lib/research.d.ts +158 -0
- package/lib/research.js +393 -0
- package/lib/verticals.d.ts +240 -3
- package/lib/verticals.js +1185 -78
- package/package.json +3 -2
package/lib/research.js
ADDED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
/** The name this tool prefers. */
|
|
2
|
+
export const RESEARCH_TOOL_NAME = 'research';
|
|
3
|
+
/** The name used when another plugin already registered `research`. */
|
|
4
|
+
export const RESEARCH_TOOL_FALLBACK_NAME = 'web_research';
|
|
5
|
+
/** Pages fetched when the call names no budget. */
|
|
6
|
+
export const RESEARCH_DEFAULT_SOURCES = 3;
|
|
7
|
+
/** Smallest fetch budget a call may ask for. */
|
|
8
|
+
export const RESEARCH_MIN_SOURCES = 1;
|
|
9
|
+
/** Largest fetch budget a call may ask for. */
|
|
10
|
+
export const RESEARCH_MAX_SOURCES = 5;
|
|
11
|
+
/** Distinct queries accepted in one call. */
|
|
12
|
+
export const RESEARCH_MAX_QUERIES = 3;
|
|
13
|
+
/** Characters kept from each fetched page. */
|
|
14
|
+
export const RESEARCH_EXCERPT_CHARS = 1_200;
|
|
15
|
+
/** Pages fetched at the same time. */
|
|
16
|
+
export const RESEARCH_PAGE_CONCURRENCY = 3;
|
|
17
|
+
/** Wall-clock budget for the whole call, below the tool's host timeout. */
|
|
18
|
+
export const RESEARCH_BUDGET_MS = 45_000;
|
|
19
|
+
/** Below this much budget a page fetch is not worth starting. */
|
|
20
|
+
const RESEARCH_MIN_PAGE_MS = 2_500;
|
|
21
|
+
/**
|
|
22
|
+
* Canonical form used to decide whether two rows are the same page.
|
|
23
|
+
*
|
|
24
|
+
* Search engines disagree about `www.`, trailing slashes and fragments; two
|
|
25
|
+
* rows that differ only in those are one page, and fetching both wastes the
|
|
26
|
+
* whole budget.
|
|
27
|
+
* @param raw - candidate URL.
|
|
28
|
+
* @returns a stable key, or `undefined` for a non-http(s) URL.
|
|
29
|
+
*/
|
|
30
|
+
export function canonicalKey(raw) {
|
|
31
|
+
try {
|
|
32
|
+
const url = new URL(raw);
|
|
33
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:')
|
|
34
|
+
return undefined;
|
|
35
|
+
const host = url.hostname.toLowerCase().replace(/^www\./u, '');
|
|
36
|
+
const path = url.pathname.replace(/\/+$/u, '') || '/';
|
|
37
|
+
return `${host}${path}${url.search}`;
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return undefined;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/** Collapse page text into a readable excerpt. */
|
|
44
|
+
function excerptOf(text, max) {
|
|
45
|
+
const flattened = text
|
|
46
|
+
.replace(/\r/g, '')
|
|
47
|
+
.replace(/[ \t]+/g, ' ')
|
|
48
|
+
.replace(/\n{3,}/g, '\n\n')
|
|
49
|
+
.trim();
|
|
50
|
+
if (flattened.length <= max)
|
|
51
|
+
return flattened;
|
|
52
|
+
return `${flattened.slice(0, max - 1).trimEnd()}…`;
|
|
53
|
+
}
|
|
54
|
+
/** Walk `worker` over items with at most `limit` in flight, preserving order. */
|
|
55
|
+
async function mapLimit(items, limit, worker) {
|
|
56
|
+
const results = new Array(items.length);
|
|
57
|
+
let next = 0;
|
|
58
|
+
const runners = Array.from({ length: Math.min(Math.max(1, limit), items.length) }, async () => {
|
|
59
|
+
for (;;) {
|
|
60
|
+
const index = next;
|
|
61
|
+
next += 1;
|
|
62
|
+
if (index >= items.length)
|
|
63
|
+
return;
|
|
64
|
+
results[index] = await worker(items[index]);
|
|
65
|
+
}
|
|
66
|
+
});
|
|
67
|
+
await Promise.all(runners);
|
|
68
|
+
return results;
|
|
69
|
+
}
|
|
70
|
+
/** Interleave per-query result lists by rank, so one query cannot fill the pack. */
|
|
71
|
+
function interleave(sourcesByQuery) {
|
|
72
|
+
const merged = [];
|
|
73
|
+
const seen = new Set();
|
|
74
|
+
let depth = 0;
|
|
75
|
+
for (const list of sourcesByQuery)
|
|
76
|
+
depth = Math.max(depth, list.length);
|
|
77
|
+
for (let rank = 0; rank < depth; rank += 1) {
|
|
78
|
+
for (const list of sourcesByQuery) {
|
|
79
|
+
const source = list[rank];
|
|
80
|
+
if (source === undefined)
|
|
81
|
+
continue;
|
|
82
|
+
const key = canonicalKey(source.url);
|
|
83
|
+
if (key === undefined || seen.has(key))
|
|
84
|
+
continue;
|
|
85
|
+
seen.add(key);
|
|
86
|
+
merged.push(source);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return merged;
|
|
90
|
+
}
|
|
91
|
+
/** Read the argument object, rejecting the shapes the model gets wrong. */
|
|
92
|
+
function parseArgs(args) {
|
|
93
|
+
if (typeof args !== 'object' || args === null)
|
|
94
|
+
throw new Error('research requires an object argument');
|
|
95
|
+
const record = args;
|
|
96
|
+
const query = typeof record['query'] === 'string' ? record['query'].trim() : '';
|
|
97
|
+
const extra = record['queries'];
|
|
98
|
+
const queries = [];
|
|
99
|
+
if (query.length > 0)
|
|
100
|
+
queries.push(query);
|
|
101
|
+
if (extra !== undefined) {
|
|
102
|
+
if (!Array.isArray(extra))
|
|
103
|
+
throw new Error('research queries must be an array of strings');
|
|
104
|
+
for (const item of extra) {
|
|
105
|
+
if (typeof item !== 'string' || item.trim().length === 0) {
|
|
106
|
+
throw new Error('research queries must be non-empty strings');
|
|
107
|
+
}
|
|
108
|
+
const trimmed = item.trim();
|
|
109
|
+
if (!queries.includes(trimmed))
|
|
110
|
+
queries.push(trimmed);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
if (queries.length === 0)
|
|
114
|
+
throw new Error('research requires a non-empty query');
|
|
115
|
+
if (queries.length > RESEARCH_MAX_QUERIES) {
|
|
116
|
+
throw new Error(`research accepts at most ${RESEARCH_MAX_QUERIES} queries, got ${queries.length}`);
|
|
117
|
+
}
|
|
118
|
+
const rawSources = record['maxSources'];
|
|
119
|
+
let maxSources = RESEARCH_DEFAULT_SOURCES;
|
|
120
|
+
if (rawSources !== undefined) {
|
|
121
|
+
if (typeof rawSources !== 'number' || !Number.isInteger(rawSources)) {
|
|
122
|
+
throw new Error('research maxSources must be an integer');
|
|
123
|
+
}
|
|
124
|
+
maxSources = Math.min(RESEARCH_MAX_SOURCES, Math.max(RESEARCH_MIN_SOURCES, rawSources));
|
|
125
|
+
}
|
|
126
|
+
return { queries, maxSources };
|
|
127
|
+
}
|
|
128
|
+
/** The parameter schema, kept as plain JSON Schema like the other tools. */
|
|
129
|
+
function parameterSchema() {
|
|
130
|
+
return {
|
|
131
|
+
type: 'object',
|
|
132
|
+
additionalProperties: false,
|
|
133
|
+
properties: {
|
|
134
|
+
query: {
|
|
135
|
+
type: 'string',
|
|
136
|
+
description: 'The question to research. It is sent to the search engines verbatim.',
|
|
137
|
+
},
|
|
138
|
+
queries: {
|
|
139
|
+
type: 'array',
|
|
140
|
+
description: `Up to ${RESEARCH_MAX_QUERIES} additional distinct queries to widen the evidence; results are merged with the main query.`,
|
|
141
|
+
items: { type: 'string' },
|
|
142
|
+
},
|
|
143
|
+
maxSources: {
|
|
144
|
+
type: 'integer',
|
|
145
|
+
description: `How many pages to fetch and quote (${RESEARCH_MIN_SOURCES}–${RESEARCH_MAX_SOURCES}, default ${RESEARCH_DEFAULT_SOURCES}).`,
|
|
146
|
+
},
|
|
147
|
+
},
|
|
148
|
+
required: ['query'],
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
/** The output schema: pages carry their own citation. */
|
|
152
|
+
function outputSchema() {
|
|
153
|
+
return {
|
|
154
|
+
type: 'object',
|
|
155
|
+
additionalProperties: false,
|
|
156
|
+
properties: {
|
|
157
|
+
query: { type: 'string' },
|
|
158
|
+
queries: { type: 'array', items: { type: 'string' } },
|
|
159
|
+
pages: {
|
|
160
|
+
type: 'array',
|
|
161
|
+
items: {
|
|
162
|
+
type: 'object',
|
|
163
|
+
additionalProperties: false,
|
|
164
|
+
properties: {
|
|
165
|
+
url: { type: 'string' },
|
|
166
|
+
title: { type: 'string' },
|
|
167
|
+
status: { type: 'integer' },
|
|
168
|
+
chars: { type: 'integer' },
|
|
169
|
+
excerpt: { type: 'string' },
|
|
170
|
+
truncated: { type: 'boolean' },
|
|
171
|
+
},
|
|
172
|
+
required: ['url', 'status', 'chars', 'excerpt', 'truncated'],
|
|
173
|
+
},
|
|
174
|
+
},
|
|
175
|
+
sources: {
|
|
176
|
+
type: 'array',
|
|
177
|
+
items: {
|
|
178
|
+
type: 'object',
|
|
179
|
+
additionalProperties: false,
|
|
180
|
+
properties: { url: { type: 'string' }, title: { type: 'string' } },
|
|
181
|
+
required: ['url'],
|
|
182
|
+
},
|
|
183
|
+
},
|
|
184
|
+
failures: {
|
|
185
|
+
type: 'array',
|
|
186
|
+
items: {
|
|
187
|
+
type: 'object',
|
|
188
|
+
additionalProperties: false,
|
|
189
|
+
properties: { url: { type: 'string' }, reason: { type: 'string' } },
|
|
190
|
+
required: ['url', 'reason'],
|
|
191
|
+
},
|
|
192
|
+
},
|
|
193
|
+
found: { type: 'integer' },
|
|
194
|
+
truncated: { type: 'boolean' },
|
|
195
|
+
},
|
|
196
|
+
required: ['query', 'queries', 'pages', 'sources', 'failures', 'found', 'truncated'],
|
|
197
|
+
};
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Format the evidence pack the model reads.
|
|
201
|
+
* @param value - what the call produced.
|
|
202
|
+
* @returns a Markdown document with one section per fetched page.
|
|
203
|
+
*/
|
|
204
|
+
export function formatResearchValue(value) {
|
|
205
|
+
const head = [
|
|
206
|
+
`Research: ${JSON.stringify(value.query)}`,
|
|
207
|
+
`Searched ${value.queries.length} quer${value.queries.length === 1 ? 'y' : 'ies'}, `
|
|
208
|
+
+ `${value.found} source(s) found, ${value.pages.length} page(s) fetched.`,
|
|
209
|
+
];
|
|
210
|
+
if (value.queries.length > 1) {
|
|
211
|
+
head.push(`Queries: ${value.queries.map((query) => JSON.stringify(query)).join(', ')}`);
|
|
212
|
+
}
|
|
213
|
+
const sections = value.pages.map((page, index) => {
|
|
214
|
+
const lines = [
|
|
215
|
+
`## ${index + 1}. ${page.title !== undefined && page.title.length > 0 ? page.title : page.url}`,
|
|
216
|
+
page.url,
|
|
217
|
+
'',
|
|
218
|
+
page.excerpt,
|
|
219
|
+
];
|
|
220
|
+
if (page.truncated)
|
|
221
|
+
lines.push('', '(excerpt truncated)');
|
|
222
|
+
return lines.join('\n');
|
|
223
|
+
});
|
|
224
|
+
const notFetched = value.sources.length === 0
|
|
225
|
+
? []
|
|
226
|
+
: [
|
|
227
|
+
'## Found but not fetched',
|
|
228
|
+
...value.sources.map((source) => `- ${source.title !== undefined && source.title.length > 0 ? `${source.title} — ` : ''}${source.url}`),
|
|
229
|
+
];
|
|
230
|
+
const failures = value.failures.length === 0
|
|
231
|
+
? []
|
|
232
|
+
: ['## Fetch failures', ...value.failures.map((failure) => `- ${failure.url}: ${failure.reason}`)];
|
|
233
|
+
return [...head, '', ...sections, ...(notFetched.length === 0 ? [] : ['', ...notFetched]), ...(failures.length === 0 ? [] : ['', ...failures])]
|
|
234
|
+
.join('\n')
|
|
235
|
+
.trimEnd();
|
|
236
|
+
}
|
|
237
|
+
/**
|
|
238
|
+
* Build the tool definition the host registry accepts.
|
|
239
|
+
* @param name - `research`, or the fallback when that name is taken.
|
|
240
|
+
* @param search - runs one query through the plugin's search path.
|
|
241
|
+
* @param fetchPage - fetches one URL through the plugin's fetch ladder.
|
|
242
|
+
* @returns a registry-ready definition.
|
|
243
|
+
*/
|
|
244
|
+
export function createResearchDefinition(name, search, fetchPage) {
|
|
245
|
+
return {
|
|
246
|
+
name,
|
|
247
|
+
description: 'Research a question end to end: run up to three searches, open the leading pages, and return an evidence pack in which every excerpt sits under its URL. Use it when the answer needs citations or several sources, not when one search row is enough. Pages that fail to load are reported by URL and reason instead of aborting the call.',
|
|
248
|
+
parameters: parameterSchema(),
|
|
249
|
+
output: {
|
|
250
|
+
schema: outputSchema(),
|
|
251
|
+
render: (_args, value) => [{ type: 'text', text: formatResearchValue(value) }],
|
|
252
|
+
},
|
|
253
|
+
timeoutMs: 60_000,
|
|
254
|
+
isConcurrencySafe: () => true,
|
|
255
|
+
async execute(args, exec) {
|
|
256
|
+
const parsed = parseArgs(args);
|
|
257
|
+
const signal = exec.signal;
|
|
258
|
+
const startedAt = Date.now();
|
|
259
|
+
const remaining = () => Math.max(0, RESEARCH_BUDGET_MS - (Date.now() - startedAt));
|
|
260
|
+
const cancelled = () => signal !== undefined && signal.aborted;
|
|
261
|
+
const searchLimit = Math.max(5, parsed.maxSources);
|
|
262
|
+
const outcomes = await mapLimit(parsed.queries, 2, async (query) => {
|
|
263
|
+
try {
|
|
264
|
+
return await search(query, searchLimit, signal);
|
|
265
|
+
}
|
|
266
|
+
catch (error) {
|
|
267
|
+
return {
|
|
268
|
+
sources: [],
|
|
269
|
+
failures: [error instanceof Error ? error.message : String(error)],
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
});
|
|
273
|
+
if (cancelled())
|
|
274
|
+
throw new Error('research was cancelled');
|
|
275
|
+
const candidates = interleave(outcomes.map((outcome) => outcome.sources));
|
|
276
|
+
const failures = [];
|
|
277
|
+
const wanted = candidates.slice(0, parsed.maxSources);
|
|
278
|
+
const pages = await mapLimit(wanted, RESEARCH_PAGE_CONCURRENCY, async (source) => {
|
|
279
|
+
if (cancelled())
|
|
280
|
+
return undefined;
|
|
281
|
+
const budget = Math.min(20_000, remaining());
|
|
282
|
+
if (budget < RESEARCH_MIN_PAGE_MS) {
|
|
283
|
+
failures.push({ url: source.url, reason: 'no time left in the research budget' });
|
|
284
|
+
return undefined;
|
|
285
|
+
}
|
|
286
|
+
const bound = signal === undefined
|
|
287
|
+
? AbortSignal.timeout(budget)
|
|
288
|
+
: AbortSignal.any([signal, AbortSignal.timeout(budget)]);
|
|
289
|
+
try {
|
|
290
|
+
const fetched = await fetchPage(source.url, bound);
|
|
291
|
+
const excerpt = excerptOf(fetched.text, RESEARCH_EXCERPT_CHARS);
|
|
292
|
+
if (excerpt.length === 0) {
|
|
293
|
+
failures.push({ url: source.url, reason: 'the page carried no readable text' });
|
|
294
|
+
return undefined;
|
|
295
|
+
}
|
|
296
|
+
const page = {
|
|
297
|
+
url: fetched.url.length > 0 ? fetched.url : source.url,
|
|
298
|
+
...(source.title === undefined || source.title.length === 0 ? {} : { title: source.title }),
|
|
299
|
+
status: fetched.status,
|
|
300
|
+
chars: fetched.text.length,
|
|
301
|
+
excerpt,
|
|
302
|
+
truncated: fetched.truncated || fetched.text.length > RESEARCH_EXCERPT_CHARS,
|
|
303
|
+
};
|
|
304
|
+
return page;
|
|
305
|
+
}
|
|
306
|
+
catch (error) {
|
|
307
|
+
if (cancelled())
|
|
308
|
+
throw error;
|
|
309
|
+
failures.push({ url: source.url, reason: error instanceof Error ? error.message : String(error) });
|
|
310
|
+
return undefined;
|
|
311
|
+
}
|
|
312
|
+
});
|
|
313
|
+
if (cancelled())
|
|
314
|
+
throw new Error('research was cancelled');
|
|
315
|
+
const fetchedPages = pages.filter((page) => page !== undefined);
|
|
316
|
+
const fetchedUrls = new Set(fetchedPages.map((page) => page.url));
|
|
317
|
+
return {
|
|
318
|
+
query: parsed.queries[0],
|
|
319
|
+
queries: parsed.queries,
|
|
320
|
+
pages: fetchedPages,
|
|
321
|
+
sources: candidates
|
|
322
|
+
.filter((source) => !fetchedUrls.has(source.url))
|
|
323
|
+
.slice(0, 20)
|
|
324
|
+
.map((source) => ({
|
|
325
|
+
url: source.url,
|
|
326
|
+
...(source.title === undefined || source.title.length === 0 ? {} : { title: source.title }),
|
|
327
|
+
})),
|
|
328
|
+
failures,
|
|
329
|
+
found: candidates.length,
|
|
330
|
+
truncated: candidates.length > parsed.maxSources,
|
|
331
|
+
};
|
|
332
|
+
},
|
|
333
|
+
};
|
|
334
|
+
}
|
|
335
|
+
/**
|
|
336
|
+
* Pick the tool name this deployment can accept.
|
|
337
|
+
* @param tools - the live registry.
|
|
338
|
+
* @returns the preferred or fallback name.
|
|
339
|
+
*/
|
|
340
|
+
export function pickResearchToolName(tools) {
|
|
341
|
+
if (typeof tools.get !== 'function')
|
|
342
|
+
return RESEARCH_TOOL_NAME;
|
|
343
|
+
try {
|
|
344
|
+
return tools.get(RESEARCH_TOOL_NAME) === undefined ? RESEARCH_TOOL_NAME : RESEARCH_TOOL_FALLBACK_NAME;
|
|
345
|
+
}
|
|
346
|
+
catch {
|
|
347
|
+
return RESEARCH_TOOL_NAME;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
/** Report a registration failure without throwing out of the plugin. */
|
|
351
|
+
function report(options, error) {
|
|
352
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
353
|
+
try {
|
|
354
|
+
options.warn?.(`research registration skipped: ${message}`);
|
|
355
|
+
}
|
|
356
|
+
catch {
|
|
357
|
+
// A diagnostic callback must never break plugin loading.
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
/**
|
|
361
|
+
* Register the tool when a registry is available.
|
|
362
|
+
* @param tools - the live registry.
|
|
363
|
+
* @param search - executes one search through the plugin's normal path.
|
|
364
|
+
* @param fetchPage - fetches one URL through the plugin's fetch ladder.
|
|
365
|
+
* @param options - diagnostics.
|
|
366
|
+
* @returns the registered name and its disposer, or `undefined` when skipped.
|
|
367
|
+
*/
|
|
368
|
+
export function registerResearchTool(tools, search, fetchPage, options = {}) {
|
|
369
|
+
const name = pickResearchToolName(tools);
|
|
370
|
+
try {
|
|
371
|
+
const registered = tools.register(createResearchDefinition(name, search, fetchPage));
|
|
372
|
+
const disposeHost = typeof registered === 'function' ? registered : undefined;
|
|
373
|
+
let disposed = false;
|
|
374
|
+
return {
|
|
375
|
+
name,
|
|
376
|
+
dispose: () => {
|
|
377
|
+
if (disposed)
|
|
378
|
+
return;
|
|
379
|
+
disposed = true;
|
|
380
|
+
try {
|
|
381
|
+
disposeHost?.();
|
|
382
|
+
}
|
|
383
|
+
catch {
|
|
384
|
+
// Disposal must stay contained; the plugin is already going away.
|
|
385
|
+
}
|
|
386
|
+
},
|
|
387
|
+
};
|
|
388
|
+
}
|
|
389
|
+
catch (error) {
|
|
390
|
+
report(options, error);
|
|
391
|
+
return undefined;
|
|
392
|
+
}
|
|
393
|
+
}
|
package/lib/verticals.d.ts
CHANGED
|
@@ -13,12 +13,25 @@
|
|
|
13
13
|
* sources still answer. Timeouts, status classification and failure isolation
|
|
14
14
|
* live in {@link searchVerticals}, not in the parsers.
|
|
15
15
|
*
|
|
16
|
+
* The original eight sources answer by default. The expanded repositories,
|
|
17
|
+
* package registries, paper indexes and data sources are opt-in
|
|
18
|
+
* (`defaultOn: false`): they are queried only when the caller names them, so a
|
|
19
|
+
* bare `domain_search` stays a cheap eight-request fan-out instead of a
|
|
20
|
+
* thirty-request one (GitHub alone allows ten unauthenticated searches a
|
|
21
|
+
* minute). {@link DEFAULT_VERTICAL_IDS} is the default set and
|
|
22
|
+
* {@link VERTICAL_IDS} is every supported source.
|
|
23
|
+
*
|
|
16
24
|
* @module @fangchenghao/dsh-web-search/verticals
|
|
17
25
|
*/
|
|
18
26
|
import type { Source } from './types.ts';
|
|
19
27
|
/** Identifiers accepted in `sources` and in the `domainSources` config field. */
|
|
20
|
-
export type VerticalId = 'github' | 'npm' | 'pypi' | 'arxiv' | 'wikipedia' | 'hn' | 'stackexchange' | 'crates';
|
|
21
|
-
/**
|
|
28
|
+
export type VerticalId = 'github' | 'npm' | 'pypi' | 'arxiv' | 'wikipedia' | 'hn' | 'stackexchange' | 'crates' | 'githubissues' | 'gitlab' | 'codeberg' | 'dockerhub' | 'maven' | 'nuget' | 'rubygems' | 'packagist' | 'pubdev' | 'huggingface' | 'hfdatasets' | 'openvsx' | 'terraform' | 'terraformproviders' | 'wordpress' | 'jetbrains' | 'openalex' | 'crossref' | 'europepmc' | 'uniprot' | 'clinicaltrials' | 'nvd' | 'coingecko' | 'internetarchive' | 'wikidata' | 'nominatim';
|
|
29
|
+
/**
|
|
30
|
+
* Every supported source, in the order results are interleaved.
|
|
31
|
+
*
|
|
32
|
+
* The eight original ids come first so the default set keeps its historical
|
|
33
|
+
* order; expanded sources follow, grouped by kind.
|
|
34
|
+
*/
|
|
22
35
|
export declare const VERTICAL_IDS: readonly VerticalId[];
|
|
23
36
|
/** Default and maximum result budget for one source. */
|
|
24
37
|
export declare const DEFAULT_VERTICAL_LIMIT = 5;
|
|
@@ -32,6 +45,12 @@ export interface VerticalSpec {
|
|
|
32
45
|
label: string;
|
|
33
46
|
/** `Accept` header this endpoint serves. */
|
|
34
47
|
accept: string;
|
|
48
|
+
/**
|
|
49
|
+
* Include this source in {@link DEFAULT_VERTICAL_IDS}. The original eight are
|
|
50
|
+
* on by default; expanded sources set `false` so a bare `domain_search` stays
|
|
51
|
+
* a cheap eight-request fan-out. A missing value counts as on.
|
|
52
|
+
*/
|
|
53
|
+
defaultOn?: boolean;
|
|
35
54
|
/**
|
|
36
55
|
* Treat HTTP 404 as "no such record" rather than a failure. Only PyPI needs
|
|
37
56
|
* this: its endpoint is an exact-package lookup, and a missing package is a
|
|
@@ -49,7 +68,7 @@ export interface VerticalSpec {
|
|
|
49
68
|
export type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
|
|
50
69
|
/** Options for {@link searchVerticals}. */
|
|
51
70
|
export interface VerticalSearchOptions {
|
|
52
|
-
/** Source ids to query; empty or absent means
|
|
71
|
+
/** Source ids to query; empty or absent means {@link DEFAULT_VERTICAL_IDS}. */
|
|
53
72
|
ids?: readonly string[];
|
|
54
73
|
/** Results wanted per source; clamped to 1…20. */
|
|
55
74
|
limit?: number;
|
|
@@ -142,12 +161,230 @@ export declare function parseStackExchange(payload: string, _url: string, limit?
|
|
|
142
161
|
* @returns crates as sources.
|
|
143
162
|
*/
|
|
144
163
|
export declare function parseCrates(payload: string, _url: string, limit?: number): Source[];
|
|
164
|
+
/**
|
|
165
|
+
* Parse GitHub's issue-and-pull-request search envelope.
|
|
166
|
+
* @param payload - `api.github.com/search/issues` JSON.
|
|
167
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
168
|
+
* @param limit - rows to keep.
|
|
169
|
+
* @returns issues and pull requests as sources, in API order.
|
|
170
|
+
*/
|
|
171
|
+
export declare function parseGithubIssues(payload: string, _url: string, limit?: number): Source[];
|
|
172
|
+
/**
|
|
173
|
+
* Parse GitLab's project search envelope.
|
|
174
|
+
* @param payload - `gitlab.com/api/v4/projects` JSON array.
|
|
175
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
176
|
+
* @param limit - rows to keep.
|
|
177
|
+
* @returns projects as sources.
|
|
178
|
+
*/
|
|
179
|
+
export declare function parseGitlab(payload: string, _url: string, limit?: number): Source[];
|
|
180
|
+
/**
|
|
181
|
+
* Parse Codeberg's Gitea repository search envelope.
|
|
182
|
+
* @param payload - `codeberg.org/api/v1/repos/search` JSON.
|
|
183
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
184
|
+
* @param limit - rows to keep.
|
|
185
|
+
* @returns repositories as sources.
|
|
186
|
+
*/
|
|
187
|
+
export declare function parseCodeberg(payload: string, _url: string, limit?: number): Source[];
|
|
188
|
+
/**
|
|
189
|
+
* Parse Docker Hub's repository search envelope.
|
|
190
|
+
* @param payload - `hub.docker.com/v2/search/repositories` JSON.
|
|
191
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
192
|
+
* @param limit - rows to keep.
|
|
193
|
+
* @returns container repositories as sources.
|
|
194
|
+
*/
|
|
195
|
+
export declare function parseDockerHub(payload: string, _url: string, limit?: number): Source[];
|
|
196
|
+
/**
|
|
197
|
+
* Parse Maven Central's Solr search envelope.
|
|
198
|
+
* @param payload - `search.maven.org/solrsearch/select` JSON.
|
|
199
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
200
|
+
* @param limit - rows to keep.
|
|
201
|
+
* @returns artifacts as sources.
|
|
202
|
+
*/
|
|
203
|
+
export declare function parseMaven(payload: string, _url: string, limit?: number): Source[];
|
|
204
|
+
/**
|
|
205
|
+
* Parse NuGet's search envelope.
|
|
206
|
+
* @param payload - `azuresearch-usnc.nuget.org/query` JSON.
|
|
207
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
208
|
+
* @param limit - rows to keep.
|
|
209
|
+
* @returns packages as sources.
|
|
210
|
+
*/
|
|
211
|
+
export declare function parseNuget(payload: string, _url: string, limit?: number): Source[];
|
|
212
|
+
/**
|
|
213
|
+
* Parse RubyGems' search envelope.
|
|
214
|
+
* @param payload - `rubygems.org/api/v1/search.json` JSON array.
|
|
215
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
216
|
+
* @param limit - rows to keep (the endpoint sends a fixed page; rows are capped).
|
|
217
|
+
* @returns gems as sources.
|
|
218
|
+
*/
|
|
219
|
+
export declare function parseRubyGems(payload: string, _url: string, limit?: number): Source[];
|
|
220
|
+
/**
|
|
221
|
+
* Parse Packagist's search envelope.
|
|
222
|
+
* @param payload - `packagist.org/search.json` JSON.
|
|
223
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
224
|
+
* @param limit - rows to keep (the endpoint sends a fixed page; rows are capped).
|
|
225
|
+
* @returns Composer packages as sources.
|
|
226
|
+
*/
|
|
227
|
+
export declare function parsePackagist(payload: string, _url: string, limit?: number): Source[];
|
|
228
|
+
/**
|
|
229
|
+
* Parse pub.dev's package search envelope.
|
|
230
|
+
* @param payload - `pub.dev/api/search` JSON.
|
|
231
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
232
|
+
* @param limit - rows to keep (the endpoint sends a fixed page; rows are capped).
|
|
233
|
+
* @returns Dart packages as sources.
|
|
234
|
+
*/
|
|
235
|
+
export declare function parsePubDev(payload: string, _url: string, limit?: number): Source[];
|
|
236
|
+
/**
|
|
237
|
+
* Parse Hugging Face's model search envelope.
|
|
238
|
+
* @param payload - `huggingface.co/api/models` JSON array.
|
|
239
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
240
|
+
* @param limit - rows to keep.
|
|
241
|
+
* @returns models as sources.
|
|
242
|
+
*/
|
|
243
|
+
export declare function parseHuggingFaceModels(payload: string, _url: string, limit?: number): Source[];
|
|
244
|
+
/**
|
|
245
|
+
* Parse Hugging Face's dataset search envelope.
|
|
246
|
+
* @param payload - `huggingface.co/api/datasets` JSON array.
|
|
247
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
248
|
+
* @param limit - rows to keep.
|
|
249
|
+
* @returns datasets as sources.
|
|
250
|
+
*/
|
|
251
|
+
export declare function parseHuggingFaceDatasets(payload: string, _url: string, limit?: number): Source[];
|
|
252
|
+
/**
|
|
253
|
+
* Parse the Open VSX extension search envelope.
|
|
254
|
+
* @param payload - `open-vsx.org/api/-/search` JSON.
|
|
255
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
256
|
+
* @param limit - rows to keep.
|
|
257
|
+
* @returns extensions as sources.
|
|
258
|
+
*/
|
|
259
|
+
export declare function parseOpenVsx(payload: string, _url: string, limit?: number): Source[];
|
|
260
|
+
/**
|
|
261
|
+
* Parse the Terraform Registry module search envelope.
|
|
262
|
+
* @param payload - `registry.terraform.io/v1/modules` JSON.
|
|
263
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
264
|
+
* @param limit - rows to keep.
|
|
265
|
+
* @returns registry modules as sources.
|
|
266
|
+
*/
|
|
267
|
+
export declare function parseTerraformModules(payload: string, _url: string, limit?: number): Source[];
|
|
268
|
+
/**
|
|
269
|
+
* Parse the Terraform Registry provider search envelope.
|
|
270
|
+
* @param payload - `registry.terraform.io/v1/providers` JSON.
|
|
271
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
272
|
+
* @param limit - rows to keep.
|
|
273
|
+
* @returns registry providers as sources.
|
|
274
|
+
*/
|
|
275
|
+
export declare function parseTerraformProviders(payload: string, _url: string, limit?: number): Source[];
|
|
276
|
+
/**
|
|
277
|
+
* Parse WordPress.org's plugin search envelope.
|
|
278
|
+
* @param payload - `api.wordpress.org/plugins/info/1.2` JSON.
|
|
279
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
280
|
+
* @param limit - rows to keep.
|
|
281
|
+
* @returns plugins as sources.
|
|
282
|
+
*/
|
|
283
|
+
export declare function parseWordpress(payload: string, _url: string, limit?: number): Source[];
|
|
284
|
+
/**
|
|
285
|
+
* Parse the JetBrains Marketplace plugin search envelope.
|
|
286
|
+
* @param payload - `plugins.jetbrains.com/api/searchPlugins` JSON.
|
|
287
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
288
|
+
* @param limit - rows to keep.
|
|
289
|
+
* @returns plugins as sources.
|
|
290
|
+
*/
|
|
291
|
+
export declare function parseJetbrains(payload: string, _url: string, limit?: number): Source[];
|
|
292
|
+
/**
|
|
293
|
+
* Parse OpenAlex's work search envelope.
|
|
294
|
+
* @param payload - `api.openalex.org/works` JSON (`select`ed fields).
|
|
295
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
296
|
+
* @param limit - rows to keep.
|
|
297
|
+
* @returns works as sources.
|
|
298
|
+
*/
|
|
299
|
+
export declare function parseOpenAlex(payload: string, _url: string, limit?: number): Source[];
|
|
300
|
+
/**
|
|
301
|
+
* Parse Crossref's work search envelope.
|
|
302
|
+
* @param payload - `api.crossref.org/works` JSON.
|
|
303
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
304
|
+
* @param limit - rows to keep.
|
|
305
|
+
* @returns DOIs as sources.
|
|
306
|
+
*/
|
|
307
|
+
export declare function parseCrossref(payload: string, _url: string, limit?: number): Source[];
|
|
308
|
+
/**
|
|
309
|
+
* Parse Europe PMC's search envelope.
|
|
310
|
+
* @param payload - `ebi.ac.uk/europepmc/webservices/rest/search` JSON.
|
|
311
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
312
|
+
* @param limit - rows to keep.
|
|
313
|
+
* @returns articles and preprints as sources.
|
|
314
|
+
*/
|
|
315
|
+
export declare function parseEuropepmc(payload: string, _url: string, limit?: number): Source[];
|
|
316
|
+
/**
|
|
317
|
+
* Parse UniProt's protein search envelope.
|
|
318
|
+
* @param payload - `rest.uniprot.org/uniprotkb/search` JSON (`select`ed fields).
|
|
319
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
320
|
+
* @param limit - rows to keep.
|
|
321
|
+
* @returns protein records as sources.
|
|
322
|
+
*/
|
|
323
|
+
export declare function parseUniprot(payload: string, _url: string, limit?: number): Source[];
|
|
324
|
+
/**
|
|
325
|
+
* Parse ClinicalTrials.gov's study search envelope.
|
|
326
|
+
* @param payload - `clinicaltrials.gov/api/v2/studies` JSON (`fields` narrowed).
|
|
327
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
328
|
+
* @param limit - rows to keep.
|
|
329
|
+
* @returns trials as sources.
|
|
330
|
+
*/
|
|
331
|
+
export declare function parseClinicaltrials(payload: string, _url: string, limit?: number): Source[];
|
|
332
|
+
/**
|
|
333
|
+
* Parse NVD's CVE search envelope.
|
|
334
|
+
* @param payload - `services.nvd.nist.gov/rest/json/cves/2.0` JSON.
|
|
335
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
336
|
+
* @param limit - rows to keep.
|
|
337
|
+
* @returns CVEs as sources.
|
|
338
|
+
*/
|
|
339
|
+
export declare function parseNvd(payload: string, _url: string, limit?: number): Source[];
|
|
340
|
+
/**
|
|
341
|
+
* Parse CoinGecko's coin search envelope.
|
|
342
|
+
* @param payload - `api.coingecko.com/api/v3/search` JSON.
|
|
343
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
344
|
+
* @param limit - rows to keep (the endpoint sends no row cap; rows are capped).
|
|
345
|
+
* @returns coins as sources.
|
|
346
|
+
*/
|
|
347
|
+
export declare function parseCoinGecko(payload: string, _url: string, limit?: number): Source[];
|
|
348
|
+
/**
|
|
349
|
+
* Parse the Internet Archive's advanced-search envelope.
|
|
350
|
+
* @param payload - `archive.org/advancedsearch.php` JSON.
|
|
351
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
352
|
+
* @param limit - rows to keep.
|
|
353
|
+
* @returns archive items as sources.
|
|
354
|
+
*/
|
|
355
|
+
export declare function parseInternetarchive(payload: string, _url: string, limit?: number): Source[];
|
|
356
|
+
/**
|
|
357
|
+
* Parse Wikidata's entity search envelope.
|
|
358
|
+
* @param payload - `www.wikidata.org/w/api.php?action=wbsearchentities` JSON.
|
|
359
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
360
|
+
* @param limit - rows to keep.
|
|
361
|
+
* @returns entities as sources.
|
|
362
|
+
*/
|
|
363
|
+
export declare function parseWikidata(payload: string, _url: string, limit?: number): Source[];
|
|
364
|
+
/**
|
|
365
|
+
* Parse Nominatim's place search envelope.
|
|
366
|
+
* @param payload - `nominatim.openstreetmap.org/search?format=jsonv2` JSON array.
|
|
367
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
368
|
+
* @param limit - rows to keep.
|
|
369
|
+
* @returns places as sources.
|
|
370
|
+
*/
|
|
371
|
+
export declare function parseNominatim(payload: string, _url: string, limit?: number): Source[];
|
|
145
372
|
/** A source id that this module actually knows. */
|
|
146
373
|
export declare function isVerticalId(value: unknown): value is VerticalId;
|
|
147
374
|
/** Every adapter, keyed by id. */
|
|
148
375
|
export declare const VERTICALS: readonly VerticalSpec[];
|
|
149
376
|
/** One adapter by id, or `undefined` for a typo. */
|
|
150
377
|
export declare function verticalById(id: string): VerticalSpec | undefined;
|
|
378
|
+
/**
|
|
379
|
+
* Sources queried when the caller names none.
|
|
380
|
+
*
|
|
381
|
+
* The original eight answer by default; every expanded source is
|
|
382
|
+
* `defaultOn: false`. Fanning a bare tool call out to all of
|
|
383
|
+
* {@link VERTICAL_IDS} would spend the whole GitHub unauthenticated search
|
|
384
|
+
* budget in one request and make the common case slow, so participation is
|
|
385
|
+
* declared per spec rather than implied by membership.
|
|
386
|
+
*/
|
|
387
|
+
export declare const DEFAULT_VERTICAL_IDS: readonly VerticalId[];
|
|
151
388
|
/**
|
|
152
389
|
* Query every requested vertical concurrently and merge the answers.
|
|
153
390
|
*
|