@fffangchenghao/dsh-web-search 0.14.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +43 -35
- package/README.zh-CN.md +37 -33
- package/SECURITY.md +4 -0
- package/client.js +72 -5
- package/docs/rrf-ranking.md +40 -22
- package/lib/config.js +5 -5
- package/lib/domain-tool.js +2 -2
- package/lib/engines.js +96 -6
- package/lib/fetch.js +43 -3
- package/lib/index.d.ts +3 -1
- package/lib/index.js +30 -1
- package/lib/keys.d.ts +101 -0
- package/lib/keys.js +163 -0
- package/lib/parsers.d.ts +25 -0
- package/lib/parsers.js +139 -0
- package/lib/research.d.ts +158 -0
- package/lib/research.js +393 -0
- package/lib/verticals.d.ts +240 -3
- package/lib/verticals.js +1185 -78
- package/package.json +2 -1
- package/skills/dsh-web-search/SKILL.md +47 -0
package/lib/verticals.js
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
1
|
import { SearchError } from "./errors.js";
|
|
2
2
|
import { text } from "./parsers.js";
|
|
3
3
|
import { proxyGet } from "./proxy.js";
|
|
4
|
-
/**
|
|
4
|
+
/**
|
|
5
|
+
* Every supported source, in the order results are interleaved.
|
|
6
|
+
*
|
|
7
|
+
* The eight original ids come first so the default set keeps its historical
|
|
8
|
+
* order; expanded sources follow, grouped by kind.
|
|
9
|
+
*/
|
|
5
10
|
export const VERTICAL_IDS = [
|
|
6
11
|
'github',
|
|
7
12
|
'npm',
|
|
@@ -11,13 +16,39 @@ export const VERTICAL_IDS = [
|
|
|
11
16
|
'hn',
|
|
12
17
|
'stackexchange',
|
|
13
18
|
'crates',
|
|
19
|
+
'githubissues',
|
|
20
|
+
'gitlab',
|
|
21
|
+
'codeberg',
|
|
22
|
+
'dockerhub',
|
|
23
|
+
'maven',
|
|
24
|
+
'nuget',
|
|
25
|
+
'rubygems',
|
|
26
|
+
'packagist',
|
|
27
|
+
'pubdev',
|
|
28
|
+
'huggingface',
|
|
29
|
+
'hfdatasets',
|
|
30
|
+
'openvsx',
|
|
31
|
+
'terraform',
|
|
32
|
+
'terraformproviders',
|
|
33
|
+
'wordpress',
|
|
34
|
+
'jetbrains',
|
|
35
|
+
'openalex',
|
|
36
|
+
'crossref',
|
|
37
|
+
'europepmc',
|
|
38
|
+
'uniprot',
|
|
39
|
+
'clinicaltrials',
|
|
40
|
+
'nvd',
|
|
41
|
+
'coingecko',
|
|
42
|
+
'internetarchive',
|
|
43
|
+
'wikidata',
|
|
44
|
+
'nominatim',
|
|
14
45
|
];
|
|
15
46
|
/** Default and maximum result budget for one source. */
|
|
16
47
|
export const DEFAULT_VERTICAL_LIMIT = 5;
|
|
17
48
|
/** Upper bound the tool and the adapters both respect. */
|
|
18
49
|
export const MAX_VERTICAL_LIMIT = 20;
|
|
19
50
|
/** The user agent every source sees. GitHub and crates.io both require one. */
|
|
20
|
-
const USER_AGENT = 'dsh-web-search/0.
|
|
51
|
+
const USER_AGENT = 'dsh-web-search/0.15 (+https://github.com/fangchenghao/dsh-web-search)';
|
|
21
52
|
/** Scripts that make a query CJK rather than ASCII. */
|
|
22
53
|
const CJK = /[\u3000-\u303f\u3040-\u30ff\u3400-\u9fff\uf900-\ufaff\uff00-\uffef]/u;
|
|
23
54
|
/** True for a value usable as a JSON record. */
|
|
@@ -56,6 +87,78 @@ function snippetOf(value, max = 400) {
|
|
|
56
87
|
const clean = text(value);
|
|
57
88
|
return clean.length > max ? `${clean.slice(0, max).trimEnd()}…` : clean;
|
|
58
89
|
}
|
|
90
|
+
/** Read a finite number, or `undefined`. */
|
|
91
|
+
function num(value) {
|
|
92
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
93
|
+
}
|
|
94
|
+
/** Read a scalar (string or number) as trimmed text; anything else is empty. */
|
|
95
|
+
function scalarText(value) {
|
|
96
|
+
if (typeof value === 'string')
|
|
97
|
+
return value.trim();
|
|
98
|
+
const number = num(value);
|
|
99
|
+
return number === undefined ? '' : String(number);
|
|
100
|
+
}
|
|
101
|
+
/** Turn unix seconds, milliseconds or a date string into a timestamp. */
|
|
102
|
+
function isoFrom(value) {
|
|
103
|
+
if (typeof value === 'string') {
|
|
104
|
+
const trimmed = value.trim();
|
|
105
|
+
return trimmed.length === 0 ? undefined : trimmed;
|
|
106
|
+
}
|
|
107
|
+
const number = num(value);
|
|
108
|
+
if (number === undefined)
|
|
109
|
+
return undefined;
|
|
110
|
+
const date = new Date(number > 1e12 ? number : number * 1000);
|
|
111
|
+
return Number.isNaN(date.getTime()) ? undefined : date.toISOString();
|
|
112
|
+
}
|
|
113
|
+
/** Read the first string of a `string | string[]` field. */
|
|
114
|
+
function firstText(value) {
|
|
115
|
+
return Array.isArray(value) ? str(value[0]) : str(value);
|
|
116
|
+
}
|
|
117
|
+
/** Join a `string | string[]` field into one plain-text line. */
|
|
118
|
+
function joinedText(value) {
|
|
119
|
+
if (!Array.isArray(value))
|
|
120
|
+
return text(value);
|
|
121
|
+
return value
|
|
122
|
+
.map((item) => text(item))
|
|
123
|
+
.filter((item) => item.length > 0)
|
|
124
|
+
.join(' ');
|
|
125
|
+
}
|
|
126
|
+
/** Read a Crossref `date-parts` stamp as `YYYY-MM-DD`. */
|
|
127
|
+
function crossrefDate(row) {
|
|
128
|
+
for (const key of ['published', 'published-print', 'published-online', 'issued', 'created']) {
|
|
129
|
+
const stamp = row[key];
|
|
130
|
+
if (!isRecord(stamp))
|
|
131
|
+
continue;
|
|
132
|
+
const parts = stamp['date-parts'];
|
|
133
|
+
if (!Array.isArray(parts) || !Array.isArray(parts[0]))
|
|
134
|
+
continue;
|
|
135
|
+
const year = num(parts[0][0]);
|
|
136
|
+
if (year === undefined)
|
|
137
|
+
continue;
|
|
138
|
+
const month = num(parts[0][1]);
|
|
139
|
+
const day = num(parts[0][2]);
|
|
140
|
+
const mm = month === undefined ? '01' : String(month).padStart(2, '0');
|
|
141
|
+
const dd = day === undefined ? '01' : String(day).padStart(2, '0');
|
|
142
|
+
return `${String(year).padStart(4, '0')}-${mm}-${dd}`;
|
|
143
|
+
}
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
/** Rebuild an OpenAlex abstract from its inverted index, in reading order. */
|
|
147
|
+
function openAlexAbstract(value) {
|
|
148
|
+
if (!isRecord(value))
|
|
149
|
+
return '';
|
|
150
|
+
const words = [];
|
|
151
|
+
for (const [word, positions] of Object.entries(value)) {
|
|
152
|
+
if (!Array.isArray(positions) || word.trim().length === 0)
|
|
153
|
+
continue;
|
|
154
|
+
const position = positions.find((item) => typeof item === 'number' && Number.isFinite(item));
|
|
155
|
+
if (position === undefined)
|
|
156
|
+
continue;
|
|
157
|
+
words.push({ word: word.trim(), position });
|
|
158
|
+
}
|
|
159
|
+
words.sort((a, b) => a.position - b.position);
|
|
160
|
+
return words.map((entry) => entry.word).join(' ');
|
|
161
|
+
}
|
|
59
162
|
/** Build one source, dropping rows that carry no citable URL. */
|
|
60
163
|
function sourceOf(rawUrl, title, snippet, publishedAt) {
|
|
61
164
|
const url = webUrl(rawUrl);
|
|
@@ -307,83 +410,1084 @@ export function parseCrates(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
|
307
410
|
}
|
|
308
411
|
return sources;
|
|
309
412
|
}
|
|
310
|
-
/**
|
|
311
|
-
|
|
312
|
-
|
|
413
|
+
/**
|
|
414
|
+
* Parse GitHub's issue-and-pull-request search envelope.
|
|
415
|
+
* @param payload - `api.github.com/search/issues` JSON.
|
|
416
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
417
|
+
* @param limit - rows to keep.
|
|
418
|
+
* @returns issues and pull requests as sources, in API order.
|
|
419
|
+
*/
|
|
420
|
+
export function parseGithubIssues(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
421
|
+
const body = parseJson(payload);
|
|
422
|
+
if (!isRecord(body) || !Array.isArray(body.items))
|
|
423
|
+
return [];
|
|
424
|
+
const sources = [];
|
|
425
|
+
for (const row of body.items) {
|
|
426
|
+
if (!isRecord(row))
|
|
427
|
+
continue;
|
|
428
|
+
const source = sourceOf(row.html_url, str(row.title), snippetOf(row.body), str(row.created_at) || str(row.updated_at));
|
|
429
|
+
if (source !== undefined)
|
|
430
|
+
sources.push(source);
|
|
431
|
+
if (sources.length >= limit)
|
|
432
|
+
break;
|
|
433
|
+
}
|
|
434
|
+
return sources;
|
|
313
435
|
}
|
|
314
|
-
/**
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
436
|
+
/**
|
|
437
|
+
* Parse GitLab's project search envelope.
|
|
438
|
+
* @param payload - `gitlab.com/api/v4/projects` JSON array.
|
|
439
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
440
|
+
* @param limit - rows to keep.
|
|
441
|
+
* @returns projects as sources.
|
|
442
|
+
*/
|
|
443
|
+
export function parseGitlab(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
444
|
+
const body = parseJson(payload);
|
|
445
|
+
if (!Array.isArray(body))
|
|
446
|
+
return [];
|
|
447
|
+
const sources = [];
|
|
448
|
+
for (const row of body) {
|
|
449
|
+
if (!isRecord(row))
|
|
450
|
+
continue;
|
|
451
|
+
const name = str(row.path_with_namespace) || str(row.name);
|
|
452
|
+
const source = sourceOf(str(row.web_url), name, snippetOf(row.description), str(row.last_activity_at));
|
|
453
|
+
if (source !== undefined)
|
|
454
|
+
sources.push(source);
|
|
455
|
+
if (sources.length >= limit)
|
|
456
|
+
break;
|
|
457
|
+
}
|
|
458
|
+
return sources;
|
|
459
|
+
}
|
|
460
|
+
/**
|
|
461
|
+
* Parse Codeberg's Gitea repository search envelope.
|
|
462
|
+
* @param payload - `codeberg.org/api/v1/repos/search` JSON.
|
|
463
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
464
|
+
* @param limit - rows to keep.
|
|
465
|
+
* @returns repositories as sources.
|
|
466
|
+
*/
|
|
467
|
+
export function parseCodeberg(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
468
|
+
const body = parseJson(payload);
|
|
469
|
+
if (!isRecord(body) || !Array.isArray(body.data))
|
|
470
|
+
return [];
|
|
471
|
+
const sources = [];
|
|
472
|
+
for (const row of body.data) {
|
|
473
|
+
if (!isRecord(row))
|
|
474
|
+
continue;
|
|
475
|
+
const name = str(row.full_name) || str(row.name);
|
|
476
|
+
const source = sourceOf(str(row.html_url), name, snippetOf(row.description), str(row.updated_at));
|
|
477
|
+
if (source !== undefined)
|
|
478
|
+
sources.push(source);
|
|
479
|
+
if (sources.length >= limit)
|
|
480
|
+
break;
|
|
481
|
+
}
|
|
482
|
+
return sources;
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* Parse Docker Hub's repository search envelope.
|
|
486
|
+
* @param payload - `hub.docker.com/v2/search/repositories` JSON.
|
|
487
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
488
|
+
* @param limit - rows to keep.
|
|
489
|
+
* @returns container repositories as sources.
|
|
490
|
+
*/
|
|
491
|
+
export function parseDockerHub(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
492
|
+
const body = parseJson(payload);
|
|
493
|
+
if (!isRecord(body) || !Array.isArray(body.results))
|
|
494
|
+
return [];
|
|
495
|
+
const sources = [];
|
|
496
|
+
for (const row of body.results) {
|
|
497
|
+
if (!isRecord(row))
|
|
498
|
+
continue;
|
|
499
|
+
const name = str(row.repo_name);
|
|
500
|
+
if (name.length === 0)
|
|
501
|
+
continue;
|
|
502
|
+
const stars = num(row.star_count);
|
|
503
|
+
const pulls = num(row.pull_count);
|
|
504
|
+
const meta = [
|
|
505
|
+
stars === undefined ? '' : `${stars} stars`,
|
|
506
|
+
pulls === undefined ? '' : `${pulls} pulls`,
|
|
507
|
+
].filter((value) => value.length > 0).join(' · ');
|
|
508
|
+
const source = sourceOf(`https://hub.docker.com/r/${name}`, name, snippetOf(row.short_description) || meta);
|
|
509
|
+
if (source !== undefined)
|
|
510
|
+
sources.push(source);
|
|
511
|
+
if (sources.length >= limit)
|
|
512
|
+
break;
|
|
513
|
+
}
|
|
514
|
+
return sources;
|
|
515
|
+
}
|
|
516
|
+
/**
|
|
517
|
+
* Parse Maven Central's Solr search envelope.
|
|
518
|
+
* @param payload - `search.maven.org/solrsearch/select` JSON.
|
|
519
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
520
|
+
* @param limit - rows to keep.
|
|
521
|
+
* @returns artifacts as sources.
|
|
522
|
+
*/
|
|
523
|
+
export function parseMaven(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
524
|
+
const body = parseJson(payload);
|
|
525
|
+
if (!isRecord(body) || !isRecord(body.response) || !Array.isArray(body.response.docs))
|
|
526
|
+
return [];
|
|
527
|
+
const sources = [];
|
|
528
|
+
for (const row of body.response.docs) {
|
|
529
|
+
if (!isRecord(row))
|
|
530
|
+
continue;
|
|
531
|
+
const group = str(row.g);
|
|
532
|
+
const artifact = str(row.a);
|
|
533
|
+
if (group.length === 0 || artifact.length === 0)
|
|
534
|
+
continue;
|
|
535
|
+
const version = str(row.latestVersion);
|
|
536
|
+
const versions = num(row.versionCount);
|
|
537
|
+
const source = sourceOf(`https://central.sonatype.com/artifact/${encodeURIComponent(group)}/${encodeURIComponent(artifact)}`, version.length > 0 ? `${group}:${artifact} ${version}` : `${group}:${artifact}`, versions === undefined ? '' : `${versions} versions`, isoFrom(row.timestamp));
|
|
538
|
+
if (source !== undefined)
|
|
539
|
+
sources.push(source);
|
|
540
|
+
if (sources.length >= limit)
|
|
541
|
+
break;
|
|
542
|
+
}
|
|
543
|
+
return sources;
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* Parse NuGet's search envelope.
|
|
547
|
+
* @param payload - `azuresearch-usnc.nuget.org/query` JSON.
|
|
548
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
549
|
+
* @param limit - rows to keep.
|
|
550
|
+
* @returns packages as sources.
|
|
551
|
+
*/
|
|
552
|
+
export function parseNuget(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
553
|
+
const body = parseJson(payload);
|
|
554
|
+
if (!isRecord(body) || !Array.isArray(body.data))
|
|
555
|
+
return [];
|
|
556
|
+
const sources = [];
|
|
557
|
+
for (const row of body.data) {
|
|
558
|
+
if (!isRecord(row))
|
|
559
|
+
continue;
|
|
560
|
+
const name = str(row.id);
|
|
561
|
+
if (name.length === 0)
|
|
562
|
+
continue;
|
|
563
|
+
const version = str(row.version);
|
|
564
|
+
const downloads = num(row.totalDownloads);
|
|
565
|
+
const source = sourceOf(`https://www.nuget.org/packages/${encodeURIComponent(name)}`, version.length > 0 ? `${name} ${version}` : name, snippetOf(row.description) || (downloads === undefined ? '' : `${downloads} downloads`));
|
|
566
|
+
if (source !== undefined)
|
|
567
|
+
sources.push(source);
|
|
568
|
+
if (sources.length >= limit)
|
|
569
|
+
break;
|
|
570
|
+
}
|
|
571
|
+
return sources;
|
|
572
|
+
}
|
|
573
|
+
/**
|
|
574
|
+
* Parse RubyGems' search envelope.
|
|
575
|
+
* @param payload - `rubygems.org/api/v1/search.json` JSON array.
|
|
576
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
577
|
+
* @param limit - rows to keep (the endpoint sends a fixed page; rows are capped).
|
|
578
|
+
* @returns gems as sources.
|
|
579
|
+
*/
|
|
580
|
+
export function parseRubyGems(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
581
|
+
const body = parseJson(payload);
|
|
582
|
+
if (!Array.isArray(body))
|
|
583
|
+
return [];
|
|
584
|
+
const sources = [];
|
|
585
|
+
for (const row of body) {
|
|
586
|
+
if (!isRecord(row))
|
|
587
|
+
continue;
|
|
588
|
+
const name = str(row.name);
|
|
589
|
+
if (name.length === 0)
|
|
590
|
+
continue;
|
|
591
|
+
const version = str(row.version);
|
|
592
|
+
const source = sourceOf(str(row.project_uri) || `https://rubygems.org/gems/${encodeURIComponent(name)}`, version.length > 0 ? `${name} ${version}` : name, snippetOf(row.info));
|
|
593
|
+
if (source !== undefined)
|
|
594
|
+
sources.push(source);
|
|
595
|
+
if (sources.length >= limit)
|
|
596
|
+
break;
|
|
597
|
+
}
|
|
598
|
+
return sources;
|
|
599
|
+
}
|
|
600
|
+
/**
|
|
601
|
+
* Parse Packagist's search envelope.
|
|
602
|
+
* @param payload - `packagist.org/search.json` JSON.
|
|
603
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
604
|
+
* @param limit - rows to keep (the endpoint sends a fixed page; rows are capped).
|
|
605
|
+
* @returns Composer packages as sources.
|
|
606
|
+
*/
|
|
607
|
+
export function parsePackagist(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
608
|
+
const body = parseJson(payload);
|
|
609
|
+
if (!isRecord(body) || !Array.isArray(body.results))
|
|
610
|
+
return [];
|
|
611
|
+
const sources = [];
|
|
612
|
+
for (const row of body.results) {
|
|
613
|
+
if (!isRecord(row))
|
|
614
|
+
continue;
|
|
615
|
+
const name = str(row.name);
|
|
616
|
+
if (name.length === 0)
|
|
617
|
+
continue;
|
|
618
|
+
const source = sourceOf(str(row.url) || `https://packagist.org/packages/${name}`, name, snippetOf(row.description));
|
|
619
|
+
if (source !== undefined)
|
|
620
|
+
sources.push(source);
|
|
621
|
+
if (sources.length >= limit)
|
|
622
|
+
break;
|
|
623
|
+
}
|
|
624
|
+
return sources;
|
|
625
|
+
}
|
|
626
|
+
/**
|
|
627
|
+
* Parse pub.dev's package search envelope.
|
|
628
|
+
* @param payload - `pub.dev/api/search` JSON.
|
|
629
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
630
|
+
* @param limit - rows to keep (the endpoint sends a fixed page; rows are capped).
|
|
631
|
+
* @returns Dart packages as sources.
|
|
632
|
+
*/
|
|
633
|
+
export function parsePubDev(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
634
|
+
const body = parseJson(payload);
|
|
635
|
+
if (!isRecord(body) || !Array.isArray(body.packages))
|
|
636
|
+
return [];
|
|
637
|
+
const sources = [];
|
|
638
|
+
for (const row of body.packages) {
|
|
639
|
+
if (!isRecord(row))
|
|
640
|
+
continue;
|
|
641
|
+
const name = str(row.package);
|
|
642
|
+
if (name.length === 0)
|
|
643
|
+
continue;
|
|
644
|
+
const source = sourceOf(`https://pub.dev/packages/${encodeURIComponent(name)}`, name, '');
|
|
645
|
+
if (source !== undefined)
|
|
646
|
+
sources.push(source);
|
|
647
|
+
if (sources.length >= limit)
|
|
648
|
+
break;
|
|
649
|
+
}
|
|
650
|
+
return sources;
|
|
386
651
|
}
|
|
652
|
+
/** Shared parse for Hugging Face's model and dataset listings. */
|
|
653
|
+
function huggingFaceList(payload, limit, base) {
|
|
654
|
+
const body = parseJson(payload);
|
|
655
|
+
if (!Array.isArray(body))
|
|
656
|
+
return [];
|
|
657
|
+
const sources = [];
|
|
658
|
+
for (const row of body) {
|
|
659
|
+
if (!isRecord(row))
|
|
660
|
+
continue;
|
|
661
|
+
const id = str(row.id) || str(row.modelId);
|
|
662
|
+
if (id.length === 0)
|
|
663
|
+
continue;
|
|
664
|
+
const likes = num(row.likes);
|
|
665
|
+
const downloads = num(row.downloads);
|
|
666
|
+
const meta = [
|
|
667
|
+
str(row.pipeline_tag),
|
|
668
|
+
downloads === undefined ? '' : `${downloads} downloads`,
|
|
669
|
+
likes === undefined || likes === 0 ? '' : `${likes} likes`,
|
|
670
|
+
].filter((value) => value.length > 0).join(' · ');
|
|
671
|
+
const source = sourceOf(`${base}/${id}`, id, snippetOf(row.description) || meta, str(row.createdAt) || str(row.lastModified));
|
|
672
|
+
if (source !== undefined)
|
|
673
|
+
sources.push(source);
|
|
674
|
+
if (sources.length >= limit)
|
|
675
|
+
break;
|
|
676
|
+
}
|
|
677
|
+
return sources;
|
|
678
|
+
}
|
|
679
|
+
/**
|
|
680
|
+
* Parse Hugging Face's model search envelope.
|
|
681
|
+
* @param payload - `huggingface.co/api/models` JSON array.
|
|
682
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
683
|
+
* @param limit - rows to keep.
|
|
684
|
+
* @returns models as sources.
|
|
685
|
+
*/
|
|
686
|
+
export function parseHuggingFaceModels(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
687
|
+
return huggingFaceList(payload, limit, 'https://huggingface.co');
|
|
688
|
+
}
|
|
689
|
+
/**
|
|
690
|
+
* Parse Hugging Face's dataset search envelope.
|
|
691
|
+
* @param payload - `huggingface.co/api/datasets` JSON array.
|
|
692
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
693
|
+
* @param limit - rows to keep.
|
|
694
|
+
* @returns datasets as sources.
|
|
695
|
+
*/
|
|
696
|
+
export function parseHuggingFaceDatasets(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
697
|
+
return huggingFaceList(payload, limit, 'https://huggingface.co/datasets');
|
|
698
|
+
}
|
|
699
|
+
/**
|
|
700
|
+
* Parse the Open VSX extension search envelope.
|
|
701
|
+
* @param payload - `open-vsx.org/api/-/search` JSON.
|
|
702
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
703
|
+
* @param limit - rows to keep.
|
|
704
|
+
* @returns extensions as sources.
|
|
705
|
+
*/
|
|
706
|
+
export function parseOpenVsx(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
707
|
+
const body = parseJson(payload);
|
|
708
|
+
if (!isRecord(body) || !Array.isArray(body.extensions))
|
|
709
|
+
return [];
|
|
710
|
+
const sources = [];
|
|
711
|
+
for (const row of body.extensions) {
|
|
712
|
+
if (!isRecord(row))
|
|
713
|
+
continue;
|
|
714
|
+
const namespace = str(row.namespace);
|
|
715
|
+
const name = str(row.name);
|
|
716
|
+
if (namespace.length === 0 || name.length === 0)
|
|
717
|
+
continue;
|
|
718
|
+
const version = str(row.version);
|
|
719
|
+
const page = `https://open-vsx.org/extension/${encodeURIComponent(namespace)}/${encodeURIComponent(name)}`;
|
|
720
|
+
const source = sourceOf(version.length > 0 ? `${page}/${encodeURIComponent(version)}` : page, str(row.displayName) || `${namespace}.${name}`, snippetOf(row.description), isoFrom(row.timestamp));
|
|
721
|
+
if (source !== undefined)
|
|
722
|
+
sources.push(source);
|
|
723
|
+
if (sources.length >= limit)
|
|
724
|
+
break;
|
|
725
|
+
}
|
|
726
|
+
return sources;
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* Parse the Terraform Registry module search envelope.
|
|
730
|
+
* @param payload - `registry.terraform.io/v1/modules` JSON.
|
|
731
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
732
|
+
* @param limit - rows to keep.
|
|
733
|
+
* @returns registry modules as sources.
|
|
734
|
+
*/
|
|
735
|
+
export function parseTerraformModules(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
736
|
+
const body = parseJson(payload);
|
|
737
|
+
if (!isRecord(body) || !Array.isArray(body.modules))
|
|
738
|
+
return [];
|
|
739
|
+
const sources = [];
|
|
740
|
+
for (const row of body.modules) {
|
|
741
|
+
if (!isRecord(row))
|
|
742
|
+
continue;
|
|
743
|
+
const namespace = str(row.namespace);
|
|
744
|
+
const name = str(row.name);
|
|
745
|
+
if (namespace.length === 0 || name.length === 0)
|
|
746
|
+
continue;
|
|
747
|
+
const provider = str(row.provider);
|
|
748
|
+
const version = str(row.version);
|
|
749
|
+
const page = [
|
|
750
|
+
'https://registry.terraform.io/modules',
|
|
751
|
+
encodeURIComponent(namespace),
|
|
752
|
+
encodeURIComponent(name),
|
|
753
|
+
encodeURIComponent(provider.length > 0 ? provider : 'latest'),
|
|
754
|
+
encodeURIComponent(version.length > 0 ? version : 'latest'),
|
|
755
|
+
].join('/');
|
|
756
|
+
const label = [namespace, name, provider].filter((value) => value.length > 0).join('/');
|
|
757
|
+
const source = sourceOf(page, version.length > 0 ? `${label} ${version}` : label, snippetOf(row.description), str(row.published_at));
|
|
758
|
+
if (source !== undefined)
|
|
759
|
+
sources.push(source);
|
|
760
|
+
if (sources.length >= limit)
|
|
761
|
+
break;
|
|
762
|
+
}
|
|
763
|
+
return sources;
|
|
764
|
+
}
|
|
765
|
+
/**
|
|
766
|
+
* Parse the Terraform Registry provider search envelope.
|
|
767
|
+
* @param payload - `registry.terraform.io/v1/providers` JSON.
|
|
768
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
769
|
+
* @param limit - rows to keep.
|
|
770
|
+
* @returns registry providers as sources.
|
|
771
|
+
*/
|
|
772
|
+
export function parseTerraformProviders(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
773
|
+
const body = parseJson(payload);
|
|
774
|
+
if (!isRecord(body) || !Array.isArray(body.providers))
|
|
775
|
+
return [];
|
|
776
|
+
const sources = [];
|
|
777
|
+
for (const row of body.providers) {
|
|
778
|
+
if (!isRecord(row))
|
|
779
|
+
continue;
|
|
780
|
+
const namespace = str(row.namespace);
|
|
781
|
+
const name = str(row.name);
|
|
782
|
+
if (namespace.length === 0 || name.length === 0)
|
|
783
|
+
continue;
|
|
784
|
+
const version = str(row.version);
|
|
785
|
+
const page = [
|
|
786
|
+
'https://registry.terraform.io/providers',
|
|
787
|
+
encodeURIComponent(namespace),
|
|
788
|
+
encodeURIComponent(name),
|
|
789
|
+
encodeURIComponent(version.length > 0 ? version : 'latest'),
|
|
790
|
+
].join('/');
|
|
791
|
+
const label = `${namespace}/${name}`;
|
|
792
|
+
const source = sourceOf(page, version.length > 0 ? `${label} ${version}` : label, snippetOf(row.description), str(row.published_at));
|
|
793
|
+
if (source !== undefined)
|
|
794
|
+
sources.push(source);
|
|
795
|
+
if (sources.length >= limit)
|
|
796
|
+
break;
|
|
797
|
+
}
|
|
798
|
+
return sources;
|
|
799
|
+
}
|
|
800
|
+
/**
|
|
801
|
+
* Parse WordPress.org's plugin search envelope.
|
|
802
|
+
* @param payload - `api.wordpress.org/plugins/info/1.2` JSON.
|
|
803
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
804
|
+
* @param limit - rows to keep.
|
|
805
|
+
* @returns plugins as sources.
|
|
806
|
+
*/
|
|
807
|
+
export function parseWordpress(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
808
|
+
const body = parseJson(payload);
|
|
809
|
+
if (!isRecord(body) || !Array.isArray(body.plugins))
|
|
810
|
+
return [];
|
|
811
|
+
const sources = [];
|
|
812
|
+
for (const row of body.plugins) {
|
|
813
|
+
if (!isRecord(row))
|
|
814
|
+
continue;
|
|
815
|
+
const slug = str(row.slug);
|
|
816
|
+
const name = text(row.name);
|
|
817
|
+
if (slug.length === 0 || name.length === 0)
|
|
818
|
+
continue;
|
|
819
|
+
const version = str(row.version);
|
|
820
|
+
const source = sourceOf(`https://wordpress.org/plugins/${encodeURIComponent(slug)}/`, version.length > 0 ? `${name} ${version}` : name, snippetOf(row.short_description));
|
|
821
|
+
if (source !== undefined)
|
|
822
|
+
sources.push(source);
|
|
823
|
+
if (sources.length >= limit)
|
|
824
|
+
break;
|
|
825
|
+
}
|
|
826
|
+
return sources;
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* Parse the JetBrains Marketplace plugin search envelope.
|
|
830
|
+
* @param payload - `plugins.jetbrains.com/api/searchPlugins` JSON.
|
|
831
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
832
|
+
* @param limit - rows to keep.
|
|
833
|
+
* @returns plugins as sources.
|
|
834
|
+
*/
|
|
835
|
+
export function parseJetbrains(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
836
|
+
const body = parseJson(payload);
|
|
837
|
+
if (!isRecord(body) || !Array.isArray(body.plugins))
|
|
838
|
+
return [];
|
|
839
|
+
const sources = [];
|
|
840
|
+
for (const row of body.plugins) {
|
|
841
|
+
if (!isRecord(row))
|
|
842
|
+
continue;
|
|
843
|
+
const name = str(row.name);
|
|
844
|
+
const link = str(row.link);
|
|
845
|
+
if (name.length === 0 || link.length === 0)
|
|
846
|
+
continue;
|
|
847
|
+
const page = `https://plugins.jetbrains.com${link.startsWith('/') ? link : `/${link}`}`;
|
|
848
|
+
const source = sourceOf(page, name, snippetOf(row.preview), isoFrom(row.cdate));
|
|
849
|
+
if (source !== undefined)
|
|
850
|
+
sources.push(source);
|
|
851
|
+
if (sources.length >= limit)
|
|
852
|
+
break;
|
|
853
|
+
}
|
|
854
|
+
return sources;
|
|
855
|
+
}
|
|
856
|
+
/**
|
|
857
|
+
* Parse OpenAlex's work search envelope.
|
|
858
|
+
* @param payload - `api.openalex.org/works` JSON (`select`ed fields).
|
|
859
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
860
|
+
* @param limit - rows to keep.
|
|
861
|
+
* @returns works as sources.
|
|
862
|
+
*/
|
|
863
|
+
export function parseOpenAlex(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
864
|
+
const body = parseJson(payload);
|
|
865
|
+
if (!isRecord(body) || !Array.isArray(body.results))
|
|
866
|
+
return [];
|
|
867
|
+
const sources = [];
|
|
868
|
+
for (const row of body.results) {
|
|
869
|
+
if (!isRecord(row))
|
|
870
|
+
continue;
|
|
871
|
+
const primary = isRecord(row.primary_location) ? row.primary_location : undefined;
|
|
872
|
+
const venueRecord = primary !== undefined && isRecord(primary.source) ? primary.source : undefined;
|
|
873
|
+
const venue = venueRecord === undefined ? '' : str(venueRecord.display_name);
|
|
874
|
+
const year = num(row.publication_year);
|
|
875
|
+
const meta = [
|
|
876
|
+
str(row.type),
|
|
877
|
+
venue,
|
|
878
|
+
year === undefined ? '' : String(year),
|
|
879
|
+
].filter((value) => value.length > 0).join(' · ');
|
|
880
|
+
const title = str(row.display_name) || str(row.title);
|
|
881
|
+
if (title.length === 0)
|
|
882
|
+
continue;
|
|
883
|
+
const source = sourceOf(str(row.doi) || str(row.id) || (primary === undefined ? '' : str(primary.landing_page_url)), title, snippetOf(openAlexAbstract(row.abstract_inverted_index)) || snippetOf(meta), str(row.publication_date) || (year === undefined ? undefined : String(year)));
|
|
884
|
+
if (source !== undefined)
|
|
885
|
+
sources.push(source);
|
|
886
|
+
if (sources.length >= limit)
|
|
887
|
+
break;
|
|
888
|
+
}
|
|
889
|
+
return sources;
|
|
890
|
+
}
|
|
891
|
+
/**
|
|
892
|
+
* Parse Crossref's work search envelope.
|
|
893
|
+
* @param payload - `api.crossref.org/works` JSON.
|
|
894
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
895
|
+
* @param limit - rows to keep.
|
|
896
|
+
* @returns DOIs as sources.
|
|
897
|
+
*/
|
|
898
|
+
export function parseCrossref(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
899
|
+
const body = parseJson(payload);
|
|
900
|
+
if (!isRecord(body) || !isRecord(body.message) || !Array.isArray(body.message.items))
|
|
901
|
+
return [];
|
|
902
|
+
const sources = [];
|
|
903
|
+
for (const row of body.message.items) {
|
|
904
|
+
if (!isRecord(row))
|
|
905
|
+
continue;
|
|
906
|
+
const doi = str(row.DOI);
|
|
907
|
+
const title = firstText(row.title) || doi;
|
|
908
|
+
if (title.length === 0)
|
|
909
|
+
continue;
|
|
910
|
+
const published = crossrefDate(row);
|
|
911
|
+
const source = sourceOf(str(row.URL) || (doi.length > 0 ? `https://doi.org/${doi}` : undefined), title, snippetOf(row.abstract) || snippetOf(joinedText(row['container-title'])) || snippetOf(row.type), published);
|
|
912
|
+
if (source !== undefined)
|
|
913
|
+
sources.push(source);
|
|
914
|
+
if (sources.length >= limit)
|
|
915
|
+
break;
|
|
916
|
+
}
|
|
917
|
+
return sources;
|
|
918
|
+
}
|
|
919
|
+
/**
|
|
920
|
+
* Parse Europe PMC's search envelope.
|
|
921
|
+
* @param payload - `ebi.ac.uk/europepmc/webservices/rest/search` JSON.
|
|
922
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
923
|
+
* @param limit - rows to keep.
|
|
924
|
+
* @returns articles and preprints as sources.
|
|
925
|
+
*/
|
|
926
|
+
export function parseEuropepmc(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
927
|
+
const body = parseJson(payload);
|
|
928
|
+
if (!isRecord(body) || !isRecord(body.resultList) || !Array.isArray(body.resultList.result))
|
|
929
|
+
return [];
|
|
930
|
+
const sources = [];
|
|
931
|
+
for (const row of body.resultList.result) {
|
|
932
|
+
if (!isRecord(row))
|
|
933
|
+
continue;
|
|
934
|
+
const id = str(row.id);
|
|
935
|
+
const doi = str(row.doi);
|
|
936
|
+
const database = str(row.source) || 'MED';
|
|
937
|
+
const fallback = id.length > 0
|
|
938
|
+
? `https://europepmc.org/article/${encodeURIComponent(database)}/${encodeURIComponent(id)}`
|
|
939
|
+
: undefined;
|
|
940
|
+
const title = text(row.title);
|
|
941
|
+
if (title.length === 0)
|
|
942
|
+
continue;
|
|
943
|
+
const source = sourceOf(doi.length > 0 ? `https://doi.org/${doi}` : fallback, title, snippetOf(row.abstractText), str(row.firstPublicationDate) || str(row.pubYear));
|
|
944
|
+
if (source !== undefined)
|
|
945
|
+
sources.push(source);
|
|
946
|
+
if (sources.length >= limit)
|
|
947
|
+
break;
|
|
948
|
+
}
|
|
949
|
+
return sources;
|
|
950
|
+
}
|
|
951
|
+
/**
|
|
952
|
+
* Parse UniProt's protein search envelope.
|
|
953
|
+
* @param payload - `rest.uniprot.org/uniprotkb/search` JSON (`select`ed fields).
|
|
954
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
955
|
+
* @param limit - rows to keep.
|
|
956
|
+
* @returns protein records as sources.
|
|
957
|
+
*/
|
|
958
|
+
export function parseUniprot(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
959
|
+
const body = parseJson(payload);
|
|
960
|
+
if (!isRecord(body) || !Array.isArray(body.results))
|
|
961
|
+
return [];
|
|
962
|
+
const sources = [];
|
|
963
|
+
for (const row of body.results) {
|
|
964
|
+
if (!isRecord(row))
|
|
965
|
+
continue;
|
|
966
|
+
const accession = str(row.primaryAccession);
|
|
967
|
+
if (accession.length === 0)
|
|
968
|
+
continue;
|
|
969
|
+
const description = isRecord(row.proteinDescription) ? row.proteinDescription : undefined;
|
|
970
|
+
const recommended = description !== undefined && isRecord(description.recommendedName)
|
|
971
|
+
? description.recommendedName
|
|
972
|
+
: undefined;
|
|
973
|
+
const fullName = recommended !== undefined && isRecord(recommended.fullName)
|
|
974
|
+
? str(recommended.fullName.value)
|
|
975
|
+
: '';
|
|
976
|
+
const organism = isRecord(row.organism) ? str(row.organism.scientificName) : '';
|
|
977
|
+
const length = isRecord(row.sequence) ? num(row.sequence.length) : undefined;
|
|
978
|
+
const meta = [organism, length === undefined ? '' : `${length} aa`]
|
|
979
|
+
.filter((value) => value.length > 0)
|
|
980
|
+
.join(' · ');
|
|
981
|
+
const source = sourceOf(`https://www.uniprot.org/uniprotkb/${encodeURIComponent(accession)}/entry`, fullName.length > 0 ? `${fullName} (${accession})` : accession, meta);
|
|
982
|
+
if (source !== undefined)
|
|
983
|
+
sources.push(source);
|
|
984
|
+
if (sources.length >= limit)
|
|
985
|
+
break;
|
|
986
|
+
}
|
|
987
|
+
return sources;
|
|
988
|
+
}
|
|
989
|
+
/**
|
|
990
|
+
* Parse ClinicalTrials.gov's study search envelope.
|
|
991
|
+
* @param payload - `clinicaltrials.gov/api/v2/studies` JSON (`fields` narrowed).
|
|
992
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
993
|
+
* @param limit - rows to keep.
|
|
994
|
+
* @returns trials as sources.
|
|
995
|
+
*/
|
|
996
|
+
export function parseClinicaltrials(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
997
|
+
const body = parseJson(payload);
|
|
998
|
+
if (!isRecord(body) || !Array.isArray(body.studies))
|
|
999
|
+
return [];
|
|
1000
|
+
const sources = [];
|
|
1001
|
+
for (const row of body.studies) {
|
|
1002
|
+
if (!isRecord(row) || !isRecord(row.protocolSection))
|
|
1003
|
+
continue;
|
|
1004
|
+
const protocol = row.protocolSection;
|
|
1005
|
+
const identification = isRecord(protocol.identificationModule) ? protocol.identificationModule : undefined;
|
|
1006
|
+
const trialId = identification === undefined ? '' : str(identification.nctId);
|
|
1007
|
+
if (trialId.length === 0)
|
|
1008
|
+
continue;
|
|
1009
|
+
const status = isRecord(protocol.statusModule) ? protocol.statusModule : undefined;
|
|
1010
|
+
const summary = isRecord(protocol.descriptionModule) ? protocol.descriptionModule : undefined;
|
|
1011
|
+
const start = status !== undefined && isRecord(status.startDateStruct)
|
|
1012
|
+
? str(status.startDateStruct.date)
|
|
1013
|
+
: '';
|
|
1014
|
+
const title = identification !== undefined ? str(identification.briefTitle) : '';
|
|
1015
|
+
const overallStatus = status === undefined ? '' : str(status.overallStatus);
|
|
1016
|
+
const source = sourceOf(`https://clinicaltrials.gov/study/${encodeURIComponent(trialId)}`, title.length > 0 ? title : trialId, snippetOf(summary === undefined ? '' : summary.briefSummary)
|
|
1017
|
+
|| (overallStatus.length > 0 ? `Status: ${overallStatus}` : ''), start);
|
|
1018
|
+
if (source !== undefined)
|
|
1019
|
+
sources.push(source);
|
|
1020
|
+
if (sources.length >= limit)
|
|
1021
|
+
break;
|
|
1022
|
+
}
|
|
1023
|
+
return sources;
|
|
1024
|
+
}
|
|
1025
|
+
/**
|
|
1026
|
+
* Parse NVD's CVE search envelope.
|
|
1027
|
+
* @param payload - `services.nvd.nist.gov/rest/json/cves/2.0` JSON.
|
|
1028
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
1029
|
+
* @param limit - rows to keep.
|
|
1030
|
+
* @returns CVEs as sources.
|
|
1031
|
+
*/
|
|
1032
|
+
export function parseNvd(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
1033
|
+
const body = parseJson(payload);
|
|
1034
|
+
if (!isRecord(body) || !Array.isArray(body.vulnerabilities))
|
|
1035
|
+
return [];
|
|
1036
|
+
const sources = [];
|
|
1037
|
+
for (const row of body.vulnerabilities) {
|
|
1038
|
+
if (!isRecord(row) || !isRecord(row.cve))
|
|
1039
|
+
continue;
|
|
1040
|
+
const cve = row.cve;
|
|
1041
|
+
const id = str(cve.id);
|
|
1042
|
+
if (id.length === 0)
|
|
1043
|
+
continue;
|
|
1044
|
+
const descriptions = Array.isArray(cve.descriptions) ? cve.descriptions : [];
|
|
1045
|
+
const english = descriptions.find((item) => isRecord(item) && str(item.lang) === 'en');
|
|
1046
|
+
const chosen = isRecord(english) ? english : descriptions.find((item) => isRecord(item));
|
|
1047
|
+
const description = isRecord(chosen) ? snippetOf(chosen.value) : '';
|
|
1048
|
+
const source = sourceOf(`https://nvd.nist.gov/vuln/detail/${encodeURIComponent(id)}`, id, description, str(cve.published));
|
|
1049
|
+
if (source !== undefined)
|
|
1050
|
+
sources.push(source);
|
|
1051
|
+
if (sources.length >= limit)
|
|
1052
|
+
break;
|
|
1053
|
+
}
|
|
1054
|
+
return sources;
|
|
1055
|
+
}
|
|
1056
|
+
/**
|
|
1057
|
+
* Parse CoinGecko's coin search envelope.
|
|
1058
|
+
* @param payload - `api.coingecko.com/api/v3/search` JSON.
|
|
1059
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
1060
|
+
* @param limit - rows to keep (the endpoint sends no row cap; rows are capped).
|
|
1061
|
+
* @returns coins as sources.
|
|
1062
|
+
*/
|
|
1063
|
+
export function parseCoinGecko(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
1064
|
+
const body = parseJson(payload);
|
|
1065
|
+
if (!isRecord(body) || !Array.isArray(body.coins))
|
|
1066
|
+
return [];
|
|
1067
|
+
const sources = [];
|
|
1068
|
+
for (const row of body.coins) {
|
|
1069
|
+
if (!isRecord(row))
|
|
1070
|
+
continue;
|
|
1071
|
+
const id = str(row.id);
|
|
1072
|
+
if (id.length === 0)
|
|
1073
|
+
continue;
|
|
1074
|
+
const symbol = str(row.symbol);
|
|
1075
|
+
const rank = num(row.market_cap_rank);
|
|
1076
|
+
const meta = [
|
|
1077
|
+
symbol.length > 0 ? `$${symbol}` : '',
|
|
1078
|
+
rank === undefined ? '' : `rank #${rank}`,
|
|
1079
|
+
].filter((value) => value.length > 0).join(' · ');
|
|
1080
|
+
const source = sourceOf(`https://www.coingecko.com/en/coins/${encodeURIComponent(id)}`, str(row.name) || id, meta);
|
|
1081
|
+
if (source !== undefined)
|
|
1082
|
+
sources.push(source);
|
|
1083
|
+
if (sources.length >= limit)
|
|
1084
|
+
break;
|
|
1085
|
+
}
|
|
1086
|
+
return sources;
|
|
1087
|
+
}
|
|
1088
|
+
/**
|
|
1089
|
+
* Parse the Internet Archive's advanced-search envelope.
|
|
1090
|
+
* @param payload - `archive.org/advancedsearch.php` JSON.
|
|
1091
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
1092
|
+
* @param limit - rows to keep.
|
|
1093
|
+
* @returns archive items as sources.
|
|
1094
|
+
*/
|
|
1095
|
+
export function parseInternetarchive(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
1096
|
+
const body = parseJson(payload);
|
|
1097
|
+
if (!isRecord(body) || !isRecord(body.response) || !Array.isArray(body.response.docs))
|
|
1098
|
+
return [];
|
|
1099
|
+
const sources = [];
|
|
1100
|
+
for (const row of body.response.docs) {
|
|
1101
|
+
if (!isRecord(row))
|
|
1102
|
+
continue;
|
|
1103
|
+
const identifier = scalarText(row.identifier);
|
|
1104
|
+
if (identifier.length === 0)
|
|
1105
|
+
continue;
|
|
1106
|
+
const source = sourceOf(`https://archive.org/details/${encodeURIComponent(identifier)}`, joinedText(row.title) || identifier, joinedText(row.description), scalarText(row.year));
|
|
1107
|
+
if (source !== undefined)
|
|
1108
|
+
sources.push(source);
|
|
1109
|
+
if (sources.length >= limit)
|
|
1110
|
+
break;
|
|
1111
|
+
}
|
|
1112
|
+
return sources;
|
|
1113
|
+
}
|
|
1114
|
+
/**
|
|
1115
|
+
* Parse Wikidata's entity search envelope.
|
|
1116
|
+
* @param payload - `www.wikidata.org/w/api.php?action=wbsearchentities` JSON.
|
|
1117
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
1118
|
+
* @param limit - rows to keep.
|
|
1119
|
+
* @returns entities as sources.
|
|
1120
|
+
*/
|
|
1121
|
+
export function parseWikidata(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
1122
|
+
const body = parseJson(payload);
|
|
1123
|
+
if (!isRecord(body) || !Array.isArray(body.search))
|
|
1124
|
+
return [];
|
|
1125
|
+
const sources = [];
|
|
1126
|
+
for (const row of body.search) {
|
|
1127
|
+
if (!isRecord(row))
|
|
1128
|
+
continue;
|
|
1129
|
+
const id = str(row.id);
|
|
1130
|
+
if (id.length === 0)
|
|
1131
|
+
continue;
|
|
1132
|
+
let page = str(row.url);
|
|
1133
|
+
if (page.startsWith('//'))
|
|
1134
|
+
page = `https:${page}`;
|
|
1135
|
+
const concept = str(row.concepturi).replace(/^http:/i, 'https:');
|
|
1136
|
+
const source = sourceOf(page || concept || `https://www.wikidata.org/wiki/${encodeURIComponent(id)}`, str(row.label) || id, str(row.description));
|
|
1137
|
+
if (source !== undefined)
|
|
1138
|
+
sources.push(source);
|
|
1139
|
+
if (sources.length >= limit)
|
|
1140
|
+
break;
|
|
1141
|
+
}
|
|
1142
|
+
return sources;
|
|
1143
|
+
}
|
|
1144
|
+
/**
|
|
1145
|
+
* Parse Nominatim's place search envelope.
|
|
1146
|
+
* @param payload - `nominatim.openstreetmap.org/search?format=jsonv2` JSON array.
|
|
1147
|
+
* @param _url - request URL, unused by this JSON shape.
|
|
1148
|
+
* @param limit - rows to keep.
|
|
1149
|
+
* @returns places as sources.
|
|
1150
|
+
*/
|
|
1151
|
+
export function parseNominatim(payload, _url, limit = MAX_VERTICAL_LIMIT) {
|
|
1152
|
+
const body = parseJson(payload);
|
|
1153
|
+
if (!Array.isArray(body))
|
|
1154
|
+
return [];
|
|
1155
|
+
const sources = [];
|
|
1156
|
+
for (const row of body) {
|
|
1157
|
+
if (!isRecord(row))
|
|
1158
|
+
continue;
|
|
1159
|
+
const osmType = str(row.osm_type);
|
|
1160
|
+
const osmId = scalarText(row.osm_id);
|
|
1161
|
+
if (osmType.length === 0 || osmId.length === 0)
|
|
1162
|
+
continue;
|
|
1163
|
+
const display = str(row.display_name);
|
|
1164
|
+
const first = display.split(',')[0]?.trim() ?? '';
|
|
1165
|
+
const source = sourceOf(`https://www.openstreetmap.org/${encodeURIComponent(osmType)}/${encodeURIComponent(osmId)}`, str(row.name) || first || osmId, display || [str(row.category), str(row.type)].filter((value) => value.length > 0).join('/'));
|
|
1166
|
+
if (source !== undefined)
|
|
1167
|
+
sources.push(source);
|
|
1168
|
+
if (sources.length >= limit)
|
|
1169
|
+
break;
|
|
1170
|
+
}
|
|
1171
|
+
return sources;
|
|
1172
|
+
}
|
|
1173
|
+
/** A source id that this module actually knows. */
|
|
1174
|
+
export function isVerticalId(value) {
|
|
1175
|
+
return typeof value === 'string' && VERTICAL_IDS.includes(value);
|
|
1176
|
+
}
|
|
1177
|
+
/** Every adapter, keyed by id. */
|
|
1178
|
+
export const VERTICALS = [
|
|
1179
|
+
{
|
|
1180
|
+
id: 'github',
|
|
1181
|
+
label: 'GitHub',
|
|
1182
|
+
accept: 'application/vnd.github+json',
|
|
1183
|
+
headers: { 'x-github-api-version': '2022-11-28' },
|
|
1184
|
+
defaultOn: true,
|
|
1185
|
+
build: (query, limit) => `https://api.github.com/search/repositories?q=${encodeURIComponent(query)}&per_page=${limit}`,
|
|
1186
|
+
parse: parseGithub,
|
|
1187
|
+
},
|
|
1188
|
+
{
|
|
1189
|
+
id: 'npm',
|
|
1190
|
+
label: 'npm',
|
|
1191
|
+
accept: 'application/json',
|
|
1192
|
+
defaultOn: true,
|
|
1193
|
+
build: (query, limit) => `https://registry.npmjs.org/-/v1/search?text=${encodeURIComponent(query)}&size=${limit}`,
|
|
1194
|
+
parse: parseNpm,
|
|
1195
|
+
},
|
|
1196
|
+
{
|
|
1197
|
+
id: 'pypi',
|
|
1198
|
+
label: 'PyPI',
|
|
1199
|
+
accept: 'application/json',
|
|
1200
|
+
notFoundIsEmpty: true,
|
|
1201
|
+
defaultOn: true,
|
|
1202
|
+
// PyPI has no credential-free search API. The first token is the package
|
|
1203
|
+
// name; a descriptive query (`requests http`) still resolves `requests`.
|
|
1204
|
+
build: (query, _limit) => {
|
|
1205
|
+
const name = query.trim().split(/\s+/)[0] ?? query.trim();
|
|
1206
|
+
return `https://pypi.org/pypi/${encodeURIComponent(name)}/json`;
|
|
1207
|
+
},
|
|
1208
|
+
parse: parsePypi,
|
|
1209
|
+
},
|
|
1210
|
+
{
|
|
1211
|
+
id: 'arxiv',
|
|
1212
|
+
label: 'arXiv',
|
|
1213
|
+
accept: 'application/atom+xml',
|
|
1214
|
+
defaultOn: true,
|
|
1215
|
+
build: (query, limit) => `https://export.arxiv.org/api/query?search_query=all:${encodeURIComponent(query)}&start=0&max_results=${limit}`,
|
|
1216
|
+
parse: parseArxiv,
|
|
1217
|
+
},
|
|
1218
|
+
{
|
|
1219
|
+
id: 'wikipedia',
|
|
1220
|
+
label: 'Wikipedia',
|
|
1221
|
+
accept: 'application/json',
|
|
1222
|
+
defaultOn: true,
|
|
1223
|
+
build: (query, limit) => {
|
|
1224
|
+
const language = CJK.test(query) ? 'zh' : 'en';
|
|
1225
|
+
return `https://${language}.wikipedia.org/w/rest.php/v1/search/page?q=${encodeURIComponent(query)}&limit=${limit}`;
|
|
1226
|
+
},
|
|
1227
|
+
parse: parseWikipedia,
|
|
1228
|
+
},
|
|
1229
|
+
{
|
|
1230
|
+
id: 'hn',
|
|
1231
|
+
label: 'Hacker News',
|
|
1232
|
+
accept: 'application/json',
|
|
1233
|
+
defaultOn: true,
|
|
1234
|
+
build: (query, limit) => `https://hn.algolia.com/api/v1/search?query=${encodeURIComponent(query)}&hitsPerPage=${limit}`,
|
|
1235
|
+
parse: parseHn,
|
|
1236
|
+
},
|
|
1237
|
+
{
|
|
1238
|
+
id: 'stackexchange',
|
|
1239
|
+
label: 'Stack Exchange',
|
|
1240
|
+
accept: 'application/json',
|
|
1241
|
+
defaultOn: true,
|
|
1242
|
+
build: (query, limit) => `https://api.stackexchange.com/2.3/search/advanced?order=desc&sort=relevance&q=${encodeURIComponent(query)}&site=stackoverflow&pagesize=${limit}&filter=withbody`,
|
|
1243
|
+
parse: parseStackExchange,
|
|
1244
|
+
},
|
|
1245
|
+
{
|
|
1246
|
+
id: 'crates',
|
|
1247
|
+
label: 'crates.io',
|
|
1248
|
+
accept: 'application/json',
|
|
1249
|
+
defaultOn: true,
|
|
1250
|
+
build: (query, limit) => `https://crates.io/api/v1/crates?q=${encodeURIComponent(query)}&per_page=${limit}`,
|
|
1251
|
+
parse: parseCrates,
|
|
1252
|
+
},
|
|
1253
|
+
// Code hosts: repositories, issues and merge requests.
|
|
1254
|
+
{
|
|
1255
|
+
id: 'githubissues',
|
|
1256
|
+
label: 'GitHub issues',
|
|
1257
|
+
accept: 'application/vnd.github+json',
|
|
1258
|
+
headers: { 'x-github-api-version': '2022-11-28' },
|
|
1259
|
+
defaultOn: false,
|
|
1260
|
+
build: (query, limit) => `https://api.github.com/search/issues?q=${encodeURIComponent(query)}&per_page=${limit}`,
|
|
1261
|
+
parse: parseGithubIssues,
|
|
1262
|
+
},
|
|
1263
|
+
{
|
|
1264
|
+
id: 'gitlab',
|
|
1265
|
+
label: 'GitLab',
|
|
1266
|
+
accept: 'application/json',
|
|
1267
|
+
defaultOn: false,
|
|
1268
|
+
build: (query, limit) => `https://gitlab.com/api/v4/projects?search=${encodeURIComponent(query)}&per_page=${limit}`,
|
|
1269
|
+
parse: parseGitlab,
|
|
1270
|
+
},
|
|
1271
|
+
{
|
|
1272
|
+
id: 'codeberg',
|
|
1273
|
+
label: 'Codeberg',
|
|
1274
|
+
accept: 'application/json',
|
|
1275
|
+
defaultOn: false,
|
|
1276
|
+
build: (query, limit) => `https://codeberg.org/api/v1/repos/search?q=${encodeURIComponent(query)}&limit=${limit}`,
|
|
1277
|
+
parse: parseCodeberg,
|
|
1278
|
+
},
|
|
1279
|
+
// Package, container, model and extension registries.
|
|
1280
|
+
{
|
|
1281
|
+
id: 'dockerhub',
|
|
1282
|
+
label: 'Docker Hub',
|
|
1283
|
+
accept: 'application/json',
|
|
1284
|
+
defaultOn: false,
|
|
1285
|
+
build: (query, limit) => `https://hub.docker.com/v2/search/repositories/?query=${encodeURIComponent(query)}&page_size=${limit}`,
|
|
1286
|
+
parse: parseDockerHub,
|
|
1287
|
+
},
|
|
1288
|
+
{
|
|
1289
|
+
id: 'maven',
|
|
1290
|
+
label: 'Maven Central',
|
|
1291
|
+
accept: 'application/json',
|
|
1292
|
+
defaultOn: false,
|
|
1293
|
+
build: (query, limit) => `https://search.maven.org/solrsearch/select?q=${encodeURIComponent(query)}&rows=${limit}&wt=json`,
|
|
1294
|
+
parse: parseMaven,
|
|
1295
|
+
},
|
|
1296
|
+
{
|
|
1297
|
+
id: 'nuget',
|
|
1298
|
+
label: 'NuGet',
|
|
1299
|
+
accept: 'application/json',
|
|
1300
|
+
defaultOn: false,
|
|
1301
|
+
build: (query, limit) => `https://azuresearch-usnc.nuget.org/query?q=${encodeURIComponent(query)}&take=${limit}`,
|
|
1302
|
+
parse: parseNuget,
|
|
1303
|
+
},
|
|
1304
|
+
{
|
|
1305
|
+
id: 'rubygems',
|
|
1306
|
+
label: 'RubyGems',
|
|
1307
|
+
accept: 'application/json',
|
|
1308
|
+
defaultOn: false,
|
|
1309
|
+
// RubyGems answers a fixed page; the parser caps it at `limit`.
|
|
1310
|
+
build: (query) => `https://rubygems.org/api/v1/search.json?query=${encodeURIComponent(query)}`,
|
|
1311
|
+
parse: parseRubyGems,
|
|
1312
|
+
},
|
|
1313
|
+
{
|
|
1314
|
+
id: 'packagist',
|
|
1315
|
+
label: 'Packagist',
|
|
1316
|
+
accept: 'application/json',
|
|
1317
|
+
defaultOn: false,
|
|
1318
|
+
// Packagist answers a fixed page; the parser caps it at `limit`.
|
|
1319
|
+
build: (query) => `https://packagist.org/search.json?q=${encodeURIComponent(query)}`,
|
|
1320
|
+
parse: parsePackagist,
|
|
1321
|
+
},
|
|
1322
|
+
{
|
|
1323
|
+
id: 'pubdev',
|
|
1324
|
+
label: 'pub.dev',
|
|
1325
|
+
accept: 'application/json',
|
|
1326
|
+
defaultOn: false,
|
|
1327
|
+
// pub.dev answers a fixed page; the parser caps it at `limit`.
|
|
1328
|
+
build: (query) => `https://pub.dev/api/search?q=${encodeURIComponent(query)}`,
|
|
1329
|
+
parse: parsePubDev,
|
|
1330
|
+
},
|
|
1331
|
+
{
|
|
1332
|
+
id: 'huggingface',
|
|
1333
|
+
label: 'Hugging Face models',
|
|
1334
|
+
accept: 'application/json',
|
|
1335
|
+
defaultOn: false,
|
|
1336
|
+
build: (query, limit) => `https://huggingface.co/api/models?search=${encodeURIComponent(query)}&limit=${limit}`,
|
|
1337
|
+
parse: parseHuggingFaceModels,
|
|
1338
|
+
},
|
|
1339
|
+
{
|
|
1340
|
+
id: 'hfdatasets',
|
|
1341
|
+
label: 'Hugging Face datasets',
|
|
1342
|
+
accept: 'application/json',
|
|
1343
|
+
defaultOn: false,
|
|
1344
|
+
build: (query, limit) => `https://huggingface.co/api/datasets?search=${encodeURIComponent(query)}&limit=${limit}`,
|
|
1345
|
+
parse: parseHuggingFaceDatasets,
|
|
1346
|
+
},
|
|
1347
|
+
{
|
|
1348
|
+
id: 'openvsx',
|
|
1349
|
+
label: 'Open VSX',
|
|
1350
|
+
accept: 'application/json',
|
|
1351
|
+
defaultOn: false,
|
|
1352
|
+
build: (query, limit) => `https://open-vsx.org/api/-/search?query=${encodeURIComponent(query)}&size=${limit}`,
|
|
1353
|
+
parse: parseOpenVsx,
|
|
1354
|
+
},
|
|
1355
|
+
{
|
|
1356
|
+
id: 'terraform',
|
|
1357
|
+
label: 'Terraform Registry',
|
|
1358
|
+
accept: 'application/json',
|
|
1359
|
+
defaultOn: false,
|
|
1360
|
+
build: (query, limit) => `https://registry.terraform.io/v1/modules?q=${encodeURIComponent(query)}&limit=${limit}`,
|
|
1361
|
+
parse: parseTerraformModules,
|
|
1362
|
+
},
|
|
1363
|
+
{
|
|
1364
|
+
id: 'terraformproviders',
|
|
1365
|
+
label: 'Terraform providers',
|
|
1366
|
+
accept: 'application/json',
|
|
1367
|
+
defaultOn: false,
|
|
1368
|
+
build: (query, limit) => `https://registry.terraform.io/v1/providers?q=${encodeURIComponent(query)}&limit=${limit}`,
|
|
1369
|
+
parse: parseTerraformProviders,
|
|
1370
|
+
},
|
|
1371
|
+
{
|
|
1372
|
+
id: 'wordpress',
|
|
1373
|
+
label: 'WordPress plugins',
|
|
1374
|
+
accept: 'application/json',
|
|
1375
|
+
defaultOn: false,
|
|
1376
|
+
build: (query, limit) => `https://api.wordpress.org/plugins/info/1.2/?action=query_plugins&request%5Bsearch%5D=${encodeURIComponent(query)}&request%5Bper_page%5D=${limit}`,
|
|
1377
|
+
parse: parseWordpress,
|
|
1378
|
+
},
|
|
1379
|
+
{
|
|
1380
|
+
id: 'jetbrains',
|
|
1381
|
+
label: 'JetBrains Marketplace',
|
|
1382
|
+
accept: 'application/json',
|
|
1383
|
+
defaultOn: false,
|
|
1384
|
+
build: (query, limit) => `https://plugins.jetbrains.com/api/searchPlugins?search=${encodeURIComponent(query)}&max=${limit}`,
|
|
1385
|
+
parse: parseJetbrains,
|
|
1386
|
+
},
|
|
1387
|
+
// Papers and life-science records.
|
|
1388
|
+
{
|
|
1389
|
+
id: 'openalex',
|
|
1390
|
+
label: 'OpenAlex',
|
|
1391
|
+
accept: 'application/json',
|
|
1392
|
+
defaultOn: false,
|
|
1393
|
+
// `select` keeps the response to the fields the parser reads.
|
|
1394
|
+
build: (query, limit) => `https://api.openalex.org/works?search=${encodeURIComponent(query)}&per-page=${limit}&select=id,display_name,doi,publication_year,publication_date,type,primary_location,abstract_inverted_index`,
|
|
1395
|
+
parse: parseOpenAlex,
|
|
1396
|
+
},
|
|
1397
|
+
{
|
|
1398
|
+
id: 'crossref',
|
|
1399
|
+
label: 'Crossref',
|
|
1400
|
+
accept: 'application/json',
|
|
1401
|
+
defaultOn: false,
|
|
1402
|
+
build: (query, limit) => `https://api.crossref.org/works?query=${encodeURIComponent(query)}&rows=${limit}`,
|
|
1403
|
+
parse: parseCrossref,
|
|
1404
|
+
},
|
|
1405
|
+
{
|
|
1406
|
+
id: 'europepmc',
|
|
1407
|
+
label: 'Europe PMC',
|
|
1408
|
+
accept: 'application/json',
|
|
1409
|
+
defaultOn: false,
|
|
1410
|
+
build: (query, limit) => `https://www.ebi.ac.uk/europepmc/webservices/rest/search?query=${encodeURIComponent(query)}&format=json&pageSize=${limit}`,
|
|
1411
|
+
parse: parseEuropepmc,
|
|
1412
|
+
},
|
|
1413
|
+
{
|
|
1414
|
+
id: 'uniprot',
|
|
1415
|
+
label: 'UniProt',
|
|
1416
|
+
accept: 'application/json',
|
|
1417
|
+
defaultOn: false,
|
|
1418
|
+
// `fields` keeps the response to the fields the parser reads.
|
|
1419
|
+
build: (query, limit) => `https://rest.uniprot.org/uniprotkb/search?query=${encodeURIComponent(query)}&format=json&size=${limit}&fields=accession,id,protein_name,organism_name,length`,
|
|
1420
|
+
parse: parseUniprot,
|
|
1421
|
+
},
|
|
1422
|
+
{
|
|
1423
|
+
id: 'clinicaltrials',
|
|
1424
|
+
label: 'ClinicalTrials.gov',
|
|
1425
|
+
accept: 'application/json',
|
|
1426
|
+
defaultOn: false,
|
|
1427
|
+
// `fields` keeps the response to the fields the parser reads.
|
|
1428
|
+
build: (query, limit) => `https://clinicaltrials.gov/api/v2/studies?query.term=${encodeURIComponent(query)}&pageSize=${limit}&fields=NCTId,BriefTitle,BriefSummary,OverallStatus,StartDate`,
|
|
1429
|
+
parse: parseClinicaltrials,
|
|
1430
|
+
},
|
|
1431
|
+
// Structured data: vulnerabilities, coins, archive, entities and places.
|
|
1432
|
+
{
|
|
1433
|
+
id: 'nvd',
|
|
1434
|
+
label: 'NVD',
|
|
1435
|
+
accept: 'application/json',
|
|
1436
|
+
defaultOn: false,
|
|
1437
|
+
build: (query, limit) => `https://services.nvd.nist.gov/rest/json/cves/2.0?keywordSearch=${encodeURIComponent(query)}&resultsPerPage=${limit}`,
|
|
1438
|
+
parse: parseNvd,
|
|
1439
|
+
},
|
|
1440
|
+
{
|
|
1441
|
+
id: 'coingecko',
|
|
1442
|
+
label: 'CoinGecko',
|
|
1443
|
+
accept: 'application/json',
|
|
1444
|
+
defaultOn: false,
|
|
1445
|
+
// CoinGecko has no row cap; the parser caps the answer at `limit`.
|
|
1446
|
+
build: (query) => `https://api.coingecko.com/api/v3/search?query=${encodeURIComponent(query)}`,
|
|
1447
|
+
parse: parseCoinGecko,
|
|
1448
|
+
},
|
|
1449
|
+
{
|
|
1450
|
+
id: 'internetarchive',
|
|
1451
|
+
label: 'Internet Archive',
|
|
1452
|
+
accept: 'application/json',
|
|
1453
|
+
defaultOn: false,
|
|
1454
|
+
build: (query, limit) => `https://archive.org/advancedsearch.php?q=${encodeURIComponent(query)}&fl%5B%5D=identifier&fl%5B%5D=title&fl%5B%5D=description&fl%5B%5D=year&rows=${limit}&output=json`,
|
|
1455
|
+
parse: parseInternetarchive,
|
|
1456
|
+
},
|
|
1457
|
+
{
|
|
1458
|
+
id: 'wikidata',
|
|
1459
|
+
label: 'Wikidata',
|
|
1460
|
+
accept: 'application/json',
|
|
1461
|
+
defaultOn: false,
|
|
1462
|
+
build: (query, limit) => {
|
|
1463
|
+
const language = CJK.test(query) ? 'zh' : 'en';
|
|
1464
|
+
return `https://www.wikidata.org/w/api.php?action=wbsearchentities&search=${encodeURIComponent(query)}&language=${language}&format=json&limit=${limit}`;
|
|
1465
|
+
},
|
|
1466
|
+
parse: parseWikidata,
|
|
1467
|
+
},
|
|
1468
|
+
{
|
|
1469
|
+
id: 'nominatim',
|
|
1470
|
+
label: 'OpenStreetMap Nominatim',
|
|
1471
|
+
accept: 'application/json',
|
|
1472
|
+
defaultOn: false,
|
|
1473
|
+
build: (query, limit) => `https://nominatim.openstreetmap.org/search?q=${encodeURIComponent(query)}&format=jsonv2&limit=${limit}`,
|
|
1474
|
+
parse: parseNominatim,
|
|
1475
|
+
},
|
|
1476
|
+
];
|
|
1477
|
+
/** One adapter by id, or `undefined` for a typo. */
|
|
1478
|
+
export function verticalById(id) {
|
|
1479
|
+
return VERTICALS.find((spec) => spec.id === id);
|
|
1480
|
+
}
|
|
1481
|
+
/**
|
|
1482
|
+
* Sources queried when the caller names none.
|
|
1483
|
+
*
|
|
1484
|
+
* The original eight answer by default; every expanded source is
|
|
1485
|
+
* `defaultOn: false`. Fanning a bare tool call out to all of
|
|
1486
|
+
* {@link VERTICAL_IDS} would spend the whole GitHub unauthenticated search
|
|
1487
|
+
* budget in one request and make the common case slow, so participation is
|
|
1488
|
+
* declared per spec rather than implied by membership.
|
|
1489
|
+
*/
|
|
1490
|
+
export const DEFAULT_VERTICAL_IDS = VERTICAL_IDS.filter((id) => verticalById(id)?.defaultOn !== false);
|
|
387
1491
|
/** Classify a non-2xx status into the shared business codes. */
|
|
388
1492
|
function statusError(status) {
|
|
389
1493
|
if (status === 429)
|
|
@@ -490,7 +1594,10 @@ export async function searchVerticals(query, options = {}) {
|
|
|
490
1594
|
? options.timeoutMs
|
|
491
1595
|
: 8000;
|
|
492
1596
|
const requested = (options.ids ?? []).filter(isVerticalId);
|
|
493
|
-
const
|
|
1597
|
+
const selected = requested.length > 0
|
|
1598
|
+
? VERTICAL_IDS.filter((id) => requested.includes(id))
|
|
1599
|
+
: DEFAULT_VERTICAL_IDS;
|
|
1600
|
+
const specs = selected
|
|
494
1601
|
.map((id) => verticalById(id))
|
|
495
1602
|
.filter((spec) => spec !== undefined);
|
|
496
1603
|
const answers = await Promise.all(specs.map(async (spec) => {
|