@michaelborck/honeyguide-core 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/align/outcomes.d.ts +75 -0
- package/dist/align/outcomes.js +266 -0
- package/dist/align/outcomes.js.map +1 -0
- package/dist/align/reasons.d.ts +11 -0
- package/dist/align/reasons.js +84 -0
- package/dist/align/reasons.js.map +1 -0
- package/dist/annotate/ladder.d.ts +1 -1
- package/dist/annotate/ladder.js +2 -2
- package/dist/annotate/ladder.js.map +1 -1
- package/dist/annotate/ollama.d.ts +3 -1
- package/dist/annotate/ollama.js +17 -11
- package/dist/annotate/ollama.js.map +1 -1
- package/dist/batch.d.ts +0 -0
- package/dist/batch.js +0 -0
- package/dist/batch.js.map +0 -0
- package/dist/brief.d.ts +30 -0
- package/dist/brief.js +70 -0
- package/dist/brief.js.map +1 -0
- package/dist/collection.d.ts +27 -0
- package/dist/collection.js +98 -0
- package/dist/collection.js.map +1 -0
- package/dist/config.d.ts +31 -1
- package/dist/config.js +10 -0
- package/dist/config.js.map +1 -1
- package/dist/enrich/page.d.ts +0 -0
- package/dist/enrich/page.js +0 -0
- package/dist/enrich/page.js.map +0 -0
- package/dist/enrich/sourceType.d.ts +0 -0
- package/dist/enrich/sourceType.js +0 -0
- package/dist/enrich/sourceType.js.map +0 -0
- package/dist/enrich/unpaywall.d.ts +28 -0
- package/dist/enrich/unpaywall.js +36 -0
- package/dist/enrich/unpaywall.js.map +1 -0
- package/dist/export/citations.d.ts +0 -0
- package/dist/export/citations.js +0 -0
- package/dist/export/citations.js.map +0 -0
- package/dist/export/lmsHtml.d.ts +0 -0
- package/dist/export/lmsHtml.js +16 -2
- package/dist/export/lmsHtml.js.map +1 -1
- package/dist/export/markdown.d.ts +0 -0
- package/dist/export/markdown.js +11 -3
- package/dist/export/markdown.js.map +1 -1
- package/dist/index.d.ts +18 -1
- package/dist/index.js +17 -1
- package/dist/index.js.map +1 -1
- package/dist/pipeline.d.ts +2 -2
- package/dist/pipeline.js +117 -12
- package/dist/pipeline.js.map +1 -1
- package/dist/rank/decision.d.ts +59 -0
- package/dist/rank/decision.js +124 -0
- package/dist/rank/decision.js.map +1 -0
- package/dist/rank/diversity.d.ts +0 -0
- package/dist/rank/diversity.js +0 -0
- package/dist/rank/diversity.js.map +0 -0
- package/dist/rank/embeddings.d.ts +41 -0
- package/dist/rank/embeddings.js +98 -0
- package/dist/rank/embeddings.js.map +1 -0
- package/dist/rank/reranker.d.ts +21 -0
- package/dist/rank/reranker.js +30 -0
- package/dist/rank/reranker.js.map +1 -0
- package/dist/rank/semantic.d.ts +14 -0
- package/dist/rank/semantic.js +58 -0
- package/dist/rank/semantic.js.map +1 -0
- package/dist/recheck.d.ts +0 -0
- package/dist/recheck.js +0 -0
- package/dist/recheck.js.map +0 -0
- package/dist/retrieval/arxiv.d.ts +24 -0
- package/dist/retrieval/arxiv.js +94 -0
- package/dist/retrieval/arxiv.js.map +1 -0
- package/dist/retrieval/books.d.ts +0 -0
- package/dist/retrieval/books.js +0 -0
- package/dist/retrieval/books.js.map +0 -0
- package/dist/retrieval/breaker.d.ts +16 -0
- package/dist/retrieval/breaker.js +42 -0
- package/dist/retrieval/breaker.js.map +1 -0
- package/dist/retrieval/cache.d.ts +8 -0
- package/dist/retrieval/cache.js +33 -0
- package/dist/retrieval/cache.js.map +1 -0
- package/dist/retrieval/candidates.d.ts +1 -1
- package/dist/retrieval/candidates.js +31 -7
- package/dist/retrieval/candidates.js.map +1 -1
- package/dist/retrieval/counterpoint.d.ts +0 -0
- package/dist/retrieval/counterpoint.js +1 -0
- package/dist/retrieval/counterpoint.js.map +1 -1
- package/dist/retrieval/europePmc.d.ts +32 -0
- package/dist/retrieval/europePmc.js +62 -0
- package/dist/retrieval/europePmc.js.map +1 -0
- package/dist/retrieval/openalexTopical.d.ts +0 -0
- package/dist/retrieval/openalexTopical.js +0 -0
- package/dist/retrieval/openalexTopical.js.map +0 -0
- package/dist/retrieval/paperMerge.d.ts +16 -0
- package/dist/retrieval/paperMerge.js +77 -0
- package/dist/retrieval/paperMerge.js.map +1 -0
- package/dist/retrieval/papers.d.ts +10 -5
- package/dist/retrieval/papers.js +51 -32
- package/dist/retrieval/papers.js.map +1 -1
- package/dist/retrieval/podcasts.d.ts +0 -0
- package/dist/retrieval/podcasts.js +0 -0
- package/dist/retrieval/podcasts.js.map +0 -0
- package/dist/retrieval/query.d.ts +0 -0
- package/dist/retrieval/query.js +1 -0
- package/dist/retrieval/query.js.map +1 -1
- package/dist/retrieval/rateLimit.d.ts +6 -0
- package/dist/retrieval/rateLimit.js +15 -0
- package/dist/retrieval/rateLimit.js.map +1 -0
- package/dist/retrieval/search/brave.d.ts +13 -0
- package/dist/retrieval/search/brave.js +32 -0
- package/dist/retrieval/search/brave.js.map +1 -0
- package/dist/retrieval/search/index.d.ts +13 -0
- package/dist/retrieval/search/index.js +22 -0
- package/dist/retrieval/search/index.js.map +1 -0
- package/dist/retrieval/search/searxng.d.ts +14 -0
- package/dist/retrieval/search/searxng.js +24 -0
- package/dist/retrieval/search/searxng.js.map +1 -0
- package/dist/retrieval/search/serper.d.ts +14 -0
- package/dist/retrieval/search/serper.js +38 -0
- package/dist/retrieval/search/serper.js.map +1 -0
- package/dist/retrieval/search/types.d.ts +17 -0
- package/dist/retrieval/search/types.js +2 -0
- package/dist/retrieval/search/types.js.map +1 -0
- package/dist/retrieval/videos.d.ts +0 -0
- package/dist/retrieval/videos.js +0 -0
- package/dist/retrieval/videos.js.map +0 -0
- package/dist/retrieval/web.d.ts +5 -6
- package/dist/retrieval/web.js +14 -27
- package/dist/retrieval/web.js.map +1 -1
- package/dist/types.d.ts +62 -0
- package/dist/types.js +0 -0
- package/dist/types.js.map +0 -0
- package/dist/verify/verify.d.ts +11 -2
- package/dist/verify/verify.js +13 -4
- package/dist/verify/verify.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Canonical DOI form for matching: no resolver prefix, no `doi:` label,
|
|
3
|
+
* lower-case. Everything downstream compares on this.
|
|
4
|
+
*/
|
|
5
|
+
export function normalizeDoi(doi) {
|
|
6
|
+
if (!doi)
|
|
7
|
+
return undefined;
|
|
8
|
+
const cleaned = doi
|
|
9
|
+
.trim()
|
|
10
|
+
.replace(/^https?:\/\/(?:dx\.)?doi\.org\//i, '')
|
|
11
|
+
.replace(/^doi:\s*/i, '')
|
|
12
|
+
.trim();
|
|
13
|
+
return cleaned ? cleaned.toLowerCase() : undefined;
|
|
14
|
+
}
|
|
15
|
+
/** Title key for matching when a DOI is unavailable. */
|
|
16
|
+
export function normalizeTitle(title) {
|
|
17
|
+
return title
|
|
18
|
+
.toLowerCase()
|
|
19
|
+
.replace(/[^a-z0-9]+/g, ' ')
|
|
20
|
+
.trim()
|
|
21
|
+
.replace(/\s+/g, ' ');
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Merge paper candidates from several databases (ADR-0013). Duplicates are
|
|
25
|
+
* matched by normalised DOI first, then by normalised title, so a DOI-less
|
|
26
|
+
* arXiv record and a Crossref record for the same paper collapse into one.
|
|
27
|
+
* The first occurrence wins for conflicting fields; missing fields are filled
|
|
28
|
+
* from later duplicates.
|
|
29
|
+
*/
|
|
30
|
+
export function mergePaperCandidates(lists) {
|
|
31
|
+
const out = [];
|
|
32
|
+
const byDoi = new Map();
|
|
33
|
+
const byTitle = new Map();
|
|
34
|
+
for (const list of lists) {
|
|
35
|
+
for (const candidate of list) {
|
|
36
|
+
const titleKey = normalizeTitle(candidate.title);
|
|
37
|
+
if (!titleKey)
|
|
38
|
+
continue;
|
|
39
|
+
const doi = normalizeDoi(candidate.doi);
|
|
40
|
+
const existing = (doi ? byDoi.get(doi) : undefined) ?? byTitle.get(titleKey);
|
|
41
|
+
if (!existing) {
|
|
42
|
+
const copy = {
|
|
43
|
+
...candidate,
|
|
44
|
+
doi,
|
|
45
|
+
accessibilityNotes: candidate.accessibilityNotes
|
|
46
|
+
? [...candidate.accessibilityNotes]
|
|
47
|
+
: undefined,
|
|
48
|
+
};
|
|
49
|
+
out.push(copy);
|
|
50
|
+
byTitle.set(titleKey, copy);
|
|
51
|
+
if (doi)
|
|
52
|
+
byDoi.set(doi, copy);
|
|
53
|
+
continue;
|
|
54
|
+
}
|
|
55
|
+
mergeInto(existing, candidate, doi);
|
|
56
|
+
byTitle.set(titleKey, existing);
|
|
57
|
+
if (doi)
|
|
58
|
+
byDoi.set(doi, existing);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return out;
|
|
62
|
+
}
|
|
63
|
+
function mergeInto(target, extra, doi) {
|
|
64
|
+
if (!target.authors.length && extra.authors.length)
|
|
65
|
+
target.authors = extra.authors;
|
|
66
|
+
if (target.year == null && extra.year != null)
|
|
67
|
+
target.year = extra.year;
|
|
68
|
+
target.doi ??= doi ?? extra.doi;
|
|
69
|
+
target.venue ??= extra.venue;
|
|
70
|
+
target.licensing ??= extra.licensing;
|
|
71
|
+
target.description ??= extra.description;
|
|
72
|
+
target.thumbnailUrl ??= extra.thumbnailUrl;
|
|
73
|
+
if (!target.accessibilityNotes?.length && extra.accessibilityNotes?.length) {
|
|
74
|
+
target.accessibilityNotes = extra.accessibilityNotes;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
//# sourceMappingURL=paperMerge.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paperMerge.js","sourceRoot":"","sources":["../../src/retrieval/paperMerge.ts"],"names":[],"mappings":"AAEA;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAmB;IAC9C,IAAI,CAAC,GAAG;QAAE,OAAO,SAAS,CAAC;IAC3B,MAAM,OAAO,GAAG,GAAG;SAChB,IAAI,EAAE;SACN,OAAO,CAAC,kCAAkC,EAAE,EAAE,CAAC;SAC/C,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;SACxB,IAAI,EAAE,CAAC;IACV,OAAO,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;AACrD,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,cAAc,CAAC,KAAa;IAC1C,OAAO,KAAK;SACT,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,IAAI,EAAE;SACN,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAoB;IACvD,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC3C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAC;IAE7C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,MAAM,SAAS,IAAI,IAAI,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACjD,IAAI,CAAC,QAAQ;gBAAE,SAAS;YACxB,MAAM,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAExC,MAAM,QAAQ,GACZ,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,MAAM,IAAI,GAAc;oBACtB,GAAG,SAAS;oBACZ,GAAG;oBACH,kBAAkB,EAAE,SAAS,CAAC,kBAAkB;wBAC9C,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC,kBAAkB,CAAC;wBACnC,CAAC,CAAC,SAAS;iBACd,CAAC;gBACF,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAC5B,IAAI,GAAG;oBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC9B,SAAS;YACX,CAAC;YAED,SAAS,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAChC,IAAI,GAAG;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,SAAS,CAAC,MAAiB,EAAE,KAAgB,EAAE,GAAY;IAClE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM;QAAE,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC;IACnF,IAAI,MAAM,CAAC,IAAI,IAAI,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxE,MAAM,CAAC,GAAG,KAAK,GAAG,IAAI,KAAK,CAAC,GAAG,CAAC;IAChC,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,KAAK,CAAC;IAC7B,MAAM,CAAC,SAAS,KAAK,KAAK,CAAC,SAAS,CAAC;IACrC,MAAM,CAAC,WAAW,KAAK,KAAK,CAAC,WAAW,CAAC;IACzC,MAAM,CAAC,YAAY,KAAK,KAAK,CAAC,YAAY,CAAC;IAC3C,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,EAAE,MAAM,EAAE,CAAC;QAC3E,MAAM,CAAC,kBAAkB,GAAG,KAAK,CAAC,kBAAkB,CAAC;IACvD,CAAC;AACH,CAAC"}
|
|
@@ -1,8 +1,13 @@
|
|
|
1
1
|
import type { Candidate } from './candidates.js';
|
|
2
|
+
export interface PaperRetrievalOptions {
|
|
3
|
+
mailto?: string;
|
|
4
|
+
/** Lifts Semantic Scholar's keyless shared-pool throttling. */
|
|
5
|
+
semanticScholarApiKey?: string;
|
|
6
|
+
}
|
|
2
7
|
/**
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
8
|
+
* Multi-source paper discovery (ADR-0013). OpenAlex, Semantic Scholar, Europe
|
|
9
|
+
* PMC and arXiv run in parallel and are merged; Crossref is the last-resort
|
|
10
|
+
* fallback when every source returns nothing. One source failing never fails
|
|
11
|
+
* the request (ADR-0006), and results are database-attested by construction.
|
|
7
12
|
*/
|
|
8
|
-
export declare function retrievePapers(topic: string,
|
|
13
|
+
export declare function retrievePapers(topic: string, options?: PaperRetrievalOptions): Promise<Candidate[]>;
|
package/dist/retrieval/papers.js
CHANGED
|
@@ -1,47 +1,66 @@
|
|
|
1
|
-
import { searchCrossref } from '@michaelborck/cite-sight-core';
|
|
1
|
+
import { searchCrossref, searchSemanticScholar, } from '@michaelborck/cite-sight-core';
|
|
2
2
|
import { searchOpenAlexTopical } from './openalexTopical.js';
|
|
3
|
+
import { searchEuropePmc } from './europePmc.js';
|
|
4
|
+
import { searchArxiv } from './arxiv.js';
|
|
5
|
+
import { mergePaperCandidates, normalizeDoi } from './paperMerge.js';
|
|
3
6
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* OpenAlex is unreachable. Results are database-attested: existence is
|
|
7
|
-
* verified by construction, no separate URL probe needed.
|
|
7
|
+
* Bound each source so a slow or rate-limited API cannot dominate latency.
|
|
8
|
+
* Paper sources run in parallel, so the request costs roughly the slowest one.
|
|
8
9
|
*/
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
10
|
+
const SOURCE_TIMEOUT_MS = 5000;
|
|
11
|
+
function withTimeout(promise, ms, fallback) {
|
|
12
|
+
let timer;
|
|
13
|
+
const guarded = promise.catch(() => fallback);
|
|
14
|
+
const timeout = new Promise((resolve) => {
|
|
15
|
+
timer = setTimeout(() => resolve(fallback), ms);
|
|
16
|
+
});
|
|
17
|
+
return Promise.race([guarded, timeout]).finally(() => {
|
|
18
|
+
if (timer)
|
|
19
|
+
clearTimeout(timer);
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Multi-source paper discovery (ADR-0013). OpenAlex, Semantic Scholar, Europe
|
|
24
|
+
* PMC and arXiv run in parallel and are merged; Crossref is the last-resort
|
|
25
|
+
* fallback when every source returns nothing. One source failing never fails
|
|
26
|
+
* the request (ADR-0006), and results are database-attested by construction.
|
|
27
|
+
*/
|
|
28
|
+
export async function retrievePapers(topic, options = {}) {
|
|
29
|
+
const settled = await Promise.allSettled([
|
|
30
|
+
withTimeout(searchOpenAlexTopical(topic, options.mailto), SOURCE_TIMEOUT_MS, []),
|
|
31
|
+
withTimeout(searchSemanticScholar(topic, options.semanticScholarApiKey).then((works) => worksToCandidates(works, 'semantic-scholar')), SOURCE_TIMEOUT_MS, []),
|
|
32
|
+
withTimeout(searchEuropePmc(topic), SOURCE_TIMEOUT_MS, []),
|
|
33
|
+
withTimeout(searchArxiv(topic), SOURCE_TIMEOUT_MS, []),
|
|
34
|
+
]);
|
|
35
|
+
const lists = settled.flatMap((outcome) => outcome.status === 'fulfilled' ? [outcome.value] : []);
|
|
36
|
+
const merged = mergePaperCandidates(lists);
|
|
37
|
+
if (merged.length > 0)
|
|
38
|
+
return merged;
|
|
39
|
+
const works = await searchCrossref(topic, options.mailto);
|
|
40
|
+
return mergePaperCandidates([worksToCandidates(works, 'crossref')]);
|
|
41
|
+
}
|
|
42
|
+
function worksToCandidates(works, origin) {
|
|
43
|
+
return works.flatMap((work) => {
|
|
44
|
+
const title = work.title?.trim();
|
|
45
|
+
if (!title)
|
|
46
|
+
return [];
|
|
47
|
+
const doi = normalizeDoi(work.doi);
|
|
48
|
+
const url = (doi ? `https://doi.org/${doi}` : undefined) ?? work.url;
|
|
21
49
|
if (!url)
|
|
22
50
|
return [];
|
|
23
51
|
return [
|
|
24
52
|
{
|
|
25
|
-
title
|
|
53
|
+
title,
|
|
26
54
|
url,
|
|
27
55
|
format: 'paper',
|
|
28
|
-
authors: work.authors,
|
|
29
|
-
year: work.year,
|
|
30
|
-
doi
|
|
31
|
-
|
|
56
|
+
authors: work.authors ?? [],
|
|
57
|
+
year: work.year ?? null,
|
|
58
|
+
doi,
|
|
59
|
+
venue: work.journal,
|
|
60
|
+
origin,
|
|
32
61
|
databaseAttested: true,
|
|
33
62
|
},
|
|
34
63
|
];
|
|
35
|
-
}));
|
|
36
|
-
}
|
|
37
|
-
function dedupe(candidates) {
|
|
38
|
-
const seen = new Set();
|
|
39
|
-
return candidates.filter((c) => {
|
|
40
|
-
const key = (c.doi ?? c.title).toLowerCase();
|
|
41
|
-
if (seen.has(key))
|
|
42
|
-
return false;
|
|
43
|
-
seen.add(key);
|
|
44
|
-
return true;
|
|
45
64
|
});
|
|
46
65
|
}
|
|
47
66
|
//# sourceMappingURL=papers.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"papers.js","sourceRoot":"","sources":["../../src/retrieval/papers.ts"],"names":[],"mappings":"AAAA,OAAO,
|
|
1
|
+
{"version":3,"file":"papers.js","sourceRoot":"","sources":["../../src/retrieval/papers.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,qBAAqB,GAEtB,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AASrE;;;GAGG;AACH,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAE/B,SAAS,WAAW,CAAI,OAAmB,EAAE,EAAU,EAAE,QAAW;IAClE,IAAI,KAAgD,CAAC;IACrD,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,EAAE;QACzC,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IACH,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE;QACnD,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IACjC,CAAC,CAAC,CAAC;AACL,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,KAAa,EACb,UAAiC,EAAE;IAEnC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;QACvC,WAAW,CAAC,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,iBAAiB,EAAE,EAAiB,CAAC;QAC/F,WAAW,CACT,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CACzE,iBAAiB,CAAC,KAAK,EAAE,kBAAkB,CAAC,CAC7C,EACD,iBAAiB,EACjB,EAAiB,CAClB;QACD,WAAW,CAAC,eAAe,CAAC,KAAK,CAAC,EAAE,iBAAiB,EAAE,EAAiB,CAAC;QACzE,WAAW,CAAC,WAAW,CAAC,KAAK,CAAC,EAAE,iBAAiB,EAAE,EAAiB,CAAC;KACtE,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CACxC,OAAO,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CACtD,CAAC;IACF,MAAM,MAAM,GAAG,oBAAoB,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC;IAErC,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1D,OAAO,oBAAoB,CAAC,CAAC,iBAAiB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,iBAAiB,CACxB,KAAqB,EACrB,MAAuC;IAEvC,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAe,EAAE;QACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAC;QACtB,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC;QACrE,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,CAAC;QACpB,OAAO;YACL;gBACE,KAAK;gBACL,GAAG;gBACH,MAAM,EAAE,OAAO;gBACf,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;gBAC3B,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,IAAI;gBACvB,GAAG;gBACH,KAAK,EAAE,IAAI,CAAC,OAAO;gBACnB,MAAM;gBACN,gBAAgB,EAAE,IAAI;aACvB;SACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
package/dist/retrieval/query.js
CHANGED
|
@@ -40,6 +40,7 @@ async function llmKeywords(text, config) {
|
|
|
40
40
|
`Reply with a 3-8 word search query capturing its main topic, ` +
|
|
41
41
|
`suitable for an academic search engine. Reply with the query only.`,
|
|
42
42
|
stream: false,
|
|
43
|
+
think: false,
|
|
43
44
|
options: { num_predict: 30, temperature: 0.1 },
|
|
44
45
|
}),
|
|
45
46
|
signal: AbortSignal.timeout(QUERY_TIMEOUT_MS),
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/retrieval/query.ts"],"names":[],"mappings":"AAEA,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,CACE,+EAA+E;IAC/E,4EAA4E;IAC5E,iFAAiF;IACjF,8EAA8E;IAC9E,+EAA+E;IAC/E,yEAAyE,CAC1E,CAAC,KAAK,CAAC,GAAG,CAAC,CACb,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAa,EAAE,MAAsB;IACrE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAClD,IAAI,OAAO,CAAC,MAAM,IAAI,iBAAiB;QAAE,OAAO,OAAO,CAAC;IACxD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;IACH,CAAC;IACD,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,MAAsB;IAC7D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,SAAU,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,eAAe,EAAE;QAC9E,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnF;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,aAAa;YAC1C,MAAM,EACJ,iCAAiC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM;gBAC1D,+DAA+D;gBAC/D,oEAAoE;YACtE,MAAM,EAAE,KAAK;YACb,OAAO,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE;SAC/C,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,gBAAgB,CAAC;KAC9C,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA0B,CAAC;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,UAAU,GAAG,CAAC;IACzD,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,GAAG,EAAE,CAAC;QACf,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC/B,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AACpD,CAAC;AAED;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAClC,CACE,2EAA2E;IAC3E,8EAA8E;IAC9E,0EAA0E;IAC1E,gDAAgD,CACjD,CAAC,KAAK,CAAC,GAAG,CAAC,CACb,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,UAAU,GAAG,CAAC;IAC5D,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5F,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,IAAI,CAAC;YACf,CAAC,CAAC,CAAC,KAAK,CAAC;YACT,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5D,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAC9B,CAAC;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;aAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC;aACpB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;SACvB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;SACf,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC"}
|
|
1
|
+
{"version":3,"file":"query.js","sourceRoot":"","sources":["../../src/retrieval/query.ts"],"names":[],"mappings":"AAEA,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC,MAAM,SAAS,GAAG,IAAI,GAAG,CACvB,CACE,+EAA+E;IAC/E,4EAA4E;IAC5E,iFAAiF;IACjF,8EAA8E;IAC9E,+EAA+E;IAC/E,yEAAyE,CAC1E,CAAC,KAAK,CAAC,GAAG,CAAC,CACb,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAa,EAAE,MAAsB;IACrE,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAClD,IAAI,OAAO,CAAC,MAAM,IAAI,iBAAiB;QAAE,OAAO,OAAO,CAAC;IACxD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,MAAM,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,gCAAgC;QAClC,CAAC;IACH,CAAC;IACD,OAAO,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACpC,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAY,EAAE,MAAsB;IAC7D,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,MAAM,CAAC,SAAU,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,eAAe,EAAE;QAC9E,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,UAAU,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACnF;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YACnB,KAAK,EAAE,MAAM,CAAC,WAAW,IAAI,aAAa;YAC1C,MAAM,EACJ,iCAAiC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM;gBAC1D,+DAA+D;gBAC/D,oEAAoE;YACtE,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,KAAK;YACZ,OAAO,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE;SAC/C,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,gBAAgB,CAAC;KAC9C,CAAC,CAAC;IACH,IAAI,CAAC,GAAG,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA0B,CAAC;IACzD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/E,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAC7E,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,UAAU,GAAG,CAAC;IACzD,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,GAAG,IAAI,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,GAAG,EAAE,CAAC;QACf,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC/B,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7C,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;AACpD,CAAC;AAED;;;;GAIG;AACH,MAAM,oBAAoB,GAAG,IAAI,GAAG,CAClC,CACE,2EAA2E;IAC3E,8EAA8E;IAC9E,0EAA0E;IAC1E,gDAAgD,CACjD,CAAC,KAAK,CAAC,GAAG,CAAC,CACb,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY,EAAE,UAAU,GAAG,CAAC;IAC5D,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,oBAAoB,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE5F,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,MAAM,MAAM,GAAe,EAAE,CAAC;IAC9B,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,cAAc,CAAC,EAAE,CAAC;QAC3D,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,IAAI,OAAO,CAAC,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,IAAI,OAAO,CAAC,MAAM;QAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAC/B,MAAM,KAAK,GACT,KAAK,CAAC,MAAM,IAAI,CAAC;YACf,CAAC,CAAC,CAAC,KAAK,CAAC;YACT,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5D,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAC9B,CAAC;QACR,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACnE,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QAC3D,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;aAC1B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;aAC3B,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC;aACpB,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;aACf,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;SACvB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;SACX,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;SACf,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Minimal per-service pacer for the keyless academic APIs, which ask for
|
|
3
|
+
* polite request spacing (arXiv in particular). The first call for a service
|
|
4
|
+
* returns immediately.
|
|
5
|
+
*/
|
|
6
|
+
export declare function throttle(service: string, minIntervalMs: number): Promise<void>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const lastCall = new Map();
|
|
2
|
+
/**
|
|
3
|
+
* Minimal per-service pacer for the keyless academic APIs, which ask for
|
|
4
|
+
* polite request spacing (arXiv in particular). The first call for a service
|
|
5
|
+
* returns immediately.
|
|
6
|
+
*/
|
|
7
|
+
export async function throttle(service, minIntervalMs) {
|
|
8
|
+
const now = Date.now();
|
|
9
|
+
const previous = lastCall.get(service) ?? 0;
|
|
10
|
+
const wait = Math.max(0, previous + minIntervalMs - now);
|
|
11
|
+
lastCall.set(service, Math.max(now, previous + minIntervalMs));
|
|
12
|
+
if (wait > 0)
|
|
13
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=rateLimit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rateLimit.js","sourceRoot":"","sources":["../../src/retrieval/rateLimit.ts"],"names":[],"mappings":"AAAA,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;AAE3C;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,OAAe,EAAE,aAAqB;IACnE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,aAAa,GAAG,GAAG,CAAC,CAAC;IACzD,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,GAAG,aAAa,CAAC,CAAC,CAAC;IAC/D,IAAI,IAAI,GAAG,CAAC;QAAE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC;AAC1E,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { SearchProvider, WebSearchResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Brave Search API (ADR-0013): keyed fallback when SearXNG is unavailable or
|
|
4
|
+
* blocked. Requires `BRAVE_API_KEY`.
|
|
5
|
+
*/
|
|
6
|
+
export declare class BraveSearchProvider implements SearchProvider {
|
|
7
|
+
private readonly apiKey;
|
|
8
|
+
readonly name: "brave";
|
|
9
|
+
constructor(apiKey: string);
|
|
10
|
+
search(query: string, options?: {
|
|
11
|
+
limit?: number;
|
|
12
|
+
}): Promise<WebSearchResult[]>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { fetchJson } from '../candidates.js';
|
|
2
|
+
/**
|
|
3
|
+
* Brave Search API (ADR-0013): keyed fallback when SearXNG is unavailable or
|
|
4
|
+
* blocked. Requires `BRAVE_API_KEY`.
|
|
5
|
+
*/
|
|
6
|
+
export class BraveSearchProvider {
|
|
7
|
+
apiKey;
|
|
8
|
+
name = 'brave';
|
|
9
|
+
constructor(apiKey) {
|
|
10
|
+
this.apiKey = apiKey;
|
|
11
|
+
}
|
|
12
|
+
async search(query, options = {}) {
|
|
13
|
+
const count = clamp(options.limit ?? 8, 1, 20);
|
|
14
|
+
const url = 'https://api.search.brave.com/res/v1/web/search?' +
|
|
15
|
+
new URLSearchParams({
|
|
16
|
+
q: query,
|
|
17
|
+
count: String(count),
|
|
18
|
+
safesearch: 'strict',
|
|
19
|
+
}).toString();
|
|
20
|
+
const data = (await fetchJson(url, {
|
|
21
|
+
'x-subscription-token': this.apiKey,
|
|
22
|
+
}));
|
|
23
|
+
return (data.web?.results ?? [])
|
|
24
|
+
.filter((r) => Boolean(r.title && r.url))
|
|
25
|
+
.slice(0, count)
|
|
26
|
+
.map((r) => ({ title: r.title, url: r.url, content: r.description || undefined }));
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
function clamp(n, min, max) {
|
|
30
|
+
return Math.max(min, Math.min(max, Number.isFinite(n) ? n : min));
|
|
31
|
+
}
|
|
32
|
+
//# sourceMappingURL=brave.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"brave.js","sourceRoot":"","sources":["../../../src/retrieval/search/brave.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAS7C;;;GAGG;AACH,MAAM,OAAO,mBAAmB;IAGD;IAFpB,IAAI,GAAG,OAAgB,CAAC;IAEjC,YAA6B,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;IAAG,CAAC;IAE/C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA8B,EAAE;QAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/C,MAAM,GAAG,GACP,iDAAiD;YACjD,IAAI,eAAe,CAAC;gBAClB,CAAC,EAAE,KAAK;gBACR,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC;gBACpB,UAAU,EAAE,QAAQ;aACrB,CAAC,CAAC,QAAQ,EAAE,CAAC;QAChB,MAAM,IAAI,GAAG,CAAC,MAAM,SAAS,CAAC,GAAG,EAAE;YACjC,sBAAsB,EAAE,IAAI,CAAC,MAAM;SACpC,CAAC,CAA0C,CAAC;QAC7C,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,IAAI,EAAE,CAAC;aAC7B,MAAM,CAAC,CAAC,CAAC,EAAqD,EAAE,CAC/D,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAC1B;aACA,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,WAAW,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC;IACvF,CAAC;CACF;AAED,SAAS,KAAK,CAAC,CAAS,EAAE,GAAW,EAAE,GAAW;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { PipelineConfig } from '../../config.js';
|
|
2
|
+
import type { SearchProvider } from './types.js';
|
|
3
|
+
export type { SearchProvider, WebSearchResult, WebSearchProviderName } from './types.js';
|
|
4
|
+
export { SearxngSearchProvider } from './searxng.js';
|
|
5
|
+
export { BraveSearchProvider } from './brave.js';
|
|
6
|
+
export { SerperSearchProvider } from './serper.js';
|
|
7
|
+
/**
|
|
8
|
+
* Pick the configured web-search provider (ADR-0013). A self-hosted SearXNG
|
|
9
|
+
* instance is preferred because it is free and keyless; keyed providers are
|
|
10
|
+
* the fallback for rules or environments that block metasearch. Returns null
|
|
11
|
+
* when none is configured, which the pipeline reports as "unavailable".
|
|
12
|
+
*/
|
|
13
|
+
export declare function createSearchProvider(config: PipelineConfig): SearchProvider | null;
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { SearxngSearchProvider } from './searxng.js';
|
|
2
|
+
import { BraveSearchProvider } from './brave.js';
|
|
3
|
+
import { SerperSearchProvider } from './serper.js';
|
|
4
|
+
export { SearxngSearchProvider } from './searxng.js';
|
|
5
|
+
export { BraveSearchProvider } from './brave.js';
|
|
6
|
+
export { SerperSearchProvider } from './serper.js';
|
|
7
|
+
/**
|
|
8
|
+
* Pick the configured web-search provider (ADR-0013). A self-hosted SearXNG
|
|
9
|
+
* instance is preferred because it is free and keyless; keyed providers are
|
|
10
|
+
* the fallback for rules or environments that block metasearch. Returns null
|
|
11
|
+
* when none is configured, which the pipeline reports as "unavailable".
|
|
12
|
+
*/
|
|
13
|
+
export function createSearchProvider(config) {
|
|
14
|
+
if (config.searxngUrl)
|
|
15
|
+
return new SearxngSearchProvider(config.searxngUrl);
|
|
16
|
+
if (config.braveApiKey)
|
|
17
|
+
return new BraveSearchProvider(config.braveApiKey);
|
|
18
|
+
if (config.serperApiKey)
|
|
19
|
+
return new SerperSearchProvider(config.serperApiKey);
|
|
20
|
+
return null;
|
|
21
|
+
}
|
|
22
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/retrieval/search/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAInD,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACjD,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAEnD;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,MAAsB;IACzD,IAAI,MAAM,CAAC,UAAU;QAAE,OAAO,IAAI,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC3E,IAAI,MAAM,CAAC,WAAW;QAAE,OAAO,IAAI,mBAAmB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAC3E,IAAI,MAAM,CAAC,YAAY;QAAE,OAAO,IAAI,oBAAoB,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IAC9E,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SearchProvider, WebSearchResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Self-hosted metasearch (ADR-0013). Free and keyless, but it depends on
|
|
4
|
+
* upstream engines that may rate-limit or block the instance — hence the
|
|
5
|
+
* pluggable provider interface.
|
|
6
|
+
*/
|
|
7
|
+
export declare class SearxngSearchProvider implements SearchProvider {
|
|
8
|
+
private readonly baseUrl;
|
|
9
|
+
readonly name: "searxng";
|
|
10
|
+
constructor(baseUrl: string);
|
|
11
|
+
search(query: string, options?: {
|
|
12
|
+
limit?: number;
|
|
13
|
+
}): Promise<WebSearchResult[]>;
|
|
14
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { fetchJson } from '../candidates.js';
|
|
2
|
+
/**
|
|
3
|
+
* Self-hosted metasearch (ADR-0013). Free and keyless, but it depends on
|
|
4
|
+
* upstream engines that may rate-limit or block the instance — hence the
|
|
5
|
+
* pluggable provider interface.
|
|
6
|
+
*/
|
|
7
|
+
export class SearxngSearchProvider {
|
|
8
|
+
baseUrl;
|
|
9
|
+
name = 'searxng';
|
|
10
|
+
constructor(baseUrl) {
|
|
11
|
+
this.baseUrl = baseUrl;
|
|
12
|
+
}
|
|
13
|
+
async search(query, options = {}) {
|
|
14
|
+
const base = this.baseUrl.replace(/\/$/, '');
|
|
15
|
+
const url = `${base}/search?` +
|
|
16
|
+
new URLSearchParams({ q: query, format: 'json', safesearch: '1' }).toString();
|
|
17
|
+
const data = (await fetchJson(url));
|
|
18
|
+
return (data.results ?? [])
|
|
19
|
+
.filter((r) => Boolean(r.title && r.url))
|
|
20
|
+
.slice(0, options.limit ?? 8)
|
|
21
|
+
.map((r) => ({ title: r.title, url: r.url, content: r.content || undefined }));
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=searxng.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"searxng.js","sourceRoot":"","sources":["../../../src/retrieval/search/searxng.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAS7C;;;;GAIG;AACH,MAAM,OAAO,qBAAqB;IAGH;IAFpB,IAAI,GAAG,SAAkB,CAAC;IAEnC,YAA6B,OAAe;QAAf,YAAO,GAAP,OAAO,CAAQ;IAAG,CAAC;IAEhD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA8B,EAAE;QAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7C,MAAM,GAAG,GACP,GAAG,IAAI,UAAU;YACjB,IAAI,eAAe,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;QAChF,MAAM,IAAI,GAAG,CAAC,MAAM,SAAS,CAAC,GAAG,CAAC,CAAkC,CAAC;QACrE,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;aACxB,MAAM,CAAC,CAAC,CAAC,EAAuD,EAAE,CACjE,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,CAC1B;aACA,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC,CAAC;aAC5B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC;IACnF,CAAC;CACF"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { SearchProvider, WebSearchResult } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Serper (Google SERP) adapter (ADR-0013). Keyed fallback for hosted
|
|
4
|
+
* deployments where a self-hosted SearXNG is an abuse surface. Requires
|
|
5
|
+
* `SERPER_API_KEY`.
|
|
6
|
+
*/
|
|
7
|
+
export declare class SerperSearchProvider implements SearchProvider {
|
|
8
|
+
private readonly apiKey;
|
|
9
|
+
readonly name: "serper";
|
|
10
|
+
constructor(apiKey: string);
|
|
11
|
+
search(query: string, options?: {
|
|
12
|
+
limit?: number;
|
|
13
|
+
}): Promise<WebSearchResult[]>;
|
|
14
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
const ENDPOINT = 'https://google.serper.dev/search';
|
|
2
|
+
const TIMEOUT_MS = 8000;
|
|
3
|
+
/**
|
|
4
|
+
* Serper (Google SERP) adapter (ADR-0013). Keyed fallback for hosted
|
|
5
|
+
* deployments where a self-hosted SearXNG is an abuse surface. Requires
|
|
6
|
+
* `SERPER_API_KEY`.
|
|
7
|
+
*/
|
|
8
|
+
export class SerperSearchProvider {
|
|
9
|
+
apiKey;
|
|
10
|
+
name = 'serper';
|
|
11
|
+
constructor(apiKey) {
|
|
12
|
+
this.apiKey = apiKey;
|
|
13
|
+
}
|
|
14
|
+
async search(query, options = {}) {
|
|
15
|
+
const num = clamp(options.limit ?? 8, 1, 10);
|
|
16
|
+
const res = await fetch(ENDPOINT, {
|
|
17
|
+
method: 'POST',
|
|
18
|
+
headers: {
|
|
19
|
+
'content-type': 'application/json',
|
|
20
|
+
accept: 'application/json',
|
|
21
|
+
'x-api-key': this.apiKey,
|
|
22
|
+
},
|
|
23
|
+
body: JSON.stringify({ q: query, num }),
|
|
24
|
+
signal: AbortSignal.timeout(TIMEOUT_MS),
|
|
25
|
+
});
|
|
26
|
+
if (!res.ok)
|
|
27
|
+
throw new Error(`HTTP ${res.status} from serper.dev`);
|
|
28
|
+
const data = (await res.json());
|
|
29
|
+
return (data.organic ?? [])
|
|
30
|
+
.filter((r) => Boolean(r.title && r.link))
|
|
31
|
+
.slice(0, num)
|
|
32
|
+
.map((r) => ({ title: r.title, url: r.link, content: r.snippet || undefined }));
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
function clamp(n, min, max) {
|
|
36
|
+
return Math.max(min, Math.min(max, Number.isFinite(n) ? n : min));
|
|
37
|
+
}
|
|
38
|
+
//# sourceMappingURL=serper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"serper.js","sourceRoot":"","sources":["../../../src/retrieval/search/serper.ts"],"names":[],"mappings":"AAQA,MAAM,QAAQ,GAAG,kCAAkC,CAAC;AACpD,MAAM,UAAU,GAAG,IAAI,CAAC;AAExB;;;;GAIG;AACH,MAAM,OAAO,oBAAoB;IAGF;IAFpB,IAAI,GAAG,QAAiB,CAAC;IAElC,YAA6B,MAAc;QAAd,WAAM,GAAN,MAAM,CAAQ;IAAG,CAAC;IAE/C,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAA8B,EAAE;QAC1D,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7C,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE;YAChC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,MAAM,EAAE,kBAAkB;gBAC1B,WAAW,EAAE,IAAI,CAAC,MAAM;aACzB;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACvC,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,UAAU,CAAC;SACxC,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,MAAM,kBAAkB,CAAC,CAAC;QACnE,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAiC,CAAC;QAChE,OAAO,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;aACxB,MAAM,CAAC,CAAC,CAAC,EAAuD,EAAE,CACjE,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,CAC3B;aACA,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS,EAAE,CAAC,CAAC,CAAC;IACpF,CAAC;CACF;AAED,SAAS,KAAK,CAAC,CAAS,EAAE,GAAW,EAAE,GAAW;IAChD,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACpE,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Web-search provider contract (ADR-0013). One adapter per engine, chosen by
|
|
3
|
+
* configuration; the pipeline only sees this interface.
|
|
4
|
+
*/
|
|
5
|
+
export interface WebSearchResult {
|
|
6
|
+
title: string;
|
|
7
|
+
url: string;
|
|
8
|
+
/** Snippet or summary supplied by the engine, when available. */
|
|
9
|
+
content?: string;
|
|
10
|
+
}
|
|
11
|
+
export type WebSearchProviderName = 'searxng' | 'brave' | 'serper';
|
|
12
|
+
export interface SearchProvider {
|
|
13
|
+
readonly name: WebSearchProviderName;
|
|
14
|
+
search(query: string, options?: {
|
|
15
|
+
limit?: number;
|
|
16
|
+
}): Promise<WebSearchResult[]>;
|
|
17
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/retrieval/search/types.ts"],"names":[],"mappings":""}
|
|
File without changes
|
package/dist/retrieval/videos.js
CHANGED
|
File without changes
|
|
File without changes
|
package/dist/retrieval/web.d.ts
CHANGED
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import type { Candidate } from './candidates.js';
|
|
2
|
+
import type { SearchProvider } from './search/types.js';
|
|
2
3
|
/**
|
|
3
|
-
* General-web retrieval
|
|
4
|
-
*
|
|
5
|
-
* configured instance this component reports unavailable (ADR-0006).
|
|
6
|
-
* Web results are NOT database-attested: they get a full URL check.
|
|
4
|
+
* General-web retrieval through the configured search provider (ADR-0013).
|
|
5
|
+
* Web results are NOT database-attested: they get a full URL check later.
|
|
7
6
|
*/
|
|
8
|
-
export declare function retrieveWeb(topic: string,
|
|
7
|
+
export declare function retrieveWeb(topic: string, provider: SearchProvider): Promise<Candidate[]>;
|
package/dist/retrieval/web.js
CHANGED
|
@@ -1,31 +1,18 @@
|
|
|
1
|
-
import { fetchJson } from './candidates.js';
|
|
2
1
|
/**
|
|
3
|
-
* General-web retrieval
|
|
4
|
-
*
|
|
5
|
-
* configured instance this component reports unavailable (ADR-0006).
|
|
6
|
-
* Web results are NOT database-attested: they get a full URL check.
|
|
2
|
+
* General-web retrieval through the configured search provider (ADR-0013).
|
|
3
|
+
* Web results are NOT database-attested: they get a full URL check later.
|
|
7
4
|
*/
|
|
8
|
-
export async function retrieveWeb(topic,
|
|
9
|
-
const
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
url: r.url,
|
|
21
|
-
format: 'website',
|
|
22
|
-
authors: [],
|
|
23
|
-
year: null,
|
|
24
|
-
description: r.content || undefined,
|
|
25
|
-
origin: 'searxng',
|
|
26
|
-
databaseAttested: false,
|
|
27
|
-
};
|
|
28
|
-
})
|
|
29
|
-
.filter((c) => c !== null);
|
|
5
|
+
export async function retrieveWeb(topic, provider) {
|
|
6
|
+
const results = await provider.search(topic, { limit: 8 });
|
|
7
|
+
return results.map((r) => ({
|
|
8
|
+
title: r.title,
|
|
9
|
+
url: r.url,
|
|
10
|
+
format: 'website',
|
|
11
|
+
authors: [],
|
|
12
|
+
year: null,
|
|
13
|
+
description: r.content || undefined,
|
|
14
|
+
origin: provider.name,
|
|
15
|
+
databaseAttested: false,
|
|
16
|
+
}));
|
|
30
17
|
}
|
|
31
18
|
//# sourceMappingURL=web.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"web.js","sourceRoot":"","sources":["../../src/retrieval/web.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"web.js","sourceRoot":"","sources":["../../src/retrieval/web.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAa,EACb,QAAwB;IAExB,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;IAC3D,OAAO,OAAO,CAAC,GAAG,CAChB,CAAC,CAAC,EAAa,EAAE,CAAC,CAAC;QACjB,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI;QACV,WAAW,EAAE,CAAC,CAAC,OAAO,IAAI,SAAS;QACnC,MAAM,EAAE,QAAQ,CAAC,IAAI;QACrB,gBAAgB,EAAE,KAAK;KACxB,CAAC,CACH,CAAC;AACJ,CAAC"}
|