@zhixuan92/multi-model-agent-core 3.11.1 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/read-only-review-flag.d.ts +1 -1
- package/dist/config/read-only-review-flag.d.ts.map +1 -1
- package/dist/config/read-only-review-flag.js +1 -0
- package/dist/config/read-only-review-flag.js.map +1 -1
- package/dist/config/schema.d.ts +47 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +102 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/delegate-with-escalation.d.ts.map +1 -1
- package/dist/delegate-with-escalation.js +1 -0
- package/dist/delegate-with-escalation.js.map +1 -1
- package/dist/executors/explore.d.ts +13 -0
- package/dist/executors/explore.d.ts.map +1 -0
- package/dist/executors/explore.js +352 -0
- package/dist/executors/explore.js.map +1 -0
- package/dist/executors/index.d.ts +1 -0
- package/dist/executors/index.d.ts.map +1 -1
- package/dist/executors/index.js +1 -0
- package/dist/executors/index.js.map +1 -1
- package/dist/intake/compilers/explore.d.ts +29 -0
- package/dist/intake/compilers/explore.d.ts.map +1 -0
- package/dist/intake/compilers/explore.js +108 -0
- package/dist/intake/compilers/explore.js.map +1 -0
- package/dist/observability/events.d.ts +92 -0
- package/dist/observability/events.d.ts.map +1 -1
- package/dist/observability/events.js +49 -0
- package/dist/observability/events.js.map +1 -1
- package/dist/reporting/compose-explore-headline.d.ts +14 -0
- package/dist/reporting/compose-explore-headline.d.ts.map +1 -0
- package/dist/reporting/compose-explore-headline.js +14 -0
- package/dist/reporting/compose-explore-headline.js.map +1 -0
- package/dist/reporting/derive-explore-status.d.ts +18 -0
- package/dist/reporting/derive-explore-status.d.ts.map +1 -0
- package/dist/reporting/derive-explore-status.js +19 -0
- package/dist/reporting/derive-explore-status.js.map +1 -0
- package/dist/reporting/parse-explore-report.d.ts +38 -0
- package/dist/reporting/parse-explore-report.d.ts.map +1 -0
- package/dist/reporting/parse-explore-report.js +185 -0
- package/dist/reporting/parse-explore-report.js.map +1 -0
- package/dist/research/adapters/arxiv.d.ts +6 -0
- package/dist/research/adapters/arxiv.d.ts.map +1 -0
- package/dist/research/adapters/arxiv.js +36 -0
- package/dist/research/adapters/arxiv.js.map +1 -0
- package/dist/research/adapters/generic-rss.d.ts +8 -0
- package/dist/research/adapters/generic-rss.d.ts.map +1 -0
- package/dist/research/adapters/generic-rss.js +26 -0
- package/dist/research/adapters/generic-rss.js.map +1 -0
- package/dist/research/adapters/github-search.d.ts +7 -0
- package/dist/research/adapters/github-search.d.ts.map +1 -0
- package/dist/research/adapters/github-search.js +95 -0
- package/dist/research/adapters/github-search.js.map +1 -0
- package/dist/research/adapters/index.d.ts +8 -0
- package/dist/research/adapters/index.d.ts.map +1 -0
- package/dist/research/adapters/index.js +17 -0
- package/dist/research/adapters/index.js.map +1 -0
- package/dist/research/adapters/semantic-scholar.d.ts +6 -0
- package/dist/research/adapters/semantic-scholar.d.ts.map +1 -0
- package/dist/research/adapters/semantic-scholar.js +54 -0
- package/dist/research/adapters/semantic-scholar.js.map +1 -0
- package/dist/research/adapters/types.d.ts +15 -0
- package/dist/research/adapters/types.d.ts.map +1 -0
- package/dist/research/adapters/types.js +2 -0
- package/dist/research/adapters/types.js.map +1 -0
- package/dist/research/allowlist.d.ts +25 -0
- package/dist/research/allowlist.d.ts.map +1 -0
- package/dist/research/allowlist.js +102 -0
- package/dist/research/allowlist.js.map +1 -0
- package/dist/research/ssrf-guard.d.ts +12 -0
- package/dist/research/ssrf-guard.d.ts.map +1 -0
- package/dist/research/ssrf-guard.js +209 -0
- package/dist/research/ssrf-guard.js.map +1 -0
- package/dist/research/types.d.ts +14 -0
- package/dist/research/types.d.ts.map +1 -0
- package/dist/research/types.js +2 -0
- package/dist/research/types.js.map +1 -0
- package/dist/research/untrusted-content.d.ts +13 -0
- package/dist/research/untrusted-content.d.ts.map +1 -0
- package/dist/research/untrusted-content.js +9 -0
- package/dist/research/untrusted-content.js.map +1 -0
- package/dist/research/web-fetch.d.ts +50 -0
- package/dist/research/web-fetch.d.ts.map +1 -0
- package/dist/research/web-fetch.js +411 -0
- package/dist/research/web-fetch.js.map +1 -0
- package/dist/research/web-search.d.ts +28 -0
- package/dist/research/web-search.d.ts.map +1 -0
- package/dist/research/web-search.js +134 -0
- package/dist/research/web-search.js.map +1 -0
- package/dist/runners/base/research-tools.d.ts +47 -0
- package/dist/runners/base/research-tools.d.ts.map +1 -0
- package/dist/runners/base/research-tools.js +67 -0
- package/dist/runners/base/research-tools.js.map +1 -0
- package/dist/runners/claude-runner.d.ts.map +1 -1
- package/dist/runners/claude-runner.js +27 -1
- package/dist/runners/claude-runner.js.map +1 -1
- package/dist/runners/codex-runner.d.ts.map +1 -1
- package/dist/runners/codex-runner.js +25 -0
- package/dist/runners/codex-runner.js.map +1 -1
- package/dist/runners/openai-runner.d.ts.map +1 -1
- package/dist/runners/openai-runner.js +34 -1
- package/dist/runners/openai-runner.js.map +1 -1
- package/dist/runners/types.d.ts +6 -0
- package/dist/runners/types.d.ts.map +1 -1
- package/dist/tool-schemas/explore.d.ts +9 -0
- package/dist/tool-schemas/explore.d.ts.map +1 -0
- package/dist/tool-schemas/explore.js +64 -0
- package/dist/tool-schemas/explore.js.map +1 -0
- package/dist/types.d.ts +33 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +16 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arxiv.d.ts","sourceRoot":"","sources":["../../../src/research/adapters/arxiv.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAIhD,MAAM,WAAW,SAAS;IAAG,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AAEnD,wBAAsB,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,SAAc,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAgC/F"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { request } from 'undici';
|
|
2
|
+
import { XMLParser } from 'fast-xml-parser';
|
|
3
|
+
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_' });
|
|
4
|
+
export async function arxivSearch(query, opts = {}) {
|
|
5
|
+
const max = Math.min(25, Math.max(1, opts.maxResults ?? 10));
|
|
6
|
+
const url = new URL('https://export.arxiv.org/api/query');
|
|
7
|
+
url.searchParams.set('search_query', `all:${query}`);
|
|
8
|
+
url.searchParams.set('max_results', String(max));
|
|
9
|
+
const res = await request(url.toString(), { method: 'GET' });
|
|
10
|
+
if (res.statusCode >= 300 && res.statusCode < 400) {
|
|
11
|
+
throw new Error('adapter_unexpected_redirect: arxiv');
|
|
12
|
+
}
|
|
13
|
+
if (res.statusCode !== 200) {
|
|
14
|
+
throw new Error(`arxiv_http_${res.statusCode}`);
|
|
15
|
+
}
|
|
16
|
+
const xml = await res.body.text();
|
|
17
|
+
const parsed = parser.parse(xml);
|
|
18
|
+
const entriesRaw = parsed.feed?.entry;
|
|
19
|
+
const entries = Array.isArray(entriesRaw) ? entriesRaw : entriesRaw ? [entriesRaw] : [];
|
|
20
|
+
return entries.slice(0, max).map((e) => {
|
|
21
|
+
const id = String(e.id ?? '');
|
|
22
|
+
const m = id.match(/(\d{4}\.\d{4,5})(v\d+)?$/);
|
|
23
|
+
const rawUrl = id || String(e.link?.['@_href'] ?? '');
|
|
24
|
+
const url = rawUrl.replace(/^http:\/\//i, 'https://');
|
|
25
|
+
return {
|
|
26
|
+
adapterId: 'arxiv',
|
|
27
|
+
recordId: m?.[1] ?? id,
|
|
28
|
+
title: String(e.title ?? '').trim(),
|
|
29
|
+
url,
|
|
30
|
+
snippet: String(e.summary ?? '').trim().slice(0, 500),
|
|
31
|
+
publishedAt: String(e.published ?? ''),
|
|
32
|
+
raw: e,
|
|
33
|
+
};
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=arxiv.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"arxiv.js","sourceRoot":"","sources":["../../../src/research/adapters/arxiv.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAG5C,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;AAIrF,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAa,EAAE,OAAkB,EAAE;IACnE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAC1D,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,cAAc,EAAE,OAAO,KAAK,EAAE,CAAC,CAAC;IACrD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,aAAa,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAEjD,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IAC7D,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAmC,CAAC;IACnE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;IACtC,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACxF,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE;QAC1C,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAC/C,MAAM,MAAM,GAAG,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;QACtD,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,UAAU,CAAC,CAAC;QACtD,OAAO;YACL,SAAS,EAAE,OAAgB;YAC3B,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE;YACtB,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;YACnC,GAAG;YACH,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YACrD,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC;YACtC,GAAG,EAAE,CAAC;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { AdapterResult } from './types.js';
|
|
2
|
+
import type { WebFetchResult } from '../web-fetch.js';
|
|
3
|
+
export interface RssOpts {
|
|
4
|
+
webFetch: (url: string) => Promise<WebFetchResult>;
|
|
5
|
+
maxResults?: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function rssAdapter(url: string, opts: RssOpts): Promise<AdapterResult[]>;
|
|
8
|
+
//# sourceMappingURL=generic-rss.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generic-rss.d.ts","sourceRoot":"","sources":["../../../src/research/adapters/generic-rss.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AAItD,MAAM,WAAW,OAAO;IACtB,QAAQ,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IACnD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAsBrF"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { XMLParser } from 'fast-xml-parser';
|
|
2
|
+
const parser = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: '@_' });
|
|
3
|
+
export async function rssAdapter(url, opts) {
|
|
4
|
+
const r = await opts.webFetch(url);
|
|
5
|
+
if (r.status !== 'ok')
|
|
6
|
+
throw new Error(`rss_fetch_failed:${r.reasonCode}`);
|
|
7
|
+
if (r.textTruncated)
|
|
8
|
+
throw new Error('rss_text_truncated_skip');
|
|
9
|
+
const parsed = parser.parse(r.rawText);
|
|
10
|
+
const rdfItems = parsed?.['rdf:RDF']?.item ?? parsed?.RDF?.item;
|
|
11
|
+
const items = parsed?.rss?.channel?.item ?? parsed?.feed?.entry ?? rdfItems ?? [];
|
|
12
|
+
const arr = Array.isArray(items) ? items : [items];
|
|
13
|
+
const max = Math.min(50, Math.max(1, opts.maxResults ?? 25));
|
|
14
|
+
return arr.slice(0, max).map((it) => ({
|
|
15
|
+
adapterId: 'rss',
|
|
16
|
+
recordId: String(it.guid?.['#text'] ?? it.guid ??
|
|
17
|
+
it['rdf:about'] ?? it['@_rdf:about'] ??
|
|
18
|
+
it.link ?? it.title),
|
|
19
|
+
title: String(it.title?.['#text'] ?? it.title ?? '').trim(),
|
|
20
|
+
url: String(it.link?.['@_href'] ?? it.link ?? ''),
|
|
21
|
+
snippet: String(it.description?.['#text'] ?? it.description ?? it.summary ?? '').replace(/<[^>]+>/g, '').slice(0, 500),
|
|
22
|
+
publishedAt: String(it.pubDate ?? it.published ?? it['dc:date'] ?? ''),
|
|
23
|
+
raw: it,
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=generic-rss.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"generic-rss.js","sourceRoot":"","sources":["../../../src/research/adapters/generic-rss.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAI5C,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,gBAAgB,EAAE,KAAK,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;AAOrF,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,IAAa;IACzD,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC;IAC3E,IAAI,CAAC,CAAC,aAAa;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAQ,CAAC;IAC9C,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,SAAS,CAAC,EAAE,IAAI,IAAI,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC;IAChE,MAAM,KAAK,GAAG,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,IAAI,EAAE,KAAK,IAAI,QAAQ,IAAI,EAAE,CAAC;IAClF,MAAM,GAAG,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IACnD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAO,EAAE,EAAE,CAAC,CAAC;QACzC,SAAS,EAAE,KAAc;QACzB,QAAQ,EAAE,MAAM,CACd,EAAE,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI;YAC7B,EAAE,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,aAAa,CAAC;YACpC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,KAAK,CACpB;QACD,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;QAC3D,GAAG,EAAE,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QACjD,OAAO,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACtH,WAAW,EAAE,MAAM,CAAC,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,SAAS,IAAI,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACtE,GAAG,EAAE,EAAE;KACR,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { AdapterResult } from './types.js';
|
|
2
|
+
export interface GitHubOpts {
|
|
3
|
+
kind: 'repo' | 'code';
|
|
4
|
+
maxResults?: number;
|
|
5
|
+
}
|
|
6
|
+
export declare function githubSearch(query: string, opts: GitHubOpts): Promise<AdapterResult[]>;
|
|
7
|
+
//# sourceMappingURL=github-search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github-search.d.ts","sourceRoot":"","sources":["../../../src/research/adapters/github-search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,MAAM,WAAW,UAAU;IAAG,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AA6B3E,wBAAsB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAsF5F"}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { request } from 'undici';
|
|
2
|
+
function isGitHubKind(kind) {
|
|
3
|
+
return kind === 'repo' || kind === 'code';
|
|
4
|
+
}
|
|
5
|
+
function firstTextMatchFragment(textMatches) {
|
|
6
|
+
if (!Array.isArray(textMatches))
|
|
7
|
+
return '';
|
|
8
|
+
const fragment = textMatches[0]?.fragment;
|
|
9
|
+
return typeof fragment === 'string' ? fragment : '';
|
|
10
|
+
}
|
|
11
|
+
export async function githubSearch(query, opts) {
|
|
12
|
+
if (!isGitHubKind(opts.kind))
|
|
13
|
+
throw new Error('github_invalid_kind');
|
|
14
|
+
const max = Math.min(25, Math.max(1, opts.maxResults ?? 10));
|
|
15
|
+
const path = opts.kind === 'repo' ? '/search/repositories' : '/search/code';
|
|
16
|
+
const url = new URL(`https://api.github.com${path}`);
|
|
17
|
+
url.searchParams.set('q', query);
|
|
18
|
+
url.searchParams.set('per_page', String(max));
|
|
19
|
+
const accept = opts.kind === 'code'
|
|
20
|
+
? 'application/vnd.github.v3.text-match+json'
|
|
21
|
+
: 'application/vnd.github+json';
|
|
22
|
+
const res = await request(url.toString(), {
|
|
23
|
+
method: 'GET',
|
|
24
|
+
headers: { accept, 'user-agent': 'mma-explore' },
|
|
25
|
+
});
|
|
26
|
+
if (res.statusCode === 403) {
|
|
27
|
+
const remaining = res.headers['x-ratelimit-remaining'];
|
|
28
|
+
if (remaining === '0')
|
|
29
|
+
throw new Error('github_rate_limited');
|
|
30
|
+
try {
|
|
31
|
+
const text = await res.body.text();
|
|
32
|
+
const body = JSON.parse(text);
|
|
33
|
+
if (typeof body.message === 'string') {
|
|
34
|
+
const msg = body.message.toLowerCase();
|
|
35
|
+
if (msg.includes('rate limit') || msg.includes('abuse')) {
|
|
36
|
+
throw new Error('github_rate_limited');
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
catch (err) {
|
|
41
|
+
if (err instanceof Error && err.message === 'github_rate_limited')
|
|
42
|
+
throw err;
|
|
43
|
+
}
|
|
44
|
+
throw new Error(`github_http_${res.statusCode}`);
|
|
45
|
+
}
|
|
46
|
+
if (res.statusCode >= 300 && res.statusCode < 400) {
|
|
47
|
+
throw new Error('adapter_unexpected_redirect: github_search');
|
|
48
|
+
}
|
|
49
|
+
if (res.statusCode !== 200)
|
|
50
|
+
throw new Error(`github_http_${res.statusCode}`);
|
|
51
|
+
let body;
|
|
52
|
+
try {
|
|
53
|
+
body = await res.body.json();
|
|
54
|
+
}
|
|
55
|
+
catch {
|
|
56
|
+
throw new Error('github_invalid_json');
|
|
57
|
+
}
|
|
58
|
+
const items = typeof body === 'object' && body !== null && 'items' in body && Array.isArray(body.items)
|
|
59
|
+
? body.items
|
|
60
|
+
: [];
|
|
61
|
+
if (opts.kind === 'repo') {
|
|
62
|
+
return items.slice(0, max).map((raw) => {
|
|
63
|
+
const item = raw;
|
|
64
|
+
const recordId = typeof item.full_name === 'string' && item.full_name.length > 0
|
|
65
|
+
? item.full_name
|
|
66
|
+
: String(item.id ?? '');
|
|
67
|
+
return {
|
|
68
|
+
adapterId: 'github_search',
|
|
69
|
+
recordId,
|
|
70
|
+
title: recordId,
|
|
71
|
+
url: String(item.html_url ?? item.url ?? ''),
|
|
72
|
+
snippet: String(item.description ?? '').slice(0, 500),
|
|
73
|
+
raw: item,
|
|
74
|
+
};
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
return items.slice(0, max).map((raw) => {
|
|
78
|
+
const item = raw;
|
|
79
|
+
const repoName = typeof item.repository?.full_name === 'string' ? item.repository.full_name : '';
|
|
80
|
+
const filePath = typeof item.path === 'string' ? item.path : '';
|
|
81
|
+
const fallbackId = String(item.url ?? item.html_url ?? '');
|
|
82
|
+
const recordId = repoName || filePath ? `${repoName}:${filePath}` : fallbackId;
|
|
83
|
+
const title = repoName || filePath ? `${repoName} — ${filePath}` : fallbackId;
|
|
84
|
+
const snippet = firstTextMatchFragment(item.text_matches);
|
|
85
|
+
return {
|
|
86
|
+
adapterId: 'github_search',
|
|
87
|
+
recordId,
|
|
88
|
+
title,
|
|
89
|
+
url: String(item.html_url ?? item.url ?? ''),
|
|
90
|
+
snippet: snippet.slice(0, 500),
|
|
91
|
+
raw: item,
|
|
92
|
+
};
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
//# sourceMappingURL=github-search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"github-search.js","sourceRoot":"","sources":["../../../src/research/adapters/github-search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAsBjC,SAAS,YAAY,CAAC,IAAa;IACjC,OAAO,IAAI,KAAK,MAAM,IAAI,IAAI,KAAK,MAAM,CAAC;AAC5C,CAAC;AAED,SAAS,sBAAsB,CAAC,WAAoB;IAClD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;QAAE,OAAO,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC;IAC1C,OAAO,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;AACtD,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,KAAa,EAAE,IAAgB;IAChE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IAErE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,cAAc,CAAC;IAC5E,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC;IACrD,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACjC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,KAAK,MAAM;QACjC,CAAC,CAAC,2CAA2C;QAC7C,CAAC,CAAC,6BAA6B,CAAC;IAElC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;QACxC,MAAM,EAAE,KAAK;QAEb,OAAO,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE;KACjD,CAAC,CAAC;IAEH,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;QACvD,IAAI,SAAS,KAAK,GAAG;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC9D,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAyB,CAAC;YACtD,IAAI,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;gBACrC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBACvC,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACxD,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;gBACzC,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,GAAY,EAAE,CAAC;YACtB,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,KAAK,qBAAqB;gBAAE,MAAM,GAAG,CAAC;QAC/E,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,eAAe,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IAE7E,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC;QACrG,CAAC,CAAC,IAAI,CAAC,KAAK;QACZ,CAAC,CAAC,EAAE,CAAC;IAEP,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAY,EAAE,EAAE;YAC9C,MAAM,IAAI,GAAG,GAAqB,CAAC;YACnC,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,SAAS,KAAK,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;gBAC9E,CAAC,CAAC,IAAI,CAAC,SAAS;gBAChB,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1B,OAAO;gBACL,SAAS,EAAE,eAAwB;gBACnC,QAAQ;gBACR,KAAK,EAAE,QAAQ;gBACf,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;gBAC5C,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACrD,GAAG,EAAE,IAAI;aACV,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAY,EAAE,EAAE;QAC9C,MAAM,IAAI,GAAG,GAAqB,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,UAAU,EAAE,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QACjG,MAAM,QAAQ,GAAG,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;QAChE,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,QAAQ,GAAG,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,QAAQ,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;QAC/E,MAAM,KAAK,GAAG,QAAQ,IAAI,QAAQ,CAAC,CAAC,CAAC,GAAG,QAAQ,MAAM,QAAQ,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC;QAC9E,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1D,OAAO;YACL,SAAS,EAAE,eAAwB;YACnC,QAAQ;YACR,KAAK;YACL,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;YAC5C,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC9B,GAAG,EAAE,IAAI;SACV,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { ResearchConfig } from '../../config/schema.js';
|
|
2
|
+
import type { AdapterId } from './types.js';
|
|
3
|
+
export declare function resolveEnabledAdapters(cfg: ResearchConfig['builtinAdapters']): AdapterId[];
|
|
4
|
+
export { arxivSearch } from './arxiv.js';
|
|
5
|
+
export { semanticScholarSearch } from './semantic-scholar.js';
|
|
6
|
+
export { githubSearch } from './github-search.js';
|
|
7
|
+
export { rssAdapter } from './generic-rss.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/research/adapters/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE5C,wBAAgB,sBAAsB,CAAC,GAAG,EAAE,cAAc,CAAC,iBAAiB,CAAC,GAAG,SAAS,EAAE,CAO1F;AAED,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export function resolveEnabledAdapters(cfg) {
|
|
2
|
+
const out = [];
|
|
3
|
+
if (cfg.arxiv)
|
|
4
|
+
out.push('arxiv');
|
|
5
|
+
if (cfg.semanticScholar)
|
|
6
|
+
out.push('semantic_scholar');
|
|
7
|
+
if (cfg.githubSearch)
|
|
8
|
+
out.push('github_search');
|
|
9
|
+
if (cfg.genericRss)
|
|
10
|
+
out.push('rss');
|
|
11
|
+
return out;
|
|
12
|
+
}
|
|
13
|
+
export { arxivSearch } from './arxiv.js';
|
|
14
|
+
export { semanticScholarSearch } from './semantic-scholar.js';
|
|
15
|
+
export { githubSearch } from './github-search.js';
|
|
16
|
+
export { rssAdapter } from './generic-rss.js';
|
|
17
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/research/adapters/index.ts"],"names":[],"mappings":"AAGA,MAAM,UAAU,sBAAsB,CAAC,GAAsC;IAC3E,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,IAAI,GAAG,CAAC,KAAK;QAAE,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACjC,IAAI,GAAG,CAAC,eAAe;QAAE,GAAG,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACtD,IAAI,GAAG,CAAC,YAAY;QAAE,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAChD,IAAI,GAAG,CAAC,UAAU;QAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"semantic-scholar.d.ts","sourceRoot":"","sources":["../../../src/research/adapters/semantic-scholar.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,MAAM,WAAW,MAAM;IAAG,UAAU,CAAC,EAAE,MAAM,CAAC;CAAE;AA8BhD,wBAAsB,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,MAAW,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC,CAiCtG"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { request } from 'undici';
|
|
2
|
+
function normalizeRecord(r) {
|
|
3
|
+
const paperId = typeof r.paperId === 'string' && r.paperId.length > 0 ? r.paperId : null;
|
|
4
|
+
const title = typeof r.title === 'string' ? r.title.trim() : '';
|
|
5
|
+
if (!paperId || !title)
|
|
6
|
+
return null;
|
|
7
|
+
const abstract = typeof r.abstract === 'string' ? r.abstract : '';
|
|
8
|
+
const apiUrl = typeof r.url === 'string' && r.url.length > 0 ? r.url : null;
|
|
9
|
+
const year = typeof r.year === 'number' && Number.isFinite(r.year) && r.year > 1900 ? r.year : null;
|
|
10
|
+
return {
|
|
11
|
+
adapterId: 'semantic_scholar',
|
|
12
|
+
recordId: paperId,
|
|
13
|
+
title,
|
|
14
|
+
url: apiUrl ?? `https://www.semanticscholar.org/paper/${paperId}`,
|
|
15
|
+
snippet: abstract.slice(0, 500),
|
|
16
|
+
publishedAt: year ? `${year}-01-01` : undefined,
|
|
17
|
+
raw: r,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
export async function semanticScholarSearch(query, opts = {}) {
|
|
21
|
+
const max = Math.min(25, Math.max(1, opts.maxResults ?? 10));
|
|
22
|
+
const url = new URL('https://api.semanticscholar.org/graph/v1/paper/search');
|
|
23
|
+
url.searchParams.set('query', query);
|
|
24
|
+
url.searchParams.set('limit', String(max));
|
|
25
|
+
url.searchParams.set('fields', 'paperId,title,abstract,year,authors,url');
|
|
26
|
+
let res;
|
|
27
|
+
try {
|
|
28
|
+
res = await request(url.toString(), { method: 'GET' });
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
throw new Error(`semantic_scholar_request_failed: ${err.message}`);
|
|
32
|
+
}
|
|
33
|
+
if (res.statusCode === 429)
|
|
34
|
+
throw new Error('semantic_scholar_rate_limited');
|
|
35
|
+
if (res.statusCode >= 300 && res.statusCode < 400)
|
|
36
|
+
throw new Error('adapter_unexpected_redirect: semantic_scholar');
|
|
37
|
+
if (res.statusCode !== 200)
|
|
38
|
+
throw new Error(`semantic_scholar_http_${res.statusCode}`);
|
|
39
|
+
let body;
|
|
40
|
+
try {
|
|
41
|
+
body = await res.body.json();
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
throw new Error(`semantic_scholar_parse_error: ${err.message}`);
|
|
45
|
+
}
|
|
46
|
+
const data = Array.isArray(body?.data)
|
|
47
|
+
? body.data
|
|
48
|
+
: [];
|
|
49
|
+
return data
|
|
50
|
+
.slice(0, max)
|
|
51
|
+
.map(normalizeRecord)
|
|
52
|
+
.filter((r) => r !== null);
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=semantic-scholar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"semantic-scholar.js","sourceRoot":"","sources":["../../../src/research/adapters/semantic-scholar.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAajC,SAAS,eAAe,CAAC,CAAY;IACnC,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC;IACzF,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAEpC,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpG,OAAO;QACL,SAAS,EAAE,kBAA2B;QACtC,QAAQ,EAAE,OAAO;QACjB,KAAK;QACL,GAAG,EAAE,MAAM,IAAI,yCAAyC,OAAO,EAAE;QACjE,OAAO,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAC/B,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC,CAAC,SAAS;QAC/C,GAAG,EAAE,CAAC;KACP,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,KAAa,EAAE,OAAe,EAAE;IAC1E,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7D,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,uDAAuD,CAAC,CAAC;IAC7E,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;IACrC,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3C,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,yCAAyC,CAAC,CAAC;IAE1E,IAAI,GAAG,CAAC;IACR,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,oCAAqC,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAChF,CAAC;IAED,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IAC7E,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,GAAG,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACpH,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG;QAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;IAEvF,IAAI,IAAa,CAAC;IAClB,IAAI,CAAC;QACH,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,iCAAkC,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAE,IAAgC,EAAE,IAAI,CAAC;QACjE,CAAC,CAAE,IAAgC,CAAC,IAAmB;QACvD,CAAC,CAAC,EAAE,CAAC;IAEP,OAAO,IAAI;SACR,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;SACb,GAAG,CAAC,eAAe,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAsB,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;AACnD,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type AdapterId = 'arxiv' | 'semantic_scholar' | 'github_search' | 'rss';
|
|
2
|
+
export interface AdapterResult {
|
|
3
|
+
adapterId: AdapterId;
|
|
4
|
+
recordId: string;
|
|
5
|
+
title: string;
|
|
6
|
+
url: string;
|
|
7
|
+
snippet: string;
|
|
8
|
+
publishedAt?: string;
|
|
9
|
+
raw: unknown;
|
|
10
|
+
}
|
|
11
|
+
export interface AdapterCallContext {
|
|
12
|
+
abortSignal: AbortSignal;
|
|
13
|
+
hostAllowlist?: ReadonlySet<string>;
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/research/adapters/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,kBAAkB,GAAG,eAAe,GAAG,KAAK,CAAC;AAE/E,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,GAAG,EAAE,OAAO,CAAC;CACd;AAED,MAAM,WAAW,kBAAkB;IACjC,WAAW,EAAE,WAAW,CAAC;IAEzB,aAAa,CAAC,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CACrC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/research/adapters/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
export declare function extractURLHosts(strings: readonly string[]): string[];
|
|
2
|
+
export interface AllowlistInput {
|
|
3
|
+
fetchAllowlistExtra: readonly string[];
|
|
4
|
+
userSources: readonly string[];
|
|
5
|
+
}
|
|
6
|
+
export type AllowlistProvenance = 'extra' | 'user_source';
|
|
7
|
+
export type HostAllowlist = ReadonlyMap<string, AllowlistProvenance>;
|
|
8
|
+
/**
|
|
9
|
+
* Build the per-task fetch allowlist as a provenance-aware map. Per spec
|
|
10
|
+
* §6.2 / §7.1 step 8, the private-network opt-in is gated on
|
|
11
|
+
* `provenance === 'extra'` — only operator-declared hosts in
|
|
12
|
+
* `fetchAllowlistExtra` are eligible, never hosts harvested from
|
|
13
|
+
* `userSources`. Build order: userSources first (lower precedence), then
|
|
14
|
+
* fetchAllowlistExtra overwrites — so collisions resolve to the operator's
|
|
15
|
+
* deliberate intent.
|
|
16
|
+
*
|
|
17
|
+
* Both sources are validated through canonicalizeHostname; invalid entries
|
|
18
|
+
* (IP literals, single-label names, malformed hostnames, overlong hosts)
|
|
19
|
+
* are silently skipped. This mirrors ResearchConfigSchema's HostString
|
|
20
|
+
* canonicalization as defense-in-depth for callers that have not parsed
|
|
21
|
+
* configuration first; silent skipping preserves URL-extraction behavior
|
|
22
|
+
* for untrusted userSources while keeping this builder non-throwing.
|
|
23
|
+
*/
|
|
24
|
+
export declare function buildHostAllowlist(input: AllowlistInput): HostAllowlist;
|
|
25
|
+
//# sourceMappingURL=allowlist.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"allowlist.d.ts","sourceRoot":"","sources":["../../src/research/allowlist.ts"],"names":[],"mappings":"AAyCA,wBAAgB,eAAe,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAsBpE;AAED,MAAM,WAAW,cAAc;IAC7B,mBAAmB,EAAE,SAAS,MAAM,EAAE,CAAC;IACvC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;CAChC;AAED,MAAM,MAAM,mBAAmB,GAAG,OAAO,GAAG,aAAa,CAAC;AAC1D,MAAM,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,EAAE,mBAAmB,CAAC,CAAC;AAErE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,cAAc,GAAG,aAAa,CAYvE"}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { isIP } from 'node:net';
|
|
2
|
+
const URL_REGEX = /\bhttps?:\/\/[^\s'"<>()]+/gi;
|
|
3
|
+
const FORBIDDEN_HOST_CHARS = /[\/:@?#]|:\/\//;
|
|
4
|
+
const MAX_HOSTNAME_LENGTH = 253;
|
|
5
|
+
const TRAILING_URL_PUNCTUATION = /[.,;:!?]+$/;
|
|
6
|
+
/**
|
|
7
|
+
* Validate and canonicalize a hostname string. Returns the canonical
|
|
8
|
+
* (lowercase, IDNA-encoded) form or null if the input is not a valid
|
|
9
|
+
* DNS FQDN. Mirrors the HostString rules in config/schema.ts as a
|
|
10
|
+
* defense-in-depth layer: even though ResearchConfigSchema already
|
|
11
|
+
* validates fetchAllowlistExtra via HostString, this function ensures
|
|
12
|
+
* that any caller of buildHostAllowlist — including callers that bypass
|
|
13
|
+
* schema parsing — cannot inject non-FQDN entries with "extra" provenance.
|
|
14
|
+
*/
|
|
15
|
+
function canonicalizeHostname(raw) {
|
|
16
|
+
const trimmed = raw.trim();
|
|
17
|
+
if (trimmed.length === 0 || trimmed.length > MAX_HOSTNAME_LENGTH)
|
|
18
|
+
return null;
|
|
19
|
+
if (FORBIDDEN_HOST_CHARS.test(trimmed))
|
|
20
|
+
return null;
|
|
21
|
+
// Strip trailing dot (DNS root label) — example.com. ≡ example.com
|
|
22
|
+
const stripped = trimmed.endsWith('.') ? trimmed.slice(0, -1) : trimmed;
|
|
23
|
+
if (stripped.length === 0 || stripped.length > MAX_HOSTNAME_LENGTH)
|
|
24
|
+
return null;
|
|
25
|
+
let canonical;
|
|
26
|
+
try {
|
|
27
|
+
canonical = new URL(`https://${stripped}`).hostname;
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return null;
|
|
31
|
+
}
|
|
32
|
+
if (canonical.length === 0 || canonical.length > MAX_HOSTNAME_LENGTH)
|
|
33
|
+
return null;
|
|
34
|
+
if (isIP(canonical) !== 0)
|
|
35
|
+
return null;
|
|
36
|
+
const labels = canonical.split('.');
|
|
37
|
+
if (labels.length < 2)
|
|
38
|
+
return null;
|
|
39
|
+
for (const label of labels) {
|
|
40
|
+
if (label.length === 0 || label.length > 63)
|
|
41
|
+
return null;
|
|
42
|
+
if (label.startsWith('-') || label.endsWith('-'))
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
return canonical;
|
|
46
|
+
}
|
|
47
|
+
export function extractURLHosts(strings) {
|
|
48
|
+
const out = [];
|
|
49
|
+
const seen = new Set();
|
|
50
|
+
for (const s of strings) {
|
|
51
|
+
const matches = s.match(URL_REGEX) ?? [];
|
|
52
|
+
for (const m of matches) {
|
|
53
|
+
try {
|
|
54
|
+
const candidate = m.replace(TRAILING_URL_PUNCTUATION, '');
|
|
55
|
+
const u = new URL(candidate);
|
|
56
|
+
if (!u.hostname)
|
|
57
|
+
continue;
|
|
58
|
+
const canonical = canonicalizeHostname(u.hostname);
|
|
59
|
+
if (canonical === null)
|
|
60
|
+
continue;
|
|
61
|
+
if (!seen.has(canonical)) {
|
|
62
|
+
seen.add(canonical);
|
|
63
|
+
out.push(canonical);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
catch {
|
|
67
|
+
// malformed URL — silently skip
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return out;
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Build the per-task fetch allowlist as a provenance-aware map. Per spec
|
|
75
|
+
* §6.2 / §7.1 step 8, the private-network opt-in is gated on
|
|
76
|
+
* `provenance === 'extra'` — only operator-declared hosts in
|
|
77
|
+
* `fetchAllowlistExtra` are eligible, never hosts harvested from
|
|
78
|
+
* `userSources`. Build order: userSources first (lower precedence), then
|
|
79
|
+
* fetchAllowlistExtra overwrites — so collisions resolve to the operator's
|
|
80
|
+
* deliberate intent.
|
|
81
|
+
*
|
|
82
|
+
* Both sources are validated through canonicalizeHostname; invalid entries
|
|
83
|
+
* (IP literals, single-label names, malformed hostnames, overlong hosts)
|
|
84
|
+
* are silently skipped. This mirrors ResearchConfigSchema's HostString
|
|
85
|
+
* canonicalization as defense-in-depth for callers that have not parsed
|
|
86
|
+
* configuration first; silent skipping preserves URL-extraction behavior
|
|
87
|
+
* for untrusted userSources while keeping this builder non-throwing.
|
|
88
|
+
*/
|
|
89
|
+
export function buildHostAllowlist(input) {
|
|
90
|
+
const map = new Map();
|
|
91
|
+
for (const h of extractURLHosts(input.userSources)) {
|
|
92
|
+
map.set(h, 'user_source');
|
|
93
|
+
}
|
|
94
|
+
for (const h of input.fetchAllowlistExtra) {
|
|
95
|
+
const canonical = canonicalizeHostname(h);
|
|
96
|
+
if (canonical !== null) {
|
|
97
|
+
map.set(canonical, 'extra');
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
return map;
|
|
101
|
+
}
|
|
102
|
+
//# sourceMappingURL=allowlist.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"allowlist.js","sourceRoot":"","sources":["../../src/research/allowlist.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAEhC,MAAM,SAAS,GAAG,6BAA6B,CAAC;AAEhD,MAAM,oBAAoB,GAAG,gBAAgB,CAAC;AAC9C,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,wBAAwB,GAAG,YAAY,CAAC;AAE9C;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAAC,GAAW;IACvC,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,mBAAmB;QAAE,OAAO,IAAI,CAAC;IAC9E,IAAI,oBAAoB,CAAC,IAAI,CAAC,OAAO,CAAC;QAAE,OAAO,IAAI,CAAC;IACpD,mEAAmE;IACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACxE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,mBAAmB;QAAE,OAAO,IAAI,CAAC;IAChF,IAAI,SAAiB,CAAC;IACtB,IAAI,CAAC;QACH,SAAS,GAAG,IAAI,GAAG,CAAC,WAAW,QAAQ,EAAE,CAAC,CAAC,QAAQ,CAAC;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,mBAAmB;QAAE,OAAO,IAAI,CAAC;IAClF,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACpC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,EAAE;YAAE,OAAO,IAAI,CAAC;QACzD,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC;YAAE,OAAO,IAAI,CAAC;IAChE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAA0B;IACxD,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,wBAAwB,EAAE,EAAE,CAAC,CAAC;gBAC1D,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;gBAC7B,IAAI,CAAC,CAAC,CAAC,QAAQ;oBAAE,SAAS;gBAC1B,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;gBACnD,IAAI,SAAS,KAAK,IAAI;oBAAE,SAAS;gBACjC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;oBACzB,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;oBACpB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,gCAAgC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAUD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,kBAAkB,CAAC,KAAqB;IACtD,MAAM,GAAG,GAAG,IAAI,GAAG,EAA+B,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,WAAW,CAAC,EAAE,CAAC;QACnD,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IAC5B,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,EAAE,CAAC;QAC1C,MAAM,SAAS,GAAG,oBAAoB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;YACvB,GAAG,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export type IPClass = 'public' | 'loopback' | 'private' | 'link-local-or-metadata' | 'unspecified' | 'multicast' | 'broadcast-or-reserved';
|
|
2
|
+
export declare class SsrfBlocked extends Error {
|
|
3
|
+
readonly code: string;
|
|
4
|
+
constructor(code: string, message: string);
|
|
5
|
+
}
|
|
6
|
+
export declare function classifyIP(ip: string): IPClass;
|
|
7
|
+
export interface ResolveOptions {
|
|
8
|
+
resolve?: (host: string) => Promise<string[]>;
|
|
9
|
+
allowPrivateForHost: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare function resolveAndPin(host: string, options: ResolveOptions): Promise<string>;
|
|
12
|
+
//# sourceMappingURL=ssrf-guard.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ssrf-guard.d.ts","sourceRoot":"","sources":["../../src/research/ssrf-guard.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,OAAO,GACf,QAAQ,GACR,UAAU,GACV,SAAS,GACT,wBAAwB,GACxB,aAAa,GACb,WAAW,GACX,uBAAuB,CAAC;AAE5B,qBAAa,WAAY,SAAQ,KAAK;aACR,IAAI,EAAE,MAAM;gBAAZ,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAI1D;AA8DD,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAyD9C;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9C,mBAAmB,EAAE,OAAO,CAAC;CAC9B;AA+CD,wBAAsB,aAAa,CACjC,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,MAAM,CAAC,CAyCjB"}
|