@zhixuan92/multi-model-agent-core 3.11.1 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/config/read-only-review-flag.d.ts +1 -1
- package/dist/config/read-only-review-flag.d.ts.map +1 -1
- package/dist/config/read-only-review-flag.js +1 -0
- package/dist/config/read-only-review-flag.js.map +1 -1
- package/dist/config/schema.d.ts +47 -0
- package/dist/config/schema.d.ts.map +1 -1
- package/dist/config/schema.js +102 -0
- package/dist/config/schema.js.map +1 -1
- package/dist/delegate-with-escalation.d.ts.map +1 -1
- package/dist/delegate-with-escalation.js +1 -0
- package/dist/delegate-with-escalation.js.map +1 -1
- package/dist/executors/explore.d.ts +13 -0
- package/dist/executors/explore.d.ts.map +1 -0
- package/dist/executors/explore.js +352 -0
- package/dist/executors/explore.js.map +1 -0
- package/dist/executors/index.d.ts +1 -0
- package/dist/executors/index.d.ts.map +1 -1
- package/dist/executors/index.js +1 -0
- package/dist/executors/index.js.map +1 -1
- package/dist/intake/compilers/explore.d.ts +29 -0
- package/dist/intake/compilers/explore.d.ts.map +1 -0
- package/dist/intake/compilers/explore.js +108 -0
- package/dist/intake/compilers/explore.js.map +1 -0
- package/dist/observability/events.d.ts +92 -0
- package/dist/observability/events.d.ts.map +1 -1
- package/dist/observability/events.js +49 -0
- package/dist/observability/events.js.map +1 -1
- package/dist/reporting/compose-explore-headline.d.ts +14 -0
- package/dist/reporting/compose-explore-headline.d.ts.map +1 -0
- package/dist/reporting/compose-explore-headline.js +14 -0
- package/dist/reporting/compose-explore-headline.js.map +1 -0
- package/dist/reporting/derive-explore-status.d.ts +18 -0
- package/dist/reporting/derive-explore-status.d.ts.map +1 -0
- package/dist/reporting/derive-explore-status.js +19 -0
- package/dist/reporting/derive-explore-status.js.map +1 -0
- package/dist/reporting/parse-explore-report.d.ts +38 -0
- package/dist/reporting/parse-explore-report.d.ts.map +1 -0
- package/dist/reporting/parse-explore-report.js +185 -0
- package/dist/reporting/parse-explore-report.js.map +1 -0
- package/dist/research/adapters/arxiv.d.ts +6 -0
- package/dist/research/adapters/arxiv.d.ts.map +1 -0
- package/dist/research/adapters/arxiv.js +36 -0
- package/dist/research/adapters/arxiv.js.map +1 -0
- package/dist/research/adapters/generic-rss.d.ts +8 -0
- package/dist/research/adapters/generic-rss.d.ts.map +1 -0
- package/dist/research/adapters/generic-rss.js +26 -0
- package/dist/research/adapters/generic-rss.js.map +1 -0
- package/dist/research/adapters/github-search.d.ts +7 -0
- package/dist/research/adapters/github-search.d.ts.map +1 -0
- package/dist/research/adapters/github-search.js +95 -0
- package/dist/research/adapters/github-search.js.map +1 -0
- package/dist/research/adapters/index.d.ts +8 -0
- package/dist/research/adapters/index.d.ts.map +1 -0
- package/dist/research/adapters/index.js +17 -0
- package/dist/research/adapters/index.js.map +1 -0
- package/dist/research/adapters/semantic-scholar.d.ts +6 -0
- package/dist/research/adapters/semantic-scholar.d.ts.map +1 -0
- package/dist/research/adapters/semantic-scholar.js +54 -0
- package/dist/research/adapters/semantic-scholar.js.map +1 -0
- package/dist/research/adapters/types.d.ts +15 -0
- package/dist/research/adapters/types.d.ts.map +1 -0
- package/dist/research/adapters/types.js +2 -0
- package/dist/research/adapters/types.js.map +1 -0
- package/dist/research/allowlist.d.ts +25 -0
- package/dist/research/allowlist.d.ts.map +1 -0
- package/dist/research/allowlist.js +102 -0
- package/dist/research/allowlist.js.map +1 -0
- package/dist/research/ssrf-guard.d.ts +12 -0
- package/dist/research/ssrf-guard.d.ts.map +1 -0
- package/dist/research/ssrf-guard.js +209 -0
- package/dist/research/ssrf-guard.js.map +1 -0
- package/dist/research/types.d.ts +14 -0
- package/dist/research/types.d.ts.map +1 -0
- package/dist/research/types.js +2 -0
- package/dist/research/types.js.map +1 -0
- package/dist/research/untrusted-content.d.ts +13 -0
- package/dist/research/untrusted-content.d.ts.map +1 -0
- package/dist/research/untrusted-content.js +9 -0
- package/dist/research/untrusted-content.js.map +1 -0
- package/dist/research/web-fetch.d.ts +50 -0
- package/dist/research/web-fetch.d.ts.map +1 -0
- package/dist/research/web-fetch.js +411 -0
- package/dist/research/web-fetch.js.map +1 -0
- package/dist/research/web-search.d.ts +28 -0
- package/dist/research/web-search.d.ts.map +1 -0
- package/dist/research/web-search.js +134 -0
- package/dist/research/web-search.js.map +1 -0
- package/dist/runners/base/research-tools.d.ts +47 -0
- package/dist/runners/base/research-tools.d.ts.map +1 -0
- package/dist/runners/base/research-tools.js +67 -0
- package/dist/runners/base/research-tools.js.map +1 -0
- package/dist/runners/claude-runner.d.ts.map +1 -1
- package/dist/runners/claude-runner.js +27 -1
- package/dist/runners/claude-runner.js.map +1 -1
- package/dist/runners/codex-runner.d.ts.map +1 -1
- package/dist/runners/codex-runner.js +25 -0
- package/dist/runners/codex-runner.js.map +1 -1
- package/dist/runners/openai-runner.d.ts.map +1 -1
- package/dist/runners/openai-runner.js +34 -1
- package/dist/runners/openai-runner.js.map +1 -1
- package/dist/runners/types.d.ts +6 -0
- package/dist/runners/types.d.ts.map +1 -1
- package/dist/tool-schemas/explore.d.ts +9 -0
- package/dist/tool-schemas/explore.d.ts.map +1 -0
- package/dist/tool-schemas/explore.js +64 -0
- package/dist/tool-schemas/explore.js.map +1 -0
- package/dist/types.d.ts +33 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +16 -1
|
@@ -0,0 +1,411 @@
|
|
|
1
|
+
import { request, Agent } from 'undici';
|
|
2
|
+
import { isIP } from 'node:net';
|
|
3
|
+
import { Readability } from '@mozilla/readability';
|
|
4
|
+
import { JSDOM } from 'jsdom';
|
|
5
|
+
import { wrapFetchedContent } from './untrusted-content.js';
|
|
6
|
+
import { resolveAndPin, SsrfBlocked } from './ssrf-guard.js';
|
|
7
|
+
export const defaultIPPinningDispatcher = (host, pinnedIP, cfg) => new Agent({
|
|
8
|
+
connect: {
|
|
9
|
+
lookup: (_h, _o, cb) => cb(null, pinnedIP, pinnedIP.includes(':') ? 6 : 4),
|
|
10
|
+
servername: host,
|
|
11
|
+
},
|
|
12
|
+
connectTimeout: cfg.connectTimeoutMs,
|
|
13
|
+
});
|
|
14
|
+
const ALLOWED_CT = new Set([
|
|
15
|
+
'text/html',
|
|
16
|
+
'text/plain',
|
|
17
|
+
'application/xml',
|
|
18
|
+
'application/atom+xml',
|
|
19
|
+
'application/rss+xml',
|
|
20
|
+
'application/json',
|
|
21
|
+
]);
|
|
22
|
+
/** Post-extraction text cap — default 64 KiB. */
|
|
23
|
+
const RETURNED_TEXT_CAP = 64 * 1024;
|
|
24
|
+
/** Max bytes to drain from a redirect response body before giving up. */
|
|
25
|
+
const REDIRECT_DRAIN_CAP = 64 * 1024;
|
|
26
|
+
/**
|
|
27
|
+
* Races a promise against an AbortSignal. If the signal fires first, throws
|
|
28
|
+
* a DOMException with name 'AbortError' so callers can map it to timeout.
|
|
29
|
+
*/
|
|
30
|
+
async function withDeadline(promise, signal) {
|
|
31
|
+
if (signal.aborted)
|
|
32
|
+
throw new DOMException('Aborted', 'AbortError');
|
|
33
|
+
return new Promise((resolve, reject) => {
|
|
34
|
+
const onAbort = () => reject(new DOMException('Aborted', 'AbortError'));
|
|
35
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
36
|
+
promise.then((result) => {
|
|
37
|
+
signal.removeEventListener('abort', onAbort);
|
|
38
|
+
resolve(result);
|
|
39
|
+
}, (err) => {
|
|
40
|
+
signal.removeEventListener('abort', onAbort);
|
|
41
|
+
reject(err);
|
|
42
|
+
});
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
async function validateAndPinURL(raw, hostAllowlist, privateNetworkHosts, resolveIP, signal) {
|
|
46
|
+
let url;
|
|
47
|
+
try {
|
|
48
|
+
url = new URL(raw);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
return { ok: false, reasonCode: 'web_fetch_invalid_url' };
|
|
52
|
+
}
|
|
53
|
+
if (url.protocol !== 'https:') {
|
|
54
|
+
return { ok: false, reasonCode: 'web_fetch_invalid_scheme' };
|
|
55
|
+
}
|
|
56
|
+
// Strip brackets for IPv6 isIP check
|
|
57
|
+
const stripped = url.hostname.replace(/^\[|\]$/g, '');
|
|
58
|
+
if (isIP(stripped) !== 0) {
|
|
59
|
+
return { ok: false, reasonCode: 'web_fetch_ip_literal_blocked' };
|
|
60
|
+
}
|
|
61
|
+
const host = url.hostname.toLowerCase();
|
|
62
|
+
if (!hostAllowlist.has(host)) {
|
|
63
|
+
return { ok: false, reasonCode: 'web_fetch_off_allowlist', host };
|
|
64
|
+
}
|
|
65
|
+
const allowPrivate = privateNetworkHosts.has(host);
|
|
66
|
+
let pinnedIP;
|
|
67
|
+
try {
|
|
68
|
+
pinnedIP = await withDeadline(resolveAndPin(host, {
|
|
69
|
+
resolve: resolveIP ? async (h) => [await resolveIP(h)] : undefined,
|
|
70
|
+
allowPrivateForHost: allowPrivate,
|
|
71
|
+
}), signal);
|
|
72
|
+
}
|
|
73
|
+
catch (e) {
|
|
74
|
+
if (e instanceof DOMException && e.name === 'AbortError') {
|
|
75
|
+
throw e; // let caller map to timeout
|
|
76
|
+
}
|
|
77
|
+
if (e instanceof SsrfBlocked) {
|
|
78
|
+
return { ok: false, reasonCode: e.code, host };
|
|
79
|
+
}
|
|
80
|
+
// DNS resolution threw before SsrfBlocked could wrap it (e.g. signals, system errors).
|
|
81
|
+
// resolveAndPin wraps its own resolver errors as web_fetch_dns_resolution_failed,
|
|
82
|
+
// so this path is only reached for truly unexpected failures.
|
|
83
|
+
return { ok: false, reasonCode: 'web_fetch_dns_resolution_failed', host };
|
|
84
|
+
}
|
|
85
|
+
return { ok: true, url, host, pinnedIP };
|
|
86
|
+
}
|
|
87
|
+
function extractContentType(headers) {
|
|
88
|
+
const raw = headers['content-type'];
|
|
89
|
+
if (typeof raw === 'string') {
|
|
90
|
+
return raw.split(';')[0].trim().toLowerCase();
|
|
91
|
+
}
|
|
92
|
+
if (Array.isArray(raw) && raw.length > 0) {
|
|
93
|
+
return String(raw[0]).split(';')[0].trim().toLowerCase();
|
|
94
|
+
}
|
|
95
|
+
return '';
|
|
96
|
+
}
|
|
97
|
+
function isRedirect(status) {
|
|
98
|
+
return status === 301 || status === 302 || status === 303 || status === 307 || status === 308;
|
|
99
|
+
}
|
|
100
|
+
function extractLocation(headers) {
|
|
101
|
+
const loc = headers['location'];
|
|
102
|
+
if (typeof loc === 'string')
|
|
103
|
+
return loc;
|
|
104
|
+
if (Array.isArray(loc) && loc.length > 0)
|
|
105
|
+
return String(loc[0]);
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
function extractBodyFromHTML(html) {
|
|
109
|
+
let dom;
|
|
110
|
+
try {
|
|
111
|
+
dom = new JSDOM(html, { url: 'https://localhost/' });
|
|
112
|
+
const reader = new Readability(dom.window.document);
|
|
113
|
+
const article = reader.parse();
|
|
114
|
+
if (article?.textContent) {
|
|
115
|
+
return article.textContent;
|
|
116
|
+
}
|
|
117
|
+
return dom.window.document.body?.textContent?.trim() ?? html;
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
try {
|
|
121
|
+
return dom?.window.document.body?.textContent?.trim() || html;
|
|
122
|
+
}
|
|
123
|
+
catch {
|
|
124
|
+
return html;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
function stripCredentialsFromURL(url) {
|
|
129
|
+
if (!url.username && !url.password)
|
|
130
|
+
return false;
|
|
131
|
+
url.username = '';
|
|
132
|
+
url.password = '';
|
|
133
|
+
return true;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Read a response body with a byte cap and abort-signal awareness.
|
|
137
|
+
* Stream errors (non-abort) are re-thrown so the caller can map them to
|
|
138
|
+
* web_fetch_body_read_failed rather than silently returning partial content.
|
|
139
|
+
*/
|
|
140
|
+
async function readBody(body, maxBytes, signal) {
|
|
141
|
+
if (!body)
|
|
142
|
+
return { text: '', bytesReturned: 0, truncated: false };
|
|
143
|
+
const chunks = [];
|
|
144
|
+
let total = 0;
|
|
145
|
+
let truncated = false;
|
|
146
|
+
for await (const chunk of body) {
|
|
147
|
+
if (signal.aborted) {
|
|
148
|
+
throw new DOMException('Aborted', 'AbortError');
|
|
149
|
+
}
|
|
150
|
+
const remaining = maxBytes - total;
|
|
151
|
+
if (remaining <= 0) {
|
|
152
|
+
truncated = true;
|
|
153
|
+
break;
|
|
154
|
+
}
|
|
155
|
+
const value = typeof chunk === 'string' ? Buffer.from(chunk) : chunk;
|
|
156
|
+
if (value.length > remaining) {
|
|
157
|
+
chunks.push(value.subarray(0, remaining));
|
|
158
|
+
total += remaining;
|
|
159
|
+
truncated = true;
|
|
160
|
+
break;
|
|
161
|
+
}
|
|
162
|
+
chunks.push(value);
|
|
163
|
+
total += value.length;
|
|
164
|
+
}
|
|
165
|
+
const decoder = new TextDecoder();
|
|
166
|
+
let text = '';
|
|
167
|
+
for (const chunk of chunks) {
|
|
168
|
+
text += decoder.decode(chunk, { stream: true });
|
|
169
|
+
}
|
|
170
|
+
text += decoder.decode();
|
|
171
|
+
return { text, bytesReturned: total, truncated };
|
|
172
|
+
}
|
|
173
|
+
/**
|
|
174
|
+
* Drain a response body with a size cap and abort-signal awareness.
|
|
175
|
+
* Used for redirect responses and rejected content types.
|
|
176
|
+
*/
|
|
177
|
+
async function drainBody(body, signal) {
|
|
178
|
+
if (!body)
|
|
179
|
+
return;
|
|
180
|
+
let drained = 0;
|
|
181
|
+
try {
|
|
182
|
+
for await (const chunk of body) {
|
|
183
|
+
if (signal.aborted)
|
|
184
|
+
break;
|
|
185
|
+
let len = 0;
|
|
186
|
+
if (typeof chunk === 'string') {
|
|
187
|
+
len = Buffer.byteLength(chunk);
|
|
188
|
+
}
|
|
189
|
+
else if (chunk instanceof Uint8Array) {
|
|
190
|
+
len = chunk.length;
|
|
191
|
+
}
|
|
192
|
+
else if (Buffer.isBuffer(chunk)) {
|
|
193
|
+
len = chunk.length;
|
|
194
|
+
}
|
|
195
|
+
drained += len;
|
|
196
|
+
if (drained > REDIRECT_DRAIN_CAP)
|
|
197
|
+
break;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
catch {
|
|
201
|
+
// drain errors are ignorable
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
const REDIRECT_ERR_CODE_MAP = {
|
|
205
|
+
web_fetch_off_allowlist: 'web_fetch_redirect_off_allowlist',
|
|
206
|
+
web_fetch_invalid_url: 'web_fetch_redirect_invalid_url',
|
|
207
|
+
web_fetch_invalid_scheme: 'web_fetch_redirect_scheme_downgrade',
|
|
208
|
+
web_fetch_ip_literal_blocked: 'web_fetch_redirect_ip_literal_blocked',
|
|
209
|
+
web_fetch_private_ip_blocked: 'web_fetch_redirect_private_ip_blocked',
|
|
210
|
+
web_fetch_reserved_ip_blocked: 'web_fetch_redirect_reserved_ip_blocked',
|
|
211
|
+
};
|
|
212
|
+
function isUndiciTimeout(err) {
|
|
213
|
+
if (!(err instanceof Error))
|
|
214
|
+
return false;
|
|
215
|
+
const code = err.code;
|
|
216
|
+
return code === 'UND_ERR_CONNECT_TIMEOUT'
|
|
217
|
+
|| code === 'UND_ERR_HEADERS_TIMEOUT'
|
|
218
|
+
|| code === 'UND_ERR_BODY_TIMEOUT';
|
|
219
|
+
}
|
|
220
|
+
/** Close/destroy a dispatcher if it supports it (Agent instances do). */
|
|
221
|
+
function closeDispatcher(d) {
|
|
222
|
+
if (!d)
|
|
223
|
+
return;
|
|
224
|
+
try {
|
|
225
|
+
if (typeof d.destroy === 'function') {
|
|
226
|
+
d.destroy();
|
|
227
|
+
}
|
|
228
|
+
else if (typeof d.close === 'function') {
|
|
229
|
+
d.close().catch(() => { });
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
catch {
|
|
233
|
+
// best-effort cleanup
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
function mapRequestError(err, signal, host) {
|
|
237
|
+
if (signal.aborted) {
|
|
238
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', host, credentialsStripped: false };
|
|
239
|
+
}
|
|
240
|
+
if (err instanceof DOMException && err.name === 'AbortError') {
|
|
241
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', host, credentialsStripped: false };
|
|
242
|
+
}
|
|
243
|
+
if (isUndiciTimeout(err)) {
|
|
244
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', host, credentialsStripped: false };
|
|
245
|
+
}
|
|
246
|
+
return { status: 'error', reasonCode: 'web_fetch_request_failed', host, credentialsStripped: false };
|
|
247
|
+
}
|
|
248
|
+
export async function webFetch(input) {
|
|
249
|
+
const { cfg, hostAllowlist } = input;
|
|
250
|
+
const privateNetworkHosts = input.privateNetworkHosts ?? new Set();
|
|
251
|
+
let credentialsStripped = false;
|
|
252
|
+
// Strip credentials BEFORE any logging. Mutate to avoid leaking via toString.
|
|
253
|
+
let initial;
|
|
254
|
+
try {
|
|
255
|
+
initial = new URL(input.url);
|
|
256
|
+
}
|
|
257
|
+
catch {
|
|
258
|
+
return { status: 'error', reasonCode: 'web_fetch_invalid_url', credentialsStripped };
|
|
259
|
+
}
|
|
260
|
+
credentialsStripped = stripCredentialsFromURL(initial);
|
|
261
|
+
// Total-deadline AbortController scopes the entire op (including DNS, redirects, body reads).
|
|
262
|
+
const totalCtrl = new AbortController();
|
|
263
|
+
const totalTimer = setTimeout(() => totalCtrl.abort(), cfg.totalDeadlineMs);
|
|
264
|
+
try {
|
|
265
|
+
let currentURL = initial.toString();
|
|
266
|
+
let redirects = 0;
|
|
267
|
+
while (true) {
|
|
268
|
+
// validateAndPinURL is raced against totalCtrl.signal so a hanging DNS
|
|
269
|
+
// resolver (including injected resolveIP) cannot exceed totalDeadlineMs.
|
|
270
|
+
let v;
|
|
271
|
+
try {
|
|
272
|
+
v = await validateAndPinURL(currentURL, hostAllowlist, privateNetworkHosts, input.resolveIP, totalCtrl.signal);
|
|
273
|
+
}
|
|
274
|
+
catch (e) {
|
|
275
|
+
if (e instanceof DOMException && e.name === 'AbortError') {
|
|
276
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', credentialsStripped };
|
|
277
|
+
}
|
|
278
|
+
return { status: 'error', reasonCode: 'web_fetch_dns_resolution_failed', credentialsStripped };
|
|
279
|
+
}
|
|
280
|
+
if (!v.ok) {
|
|
281
|
+
if (redirects > 0) {
|
|
282
|
+
const mapped = REDIRECT_ERR_CODE_MAP[v.reasonCode] ?? v.reasonCode;
|
|
283
|
+
return { status: 'error', reasonCode: mapped, host: v.host, credentialsStripped };
|
|
284
|
+
}
|
|
285
|
+
return { status: 'error', reasonCode: v.reasonCode, host: v.host, credentialsStripped };
|
|
286
|
+
}
|
|
287
|
+
// Build IP-pinning dispatcher. In tests createDispatcher returns undefined
|
|
288
|
+
// so undici falls back to the global dispatcher (MockAgent).
|
|
289
|
+
const usingDefaultDispatcher = input.createDispatcher === undefined;
|
|
290
|
+
const dispatcher = (input.createDispatcher ?? defaultIPPinningDispatcher)(v.host, v.pinnedIP, cfg);
|
|
291
|
+
let res;
|
|
292
|
+
try {
|
|
293
|
+
// undici 8 `request()` does not follow redirects by default; we handle
|
|
294
|
+
// them manually below to re-validate against the per-task allowlist.
|
|
295
|
+
res = await request(v.url.toString(), {
|
|
296
|
+
method: 'GET',
|
|
297
|
+
headersTimeout: cfg.connectTimeoutMs,
|
|
298
|
+
...(dispatcher ? { dispatcher } : {}),
|
|
299
|
+
signal: totalCtrl.signal,
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
catch (e) {
|
|
303
|
+
if (usingDefaultDispatcher)
|
|
304
|
+
closeDispatcher(dispatcher);
|
|
305
|
+
return { ...mapRequestError(e, totalCtrl.signal, v.host), credentialsStripped };
|
|
306
|
+
}
|
|
307
|
+
// Handle redirects manually (maxRedirections: 0 on undici)
|
|
308
|
+
if (isRedirect(res.statusCode)) {
|
|
309
|
+
redirects++;
|
|
310
|
+
if (redirects > cfg.maxRedirects) {
|
|
311
|
+
if (usingDefaultDispatcher)
|
|
312
|
+
closeDispatcher(dispatcher);
|
|
313
|
+
return { status: 'error', reasonCode: 'web_fetch_too_many_redirects', host: v.host, credentialsStripped };
|
|
314
|
+
}
|
|
315
|
+
const location = extractLocation(res.headers);
|
|
316
|
+
if (!location) {
|
|
317
|
+
if (usingDefaultDispatcher)
|
|
318
|
+
closeDispatcher(dispatcher);
|
|
319
|
+
return { status: 'error', reasonCode: 'web_fetch_redirect_missing_location', host: v.host, credentialsStripped };
|
|
320
|
+
}
|
|
321
|
+
let nextURL;
|
|
322
|
+
try {
|
|
323
|
+
nextURL = new URL(location, v.url);
|
|
324
|
+
}
|
|
325
|
+
catch {
|
|
326
|
+
if (usingDefaultDispatcher)
|
|
327
|
+
closeDispatcher(dispatcher);
|
|
328
|
+
return { status: 'error', reasonCode: 'web_fetch_redirect_invalid_url', host: v.host, credentialsStripped };
|
|
329
|
+
}
|
|
330
|
+
credentialsStripped = stripCredentialsFromURL(nextURL) || credentialsStripped;
|
|
331
|
+
currentURL = nextURL.toString();
|
|
332
|
+
// Drain redirect body with cap to free connection
|
|
333
|
+
await drainBody(res.body, totalCtrl.signal);
|
|
334
|
+
if (usingDefaultDispatcher)
|
|
335
|
+
closeDispatcher(dispatcher);
|
|
336
|
+
if (totalCtrl.signal.aborted) {
|
|
337
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
338
|
+
}
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
// Check content type. Missing content-type (empty) is allowed through;
|
|
342
|
+
// only explicit unsupported types are rejected.
|
|
343
|
+
const contentType = extractContentType(res.headers);
|
|
344
|
+
if (contentType && !ALLOWED_CT.has(contentType)) {
|
|
345
|
+
await drainBody(res.body, totalCtrl.signal);
|
|
346
|
+
if (usingDefaultDispatcher)
|
|
347
|
+
closeDispatcher(dispatcher);
|
|
348
|
+
if (totalCtrl.signal.aborted) {
|
|
349
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
350
|
+
}
|
|
351
|
+
return { status: 'error', reasonCode: 'web_fetch_unsupported_content_type', host: v.host, credentialsStripped };
|
|
352
|
+
}
|
|
353
|
+
// Read body with size cap. Stream errors map to web_fetch_body_read_failed.
|
|
354
|
+
let rawText;
|
|
355
|
+
let bytesReturned;
|
|
356
|
+
let truncated;
|
|
357
|
+
try {
|
|
358
|
+
const rawBody = res.body;
|
|
359
|
+
const result = await readBody(rawBody, cfg.maxBodyBytes, totalCtrl.signal);
|
|
360
|
+
rawText = result.text;
|
|
361
|
+
bytesReturned = result.bytesReturned;
|
|
362
|
+
truncated = result.truncated;
|
|
363
|
+
}
|
|
364
|
+
catch (e) {
|
|
365
|
+
if (usingDefaultDispatcher)
|
|
366
|
+
closeDispatcher(dispatcher);
|
|
367
|
+
if (e instanceof DOMException && e.name === 'AbortError') {
|
|
368
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
369
|
+
}
|
|
370
|
+
if (totalCtrl.signal.aborted) {
|
|
371
|
+
return { status: 'error', reasonCode: 'web_fetch_timeout', host: v.host, credentialsStripped };
|
|
372
|
+
}
|
|
373
|
+
return { status: 'error', reasonCode: 'web_fetch_body_read_failed', host: v.host, credentialsStripped };
|
|
374
|
+
}
|
|
375
|
+
if (usingDefaultDispatcher)
|
|
376
|
+
closeDispatcher(dispatcher);
|
|
377
|
+
// Extract content for the worker-facing body
|
|
378
|
+
let extracted = rawText;
|
|
379
|
+
if (contentType === 'text/html') {
|
|
380
|
+
extracted = extractBodyFromHTML(rawText);
|
|
381
|
+
}
|
|
382
|
+
// text/plain, application/json, application/xml, application/atom+xml,
|
|
383
|
+
// application/rss+xml all keep rawText as-is for the wrapped body too
|
|
384
|
+
// Apply post-extraction text cap
|
|
385
|
+
let textTruncated = false;
|
|
386
|
+
if (extracted.length > RETURNED_TEXT_CAP) {
|
|
387
|
+
extracted = extracted.slice(0, RETURNED_TEXT_CAP);
|
|
388
|
+
textTruncated = true;
|
|
389
|
+
}
|
|
390
|
+
const wrapped = wrapFetchedContent({
|
|
391
|
+
url: v.url.toString(),
|
|
392
|
+
host: v.host,
|
|
393
|
+
content: extracted,
|
|
394
|
+
});
|
|
395
|
+
return {
|
|
396
|
+
status: 'ok',
|
|
397
|
+
body: wrapped,
|
|
398
|
+
rawText,
|
|
399
|
+
host: v.host,
|
|
400
|
+
bytesReturned,
|
|
401
|
+
truncated,
|
|
402
|
+
textTruncated,
|
|
403
|
+
credentialsStripped,
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
finally {
|
|
408
|
+
clearTimeout(totalTimer);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
//# sourceMappingURL=web-fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-fetch.js","sourceRoot":"","sources":["../../src/research/web-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,QAAQ,CAAC;AACxC,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAChC,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAE9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAsB7D,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC,IAAY,EAAE,QAAgB,EAAE,GAA4B,EAAE,EAAE,CACzG,IAAI,KAAK,CAAC;IACR,OAAO,EAAE;QACP,MAAM,EAAE,CAAC,EAAU,EAAE,EAAW,EAAE,EAA+D,EAAE,EAAE,CACnG,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,UAAU,EAAE,IAAI;KACjB;IACD,cAAc,EAAE,GAAG,CAAC,gBAAgB;CACrC,CAAC,CAAC;AA+BL,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC;IACzB,WAAW;IACX,YAAY;IACZ,iBAAiB;IACjB,sBAAsB;IACtB,qBAAqB;IACrB,kBAAkB;CACnB,CAAC,CAAC;AAEH,iDAAiD;AACjD,MAAM,iBAAiB,GAAG,EAAE,GAAG,IAAI,CAAC;AAEpC,yEAAyE;AACzE,MAAM,kBAAkB,GAAG,EAAE,GAAG,IAAI,CAAC;AAerC;;;GAGG;AACH,KAAK,UAAU,YAAY,CAAI,OAAmB,EAAE,MAAmB;IACrE,IAAI,MAAM,CAAC,OAAO;QAAE,MAAM,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;IACpE,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC,CAAC;QACxE,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CACV,CAAC,MAAM,EAAE,EAAE;YACT,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,CAAC,MAAM,CAAC,CAAC;QAClB,CAAC,EACD,CAAC,GAAG,EAAE,EAAE;YACN,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC7C,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,GAAW,EACX,aAAkC,EAClC,mBAAwC,EACxC,SAAqC,EACrC,MAAmB;IAEnB,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,uBAAuB,EAAE,CAAC;IAC5D,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,0BAA0B,EAAE,CAAC;IAC/D,CAAC;IACD,qCAAqC;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IACtD,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,8BAA8B,EAAE,CAAC;IACnE,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IACxC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,yBAAyB,EAAE,IAAI,EAAE,CAAC;IACpE,CAAC;IACD,MAAM,YAAY,GAAG,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IACnD,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,YAAY,CAC3B,aAAa,CAAC,IAAI,EAAE;YAClB,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS;YAClE,mBAAmB,EAAE,YAAY;SAClC,CAAC,EACF,MAAM,CACP,CAAC;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;YACzD,MAAM,CAAC,CAAC,CAAC,4BAA4B;QACvC,CAAC;QACD,IAAI,CAAC,YAAY,WAAW,EAAE,CAAC;YAC7B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;QACjD,CAAC;QACD,uFAAuF;QACvF,kFAAkF;QAClF,8DAA8D;QAC9D,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,iCAAiC,EAAE,IAAI,EAAE,CAAC;IAC5E,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;AAC3C,CAAC;AAED,SAAS,kBAAkB,CAAC,OAAsD;IAChF,MAAM,GAAG,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpC,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,OAAO,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IACjD,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC5D,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,UAAU,CAAC,MAAc;IAChC,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;AAChG,CAAC;AAED,SAAS,eAAe,CAAC,OAAsD;IAC7E,MAAM,GAAG,GAAG,OAAO,CAAC,UAAU,CAAC,CAAC;IAChC,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,GAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,OAAO,EAAE,WAAW,EAAE,CAAC;YACzB,OAAO,OAAO,CAAC,WAAW,CAAC;QAC7B,CAAC;QACD,OAAO,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,IAAI,CAAC;QAChE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,GAAQ;IACvC,IAAI,CAAC,GAAG,CAAC,QAAQ,IAAI,CAAC,GAAG,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAC;IACjD,GAAG,CAAC,QAAQ,GAAG,EAAE,CAAC;IAClB,GAAG,CAAC,QAAQ,GAAG,EAAE,CAAC;IAClB,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CACrB,IAA+C,EAC/C,QAAgB,EAChB,MAAmB;IAEnB,IAAI,CAAC,IAAI;QAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IAEnE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,SAAS,GAAG,KAAK,CAAC;IAEtB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,IAAI,YAAY,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,SAAS,GAAG,QAAQ,GAAG,KAAK,CAAC;QACnC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACnB,SAAS,GAAG,IAAI,CAAC;YACjB,MAAM;QACR,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QACrE,IAAI,KAAK,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC;YAC1C,KAAK,IAAI,SAAS,CAAC;YACnB,SAAS,GAAG,IAAI,CAAC;YACjB,MAAM;QACR,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnB,KAAK,IAAI,KAAK,CAAC,MAAM,CAAC;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;IAClC,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,IAAI,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;IACzB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;AACnD,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,SAAS,CACtB,IAAmC,EACnC,MAAmB;IAEnB,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,IAAI,EAAE,CAAC;YAC/B,IAAI,MAAM,CAAC,OAAO;gBAAE,MAAM;YAC1B,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC9B,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YACjC,CAAC;iBAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;gBACvC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;YACrB,CAAC;iBAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBAClC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC;YACrB,CAAC;YACD,OAAO,IAAI,GAAG,CAAC;YACf,IAAI,OAAO,GAAG,kBAAkB;gBAAE,MAAM;QAC1C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,6BAA6B;IAC/B,CAAC;AACH,CAAC;AAED,MAAM,qBAAqB,GAA2B;IACpD,uBAAuB,EAAE,kCAAkC;IAC3D,qBAAqB,EAAE,gCAAgC;IACvD,wBAAwB,EAAE,qCAAqC;IAC/D,4BAA4B,EAAE,uCAAuC;IACrE,4BAA4B,EAAE,uCAAuC;IACrE,6BAA6B,EAAE,wCAAwC;CACxE,CAAC;AAEF,SAAS,eAAe,CAAC,GAAY;IACnC,IAAI,CAAC,CAAC,GAAG,YAAY,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC1C,MAAM,IAAI,GAAI,GAAyB,CAAC,IAAI,CAAC;IAC7C,OAAO,IAAI,KAAK,yBAAyB;WAClC,IAAI,KAAK,yBAAyB;WAClC,IAAI,KAAK,sBAAsB,CAAC;AACzC,CAAC;AAED,yEAAyE;AACzE,SAAS,eAAe,CAAC,CAA0C;IACjE,IAAI,CAAC,CAAC;QAAE,OAAO;IACf,IAAI,CAAC;QACH,IAAI,OAAQ,CAA8B,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;YACjE,CAA6B,CAAC,OAAO,EAAE,CAAC;QAC3C,CAAC;aAAM,IAAI,OAAQ,CAAqC,CAAC,KAAK,KAAK,UAAU,EAAE,CAAC;YAC7E,CAAoC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,sBAAsB;IACxB,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,GAAY,EAAE,MAAmB,EAAE,IAAY;IACtE,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;IAChG,CAAC;IACD,IAAI,GAAG,YAAY,YAAY,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;QAC7D,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;IAChG,CAAC;IACD,IAAI,eAAe,CAAC,GAAG,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;IAChG,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,0BAA0B,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;AACvG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,KAAoB;IACjD,MAAM,EAAE,GAAG,EAAE,aAAa,EAAE,GAAG,KAAK,CAAC;IACrC,MAAM,mBAAmB,GAAG,KAAK,CAAC,mBAAmB,IAAI,IAAI,GAAG,EAAU,CAAC;IAC3E,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAEhC,8EAA8E;IAC9E,IAAI,OAAY,CAAC;IACjB,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,uBAAuB,EAAE,mBAAmB,EAAE,CAAC;IACvF,CAAC;IACD,mBAAmB,GAAG,uBAAuB,CAAC,OAAO,CAAC,CAAC;IAEvD,8FAA8F;IAC9F,MAAM,SAAS,GAAG,IAAI,eAAe,EAAE,CAAC;IACxC,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,GAAG,CAAC,eAAe,CAAC,CAAC;IAE5E,IAAI,CAAC;QACH,IAAI,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QACpC,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,OAAO,IAAI,EAAE,CAAC;YACZ,uEAAuE;YACvE,yEAAyE;YACzE,IAAI,CAAkC,CAAC;YACvC,IAAI,CAAC;gBACH,CAAC,GAAG,MAAM,iBAAiB,CACzB,UAAU,EACV,aAAa,EACb,mBAAmB,EACnB,KAAK,CAAC,SAAS,EACf,SAAS,CAAC,MAAM,CACjB,CAAC;YACJ,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACzD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,CAAC;gBACnF,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,iCAAiC,EAAE,mBAAmB,EAAE,CAAC;YACjG,CAAC;YAED,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;gBACV,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;oBAClB,MAAM,MAAM,GAAG,qBAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC;oBACnE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACpF,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAC1F,CAAC;YAED,2EAA2E;YAC3E,6DAA6D;YAC7D,MAAM,sBAAsB,GAAG,KAAK,CAAC,gBAAgB,KAAK,SAAS,CAAC;YACpE,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,gBAAgB,IAAI,0BAA0B,CAAC,CACvE,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,QAAQ,EACV,GAAG,CACJ,CAAC;YAEF,IAAI,GAAG,CAAC;YACR,IAAI,CAAC;gBACH,uEAAuE;gBACvE,qEAAqE;gBACrE,GAAG,GAAG,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;oBACpC,MAAM,EAAE,KAAK;oBACb,cAAc,EAAE,GAAG,CAAC,gBAAgB;oBACpC,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;oBACrC,MAAM,EAAE,SAAS,CAAC,MAAM;iBACzB,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAU,EAAE,CAAC;gBACpB,IAAI,sBAAsB;oBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;gBACxD,OAAO,EAAE,GAAG,eAAe,CAAC,CAAC,EAAE,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,mBAAmB,EAAE,CAAC;YAClF,CAAC;YAED,2DAA2D;YAC3D,IAAI,UAAU,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC/B,SAAS,EAAE,CAAC;gBACZ,IAAI,SAAS,GAAG,GAAG,CAAC,YAAY,EAAE,CAAC;oBACjC,IAAI,sBAAsB;wBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;oBACxD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,8BAA8B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAC5G,CAAC;gBACD,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,OAA4C,CAAC,CAAC;gBACnF,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,IAAI,sBAAsB;wBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;oBACxD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,qCAAqC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACnH,CAAC;gBAED,IAAI,OAAY,CAAC;gBACjB,IAAI,CAAC;oBACH,OAAO,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;gBACrC,CAAC;gBAAC,MAAM,CAAC;oBACP,IAAI,sBAAsB;wBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;oBACxD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,gCAAgC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBAC9G,CAAC;gBACD,mBAAmB,GAAG,uBAAuB,CAAC,OAAO,CAAC,IAAI,mBAAmB,CAAC;gBAC9E,UAAU,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAEhC,kDAAkD;gBAClD,MAAM,SAAS,CAAC,GAAG,CAAC,IAAqC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7E,IAAI,sBAAsB;oBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;gBACxD,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,SAAS;YACX,CAAC;YAED,uEAAuE;YACvE,gDAAgD;YAChD,MAAM,WAAW,GAAG,kBAAkB,CAAC,GAAG,CAAC,OAA4C,CAAC,CAAC;YACzF,IAAI,WAAW,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;gBAChD,MAAM,SAAS,CAAC,GAAG,CAAC,IAAqC,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC7E,IAAI,sBAAsB;oBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;gBACxD,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,oCAAoC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAClH,CAAC;YAED,4EAA4E;YAC5E,IAAI,OAAe,CAAC;YACpB,IAAI,aAAqB,CAAC;YAC1B,IAAI,SAAkB,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,OAAO,GAAG,GAAG,CAAC,IAAwC,CAAC;gBAC7D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,GAAG,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;gBAC3E,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC;gBACtB,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;gBACrC,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAC/B,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,IAAI,sBAAsB;oBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;gBACxD,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;oBACzD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;oBAC7B,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;gBACjG,CAAC;gBACD,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,UAAU,EAAE,4BAA4B,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,mBAAmB,EAAE,CAAC;YAC1G,CAAC;YAED,IAAI,sBAAsB;gBAAE,eAAe,CAAC,UAAU,CAAC,CAAC;YAExD,6CAA6C;YAC7C,IAAI,SAAS,GAAG,OAAO,CAAC;YACxB,IAAI,WAAW,KAAK,WAAW,EAAE,CAAC;gBAChC,SAAS,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;YAC3C,CAAC;YACD,uEAAuE;YACvE,sEAAsE;YAEtE,iCAAiC;YACjC,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBACzC,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;gBAClD,aAAa,GAAG,IAAI,CAAC;YACvB,CAAC;YAED,MAAM,OAAO,GAAG,kBAAkB,CAAC;gBACjC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE;gBACrB,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,OAAO;gBACL,MAAM,EAAE,IAAI;gBACZ,IAAI,EAAE,OAAO;gBACb,OAAO;gBACP,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,aAAa;gBACb,SAAS;gBACT,aAAa;gBACb,mBAAmB;aACpB,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,UAAU,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { ResearchConfig } from '../config/schema.js';
|
|
2
|
+
export interface BraveSearchResult {
|
|
3
|
+
title: string;
|
|
4
|
+
url: string;
|
|
5
|
+
snippet: string;
|
|
6
|
+
}
|
|
7
|
+
export interface BraveSearchResponse {
|
|
8
|
+
results: BraveSearchResult[];
|
|
9
|
+
keyIndex: number;
|
|
10
|
+
attempts: Array<{
|
|
11
|
+
keyIndex: number;
|
|
12
|
+
status: number | 'error';
|
|
13
|
+
}>;
|
|
14
|
+
}
|
|
15
|
+
export declare class BraveClient {
|
|
16
|
+
private readonly cfg;
|
|
17
|
+
private nextKeyIndex;
|
|
18
|
+
private lockChain;
|
|
19
|
+
private readonly _sleep;
|
|
20
|
+
private readonly _random;
|
|
21
|
+
constructor(cfg: ResearchConfig['brave'], opts?: {
|
|
22
|
+
sleep?: (ms: number) => Promise<void>;
|
|
23
|
+
random?: () => number;
|
|
24
|
+
});
|
|
25
|
+
private takeNextKeyIndex;
|
|
26
|
+
search(query: string, siteFilter?: string): Promise<BraveSearchResponse>;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=web-search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-search.d.ts","sourceRoot":"","sources":["../../src/research/web-search.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D,MAAM,WAAW,iBAAiB;IAAG,KAAK,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAC;CAAE;AACnF,MAAM,WAAW,mBAAmB;IAClC,OAAO,EAAE,iBAAiB,EAAE,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAA;KAAE,CAAC,CAAC;CACjE;AAsBD,qBAAa,WAAW;IAcpB,OAAO,CAAC,QAAQ,CAAC,GAAG;IAbtB,OAAO,CAAC,YAAY,CAAK;IAMzB,OAAO,CAAC,SAAS,CAAoC;IAGrD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgC;IACvD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;gBAGpB,GAAG,EAAE,cAAc,CAAC,OAAO,CAAC,EAC7C,IAAI,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;KAAE;YAM3D,gBAAgB;IAmBxB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,mBAAmB,CAAC;CAyE/E"}
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { request } from 'undici';
|
|
2
|
+
function validateBraveResults(body) {
|
|
3
|
+
if (body == null || typeof body !== 'object')
|
|
4
|
+
return [];
|
|
5
|
+
const b = body;
|
|
6
|
+
const web = b.web;
|
|
7
|
+
if (web == null || typeof web !== 'object')
|
|
8
|
+
return [];
|
|
9
|
+
const results = web.results;
|
|
10
|
+
if (!Array.isArray(results))
|
|
11
|
+
return [];
|
|
12
|
+
return results.map((r, i) => {
|
|
13
|
+
if (r == null || typeof r !== 'object') {
|
|
14
|
+
return { title: `[invalid entry ${i}]`, url: '', snippet: '' };
|
|
15
|
+
}
|
|
16
|
+
const item = r;
|
|
17
|
+
return {
|
|
18
|
+
title: typeof item.title === 'string' ? item.title : `[missing title ${i}]`,
|
|
19
|
+
url: typeof item.url === 'string' ? item.url : '',
|
|
20
|
+
snippet: typeof item.snippet === 'string' ? item.snippet : '',
|
|
21
|
+
};
|
|
22
|
+
});
|
|
23
|
+
}
|
|
24
|
+
export class BraveClient {
|
|
25
|
+
cfg;
|
|
26
|
+
nextKeyIndex = 0;
|
|
27
|
+
// Promise-chain critical-section lock: each caller waits for its
|
|
28
|
+
// predecessor, then atomically reads+advances nextKeyIndex.
|
|
29
|
+
// try/finally is load-bearing — without it an exception in the
|
|
30
|
+
// critical section would stall the chain permanently, hanging every
|
|
31
|
+
// subsequent search() call.
|
|
32
|
+
lockChain = Promise.resolve();
|
|
33
|
+
// Injectable for deterministic testing (backoff/jitter).
|
|
34
|
+
_sleep;
|
|
35
|
+
_random;
|
|
36
|
+
constructor(cfg, opts) {
|
|
37
|
+
this.cfg = cfg;
|
|
38
|
+
this._sleep = opts?.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
39
|
+
this._random = opts?.random ?? (() => Math.random());
|
|
40
|
+
}
|
|
41
|
+
async takeNextKeyIndex() {
|
|
42
|
+
let release;
|
|
43
|
+
const next = new Promise((r) => (release = r));
|
|
44
|
+
const prev = this.lockChain;
|
|
45
|
+
this.lockChain = next;
|
|
46
|
+
try {
|
|
47
|
+
await prev;
|
|
48
|
+
const n = this.cfg.apiKeys.length;
|
|
49
|
+
if (n === 0) {
|
|
50
|
+
throw new Error('brave_internal_no_keys');
|
|
51
|
+
}
|
|
52
|
+
const idx = this.nextKeyIndex;
|
|
53
|
+
this.nextKeyIndex = (this.nextKeyIndex + 1) % n;
|
|
54
|
+
return idx;
|
|
55
|
+
}
|
|
56
|
+
finally {
|
|
57
|
+
release();
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
async search(query, siteFilter) {
|
|
61
|
+
if (this.cfg.apiKeys.length === 0) {
|
|
62
|
+
throw new Error('brave_not_configured: no API keys configured');
|
|
63
|
+
}
|
|
64
|
+
const url = new URL('https://api.search.brave.com/res/v1/web/search');
|
|
65
|
+
url.searchParams.set('q', siteFilter ? `${siteFilter} ${query}` : query);
|
|
66
|
+
url.searchParams.set('count', String(this.cfg.maxResultsPerQuery));
|
|
67
|
+
const maxAttempts = Math.min(this.cfg.apiKeys.length, 4);
|
|
68
|
+
const attempts = [];
|
|
69
|
+
const deadline = Date.now() + this.cfg.timeoutMs;
|
|
70
|
+
let lastIndex = -1;
|
|
71
|
+
for (let attempt = 0; attempt < maxAttempts; attempt++) {
|
|
72
|
+
if (Date.now() > deadline) {
|
|
73
|
+
throw new Error('brave_deadline_exceeded');
|
|
74
|
+
}
|
|
75
|
+
const idx = await this.takeNextKeyIndex();
|
|
76
|
+
lastIndex = idx;
|
|
77
|
+
// Re-check deadline after lock acquisition — takeNextKeyIndex() may
|
|
78
|
+
// have waited behind a slow predecessor.
|
|
79
|
+
if (Date.now() > deadline) {
|
|
80
|
+
throw new Error('brave_deadline_exceeded');
|
|
81
|
+
}
|
|
82
|
+
const key = this.cfg.apiKeys[idx];
|
|
83
|
+
const ctrl = new AbortController();
|
|
84
|
+
const remaining = deadline - Date.now();
|
|
85
|
+
if (remaining <= 0) {
|
|
86
|
+
throw new Error('brave_deadline_exceeded');
|
|
87
|
+
}
|
|
88
|
+
const timer = setTimeout(() => ctrl.abort(), remaining);
|
|
89
|
+
let res;
|
|
90
|
+
try {
|
|
91
|
+
res = await request(url.toString(), {
|
|
92
|
+
method: 'GET',
|
|
93
|
+
headers: {
|
|
94
|
+
'accept': 'application/json',
|
|
95
|
+
'x-subscription-token': key,
|
|
96
|
+
},
|
|
97
|
+
signal: ctrl.signal,
|
|
98
|
+
});
|
|
99
|
+
if (res.statusCode === 200) {
|
|
100
|
+
const body = await res.body.json();
|
|
101
|
+
return {
|
|
102
|
+
results: validateBraveResults(body),
|
|
103
|
+
keyIndex: idx,
|
|
104
|
+
attempts: [...attempts, { keyIndex: idx, status: 200 }],
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
attempts.push({ keyIndex: idx, status: res.statusCode });
|
|
108
|
+
}
|
|
109
|
+
catch {
|
|
110
|
+
attempts.push({ keyIndex: idx, status: 'error' });
|
|
111
|
+
}
|
|
112
|
+
finally {
|
|
113
|
+
clearTimeout(timer);
|
|
114
|
+
if (res && res.statusCode !== 200) {
|
|
115
|
+
try {
|
|
116
|
+
await res.body.dump?.();
|
|
117
|
+
}
|
|
118
|
+
catch { /* nothing */ }
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
// No sleep after the final attempt.
|
|
122
|
+
if (attempt < maxAttempts - 1) {
|
|
123
|
+
const base = this.cfg.perCallBackoffMs * (2 ** attempt);
|
|
124
|
+
const jitter = base * (0.75 + this._random() * 0.5);
|
|
125
|
+
// Cap to remaining deadline so backoff alone never exceeds timeoutMs.
|
|
126
|
+
const capped = Math.min(jitter, Math.max(0, deadline - Date.now()));
|
|
127
|
+
await this._sleep(capped);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const summary = attempts.map(a => a.status).join(',');
|
|
131
|
+
throw new Error(`brave_keys_exhausted: attempts=[${summary}] lastKeyIndex=${lastIndex}`);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
//# sourceMappingURL=web-search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"web-search.js","sourceRoot":"","sources":["../../src/research/web-search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,QAAQ,CAAC;AAUjC,SAAS,oBAAoB,CAAC,IAAa;IACzC,IAAI,IAAI,IAAI,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACxD,MAAM,CAAC,GAAG,IAA+B,CAAC;IAC1C,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;IAClB,IAAI,GAAG,IAAI,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IACtD,MAAM,OAAO,GAAI,GAA+B,CAAC,OAAO,CAAC;IACzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAC;IACvC,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAU,EAAE,CAAS,EAAE,EAAE;QAC3C,IAAI,CAAC,IAAI,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YACvC,OAAO,EAAE,KAAK,EAAE,kBAAkB,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;QACjE,CAAC;QACD,MAAM,IAAI,GAAG,CAA4B,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG;YAC3E,GAAG,EAAE,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE;YACjD,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;SAC9D,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,OAAO,WAAW;IAcH;IAbX,YAAY,GAAG,CAAC,CAAC;IACzB,iEAAiE;IACjE,4DAA4D;IAC5D,+DAA+D;IAC/D,oEAAoE;IACpE,4BAA4B;IACpB,SAAS,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAErD,yDAAyD;IACxC,MAAM,CAAgC;IACtC,OAAO,CAAe;IAEvC,YACmB,GAA4B,EAC7C,IAAuE;QADtD,QAAG,GAAH,GAAG,CAAyB;QAG7C,IAAI,CAAC,MAAM,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,IAAI,EAAE,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,gBAAgB;QAC5B,IAAI,OAAoB,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC;YACX,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;YAClC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;YAC5C,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,CAAC;YAC9B,IAAI,CAAC,YAAY,GAAG,CAAC,IAAI,CAAC,YAAY,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;YAChD,OAAO,GAAG,CAAC;QACb,CAAC;gBAAS,CAAC;YACT,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAmB;QAC7C,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,gDAAgD,CAAC,CAAC;QACtE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,IAAI,KAAK,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACzE,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC;QAEnE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAoC,EAAE,CAAC;QACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC;QAEjD,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1C,SAAS,GAAG,GAAG,CAAC;YAEhB,oEAAoE;YACpE,yCAAyC;YACzC,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAE,CAAC;YACnC,MAAM,IAAI,GAAG,IAAI,eAAe,EAAE,CAAC;YACnC,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;gBACnB,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAC7C,CAAC;YACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;YACxD,IAAI,GAAoD,CAAC;YACzD,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE;oBAClC,MAAM,EAAE,KAAK;oBACb,OAAO,EAAE;wBACP,QAAQ,EAAE,kBAAkB;wBAC5B,sBAAsB,EAAE,GAAG;qBAC5B;oBACD,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;gBACH,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAC3B,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAa,CAAC;oBAC9C,OAAO;wBACL,OAAO,EAAE,oBAAoB,CAAC,IAAI,CAAC;wBACnC,QAAQ,EAAE,GAAG;wBACb,QAAQ,EAAE,CAAC,GAAG,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;qBACxD,CAAC;gBACJ,CAAC;gBACD,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;YAC3D,CAAC;YAAC,MAAM,CAAC;gBACP,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;YACpD,CAAC;oBAAS,CAAC;gBACT,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,IAAI,GAAG,IAAI,GAAG,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;oBAClC,IAAI,CAAC;wBAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;oBAAC,CAAC;oBAAC,MAAM,CAAC,CAAC,aAAa,CAAC,CAAC;gBAC1D,CAAC;YACH,CAAC;YAED,oCAAoC;YACpC,IAAI,OAAO,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;gBAC9B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC;gBACxD,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,GAAG,GAAG,CAAC,CAAC;gBACpD,sEAAsE;gBACtE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;gBACpE,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QACD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtD,MAAM,IAAI,KAAK,CAAC,mCAAmC,OAAO,kBAAkB,SAAS,EAAE,CAAC,CAAC;IAC3F,CAAC;CACF"}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import type { ResearchConfig } from '../../config/schema.js';
|
|
3
|
+
export interface ToolDef<I, O> {
|
|
4
|
+
name: string;
|
|
5
|
+
description: string;
|
|
6
|
+
inputSchema: z.ZodType<I>;
|
|
7
|
+
handler: (input: unknown) => Promise<{
|
|
8
|
+
ok: true;
|
|
9
|
+
result: O;
|
|
10
|
+
} | {
|
|
11
|
+
ok: false;
|
|
12
|
+
code: string;
|
|
13
|
+
message: string;
|
|
14
|
+
}>;
|
|
15
|
+
}
|
|
16
|
+
export interface BuildToolsInput {
|
|
17
|
+
cfg: ResearchConfig;
|
|
18
|
+
hostAllowlist: ReadonlySet<string>;
|
|
19
|
+
privateNetworkHosts: ReadonlySet<string>;
|
|
20
|
+
}
|
|
21
|
+
export interface ResearchTools {
|
|
22
|
+
arxiv?: ToolDef<{
|
|
23
|
+
query: string;
|
|
24
|
+
maxResults?: number;
|
|
25
|
+
}, unknown>;
|
|
26
|
+
semantic_scholar?: ToolDef<{
|
|
27
|
+
query: string;
|
|
28
|
+
maxResults?: number;
|
|
29
|
+
}, unknown>;
|
|
30
|
+
github_search?: ToolDef<{
|
|
31
|
+
query: string;
|
|
32
|
+
kind: 'repo' | 'code';
|
|
33
|
+
maxResults?: number;
|
|
34
|
+
}, unknown>;
|
|
35
|
+
rss?: ToolDef<{
|
|
36
|
+
url: string;
|
|
37
|
+
}, unknown>;
|
|
38
|
+
web_search?: ToolDef<{
|
|
39
|
+
query: string;
|
|
40
|
+
siteFilter?: string;
|
|
41
|
+
}, string>;
|
|
42
|
+
web_fetch: ToolDef<{
|
|
43
|
+
url: string;
|
|
44
|
+
}, string>;
|
|
45
|
+
}
|
|
46
|
+
export declare function buildResearchTools(opts: BuildToolsInput): ResearchTools;
|
|
47
|
+
//# sourceMappingURL=research-tools.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"research-tools.d.ts","sourceRoot":"","sources":["../../../src/runners/base/research-tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAM7D,MAAM,WAAW,OAAO,CAAC,CAAC,EAAE,CAAC;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1B,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,MAAM,EAAE,CAAC,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC9G;AAED,MAAM,WAAW,eAAe;IAC9B,GAAG,EAAE,cAAc,CAAC;IACpB,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IACnC,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;CAC1C;AAED,MAAM,WAAW,aAAa;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,CAAC,CAAC;IACjE,gBAAgB,CAAC,EAAE,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,CAAC,CAAC;IAC5E,aAAa,CAAC,EAAE,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,CAAC,CAAC;IAChG,GAAG,CAAC,EAAE,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,OAAO,CAAC,CAAC;IACxC,UAAU,CAAC,EAAE,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,CAAC;IACrE,SAAS,EAAE,OAAO,CAAC;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,EAAE,MAAM,CAAC,CAAC;CAC7C;AAsBD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,eAAe,GAAG,aAAa,CA4EvE"}
|