@open-mercato/web-research 0.7.1-develop.7194.1.ab4fc81f82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +2 -0
- package/AGENTS.md +67 -0
- package/build.mjs +10 -0
- package/dist/contract/adapter.js +13 -0
- package/dist/contract/adapter.js.map +7 -0
- package/dist/contract/errors.js +28 -0
- package/dist/contract/errors.js.map +7 -0
- package/dist/contract/http.js +7 -0
- package/dist/contract/http.js.map +7 -0
- package/dist/contract/index.js +9 -0
- package/dist/contract/index.js.map +7 -0
- package/dist/contract/outcomes.js +58 -0
- package/dist/contract/outcomes.js.map +7 -0
- package/dist/contract/policy.js +91 -0
- package/dist/contract/policy.js.map +7 -0
- package/dist/contract/results.js +13 -0
- package/dist/contract/results.js.map +7 -0
- package/dist/contract/steps.js +1 -0
- package/dist/contract/steps.js.map +7 -0
- package/dist/contract/version.js +11 -0
- package/dist/contract/version.js.map +7 -0
- package/dist/engine/cache.js +29 -0
- package/dist/engine/cache.js.map +7 -0
- package/dist/engine/deadline.js +54 -0
- package/dist/engine/deadline.js.map +7 -0
- package/dist/engine/engine.js +481 -0
- package/dist/engine/engine.js.map +7 -0
- package/dist/engine/index.js +20 -0
- package/dist/engine/index.js.map +7 -0
- package/dist/engine/runAdapter.js +57 -0
- package/dist/engine/runAdapter.js.map +7 -0
- package/dist/engine/types.js +1 -0
- package/dist/engine/types.js.map +7 -0
- package/dist/extract/classify.js +51 -0
- package/dist/extract/classify.js.map +7 -0
- package/dist/extract/content.js +104 -0
- package/dist/extract/content.js.map +7 -0
- package/dist/extract/entities.js +95 -0
- package/dist/extract/entities.js.map +7 -0
- package/dist/extract/index.js +24 -0
- package/dist/extract/index.js.map +7 -0
- package/dist/extract/text.js +114 -0
- package/dist/extract/text.js.map +7 -0
- package/dist/extract/tokenizer.js +141 -0
- package/dist/extract/tokenizer.js.map +7 -0
- package/dist/fusion/canonicalize.js +88 -0
- package/dist/fusion/canonicalize.js.map +7 -0
- package/dist/fusion/fuse.js +88 -0
- package/dist/fusion/fuse.js.map +7 -0
- package/dist/fusion/index.js +11 -0
- package/dist/fusion/index.js.map +7 -0
- package/dist/index.js +21 -0
- package/dist/index.js.map +7 -0
- package/dist/loader/describe.js +92 -0
- package/dist/loader/describe.js.map +7 -0
- package/dist/loader/index.js +31 -0
- package/dist/loader/index.js.map +7 -0
- package/dist/loader/manifest.js +32 -0
- package/dist/loader/manifest.js.map +7 -0
- package/dist/loader/registry.js +98 -0
- package/dist/loader/registry.js.map +7 -0
- package/dist/net/client.js +298 -0
- package/dist/net/client.js.map +7 -0
- package/dist/net/ssrf.js +165 -0
- package/dist/net/ssrf.js.map +7 -0
- package/dist/net/url.js +9 -0
- package/dist/net/url.js.map +7 -0
- package/dist/testing/contract.js +118 -0
- package/dist/testing/contract.js.map +7 -0
- package/dist/testing/fakeAdapter.js +44 -0
- package/dist/testing/fakeAdapter.js.map +7 -0
- package/dist/testing/index.js +15 -0
- package/dist/testing/index.js.map +7 -0
- package/jest.config.cjs +24 -0
- package/package.json +65 -0
- package/src/contract/adapter.ts +100 -0
- package/src/contract/errors.ts +48 -0
- package/src/contract/http.ts +46 -0
- package/src/contract/index.ts +8 -0
- package/src/contract/outcomes.ts +91 -0
- package/src/contract/policy.ts +157 -0
- package/src/contract/results.ts +79 -0
- package/src/contract/steps.ts +45 -0
- package/src/contract/version.ts +25 -0
- package/src/engine/__tests__/engine.test.ts +597 -0
- package/src/engine/cache.ts +46 -0
- package/src/engine/deadline.ts +65 -0
- package/src/engine/engine.ts +633 -0
- package/src/engine/index.ts +22 -0
- package/src/engine/runAdapter.ts +89 -0
- package/src/engine/types.ts +137 -0
- package/src/extract/__tests__/content.test.ts +81 -0
- package/src/extract/__tests__/text.test.ts +61 -0
- package/src/extract/__tests__/tokenizer.test.ts +85 -0
- package/src/extract/classify.ts +81 -0
- package/src/extract/content.ts +136 -0
- package/src/extract/entities.ts +98 -0
- package/src/extract/index.ts +11 -0
- package/src/extract/text.ts +124 -0
- package/src/extract/tokenizer.ts +197 -0
- package/src/fusion/__tests__/fusion.test.ts +198 -0
- package/src/fusion/canonicalize.ts +119 -0
- package/src/fusion/fuse.ts +145 -0
- package/src/fusion/index.ts +8 -0
- package/src/index.ts +15 -0
- package/src/loader/__tests__/describe.test.ts +119 -0
- package/src/loader/__tests__/registry.test.ts +134 -0
- package/src/loader/describe.ts +145 -0
- package/src/loader/index.ts +25 -0
- package/src/loader/manifest.ts +44 -0
- package/src/loader/registry.ts +156 -0
- package/src/net/__tests__/client.test.ts +273 -0
- package/src/net/__tests__/ssrf.test.ts +165 -0
- package/src/net/client.ts +434 -0
- package/src/net/ssrf.ts +235 -0
- package/src/net/url.ts +13 -0
- package/src/testing/contract.ts +172 -0
- package/src/testing/fakeAdapter.ts +69 -0
- package/src/testing/index.ts +9 -0
- package/tsconfig.json +10 -0
- package/watch.mjs +7 -0
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
import { Agent as HttpAgent, request as sendHttp } from "node:http";
|
|
2
|
+
import { Agent as HttpsAgent, request as sendHttps } from "node:https";
|
|
3
|
+
import { isIP } from "node:net";
|
|
4
|
+
import { createBrotliDecompress, createGunzip, createInflate } from "node:zlib";
|
|
5
|
+
import { WebResearchError, errorMessage, isWebResearchError } from "../contract/errors.js";
|
|
6
|
+
import { assertPublicUrl } from "./ssrf.js";
|
|
7
|
+
const DEFAULT_USER_AGENT = "Mozilla/5.0 (compatible; OpenMercatoResearch/1.0; +https://github.com/open-mercato/open-mercato)";
|
|
8
|
+
const DEFAULT_MAX_BYTES = 512 * 1024;
|
|
9
|
+
const DEFAULT_TIMEOUT_MS = 1e4;
|
|
10
|
+
const MAX_REDIRECTS = 5;
|
|
11
|
+
const MAX_RETRIES = 2;
|
|
12
|
+
const MAX_BACKOFF_MS = 5e3;
|
|
13
|
+
const MAX_SLEEP_MS = 6e4;
|
|
14
|
+
const DEFAULT_TOTAL_TIMEOUT_MS = 3e4;
|
|
15
|
+
function createPinnedLookup(addresses) {
|
|
16
|
+
const entries = addresses.map((address) => ({ address, family: isIP(address) }));
|
|
17
|
+
return (hostname, options, callback) => {
|
|
18
|
+
if (entries.length === 0) {
|
|
19
|
+
callback(new Error(`No pinned address for ${hostname}`), "", 0);
|
|
20
|
+
return;
|
|
21
|
+
}
|
|
22
|
+
const requestedFamily = typeof options === "number" ? options : options?.family ?? 0;
|
|
23
|
+
const matching = requestedFamily ? entries.filter((entry) => entry.family === requestedFamily) : entries;
|
|
24
|
+
const selected = matching.length > 0 ? matching : entries;
|
|
25
|
+
if (typeof options === "object" && options?.all) {
|
|
26
|
+
callback(null, selected);
|
|
27
|
+
return;
|
|
28
|
+
}
|
|
29
|
+
callback(null, selected[0].address, selected[0].family);
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
function defaultSleep(ms, signal) {
|
|
33
|
+
if (!Number.isFinite(ms) || ms <= 0) return Promise.resolve();
|
|
34
|
+
const bounded = ms > MAX_SLEEP_MS ? MAX_SLEEP_MS : ms;
|
|
35
|
+
return new Promise((resolve) => {
|
|
36
|
+
const timer = setTimeout(finish, bounded);
|
|
37
|
+
function finish() {
|
|
38
|
+
clearTimeout(timer);
|
|
39
|
+
signal?.removeEventListener("abort", finish);
|
|
40
|
+
resolve();
|
|
41
|
+
}
|
|
42
|
+
signal?.addEventListener("abort", finish, { once: true });
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
function decompress(response) {
|
|
46
|
+
const encoding = String(response.headers["content-encoding"] ?? "").toLowerCase();
|
|
47
|
+
if (encoding === "gzip" || encoding === "x-gzip") return response.pipe(createGunzip());
|
|
48
|
+
if (encoding === "deflate") return response.pipe(createInflate());
|
|
49
|
+
if (encoding === "br") return response.pipe(createBrotliDecompress());
|
|
50
|
+
return response;
|
|
51
|
+
}
|
|
52
|
+
async function readCapped(stream, maxBytes) {
|
|
53
|
+
const decoder = new TextDecoder("utf-8");
|
|
54
|
+
let text = "";
|
|
55
|
+
let bytes = 0;
|
|
56
|
+
let truncated = false;
|
|
57
|
+
try {
|
|
58
|
+
for await (const chunk of stream) {
|
|
59
|
+
const buffer = chunk;
|
|
60
|
+
if (bytes >= maxBytes) {
|
|
61
|
+
truncated = true;
|
|
62
|
+
break;
|
|
63
|
+
}
|
|
64
|
+
const remaining = maxBytes - bytes;
|
|
65
|
+
if (buffer.byteLength > remaining) {
|
|
66
|
+
text += decoder.decode(buffer.subarray(0, remaining), { stream: true });
|
|
67
|
+
bytes = maxBytes;
|
|
68
|
+
truncated = true;
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
text += decoder.decode(buffer, { stream: true });
|
|
72
|
+
bytes += buffer.byteLength;
|
|
73
|
+
}
|
|
74
|
+
} finally {
|
|
75
|
+
stream.destroy();
|
|
76
|
+
}
|
|
77
|
+
text += decoder.decode();
|
|
78
|
+
return { text, truncated, bytes };
|
|
79
|
+
}
|
|
80
|
+
function send(target, addresses, options) {
|
|
81
|
+
return new Promise((resolve, reject) => {
|
|
82
|
+
const secure = target.protocol === "https:";
|
|
83
|
+
const agent = secure ? new HttpsAgent({ keepAlive: false, lookup: createPinnedLookup(addresses) }) : new HttpAgent({ keepAlive: false, lookup: createPinnedLookup(addresses) });
|
|
84
|
+
const transport = secure ? sendHttps : sendHttp;
|
|
85
|
+
const request = transport(
|
|
86
|
+
{
|
|
87
|
+
protocol: target.protocol,
|
|
88
|
+
hostname: target.hostname,
|
|
89
|
+
port: target.port || (secure ? 443 : 80),
|
|
90
|
+
path: `${target.pathname}${target.search}`,
|
|
91
|
+
method: options.method,
|
|
92
|
+
headers: options.headers,
|
|
93
|
+
agent
|
|
94
|
+
},
|
|
95
|
+
(response) => {
|
|
96
|
+
clearTimeout(headerTimer);
|
|
97
|
+
resolve({ response, dispose, terminalError: () => terminal });
|
|
98
|
+
}
|
|
99
|
+
);
|
|
100
|
+
let terminal = null;
|
|
101
|
+
const fail = (error) => {
|
|
102
|
+
terminal = error;
|
|
103
|
+
request.destroy(error);
|
|
104
|
+
};
|
|
105
|
+
const dispose = () => {
|
|
106
|
+
clearTimeout(headerTimer);
|
|
107
|
+
request.setTimeout(0);
|
|
108
|
+
options.signal?.removeEventListener("abort", onAbort);
|
|
109
|
+
request.destroy();
|
|
110
|
+
agent.destroy();
|
|
111
|
+
};
|
|
112
|
+
const headerTimer = setTimeout(() => {
|
|
113
|
+
fail(new WebResearchError("timeout", `Request to ${target.href} timed out`));
|
|
114
|
+
}, options.timeoutMs);
|
|
115
|
+
request.setTimeout(options.timeoutMs, () => {
|
|
116
|
+
fail(new WebResearchError("timeout", `Request to ${target.href} stalled`));
|
|
117
|
+
});
|
|
118
|
+
function onAbort() {
|
|
119
|
+
fail(new WebResearchError("aborted", `Request to ${target.href} was aborted`));
|
|
120
|
+
}
|
|
121
|
+
options.signal?.addEventListener("abort", onAbort, { once: true });
|
|
122
|
+
request.on("error", (error) => {
|
|
123
|
+
dispose();
|
|
124
|
+
reject(
|
|
125
|
+
terminal ?? (isWebResearchError(error) ? error : new WebResearchError("network", `Request to ${target.href} failed: ${errorMessage(error)}`, {
|
|
126
|
+
cause: error
|
|
127
|
+
}))
|
|
128
|
+
);
|
|
129
|
+
});
|
|
130
|
+
if (options.body !== void 0) request.write(options.body);
|
|
131
|
+
request.end();
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
function headerMap(response) {
|
|
135
|
+
const headers = /* @__PURE__ */ new Map();
|
|
136
|
+
for (const [key, value] of Object.entries(response.headers)) {
|
|
137
|
+
if (value === void 0) continue;
|
|
138
|
+
headers.set(key.toLowerCase(), Array.isArray(value) ? value.join(", ") : value);
|
|
139
|
+
}
|
|
140
|
+
return headers;
|
|
141
|
+
}
|
|
142
|
+
function retryAfterMs(headers) {
|
|
143
|
+
const raw = headers.get("retry-after");
|
|
144
|
+
if (!raw) return void 0;
|
|
145
|
+
const seconds = Number.parseInt(raw.trim(), 10);
|
|
146
|
+
if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1e3;
|
|
147
|
+
const date = Date.parse(raw);
|
|
148
|
+
if (Number.isFinite(date)) return Math.max(0, date - Date.now());
|
|
149
|
+
return void 0;
|
|
150
|
+
}
|
|
151
|
+
function contentTypeOf(headers) {
|
|
152
|
+
const raw = headers.get("content-type");
|
|
153
|
+
if (!raw) return null;
|
|
154
|
+
return raw.split(";")[0].trim().toLowerCase();
|
|
155
|
+
}
|
|
156
|
+
function isAcceptable(contentType, accept) {
|
|
157
|
+
if (!accept || accept.length === 0) return true;
|
|
158
|
+
if (contentType === null) return true;
|
|
159
|
+
return accept.some((prefix) => contentType.startsWith(prefix));
|
|
160
|
+
}
|
|
161
|
+
function statusError(status, url, headers) {
|
|
162
|
+
if (status === 429) {
|
|
163
|
+
return new WebResearchError("blocked", `Rate limited by ${url} (HTTP 429)`, {
|
|
164
|
+
status,
|
|
165
|
+
retryAfterMs: retryAfterMs(headers)
|
|
166
|
+
});
|
|
167
|
+
}
|
|
168
|
+
if (status === 403 || status === 401) {
|
|
169
|
+
return new WebResearchError("blocked", `Access denied by ${url} (HTTP ${status})`, { status });
|
|
170
|
+
}
|
|
171
|
+
return new WebResearchError("bad_response", `Request to ${url} failed with HTTP ${status}`, { status });
|
|
172
|
+
}
|
|
173
|
+
function isRetriable(error) {
|
|
174
|
+
if (error.code === "blocked") return error.status === 429;
|
|
175
|
+
if (error.code === "network") return true;
|
|
176
|
+
return error.code === "bad_response" && (error.status ?? 0) >= 500;
|
|
177
|
+
}
|
|
178
|
+
function createHttpClient(options = {}) {
|
|
179
|
+
const userAgent = options.userAgent ?? DEFAULT_USER_AGENT;
|
|
180
|
+
const acceptLanguage = options.acceptLanguage ?? "en-US,en;q=0.9";
|
|
181
|
+
const defaultMaxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES;
|
|
182
|
+
const defaultTimeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
183
|
+
const maxRedirects = options.maxRedirects ?? MAX_REDIRECTS;
|
|
184
|
+
const maxRetries = options.maxRetries ?? MAX_RETRIES;
|
|
185
|
+
const maxTotalMs = options.maxTotalMs ?? DEFAULT_TOTAL_TIMEOUT_MS;
|
|
186
|
+
const politenessMs = options.politenessMs ?? 0;
|
|
187
|
+
const sleep = options.sleep ?? defaultSleep;
|
|
188
|
+
const now = options.now ?? Date.now;
|
|
189
|
+
const nextAllowedAt = /* @__PURE__ */ new Map();
|
|
190
|
+
async function waitForHost(host, signal) {
|
|
191
|
+
if (politenessMs <= 0) return;
|
|
192
|
+
const earliest = nextAllowedAt.get(host) ?? 0;
|
|
193
|
+
const current = now();
|
|
194
|
+
const delay = earliest - current;
|
|
195
|
+
nextAllowedAt.set(host, Math.max(current, earliest) + politenessMs);
|
|
196
|
+
if (delay > 0) await sleep(delay, signal);
|
|
197
|
+
}
|
|
198
|
+
async function attempt(target, addresses, init, carryCallerHeaders) {
|
|
199
|
+
const headers = {
|
|
200
|
+
"user-agent": userAgent,
|
|
201
|
+
"accept-language": acceptLanguage,
|
|
202
|
+
"accept-encoding": "gzip, deflate, br",
|
|
203
|
+
accept: init.accept?.length ? `${init.accept.join(", ")}, */*;q=0.1` : "*/*",
|
|
204
|
+
...init.body === void 0 ? {} : { "content-type": "application/json" },
|
|
205
|
+
...carryCallerHeaders ? init.headers ?? {} : {}
|
|
206
|
+
};
|
|
207
|
+
return send(target, addresses, {
|
|
208
|
+
method: init.method ?? "GET",
|
|
209
|
+
headers,
|
|
210
|
+
...init.body === void 0 ? {} : { body: init.body },
|
|
211
|
+
timeoutMs: init.timeoutMs ?? defaultTimeoutMs,
|
|
212
|
+
...init.signal ? { signal: init.signal } : {}
|
|
213
|
+
});
|
|
214
|
+
}
|
|
215
|
+
async function request(rawUrl, init = {}) {
|
|
216
|
+
const maxBytes = init.maxBytes ?? defaultMaxBytes;
|
|
217
|
+
const expiresAt = now() + maxTotalMs;
|
|
218
|
+
const outOfTime = () => now() >= expiresAt;
|
|
219
|
+
let currentUrl = rawUrl;
|
|
220
|
+
let sameOrigin = true;
|
|
221
|
+
for (let hop = 0; hop <= maxRedirects; hop += 1) {
|
|
222
|
+
if (outOfTime()) {
|
|
223
|
+
throw new WebResearchError("timeout", `Request to ${rawUrl} exceeded its ${maxTotalMs}ms budget`);
|
|
224
|
+
}
|
|
225
|
+
const vetted = await assertPublicUrl(currentUrl, "web research fetch", {
|
|
226
|
+
...options.lookup ? { lookup: options.lookup } : {},
|
|
227
|
+
...options.allowPrivateHosts ? { allowPrivateHosts: options.allowPrivateHosts } : {}
|
|
228
|
+
});
|
|
229
|
+
await waitForHost(vetted.url.host, init.signal);
|
|
230
|
+
let sent = null;
|
|
231
|
+
let lastError = null;
|
|
232
|
+
for (let retry = 0; retry <= maxRetries; retry += 1) {
|
|
233
|
+
try {
|
|
234
|
+
sent = await attempt(vetted.url, vetted.addresses, init, sameOrigin);
|
|
235
|
+
const status2 = sent.response.statusCode ?? 0;
|
|
236
|
+
if (status2 >= 400) {
|
|
237
|
+
const error = statusError(status2, vetted.url.href, headerMap(sent.response));
|
|
238
|
+
sent.response.resume();
|
|
239
|
+
sent.dispose();
|
|
240
|
+
sent = null;
|
|
241
|
+
throw error;
|
|
242
|
+
}
|
|
243
|
+
break;
|
|
244
|
+
} catch (error) {
|
|
245
|
+
const typed = isWebResearchError(error) ? error : new WebResearchError("network", errorMessage(error), { cause: error });
|
|
246
|
+
lastError = typed;
|
|
247
|
+
if (typed.code === "aborted" || retry === maxRetries || !isRetriable(typed)) throw typed;
|
|
248
|
+
if (outOfTime()) throw typed;
|
|
249
|
+
const backoff = Math.min(typed.retryAfterMs ?? 250 * 2 ** retry, MAX_BACKOFF_MS);
|
|
250
|
+
await sleep(backoff, init.signal);
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
if (!sent) throw lastError ?? new WebResearchError("network", `Request to ${currentUrl} failed`);
|
|
254
|
+
const headers = headerMap(sent.response);
|
|
255
|
+
const status = sent.response.statusCode ?? 0;
|
|
256
|
+
if (status >= 300 && status < 400) {
|
|
257
|
+
const location = headers.get("location");
|
|
258
|
+
sent.response.resume();
|
|
259
|
+
sent.dispose();
|
|
260
|
+
if (!location) {
|
|
261
|
+
throw new WebResearchError("bad_response", `Redirect from ${currentUrl} without a Location header`);
|
|
262
|
+
}
|
|
263
|
+
const next = new URL(location, vetted.url);
|
|
264
|
+
if (next.origin !== vetted.url.origin) sameOrigin = false;
|
|
265
|
+
currentUrl = next.toString();
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
const contentType = contentTypeOf(headers);
|
|
269
|
+
if (!isAcceptable(contentType, init.accept)) {
|
|
270
|
+
sent.response.resume();
|
|
271
|
+
sent.dispose();
|
|
272
|
+
throw new WebResearchError(
|
|
273
|
+
"unsupported_content_type",
|
|
274
|
+
`Refusing to decode ${contentType ?? "unknown"} from ${vetted.url.href} as text`
|
|
275
|
+
);
|
|
276
|
+
}
|
|
277
|
+
try {
|
|
278
|
+
const { text, truncated } = await readCapped(decompress(sent.response), maxBytes);
|
|
279
|
+
return { url: vetted.url.href, status, headers, contentType, body: text, truncated };
|
|
280
|
+
} catch (error) {
|
|
281
|
+
const terminal = sent.terminalError();
|
|
282
|
+
if (terminal) throw terminal;
|
|
283
|
+
throw isWebResearchError(error) ? error : new WebResearchError("network", `Reading ${vetted.url.href} failed: ${errorMessage(error)}`, {
|
|
284
|
+
cause: error
|
|
285
|
+
});
|
|
286
|
+
} finally {
|
|
287
|
+
sent.dispose();
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
throw new WebResearchError("too_many_redirects", `Exceeded ${maxRedirects} redirects from ${rawUrl}`);
|
|
291
|
+
}
|
|
292
|
+
return { request };
|
|
293
|
+
}
|
|
294
|
+
export {
|
|
295
|
+
createHttpClient,
|
|
296
|
+
createPinnedLookup
|
|
297
|
+
};
|
|
298
|
+
//# sourceMappingURL=client.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/net/client.ts"],
|
|
4
|
+
"sourcesContent": ["import { Agent as HttpAgent, request as sendHttp, type IncomingMessage } from 'node:http'\nimport { Agent as HttpsAgent, request as sendHttps } from 'node:https'\nimport { isIP, type LookupFunction } from 'node:net'\nimport type { Readable } from 'node:stream'\nimport { createBrotliDecompress, createGunzip, createInflate } from 'node:zlib'\nimport { WebResearchError, errorMessage, isWebResearchError } from '../contract/errors'\nimport type { HttpClient, HttpRequestInit, HttpResponse } from '../contract/http'\nimport { assertPublicUrl, type LookupFn } from './ssrf'\n\nconst DEFAULT_USER_AGENT =\n 'Mozilla/5.0 (compatible; OpenMercatoResearch/1.0; +https://github.com/open-mercato/open-mercato)'\nconst DEFAULT_MAX_BYTES = 512 * 1024\nconst DEFAULT_TIMEOUT_MS = 10_000\nconst MAX_REDIRECTS = 5\nconst MAX_RETRIES = 2\nconst MAX_BACKOFF_MS = 5_000\n/**\n * Hard ceiling on a single `sleep()`. Backoff is already clamped to\n * `MAX_BACKOFF_MS`, but the politeness delay is derived from configuration and\n * a `Retry-After` header is remote input, so the timer itself is capped too \u2014\n * no caller can park a request for an unbounded stretch.\n */\nconst MAX_SLEEP_MS = 60_000\n/**\n * Ceiling on one `request()` call, redirects and retries included.\n *\n * `timeoutMs` bounds a single attempt, so without this a call could legitimately\n * run for redirects x retries x timeout plus backoff \u2014 minutes, inside an adapter\n * budget of seconds. The adapter's signal would eventually cut it, but only after\n * the work was already spent.\n */\nconst DEFAULT_TOTAL_TIMEOUT_MS = 30_000\n\nexport type HttpClientOptions = {\n readonly userAgent?: string\n readonly acceptLanguage?: string\n readonly maxBytes?: number\n readonly timeoutMs?: number\n readonly maxRedirects?: number\n readonly maxRetries?: number\n /** Ceiling on one call including every redirect hop and retry. */\n readonly maxTotalMs?: number\n /** Minimum gap between requests to the same host. */\n readonly politenessMs?: number\n readonly lookup?: LookupFn\n /**\n * Hosts allowed to resolve to a non-public address. Empty by default; see\n * `assertPublicUrl` for why this is an allowlist and not a boolean.\n */\n readonly allowPrivateHosts?: readonly string[]\n readonly sleep?: (ms: number, signal?: AbortSignal) => Promise<void>\n readonly now?: () => number\n}\n\n/**\n * Resolves a hostname to exactly the addresses the SSRF guard already vetted.\n * Without this the kernel re-resolves at connect time, reopening the rebinding\n * window between check and connect. Only name resolution is overridden, so SNI\n * and certificate validation still see the real hostname.\n */\nexport function createPinnedLookup(addresses: readonly string[]): LookupFunction {\n const entries = addresses.map((address) => ({ address, family: isIP(address) }))\n return (hostname, options, callback) => {\n if (entries.length === 0) {\n callback(new Error(`No pinned address for ${hostname}`), '', 0)\n return\n }\n const requestedFamily = typeof options === 'number' ? options : (options?.family ?? 0)\n const matching = requestedFamily\n ? entries.filter((entry) => entry.family === requestedFamily)\n : entries\n const selected = matching.length > 0 ? matching : entries\n if (typeof options === 'object' && options?.all) {\n callback(null, selected)\n return\n }\n callback(null, selected[0].address, selected[0].family)\n }\n}\n\nfunction defaultSleep(ms: number, signal?: AbortSignal): Promise<void> {\n // Explicit finite check + relational clamp rather than `Math.min`: a NaN\n // duration must not reach `setTimeout`, and the upper bound has to be\n // provable at the call site.\n if (!Number.isFinite(ms) || ms <= 0) return Promise.resolve()\n const bounded = ms > MAX_SLEEP_MS ? MAX_SLEEP_MS : ms\n return new Promise((resolve) => {\n const timer = setTimeout(finish, bounded)\n function finish(): void {\n clearTimeout(timer)\n signal?.removeEventListener('abort', finish)\n resolve()\n }\n signal?.addEventListener('abort', finish, { once: true })\n })\n}\n\nfunction decompress(response: IncomingMessage): Readable {\n const encoding = String(response.headers['content-encoding'] ?? '').toLowerCase()\n if (encoding === 'gzip' || encoding === 'x-gzip') return response.pipe(createGunzip())\n if (encoding === 'deflate') return response.pipe(createInflate())\n if (encoding === 'br') return response.pipe(createBrotliDecompress())\n return response\n}\n\n/**\n * Reads at most `maxBytes`, decoding incrementally so a cap landing inside a\n * multi-byte sequence does not produce a replacement character at the tail.\n */\nasync function readCapped(\n stream: Readable,\n maxBytes: number,\n): Promise<{ text: string; truncated: boolean; bytes: number }> {\n const decoder = new TextDecoder('utf-8')\n let text = ''\n let bytes = 0\n let truncated = false\n try {\n for await (const chunk of stream) {\n const buffer = chunk as Buffer\n if (bytes >= maxBytes) {\n truncated = true\n break\n }\n const remaining = maxBytes - bytes\n if (buffer.byteLength > remaining) {\n text += decoder.decode(buffer.subarray(0, remaining), { stream: true })\n bytes = maxBytes\n truncated = true\n break\n }\n text += decoder.decode(buffer, { stream: true })\n bytes += buffer.byteLength\n }\n } finally {\n stream.destroy()\n }\n text += decoder.decode()\n return { text, truncated, bytes }\n}\n\ntype SentRequest = {\n readonly response: IncomingMessage\n readonly dispose: () => void\n /**\n * Why we tore the exchange down, if we did.\n *\n * Destroying a request does not deliver our reason to the response stream \u2014\n * Node surfaces its own socket error there \u2014 so a read cut short by our own\n * timeout or abort would otherwise be reported as a generic network fault,\n * and an adapter would call a deadline breach a retriable error.\n */\n readonly terminalError: () => WebResearchError | null\n}\n\ntype SendOptions = {\n readonly method: string\n readonly headers: Record<string, string>\n readonly body?: string\n readonly timeoutMs: number\n readonly signal?: AbortSignal\n}\n\nfunction send(target: URL, addresses: readonly string[], options: SendOptions): Promise<SentRequest> {\n return new Promise((resolve, reject) => {\n const secure = target.protocol === 'https:'\n const agent = secure\n ? new HttpsAgent({ keepAlive: false, lookup: createPinnedLookup(addresses) })\n : new HttpAgent({ keepAlive: false, lookup: createPinnedLookup(addresses) })\n const transport = secure ? sendHttps : sendHttp\n const request = transport(\n {\n protocol: target.protocol,\n hostname: target.hostname,\n port: target.port || (secure ? 443 : 80),\n path: `${target.pathname}${target.search}`,\n method: options.method,\n headers: options.headers,\n agent,\n },\n (response) => {\n clearTimeout(headerTimer)\n resolve({ response, dispose, terminalError: () => terminal })\n },\n )\n\n let terminal: WebResearchError | null = null\n const fail = (error: WebResearchError): void => {\n terminal = error\n request.destroy(error)\n }\n\n /**\n * Teardown for the whole exchange, headers *and* body. The abort listener and\n * the socket idle timer deliberately stay armed past the response callback:\n * releasing them there leaves a body read that no signal and no deadline can\n * interrupt, so a server that trickles bytes forever pins a socket and an\n * adapter promise for the life of the process.\n */\n const dispose = (): void => {\n clearTimeout(headerTimer)\n request.setTimeout(0)\n options.signal?.removeEventListener('abort', onAbort)\n request.destroy()\n agent.destroy()\n }\n\n const headerTimer = setTimeout(() => {\n fail(new WebResearchError('timeout', `Request to ${target.href} timed out`))\n }, options.timeoutMs)\n\n request.setTimeout(options.timeoutMs, () => {\n fail(new WebResearchError('timeout', `Request to ${target.href} stalled`))\n })\n\n function onAbort(): void {\n fail(new WebResearchError('aborted', `Request to ${target.href} was aborted`))\n }\n options.signal?.addEventListener('abort', onAbort, { once: true })\n\n request.on('error', (error) => {\n dispose()\n reject(\n terminal ??\n (isWebResearchError(error)\n ? error\n : new WebResearchError('network', `Request to ${target.href} failed: ${errorMessage(error)}`, {\n cause: error,\n })),\n )\n })\n\n if (options.body !== undefined) request.write(options.body)\n request.end()\n })\n}\n\nfunction headerMap(response: IncomingMessage): Map<string, string> {\n const headers = new Map<string, string>()\n for (const [key, value] of Object.entries(response.headers)) {\n if (value === undefined) continue\n headers.set(key.toLowerCase(), Array.isArray(value) ? value.join(', ') : value)\n }\n return headers\n}\n\nfunction retryAfterMs(headers: Map<string, string>): number | undefined {\n const raw = headers.get('retry-after')\n if (!raw) return undefined\n const seconds = Number.parseInt(raw.trim(), 10)\n if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000\n const date = Date.parse(raw)\n if (Number.isFinite(date)) return Math.max(0, date - Date.now())\n return undefined\n}\n\nfunction contentTypeOf(headers: Map<string, string>): string | null {\n const raw = headers.get('content-type')\n if (!raw) return null\n return raw.split(';')[0].trim().toLowerCase()\n}\n\nfunction isAcceptable(contentType: string | null, accept: readonly string[] | undefined): boolean {\n if (!accept || accept.length === 0) return true\n if (contentType === null) return true\n return accept.some((prefix) => contentType.startsWith(prefix))\n}\n\nfunction statusError(status: number, url: string, headers: Map<string, string>): WebResearchError {\n if (status === 429) {\n return new WebResearchError('blocked', `Rate limited by ${url} (HTTP 429)`, {\n status,\n retryAfterMs: retryAfterMs(headers),\n })\n }\n if (status === 403 || status === 401) {\n return new WebResearchError('blocked', `Access denied by ${url} (HTTP ${status})`, { status })\n }\n return new WebResearchError('bad_response', `Request to ${url} failed with HTTP ${status}`, { status })\n}\n\nfunction isRetriable(error: WebResearchError): boolean {\n if (error.code === 'blocked') return error.status === 429\n if (error.code === 'network') return true\n return error.code === 'bad_response' && (error.status ?? 0) >= 500\n}\n\n/**\n * The single hardened egress path: SSRF-vetted, DNS-pinned per redirect hop,\n * manually redirected so every hop is re-vetted, byte-capped, content-type\n * gated, and polite to each host.\n */\nexport function createHttpClient(options: HttpClientOptions = {}): HttpClient {\n const userAgent = options.userAgent ?? DEFAULT_USER_AGENT\n const acceptLanguage = options.acceptLanguage ?? 'en-US,en;q=0.9'\n const defaultMaxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES\n const defaultTimeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS\n const maxRedirects = options.maxRedirects ?? MAX_REDIRECTS\n const maxRetries = options.maxRetries ?? MAX_RETRIES\n const maxTotalMs = options.maxTotalMs ?? DEFAULT_TOTAL_TIMEOUT_MS\n const politenessMs = options.politenessMs ?? 0\n const sleep = options.sleep ?? defaultSleep\n const now = options.now ?? Date.now\n const nextAllowedAt = new Map<string, number>()\n\n async function waitForHost(host: string, signal?: AbortSignal): Promise<void> {\n if (politenessMs <= 0) return\n const earliest = nextAllowedAt.get(host) ?? 0\n const current = now()\n const delay = earliest - current\n nextAllowedAt.set(host, Math.max(current, earliest) + politenessMs)\n if (delay > 0) await sleep(delay, signal)\n }\n\n async function attempt(\n target: URL,\n addresses: readonly string[],\n init: HttpRequestInit,\n carryCallerHeaders: boolean,\n ) {\n const headers: Record<string, string> = {\n 'user-agent': userAgent,\n 'accept-language': acceptLanguage,\n 'accept-encoding': 'gzip, deflate, br',\n accept: init.accept?.length ? `${init.accept.join(', ')}, */*;q=0.1` : '*/*',\n ...(init.body === undefined ? {} : { 'content-type': 'application/json' }),\n ...(carryCallerHeaders ? (init.headers ?? {}) : {}),\n }\n return send(target, addresses, {\n method: init.method ?? 'GET',\n headers,\n ...(init.body === undefined ? {} : { body: init.body }),\n timeoutMs: init.timeoutMs ?? defaultTimeoutMs,\n ...(init.signal ? { signal: init.signal } : {}),\n })\n }\n\n async function request(rawUrl: string, init: HttpRequestInit = {}): Promise<HttpResponse> {\n const maxBytes = init.maxBytes ?? defaultMaxBytes\n const expiresAt = now() + maxTotalMs\n const outOfTime = (): boolean => now() >= expiresAt\n let currentUrl = rawUrl\n // Caller headers carry vendor credentials (`authorization`, an API-key header).\n // A redirect to another origin must not replay them, or a compromised or\n // merely sloppy endpoint can harvest a tenant's key with a single 302.\n let sameOrigin = true\n\n for (let hop = 0; hop <= maxRedirects; hop += 1) {\n if (outOfTime()) {\n throw new WebResearchError('timeout', `Request to ${rawUrl} exceeded its ${maxTotalMs}ms budget`)\n }\n const vetted = await assertPublicUrl(currentUrl, 'web research fetch', {\n ...(options.lookup ? { lookup: options.lookup } : {}),\n ...(options.allowPrivateHosts ? { allowPrivateHosts: options.allowPrivateHosts } : {}),\n })\n await waitForHost(vetted.url.host, init.signal)\n\n let sent: SentRequest | null = null\n let lastError: WebResearchError | null = null\n for (let retry = 0; retry <= maxRetries; retry += 1) {\n try {\n sent = await attempt(vetted.url, vetted.addresses, init, sameOrigin)\n const status = sent.response.statusCode ?? 0\n if (status >= 400) {\n const error = statusError(status, vetted.url.href, headerMap(sent.response))\n sent.response.resume()\n sent.dispose()\n sent = null\n throw error\n }\n break\n } catch (error) {\n // Branded rather than `instanceof`: this package typechecks as `src`\n // and runs as `dist`, so the two class identities can differ.\n const typed = isWebResearchError(error)\n ? error\n : new WebResearchError('network', errorMessage(error), { cause: error })\n lastError = typed\n if (typed.code === 'aborted' || retry === maxRetries || !isRetriable(typed)) throw typed\n // Retrying past the budget only delays the failure the caller already earned.\n if (outOfTime()) throw typed\n const backoff = Math.min(typed.retryAfterMs ?? 250 * 2 ** retry, MAX_BACKOFF_MS)\n await sleep(backoff, init.signal)\n }\n }\n if (!sent) throw lastError ?? new WebResearchError('network', `Request to ${currentUrl} failed`)\n\n const headers = headerMap(sent.response)\n const status = sent.response.statusCode ?? 0\n\n if (status >= 300 && status < 400) {\n const location = headers.get('location')\n sent.response.resume()\n sent.dispose()\n if (!location) {\n throw new WebResearchError('bad_response', `Redirect from ${currentUrl} without a Location header`)\n }\n const next = new URL(location, vetted.url)\n if (next.origin !== vetted.url.origin) sameOrigin = false\n currentUrl = next.toString()\n continue\n }\n\n const contentType = contentTypeOf(headers)\n if (!isAcceptable(contentType, init.accept)) {\n sent.response.resume()\n sent.dispose()\n throw new WebResearchError(\n 'unsupported_content_type',\n `Refusing to decode ${contentType ?? 'unknown'} from ${vetted.url.href} as text`,\n )\n }\n\n try {\n const { text, truncated } = await readCapped(decompress(sent.response), maxBytes)\n return { url: vetted.url.href, status, headers, contentType, body: text, truncated }\n } catch (error) {\n const terminal = sent.terminalError()\n if (terminal) throw terminal\n throw isWebResearchError(error)\n ? error\n : new WebResearchError('network', `Reading ${vetted.url.href} failed: ${errorMessage(error)}`, {\n cause: error,\n })\n } finally {\n sent.dispose()\n }\n }\n\n throw new WebResearchError('too_many_redirects', `Exceeded ${maxRedirects} redirects from ${rawUrl}`)\n }\n\n return { request }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,SAAS,WAAW,WAAW,gBAAsC;AAC9E,SAAS,SAAS,YAAY,WAAW,iBAAiB;AAC1D,SAAS,YAAiC;AAE1C,SAAS,wBAAwB,cAAc,qBAAqB;AACpE,SAAS,kBAAkB,cAAc,0BAA0B;AAEnE,SAAS,uBAAsC;AAE/C,MAAM,qBACJ;AACF,MAAM,oBAAoB,MAAM;AAChC,MAAM,qBAAqB;AAC3B,MAAM,gBAAgB;AACtB,MAAM,cAAc;AACpB,MAAM,iBAAiB;AAOvB,MAAM,eAAe;AASrB,MAAM,2BAA2B;AA6B1B,SAAS,mBAAmB,WAA8C;AAC/E,QAAM,UAAU,UAAU,IAAI,CAAC,aAAa,EAAE,SAAS,QAAQ,KAAK,OAAO,EAAE,EAAE;AAC/E,SAAO,CAAC,UAAU,SAAS,aAAa;AACtC,QAAI,QAAQ,WAAW,GAAG;AACxB,eAAS,IAAI,MAAM,yBAAyB,QAAQ,EAAE,GAAG,IAAI,CAAC;AAC9D;AAAA,IACF;AACA,UAAM,kBAAkB,OAAO,YAAY,WAAW,UAAW,SAAS,UAAU;AACpF,UAAM,WAAW,kBACb,QAAQ,OAAO,CAAC,UAAU,MAAM,WAAW,eAAe,IAC1D;AACJ,UAAM,WAAW,SAAS,SAAS,IAAI,WAAW;AAClD,QAAI,OAAO,YAAY,YAAY,SAAS,KAAK;AAC/C,eAAS,MAAM,QAAQ;AACvB;AAAA,IACF;AACA,aAAS,MAAM,SAAS,CAAC,EAAE,SAAS,SAAS,CAAC,EAAE,MAAM;AAAA,EACxD;AACF;AAEA,SAAS,aAAa,IAAY,QAAqC;AAIrE,MAAI,CAAC,OAAO,SAAS,EAAE,KAAK,MAAM,EAAG,QAAO,QAAQ,QAAQ;AAC5D,QAAM,UAAU,KAAK,eAAe,eAAe;AACnD,SAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,UAAM,QAAQ,WAAW,QAAQ,OAAO;AACxC,aAAS,SAAe;AACtB,mBAAa,KAAK;AAClB,cAAQ,oBAAoB,SAAS,MAAM;AAC3C,cAAQ;AAAA,IACV;AACA,YAAQ,iBAAiB,SAAS,QAAQ,EAAE,MAAM,KAAK,CAAC;AAAA,EAC1D,CAAC;AACH;AAEA,SAAS,WAAW,UAAqC;AACvD,QAAM,WAAW,OAAO,SAAS,QAAQ,kBAAkB,KAAK,EAAE,EAAE,YAAY;AAChF,MAAI,aAAa,UAAU,aAAa,SAAU,QAAO,SAAS,KAAK,aAAa,CAAC;AACrF,MAAI,aAAa,UAAW,QAAO,SAAS,KAAK,cAAc,CAAC;AAChE,MAAI,aAAa,KAAM,QAAO,SAAS,KAAK,uBAAuB,CAAC;AACpE,SAAO;AACT;AAMA,eAAe,WACb,QACA,UAC8D;AAC9D,QAAM,UAAU,IAAI,YAAY,OAAO;AACvC,MAAI,OAAO;AACX,MAAI,QAAQ;AACZ,MAAI,YAAY;AAChB,MAAI;AACF,qBAAiB,SAAS,QAAQ;AAChC,YAAM,SAAS;AACf,UAAI,SAAS,UAAU;AACrB,oBAAY;AACZ;AAAA,MACF;AACA,YAAM,YAAY,WAAW;AAC7B,UAAI,OAAO,aAAa,WAAW;AACjC,gBAAQ,QAAQ,OAAO,OAAO,SAAS,GAAG,SAAS,GAAG,EAAE,QAAQ,KAAK,CAAC;AACtE,gBAAQ;AACR,oBAAY;AACZ;AAAA,MACF;AACA,cAAQ,QAAQ,OAAO,QAAQ,EAAE,QAAQ,KAAK,CAAC;AAC/C,eAAS,OAAO;AAAA,IAClB;AAAA,EACF,UAAE;AACA,WAAO,QAAQ;AAAA,EACjB;AACA,UAAQ,QAAQ,OAAO;AACvB,SAAO,EAAE,MAAM,WAAW,MAAM;AAClC;AAwBA,SAAS,KAAK,QAAa,WAA8B,SAA4C;AACnG,SAAO,IAAI,QAAQ,CAAC,SAAS,WAAW;AACtC,UAAM,SAAS,OAAO,aAAa;AACnC,UAAM,QAAQ,SACV,IAAI,WAAW,EAAE,WAAW,OAAO,QAAQ,mBAAmB,SAAS,EAAE,CAAC,IAC1E,IAAI,UAAU,EAAE,WAAW,OAAO,QAAQ,mBAAmB,SAAS,EAAE,CAAC;AAC7E,UAAM,YAAY,SAAS,YAAY;AACvC,UAAM,UAAU;AAAA,MACd;AAAA,QACE,UAAU,OAAO;AAAA,QACjB,UAAU,OAAO;AAAA,QACjB,MAAM,OAAO,SAAS,SAAS,MAAM;AAAA,QACrC,MAAM,GAAG,OAAO,QAAQ,GAAG,OAAO,MAAM;AAAA,QACxC,QAAQ,QAAQ;AAAA,QAChB,SAAS,QAAQ;AAAA,QACjB;AAAA,MACF;AAAA,MACA,CAAC,aAAa;AACZ,qBAAa,WAAW;AACxB,gBAAQ,EAAE,UAAU,SAAS,eAAe,MAAM,SAAS,CAAC;AAAA,MAC9D;AAAA,IACF;AAEA,QAAI,WAAoC;AACxC,UAAM,OAAO,CAAC,UAAkC;AAC9C,iBAAW;AACX,cAAQ,QAAQ,KAAK;AAAA,IACvB;AASA,UAAM,UAAU,MAAY;AAC1B,mBAAa,WAAW;AACxB,cAAQ,WAAW,CAAC;AACpB,cAAQ,QAAQ,oBAAoB,SAAS,OAAO;AACpD,cAAQ,QAAQ;AAChB,YAAM,QAAQ;AAAA,IAChB;AAEA,UAAM,cAAc,WAAW,MAAM;AACnC,WAAK,IAAI,iBAAiB,WAAW,cAAc,OAAO,IAAI,YAAY,CAAC;AAAA,IAC7E,GAAG,QAAQ,SAAS;AAEpB,YAAQ,WAAW,QAAQ,WAAW,MAAM;AAC1C,WAAK,IAAI,iBAAiB,WAAW,cAAc,OAAO,IAAI,UAAU,CAAC;AAAA,IAC3E,CAAC;AAED,aAAS,UAAgB;AACvB,WAAK,IAAI,iBAAiB,WAAW,cAAc,OAAO,IAAI,cAAc,CAAC;AAAA,IAC/E;AACA,YAAQ,QAAQ,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AAEjE,YAAQ,GAAG,SAAS,CAAC,UAAU;AAC7B,cAAQ;AACR;AAAA,QACE,aACG,mBAAmB,KAAK,IACrB,QACA,IAAI,iBAAiB,WAAW,cAAc,OAAO,IAAI,YAAY,aAAa,KAAK,CAAC,IAAI;AAAA,UAC1F,OAAO;AAAA,QACT,CAAC;AAAA,MACT;AAAA,IACF,CAAC;AAED,QAAI,QAAQ,SAAS,OAAW,SAAQ,MAAM,QAAQ,IAAI;AAC1D,YAAQ,IAAI;AAAA,EACd,CAAC;AACH;AAEA,SAAS,UAAU,UAAgD;AACjE,QAAM,UAAU,oBAAI,IAAoB;AACxC,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,SAAS,OAAO,GAAG;AAC3D,QAAI,UAAU,OAAW;AACzB,YAAQ,IAAI,IAAI,YAAY,GAAG,MAAM,QAAQ,KAAK,IAAI,MAAM,KAAK,IAAI,IAAI,KAAK;AAAA,EAChF;AACA,SAAO;AACT;AAEA,SAAS,aAAa,SAAkD;AACtE,QAAM,MAAM,QAAQ,IAAI,aAAa;AACrC,MAAI,CAAC,IAAK,QAAO;AACjB,QAAM,UAAU,OAAO,SAAS,IAAI,KAAK,GAAG,EAAE;AAC9C,MAAI,OAAO,SAAS,OAAO,KAAK,WAAW,EAAG,QAAO,UAAU;AAC/D,QAAM,OAAO,KAAK,MAAM,GAAG;AAC3B,MAAI,OAAO,SAAS,IAAI,EAAG,QAAO,KAAK,IAAI,GAAG,OAAO,KAAK,IAAI,CAAC;AAC/D,SAAO;AACT;AAEA,SAAS,cAAc,SAA6C;AAClE,QAAM,MAAM,QAAQ,IAAI,cAAc;AACtC,MAAI,CAAC,IAAK,QAAO;AACjB,SAAO,IAAI,MAAM,GAAG,EAAE,CAAC,EAAE,KAAK,EAAE,YAAY;AAC9C;AAEA,SAAS,aAAa,aAA4B,QAAgD;AAChG,MAAI,CAAC,UAAU,OAAO,WAAW,EAAG,QAAO;AAC3C,MAAI,gBAAgB,KAAM,QAAO;AACjC,SAAO,OAAO,KAAK,CAAC,WAAW,YAAY,WAAW,MAAM,CAAC;AAC/D;AAEA,SAAS,YAAY,QAAgB,KAAa,SAAgD;AAChG,MAAI,WAAW,KAAK;AAClB,WAAO,IAAI,iBAAiB,WAAW,mBAAmB,GAAG,eAAe;AAAA,MAC1E;AAAA,MACA,cAAc,aAAa,OAAO;AAAA,IACpC,CAAC;AAAA,EACH;AACA,MAAI,WAAW,OAAO,WAAW,KAAK;AACpC,WAAO,IAAI,iBAAiB,WAAW,oBAAoB,GAAG,UAAU,MAAM,KAAK,EAAE,OAAO,CAAC;AAAA,EAC/F;AACA,SAAO,IAAI,iBAAiB,gBAAgB,cAAc,GAAG,qBAAqB,MAAM,IAAI,EAAE,OAAO,CAAC;AACxG;AAEA,SAAS,YAAY,OAAkC;AACrD,MAAI,MAAM,SAAS,UAAW,QAAO,MAAM,WAAW;AACtD,MAAI,MAAM,SAAS,UAAW,QAAO;AACrC,SAAO,MAAM,SAAS,mBAAmB,MAAM,UAAU,MAAM;AACjE;AAOO,SAAS,iBAAiB,UAA6B,CAAC,GAAe;AAC5E,QAAM,YAAY,QAAQ,aAAa;AACvC,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,kBAAkB,QAAQ,YAAY;AAC5C,QAAM,mBAAmB,QAAQ,aAAa;AAC9C,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,MAAM,QAAQ,OAAO,KAAK;AAChC,QAAM,gBAAgB,oBAAI,IAAoB;AAE9C,iBAAe,YAAY,MAAc,QAAqC;AAC5E,QAAI,gBAAgB,EAAG;AACvB,UAAM,WAAW,cAAc,IAAI,IAAI,KAAK;AAC5C,UAAM,UAAU,IAAI;AACpB,UAAM,QAAQ,WAAW;AACzB,kBAAc,IAAI,MAAM,KAAK,IAAI,SAAS,QAAQ,IAAI,YAAY;AAClE,QAAI,QAAQ,EAAG,OAAM,MAAM,OAAO,MAAM;AAAA,EAC1C;AAEA,iBAAe,QACb,QACA,WACA,MACA,oBACA;AACA,UAAM,UAAkC;AAAA,MACtC,cAAc;AAAA,MACd,mBAAmB;AAAA,MACnB,mBAAmB;AAAA,MACnB,QAAQ,KAAK,QAAQ,SAAS,GAAG,KAAK,OAAO,KAAK,IAAI,CAAC,gBAAgB;AAAA,MACvE,GAAI,KAAK,SAAS,SAAY,CAAC,IAAI,EAAE,gBAAgB,mBAAmB;AAAA,MACxE,GAAI,qBAAsB,KAAK,WAAW,CAAC,IAAK,CAAC;AAAA,IACnD;AACA,WAAO,KAAK,QAAQ,WAAW;AAAA,MAC7B,QAAQ,KAAK,UAAU;AAAA,MACvB;AAAA,MACA,GAAI,KAAK,SAAS,SAAY,CAAC,IAAI,EAAE,MAAM,KAAK,KAAK;AAAA,MACrD,WAAW,KAAK,aAAa;AAAA,MAC7B,GAAI,KAAK,SAAS,EAAE,QAAQ,KAAK,OAAO,IAAI,CAAC;AAAA,IAC/C,CAAC;AAAA,EACH;AAEA,iBAAe,QAAQ,QAAgB,OAAwB,CAAC,GAA0B;AACxF,UAAM,WAAW,KAAK,YAAY;AAClC,UAAM,YAAY,IAAI,IAAI;AAC1B,UAAM,YAAY,MAAe,IAAI,KAAK;AAC1C,QAAI,aAAa;AAIjB,QAAI,aAAa;AAEjB,aAAS,MAAM,GAAG,OAAO,cAAc,OAAO,GAAG;AAC/C,UAAI,UAAU,GAAG;AACf,cAAM,IAAI,iBAAiB,WAAW,cAAc,MAAM,iBAAiB,UAAU,WAAW;AAAA,MAClG;AACA,YAAM,SAAS,MAAM,gBAAgB,YAAY,sBAAsB;AAAA,QACrE,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;AAAA,QACnD,GAAI,QAAQ,oBAAoB,EAAE,mBAAmB,QAAQ,kBAAkB,IAAI,CAAC;AAAA,MACtF,CAAC;AACD,YAAM,YAAY,OAAO,IAAI,MAAM,KAAK,MAAM;AAE9C,UAAI,OAA2B;AAC/B,UAAI,YAAqC;AACzC,eAAS,QAAQ,GAAG,SAAS,YAAY,SAAS,GAAG;AACnD,YAAI;AACF,iBAAO,MAAM,QAAQ,OAAO,KAAK,OAAO,WAAW,MAAM,UAAU;AACnE,gBAAMA,UAAS,KAAK,SAAS,cAAc;AAC3C,cAAIA,WAAU,KAAK;AACjB,kBAAM,QAAQ,YAAYA,SAAQ,OAAO,IAAI,MAAM,UAAU,KAAK,QAAQ,CAAC;AAC3E,iBAAK,SAAS,OAAO;AACrB,iBAAK,QAAQ;AACb,mBAAO;AACP,kBAAM;AAAA,UACR;AACA;AAAA,QACF,SAAS,OAAO;AAGd,gBAAM,QAAQ,mBAAmB,KAAK,IAClC,QACA,IAAI,iBAAiB,WAAW,aAAa,KAAK,GAAG,EAAE,OAAO,MAAM,CAAC;AACzE,sBAAY;AACZ,cAAI,MAAM,SAAS,aAAa,UAAU,cAAc,CAAC,YAAY,KAAK,EAAG,OAAM;AAEnF,cAAI,UAAU,EAAG,OAAM;AACvB,gBAAM,UAAU,KAAK,IAAI,MAAM,gBAAgB,MAAM,KAAK,OAAO,cAAc;AAC/E,gBAAM,MAAM,SAAS,KAAK,MAAM;AAAA,QAClC;AAAA,MACF;AACA,UAAI,CAAC,KAAM,OAAM,aAAa,IAAI,iBAAiB,WAAW,cAAc,UAAU,SAAS;AAE/F,YAAM,UAAU,UAAU,KAAK,QAAQ;AACvC,YAAM,SAAS,KAAK,SAAS,cAAc;AAE3C,UAAI,UAAU,OAAO,SAAS,KAAK;AACjC,cAAM,WAAW,QAAQ,IAAI,UAAU;AACvC,aAAK,SAAS,OAAO;AACrB,aAAK,QAAQ;AACb,YAAI,CAAC,UAAU;AACb,gBAAM,IAAI,iBAAiB,gBAAgB,iBAAiB,UAAU,4BAA4B;AAAA,QACpG;AACA,cAAM,OAAO,IAAI,IAAI,UAAU,OAAO,GAAG;AACzC,YAAI,KAAK,WAAW,OAAO,IAAI,OAAQ,cAAa;AACpD,qBAAa,KAAK,SAAS;AAC3B;AAAA,MACF;AAEA,YAAM,cAAc,cAAc,OAAO;AACzC,UAAI,CAAC,aAAa,aAAa,KAAK,MAAM,GAAG;AAC3C,aAAK,SAAS,OAAO;AACrB,aAAK,QAAQ;AACb,cAAM,IAAI;AAAA,UACR;AAAA,UACA,sBAAsB,eAAe,SAAS,SAAS,OAAO,IAAI,IAAI;AAAA,QACxE;AAAA,MACF;AAEA,UAAI;AACF,cAAM,EAAE,MAAM,UAAU,IAAI,MAAM,WAAW,WAAW,KAAK,QAAQ,GAAG,QAAQ;AAChF,eAAO,EAAE,KAAK,OAAO,IAAI,MAAM,QAAQ,SAAS,aAAa,MAAM,MAAM,UAAU;AAAA,MACrF,SAAS,OAAO;AACd,cAAM,WAAW,KAAK,cAAc;AACpC,YAAI,SAAU,OAAM;AACpB,cAAM,mBAAmB,KAAK,IAC1B,QACA,IAAI,iBAAiB,WAAW,WAAW,OAAO,IAAI,IAAI,YAAY,aAAa,KAAK,CAAC,IAAI;AAAA,UAC3F,OAAO;AAAA,QACT,CAAC;AAAA,MACP,UAAE;AACA,aAAK,QAAQ;AAAA,MACf;AAAA,IACF;AAEA,UAAM,IAAI,iBAAiB,sBAAsB,YAAY,YAAY,mBAAmB,MAAM,EAAE;AAAA,EACtG;AAEA,SAAO,EAAE,QAAQ;AACnB;",
|
|
6
|
+
"names": ["status"]
|
|
7
|
+
}
|
package/dist/net/ssrf.js
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import { lookup as dnsLookup } from "node:dns/promises";
|
|
2
|
+
import { isIP } from "node:net";
|
|
3
|
+
import { WebResearchError } from "../contract/errors.js";
|
|
4
|
+
const defaultLookup = async (hostname) => {
|
|
5
|
+
const records = await dnsLookup(hostname, { all: true });
|
|
6
|
+
return records.map((record) => ({ address: record.address, family: record.family }));
|
|
7
|
+
};
|
|
8
|
+
function ipv4ToInt(address) {
|
|
9
|
+
const parts = address.split(".");
|
|
10
|
+
if (parts.length !== 4) return null;
|
|
11
|
+
let value = 0;
|
|
12
|
+
for (const part of parts) {
|
|
13
|
+
if (!/^\d{1,3}$/.test(part)) return null;
|
|
14
|
+
const octet = Number(part);
|
|
15
|
+
if (octet > 255) return null;
|
|
16
|
+
value = value * 256 + octet;
|
|
17
|
+
}
|
|
18
|
+
return value >>> 0;
|
|
19
|
+
}
|
|
20
|
+
function isPrivateIpv4(address) {
|
|
21
|
+
const value = ipv4ToInt(address);
|
|
22
|
+
if (value === null) return true;
|
|
23
|
+
const inRange = (base, maskBits) => {
|
|
24
|
+
const baseValue = ipv4ToInt(base);
|
|
25
|
+
if (baseValue === null) return false;
|
|
26
|
+
const mask = maskBits === 0 ? 0 : 4294967295 << 32 - maskBits >>> 0;
|
|
27
|
+
return (value & mask) === (baseValue & mask);
|
|
28
|
+
};
|
|
29
|
+
return inRange("0.0.0.0", 8) || inRange("10.0.0.0", 8) || inRange("100.64.0.0", 10) || inRange("127.0.0.0", 8) || inRange("169.254.0.0", 16) || inRange("172.16.0.0", 12) || inRange("192.0.0.0", 24) || inRange("192.168.0.0", 16) || inRange("198.18.0.0", 15) || inRange("224.0.0.0", 4) || inRange("240.0.0.0", 4);
|
|
30
|
+
}
|
|
31
|
+
function normalizeIpv6(address) {
|
|
32
|
+
const zoneIndex = address.indexOf("%");
|
|
33
|
+
return (zoneIndex === -1 ? address : address.slice(0, zoneIndex)).toLowerCase();
|
|
34
|
+
}
|
|
35
|
+
function parseHextets(segment) {
|
|
36
|
+
if (segment === "") return [];
|
|
37
|
+
const parsed = [];
|
|
38
|
+
for (const part of segment.split(":")) {
|
|
39
|
+
if (!/^[0-9a-f]{1,4}$/.test(part)) return null;
|
|
40
|
+
parsed.push(Number.parseInt(part, 16));
|
|
41
|
+
}
|
|
42
|
+
return parsed;
|
|
43
|
+
}
|
|
44
|
+
function expandIpv6(raw) {
|
|
45
|
+
let address = raw;
|
|
46
|
+
const dotted = /:(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})$/.exec(address);
|
|
47
|
+
if (dotted) {
|
|
48
|
+
const value = ipv4ToInt(dotted[1]);
|
|
49
|
+
if (value === null) return null;
|
|
50
|
+
const high = (value >>> 16 & 65535).toString(16);
|
|
51
|
+
const low = (value & 65535).toString(16);
|
|
52
|
+
address = `${address.slice(0, dotted.index)}:${high}:${low}`;
|
|
53
|
+
}
|
|
54
|
+
const halves = address.split("::");
|
|
55
|
+
if (halves.length > 2) return null;
|
|
56
|
+
const head = parseHextets(halves[0] ?? "");
|
|
57
|
+
if (head === null) return null;
|
|
58
|
+
if (halves.length === 1) return head.length === 8 ? head : null;
|
|
59
|
+
const tail = parseHextets(halves[1] ?? "");
|
|
60
|
+
if (tail === null) return null;
|
|
61
|
+
const missing = 8 - head.length - tail.length;
|
|
62
|
+
if (missing < 0) return null;
|
|
63
|
+
return [...head, ...Array(missing).fill(0), ...tail];
|
|
64
|
+
}
|
|
65
|
+
function hextetsToIpv4(high, low) {
|
|
66
|
+
return `${high >>> 8}.${high & 255}.${low >>> 8}.${low & 255}`;
|
|
67
|
+
}
|
|
68
|
+
function embeddedIpv4(hextets) {
|
|
69
|
+
const [first, second, third, fourth, fifth, sixth, seventh, eighth] = hextets;
|
|
70
|
+
const zeroPrefix = first === 0 && second === 0 && third === 0 && fourth === 0 && fifth === 0;
|
|
71
|
+
if (zeroPrefix && (sixth === 65535 || sixth === 0)) return hextetsToIpv4(seventh, eighth);
|
|
72
|
+
const nat64 = first === 100 && second === 65435 && third === 0 && fourth === 0 && fifth === 0 && sixth === 0;
|
|
73
|
+
if (nat64) return hextetsToIpv4(seventh, eighth);
|
|
74
|
+
return null;
|
|
75
|
+
}
|
|
76
|
+
function isPrivateIpv6(address) {
|
|
77
|
+
const normalized = normalizeIpv6(address);
|
|
78
|
+
if (normalized === "::1" || normalized === "::") return true;
|
|
79
|
+
const hextets = expandIpv6(normalized);
|
|
80
|
+
if (hextets === null) return true;
|
|
81
|
+
const embedded = embeddedIpv4(hextets);
|
|
82
|
+
if (embedded !== null) return isPrivateIpv4(embedded);
|
|
83
|
+
const first = hextets[0];
|
|
84
|
+
if ((first & 65024) === 64512) return true;
|
|
85
|
+
if ((first & 65472) === 65152) return true;
|
|
86
|
+
if ((first & 65280) === 65280) return true;
|
|
87
|
+
return false;
|
|
88
|
+
}
|
|
89
|
+
function isPrivateAddress(address) {
|
|
90
|
+
const family = isIP(address);
|
|
91
|
+
if (family === 4) return isPrivateIpv4(address);
|
|
92
|
+
if (family === 6) return isPrivateIpv6(address);
|
|
93
|
+
return true;
|
|
94
|
+
}
|
|
95
|
+
function isLoopbackHostname(hostname) {
|
|
96
|
+
const host = hostname.toLowerCase();
|
|
97
|
+
return host === "localhost" || host.endsWith(".localhost");
|
|
98
|
+
}
|
|
99
|
+
function isPrivateHostAllowed(hostname, allowlist) {
|
|
100
|
+
if (!allowlist || allowlist.length === 0) return false;
|
|
101
|
+
const host = hostname.toLowerCase();
|
|
102
|
+
return allowlist.some((entry) => {
|
|
103
|
+
const allowed = entry.trim().toLowerCase();
|
|
104
|
+
if (allowed.length === 0) return false;
|
|
105
|
+
return host === allowed || host.endsWith(`.${allowed}`);
|
|
106
|
+
});
|
|
107
|
+
}
|
|
108
|
+
async function assertPublicUrl(rawUrl, label = "request", options = {}) {
|
|
109
|
+
let parsed;
|
|
110
|
+
try {
|
|
111
|
+
parsed = new URL(rawUrl);
|
|
112
|
+
} catch {
|
|
113
|
+
throw new WebResearchError("ssrf_blocked", `Invalid URL for ${label}`);
|
|
114
|
+
}
|
|
115
|
+
if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
|
|
116
|
+
throw new WebResearchError("ssrf_blocked", `Blocked URL scheme for ${label}: ${parsed.protocol}`);
|
|
117
|
+
}
|
|
118
|
+
if (parsed.username || parsed.password) {
|
|
119
|
+
throw new WebResearchError("ssrf_blocked", `Credentials in URL are not allowed for ${label}`);
|
|
120
|
+
}
|
|
121
|
+
const hostname = parsed.hostname.replace(/^\[|\]$/g, "");
|
|
122
|
+
const privateAllowed = isPrivateHostAllowed(hostname, options.allowPrivateHosts);
|
|
123
|
+
if (!privateAllowed && isLoopbackHostname(hostname)) {
|
|
124
|
+
throw new WebResearchError("ssrf_blocked", `Blocked loopback host for ${label}: ${hostname}`);
|
|
125
|
+
}
|
|
126
|
+
if (isIP(hostname)) {
|
|
127
|
+
if (!privateAllowed && isPrivateAddress(hostname)) {
|
|
128
|
+
throw new WebResearchError("ssrf_blocked", `Blocked non-public address for ${label}: ${hostname}`);
|
|
129
|
+
}
|
|
130
|
+
return { url: parsed, addresses: [hostname] };
|
|
131
|
+
}
|
|
132
|
+
const lookup = options.lookup ?? defaultLookup;
|
|
133
|
+
const failClosed = options.failClosed ?? true;
|
|
134
|
+
let records;
|
|
135
|
+
try {
|
|
136
|
+
records = await lookup(hostname);
|
|
137
|
+
} catch {
|
|
138
|
+
if (failClosed) {
|
|
139
|
+
throw new WebResearchError("ssrf_blocked", `Could not resolve host for ${label}: ${hostname}`);
|
|
140
|
+
}
|
|
141
|
+
return { url: parsed, addresses: [] };
|
|
142
|
+
}
|
|
143
|
+
if (records.length === 0) {
|
|
144
|
+
if (failClosed) {
|
|
145
|
+
throw new WebResearchError("ssrf_blocked", `Host did not resolve for ${label}: ${hostname}`);
|
|
146
|
+
}
|
|
147
|
+
return { url: parsed, addresses: [] };
|
|
148
|
+
}
|
|
149
|
+
for (const { address } of records) {
|
|
150
|
+
if (!privateAllowed && isPrivateAddress(address)) {
|
|
151
|
+
throw new WebResearchError(
|
|
152
|
+
"ssrf_blocked",
|
|
153
|
+
`Host ${hostname} resolves to non-public address ${address} for ${label}`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
return { url: parsed, addresses: records.map((record) => record.address) };
|
|
158
|
+
}
|
|
159
|
+
export {
|
|
160
|
+
assertPublicUrl,
|
|
161
|
+
isLoopbackHostname,
|
|
162
|
+
isPrivateAddress,
|
|
163
|
+
isPrivateHostAllowed
|
|
164
|
+
};
|
|
165
|
+
//# sourceMappingURL=ssrf.js.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/net/ssrf.ts"],
|
|
4
|
+
"sourcesContent": ["import { lookup as dnsLookup } from 'node:dns/promises'\nimport { isIP } from 'node:net'\nimport { WebResearchError } from '../contract/errors'\n\nexport type LookupFn = (hostname: string) => Promise<Array<{ address: string; family: number }>>\n\nconst defaultLookup: LookupFn = async (hostname) => {\n const records = await dnsLookup(hostname, { all: true })\n return records.map((record) => ({ address: record.address, family: record.family }))\n}\n\nfunction ipv4ToInt(address: string): number | null {\n const parts = address.split('.')\n if (parts.length !== 4) return null\n let value = 0\n for (const part of parts) {\n if (!/^\\d{1,3}$/.test(part)) return null\n const octet = Number(part)\n if (octet > 255) return null\n value = value * 256 + octet\n }\n return value >>> 0\n}\n\nfunction isPrivateIpv4(address: string): boolean {\n const value = ipv4ToInt(address)\n if (value === null) return true\n const inRange = (base: string, maskBits: number): boolean => {\n const baseValue = ipv4ToInt(base)\n if (baseValue === null) return false\n const mask = maskBits === 0 ? 0 : (0xffffffff << (32 - maskBits)) >>> 0\n return (value & mask) === (baseValue & mask)\n }\n return (\n inRange('0.0.0.0', 8) ||\n inRange('10.0.0.0', 8) ||\n inRange('100.64.0.0', 10) ||\n inRange('127.0.0.0', 8) ||\n inRange('169.254.0.0', 16) ||\n inRange('172.16.0.0', 12) ||\n inRange('192.0.0.0', 24) ||\n inRange('192.168.0.0', 16) ||\n inRange('198.18.0.0', 15) ||\n inRange('224.0.0.0', 4) ||\n inRange('240.0.0.0', 4)\n )\n}\n\nfunction normalizeIpv6(address: string): string {\n const zoneIndex = address.indexOf('%')\n return (zoneIndex === -1 ? address : address.slice(0, zoneIndex)).toLowerCase()\n}\n\nfunction parseHextets(segment: string): number[] | null {\n if (segment === '') return []\n const parsed: number[] = []\n for (const part of segment.split(':')) {\n if (!/^[0-9a-f]{1,4}$/.test(part)) return null\n parsed.push(Number.parseInt(part, 16))\n }\n return parsed\n}\n\n/** Expands any IPv6 spelling (`::`, embedded dotted-quad) to its eight hextets. */\nfunction expandIpv6(raw: string): number[] | null {\n let address = raw\n const dotted = /:(\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3})$/.exec(address)\n if (dotted) {\n const value = ipv4ToInt(dotted[1])\n if (value === null) return null\n const high = ((value >>> 16) & 0xffff).toString(16)\n const low = (value & 0xffff).toString(16)\n address = `${address.slice(0, dotted.index)}:${high}:${low}`\n }\n const halves = address.split('::')\n if (halves.length > 2) return null\n const head = parseHextets(halves[0] ?? '')\n if (head === null) return null\n if (halves.length === 1) return head.length === 8 ? head : null\n const tail = parseHextets(halves[1] ?? '')\n if (tail === null) return null\n const missing = 8 - head.length - tail.length\n if (missing < 0) return null\n return [...head, ...Array<number>(missing).fill(0), ...tail]\n}\n\nfunction hextetsToIpv4(high: number, low: number): string {\n return `${high >>> 8}.${high & 0xff}.${low >>> 8}.${low & 0xff}`\n}\n\n/**\n * Recovers the IPv4 address tunnelled inside an IPv6 one. This is load-bearing:\n * `new URL('http://[::ffff:127.0.0.1]/').hostname` is `::ffff:7f00:1`, so a guard\n * that only pattern-matches the dotted-quad spelling never fires and loopback \u2014\n * and the cloud metadata endpoint \u2014 walk straight through.\n */\nfunction embeddedIpv4(hextets: readonly number[]): string | null {\n const [first, second, third, fourth, fifth, sixth, seventh, eighth] = hextets\n const zeroPrefix = first === 0 && second === 0 && third === 0 && fourth === 0 && fifth === 0\n if (zeroPrefix && (sixth === 0xffff || sixth === 0)) return hextetsToIpv4(seventh, eighth)\n const nat64 =\n first === 0x0064 && second === 0xff9b && third === 0 && fourth === 0 && fifth === 0 && sixth === 0\n if (nat64) return hextetsToIpv4(seventh, eighth)\n return null\n}\n\nfunction isPrivateIpv6(address: string): boolean {\n const normalized = normalizeIpv6(address)\n if (normalized === '::1' || normalized === '::') return true\n const hextets = expandIpv6(normalized)\n if (hextets === null) return true\n const embedded = embeddedIpv4(hextets)\n if (embedded !== null) return isPrivateIpv4(embedded)\n const first = hextets[0]\n if ((first & 0xfe00) === 0xfc00) return true\n if ((first & 0xffc0) === 0xfe80) return true\n if ((first & 0xff00) === 0xff00) return true\n return false\n}\n\nexport function isPrivateAddress(address: string): boolean {\n const family = isIP(address)\n if (family === 4) return isPrivateIpv4(address)\n if (family === 6) return isPrivateIpv6(address)\n return true\n}\n\n/** `localhost` and anything under it never reach DNS. */\nexport function isLoopbackHostname(hostname: string): boolean {\n const host = hostname.toLowerCase()\n return host === 'localhost' || host.endsWith('.localhost')\n}\n\nexport type AssertPublicUrlOptions = {\n readonly lookup?: LookupFn\n /**\n * Whether an unresolvable host is a hard failure. Defaults to `true` \u2014 unlike\n * reference implementations that fail open here, we pin the connection to the\n * addresses this check returns, so no addresses means no pinning, and letting\n * the request proceed would silently drop a control rather than merely fail.\n */\n readonly failClosed?: boolean\n /**\n * Hosts permitted to resolve to a non-public address.\n *\n * An allowlist rather than a blanket \"allow private\", because the two are not\n * remotely equivalent: a boolean opens every internal address to whatever URL\n * a model or a search result happens to produce, while this only unblocks\n * destinations an operator named. It exists because a service the operator\n * runs themselves \u2014 a SearXNG instance on the container network, a self-hosted\n * scraper \u2014 is otherwise unreachable by design.\n *\n * Matching is exact host or a dot-boundary suffix, so `internal.example.com`\n * covers `search.internal.example.com` but never `notinternal.example.com`.\n */\n readonly allowPrivateHosts?: readonly string[]\n}\n\n/** Exact host or a dot-boundary suffix; never a bare substring. */\nexport function isPrivateHostAllowed(hostname: string, allowlist: readonly string[] | undefined): boolean {\n if (!allowlist || allowlist.length === 0) return false\n const host = hostname.toLowerCase()\n return allowlist.some((entry) => {\n const allowed = entry.trim().toLowerCase()\n if (allowed.length === 0) return false\n return host === allowed || host.endsWith(`.${allowed}`)\n })\n}\n\nexport type VettedUrl = {\n readonly url: URL\n /** Vetted addresses to pin the connection to; empty only when failClosed is off. */\n readonly addresses: readonly string[]\n}\n\nexport async function assertPublicUrl(\n rawUrl: string,\n label = 'request',\n options: AssertPublicUrlOptions = {},\n): Promise<VettedUrl> {\n let parsed: URL\n try {\n parsed = new URL(rawUrl)\n } catch {\n throw new WebResearchError('ssrf_blocked', `Invalid URL for ${label}`)\n }\n\n if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {\n throw new WebResearchError('ssrf_blocked', `Blocked URL scheme for ${label}: ${parsed.protocol}`)\n }\n if (parsed.username || parsed.password) {\n throw new WebResearchError('ssrf_blocked', `Credentials in URL are not allowed for ${label}`)\n }\n\n const hostname = parsed.hostname.replace(/^\\[|\\]$/g, '')\n const privateAllowed = isPrivateHostAllowed(hostname, options.allowPrivateHosts)\n\n if (!privateAllowed && isLoopbackHostname(hostname)) {\n throw new WebResearchError('ssrf_blocked', `Blocked loopback host for ${label}: ${hostname}`)\n }\n\n if (isIP(hostname)) {\n if (!privateAllowed && isPrivateAddress(hostname)) {\n throw new WebResearchError('ssrf_blocked', `Blocked non-public address for ${label}: ${hostname}`)\n }\n return { url: parsed, addresses: [hostname] }\n }\n\n const lookup = options.lookup ?? defaultLookup\n const failClosed = options.failClosed ?? true\n let records: Array<{ address: string; family: number }>\n try {\n records = await lookup(hostname)\n } catch {\n if (failClosed) {\n throw new WebResearchError('ssrf_blocked', `Could not resolve host for ${label}: ${hostname}`)\n }\n return { url: parsed, addresses: [] }\n }\n if (records.length === 0) {\n if (failClosed) {\n throw new WebResearchError('ssrf_blocked', `Host did not resolve for ${label}: ${hostname}`)\n }\n return { url: parsed, addresses: [] }\n }\n for (const { address } of records) {\n if (!privateAllowed && isPrivateAddress(address)) {\n throw new WebResearchError(\n 'ssrf_blocked',\n `Host ${hostname} resolves to non-public address ${address} for ${label}`,\n )\n }\n }\n return { url: parsed, addresses: records.map((record) => record.address) }\n}\n"],
|
|
5
|
+
"mappings": "AAAA,SAAS,UAAU,iBAAiB;AACpC,SAAS,YAAY;AACrB,SAAS,wBAAwB;AAIjC,MAAM,gBAA0B,OAAO,aAAa;AAClD,QAAM,UAAU,MAAM,UAAU,UAAU,EAAE,KAAK,KAAK,CAAC;AACvD,SAAO,QAAQ,IAAI,CAAC,YAAY,EAAE,SAAS,OAAO,SAAS,QAAQ,OAAO,OAAO,EAAE;AACrF;AAEA,SAAS,UAAU,SAAgC;AACjD,QAAM,QAAQ,QAAQ,MAAM,GAAG;AAC/B,MAAI,MAAM,WAAW,EAAG,QAAO;AAC/B,MAAI,QAAQ;AACZ,aAAW,QAAQ,OAAO;AACxB,QAAI,CAAC,YAAY,KAAK,IAAI,EAAG,QAAO;AACpC,UAAM,QAAQ,OAAO,IAAI;AACzB,QAAI,QAAQ,IAAK,QAAO;AACxB,YAAQ,QAAQ,MAAM;AAAA,EACxB;AACA,SAAO,UAAU;AACnB;AAEA,SAAS,cAAc,SAA0B;AAC/C,QAAM,QAAQ,UAAU,OAAO;AAC/B,MAAI,UAAU,KAAM,QAAO;AAC3B,QAAM,UAAU,CAAC,MAAc,aAA8B;AAC3D,UAAM,YAAY,UAAU,IAAI;AAChC,QAAI,cAAc,KAAM,QAAO;AAC/B,UAAM,OAAO,aAAa,IAAI,IAAK,cAAe,KAAK,aAAe;AACtE,YAAQ,QAAQ,WAAW,YAAY;AAAA,EACzC;AACA,SACE,QAAQ,WAAW,CAAC,KACpB,QAAQ,YAAY,CAAC,KACrB,QAAQ,cAAc,EAAE,KACxB,QAAQ,aAAa,CAAC,KACtB,QAAQ,eAAe,EAAE,KACzB,QAAQ,cAAc,EAAE,KACxB,QAAQ,aAAa,EAAE,KACvB,QAAQ,eAAe,EAAE,KACzB,QAAQ,cAAc,EAAE,KACxB,QAAQ,aAAa,CAAC,KACtB,QAAQ,aAAa,CAAC;AAE1B;AAEA,SAAS,cAAc,SAAyB;AAC9C,QAAM,YAAY,QAAQ,QAAQ,GAAG;AACrC,UAAQ,cAAc,KAAK,UAAU,QAAQ,MAAM,GAAG,SAAS,GAAG,YAAY;AAChF;AAEA,SAAS,aAAa,SAAkC;AACtD,MAAI,YAAY,GAAI,QAAO,CAAC;AAC5B,QAAM,SAAmB,CAAC;AAC1B,aAAW,QAAQ,QAAQ,MAAM,GAAG,GAAG;AACrC,QAAI,CAAC,kBAAkB,KAAK,IAAI,EAAG,QAAO;AAC1C,WAAO,KAAK,OAAO,SAAS,MAAM,EAAE,CAAC;AAAA,EACvC;AACA,SAAO;AACT;AAGA,SAAS,WAAW,KAA8B;AAChD,MAAI,UAAU;AACd,QAAM,SAAS,yCAAyC,KAAK,OAAO;AACpE,MAAI,QAAQ;AACV,UAAM,QAAQ,UAAU,OAAO,CAAC,CAAC;AACjC,QAAI,UAAU,KAAM,QAAO;AAC3B,UAAM,QAAS,UAAU,KAAM,OAAQ,SAAS,EAAE;AAClD,UAAM,OAAO,QAAQ,OAAQ,SAAS,EAAE;AACxC,cAAU,GAAG,QAAQ,MAAM,GAAG,OAAO,KAAK,CAAC,IAAI,IAAI,IAAI,GAAG;AAAA,EAC5D;AACA,QAAM,SAAS,QAAQ,MAAM,IAAI;AACjC,MAAI,OAAO,SAAS,EAAG,QAAO;AAC9B,QAAM,OAAO,aAAa,OAAO,CAAC,KAAK,EAAE;AACzC,MAAI,SAAS,KAAM,QAAO;AAC1B,MAAI,OAAO,WAAW,EAAG,QAAO,KAAK,WAAW,IAAI,OAAO;AAC3D,QAAM,OAAO,aAAa,OAAO,CAAC,KAAK,EAAE;AACzC,MAAI,SAAS,KAAM,QAAO;AAC1B,QAAM,UAAU,IAAI,KAAK,SAAS,KAAK;AACvC,MAAI,UAAU,EAAG,QAAO;AACxB,SAAO,CAAC,GAAG,MAAM,GAAG,MAAc,OAAO,EAAE,KAAK,CAAC,GAAG,GAAG,IAAI;AAC7D;AAEA,SAAS,cAAc,MAAc,KAAqB;AACxD,SAAO,GAAG,SAAS,CAAC,IAAI,OAAO,GAAI,IAAI,QAAQ,CAAC,IAAI,MAAM,GAAI;AAChE;AAQA,SAAS,aAAa,SAA2C;AAC/D,QAAM,CAAC,OAAO,QAAQ,OAAO,QAAQ,OAAO,OAAO,SAAS,MAAM,IAAI;AACtE,QAAM,aAAa,UAAU,KAAK,WAAW,KAAK,UAAU,KAAK,WAAW,KAAK,UAAU;AAC3F,MAAI,eAAe,UAAU,SAAU,UAAU,GAAI,QAAO,cAAc,SAAS,MAAM;AACzF,QAAM,QACJ,UAAU,OAAU,WAAW,SAAU,UAAU,KAAK,WAAW,KAAK,UAAU,KAAK,UAAU;AACnG,MAAI,MAAO,QAAO,cAAc,SAAS,MAAM;AAC/C,SAAO;AACT;AAEA,SAAS,cAAc,SAA0B;AAC/C,QAAM,aAAa,cAAc,OAAO;AACxC,MAAI,eAAe,SAAS,eAAe,KAAM,QAAO;AACxD,QAAM,UAAU,WAAW,UAAU;AACrC,MAAI,YAAY,KAAM,QAAO;AAC7B,QAAM,WAAW,aAAa,OAAO;AACrC,MAAI,aAAa,KAAM,QAAO,cAAc,QAAQ;AACpD,QAAM,QAAQ,QAAQ,CAAC;AACvB,OAAK,QAAQ,WAAY,MAAQ,QAAO;AACxC,OAAK,QAAQ,WAAY,MAAQ,QAAO;AACxC,OAAK,QAAQ,WAAY,MAAQ,QAAO;AACxC,SAAO;AACT;AAEO,SAAS,iBAAiB,SAA0B;AACzD,QAAM,SAAS,KAAK,OAAO;AAC3B,MAAI,WAAW,EAAG,QAAO,cAAc,OAAO;AAC9C,MAAI,WAAW,EAAG,QAAO,cAAc,OAAO;AAC9C,SAAO;AACT;AAGO,SAAS,mBAAmB,UAA2B;AAC5D,QAAM,OAAO,SAAS,YAAY;AAClC,SAAO,SAAS,eAAe,KAAK,SAAS,YAAY;AAC3D;AA4BO,SAAS,qBAAqB,UAAkB,WAAmD;AACxG,MAAI,CAAC,aAAa,UAAU,WAAW,EAAG,QAAO;AACjD,QAAM,OAAO,SAAS,YAAY;AAClC,SAAO,UAAU,KAAK,CAAC,UAAU;AAC/B,UAAM,UAAU,MAAM,KAAK,EAAE,YAAY;AACzC,QAAI,QAAQ,WAAW,EAAG,QAAO;AACjC,WAAO,SAAS,WAAW,KAAK,SAAS,IAAI,OAAO,EAAE;AAAA,EACxD,CAAC;AACH;AAQA,eAAsB,gBACpB,QACA,QAAQ,WACR,UAAkC,CAAC,GACf;AACpB,MAAI;AACJ,MAAI;AACF,aAAS,IAAI,IAAI,MAAM;AAAA,EACzB,QAAQ;AACN,UAAM,IAAI,iBAAiB,gBAAgB,mBAAmB,KAAK,EAAE;AAAA,EACvE;AAEA,MAAI,OAAO,aAAa,WAAW,OAAO,aAAa,UAAU;AAC/D,UAAM,IAAI,iBAAiB,gBAAgB,0BAA0B,KAAK,KAAK,OAAO,QAAQ,EAAE;AAAA,EAClG;AACA,MAAI,OAAO,YAAY,OAAO,UAAU;AACtC,UAAM,IAAI,iBAAiB,gBAAgB,0CAA0C,KAAK,EAAE;AAAA,EAC9F;AAEA,QAAM,WAAW,OAAO,SAAS,QAAQ,YAAY,EAAE;AACvD,QAAM,iBAAiB,qBAAqB,UAAU,QAAQ,iBAAiB;AAE/E,MAAI,CAAC,kBAAkB,mBAAmB,QAAQ,GAAG;AACnD,UAAM,IAAI,iBAAiB,gBAAgB,6BAA6B,KAAK,KAAK,QAAQ,EAAE;AAAA,EAC9F;AAEA,MAAI,KAAK,QAAQ,GAAG;AAClB,QAAI,CAAC,kBAAkB,iBAAiB,QAAQ,GAAG;AACjD,YAAM,IAAI,iBAAiB,gBAAgB,kCAAkC,KAAK,KAAK,QAAQ,EAAE;AAAA,IACnG;AACA,WAAO,EAAE,KAAK,QAAQ,WAAW,CAAC,QAAQ,EAAE;AAAA,EAC9C;AAEA,QAAM,SAAS,QAAQ,UAAU;AACjC,QAAM,aAAa,QAAQ,cAAc;AACzC,MAAI;AACJ,MAAI;AACF,cAAU,MAAM,OAAO,QAAQ;AAAA,EACjC,QAAQ;AACN,QAAI,YAAY;AACd,YAAM,IAAI,iBAAiB,gBAAgB,8BAA8B,KAAK,KAAK,QAAQ,EAAE;AAAA,IAC/F;AACA,WAAO,EAAE,KAAK,QAAQ,WAAW,CAAC,EAAE;AAAA,EACtC;AACA,MAAI,QAAQ,WAAW,GAAG;AACxB,QAAI,YAAY;AACd,YAAM,IAAI,iBAAiB,gBAAgB,4BAA4B,KAAK,KAAK,QAAQ,EAAE;AAAA,IAC7F;AACA,WAAO,EAAE,KAAK,QAAQ,WAAW,CAAC,EAAE;AAAA,EACtC;AACA,aAAW,EAAE,QAAQ,KAAK,SAAS;AACjC,QAAI,CAAC,kBAAkB,iBAAiB,OAAO,GAAG;AAChD,YAAM,IAAI;AAAA,QACR;AAAA,QACA,QAAQ,QAAQ,mCAAmC,OAAO,QAAQ,KAAK;AAAA,MACzE;AAAA,IACF;AAAA,EACF;AACA,SAAO,EAAE,KAAK,QAAQ,WAAW,QAAQ,IAAI,CAAC,WAAW,OAAO,OAAO,EAAE;AAC3E;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|
package/dist/net/url.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../src/net/url.ts"],
|
|
4
|
+
"sourcesContent": ["/**\n * Trims every trailing `/` from a base URL.\n *\n * Written as a scan rather than `replace(/\\/+$/, '')` on purpose: the regex\n * form backtracks quadratically on a configured base URL that ends in a long\n * run of slashes, which CodeQL flags as a polynomial ReDoS. This walks the\n * string once from the end.\n */\nexport function stripTrailingSlashes(value: string): string {\n let end = value.length\n while (end > 0 && value.charCodeAt(end - 1) === 0x2f) end -= 1\n return value.slice(0, end)\n}\n"],
|
|
5
|
+
"mappings": "AAQO,SAAS,qBAAqB,OAAuB;AAC1D,MAAI,MAAM,MAAM;AAChB,SAAO,MAAM,KAAK,MAAM,WAAW,MAAM,CAAC,MAAM,GAAM,QAAO;AAC7D,SAAO,MAAM,MAAM,GAAG,GAAG;AAC3B;",
|
|
6
|
+
"names": []
|
|
7
|
+
}
|