@only-cli/oc 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +118 -29
- package/clis/developer.mozilla.org.json +9 -0
- package/clis/doc.rust-lang.org.json +8 -0
- package/clis/docs.oracle.com.json +7 -0
- package/clis/docs.python.org.json +8 -0
- package/clis/docs.ruby-lang.org.json +7 -0
- package/clis/en.cppreference.com.json +8 -0
- package/clis/github.com.json +2 -2
- package/clis/learn.microsoft.com.json +1 -0
- package/clis/nodejs.org.json +7 -0
- package/clis/php.net.json +8 -0
- package/clis/pkg.go.dev.json +7 -0
- package/clis/reddit.com.json +1 -1
- package/clis/typescriptlang.org.json +7 -0
- package/clis/wikipedia.org.json +8 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +55 -19
- package/src/act.js +9 -0
- package/src/apisearch.js +80 -0
- package/src/auth.js +46 -0
- package/src/cache.js +43 -0
- package/src/cli.js +223 -10
- package/src/cookies.js +530 -0
- package/src/fetch.js +418 -18
- package/src/nodedocs.js +177 -0
- package/src/rdoc.js +98 -0
- package/src/render.js +64 -3
- package/src/session.js +43 -4
- package/src/sites.js +173 -0
- package/src/sphinx.js +211 -0
package/src/fetch.js
CHANGED
|
@@ -8,7 +8,12 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import dns from 'node:dns/promises';
|
|
11
|
+
import http from 'node:http';
|
|
12
|
+
import https from 'node:https';
|
|
11
13
|
import net from 'node:net';
|
|
14
|
+
import tls from 'node:tls';
|
|
15
|
+
|
|
16
|
+
import { getSetCookieHeaders } from './cookies.js';
|
|
12
17
|
|
|
13
18
|
// The fetch fallback can't fake a TLS fingerprint like impers does, but it
|
|
14
19
|
// should at least send the same Chrome identity in its headers.
|
|
@@ -23,6 +28,8 @@ const loadImpers = () => {
|
|
|
23
28
|
|
|
24
29
|
const BLOCKED_MESSAGE = 'blocked: private or internal URL';
|
|
25
30
|
const MAX_REDIRECTS = 20;
|
|
31
|
+
// Match undici's default so proxy transport does not hang indefinitely.
|
|
32
|
+
const PROXY_TIMEOUT_MS = 300_000;
|
|
26
33
|
|
|
27
34
|
// What oc can turn into text: any text/* type, plus the application/* types
|
|
28
35
|
// that are really text (json, xml, and the +json / +xml families a feed or an
|
|
@@ -30,6 +37,29 @@ const MAX_REDIRECTS = 20;
|
|
|
30
37
|
// pages of mojibake an agent then pays for, so it is refused by name instead.
|
|
31
38
|
const READABLE_TYPE = /^\s*(?:text\/|application\/(?:json|xml|javascript|x-ndjson|[\w.+-]*\+(?:json|xml)))/i;
|
|
32
39
|
|
|
40
|
+
// The whole decoded body is buffered before the distiller sees it, so an
|
|
41
|
+
// unbounded response is an unbounded allocation, and a URL is often the
|
|
42
|
+
// page's to name, not the caller's. The cap is generous because oc fetches
|
|
43
|
+
// some large corpora on purpose (the Node.js docs reference is 8.5MB
|
|
44
|
+
// decoded); three times that and a response is not a page anyone reads.
|
|
45
|
+
// Content-Length rejects a known-large response before its bytes arrive, but
|
|
46
|
+
// the header is optional and untrusted, so every transport also counts what
|
|
47
|
+
// actually lands, after decoding, which is what stops a decompression bomb.
|
|
48
|
+
export const MAX_BODY = 25 * 1024 * 1024;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Refuse a body larger than oc will buffer. Called on the Content-Length
|
|
52
|
+
* header first and again on the bytes as they arrive, since only the second
|
|
53
|
+
* count is trustworthy.
|
|
54
|
+
* @param {number} size - bytes seen so far, or claimed by the header
|
|
55
|
+
* @param {string} url
|
|
56
|
+
*/
|
|
57
|
+
export function assertBodySize(size, url) {
|
|
58
|
+
if (size > MAX_BODY) {
|
|
59
|
+
throw new Error(`response body over ${MAX_BODY / 1048576}MB for ${url}, more than oc will read`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
33
63
|
/**
|
|
34
64
|
* Refuse a response oc cannot read as text. Both transports call this: the
|
|
35
65
|
* gate has to live on whichever client got the page, or the same URL renders
|
|
@@ -139,25 +169,338 @@ async function assertSafeTarget(urlStr) {
|
|
|
139
169
|
return;
|
|
140
170
|
}
|
|
141
171
|
if (hostname === 'localhost') throw new Error(BLOCKED_MESSAGE);
|
|
142
|
-
const addresses = await dns.lookup(hostname, { all: true }).catch(() =>
|
|
172
|
+
const addresses = await dns.lookup(hostname, { all: true }).catch(() => {
|
|
173
|
+
// With a proxy the client never resolves the target; the proxy does, often
|
|
174
|
+
// on corporate DNS where internal names are NXDOMAIN locally. Fail closed.
|
|
175
|
+
if (resolveProxy(urlStr)) throw new Error(BLOCKED_MESSAGE);
|
|
176
|
+
return [];
|
|
177
|
+
});
|
|
143
178
|
for (const { address, family } of addresses) {
|
|
144
179
|
if (family === 4 && isBlockedIPv4(address)) throw new Error(BLOCKED_MESSAGE);
|
|
145
180
|
if (family === 6 && isBlockedIPv6(address)) throw new Error(BLOCKED_MESSAGE);
|
|
146
181
|
}
|
|
147
182
|
}
|
|
148
183
|
|
|
184
|
+
function envFirst(env, ...names) {
|
|
185
|
+
for (const name of names) {
|
|
186
|
+
const value = env[name];
|
|
187
|
+
if (value) return value;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function normalizeProxy(value) {
|
|
192
|
+
if (value == null) return null;
|
|
193
|
+
const trimmed = String(value).trim();
|
|
194
|
+
if (!trimmed) return null;
|
|
195
|
+
return /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `http://${trimmed}`;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function assertHttpProxyProtocol(proxyUrl) {
|
|
199
|
+
if (!/^https?:$/.test(proxyUrl.protocol)) {
|
|
200
|
+
throw new Error(`unsupported proxy protocol (${proxyUrl.protocol.slice(0, -1)}), oc honors HTTP and HTTPS proxies`);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// Split host[:port], including [IPv6]:port. Unbracketed IPv6 literals never
|
|
205
|
+
// carry a port suffix (use [addr]:port); a trailing :digits on ::1 is part of
|
|
206
|
+
// the address, not a port.
|
|
207
|
+
function splitHostPort(entry) {
|
|
208
|
+
if (entry.startsWith('[')) {
|
|
209
|
+
const end = entry.indexOf(']');
|
|
210
|
+
if (end === -1) return { host: entry, port: '' };
|
|
211
|
+
const rest = entry.slice(end + 1);
|
|
212
|
+
return { host: entry.slice(1, end), port: rest.startsWith(':') ? rest.slice(1) : '' };
|
|
213
|
+
}
|
|
214
|
+
if (net.isIP(entry) === 6) return { host: entry, port: '' };
|
|
215
|
+
const colon = entry.lastIndexOf(':');
|
|
216
|
+
if (colon !== -1 && /^\d+$/.test(entry.slice(colon + 1))) {
|
|
217
|
+
const host = entry.slice(0, colon);
|
|
218
|
+
if (net.isIP(host) === 6) return { host: entry, port: '' };
|
|
219
|
+
return { host, port: entry.slice(colon + 1) };
|
|
220
|
+
}
|
|
221
|
+
return { host: entry, port: '' };
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function ipv4InCidr(ip, base, bits) {
|
|
225
|
+
const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0;
|
|
226
|
+
return (ipv4ToInt(ip) & mask) === (ipv4ToInt(base) & mask);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
function ipv6InCidr(ip, base, bits) {
|
|
230
|
+
const g = expandIPv6(ip);
|
|
231
|
+
const b = expandIPv6(base);
|
|
232
|
+
if (!g || !b) return false;
|
|
233
|
+
let remaining = bits;
|
|
234
|
+
for (let i = 0; i < 8 && remaining > 0; i++) {
|
|
235
|
+
if (remaining >= 16) {
|
|
236
|
+
if (g[i] !== b[i]) return false;
|
|
237
|
+
remaining -= 16;
|
|
238
|
+
} else {
|
|
239
|
+
const mask = (0xffff << (16 - remaining)) & 0xffff;
|
|
240
|
+
if ((g[i] & mask) !== (b[i] & mask)) return false;
|
|
241
|
+
remaining = 0;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
return true;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function ipInCidr(ip, base, bits) {
|
|
248
|
+
const family = net.isIP(ip);
|
|
249
|
+
if (family === 4) return ipv4InCidr(ip, base, bits);
|
|
250
|
+
if (family === 6) return ipv6InCidr(ip, base, bits);
|
|
251
|
+
return false;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
function bypassesProxy(target, noProxy) {
|
|
255
|
+
const list = noProxy.trim();
|
|
256
|
+
if (!list) return false;
|
|
257
|
+
const hostname = target.hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
|
258
|
+
const port = target.port || (target.protocol === 'https:' ? '443' : '80');
|
|
259
|
+
for (let entry of list.split(',')) {
|
|
260
|
+
entry = entry.trim();
|
|
261
|
+
if (!entry) continue;
|
|
262
|
+
if (entry === '*') return true;
|
|
263
|
+
const { host: rawHost, port: entryPort } = splitHostPort(entry);
|
|
264
|
+
if (entryPort && entryPort !== port) continue;
|
|
265
|
+
let pattern = rawHost.toLowerCase().replace(/^\[|\]$/g, '');
|
|
266
|
+
const wildcard = pattern.startsWith('*.');
|
|
267
|
+
if (wildcard) pattern = pattern.slice(2);
|
|
268
|
+
const slash = pattern.indexOf('/');
|
|
269
|
+
if (slash !== -1 && net.isIP(pattern.slice(0, slash))) {
|
|
270
|
+
const bits = Number(pattern.slice(slash + 1));
|
|
271
|
+
if (Number.isInteger(bits) && net.isIP(hostname) && ipInCidr(hostname, pattern.slice(0, slash), bits)) {
|
|
272
|
+
return true;
|
|
273
|
+
}
|
|
274
|
+
continue;
|
|
275
|
+
}
|
|
276
|
+
const host = pattern.replace(/^\./, '');
|
|
277
|
+
if (!host) continue;
|
|
278
|
+
if (wildcard) {
|
|
279
|
+
if (hostname.endsWith(`.${host}`)) return true;
|
|
280
|
+
continue;
|
|
281
|
+
}
|
|
282
|
+
if (hostname === host || hostname.endsWith(`.${host}`)) return true;
|
|
283
|
+
}
|
|
284
|
+
return false;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
/**
|
|
288
|
+
* Pick a proxy for this URL from HTTP_PROXY / HTTPS_PROXY / NO_PROXY (and
|
|
289
|
+
* their lowercase forms). The proxy host is not run through assertSafeTarget:
|
|
290
|
+
* corporate proxies live on loopback or RFC 1918 addresses, and they are not
|
|
291
|
+
* the page being fetched. Redirect hops still go through that check.
|
|
292
|
+
* @param {string} url
|
|
293
|
+
* @param {NodeJS.ProcessEnv} [env]
|
|
294
|
+
* @returns {string | null} proxy URL, or null to connect directly
|
|
295
|
+
*/
|
|
296
|
+
export function resolveProxy(url, env = process.env) {
|
|
297
|
+
const target = new URL(url);
|
|
298
|
+
if (bypassesProxy(target, envFirst(env, 'NO_PROXY', 'no_proxy') ?? '')) return null;
|
|
299
|
+
const httpsProxy = envFirst(env, 'HTTPS_PROXY', 'https_proxy');
|
|
300
|
+
const httpProxy = envFirst(env, 'HTTP_PROXY', 'http_proxy');
|
|
301
|
+
const chosen = target.protocol === 'https:' ? (httpsProxy || httpProxy) : httpProxy;
|
|
302
|
+
const normalized = normalizeProxy(chosen);
|
|
303
|
+
if (!normalized) return null;
|
|
304
|
+
assertHttpProxyProtocol(new URL(normalized));
|
|
305
|
+
return normalized;
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
function decodeCredential(value) {
|
|
309
|
+
try {
|
|
310
|
+
return decodeURIComponent(value);
|
|
311
|
+
} catch {
|
|
312
|
+
return value;
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
function proxyAuthHeader(proxy) {
|
|
317
|
+
if (!proxy.username) return undefined;
|
|
318
|
+
const token = Buffer.from(`${decodeCredential(proxy.username)}:${decodeCredential(proxy.password)}`).toString('base64');
|
|
319
|
+
return `Basic ${token}`;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
function authority(target) {
|
|
323
|
+
const host = net.isIP(target.hostname) === 6 ? `[${target.hostname}]` : target.hostname;
|
|
324
|
+
const port = target.port || (target.protocol === 'https:' ? '443' : '80');
|
|
325
|
+
return `${host}:${port}`;
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
function wrapNodeResponse(res, url) {
|
|
329
|
+
const headers = {
|
|
330
|
+
get(name) {
|
|
331
|
+
const v = res.headers[name.toLowerCase()];
|
|
332
|
+
if (v == null) return null;
|
|
333
|
+
return Array.isArray(v) ? v.join(', ') : v;
|
|
334
|
+
},
|
|
335
|
+
// Node keeps Set-Cookie as an array of raw values. Expose it unjoined so
|
|
336
|
+
// the cookie jar reads each header intact: a comma in an Expires date
|
|
337
|
+
// makes the joined form ambiguous to split back apart.
|
|
338
|
+
getSetCookie() {
|
|
339
|
+
const v = res.headers['set-cookie'];
|
|
340
|
+
if (v == null) return [];
|
|
341
|
+
return Array.isArray(v) ? v : [v];
|
|
342
|
+
},
|
|
343
|
+
};
|
|
344
|
+
const text = () => new Promise((resolve, reject) => {
|
|
345
|
+
const chunks = [];
|
|
346
|
+
let size = 0;
|
|
347
|
+
res.on('data', (c) => {
|
|
348
|
+
size += c.length;
|
|
349
|
+
try {
|
|
350
|
+
assertBodySize(size, url);
|
|
351
|
+
} catch (err) {
|
|
352
|
+
// destroy surfaces the refusal through 'error', and stops the read.
|
|
353
|
+
res.destroy(err);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
chunks.push(c);
|
|
357
|
+
});
|
|
358
|
+
res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
|
359
|
+
res.on('error', reject);
|
|
360
|
+
});
|
|
361
|
+
const status = res.statusCode ?? 0;
|
|
362
|
+
return {
|
|
363
|
+
status,
|
|
364
|
+
statusText: res.statusMessage || '',
|
|
365
|
+
headers,
|
|
366
|
+
ok: status >= 200 && status < 300,
|
|
367
|
+
url,
|
|
368
|
+
text,
|
|
369
|
+
};
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
function proxyTransport(proxy) {
|
|
373
|
+
return proxy.protocol === 'https:' ? https : http;
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
function proxyPort(proxy) {
|
|
377
|
+
return Number(proxy.port) || (proxy.protocol === 'https:' ? 443 : 80);
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
function armRequestTimeout(req, reject, label) {
|
|
381
|
+
req.setTimeout(PROXY_TIMEOUT_MS, () => {
|
|
382
|
+
req.destroy();
|
|
383
|
+
reject(new Error(`proxy timed out after ${PROXY_TIMEOUT_MS / 1000}s for ${label}`));
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
function pickTlsCa(tlsOpts) {
|
|
388
|
+
return tlsOpts.ca != null ? { ca: tlsOpts.ca } : {};
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
function httpViaProxy(target, proxy, headers) {
|
|
392
|
+
const auth = proxyAuthHeader(proxy);
|
|
393
|
+
return new Promise((resolve, reject) => {
|
|
394
|
+
const req = proxyTransport(proxy).request({
|
|
395
|
+
hostname: proxy.hostname,
|
|
396
|
+
port: proxyPort(proxy),
|
|
397
|
+
method: 'GET',
|
|
398
|
+
path: `${target.protocol}//${target.host}${target.pathname}${target.search}`,
|
|
399
|
+
headers: {
|
|
400
|
+
...headers,
|
|
401
|
+
host: target.host,
|
|
402
|
+
...(auth && { 'proxy-authorization': auth }),
|
|
403
|
+
},
|
|
404
|
+
}, (res) => resolve(wrapNodeResponse(res, target.href)));
|
|
405
|
+
armRequestTimeout(req, reject, target.href);
|
|
406
|
+
req.on('error', (err) => reject(new Error(`proxy failed: ${err.message} for ${target.href}`)));
|
|
407
|
+
req.end();
|
|
408
|
+
});
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
function httpsViaConnect(target, proxy, headers, tlsOpts = {}) {
|
|
412
|
+
const dest = authority(target);
|
|
413
|
+
const auth = proxyAuthHeader(proxy);
|
|
414
|
+
return new Promise((resolve, reject) => {
|
|
415
|
+
let settled = false;
|
|
416
|
+
const fail = (err) => {
|
|
417
|
+
if (settled) return;
|
|
418
|
+
settled = true;
|
|
419
|
+
req.destroy();
|
|
420
|
+
reject(err instanceof Error ? err : new Error(String(err)));
|
|
421
|
+
};
|
|
422
|
+
const req = proxyTransport(proxy).request({
|
|
423
|
+
hostname: proxy.hostname,
|
|
424
|
+
port: proxyPort(proxy),
|
|
425
|
+
method: 'CONNECT',
|
|
426
|
+
path: dest,
|
|
427
|
+
headers: {
|
|
428
|
+
host: dest,
|
|
429
|
+
...(auth && { 'proxy-authorization': auth }),
|
|
430
|
+
},
|
|
431
|
+
});
|
|
432
|
+
armRequestTimeout(req, fail, target.href);
|
|
433
|
+
req.on('connect', (res, socket, head) => {
|
|
434
|
+
if (res.statusCode !== 200) {
|
|
435
|
+
socket.destroy();
|
|
436
|
+
fail(new Error(`proxy CONNECT failed: ${res.statusCode} for ${target.href}`));
|
|
437
|
+
return;
|
|
438
|
+
}
|
|
439
|
+
if (head.length) socket.unshift(head);
|
|
440
|
+
// tls.connect already opened the tunnel. https.request would wrap TLS
|
|
441
|
+
// again, and the origin would see a second ClientHello as garbage.
|
|
442
|
+
// SNI is a hostname; an IP literal is only used for the cert check.
|
|
443
|
+
// URL.hostname keeps the brackets around an IPv6 literal ("[::1]"), which
|
|
444
|
+
// tls.connect would treat as a DNS name; strip them like assertSafeTarget.
|
|
445
|
+
const hostname = target.hostname.replace(/^\[|\]$/g, '');
|
|
446
|
+
const tlsSocket = tls.connect({
|
|
447
|
+
socket,
|
|
448
|
+
host: hostname,
|
|
449
|
+
...(net.isIP(hostname) ? {} : { servername: hostname }),
|
|
450
|
+
...pickTlsCa(tlsOpts),
|
|
451
|
+
}, () => {
|
|
452
|
+
const tunneled = http.request({
|
|
453
|
+
createConnection: () => tlsSocket,
|
|
454
|
+
path: `${target.pathname}${target.search}`,
|
|
455
|
+
method: 'GET',
|
|
456
|
+
headers: { ...headers, host: target.host },
|
|
457
|
+
}, (httpsRes) => {
|
|
458
|
+
if (settled) return;
|
|
459
|
+
settled = true;
|
|
460
|
+
resolve(wrapNodeResponse(httpsRes, target.href));
|
|
461
|
+
});
|
|
462
|
+
armRequestTimeout(tunneled, fail, target.href);
|
|
463
|
+
tunneled.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`)));
|
|
464
|
+
tunneled.end();
|
|
465
|
+
});
|
|
466
|
+
tlsSocket.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`)));
|
|
467
|
+
});
|
|
468
|
+
req.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`)));
|
|
469
|
+
req.end();
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
/**
|
|
474
|
+
* One GET through an HTTP(S) proxy. HTTP targets use the absolute-URI form;
|
|
475
|
+
* HTTPS targets open a CONNECT tunnel first. Redirects are not followed:
|
|
476
|
+
* followRedirects owns that so each hop still goes through assertSafeTarget.
|
|
477
|
+
* @param {string} url
|
|
478
|
+
* @param {string} proxy
|
|
479
|
+
* @param {Record<string, string>} [headers]
|
|
480
|
+
* @param {import('node:tls').ConnectionOptions} [tlsOpts]
|
|
481
|
+
*/
|
|
482
|
+
export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
|
|
483
|
+
const target = new URL(url);
|
|
484
|
+
const proxyUrl = new URL(proxy);
|
|
485
|
+
assertHttpProxyProtocol(proxyUrl);
|
|
486
|
+
return target.protocol === 'https:'
|
|
487
|
+
? httpsViaConnect(target, proxyUrl, headers, tlsOpts)
|
|
488
|
+
: httpViaProxy(target, proxyUrl, headers);
|
|
489
|
+
}
|
|
490
|
+
|
|
149
491
|
/**
|
|
150
492
|
* Fetch a page.
|
|
151
493
|
* @param {string} url - with or without a scheme, https is assumed
|
|
494
|
+
* @param {{ jar?: { cookieHeaderFor(url: string): string|undefined, storeFromResponse(url: string, headers: string[]): void } }} [opts]
|
|
152
495
|
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
|
153
496
|
* final URL after redirects, the body, the HTTP status, and which client
|
|
154
497
|
* identity got the page (impers:chrome, impers:firefox, or fetch)
|
|
155
498
|
*/
|
|
156
|
-
export async function fetchPage(url) {
|
|
499
|
+
export async function fetchPage(url, { jar } = {}) {
|
|
157
500
|
const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
|
158
501
|
await assertSafeTarget(target);
|
|
159
502
|
const impers = await loadImpers();
|
|
160
|
-
return impers ? viaImpers(impers, target) : viaFetch(target);
|
|
503
|
+
return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
|
|
161
504
|
}
|
|
162
505
|
|
|
163
506
|
/**
|
|
@@ -174,11 +517,12 @@ export async function fetchPage(url) {
|
|
|
174
517
|
* @param {string} start
|
|
175
518
|
* @returns {Promise<{res: any, url: string}>} the first non-redirect response
|
|
176
519
|
*/
|
|
177
|
-
export async function followRedirects(get, start) {
|
|
520
|
+
export async function followRedirects(get, start, { onResponse } = {}) {
|
|
178
521
|
let current = start;
|
|
179
522
|
for (let i = 0; ; i++) {
|
|
180
523
|
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
|
|
181
524
|
const res = await get(current);
|
|
525
|
+
onResponse?.(current, res);
|
|
182
526
|
const status = res.status ?? res.statusCode ?? 0;
|
|
183
527
|
const location = res.headers.get('location');
|
|
184
528
|
if (status >= 300 && status < 400 && location) {
|
|
@@ -190,36 +534,92 @@ export async function followRedirects(get, start) {
|
|
|
190
534
|
}
|
|
191
535
|
}
|
|
192
536
|
|
|
193
|
-
|
|
537
|
+
const FETCH_HEADERS = {
|
|
538
|
+
'user-agent': UA,
|
|
539
|
+
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
540
|
+
'accept-language': 'en-US,en;q=0.9',
|
|
541
|
+
};
|
|
542
|
+
|
|
543
|
+
function mergeHeaders(base, extra) {
|
|
544
|
+
return extra ? { ...base, ...extra } : base;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function jarHeaders(jar, url, base) {
|
|
548
|
+
if (!jar) return base;
|
|
549
|
+
const cookie = jar.cookieHeaderFor(url);
|
|
550
|
+
return cookie ? mergeHeaders(base, { cookie }) : base;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
function captureSetCookie(jar, url, res) {
|
|
554
|
+
if (!jar) return;
|
|
555
|
+
jar.storeFromResponse(url, getSetCookieHeaders(res));
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
async function viaImpers(impers, target, jar) {
|
|
194
559
|
// Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a
|
|
195
560
|
// blocked first attempt gets one cheap retry with a second identity.
|
|
196
|
-
const asking = (impersonate) => (url) =>
|
|
561
|
+
const asking = (impersonate) => (url) =>
|
|
562
|
+
impers.get(url, {
|
|
563
|
+
impersonate,
|
|
564
|
+
allowRedirects: false,
|
|
565
|
+
proxy: resolveProxy(url) ?? '',
|
|
566
|
+
headers: jarHeaders(jar, url, {}),
|
|
567
|
+
});
|
|
568
|
+
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
197
569
|
let via = 'impers:chrome';
|
|
198
|
-
let { res } = await followRedirects(asking('chrome'), target);
|
|
570
|
+
let { res } = await followRedirects(asking('chrome'), target, { onResponse });
|
|
199
571
|
let status = res.status ?? res.statusCode ?? 0;
|
|
200
572
|
if (status >= 400) {
|
|
201
573
|
via = 'impers:firefox';
|
|
202
|
-
({ res } = await followRedirects(asking('firefox'), target));
|
|
574
|
+
({ res } = await followRedirects(asking('firefox'), target, { onResponse }));
|
|
203
575
|
status = res.status ?? res.statusCode ?? 0;
|
|
204
576
|
}
|
|
205
577
|
if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
|
|
206
578
|
assertReadableType(res.headers.get('content-type'));
|
|
579
|
+
assertBodySize(Number(res.headers.get('content-length')) || 0, target);
|
|
580
|
+
// impers buffers inside its own binding, so the size of what it already
|
|
581
|
+
// holds is all there is to check; the bound still stops an oversized body
|
|
582
|
+
// from travelling any further.
|
|
207
583
|
const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
|
|
584
|
+
assertBodySize(html.length, target);
|
|
208
585
|
return { url: res.url ?? target, html, status, via };
|
|
209
586
|
}
|
|
210
587
|
|
|
211
|
-
async function viaFetch(target) {
|
|
212
|
-
const
|
|
213
|
-
|
|
214
|
-
headers
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
588
|
+
async function viaFetch(target, jar) {
|
|
589
|
+
const get = (url) => {
|
|
590
|
+
const proxy = resolveProxy(url);
|
|
591
|
+
const headers = jarHeaders(jar, url, FETCH_HEADERS);
|
|
592
|
+
return proxy
|
|
593
|
+
? proxyGet(url, proxy, headers)
|
|
594
|
+
: fetch(url, { redirect: 'manual', headers });
|
|
595
|
+
};
|
|
596
|
+
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
597
|
+
const { res, url: current } = await followRedirects(get, target, { onResponse });
|
|
220
598
|
if (!res.ok) {
|
|
221
599
|
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
|
222
600
|
}
|
|
223
601
|
assertReadableType(res.headers.get('content-type'));
|
|
224
|
-
|
|
602
|
+
assertBodySize(Number(res.headers.get('content-length')) || 0, current);
|
|
603
|
+
return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch' };
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
/**
|
|
607
|
+
* The decoded body as text, counted as it arrives so crossing the cap aborts
|
|
608
|
+
* the transfer instead of finishing it. Throwing mid-iteration cancels the
|
|
609
|
+
* stream. A proxy response has no web stream to iterate; its text() counts
|
|
610
|
+
* inside wrapNodeResponse instead.
|
|
611
|
+
* @param {any} res
|
|
612
|
+
* @param {string} url
|
|
613
|
+
* @returns {Promise<string>}
|
|
614
|
+
*/
|
|
615
|
+
export async function readBody(res, url) {
|
|
616
|
+
if (!res.body?.getReader) return res.text();
|
|
617
|
+
const chunks = [];
|
|
618
|
+
let size = 0;
|
|
619
|
+
for await (const chunk of res.body) {
|
|
620
|
+
size += chunk.byteLength;
|
|
621
|
+
assertBodySize(size, url);
|
|
622
|
+
chunks.push(chunk);
|
|
623
|
+
}
|
|
624
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
225
625
|
}
|
package/src/nodedocs.js
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Node.js docs search backend. nodejs.org has no search results page at all:
|
|
3
|
+
* the site's search box is a JavaScript modal asking a third-party service,
|
|
4
|
+
* so there is nothing for oc to fetch or call directly. But the API docs
|
|
5
|
+
* publish their entire reference as one static JSON file, all.json, much the
|
|
6
|
+
* way a Sphinx site publishes its search index, so `search` ranks that file
|
|
7
|
+
* locally: every module, class, method, property, and event heading becomes
|
|
8
|
+
* a result linking to its own anchor. The file is ~8MB (~1MB over the wire)
|
|
9
|
+
* and static, so it shares the Sphinx backend's day cache and, like the
|
|
10
|
+
* index, is never printed: what reaches the agent is the ranked list only.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { cachedFile } from './cache.js';
|
|
14
|
+
import { escapeHTML } from './sphinx.js';
|
|
15
|
+
|
|
16
|
+
const MAX_RESULTS = 20;
|
|
17
|
+
|
|
18
|
+
// The list-valued keys that hold entries with headings of their own on the
|
|
19
|
+
// page. The others (params, options) describe one signature's arguments and
|
|
20
|
+
// have no heading or anchor; a class's constructor rides in `signatures`.
|
|
21
|
+
const CHILD_KEYS = [
|
|
22
|
+
'modules', 'globals', 'miscs', 'classes', 'classMethods',
|
|
23
|
+
'signatures', 'methods', 'properties', 'events',
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
// What each heading's `type` is called on the results page. Anything else is
|
|
27
|
+
// a property whose type field holds its value type ({number}, {boolean}).
|
|
28
|
+
const KIND = {
|
|
29
|
+
module: 'module', misc: 'section', global: 'global', class: 'class',
|
|
30
|
+
ctor: 'constructor', classMethod: 'static method', method: 'method',
|
|
31
|
+
event: 'event', property: 'property',
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// The docs derive a heading's anchor from its text the github-slugger way:
|
|
35
|
+
// lowercase, drop everything but letters, digits, spaces, hyphens, and
|
|
36
|
+
// underscores, then spaces become hyphens. 'fs.readFile(path[, options],
|
|
37
|
+
// callback)' is #fsreadfilepath-options-callback.
|
|
38
|
+
const slug = (text) =>
|
|
39
|
+
text.toLowerCase().replace(/[^a-z0-9 _-]/g, '').trim().replaceAll(' ', '-');
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Anything that parses but is not the docs corpus (an error page served as
|
|
43
|
+
* JSON, a moved file) fails here, which keeps it out of the cache too.
|
|
44
|
+
* @param {string} text
|
|
45
|
+
* @returns {any}
|
|
46
|
+
*/
|
|
47
|
+
export function parseAll(text) {
|
|
48
|
+
let all;
|
|
49
|
+
try {
|
|
50
|
+
all = JSON.parse(text);
|
|
51
|
+
} catch {
|
|
52
|
+
all = null;
|
|
53
|
+
}
|
|
54
|
+
if (!Array.isArray(all?.modules)) throw new Error('not the Node.js docs corpus');
|
|
55
|
+
return all;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Flatten the docs tree into the headings a query can hit. Only a top-level
|
|
60
|
+
* entry names its page (source: 'doc/api/fs.md'); everything nested under it
|
|
61
|
+
* inherits that page and contributes its own heading and anchor. A nested
|
|
62
|
+
* entry whose textRaw does not name it is not a heading (a bare 'Type:
|
|
63
|
+
* {number}' line under a property) and is skipped; its parent still stands.
|
|
64
|
+
* A heading repeated on one page is kept once: the corpus lists some methods
|
|
65
|
+
* twice for one heading, and where a page really repeats one (each stream
|
|
66
|
+
* class has an Event: 'close') the rows would be indistinguishable anyway.
|
|
67
|
+
* @param {any} all - parsed all.json
|
|
68
|
+
* @returns {{text: string, name: string, type: string, page: string, anchor: string}[]}
|
|
69
|
+
*/
|
|
70
|
+
export function buildEntries(all) {
|
|
71
|
+
const entries = [];
|
|
72
|
+
const seen = new Set();
|
|
73
|
+
const add = (node, page, top) => {
|
|
74
|
+
const text = String(node?.textRaw ?? '').replaceAll('`', '').trim();
|
|
75
|
+
const name = String(node?.name ?? '');
|
|
76
|
+
const type = String(node?.type ?? '');
|
|
77
|
+
const heading = text && name
|
|
78
|
+
&& (top || text.toLowerCase().includes(name.toLowerCase()));
|
|
79
|
+
if (heading && !seen.has(`${page}#${text}`)) {
|
|
80
|
+
seen.add(`${page}#${text}`);
|
|
81
|
+
// A module or section heading is the page's own title, so its entry
|
|
82
|
+
// links to the page top; every other heading has an anchor worth keeping.
|
|
83
|
+
const anchor = ['module', 'misc', 'global'].includes(type) ? '' : slug(text);
|
|
84
|
+
entries.push({ text, name, type, page, anchor });
|
|
85
|
+
}
|
|
86
|
+
for (const key of CHILD_KEYS) {
|
|
87
|
+
for (const child of node?.[key] ?? []) add(child, page, false);
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
for (const key of CHILD_KEYS) {
|
|
91
|
+
for (const node of all?.[key] ?? []) {
|
|
92
|
+
const page = String(node?.source ?? '')
|
|
93
|
+
.replace(/^doc\/api\//, '').replace(/\.md$/, '');
|
|
94
|
+
if (page) add(node, `${page}.html`, true);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return entries;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Rank the headings against a query. A word that is an entry's own name (the
|
|
102
|
+
* bare symbol: 'readFile') weighs most, a whole word of its heading next, a
|
|
103
|
+
* substring of it least, and an entry must match every word before any-word
|
|
104
|
+
* matching kicks in, the same policy the Sphinx backend follows.
|
|
105
|
+
* @param {ReturnType<typeof buildEntries>} entries
|
|
106
|
+
* @param {string} query
|
|
107
|
+
*/
|
|
108
|
+
export function searchEntries(entries, query) {
|
|
109
|
+
const words = [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))];
|
|
110
|
+
const scored = [];
|
|
111
|
+
for (const e of entries) {
|
|
112
|
+
const nameL = e.name.toLowerCase();
|
|
113
|
+
const textL = e.text.toLowerCase();
|
|
114
|
+
const tokens = new Set(textL.split(/[^a-z0-9_]+/));
|
|
115
|
+
let score = 0;
|
|
116
|
+
let hit = 0;
|
|
117
|
+
for (const word of words) {
|
|
118
|
+
const s = nameL === word ? 10 : tokens.has(word) ? 5 : textL.includes(word) ? 2 : 0;
|
|
119
|
+
if (s) {
|
|
120
|
+
score += s;
|
|
121
|
+
hit += 1;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (hit) scored.push({ e, score, hit });
|
|
125
|
+
}
|
|
126
|
+
let hits = scored.filter((s) => s.hit === words.length);
|
|
127
|
+
const partial = !hits.length && words.length > 1 && scored.length > 0;
|
|
128
|
+
if (partial) hits = scored;
|
|
129
|
+
// Among equal scores the shorter heading is the plainer API, so it leads.
|
|
130
|
+
hits.sort((a, b) => b.score - a.score
|
|
131
|
+
|| a.e.text.length - b.e.text.length
|
|
132
|
+
|| a.e.text.localeCompare(b.e.text));
|
|
133
|
+
return { words, partial, total: hits.length, hits: hits.slice(0, MAX_RESULTS).map((s) => s.e) };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The result list becomes the same small synthetic page the other search
|
|
138
|
+
* backends emit, so it distills, renders, numbers, and remembers like any
|
|
139
|
+
* fetched page and `do <n>` follows a result.
|
|
140
|
+
* @param {string} base
|
|
141
|
+
* @param {string} query
|
|
142
|
+
* @param {ReturnType<typeof searchEntries>} found
|
|
143
|
+
* @returns {string}
|
|
144
|
+
*/
|
|
145
|
+
export function resultsToHTML(base, query, found) {
|
|
146
|
+
const host = new URL(base).host;
|
|
147
|
+
const items = found.hits.map((e) => {
|
|
148
|
+
const href = new URL(e.anchor ? `${e.page}#${e.anchor}` : e.page, base).href;
|
|
149
|
+
return `<li><a href="${escapeHTML(href)}">${escapeHTML(e.text)}</a>`
|
|
150
|
+
+ ` ${escapeHTML(KIND[e.type] ?? 'property')}, in ${escapeHTML(e.page.replace(/\.html$/, ''))}</li>`;
|
|
151
|
+
});
|
|
152
|
+
const partial = found.partial ? '; no heading matches every word, so these match some' : '';
|
|
153
|
+
const summary = items.length
|
|
154
|
+
? `${found.total} heading${found.total === 1 ? '' : 's'} match in the docs' own reference,`
|
|
155
|
+
+ ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
|
|
156
|
+
: `nothing in the docs' own reference matches; try fewer or different words`;
|
|
157
|
+
return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
|
|
158
|
+
+ `<p>${summary}</p>`
|
|
159
|
+
+ (items.length ? `<ol>${items.join('')}</ol>` : '')
|
|
160
|
+
+ `</main></body></html>`;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Search the Node.js API docs. The site has no human search URL to remember,
|
|
165
|
+
* so the session keeps the docs index, the page a reader would start from.
|
|
166
|
+
* @param {string} base - docs root ending in '/', e.g. https://nodejs.org/api/
|
|
167
|
+
* @param {string} query
|
|
168
|
+
*/
|
|
169
|
+
export async function nodeSearch(base, query) {
|
|
170
|
+
if (!query.trim()) throw new Error('usage: search <query>');
|
|
171
|
+
const { data, via } = await cachedFile('nodedoc', new URL('all.json', base).href, parseAll);
|
|
172
|
+
return {
|
|
173
|
+
url: new URL('index.html', base).href,
|
|
174
|
+
html: resultsToHTML(base, query, searchEntries(buildEntries(data), query)),
|
|
175
|
+
via,
|
|
176
|
+
};
|
|
177
|
+
}
|