@only-cli/oc 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/fetch.js CHANGED
@@ -8,7 +8,12 @@
8
8
  */
9
9
 
10
10
  import dns from 'node:dns/promises';
11
+ import http from 'node:http';
12
+ import https from 'node:https';
11
13
  import net from 'node:net';
14
+ import tls from 'node:tls';
15
+
16
+ import { getSetCookieHeaders } from './cookies.js';
12
17
 
13
18
  // The fetch fallback can't fake a TLS fingerprint like impers does, but it
14
19
  // should at least send the same Chrome identity in its headers.
@@ -23,6 +28,8 @@ const loadImpers = () => {
23
28
 
24
29
  const BLOCKED_MESSAGE = 'blocked: private or internal URL';
25
30
  const MAX_REDIRECTS = 20;
31
+ // Match undici's default so proxy transport does not hang indefinitely.
32
+ const PROXY_TIMEOUT_MS = 300_000;
26
33
 
27
34
  // What oc can turn into text: any text/* type, plus the application/* types
28
35
  // that are really text (json, xml, and the +json / +xml families a feed or an
@@ -30,6 +37,29 @@ const MAX_REDIRECTS = 20;
30
37
  // pages of mojibake an agent then pays for, so it is refused by name instead.
31
38
  const READABLE_TYPE = /^\s*(?:text\/|application\/(?:json|xml|javascript|x-ndjson|[\w.+-]*\+(?:json|xml)))/i;
32
39
 
40
+ // The whole decoded body is buffered before the distiller sees it, so an
41
+ // unbounded response is an unbounded allocation, and a URL is often the
42
+ // page's to name, not the caller's. The cap is generous because oc fetches
43
+ // some large corpora on purpose (the Node.js docs reference is 8.5MB
44
+ // decoded); three times that and a response is not a page anyone reads.
45
+ // Content-Length rejects a known-large response before its bytes arrive, but
46
+ // the header is optional and untrusted, so every transport also counts what
47
+ // actually lands, after decoding, which is what stops a decompression bomb.
48
+ export const MAX_BODY = 25 * 1024 * 1024;
49
+
50
+ /**
51
+ * Refuse a body larger than oc will buffer. Called on the Content-Length
52
+ * header first and again on the bytes as they arrive, since only the second
53
+ * count is trustworthy.
54
+ * @param {number} size - bytes seen so far, or claimed by the header
55
+ * @param {string} url
56
+ */
57
+ export function assertBodySize(size, url) {
58
+ if (size > MAX_BODY) {
59
+ throw new Error(`response body over ${MAX_BODY / 1048576}MB for ${url}, more than oc will read`);
60
+ }
61
+ }
62
+
33
63
  /**
34
64
  * Refuse a response oc cannot read as text. Both transports call this: the
35
65
  * gate has to live on whichever client got the page, or the same URL renders
@@ -139,25 +169,338 @@ async function assertSafeTarget(urlStr) {
139
169
  return;
140
170
  }
141
171
  if (hostname === 'localhost') throw new Error(BLOCKED_MESSAGE);
142
- const addresses = await dns.lookup(hostname, { all: true }).catch(() => []);
172
+ const addresses = await dns.lookup(hostname, { all: true }).catch(() => {
173
+ // With a proxy the client never resolves the target; the proxy does, often
174
+ // on corporate DNS where internal names are NXDOMAIN locally. Fail closed.
175
+ if (resolveProxy(urlStr)) throw new Error(BLOCKED_MESSAGE);
176
+ return [];
177
+ });
143
178
  for (const { address, family } of addresses) {
144
179
  if (family === 4 && isBlockedIPv4(address)) throw new Error(BLOCKED_MESSAGE);
145
180
  if (family === 6 && isBlockedIPv6(address)) throw new Error(BLOCKED_MESSAGE);
146
181
  }
147
182
  }
148
183
 
184
+ function envFirst(env, ...names) {
185
+ for (const name of names) {
186
+ const value = env[name];
187
+ if (value) return value;
188
+ }
189
+ }
190
+
191
+ function normalizeProxy(value) {
192
+ if (value == null) return null;
193
+ const trimmed = String(value).trim();
194
+ if (!trimmed) return null;
195
+ return /^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed) ? trimmed : `http://${trimmed}`;
196
+ }
197
+
198
+ function assertHttpProxyProtocol(proxyUrl) {
199
+ if (!/^https?:$/.test(proxyUrl.protocol)) {
200
+ throw new Error(`unsupported proxy protocol (${proxyUrl.protocol.slice(0, -1)}), oc honors HTTP and HTTPS proxies`);
201
+ }
202
+ }
203
+
204
+ // Split host[:port], including [IPv6]:port. Unbracketed IPv6 literals never
205
+ // carry a port suffix (use [addr]:port); a trailing :digits on ::1 is part of
206
+ // the address, not a port.
207
+ function splitHostPort(entry) {
208
+ if (entry.startsWith('[')) {
209
+ const end = entry.indexOf(']');
210
+ if (end === -1) return { host: entry, port: '' };
211
+ const rest = entry.slice(end + 1);
212
+ return { host: entry.slice(1, end), port: rest.startsWith(':') ? rest.slice(1) : '' };
213
+ }
214
+ if (net.isIP(entry) === 6) return { host: entry, port: '' };
215
+ const colon = entry.lastIndexOf(':');
216
+ if (colon !== -1 && /^\d+$/.test(entry.slice(colon + 1))) {
217
+ const host = entry.slice(0, colon);
218
+ if (net.isIP(host) === 6) return { host: entry, port: '' };
219
+ return { host, port: entry.slice(colon + 1) };
220
+ }
221
+ return { host: entry, port: '' };
222
+ }
223
+
224
+ function ipv4InCidr(ip, base, bits) {
225
+ const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0;
226
+ return (ipv4ToInt(ip) & mask) === (ipv4ToInt(base) & mask);
227
+ }
228
+
229
+ function ipv6InCidr(ip, base, bits) {
230
+ const g = expandIPv6(ip);
231
+ const b = expandIPv6(base);
232
+ if (!g || !b) return false;
233
+ let remaining = bits;
234
+ for (let i = 0; i < 8 && remaining > 0; i++) {
235
+ if (remaining >= 16) {
236
+ if (g[i] !== b[i]) return false;
237
+ remaining -= 16;
238
+ } else {
239
+ const mask = (0xffff << (16 - remaining)) & 0xffff;
240
+ if ((g[i] & mask) !== (b[i] & mask)) return false;
241
+ remaining = 0;
242
+ }
243
+ }
244
+ return true;
245
+ }
246
+
247
+ function ipInCidr(ip, base, bits) {
248
+ const family = net.isIP(ip);
249
+ if (family === 4) return ipv4InCidr(ip, base, bits);
250
+ if (family === 6) return ipv6InCidr(ip, base, bits);
251
+ return false;
252
+ }
253
+
254
+ function bypassesProxy(target, noProxy) {
255
+ const list = noProxy.trim();
256
+ if (!list) return false;
257
+ const hostname = target.hostname.toLowerCase().replace(/^\[|\]$/g, '');
258
+ const port = target.port || (target.protocol === 'https:' ? '443' : '80');
259
+ for (let entry of list.split(',')) {
260
+ entry = entry.trim();
261
+ if (!entry) continue;
262
+ if (entry === '*') return true;
263
+ const { host: rawHost, port: entryPort } = splitHostPort(entry);
264
+ if (entryPort && entryPort !== port) continue;
265
+ let pattern = rawHost.toLowerCase().replace(/^\[|\]$/g, '');
266
+ const wildcard = pattern.startsWith('*.');
267
+ if (wildcard) pattern = pattern.slice(2);
268
+ const slash = pattern.indexOf('/');
269
+ if (slash !== -1 && net.isIP(pattern.slice(0, slash))) {
270
+ const bits = Number(pattern.slice(slash + 1));
271
+ if (Number.isInteger(bits) && net.isIP(hostname) && ipInCidr(hostname, pattern.slice(0, slash), bits)) {
272
+ return true;
273
+ }
274
+ continue;
275
+ }
276
+ const host = pattern.replace(/^\./, '');
277
+ if (!host) continue;
278
+ if (wildcard) {
279
+ if (hostname.endsWith(`.${host}`)) return true;
280
+ continue;
281
+ }
282
+ if (hostname === host || hostname.endsWith(`.${host}`)) return true;
283
+ }
284
+ return false;
285
+ }
286
+
287
+ /**
288
+ * Pick a proxy for this URL from HTTP_PROXY / HTTPS_PROXY / NO_PROXY (and
289
+ * their lowercase forms). The proxy host is not run through assertSafeTarget:
290
+ * corporate proxies live on loopback or RFC 1918 addresses, and they are not
291
+ * the page being fetched. Redirect hops still go through that check.
292
+ * @param {string} url
293
+ * @param {NodeJS.ProcessEnv} [env]
294
+ * @returns {string | null} proxy URL, or null to connect directly
295
+ */
296
+ export function resolveProxy(url, env = process.env) {
297
+ const target = new URL(url);
298
+ if (bypassesProxy(target, envFirst(env, 'NO_PROXY', 'no_proxy') ?? '')) return null;
299
+ const httpsProxy = envFirst(env, 'HTTPS_PROXY', 'https_proxy');
300
+ const httpProxy = envFirst(env, 'HTTP_PROXY', 'http_proxy');
301
+ const chosen = target.protocol === 'https:' ? (httpsProxy || httpProxy) : httpProxy;
302
+ const normalized = normalizeProxy(chosen);
303
+ if (!normalized) return null;
304
+ assertHttpProxyProtocol(new URL(normalized));
305
+ return normalized;
306
+ }
307
+
308
+ function decodeCredential(value) {
309
+ try {
310
+ return decodeURIComponent(value);
311
+ } catch {
312
+ return value;
313
+ }
314
+ }
315
+
316
+ function proxyAuthHeader(proxy) {
317
+ if (!proxy.username) return undefined;
318
+ const token = Buffer.from(`${decodeCredential(proxy.username)}:${decodeCredential(proxy.password)}`).toString('base64');
319
+ return `Basic ${token}`;
320
+ }
321
+
322
+ function authority(target) {
323
+ const host = net.isIP(target.hostname) === 6 ? `[${target.hostname}]` : target.hostname;
324
+ const port = target.port || (target.protocol === 'https:' ? '443' : '80');
325
+ return `${host}:${port}`;
326
+ }
327
+
328
+ function wrapNodeResponse(res, url) {
329
+ const headers = {
330
+ get(name) {
331
+ const v = res.headers[name.toLowerCase()];
332
+ if (v == null) return null;
333
+ return Array.isArray(v) ? v.join(', ') : v;
334
+ },
335
+ // Node keeps Set-Cookie as an array of raw values. Expose it unjoined so
336
+ // the cookie jar reads each header intact: a comma in an Expires date
337
+ // makes the joined form ambiguous to split back apart.
338
+ getSetCookie() {
339
+ const v = res.headers['set-cookie'];
340
+ if (v == null) return [];
341
+ return Array.isArray(v) ? v : [v];
342
+ },
343
+ };
344
+ const text = () => new Promise((resolve, reject) => {
345
+ const chunks = [];
346
+ let size = 0;
347
+ res.on('data', (c) => {
348
+ size += c.length;
349
+ try {
350
+ assertBodySize(size, url);
351
+ } catch (err) {
352
+ // destroy surfaces the refusal through 'error', and stops the read.
353
+ res.destroy(err);
354
+ return;
355
+ }
356
+ chunks.push(c);
357
+ });
358
+ res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
359
+ res.on('error', reject);
360
+ });
361
+ const status = res.statusCode ?? 0;
362
+ return {
363
+ status,
364
+ statusText: res.statusMessage || '',
365
+ headers,
366
+ ok: status >= 200 && status < 300,
367
+ url,
368
+ text,
369
+ };
370
+ }
371
+
372
+ function proxyTransport(proxy) {
373
+ return proxy.protocol === 'https:' ? https : http;
374
+ }
375
+
376
+ function proxyPort(proxy) {
377
+ return Number(proxy.port) || (proxy.protocol === 'https:' ? 443 : 80);
378
+ }
379
+
380
+ function armRequestTimeout(req, reject, label) {
381
+ req.setTimeout(PROXY_TIMEOUT_MS, () => {
382
+ req.destroy();
383
+ reject(new Error(`proxy timed out after ${PROXY_TIMEOUT_MS / 1000}s for ${label}`));
384
+ });
385
+ }
386
+
387
+ function pickTlsCa(tlsOpts) {
388
+ return tlsOpts.ca != null ? { ca: tlsOpts.ca } : {};
389
+ }
390
+
391
+ function httpViaProxy(target, proxy, headers) {
392
+ const auth = proxyAuthHeader(proxy);
393
+ return new Promise((resolve, reject) => {
394
+ const req = proxyTransport(proxy).request({
395
+ hostname: proxy.hostname,
396
+ port: proxyPort(proxy),
397
+ method: 'GET',
398
+ path: `${target.protocol}//${target.host}${target.pathname}${target.search}`,
399
+ headers: {
400
+ ...headers,
401
+ host: target.host,
402
+ ...(auth && { 'proxy-authorization': auth }),
403
+ },
404
+ }, (res) => resolve(wrapNodeResponse(res, target.href)));
405
+ armRequestTimeout(req, reject, target.href);
406
+ req.on('error', (err) => reject(new Error(`proxy failed: ${err.message} for ${target.href}`)));
407
+ req.end();
408
+ });
409
+ }
410
+
411
+ function httpsViaConnect(target, proxy, headers, tlsOpts = {}) {
412
+ const dest = authority(target);
413
+ const auth = proxyAuthHeader(proxy);
414
+ return new Promise((resolve, reject) => {
415
+ let settled = false;
416
+ const fail = (err) => {
417
+ if (settled) return;
418
+ settled = true;
419
+ req.destroy();
420
+ reject(err instanceof Error ? err : new Error(String(err)));
421
+ };
422
+ const req = proxyTransport(proxy).request({
423
+ hostname: proxy.hostname,
424
+ port: proxyPort(proxy),
425
+ method: 'CONNECT',
426
+ path: dest,
427
+ headers: {
428
+ host: dest,
429
+ ...(auth && { 'proxy-authorization': auth }),
430
+ },
431
+ });
432
+ armRequestTimeout(req, fail, target.href);
433
+ req.on('connect', (res, socket, head) => {
434
+ if (res.statusCode !== 200) {
435
+ socket.destroy();
436
+ fail(new Error(`proxy CONNECT failed: ${res.statusCode} for ${target.href}`));
437
+ return;
438
+ }
439
+ if (head.length) socket.unshift(head);
440
+ // tls.connect already opened the tunnel. https.request would wrap TLS
441
+ // again, and the origin would see a second ClientHello as garbage.
442
+ // SNI is a hostname; an IP literal is only used for the cert check.
443
+ // URL.hostname keeps the brackets around an IPv6 literal ("[::1]"), which
444
+ // tls.connect would treat as a DNS name; strip them like assertSafeTarget.
445
+ const hostname = target.hostname.replace(/^\[|\]$/g, '');
446
+ const tlsSocket = tls.connect({
447
+ socket,
448
+ host: hostname,
449
+ ...(net.isIP(hostname) ? {} : { servername: hostname }),
450
+ ...pickTlsCa(tlsOpts),
451
+ }, () => {
452
+ const tunneled = http.request({
453
+ createConnection: () => tlsSocket,
454
+ path: `${target.pathname}${target.search}`,
455
+ method: 'GET',
456
+ headers: { ...headers, host: target.host },
457
+ }, (httpsRes) => {
458
+ if (settled) return;
459
+ settled = true;
460
+ resolve(wrapNodeResponse(httpsRes, target.href));
461
+ });
462
+ armRequestTimeout(tunneled, fail, target.href);
463
+ tunneled.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`)));
464
+ tunneled.end();
465
+ });
466
+ tlsSocket.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`)));
467
+ });
468
+ req.on('error', (err) => fail(new Error(`proxy failed: ${err.message || err.code} for ${target.href}`)));
469
+ req.end();
470
+ });
471
+ }
472
+
473
+ /**
474
+ * One GET through an HTTP(S) proxy. HTTP targets use the absolute-URI form;
475
+ * HTTPS targets open a CONNECT tunnel first. Redirects are not followed:
476
+ * followRedirects owns that so each hop still goes through assertSafeTarget.
477
+ * @param {string} url
478
+ * @param {string} proxy
479
+ * @param {Record<string, string>} [headers]
480
+ * @param {import('node:tls').ConnectionOptions} [tlsOpts]
481
+ */
482
+ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
483
+ const target = new URL(url);
484
+ const proxyUrl = new URL(proxy);
485
+ assertHttpProxyProtocol(proxyUrl);
486
+ return target.protocol === 'https:'
487
+ ? httpsViaConnect(target, proxyUrl, headers, tlsOpts)
488
+ : httpViaProxy(target, proxyUrl, headers);
489
+ }
490
+
149
491
  /**
150
492
  * Fetch a page.
151
493
  * @param {string} url - with or without a scheme, https is assumed
494
+ * @param {{ jar?: { cookieHeaderFor(url: string): string|undefined, storeFromResponse(url: string, headers: string[]): void } }} [opts]
152
495
  * @returns {Promise<{url: string, html: string, status: number, via: string}>}
153
496
  * final URL after redirects, the body, the HTTP status, and which client
154
497
  * identity got the page (impers:chrome, impers:firefox, or fetch)
155
498
  */
156
- export async function fetchPage(url) {
499
+ export async function fetchPage(url, { jar } = {}) {
157
500
  const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
158
501
  await assertSafeTarget(target);
159
502
  const impers = await loadImpers();
160
- return impers ? viaImpers(impers, target) : viaFetch(target);
503
+ return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
161
504
  }
162
505
 
163
506
  /**
@@ -174,11 +517,12 @@ export async function fetchPage(url) {
174
517
  * @param {string} start
175
518
  * @returns {Promise<{res: any, url: string}>} the first non-redirect response
176
519
  */
177
- export async function followRedirects(get, start) {
520
+ export async function followRedirects(get, start, { onResponse } = {}) {
178
521
  let current = start;
179
522
  for (let i = 0; ; i++) {
180
523
  if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
181
524
  const res = await get(current);
525
+ onResponse?.(current, res);
182
526
  const status = res.status ?? res.statusCode ?? 0;
183
527
  const location = res.headers.get('location');
184
528
  if (status >= 300 && status < 400 && location) {
@@ -190,36 +534,92 @@ export async function followRedirects(get, start) {
190
534
  }
191
535
  }
192
536
 
193
- async function viaImpers(impers, target) {
537
+ const FETCH_HEADERS = {
538
+ 'user-agent': UA,
539
+ accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
540
+ 'accept-language': 'en-US,en;q=0.9',
541
+ };
542
+
543
+ function mergeHeaders(base, extra) {
544
+ return extra ? { ...base, ...extra } : base;
545
+ }
546
+
547
+ function jarHeaders(jar, url, base) {
548
+ if (!jar) return base;
549
+ const cookie = jar.cookieHeaderFor(url);
550
+ return cookie ? mergeHeaders(base, { cookie }) : base;
551
+ }
552
+
553
+ function captureSetCookie(jar, url, res) {
554
+ if (!jar) return;
555
+ jar.storeFromResponse(url, getSetCookieHeaders(res));
556
+ }
557
+
558
+ async function viaImpers(impers, target, jar) {
194
559
  // Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a
195
560
  // blocked first attempt gets one cheap retry with a second identity.
196
- const asking = (impersonate) => (url) => impers.get(url, { impersonate, allowRedirects: false });
561
+ const asking = (impersonate) => (url) =>
562
+ impers.get(url, {
563
+ impersonate,
564
+ allowRedirects: false,
565
+ proxy: resolveProxy(url) ?? '',
566
+ headers: jarHeaders(jar, url, {}),
567
+ });
568
+ const onResponse = (url, res) => captureSetCookie(jar, url, res);
197
569
  let via = 'impers:chrome';
198
- let { res } = await followRedirects(asking('chrome'), target);
570
+ let { res } = await followRedirects(asking('chrome'), target, { onResponse });
199
571
  let status = res.status ?? res.statusCode ?? 0;
200
572
  if (status >= 400) {
201
573
  via = 'impers:firefox';
202
- ({ res } = await followRedirects(asking('firefox'), target));
574
+ ({ res } = await followRedirects(asking('firefox'), target, { onResponse }));
203
575
  status = res.status ?? res.statusCode ?? 0;
204
576
  }
205
577
  if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
206
578
  assertReadableType(res.headers.get('content-type'));
579
+ assertBodySize(Number(res.headers.get('content-length')) || 0, target);
580
+ // impers buffers inside its own binding, so the size of what it already
581
+ // holds is all there is to check; the bound still stops an oversized body
582
+ // from travelling any further.
207
583
  const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
584
+ assertBodySize(html.length, target);
208
585
  return { url: res.url ?? target, html, status, via };
209
586
  }
210
587
 
211
- async function viaFetch(target) {
212
- const { res, url: current } = await followRedirects((url) => fetch(url, {
213
- redirect: 'manual',
214
- headers: {
215
- 'user-agent': UA,
216
- accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
217
- 'accept-language': 'en-US,en;q=0.9',
218
- },
219
- }), target);
588
+ async function viaFetch(target, jar) {
589
+ const get = (url) => {
590
+ const proxy = resolveProxy(url);
591
+ const headers = jarHeaders(jar, url, FETCH_HEADERS);
592
+ return proxy
593
+ ? proxyGet(url, proxy, headers)
594
+ : fetch(url, { redirect: 'manual', headers });
595
+ };
596
+ const onResponse = (url, res) => captureSetCookie(jar, url, res);
597
+ const { res, url: current } = await followRedirects(get, target, { onResponse });
220
598
  if (!res.ok) {
221
599
  throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
222
600
  }
223
601
  assertReadableType(res.headers.get('content-type'));
224
- return { url: res.url || current, html: await res.text(), status: res.status, via: 'fetch' };
602
+ assertBodySize(Number(res.headers.get('content-length')) || 0, current);
603
+ return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch' };
604
+ }
605
+
606
+ /**
607
+ * The decoded body as text, counted as it arrives so crossing the cap aborts
608
+ * the transfer instead of finishing it. Throwing mid-iteration cancels the
609
+ * stream. A proxy response has no web stream to iterate; its text() counts
610
+ * inside wrapNodeResponse instead.
611
+ * @param {any} res
612
+ * @param {string} url
613
+ * @returns {Promise<string>}
614
+ */
615
+ export async function readBody(res, url) {
616
+ if (!res.body?.getReader) return res.text();
617
+ const chunks = [];
618
+ let size = 0;
619
+ for await (const chunk of res.body) {
620
+ size += chunk.byteLength;
621
+ assertBodySize(size, url);
622
+ chunks.push(chunk);
623
+ }
624
+ return Buffer.concat(chunks).toString('utf8');
225
625
  }
@@ -0,0 +1,177 @@
1
+ /**
2
+ * Node.js docs search backend. nodejs.org has no search results page at all:
3
+ * the site's search box is a JavaScript modal asking a third-party service,
4
+ * so there is nothing for oc to fetch or call directly. But the API docs
5
+ * publish their entire reference as one static JSON file, all.json, much the
6
+ * way a Sphinx site publishes its search index, so `search` ranks that file
7
+ * locally: every module, class, method, property, and event heading becomes
8
+ * a result linking to its own anchor. The file is ~8MB (~1MB over the wire)
9
+ * and static, so it shares the Sphinx backend's day cache and, like the
10
+ * index, is never printed: what reaches the agent is the ranked list only.
11
+ */
12
+
13
+ import { cachedFile } from './cache.js';
14
+ import { escapeHTML } from './sphinx.js';
15
+
16
+ const MAX_RESULTS = 20;
17
+
18
+ // The list-valued keys that hold entries with headings of their own on the
19
+ // page. The others (params, options) describe one signature's arguments and
20
+ // have no heading or anchor; a class's constructor rides in `signatures`.
21
+ const CHILD_KEYS = [
22
+ 'modules', 'globals', 'miscs', 'classes', 'classMethods',
23
+ 'signatures', 'methods', 'properties', 'events',
24
+ ];
25
+
26
+ // What each heading's `type` is called on the results page. Anything else is
27
+ // a property whose type field holds its value type ({number}, {boolean}).
28
+ const KIND = {
29
+ module: 'module', misc: 'section', global: 'global', class: 'class',
30
+ ctor: 'constructor', classMethod: 'static method', method: 'method',
31
+ event: 'event', property: 'property',
32
+ };
33
+
34
+ // The docs derive a heading's anchor from its text the github-slugger way:
35
+ // lowercase, drop everything but letters, digits, spaces, hyphens, and
36
+ // underscores, then spaces become hyphens. 'fs.readFile(path[, options],
37
+ // callback)' is #fsreadfilepath-options-callback.
38
+ const slug = (text) =>
39
+ text.toLowerCase().replace(/[^a-z0-9 _-]/g, '').trim().replaceAll(' ', '-');
40
+
41
+ /**
42
+ * Anything that parses but is not the docs corpus (an error page served as
43
+ * JSON, a moved file) fails here, which keeps it out of the cache too.
44
+ * @param {string} text
45
+ * @returns {any}
46
+ */
47
+ export function parseAll(text) {
48
+ let all;
49
+ try {
50
+ all = JSON.parse(text);
51
+ } catch {
52
+ all = null;
53
+ }
54
+ if (!Array.isArray(all?.modules)) throw new Error('not the Node.js docs corpus');
55
+ return all;
56
+ }
57
+
58
+ /**
59
+ * Flatten the docs tree into the headings a query can hit. Only a top-level
60
+ * entry names its page (source: 'doc/api/fs.md'); everything nested under it
61
+ * inherits that page and contributes its own heading and anchor. A nested
62
+ * entry whose textRaw does not name it is not a heading (a bare 'Type:
63
+ * {number}' line under a property) and is skipped; its parent still stands.
64
+ * A heading repeated on one page is kept once: the corpus lists some methods
65
+ * twice for one heading, and where a page really repeats one (each stream
66
+ * class has an Event: 'close') the rows would be indistinguishable anyway.
67
+ * @param {any} all - parsed all.json
68
+ * @returns {{text: string, name: string, type: string, page: string, anchor: string}[]}
69
+ */
70
+ export function buildEntries(all) {
71
+ const entries = [];
72
+ const seen = new Set();
73
+ const add = (node, page, top) => {
74
+ const text = String(node?.textRaw ?? '').replaceAll('`', '').trim();
75
+ const name = String(node?.name ?? '');
76
+ const type = String(node?.type ?? '');
77
+ const heading = text && name
78
+ && (top || text.toLowerCase().includes(name.toLowerCase()));
79
+ if (heading && !seen.has(`${page}#${text}`)) {
80
+ seen.add(`${page}#${text}`);
81
+ // A module or section heading is the page's own title, so its entry
82
+ // links to the page top; every other heading has an anchor worth keeping.
83
+ const anchor = ['module', 'misc', 'global'].includes(type) ? '' : slug(text);
84
+ entries.push({ text, name, type, page, anchor });
85
+ }
86
+ for (const key of CHILD_KEYS) {
87
+ for (const child of node?.[key] ?? []) add(child, page, false);
88
+ }
89
+ };
90
+ for (const key of CHILD_KEYS) {
91
+ for (const node of all?.[key] ?? []) {
92
+ const page = String(node?.source ?? '')
93
+ .replace(/^doc\/api\//, '').replace(/\.md$/, '');
94
+ if (page) add(node, `${page}.html`, true);
95
+ }
96
+ }
97
+ return entries;
98
+ }
99
+
100
+ /**
101
+ * Rank the headings against a query. A word that is an entry's own name (the
102
+ * bare symbol: 'readFile') weighs most, a whole word of its heading next, a
103
+ * substring of it least, and an entry must match every word before any-word
104
+ * matching kicks in, the same policy the Sphinx backend follows.
105
+ * @param {ReturnType<typeof buildEntries>} entries
106
+ * @param {string} query
107
+ */
108
+ export function searchEntries(entries, query) {
109
+ const words = [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))];
110
+ const scored = [];
111
+ for (const e of entries) {
112
+ const nameL = e.name.toLowerCase();
113
+ const textL = e.text.toLowerCase();
114
+ const tokens = new Set(textL.split(/[^a-z0-9_]+/));
115
+ let score = 0;
116
+ let hit = 0;
117
+ for (const word of words) {
118
+ const s = nameL === word ? 10 : tokens.has(word) ? 5 : textL.includes(word) ? 2 : 0;
119
+ if (s) {
120
+ score += s;
121
+ hit += 1;
122
+ }
123
+ }
124
+ if (hit) scored.push({ e, score, hit });
125
+ }
126
+ let hits = scored.filter((s) => s.hit === words.length);
127
+ const partial = !hits.length && words.length > 1 && scored.length > 0;
128
+ if (partial) hits = scored;
129
+ // Among equal scores the shorter heading is the plainer API, so it leads.
130
+ hits.sort((a, b) => b.score - a.score
131
+ || a.e.text.length - b.e.text.length
132
+ || a.e.text.localeCompare(b.e.text));
133
+ return { words, partial, total: hits.length, hits: hits.slice(0, MAX_RESULTS).map((s) => s.e) };
134
+ }
135
+
136
+ /**
137
+ * The result list becomes the same small synthetic page the other search
138
+ * backends emit, so it distills, renders, numbers, and remembers like any
139
+ * fetched page and `do <n>` follows a result.
140
+ * @param {string} base
141
+ * @param {string} query
142
+ * @param {ReturnType<typeof searchEntries>} found
143
+ * @returns {string}
144
+ */
145
+ export function resultsToHTML(base, query, found) {
146
+ const host = new URL(base).host;
147
+ const items = found.hits.map((e) => {
148
+ const href = new URL(e.anchor ? `${e.page}#${e.anchor}` : e.page, base).href;
149
+ return `<li><a href="${escapeHTML(href)}">${escapeHTML(e.text)}</a>`
150
+ + ` ${escapeHTML(KIND[e.type] ?? 'property')}, in ${escapeHTML(e.page.replace(/\.html$/, ''))}</li>`;
151
+ });
152
+ const partial = found.partial ? '; no heading matches every word, so these match some' : '';
153
+ const summary = items.length
154
+ ? `${found.total} heading${found.total === 1 ? '' : 's'} match in the docs' own reference,`
155
+ + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
156
+ : `nothing in the docs' own reference matches; try fewer or different words`;
157
+ return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
158
+ + `<p>${summary}</p>`
159
+ + (items.length ? `<ol>${items.join('')}</ol>` : '')
160
+ + `</main></body></html>`;
161
+ }
162
+
163
+ /**
164
+ * Search the Node.js API docs. The site has no human search URL to remember,
165
+ * so the session keeps the docs index, the page a reader would start from.
166
+ * @param {string} base - docs root ending in '/', e.g. https://nodejs.org/api/
167
+ * @param {string} query
168
+ */
169
+ export async function nodeSearch(base, query) {
170
+ if (!query.trim()) throw new Error('usage: search <query>');
171
+ const { data, via } = await cachedFile('nodedoc', new URL('all.json', base).href, parseAll);
172
+ return {
173
+ url: new URL('index.html', base).href,
174
+ html: resultsToHTML(base, query, searchEntries(buildEntries(data), query)),
175
+ via,
176
+ };
177
+ }