@only-cli/oc 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +107 -38
- package/clis/developer.mozilla.org.json +9 -0
- package/clis/doc.rust-lang.org.json +8 -0
- package/clis/docs.oracle.com.json +7 -0
- package/clis/docs.python.org.json +8 -0
- package/clis/docs.ruby-lang.org.json +7 -0
- package/clis/en.cppreference.com.json +8 -0
- package/clis/learn.microsoft.com.json +1 -0
- package/clis/nodejs.org.json +7 -0
- package/clis/php.net.json +8 -0
- package/clis/pkg.go.dev.json +7 -0
- package/clis/typescriptlang.org.json +7 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +42 -23
- package/src/act.js +13 -2
- package/src/apisearch.js +80 -0
- package/src/auth.js +46 -0
- package/src/cache.js +43 -0
- package/src/cli.js +167 -11
- package/src/cookies.js +530 -0
- package/src/fetch.js +142 -19
- package/src/nodedocs.js +177 -0
- package/src/rdoc.js +98 -0
- package/src/render.js +21 -10
- package/src/session.js +43 -4
- package/src/sites.js +30 -5
- package/src/sphinx.js +211 -0
package/src/fetch.js
CHANGED
|
@@ -13,6 +13,8 @@ import https from 'node:https';
|
|
|
13
13
|
import net from 'node:net';
|
|
14
14
|
import tls from 'node:tls';
|
|
15
15
|
|
|
16
|
+
import { getSetCookieHeaders } from './cookies.js';
|
|
17
|
+
|
|
16
18
|
// The fetch fallback can't fake a TLS fingerprint like impers does, but it
|
|
17
19
|
// should at least send the same Chrome identity in its headers.
|
|
18
20
|
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36';
|
|
@@ -35,6 +37,29 @@ const PROXY_TIMEOUT_MS = 300_000;
|
|
|
35
37
|
// pages of mojibake an agent then pays for, so it is refused by name instead.
|
|
36
38
|
const READABLE_TYPE = /^\s*(?:text\/|application\/(?:json|xml|javascript|x-ndjson|[\w.+-]*\+(?:json|xml)))/i;
|
|
37
39
|
|
|
40
|
+
// The whole decoded body is buffered before the distiller sees it, so an
|
|
41
|
+
// unbounded response is an unbounded allocation, and a URL is often the
|
|
42
|
+
// page's to name, not the caller's. The cap is generous because oc fetches
|
|
43
|
+
// some large corpora on purpose (the Node.js docs reference is 8.5MB
|
|
44
|
+
// decoded); three times that and a response is not a page anyone reads.
|
|
45
|
+
// Content-Length rejects a known-large response before its bytes arrive, but
|
|
46
|
+
// the header is optional and untrusted, so every transport also counts what
|
|
47
|
+
// actually lands, after decoding, which is what stops a decompression bomb.
|
|
48
|
+
export const MAX_BODY = 25 * 1024 * 1024;
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Refuse a body larger than oc will buffer. Called on the Content-Length
|
|
52
|
+
* header first and again on the bytes as they arrive, since only the second
|
|
53
|
+
* count is trustworthy.
|
|
54
|
+
* @param {number} size - bytes seen so far, or claimed by the header
|
|
55
|
+
* @param {string} url
|
|
56
|
+
*/
|
|
57
|
+
export function assertBodySize(size, url) {
|
|
58
|
+
if (size > MAX_BODY) {
|
|
59
|
+
throw new Error(`response body over ${MAX_BODY / 1048576}MB for ${url}, more than oc will read`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
38
63
|
/**
|
|
39
64
|
* Refuse a response oc cannot read as text. Both transports call this: the
|
|
40
65
|
* gate has to live on whichever client got the page, or the same URL renders
|
|
@@ -307,10 +332,29 @@ function wrapNodeResponse(res, url) {
|
|
|
307
332
|
if (v == null) return null;
|
|
308
333
|
return Array.isArray(v) ? v.join(', ') : v;
|
|
309
334
|
},
|
|
335
|
+
// Node keeps Set-Cookie as an array of raw values. Expose it unjoined so
|
|
336
|
+
// the cookie jar reads each header intact: a comma in an Expires date
|
|
337
|
+
// makes the joined form ambiguous to split back apart.
|
|
338
|
+
getSetCookie() {
|
|
339
|
+
const v = res.headers['set-cookie'];
|
|
340
|
+
if (v == null) return [];
|
|
341
|
+
return Array.isArray(v) ? v : [v];
|
|
342
|
+
},
|
|
310
343
|
};
|
|
311
344
|
const text = () => new Promise((resolve, reject) => {
|
|
312
345
|
const chunks = [];
|
|
313
|
-
|
|
346
|
+
let size = 0;
|
|
347
|
+
res.on('data', (c) => {
|
|
348
|
+
size += c.length;
|
|
349
|
+
try {
|
|
350
|
+
assertBodySize(size, url);
|
|
351
|
+
} catch (err) {
|
|
352
|
+
// destroy surfaces the refusal through 'error', and stops the read.
|
|
353
|
+
res.destroy(err);
|
|
354
|
+
return;
|
|
355
|
+
}
|
|
356
|
+
chunks.push(c);
|
|
357
|
+
});
|
|
314
358
|
res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
|
315
359
|
res.on('error', reject);
|
|
316
360
|
});
|
|
@@ -396,7 +440,9 @@ function httpsViaConnect(target, proxy, headers, tlsOpts = {}) {
|
|
|
396
440
|
// tls.connect already opened the tunnel. https.request would wrap TLS
|
|
397
441
|
// again, and the origin would see a second ClientHello as garbage.
|
|
398
442
|
// SNI is a hostname; an IP literal is only used for the cert check.
|
|
399
|
-
|
|
443
|
+
// URL.hostname keeps the brackets around an IPv6 literal ("[::1]"), which
|
|
444
|
+
// tls.connect would treat as a DNS name; strip them like assertSafeTarget.
|
|
445
|
+
const hostname = target.hostname.replace(/^\[|\]$/g, '');
|
|
400
446
|
const tlsSocket = tls.connect({
|
|
401
447
|
socket,
|
|
402
448
|
host: hostname,
|
|
@@ -445,15 +491,16 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
|
|
|
445
491
|
/**
|
|
446
492
|
* Fetch a page.
|
|
447
493
|
* @param {string} url - with or without a scheme, https is assumed
|
|
494
|
+
* @param {{ jar?: { cookieHeaderFor(url: string): string|undefined, storeFromResponse(url: string, headers: string[]): void } }} [opts]
|
|
448
495
|
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
|
449
496
|
* final URL after redirects, the body, the HTTP status, and which client
|
|
450
497
|
* identity got the page (impers:chrome, impers:firefox, or fetch)
|
|
451
498
|
*/
|
|
452
|
-
export async function fetchPage(url) {
|
|
499
|
+
export async function fetchPage(url, { jar } = {}) {
|
|
453
500
|
const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
|
454
501
|
await assertSafeTarget(target);
|
|
455
502
|
const impers = await loadImpers();
|
|
456
|
-
return impers ? viaImpers(impers, target) : viaFetch(target);
|
|
503
|
+
return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
|
|
457
504
|
}
|
|
458
505
|
|
|
459
506
|
/**
|
|
@@ -470,11 +517,12 @@ export async function fetchPage(url) {
|
|
|
470
517
|
* @param {string} start
|
|
471
518
|
* @returns {Promise<{res: any, url: string}>} the first non-redirect response
|
|
472
519
|
*/
|
|
473
|
-
export async function followRedirects(get, start) {
|
|
520
|
+
export async function followRedirects(get, start, { onResponse } = {}) {
|
|
474
521
|
let current = start;
|
|
475
522
|
for (let i = 0; ; i++) {
|
|
476
523
|
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
|
|
477
524
|
const res = await get(current);
|
|
525
|
+
onResponse?.(current, res);
|
|
478
526
|
const status = res.status ?? res.statusCode ?? 0;
|
|
479
527
|
const location = res.headers.get('location');
|
|
480
528
|
if (status >= 300 && status < 400 && location) {
|
|
@@ -492,35 +540,110 @@ const FETCH_HEADERS = {
|
|
|
492
540
|
'accept-language': 'en-US,en;q=0.9',
|
|
493
541
|
};
|
|
494
542
|
|
|
495
|
-
|
|
543
|
+
function mergeHeaders(base, extra) {
|
|
544
|
+
return extra ? { ...base, ...extra } : base;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
function jarHeaders(jar, url, base) {
|
|
548
|
+
if (!jar) return base;
|
|
549
|
+
const cookie = jar.cookieHeaderFor(url);
|
|
550
|
+
return cookie ? mergeHeaders(base, { cookie }) : base;
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
function captureSetCookie(jar, url, res) {
|
|
554
|
+
if (!jar) return;
|
|
555
|
+
jar.storeFromResponse(url, getSetCookieHeaders(res));
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Fetch a page through impers, downgrading identity when one is refused.
|
|
560
|
+
* Exported so the downgrade chain can be proven against a fake impers; the
|
|
561
|
+
* real entry point is fetchPage.
|
|
562
|
+
* @param {any} impers - the impers module (or a stand-in with a get method)
|
|
563
|
+
* @param {string} target
|
|
564
|
+
* @param {object} [jar]
|
|
565
|
+
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
|
566
|
+
*/
|
|
567
|
+
export async function viaImpers(impers, target, jar) {
|
|
496
568
|
// Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a
|
|
497
|
-
// blocked first attempt gets one cheap retry with a second identity.
|
|
569
|
+
// blocked first attempt gets one cheap retry with a second identity. An
|
|
570
|
+
// ImpersonateError is the same story one layer down: impers resolves the
|
|
571
|
+
// 'chrome' alias to its newest fingerprint, but the native library it loads
|
|
572
|
+
// can be an older system copy of libcurl-impersonate that predates that
|
|
573
|
+
// fingerprint and refuses it before any request leaves. Firefox aliases to
|
|
574
|
+
// an older target that such a library usually still knows, and when both
|
|
575
|
+
// identities are refused the plain fetch transport still gets the page.
|
|
498
576
|
const asking = (impersonate) => (url) =>
|
|
499
|
-
impers.get(url, {
|
|
577
|
+
impers.get(url, {
|
|
578
|
+
impersonate,
|
|
579
|
+
allowRedirects: false,
|
|
580
|
+
proxy: resolveProxy(url) ?? '',
|
|
581
|
+
headers: jarHeaders(jar, url, {}),
|
|
582
|
+
});
|
|
583
|
+
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
584
|
+
const attempt = async (impersonate) => {
|
|
585
|
+
try {
|
|
586
|
+
const { res } = await followRedirects(asking(impersonate), target, { onResponse });
|
|
587
|
+
return { res, status: res.status ?? res.statusCode ?? 0 };
|
|
588
|
+
} catch (err) {
|
|
589
|
+
if (err?.name !== 'ImpersonateError') throw err;
|
|
590
|
+
return null;
|
|
591
|
+
}
|
|
592
|
+
};
|
|
500
593
|
let via = 'impers:chrome';
|
|
501
|
-
let
|
|
502
|
-
|
|
503
|
-
if (status >= 400) {
|
|
594
|
+
let got = await attempt('chrome');
|
|
595
|
+
if (!got || got.status >= 400) {
|
|
504
596
|
via = 'impers:firefox';
|
|
505
|
-
|
|
506
|
-
status = res.status ?? res.statusCode ?? 0;
|
|
597
|
+
got = (await attempt('firefox')) ?? got;
|
|
507
598
|
}
|
|
599
|
+
if (!got) return viaFetch(target, jar);
|
|
600
|
+
const { res, status } = got;
|
|
508
601
|
if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
|
|
509
602
|
assertReadableType(res.headers.get('content-type'));
|
|
603
|
+
assertBodySize(Number(res.headers.get('content-length')) || 0, target);
|
|
604
|
+
// impers buffers inside its own binding, so the size of what it already
|
|
605
|
+
// holds is all there is to check; the bound still stops an oversized body
|
|
606
|
+
// from travelling any further.
|
|
510
607
|
const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
|
|
608
|
+
assertBodySize(html.length, target);
|
|
511
609
|
return { url: res.url ?? target, html, status, via };
|
|
512
610
|
}
|
|
513
611
|
|
|
514
|
-
async function viaFetch(target) {
|
|
515
|
-
const
|
|
612
|
+
async function viaFetch(target, jar) {
|
|
613
|
+
const get = (url) => {
|
|
516
614
|
const proxy = resolveProxy(url);
|
|
615
|
+
const headers = jarHeaders(jar, url, FETCH_HEADERS);
|
|
517
616
|
return proxy
|
|
518
|
-
? proxyGet(url, proxy,
|
|
519
|
-
: fetch(url, { redirect: 'manual', headers
|
|
520
|
-
}
|
|
617
|
+
? proxyGet(url, proxy, headers)
|
|
618
|
+
: fetch(url, { redirect: 'manual', headers });
|
|
619
|
+
};
|
|
620
|
+
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
621
|
+
const { res, url: current } = await followRedirects(get, target, { onResponse });
|
|
521
622
|
if (!res.ok) {
|
|
522
623
|
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
|
523
624
|
}
|
|
524
625
|
assertReadableType(res.headers.get('content-type'));
|
|
525
|
-
|
|
626
|
+
assertBodySize(Number(res.headers.get('content-length')) || 0, current);
|
|
627
|
+
return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch' };
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
/**
|
|
631
|
+
* The decoded body as text, counted as it arrives so crossing the cap aborts
|
|
632
|
+
* the transfer instead of finishing it. Throwing mid-iteration cancels the
|
|
633
|
+
* stream. A proxy response has no web stream to iterate; its text() counts
|
|
634
|
+
* inside wrapNodeResponse instead.
|
|
635
|
+
* @param {any} res
|
|
636
|
+
* @param {string} url
|
|
637
|
+
* @returns {Promise<string>}
|
|
638
|
+
*/
|
|
639
|
+
export async function readBody(res, url) {
|
|
640
|
+
if (!res.body?.getReader) return res.text();
|
|
641
|
+
const chunks = [];
|
|
642
|
+
let size = 0;
|
|
643
|
+
for await (const chunk of res.body) {
|
|
644
|
+
size += chunk.byteLength;
|
|
645
|
+
assertBodySize(size, url);
|
|
646
|
+
chunks.push(chunk);
|
|
647
|
+
}
|
|
648
|
+
return Buffer.concat(chunks).toString('utf8');
|
|
526
649
|
}
|
package/src/nodedocs.js
ADDED
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Node.js docs search backend. nodejs.org has no search results page at all:
|
|
3
|
+
* the site's search box is a JavaScript modal asking a third-party service,
|
|
4
|
+
* so there is nothing for oc to fetch or call directly. But the API docs
|
|
5
|
+
* publish their entire reference as one static JSON file, all.json, much the
|
|
6
|
+
* way a Sphinx site publishes its search index, so `search` ranks that file
|
|
7
|
+
* locally: every module, class, method, property, and event heading becomes
|
|
8
|
+
* a result linking to its own anchor. The file is ~8MB (~1MB over the wire)
|
|
9
|
+
* and static, so it shares the Sphinx backend's day cache and, like the
|
|
10
|
+
* index, is never printed: what reaches the agent is the ranked list only.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { cachedFile } from './cache.js';
|
|
14
|
+
import { escapeHTML } from './sphinx.js';
|
|
15
|
+
|
|
16
|
+
const MAX_RESULTS = 20;
|
|
17
|
+
|
|
18
|
+
// The list-valued keys that hold entries with headings of their own on the
|
|
19
|
+
// page. The others (params, options) describe one signature's arguments and
|
|
20
|
+
// have no heading or anchor; a class's constructor rides in `signatures`.
|
|
21
|
+
const CHILD_KEYS = [
|
|
22
|
+
'modules', 'globals', 'miscs', 'classes', 'classMethods',
|
|
23
|
+
'signatures', 'methods', 'properties', 'events',
|
|
24
|
+
];
|
|
25
|
+
|
|
26
|
+
// What each heading's `type` is called on the results page. Anything else is
|
|
27
|
+
// a property whose type field holds its value type ({number}, {boolean}).
|
|
28
|
+
const KIND = {
|
|
29
|
+
module: 'module', misc: 'section', global: 'global', class: 'class',
|
|
30
|
+
ctor: 'constructor', classMethod: 'static method', method: 'method',
|
|
31
|
+
event: 'event', property: 'property',
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
// The docs derive a heading's anchor from its text the github-slugger way:
|
|
35
|
+
// lowercase, drop everything but letters, digits, spaces, hyphens, and
|
|
36
|
+
// underscores, then spaces become hyphens. 'fs.readFile(path[, options],
|
|
37
|
+
// callback)' is #fsreadfilepath-options-callback.
|
|
38
|
+
const slug = (text) =>
|
|
39
|
+
text.toLowerCase().replace(/[^a-z0-9 _-]/g, '').trim().replaceAll(' ', '-');
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Anything that parses but is not the docs corpus (an error page served as
|
|
43
|
+
* JSON, a moved file) fails here, which keeps it out of the cache too.
|
|
44
|
+
* @param {string} text
|
|
45
|
+
* @returns {any}
|
|
46
|
+
*/
|
|
47
|
+
export function parseAll(text) {
|
|
48
|
+
let all;
|
|
49
|
+
try {
|
|
50
|
+
all = JSON.parse(text);
|
|
51
|
+
} catch {
|
|
52
|
+
all = null;
|
|
53
|
+
}
|
|
54
|
+
if (!Array.isArray(all?.modules)) throw new Error('not the Node.js docs corpus');
|
|
55
|
+
return all;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Flatten the docs tree into the headings a query can hit. Only a top-level
|
|
60
|
+
* entry names its page (source: 'doc/api/fs.md'); everything nested under it
|
|
61
|
+
* inherits that page and contributes its own heading and anchor. A nested
|
|
62
|
+
* entry whose textRaw does not name it is not a heading (a bare 'Type:
|
|
63
|
+
* {number}' line under a property) and is skipped; its parent still stands.
|
|
64
|
+
* A heading repeated on one page is kept once: the corpus lists some methods
|
|
65
|
+
* twice for one heading, and where a page really repeats one (each stream
|
|
66
|
+
* class has an Event: 'close') the rows would be indistinguishable anyway.
|
|
67
|
+
* @param {any} all - parsed all.json
|
|
68
|
+
* @returns {{text: string, name: string, type: string, page: string, anchor: string}[]}
|
|
69
|
+
*/
|
|
70
|
+
export function buildEntries(all) {
|
|
71
|
+
const entries = [];
|
|
72
|
+
const seen = new Set();
|
|
73
|
+
const add = (node, page, top) => {
|
|
74
|
+
const text = String(node?.textRaw ?? '').replaceAll('`', '').trim();
|
|
75
|
+
const name = String(node?.name ?? '');
|
|
76
|
+
const type = String(node?.type ?? '');
|
|
77
|
+
const heading = text && name
|
|
78
|
+
&& (top || text.toLowerCase().includes(name.toLowerCase()));
|
|
79
|
+
if (heading && !seen.has(`${page}#${text}`)) {
|
|
80
|
+
seen.add(`${page}#${text}`);
|
|
81
|
+
// A module or section heading is the page's own title, so its entry
|
|
82
|
+
// links to the page top; every other heading has an anchor worth keeping.
|
|
83
|
+
const anchor = ['module', 'misc', 'global'].includes(type) ? '' : slug(text);
|
|
84
|
+
entries.push({ text, name, type, page, anchor });
|
|
85
|
+
}
|
|
86
|
+
for (const key of CHILD_KEYS) {
|
|
87
|
+
for (const child of node?.[key] ?? []) add(child, page, false);
|
|
88
|
+
}
|
|
89
|
+
};
|
|
90
|
+
for (const key of CHILD_KEYS) {
|
|
91
|
+
for (const node of all?.[key] ?? []) {
|
|
92
|
+
const page = String(node?.source ?? '')
|
|
93
|
+
.replace(/^doc\/api\//, '').replace(/\.md$/, '');
|
|
94
|
+
if (page) add(node, `${page}.html`, true);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return entries;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Rank the headings against a query. A word that is an entry's own name (the
|
|
102
|
+
* bare symbol: 'readFile') weighs most, a whole word of its heading next, a
|
|
103
|
+
* substring of it least, and an entry must match every word before any-word
|
|
104
|
+
* matching kicks in, the same policy the Sphinx backend follows.
|
|
105
|
+
* @param {ReturnType<typeof buildEntries>} entries
|
|
106
|
+
* @param {string} query
|
|
107
|
+
*/
|
|
108
|
+
export function searchEntries(entries, query) {
|
|
109
|
+
const words = [...new Set(query.toLowerCase().split(/\s+/).filter(Boolean))];
|
|
110
|
+
const scored = [];
|
|
111
|
+
for (const e of entries) {
|
|
112
|
+
const nameL = e.name.toLowerCase();
|
|
113
|
+
const textL = e.text.toLowerCase();
|
|
114
|
+
const tokens = new Set(textL.split(/[^a-z0-9_]+/));
|
|
115
|
+
let score = 0;
|
|
116
|
+
let hit = 0;
|
|
117
|
+
for (const word of words) {
|
|
118
|
+
const s = nameL === word ? 10 : tokens.has(word) ? 5 : textL.includes(word) ? 2 : 0;
|
|
119
|
+
if (s) {
|
|
120
|
+
score += s;
|
|
121
|
+
hit += 1;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
if (hit) scored.push({ e, score, hit });
|
|
125
|
+
}
|
|
126
|
+
let hits = scored.filter((s) => s.hit === words.length);
|
|
127
|
+
const partial = !hits.length && words.length > 1 && scored.length > 0;
|
|
128
|
+
if (partial) hits = scored;
|
|
129
|
+
// Among equal scores the shorter heading is the plainer API, so it leads.
|
|
130
|
+
hits.sort((a, b) => b.score - a.score
|
|
131
|
+
|| a.e.text.length - b.e.text.length
|
|
132
|
+
|| a.e.text.localeCompare(b.e.text));
|
|
133
|
+
return { words, partial, total: hits.length, hits: hits.slice(0, MAX_RESULTS).map((s) => s.e) };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The result list becomes the same small synthetic page the other search
|
|
138
|
+
* backends emit, so it distills, renders, numbers, and remembers like any
|
|
139
|
+
* fetched page and `do <n>` follows a result.
|
|
140
|
+
* @param {string} base
|
|
141
|
+
* @param {string} query
|
|
142
|
+
* @param {ReturnType<typeof searchEntries>} found
|
|
143
|
+
* @returns {string}
|
|
144
|
+
*/
|
|
145
|
+
export function resultsToHTML(base, query, found) {
|
|
146
|
+
const host = new URL(base).host;
|
|
147
|
+
const items = found.hits.map((e) => {
|
|
148
|
+
const href = new URL(e.anchor ? `${e.page}#${e.anchor}` : e.page, base).href;
|
|
149
|
+
return `<li><a href="${escapeHTML(href)}">${escapeHTML(e.text)}</a>`
|
|
150
|
+
+ ` ${escapeHTML(KIND[e.type] ?? 'property')}, in ${escapeHTML(e.page.replace(/\.html$/, ''))}</li>`;
|
|
151
|
+
});
|
|
152
|
+
const partial = found.partial ? '; no heading matches every word, so these match some' : '';
|
|
153
|
+
const summary = items.length
|
|
154
|
+
? `${found.total} heading${found.total === 1 ? '' : 's'} match in the docs' own reference,`
|
|
155
|
+
+ ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
|
|
156
|
+
: `nothing in the docs' own reference matches; try fewer or different words`;
|
|
157
|
+
return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
|
|
158
|
+
+ `<p>${summary}</p>`
|
|
159
|
+
+ (items.length ? `<ol>${items.join('')}</ol>` : '')
|
|
160
|
+
+ `</main></body></html>`;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Search the Node.js API docs. The site has no human search URL to remember,
|
|
165
|
+
* so the session keeps the docs index, the page a reader would start from.
|
|
166
|
+
* @param {string} base - docs root ending in '/', e.g. https://nodejs.org/api/
|
|
167
|
+
* @param {string} query
|
|
168
|
+
*/
|
|
169
|
+
export async function nodeSearch(base, query) {
|
|
170
|
+
if (!query.trim()) throw new Error('usage: search <query>');
|
|
171
|
+
const { data, via } = await cachedFile('nodedoc', new URL('all.json', base).href, parseAll);
|
|
172
|
+
return {
|
|
173
|
+
url: new URL('index.html', base).href,
|
|
174
|
+
html: resultsToHTML(base, query, searchEntries(buildEntries(data), query)),
|
|
175
|
+
via,
|
|
176
|
+
};
|
|
177
|
+
}
|
package/src/rdoc.js
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RDoc search backend. The Ruby docs (docs.ruby-lang.org) are built with
|
|
3
|
+
* RDoc, which like Sphinx has no search server: the generated site ships its
|
|
4
|
+
* whole search index as one static file, js/search_index.js, and matches in
|
|
5
|
+
* the visitor's browser. So `search` ranks that file locally: every class,
|
|
6
|
+
* module, method, and guide page in the index becomes a result linking to
|
|
7
|
+
* its own anchor. The file is ~3.4MB (~560KB over the wire) and static, so
|
|
8
|
+
* it lives in the same day cache the other local backends use, and what
|
|
9
|
+
* reaches the agent is the ranked result list only.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { cachedFile } from './cache.js';
|
|
13
|
+
import { escapeHTML } from './sphinx.js';
|
|
14
|
+
import { searchEntries } from './nodedocs.js';
|
|
15
|
+
|
|
16
|
+
const MAX_RESULTS = 20;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The index file is `var search_data = {...}`: JSON behind one assignment
|
|
20
|
+
* for the browser's benefit. Anything that does not parse that way, or that
|
|
21
|
+
* lacks the info rows, is not an RDoc index, which on a wrong or moved URL
|
|
22
|
+
* is the honest error, and it keeps a block page out of the cache too.
|
|
23
|
+
* @param {string} js
|
|
24
|
+
* @returns {any}
|
|
25
|
+
*/
|
|
26
|
+
export function parseRdocIndex(js) {
|
|
27
|
+
const start = js.indexOf('=');
|
|
28
|
+
if (start >= 0) {
|
|
29
|
+
try {
|
|
30
|
+
const data = JSON.parse(js.slice(start + 1));
|
|
31
|
+
if (Array.isArray(data?.index?.info)) return data;
|
|
32
|
+
} catch {}
|
|
33
|
+
}
|
|
34
|
+
throw new Error('not an RDoc search index');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Flatten the index's info rows into the entries the shared ranker scores.
|
|
39
|
+
* A row is [name, namespace, path, params, snippet]; the path's own anchor
|
|
40
|
+
* says what the row is, so 'dig' in 'Array' with anchor method-i-dig reads
|
|
41
|
+
* back as the heading a rubyist expects, Array#dig(*args). Snippets stay
|
|
42
|
+
* behind: matching on them would rank prose over the symbol asked for.
|
|
43
|
+
* @param {any} data - parsed search_index.js
|
|
44
|
+
* @returns {{text: string, name: string, kind: string, path: string}[]}
|
|
45
|
+
*/
|
|
46
|
+
export function buildRdocEntries(data) {
|
|
47
|
+
return data.index.info.map(([name, namespace, path, params]) => {
|
|
48
|
+
const p = String(path ?? '');
|
|
49
|
+
const kind = p.includes('#method-c-') ? 'class method'
|
|
50
|
+
: p.includes('#method-i-') ? 'method'
|
|
51
|
+
: /^[A-Z]/.test(String(name)) ? 'class' : 'page';
|
|
52
|
+
const text = kind === 'class method' ? `${namespace}.${name}${params}`
|
|
53
|
+
: kind === 'method' ? `${namespace}#${name}${params}`
|
|
54
|
+
: namespace ? `${namespace}::${name}` : String(name ?? '');
|
|
55
|
+
return { text, name: String(name ?? ''), kind, path: p };
|
|
56
|
+
}).filter((e) => e.text && e.path);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The result list becomes the same small synthetic page the other search
|
|
61
|
+
* backends emit, so it distills, renders, numbers, and remembers like any
|
|
62
|
+
* fetched page and `do <n>` follows a result.
|
|
63
|
+
* @param {string} base
|
|
64
|
+
* @param {string} query
|
|
65
|
+
* @param {ReturnType<typeof searchEntries>} found
|
|
66
|
+
* @returns {string}
|
|
67
|
+
*/
|
|
68
|
+
export function resultsToHTML(base, query, found) {
|
|
69
|
+
const host = new URL(base).host;
|
|
70
|
+
const items = found.hits.map((e) =>
|
|
71
|
+
`<li><a href="${escapeHTML(new URL(e.path, base).href)}">${escapeHTML(e.text)}</a>`
|
|
72
|
+
+ ` ${escapeHTML(e.kind)}</li>`);
|
|
73
|
+
const partial = found.partial ? '; no entry matches every word, so these match some' : '';
|
|
74
|
+
const summary = items.length
|
|
75
|
+
? `${found.total} entr${found.total === 1 ? 'y matches' : 'ies match'} in the docs' own index,`
|
|
76
|
+
+ ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
|
|
77
|
+
: `nothing in the docs' own index matches; try fewer or different words`;
|
|
78
|
+
return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
|
|
79
|
+
+ `<p>${summary}</p>`
|
|
80
|
+
+ (items.length ? `<ol>${items.join('')}</ol>` : '')
|
|
81
|
+
+ `</main></body></html>`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Search one RDoc site. The site has no search URL of its own to remember,
|
|
86
|
+
* so the session keeps the docs root, the page a reader would start from.
|
|
87
|
+
* @param {string} base - docs root ending in '/', e.g. https://docs.ruby-lang.org/en/3.4/
|
|
88
|
+
* @param {string} query
|
|
89
|
+
*/
|
|
90
|
+
export async function rdocSearch(base, query) {
|
|
91
|
+
if (!query.trim()) throw new Error('usage: search <query>');
|
|
92
|
+
const { data, via } = await cachedFile('rdoc', new URL('js/search_index.js', base).href, parseRdocIndex);
|
|
93
|
+
return {
|
|
94
|
+
url: new URL('index.html', base).href,
|
|
95
|
+
html: resultsToHTML(base, query, searchEntries(buildRdocEntries(data), query)),
|
|
96
|
+
via,
|
|
97
|
+
};
|
|
98
|
+
}
|
package/src/render.js
CHANGED
|
@@ -35,8 +35,9 @@ const num = (v) => v.toLocaleString('en-US');
|
|
|
35
35
|
// which is what tells a link-list page (Hacker News, search results) from a
|
|
36
36
|
// page whose only links are its own menu.
|
|
37
37
|
const CONTENT_LABEL = 25;
|
|
38
|
-
// Below this
|
|
39
|
-
//
|
|
38
|
+
// Below this a render is suspiciously thin, but thin is only a verdict when
|
|
39
|
+
// the page's own size says there should have been more. A terse page that
|
|
40
|
+
// arrived terse (a status endpoint, a one-line answer) distilled fine.
|
|
40
41
|
export const MIN_CONTENT = 25;
|
|
41
42
|
// Below this, with markup that large behind it, the fetch worked and the render
|
|
42
43
|
// did not: a real page of that weight always distills to more. A genuinely
|
|
@@ -65,7 +66,11 @@ export const contentTokens = (page) =>
|
|
|
65
66
|
* @returns {string|null}
|
|
66
67
|
*/
|
|
67
68
|
export function contentFailure(content, htmlTokens) {
|
|
68
|
-
|
|
69
|
+
// Nothing extracted is empty whatever the page weighed. Anything more is
|
|
70
|
+
// only a failure with evidence: a small page that renders small is not
|
|
71
|
+
// gated, it is small, and exit 2 on it would send an agent to a browser
|
|
72
|
+
// for a page it was already holding.
|
|
73
|
+
if (content === 0) return 'no text on the whole page';
|
|
69
74
|
if (content < THIN_CONTENT && htmlTokens > THIN_HTML) {
|
|
70
75
|
return `~${content} tokens of text out of ~${htmlTokens} of HTML`;
|
|
71
76
|
}
|
|
@@ -92,11 +97,10 @@ export const FINISH = 4;
|
|
|
92
97
|
*/
|
|
93
98
|
export function render(page, { budget = 500, from = 0 } = {}) {
|
|
94
99
|
const blocks = collapseRuns(page.blocks);
|
|
95
|
-
const head = page.title ? [from > 0 ? `# ${page.title} (continued)` : `# ${page.title}`] : [];
|
|
100
|
+
const head = page.title ? [from > 0 ? `# ${truncate(page.title)} (continued)` : `# ${truncate(page.title)}`] : [];
|
|
96
101
|
const lines = [...head];
|
|
97
102
|
let spent = estimateTokens(lines.join('\n'));
|
|
98
103
|
let hasLinks = false;
|
|
99
|
-
let hasInputs = false;
|
|
100
104
|
let i = Math.max(0, from);
|
|
101
105
|
|
|
102
106
|
// What the rest of the page would cost if it were all printed. When that is
|
|
@@ -124,7 +128,6 @@ export function render(page, { budget = 500, from = 0 } = {}) {
|
|
|
124
128
|
spent += cost;
|
|
125
129
|
lines.push(line);
|
|
126
130
|
if (block.type === 'link' || block.type === 'button') hasLinks = true;
|
|
127
|
-
if (block.type === 'input') hasInputs = true;
|
|
128
131
|
}
|
|
129
132
|
|
|
130
133
|
const rest = blocks.slice(i);
|
|
@@ -135,10 +138,18 @@ export function render(page, { budget = 500, from = 0 } = {}) {
|
|
|
135
138
|
if (rest.length) {
|
|
136
139
|
lines.push(`... ${num(rest.length)} more blocks (~${num(leftTokens)} tokens): 'oc next' for the next ~${num(budget)}, 'oc raw' for all`);
|
|
137
140
|
}
|
|
141
|
+
// An input on the page adds no action. 'fill' and 'submit' are still stubs
|
|
142
|
+
// that throw, and this footer is the line an agent trusts for what to run
|
|
143
|
+
// next, so naming one of them costs a turn and returns nothing.
|
|
144
|
+
//
|
|
145
|
+
// 'find' comes before 'read' because it is the cheaper way to go deeper: one
|
|
146
|
+
// command lands on the block that matters, where 'read' needs the right
|
|
147
|
+
// number first and 'next' pages toward it. The entry costs about three
|
|
148
|
+
// tokens on every render, and skipping one 'next' on a long page pays for
|
|
149
|
+
// a hundred of them.
|
|
138
150
|
const actions = [
|
|
139
151
|
hasLinks && 'do <n>',
|
|
140
|
-
|
|
141
|
-
hasInputs && 'submit',
|
|
152
|
+
'find <query>',
|
|
142
153
|
'read <n>',
|
|
143
154
|
rest.length && 'next',
|
|
144
155
|
'raw',
|
|
@@ -204,13 +215,13 @@ export function formatBlock(b, { full = false } = {}) {
|
|
|
204
215
|
const tag = b.n == null ? '' : `[${b.n}] `;
|
|
205
216
|
switch (b.type) {
|
|
206
217
|
case 'heading':
|
|
207
|
-
return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${tag}${b.text}`;
|
|
218
|
+
return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${tag}${full ? b.text : truncate(b.text)}`;
|
|
208
219
|
case 'link':
|
|
209
220
|
return `${tag}${full ? b.text : truncate(b.text)}`;
|
|
210
221
|
case 'button':
|
|
211
222
|
return `${tag}button "${full ? b.text : truncate(b.text)}"`;
|
|
212
223
|
case 'input':
|
|
213
|
-
return `${tag}input ${b.name} (${b.text})`;
|
|
224
|
+
return `${tag}input ${truncate(b.name ?? '')} (${truncate(b.text ?? '')})`;
|
|
214
225
|
case 'divider':
|
|
215
226
|
return b.text;
|
|
216
227
|
default:
|
package/src/session.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Sessions are plain JSON files on disk, one per name: the current URL, the
|
|
3
3
|
* distilled blocks of the page it holds, how far the last render got through
|
|
4
|
-
* them, and a short history. No daemon, no background process
|
|
4
|
+
* them, and a short history. No daemon, no background process; cookies live in
|
|
5
|
+
* a separate sidecar file (see cookies.js).
|
|
5
6
|
*
|
|
6
7
|
* The file exists so `oc do <n>` can follow a link the compact view never
|
|
7
8
|
* printed the URL of. Hiding URLs is what makes `oc open` cheap; this is what
|
|
@@ -11,18 +12,35 @@
|
|
|
11
12
|
|
|
12
13
|
import { homedir } from 'node:os';
|
|
13
14
|
import { join } from 'node:path';
|
|
14
|
-
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { mkdirSync, readFileSync, writeFileSync, chmodSync, unlinkSync } from 'node:fs';
|
|
15
16
|
|
|
16
17
|
export const DEFAULT_SESSION = 'default';
|
|
17
18
|
|
|
18
19
|
// OC_HOME relocates the whole state directory, for sandboxes, CI, and tests.
|
|
19
20
|
export const sessionDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sessions');
|
|
20
21
|
|
|
22
|
+
// A session name is interpolated straight into a filename, and the cookie
|
|
23
|
+
// sidecar it names now holds real credentials, so a name that is a path
|
|
24
|
+
// (absolute, or with a separator or '..') could write or delete a file outside
|
|
25
|
+
// the store. Names are user-facing labels, so this charset loses nothing real.
|
|
26
|
+
const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @param {string} name
|
|
30
|
+
* @returns {string} the same name, once it is known to be a safe filename
|
|
31
|
+
*/
|
|
32
|
+
export function assertSafeName(name) {
|
|
33
|
+
if (typeof name !== 'string' || name === '.' || name === '..' || !SAFE_NAME.test(name)) {
|
|
34
|
+
throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_'`);
|
|
35
|
+
}
|
|
36
|
+
return name;
|
|
37
|
+
}
|
|
38
|
+
|
|
21
39
|
/**
|
|
22
40
|
* @param {string} name
|
|
23
41
|
* @returns {string}
|
|
24
42
|
*/
|
|
25
|
-
export const sessionPath = (name) => join(sessionDir(), `${name}.json`);
|
|
43
|
+
export const sessionPath = (name) => join(sessionDir(), `${assertSafeName(name)}.json`);
|
|
26
44
|
|
|
27
45
|
// Search engines and link aggregators wrap outbound links in a tracking
|
|
28
46
|
// redirector whose landing page is a script, not content, so following one
|
|
@@ -136,7 +154,28 @@ export function handleNumbers(state) {
|
|
|
136
154
|
*/
|
|
137
155
|
export function saveSession(name, state) {
|
|
138
156
|
mkdirSync(sessionDir(), { recursive: true });
|
|
139
|
-
|
|
157
|
+
// A snapshot of an authenticated page holds that page's text, so it gets the
|
|
158
|
+
// same owner-only mode as the cookie sidecar. writeFileSync only sets the
|
|
159
|
+
// mode on create, so a snapshot left world-readable by an older version is
|
|
160
|
+
// tightened explicitly on the next save.
|
|
161
|
+
const path = sessionPath(name);
|
|
162
|
+
writeFileSync(path, JSON.stringify(state), { mode: 0o600 });
|
|
163
|
+
chmodSync(path, 0o600);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Drop a saved page. `oc logout` calls this alongside clearing the cookie jar:
|
|
168
|
+
* a snapshot taken under a login holds that page's text, so leaving it behind
|
|
169
|
+
* would make logout mean "the cookies are gone" rather than "nothing of this
|
|
170
|
+
* login remains".
|
|
171
|
+
* @param {string} name
|
|
172
|
+
*/
|
|
173
|
+
export function clearSession(name) {
|
|
174
|
+
try {
|
|
175
|
+
unlinkSync(sessionPath(name));
|
|
176
|
+
} catch {
|
|
177
|
+
// nothing saved under that name is fine
|
|
178
|
+
}
|
|
140
179
|
}
|
|
141
180
|
|
|
142
181
|
/**
|