@only-cli/oc 0.5.7 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/clis/caniuse.com.json +7 -0
- package/clis/codeberg.org.json +13 -0
- package/clis/crates.io.json +9 -0
- package/clis/discourse.nixos.org.json +8 -0
- package/clis/docs.astral.sh.json +8 -0
- package/clis/docs.pola.rs.json +6 -0
- package/clis/endoflife.date.json +8 -0
- package/clis/forums.swift.org.json +8 -0
- package/clis/github.com.json +6 -1
- package/clis/gitlab.com.json +13 -0
- package/clis/hub.docker.com.json +9 -0
- package/clis/man7.org.json +7 -0
- package/clis/npmjs.com.json +9 -0
- package/clis/osv.dev.json +6 -0
- package/clis/pypi.org.json +8 -0
- package/clis/rfc-editor.org.json +7 -0
- package/clis/rubygems.org.json +8 -0
- package/clis/squidfunk.github.io.json +6 -0
- package/clis/tldr.sh.json +9 -0
- package/clis/users.rust-lang.org.json +8 -0
- package/clis/www.mkdocs.org.json +6 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +11 -11
- package/src/act.js +14 -6
- package/src/apisearch.js +20 -10
- package/src/cache.js +9 -4
- package/src/cli.js +25 -8
- package/src/distill.js +382 -26
- package/src/fetch.js +34 -17
- package/src/mkdocs.js +190 -0
- package/src/nodedocs.js +15 -7
- package/src/render.js +7 -3
- package/src/sites.js +31 -11
package/src/fetch.js
CHANGED
|
@@ -617,20 +617,30 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
|
|
|
617
617
|
* Fetch a page.
|
|
618
618
|
* @param {string} url - with or without a scheme, https is assumed
|
|
619
619
|
* @param {{ jar?: { cookieHeaderFor(url: string): string|undefined, storeFromResponse(url: string, headers: string[]): void } }} [opts]
|
|
620
|
-
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
|
621
|
-
* final URL after redirects, the body, the HTTP status,
|
|
622
|
-
* identity got the page (impers:chrome, impers:firefox, or fetch)
|
|
620
|
+
* @returns {Promise<{url: string, html: string, status: number, via: string, type: string}>}
|
|
621
|
+
* final URL after redirects, the body, the HTTP status, which client
|
|
622
|
+
* identity got the page (impers:chrome, impers:firefox, or fetch), and the
|
|
623
|
+
* Content-Type the server declared
|
|
623
624
|
*/
|
|
624
625
|
export async function fetchPage(url, { jar } = {}) {
|
|
625
|
-
|
|
626
|
-
// A reader with no cookies for reddit.com gets the feed where the page
|
|
627
|
-
// would be a login wall; one who logged in gets the page it asked for.
|
|
628
|
-
if (!jar?.cookieHeaderFor(target)) target = redditFeedURL(target) ?? target;
|
|
626
|
+
const target = readableURL(/^https?:\/\//i.test(url) ? url : `https://${url}`, jar);
|
|
629
627
|
await assertSafeTarget(target);
|
|
630
628
|
const impers = await loadImpers();
|
|
631
629
|
return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
|
|
632
630
|
}
|
|
633
631
|
|
|
632
|
+
/**
|
|
633
|
+
* The URL to ask for in place of this one. A reader with no cookies for
|
|
634
|
+
* reddit.com gets the feed where the page would be a login wall; one who
|
|
635
|
+
* logged in gets the page it asked for. Every redirect hop goes through this
|
|
636
|
+
* too: redd.it short links and the app's /r/<sub>/s/<code> share links answer
|
|
637
|
+
* with a 301 to the post page, the very wall the feed steps around.
|
|
638
|
+
* @param {string} url
|
|
639
|
+
* @param {{ cookieHeaderFor(url: string): string|undefined }} [jar]
|
|
640
|
+
* @returns {string}
|
|
641
|
+
*/
|
|
642
|
+
const readableURL = (url, jar) => (jar?.cookieHeaderFor(url) ? url : redditFeedURL(url) ?? url);
|
|
643
|
+
|
|
634
644
|
/**
|
|
635
645
|
* Follow redirects one hop at a time, validating each destination before the
|
|
636
646
|
* next request goes out.
|
|
@@ -643,9 +653,11 @@ export async function fetchPage(url, { jar } = {}) {
|
|
|
643
653
|
* process.
|
|
644
654
|
* @param {(url: string) => Promise<any>} get - one request, redirects not followed
|
|
645
655
|
* @param {string} start
|
|
656
|
+
* @param {{onResponse?: (url: string, res: any) => void, rewrite?: (url: string) => string}} [opts] -
|
|
657
|
+
* rewrite names the URL to ask for in place of a hop's Location
|
|
646
658
|
* @returns {Promise<{res: any, url: string}>} the first non-redirect response
|
|
647
659
|
*/
|
|
648
|
-
export async function followRedirects(get, start, { onResponse } = {}) {
|
|
660
|
+
export async function followRedirects(get, start, { onResponse, rewrite } = {}) {
|
|
649
661
|
let current = start;
|
|
650
662
|
for (let i = 0; ; i++) {
|
|
651
663
|
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
|
|
@@ -654,7 +666,8 @@ export async function followRedirects(get, start, { onResponse } = {}) {
|
|
|
654
666
|
const status = res.status ?? res.statusCode ?? 0;
|
|
655
667
|
const location = res.headers.get('location');
|
|
656
668
|
if (status >= 300 && status < 400 && location) {
|
|
657
|
-
|
|
669
|
+
const next = new URL(location, current).toString();
|
|
670
|
+
current = rewrite ? rewrite(next) : next;
|
|
658
671
|
await assertSafeTarget(current);
|
|
659
672
|
continue;
|
|
660
673
|
}
|
|
@@ -687,8 +700,9 @@ function captureSetCookie(jar, url, res) {
|
|
|
687
700
|
// letting firefox through. reddit.com started doing this in 2026 (#52), so
|
|
688
701
|
// starting with chrome there would turn every read into two requests against
|
|
689
702
|
// a per-address rate limit of about ten a minute. Subdomains inherit the
|
|
690
|
-
// entry.
|
|
691
|
-
|
|
703
|
+
// entry. A redd.it short link is a 301 to a reddit.com post, and the identity
|
|
704
|
+
// is picked once for the whole chain, so it starts the same way.
|
|
705
|
+
const FIREFOX_FIRST_HOSTS = ['reddit.com', 'redd.it'];
|
|
692
706
|
|
|
693
707
|
// Reddit has sent logged-out readers of its HTML pages to a login page since
|
|
694
708
|
// June 2026 (#52), while the Atom feed beside each of those pages still
|
|
@@ -757,7 +771,7 @@ const bufferedContent = (res) => {
|
|
|
757
771
|
* @param {any} impers - the impers module (or a stand-in with a get method)
|
|
758
772
|
* @param {string} target
|
|
759
773
|
* @param {object} [jar]
|
|
760
|
-
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
|
774
|
+
* @returns {Promise<{url: string, html: string, status: number, via: string, type: string}>}
|
|
761
775
|
*/
|
|
762
776
|
export async function viaImpers(impers, target, jar) {
|
|
763
777
|
// A blocked first attempt gets one cheap retry with the other identity. An
|
|
@@ -777,9 +791,10 @@ export async function viaImpers(impers, target, jar) {
|
|
|
777
791
|
headers: jarHeaders(jar, url, {}),
|
|
778
792
|
});
|
|
779
793
|
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
794
|
+
const rewrite = (url) => readableURL(url, jar);
|
|
780
795
|
const attempt = async (impersonate) => {
|
|
781
796
|
try {
|
|
782
|
-
const { res } = await followRedirects(asking(impersonate), target, { onResponse });
|
|
797
|
+
const { res } = await followRedirects(asking(impersonate), target, { onResponse, rewrite });
|
|
783
798
|
return { res, status: res.status ?? res.statusCode ?? 0 };
|
|
784
799
|
} catch (err) {
|
|
785
800
|
if (err?.name !== 'ImpersonateError') throw err;
|
|
@@ -810,11 +825,11 @@ export async function viaImpers(impers, target, jar) {
|
|
|
810
825
|
// this is the only size this transport ever really checks. Refused before
|
|
811
826
|
// the decode so an oversized body is not allocated twice over.
|
|
812
827
|
assertBodySize(bytes.length, target);
|
|
813
|
-
return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via };
|
|
828
|
+
return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via, type: res.headers.get('content-type') ?? '' };
|
|
814
829
|
}
|
|
815
830
|
const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
|
|
816
831
|
assertBodySize(html.length, target);
|
|
817
|
-
return { url: res.url ?? target, html, status, via };
|
|
832
|
+
return { url: res.url ?? target, html, status, via, type: res.headers.get('content-type') ?? '' };
|
|
818
833
|
}
|
|
819
834
|
|
|
820
835
|
async function viaFetch(target, jar) {
|
|
@@ -826,13 +841,15 @@ async function viaFetch(target, jar) {
|
|
|
826
841
|
: fetch(url, { redirect: 'manual', headers });
|
|
827
842
|
};
|
|
828
843
|
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
829
|
-
const
|
|
844
|
+
const rewrite = (url) => readableURL(url, jar);
|
|
845
|
+
const { res, url: current } = await followRedirects(get, target, { onResponse, rewrite });
|
|
830
846
|
if (!res.ok) {
|
|
831
847
|
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
|
832
848
|
}
|
|
833
849
|
assertReadableType(res.headers.get('content-type'));
|
|
834
850
|
assertBodySize(Number(res.headers.get('content-length')) || 0, current);
|
|
835
|
-
return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch'
|
|
851
|
+
return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch',
|
|
852
|
+
type: res.headers.get('content-type') ?? '' };
|
|
836
853
|
}
|
|
837
854
|
|
|
838
855
|
/**
|
package/src/mkdocs.js
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MkDocs search backend. MkDocs, and Material for MkDocs on top of it, has no
|
|
3
|
+
* search server either: the build writes search/search_index.json, one row
|
|
4
|
+
* per page and per section with its title and body text, and the browser
|
|
5
|
+
* searches it with lunr. So `search` ranks that file locally, under the same
|
|
6
|
+
* day cache as the other local backends, and prints only the result list.
|
|
7
|
+
*
|
|
8
|
+
* A section title here is a short prose heading ("Installation", "Adding a
|
|
9
|
+
* dependency"), not an API symbol, so titles alone miss most questions a
|
|
10
|
+
* reader asks in their own words. A match in a title outweighs one in the
|
|
11
|
+
* body, and a section matched on its body shows a line of the text around
|
|
12
|
+
* the match, so the agent can pick one without opening it.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { cachedFile } from './cache.js';
|
|
16
|
+
import { escapeHTML } from './sphinx.js';
|
|
17
|
+
|
|
18
|
+
const MAX_RESULTS = 20;
|
|
19
|
+
const EXCERPT = 110;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* The index is `{config, docs: [{location, title, text}]}`. Anything else is
|
|
23
|
+
* not an MkDocs index, which on a site that moved to another generator (as
|
|
24
|
+
* FastAPI's did) is the honest error, and it keeps a 404 page out of the
|
|
25
|
+
* cache.
|
|
26
|
+
* @param {string} json
|
|
27
|
+
* @returns {{docs: {location: string, title: string, text: string}[]}}
|
|
28
|
+
*/
|
|
29
|
+
export function parseMkdocsIndex(json) {
|
|
30
|
+
try {
|
|
31
|
+
const data = JSON.parse(json);
|
|
32
|
+
if (Array.isArray(data?.docs) && data.docs.every((d) => typeof d?.location === 'string')) return data;
|
|
33
|
+
} catch {}
|
|
34
|
+
throw new Error('not an MkDocs search index');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Material stores the body as HTML; plain MkDocs stores text. Either way the
|
|
38
|
+
// ranker and the excerpt want words.
|
|
39
|
+
const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', '#39': "'", nbsp: ' ' };
|
|
40
|
+
const plain = (s) => String(s ?? '').replace(/<[^>]*>/g, ' ')
|
|
41
|
+
.replace(/&(amp|lt|gt|quot|#39|nbsp);/g, (_, e) => ENTITIES[e]).replace(/\s+/g, ' ').trim();
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* One entry per row. The page a section sits on is named by the page row's
|
|
45
|
+
* title, so a result reads "Installation, in Getting Started".
|
|
46
|
+
* @param {ReturnType<typeof parseMkdocsIndex>} data
|
|
47
|
+
* @returns {{text: string, name: string, kind: string, path: string, page: string, body: string}[]}
|
|
48
|
+
*/
|
|
49
|
+
export function buildMkdocsEntries(data) {
|
|
50
|
+
const pages = new Map();
|
|
51
|
+
for (const d of data.docs) if (!d.location.includes('#')) pages.set(d.location, plain(d.title));
|
|
52
|
+
return data.docs.map((d) => {
|
|
53
|
+
const title = plain(d.title);
|
|
54
|
+
const [page, anchor] = d.location.split('#');
|
|
55
|
+
return {
|
|
56
|
+
text: title,
|
|
57
|
+
name: title,
|
|
58
|
+
kind: anchor ? 'section' : 'page',
|
|
59
|
+
path: d.location,
|
|
60
|
+
page: anchor ? pages.get(page) ?? page : '',
|
|
61
|
+
body: plain(d.text),
|
|
62
|
+
};
|
|
63
|
+
}).filter((e) => e.text);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
// Words that carry no topic of their own. A question typed in prose ("how
|
|
67
|
+
// do I add a dependency") would otherwise match every section on "a".
|
|
68
|
+
const STOP = new Set(['a', 'an', 'the', 'to', 'of', 'in', 'on', 'for', 'and', 'or', 'is', 'how', 'do', 'i', 'my', 'with', 'what', 'can']);
|
|
69
|
+
// "dependency" should find "Managing dependencies", so a word also matches
|
|
70
|
+
// by its stem once a plural or verb ending is off.
|
|
71
|
+
const stem = (w) => (w.length > 4 ? w.replace(/(ies|es|s|y|ing|ed)$/, '') : w);
|
|
72
|
+
// A title, a body, and the query are all cut into words at the same places,
|
|
73
|
+
// so "pyproject.toml", "uv.lock", "--group", or a question's closing "?"
|
|
74
|
+
// meet the words the index holds, and a letter outside ASCII stays in its
|
|
75
|
+
// word. A contraction's tail goes first: "what's" is "what", not "what" and
|
|
76
|
+
// a stray "s" every section would have to match.
|
|
77
|
+
const wordsOf = (s) => s.replace(/(?<=\p{L})['’](s|t|re|ve|ll|d|m)(?!\p{L})/gu, '')
|
|
78
|
+
.split(/[^\p{L}\p{N}_]+/u).filter(Boolean);
|
|
79
|
+
const tokensOf = (s) => new Set(wordsOf(s));
|
|
80
|
+
// A stem matches a word it starts, give or take an ending: "add" finds
|
|
81
|
+
// "adding" but not "additional".
|
|
82
|
+
const rooted = (tokens, root) => [...tokens].some((t) => t.startsWith(root) && t.length - root.length <= 3);
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Rank sections for a query. A word in a section's title weighs far more
|
|
86
|
+
* than one in its body, a whole word more than a stem, and a section must
|
|
87
|
+
* hold every word before sections holding only some are shown, and one
|
|
88
|
+
* whose title holds every word leads the ones matched on their body. Among equal
|
|
89
|
+
* scores the shorter body leads: a page row's body is every section on it,
|
|
90
|
+
* so the section that is about the words beats the page that mentions them.
|
|
91
|
+
* @param {ReturnType<typeof buildMkdocsEntries>} entries
|
|
92
|
+
* @param {string} query
|
|
93
|
+
*/
|
|
94
|
+
export function searchMkdocs(entries, query) {
|
|
95
|
+
const all = [...new Set(wordsOf(query.toLowerCase()))];
|
|
96
|
+
const topical = all.filter((w) => !STOP.has(w));
|
|
97
|
+
const words = topical.length ? topical : all;
|
|
98
|
+
const scored = [];
|
|
99
|
+
for (const e of entries) {
|
|
100
|
+
const title = e.text.toLowerCase();
|
|
101
|
+
const body = e.body.toLowerCase();
|
|
102
|
+
const titleTokens = tokensOf(title);
|
|
103
|
+
const bodyTokens = tokensOf(body);
|
|
104
|
+
let score = 0;
|
|
105
|
+
let hit = 0;
|
|
106
|
+
let inTitle = 0;
|
|
107
|
+
for (const word of words) {
|
|
108
|
+
const root = stem(word);
|
|
109
|
+
const s = titleTokens.has(word) ? 10 : rooted(titleTokens, root) ? 6
|
|
110
|
+
: bodyTokens.has(word) ? 2 : rooted(bodyTokens, root) ? 1 : 0;
|
|
111
|
+
if (s) {
|
|
112
|
+
score += s;
|
|
113
|
+
hit += 1;
|
|
114
|
+
if (s >= 6) inTitle += 1;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
if (hit) scored.push({ e, score, hit, titled: inTitle === words.length });
|
|
118
|
+
}
|
|
119
|
+
let hits = scored.filter((s) => s.hit === words.length);
|
|
120
|
+
const partial = !hits.length && words.length > 1 && scored.length > 0;
|
|
121
|
+
if (partial) hits = scored.filter((s) => s.hit === Math.max(...scored.map((x) => x.hit)));
|
|
122
|
+
hits.sort((a, b) => b.hit - a.hit || Number(b.titled) - Number(a.titled) || b.score - a.score
|
|
123
|
+
|| a.e.body.length - b.e.body.length);
|
|
124
|
+
const top = hits.slice(0, MAX_RESULTS);
|
|
125
|
+
return {
|
|
126
|
+
words,
|
|
127
|
+
partial,
|
|
128
|
+
total: hits.length,
|
|
129
|
+
hits: top.map((s) => s.e),
|
|
130
|
+
// Only a section whose title does not already say why it matched gets
|
|
131
|
+
// a line of its text.
|
|
132
|
+
body: new Set(top.filter((s) => !s.titled).map((s) => s.e)),
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The text around the first query word, clipped at word boundaries. */
|
|
137
|
+
function excerpt(body, words) {
|
|
138
|
+
const lower = body.toLowerCase();
|
|
139
|
+
const at = Math.min(...words.map((w) => lower.indexOf(stem(w))).filter((i) => i >= 0));
|
|
140
|
+
if (!Number.isFinite(at)) return '';
|
|
141
|
+
let start = Math.max(0, at - 30);
|
|
142
|
+
if (start) start = body.indexOf(' ', start) + 1;
|
|
143
|
+
let end = Math.min(body.length, start + EXCERPT);
|
|
144
|
+
if (end < body.length) end = body.lastIndexOf(' ', end);
|
|
145
|
+
return `${start ? '...' : ''}${body.slice(start, end)}${end < body.length ? '...' : ''}`;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The result list as the same small synthetic page the other search
|
|
150
|
+
* backends emit, so `do <n>` follows a result.
|
|
151
|
+
* @param {string} base
|
|
152
|
+
* @param {string} query
|
|
153
|
+
* @param {ReturnType<typeof searchMkdocs>} found
|
|
154
|
+
* @returns {string}
|
|
155
|
+
*/
|
|
156
|
+
export function resultsToHTML(base, query, found) {
|
|
157
|
+
const host = new URL(base).host;
|
|
158
|
+
// Reference pages repeat their section titles ("Example", "Fix safety"),
|
|
159
|
+
// and identical link text on one page reads as repeated controls and is
|
|
160
|
+
// hidden, so a title that repeats in the list carries its page in the link.
|
|
161
|
+
const count = new Map();
|
|
162
|
+
for (const e of found.hits) count.set(e.text, (count.get(e.text) ?? 0) + 1);
|
|
163
|
+
const items = found.hits.map((e) => {
|
|
164
|
+
const shared = count.get(e.text) > 1 && e.page;
|
|
165
|
+
const label = shared ? `${e.text} (${e.page})` : e.text;
|
|
166
|
+
const where = e.page && !shared ? `, in ${escapeHTML(e.page)}` : '';
|
|
167
|
+
const text = found.body.has(e) ? `: ${escapeHTML(excerpt(e.body, found.words))}` : '';
|
|
168
|
+
return `<li><a href="${escapeHTML(new URL(e.path, base).href)}">${escapeHTML(label)}</a> ${e.kind}${where}${text}</li>`;
|
|
169
|
+
});
|
|
170
|
+
const partial = found.partial ? '; no section matches every word, so these match some' : '';
|
|
171
|
+
const summary = items.length
|
|
172
|
+
? `${found.total} section${found.total === 1 ? ' matches' : 's match'} in the docs' own index,`
|
|
173
|
+
+ ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
|
|
174
|
+
: `nothing in the docs' own index matches; try fewer or different words`;
|
|
175
|
+
return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
|
|
176
|
+
+ `<p>${summary}</p>`
|
|
177
|
+
+ (items.length ? `<ol>${items.join('')}</ol>` : '')
|
|
178
|
+
+ `</main></body></html>`;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Search one MkDocs site. The session keeps the docs root as the page URL.
|
|
183
|
+
* @param {string} base - docs root ending in '/', e.g. https://docs.astral.sh/uv/
|
|
184
|
+
* @param {string} query
|
|
185
|
+
*/
|
|
186
|
+
export async function mkdocsSearch(base, query) {
|
|
187
|
+
if (!query.trim()) throw new Error('usage: search <query>');
|
|
188
|
+
const { data, via } = await cachedFile('mkdocs', new URL('search/search_index.json', base).href, parseMkdocsIndex);
|
|
189
|
+
return { url: base, html: resultsToHTML(base, query, searchMkdocs(buildMkdocsEntries(data), query)), via };
|
|
190
|
+
}
|
package/src/nodedocs.js
CHANGED
|
@@ -59,8 +59,8 @@ export function parseAll(text) {
|
|
|
59
59
|
* Flatten the docs tree into the headings a query can hit. Only a top-level
|
|
60
60
|
* entry names its page (source: 'doc/api/fs.md'); everything nested under it
|
|
61
61
|
* inherits that page and contributes its own heading and anchor. A nested
|
|
62
|
-
* entry whose textRaw
|
|
63
|
-
*
|
|
62
|
+
* entry whose textRaw is a property's type line, or does not name the entry,
|
|
63
|
+
* is not a heading and is skipped; its parent still stands.
|
|
64
64
|
* A heading repeated on one page is kept once: the corpus lists some methods
|
|
65
65
|
* twice for one heading, and where a page really repeats one (each stream
|
|
66
66
|
* class has an Event: 'close') the rows would be indistinguishable anyway.
|
|
@@ -72,15 +72,23 @@ export function buildEntries(all) {
|
|
|
72
72
|
const seen = new Set();
|
|
73
73
|
const add = (node, page, top) => {
|
|
74
74
|
const text = String(node?.textRaw ?? '').replaceAll('`', '').trim();
|
|
75
|
-
|
|
75
|
+
// A section's name keeps the heading's backticks ('`fetch`', every N-API
|
|
76
|
+
// function), so the two are compared with those stripped from both.
|
|
77
|
+
const name = String(node?.name ?? '').replaceAll('`', '');
|
|
76
78
|
const type = String(node?.type ?? '');
|
|
77
|
-
|
|
79
|
+
// A property's value line, 'Type: {string}', 'Returns: {Object}', or a
|
|
80
|
+
// bare '{number}', stands where its heading would; it is never one, even
|
|
81
|
+
// when the property's own name is a word in it (blob.type,
|
|
82
|
+
// server.listening).
|
|
83
|
+
const typeLine = /^(?:\w+: )?\{/.test(text);
|
|
84
|
+
const heading = text && name && !typeLine
|
|
78
85
|
&& (top || text.toLowerCase().includes(name.toLowerCase()));
|
|
79
86
|
if (heading && !seen.has(`${page}#${text}`)) {
|
|
80
87
|
seen.add(`${page}#${text}`);
|
|
81
|
-
//
|
|
82
|
-
// links to the page top; every other heading
|
|
83
|
-
|
|
88
|
+
// The top-level module or section heading is the page's own title, so
|
|
89
|
+
// its entry links to the page top; every other heading, a section
|
|
90
|
+
// within the page included, has an anchor worth keeping.
|
|
91
|
+
const anchor = top && ['module', 'misc', 'global'].includes(type) ? '' : slug(text);
|
|
84
92
|
entries.push({ text, name, type, page, anchor });
|
|
85
93
|
}
|
|
86
94
|
for (const key of CHILD_KEYS) {
|
package/src/render.js
CHANGED
|
@@ -128,8 +128,9 @@ export function render(page, { budget = 500, from = 0 } = {}) {
|
|
|
128
128
|
spent += cost;
|
|
129
129
|
lines.push(line);
|
|
130
130
|
// A heading that is a link (search result titles) is what do follows, so
|
|
131
|
-
// it has to count here or the footer omits do on a page of only those.
|
|
132
|
-
|
|
131
|
+
// it has to count here or the footer omits do on a page of only those. A
|
|
132
|
+
// link with no href is a fragment or mailto:, which do only reads.
|
|
133
|
+
if (block.type === 'button' || block.href) hasLinks = true;
|
|
133
134
|
}
|
|
134
135
|
|
|
135
136
|
const rest = blocks.slice(i);
|
|
@@ -249,7 +250,10 @@ const truncate = (s) => {
|
|
|
249
250
|
// instead of falling through to the sentence rule below.
|
|
250
251
|
const line = s.slice(0, TEXT_CAP).lastIndexOf('\n');
|
|
251
252
|
if (line >= TEXT_CAP * SENTENCE_FLOOR) return `${s.slice(0, line)} ... +${num(s.length - line)} chars`;
|
|
252
|
-
|
|
253
|
+
// A character past U+FFFF (an emoji, a rare CJK ideograph) is two UTF-16
|
|
254
|
+
// units. A cut between them printed half of one, which a terminal shows as
|
|
255
|
+
// the replacement character, so the cut steps back to before it.
|
|
256
|
+
let cut = /[\uD800-\uDBFF]/.test(s[TEXT_CAP - 1]) ? TEXT_CAP - 1 : TEXT_CAP;
|
|
253
257
|
for (const m of s.slice(0, TEXT_CAP).matchAll(SENTENCE_END)) {
|
|
254
258
|
const end = (m.index ?? 0) + m[0].length;
|
|
255
259
|
if (end >= TEXT_CAP * SENTENCE_FLOOR) cut = end;
|
package/src/sites.js
CHANGED
|
@@ -3,9 +3,10 @@
|
|
|
3
3
|
* directly, so `oc hn item 4711` gets there without the agent knowing that
|
|
4
4
|
* Hacker News spells it /item?id=. A shortcut is almost always a URL: it
|
|
5
5
|
* resolves to one and hands off to the same fetch and render path `oc open`
|
|
6
|
-
* uses, so nothing here can change what a page costs or how it reads
|
|
6
|
+
* uses, so nothing here can change what a page costs or how it reads, except
|
|
7
|
+
* that a shortcut at a JSON endpoint may name the fields worth showing. The
|
|
7
8
|
* other shapes are searches cli.js runs itself and renders like any other
|
|
8
|
-
* page: `sphinx` and `
|
|
9
|
+
* page: `sphinx`, `rdoc`, and `mkdocs`, for docs sites whose search only exists as a
|
|
9
10
|
* static index file, `nodedoc`, for the Node.js API docs, which ship their
|
|
10
11
|
* reference the same way, and `api`, for a site whose search answers as
|
|
11
12
|
* JSON.
|
|
@@ -38,9 +39,20 @@ const ALIASES = {
|
|
|
38
39
|
gcp: 'cloud.google.com',
|
|
39
40
|
learn: 'learn.microsoft.com',
|
|
40
41
|
wiki: 'wikipedia.org',
|
|
42
|
+
npm: 'npmjs.com',
|
|
43
|
+
pip: 'pypi.org',
|
|
44
|
+
cargo: 'crates.io',
|
|
45
|
+
gem: 'rubygems.org',
|
|
46
|
+
eol: 'endoflife.date',
|
|
47
|
+
man: 'man7.org',
|
|
48
|
+
rfc: 'rfc-editor.org',
|
|
49
|
+
gl: 'gitlab.com',
|
|
50
|
+
material: 'squidfunk.github.io',
|
|
51
|
+
polars: 'docs.pola.rs',
|
|
52
|
+
urlo: 'users.rust-lang.org',
|
|
41
53
|
};
|
|
42
54
|
|
|
43
|
-
/** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
|
|
55
|
+
/** @typedef {{open?: string, keep?: string[], list?: string, sphinx?: string, nodedoc?: string, rdoc?: string, mkdocs?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
|
|
44
56
|
/** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */
|
|
45
57
|
|
|
46
58
|
/** @type {Map<string, Site>|null} */
|
|
@@ -97,14 +109,16 @@ const verbs = (site) =>
|
|
|
97
109
|
* instead, since the agent has the right site and only needs the verb list.
|
|
98
110
|
* @param {string} name
|
|
99
111
|
* @param {string[]} args
|
|
100
|
-
* @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
|
|
112
|
+
* @returns {{url?: string, view?: {keep: string[], list?: string}, sphinx?: string, nodedoc?: string, rdoc?: string, mkdocs?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
|
|
101
113
|
*/
|
|
102
114
|
export function resolveSite(name, args) {
|
|
103
115
|
const site = sites().get(name.toLowerCase());
|
|
104
116
|
if (!site) return null;
|
|
105
117
|
const [verb, ...rest] = args;
|
|
106
118
|
if (!verb) throw new Error(`usage: oc ${name} <verb>, one of: ${verbs(site)}`);
|
|
107
|
-
|
|
119
|
+
// commands comes from JSON.parse, so a plain lookup would also find
|
|
120
|
+
// inherited Object members such as toString, which name no URL.
|
|
121
|
+
const def = Object.hasOwn(site.commands, verb) ? site.commands[verb] : undefined;
|
|
108
122
|
if (!def) throw new Error(`'${verb}' is not a ${site.domain} shortcut, try: ${verbs(site)}`);
|
|
109
123
|
|
|
110
124
|
const need = def.args ?? [];
|
|
@@ -119,7 +133,7 @@ export function resolveSite(name, args) {
|
|
|
119
133
|
// back whole for cli.js to run against the site's own search. The local
|
|
120
134
|
// backends need only their docs root; the API shape needs its whole
|
|
121
135
|
// definition, since it names the endpoint and the response fields.
|
|
122
|
-
for (const kind of ['sphinx', 'nodedoc', 'rdoc']) {
|
|
136
|
+
for (const kind of ['sphinx', 'nodedoc', 'rdoc', 'mkdocs']) {
|
|
123
137
|
if (def[kind]) {
|
|
124
138
|
return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
|
|
125
139
|
}
|
|
@@ -130,7 +144,11 @@ export function resolveSite(name, args) {
|
|
|
130
144
|
const url = need.reduce(
|
|
131
145
|
(open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)),
|
|
132
146
|
def.open);
|
|
133
|
-
|
|
147
|
+
// A shortcut at a JSON endpoint may say which fields to show. The choice
|
|
148
|
+
// rides along to the render and touches nothing else: the URL is fetched,
|
|
149
|
+
// cached, and remembered exactly as `oc open` would.
|
|
150
|
+
const view = def.keep?.length ? { keep: def.keep, list: def.list } : undefined;
|
|
151
|
+
return { url, view, domain: site.domain, command: verb };
|
|
134
152
|
}
|
|
135
153
|
|
|
136
154
|
/**
|
|
@@ -148,10 +166,12 @@ function encode(value, template, arg) {
|
|
|
148
166
|
// A path slot that starts with @ is a copied handle (@name, @channel).
|
|
149
167
|
// encodeURIComponent would turn it into %40 and miss the user, and YouTube's
|
|
150
168
|
// template already has the @ in the URL. Search queries keep it, so a
|
|
151
|
-
// mention in a query is still the thing being searched for.
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
169
|
+
// mention in a query is still the thing being searched for. A handle never
|
|
170
|
+
// contains a slash, so @scope/name is a scoped npm package, not a handle:
|
|
171
|
+
// stripping its @ asked the registry for a different package.
|
|
172
|
+
const handle = !query && value.startsWith('@') && value.length > 1 && !value.includes('/');
|
|
173
|
+
const encoded = encodeURIComponent(handle ? value.slice(1) : value);
|
|
174
|
+
return query ? encoded : encoded.replaceAll('%2F', '/').replaceAll('%40', '@');
|
|
155
175
|
}
|
|
156
176
|
|
|
157
177
|
/**
|