@only-cli/oc 0.5.8 → 0.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/mkdocs.js ADDED
@@ -0,0 +1,190 @@
1
+ /**
2
+ * MkDocs search backend. MkDocs, and Material for MkDocs on top of it, has no
3
+ * search server either: the build writes search/search_index.json, one row
4
+ * per page and per section with its title and body text, and the browser
5
+ * searches it with lunr. So `search` ranks that file locally, under the same
6
+ * day cache as the other local backends, and prints only the result list.
7
+ *
8
+ * A section title here is a short prose heading ("Installation", "Adding a
9
+ * dependency"), not an API symbol, so titles alone miss most questions a
10
+ * reader asks in their own words. A match in a title outweighs one in the
11
+ * body, and a section matched on its body shows a line of the text around
12
+ * the match, so the agent can pick one without opening it.
13
+ */
14
+
15
+ import { cachedFile } from './cache.js';
16
+ import { escapeHTML } from './sphinx.js';
17
+
18
+ const MAX_RESULTS = 20;
19
+ const EXCERPT = 110;
20
+
21
+ /**
22
+ * The index is `{config, docs: [{location, title, text}]}`. Anything else is
23
+ * not an MkDocs index, which on a site that moved to another generator (as
24
+ * FastAPI's did) is the honest error, and it keeps a 404 page out of the
25
+ * cache.
26
+ * @param {string} json
27
+ * @returns {{docs: {location: string, title: string, text: string}[]}}
28
+ */
29
+ export function parseMkdocsIndex(json) {
30
+ try {
31
+ const data = JSON.parse(json);
32
+ if (Array.isArray(data?.docs) && data.docs.every((d) => typeof d?.location === 'string')) return data;
33
+ } catch {}
34
+ throw new Error('not an MkDocs search index');
35
+ }
36
+
37
+ // Material stores the body as HTML; plain MkDocs stores text. Either way the
38
+ // ranker and the excerpt want words.
39
+ const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', '#39': "'", nbsp: ' ' };
40
+ const plain = (s) => String(s ?? '').replace(/<[^>]*>/g, ' ')
41
+ .replace(/&(amp|lt|gt|quot|#39|nbsp);/g, (_, e) => ENTITIES[e]).replace(/\s+/g, ' ').trim();
42
+
43
+ /**
44
+ * One entry per row. The page a section sits on is named by the page row's
45
+ * title, so a result reads "Installation, in Getting Started".
46
+ * @param {ReturnType<typeof parseMkdocsIndex>} data
47
+ * @returns {{text: string, name: string, kind: string, path: string, page: string, body: string}[]}
48
+ */
49
+ export function buildMkdocsEntries(data) {
50
+ const pages = new Map();
51
+ for (const d of data.docs) if (!d.location.includes('#')) pages.set(d.location, plain(d.title));
52
+ return data.docs.map((d) => {
53
+ const title = plain(d.title);
54
+ const [page, anchor] = d.location.split('#');
55
+ return {
56
+ text: title,
57
+ name: title,
58
+ kind: anchor ? 'section' : 'page',
59
+ path: d.location,
60
+ page: anchor ? pages.get(page) ?? page : '',
61
+ body: plain(d.text),
62
+ };
63
+ }).filter((e) => e.text);
64
+ }
65
+
66
+ // Words that carry no topic of their own. A question typed in prose ("how
67
+ // do I add a dependency") would otherwise match every section on "a".
68
+ const STOP = new Set(['a', 'an', 'the', 'to', 'of', 'in', 'on', 'for', 'and', 'or', 'is', 'how', 'do', 'i', 'my', 'with', 'what', 'can']);
69
+ // "dependency" should find "Managing dependencies", so a word also matches
70
+ // by its stem once a plural or verb ending is off.
71
+ const stem = (w) => (w.length > 4 ? w.replace(/(ies|es|s|y|ing|ed)$/, '') : w);
72
+ // A title, a body, and the query are all cut into words at the same places,
73
+ // so "pyproject.toml", "uv.lock", "--group", or a question's closing "?"
74
+ // meet the words the index holds, and a letter outside ASCII stays in its
75
+ // word. A contraction's tail goes first: "what's" is "what", not "what" and
76
+ // a stray "s" every section would have to match.
77
+ const wordsOf = (s) => s.replace(/(?<=\p{L})['’](s|t|re|ve|ll|d|m)(?!\p{L})/gu, '')
78
+ .split(/[^\p{L}\p{N}_]+/u).filter(Boolean);
79
+ const tokensOf = (s) => new Set(wordsOf(s));
80
+ // A stem matches a word it starts, give or take an ending: "add" finds
81
+ // "adding" but not "additional".
82
+ const rooted = (tokens, root) => [...tokens].some((t) => t.startsWith(root) && t.length - root.length <= 3);
83
+
84
+ /**
85
+ * Rank sections for a query. A word in a section's title weighs far more
86
+ * than one in its body, a whole word more than a stem, and a section must
87
+ * hold every word before sections holding only some are shown, and one
88
+ * whose title holds every word leads the ones matched on their body. Among equal
89
+ * scores the shorter body leads: a page row's body is every section on it,
90
+ * so the section that is about the words beats the page that mentions them.
91
+ * @param {ReturnType<typeof buildMkdocsEntries>} entries
92
+ * @param {string} query
93
+ */
94
+ export function searchMkdocs(entries, query) {
95
+ const all = [...new Set(wordsOf(query.toLowerCase()))];
96
+ const topical = all.filter((w) => !STOP.has(w));
97
+ const words = topical.length ? topical : all;
98
+ const scored = [];
99
+ for (const e of entries) {
100
+ const title = e.text.toLowerCase();
101
+ const body = e.body.toLowerCase();
102
+ const titleTokens = tokensOf(title);
103
+ const bodyTokens = tokensOf(body);
104
+ let score = 0;
105
+ let hit = 0;
106
+ let inTitle = 0;
107
+ for (const word of words) {
108
+ const root = stem(word);
109
+ const s = titleTokens.has(word) ? 10 : rooted(titleTokens, root) ? 6
110
+ : bodyTokens.has(word) ? 2 : rooted(bodyTokens, root) ? 1 : 0;
111
+ if (s) {
112
+ score += s;
113
+ hit += 1;
114
+ if (s >= 6) inTitle += 1;
115
+ }
116
+ }
117
+ if (hit) scored.push({ e, score, hit, titled: inTitle === words.length });
118
+ }
119
+ let hits = scored.filter((s) => s.hit === words.length);
120
+ const partial = !hits.length && words.length > 1 && scored.length > 0;
121
+ if (partial) hits = scored.filter((s) => s.hit === Math.max(...scored.map((x) => x.hit)));
122
+ hits.sort((a, b) => b.hit - a.hit || Number(b.titled) - Number(a.titled) || b.score - a.score
123
+ || a.e.body.length - b.e.body.length);
124
+ const top = hits.slice(0, MAX_RESULTS);
125
+ return {
126
+ words,
127
+ partial,
128
+ total: hits.length,
129
+ hits: top.map((s) => s.e),
130
+ // Only a section whose title does not already say why it matched gets
131
+ // a line of its text.
132
+ body: new Set(top.filter((s) => !s.titled).map((s) => s.e)),
133
+ };
134
+ }
135
+
136
+ /** The text around the first query word, clipped at word boundaries. */
137
+ function excerpt(body, words) {
138
+ const lower = body.toLowerCase();
139
+ const at = Math.min(...words.map((w) => lower.indexOf(stem(w))).filter((i) => i >= 0));
140
+ if (!Number.isFinite(at)) return '';
141
+ let start = Math.max(0, at - 30);
142
+ if (start) start = body.indexOf(' ', start) + 1;
143
+ let end = Math.min(body.length, start + EXCERPT);
144
+ if (end < body.length) end = body.lastIndexOf(' ', end);
145
+ return `${start ? '...' : ''}${body.slice(start, end)}${end < body.length ? '...' : ''}`;
146
+ }
147
+
148
+ /**
149
+ * The result list as the same small synthetic page the other search
150
+ * backends emit, so `do <n>` follows a result.
151
+ * @param {string} base
152
+ * @param {string} query
153
+ * @param {ReturnType<typeof searchMkdocs>} found
154
+ * @returns {string}
155
+ */
156
+ export function resultsToHTML(base, query, found) {
157
+ const host = new URL(base).host;
158
+ // Reference pages repeat their section titles ("Example", "Fix safety"),
159
+ // and identical link text on one page reads as repeated controls and is
160
+ // hidden, so a title that repeats in the list carries its page in the link.
161
+ const count = new Map();
162
+ for (const e of found.hits) count.set(e.text, (count.get(e.text) ?? 0) + 1);
163
+ const items = found.hits.map((e) => {
164
+ const shared = count.get(e.text) > 1 && e.page;
165
+ const label = shared ? `${e.text} (${e.page})` : e.text;
166
+ const where = e.page && !shared ? `, in ${escapeHTML(e.page)}` : '';
167
+ const text = found.body.has(e) ? `: ${escapeHTML(excerpt(e.body, found.words))}` : '';
168
+ return `<li><a href="${escapeHTML(new URL(e.path, base).href)}">${escapeHTML(label)}</a> ${e.kind}${where}${text}</li>`;
169
+ });
170
+ const partial = found.partial ? '; no section matches every word, so these match some' : '';
171
+ const summary = items.length
172
+ ? `${found.total} section${found.total === 1 ? ' matches' : 's match'} in the docs' own index,`
173
+ + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
174
+ : `nothing in the docs' own index matches; try fewer or different words`;
175
+ return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
176
+ + `<p>${summary}</p>`
177
+ + (items.length ? `<ol>${items.join('')}</ol>` : '')
178
+ + `</main></body></html>`;
179
+ }
180
+
181
+ /**
182
+ * Search one MkDocs site. The session keeps the docs root as the page URL.
183
+ * @param {string} base - docs root ending in '/', e.g. https://docs.astral.sh/uv/
184
+ * @param {string} query
185
+ */
186
+ export async function mkdocsSearch(base, query) {
187
+ if (!query.trim()) throw new Error('usage: search <query>');
188
+ const { data, via } = await cachedFile('mkdocs', new URL('search/search_index.json', base).href, parseMkdocsIndex);
189
+ return { url: base, html: resultsToHTML(base, query, searchMkdocs(buildMkdocsEntries(data), query)), via };
190
+ }
package/src/nodedocs.js CHANGED
@@ -59,8 +59,8 @@ export function parseAll(text) {
59
59
  * Flatten the docs tree into the headings a query can hit. Only a top-level
60
60
  * entry names its page (source: 'doc/api/fs.md'); everything nested under it
61
61
  * inherits that page and contributes its own heading and anchor. A nested
62
- * entry whose textRaw does not name it is not a heading (a bare 'Type:
63
- * {number}' line under a property) and is skipped; its parent still stands.
62
+ * entry whose textRaw is a property's type line, or does not name the entry,
63
+ * is not a heading and is skipped; its parent still stands.
64
64
  * A heading repeated on one page is kept once: the corpus lists some methods
65
65
  * twice for one heading, and where a page really repeats one (each stream
66
66
  * class has an Event: 'close') the rows would be indistinguishable anyway.
@@ -72,15 +72,23 @@ export function buildEntries(all) {
72
72
  const seen = new Set();
73
73
  const add = (node, page, top) => {
74
74
  const text = String(node?.textRaw ?? '').replaceAll('`', '').trim();
75
- const name = String(node?.name ?? '');
75
+ // A section's name keeps the heading's backticks ('`fetch`', every N-API
76
+ // function), so the two are compared with those stripped from both.
77
+ const name = String(node?.name ?? '').replaceAll('`', '');
76
78
  const type = String(node?.type ?? '');
77
- const heading = text && name
79
+ // A property's value line, 'Type: {string}', 'Returns: {Object}', or a
80
+ // bare '{number}', stands where its heading would; it is never one, even
81
+ // when the property's own name is a word in it (blob.type,
82
+ // server.listening).
83
+ const typeLine = /^(?:\w+: )?\{/.test(text);
84
+ const heading = text && name && !typeLine
78
85
  && (top || text.toLowerCase().includes(name.toLowerCase()));
79
86
  if (heading && !seen.has(`${page}#${text}`)) {
80
87
  seen.add(`${page}#${text}`);
81
- // A module or section heading is the page's own title, so its entry
82
- // links to the page top; every other heading has an anchor worth keeping.
83
- const anchor = ['module', 'misc', 'global'].includes(type) ? '' : slug(text);
88
+ // The top-level module or section heading is the page's own title, so
89
+ // its entry links to the page top; every other heading, a section
90
+ // within the page included, has an anchor worth keeping.
91
+ const anchor = top && ['module', 'misc', 'global'].includes(type) ? '' : slug(text);
84
92
  entries.push({ text, name, type, page, anchor });
85
93
  }
86
94
  for (const key of CHILD_KEYS) {
package/src/render.js CHANGED
@@ -250,7 +250,10 @@ const truncate = (s) => {
250
250
  // instead of falling through to the sentence rule below.
251
251
  const line = s.slice(0, TEXT_CAP).lastIndexOf('\n');
252
252
  if (line >= TEXT_CAP * SENTENCE_FLOOR) return `${s.slice(0, line)} ... +${num(s.length - line)} chars`;
253
- let cut = TEXT_CAP;
253
+ // A character past U+FFFF (an emoji, a rare CJK ideograph) is two UTF-16
254
+ // units. A cut between them printed half of one, which a terminal shows as
255
+ // the replacement character, so the cut steps back to before it.
256
+ let cut = /[\uD800-\uDBFF]/.test(s[TEXT_CAP - 1]) ? TEXT_CAP - 1 : TEXT_CAP;
254
257
  for (const m of s.slice(0, TEXT_CAP).matchAll(SENTENCE_END)) {
255
258
  const end = (m.index ?? 0) + m[0].length;
256
259
  if (end >= TEXT_CAP * SENTENCE_FLOOR) cut = end;
package/src/sites.js CHANGED
@@ -3,9 +3,10 @@
3
3
  * directly, so `oc hn item 4711` gets there without the agent knowing that
4
4
  * Hacker News spells it /item?id=. A shortcut is almost always a URL: it
5
5
  * resolves to one and hands off to the same fetch and render path `oc open`
6
- * uses, so nothing here can change what a page costs or how it reads. The
6
+ * uses, so nothing here can change what a page costs or how it reads, except
7
+ * that a shortcut at a JSON endpoint may name the fields worth showing. The
7
8
  * other shapes are searches cli.js runs itself and renders like any other
8
- * page: `sphinx` and `rdoc`, for docs sites whose search only exists as a
9
+ * page: `sphinx`, `rdoc`, and `mkdocs`, for docs sites whose search only exists as a
9
10
  * static index file, `nodedoc`, for the Node.js API docs, which ship their
10
11
  * reference the same way, and `api`, for a site whose search answers as
11
12
  * JSON.
@@ -38,9 +39,20 @@ const ALIASES = {
38
39
  gcp: 'cloud.google.com',
39
40
  learn: 'learn.microsoft.com',
40
41
  wiki: 'wikipedia.org',
42
+ npm: 'npmjs.com',
43
+ pip: 'pypi.org',
44
+ cargo: 'crates.io',
45
+ gem: 'rubygems.org',
46
+ eol: 'endoflife.date',
47
+ man: 'man7.org',
48
+ rfc: 'rfc-editor.org',
49
+ gl: 'gitlab.com',
50
+ material: 'squidfunk.github.io',
51
+ polars: 'docs.pola.rs',
52
+ urlo: 'users.rust-lang.org',
41
53
  };
42
54
 
43
- /** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
55
+ /** @typedef {{open?: string, keep?: string[], list?: string, sphinx?: string, nodedoc?: string, rdoc?: string, mkdocs?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
44
56
  /** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */
45
57
 
46
58
  /** @type {Map<string, Site>|null} */
@@ -97,14 +109,16 @@ const verbs = (site) =>
97
109
  * instead, since the agent has the right site and only needs the verb list.
98
110
  * @param {string} name
99
111
  * @param {string[]} args
100
- * @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
112
+ * @returns {{url?: string, view?: {keep: string[], list?: string}, sphinx?: string, nodedoc?: string, rdoc?: string, mkdocs?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
101
113
  */
102
114
  export function resolveSite(name, args) {
103
115
  const site = sites().get(name.toLowerCase());
104
116
  if (!site) return null;
105
117
  const [verb, ...rest] = args;
106
118
  if (!verb) throw new Error(`usage: oc ${name} <verb>, one of: ${verbs(site)}`);
107
- const def = site.commands[verb];
119
+ // commands comes from JSON.parse, so a plain lookup would also find
120
+ // inherited Object members such as toString, which name no URL.
121
+ const def = Object.hasOwn(site.commands, verb) ? site.commands[verb] : undefined;
108
122
  if (!def) throw new Error(`'${verb}' is not a ${site.domain} shortcut, try: ${verbs(site)}`);
109
123
 
110
124
  const need = def.args ?? [];
@@ -119,7 +133,7 @@ export function resolveSite(name, args) {
119
133
  // back whole for cli.js to run against the site's own search. The local
120
134
  // backends need only their docs root; the API shape needs its whole
121
135
  // definition, since it names the endpoint and the response fields.
122
- for (const kind of ['sphinx', 'nodedoc', 'rdoc']) {
136
+ for (const kind of ['sphinx', 'nodedoc', 'rdoc', 'mkdocs']) {
123
137
  if (def[kind]) {
124
138
  return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
125
139
  }
@@ -130,7 +144,11 @@ export function resolveSite(name, args) {
130
144
  const url = need.reduce(
131
145
  (open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)),
132
146
  def.open);
133
- return { url, domain: site.domain, command: verb };
147
+ // A shortcut at a JSON endpoint may say which fields to show. The choice
148
+ // rides along to the render and touches nothing else: the URL is fetched,
149
+ // cached, and remembered exactly as `oc open` would.
150
+ const view = def.keep?.length ? { keep: def.keep, list: def.list } : undefined;
151
+ return { url, view, domain: site.domain, command: verb };
134
152
  }
135
153
 
136
154
  /**
@@ -148,10 +166,12 @@ function encode(value, template, arg) {
148
166
  // A path slot that starts with @ is a copied handle (@name, @channel).
149
167
  // encodeURIComponent would turn it into %40 and miss the user, and YouTube's
150
168
  // template already has the @ in the URL. Search queries keep it, so a
151
- // mention in a query is still the thing being searched for.
152
- const raw = !query && value.startsWith('@') && value.length > 1 ? value.slice(1) : value;
153
- const encoded = encodeURIComponent(raw);
154
- return query ? encoded : encoded.replaceAll('%2F', '/');
169
+ // mention in a query is still the thing being searched for. A handle never
170
+ // contains a slash, so @scope/name is a scoped npm package, not a handle:
171
+ // stripping its @ asked the registry for a different package.
172
+ const handle = !query && value.startsWith('@') && value.length > 1 && !value.includes('/');
173
+ const encoded = encodeURIComponent(handle ? value.slice(1) : value);
174
+ return query ? encoded : encoded.replaceAll('%2F', '/').replaceAll('%40', '@');
155
175
  }
156
176
 
157
177
  /**