@only-cli/oc 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/sites.js CHANGED
@@ -1,9 +1,14 @@
1
1
  /**
2
2
  * Site shortcuts. `clis/*.json` names the URLs on a site worth reaching
3
3
  * directly, so `oc hn item 4711` gets there without the agent knowing that
4
- * Hacker News spells it /item?id=. A shortcut is only ever a URL: it resolves
5
- * to one and hands off to the same fetch and render path `oc open` uses, so
6
- * nothing here can change what a page costs or how it reads.
4
+ * Hacker News spells it /item?id=. A shortcut is almost always a URL: it
5
+ * resolves to one and hands off to the same fetch and render path `oc open`
6
+ * uses, so nothing here can change what a page costs or how it reads. The
7
+ * other shapes are searches cli.js runs itself and renders like any other
8
+ * page: `sphinx` and `rdoc`, for docs sites whose search only exists as a
9
+ * static index file, `nodedoc`, for the Node.js API docs, which ship their
10
+ * reference the same way, and `api`, for a site whose search answers as
11
+ * JSON.
7
12
  */
8
13
 
9
14
  import { readdirSync, readFileSync } from 'node:fs';
@@ -22,12 +27,20 @@ const ALIASES = {
22
27
  finance: 'finance.yahoo.com',
23
28
  twitter: 'x.com',
24
29
  aws: 'docs.aws.amazon.com',
30
+ py: 'docs.python.org',
31
+ mdn: 'developer.mozilla.org',
32
+ node: 'nodejs.org',
33
+ rust: 'doc.rust-lang.org',
34
+ java: 'docs.oracle.com',
35
+ ruby: 'docs.ruby-lang.org',
36
+ cpp: 'en.cppreference.com',
37
+ ts: 'typescriptlang.org',
25
38
  gcp: 'cloud.google.com',
26
39
  learn: 'learn.microsoft.com',
27
40
  wiki: 'wikipedia.org',
28
41
  };
29
42
 
30
- /** @typedef {{open: string, args?: string[]}} Shortcut */
43
+ /** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
31
44
  /** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */
32
45
 
33
46
  /** @type {Map<string, Site>|null} */
@@ -84,7 +97,7 @@ const verbs = (site) =>
84
97
  * instead, since the agent has the right site and only needs the verb list.
85
98
  * @param {string} name
86
99
  * @param {string[]} args
87
- * @returns {{url: string, domain: string, command: string}|null}
100
+ * @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
88
101
  */
89
102
  export function resolveSite(name, args) {
90
103
  const site = sites().get(name.toLowerCase());
@@ -102,6 +115,18 @@ export function resolveSite(name, args) {
102
115
  // separate words ('oc ddg search claude code cli') works unquoted.
103
116
  const values = need.map((_, i) =>
104
117
  i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]);
118
+ // A search oc runs itself has no page URL to build: the query is handed
119
+ // back whole for cli.js to run against the site's own search. The local
120
+ // backends need only their docs root; the API shape needs its whole
121
+ // definition, since it names the endpoint and the response fields.
122
+ for (const kind of ['sphinx', 'nodedoc', 'rdoc']) {
123
+ if (def[kind]) {
124
+ return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
125
+ }
126
+ }
127
+ if (def.api) {
128
+ return { api: def, query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
129
+ }
105
130
  const url = need.reduce(
106
131
  (open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)),
107
132
  def.open);
package/src/sphinx.js ADDED
@@ -0,0 +1,211 @@
1
+ /**
2
+ * Sphinx search backend. A Sphinx-built documentation site (docs.python.org,
3
+ * most Read the Docs projects) has no search server: its search page ships
4
+ * the site's entire full-text index as one static file, searchindex.js, and
5
+ * ranks matches in the visitor's browser. oc can run the same ranking here,
6
+ * so `search` on such a site answers from the site's own index instead of a
7
+ * third-party engine. The index is big (docs.python.org's is ~4MB, ~900KB
8
+ * over the wire) but static, so it is cached on disk for a day and never
9
+ * printed: what reaches the agent is only the ranked result list.
10
+ */
11
+
12
+ import { cachedFile } from './cache.js';
13
+
14
+ const MAX_RESULTS = 20;
15
+
16
+ /**
17
+ * The index file is `Search.setIndex({...})`: JSON wrapped in one function
18
+ * call for the browser's benefit. Anything that does not parse that way is
19
+ * not a Sphinx index, which on a wrong or moved URL is the honest error.
20
+ * @param {string} js
21
+ * @returns {any}
22
+ */
23
+ export function parseIndex(js) {
24
+ const start = js.indexOf('(');
25
+ const end = js.lastIndexOf(')');
26
+ if (start >= 0 && end > start) {
27
+ try {
28
+ return JSON.parse(js.slice(start + 1, end));
29
+ } catch {}
30
+ }
31
+ throw new Error('not a Sphinx search index');
32
+ }
33
+
34
+ // terms and titleterms store a bare number when a word appears in one
35
+ // document and an array when it appears in several.
36
+ const docsFor = (table, word) => {
37
+ const hit = table?.[word];
38
+ return hit == null ? null : Array.isArray(hit) ? hit : [hit];
39
+ };
40
+
41
+ /**
42
+ * Sphinx stems words before indexing ('threading' is stored as 'thread'), so
43
+ * an exact lookup misses common query spellings. Rather than shipping the
44
+ * Porter stemmer, try the word with common suffixes stripped, and only then
45
+ * a prefix scan: an index key that extends the word, or that the word
46
+ * extends, counts at reduced weight.
47
+ * @param {Record<string, number|number[]>} table
48
+ * @param {string} word
49
+ * @returns {{docs: number[], exact: boolean}|null}
50
+ */
51
+ function lookup(table, word) {
52
+ const exact = docsFor(table, word);
53
+ if (exact) return { docs: exact, exact: true };
54
+ for (const suffix of ['ing', 'ed', 'es', 's', 'e']) {
55
+ if (word.length - suffix.length >= 3 && word.endsWith(suffix)) {
56
+ const hit = docsFor(table, word.slice(0, -suffix.length));
57
+ if (hit) return { docs: hit, exact: false };
58
+ }
59
+ }
60
+ if (word.length >= 4) {
61
+ const docs = new Set();
62
+ for (const key of Object.keys(table ?? {})) {
63
+ if (key.length >= 4 && (key.startsWith(word) || word.startsWith(key))) {
64
+ for (const d of docsFor(table, key)) docs.add(d);
65
+ }
66
+ }
67
+ if (docs.size) return { docs: [...docs], exact: false };
68
+ }
69
+ return null;
70
+ }
71
+
72
+ /**
73
+ * An exact object hit ('json.dumps', or just 'dumps') beats any full-text
74
+ * rank: the index maps the symbol straight to its anchor on the page, so it
75
+ * goes at the top as a direct link. Only single-word queries can be symbols.
76
+ * @param {any} index
77
+ * @param {string} query
78
+ */
79
+ function objectHits(index, query) {
80
+ const q = query.trim().toLowerCase();
81
+ if (!q || q.includes(' ')) return [];
82
+ const hits = [];
83
+ for (const [prefix, entries] of Object.entries(index.objects ?? {})) {
84
+ for (const [doc, typeIdx, priority, anchor, name] of entries) {
85
+ const full = prefix ? `${prefix}.${name}` : name;
86
+ if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
87
+ hits.push({
88
+ name: full,
89
+ type: index.objnames?.[typeIdx]?.[2] ?? '',
90
+ doc,
91
+ anchor: anchor === '' ? full : anchor,
92
+ priority,
93
+ });
94
+ }
95
+ }
96
+ return hits.sort((a, b) => a.priority - b.priority).slice(0, 5);
97
+ }
98
+
99
+ /**
100
+ * Rank the index against a query the way the site's own search page would:
101
+ * a document must match every word, a title hit weighs far more than a body
102
+ * hit, and only when nothing matches every word does any-word matching kick
103
+ * in, and then the result page says so.
104
+ * @param {any} index
105
+ * @param {string} query
106
+ */
107
+ export function searchIndex(index, query) {
108
+ const words = [...new Set(
109
+ query.toLowerCase().split(/\s+/)
110
+ .map((w) => w.replace(/^[^\w.]+|[^\w.]+$/g, ''))
111
+ .filter(Boolean))];
112
+ const scores = new Map();
113
+ const matched = new Map();
114
+ for (const word of words) {
115
+ const perDoc = new Map();
116
+ const body = lookup(index.terms, word);
117
+ if (body) for (const d of body.docs) perDoc.set(d, body.exact ? 5 : 2);
118
+ const title = lookup(index.titleterms, word);
119
+ if (title) for (const d of title.docs) perDoc.set(d, (perDoc.get(d) ?? 0) + (title.exact ? 15 : 5));
120
+ for (const [d, score] of perDoc) {
121
+ scores.set(d, (scores.get(d) ?? 0) + score);
122
+ matched.set(d, (matched.get(d) ?? 0) + 1);
123
+ }
124
+ }
125
+ let docs = [...scores.keys()].filter((d) => matched.get(d) === words.length);
126
+ const partial = !docs.length && words.length > 1 && scores.size > 0;
127
+ if (partial) docs = [...scores.keys()];
128
+ docs.sort((a, b) =>
129
+ scores.get(b) - scores.get(a)
130
+ || String(index.titles[a]).localeCompare(String(index.titles[b])));
131
+ return {
132
+ words,
133
+ partial,
134
+ total: docs.length,
135
+ objects: objectHits(index, query),
136
+ docs: docs.slice(0, MAX_RESULTS).map((d) => ({
137
+ doc: d,
138
+ title: plainTitle(index.titles[d]) || index.docnames[d],
139
+ })),
140
+ };
141
+ }
142
+
143
+ // Titles in the index arrive as the HTML of the page's <h1>, markup and all
144
+ // (docs.python.org wraps module names in <code> spans), so they are flattened
145
+ // to text before they are placed on the results page. A title is index data
146
+ // from the network, so the walk keeps only what stands outside a bracket: no
147
+ // '<' or '>' can survive it, even from a tag the title never closes. A
148
+ // literal angle bracket in a real title arrives as an entity, so nothing
149
+ // legitimate is lost.
150
+ const plainTitle = (t) => {
151
+ let text = '';
152
+ let inTag = false;
153
+ for (const ch of String(t)) {
154
+ if (ch === '<') inTag = true;
155
+ else if (ch === '>') inTag = false;
156
+ else if (!inTag) text += ch;
157
+ }
158
+ return text.trim();
159
+ };
160
+
161
+ export const escapeHTML = (s) => String(s)
162
+ .replaceAll('&', '&amp;').replaceAll('<', '&lt;')
163
+ .replaceAll('>', '&gt;').replaceAll('"', '&quot;');
164
+
165
+ /**
166
+ * The result list becomes a small HTML page and rides the same distill and
167
+ * render path a fetched page does. That is what makes results numbered,
168
+ * followable with `do <n>`, and saved as session state, with nothing new for
169
+ * an agent to learn.
170
+ * @param {string} base
171
+ * @param {string} query
172
+ * @param {ReturnType<typeof searchIndex>} found
173
+ * @param {any} index
174
+ * @returns {string}
175
+ */
176
+ export function resultsToHTML(base, query, found, index) {
177
+ const host = new URL(base).host;
178
+ const pageURL = (doc) => new URL(`${index.docnames[doc]}.html`, base).href;
179
+ const items = found.objects.map((o) =>
180
+ `<li><a href="${escapeHTML(`${pageURL(o.doc)}#${o.anchor}`)}">${escapeHTML(o.name)}</a>`
181
+ + ` ${escapeHTML(o.type)}, in ${escapeHTML(plainTitle(index.titles[o.doc]) || index.docnames[o.doc])}</li>`);
182
+ for (const r of found.docs) {
183
+ items.push(`<li><a href="${escapeHTML(pageURL(r.doc))}">${escapeHTML(r.title)}</a></li>`);
184
+ }
185
+ const partial = found.partial ? '; no page matches every word, so these match some' : '';
186
+ const summary = items.length
187
+ ? `${found.total} page${found.total === 1 ? '' : 's'} match in the site's own search index,`
188
+ + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
189
+ : `nothing in the site's own search index matches; try fewer or different words`;
190
+ return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
191
+ + `<p>${summary}</p>`
192
+ + (items.length ? `<ol>${items.join('')}</ol>` : '')
193
+ + `</main></body></html>`;
194
+ }
195
+
196
+ /**
197
+ * Search one Sphinx site. Returns the synthetic results page plus the URL the
198
+ * session should remember: the site's human search URL, so the state reads
199
+ * sensibly in `oc session` listings and error messages.
200
+ * @param {string} base - site root ending in '/', e.g. https://docs.python.org/3/
201
+ * @param {string} query
202
+ */
203
+ export async function sphinxSearch(base, query) {
204
+ if (!query.trim()) throw new Error('usage: search <query>');
205
+ const { data: index, via } = await cachedFile('sphinx', new URL('searchindex.js', base).href, parseIndex);
206
+ return {
207
+ url: new URL(`search.html?q=${encodeURIComponent(query)}`, base).href,
208
+ html: resultsToHTML(base, query, searchIndex(index, query), index),
209
+ via,
210
+ };
211
+ }