@only-cli/oc 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/rdoc.js ADDED
@@ -0,0 +1,98 @@
1
+ /**
2
+ * RDoc search backend. The Ruby docs (docs.ruby-lang.org) are built with
3
+ * RDoc, which like Sphinx has no search server: the generated site ships its
4
+ * whole search index as one static file, js/search_index.js, and matches in
5
+ * the visitor's browser. So `search` ranks that file locally: every class,
6
+ * module, method, and guide page in the index becomes a result linking to
7
+ * its own anchor. The file is ~3.4MB (~560KB over the wire) and static, so
8
+ * it lives in the same day cache the other local backends use, and what
9
+ * reaches the agent is the ranked result list only.
10
+ */
11
+
12
+ import { cachedFile } from './cache.js';
13
+ import { escapeHTML } from './sphinx.js';
14
+ import { searchEntries } from './nodedocs.js';
15
+
16
+ const MAX_RESULTS = 20;
17
+
18
+ /**
19
+ * The index file is `var search_data = {...}`: JSON behind one assignment
20
+ * for the browser's benefit. Anything that does not parse that way, or that
21
+ * lacks the info rows, is not an RDoc index, which on a wrong or moved URL
22
+ * is the honest error, and it keeps a block page out of the cache too.
23
+ * @param {string} js
24
+ * @returns {any}
25
+ */
26
+ export function parseRdocIndex(js) {
27
+ const start = js.indexOf('=');
28
+ if (start >= 0) {
29
+ try {
30
+ const data = JSON.parse(js.slice(start + 1));
31
+ if (Array.isArray(data?.index?.info)) return data;
32
+ } catch {}
33
+ }
34
+ throw new Error('not an RDoc search index');
35
+ }
36
+
37
+ /**
38
+ * Flatten the index's info rows into the entries the shared ranker scores.
39
+ * A row is [name, namespace, path, params, snippet]; the path's own anchor
40
+ * says what the row is, so 'dig' in 'Array' with anchor method-i-dig reads
41
+ * back as the heading a rubyist expects, Array#dig(*args). Snippets stay
42
+ * behind: matching on them would rank prose over the symbol asked for.
43
+ * @param {any} data - parsed search_index.js
44
+ * @returns {{text: string, name: string, kind: string, path: string}[]}
45
+ */
46
+ export function buildRdocEntries(data) {
47
+ return data.index.info.map(([name, namespace, path, params]) => {
48
+ const p = String(path ?? '');
49
+ const kind = p.includes('#method-c-') ? 'class method'
50
+ : p.includes('#method-i-') ? 'method'
51
+ : /^[A-Z]/.test(String(name)) ? 'class' : 'page';
52
+ const text = kind === 'class method' ? `${namespace}.${name}${params}`
53
+ : kind === 'method' ? `${namespace}#${name}${params}`
54
+ : namespace ? `${namespace}::${name}` : String(name ?? '');
55
+ return { text, name: String(name ?? ''), kind, path: p };
56
+ }).filter((e) => e.text && e.path);
57
+ }
58
+
59
+ /**
60
+ * The result list becomes the same small synthetic page the other search
61
+ * backends emit, so it distills, renders, numbers, and remembers like any
62
+ * fetched page and `do <n>` follows a result.
63
+ * @param {string} base
64
+ * @param {string} query
65
+ * @param {ReturnType<typeof searchEntries>} found
66
+ * @returns {string}
67
+ */
68
+ export function resultsToHTML(base, query, found) {
69
+ const host = new URL(base).host;
70
+ const items = found.hits.map((e) =>
71
+ `<li><a href="${escapeHTML(new URL(e.path, base).href)}">${escapeHTML(e.text)}</a>`
72
+ + ` ${escapeHTML(e.kind)}</li>`);
73
+ const partial = found.partial ? '; no entry matches every word, so these match some' : '';
74
+ const summary = items.length
75
+ ? `${found.total} entr${found.total === 1 ? 'y matches' : 'ies match'} in the docs' own index,`
76
+ + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
77
+ : `nothing in the docs' own index matches; try fewer or different words`;
78
+ return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
79
+ + `<p>${summary}</p>`
80
+ + (items.length ? `<ol>${items.join('')}</ol>` : '')
81
+ + `</main></body></html>`;
82
+ }
83
+
84
+ /**
85
+ * Search one RDoc site. The site has no search URL of its own to remember,
86
+ * so the session keeps the docs root, the page a reader would start from.
87
+ * @param {string} base - docs root ending in '/', e.g. https://docs.ruby-lang.org/en/3.4/
88
+ * @param {string} query
89
+ */
90
+ export async function rdocSearch(base, query) {
91
+ if (!query.trim()) throw new Error('usage: search <query>');
92
+ const { data, via } = await cachedFile('rdoc', new URL('js/search_index.js', base).href, parseRdocIndex);
93
+ return {
94
+ url: new URL('index.html', base).href,
95
+ html: resultsToHTML(base, query, searchEntries(buildRdocEntries(data), query)),
96
+ via,
97
+ };
98
+ }
package/src/render.js CHANGED
@@ -16,6 +16,67 @@ export const estimateTokens = (s) => Math.ceil(s.length / 4);
16
16
 
17
17
  const num = (v) => v.toLocaleString('en-US');
18
18
 
19
+ // A page that distilled to nothing must not read like a page with nothing on
20
+ // it. JS-only pages, consent walls, and bot challenges all answer HTTP 200 with
21
+ // markup carrying no text, and "100% saved" is technically true of a render
22
+ // that saved every token by extracting none. From the output alone an agent
23
+ // cannot tell that from a genuinely empty page, so it never falls back to
24
+ // something heavier and an empty result travels on as evidence. oc has to fail
25
+ // loud and cheap there instead, per principle 5 in CONTRIBUTING.
26
+ //
27
+ // The numbers below are measured, not guessed. Against live pages, the thinnest
28
+ // render the README claims (an X profile) carries about 460 tokens of content,
29
+ // while a JS-only or gated one carries under 55 (reddit.com/r/*,
30
+ // instagram.com), so a threshold in between never has to be a close call.
31
+
32
+ // Text the page itself wrote, in tokens: prose and headings, plus link and
33
+ // button labels long enough to be content rather than furniture. Nav chrome is
34
+ // short by nature ("Help", "Log in") and a headline or a post title is not,
35
+ // which is what tells a link-list page (Hacker News, search results) from a
36
+ // page whose only links are its own menu.
37
+ const CONTENT_LABEL = 25;
38
+ // Below this a render is suspiciously thin, but thin is only a verdict when
39
+ // the page's own size says there should have been more. A terse page that
40
+ // arrived terse (a status endpoint, a one-line answer) distilled fine.
41
+ export const MIN_CONTENT = 25;
42
+ // Below this, with markup that large behind it, the fetch worked and the render
43
+ // did not: a real page of that weight always distills to more. A genuinely
44
+ // short page is exempt, because its HTML never reaches THIN_HTML.
45
+ const THIN_CONTENT = 100;
46
+ const THIN_HTML = 2500;
47
+
48
+ /**
49
+ * How much of a distilled page is text the page itself wrote.
50
+ * @param {import('./distill.js').Page} page
51
+ * @returns {number}
52
+ */
53
+ export const contentTokens = (page) =>
54
+ page.blocks.reduce((sum, b) => {
55
+ if (b.type === 'heading' || b.type === 'text') return sum + estimateTokens(b.text ?? '');
56
+ const label = (b.type === 'link' || b.type === 'button') && (b.text ?? '').length > CONTENT_LABEL;
57
+ return label ? sum + estimateTokens(b.text) : sum;
58
+ }, 0);
59
+
60
+ /**
61
+ * Why this render carries no content worth printing, as a phrase for the
62
+ * failure line, or null when it does carry some. Both figures are token counts,
63
+ * which is the unit the caller is paying in.
64
+ * @param {number} content
65
+ * @param {number} htmlTokens
66
+ * @returns {string|null}
67
+ */
68
+ export function contentFailure(content, htmlTokens) {
69
+ // Nothing extracted is empty whatever the page weighed. Anything more is
70
+ // only a failure with evidence: a small page that renders small is not
71
+ // gated, it is small, and exit 2 on it would send an agent to a browser
72
+ // for a page it was already holding.
73
+ if (content === 0) return 'no text on the whole page';
74
+ if (content < THIN_CONTENT && htmlTokens > THIN_HTML) {
75
+ return `~${content} tokens of text out of ~${htmlTokens} of HTML`;
76
+ }
77
+ return null;
78
+ }
79
+
19
80
  // How far past the budget a page may run and still be printed whole. Cutting a
20
81
  // page that was nearly done costs the agent a second command, and a command is
21
82
  // dear: measured inside Claude Code, one tool call is 23,000 to 33,000 tokens of
@@ -36,7 +97,7 @@ export const FINISH = 4;
36
97
  */
37
98
  export function render(page, { budget = 500, from = 0 } = {}) {
38
99
  const blocks = collapseRuns(page.blocks);
39
- const head = page.title ? [from > 0 ? `# ${page.title} (continued)` : `# ${page.title}`] : [];
100
+ const head = page.title ? [from > 0 ? `# ${truncate(page.title)} (continued)` : `# ${truncate(page.title)}`] : [];
40
101
  const lines = [...head];
41
102
  let spent = estimateTokens(lines.join('\n'));
42
103
  let hasLinks = false;
@@ -148,13 +209,13 @@ export function formatBlock(b, { full = false } = {}) {
148
209
  const tag = b.n == null ? '' : `[${b.n}] `;
149
210
  switch (b.type) {
150
211
  case 'heading':
151
- return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${tag}${b.text}`;
212
+ return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${tag}${full ? b.text : truncate(b.text)}`;
152
213
  case 'link':
153
214
  return `${tag}${full ? b.text : truncate(b.text)}`;
154
215
  case 'button':
155
216
  return `${tag}button "${full ? b.text : truncate(b.text)}"`;
156
217
  case 'input':
157
- return `${tag}input ${b.name} (${b.text})`;
218
+ return `${tag}input ${truncate(b.name ?? '')} (${truncate(b.text ?? '')})`;
158
219
  case 'divider':
159
220
  return b.text;
160
221
  default:
package/src/session.js CHANGED
@@ -1,7 +1,8 @@
1
1
  /**
2
2
  * Sessions are plain JSON files on disk, one per name: the current URL, the
3
3
  * distilled blocks of the page it holds, how far the last render got through
4
- * them, and a short history. No daemon, no background process, no cookies yet.
4
+ * them, and a short history. No daemon, no background process; cookies live in
5
+ * a separate sidecar file (see cookies.js).
5
6
  *
6
7
  * The file exists so `oc do <n>` can follow a link the compact view never
7
8
  * printed the URL of. Hiding URLs is what makes `oc open` cheap; this is what
@@ -11,18 +12,35 @@
11
12
 
12
13
  import { homedir } from 'node:os';
13
14
  import { join } from 'node:path';
14
- import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
15
+ import { mkdirSync, readFileSync, writeFileSync, chmodSync, unlinkSync } from 'node:fs';
15
16
 
16
17
  export const DEFAULT_SESSION = 'default';
17
18
 
18
19
  // OC_HOME relocates the whole state directory, for sandboxes, CI, and tests.
19
20
  export const sessionDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sessions');
20
21
 
22
+ // A session name is interpolated straight into a filename, and the cookie
23
+ // sidecar it names now holds real credentials, so a name that is a path
24
+ // (absolute, or with a separator or '..') could write or delete a file outside
25
+ // the store. Names are user-facing labels, so this charset loses nothing real.
26
+ const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
27
+
28
+ /**
29
+ * @param {string} name
30
+ * @returns {string} the same name, once it is known to be a safe filename
31
+ */
32
+ export function assertSafeName(name) {
33
+ if (typeof name !== 'string' || name === '.' || name === '..' || !SAFE_NAME.test(name)) {
34
+ throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_'`);
35
+ }
36
+ return name;
37
+ }
38
+
21
39
  /**
22
40
  * @param {string} name
23
41
  * @returns {string}
24
42
  */
25
- export const sessionPath = (name) => join(sessionDir(), `${name}.json`);
43
+ export const sessionPath = (name) => join(sessionDir(), `${assertSafeName(name)}.json`);
26
44
 
27
45
  // Search engines and link aggregators wrap outbound links in a tracking
28
46
  // redirector whose landing page is a script, not content, so following one
@@ -136,7 +154,28 @@ export function handleNumbers(state) {
136
154
  */
137
155
  export function saveSession(name, state) {
138
156
  mkdirSync(sessionDir(), { recursive: true });
139
- writeFileSync(sessionPath(name), JSON.stringify(state));
157
+ // A snapshot of an authenticated page holds that page's text, so it gets the
158
+ // same owner-only mode as the cookie sidecar. writeFileSync only sets the
159
+ // mode on create, so a snapshot left world-readable by an older version is
160
+ // tightened explicitly on the next save.
161
+ const path = sessionPath(name);
162
+ writeFileSync(path, JSON.stringify(state), { mode: 0o600 });
163
+ chmodSync(path, 0o600);
164
+ }
165
+
166
+ /**
167
+ * Drop a saved page. `oc logout` calls this alongside clearing the cookie jar:
168
+ * a snapshot taken under a login holds that page's text, so leaving it behind
169
+ * would make logout mean "the cookies are gone" rather than "nothing of this
170
+ * login remains".
171
+ * @param {string} name
172
+ */
173
+ export function clearSession(name) {
174
+ try {
175
+ unlinkSync(sessionPath(name));
176
+ } catch {
177
+ // nothing saved under that name is fine
178
+ }
140
179
  }
141
180
 
142
181
  /**
package/src/sites.js ADDED
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Site shortcuts. `clis/*.json` names the URLs on a site worth reaching
3
+ * directly, so `oc hn item 4711` gets there without the agent knowing that
4
+ * Hacker News spells it /item?id=. A shortcut is almost always a URL: it
5
+ * resolves to one and hands off to the same fetch and render path `oc open`
6
+ * uses, so nothing here can change what a page costs or how it reads. The
7
+ * other shapes are searches cli.js runs itself and renders like any other
8
+ * page: `sphinx` and `rdoc`, for docs sites whose search only exists as a
9
+ * static index file, `nodedoc`, for the Node.js API docs, which ship their
10
+ * reference the same way, and `api`, for a site whose search answers as
11
+ * JSON.
12
+ */
13
+
14
+ import { readdirSync, readFileSync } from 'node:fs';
15
+ import { fileURLToPath } from 'node:url';
16
+
17
+ const DIR = fileURLToPath(new URL('../clis/', import.meta.url));
18
+
19
+ // Short names an agent is likely to reach for. The domain itself and its
20
+ // registrable label always resolve, so this only covers what those miss.
21
+ const ALIASES = {
22
+ hn: 'news.ycombinator.com',
23
+ gh: 'github.com',
24
+ so: 'stackoverflow.com',
25
+ ddg: 'duckduckgo.com',
26
+ yt: 'youtube.com',
27
+ finance: 'finance.yahoo.com',
28
+ twitter: 'x.com',
29
+ aws: 'docs.aws.amazon.com',
30
+ py: 'docs.python.org',
31
+ mdn: 'developer.mozilla.org',
32
+ node: 'nodejs.org',
33
+ rust: 'doc.rust-lang.org',
34
+ java: 'docs.oracle.com',
35
+ ruby: 'docs.ruby-lang.org',
36
+ cpp: 'en.cppreference.com',
37
+ ts: 'typescriptlang.org',
38
+ gcp: 'cloud.google.com',
39
+ learn: 'learn.microsoft.com',
40
+ wiki: 'wikipedia.org',
41
+ };
42
+
43
+ /** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
44
+ /** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */
45
+
46
+ /** @type {Map<string, Site>|null} */
47
+ let cache = null;
48
+
49
+ /**
50
+ * Every site definition that ships with oc, keyed by each name that resolves
51
+ * to it. A definition that will not parse is skipped rather than fatal: a bad
52
+ * file costs its own shortcuts and leaves every other command working.
53
+ * @returns {Map<string, Site>}
54
+ */
55
+ export function sites() {
56
+ if (cache) return cache;
57
+ cache = new Map();
58
+ /** @type {string[]} */
59
+ let files = [];
60
+ try {
61
+ files = readdirSync(DIR).filter((f) => f.endsWith('.json')).sort();
62
+ } catch {
63
+ return cache;
64
+ }
65
+ for (const file of files) {
66
+ /** @type {Site} */
67
+ let site;
68
+ try {
69
+ site = JSON.parse(readFileSync(DIR + file, 'utf8'));
70
+ } catch {
71
+ continue;
72
+ }
73
+ if (!site?.domain || !site.commands) continue;
74
+ const labels = site.domain.split('.');
75
+ for (const key of [site.domain, labels[labels.length - 2]]) {
76
+ // First definition wins, in sorted filename order, so which site owns a
77
+ // shared label never depends on how the directory happens to be read.
78
+ if (key && !cache.has(key)) cache.set(key, site);
79
+ }
80
+ }
81
+ for (const [alias, domain] of Object.entries(ALIASES)) {
82
+ const site = cache.get(domain);
83
+ if (site && !cache.has(alias)) cache.set(alias, site);
84
+ }
85
+ return cache;
86
+ }
87
+
88
+ const verbs = (site) =>
89
+ Object.entries(site.commands)
90
+ .map(([name, def]) => (def.args?.length ? `${name} <${def.args.join('> <')}>` : name))
91
+ .join(' | ');
92
+
93
+ /**
94
+ * Resolve `oc <site> <verb> [args]` to a URL, or null when the first word
95
+ * names no site oc ships, which is the caller's cue to report an unknown
96
+ * command. A site that exists with a verb that does not is an error here
97
+ * instead, since the agent has the right site and only needs the verb list.
98
+ * @param {string} name
99
+ * @param {string[]} args
100
+ * @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
101
+ */
102
+ export function resolveSite(name, args) {
103
+ const site = sites().get(name.toLowerCase());
104
+ if (!site) return null;
105
+ const [verb, ...rest] = args;
106
+ if (!verb) throw new Error(`usage: oc ${name} <verb>, one of: ${verbs(site)}`);
107
+ const def = site.commands[verb];
108
+ if (!def) throw new Error(`'${verb}' is not a ${site.domain} shortcut, try: ${verbs(site)}`);
109
+
110
+ const need = def.args ?? [];
111
+ if (rest.length < need.length) {
112
+ throw new Error(`usage: oc ${name} ${verb} <${need.join('> <')}>`);
113
+ }
114
+ // The last argument soaks up everything left, so a query an agent typed as
115
+ // separate words ('oc ddg search claude code cli') works unquoted.
116
+ const values = need.map((_, i) =>
117
+ i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]);
118
+ // A search oc runs itself has no page URL to build: the query is handed
119
+ // back whole for cli.js to run against the site's own search. The local
120
+ // backends need only their docs root; the API shape needs its whole
121
+ // definition, since it names the endpoint and the response fields.
122
+ for (const kind of ['sphinx', 'nodedoc', 'rdoc']) {
123
+ if (def[kind]) {
124
+ return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
125
+ }
126
+ }
127
+ if (def.api) {
128
+ return { api: def, query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
129
+ }
130
+ const url = need.reduce(
131
+ (open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)),
132
+ def.open);
133
+ return { url, domain: site.domain, command: verb };
134
+ }
135
+
136
+ /**
137
+ * Percent-encode one value for the slot it fills. A value in the query string
138
+ * is encoded outright, but a docs path is often several segments deep, so
139
+ * 'oc learn doc azure/aks/intro' has to keep its slashes: escaping them would
140
+ * ask the site for one impossible segment instead of the page.
141
+ * @param {string} value
142
+ * @param {string} template
143
+ * @param {string} arg
144
+ * @returns {string}
145
+ */
146
+ function encode(value, template, arg) {
147
+ const query = template.includes('?') && template.indexOf(`{${arg}}`) > template.indexOf('?');
148
+ const encoded = encodeURIComponent(value);
149
+ return query ? encoded : encoded.replaceAll('%2F', '/');
150
+ }
151
+
152
+ /**
153
+ * One line per site for `oc sites`, shortest name first, since that is what an
154
+ * agent will type. Discovery has to cost less than a wrong guess does, so this
155
+ * stays one line each rather than a table.
156
+ * @returns {string}
157
+ */
158
+ export function listSites() {
159
+ const byDomain = new Map();
160
+ for (const [key, site] of sites()) {
161
+ if (!byDomain.has(site.domain)) byDomain.set(site.domain, { site, keys: [] });
162
+ byDomain.get(site.domain).keys.push(key);
163
+ }
164
+ const lines = [...byDomain.values()].map(({ site, keys }) => {
165
+ const names = keys
166
+ .filter((k) => k !== site.domain)
167
+ .sort((a, b) => a.length - b.length || a.localeCompare(b));
168
+ return `oc ${names[0] ?? site.domain} <verb> (${[...names.slice(1), site.domain].join(', ')}): ${verbs(site)}`;
169
+ });
170
+ return lines.length
171
+ ? `${lines.join('\n')}\nany other site: oc open <url>`
172
+ : 'no site definitions found; oc open <url> works on any site';
173
+ }
package/src/sphinx.js ADDED
@@ -0,0 +1,211 @@
1
+ /**
2
+ * Sphinx search backend. A Sphinx-built documentation site (docs.python.org,
3
+ * most Read the Docs projects) has no search server: its search page ships
4
+ * the site's entire full-text index as one static file, searchindex.js, and
5
+ * ranks matches in the visitor's browser. oc can run the same ranking here,
6
+ * so `search` on such a site answers from the site's own index instead of a
7
+ * third-party engine. The index is big (docs.python.org's is ~4MB, ~900KB
8
+ * over the wire) but static, so it is cached on disk for a day and never
9
+ * printed: what reaches the agent is only the ranked result list.
10
+ */
11
+
12
+ import { cachedFile } from './cache.js';
13
+
14
+ const MAX_RESULTS = 20;
15
+
16
+ /**
17
+ * The index file is `Search.setIndex({...})`: JSON wrapped in one function
18
+ * call for the browser's benefit. Anything that does not parse that way is
19
+ * not a Sphinx index, which on a wrong or moved URL is the honest error.
20
+ * @param {string} js
21
+ * @returns {any}
22
+ */
23
+ export function parseIndex(js) {
24
+ const start = js.indexOf('(');
25
+ const end = js.lastIndexOf(')');
26
+ if (start >= 0 && end > start) {
27
+ try {
28
+ return JSON.parse(js.slice(start + 1, end));
29
+ } catch {}
30
+ }
31
+ throw new Error('not a Sphinx search index');
32
+ }
33
+
34
+ // terms and titleterms store a bare number when a word appears in one
35
+ // document and an array when it appears in several.
36
+ const docsFor = (table, word) => {
37
+ const hit = table?.[word];
38
+ return hit == null ? null : Array.isArray(hit) ? hit : [hit];
39
+ };
40
+
41
+ /**
42
+ * Sphinx stems words before indexing ('threading' is stored as 'thread'), so
43
+ * an exact lookup misses common query spellings. Rather than shipping the
44
+ * Porter stemmer, try the word with common suffixes stripped, and only then
45
+ * a prefix scan: an index key that extends the word, or that the word
46
+ * extends, counts at reduced weight.
47
+ * @param {Record<string, number|number[]>} table
48
+ * @param {string} word
49
+ * @returns {{docs: number[], exact: boolean}|null}
50
+ */
51
+ function lookup(table, word) {
52
+ const exact = docsFor(table, word);
53
+ if (exact) return { docs: exact, exact: true };
54
+ for (const suffix of ['ing', 'ed', 'es', 's', 'e']) {
55
+ if (word.length - suffix.length >= 3 && word.endsWith(suffix)) {
56
+ const hit = docsFor(table, word.slice(0, -suffix.length));
57
+ if (hit) return { docs: hit, exact: false };
58
+ }
59
+ }
60
+ if (word.length >= 4) {
61
+ const docs = new Set();
62
+ for (const key of Object.keys(table ?? {})) {
63
+ if (key.length >= 4 && (key.startsWith(word) || word.startsWith(key))) {
64
+ for (const d of docsFor(table, key)) docs.add(d);
65
+ }
66
+ }
67
+ if (docs.size) return { docs: [...docs], exact: false };
68
+ }
69
+ return null;
70
+ }
71
+
72
+ /**
73
+ * An exact object hit ('json.dumps', or just 'dumps') beats any full-text
74
+ * rank: the index maps the symbol straight to its anchor on the page, so it
75
+ * goes at the top as a direct link. Only single-word queries can be symbols.
76
+ * @param {any} index
77
+ * @param {string} query
78
+ */
79
+ function objectHits(index, query) {
80
+ const q = query.trim().toLowerCase();
81
+ if (!q || q.includes(' ')) return [];
82
+ const hits = [];
83
+ for (const [prefix, entries] of Object.entries(index.objects ?? {})) {
84
+ for (const [doc, typeIdx, priority, anchor, name] of entries) {
85
+ const full = prefix ? `${prefix}.${name}` : name;
86
+ if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
87
+ hits.push({
88
+ name: full,
89
+ type: index.objnames?.[typeIdx]?.[2] ?? '',
90
+ doc,
91
+ anchor: anchor === '' ? full : anchor,
92
+ priority,
93
+ });
94
+ }
95
+ }
96
+ return hits.sort((a, b) => a.priority - b.priority).slice(0, 5);
97
+ }
98
+
99
+ /**
100
+ * Rank the index against a query the way the site's own search page would:
101
+ * a document must match every word, a title hit weighs far more than a body
102
+ * hit, and only when nothing matches every word does any-word matching kick
103
+ * in, and then the result page says so.
104
+ * @param {any} index
105
+ * @param {string} query
106
+ */
107
+ export function searchIndex(index, query) {
108
+ const words = [...new Set(
109
+ query.toLowerCase().split(/\s+/)
110
+ .map((w) => w.replace(/^[^\w.]+|[^\w.]+$/g, ''))
111
+ .filter(Boolean))];
112
+ const scores = new Map();
113
+ const matched = new Map();
114
+ for (const word of words) {
115
+ const perDoc = new Map();
116
+ const body = lookup(index.terms, word);
117
+ if (body) for (const d of body.docs) perDoc.set(d, body.exact ? 5 : 2);
118
+ const title = lookup(index.titleterms, word);
119
+ if (title) for (const d of title.docs) perDoc.set(d, (perDoc.get(d) ?? 0) + (title.exact ? 15 : 5));
120
+ for (const [d, score] of perDoc) {
121
+ scores.set(d, (scores.get(d) ?? 0) + score);
122
+ matched.set(d, (matched.get(d) ?? 0) + 1);
123
+ }
124
+ }
125
+ let docs = [...scores.keys()].filter((d) => matched.get(d) === words.length);
126
+ const partial = !docs.length && words.length > 1 && scores.size > 0;
127
+ if (partial) docs = [...scores.keys()];
128
+ docs.sort((a, b) =>
129
+ scores.get(b) - scores.get(a)
130
+ || String(index.titles[a]).localeCompare(String(index.titles[b])));
131
+ return {
132
+ words,
133
+ partial,
134
+ total: docs.length,
135
+ objects: objectHits(index, query),
136
+ docs: docs.slice(0, MAX_RESULTS).map((d) => ({
137
+ doc: d,
138
+ title: plainTitle(index.titles[d]) || index.docnames[d],
139
+ })),
140
+ };
141
+ }
142
+
143
+ // Titles in the index arrive as the HTML of the page's <h1>, markup and all
144
+ // (docs.python.org wraps module names in <code> spans), so they are flattened
145
+ // to text before they are placed on the results page. A title is index data
146
+ // from the network, so the walk keeps only what stands outside a bracket: no
147
+ // '<' or '>' can survive it, even from a tag the title never closes. A
148
+ // literal angle bracket in a real title arrives as an entity, so nothing
149
+ // legitimate is lost.
150
+ const plainTitle = (t) => {
151
+ let text = '';
152
+ let inTag = false;
153
+ for (const ch of String(t)) {
154
+ if (ch === '<') inTag = true;
155
+ else if (ch === '>') inTag = false;
156
+ else if (!inTag) text += ch;
157
+ }
158
+ return text.trim();
159
+ };
160
+
161
+ export const escapeHTML = (s) => String(s)
162
+ .replaceAll('&', '&amp;').replaceAll('<', '&lt;')
163
+ .replaceAll('>', '&gt;').replaceAll('"', '&quot;');
164
+
165
+ /**
166
+ * The result list becomes a small HTML page and rides the same distill and
167
+ * render path a fetched page does. That is what makes results numbered,
168
+ * followable with `do <n>`, and saved as session state, with nothing new for
169
+ * an agent to learn.
170
+ * @param {string} base
171
+ * @param {string} query
172
+ * @param {ReturnType<typeof searchIndex>} found
173
+ * @param {any} index
174
+ * @returns {string}
175
+ */
176
+ export function resultsToHTML(base, query, found, index) {
177
+ const host = new URL(base).host;
178
+ const pageURL = (doc) => new URL(`${index.docnames[doc]}.html`, base).href;
179
+ const items = found.objects.map((o) =>
180
+ `<li><a href="${escapeHTML(`${pageURL(o.doc)}#${o.anchor}`)}">${escapeHTML(o.name)}</a>`
181
+ + ` ${escapeHTML(o.type)}, in ${escapeHTML(plainTitle(index.titles[o.doc]) || index.docnames[o.doc])}</li>`);
182
+ for (const r of found.docs) {
183
+ items.push(`<li><a href="${escapeHTML(pageURL(r.doc))}">${escapeHTML(r.title)}</a></li>`);
184
+ }
185
+ const partial = found.partial ? '; no page matches every word, so these match some' : '';
186
+ const summary = items.length
187
+ ? `${found.total} page${found.total === 1 ? '' : 's'} match in the site's own search index,`
188
+ + ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
189
+ : `nothing in the site's own search index matches; try fewer or different words`;
190
+ return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
191
+ + `<p>${summary}</p>`
192
+ + (items.length ? `<ol>${items.join('')}</ol>` : '')
193
+ + `</main></body></html>`;
194
+ }
195
+
196
+ /**
197
+ * Search one Sphinx site. Returns the synthetic results page plus the URL the
198
+ * session should remember: the site's human search URL, so the state reads
199
+ * sensibly in `oc session` listings and error messages.
200
+ * @param {string} base - site root ending in '/', e.g. https://docs.python.org/3/
201
+ * @param {string} query
202
+ */
203
+ export async function sphinxSearch(base, query) {
204
+ if (!query.trim()) throw new Error('usage: search <query>');
205
+ const { data: index, via } = await cachedFile('sphinx', new URL('searchindex.js', base).href, parseIndex);
206
+ return {
207
+ url: new URL(`search.html?q=${encodeURIComponent(query)}`, base).href,
208
+ html: resultsToHTML(base, query, searchIndex(index, query), index),
209
+ via,
210
+ };
211
+ }