@only-cli/oc 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +118 -29
- package/clis/developer.mozilla.org.json +9 -0
- package/clis/doc.rust-lang.org.json +8 -0
- package/clis/docs.oracle.com.json +7 -0
- package/clis/docs.python.org.json +8 -0
- package/clis/docs.ruby-lang.org.json +7 -0
- package/clis/en.cppreference.com.json +8 -0
- package/clis/github.com.json +2 -2
- package/clis/learn.microsoft.com.json +1 -0
- package/clis/nodejs.org.json +7 -0
- package/clis/php.net.json +8 -0
- package/clis/pkg.go.dev.json +7 -0
- package/clis/reddit.com.json +1 -1
- package/clis/typescriptlang.org.json +7 -0
- package/clis/wikipedia.org.json +8 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +55 -19
- package/src/act.js +9 -0
- package/src/apisearch.js +80 -0
- package/src/auth.js +46 -0
- package/src/cache.js +43 -0
- package/src/cli.js +223 -10
- package/src/cookies.js +530 -0
- package/src/fetch.js +418 -18
- package/src/nodedocs.js +177 -0
- package/src/rdoc.js +98 -0
- package/src/render.js +64 -3
- package/src/session.js +43 -4
- package/src/sites.js +173 -0
- package/src/sphinx.js +211 -0
package/src/rdoc.js
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RDoc search backend. The Ruby docs (docs.ruby-lang.org) are built with
|
|
3
|
+
* RDoc, which like Sphinx has no search server: the generated site ships its
|
|
4
|
+
* whole search index as one static file, js/search_index.js, and matches in
|
|
5
|
+
* the visitor's browser. So `search` ranks that file locally: every class,
|
|
6
|
+
* module, method, and guide page in the index becomes a result linking to
|
|
7
|
+
* its own anchor. The file is ~3.4MB (~560KB over the wire) and static, so
|
|
8
|
+
* it lives in the same day cache the other local backends use, and what
|
|
9
|
+
* reaches the agent is the ranked result list only.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { cachedFile } from './cache.js';
|
|
13
|
+
import { escapeHTML } from './sphinx.js';
|
|
14
|
+
import { searchEntries } from './nodedocs.js';
|
|
15
|
+
|
|
16
|
+
const MAX_RESULTS = 20;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The index file is `var search_data = {...}`: JSON behind one assignment
|
|
20
|
+
* for the browser's benefit. Anything that does not parse that way, or that
|
|
21
|
+
* lacks the info rows, is not an RDoc index, which on a wrong or moved URL
|
|
22
|
+
* is the honest error, and it keeps a block page out of the cache too.
|
|
23
|
+
* @param {string} js
|
|
24
|
+
* @returns {any}
|
|
25
|
+
*/
|
|
26
|
+
export function parseRdocIndex(js) {
|
|
27
|
+
const start = js.indexOf('=');
|
|
28
|
+
if (start >= 0) {
|
|
29
|
+
try {
|
|
30
|
+
const data = JSON.parse(js.slice(start + 1));
|
|
31
|
+
if (Array.isArray(data?.index?.info)) return data;
|
|
32
|
+
} catch {}
|
|
33
|
+
}
|
|
34
|
+
throw new Error('not an RDoc search index');
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Flatten the index's info rows into the entries the shared ranker scores.
|
|
39
|
+
* A row is [name, namespace, path, params, snippet]; the path's own anchor
|
|
40
|
+
* says what the row is, so 'dig' in 'Array' with anchor method-i-dig reads
|
|
41
|
+
* back as the heading a rubyist expects, Array#dig(*args). Snippets stay
|
|
42
|
+
* behind: matching on them would rank prose over the symbol asked for.
|
|
43
|
+
* @param {any} data - parsed search_index.js
|
|
44
|
+
* @returns {{text: string, name: string, kind: string, path: string}[]}
|
|
45
|
+
*/
|
|
46
|
+
export function buildRdocEntries(data) {
|
|
47
|
+
return data.index.info.map(([name, namespace, path, params]) => {
|
|
48
|
+
const p = String(path ?? '');
|
|
49
|
+
const kind = p.includes('#method-c-') ? 'class method'
|
|
50
|
+
: p.includes('#method-i-') ? 'method'
|
|
51
|
+
: /^[A-Z]/.test(String(name)) ? 'class' : 'page';
|
|
52
|
+
const text = kind === 'class method' ? `${namespace}.${name}${params}`
|
|
53
|
+
: kind === 'method' ? `${namespace}#${name}${params}`
|
|
54
|
+
: namespace ? `${namespace}::${name}` : String(name ?? '');
|
|
55
|
+
return { text, name: String(name ?? ''), kind, path: p };
|
|
56
|
+
}).filter((e) => e.text && e.path);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* The result list becomes the same small synthetic page the other search
|
|
61
|
+
* backends emit, so it distills, renders, numbers, and remembers like any
|
|
62
|
+
* fetched page and `do <n>` follows a result.
|
|
63
|
+
* @param {string} base
|
|
64
|
+
* @param {string} query
|
|
65
|
+
* @param {ReturnType<typeof searchEntries>} found
|
|
66
|
+
* @returns {string}
|
|
67
|
+
*/
|
|
68
|
+
export function resultsToHTML(base, query, found) {
|
|
69
|
+
const host = new URL(base).host;
|
|
70
|
+
const items = found.hits.map((e) =>
|
|
71
|
+
`<li><a href="${escapeHTML(new URL(e.path, base).href)}">${escapeHTML(e.text)}</a>`
|
|
72
|
+
+ ` ${escapeHTML(e.kind)}</li>`);
|
|
73
|
+
const partial = found.partial ? '; no entry matches every word, so these match some' : '';
|
|
74
|
+
const summary = items.length
|
|
75
|
+
? `${found.total} entr${found.total === 1 ? 'y matches' : 'ies match'} in the docs' own index,`
|
|
76
|
+
+ ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
|
|
77
|
+
: `nothing in the docs' own index matches; try fewer or different words`;
|
|
78
|
+
return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
|
|
79
|
+
+ `<p>${summary}</p>`
|
|
80
|
+
+ (items.length ? `<ol>${items.join('')}</ol>` : '')
|
|
81
|
+
+ `</main></body></html>`;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Search one RDoc site. The site has no search URL of its own to remember,
|
|
86
|
+
* so the session keeps the docs root, the page a reader would start from.
|
|
87
|
+
* @param {string} base - docs root ending in '/', e.g. https://docs.ruby-lang.org/en/3.4/
|
|
88
|
+
* @param {string} query
|
|
89
|
+
*/
|
|
90
|
+
export async function rdocSearch(base, query) {
|
|
91
|
+
if (!query.trim()) throw new Error('usage: search <query>');
|
|
92
|
+
const { data, via } = await cachedFile('rdoc', new URL('js/search_index.js', base).href, parseRdocIndex);
|
|
93
|
+
return {
|
|
94
|
+
url: new URL('index.html', base).href,
|
|
95
|
+
html: resultsToHTML(base, query, searchEntries(buildRdocEntries(data), query)),
|
|
96
|
+
via,
|
|
97
|
+
};
|
|
98
|
+
}
|
package/src/render.js
CHANGED
|
@@ -16,6 +16,67 @@ export const estimateTokens = (s) => Math.ceil(s.length / 4);
|
|
|
16
16
|
|
|
17
17
|
const num = (v) => v.toLocaleString('en-US');
|
|
18
18
|
|
|
19
|
+
// A page that distilled to nothing must not read like a page with nothing on
|
|
20
|
+
// it. JS-only pages, consent walls, and bot challenges all answer HTTP 200 with
|
|
21
|
+
// markup carrying no text, and "100% saved" is technically true of a render
|
|
22
|
+
// that saved every token by extracting none. From the output alone an agent
|
|
23
|
+
// cannot tell that from a genuinely empty page, so it never falls back to
|
|
24
|
+
// something heavier and an empty result travels on as evidence. oc has to fail
|
|
25
|
+
// loud and cheap there instead, per principle 5 in CONTRIBUTING.
|
|
26
|
+
//
|
|
27
|
+
// The numbers below are measured, not guessed. Against live pages, the thinnest
|
|
28
|
+
// render the README claims (an X profile) carries about 460 tokens of content,
|
|
29
|
+
// while a JS-only or gated one carries under 55 (reddit.com/r/*,
|
|
30
|
+
// instagram.com), so a threshold in between never has to be a close call.
|
|
31
|
+
|
|
32
|
+
// Text the page itself wrote, in tokens: prose and headings, plus link and
|
|
33
|
+
// button labels long enough to be content rather than furniture. Nav chrome is
|
|
34
|
+
// short by nature ("Help", "Log in") and a headline or a post title is not,
|
|
35
|
+
// which is what tells a link-list page (Hacker News, search results) from a
|
|
36
|
+
// page whose only links are its own menu.
|
|
37
|
+
const CONTENT_LABEL = 25;
|
|
38
|
+
// Below this a render is suspiciously thin, but thin is only a verdict when
|
|
39
|
+
// the page's own size says there should have been more. A terse page that
|
|
40
|
+
// arrived terse (a status endpoint, a one-line answer) distilled fine.
|
|
41
|
+
export const MIN_CONTENT = 25;
|
|
42
|
+
// Below this, with markup that large behind it, the fetch worked and the render
|
|
43
|
+
// did not: a real page of that weight always distills to more. A genuinely
|
|
44
|
+
// short page is exempt, because its HTML never reaches THIN_HTML.
|
|
45
|
+
const THIN_CONTENT = 100;
|
|
46
|
+
const THIN_HTML = 2500;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* How much of a distilled page is text the page itself wrote.
|
|
50
|
+
* @param {import('./distill.js').Page} page
|
|
51
|
+
* @returns {number}
|
|
52
|
+
*/
|
|
53
|
+
export const contentTokens = (page) =>
|
|
54
|
+
page.blocks.reduce((sum, b) => {
|
|
55
|
+
if (b.type === 'heading' || b.type === 'text') return sum + estimateTokens(b.text ?? '');
|
|
56
|
+
const label = (b.type === 'link' || b.type === 'button') && (b.text ?? '').length > CONTENT_LABEL;
|
|
57
|
+
return label ? sum + estimateTokens(b.text) : sum;
|
|
58
|
+
}, 0);
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Why this render carries no content worth printing, as a phrase for the
|
|
62
|
+
* failure line, or null when it does carry some. Both figures are token counts,
|
|
63
|
+
* which is the unit the caller is paying in.
|
|
64
|
+
* @param {number} content
|
|
65
|
+
* @param {number} htmlTokens
|
|
66
|
+
* @returns {string|null}
|
|
67
|
+
*/
|
|
68
|
+
export function contentFailure(content, htmlTokens) {
|
|
69
|
+
// Nothing extracted is empty whatever the page weighed. Anything more is
|
|
70
|
+
// only a failure with evidence: a small page that renders small is not
|
|
71
|
+
// gated, it is small, and exit 2 on it would send an agent to a browser
|
|
72
|
+
// for a page it was already holding.
|
|
73
|
+
if (content === 0) return 'no text on the whole page';
|
|
74
|
+
if (content < THIN_CONTENT && htmlTokens > THIN_HTML) {
|
|
75
|
+
return `~${content} tokens of text out of ~${htmlTokens} of HTML`;
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
|
|
19
80
|
// How far past the budget a page may run and still be printed whole. Cutting a
|
|
20
81
|
// page that was nearly done costs the agent a second command, and a command is
|
|
21
82
|
// dear: measured inside Claude Code, one tool call is 23,000 to 33,000 tokens of
|
|
@@ -36,7 +97,7 @@ export const FINISH = 4;
|
|
|
36
97
|
*/
|
|
37
98
|
export function render(page, { budget = 500, from = 0 } = {}) {
|
|
38
99
|
const blocks = collapseRuns(page.blocks);
|
|
39
|
-
const head = page.title ? [from > 0 ? `# ${page.title} (continued)` : `# ${page.title}`] : [];
|
|
100
|
+
const head = page.title ? [from > 0 ? `# ${truncate(page.title)} (continued)` : `# ${truncate(page.title)}`] : [];
|
|
40
101
|
const lines = [...head];
|
|
41
102
|
let spent = estimateTokens(lines.join('\n'));
|
|
42
103
|
let hasLinks = false;
|
|
@@ -148,13 +209,13 @@ export function formatBlock(b, { full = false } = {}) {
|
|
|
148
209
|
const tag = b.n == null ? '' : `[${b.n}] `;
|
|
149
210
|
switch (b.type) {
|
|
150
211
|
case 'heading':
|
|
151
|
-
return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${tag}${b.text}`;
|
|
212
|
+
return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${tag}${full ? b.text : truncate(b.text)}`;
|
|
152
213
|
case 'link':
|
|
153
214
|
return `${tag}${full ? b.text : truncate(b.text)}`;
|
|
154
215
|
case 'button':
|
|
155
216
|
return `${tag}button "${full ? b.text : truncate(b.text)}"`;
|
|
156
217
|
case 'input':
|
|
157
|
-
return `${tag}input ${b.name} (${b.text})`;
|
|
218
|
+
return `${tag}input ${truncate(b.name ?? '')} (${truncate(b.text ?? '')})`;
|
|
158
219
|
case 'divider':
|
|
159
220
|
return b.text;
|
|
160
221
|
default:
|
package/src/session.js
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Sessions are plain JSON files on disk, one per name: the current URL, the
|
|
3
3
|
* distilled blocks of the page it holds, how far the last render got through
|
|
4
|
-
* them, and a short history. No daemon, no background process
|
|
4
|
+
* them, and a short history. No daemon, no background process; cookies live in
|
|
5
|
+
* a separate sidecar file (see cookies.js).
|
|
5
6
|
*
|
|
6
7
|
* The file exists so `oc do <n>` can follow a link the compact view never
|
|
7
8
|
* printed the URL of. Hiding URLs is what makes `oc open` cheap; this is what
|
|
@@ -11,18 +12,35 @@
|
|
|
11
12
|
|
|
12
13
|
import { homedir } from 'node:os';
|
|
13
14
|
import { join } from 'node:path';
|
|
14
|
-
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
15
|
+
import { mkdirSync, readFileSync, writeFileSync, chmodSync, unlinkSync } from 'node:fs';
|
|
15
16
|
|
|
16
17
|
export const DEFAULT_SESSION = 'default';
|
|
17
18
|
|
|
18
19
|
// OC_HOME relocates the whole state directory, for sandboxes, CI, and tests.
|
|
19
20
|
export const sessionDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sessions');
|
|
20
21
|
|
|
22
|
+
// A session name is interpolated straight into a filename, and the cookie
|
|
23
|
+
// sidecar it names now holds real credentials, so a name that is a path
|
|
24
|
+
// (absolute, or with a separator or '..') could write or delete a file outside
|
|
25
|
+
// the store. Names are user-facing labels, so this charset loses nothing real.
|
|
26
|
+
const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @param {string} name
|
|
30
|
+
* @returns {string} the same name, once it is known to be a safe filename
|
|
31
|
+
*/
|
|
32
|
+
export function assertSafeName(name) {
|
|
33
|
+
if (typeof name !== 'string' || name === '.' || name === '..' || !SAFE_NAME.test(name)) {
|
|
34
|
+
throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_'`);
|
|
35
|
+
}
|
|
36
|
+
return name;
|
|
37
|
+
}
|
|
38
|
+
|
|
21
39
|
/**
|
|
22
40
|
* @param {string} name
|
|
23
41
|
* @returns {string}
|
|
24
42
|
*/
|
|
25
|
-
export const sessionPath = (name) => join(sessionDir(), `${name}.json`);
|
|
43
|
+
export const sessionPath = (name) => join(sessionDir(), `${assertSafeName(name)}.json`);
|
|
26
44
|
|
|
27
45
|
// Search engines and link aggregators wrap outbound links in a tracking
|
|
28
46
|
// redirector whose landing page is a script, not content, so following one
|
|
@@ -136,7 +154,28 @@ export function handleNumbers(state) {
|
|
|
136
154
|
*/
|
|
137
155
|
export function saveSession(name, state) {
|
|
138
156
|
mkdirSync(sessionDir(), { recursive: true });
|
|
139
|
-
|
|
157
|
+
// A snapshot of an authenticated page holds that page's text, so it gets the
|
|
158
|
+
// same owner-only mode as the cookie sidecar. writeFileSync only sets the
|
|
159
|
+
// mode on create, so a snapshot left world-readable by an older version is
|
|
160
|
+
// tightened explicitly on the next save.
|
|
161
|
+
const path = sessionPath(name);
|
|
162
|
+
writeFileSync(path, JSON.stringify(state), { mode: 0o600 });
|
|
163
|
+
chmodSync(path, 0o600);
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* Drop a saved page. `oc logout` calls this alongside clearing the cookie jar:
|
|
168
|
+
* a snapshot taken under a login holds that page's text, so leaving it behind
|
|
169
|
+
* would make logout mean "the cookies are gone" rather than "nothing of this
|
|
170
|
+
* login remains".
|
|
171
|
+
* @param {string} name
|
|
172
|
+
*/
|
|
173
|
+
export function clearSession(name) {
|
|
174
|
+
try {
|
|
175
|
+
unlinkSync(sessionPath(name));
|
|
176
|
+
} catch {
|
|
177
|
+
// nothing saved under that name is fine
|
|
178
|
+
}
|
|
140
179
|
}
|
|
141
180
|
|
|
142
181
|
/**
|
package/src/sites.js
ADDED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Site shortcuts. `clis/*.json` names the URLs on a site worth reaching
|
|
3
|
+
* directly, so `oc hn item 4711` gets there without the agent knowing that
|
|
4
|
+
* Hacker News spells it /item?id=. A shortcut is almost always a URL: it
|
|
5
|
+
* resolves to one and hands off to the same fetch and render path `oc open`
|
|
6
|
+
* uses, so nothing here can change what a page costs or how it reads. The
|
|
7
|
+
* other shapes are searches cli.js runs itself and renders like any other
|
|
8
|
+
* page: `sphinx` and `rdoc`, for docs sites whose search only exists as a
|
|
9
|
+
* static index file, `nodedoc`, for the Node.js API docs, which ship their
|
|
10
|
+
* reference the same way, and `api`, for a site whose search answers as
|
|
11
|
+
* JSON.
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
15
|
+
import { fileURLToPath } from 'node:url';
|
|
16
|
+
|
|
17
|
+
const DIR = fileURLToPath(new URL('../clis/', import.meta.url));
|
|
18
|
+
|
|
19
|
+
// Short names an agent is likely to reach for. The domain itself and its
|
|
20
|
+
// registrable label always resolve, so this only covers what those miss.
|
|
21
|
+
const ALIASES = {
|
|
22
|
+
hn: 'news.ycombinator.com',
|
|
23
|
+
gh: 'github.com',
|
|
24
|
+
so: 'stackoverflow.com',
|
|
25
|
+
ddg: 'duckduckgo.com',
|
|
26
|
+
yt: 'youtube.com',
|
|
27
|
+
finance: 'finance.yahoo.com',
|
|
28
|
+
twitter: 'x.com',
|
|
29
|
+
aws: 'docs.aws.amazon.com',
|
|
30
|
+
py: 'docs.python.org',
|
|
31
|
+
mdn: 'developer.mozilla.org',
|
|
32
|
+
node: 'nodejs.org',
|
|
33
|
+
rust: 'doc.rust-lang.org',
|
|
34
|
+
java: 'docs.oracle.com',
|
|
35
|
+
ruby: 'docs.ruby-lang.org',
|
|
36
|
+
cpp: 'en.cppreference.com',
|
|
37
|
+
ts: 'typescriptlang.org',
|
|
38
|
+
gcp: 'cloud.google.com',
|
|
39
|
+
learn: 'learn.microsoft.com',
|
|
40
|
+
wiki: 'wikipedia.org',
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
|
|
44
|
+
/** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */
|
|
45
|
+
|
|
46
|
+
/** @type {Map<string, Site>|null} */
|
|
47
|
+
let cache = null;
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Every site definition that ships with oc, keyed by each name that resolves
|
|
51
|
+
* to it. A definition that will not parse is skipped rather than fatal: a bad
|
|
52
|
+
* file costs its own shortcuts and leaves every other command working.
|
|
53
|
+
* @returns {Map<string, Site>}
|
|
54
|
+
*/
|
|
55
|
+
export function sites() {
|
|
56
|
+
if (cache) return cache;
|
|
57
|
+
cache = new Map();
|
|
58
|
+
/** @type {string[]} */
|
|
59
|
+
let files = [];
|
|
60
|
+
try {
|
|
61
|
+
files = readdirSync(DIR).filter((f) => f.endsWith('.json')).sort();
|
|
62
|
+
} catch {
|
|
63
|
+
return cache;
|
|
64
|
+
}
|
|
65
|
+
for (const file of files) {
|
|
66
|
+
/** @type {Site} */
|
|
67
|
+
let site;
|
|
68
|
+
try {
|
|
69
|
+
site = JSON.parse(readFileSync(DIR + file, 'utf8'));
|
|
70
|
+
} catch {
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (!site?.domain || !site.commands) continue;
|
|
74
|
+
const labels = site.domain.split('.');
|
|
75
|
+
for (const key of [site.domain, labels[labels.length - 2]]) {
|
|
76
|
+
// First definition wins, in sorted filename order, so which site owns a
|
|
77
|
+
// shared label never depends on how the directory happens to be read.
|
|
78
|
+
if (key && !cache.has(key)) cache.set(key, site);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
for (const [alias, domain] of Object.entries(ALIASES)) {
|
|
82
|
+
const site = cache.get(domain);
|
|
83
|
+
if (site && !cache.has(alias)) cache.set(alias, site);
|
|
84
|
+
}
|
|
85
|
+
return cache;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const verbs = (site) =>
|
|
89
|
+
Object.entries(site.commands)
|
|
90
|
+
.map(([name, def]) => (def.args?.length ? `${name} <${def.args.join('> <')}>` : name))
|
|
91
|
+
.join(' | ');
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Resolve `oc <site> <verb> [args]` to a URL, or null when the first word
|
|
95
|
+
* names no site oc ships, which is the caller's cue to report an unknown
|
|
96
|
+
* command. A site that exists with a verb that does not is an error here
|
|
97
|
+
* instead, since the agent has the right site and only needs the verb list.
|
|
98
|
+
* @param {string} name
|
|
99
|
+
* @param {string[]} args
|
|
100
|
+
* @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
|
|
101
|
+
*/
|
|
102
|
+
export function resolveSite(name, args) {
|
|
103
|
+
const site = sites().get(name.toLowerCase());
|
|
104
|
+
if (!site) return null;
|
|
105
|
+
const [verb, ...rest] = args;
|
|
106
|
+
if (!verb) throw new Error(`usage: oc ${name} <verb>, one of: ${verbs(site)}`);
|
|
107
|
+
const def = site.commands[verb];
|
|
108
|
+
if (!def) throw new Error(`'${verb}' is not a ${site.domain} shortcut, try: ${verbs(site)}`);
|
|
109
|
+
|
|
110
|
+
const need = def.args ?? [];
|
|
111
|
+
if (rest.length < need.length) {
|
|
112
|
+
throw new Error(`usage: oc ${name} ${verb} <${need.join('> <')}>`);
|
|
113
|
+
}
|
|
114
|
+
// The last argument soaks up everything left, so a query an agent typed as
|
|
115
|
+
// separate words ('oc ddg search claude code cli') works unquoted.
|
|
116
|
+
const values = need.map((_, i) =>
|
|
117
|
+
i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]);
|
|
118
|
+
// A search oc runs itself has no page URL to build: the query is handed
|
|
119
|
+
// back whole for cli.js to run against the site's own search. The local
|
|
120
|
+
// backends need only their docs root; the API shape needs its whole
|
|
121
|
+
// definition, since it names the endpoint and the response fields.
|
|
122
|
+
for (const kind of ['sphinx', 'nodedoc', 'rdoc']) {
|
|
123
|
+
if (def[kind]) {
|
|
124
|
+
return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
if (def.api) {
|
|
128
|
+
return { api: def, query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
|
|
129
|
+
}
|
|
130
|
+
const url = need.reduce(
|
|
131
|
+
(open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)),
|
|
132
|
+
def.open);
|
|
133
|
+
return { url, domain: site.domain, command: verb };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Percent-encode one value for the slot it fills. A value in the query string
|
|
138
|
+
* is encoded outright, but a docs path is often several segments deep, so
|
|
139
|
+
* 'oc learn doc azure/aks/intro' has to keep its slashes: escaping them would
|
|
140
|
+
* ask the site for one impossible segment instead of the page.
|
|
141
|
+
* @param {string} value
|
|
142
|
+
* @param {string} template
|
|
143
|
+
* @param {string} arg
|
|
144
|
+
* @returns {string}
|
|
145
|
+
*/
|
|
146
|
+
function encode(value, template, arg) {
|
|
147
|
+
const query = template.includes('?') && template.indexOf(`{${arg}}`) > template.indexOf('?');
|
|
148
|
+
const encoded = encodeURIComponent(value);
|
|
149
|
+
return query ? encoded : encoded.replaceAll('%2F', '/');
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* One line per site for `oc sites`, shortest name first, since that is what an
|
|
154
|
+
* agent will type. Discovery has to cost less than a wrong guess does, so this
|
|
155
|
+
* stays one line each rather than a table.
|
|
156
|
+
* @returns {string}
|
|
157
|
+
*/
|
|
158
|
+
export function listSites() {
|
|
159
|
+
const byDomain = new Map();
|
|
160
|
+
for (const [key, site] of sites()) {
|
|
161
|
+
if (!byDomain.has(site.domain)) byDomain.set(site.domain, { site, keys: [] });
|
|
162
|
+
byDomain.get(site.domain).keys.push(key);
|
|
163
|
+
}
|
|
164
|
+
const lines = [...byDomain.values()].map(({ site, keys }) => {
|
|
165
|
+
const names = keys
|
|
166
|
+
.filter((k) => k !== site.domain)
|
|
167
|
+
.sort((a, b) => a.length - b.length || a.localeCompare(b));
|
|
168
|
+
return `oc ${names[0] ?? site.domain} <verb> (${[...names.slice(1), site.domain].join(', ')}): ${verbs(site)}`;
|
|
169
|
+
});
|
|
170
|
+
return lines.length
|
|
171
|
+
? `${lines.join('\n')}\nany other site: oc open <url>`
|
|
172
|
+
: 'no site definitions found; oc open <url> works on any site';
|
|
173
|
+
}
|
package/src/sphinx.js
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sphinx search backend. A Sphinx-built documentation site (docs.python.org,
|
|
3
|
+
* most Read the Docs projects) has no search server: its search page ships
|
|
4
|
+
* the site's entire full-text index as one static file, searchindex.js, and
|
|
5
|
+
* ranks matches in the visitor's browser. oc can run the same ranking here,
|
|
6
|
+
* so `search` on such a site answers from the site's own index instead of a
|
|
7
|
+
* third-party engine. The index is big (docs.python.org's is ~4MB, ~900KB
|
|
8
|
+
* over the wire) but static, so it is cached on disk for a day and never
|
|
9
|
+
* printed: what reaches the agent is only the ranked result list.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { cachedFile } from './cache.js';
|
|
13
|
+
|
|
14
|
+
const MAX_RESULTS = 20;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The index file is `Search.setIndex({...})`: JSON wrapped in one function
|
|
18
|
+
* call for the browser's benefit. Anything that does not parse that way is
|
|
19
|
+
* not a Sphinx index, which on a wrong or moved URL is the honest error.
|
|
20
|
+
* @param {string} js
|
|
21
|
+
* @returns {any}
|
|
22
|
+
*/
|
|
23
|
+
export function parseIndex(js) {
|
|
24
|
+
const start = js.indexOf('(');
|
|
25
|
+
const end = js.lastIndexOf(')');
|
|
26
|
+
if (start >= 0 && end > start) {
|
|
27
|
+
try {
|
|
28
|
+
return JSON.parse(js.slice(start + 1, end));
|
|
29
|
+
} catch {}
|
|
30
|
+
}
|
|
31
|
+
throw new Error('not a Sphinx search index');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// terms and titleterms store a bare number when a word appears in one
|
|
35
|
+
// document and an array when it appears in several.
|
|
36
|
+
const docsFor = (table, word) => {
|
|
37
|
+
const hit = table?.[word];
|
|
38
|
+
return hit == null ? null : Array.isArray(hit) ? hit : [hit];
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Sphinx stems words before indexing ('threading' is stored as 'thread'), so
|
|
43
|
+
* an exact lookup misses common query spellings. Rather than shipping the
|
|
44
|
+
* Porter stemmer, try the word with common suffixes stripped, and only then
|
|
45
|
+
* a prefix scan: an index key that extends the word, or that the word
|
|
46
|
+
* extends, counts at reduced weight.
|
|
47
|
+
* @param {Record<string, number|number[]>} table
|
|
48
|
+
* @param {string} word
|
|
49
|
+
* @returns {{docs: number[], exact: boolean}|null}
|
|
50
|
+
*/
|
|
51
|
+
function lookup(table, word) {
|
|
52
|
+
const exact = docsFor(table, word);
|
|
53
|
+
if (exact) return { docs: exact, exact: true };
|
|
54
|
+
for (const suffix of ['ing', 'ed', 'es', 's', 'e']) {
|
|
55
|
+
if (word.length - suffix.length >= 3 && word.endsWith(suffix)) {
|
|
56
|
+
const hit = docsFor(table, word.slice(0, -suffix.length));
|
|
57
|
+
if (hit) return { docs: hit, exact: false };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (word.length >= 4) {
|
|
61
|
+
const docs = new Set();
|
|
62
|
+
for (const key of Object.keys(table ?? {})) {
|
|
63
|
+
if (key.length >= 4 && (key.startsWith(word) || word.startsWith(key))) {
|
|
64
|
+
for (const d of docsFor(table, key)) docs.add(d);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (docs.size) return { docs: [...docs], exact: false };
|
|
68
|
+
}
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* An exact object hit ('json.dumps', or just 'dumps') beats any full-text
|
|
74
|
+
* rank: the index maps the symbol straight to its anchor on the page, so it
|
|
75
|
+
* goes at the top as a direct link. Only single-word queries can be symbols.
|
|
76
|
+
* @param {any} index
|
|
77
|
+
* @param {string} query
|
|
78
|
+
*/
|
|
79
|
+
function objectHits(index, query) {
|
|
80
|
+
const q = query.trim().toLowerCase();
|
|
81
|
+
if (!q || q.includes(' ')) return [];
|
|
82
|
+
const hits = [];
|
|
83
|
+
for (const [prefix, entries] of Object.entries(index.objects ?? {})) {
|
|
84
|
+
for (const [doc, typeIdx, priority, anchor, name] of entries) {
|
|
85
|
+
const full = prefix ? `${prefix}.${name}` : name;
|
|
86
|
+
if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
|
|
87
|
+
hits.push({
|
|
88
|
+
name: full,
|
|
89
|
+
type: index.objnames?.[typeIdx]?.[2] ?? '',
|
|
90
|
+
doc,
|
|
91
|
+
anchor: anchor === '' ? full : anchor,
|
|
92
|
+
priority,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return hits.sort((a, b) => a.priority - b.priority).slice(0, 5);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Rank the index against a query the way the site's own search page would:
|
|
101
|
+
* a document must match every word, a title hit weighs far more than a body
|
|
102
|
+
* hit, and only when nothing matches every word does any-word matching kick
|
|
103
|
+
* in, and then the result page says so.
|
|
104
|
+
* @param {any} index
|
|
105
|
+
* @param {string} query
|
|
106
|
+
*/
|
|
107
|
+
export function searchIndex(index, query) {
|
|
108
|
+
const words = [...new Set(
|
|
109
|
+
query.toLowerCase().split(/\s+/)
|
|
110
|
+
.map((w) => w.replace(/^[^\w.]+|[^\w.]+$/g, ''))
|
|
111
|
+
.filter(Boolean))];
|
|
112
|
+
const scores = new Map();
|
|
113
|
+
const matched = new Map();
|
|
114
|
+
for (const word of words) {
|
|
115
|
+
const perDoc = new Map();
|
|
116
|
+
const body = lookup(index.terms, word);
|
|
117
|
+
if (body) for (const d of body.docs) perDoc.set(d, body.exact ? 5 : 2);
|
|
118
|
+
const title = lookup(index.titleterms, word);
|
|
119
|
+
if (title) for (const d of title.docs) perDoc.set(d, (perDoc.get(d) ?? 0) + (title.exact ? 15 : 5));
|
|
120
|
+
for (const [d, score] of perDoc) {
|
|
121
|
+
scores.set(d, (scores.get(d) ?? 0) + score);
|
|
122
|
+
matched.set(d, (matched.get(d) ?? 0) + 1);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
let docs = [...scores.keys()].filter((d) => matched.get(d) === words.length);
|
|
126
|
+
const partial = !docs.length && words.length > 1 && scores.size > 0;
|
|
127
|
+
if (partial) docs = [...scores.keys()];
|
|
128
|
+
docs.sort((a, b) =>
|
|
129
|
+
scores.get(b) - scores.get(a)
|
|
130
|
+
|| String(index.titles[a]).localeCompare(String(index.titles[b])));
|
|
131
|
+
return {
|
|
132
|
+
words,
|
|
133
|
+
partial,
|
|
134
|
+
total: docs.length,
|
|
135
|
+
objects: objectHits(index, query),
|
|
136
|
+
docs: docs.slice(0, MAX_RESULTS).map((d) => ({
|
|
137
|
+
doc: d,
|
|
138
|
+
title: plainTitle(index.titles[d]) || index.docnames[d],
|
|
139
|
+
})),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Titles in the index arrive as the HTML of the page's <h1>, markup and all
|
|
144
|
+
// (docs.python.org wraps module names in <code> spans), so they are flattened
|
|
145
|
+
// to text before they are placed on the results page. A title is index data
|
|
146
|
+
// from the network, so the walk keeps only what stands outside a bracket: no
|
|
147
|
+
// '<' or '>' can survive it, even from a tag the title never closes. A
|
|
148
|
+
// literal angle bracket in a real title arrives as an entity, so nothing
|
|
149
|
+
// legitimate is lost.
|
|
150
|
+
const plainTitle = (t) => {
|
|
151
|
+
let text = '';
|
|
152
|
+
let inTag = false;
|
|
153
|
+
for (const ch of String(t)) {
|
|
154
|
+
if (ch === '<') inTag = true;
|
|
155
|
+
else if (ch === '>') inTag = false;
|
|
156
|
+
else if (!inTag) text += ch;
|
|
157
|
+
}
|
|
158
|
+
return text.trim();
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
export const escapeHTML = (s) => String(s)
|
|
162
|
+
.replaceAll('&', '&').replaceAll('<', '<')
|
|
163
|
+
.replaceAll('>', '>').replaceAll('"', '"');
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The result list becomes a small HTML page and rides the same distill and
|
|
167
|
+
* render path a fetched page does. That is what makes results numbered,
|
|
168
|
+
* followable with `do <n>`, and saved as session state, with nothing new for
|
|
169
|
+
* an agent to learn.
|
|
170
|
+
* @param {string} base
|
|
171
|
+
* @param {string} query
|
|
172
|
+
* @param {ReturnType<typeof searchIndex>} found
|
|
173
|
+
* @param {any} index
|
|
174
|
+
* @returns {string}
|
|
175
|
+
*/
|
|
176
|
+
export function resultsToHTML(base, query, found, index) {
|
|
177
|
+
const host = new URL(base).host;
|
|
178
|
+
const pageURL = (doc) => new URL(`${index.docnames[doc]}.html`, base).href;
|
|
179
|
+
const items = found.objects.map((o) =>
|
|
180
|
+
`<li><a href="${escapeHTML(`${pageURL(o.doc)}#${o.anchor}`)}">${escapeHTML(o.name)}</a>`
|
|
181
|
+
+ ` ${escapeHTML(o.type)}, in ${escapeHTML(plainTitle(index.titles[o.doc]) || index.docnames[o.doc])}</li>`);
|
|
182
|
+
for (const r of found.docs) {
|
|
183
|
+
items.push(`<li><a href="${escapeHTML(pageURL(r.doc))}">${escapeHTML(r.title)}</a></li>`);
|
|
184
|
+
}
|
|
185
|
+
const partial = found.partial ? '; no page matches every word, so these match some' : '';
|
|
186
|
+
const summary = items.length
|
|
187
|
+
? `${found.total} page${found.total === 1 ? '' : 's'} match in the site's own search index,`
|
|
188
|
+
+ ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
|
|
189
|
+
: `nothing in the site's own search index matches; try fewer or different words`;
|
|
190
|
+
return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
|
|
191
|
+
+ `<p>${summary}</p>`
|
|
192
|
+
+ (items.length ? `<ol>${items.join('')}</ol>` : '')
|
|
193
|
+
+ `</main></body></html>`;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Search one Sphinx site. Returns the synthetic results page plus the URL the
|
|
198
|
+
* session should remember: the site's human search URL, so the state reads
|
|
199
|
+
* sensibly in `oc session` listings and error messages.
|
|
200
|
+
* @param {string} base - site root ending in '/', e.g. https://docs.python.org/3/
|
|
201
|
+
* @param {string} query
|
|
202
|
+
*/
|
|
203
|
+
export async function sphinxSearch(base, query) {
|
|
204
|
+
if (!query.trim()) throw new Error('usage: search <query>');
|
|
205
|
+
const { data: index, via } = await cachedFile('sphinx', new URL('searchindex.js', base).href, parseIndex);
|
|
206
|
+
return {
|
|
207
|
+
url: new URL(`search.html?q=${encodeURIComponent(query)}`, base).href,
|
|
208
|
+
html: resultsToHTML(base, query, searchIndex(index, query), index),
|
|
209
|
+
via,
|
|
210
|
+
};
|
|
211
|
+
}
|