@only-cli/oc 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +107 -38
- package/clis/developer.mozilla.org.json +9 -0
- package/clis/doc.rust-lang.org.json +8 -0
- package/clis/docs.oracle.com.json +7 -0
- package/clis/docs.python.org.json +8 -0
- package/clis/docs.ruby-lang.org.json +7 -0
- package/clis/en.cppreference.com.json +8 -0
- package/clis/learn.microsoft.com.json +1 -0
- package/clis/nodejs.org.json +7 -0
- package/clis/php.net.json +8 -0
- package/clis/pkg.go.dev.json +7 -0
- package/clis/typescriptlang.org.json +7 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +42 -23
- package/src/act.js +13 -2
- package/src/apisearch.js +80 -0
- package/src/auth.js +46 -0
- package/src/cache.js +43 -0
- package/src/cli.js +167 -11
- package/src/cookies.js +530 -0
- package/src/fetch.js +142 -19
- package/src/nodedocs.js +177 -0
- package/src/rdoc.js +98 -0
- package/src/render.js +21 -10
- package/src/session.js +43 -4
- package/src/sites.js +30 -5
- package/src/sphinx.js +211 -0
package/src/sites.js
CHANGED
|
@@ -1,9 +1,14 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Site shortcuts. `clis/*.json` names the URLs on a site worth reaching
|
|
3
3
|
* directly, so `oc hn item 4711` gets there without the agent knowing that
|
|
4
|
-
* Hacker News spells it /item?id=. A shortcut is
|
|
5
|
-
* to one and hands off to the same fetch and render path `oc open`
|
|
6
|
-
* nothing here can change what a page costs or how it reads.
|
|
4
|
+
* Hacker News spells it /item?id=. A shortcut is almost always a URL: it
|
|
5
|
+
* resolves to one and hands off to the same fetch and render path `oc open`
|
|
6
|
+
* uses, so nothing here can change what a page costs or how it reads. The
|
|
7
|
+
* other shapes are searches cli.js runs itself and renders like any other
|
|
8
|
+
* page: `sphinx` and `rdoc`, for docs sites whose search only exists as a
|
|
9
|
+
* static index file, `nodedoc`, for the Node.js API docs, which ship their
|
|
10
|
+
* reference the same way, and `api`, for a site whose search answers as
|
|
11
|
+
* JSON.
|
|
7
12
|
*/
|
|
8
13
|
|
|
9
14
|
import { readdirSync, readFileSync } from 'node:fs';
|
|
@@ -22,12 +27,20 @@ const ALIASES = {
|
|
|
22
27
|
finance: 'finance.yahoo.com',
|
|
23
28
|
twitter: 'x.com',
|
|
24
29
|
aws: 'docs.aws.amazon.com',
|
|
30
|
+
py: 'docs.python.org',
|
|
31
|
+
mdn: 'developer.mozilla.org',
|
|
32
|
+
node: 'nodejs.org',
|
|
33
|
+
rust: 'doc.rust-lang.org',
|
|
34
|
+
java: 'docs.oracle.com',
|
|
35
|
+
ruby: 'docs.ruby-lang.org',
|
|
36
|
+
cpp: 'en.cppreference.com',
|
|
37
|
+
ts: 'typescriptlang.org',
|
|
25
38
|
gcp: 'cloud.google.com',
|
|
26
39
|
learn: 'learn.microsoft.com',
|
|
27
40
|
wiki: 'wikipedia.org',
|
|
28
41
|
};
|
|
29
42
|
|
|
30
|
-
/** @typedef {{open
|
|
43
|
+
/** @typedef {{open?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: string, page?: string, results?: string, fields?: Record<string, string>, total?: string, args?: string[]}} Shortcut */
|
|
31
44
|
/** @typedef {{domain: string, commands: Record<string, Shortcut>}} Site */
|
|
32
45
|
|
|
33
46
|
/** @type {Map<string, Site>|null} */
|
|
@@ -84,7 +97,7 @@ const verbs = (site) =>
|
|
|
84
97
|
* instead, since the agent has the right site and only needs the verb list.
|
|
85
98
|
* @param {string} name
|
|
86
99
|
* @param {string[]} args
|
|
87
|
-
* @returns {{url
|
|
100
|
+
* @returns {{url?: string, sphinx?: string, nodedoc?: string, rdoc?: string, api?: Shortcut, query?: string, domain: string, command: string}|null}
|
|
88
101
|
*/
|
|
89
102
|
export function resolveSite(name, args) {
|
|
90
103
|
const site = sites().get(name.toLowerCase());
|
|
@@ -102,6 +115,18 @@ export function resolveSite(name, args) {
|
|
|
102
115
|
// separate words ('oc ddg search claude code cli') works unquoted.
|
|
103
116
|
const values = need.map((_, i) =>
|
|
104
117
|
i === need.length - 1 ? rest.slice(i).join(' ') : rest[i]);
|
|
118
|
+
// A search oc runs itself has no page URL to build: the query is handed
|
|
119
|
+
// back whole for cli.js to run against the site's own search. The local
|
|
120
|
+
// backends need only their docs root; the API shape needs its whole
|
|
121
|
+
// definition, since it names the endpoint and the response fields.
|
|
122
|
+
for (const kind of ['sphinx', 'nodedoc', 'rdoc']) {
|
|
123
|
+
if (def[kind]) {
|
|
124
|
+
return { [kind]: def[kind], query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
if (def.api) {
|
|
128
|
+
return { api: def, query: values[values.length - 1] ?? '', domain: site.domain, command: verb };
|
|
129
|
+
}
|
|
105
130
|
const url = need.reduce(
|
|
106
131
|
(open, arg, i) => open.replaceAll(`{${arg}}`, encode(values[i], def.open, arg)),
|
|
107
132
|
def.open);
|
package/src/sphinx.js
ADDED
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Sphinx search backend. A Sphinx-built documentation site (docs.python.org,
|
|
3
|
+
* most Read the Docs projects) has no search server: its search page ships
|
|
4
|
+
* the site's entire full-text index as one static file, searchindex.js, and
|
|
5
|
+
* ranks matches in the visitor's browser. oc can run the same ranking here,
|
|
6
|
+
* so `search` on such a site answers from the site's own index instead of a
|
|
7
|
+
* third-party engine. The index is big (docs.python.org's is ~4MB, ~900KB
|
|
8
|
+
* over the wire) but static, so it is cached on disk for a day and never
|
|
9
|
+
* printed: what reaches the agent is only the ranked result list.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { cachedFile } from './cache.js';
|
|
13
|
+
|
|
14
|
+
const MAX_RESULTS = 20;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The index file is `Search.setIndex({...})`: JSON wrapped in one function
|
|
18
|
+
* call for the browser's benefit. Anything that does not parse that way is
|
|
19
|
+
* not a Sphinx index, which on a wrong or moved URL is the honest error.
|
|
20
|
+
* @param {string} js
|
|
21
|
+
* @returns {any}
|
|
22
|
+
*/
|
|
23
|
+
export function parseIndex(js) {
|
|
24
|
+
const start = js.indexOf('(');
|
|
25
|
+
const end = js.lastIndexOf(')');
|
|
26
|
+
if (start >= 0 && end > start) {
|
|
27
|
+
try {
|
|
28
|
+
return JSON.parse(js.slice(start + 1, end));
|
|
29
|
+
} catch {}
|
|
30
|
+
}
|
|
31
|
+
throw new Error('not a Sphinx search index');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// terms and titleterms store a bare number when a word appears in one
|
|
35
|
+
// document and an array when it appears in several.
|
|
36
|
+
const docsFor = (table, word) => {
|
|
37
|
+
const hit = table?.[word];
|
|
38
|
+
return hit == null ? null : Array.isArray(hit) ? hit : [hit];
|
|
39
|
+
};
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Sphinx stems words before indexing ('threading' is stored as 'thread'), so
|
|
43
|
+
* an exact lookup misses common query spellings. Rather than shipping the
|
|
44
|
+
* Porter stemmer, try the word with common suffixes stripped, and only then
|
|
45
|
+
* a prefix scan: an index key that extends the word, or that the word
|
|
46
|
+
* extends, counts at reduced weight.
|
|
47
|
+
* @param {Record<string, number|number[]>} table
|
|
48
|
+
* @param {string} word
|
|
49
|
+
* @returns {{docs: number[], exact: boolean}|null}
|
|
50
|
+
*/
|
|
51
|
+
function lookup(table, word) {
|
|
52
|
+
const exact = docsFor(table, word);
|
|
53
|
+
if (exact) return { docs: exact, exact: true };
|
|
54
|
+
for (const suffix of ['ing', 'ed', 'es', 's', 'e']) {
|
|
55
|
+
if (word.length - suffix.length >= 3 && word.endsWith(suffix)) {
|
|
56
|
+
const hit = docsFor(table, word.slice(0, -suffix.length));
|
|
57
|
+
if (hit) return { docs: hit, exact: false };
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (word.length >= 4) {
|
|
61
|
+
const docs = new Set();
|
|
62
|
+
for (const key of Object.keys(table ?? {})) {
|
|
63
|
+
if (key.length >= 4 && (key.startsWith(word) || word.startsWith(key))) {
|
|
64
|
+
for (const d of docsFor(table, key)) docs.add(d);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
if (docs.size) return { docs: [...docs], exact: false };
|
|
68
|
+
}
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* An exact object hit ('json.dumps', or just 'dumps') beats any full-text
|
|
74
|
+
* rank: the index maps the symbol straight to its anchor on the page, so it
|
|
75
|
+
* goes at the top as a direct link. Only single-word queries can be symbols.
|
|
76
|
+
* @param {any} index
|
|
77
|
+
* @param {string} query
|
|
78
|
+
*/
|
|
79
|
+
function objectHits(index, query) {
|
|
80
|
+
const q = query.trim().toLowerCase();
|
|
81
|
+
if (!q || q.includes(' ')) return [];
|
|
82
|
+
const hits = [];
|
|
83
|
+
for (const [prefix, entries] of Object.entries(index.objects ?? {})) {
|
|
84
|
+
for (const [doc, typeIdx, priority, anchor, name] of entries) {
|
|
85
|
+
const full = prefix ? `${prefix}.${name}` : name;
|
|
86
|
+
if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
|
|
87
|
+
hits.push({
|
|
88
|
+
name: full,
|
|
89
|
+
type: index.objnames?.[typeIdx]?.[2] ?? '',
|
|
90
|
+
doc,
|
|
91
|
+
anchor: anchor === '' ? full : anchor,
|
|
92
|
+
priority,
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
return hits.sort((a, b) => a.priority - b.priority).slice(0, 5);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/**
|
|
100
|
+
* Rank the index against a query the way the site's own search page would:
|
|
101
|
+
* a document must match every word, a title hit weighs far more than a body
|
|
102
|
+
* hit, and only when nothing matches every word does any-word matching kick
|
|
103
|
+
* in, and then the result page says so.
|
|
104
|
+
* @param {any} index
|
|
105
|
+
* @param {string} query
|
|
106
|
+
*/
|
|
107
|
+
export function searchIndex(index, query) {
|
|
108
|
+
const words = [...new Set(
|
|
109
|
+
query.toLowerCase().split(/\s+/)
|
|
110
|
+
.map((w) => w.replace(/^[^\w.]+|[^\w.]+$/g, ''))
|
|
111
|
+
.filter(Boolean))];
|
|
112
|
+
const scores = new Map();
|
|
113
|
+
const matched = new Map();
|
|
114
|
+
for (const word of words) {
|
|
115
|
+
const perDoc = new Map();
|
|
116
|
+
const body = lookup(index.terms, word);
|
|
117
|
+
if (body) for (const d of body.docs) perDoc.set(d, body.exact ? 5 : 2);
|
|
118
|
+
const title = lookup(index.titleterms, word);
|
|
119
|
+
if (title) for (const d of title.docs) perDoc.set(d, (perDoc.get(d) ?? 0) + (title.exact ? 15 : 5));
|
|
120
|
+
for (const [d, score] of perDoc) {
|
|
121
|
+
scores.set(d, (scores.get(d) ?? 0) + score);
|
|
122
|
+
matched.set(d, (matched.get(d) ?? 0) + 1);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
let docs = [...scores.keys()].filter((d) => matched.get(d) === words.length);
|
|
126
|
+
const partial = !docs.length && words.length > 1 && scores.size > 0;
|
|
127
|
+
if (partial) docs = [...scores.keys()];
|
|
128
|
+
docs.sort((a, b) =>
|
|
129
|
+
scores.get(b) - scores.get(a)
|
|
130
|
+
|| String(index.titles[a]).localeCompare(String(index.titles[b])));
|
|
131
|
+
return {
|
|
132
|
+
words,
|
|
133
|
+
partial,
|
|
134
|
+
total: docs.length,
|
|
135
|
+
objects: objectHits(index, query),
|
|
136
|
+
docs: docs.slice(0, MAX_RESULTS).map((d) => ({
|
|
137
|
+
doc: d,
|
|
138
|
+
title: plainTitle(index.titles[d]) || index.docnames[d],
|
|
139
|
+
})),
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Titles in the index arrive as the HTML of the page's <h1>, markup and all
|
|
144
|
+
// (docs.python.org wraps module names in <code> spans), so they are flattened
|
|
145
|
+
// to text before they are placed on the results page. A title is index data
|
|
146
|
+
// from the network, so the walk keeps only what stands outside a bracket: no
|
|
147
|
+
// '<' or '>' can survive it, even from a tag the title never closes. A
|
|
148
|
+
// literal angle bracket in a real title arrives as an entity, so nothing
|
|
149
|
+
// legitimate is lost.
|
|
150
|
+
const plainTitle = (t) => {
|
|
151
|
+
let text = '';
|
|
152
|
+
let inTag = false;
|
|
153
|
+
for (const ch of String(t)) {
|
|
154
|
+
if (ch === '<') inTag = true;
|
|
155
|
+
else if (ch === '>') inTag = false;
|
|
156
|
+
else if (!inTag) text += ch;
|
|
157
|
+
}
|
|
158
|
+
return text.trim();
|
|
159
|
+
};
|
|
160
|
+
|
|
161
|
+
export const escapeHTML = (s) => String(s)
|
|
162
|
+
.replaceAll('&', '&').replaceAll('<', '<')
|
|
163
|
+
.replaceAll('>', '>').replaceAll('"', '"');
|
|
164
|
+
|
|
165
|
+
/**
|
|
166
|
+
* The result list becomes a small HTML page and rides the same distill and
|
|
167
|
+
* render path a fetched page does. That is what makes results numbered,
|
|
168
|
+
* followable with `do <n>`, and saved as session state, with nothing new for
|
|
169
|
+
* an agent to learn.
|
|
170
|
+
* @param {string} base
|
|
171
|
+
* @param {string} query
|
|
172
|
+
* @param {ReturnType<typeof searchIndex>} found
|
|
173
|
+
* @param {any} index
|
|
174
|
+
* @returns {string}
|
|
175
|
+
*/
|
|
176
|
+
export function resultsToHTML(base, query, found, index) {
|
|
177
|
+
const host = new URL(base).host;
|
|
178
|
+
const pageURL = (doc) => new URL(`${index.docnames[doc]}.html`, base).href;
|
|
179
|
+
const items = found.objects.map((o) =>
|
|
180
|
+
`<li><a href="${escapeHTML(`${pageURL(o.doc)}#${o.anchor}`)}">${escapeHTML(o.name)}</a>`
|
|
181
|
+
+ ` ${escapeHTML(o.type)}, in ${escapeHTML(plainTitle(index.titles[o.doc]) || index.docnames[o.doc])}</li>`);
|
|
182
|
+
for (const r of found.docs) {
|
|
183
|
+
items.push(`<li><a href="${escapeHTML(pageURL(r.doc))}">${escapeHTML(r.title)}</a></li>`);
|
|
184
|
+
}
|
|
185
|
+
const partial = found.partial ? '; no page matches every word, so these match some' : '';
|
|
186
|
+
const summary = items.length
|
|
187
|
+
? `${found.total} page${found.total === 1 ? '' : 's'} match in the site's own search index,`
|
|
188
|
+
+ ` ranked locally${found.total > MAX_RESULTS ? `, top ${MAX_RESULTS} shown` : ''}${partial}:`
|
|
189
|
+
: `nothing in the site's own search index matches; try fewer or different words`;
|
|
190
|
+
return `<html><head><title>${escapeHTML(host)} search: ${escapeHTML(query)}</title></head><body><main>`
|
|
191
|
+
+ `<p>${summary}</p>`
|
|
192
|
+
+ (items.length ? `<ol>${items.join('')}</ol>` : '')
|
|
193
|
+
+ `</main></body></html>`;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Search one Sphinx site. Returns the synthetic results page plus the URL the
|
|
198
|
+
* session should remember: the site's human search URL, so the state reads
|
|
199
|
+
* sensibly in `oc session` listings and error messages.
|
|
200
|
+
* @param {string} base - site root ending in '/', e.g. https://docs.python.org/3/
|
|
201
|
+
* @param {string} query
|
|
202
|
+
*/
|
|
203
|
+
export async function sphinxSearch(base, query) {
|
|
204
|
+
if (!query.trim()) throw new Error('usage: search <query>');
|
|
205
|
+
const { data: index, via } = await cachedFile('sphinx', new URL('searchindex.js', base).href, parseIndex);
|
|
206
|
+
return {
|
|
207
|
+
url: new URL(`search.html?q=${encodeURIComponent(query)}`, base).href,
|
|
208
|
+
html: resultsToHTML(base, query, searchIndex(index, query), index),
|
|
209
|
+
via,
|
|
210
|
+
};
|
|
211
|
+
}
|