@only-cli/oc 0.4.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +107 -38
- package/clis/developer.mozilla.org.json +9 -0
- package/clis/doc.rust-lang.org.json +8 -0
- package/clis/docs.oracle.com.json +7 -0
- package/clis/docs.python.org.json +8 -0
- package/clis/docs.ruby-lang.org.json +7 -0
- package/clis/en.cppreference.com.json +8 -0
- package/clis/learn.microsoft.com.json +1 -0
- package/clis/nodejs.org.json +7 -0
- package/clis/php.net.json +8 -0
- package/clis/pkg.go.dev.json +7 -0
- package/clis/typescriptlang.org.json +7 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +42 -23
- package/src/act.js +13 -2
- package/src/apisearch.js +80 -0
- package/src/auth.js +46 -0
- package/src/cache.js +43 -0
- package/src/cli.js +167 -11
- package/src/cookies.js +530 -0
- package/src/fetch.js +142 -19
- package/src/nodedocs.js +177 -0
- package/src/rdoc.js +98 -0
- package/src/render.js +21 -10
- package/src/session.js +43 -4
- package/src/sites.js +30 -5
- package/src/sphinx.js +211 -0
package/src/cache.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Day cache for the big static files a local search ranks: a Sphinx site's
|
|
3
|
+
* searchindex.js, the Node.js docs' all.json. Each is megabytes over the
|
|
4
|
+
* wire but rebuilds at most a few times a day, and a stale result list still
|
|
5
|
+
* links to live pages, so a day-old copy is a fair trade against moving the
|
|
6
|
+
* file again on every search.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
10
|
+
import { homedir } from 'node:os';
|
|
11
|
+
import { extname, join } from 'node:path';
|
|
12
|
+
import { fetchPage } from './fetch.js';
|
|
13
|
+
|
|
14
|
+
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The file at `url`, parsed, from the disk cache while it is fresh and from
|
|
18
|
+
* the network otherwise. One directory per backend, one file per host, kept
|
|
19
|
+
* under the URL's own extension so the cache directory reads plainly. The
|
|
20
|
+
* file is parsed before it is written, so a block page or an error never
|
|
21
|
+
* poisons the cache, and a cache that cannot be written costs nothing but
|
|
22
|
+
* the refetch, the same policy session state follows.
|
|
23
|
+
* @param {string} kind - cache subdirectory, one per backend ('sphinx')
|
|
24
|
+
* @param {string} url
|
|
25
|
+
* @param {(text: string) => any} parse - throws on anything but the real file
|
|
26
|
+
* @returns {Promise<{data: any, via: 'cache'|'network'}>}
|
|
27
|
+
*/
|
|
28
|
+
export async function cachedFile(kind, url, parse) {
|
|
29
|
+
const dir = join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), kind);
|
|
30
|
+
const file = join(dir, `${new URL(url).host}${extname(new URL(url).pathname)}`);
|
|
31
|
+
try {
|
|
32
|
+
if (Date.now() - statSync(file).mtimeMs < CACHE_TTL_MS) {
|
|
33
|
+
return { data: parse(readFileSync(file, 'utf8')), via: 'cache' };
|
|
34
|
+
}
|
|
35
|
+
} catch {}
|
|
36
|
+
const { html } = await fetchPage(url);
|
|
37
|
+
const data = parse(html);
|
|
38
|
+
try {
|
|
39
|
+
mkdirSync(dir, { recursive: true });
|
|
40
|
+
writeFileSync(file, html);
|
|
41
|
+
} catch {}
|
|
42
|
+
return { data, via: 'network' };
|
|
43
|
+
}
|
package/src/cli.js
CHANGED
|
@@ -1,11 +1,28 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { parseArgs } from 'node:util';
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
3
4
|
import { fetchPage } from './fetch.js';
|
|
4
5
|
import { distill, toMarkdown, toHTML } from './distill.js';
|
|
5
6
|
import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js';
|
|
6
7
|
import { resolveSite, listSites } from './sites.js';
|
|
8
|
+
import { sphinxSearch } from './sphinx.js';
|
|
9
|
+
import { nodeSearch } from './nodedocs.js';
|
|
10
|
+
import { rdocSearch } from './rdoc.js';
|
|
11
|
+
import { apiSearch } from './apisearch.js';
|
|
7
12
|
import * as act from './act.js';
|
|
8
|
-
import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
|
|
13
|
+
import { DEFAULT_SESSION, assertSafeName, clearSession, loadSession, saveSession, sessionFromPage } from './session.js';
|
|
14
|
+
import { authFailure, sessionExpiredMessage } from './auth.js';
|
|
15
|
+
import {
|
|
16
|
+
loadCookieJar,
|
|
17
|
+
saveCookieJar,
|
|
18
|
+
clearCookieJar,
|
|
19
|
+
createJarHandle,
|
|
20
|
+
loginCookieJar,
|
|
21
|
+
parseExpires,
|
|
22
|
+
withheldForScheme,
|
|
23
|
+
DEFAULT_EXPIRES_MS,
|
|
24
|
+
JAR_EXPIRED,
|
|
25
|
+
} from './cookies.js';
|
|
9
26
|
|
|
10
27
|
const HELP = `only-cli: the web as a compact terminal, built for AI agents.
|
|
11
28
|
|
|
@@ -23,6 +40,8 @@ usage: oc <command> [args] [flags]
|
|
|
23
40
|
fill <n> <text> type into a numbered input (planned)
|
|
24
41
|
submit [n] submit a form (planned)
|
|
25
42
|
back return to the previous page (planned)
|
|
43
|
+
login seed cookies for a session (--cookie, --domain)
|
|
44
|
+
logout [session] forget a session: its cookies and its saved page
|
|
26
45
|
session ls|rm manage saved sessions (planned)
|
|
27
46
|
|
|
28
47
|
flags:
|
|
@@ -37,6 +56,20 @@ flags:
|
|
|
37
56
|
memory. --stats is an alias; OC_VERBOSE=1 turns it on
|
|
38
57
|
globally. Off by default because metrics cost tokens too.
|
|
39
58
|
--session <name> keep separate page state under a name (default: default)
|
|
59
|
+
--cookie <header> login only: the Cookie header to seed. '-' reads it from
|
|
60
|
+
stdin, which is the form to prefer: an argv secret is
|
|
61
|
+
visible in ps and kept in shell history
|
|
62
|
+
--domain <host> login only: the hostname those cookies belong to
|
|
63
|
+
--expires <dur> login only: how long the session lasts (default 1h)
|
|
64
|
+
--allow-http login only: let these cookies travel over plain http
|
|
65
|
+
|
|
66
|
+
Authenticated pages: run 'printf %s "session=..." | oc login --cookie - --domain
|
|
67
|
+
example.com' to seed cookies for a session (default lifetime 1h, override with
|
|
68
|
+
--expires 2h). Cookies live in a separate file from page state and are sent on
|
|
69
|
+
every fetch for that session. They are marked secure, so they go over https
|
|
70
|
+
only and a redirect down to http drops them; pass --allow-http at login if a
|
|
71
|
+
site really is http-only. 'oc logout' forgets the session early: cookies and
|
|
72
|
+
saved page both.
|
|
40
73
|
|
|
41
74
|
A page that comes back with no readable text (JavaScript-only, a consent wall,
|
|
42
75
|
a bot challenge) says so in one line on stderr and exits 2, so a caller can
|
|
@@ -78,10 +111,36 @@ const noContent = (url, detail, hint = "; 'oc raw' has the page's markdown if th
|
|
|
78
111
|
process.exitCode = NO_CONTENT_EXIT;
|
|
79
112
|
};
|
|
80
113
|
|
|
114
|
+
const LOGIN_USAGE = 'usage: printf %s "session=..." | oc login --cookie - --domain example.com'
|
|
115
|
+
+ ' [--expires 1h] [--session name] [--allow-http]';
|
|
116
|
+
|
|
117
|
+
const stripCookiePrefix = (value) => String(value).trim().replace(/^cookie\s*:\s*/i, '');
|
|
118
|
+
|
|
119
|
+
// The Cookie header a browser hands over is a live credential, and an argv
|
|
120
|
+
// secret is readable in ps for as long as oc runs and kept in shell history
|
|
121
|
+
// afterwards, so '-' reads it from stdin instead. The flag form stays, because
|
|
122
|
+
// it is what an agent already has in hand, but the docs lead with the pipe.
|
|
123
|
+
// Devtools' "copy as cURL"-style output carries the header name, and a strict
|
|
124
|
+
// cookie-name check would only report that as a puzzling parse error, so a
|
|
125
|
+
// leading 'Cookie:' is dropped here rather than refused.
|
|
126
|
+
const cookieHeaderArg = (value) => {
|
|
127
|
+
if (value !== '-') return stripCookiePrefix(value);
|
|
128
|
+
if (process.stdin.isTTY) throw new Error(`--cookie - reads the header from stdin, ${LOGIN_USAGE}`);
|
|
129
|
+
let raw;
|
|
130
|
+
try {
|
|
131
|
+
raw = readFileSync(0, 'utf8');
|
|
132
|
+
} catch (err) {
|
|
133
|
+
throw new Error(`could not read the cookie header from stdin (${err.message})`);
|
|
134
|
+
}
|
|
135
|
+
const header = stripCookiePrefix(raw);
|
|
136
|
+
if (!header) throw new Error(`nothing on stdin to read a cookie header from, ${LOGIN_USAGE}`);
|
|
137
|
+
return header;
|
|
138
|
+
};
|
|
139
|
+
|
|
81
140
|
// Anything else in the first position is tried as a site shortcut before it is
|
|
82
141
|
// called unknown, so a new clis/ definition needs no change here.
|
|
83
142
|
const COMMANDS = new Set([
|
|
84
|
-
'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'session', 'sites',
|
|
143
|
+
'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'login', 'logout', 'session', 'sites',
|
|
85
144
|
]);
|
|
86
145
|
|
|
87
146
|
async function main() {
|
|
@@ -94,6 +153,10 @@ async function main() {
|
|
|
94
153
|
verbose: { type: 'boolean', short: 'v', default: false },
|
|
95
154
|
budget: { type: 'string' },
|
|
96
155
|
session: { type: 'string' },
|
|
156
|
+
cookie: { type: 'string' },
|
|
157
|
+
domain: { type: 'string' },
|
|
158
|
+
expires: { type: 'string' },
|
|
159
|
+
'allow-http': { type: 'boolean', default: false },
|
|
97
160
|
help: { type: 'boolean', short: 'h', default: false },
|
|
98
161
|
},
|
|
99
162
|
});
|
|
@@ -111,14 +174,21 @@ async function main() {
|
|
|
111
174
|
// A first word that is not a command may still be a site oc ships a
|
|
112
175
|
// definition for, and a shortcut is only ever a URL, so it resolves to one
|
|
113
176
|
// here and the rest of this function never learns it was not typed.
|
|
177
|
+
let search = null;
|
|
114
178
|
if (!COMMANDS.has(command)) {
|
|
115
179
|
const site = resolveSite(command, args);
|
|
116
180
|
if (!site) throw new Error(`unknown command '${command}', run oc --help`);
|
|
117
|
-
|
|
118
|
-
|
|
181
|
+
// Only a search shape resolves with a query; a URL shape never has one.
|
|
182
|
+
if (site.query != null) {
|
|
183
|
+
search = site;
|
|
184
|
+
command = 'search';
|
|
185
|
+
} else {
|
|
186
|
+
args = [site.url];
|
|
187
|
+
command = 'open';
|
|
188
|
+
}
|
|
119
189
|
}
|
|
120
190
|
|
|
121
|
-
const sessionName = values.session || DEFAULT_SESSION;
|
|
191
|
+
const sessionName = assertSafeName(values.session || DEFAULT_SESSION);
|
|
122
192
|
// Zero means "whatever this command's default is", which differs: the
|
|
123
193
|
// compact view targets 500 tokens, read targets 2000.
|
|
124
194
|
const asked = values.budget ? Number(values.budget) : 0;
|
|
@@ -126,6 +196,26 @@ async function main() {
|
|
|
126
196
|
throw new Error('--budget must be a positive number');
|
|
127
197
|
}
|
|
128
198
|
|
|
199
|
+
if (command === 'login') {
|
|
200
|
+
if (!values.cookie) throw new Error(LOGIN_USAGE);
|
|
201
|
+
if (!values.domain) throw new Error('--domain is required (the site hostname your cookies belong to)');
|
|
202
|
+
const expiresMs = values.expires ? parseExpires(values.expires) : DEFAULT_EXPIRES_MS;
|
|
203
|
+
loginCookieJar(sessionName, cookieHeaderArg(values.cookie), values.domain, {
|
|
204
|
+
expiresMs,
|
|
205
|
+
allowHttp: values['allow-http'],
|
|
206
|
+
});
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
if (command === 'logout') {
|
|
211
|
+
const name = args[0] ? assertSafeName(args[0]) : sessionName;
|
|
212
|
+
clearCookieJar(name);
|
|
213
|
+
// The page saved under this name can be the distilled text of a page only
|
|
214
|
+
// the cookies could reach, so logout drops it too.
|
|
215
|
+
clearSession(name);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
|
|
129
219
|
switch (command) {
|
|
130
220
|
case 'open':
|
|
131
221
|
case 'do':
|
|
@@ -148,8 +238,24 @@ async function main() {
|
|
|
148
238
|
}
|
|
149
239
|
if (!url) throw new Error(`usage: oc ${command} <url>`);
|
|
150
240
|
const budget = asked || 500;
|
|
241
|
+
const jarData = loadCookieJar(sessionName);
|
|
242
|
+
// Clock-expired jars are wiped on load; say so in the same voice as a
|
|
243
|
+
// login-page redirect rather than fetching with no cookies and guessing.
|
|
244
|
+
if (jarData === JAR_EXPIRED) {
|
|
245
|
+
noContent(url, sessionExpiredMessage(url));
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
const hadAuth = jarData != null;
|
|
249
|
+
// Secure cookies are withheld from a plain-http request. Say so, or the
|
|
250
|
+
// fetch comes back a login page and nothing explains why.
|
|
251
|
+
if (jarData && withheldForScheme(jarData, url)) {
|
|
252
|
+
console.error(`oc: warning: session '${sessionName}' holds https-only cookies, so they are not sent to ${url}`
|
|
253
|
+
+ '; seed them with --allow-http if this site really is http-only');
|
|
254
|
+
}
|
|
255
|
+
const jar = jarData ? createJarHandle(sessionName, jarData) : null;
|
|
151
256
|
const t0 = performance.now();
|
|
152
|
-
const { url: finalUrl, html, status, via } = await fetchPage(url);
|
|
257
|
+
const { url: finalUrl, html, status, via } = await fetchPage(url, { jar: jar ?? undefined });
|
|
258
|
+
if (jar) saveCookieJar(sessionName, jar.toJSON());
|
|
153
259
|
const fetchMs = performance.now() - t0;
|
|
154
260
|
const resources = () => {
|
|
155
261
|
const processMs = performance.now() - t0 - fetchMs;
|
|
@@ -160,16 +266,29 @@ async function main() {
|
|
|
160
266
|
const htmlTokens = estimateTokens(html);
|
|
161
267
|
if (values.json) {
|
|
162
268
|
const page = distill(html, finalUrl);
|
|
163
|
-
|
|
164
|
-
const failure = contentFailure(contentTokens(page), htmlTokens);
|
|
269
|
+
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
270
|
+
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
|
165
271
|
// Always present, so a caller can branch on the field rather than on
|
|
166
272
|
// whether a field it was hoping for turned up.
|
|
167
273
|
console.log(JSON.stringify({ ...page, empty: failure != null }));
|
|
168
274
|
if (verbose) console.error(resources());
|
|
275
|
+
if (auth) {
|
|
276
|
+
if (jar) clearCookieJar(sessionName);
|
|
277
|
+
noContent(finalUrl, auth);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
remember(page, sessionName);
|
|
169
281
|
if (failure) noContent(finalUrl, failure);
|
|
170
282
|
return;
|
|
171
283
|
}
|
|
172
284
|
if (command === 'raw') {
|
|
285
|
+
const page = distill(html, finalUrl);
|
|
286
|
+
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
287
|
+
if (auth) {
|
|
288
|
+
if (jar) clearCookieJar(sessionName);
|
|
289
|
+
noContent(finalUrl, auth);
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
173
292
|
const out = values.html ? toHTML(html, finalUrl) : toMarkdown(html, finalUrl);
|
|
174
293
|
const outTokens = estimateTokens(out);
|
|
175
294
|
console.log(out);
|
|
@@ -182,12 +301,22 @@ async function main() {
|
|
|
182
301
|
// Only the blank case here. `raw` is the fallback the compact view's
|
|
183
302
|
// failure line names, so it must not fail on the same pages: a page
|
|
184
303
|
// whose only text is its menu still has markup, and printing it is the
|
|
185
|
-
// whole point of `raw`.
|
|
186
|
-
|
|
304
|
+
// whole point of `raw`. And a short page that arrived short is not
|
|
305
|
+
// blank, so the verdict needs the same evidence the compact view asks
|
|
306
|
+
// for: near-nothing distilled out of markup that promised more.
|
|
307
|
+
if (outTokens < MIN_CONTENT && contentFailure(outTokens, htmlTokens)) {
|
|
308
|
+
noContent(finalUrl, `~${outTokens} tokens of markdown`, '');
|
|
309
|
+
}
|
|
187
310
|
return;
|
|
188
311
|
}
|
|
189
312
|
const page = distill(html, finalUrl);
|
|
190
|
-
const
|
|
313
|
+
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
314
|
+
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
|
315
|
+
if (auth) {
|
|
316
|
+
if (jar) clearCookieJar(sessionName);
|
|
317
|
+
noContent(finalUrl, auth);
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
191
320
|
const { text, stats } = render(page, { budget });
|
|
192
321
|
remember(page, sessionName, stats.next);
|
|
193
322
|
console.log(text);
|
|
@@ -202,6 +331,33 @@ async function main() {
|
|
|
202
331
|
if (failure) noContent(finalUrl, failure);
|
|
203
332
|
return;
|
|
204
333
|
}
|
|
334
|
+
case 'search': {
|
|
335
|
+
// A search oc runs itself: a site's static index or docs corpus is
|
|
336
|
+
// fetched (or read back from its day cache) and ranked here, a JSON
|
|
337
|
+
// search API is asked directly. Either way the result list rides the
|
|
338
|
+
// exact `open` path: distilled, rendered, remembered, so `do <n>`
|
|
339
|
+
// follows a result. Only the list is ever printed; the index, corpus,
|
|
340
|
+
// and response stay out of context.
|
|
341
|
+
const t0 = performance.now();
|
|
342
|
+
const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch };
|
|
343
|
+
const kind = Object.keys(local).find((k) => search[k]);
|
|
344
|
+
const { url, html, via } = kind
|
|
345
|
+
? await local[kind](search[kind], search.query)
|
|
346
|
+
: await apiSearch(search.api, search.query);
|
|
347
|
+
const page = distill(html, url);
|
|
348
|
+
if (values.json) {
|
|
349
|
+
remember(page, sessionName);
|
|
350
|
+
console.log(JSON.stringify({ ...page, empty: false }));
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
const { text, stats } = render(page, { budget: asked || 500 });
|
|
354
|
+
remember(page, sessionName, stats.next);
|
|
355
|
+
console.log(text);
|
|
356
|
+
if (verbose) {
|
|
357
|
+
console.error(`~${stats.tokens} tokens, results via ${via}, ${Math.round(performance.now() - t0)}ms`);
|
|
358
|
+
}
|
|
359
|
+
return;
|
|
360
|
+
}
|
|
205
361
|
case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 }));
|
|
206
362
|
case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 }));
|
|
207
363
|
case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 }));
|