@only-cli/oc 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +118 -29
- package/clis/developer.mozilla.org.json +9 -0
- package/clis/doc.rust-lang.org.json +8 -0
- package/clis/docs.oracle.com.json +7 -0
- package/clis/docs.python.org.json +8 -0
- package/clis/docs.ruby-lang.org.json +7 -0
- package/clis/en.cppreference.com.json +8 -0
- package/clis/github.com.json +2 -2
- package/clis/learn.microsoft.com.json +1 -0
- package/clis/nodejs.org.json +7 -0
- package/clis/php.net.json +8 -0
- package/clis/pkg.go.dev.json +7 -0
- package/clis/reddit.com.json +1 -1
- package/clis/typescriptlang.org.json +7 -0
- package/clis/wikipedia.org.json +8 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +55 -19
- package/src/act.js +9 -0
- package/src/apisearch.js +80 -0
- package/src/auth.js +46 -0
- package/src/cache.js +43 -0
- package/src/cli.js +223 -10
- package/src/cookies.js +530 -0
- package/src/fetch.js +418 -18
- package/src/nodedocs.js +177 -0
- package/src/rdoc.js +98 -0
- package/src/render.js +64 -3
- package/src/session.js +43 -4
- package/src/sites.js +173 -0
- package/src/sphinx.js +211 -0
package/src/auth.js
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Detect when a fetched page is a login gate rather than the content the
|
|
3
|
+
* caller expected. Runs before contentFailure so a thin login form is not
|
|
4
|
+
* partially rendered as if it were real content.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const LOGIN_PATH = /\/(?:login|signin|sign-in|auth|oauth|sso)(?:\/|$|\?)/i;
|
|
8
|
+
const LOGIN_TITLE = /\b(?:log\s*in|sign\s*in|authenticate)\b/i;
|
|
9
|
+
const LOGIN_BUTTON = /\b(?:log\s*in|sign\s*in|continue|submit)\b/i;
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* @param {string} url
|
|
13
|
+
* @returns {string}
|
|
14
|
+
*/
|
|
15
|
+
export function sessionExpiredMessage(url) {
|
|
16
|
+
return `session expired or cookies are no longer valid for ${url}`;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {import('./distill.js').Page} page
|
|
21
|
+
* @param {string} url
|
|
22
|
+
* @param {{ hadAuth?: boolean }} [opts]
|
|
23
|
+
* @returns {string|null}
|
|
24
|
+
*/
|
|
25
|
+
export function authFailure(page, url, { hadAuth = false } = {}) {
|
|
26
|
+
const hasPassword = page.blocks.some((b) => b.type === 'input' && b.text === 'password');
|
|
27
|
+
if (!hasPassword) return null;
|
|
28
|
+
|
|
29
|
+
let pathname = '';
|
|
30
|
+
try {
|
|
31
|
+
pathname = new URL(url).pathname;
|
|
32
|
+
} catch {
|
|
33
|
+
// malformed URL: rely on other signals only
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
const loginUrl = LOGIN_PATH.test(pathname);
|
|
37
|
+
const loginTitle = LOGIN_TITLE.test(page.title ?? '');
|
|
38
|
+
const loginButton = page.blocks.some((b) =>
|
|
39
|
+
(b.type === 'button' || b.type === 'input') && LOGIN_BUTTON.test(b.text ?? ''),
|
|
40
|
+
);
|
|
41
|
+
|
|
42
|
+
if (!loginUrl && !loginTitle && !loginButton) return null;
|
|
43
|
+
|
|
44
|
+
if (hadAuth) return sessionExpiredMessage(url);
|
|
45
|
+
return 'this page requires login; run \'printf %s "session=..." | oc login --cookie - --domain example.com\'';
|
|
46
|
+
}
|
package/src/cache.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Day cache for the big static files a local search ranks: a Sphinx site's
|
|
3
|
+
* searchindex.js, the Node.js docs' all.json. Each is megabytes over the
|
|
4
|
+
* wire but rebuilds at most a few times a day, and a stale result list still
|
|
5
|
+
* links to live pages, so a day-old copy is a fair trade against moving the
|
|
6
|
+
* file again on every search.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
|
|
10
|
+
import { homedir } from 'node:os';
|
|
11
|
+
import { extname, join } from 'node:path';
|
|
12
|
+
import { fetchPage } from './fetch.js';
|
|
13
|
+
|
|
14
|
+
const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* The file at `url`, parsed, from the disk cache while it is fresh and from
|
|
18
|
+
* the network otherwise. One directory per backend, one file per host, kept
|
|
19
|
+
* under the URL's own extension so the cache directory reads plainly. The
|
|
20
|
+
* file is parsed before it is written, so a block page or an error never
|
|
21
|
+
* poisons the cache, and a cache that cannot be written costs nothing but
|
|
22
|
+
* the refetch, the same policy session state follows.
|
|
23
|
+
* @param {string} kind - cache subdirectory, one per backend ('sphinx')
|
|
24
|
+
* @param {string} url
|
|
25
|
+
* @param {(text: string) => any} parse - throws on anything but the real file
|
|
26
|
+
* @returns {Promise<{data: any, via: 'cache'|'network'}>}
|
|
27
|
+
*/
|
|
28
|
+
export async function cachedFile(kind, url, parse) {
|
|
29
|
+
const dir = join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), kind);
|
|
30
|
+
const file = join(dir, `${new URL(url).host}${extname(new URL(url).pathname)}`);
|
|
31
|
+
try {
|
|
32
|
+
if (Date.now() - statSync(file).mtimeMs < CACHE_TTL_MS) {
|
|
33
|
+
return { data: parse(readFileSync(file, 'utf8')), via: 'cache' };
|
|
34
|
+
}
|
|
35
|
+
} catch {}
|
|
36
|
+
const { html } = await fetchPage(url);
|
|
37
|
+
const data = parse(html);
|
|
38
|
+
try {
|
|
39
|
+
mkdirSync(dir, { recursive: true });
|
|
40
|
+
writeFileSync(file, html);
|
|
41
|
+
} catch {}
|
|
42
|
+
return { data, via: 'network' };
|
|
43
|
+
}
|
package/src/cli.js
CHANGED
|
@@ -1,16 +1,36 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { parseArgs } from 'node:util';
|
|
3
|
+
import { readFileSync } from 'node:fs';
|
|
3
4
|
import { fetchPage } from './fetch.js';
|
|
4
5
|
import { distill, toMarkdown, toHTML } from './distill.js';
|
|
5
|
-
import { render, estimateTokens } from './render.js';
|
|
6
|
+
import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js';
|
|
7
|
+
import { resolveSite, listSites } from './sites.js';
|
|
8
|
+
import { sphinxSearch } from './sphinx.js';
|
|
9
|
+
import { nodeSearch } from './nodedocs.js';
|
|
10
|
+
import { rdocSearch } from './rdoc.js';
|
|
11
|
+
import { apiSearch } from './apisearch.js';
|
|
6
12
|
import * as act from './act.js';
|
|
7
|
-
import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
|
|
13
|
+
import { DEFAULT_SESSION, assertSafeName, clearSession, loadSession, saveSession, sessionFromPage } from './session.js';
|
|
14
|
+
import { authFailure, sessionExpiredMessage } from './auth.js';
|
|
15
|
+
import {
|
|
16
|
+
loadCookieJar,
|
|
17
|
+
saveCookieJar,
|
|
18
|
+
clearCookieJar,
|
|
19
|
+
createJarHandle,
|
|
20
|
+
loginCookieJar,
|
|
21
|
+
parseExpires,
|
|
22
|
+
withheldForScheme,
|
|
23
|
+
DEFAULT_EXPIRES_MS,
|
|
24
|
+
JAR_EXPIRED,
|
|
25
|
+
} from './cookies.js';
|
|
8
26
|
|
|
9
27
|
const HELP = `only-cli: the web as a compact terminal, built for AI agents.
|
|
10
28
|
|
|
11
29
|
usage: oc <command> [args] [flags]
|
|
12
30
|
|
|
13
31
|
open <url> fetch and render a page with numbered actions
|
|
32
|
+
<site> <verb> ... site shortcut: 'oc hn top', 'oc reddit sub ClaudeAI'
|
|
33
|
+
sites the site shortcuts that ship with oc
|
|
14
34
|
find <query> where a string appears on the page already open, or
|
|
15
35
|
the region itself when only one place matches
|
|
16
36
|
next the next budget worth of the page already open
|
|
@@ -20,6 +40,8 @@ usage: oc <command> [args] [flags]
|
|
|
20
40
|
fill <n> <text> type into a numbered input (planned)
|
|
21
41
|
submit [n] submit a form (planned)
|
|
22
42
|
back return to the previous page (planned)
|
|
43
|
+
login seed cookies for a session (--cookie, --domain)
|
|
44
|
+
logout [session] forget a session: its cookies and its saved page
|
|
23
45
|
session ls|rm manage saved sessions (planned)
|
|
24
46
|
|
|
25
47
|
flags:
|
|
@@ -34,6 +56,24 @@ flags:
|
|
|
34
56
|
memory. --stats is an alias; OC_VERBOSE=1 turns it on
|
|
35
57
|
globally. Off by default because metrics cost tokens too.
|
|
36
58
|
--session <name> keep separate page state under a name (default: default)
|
|
59
|
+
--cookie <header> login only: the Cookie header to seed. '-' reads it from
|
|
60
|
+
stdin, which is the form to prefer: an argv secret is
|
|
61
|
+
visible in ps and kept in shell history
|
|
62
|
+
--domain <host> login only: the hostname those cookies belong to
|
|
63
|
+
--expires <dur> login only: how long the session lasts (default 1h)
|
|
64
|
+
--allow-http login only: let these cookies travel over plain http
|
|
65
|
+
|
|
66
|
+
Authenticated pages: run 'printf %s "session=..." | oc login --cookie - --domain
|
|
67
|
+
example.com' to seed cookies for a session (default lifetime 1h, override with
|
|
68
|
+
--expires 2h). Cookies live in a separate file from page state and are sent on
|
|
69
|
+
every fetch for that session. They are marked secure, so they go over https
|
|
70
|
+
only and a redirect down to http drops them; pass --allow-http at login if a
|
|
71
|
+
site really is http-only. 'oc logout' forgets the session early: cookies and
|
|
72
|
+
saved page both.
|
|
73
|
+
|
|
74
|
+
A page that comes back with no readable text (JavaScript-only, a consent wall,
|
|
75
|
+
a bot challenge) says so in one line on stderr and exits 2, so a caller can
|
|
76
|
+
tell an empty page from a page oc could not read and fall back to a browser.
|
|
37
77
|
|
|
38
78
|
'oc open' remembers the page it printed, so 'oc do 3' follows link [3] without
|
|
39
79
|
you ever handling its URL, and 'oc next' or 'oc read 12' picks up what the
|
|
@@ -60,6 +100,49 @@ const remember = (page, name, cursor) => {
|
|
|
60
100
|
const savings = (out, raw) =>
|
|
61
101
|
`~${out} tokens vs ~${raw} for the page HTML (${Math.max(0, 100 - Math.round((out / Math.max(raw, 1)) * 100))}% saved)`;
|
|
62
102
|
|
|
103
|
+
// Nonzero, and distinct from the exit 1 that every other failure uses, so a
|
|
104
|
+
// caller can branch on 'oc could not read this' without parsing prose.
|
|
105
|
+
const NO_CONTENT_EXIT = 2;
|
|
106
|
+
|
|
107
|
+
const noContent = (url, detail, hint = "; 'oc raw' has the page's markdown if there is any, otherwise this one needs a browser") => {
|
|
108
|
+
console.error(`oc: no readable content at ${url} (${detail}), so it is JavaScript-only, gated, or challenged${hint}`);
|
|
109
|
+
// Not process.exit: stdout may still be draining, and whatever did render is
|
|
110
|
+
// worth printing even when the render failed.
|
|
111
|
+
process.exitCode = NO_CONTENT_EXIT;
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
const LOGIN_USAGE = 'usage: printf %s "session=..." | oc login --cookie - --domain example.com'
|
|
115
|
+
+ ' [--expires 1h] [--session name] [--allow-http]';
|
|
116
|
+
|
|
117
|
+
const stripCookiePrefix = (value) => String(value).trim().replace(/^cookie\s*:\s*/i, '');
|
|
118
|
+
|
|
119
|
+
// The Cookie header a browser hands over is a live credential, and an argv
|
|
120
|
+
// secret is readable in ps for as long as oc runs and kept in shell history
|
|
121
|
+
// afterwards, so '-' reads it from stdin instead. The flag form stays, because
|
|
122
|
+
// it is what an agent already has in hand, but the docs lead with the pipe.
|
|
123
|
+
// Devtools' "copy as cURL"-style output carries the header name, and a strict
|
|
124
|
+
// cookie-name check would only report that as a puzzling parse error, so a
|
|
125
|
+
// leading 'Cookie:' is dropped here rather than refused.
|
|
126
|
+
const cookieHeaderArg = (value) => {
|
|
127
|
+
if (value !== '-') return stripCookiePrefix(value);
|
|
128
|
+
if (process.stdin.isTTY) throw new Error(`--cookie - reads the header from stdin, ${LOGIN_USAGE}`);
|
|
129
|
+
let raw;
|
|
130
|
+
try {
|
|
131
|
+
raw = readFileSync(0, 'utf8');
|
|
132
|
+
} catch (err) {
|
|
133
|
+
throw new Error(`could not read the cookie header from stdin (${err.message})`);
|
|
134
|
+
}
|
|
135
|
+
const header = stripCookiePrefix(raw);
|
|
136
|
+
if (!header) throw new Error(`nothing on stdin to read a cookie header from, ${LOGIN_USAGE}`);
|
|
137
|
+
return header;
|
|
138
|
+
};
|
|
139
|
+
|
|
140
|
+
// Anything else in the first position is tried as a site shortcut before it is
|
|
141
|
+
// called unknown, so a new clis/ definition needs no change here.
|
|
142
|
+
const COMMANDS = new Set([
|
|
143
|
+
'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'login', 'logout', 'session', 'sites',
|
|
144
|
+
]);
|
|
145
|
+
|
|
63
146
|
async function main() {
|
|
64
147
|
const { values, positionals } = parseArgs({
|
|
65
148
|
allowPositionals: true,
|
|
@@ -70,6 +153,10 @@ async function main() {
|
|
|
70
153
|
verbose: { type: 'boolean', short: 'v', default: false },
|
|
71
154
|
budget: { type: 'string' },
|
|
72
155
|
session: { type: 'string' },
|
|
156
|
+
cookie: { type: 'string' },
|
|
157
|
+
domain: { type: 'string' },
|
|
158
|
+
expires: { type: 'string' },
|
|
159
|
+
'allow-http': { type: 'boolean', default: false },
|
|
73
160
|
help: { type: 'boolean', short: 'h', default: false },
|
|
74
161
|
},
|
|
75
162
|
});
|
|
@@ -78,13 +165,30 @@ async function main() {
|
|
|
78
165
|
// when their own verbose mode is on, or the user exports OC_VERBOSE=1.
|
|
79
166
|
const verbose = values.stats || values.verbose || process.env.OC_VERBOSE === '1';
|
|
80
167
|
|
|
81
|
-
|
|
168
|
+
let [command, ...args] = positionals;
|
|
82
169
|
if (values.help || !command) {
|
|
83
170
|
console.log(HELP);
|
|
84
171
|
return;
|
|
85
172
|
}
|
|
86
173
|
|
|
87
|
-
|
|
174
|
+
// A first word that is not a command may still be a site oc ships a
|
|
175
|
+
// definition for, and a shortcut is only ever a URL, so it resolves to one
|
|
176
|
+
// here and the rest of this function never learns it was not typed.
|
|
177
|
+
let search = null;
|
|
178
|
+
if (!COMMANDS.has(command)) {
|
|
179
|
+
const site = resolveSite(command, args);
|
|
180
|
+
if (!site) throw new Error(`unknown command '${command}', run oc --help`);
|
|
181
|
+
// Only a search shape resolves with a query; a URL shape never has one.
|
|
182
|
+
if (site.query != null) {
|
|
183
|
+
search = site;
|
|
184
|
+
command = 'search';
|
|
185
|
+
} else {
|
|
186
|
+
args = [site.url];
|
|
187
|
+
command = 'open';
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const sessionName = assertSafeName(values.session || DEFAULT_SESSION);
|
|
88
192
|
// Zero means "whatever this command's default is", which differs: the
|
|
89
193
|
// compact view targets 500 tokens, read targets 2000.
|
|
90
194
|
const asked = values.budget ? Number(values.budget) : 0;
|
|
@@ -92,6 +196,26 @@ async function main() {
|
|
|
92
196
|
throw new Error('--budget must be a positive number');
|
|
93
197
|
}
|
|
94
198
|
|
|
199
|
+
if (command === 'login') {
|
|
200
|
+
if (!values.cookie) throw new Error(LOGIN_USAGE);
|
|
201
|
+
if (!values.domain) throw new Error('--domain is required (the site hostname your cookies belong to)');
|
|
202
|
+
const expiresMs = values.expires ? parseExpires(values.expires) : DEFAULT_EXPIRES_MS;
|
|
203
|
+
loginCookieJar(sessionName, cookieHeaderArg(values.cookie), values.domain, {
|
|
204
|
+
expiresMs,
|
|
205
|
+
allowHttp: values['allow-http'],
|
|
206
|
+
});
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
if (command === 'logout') {
|
|
211
|
+
const name = args[0] ? assertSafeName(args[0]) : sessionName;
|
|
212
|
+
clearCookieJar(name);
|
|
213
|
+
// The page saved under this name can be the distilled text of a page only
|
|
214
|
+
// the cookies could reach, so logout drops it too.
|
|
215
|
+
clearSession(name);
|
|
216
|
+
return;
|
|
217
|
+
}
|
|
218
|
+
|
|
95
219
|
switch (command) {
|
|
96
220
|
case 'open':
|
|
97
221
|
case 'do':
|
|
@@ -114,8 +238,24 @@ async function main() {
|
|
|
114
238
|
}
|
|
115
239
|
if (!url) throw new Error(`usage: oc ${command} <url>`);
|
|
116
240
|
const budget = asked || 500;
|
|
241
|
+
const jarData = loadCookieJar(sessionName);
|
|
242
|
+
// Clock-expired jars are wiped on load; say so in the same voice as a
|
|
243
|
+
// login-page redirect rather than fetching with no cookies and guessing.
|
|
244
|
+
if (jarData === JAR_EXPIRED) {
|
|
245
|
+
noContent(url, sessionExpiredMessage(url));
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
const hadAuth = jarData != null;
|
|
249
|
+
// Secure cookies are withheld from a plain-http request. Say so, or the
|
|
250
|
+
// fetch comes back a login page and nothing explains why.
|
|
251
|
+
if (jarData && withheldForScheme(jarData, url)) {
|
|
252
|
+
console.error(`oc: warning: session '${sessionName}' holds https-only cookies, so they are not sent to ${url}`
|
|
253
|
+
+ '; seed them with --allow-http if this site really is http-only');
|
|
254
|
+
}
|
|
255
|
+
const jar = jarData ? createJarHandle(sessionName, jarData) : null;
|
|
117
256
|
const t0 = performance.now();
|
|
118
|
-
const { url: finalUrl, html, status, via } = await fetchPage(url);
|
|
257
|
+
const { url: finalUrl, html, status, via } = await fetchPage(url, { jar: jar ?? undefined });
|
|
258
|
+
if (jar) saveCookieJar(sessionName, jar.toJSON());
|
|
119
259
|
const fetchMs = performance.now() - t0;
|
|
120
260
|
const resources = () => {
|
|
121
261
|
const processMs = performance.now() - t0 - fetchMs;
|
|
@@ -123,26 +263,98 @@ async function main() {
|
|
|
123
263
|
return `HTTP ${status} via ${via}, fetch ${Math.round(fetchMs)}ms, process ${Math.round(processMs)}ms, `
|
|
124
264
|
+ `${Math.round(html.length / 1024)}KB transferred, ${Math.round(rss / 1048576)}MB memory`;
|
|
125
265
|
};
|
|
266
|
+
const htmlTokens = estimateTokens(html);
|
|
126
267
|
if (values.json) {
|
|
127
268
|
const page = distill(html, finalUrl);
|
|
128
|
-
|
|
129
|
-
|
|
269
|
+
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
270
|
+
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
|
271
|
+
// Always present, so a caller can branch on the field rather than on
|
|
272
|
+
// whether a field it was hoping for turned up.
|
|
273
|
+
console.log(JSON.stringify({ ...page, empty: failure != null }));
|
|
130
274
|
if (verbose) console.error(resources());
|
|
275
|
+
if (auth) {
|
|
276
|
+
if (jar) clearCookieJar(sessionName);
|
|
277
|
+
noContent(finalUrl, auth);
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
remember(page, sessionName);
|
|
281
|
+
if (failure) noContent(finalUrl, failure);
|
|
131
282
|
return;
|
|
132
283
|
}
|
|
133
|
-
const htmlTokens = estimateTokens(html);
|
|
134
284
|
if (command === 'raw') {
|
|
285
|
+
const page = distill(html, finalUrl);
|
|
286
|
+
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
287
|
+
if (auth) {
|
|
288
|
+
if (jar) clearCookieJar(sessionName);
|
|
289
|
+
noContent(finalUrl, auth);
|
|
290
|
+
return;
|
|
291
|
+
}
|
|
135
292
|
const out = values.html ? toHTML(html, finalUrl) : toMarkdown(html, finalUrl);
|
|
293
|
+
const outTokens = estimateTokens(out);
|
|
136
294
|
console.log(out);
|
|
137
|
-
if (verbose)
|
|
295
|
+
if (verbose) {
|
|
296
|
+
const cost = outTokens < MIN_CONTENT
|
|
297
|
+
? `nothing distilled out of ~${htmlTokens} tokens of page HTML`
|
|
298
|
+
: savings(outTokens, htmlTokens);
|
|
299
|
+
console.error(`${cost}; ${resources()}`);
|
|
300
|
+
}
|
|
301
|
+
// Only the blank case here. `raw` is the fallback the compact view's
|
|
302
|
+
// failure line names, so it must not fail on the same pages: a page
|
|
303
|
+
// whose only text is its menu still has markup, and printing it is the
|
|
304
|
+
// whole point of `raw`. And a short page that arrived short is not
|
|
305
|
+
// blank, so the verdict needs the same evidence the compact view asks
|
|
306
|
+
// for: near-nothing distilled out of markup that promised more.
|
|
307
|
+
if (outTokens < MIN_CONTENT && contentFailure(outTokens, htmlTokens)) {
|
|
308
|
+
noContent(finalUrl, `~${outTokens} tokens of markdown`, '');
|
|
309
|
+
}
|
|
138
310
|
return;
|
|
139
311
|
}
|
|
140
312
|
const page = distill(html, finalUrl);
|
|
313
|
+
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
314
|
+
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
|
315
|
+
if (auth) {
|
|
316
|
+
if (jar) clearCookieJar(sessionName);
|
|
317
|
+
noContent(finalUrl, auth);
|
|
318
|
+
return;
|
|
319
|
+
}
|
|
141
320
|
const { text, stats } = render(page, { budget });
|
|
142
321
|
remember(page, sessionName, stats.next);
|
|
143
322
|
console.log(text);
|
|
144
323
|
if (verbose) {
|
|
145
|
-
|
|
324
|
+
// Reporting '100% saved' of a render that extracted nothing is the one
|
|
325
|
+
// place this line lies, and it lies in the tool's own favour.
|
|
326
|
+
const cost = failure
|
|
327
|
+
? `no content distilled out of ~${htmlTokens} tokens of page HTML`
|
|
328
|
+
: savings(stats.tokens, htmlTokens);
|
|
329
|
+
console.error(`~${stats.tokens} tokens, ${stats.rendered}/${stats.blocks} blocks rendered, ${cost}; ${resources()}`);
|
|
330
|
+
}
|
|
331
|
+
if (failure) noContent(finalUrl, failure);
|
|
332
|
+
return;
|
|
333
|
+
}
|
|
334
|
+
case 'search': {
|
|
335
|
+
// A search oc runs itself: a site's static index or docs corpus is
|
|
336
|
+
// fetched (or read back from its day cache) and ranked here, a JSON
|
|
337
|
+
// search API is asked directly. Either way the result list rides the
|
|
338
|
+
// exact `open` path: distilled, rendered, remembered, so `do <n>`
|
|
339
|
+
// follows a result. Only the list is ever printed; the index, corpus,
|
|
340
|
+
// and response stay out of context.
|
|
341
|
+
const t0 = performance.now();
|
|
342
|
+
const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch };
|
|
343
|
+
const kind = Object.keys(local).find((k) => search[k]);
|
|
344
|
+
const { url, html, via } = kind
|
|
345
|
+
? await local[kind](search[kind], search.query)
|
|
346
|
+
: await apiSearch(search.api, search.query);
|
|
347
|
+
const page = distill(html, url);
|
|
348
|
+
if (values.json) {
|
|
349
|
+
remember(page, sessionName);
|
|
350
|
+
console.log(JSON.stringify({ ...page, empty: false }));
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
const { text, stats } = render(page, { budget: asked || 500 });
|
|
354
|
+
remember(page, sessionName, stats.next);
|
|
355
|
+
console.log(text);
|
|
356
|
+
if (verbose) {
|
|
357
|
+
console.error(`~${stats.tokens} tokens, results via ${via}, ${Math.round(performance.now() - t0)}ms`);
|
|
146
358
|
}
|
|
147
359
|
return;
|
|
148
360
|
}
|
|
@@ -152,6 +364,7 @@ async function main() {
|
|
|
152
364
|
case 'fill': return act.fill(Number(args[0]), args.slice(1).join(' '));
|
|
153
365
|
case 'submit': return act.submit(args[0] ? Number(args[0]) : undefined);
|
|
154
366
|
case 'back': return act.back();
|
|
367
|
+
case 'sites': return console.log(listSites());
|
|
155
368
|
case 'session': throw new act.NotImplemented('session');
|
|
156
369
|
default:
|
|
157
370
|
throw new Error(`unknown command '${command}', run oc --help`);
|