@only-cli/oc 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/auth.js ADDED
@@ -0,0 +1,46 @@
1
+ /**
2
+ * Detect when a fetched page is a login gate rather than the content the
3
+ * caller expected. Runs before contentFailure so a thin login form is not
4
+ * partially rendered as if it were real content.
5
+ */
6
+
7
+ const LOGIN_PATH = /\/(?:login|signin|sign-in|auth|oauth|sso)(?:\/|$|\?)/i;
8
+ const LOGIN_TITLE = /\b(?:log\s*in|sign\s*in|authenticate)\b/i;
9
+ const LOGIN_BUTTON = /\b(?:log\s*in|sign\s*in|continue|submit)\b/i;
10
+
11
+ /**
12
+ * @param {string} url
13
+ * @returns {string}
14
+ */
15
+ export function sessionExpiredMessage(url) {
16
+ return `session expired or cookies are no longer valid for ${url}`;
17
+ }
18
+
19
+ /**
20
+ * @param {import('./distill.js').Page} page
21
+ * @param {string} url
22
+ * @param {{ hadAuth?: boolean }} [opts]
23
+ * @returns {string|null}
24
+ */
25
+ export function authFailure(page, url, { hadAuth = false } = {}) {
26
+ const hasPassword = page.blocks.some((b) => b.type === 'input' && b.text === 'password');
27
+ if (!hasPassword) return null;
28
+
29
+ let pathname = '';
30
+ try {
31
+ pathname = new URL(url).pathname;
32
+ } catch {
33
+ // malformed URL: rely on other signals only
34
+ }
35
+
36
+ const loginUrl = LOGIN_PATH.test(pathname);
37
+ const loginTitle = LOGIN_TITLE.test(page.title ?? '');
38
+ const loginButton = page.blocks.some((b) =>
39
+ (b.type === 'button' || b.type === 'input') && LOGIN_BUTTON.test(b.text ?? ''),
40
+ );
41
+
42
+ if (!loginUrl && !loginTitle && !loginButton) return null;
43
+
44
+ if (hadAuth) return sessionExpiredMessage(url);
45
+ return 'this page requires login; run \'printf %s "session=..." | oc login --cookie - --domain example.com\'';
46
+ }
package/src/cache.js ADDED
@@ -0,0 +1,43 @@
1
+ /**
2
+ * Day cache for the big static files a local search ranks: a Sphinx site's
3
+ * searchindex.js, the Node.js docs' all.json. Each is megabytes over the
4
+ * wire but rebuilds at most a few times a day, and a stale result list still
5
+ * links to live pages, so a day-old copy is a fair trade against moving the
6
+ * file again on every search.
7
+ */
8
+
9
+ import { mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
10
+ import { homedir } from 'node:os';
11
+ import { extname, join } from 'node:path';
12
+ import { fetchPage } from './fetch.js';
13
+
14
+ const CACHE_TTL_MS = 24 * 60 * 60 * 1000;
15
+
16
+ /**
17
+ * The file at `url`, parsed, from the disk cache while it is fresh and from
18
+ * the network otherwise. One directory per backend, one file per host, kept
19
+ * under the URL's own extension so the cache directory reads plainly. The
20
+ * file is parsed before it is written, so a block page or an error never
21
+ * poisons the cache, and a cache that cannot be written costs nothing but
22
+ * the refetch, the same policy session state follows.
23
+ * @param {string} kind - cache subdirectory, one per backend ('sphinx')
24
+ * @param {string} url
25
+ * @param {(text: string) => any} parse - throws on anything but the real file
26
+ * @returns {Promise<{data: any, via: 'cache'|'network'}>}
27
+ */
28
+ export async function cachedFile(kind, url, parse) {
29
+ const dir = join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), kind);
30
+ const file = join(dir, `${new URL(url).host}${extname(new URL(url).pathname)}`);
31
+ try {
32
+ if (Date.now() - statSync(file).mtimeMs < CACHE_TTL_MS) {
33
+ return { data: parse(readFileSync(file, 'utf8')), via: 'cache' };
34
+ }
35
+ } catch {}
36
+ const { html } = await fetchPage(url);
37
+ const data = parse(html);
38
+ try {
39
+ mkdirSync(dir, { recursive: true });
40
+ writeFileSync(file, html);
41
+ } catch {}
42
+ return { data, via: 'network' };
43
+ }
package/src/cli.js CHANGED
@@ -1,16 +1,36 @@
1
1
  #!/usr/bin/env node
2
2
  import { parseArgs } from 'node:util';
3
+ import { readFileSync } from 'node:fs';
3
4
  import { fetchPage } from './fetch.js';
4
5
  import { distill, toMarkdown, toHTML } from './distill.js';
5
- import { render, estimateTokens } from './render.js';
6
+ import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js';
7
+ import { resolveSite, listSites } from './sites.js';
8
+ import { sphinxSearch } from './sphinx.js';
9
+ import { nodeSearch } from './nodedocs.js';
10
+ import { rdocSearch } from './rdoc.js';
11
+ import { apiSearch } from './apisearch.js';
6
12
  import * as act from './act.js';
7
- import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
13
+ import { DEFAULT_SESSION, assertSafeName, clearSession, loadSession, saveSession, sessionFromPage } from './session.js';
14
+ import { authFailure, sessionExpiredMessage } from './auth.js';
15
+ import {
16
+ loadCookieJar,
17
+ saveCookieJar,
18
+ clearCookieJar,
19
+ createJarHandle,
20
+ loginCookieJar,
21
+ parseExpires,
22
+ withheldForScheme,
23
+ DEFAULT_EXPIRES_MS,
24
+ JAR_EXPIRED,
25
+ } from './cookies.js';
8
26
 
9
27
  const HELP = `only-cli: the web as a compact terminal, built for AI agents.
10
28
 
11
29
  usage: oc <command> [args] [flags]
12
30
 
13
31
  open <url> fetch and render a page with numbered actions
32
+ <site> <verb> ... site shortcut: 'oc hn top', 'oc reddit sub ClaudeAI'
33
+ sites the site shortcuts that ship with oc
14
34
  find <query> where a string appears on the page already open, or
15
35
  the region itself when only one place matches
16
36
  next the next budget worth of the page already open
@@ -20,6 +40,8 @@ usage: oc <command> [args] [flags]
20
40
  fill <n> <text> type into a numbered input (planned)
21
41
  submit [n] submit a form (planned)
22
42
  back return to the previous page (planned)
43
+ login seed cookies for a session (--cookie, --domain)
44
+ logout [session] forget a session: its cookies and its saved page
23
45
  session ls|rm manage saved sessions (planned)
24
46
 
25
47
  flags:
@@ -34,6 +56,24 @@ flags:
34
56
  memory. --stats is an alias; OC_VERBOSE=1 turns it on
35
57
  globally. Off by default because metrics cost tokens too.
36
58
  --session <name> keep separate page state under a name (default: default)
59
+ --cookie <header> login only: the Cookie header to seed. '-' reads it from
60
+ stdin, which is the form to prefer: an argv secret is
61
+ visible in ps and kept in shell history
62
+ --domain <host> login only: the hostname those cookies belong to
63
+ --expires <dur> login only: how long the session lasts (default 1h)
64
+ --allow-http login only: let these cookies travel over plain http
65
+
66
+ Authenticated pages: run 'printf %s "session=..." | oc login --cookie - --domain
67
+ example.com' to seed cookies for a session (default lifetime 1h, override with
68
+ --expires 2h). Cookies live in a separate file from page state and are sent on
69
+ every fetch for that session. They are marked secure, so they go over https
70
+ only and a redirect down to http drops them; pass --allow-http at login if a
71
+ site really is http-only. 'oc logout' forgets the session early: cookies and
72
+ saved page both.
73
+
74
+ A page that comes back with no readable text (JavaScript-only, a consent wall,
75
+ a bot challenge) says so in one line on stderr and exits 2, so a caller can
76
+ tell an empty page from a page oc could not read and fall back to a browser.
37
77
 
38
78
  'oc open' remembers the page it printed, so 'oc do 3' follows link [3] without
39
79
  you ever handling its URL, and 'oc next' or 'oc read 12' picks up what the
@@ -60,6 +100,49 @@ const remember = (page, name, cursor) => {
60
100
  const savings = (out, raw) =>
61
101
  `~${out} tokens vs ~${raw} for the page HTML (${Math.max(0, 100 - Math.round((out / Math.max(raw, 1)) * 100))}% saved)`;
62
102
 
103
+ // Nonzero, and distinct from the exit 1 that every other failure uses, so a
104
+ // caller can branch on 'oc could not read this' without parsing prose.
105
+ const NO_CONTENT_EXIT = 2;
106
+
107
+ const noContent = (url, detail, hint = "; 'oc raw' has the page's markdown if there is any, otherwise this one needs a browser") => {
108
+ console.error(`oc: no readable content at ${url} (${detail}), so it is JavaScript-only, gated, or challenged${hint}`);
109
+ // Not process.exit: stdout may still be draining, and whatever did render is
110
+ // worth printing even when the render failed.
111
+ process.exitCode = NO_CONTENT_EXIT;
112
+ };
113
+
114
+ const LOGIN_USAGE = 'usage: printf %s "session=..." | oc login --cookie - --domain example.com'
115
+ + ' [--expires 1h] [--session name] [--allow-http]';
116
+
117
+ const stripCookiePrefix = (value) => String(value).trim().replace(/^cookie\s*:\s*/i, '');
118
+
119
+ // The Cookie header a browser hands over is a live credential, and an argv
120
+ // secret is readable in ps for as long as oc runs and kept in shell history
121
+ // afterwards, so '-' reads it from stdin instead. The flag form stays, because
122
+ // it is what an agent already has in hand, but the docs lead with the pipe.
123
+ // Devtools' "copy as cURL"-style output carries the header name, and a strict
124
+ // cookie-name check would only report that as a puzzling parse error, so a
125
+ // leading 'Cookie:' is dropped here rather than refused.
126
+ const cookieHeaderArg = (value) => {
127
+ if (value !== '-') return stripCookiePrefix(value);
128
+ if (process.stdin.isTTY) throw new Error(`--cookie - reads the header from stdin, ${LOGIN_USAGE}`);
129
+ let raw;
130
+ try {
131
+ raw = readFileSync(0, 'utf8');
132
+ } catch (err) {
133
+ throw new Error(`could not read the cookie header from stdin (${err.message})`);
134
+ }
135
+ const header = stripCookiePrefix(raw);
136
+ if (!header) throw new Error(`nothing on stdin to read a cookie header from, ${LOGIN_USAGE}`);
137
+ return header;
138
+ };
139
+
140
+ // Anything else in the first position is tried as a site shortcut before it is
141
+ // called unknown, so a new clis/ definition needs no change here.
142
+ const COMMANDS = new Set([
143
+ 'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'login', 'logout', 'session', 'sites',
144
+ ]);
145
+
63
146
  async function main() {
64
147
  const { values, positionals } = parseArgs({
65
148
  allowPositionals: true,
@@ -70,6 +153,10 @@ async function main() {
70
153
  verbose: { type: 'boolean', short: 'v', default: false },
71
154
  budget: { type: 'string' },
72
155
  session: { type: 'string' },
156
+ cookie: { type: 'string' },
157
+ domain: { type: 'string' },
158
+ expires: { type: 'string' },
159
+ 'allow-http': { type: 'boolean', default: false },
73
160
  help: { type: 'boolean', short: 'h', default: false },
74
161
  },
75
162
  });
@@ -78,13 +165,30 @@ async function main() {
78
165
  // when their own verbose mode is on, or the user exports OC_VERBOSE=1.
79
166
  const verbose = values.stats || values.verbose || process.env.OC_VERBOSE === '1';
80
167
 
81
- const [command, ...args] = positionals;
168
+ let [command, ...args] = positionals;
82
169
  if (values.help || !command) {
83
170
  console.log(HELP);
84
171
  return;
85
172
  }
86
173
 
87
- const sessionName = values.session || DEFAULT_SESSION;
174
+ // A first word that is not a command may still be a site oc ships a
175
+ // definition for, and a shortcut is only ever a URL, so it resolves to one
176
+ // here and the rest of this function never learns it was not typed.
177
+ let search = null;
178
+ if (!COMMANDS.has(command)) {
179
+ const site = resolveSite(command, args);
180
+ if (!site) throw new Error(`unknown command '${command}', run oc --help`);
181
+ // Only a search shape resolves with a query; a URL shape never has one.
182
+ if (site.query != null) {
183
+ search = site;
184
+ command = 'search';
185
+ } else {
186
+ args = [site.url];
187
+ command = 'open';
188
+ }
189
+ }
190
+
191
+ const sessionName = assertSafeName(values.session || DEFAULT_SESSION);
88
192
  // Zero means "whatever this command's default is", which differs: the
89
193
  // compact view targets 500 tokens, read targets 2000.
90
194
  const asked = values.budget ? Number(values.budget) : 0;
@@ -92,6 +196,26 @@ async function main() {
92
196
  throw new Error('--budget must be a positive number');
93
197
  }
94
198
 
199
+ if (command === 'login') {
200
+ if (!values.cookie) throw new Error(LOGIN_USAGE);
201
+ if (!values.domain) throw new Error('--domain is required (the site hostname your cookies belong to)');
202
+ const expiresMs = values.expires ? parseExpires(values.expires) : DEFAULT_EXPIRES_MS;
203
+ loginCookieJar(sessionName, cookieHeaderArg(values.cookie), values.domain, {
204
+ expiresMs,
205
+ allowHttp: values['allow-http'],
206
+ });
207
+ return;
208
+ }
209
+
210
+ if (command === 'logout') {
211
+ const name = args[0] ? assertSafeName(args[0]) : sessionName;
212
+ clearCookieJar(name);
213
+ // The page saved under this name can be the distilled text of a page only
214
+ // the cookies could reach, so logout drops it too.
215
+ clearSession(name);
216
+ return;
217
+ }
218
+
95
219
  switch (command) {
96
220
  case 'open':
97
221
  case 'do':
@@ -114,8 +238,24 @@ async function main() {
114
238
  }
115
239
  if (!url) throw new Error(`usage: oc ${command} <url>`);
116
240
  const budget = asked || 500;
241
+ const jarData = loadCookieJar(sessionName);
242
+ // Clock-expired jars are wiped on load; say so in the same voice as a
243
+ // login-page redirect rather than fetching with no cookies and guessing.
244
+ if (jarData === JAR_EXPIRED) {
245
+ noContent(url, sessionExpiredMessage(url));
246
+ return;
247
+ }
248
+ const hadAuth = jarData != null;
249
+ // Secure cookies are withheld from a plain-http request. Say so, or the
250
+ // fetch comes back a login page and nothing explains why.
251
+ if (jarData && withheldForScheme(jarData, url)) {
252
+ console.error(`oc: warning: session '${sessionName}' holds https-only cookies, so they are not sent to ${url}`
253
+ + '; seed them with --allow-http if this site really is http-only');
254
+ }
255
+ const jar = jarData ? createJarHandle(sessionName, jarData) : null;
117
256
  const t0 = performance.now();
118
- const { url: finalUrl, html, status, via } = await fetchPage(url);
257
+ const { url: finalUrl, html, status, via } = await fetchPage(url, { jar: jar ?? undefined });
258
+ if (jar) saveCookieJar(sessionName, jar.toJSON());
119
259
  const fetchMs = performance.now() - t0;
120
260
  const resources = () => {
121
261
  const processMs = performance.now() - t0 - fetchMs;
@@ -123,26 +263,98 @@ async function main() {
123
263
  return `HTTP ${status} via ${via}, fetch ${Math.round(fetchMs)}ms, process ${Math.round(processMs)}ms, `
124
264
  + `${Math.round(html.length / 1024)}KB transferred, ${Math.round(rss / 1048576)}MB memory`;
125
265
  };
266
+ const htmlTokens = estimateTokens(html);
126
267
  if (values.json) {
127
268
  const page = distill(html, finalUrl);
128
- remember(page, sessionName);
129
- console.log(JSON.stringify(page));
269
+ const auth = authFailure(page, finalUrl, { hadAuth });
270
+ const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
271
+ // Always present, so a caller can branch on the field rather than on
272
+ // whether a field it was hoping for turned up.
273
+ console.log(JSON.stringify({ ...page, empty: failure != null }));
130
274
  if (verbose) console.error(resources());
275
+ if (auth) {
276
+ if (jar) clearCookieJar(sessionName);
277
+ noContent(finalUrl, auth);
278
+ return;
279
+ }
280
+ remember(page, sessionName);
281
+ if (failure) noContent(finalUrl, failure);
131
282
  return;
132
283
  }
133
- const htmlTokens = estimateTokens(html);
134
284
  if (command === 'raw') {
285
+ const page = distill(html, finalUrl);
286
+ const auth = authFailure(page, finalUrl, { hadAuth });
287
+ if (auth) {
288
+ if (jar) clearCookieJar(sessionName);
289
+ noContent(finalUrl, auth);
290
+ return;
291
+ }
135
292
  const out = values.html ? toHTML(html, finalUrl) : toMarkdown(html, finalUrl);
293
+ const outTokens = estimateTokens(out);
136
294
  console.log(out);
137
- if (verbose) console.error(`${savings(estimateTokens(out), htmlTokens)}; ${resources()}`);
295
+ if (verbose) {
296
+ const cost = outTokens < MIN_CONTENT
297
+ ? `nothing distilled out of ~${htmlTokens} tokens of page HTML`
298
+ : savings(outTokens, htmlTokens);
299
+ console.error(`${cost}; ${resources()}`);
300
+ }
301
+ // Only the blank case here. `raw` is the fallback the compact view's
302
+ // failure line names, so it must not fail on the same pages: a page
303
+ // whose only text is its menu still has markup, and printing it is the
304
+ // whole point of `raw`. And a short page that arrived short is not
305
+ // blank, so the verdict needs the same evidence the compact view asks
306
+ // for: near-nothing distilled out of markup that promised more.
307
+ if (outTokens < MIN_CONTENT && contentFailure(outTokens, htmlTokens)) {
308
+ noContent(finalUrl, `~${outTokens} tokens of markdown`, '');
309
+ }
138
310
  return;
139
311
  }
140
312
  const page = distill(html, finalUrl);
313
+ const auth = authFailure(page, finalUrl, { hadAuth });
314
+ const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
315
+ if (auth) {
316
+ if (jar) clearCookieJar(sessionName);
317
+ noContent(finalUrl, auth);
318
+ return;
319
+ }
141
320
  const { text, stats } = render(page, { budget });
142
321
  remember(page, sessionName, stats.next);
143
322
  console.log(text);
144
323
  if (verbose) {
145
- console.error(`~${stats.tokens} tokens, ${stats.rendered}/${stats.blocks} blocks rendered, ${savings(stats.tokens, htmlTokens)}; ${resources()}`);
324
+ // Reporting '100% saved' of a render that extracted nothing is the one
325
+ // place this line lies, and it lies in the tool's own favour.
326
+ const cost = failure
327
+ ? `no content distilled out of ~${htmlTokens} tokens of page HTML`
328
+ : savings(stats.tokens, htmlTokens);
329
+ console.error(`~${stats.tokens} tokens, ${stats.rendered}/${stats.blocks} blocks rendered, ${cost}; ${resources()}`);
330
+ }
331
+ if (failure) noContent(finalUrl, failure);
332
+ return;
333
+ }
334
+ case 'search': {
335
+ // A search oc runs itself: a site's static index or docs corpus is
336
+ // fetched (or read back from its day cache) and ranked here, a JSON
337
+ // search API is asked directly. Either way the result list rides the
338
+ // exact `open` path: distilled, rendered, remembered, so `do <n>`
339
+ // follows a result. Only the list is ever printed; the index, corpus,
340
+ // and response stay out of context.
341
+ const t0 = performance.now();
342
+ const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch };
343
+ const kind = Object.keys(local).find((k) => search[k]);
344
+ const { url, html, via } = kind
345
+ ? await local[kind](search[kind], search.query)
346
+ : await apiSearch(search.api, search.query);
347
+ const page = distill(html, url);
348
+ if (values.json) {
349
+ remember(page, sessionName);
350
+ console.log(JSON.stringify({ ...page, empty: false }));
351
+ return;
352
+ }
353
+ const { text, stats } = render(page, { budget: asked || 500 });
354
+ remember(page, sessionName, stats.next);
355
+ console.log(text);
356
+ if (verbose) {
357
+ console.error(`~${stats.tokens} tokens, results via ${via}, ${Math.round(performance.now() - t0)}ms`);
146
358
  }
147
359
  return;
148
360
  }
@@ -152,6 +364,7 @@ async function main() {
152
364
  case 'fill': return act.fill(Number(args[0]), args.slice(1).join(' '));
153
365
  case 'submit': return act.submit(args[0] ? Number(args[0]) : undefined);
154
366
  case 'back': return act.back();
367
+ case 'sites': return console.log(listSites());
155
368
  case 'session': throw new act.NotImplemented('session');
156
369
  default:
157
370
  throw new Error(`unknown command '${command}', run oc --help`);