@only-cli/oc 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.js CHANGED
@@ -1,11 +1,28 @@
1
1
  #!/usr/bin/env node
2
2
  import { parseArgs } from 'node:util';
3
+ import { readFileSync } from 'node:fs';
3
4
  import { fetchPage } from './fetch.js';
4
5
  import { distill, toMarkdown, toHTML } from './distill.js';
5
6
  import { render, estimateTokens, contentTokens, contentFailure, MIN_CONTENT } from './render.js';
6
7
  import { resolveSite, listSites } from './sites.js';
8
+ import { sphinxSearch } from './sphinx.js';
9
+ import { nodeSearch } from './nodedocs.js';
10
+ import { rdocSearch } from './rdoc.js';
11
+ import { apiSearch } from './apisearch.js';
7
12
  import * as act from './act.js';
8
- import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
13
+ import { DEFAULT_SESSION, assertSafeName, clearSession, loadSession, saveSession, sessionFromPage } from './session.js';
14
+ import { authFailure, sessionExpiredMessage } from './auth.js';
15
+ import {
16
+ loadCookieJar,
17
+ saveCookieJar,
18
+ clearCookieJar,
19
+ createJarHandle,
20
+ loginCookieJar,
21
+ parseExpires,
22
+ withheldForScheme,
23
+ DEFAULT_EXPIRES_MS,
24
+ JAR_EXPIRED,
25
+ } from './cookies.js';
9
26
 
10
27
  const HELP = `only-cli: the web as a compact terminal, built for AI agents.
11
28
 
@@ -23,6 +40,8 @@ usage: oc <command> [args] [flags]
23
40
  fill <n> <text> type into a numbered input (planned)
24
41
  submit [n] submit a form (planned)
25
42
  back return to the previous page (planned)
43
+ login seed cookies for a session (--cookie, --domain)
44
+ logout [session] forget a session: its cookies and its saved page
26
45
  session ls|rm manage saved sessions (planned)
27
46
 
28
47
  flags:
@@ -37,6 +56,20 @@ flags:
37
56
  memory. --stats is an alias; OC_VERBOSE=1 turns it on
38
57
  globally. Off by default because metrics cost tokens too.
39
58
  --session <name> keep separate page state under a name (default: default)
59
+ --cookie <header> login only: the Cookie header to seed. '-' reads it from
60
+ stdin, which is the form to prefer: an argv secret is
61
+ visible in ps and kept in shell history
62
+ --domain <host> login only: the hostname those cookies belong to
63
+ --expires <dur> login only: how long the session lasts (default 1h)
64
+ --allow-http login only: let these cookies travel over plain http
65
+
66
+ Authenticated pages: run 'printf %s "session=..." | oc login --cookie - --domain
67
+ example.com' to seed cookies for a session (default lifetime 1h, override with
68
+ --expires 2h). Cookies live in a separate file from page state and are sent on
69
+ every fetch for that session. They are marked secure, so they go over https
70
+ only and a redirect down to http drops them; pass --allow-http at login if a
71
+ site really is http-only. 'oc logout' forgets the session early: cookies and
72
+ saved page both.
40
73
 
41
74
  A page that comes back with no readable text (JavaScript-only, a consent wall,
42
75
  a bot challenge) says so in one line on stderr and exits 2, so a caller can
@@ -78,10 +111,36 @@ const noContent = (url, detail, hint = "; 'oc raw' has the page's markdown if th
78
111
  process.exitCode = NO_CONTENT_EXIT;
79
112
  };
80
113
 
114
+ const LOGIN_USAGE = 'usage: printf %s "session=..." | oc login --cookie - --domain example.com'
115
+ + ' [--expires 1h] [--session name] [--allow-http]';
116
+
117
+ const stripCookiePrefix = (value) => String(value).trim().replace(/^cookie\s*:\s*/i, '');
118
+
119
+ // The Cookie header a browser hands over is a live credential, and an argv
120
+ // secret is readable in ps for as long as oc runs and kept in shell history
121
+ // afterwards, so '-' reads it from stdin instead. The flag form stays, because
122
+ // it is what an agent already has in hand, but the docs lead with the pipe.
123
+ // Devtools' "copy as cURL"-style output carries the header name, and a strict
124
+ // cookie-name check would only report that as a puzzling parse error, so a
125
+ // leading 'Cookie:' is dropped here rather than refused.
126
+ const cookieHeaderArg = (value) => {
127
+ if (value !== '-') return stripCookiePrefix(value);
128
+ if (process.stdin.isTTY) throw new Error(`--cookie - reads the header from stdin, ${LOGIN_USAGE}`);
129
+ let raw;
130
+ try {
131
+ raw = readFileSync(0, 'utf8');
132
+ } catch (err) {
133
+ throw new Error(`could not read the cookie header from stdin (${err.message})`);
134
+ }
135
+ const header = stripCookiePrefix(raw);
136
+ if (!header) throw new Error(`nothing on stdin to read a cookie header from, ${LOGIN_USAGE}`);
137
+ return header;
138
+ };
139
+
81
140
  // Anything else in the first position is tried as a site shortcut before it is
82
141
  // called unknown, so a new clis/ definition needs no change here.
83
142
  const COMMANDS = new Set([
84
- 'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'session', 'sites',
143
+ 'open', 'do', 'raw', 'read', 'next', 'find', 'fill', 'submit', 'back', 'login', 'logout', 'session', 'sites',
85
144
  ]);
86
145
 
87
146
  async function main() {
@@ -94,6 +153,10 @@ async function main() {
94
153
  verbose: { type: 'boolean', short: 'v', default: false },
95
154
  budget: { type: 'string' },
96
155
  session: { type: 'string' },
156
+ cookie: { type: 'string' },
157
+ domain: { type: 'string' },
158
+ expires: { type: 'string' },
159
+ 'allow-http': { type: 'boolean', default: false },
97
160
  help: { type: 'boolean', short: 'h', default: false },
98
161
  },
99
162
  });
@@ -111,14 +174,21 @@ async function main() {
111
174
  // A first word that is not a command may still be a site oc ships a
112
175
  // definition for, and a shortcut is only ever a URL, so it resolves to one
113
176
  // here and the rest of this function never learns it was not typed.
177
+ let search = null;
114
178
  if (!COMMANDS.has(command)) {
115
179
  const site = resolveSite(command, args);
116
180
  if (!site) throw new Error(`unknown command '${command}', run oc --help`);
117
- args = [site.url];
118
- command = 'open';
181
+ // Only a search shape resolves with a query; a URL shape never has one.
182
+ if (site.query != null) {
183
+ search = site;
184
+ command = 'search';
185
+ } else {
186
+ args = [site.url];
187
+ command = 'open';
188
+ }
119
189
  }
120
190
 
121
- const sessionName = values.session || DEFAULT_SESSION;
191
+ const sessionName = assertSafeName(values.session || DEFAULT_SESSION);
122
192
  // Zero means "whatever this command's default is", which differs: the
123
193
  // compact view targets 500 tokens, read targets 2000.
124
194
  const asked = values.budget ? Number(values.budget) : 0;
@@ -126,6 +196,26 @@ async function main() {
126
196
  throw new Error('--budget must be a positive number');
127
197
  }
128
198
 
199
+ if (command === 'login') {
200
+ if (!values.cookie) throw new Error(LOGIN_USAGE);
201
+ if (!values.domain) throw new Error('--domain is required (the site hostname your cookies belong to)');
202
+ const expiresMs = values.expires ? parseExpires(values.expires) : DEFAULT_EXPIRES_MS;
203
+ loginCookieJar(sessionName, cookieHeaderArg(values.cookie), values.domain, {
204
+ expiresMs,
205
+ allowHttp: values['allow-http'],
206
+ });
207
+ return;
208
+ }
209
+
210
+ if (command === 'logout') {
211
+ const name = args[0] ? assertSafeName(args[0]) : sessionName;
212
+ clearCookieJar(name);
213
+ // The page saved under this name can be the distilled text of a page only
214
+ // the cookies could reach, so logout drops it too.
215
+ clearSession(name);
216
+ return;
217
+ }
218
+
129
219
  switch (command) {
130
220
  case 'open':
131
221
  case 'do':
@@ -148,8 +238,24 @@ async function main() {
148
238
  }
149
239
  if (!url) throw new Error(`usage: oc ${command} <url>`);
150
240
  const budget = asked || 500;
241
+ const jarData = loadCookieJar(sessionName);
242
+ // Clock-expired jars are wiped on load; say so in the same voice as a
243
+ // login-page redirect rather than fetching with no cookies and guessing.
244
+ if (jarData === JAR_EXPIRED) {
245
+ noContent(url, sessionExpiredMessage(url));
246
+ return;
247
+ }
248
+ const hadAuth = jarData != null;
249
+ // Secure cookies are withheld from a plain-http request. Say so, or the
250
+ // fetch comes back a login page and nothing explains why.
251
+ if (jarData && withheldForScheme(jarData, url)) {
252
+ console.error(`oc: warning: session '${sessionName}' holds https-only cookies, so they are not sent to ${url}`
253
+ + '; seed them with --allow-http if this site really is http-only');
254
+ }
255
+ const jar = jarData ? createJarHandle(sessionName, jarData) : null;
151
256
  const t0 = performance.now();
152
- const { url: finalUrl, html, status, via } = await fetchPage(url);
257
+ const { url: finalUrl, html, status, via } = await fetchPage(url, { jar: jar ?? undefined });
258
+ if (jar) saveCookieJar(sessionName, jar.toJSON());
153
259
  const fetchMs = performance.now() - t0;
154
260
  const resources = () => {
155
261
  const processMs = performance.now() - t0 - fetchMs;
@@ -160,16 +266,29 @@ async function main() {
160
266
  const htmlTokens = estimateTokens(html);
161
267
  if (values.json) {
162
268
  const page = distill(html, finalUrl);
163
- remember(page, sessionName);
164
- const failure = contentFailure(contentTokens(page), htmlTokens);
269
+ const auth = authFailure(page, finalUrl, { hadAuth });
270
+ const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
165
271
  // Always present, so a caller can branch on the field rather than on
166
272
  // whether a field it was hoping for turned up.
167
273
  console.log(JSON.stringify({ ...page, empty: failure != null }));
168
274
  if (verbose) console.error(resources());
275
+ if (auth) {
276
+ if (jar) clearCookieJar(sessionName);
277
+ noContent(finalUrl, auth);
278
+ return;
279
+ }
280
+ remember(page, sessionName);
169
281
  if (failure) noContent(finalUrl, failure);
170
282
  return;
171
283
  }
172
284
  if (command === 'raw') {
285
+ const page = distill(html, finalUrl);
286
+ const auth = authFailure(page, finalUrl, { hadAuth });
287
+ if (auth) {
288
+ if (jar) clearCookieJar(sessionName);
289
+ noContent(finalUrl, auth);
290
+ return;
291
+ }
173
292
  const out = values.html ? toHTML(html, finalUrl) : toMarkdown(html, finalUrl);
174
293
  const outTokens = estimateTokens(out);
175
294
  console.log(out);
@@ -182,12 +301,22 @@ async function main() {
182
301
  // Only the blank case here. `raw` is the fallback the compact view's
183
302
  // failure line names, so it must not fail on the same pages: a page
184
303
  // whose only text is its menu still has markup, and printing it is the
185
- // whole point of `raw`.
186
- if (outTokens < MIN_CONTENT) noContent(finalUrl, `~${outTokens} tokens of markdown`, '');
304
+ // whole point of `raw`. And a short page that arrived short is not
305
+ // blank, so the verdict needs the same evidence the compact view asks
306
+ // for: near-nothing distilled out of markup that promised more.
307
+ if (outTokens < MIN_CONTENT && contentFailure(outTokens, htmlTokens)) {
308
+ noContent(finalUrl, `~${outTokens} tokens of markdown`, '');
309
+ }
187
310
  return;
188
311
  }
189
312
  const page = distill(html, finalUrl);
190
- const failure = contentFailure(contentTokens(page), htmlTokens);
313
+ const auth = authFailure(page, finalUrl, { hadAuth });
314
+ const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
315
+ if (auth) {
316
+ if (jar) clearCookieJar(sessionName);
317
+ noContent(finalUrl, auth);
318
+ return;
319
+ }
191
320
  const { text, stats } = render(page, { budget });
192
321
  remember(page, sessionName, stats.next);
193
322
  console.log(text);
@@ -202,6 +331,33 @@ async function main() {
202
331
  if (failure) noContent(finalUrl, failure);
203
332
  return;
204
333
  }
334
+ case 'search': {
335
+ // A search oc runs itself: a site's static index or docs corpus is
336
+ // fetched (or read back from its day cache) and ranked here, a JSON
337
+ // search API is asked directly. Either way the result list rides the
338
+ // exact `open` path: distilled, rendered, remembered, so `do <n>`
339
+ // follows a result. Only the list is ever printed; the index, corpus,
340
+ // and response stay out of context.
341
+ const t0 = performance.now();
342
+ const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch };
343
+ const kind = Object.keys(local).find((k) => search[k]);
344
+ const { url, html, via } = kind
345
+ ? await local[kind](search[kind], search.query)
346
+ : await apiSearch(search.api, search.query);
347
+ const page = distill(html, url);
348
+ if (values.json) {
349
+ remember(page, sessionName);
350
+ console.log(JSON.stringify({ ...page, empty: false }));
351
+ return;
352
+ }
353
+ const { text, stats } = render(page, { budget: asked || 500 });
354
+ remember(page, sessionName, stats.next);
355
+ console.log(text);
356
+ if (verbose) {
357
+ console.error(`~${stats.tokens} tokens, results via ${via}, ${Math.round(performance.now() - t0)}ms`);
358
+ }
359
+ return;
360
+ }
205
361
  case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 }));
206
362
  case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 }));
207
363
  case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 }));