@only-cli/oc 0.3.0-beta.1 → 0.3.0-beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -59,7 +59,8 @@ No setup at all also works: `npx @only-cli/oc` runs without a global install, an
59
59
  ```
60
60
  oc open <url> fetch and render a page with numbered actions
61
61
  oc do <n> follow the numbered link [n], or read [n] if it is text
62
- oc find <query> where a string appears on the page already open
62
+ oc find <query> where a string appears on the page already open, or
63
+ the region itself when only one place matches
63
64
  oc read <n> full text of the region at [n], up to 2000 tokens
64
65
  oc next the next budget worth of the page already open
65
66
  oc raw [url] distilled markdown of the whole page
@@ -69,7 +70,7 @@ oc submit [n] submit a form (planned)
69
70
 
70
71
  Flags: `--budget <tokens>` (default 500), `--json`, `--html` (raw as cleaned HTML), `--session <name>`, `--verbose`/`-v` (metrics on stderr, or export `OC_VERBOSE=1`).
71
72
 
72
- `oc open` remembers the page it rendered in a JSON file per session under `~/.only-cli` (override with `OC_HOME`), so `oc do 3` follows `[3]` without the agent ever handling a URL. Pages longer than the budget say what they left out; `oc find`, `oc read <n>`, and `oc next` read the rest without refetching the page. The budget is a target rather than a hard cap: a page that would only run a little long is printed whole rather than cut, since one extra tool call costs far more than the tokens it would have saved.
73
+ `oc open` remembers the page it rendered in a JSON file per session under `~/.only-cli` (override with `OC_HOME`), so `oc do 3` follows `[3]` without the agent ever handling a URL. A result title on a search page is a link, so `oc do` on it opens the result rather than repeating the title. Pages longer than the budget say what they left out; `oc find`, `oc read <n>`, and `oc next` read the rest without refetching the page, and a `find` with a single match prints that region instead of the number to read it with. The budget is a target rather than a hard cap: a page that would only run a little long is printed whole rather than cut, since one extra tool call costs far more than the tokens it would have saved.
73
74
 
74
75
  ## Supported websites
75
76
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@only-cli/oc",
3
- "version": "0.3.0-beta.1",
3
+ "version": "0.3.0-beta.2",
4
4
  "description": "Turn websites into a compact CLI so AI agents can browse without burning tokens.",
5
5
  "type": "module",
6
6
  "bin": {
package/src/act.js CHANGED
@@ -9,7 +9,7 @@
9
9
  */
10
10
 
11
11
  import { DEFAULT_SESSION, handleFor, handleNumbers, loadSession, saveSession } from './session.js';
12
- import { estimateTokens, formatBlock, render } from './render.js';
12
+ import { FINISH, estimateTokens, formatBlock, render } from './render.js';
13
13
 
14
14
  export class NotImplemented extends Error {
15
15
  constructor(command) {
@@ -63,7 +63,11 @@ export function activate(n, { session = DEFAULT_SESSION } = {}) {
63
63
  const range = nums.length ? `1-${Math.max(...nums)}` : 'none';
64
64
  throw new Error(`no [${n}] on ${state.url} (handles ${range}), run 'oc open <url>' again to renumber`);
65
65
  }
66
- if (handle.type === 'text' || handle.type === 'heading') {
66
+ // A heading can be a link: on a search results page the result title is one,
67
+ // and opening it is what `do` was asked for. Reading the title back instead
68
+ // cost a turn, and then another to find the number that does navigate, so a
69
+ // heading that has an href falls through to the link below.
70
+ if ((handle.type === 'text' || handle.type === 'heading') && !handle.href) {
67
71
  // There is nothing to follow, but the agent asked to see what is at [n],
68
72
  // and that is what read prints. Refusing would spend a whole turn to name
69
73
  // the command that should have run, and a turn costs more than the page.
@@ -208,14 +212,41 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
208
212
  return `no match for "${query}"${tried} in ${blocks.length} blocks on ${state.url}, try fewer words or 'oc raw' for the full text`;
209
213
  }
210
214
 
211
- const lines = [`${hits.length} ${hits.length === 1 ? 'match' : 'matches'} for "${query}"${loose ? ', matching the words separately' : ''}`];
215
+ const separately = loose ? ', matching the words separately' : '';
216
+
217
+ // One match is not an index, it is the answer. Naming the number and
218
+ // stopping spends a turn to say where to look, and the agent's next command
219
+ // is always the `read` that looks, so find does the reading. Measured on an
220
+ // AWS CLI reference page, `find "Example 7"` printed a 24 token heading and
221
+ // the example it names was in the block after it.
222
+ if (hits.length === 1 && hits[0].n != null) {
223
+ const only = hits[0];
224
+ const follow = only.type === 'link' ? 'do <n> | ' : '';
225
+ return [
226
+ `1 match for "${query}"${separately}, region [${only.n}]`,
227
+ read(only.n, { session, budget: budget * FINISH }),
228
+ `actions: ${follow}read <n> | next | raw`,
229
+ ].join('\n');
230
+ }
231
+
232
+ const lines = [`${hits.length} ${hits.length === 1 ? 'match' : 'matches'} for "${query}"${separately}`];
233
+ // A snippet is short because many matches have to share one screen. When
234
+ // every match would fit whole inside the allowance a page already gets for
235
+ // finishing (render's FINISH), showing them whole makes this command the
236
+ // answer instead of an index into a `read <n>` that has to run next. The
237
+ // trade is the same one FINISH documents, and it is not close: a few
238
+ // hundred tokens against a turn.
239
+ const label = (hit, text) => `[${hit.n ?? '?'}] ${text}`;
240
+ const whole = hits.reduce((n, h) => n + estimateTokens(label(h, h.text)) + 1, estimateTokens(lines[0]));
241
+ const full = whole <= budget * FINISH;
242
+ const cap = full ? budget * FINISH : budget;
212
243
  let spent = estimateTokens(lines[0]);
213
244
  let shown = 0;
214
245
  let hasLinks = false;
215
246
  for (const hit of hits) {
216
- const line = `[${hit.n ?? '?'}] ${hit.snippet}`;
247
+ const line = label(hit, full ? hit.text : hit.snippet);
217
248
  const cost = estimateTokens(line) + 1;
218
- if (spent + cost > budget && shown) break;
249
+ if (spent + cost > cap && shown) break;
219
250
  spent += cost;
220
251
  shown++;
221
252
  if (hit.type === 'link') hasLinks = true;
@@ -252,7 +283,7 @@ function search(blocks, terms) {
252
283
  const start = Math.max(0, Math.min(...found) - BEFORE);
253
284
  const end = Math.min(block.text.length, start + SNIPPET);
254
285
  const snippet = `${start > 0 ? '... ' : ''}${block.text.slice(start, end)}${end < block.text.length ? ' ...' : ''}`;
255
- out.push({ n, type: block.type, snippet });
286
+ out.push({ n, type: block.type, snippet, text: block.text });
256
287
  }
257
288
  return out;
258
289
  }
package/src/cli.js CHANGED
@@ -11,7 +11,8 @@ const HELP = `only-cli: the web as a compact terminal, built for AI agents.
11
11
  usage: oc <command> [args] [flags]
12
12
 
13
13
  open <url> fetch and render a page with numbered actions
14
- find <query> where a string appears on the page already open
14
+ find <query> where a string appears on the page already open, or
15
+ the region itself when only one place matches
15
16
  next the next budget worth of the page already open
16
17
  read <n> full text of the region at [n], up to 2000 tokens
17
18
  raw [url] distilled markdown of the whole page
@@ -36,7 +37,8 @@ flags:
36
37
 
37
38
  'oc open' remembers the page it printed, so 'oc do 3' follows link [3] without
38
39
  you ever handling its URL, and 'oc next' or 'oc read 12' picks up what the
39
- budget left behind without fetching it again. State lives in ~/.only-cli
40
+ budget left behind without fetching it again. 'oc do' on a search result title
41
+ opens the result, because that title is a link. State lives in ~/.only-cli
40
42
  (override with OC_HOME).`;
41
43
 
42
44
  // A rendered page has to be remembered or its [3] means nothing to the next
package/src/distill.js CHANGED
@@ -7,7 +7,7 @@ import TurndownService from 'turndown';
7
7
  * @property {string} text
8
8
  * @property {number} [n] - action handle
9
9
  * @property {number} [level] - heading level 1..6
10
- * @property {string} [href] - links only
10
+ * @property {string} [href] - links, and a heading that is one
11
11
  * @property {string} [name] - inputs only
12
12
  *
13
13
  * @typedef {Object} Page
@@ -82,6 +82,29 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim();
82
82
  const asHTML = (text, url = '', opts = {}) =>
83
83
  jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text) ?? text;
84
84
 
85
+ /**
86
+ * The link a heading is, if it is one. A search engine puts the result title
87
+ * in an anchor inside an <h2>, so a heading can be the most followable thing
88
+ * on the page, and taking only its text threw that away.
89
+ *
90
+ * The heading has to BE the link, not merely contain one: exactly one anchor,
91
+ * labelled with the whole heading. Documentation fails that test on purpose.
92
+ * Every heading in the Rust book and every one on an AWS CLI reference page
93
+ * carries a permalink to its own id, so following those would refetch the
94
+ * page the agent is already reading, which is worse than the reading it
95
+ * already gets. A bare fragment is never a destination.
96
+ * @param {any} node - the heading element
97
+ * @param {string} text - its cleaned text
98
+ * @returns {string|null}
99
+ */
100
+ function headingHref(node, text) {
101
+ const anchors = node.querySelectorAll('a[href]');
102
+ if (anchors.length !== 1) return null;
103
+ const href = anchors[0].getAttribute('href') ?? '';
104
+ if (!href || href.startsWith('#')) return null;
105
+ return clean(anchors[0].textContent) === text ? href : null;
106
+ }
107
+
85
108
  /**
86
109
  * Reduce raw HTML to an interaction tree: readable text plus numbered
87
110
  * elements, in document order. The walk is deterministic and numbering is a
@@ -126,7 +149,10 @@ export function distill(html, url = '') {
126
149
 
127
150
  if (/^h[1-6]$/.test(tag)) {
128
151
  const text = clean(node.textContent);
129
- if (text) blocks.push({ type: 'heading', level: Number(tag[1]), text });
152
+ if (text) {
153
+ const href = headingHref(node, text);
154
+ blocks.push({ type: 'heading', level: Number(tag[1]), text, ...(href ? { href } : {}) });
155
+ }
130
156
  return;
131
157
  }
132
158
  if (tag === 'a' && node.getAttribute('href')) {
@@ -513,6 +539,15 @@ const TITLE_KEYS = [
513
539
  'full_name', 'summary', 'question', 'message',
514
540
  ];
515
541
 
542
+ // Keys an API is likely to put its list of results under. A response using one
543
+ // of these is a list whatever else it carries, so the name settles it before
544
+ // shape does: a sideloaded `included` array can outnumber the `items` the
545
+ // request was for without being what the request was for.
546
+ const CONTAINER_KEYS = [
547
+ 'items', 'data', 'results', 'hits', 'records', 'rows', 'entries',
548
+ 'nodes', 'edges', 'docs', 'list', 'children', 'values',
549
+ ];
550
+
516
551
  // Keys holding the item's own page. A URL under any other name is still found,
517
552
  // by looking at values rather than names, but these win when several qualify.
518
553
  const LINK_KEYS = ['link', 'url', 'html_url', 'web_url', 'permalink', 'href'];
@@ -540,6 +575,14 @@ const NESTED_PENALTY = 0.25;
540
575
  // How many names the footer lists when it says which fields it left out.
541
576
  const DROPPED_LISTED = 5;
542
577
 
578
+ // Characters of markup a field may carry before the compact view stops
579
+ // treating it as a document and starts treating it as somewhere to go. A
580
+ // question body arrives well under this and is worth rendering in place, links
581
+ // and code and all. A package readme arrives at four figures and distils into
582
+ // more blocks than the resource it is attached to has fields, which buries the
583
+ // resource the response was fetched for. `oc raw` renders either one in full.
584
+ const BODY_CAP = 4000;
585
+
543
586
  // Seconds and milliseconds since the epoch, bounded either side so an ordinary
544
587
  // count (a score, a byte size) is never mistaken for a date.
545
588
  const EPOCH_S = [1e9, 4e9];
@@ -572,6 +615,10 @@ const decodeEntities = (s) =>
572
615
  const isPlain = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
573
616
  const isURL = (v) => typeof v === 'string' && /^https?:\/\/\S+$/.test(v);
574
617
  const looksHTML = (v) => typeof v === 'string' && /<\/?(p|div|pre|code|br|ul|ol|li|h[1-6]|blockquote|table|img|a|em|strong)\b[^>]*>/i.test(v);
618
+ // Markup as one line of prose, for a body the compact view is pointing at
619
+ // rather than rendering. The distiller is what reads markup properly; this
620
+ // only has to make a line an agent can tell one body from another by.
621
+ const stripTags = (v) => clean(decodeEntities(String(v).replace(/<[^>]*>/g, ' ')));
575
622
 
576
623
  /**
577
624
  * One level of flattening, so `owner: {display_name}` becomes an
@@ -662,32 +709,40 @@ function chooseFields(rows, skip) {
662
709
  }
663
710
 
664
711
  /**
665
- * Pick the array the response is actually about: the root when it is one,
666
- * otherwise the longest array of objects at the top level, which is where
667
- * `items`, `data`, `results`, and `hits` all live. Everything beside it is
668
- * metadata about the request rather than content.
712
+ * Pick what the response is actually about: the root when it is an array or a
713
+ * single resource, otherwise the array of objects at the top level that holds
714
+ * the results. Everything beside it is metadata about the request rather than
715
+ * content.
669
716
  * @param {any} data
670
717
  * @returns {{items: any[], meta: Record<string, any>}}
671
718
  */
672
719
  function mainArray(data) {
673
720
  if (Array.isArray(data)) return { items: data, meta: {} };
674
- let key = '';
675
- /** @type {any[] | null} */
676
- let items = null;
721
+ /** @type {Map<string, any[]>} */
722
+ const arrays = new Map();
677
723
  for (const [k, v] of Object.entries(data)) {
678
724
  if (!Array.isArray(v) || !v.length) continue;
679
725
  if (!v.some(isPlain)) continue;
680
- if (!items || v.length > items.length) {
681
- items = v;
682
- key = k;
726
+ arrays.set(k, v);
727
+ }
728
+ let key = CONTAINER_KEYS.find((k) => arrays.has(k)) ?? '';
729
+ // A root carrying its own name is the resource, and an array hanging off it
730
+ // describes that resource rather than being the subject in its place. Taking
731
+ // the longest array regardless titled the npm registry's package endpoint
732
+ // after its two maintainers and demoted the package to the metadata line,
733
+ // where a 9KB readme then cost more than the rest of the page put together.
734
+ const named = TITLE_KEYS.some((k) => typeof data[k] === 'string' && data[k].trim() !== '');
735
+ if (!key && !named) {
736
+ for (const [k, v] of arrays) {
737
+ if (!key || v.length > (arrays.get(key)?.length ?? 0)) key = k;
683
738
  }
684
739
  }
685
- // A response with no array is a single resource, which renders as one item
686
- // rather than as a special case.
687
- if (!items) return { items: [data], meta: {} };
740
+ // A response with no results array of its own is a single resource, which
741
+ // renders as one item rather than as a special case.
742
+ if (!key) return { items: [data], meta: {} };
688
743
  const meta = { ...data };
689
744
  delete meta[key];
690
- return { items, meta };
745
+ return { items: arrays.get(key) ?? [data], meta };
691
746
  }
692
747
 
693
748
  /**
@@ -784,9 +839,16 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
784
839
  if (kept && !kept.has(key) && !looksHTML(value)) continue;
785
840
  // A field carrying HTML is a page in itself: `filter=withbody` on the
786
841
  // Stack Exchange API puts a whole question in one. It goes through the
787
- // distiller like any other markup instead of into a cell.
842
+ // distiller like any other markup instead of into a cell, unless it is
843
+ // longer than the compact view can afford, in which case it becomes one
844
+ // numbered line rather than a dozen blocks that bury the item it hangs
845
+ // off. `oc read <n>` opens it at a budget that fits it, `oc raw` always.
788
846
  if (looksHTML(value)) {
789
- bodies.push(String(value));
847
+ if (full || String(value).length <= BODY_CAP) {
848
+ bodies.push(String(value));
849
+ continue;
850
+ }
851
+ long.push(`<p>${escHTML(`${key}: ${stripTags(value)}`)}</p>`);
790
852
  continue;
791
853
  }
792
854
  const rendered = row.get(key);
@@ -824,12 +886,21 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
824
886
  }
825
887
 
826
888
  const metaBits = [];
889
+ const metaLong = [];
827
890
  for (const [key, value] of Object.entries(meta)) {
828
891
  if (value === null || typeof value === 'object') continue;
829
892
  const rendered = renderValue(key, value);
830
- if (rendered) metaBits.push(`${key}=${rendered}`);
893
+ if (!rendered) continue;
894
+ // A summary line has to stay a line. One long scalar at the root, a
895
+ // package readme or an endpoint description, would otherwise spend the
896
+ // whole page budget here, so it becomes a block of its own instead. The
897
+ // block is numbered, so `oc read <n>` opens it when it fits that budget
898
+ // and `oc raw` has it whatever its size.
899
+ if (rendered.length > TEXT_CAP) metaLong.push(`<p>${escHTML(`${key}: ${rendered}`)}</p>`);
900
+ else metaBits.push(`${key}=${rendered}`);
831
901
  }
832
902
  if (metaBits.length) parts.push(`<p>response: ${escHTML(metaBits.join(', '))}</p>`);
903
+ parts.push(...metaLong);
833
904
 
834
905
  const count = `${items.length} ${items.length === 1 ? 'item' : 'items'}`;
835
906
  return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
package/src/fetch.js CHANGED
@@ -24,6 +24,25 @@ const loadImpers = () => {
24
24
  const BLOCKED_MESSAGE = 'blocked: private or internal URL';
25
25
  const MAX_REDIRECTS = 20;
26
26
 
27
+ // What oc can turn into text: any text/* type, plus the application/* types
28
+ // that are really text (json, xml, and the +json / +xml families a feed or an
29
+ // API answers with). A PNG matches none of these, and rendering one produces
30
+ // pages of mojibake an agent then pays for, so it is refused by name instead.
31
+ const READABLE_TYPE = /^\s*(?:text\/|application\/(?:json|xml|javascript|x-ndjson|[\w.+-]*\+(?:json|xml)))/i;
32
+
33
+ /**
34
+ * Refuse a response oc cannot read as text. Both transports call this: the
35
+ * gate has to live on whichever client got the page, or the same URL renders
36
+ * as an error through fetch and as binary noise through impers.
37
+ * @param {string | null | undefined} type - the content-type header
38
+ */
39
+ export function assertReadableType(type) {
40
+ // No header at all is not a refusal: plenty of small servers omit it, and
41
+ // the distiller handles whatever comes back.
42
+ if (!type || READABLE_TYPE.test(type)) return;
43
+ throw new Error(`not a page oc can read (${type.split(';')[0].trim()}), it renders HTML, XML feeds, JSON, and plain text`);
44
+ }
45
+
27
46
  // IPv4 ranges with no business receiving a server-initiated fetch: loopback,
28
47
  // link-local, the three RFC 1918 private blocks, carrier-grade NAT, the
29
48
  // unspecified/broadcast addresses, and the documentation/benchmark ranges.
@@ -169,6 +188,7 @@ async function viaImpers(impers, target) {
169
188
  status = res.status ?? res.statusCode ?? 0;
170
189
  }
171
190
  if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
191
+ assertReadableType(res.headers.get('content-type'));
172
192
  const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
173
193
  return { url: res.url ?? target, html, status, via };
174
194
  }
@@ -197,12 +217,6 @@ async function viaFetch(target) {
197
217
  if (!res.ok) {
198
218
  throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
199
219
  }
200
- // JSON is a page here too: an API answer distills into one article per item.
201
- // The impers path never checked the type at all, so this is also what keeps
202
- // the two transports rendering the same URL the same way.
203
- const type = res.headers.get('content-type') ?? '';
204
- if (type && !/html|xml|json/.test(type)) {
205
- throw new Error(`not a page oc can read (${type.split(';')[0]}), it renders HTML, XML feeds, and JSON`);
206
- }
220
+ assertReadableType(res.headers.get('content-type'));
207
221
  return { url: res.url || current, html: await res.text(), status: res.status, via: 'fetch' };
208
222
  }
package/src/render.js CHANGED
@@ -24,7 +24,7 @@ const num = (v) => v.toLocaleString('en-US');
24
24
  // saving is only collected when the agent would have paged at all, while the
25
25
  // overspend is paid on every page that runs a little long, including the ones
26
26
  // answered by their first few lines. Four caps that overspend near 1,500 tokens.
27
- const FINISH = 4;
27
+ export const FINISH = 4;
28
28
 
29
29
  /**
30
30
  * Budget-aware compact view of a distilled page. `from` is a position in the
@@ -162,5 +162,22 @@ export function formatBlock(b, { full = false } = {}) {
162
162
  }
163
163
  }
164
164
 
165
- const truncate = (s) =>
166
- s.length > TEXT_CAP ? `${s.slice(0, TEXT_CAP)} ... +${num(s.length - TEXT_CAP)} chars` : s;
165
+ // A cut inside a sentence makes the half that is shown untrustworthy. Asked
166
+ // for the first sentence of a page, an agent was given it in full, followed by
167
+ // a truncation marker, and spent a turn on `read` to find out whether the
168
+ // sentence carried on. Ending on the last sentence that finished inside the cap
169
+ // answers that in the view itself. The floor bounds what the courtesy costs: a
170
+ // block whose only sentence end is early keeps the plain cut instead of
171
+ // throwing away a third of the window.
172
+ const SENTENCE_END = /[.!?]["')\]]*(?=\s)/g;
173
+ const SENTENCE_FLOOR = 0.7;
174
+
175
+ const truncate = (s) => {
176
+ if (s.length <= TEXT_CAP) return s;
177
+ let cut = TEXT_CAP;
178
+ for (const m of s.slice(0, TEXT_CAP).matchAll(SENTENCE_END)) {
179
+ const end = (m.index ?? 0) + m[0].length;
180
+ if (end >= TEXT_CAP * SENTENCE_FLOOR) cut = end;
181
+ }
182
+ return `${s.slice(0, cut).trimEnd()} ... +${num(s.length - cut)} chars`;
183
+ };