@only-cli/oc 0.2.0-beta.1 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.js CHANGED
@@ -4,30 +4,55 @@ import { fetchPage } from './fetch.js';
4
4
  import { distill, toMarkdown, toHTML } from './distill.js';
5
5
  import { render, estimateTokens } from './render.js';
6
6
  import * as act from './act.js';
7
+ import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
7
8
 
8
9
  const HELP = `only-cli: the web as a compact terminal, built for AI agents.
9
10
 
10
11
  usage: oc <command> [args] [flags]
11
12
 
12
13
  open <url> fetch and render a page with numbered actions
13
- raw <url> distilled markdown of the whole page
14
- do <n> activate a numbered element (v0.2)
14
+ find <query> where a string appears on the page already open
15
+ next the next budget worth of the page already open
16
+ read <n> full text of the region at [n], up to 2000 tokens
17
+ raw [url] distilled markdown of the whole page
18
+ do <n> follow the numbered link [n], or read [n] if it is text
15
19
  fill <n> <text> type into a numbered input (v0.2)
16
20
  submit [n] submit a form (v0.2)
17
- read [n] full text of one region (v0.2)
18
- find <query> search visible text on the current page (v0.2)
19
- back | next history and pagination (v0.2)
21
+ back return to the previous page (v0.2)
20
22
  session ls|rm manage saved sessions (v0.2)
21
23
 
22
24
  flags:
23
- --budget <tokens> tighten or loosen the render budget (default 500)
25
+ --budget <tokens> tighten or loosen the render budget (default 500,
26
+ 2000 for read). It is a target, not a ceiling: a page
27
+ that would finish within about four times it is printed
28
+ whole rather than costing you a second command
24
29
  --json machine-stable JSON output
25
30
  --html raw only: cleaned HTML instead of markdown
26
31
  --verbose, -v metrics on stderr: tokens saved vs the page HTML, HTTP
27
32
  status and client identity, timing, transfer size, and
28
33
  memory. --stats is an alias; OC_VERBOSE=1 turns it on
29
34
  globally. Off by default because metrics cost tokens too.
30
- --session <name> named session (v0.2)`;
35
+ --session <name> keep separate page state under a name (default: default)
36
+
37
+ 'oc open' remembers the page it printed, so 'oc do 3' follows link [3] without
38
+ you ever handling its URL, and 'oc next' or 'oc read 12' picks up what the
39
+ budget left behind without fetching it again. State lives in ~/.only-cli
40
+ (override with OC_HOME).`;
41
+
42
+ // A rendered page has to be remembered or its [3] means nothing to the next
43
+ // command. Saving state must never break a render, so a home directory that
44
+ // cannot be written costs the agent `do`, `read`, and `next`, and nothing
45
+ // else, but silently: a sandbox that blocks the write leaves `do` resolving
46
+ // against whatever session last saved successfully, possibly from an
47
+ // unrelated page, with no sign anything is wrong. So the failure still prints,
48
+ // on stderr where it costs nothing until something breaks.
49
+ const remember = (page, name, cursor) => {
50
+ try {
51
+ saveSession(name, sessionFromPage(page, loadSession(name), { cursor }));
52
+ } catch (err) {
53
+ console.error(`oc: warning: could not save session '${name}' (${err.message}), 'do'/'read'/'next' may act on stale state`);
54
+ }
55
+ };
31
56
 
32
57
  // What browsing costs without this tool is the raw page HTML in context.
33
58
  const savings = (out, raw) =>
@@ -57,13 +82,36 @@ async function main() {
57
82
  return;
58
83
  }
59
84
 
85
+ const sessionName = values.session || DEFAULT_SESSION;
86
+ // Zero means "whatever this command's default is", which differs: the
87
+ // compact view targets 500 tokens, read targets 2000.
88
+ const asked = values.budget ? Number(values.budget) : 0;
89
+ if (values.budget && (!Number.isFinite(asked) || asked <= 0)) {
90
+ throw new Error('--budget must be a positive number');
91
+ }
92
+
60
93
  switch (command) {
61
94
  case 'open':
95
+ case 'do':
62
96
  case 'raw': {
63
- const url = args[0];
97
+ // `do` is `open` with the URL looked up from the last render instead of
98
+ // typed, so both commands share one fetch, render, and save path. `raw`
99
+ // with no URL means the page already open, which is what the compact
100
+ // view's footer offers when it has cut something.
101
+ let url;
102
+ if (command === 'do') {
103
+ const target = act.activate(Number(args[0]), { session: sessionName });
104
+ // A number that points at text has no page behind it, so `do` reads it
105
+ // rather than making the agent pay for a second command to be told.
106
+ if (target.read != null) {
107
+ return console.log(act.read(target.read, { session: sessionName, budget: asked || 2000 }));
108
+ }
109
+ url = target.url;
110
+ } else {
111
+ url = args[0] ?? (command === 'raw' ? loadSession(sessionName)?.url : undefined);
112
+ }
64
113
  if (!url) throw new Error(`usage: oc ${command} <url>`);
65
- const budget = values.budget ? Number(values.budget) : 500;
66
- if (!Number.isFinite(budget) || budget <= 0) throw new Error('--budget must be a positive number');
114
+ const budget = asked || 500;
67
115
  const t0 = performance.now();
68
116
  const { url: finalUrl, html, status, via } = await fetchPage(url);
69
117
  const fetchMs = performance.now() - t0;
@@ -74,7 +122,9 @@ async function main() {
74
122
  + `${Math.round(html.length / 1024)}KB transferred, ${Math.round(rss / 1048576)}MB memory`;
75
123
  };
76
124
  if (values.json) {
77
- console.log(JSON.stringify(distill(html, finalUrl)));
125
+ const page = distill(html, finalUrl);
126
+ remember(page, sessionName);
127
+ console.log(JSON.stringify(page));
78
128
  if (verbose) console.error(resources());
79
129
  return;
80
130
  }
@@ -87,19 +137,19 @@ async function main() {
87
137
  }
88
138
  const page = distill(html, finalUrl);
89
139
  const { text, stats } = render(page, { budget });
140
+ remember(page, sessionName, stats.next);
90
141
  console.log(text);
91
142
  if (verbose) {
92
143
  console.error(`~${stats.tokens} tokens, ${stats.rendered}/${stats.blocks} blocks rendered, ${savings(stats.tokens, htmlTokens)}; ${resources()}`);
93
144
  }
94
145
  return;
95
146
  }
96
- case 'do': return act.activate(Number(args[0]));
147
+ case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 }));
148
+ case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 }));
149
+ case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 }));
97
150
  case 'fill': return act.fill(Number(args[0]), args.slice(1).join(' '));
98
151
  case 'submit': return act.submit(args[0] ? Number(args[0]) : undefined);
99
- case 'read': return act.read(args[0] ? Number(args[0]) : undefined);
100
- case 'find': return act.find(args.join(' '));
101
152
  case 'back': return act.back();
102
- case 'next': return act.next();
103
153
  case 'session': throw new act.NotImplemented('session');
104
154
  default:
105
155
  throw new Error(`unknown command '${command}', run oc --help`);
package/src/distill.js CHANGED
@@ -3,9 +3,9 @@ import TurndownService from 'turndown';
3
3
 
4
4
  /**
5
5
  * @typedef {Object} Block
6
- * @property {'heading'|'text'|'link'|'input'|'button'} type
6
+ * @property {'heading'|'text'|'link'|'input'|'button'|'divider'} type
7
7
  * @property {string} text
8
- * @property {number} [n] - action handle, only on interactive blocks
8
+ * @property {number} [n] - action handle
9
9
  * @property {number} [level] - heading level 1..6
10
10
  * @property {string} [href] - links only
11
11
  * @property {string} [name] - inputs only
@@ -16,6 +16,10 @@ import TurndownService from 'turndown';
16
16
  * @property {Block[]} blocks
17
17
  */
18
18
 
19
+ // Where the compact view cuts a text block. It lives here because numbering
20
+ // depends on it: a block long enough to be cut is a block that needs a handle.
21
+ export const TEXT_CAP = 200;
22
+
19
23
  // Dropped wholesale, subtree included. Nav and footer stay in v0.1: on many
20
24
  // sites they carry the only working links, and the budget in render.js is
21
25
  // what keeps them from costing anything.
@@ -24,35 +28,87 @@ const DROP = new Set([
24
28
  'link', 'meta', 'head', 'canvas', 'video', 'audio', 'object',
25
29
  ]);
26
30
 
31
+ // Elements that are furniture by definition. The main content is never one of
32
+ // them, so the search below refuses to descend into them.
33
+ const FURNITURE = new Set(['nav', 'header', 'footer', 'aside']);
34
+
35
+ // Elements that end a line on the page and so must end one here. Without this
36
+ // the text of six separate posts merges into a single block, because nothing
37
+ // between them survives distillation to keep them apart.
38
+ const BLOCKY = new Set([
39
+ 'p', 'div', 'article', 'section', 'li', 'ul', 'ol', 'tr', 'td', 'th',
40
+ 'blockquote', 'pre', 'figure', 'figcaption', 'dt', 'dd', 'form', 'br', 'hr',
41
+ 'main', 'nav', 'header', 'footer', 'aside',
42
+ ]);
43
+
44
+ // Selectors a page uses to say where its content is. One match is a claim
45
+ // worth believing; several <article> elements mean a listing, where the list
46
+ // is the content and taking the first would throw the rest away.
47
+ const MAIN_SELECTORS = ['main', '[role="main"]', 'article'];
48
+
49
+ // Reordering only matters on a page that cannot be printed whole: if the
50
+ // budget covers everything, what leads is a question with no consequences.
51
+ // Below this much text the page is left in document order.
52
+ const MIN_PAGE = 3000;
53
+ // A candidate has to hold this share of the page's prose to be believed.
54
+ const MIN_SHARE = 0.25;
55
+ // How much of its parent's prose a child must hold before the search moves
56
+ // down into it, and how much of the page's it must keep to be worth choosing.
57
+ const DESCEND_SHARE = 0.6;
58
+ const KEEP_SHARE = 0.5;
59
+ const MAX_DEPTH = 12;
60
+
61
+ // A short link or button label repeated this many times down a page is
62
+ // per-item furniture (permalink, save, reply, like, repost) rather than
63
+ // content. On a forum thread or a social timeline there is one set per item,
64
+ // which costs more than the items do.
65
+ // A link label can be content, so it gets the benefit of the doubt for longer
66
+ // than a button label, which never is.
67
+ const REPEAT_LIMIT = 5;
68
+ const REPEAT_LIMIT_BUTTON = 3;
69
+ const REPEAT_MAX_LEN = 25;
70
+
27
71
  const clean = (s) => s.replace(/\s+/g, ' ').trim();
28
72
 
29
73
  /**
30
74
  * Reduce raw HTML to an interaction tree: readable text plus numbered
31
- * interactive elements, in document order. Handles are assigned during a
32
- * single deterministic walk, so the same page always yields the same numbers.
75
+ * elements, in document order. The walk is deterministic and numbering is a
76
+ * second pass over its result, so the same page always yields the same
77
+ * numbers.
33
78
  * @param {string} html
34
79
  * @param {string} url
35
80
  * @returns {Page}
36
81
  */
37
82
  export function distill(html, url = '') {
38
- const { document } = parseHTML(html);
83
+ const { document } = parseHTML(youtubeToHTML(html) ?? transcriptToHTML(html) ?? feedToHTML(html) ?? html);
39
84
  const title = clean(document.querySelector('title')?.textContent ?? '');
40
85
  /** @type {Block[]} */
41
86
  const blocks = [];
42
- let handle = 0;
43
87
 
44
88
  const hidden = (el) =>
45
89
  el.getAttribute('hidden') !== null ||
46
90
  el.getAttribute('aria-hidden') === 'true' ||
47
91
  /display:\s*none/.test(el.getAttribute('style') ?? '');
48
92
 
93
+ /** Subtree already emitted, skipped when the rest of the page is walked. */
94
+ let done = null;
95
+
49
96
  const walk = (node) => {
50
97
  if (node.nodeType === 3) {
51
- const text = clean(node.textContent);
52
- if (text) blocks.push({ type: 'text', text });
98
+ const raw = node.textContent ?? '';
99
+ const text = clean(raw);
100
+ // A parser is free to split one run of text into several nodes, and
101
+ // linkedom does it around apostrophes, so `what's` arrives as `what`,
102
+ // `'`, `s`. Remembering the parent and whether the fragment had space
103
+ // at its edges is what lets mergeText put it back without inventing a
104
+ // space that was never in the page.
105
+ if (text) {
106
+ blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
107
+ }
53
108
  return;
54
109
  }
55
110
  if (node.nodeType !== 1) return;
111
+ if (node === done) return;
56
112
  const tag = node.localName;
57
113
  if (DROP.has(tag) || hidden(node)) return;
58
114
 
@@ -64,7 +120,7 @@ export function distill(html, url = '') {
64
120
  if (tag === 'a' && node.getAttribute('href')) {
65
121
  const text = clean(node.textContent);
66
122
  if (text) {
67
- blocks.push({ type: 'link', n: ++handle, text, href: node.getAttribute('href') });
123
+ blocks.push({ type: 'link', text, href: node.getAttribute('href') });
68
124
  }
69
125
  return;
70
126
  }
@@ -72,24 +128,164 @@ export function distill(html, url = '') {
72
128
  const kind = node.getAttribute('type') ?? 'text';
73
129
  if (kind === 'hidden') return;
74
130
  if (kind === 'submit' || kind === 'button') {
75
- blocks.push({ type: 'button', n: ++handle, text: node.getAttribute('value') ?? 'submit' });
131
+ blocks.push({ type: 'button', text: node.getAttribute('value') ?? 'submit' });
76
132
  return;
77
133
  }
78
134
  const name = node.getAttribute('name') ?? node.getAttribute('placeholder') ?? tag;
79
- blocks.push({ type: 'input', n: ++handle, text: kind, name });
135
+ blocks.push({ type: 'input', text: kind, name });
80
136
  return;
81
137
  }
82
138
  if (tag === 'button') {
83
- const text = clean(node.textContent) || 'button';
84
- blocks.push({ type: 'button', n: ++handle, text });
139
+ // An icon button carries its name in aria-label or title, and a button
140
+ // with none of the three cannot be described or pressed, so printing it
141
+ // spends tokens to say nothing. X puts seven of them under every post.
142
+ const text = clean(node.textContent)
143
+ || clean(node.getAttribute('aria-label') ?? '')
144
+ || clean(node.getAttribute('title') ?? '');
145
+ if (text) blocks.push({ type: 'button', text });
146
+ return;
147
+ }
148
+ if (BLOCKY.has(tag)) {
149
+ blocks.push({ type: 'break' });
150
+ for (const child of node.childNodes) walk(child);
151
+ blocks.push({ type: 'break' });
85
152
  return;
86
153
  }
87
154
  for (const child of node.childNodes) walk(child);
88
155
  };
89
156
 
90
157
  const body = bodyOf(document);
91
- if (body) walk(body);
92
- return { url, title, blocks: mergeText(blocks) };
158
+ const main = body ? mainOf(document, body) : null;
159
+ if (main) {
160
+ // Content first, everything else after it. The budget in render.js is
161
+ // spent top down, so whatever leads the list is what an agent gets for
162
+ // its first 500 tokens, and on most pages the top of the document is
163
+ // menus. Nothing is dropped: the chrome carries links `oc do` needs, it
164
+ // just stops being the thing that gets read.
165
+ walk(main);
166
+ done = main;
167
+ const contentBlocks = blocks.length;
168
+ if (body) walk(body);
169
+ if (blocks.length > contentBlocks) {
170
+ blocks.splice(contentBlocks, 0, { type: 'divider', text: '--- rest of page: navigation, sidebar, footer ---' });
171
+ }
172
+ } else if (body) {
173
+ walk(body);
174
+ }
175
+ return { url, title, blocks: number(mergeText(dropRepeats(blocks))) };
176
+ }
177
+
178
+ /**
179
+ * Remove link labels that repeat down the page. They are the per-item controls
180
+ * a template stamps onto every row, and on a long thread they outweigh the
181
+ * content. What went is stated in place, because a view that quietly drops
182
+ * things is a view an agent cannot trust.
183
+ * @param {Block[]} blocks
184
+ * @returns {Block[]}
185
+ */
186
+ function dropRepeats(blocks) {
187
+ const controls = (b) => b.type === 'link' || b.type === 'button';
188
+ /** @type {Map<string, {n: number, limit: number}>} */
189
+ const counts = new Map();
190
+ for (const b of blocks) {
191
+ if (!controls(b) || b.text.length > REPEAT_MAX_LEN) continue;
192
+ const key = b.text.toLowerCase();
193
+ const limit = b.type === 'button' ? REPEAT_LIMIT_BUTTON : REPEAT_LIMIT;
194
+ const seen = counts.get(key);
195
+ // A label worn by both a link and a button keeps the more patient limit.
196
+ counts.set(key, { n: (seen?.n ?? 0) + 1, limit: Math.max(seen?.limit ?? 0, limit) });
197
+ }
198
+ const repeated = new Set([...counts].filter(([, c]) => c.n >= c.limit).map(([k]) => k));
199
+ if (!repeated.size) return blocks;
200
+ const kept = blocks.filter((b) => !(controls(b) && repeated.has(b.text.toLowerCase())));
201
+ const gone = blocks.length - kept.length;
202
+ const names = [...repeated].slice(0, 3).join(', ');
203
+ const note = { type: 'divider', text: `--- ${gone} repeated controls hidden (${names}), 'oc raw' has them ---` };
204
+ // The note belongs with the content it was cut from, not after the chrome.
205
+ const boundary = kept.findIndex((b) => b.type === 'divider');
206
+ kept.splice(boundary === -1 ? kept.length : boundary, 0, note);
207
+ return kept;
208
+ }
209
+
210
+ /**
211
+ * Find the element holding the page's main content, or null when the page is
212
+ * too small or too flat for the question to have an answer.
213
+ * @param {any} document
214
+ * @param {any} body
215
+ * @returns {any}
216
+ */
217
+ function mainOf(document, body) {
218
+ if (clean(body.textContent ?? '').length < MIN_PAGE) return null;
219
+ const total = prose(body);
220
+ for (const selector of MAIN_SELECTORS) {
221
+ const found = [...document.querySelectorAll(selector)];
222
+ if (found.length !== 1) continue;
223
+ const el = found[0];
224
+ if (el !== body && prose(el) >= total * MIN_SHARE) return el;
225
+ }
226
+ return densest(body, total);
227
+ }
228
+
229
+ /**
230
+ * Walk down the tree while one child holds most of the prose of the element
231
+ * above it. That is what separates a content column from the page around it
232
+ * without knowing anything about the site.
233
+ * @param {any} body
234
+ * @param {number} total
235
+ * @returns {any}
236
+ */
237
+ function densest(body, total) {
238
+ let node = body;
239
+ for (let depth = 0; depth < MAX_DEPTH; depth++) {
240
+ let best = null;
241
+ let bestProse = 0;
242
+ for (const child of node.children ?? []) {
243
+ if (FURNITURE.has(child.localName) || DROP.has(child.localName)) continue;
244
+ const len = prose(child);
245
+ if (len > bestProse) {
246
+ best = child;
247
+ bestProse = len;
248
+ }
249
+ }
250
+ if (!best || bestProse < prose(node) * DESCEND_SHARE || bestProse < total * KEEP_SHARE) break;
251
+ node = best;
252
+ }
253
+ return node === body ? null : node;
254
+ }
255
+
256
+ /**
257
+ * Text length minus the text of controls, which is what a page is steered by
258
+ * rather than what it says.
259
+ * @param {any} el
260
+ * @returns {number}
261
+ */
262
+ function prose(el) {
263
+ const text = clean(el.textContent ?? '').length;
264
+ let controls = 0;
265
+ // Link text is what navigation is made of. Option text is worse: a country
266
+ // dropdown is hundreds of characters that no link subtraction touches, and
267
+ // on a search results page it outscores results that are themselves links.
268
+ for (const node of el.querySelectorAll?.('a, select, button') ?? []) {
269
+ controls += clean(node.textContent ?? '').length;
270
+ }
271
+ return text - controls;
272
+ }
273
+
274
+ /**
275
+ * Assign handles in document order. Interactive elements get one because they
276
+ * can be acted on, headings and long text blocks because they can be read:
277
+ * a text block over the cap is printed cut, and its number is what makes the
278
+ * rest of it reachable with `oc read <n>` instead of a second whole-page fetch.
279
+ * @param {Block[]} blocks
280
+ * @returns {Block[]}
281
+ */
282
+ function number(blocks) {
283
+ let handle = 0;
284
+ for (const block of blocks) {
285
+ if (block.type === 'divider') continue;
286
+ if (block.type !== 'text' || block.text.length > TEXT_CAP) block.n = ++handle;
287
+ }
288
+ return blocks;
93
289
  }
94
290
 
95
291
  /**
@@ -107,7 +303,9 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document
107
303
  * @param {string} html
108
304
  */
109
305
  function cleanDocument(html) {
110
- const { document } = parseHTML(html);
306
+ const { document } = parseHTML(youtubeToHTML(html) ?? transcriptToHTML(html) ?? feedToHTML(html) ?? html);
307
+ // Read the title before the sweep below removes the head with it.
308
+ const title = clean(document.querySelector('title')?.textContent ?? '');
111
309
  for (const tag of DROP) {
112
310
  for (const el of [...document.querySelectorAll(tag)]) el.remove();
113
311
  }
@@ -115,7 +313,7 @@ function cleanDocument(html) {
115
313
  for (const el of [...document.querySelectorAll('[style]')]) {
116
314
  if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
117
315
  }
118
- return document;
316
+ return { document, title };
119
317
  }
120
318
 
121
319
  /**
@@ -125,8 +323,7 @@ function cleanDocument(html) {
125
323
  * @returns {string}
126
324
  */
127
325
  export function toMarkdown(html) {
128
- const document = cleanDocument(html);
129
- const title = clean(document.querySelector('title')?.textContent ?? '');
326
+ const { document, title } = cleanDocument(html);
130
327
  const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
131
328
  const el = bodyOf(document);
132
329
  const body = el ? turndown.turndown(el.innerHTML).trim() : '';
@@ -140,11 +337,158 @@ export function toMarkdown(html) {
140
337
  * @returns {string}
141
338
  */
142
339
  export function toHTML(html) {
143
- const document = cleanDocument(html);
340
+ const { document } = cleanDocument(html);
144
341
  const el = bodyOf(document);
145
342
  return el ? el.innerHTML.trim() : '';
146
343
  }
147
344
 
345
+ /**
346
+ * Sites behind hard bot challenges often leave their Atom or RSS feeds open:
347
+ * Stack Overflow challenges every HTML page but publishes full question and
348
+ * answer bodies under /feeds. A feed is XML with the real content escaped
349
+ * inside each entry, so this converts one into a plain HTML document and
350
+ * everything downstream stays unchanged. Returns null for non-feed input.
351
+ * @param {string} text
352
+ * @returns {string | null}
353
+ */
354
+ export function feedToHTML(text) {
355
+ const head = text.slice(0, 2000);
356
+ if (!/<(feed|rss)[\s>]/i.test(head) || /<(html|body)[\s>]/i.test(head)) return null;
357
+ // RSS wraps bodies in CDATA, which an HTML parser reads as a comment and
358
+ // drops. Escaping the section turns it into ordinary text, the same shape
359
+ // Atom feeds already use.
360
+ const xml = text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, inner) =>
361
+ inner.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;'));
362
+ const { document } = parseHTML(xml);
363
+ const root = document.querySelector('feed, rss');
364
+ if (!root) return null;
365
+ const esc = (s) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
366
+ // Feed elements are unknown to the HTML parser, so self-closed ones like
367
+ // <category /> stay open and swallow their siblings. Descendant queries
368
+ // still land, because a closing </entry> or </item> pops the whole pile.
369
+ const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
370
+ const feedTitle = field(root, 'title');
371
+ const parts = [];
372
+ for (const entry of root.querySelectorAll('entry, item')) {
373
+ const title = field(entry, 'title');
374
+ const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
375
+ ?? entry.querySelector('link[href]')?.getAttribute('href')
376
+ ?? field(entry, 'guid');
377
+ const author = field(entry, 'author name') || field(entry, 'author');
378
+ const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
379
+ const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
380
+ // Atom escapes the entry body, so textContent of content/summary is the
381
+ // HTML itself, ready to be embedded and parsed like any page.
382
+ const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? '';
383
+ parts.push('<article>');
384
+ if (title) parts.push(`<h2>${esc(title)}</h2>`);
385
+ if (byline || href) {
386
+ parts.push(`<p>${esc(byline)}${href ? ` <a href="${esc(href)}">open</a>` : ''}</p>`);
387
+ }
388
+ parts.push(body, '</article>');
389
+ }
390
+ // A full skeleton, because linkedom treats the first element of a bare
391
+ // multi-rooted fragment as the whole document and drops its siblings.
392
+ return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
393
+ }
394
+
395
+ const escHTML = (s) => String(s ?? '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
396
+
397
+ /**
398
+ * Pull one embedded JSON object out of a page by its variable name, scanning
399
+ * forward from the first `{` and tracking string/escape state so a brace
400
+ * inside a quoted value never ends the object early. A regex can't do this
401
+ * safely because the JSON itself contains braces.
402
+ * @param {string} text
403
+ * @param {string} marker
404
+ * @returns {any}
405
+ */
406
+ function extractJSON(text, marker) {
407
+ const at = text.indexOf(marker);
408
+ if (at === -1) return null;
409
+ const start = text.indexOf('{', at);
410
+ if (start === -1) return null;
411
+ let depth = 0, inString = false, escaped = false;
412
+ for (let i = start; i < text.length; i++) {
413
+ const c = text[i];
414
+ if (inString) {
415
+ if (escaped) escaped = false;
416
+ else if (c === '\\') escaped = true;
417
+ else if (c === '"') inString = false;
418
+ continue;
419
+ }
420
+ if (c === '"') inString = true;
421
+ else if (c === '{') depth++;
422
+ else if (c === '}' && --depth === 0) {
423
+ try {
424
+ return JSON.parse(text.slice(start, i + 1));
425
+ } catch {
426
+ return null;
427
+ }
428
+ }
429
+ }
430
+ return null;
431
+ }
432
+
433
+ /**
434
+ * A YouTube watch page needs client JS to become interactive, but the initial
435
+ * HTML response already carries the title, description, view count, and
436
+ * caption tracks inline as `ytInitialPlayerResponse`, so none of that has to
437
+ * wait on the v0.3 headless fallback. Each caption track becomes a link to
438
+ * its timedtext URL, which `oc do` follows and transcriptToHTML turns into
439
+ * readable text. Returns null for anything that isn't a watch page.
440
+ * @param {string} text
441
+ * @returns {string | null}
442
+ */
443
+ export function youtubeToHTML(text) {
444
+ const data = extractJSON(text, 'ytInitialPlayerResponse');
445
+ const details = data?.videoDetails;
446
+ if (!details?.title) return null;
447
+ const micro = data.microformat?.playerMicroformatRenderer;
448
+ const tracks = data.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
449
+ const views = details.viewCount ? `${Number(details.viewCount).toLocaleString('en-US')} views` : '';
450
+ const byline = [details.author && `by ${details.author}`, views, micro?.publishDate].filter(Boolean).join(', ');
451
+ const channelHref = details.channelId ? `https://www.youtube.com/channel/${details.channelId}` : '';
452
+ const parts = [`<h1>${escHTML(details.title)}</h1>`];
453
+ if (byline || channelHref) {
454
+ parts.push(`<p>${escHTML(byline)}${channelHref ? ` <a href="${escHTML(channelHref)}">channel</a>` : ''}</p>`);
455
+ }
456
+ const description = details.shortDescription ?? micro?.description?.simpleText ?? '';
457
+ for (const para of description.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean)) {
458
+ parts.push(`<p>${escHTML(para)}</p>`);
459
+ }
460
+ for (const track of tracks) {
461
+ const label = track.name?.simpleText || track.languageCode || 'transcript';
462
+ const kind = track.kind === 'asr' ? ' (auto-generated)' : '';
463
+ parts.push(`<p><a href="${escHTML(track.baseUrl)}">transcript: ${escHTML(label)}${kind}</a></p>`);
464
+ }
465
+ return `<html><head><title>${escHTML(details.title)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
466
+ }
467
+
468
+ /**
469
+ * YouTube's timedtext endpoint answers plain XML, one `<text>` cue per
470
+ * caption line. Concatenated into a single block rather than one paragraph
471
+ * per cue, so a whole transcript pages through `oc next`/`oc read` like any
472
+ * other long document instead of costing one numbered block per few words.
473
+ * Consecutive duplicate cues (auto captions repeat words across overlapping
474
+ * segments) are dropped. Returns null for anything that isn't a transcript.
475
+ * @param {string} text
476
+ * @returns {string | null}
477
+ */
478
+ export function transcriptToHTML(text) {
479
+ if (!/<transcript[\s>]/i.test(text.slice(0, 200))) return null;
480
+ const { document } = parseHTML(text);
481
+ const cues = [...document.querySelectorAll('text')];
482
+ if (!cues.length) return null;
483
+ const lines = [];
484
+ for (const cue of cues) {
485
+ const line = clean(cue.textContent ?? '');
486
+ if (line && line !== lines[lines.length - 1]) lines.push(line);
487
+ }
488
+ if (!lines.length) return null;
489
+ return `<html><head><title>Transcript</title></head><body>\n<p>${escHTML(lines.join(' '))}</p>\n</body></html>`;
490
+ }
491
+
148
492
  /**
149
493
  * Adjacent text nodes arrive fragmented (one per inline element boundary).
150
494
  * Merging them is what turns DOM noise into readable lines.
@@ -155,13 +499,27 @@ function mergeText(blocks) {
155
499
  /** @type {Block[]} */
156
500
  const out = [];
157
501
  for (const b of blocks) {
502
+ if (b.type === 'break') {
503
+ // A boundary only has to stop the merge, and it does that by being the
504
+ // last thing in the list when the next text block arrives.
505
+ if (out[out.length - 1]?.type === 'text') out.push(b);
506
+ continue;
507
+ }
158
508
  const prev = out[out.length - 1];
159
509
  if (b.type === 'text' && prev?.type === 'text') {
160
- prev.text = `${prev.text} ${b.text}`;
510
+ // Two fragments of one element with no whitespace between them were one
511
+ // word before the parser split them. Anything else was separated on the
512
+ // page and stays separated here.
513
+ const glued = prev.host && prev.host === b.host && !prev.post && !b.pre;
514
+ prev.text = glued ? `${prev.text}${b.text}` : `${prev.text} ${b.text}`;
515
+ prev.post = b.post;
161
516
  } else {
162
517
  out.push(b);
163
518
  }
164
519
  }
165
520
  // Single stray characters (list bullets, separators) cost tokens and say nothing.
166
- return out.filter((b) => b.type !== 'text' || b.text.length > 1);
521
+ return out
522
+ .filter((b) => b.type !== 'break')
523
+ .filter((b) => b.type !== 'text' || b.text.length > 1)
524
+ .map(({ host, pre, post, ...block }) => block);
167
525
  }