@only-cli/oc 0.2.0-beta.1 → 0.3.0-beta.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/distill.js CHANGED
@@ -3,9 +3,9 @@ import TurndownService from 'turndown';
3
3
 
4
4
  /**
5
5
  * @typedef {Object} Block
6
- * @property {'heading'|'text'|'link'|'input'|'button'} type
6
+ * @property {'heading'|'text'|'link'|'input'|'button'|'divider'} type
7
7
  * @property {string} text
8
- * @property {number} [n] - action handle, only on interactive blocks
8
+ * @property {number} [n] - action handle
9
9
  * @property {number} [level] - heading level 1..6
10
10
  * @property {string} [href] - links only
11
11
  * @property {string} [name] - inputs only
@@ -16,6 +16,10 @@ import TurndownService from 'turndown';
16
16
  * @property {Block[]} blocks
17
17
  */
18
18
 
19
+ // Where the compact view cuts a text block. It lives here because numbering
20
+ // depends on it: a block long enough to be cut is a block that needs a handle.
21
+ export const TEXT_CAP = 200;
22
+
19
23
  // Dropped wholesale, subtree included. Nav and footer stay in v0.1: on many
20
24
  // sites they carry the only working links, and the budget in render.js is
21
25
  // what keeps them from costing anything.
@@ -24,35 +28,99 @@ const DROP = new Set([
24
28
  'link', 'meta', 'head', 'canvas', 'video', 'audio', 'object',
25
29
  ]);
26
30
 
31
+ // Elements that are furniture by definition. The main content is never one of
32
+ // them, so the search below refuses to descend into them.
33
+ const FURNITURE = new Set(['nav', 'header', 'footer', 'aside']);
34
+
35
+ // Elements that end a line on the page and so must end one here. Without this
36
+ // the text of six separate posts merges into a single block, because nothing
37
+ // between them survives distillation to keep them apart.
38
+ const BLOCKY = new Set([
39
+ 'p', 'div', 'article', 'section', 'li', 'ul', 'ol', 'tr', 'td', 'th',
40
+ 'blockquote', 'pre', 'figure', 'figcaption', 'dt', 'dd', 'form', 'br', 'hr',
41
+ 'main', 'nav', 'header', 'footer', 'aside',
42
+ ]);
43
+
44
+ // Selectors a page uses to say where its content is. One match is a claim
45
+ // worth believing; several <article> elements mean a listing, where the list
46
+ // is the content and taking the first would throw the rest away.
47
+ const MAIN_SELECTORS = ['main', '[role="main"]', 'article'];
48
+
49
+ // Reordering only matters on a page that cannot be printed whole: if the
50
+ // budget covers everything, what leads is a question with no consequences.
51
+ // Below this much text the page is left in document order.
52
+ const MIN_PAGE = 3000;
53
+ // A candidate has to hold this share of the page's prose to be believed.
54
+ const MIN_SHARE = 0.25;
55
+ // How much of its parent's prose a child must hold before the search moves
56
+ // down into it, and how much of the page's it must keep to be worth choosing.
57
+ const DESCEND_SHARE = 0.6;
58
+ const KEEP_SHARE = 0.5;
59
+ const MAX_DEPTH = 12;
60
+
61
+ // A short link or button label repeated this many times down a page is
62
+ // per-item furniture (permalink, save, reply, like, repost) rather than
63
+ // content. On a forum thread or a social timeline there is one set per item,
64
+ // which costs more than the items do.
65
+ // A link label can be content, so it gets the benefit of the doubt for longer
66
+ // than a button label, which never is.
67
+ const REPEAT_LIMIT = 5;
68
+ const REPEAT_LIMIT_BUTTON = 3;
69
+ const REPEAT_MAX_LEN = 25;
70
+
27
71
  const clean = (s) => s.replace(/\s+/g, ' ').trim();
28
72
 
73
+ /**
74
+ * Everything that is not a page, made into one. The compact view and both raw
75
+ * modes go through here, so a format is never readable in one of them and a
76
+ * blob in the other. Each converter recognises its own input and returns null
77
+ * otherwise, and HTML falls through untouched.
78
+ * @param {string} text
79
+ * @param {string} url
80
+ * @returns {string}
81
+ */
82
+ const asHTML = (text, url = '', opts = {}) =>
83
+ jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text) ?? text;
84
+
29
85
  /**
30
86
  * Reduce raw HTML to an interaction tree: readable text plus numbered
31
- * interactive elements, in document order. Handles are assigned during a
32
- * single deterministic walk, so the same page always yields the same numbers.
87
+ * elements, in document order. The walk is deterministic and numbering is a
88
+ * second pass over its result, so the same page always yields the same
89
+ * numbers.
33
90
  * @param {string} html
34
91
  * @param {string} url
35
92
  * @returns {Page}
36
93
  */
37
94
  export function distill(html, url = '') {
38
- const { document } = parseHTML(html);
95
+ const { document } = parseHTML(asHTML(html, url));
39
96
  const title = clean(document.querySelector('title')?.textContent ?? '');
40
97
  /** @type {Block[]} */
41
98
  const blocks = [];
42
- let handle = 0;
43
99
 
44
100
  const hidden = (el) =>
45
101
  el.getAttribute('hidden') !== null ||
46
102
  el.getAttribute('aria-hidden') === 'true' ||
47
103
  /display:\s*none/.test(el.getAttribute('style') ?? '');
48
104
 
105
+ /** Subtree already emitted, skipped when the rest of the page is walked. */
106
+ let done = null;
107
+
49
108
  const walk = (node) => {
50
109
  if (node.nodeType === 3) {
51
- const text = clean(node.textContent);
52
- if (text) blocks.push({ type: 'text', text });
110
+ const raw = node.textContent ?? '';
111
+ const text = clean(raw);
112
+ // A parser is free to split one run of text into several nodes, and
113
+ // linkedom does it around apostrophes, so `what's` arrives as `what`,
114
+ // `'`, `s`. Remembering the parent and whether the fragment had space
115
+ // at its edges is what lets mergeText put it back without inventing a
116
+ // space that was never in the page.
117
+ if (text) {
118
+ blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
119
+ }
53
120
  return;
54
121
  }
55
122
  if (node.nodeType !== 1) return;
123
+ if (node === done) return;
56
124
  const tag = node.localName;
57
125
  if (DROP.has(tag) || hidden(node)) return;
58
126
 
@@ -64,7 +132,7 @@ export function distill(html, url = '') {
64
132
  if (tag === 'a' && node.getAttribute('href')) {
65
133
  const text = clean(node.textContent);
66
134
  if (text) {
67
- blocks.push({ type: 'link', n: ++handle, text, href: node.getAttribute('href') });
135
+ blocks.push({ type: 'link', text, href: node.getAttribute('href') });
68
136
  }
69
137
  return;
70
138
  }
@@ -72,24 +140,164 @@ export function distill(html, url = '') {
72
140
  const kind = node.getAttribute('type') ?? 'text';
73
141
  if (kind === 'hidden') return;
74
142
  if (kind === 'submit' || kind === 'button') {
75
- blocks.push({ type: 'button', n: ++handle, text: node.getAttribute('value') ?? 'submit' });
143
+ blocks.push({ type: 'button', text: node.getAttribute('value') ?? 'submit' });
76
144
  return;
77
145
  }
78
146
  const name = node.getAttribute('name') ?? node.getAttribute('placeholder') ?? tag;
79
- blocks.push({ type: 'input', n: ++handle, text: kind, name });
147
+ blocks.push({ type: 'input', text: kind, name });
80
148
  return;
81
149
  }
82
150
  if (tag === 'button') {
83
- const text = clean(node.textContent) || 'button';
84
- blocks.push({ type: 'button', n: ++handle, text });
151
+ // An icon button carries its name in aria-label or title, and a button
152
+ // with none of the three cannot be described or pressed, so printing it
153
+ // spends tokens to say nothing. X puts seven of them under every post.
154
+ const text = clean(node.textContent)
155
+ || clean(node.getAttribute('aria-label') ?? '')
156
+ || clean(node.getAttribute('title') ?? '');
157
+ if (text) blocks.push({ type: 'button', text });
158
+ return;
159
+ }
160
+ if (BLOCKY.has(tag)) {
161
+ blocks.push({ type: 'break' });
162
+ for (const child of node.childNodes) walk(child);
163
+ blocks.push({ type: 'break' });
85
164
  return;
86
165
  }
87
166
  for (const child of node.childNodes) walk(child);
88
167
  };
89
168
 
90
169
  const body = bodyOf(document);
91
- if (body) walk(body);
92
- return { url, title, blocks: mergeText(blocks) };
170
+ const main = body ? mainOf(document, body) : null;
171
+ if (main) {
172
+ // Content first, everything else after it. The budget in render.js is
173
+ // spent top down, so whatever leads the list is what an agent gets for
174
+ // its first 500 tokens, and on most pages the top of the document is
175
+ // menus. Nothing is dropped: the chrome carries links `oc do` needs, it
176
+ // just stops being the thing that gets read.
177
+ walk(main);
178
+ done = main;
179
+ const contentBlocks = blocks.length;
180
+ if (body) walk(body);
181
+ if (blocks.length > contentBlocks) {
182
+ blocks.splice(contentBlocks, 0, { type: 'divider', text: '--- rest of page: navigation, sidebar, footer ---' });
183
+ }
184
+ } else if (body) {
185
+ walk(body);
186
+ }
187
+ return { url, title, blocks: number(mergeText(dropRepeats(blocks))) };
188
+ }
189
+
190
+ /**
191
+ * Remove link labels that repeat down the page. They are the per-item controls
192
+ * a template stamps onto every row, and on a long thread they outweigh the
193
+ * content. What went is stated in place, because a view that quietly drops
194
+ * things is a view an agent cannot trust.
195
+ * @param {Block[]} blocks
196
+ * @returns {Block[]}
197
+ */
198
+ function dropRepeats(blocks) {
199
+ const controls = (b) => b.type === 'link' || b.type === 'button';
200
+ /** @type {Map<string, {n: number, limit: number}>} */
201
+ const counts = new Map();
202
+ for (const b of blocks) {
203
+ if (!controls(b) || b.text.length > REPEAT_MAX_LEN) continue;
204
+ const key = b.text.toLowerCase();
205
+ const limit = b.type === 'button' ? REPEAT_LIMIT_BUTTON : REPEAT_LIMIT;
206
+ const seen = counts.get(key);
207
+ // A label worn by both a link and a button keeps the more patient limit.
208
+ counts.set(key, { n: (seen?.n ?? 0) + 1, limit: Math.max(seen?.limit ?? 0, limit) });
209
+ }
210
+ const repeated = new Set([...counts].filter(([, c]) => c.n >= c.limit).map(([k]) => k));
211
+ if (!repeated.size) return blocks;
212
+ const kept = blocks.filter((b) => !(controls(b) && repeated.has(b.text.toLowerCase())));
213
+ const gone = blocks.length - kept.length;
214
+ const names = [...repeated].slice(0, 3).join(', ');
215
+ const note = { type: 'divider', text: `--- ${gone} repeated controls hidden (${names}), 'oc raw' has them ---` };
216
+ // The note belongs with the content it was cut from, not after the chrome.
217
+ const boundary = kept.findIndex((b) => b.type === 'divider');
218
+ kept.splice(boundary === -1 ? kept.length : boundary, 0, note);
219
+ return kept;
220
+ }
221
+
222
+ /**
223
+ * Find the element holding the page's main content, or null when the page is
224
+ * too small or too flat for the question to have an answer.
225
+ * @param {any} document
226
+ * @param {any} body
227
+ * @returns {any}
228
+ */
229
+ function mainOf(document, body) {
230
+ if (clean(body.textContent ?? '').length < MIN_PAGE) return null;
231
+ const total = prose(body);
232
+ for (const selector of MAIN_SELECTORS) {
233
+ const found = [...document.querySelectorAll(selector)];
234
+ if (found.length !== 1) continue;
235
+ const el = found[0];
236
+ if (el !== body && prose(el) >= total * MIN_SHARE) return el;
237
+ }
238
+ return densest(body, total);
239
+ }
240
+
241
+ /**
242
+ * Walk down the tree while one child holds most of the prose of the element
243
+ * above it. That is what separates a content column from the page around it
244
+ * without knowing anything about the site.
245
+ * @param {any} body
246
+ * @param {number} total
247
+ * @returns {any}
248
+ */
249
+ function densest(body, total) {
250
+ let node = body;
251
+ for (let depth = 0; depth < MAX_DEPTH; depth++) {
252
+ let best = null;
253
+ let bestProse = 0;
254
+ for (const child of node.children ?? []) {
255
+ if (FURNITURE.has(child.localName) || DROP.has(child.localName)) continue;
256
+ const len = prose(child);
257
+ if (len > bestProse) {
258
+ best = child;
259
+ bestProse = len;
260
+ }
261
+ }
262
+ if (!best || bestProse < prose(node) * DESCEND_SHARE || bestProse < total * KEEP_SHARE) break;
263
+ node = best;
264
+ }
265
+ return node === body ? null : node;
266
+ }
267
+
268
+ /**
269
+ * Text length minus the text of controls, which is what a page is steered by
270
+ * rather than what it says.
271
+ * @param {any} el
272
+ * @returns {number}
273
+ */
274
+ function prose(el) {
275
+ const text = clean(el.textContent ?? '').length;
276
+ let controls = 0;
277
+ // Link text is what navigation is made of. Option text is worse: a country
278
+ // dropdown is hundreds of characters that no link subtraction touches, and
279
+ // on a search results page it outscores results that are themselves links.
280
+ for (const node of el.querySelectorAll?.('a, select, button') ?? []) {
281
+ controls += clean(node.textContent ?? '').length;
282
+ }
283
+ return text - controls;
284
+ }
285
+
286
+ /**
287
+ * Assign handles in document order. Interactive elements get one because they
288
+ * can be acted on, headings and long text blocks because they can be read:
289
+ * a text block over the cap is printed cut, and its number is what makes the
290
+ * rest of it reachable with `oc read <n>` instead of a second whole-page fetch.
291
+ * @param {Block[]} blocks
292
+ * @returns {Block[]}
293
+ */
294
+ function number(blocks) {
295
+ let handle = 0;
296
+ for (const block of blocks) {
297
+ if (block.type === 'divider') continue;
298
+ if (block.type !== 'text' || block.text.length > TEXT_CAP) block.n = ++handle;
299
+ }
300
+ return blocks;
93
301
  }
94
302
 
95
303
  /**
@@ -106,8 +314,12 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document
106
314
  * hidden content.
107
315
  * @param {string} html
108
316
  */
109
- function cleanDocument(html) {
110
- const { document } = parseHTML(html);
317
+ function cleanDocument(html, url = '') {
318
+ // Raw is the mode an agent reaches for when the compact view left something
319
+ // out, so it is the one place a JSON response keeps every field.
320
+ const { document } = parseHTML(asHTML(html, url, { full: true }));
321
+ // Read the title before the sweep below removes the head with it.
322
+ const title = clean(document.querySelector('title')?.textContent ?? '');
111
323
  for (const tag of DROP) {
112
324
  for (const el of [...document.querySelectorAll(tag)]) el.remove();
113
325
  }
@@ -115,18 +327,19 @@ function cleanDocument(html) {
115
327
  for (const el of [...document.querySelectorAll('[style]')]) {
116
328
  if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
117
329
  }
118
- return document;
330
+ return { document, title };
119
331
  }
120
332
 
121
333
  /**
122
334
  * Whole-page markdown for `oc raw`, produced by turndown so lists, emphasis,
123
335
  * links, and code blocks come out as real markdown instead of flat lines.
124
336
  * @param {string} html
337
+ * @param {string} url - only read when the body turns out to be JSON, whose
338
+ * title has to come from the endpoint because the payload has none
125
339
  * @returns {string}
126
340
  */
127
- export function toMarkdown(html) {
128
- const document = cleanDocument(html);
129
- const title = clean(document.querySelector('title')?.textContent ?? '');
341
+ export function toMarkdown(html, url = '') {
342
+ const { document, title } = cleanDocument(html, url);
130
343
  const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
131
344
  const el = bodyOf(document);
132
345
  const body = el ? turndown.turndown(el.innerHTML).trim() : '';
@@ -137,14 +350,513 @@ export function toMarkdown(html) {
137
350
  * Whole-page cleaned HTML for `oc raw --html`, for agents that would rather
138
351
  * work with markup than markdown. Same noise removal, no other rewriting.
139
352
  * @param {string} html
353
+ * @param {string} url - see toMarkdown
140
354
  * @returns {string}
141
355
  */
142
- export function toHTML(html) {
143
- const document = cleanDocument(html);
356
+ export function toHTML(html, url = '') {
357
+ const { document } = cleanDocument(html, url);
144
358
  const el = bodyOf(document);
145
359
  return el ? el.innerHTML.trim() : '';
146
360
  }
147
361
 
362
+ /**
363
+ * Sites behind hard bot challenges often leave their Atom or RSS feeds open:
364
+ * Stack Overflow challenges every HTML page but publishes full question and
365
+ * answer bodies under /feeds. A feed is XML with the real content escaped
366
+ * inside each entry, so this converts one into a plain HTML document and
367
+ * everything downstream stays unchanged. Returns null for non-feed input.
368
+ * @param {string} text
369
+ * @returns {string | null}
370
+ */
371
+ export function feedToHTML(text) {
372
+ const head = text.slice(0, 2000);
373
+ if (!/<(feed|rss)[\s>]/i.test(head) || /<(html|body)[\s>]/i.test(head)) return null;
374
+ // RSS wraps bodies in CDATA, which an HTML parser reads as a comment and
375
+ // drops. Escaping the section turns it into ordinary text, the same shape
376
+ // Atom feeds already use.
377
+ const xml = text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, inner) =>
378
+ inner.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;'));
379
+ const { document } = parseHTML(xml);
380
+ const root = document.querySelector('feed, rss');
381
+ if (!root) return null;
382
+ const esc = (s) => s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
383
+ // Feed elements are unknown to the HTML parser, so self-closed ones like
384
+ // <category /> stay open and swallow their siblings. Descendant queries
385
+ // still land, because a closing </entry> or </item> pops the whole pile.
386
+ const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
387
+ const feedTitle = field(root, 'title');
388
+ const parts = [];
389
+ for (const entry of root.querySelectorAll('entry, item')) {
390
+ const title = field(entry, 'title');
391
+ const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
392
+ ?? entry.querySelector('link[href]')?.getAttribute('href')
393
+ ?? field(entry, 'guid');
394
+ const author = field(entry, 'author name') || field(entry, 'author');
395
+ const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
396
+ const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
397
+ // Atom escapes the entry body, so textContent of content/summary is the
398
+ // HTML itself, ready to be embedded and parsed like any page.
399
+ const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? '';
400
+ parts.push('<article>');
401
+ if (title) parts.push(`<h2>${esc(title)}</h2>`);
402
+ if (byline || href) {
403
+ parts.push(`<p>${esc(byline)}${href ? ` <a href="${esc(href)}">open</a>` : ''}</p>`);
404
+ }
405
+ parts.push(body, '</article>');
406
+ }
407
+ // A full skeleton, because linkedom treats the first element of a bare
408
+ // multi-rooted fragment as the whole document and drops its siblings.
409
+ return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
410
+ }
411
+
412
+ const escHTML = (s) => String(s ?? '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
413
+
414
+ /**
415
+ * Pull one embedded JSON object out of a page by its variable name, scanning
416
+ * forward from the first `{` and tracking string/escape state so a brace
417
+ * inside a quoted value never ends the object early. A regex can't do this
418
+ * safely because the JSON itself contains braces.
419
+ * @param {string} text
420
+ * @param {string} marker
421
+ * @returns {any}
422
+ */
423
+ function extractJSON(text, marker) {
424
+ const at = text.indexOf(marker);
425
+ if (at === -1) return null;
426
+ const start = text.indexOf('{', at);
427
+ if (start === -1) return null;
428
+ let depth = 0, inString = false, escaped = false;
429
+ for (let i = start; i < text.length; i++) {
430
+ const c = text[i];
431
+ if (inString) {
432
+ if (escaped) escaped = false;
433
+ else if (c === '\\') escaped = true;
434
+ else if (c === '"') inString = false;
435
+ continue;
436
+ }
437
+ if (c === '"') inString = true;
438
+ else if (c === '{') depth++;
439
+ else if (c === '}' && --depth === 0) {
440
+ try {
441
+ return JSON.parse(text.slice(start, i + 1));
442
+ } catch {
443
+ return null;
444
+ }
445
+ }
446
+ }
447
+ return null;
448
+ }
449
+
450
+ /**
451
+ * A YouTube watch page needs client JS to become interactive, but the initial
452
+ * HTML response already carries the title, description, view count, and
453
+ * caption tracks inline as `ytInitialPlayerResponse`, so none of that has to
454
+ * wait on the v0.3 headless fallback. Each caption track becomes a link to
455
+ * its timedtext URL, which `oc do` follows and transcriptToHTML turns into
456
+ * readable text. Returns null for anything that isn't a watch page.
457
+ * @param {string} text
458
+ * @returns {string | null}
459
+ */
460
+ export function youtubeToHTML(text) {
461
+ const data = extractJSON(text, 'ytInitialPlayerResponse');
462
+ const details = data?.videoDetails;
463
+ if (!details?.title) return null;
464
+ const micro = data.microformat?.playerMicroformatRenderer;
465
+ const tracks = data.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
466
+ const views = details.viewCount ? `${Number(details.viewCount).toLocaleString('en-US')} views` : '';
467
+ const byline = [details.author && `by ${details.author}`, views, micro?.publishDate].filter(Boolean).join(', ');
468
+ const channelHref = details.channelId ? `https://www.youtube.com/channel/${details.channelId}` : '';
469
+ const parts = [`<h1>${escHTML(details.title)}</h1>`];
470
+ if (byline || channelHref) {
471
+ parts.push(`<p>${escHTML(byline)}${channelHref ? ` <a href="${escHTML(channelHref)}">channel</a>` : ''}</p>`);
472
+ }
473
+ const description = details.shortDescription ?? micro?.description?.simpleText ?? '';
474
+ for (const para of description.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean)) {
475
+ parts.push(`<p>${escHTML(para)}</p>`);
476
+ }
477
+ for (const track of tracks) {
478
+ const label = track.name?.simpleText || track.languageCode || 'transcript';
479
+ const kind = track.kind === 'asr' ? ' (auto-generated)' : '';
480
+ parts.push(`<p><a href="${escHTML(track.baseUrl)}">transcript: ${escHTML(label)}${kind}</a></p>`);
481
+ }
482
+ return `<html><head><title>${escHTML(details.title)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
483
+ }
484
+
485
+ /**
486
+ * YouTube's timedtext endpoint answers plain XML, one `<text>` cue per
487
+ * caption line. Concatenated into a single block rather than one paragraph
488
+ * per cue, so a whole transcript pages through `oc next`/`oc read` like any
489
+ * other long document instead of costing one numbered block per few words.
490
+ * Consecutive duplicate cues (auto captions repeat words across overlapping
491
+ * segments) are dropped. Returns null for anything that isn't a transcript.
492
+ * @param {string} text
493
+ * @returns {string | null}
494
+ */
495
+ export function transcriptToHTML(text) {
496
+ if (!/<transcript[\s>]/i.test(text.slice(0, 200))) return null;
497
+ const { document } = parseHTML(text);
498
+ const cues = [...document.querySelectorAll('text')];
499
+ if (!cues.length) return null;
500
+ const lines = [];
501
+ for (const cue of cues) {
502
+ const line = clean(cue.textContent ?? '');
503
+ if (line && line !== lines[lines.length - 1]) lines.push(line);
504
+ }
505
+ if (!lines.length) return null;
506
+ return `<html><head><title>Transcript</title></head><body>\n<p>${escHTML(lines.join(' '))}</p>\n</body></html>`;
507
+ }
508
+
509
+ // Keys an API is likely to give the human-readable name of an item, in the
510
+ // order they win when an item carries several of them.
511
+ const TITLE_KEYS = [
512
+ 'title', 'name', 'headline', 'subject', 'label', 'display_name',
513
+ 'full_name', 'summary', 'question', 'message',
514
+ ];
515
+
516
+ // Keys holding the item's own page. A URL under any other name is still found,
517
+ // by looking at values rather than names, but these win when several qualify.
518
+ const LINK_KEYS = ['link', 'url', 'html_url', 'web_url', 'permalink', 'href'];
519
+
520
+ // A title has to fit on a line to be one. Anything longer is a body that
521
+ // happens to live under a title-ish key, and belongs in a block of its own.
522
+ const TITLE_MAX = 300;
523
+
524
+ // How many constant fields the footer names before it stops counting them out.
525
+ const CONST_LISTED = 8;
526
+
527
+ // Characters of field text an item may spend in the compact view. A response
528
+ // carries far more fields than an agent asked for: one Stack Exchange result
529
+ // brings eight about the asker alone, which is the whole budget spent on who
530
+ // rather than what. Thirty items make this thirty times over, so it buys two
531
+ // or three fields, not a record. `oc raw` still has all of them.
532
+ const FIELD_BUDGET = 60;
533
+
534
+ // What a field from a flattened sub-object scores against one of the item's
535
+ // own. `owner.user_id` varies perfectly and costs little, which is enough to
536
+ // win on width and variance alone, but it identifies somebody attached to the
537
+ // result rather than the result, and no agent searched for it.
538
+ const NESTED_PENALTY = 0.25;
539
+
540
+ // How many names the footer lists when it says which fields it left out.
541
+ const DROPPED_LISTED = 5;
542
+
543
+ // Seconds and milliseconds since the epoch, bounded either side so an ordinary
544
+ // count (a score, a byte size) is never mistaken for a date.
545
+ const EPOCH_S = [1e9, 4e9];
546
+ const EPOCH_MS = [1e12, 4e12];
547
+ const DATE_KEY = /(^|_)(date|at|time|timestamp|created|updated|published|modified)$/i;
548
+
549
+ // Named entities worth knowing without a table: the five XML ones plus the
550
+ // space. Everything else arrives numeric.
551
+ const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
552
+
553
+ /**
554
+ * Undo one layer of HTML escaping in a string value. APIs that back an HTML
555
+ * site tend to escape the text they return: Stack Exchange answers with
556
+ * `Is &quot;==&quot; slower`, and re-escaping that on the way into a document
557
+ * would print the entity instead of the quote it stands for.
558
+ * @param {string} s
559
+ * @returns {string}
560
+ */
561
+ const decodeEntities = (s) =>
562
+ s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, body) => {
563
+ if (body[0] === '#') {
564
+ const code = body[1] === 'x' || body[1] === 'X'
565
+ ? parseInt(body.slice(2), 16)
566
+ : Number(body.slice(1));
567
+ return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole;
568
+ }
569
+ return ENTITIES[body.toLowerCase()] ?? whole;
570
+ });
571
+
572
+ const isPlain = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
573
+ const isURL = (v) => typeof v === 'string' && /^https?:\/\/\S+$/.test(v);
574
+ const looksHTML = (v) => typeof v === 'string' && /<\/?(p|div|pre|code|br|ul|ol|li|h[1-6]|blockquote|table|img|a|em|strong)\b[^>]*>/i.test(v);
575
+
576
+ /**
577
+ * One level of flattening, so `owner: {display_name}` becomes an
578
+ * `owner.display_name` field. Deeper than that an object stops being a set of
579
+ * fields and starts being a document, which no line-per-item view can hold.
580
+ * @param {Record<string, any>} item
581
+ * @returns {Map<string, any>}
582
+ */
583
+ function flattenItem(item) {
584
+ /** @type {Map<string, any>} */
585
+ const out = new Map();
586
+ for (const [key, value] of Object.entries(item)) {
587
+ if (!isPlain(value)) {
588
+ out.set(key, value);
589
+ continue;
590
+ }
591
+ for (const [inner, deep] of Object.entries(value)) {
592
+ if (deep !== null && typeof deep === 'object') continue;
593
+ out.set(`${key}.${inner}`, deep);
594
+ }
595
+ }
596
+ return out;
597
+ }
598
+
599
+ /**
600
+ * One field as one string. Epoch integers under a date-ish key become ISO
601
+ * dates, because an agent that has to convert one pays a turn for it. Arrays
602
+ * of scalars (tags, labels) join; arrays of objects are counted, since
603
+ * spelling them out is what the flattening above already refused to do.
604
+ * @param {string} key
605
+ * @param {any} value
606
+ * @returns {string}
607
+ */
608
+ function renderValue(key, value) {
609
+ if (value === null || value === undefined) return '';
610
+ if (Array.isArray(value)) {
611
+ if (!value.length) return '';
612
+ if (value.every((v) => v === null || typeof v !== 'object')) return value.join(', ');
613
+ return `[${value.length} items]`;
614
+ }
615
+ if (typeof value === 'object') return '';
616
+ if (typeof value === 'number' && Number.isInteger(value) && DATE_KEY.test(key)) {
617
+ const ms = value >= EPOCH_S[0] && value < EPOCH_S[1] ? value * 1000
618
+ : value >= EPOCH_MS[0] && value < EPOCH_MS[1] ? value
619
+ : null;
620
+ if (ms !== null) return new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
621
+ }
622
+ return typeof value === 'string' ? decodeEntities(value) : String(value);
623
+ }
624
+
625
+ /**
626
+ * Order fields by how much they say per character, and keep taking them while
627
+ * an item can still afford one. Variance is what carries the information: a
628
+ * field reading the same on every row has already been lifted out as a
629
+ * constant, and one reading differently every time is why the response was
630
+ * fetched. Dividing by width is what stops a long field (a profile image URL,
631
+ * a licence string) from crowding out three short ones that matter more.
632
+ * @param {Array<Map<string, string>|null>} rows
633
+ * @param {Set<string>} skip - fields already spoken for as title, link, or constant
634
+ * @returns {{kept: Set<string>, dropped: string[]}}
635
+ */
636
+ function chooseFields(rows, skip) {
637
+ const present = rows.filter(Boolean);
638
+ const keys = [];
639
+ for (const row of present) {
640
+ for (const key of row.keys()) if (!skip.has(key) && !keys.includes(key)) keys.push(key);
641
+ }
642
+ const scored = [];
643
+ for (const key of keys) {
644
+ const values = present.map((row) => row.get(key) ?? '').filter((v) => v !== '');
645
+ if (!values.length) continue;
646
+ const width = values.reduce((sum, v) => sum + v.length + key.length + 3, 0) / values.length;
647
+ const variance = new Set(values).size / values.length;
648
+ const penalty = key.includes('.') ? NESTED_PENALTY : 1;
649
+ scored.push({ key, width, score: (variance / width) * penalty });
650
+ }
651
+ scored.sort((a, b) => b.score - a.score);
652
+ const kept = new Set();
653
+ let spent = 0;
654
+ for (const field of scored) {
655
+ // The first field is taken whatever it costs, so an item of one long field
656
+ // still renders something rather than nothing.
657
+ if (spent + field.width > FIELD_BUDGET && kept.size) continue;
658
+ kept.add(field.key);
659
+ spent += field.width;
660
+ }
661
+ return { kept, dropped: scored.filter((f) => !kept.has(f.key)).map((f) => f.key) };
662
+ }
663
+
664
+ /**
665
+ * Pick the array the response is actually about: the root when it is one,
666
+ * otherwise the longest array of objects at the top level, which is where
667
+ * `items`, `data`, `results`, and `hits` all live. Everything beside it is
668
+ * metadata about the request rather than content.
669
+ * @param {any} data
670
+ * @returns {{items: any[], meta: Record<string, any>}}
671
+ */
672
+ function mainArray(data) {
673
+ if (Array.isArray(data)) return { items: data, meta: {} };
674
+ let key = '';
675
+ /** @type {any[] | null} */
676
+ let items = null;
677
+ for (const [k, v] of Object.entries(data)) {
678
+ if (!Array.isArray(v) || !v.length) continue;
679
+ if (!v.some(isPlain)) continue;
680
+ if (!items || v.length > items.length) {
681
+ items = v;
682
+ key = k;
683
+ }
684
+ }
685
+ // A response with no array is a single resource, which renders as one item
686
+ // rather than as a special case.
687
+ if (!items) return { items: [data], meta: {} };
688
+ const meta = { ...data };
689
+ delete meta[key];
690
+ return { items, meta };
691
+ }
692
+
693
+ /**
694
+ * Fields whose rendered value is the same on every item. In a list of thirty
695
+ * results they are thirty copies of one fact, so they come out of the rows and
696
+ * get stated once at the foot of the page: the saving is the point of the
697
+ * exercise, and dropping them silently would be a lie about what the API said.
698
+ * @param {Array<Map<string, string>|null>} rows
699
+ * @returns {Map<string, string>}
700
+ */
701
+ function constantFields(rows) {
702
+ const present = rows.filter(Boolean);
703
+ /** @type {Map<string, string>} */
704
+ const out = new Map();
705
+ if (present.length < 2) return out;
706
+ for (const [key, value] of present[0]) {
707
+ if (present.every((row) => row.get(key) === value)) out.set(key, value);
708
+ }
709
+ return out;
710
+ }
711
+
712
+ /**
713
+ * A JSON body is not a page, so nothing downstream can read one: the HTML
714
+ * parser turns it into a single unreadable text node and the budget truncates
715
+ * the blob. This turns a response into the same shape feedToHTML produces, one
716
+ * article per item, so numbering, the budget, `oc do`, and raw markdown all
717
+ * work on an API exactly as they do on a page.
718
+ *
719
+ * The view is a line per item rather than a table: two or three fields carry
720
+ * the signal in most responses, and turndown cannot write a markdown table
721
+ * without the GFM plugin, which is a dependency this project does not want.
722
+ * Returns null for anything that is not JSON.
723
+ * @param {string} text
724
+ * @param {string} url
725
+ * @param {{full?: boolean}} [opts] - full keeps every field, which is what the
726
+ * raw modes are for; the compact view keeps the ones that earn their tokens
727
+ * @returns {string | null}
728
+ */
729
+ export function jsonToHTML(text, url = '', { full = false } = {}) {
730
+ if (!/^\s*[[{]/.test(text.slice(0, 200))) return null;
731
+ let data;
732
+ try {
733
+ data = JSON.parse(text);
734
+ } catch {
735
+ return null;
736
+ }
737
+ if (!data || typeof data !== 'object') return null;
738
+
739
+ const { items, meta } = mainArray(data);
740
+ const flats = items.map((item) => (isPlain(item) ? flattenItem(item) : null));
741
+ const rows = flats.map((flat) => {
742
+ if (!flat) return null;
743
+ /** @type {Map<string, string>} */
744
+ const row = new Map();
745
+ for (const [key, value] of flat) row.set(key, renderValue(key, value));
746
+ return row;
747
+ });
748
+
749
+ // One item's field names stand for all of them: a ragged response still gets
750
+ // a consistent title and link column, and a field missing from an item is
751
+ // simply absent from its line.
752
+ const sample = flats.find(Boolean) ?? new Map();
753
+ const titled = (k) => {
754
+ const v = sample.get(k);
755
+ return typeof v === 'string' && v.trim() && v.length <= TITLE_MAX && !isURL(v) && !looksHTML(v);
756
+ };
757
+ const titleKey = TITLE_KEYS.find(titled) ?? [...sample.keys()].find(titled);
758
+ const linkKey = LINK_KEYS.find((k) => isURL(sample.get(k))) ?? [...sample.keys()].find((k) => isURL(sample.get(k)));
759
+
760
+ const constants = constantFields(rows);
761
+ const spoken = new Set([titleKey, linkKey, ...constants.keys()].filter(Boolean));
762
+ const { kept, dropped } = full
763
+ ? { kept: null, dropped: [] }
764
+ : chooseFields(rows, spoken);
765
+ const parts = [];
766
+
767
+ for (let i = 0; i < items.length; i++) {
768
+ const flat = flats[i];
769
+ if (!flat) {
770
+ const line = renderValue('', items[i]);
771
+ if (line) parts.push(`<p>${escHTML(line)}</p>`);
772
+ continue;
773
+ }
774
+ const row = rows[i];
775
+ const title = titleKey ? row.get(titleKey) : '';
776
+ const link = linkKey && isURL(flat.get(linkKey)) ? flat.get(linkKey) : '';
777
+ const inline = [];
778
+ const long = [];
779
+ const bodies = [];
780
+ for (const [key, value] of flat) {
781
+ if (spoken.has(key)) continue;
782
+ // A field carrying markup is a document, and is kept whatever it scored:
783
+ // it renders as its own block, so it never competes for the field line.
784
+ if (kept && !kept.has(key) && !looksHTML(value)) continue;
785
+ // A field carrying HTML is a page in itself: `filter=withbody` on the
786
+ // Stack Exchange API puts a whole question in one. It goes through the
787
+ // distiller like any other markup instead of into a cell.
788
+ if (looksHTML(value)) {
789
+ bodies.push(String(value));
790
+ continue;
791
+ }
792
+ const rendered = row.get(key);
793
+ if (!rendered) continue;
794
+ if (rendered.length > TEXT_CAP) long.push(`<p>${escHTML(`${key}: ${rendered}`)}</p>`);
795
+ else inline.push(`${key}: ${rendered}`);
796
+ }
797
+
798
+ parts.push('<article>');
799
+ if (title && link) parts.push(`<p><a href="${escHTML(link)}">${escHTML(title)}</a></p>`);
800
+ else if (title) parts.push(`<p>${escHTML(title)}</p>`);
801
+ else if (link) parts.push(`<p><a href="${escHTML(link)}">open</a></p>`);
802
+ if (inline.length) parts.push(`<p>${escHTML(inline.join(' | '))}</p>`);
803
+ parts.push(...long);
804
+ for (const body of bodies) parts.push(`<div>${body}</div>`);
805
+ parts.push('</article>');
806
+ }
807
+
808
+ // What the rows no longer carry, said once. Empty-everywhere fields are
809
+ // named but not valued, because their value is the fact that there isn't one.
810
+ const shown = [...constants].filter(([, v]) => v !== '');
811
+ const empty = [...constants].filter(([, v]) => v === '').map(([k]) => k);
812
+ const clipped = (list, render) => {
813
+ const head = list.slice(0, CONST_LISTED).map(render).join(', ');
814
+ return list.length > CONST_LISTED ? `${head}, +${list.length - CONST_LISTED} more` : head;
815
+ };
816
+ if (shown.length) {
817
+ parts.push(`<p>same on every item: ${escHTML(clipped(shown, ([k, v]) => `${k}=${v.length > 60 ? `${v.slice(0, 60)}...` : v}`))}</p>`);
818
+ }
819
+ if (empty.length) parts.push(`<p>empty on every item: ${escHTML(clipped(empty, (k) => k))}</p>`);
820
+ if (dropped.length) {
821
+ const names = dropped.slice(0, DROPPED_LISTED).join(', ');
822
+ const more = dropped.length > DROPPED_LISTED ? `, +${dropped.length - DROPPED_LISTED} more` : '';
823
+ parts.push(`<p>${dropped.length} fields per item not shown (${escHTML(names + more)}), 'oc raw' has them</p>`);
824
+ }
825
+
826
+ const metaBits = [];
827
+ for (const [key, value] of Object.entries(meta)) {
828
+ if (value === null || typeof value === 'object') continue;
829
+ const rendered = renderValue(key, value);
830
+ if (rendered) metaBits.push(`${key}=${rendered}`);
831
+ }
832
+ if (metaBits.length) parts.push(`<p>response: ${escHTML(metaBits.join(', '))}</p>`);
833
+
834
+ const count = `${items.length} ${items.length === 1 ? 'item' : 'items'}`;
835
+ return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
836
+ }
837
+
838
+ /**
839
+ * An API response has no title of its own, so the endpoint becomes one. The
840
+ * query parameter is worth the tokens it costs: it is the only part of a
841
+ * search URL that says what the page is, and without it every search a session
842
+ * runs is titled the same.
843
+ * @param {string} url
844
+ * @param {string} count
845
+ * @returns {string}
846
+ */
847
+ function jsonTitle(url, count) {
848
+ try {
849
+ const u = new URL(url);
850
+ const query = ['q', 'query', 'search', 'terms', 'keywords', 'text']
851
+ .map((k) => u.searchParams.get(k))
852
+ .find((v) => v);
853
+ const base = `${u.host}${u.pathname}`.replace(/\/+$/, '');
854
+ return query ? `${base}: "${query}" (${count})` : `${base} (${count})`;
855
+ } catch {
856
+ return `JSON (${count})`;
857
+ }
858
+ }
859
+
148
860
  /**
149
861
  * Adjacent text nodes arrive fragmented (one per inline element boundary).
150
862
  * Merging them is what turns DOM noise into readable lines.
@@ -155,13 +867,27 @@ function mergeText(blocks) {
155
867
  /** @type {Block[]} */
156
868
  const out = [];
157
869
  for (const b of blocks) {
870
+ if (b.type === 'break') {
871
+ // A boundary only has to stop the merge, and it does that by being the
872
+ // last thing in the list when the next text block arrives.
873
+ if (out[out.length - 1]?.type === 'text') out.push(b);
874
+ continue;
875
+ }
158
876
  const prev = out[out.length - 1];
159
877
  if (b.type === 'text' && prev?.type === 'text') {
160
- prev.text = `${prev.text} ${b.text}`;
878
+ // Two fragments of one element with no whitespace between them were one
879
+ // word before the parser split them. Anything else was separated on the
880
+ // page and stays separated here.
881
+ const glued = prev.host && prev.host === b.host && !prev.post && !b.pre;
882
+ prev.text = glued ? `${prev.text}${b.text}` : `${prev.text} ${b.text}`;
883
+ prev.post = b.post;
161
884
  } else {
162
885
  out.push(b);
163
886
  }
164
887
  }
165
888
  // Single stray characters (list bullets, separators) cost tokens and say nothing.
166
- return out.filter((b) => b.type !== 'text' || b.text.length > 1);
889
+ return out
890
+ .filter((b) => b.type !== 'break')
891
+ .filter((b) => b.type !== 'text' || b.text.length > 1)
892
+ .map(({ host, pre, post, ...block }) => block);
167
893
  }