@only-cli/oc 0.2.0-beta.1 → 0.3.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +52 -25
- package/clis/cloud.google.com.json +9 -0
- package/clis/docs.aws.amazon.com.json +9 -0
- package/clis/finance.yahoo.com.json +13 -0
- package/clis/learn.microsoft.com.json +9 -0
- package/clis/stackoverflow.com.json +10 -0
- package/clis/x.com.json +7 -0
- package/clis/youtube.com.json +7 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +61 -0
- package/src/act.js +235 -17
- package/src/cli.js +69 -19
- package/src/distill.js +751 -25
- package/src/fetch.js +155 -15
- package/src/render.js +75 -24
- package/src/session.js +143 -5
- package/skills/only-cli/SKILL.md +0 -37
package/src/distill.js
CHANGED
|
@@ -3,9 +3,9 @@ import TurndownService from 'turndown';
|
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* @typedef {Object} Block
|
|
6
|
-
* @property {'heading'|'text'|'link'|'input'|'button'} type
|
|
6
|
+
* @property {'heading'|'text'|'link'|'input'|'button'|'divider'} type
|
|
7
7
|
* @property {string} text
|
|
8
|
-
* @property {number} [n] - action handle
|
|
8
|
+
* @property {number} [n] - action handle
|
|
9
9
|
* @property {number} [level] - heading level 1..6
|
|
10
10
|
* @property {string} [href] - links only
|
|
11
11
|
* @property {string} [name] - inputs only
|
|
@@ -16,6 +16,10 @@ import TurndownService from 'turndown';
|
|
|
16
16
|
* @property {Block[]} blocks
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
+
// Where the compact view cuts a text block. It lives here because numbering
|
|
20
|
+
// depends on it: a block long enough to be cut is a block that needs a handle.
|
|
21
|
+
export const TEXT_CAP = 200;
|
|
22
|
+
|
|
19
23
|
// Dropped wholesale, subtree included. Nav and footer stay in v0.1: on many
|
|
20
24
|
// sites they carry the only working links, and the budget in render.js is
|
|
21
25
|
// what keeps them from costing anything.
|
|
@@ -24,35 +28,99 @@ const DROP = new Set([
|
|
|
24
28
|
'link', 'meta', 'head', 'canvas', 'video', 'audio', 'object',
|
|
25
29
|
]);
|
|
26
30
|
|
|
31
|
+
// Elements that are furniture by definition. The main content is never one of
|
|
32
|
+
// them, so the search below refuses to descend into them.
|
|
33
|
+
const FURNITURE = new Set(['nav', 'header', 'footer', 'aside']);
|
|
34
|
+
|
|
35
|
+
// Elements that end a line on the page and so must end one here. Without this
|
|
36
|
+
// the text of six separate posts merges into a single block, because nothing
|
|
37
|
+
// between them survives distillation to keep them apart.
|
|
38
|
+
const BLOCKY = new Set([
|
|
39
|
+
'p', 'div', 'article', 'section', 'li', 'ul', 'ol', 'tr', 'td', 'th',
|
|
40
|
+
'blockquote', 'pre', 'figure', 'figcaption', 'dt', 'dd', 'form', 'br', 'hr',
|
|
41
|
+
'main', 'nav', 'header', 'footer', 'aside',
|
|
42
|
+
]);
|
|
43
|
+
|
|
44
|
+
// Selectors a page uses to say where its content is. One match is a claim
|
|
45
|
+
// worth believing; several <article> elements mean a listing, where the list
|
|
46
|
+
// is the content and taking the first would throw the rest away.
|
|
47
|
+
const MAIN_SELECTORS = ['main', '[role="main"]', 'article'];
|
|
48
|
+
|
|
49
|
+
// Reordering only matters on a page that cannot be printed whole: if the
|
|
50
|
+
// budget covers everything, what leads is a question with no consequences.
|
|
51
|
+
// Below this much text the page is left in document order.
|
|
52
|
+
const MIN_PAGE = 3000;
|
|
53
|
+
// A candidate has to hold this share of the page's prose to be believed.
|
|
54
|
+
const MIN_SHARE = 0.25;
|
|
55
|
+
// How much of its parent's prose a child must hold before the search moves
|
|
56
|
+
// down into it, and how much of the page's it must keep to be worth choosing.
|
|
57
|
+
const DESCEND_SHARE = 0.6;
|
|
58
|
+
const KEEP_SHARE = 0.5;
|
|
59
|
+
const MAX_DEPTH = 12;
|
|
60
|
+
|
|
61
|
+
// A short link or button label repeated this many times down a page is
|
|
62
|
+
// per-item furniture (permalink, save, reply, like, repost) rather than
|
|
63
|
+
// content. On a forum thread or a social timeline there is one set per item,
|
|
64
|
+
// which costs more than the items do.
|
|
65
|
+
// A link label can be content, so it gets the benefit of the doubt for longer
|
|
66
|
+
// than a button label, which never is.
|
|
67
|
+
const REPEAT_LIMIT = 5;
|
|
68
|
+
const REPEAT_LIMIT_BUTTON = 3;
|
|
69
|
+
const REPEAT_MAX_LEN = 25;
|
|
70
|
+
|
|
27
71
|
const clean = (s) => s.replace(/\s+/g, ' ').trim();
|
|
28
72
|
|
|
73
|
+
/**
|
|
74
|
+
* Everything that is not a page, made into one. The compact view and both raw
|
|
75
|
+
* modes go through here, so a format is never readable in one of them and a
|
|
76
|
+
* blob in the other. Each converter recognises its own input and returns null
|
|
77
|
+
* otherwise, and HTML falls through untouched.
|
|
78
|
+
* @param {string} text
|
|
79
|
+
* @param {string} url
|
|
80
|
+
* @returns {string}
|
|
81
|
+
*/
|
|
82
|
+
const asHTML = (text, url = '', opts = {}) =>
|
|
83
|
+
jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text) ?? text;
|
|
84
|
+
|
|
29
85
|
/**
|
|
30
86
|
* Reduce raw HTML to an interaction tree: readable text plus numbered
|
|
31
|
-
*
|
|
32
|
-
*
|
|
87
|
+
* elements, in document order. The walk is deterministic and numbering is a
|
|
88
|
+
* second pass over its result, so the same page always yields the same
|
|
89
|
+
* numbers.
|
|
33
90
|
* @param {string} html
|
|
34
91
|
* @param {string} url
|
|
35
92
|
* @returns {Page}
|
|
36
93
|
*/
|
|
37
94
|
export function distill(html, url = '') {
|
|
38
|
-
const { document } = parseHTML(html);
|
|
95
|
+
const { document } = parseHTML(asHTML(html, url));
|
|
39
96
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
40
97
|
/** @type {Block[]} */
|
|
41
98
|
const blocks = [];
|
|
42
|
-
let handle = 0;
|
|
43
99
|
|
|
44
100
|
const hidden = (el) =>
|
|
45
101
|
el.getAttribute('hidden') !== null ||
|
|
46
102
|
el.getAttribute('aria-hidden') === 'true' ||
|
|
47
103
|
/display:\s*none/.test(el.getAttribute('style') ?? '');
|
|
48
104
|
|
|
105
|
+
/** Subtree already emitted, skipped when the rest of the page is walked. */
|
|
106
|
+
let done = null;
|
|
107
|
+
|
|
49
108
|
const walk = (node) => {
|
|
50
109
|
if (node.nodeType === 3) {
|
|
51
|
-
const
|
|
52
|
-
|
|
110
|
+
const raw = node.textContent ?? '';
|
|
111
|
+
const text = clean(raw);
|
|
112
|
+
// A parser is free to split one run of text into several nodes, and
|
|
113
|
+
// linkedom does it around apostrophes, so `what's` arrives as `what`,
|
|
114
|
+
// `'`, `s`. Remembering the parent and whether the fragment had space
|
|
115
|
+
// at its edges is what lets mergeText put it back without inventing a
|
|
116
|
+
// space that was never in the page.
|
|
117
|
+
if (text) {
|
|
118
|
+
blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
|
|
119
|
+
}
|
|
53
120
|
return;
|
|
54
121
|
}
|
|
55
122
|
if (node.nodeType !== 1) return;
|
|
123
|
+
if (node === done) return;
|
|
56
124
|
const tag = node.localName;
|
|
57
125
|
if (DROP.has(tag) || hidden(node)) return;
|
|
58
126
|
|
|
@@ -64,7 +132,7 @@ export function distill(html, url = '') {
|
|
|
64
132
|
if (tag === 'a' && node.getAttribute('href')) {
|
|
65
133
|
const text = clean(node.textContent);
|
|
66
134
|
if (text) {
|
|
67
|
-
blocks.push({ type: 'link',
|
|
135
|
+
blocks.push({ type: 'link', text, href: node.getAttribute('href') });
|
|
68
136
|
}
|
|
69
137
|
return;
|
|
70
138
|
}
|
|
@@ -72,24 +140,164 @@ export function distill(html, url = '') {
|
|
|
72
140
|
const kind = node.getAttribute('type') ?? 'text';
|
|
73
141
|
if (kind === 'hidden') return;
|
|
74
142
|
if (kind === 'submit' || kind === 'button') {
|
|
75
|
-
blocks.push({ type: 'button',
|
|
143
|
+
blocks.push({ type: 'button', text: node.getAttribute('value') ?? 'submit' });
|
|
76
144
|
return;
|
|
77
145
|
}
|
|
78
146
|
const name = node.getAttribute('name') ?? node.getAttribute('placeholder') ?? tag;
|
|
79
|
-
blocks.push({ type: 'input',
|
|
147
|
+
blocks.push({ type: 'input', text: kind, name });
|
|
80
148
|
return;
|
|
81
149
|
}
|
|
82
150
|
if (tag === 'button') {
|
|
83
|
-
|
|
84
|
-
|
|
151
|
+
// An icon button carries its name in aria-label or title, and a button
|
|
152
|
+
// with none of the three cannot be described or pressed, so printing it
|
|
153
|
+
// spends tokens to say nothing. X puts seven of them under every post.
|
|
154
|
+
const text = clean(node.textContent)
|
|
155
|
+
|| clean(node.getAttribute('aria-label') ?? '')
|
|
156
|
+
|| clean(node.getAttribute('title') ?? '');
|
|
157
|
+
if (text) blocks.push({ type: 'button', text });
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
if (BLOCKY.has(tag)) {
|
|
161
|
+
blocks.push({ type: 'break' });
|
|
162
|
+
for (const child of node.childNodes) walk(child);
|
|
163
|
+
blocks.push({ type: 'break' });
|
|
85
164
|
return;
|
|
86
165
|
}
|
|
87
166
|
for (const child of node.childNodes) walk(child);
|
|
88
167
|
};
|
|
89
168
|
|
|
90
169
|
const body = bodyOf(document);
|
|
91
|
-
|
|
92
|
-
|
|
170
|
+
const main = body ? mainOf(document, body) : null;
|
|
171
|
+
if (main) {
|
|
172
|
+
// Content first, everything else after it. The budget in render.js is
|
|
173
|
+
// spent top down, so whatever leads the list is what an agent gets for
|
|
174
|
+
// its first 500 tokens, and on most pages the top of the document is
|
|
175
|
+
// menus. Nothing is dropped: the chrome carries links `oc do` needs, it
|
|
176
|
+
// just stops being the thing that gets read.
|
|
177
|
+
walk(main);
|
|
178
|
+
done = main;
|
|
179
|
+
const contentBlocks = blocks.length;
|
|
180
|
+
if (body) walk(body);
|
|
181
|
+
if (blocks.length > contentBlocks) {
|
|
182
|
+
blocks.splice(contentBlocks, 0, { type: 'divider', text: '--- rest of page: navigation, sidebar, footer ---' });
|
|
183
|
+
}
|
|
184
|
+
} else if (body) {
|
|
185
|
+
walk(body);
|
|
186
|
+
}
|
|
187
|
+
return { url, title, blocks: number(mergeText(dropRepeats(blocks))) };
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Remove link labels that repeat down the page. They are the per-item controls
|
|
192
|
+
* a template stamps onto every row, and on a long thread they outweigh the
|
|
193
|
+
* content. What went is stated in place, because a view that quietly drops
|
|
194
|
+
* things is a view an agent cannot trust.
|
|
195
|
+
* @param {Block[]} blocks
|
|
196
|
+
* @returns {Block[]}
|
|
197
|
+
*/
|
|
198
|
+
function dropRepeats(blocks) {
|
|
199
|
+
const controls = (b) => b.type === 'link' || b.type === 'button';
|
|
200
|
+
/** @type {Map<string, {n: number, limit: number}>} */
|
|
201
|
+
const counts = new Map();
|
|
202
|
+
for (const b of blocks) {
|
|
203
|
+
if (!controls(b) || b.text.length > REPEAT_MAX_LEN) continue;
|
|
204
|
+
const key = b.text.toLowerCase();
|
|
205
|
+
const limit = b.type === 'button' ? REPEAT_LIMIT_BUTTON : REPEAT_LIMIT;
|
|
206
|
+
const seen = counts.get(key);
|
|
207
|
+
// A label worn by both a link and a button keeps the more patient limit.
|
|
208
|
+
counts.set(key, { n: (seen?.n ?? 0) + 1, limit: Math.max(seen?.limit ?? 0, limit) });
|
|
209
|
+
}
|
|
210
|
+
const repeated = new Set([...counts].filter(([, c]) => c.n >= c.limit).map(([k]) => k));
|
|
211
|
+
if (!repeated.size) return blocks;
|
|
212
|
+
const kept = blocks.filter((b) => !(controls(b) && repeated.has(b.text.toLowerCase())));
|
|
213
|
+
const gone = blocks.length - kept.length;
|
|
214
|
+
const names = [...repeated].slice(0, 3).join(', ');
|
|
215
|
+
const note = { type: 'divider', text: `--- ${gone} repeated controls hidden (${names}), 'oc raw' has them ---` };
|
|
216
|
+
// The note belongs with the content it was cut from, not after the chrome.
|
|
217
|
+
const boundary = kept.findIndex((b) => b.type === 'divider');
|
|
218
|
+
kept.splice(boundary === -1 ? kept.length : boundary, 0, note);
|
|
219
|
+
return kept;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Find the element holding the page's main content, or null when the page is
|
|
224
|
+
* too small or too flat for the question to have an answer.
|
|
225
|
+
* @param {any} document
|
|
226
|
+
* @param {any} body
|
|
227
|
+
* @returns {any}
|
|
228
|
+
*/
|
|
229
|
+
function mainOf(document, body) {
|
|
230
|
+
if (clean(body.textContent ?? '').length < MIN_PAGE) return null;
|
|
231
|
+
const total = prose(body);
|
|
232
|
+
for (const selector of MAIN_SELECTORS) {
|
|
233
|
+
const found = [...document.querySelectorAll(selector)];
|
|
234
|
+
if (found.length !== 1) continue;
|
|
235
|
+
const el = found[0];
|
|
236
|
+
if (el !== body && prose(el) >= total * MIN_SHARE) return el;
|
|
237
|
+
}
|
|
238
|
+
return densest(body, total);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Walk down the tree while one child holds most of the prose of the element
|
|
243
|
+
* above it. That is what separates a content column from the page around it
|
|
244
|
+
* without knowing anything about the site.
|
|
245
|
+
* @param {any} body
|
|
246
|
+
* @param {number} total
|
|
247
|
+
* @returns {any}
|
|
248
|
+
*/
|
|
249
|
+
function densest(body, total) {
|
|
250
|
+
let node = body;
|
|
251
|
+
for (let depth = 0; depth < MAX_DEPTH; depth++) {
|
|
252
|
+
let best = null;
|
|
253
|
+
let bestProse = 0;
|
|
254
|
+
for (const child of node.children ?? []) {
|
|
255
|
+
if (FURNITURE.has(child.localName) || DROP.has(child.localName)) continue;
|
|
256
|
+
const len = prose(child);
|
|
257
|
+
if (len > bestProse) {
|
|
258
|
+
best = child;
|
|
259
|
+
bestProse = len;
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
if (!best || bestProse < prose(node) * DESCEND_SHARE || bestProse < total * KEEP_SHARE) break;
|
|
263
|
+
node = best;
|
|
264
|
+
}
|
|
265
|
+
return node === body ? null : node;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Text length minus the text of controls, which is what a page is steered by
|
|
270
|
+
* rather than what it says.
|
|
271
|
+
* @param {any} el
|
|
272
|
+
* @returns {number}
|
|
273
|
+
*/
|
|
274
|
+
function prose(el) {
|
|
275
|
+
const text = clean(el.textContent ?? '').length;
|
|
276
|
+
let controls = 0;
|
|
277
|
+
// Link text is what navigation is made of. Option text is worse: a country
|
|
278
|
+
// dropdown is hundreds of characters that no link subtraction touches, and
|
|
279
|
+
// on a search results page it outscores results that are themselves links.
|
|
280
|
+
for (const node of el.querySelectorAll?.('a, select, button') ?? []) {
|
|
281
|
+
controls += clean(node.textContent ?? '').length;
|
|
282
|
+
}
|
|
283
|
+
return text - controls;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Assign handles in document order. Interactive elements get one because they
|
|
288
|
+
* can be acted on, headings and long text blocks because they can be read:
|
|
289
|
+
* a text block over the cap is printed cut, and its number is what makes the
|
|
290
|
+
* rest of it reachable with `oc read <n>` instead of a second whole-page fetch.
|
|
291
|
+
* @param {Block[]} blocks
|
|
292
|
+
* @returns {Block[]}
|
|
293
|
+
*/
|
|
294
|
+
function number(blocks) {
|
|
295
|
+
let handle = 0;
|
|
296
|
+
for (const block of blocks) {
|
|
297
|
+
if (block.type === 'divider') continue;
|
|
298
|
+
if (block.type !== 'text' || block.text.length > TEXT_CAP) block.n = ++handle;
|
|
299
|
+
}
|
|
300
|
+
return blocks;
|
|
93
301
|
}
|
|
94
302
|
|
|
95
303
|
/**
|
|
@@ -106,8 +314,12 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document
|
|
|
106
314
|
* hidden content.
|
|
107
315
|
* @param {string} html
|
|
108
316
|
*/
|
|
109
|
-
function cleanDocument(html) {
|
|
110
|
-
|
|
317
|
+
function cleanDocument(html, url = '') {
|
|
318
|
+
// Raw is the mode an agent reaches for when the compact view left something
|
|
319
|
+
// out, so it is the one place a JSON response keeps every field.
|
|
320
|
+
const { document } = parseHTML(asHTML(html, url, { full: true }));
|
|
321
|
+
// Read the title before the sweep below removes the head with it.
|
|
322
|
+
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
111
323
|
for (const tag of DROP) {
|
|
112
324
|
for (const el of [...document.querySelectorAll(tag)]) el.remove();
|
|
113
325
|
}
|
|
@@ -115,18 +327,19 @@ function cleanDocument(html) {
|
|
|
115
327
|
for (const el of [...document.querySelectorAll('[style]')]) {
|
|
116
328
|
if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
|
|
117
329
|
}
|
|
118
|
-
return document;
|
|
330
|
+
return { document, title };
|
|
119
331
|
}
|
|
120
332
|
|
|
121
333
|
/**
|
|
122
334
|
* Whole-page markdown for `oc raw`, produced by turndown so lists, emphasis,
|
|
123
335
|
* links, and code blocks come out as real markdown instead of flat lines.
|
|
124
336
|
* @param {string} html
|
|
337
|
+
* @param {string} url - only read when the body turns out to be JSON, whose
|
|
338
|
+
* title has to come from the endpoint because the payload has none
|
|
125
339
|
* @returns {string}
|
|
126
340
|
*/
|
|
127
|
-
export function toMarkdown(html) {
|
|
128
|
-
const document = cleanDocument(html);
|
|
129
|
-
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
341
|
+
export function toMarkdown(html, url = '') {
|
|
342
|
+
const { document, title } = cleanDocument(html, url);
|
|
130
343
|
const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
|
|
131
344
|
const el = bodyOf(document);
|
|
132
345
|
const body = el ? turndown.turndown(el.innerHTML).trim() : '';
|
|
@@ -137,14 +350,513 @@ export function toMarkdown(html) {
|
|
|
137
350
|
* Whole-page cleaned HTML for `oc raw --html`, for agents that would rather
|
|
138
351
|
* work with markup than markdown. Same noise removal, no other rewriting.
|
|
139
352
|
* @param {string} html
|
|
353
|
+
* @param {string} url - see toMarkdown
|
|
140
354
|
* @returns {string}
|
|
141
355
|
*/
|
|
142
|
-
export function toHTML(html) {
|
|
143
|
-
const document = cleanDocument(html);
|
|
356
|
+
export function toHTML(html, url = '') {
|
|
357
|
+
const { document } = cleanDocument(html, url);
|
|
144
358
|
const el = bodyOf(document);
|
|
145
359
|
return el ? el.innerHTML.trim() : '';
|
|
146
360
|
}
|
|
147
361
|
|
|
362
|
+
/**
|
|
363
|
+
* Sites behind hard bot challenges often leave their Atom or RSS feeds open:
|
|
364
|
+
* Stack Overflow challenges every HTML page but publishes full question and
|
|
365
|
+
* answer bodies under /feeds. A feed is XML with the real content escaped
|
|
366
|
+
* inside each entry, so this converts one into a plain HTML document and
|
|
367
|
+
* everything downstream stays unchanged. Returns null for non-feed input.
|
|
368
|
+
* @param {string} text
|
|
369
|
+
* @returns {string | null}
|
|
370
|
+
*/
|
|
371
|
+
export function feedToHTML(text) {
|
|
372
|
+
const head = text.slice(0, 2000);
|
|
373
|
+
if (!/<(feed|rss)[\s>]/i.test(head) || /<(html|body)[\s>]/i.test(head)) return null;
|
|
374
|
+
// RSS wraps bodies in CDATA, which an HTML parser reads as a comment and
|
|
375
|
+
// drops. Escaping the section turns it into ordinary text, the same shape
|
|
376
|
+
// Atom feeds already use.
|
|
377
|
+
const xml = text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, inner) =>
|
|
378
|
+
inner.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'));
|
|
379
|
+
const { document } = parseHTML(xml);
|
|
380
|
+
const root = document.querySelector('feed, rss');
|
|
381
|
+
if (!root) return null;
|
|
382
|
+
const esc = (s) => s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
383
|
+
// Feed elements are unknown to the HTML parser, so self-closed ones like
|
|
384
|
+
// <category /> stay open and swallow their siblings. Descendant queries
|
|
385
|
+
// still land, because a closing </entry> or </item> pops the whole pile.
|
|
386
|
+
const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
|
|
387
|
+
const feedTitle = field(root, 'title');
|
|
388
|
+
const parts = [];
|
|
389
|
+
for (const entry of root.querySelectorAll('entry, item')) {
|
|
390
|
+
const title = field(entry, 'title');
|
|
391
|
+
const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
|
|
392
|
+
?? entry.querySelector('link[href]')?.getAttribute('href')
|
|
393
|
+
?? field(entry, 'guid');
|
|
394
|
+
const author = field(entry, 'author name') || field(entry, 'author');
|
|
395
|
+
const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
|
|
396
|
+
const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
|
|
397
|
+
// Atom escapes the entry body, so textContent of content/summary is the
|
|
398
|
+
// HTML itself, ready to be embedded and parsed like any page.
|
|
399
|
+
const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? '';
|
|
400
|
+
parts.push('<article>');
|
|
401
|
+
if (title) parts.push(`<h2>${esc(title)}</h2>`);
|
|
402
|
+
if (byline || href) {
|
|
403
|
+
parts.push(`<p>${esc(byline)}${href ? ` <a href="${esc(href)}">open</a>` : ''}</p>`);
|
|
404
|
+
}
|
|
405
|
+
parts.push(body, '</article>');
|
|
406
|
+
}
|
|
407
|
+
// A full skeleton, because linkedom treats the first element of a bare
|
|
408
|
+
// multi-rooted fragment as the whole document and drops its siblings.
|
|
409
|
+
return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
const escHTML = (s) => String(s ?? '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
413
|
+
|
|
414
|
+
/**
|
|
415
|
+
* Pull one embedded JSON object out of a page by its variable name, scanning
|
|
416
|
+
* forward from the first `{` and tracking string/escape state so a brace
|
|
417
|
+
* inside a quoted value never ends the object early. A regex can't do this
|
|
418
|
+
* safely because the JSON itself contains braces.
|
|
419
|
+
* @param {string} text
|
|
420
|
+
* @param {string} marker
|
|
421
|
+
* @returns {any}
|
|
422
|
+
*/
|
|
423
|
+
function extractJSON(text, marker) {
|
|
424
|
+
const at = text.indexOf(marker);
|
|
425
|
+
if (at === -1) return null;
|
|
426
|
+
const start = text.indexOf('{', at);
|
|
427
|
+
if (start === -1) return null;
|
|
428
|
+
let depth = 0, inString = false, escaped = false;
|
|
429
|
+
for (let i = start; i < text.length; i++) {
|
|
430
|
+
const c = text[i];
|
|
431
|
+
if (inString) {
|
|
432
|
+
if (escaped) escaped = false;
|
|
433
|
+
else if (c === '\\') escaped = true;
|
|
434
|
+
else if (c === '"') inString = false;
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
if (c === '"') inString = true;
|
|
438
|
+
else if (c === '{') depth++;
|
|
439
|
+
else if (c === '}' && --depth === 0) {
|
|
440
|
+
try {
|
|
441
|
+
return JSON.parse(text.slice(start, i + 1));
|
|
442
|
+
} catch {
|
|
443
|
+
return null;
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
return null;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
/**
|
|
451
|
+
* A YouTube watch page needs client JS to become interactive, but the initial
|
|
452
|
+
* HTML response already carries the title, description, view count, and
|
|
453
|
+
* caption tracks inline as `ytInitialPlayerResponse`, so none of that has to
|
|
454
|
+
* wait on the v0.3 headless fallback. Each caption track becomes a link to
|
|
455
|
+
* its timedtext URL, which `oc do` follows and transcriptToHTML turns into
|
|
456
|
+
* readable text. Returns null for anything that isn't a watch page.
|
|
457
|
+
* @param {string} text
|
|
458
|
+
* @returns {string | null}
|
|
459
|
+
*/
|
|
460
|
+
export function youtubeToHTML(text) {
|
|
461
|
+
const data = extractJSON(text, 'ytInitialPlayerResponse');
|
|
462
|
+
const details = data?.videoDetails;
|
|
463
|
+
if (!details?.title) return null;
|
|
464
|
+
const micro = data.microformat?.playerMicroformatRenderer;
|
|
465
|
+
const tracks = data.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
466
|
+
const views = details.viewCount ? `${Number(details.viewCount).toLocaleString('en-US')} views` : '';
|
|
467
|
+
const byline = [details.author && `by ${details.author}`, views, micro?.publishDate].filter(Boolean).join(', ');
|
|
468
|
+
const channelHref = details.channelId ? `https://www.youtube.com/channel/${details.channelId}` : '';
|
|
469
|
+
const parts = [`<h1>${escHTML(details.title)}</h1>`];
|
|
470
|
+
if (byline || channelHref) {
|
|
471
|
+
parts.push(`<p>${escHTML(byline)}${channelHref ? ` <a href="${escHTML(channelHref)}">channel</a>` : ''}</p>`);
|
|
472
|
+
}
|
|
473
|
+
const description = details.shortDescription ?? micro?.description?.simpleText ?? '';
|
|
474
|
+
for (const para of description.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean)) {
|
|
475
|
+
parts.push(`<p>${escHTML(para)}</p>`);
|
|
476
|
+
}
|
|
477
|
+
for (const track of tracks) {
|
|
478
|
+
const label = track.name?.simpleText || track.languageCode || 'transcript';
|
|
479
|
+
const kind = track.kind === 'asr' ? ' (auto-generated)' : '';
|
|
480
|
+
parts.push(`<p><a href="${escHTML(track.baseUrl)}">transcript: ${escHTML(label)}${kind}</a></p>`);
|
|
481
|
+
}
|
|
482
|
+
return `<html><head><title>${escHTML(details.title)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
/**
|
|
486
|
+
* YouTube's timedtext endpoint answers plain XML, one `<text>` cue per
|
|
487
|
+
* caption line. Concatenated into a single block rather than one paragraph
|
|
488
|
+
* per cue, so a whole transcript pages through `oc next`/`oc read` like any
|
|
489
|
+
* other long document instead of costing one numbered block per few words.
|
|
490
|
+
* Consecutive duplicate cues (auto captions repeat words across overlapping
|
|
491
|
+
* segments) are dropped. Returns null for anything that isn't a transcript.
|
|
492
|
+
* @param {string} text
|
|
493
|
+
* @returns {string | null}
|
|
494
|
+
*/
|
|
495
|
+
export function transcriptToHTML(text) {
|
|
496
|
+
if (!/<transcript[\s>]/i.test(text.slice(0, 200))) return null;
|
|
497
|
+
const { document } = parseHTML(text);
|
|
498
|
+
const cues = [...document.querySelectorAll('text')];
|
|
499
|
+
if (!cues.length) return null;
|
|
500
|
+
const lines = [];
|
|
501
|
+
for (const cue of cues) {
|
|
502
|
+
const line = clean(cue.textContent ?? '');
|
|
503
|
+
if (line && line !== lines[lines.length - 1]) lines.push(line);
|
|
504
|
+
}
|
|
505
|
+
if (!lines.length) return null;
|
|
506
|
+
return `<html><head><title>Transcript</title></head><body>\n<p>${escHTML(lines.join(' '))}</p>\n</body></html>`;
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
// Keys an API is likely to give the human-readable name of an item, in the
|
|
510
|
+
// order they win when an item carries several of them.
|
|
511
|
+
const TITLE_KEYS = [
|
|
512
|
+
'title', 'name', 'headline', 'subject', 'label', 'display_name',
|
|
513
|
+
'full_name', 'summary', 'question', 'message',
|
|
514
|
+
];
|
|
515
|
+
|
|
516
|
+
// Keys holding the item's own page. A URL under any other name is still found,
|
|
517
|
+
// by looking at values rather than names, but these win when several qualify.
|
|
518
|
+
const LINK_KEYS = ['link', 'url', 'html_url', 'web_url', 'permalink', 'href'];
|
|
519
|
+
|
|
520
|
+
// A title has to fit on a line to be one. Anything longer is a body that
|
|
521
|
+
// happens to live under a title-ish key, and belongs in a block of its own.
|
|
522
|
+
const TITLE_MAX = 300;
|
|
523
|
+
|
|
524
|
+
// How many constant fields the footer names before it stops counting them out.
|
|
525
|
+
const CONST_LISTED = 8;
|
|
526
|
+
|
|
527
|
+
// Characters of field text an item may spend in the compact view. A response
|
|
528
|
+
// carries far more fields than an agent asked for: one Stack Exchange result
|
|
529
|
+
// brings eight about the asker alone, which is the whole budget spent on who
|
|
530
|
+
// rather than what. Thirty items make this thirty times over, so it buys two
|
|
531
|
+
// or three fields, not a record. `oc raw` still has all of them.
|
|
532
|
+
const FIELD_BUDGET = 60;
|
|
533
|
+
|
|
534
|
+
// What a field from a flattened sub-object scores against one of the item's
|
|
535
|
+
// own. `owner.user_id` varies perfectly and costs little, which is enough to
|
|
536
|
+
// win on width and variance alone, but it identifies somebody attached to the
|
|
537
|
+
// result rather than the result, and no agent searched for it.
|
|
538
|
+
const NESTED_PENALTY = 0.25;
|
|
539
|
+
|
|
540
|
+
// How many names the footer lists when it says which fields it left out.
|
|
541
|
+
const DROPPED_LISTED = 5;
|
|
542
|
+
|
|
543
|
+
// Seconds and milliseconds since the epoch, bounded either side so an ordinary
|
|
544
|
+
// count (a score, a byte size) is never mistaken for a date.
|
|
545
|
+
const EPOCH_S = [1e9, 4e9];
|
|
546
|
+
const EPOCH_MS = [1e12, 4e12];
|
|
547
|
+
const DATE_KEY = /(^|_)(date|at|time|timestamp|created|updated|published|modified)$/i;
|
|
548
|
+
|
|
549
|
+
// Named entities worth knowing without a table: the five XML ones plus the
|
|
550
|
+
// space. Everything else arrives numeric.
|
|
551
|
+
const ENTITIES = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: ' ' };
|
|
552
|
+
|
|
553
|
+
/**
|
|
554
|
+
* Undo one layer of HTML escaping in a string value. APIs that back an HTML
|
|
555
|
+
* site tend to escape the text they return: Stack Exchange answers with
|
|
556
|
+
* `Is "==" slower`, and re-escaping that on the way into a document
|
|
557
|
+
* would print the entity instead of the quote it stands for.
|
|
558
|
+
* @param {string} s
|
|
559
|
+
* @returns {string}
|
|
560
|
+
*/
|
|
561
|
+
const decodeEntities = (s) =>
|
|
562
|
+
s.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, body) => {
|
|
563
|
+
if (body[0] === '#') {
|
|
564
|
+
const code = body[1] === 'x' || body[1] === 'X'
|
|
565
|
+
? parseInt(body.slice(2), 16)
|
|
566
|
+
: Number(body.slice(1));
|
|
567
|
+
return Number.isInteger(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : whole;
|
|
568
|
+
}
|
|
569
|
+
return ENTITIES[body.toLowerCase()] ?? whole;
|
|
570
|
+
});
|
|
571
|
+
|
|
572
|
+
const isPlain = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
573
|
+
const isURL = (v) => typeof v === 'string' && /^https?:\/\/\S+$/.test(v);
|
|
574
|
+
const looksHTML = (v) => typeof v === 'string' && /<\/?(p|div|pre|code|br|ul|ol|li|h[1-6]|blockquote|table|img|a|em|strong)\b[^>]*>/i.test(v);
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* One level of flattening, so `owner: {display_name}` becomes an
|
|
578
|
+
* `owner.display_name` field. Deeper than that an object stops being a set of
|
|
579
|
+
* fields and starts being a document, which no line-per-item view can hold.
|
|
580
|
+
* @param {Record<string, any>} item
|
|
581
|
+
* @returns {Map<string, any>}
|
|
582
|
+
*/
|
|
583
|
+
function flattenItem(item) {
|
|
584
|
+
/** @type {Map<string, any>} */
|
|
585
|
+
const out = new Map();
|
|
586
|
+
for (const [key, value] of Object.entries(item)) {
|
|
587
|
+
if (!isPlain(value)) {
|
|
588
|
+
out.set(key, value);
|
|
589
|
+
continue;
|
|
590
|
+
}
|
|
591
|
+
for (const [inner, deep] of Object.entries(value)) {
|
|
592
|
+
if (deep !== null && typeof deep === 'object') continue;
|
|
593
|
+
out.set(`${key}.${inner}`, deep);
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
return out;
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* One field as one string. Epoch integers under a date-ish key become ISO
|
|
601
|
+
* dates, because an agent that has to convert one pays a turn for it. Arrays
|
|
602
|
+
* of scalars (tags, labels) join; arrays of objects are counted, since
|
|
603
|
+
* spelling them out is what the flattening above already refused to do.
|
|
604
|
+
* @param {string} key
|
|
605
|
+
* @param {any} value
|
|
606
|
+
* @returns {string}
|
|
607
|
+
*/
|
|
608
|
+
function renderValue(key, value) {
|
|
609
|
+
if (value === null || value === undefined) return '';
|
|
610
|
+
if (Array.isArray(value)) {
|
|
611
|
+
if (!value.length) return '';
|
|
612
|
+
if (value.every((v) => v === null || typeof v !== 'object')) return value.join(', ');
|
|
613
|
+
return `[${value.length} items]`;
|
|
614
|
+
}
|
|
615
|
+
if (typeof value === 'object') return '';
|
|
616
|
+
if (typeof value === 'number' && Number.isInteger(value) && DATE_KEY.test(key)) {
|
|
617
|
+
const ms = value >= EPOCH_S[0] && value < EPOCH_S[1] ? value * 1000
|
|
618
|
+
: value >= EPOCH_MS[0] && value < EPOCH_MS[1] ? value
|
|
619
|
+
: null;
|
|
620
|
+
if (ms !== null) return new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
|
|
621
|
+
}
|
|
622
|
+
return typeof value === 'string' ? decodeEntities(value) : String(value);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
/**
|
|
626
|
+
* Order fields by how much they say per character, and keep taking them while
|
|
627
|
+
* an item can still afford one. Variance is what carries the information: a
|
|
628
|
+
* field reading the same on every row has already been lifted out as a
|
|
629
|
+
* constant, and one reading differently every time is why the response was
|
|
630
|
+
* fetched. Dividing by width is what stops a long field (a profile image URL,
|
|
631
|
+
* a licence string) from crowding out three short ones that matter more.
|
|
632
|
+
* @param {Array<Map<string, string>|null>} rows
|
|
633
|
+
* @param {Set<string>} skip - fields already spoken for as title, link, or constant
|
|
634
|
+
* @returns {{kept: Set<string>, dropped: string[]}}
|
|
635
|
+
*/
|
|
636
|
+
function chooseFields(rows, skip) {
|
|
637
|
+
const present = rows.filter(Boolean);
|
|
638
|
+
const keys = [];
|
|
639
|
+
for (const row of present) {
|
|
640
|
+
for (const key of row.keys()) if (!skip.has(key) && !keys.includes(key)) keys.push(key);
|
|
641
|
+
}
|
|
642
|
+
const scored = [];
|
|
643
|
+
for (const key of keys) {
|
|
644
|
+
const values = present.map((row) => row.get(key) ?? '').filter((v) => v !== '');
|
|
645
|
+
if (!values.length) continue;
|
|
646
|
+
const width = values.reduce((sum, v) => sum + v.length + key.length + 3, 0) / values.length;
|
|
647
|
+
const variance = new Set(values).size / values.length;
|
|
648
|
+
const penalty = key.includes('.') ? NESTED_PENALTY : 1;
|
|
649
|
+
scored.push({ key, width, score: (variance / width) * penalty });
|
|
650
|
+
}
|
|
651
|
+
scored.sort((a, b) => b.score - a.score);
|
|
652
|
+
const kept = new Set();
|
|
653
|
+
let spent = 0;
|
|
654
|
+
for (const field of scored) {
|
|
655
|
+
// The first field is taken whatever it costs, so an item of one long field
|
|
656
|
+
// still renders something rather than nothing.
|
|
657
|
+
if (spent + field.width > FIELD_BUDGET && kept.size) continue;
|
|
658
|
+
kept.add(field.key);
|
|
659
|
+
spent += field.width;
|
|
660
|
+
}
|
|
661
|
+
return { kept, dropped: scored.filter((f) => !kept.has(f.key)).map((f) => f.key) };
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
/**
|
|
665
|
+
* Pick the array the response is actually about: the root when it is one,
|
|
666
|
+
* otherwise the longest array of objects at the top level, which is where
|
|
667
|
+
* `items`, `data`, `results`, and `hits` all live. Everything beside it is
|
|
668
|
+
* metadata about the request rather than content.
|
|
669
|
+
* @param {any} data
|
|
670
|
+
* @returns {{items: any[], meta: Record<string, any>}}
|
|
671
|
+
*/
|
|
672
|
+
function mainArray(data) {
|
|
673
|
+
if (Array.isArray(data)) return { items: data, meta: {} };
|
|
674
|
+
let key = '';
|
|
675
|
+
/** @type {any[] | null} */
|
|
676
|
+
let items = null;
|
|
677
|
+
for (const [k, v] of Object.entries(data)) {
|
|
678
|
+
if (!Array.isArray(v) || !v.length) continue;
|
|
679
|
+
if (!v.some(isPlain)) continue;
|
|
680
|
+
if (!items || v.length > items.length) {
|
|
681
|
+
items = v;
|
|
682
|
+
key = k;
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
// A response with no array is a single resource, which renders as one item
|
|
686
|
+
// rather than as a special case.
|
|
687
|
+
if (!items) return { items: [data], meta: {} };
|
|
688
|
+
const meta = { ...data };
|
|
689
|
+
delete meta[key];
|
|
690
|
+
return { items, meta };
|
|
691
|
+
}
|
|
692
|
+
|
|
693
|
+
/**
|
|
694
|
+
* Fields whose rendered value is the same on every item. In a list of thirty
|
|
695
|
+
* results they are thirty copies of one fact, so they come out of the rows and
|
|
696
|
+
* get stated once at the foot of the page: the saving is the point of the
|
|
697
|
+
* exercise, and dropping them silently would be a lie about what the API said.
|
|
698
|
+
* @param {Array<Map<string, string>|null>} rows
|
|
699
|
+
* @returns {Map<string, string>}
|
|
700
|
+
*/
|
|
701
|
+
function constantFields(rows) {
|
|
702
|
+
const present = rows.filter(Boolean);
|
|
703
|
+
/** @type {Map<string, string>} */
|
|
704
|
+
const out = new Map();
|
|
705
|
+
if (present.length < 2) return out;
|
|
706
|
+
for (const [key, value] of present[0]) {
|
|
707
|
+
if (present.every((row) => row.get(key) === value)) out.set(key, value);
|
|
708
|
+
}
|
|
709
|
+
return out;
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
/**
|
|
713
|
+
* A JSON body is not a page, so nothing downstream can read one: the HTML
|
|
714
|
+
* parser turns it into a single unreadable text node and the budget truncates
|
|
715
|
+
* the blob. This turns a response into the same shape feedToHTML produces, one
|
|
716
|
+
* article per item, so numbering, the budget, `oc do`, and raw markdown all
|
|
717
|
+
* work on an API exactly as they do on a page.
|
|
718
|
+
*
|
|
719
|
+
* The view is a line per item rather than a table: two or three fields carry
|
|
720
|
+
* the signal in most responses, and turndown cannot write a markdown table
|
|
721
|
+
* without the GFM plugin, which is a dependency this project does not want.
|
|
722
|
+
* Returns null for anything that is not JSON.
|
|
723
|
+
* @param {string} text
|
|
724
|
+
* @param {string} url
|
|
725
|
+
* @param {{full?: boolean}} [opts] - full keeps every field, which is what the
|
|
726
|
+
* raw modes are for; the compact view keeps the ones that earn their tokens
|
|
727
|
+
* @returns {string | null}
|
|
728
|
+
*/
|
|
729
|
+
export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
730
|
+
if (!/^\s*[[{]/.test(text.slice(0, 200))) return null;
|
|
731
|
+
let data;
|
|
732
|
+
try {
|
|
733
|
+
data = JSON.parse(text);
|
|
734
|
+
} catch {
|
|
735
|
+
return null;
|
|
736
|
+
}
|
|
737
|
+
if (!data || typeof data !== 'object') return null;
|
|
738
|
+
|
|
739
|
+
const { items, meta } = mainArray(data);
|
|
740
|
+
const flats = items.map((item) => (isPlain(item) ? flattenItem(item) : null));
|
|
741
|
+
const rows = flats.map((flat) => {
|
|
742
|
+
if (!flat) return null;
|
|
743
|
+
/** @type {Map<string, string>} */
|
|
744
|
+
const row = new Map();
|
|
745
|
+
for (const [key, value] of flat) row.set(key, renderValue(key, value));
|
|
746
|
+
return row;
|
|
747
|
+
});
|
|
748
|
+
|
|
749
|
+
// One item's field names stand for all of them: a ragged response still gets
|
|
750
|
+
// a consistent title and link column, and a field missing from an item is
|
|
751
|
+
// simply absent from its line.
|
|
752
|
+
const sample = flats.find(Boolean) ?? new Map();
|
|
753
|
+
const titled = (k) => {
|
|
754
|
+
const v = sample.get(k);
|
|
755
|
+
return typeof v === 'string' && v.trim() && v.length <= TITLE_MAX && !isURL(v) && !looksHTML(v);
|
|
756
|
+
};
|
|
757
|
+
const titleKey = TITLE_KEYS.find(titled) ?? [...sample.keys()].find(titled);
|
|
758
|
+
const linkKey = LINK_KEYS.find((k) => isURL(sample.get(k))) ?? [...sample.keys()].find((k) => isURL(sample.get(k)));
|
|
759
|
+
|
|
760
|
+
const constants = constantFields(rows);
|
|
761
|
+
const spoken = new Set([titleKey, linkKey, ...constants.keys()].filter(Boolean));
|
|
762
|
+
const { kept, dropped } = full
|
|
763
|
+
? { kept: null, dropped: [] }
|
|
764
|
+
: chooseFields(rows, spoken);
|
|
765
|
+
const parts = [];
|
|
766
|
+
|
|
767
|
+
for (let i = 0; i < items.length; i++) {
|
|
768
|
+
const flat = flats[i];
|
|
769
|
+
if (!flat) {
|
|
770
|
+
const line = renderValue('', items[i]);
|
|
771
|
+
if (line) parts.push(`<p>${escHTML(line)}</p>`);
|
|
772
|
+
continue;
|
|
773
|
+
}
|
|
774
|
+
const row = rows[i];
|
|
775
|
+
const title = titleKey ? row.get(titleKey) : '';
|
|
776
|
+
const link = linkKey && isURL(flat.get(linkKey)) ? flat.get(linkKey) : '';
|
|
777
|
+
const inline = [];
|
|
778
|
+
const long = [];
|
|
779
|
+
const bodies = [];
|
|
780
|
+
for (const [key, value] of flat) {
|
|
781
|
+
if (spoken.has(key)) continue;
|
|
782
|
+
// A field carrying markup is a document, and is kept whatever it scored:
|
|
783
|
+
// it renders as its own block, so it never competes for the field line.
|
|
784
|
+
if (kept && !kept.has(key) && !looksHTML(value)) continue;
|
|
785
|
+
// A field carrying HTML is a page in itself: `filter=withbody` on the
|
|
786
|
+
// Stack Exchange API puts a whole question in one. It goes through the
|
|
787
|
+
// distiller like any other markup instead of into a cell.
|
|
788
|
+
if (looksHTML(value)) {
|
|
789
|
+
bodies.push(String(value));
|
|
790
|
+
continue;
|
|
791
|
+
}
|
|
792
|
+
const rendered = row.get(key);
|
|
793
|
+
if (!rendered) continue;
|
|
794
|
+
if (rendered.length > TEXT_CAP) long.push(`<p>${escHTML(`${key}: ${rendered}`)}</p>`);
|
|
795
|
+
else inline.push(`${key}: ${rendered}`);
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
parts.push('<article>');
|
|
799
|
+
if (title && link) parts.push(`<p><a href="${escHTML(link)}">${escHTML(title)}</a></p>`);
|
|
800
|
+
else if (title) parts.push(`<p>${escHTML(title)}</p>`);
|
|
801
|
+
else if (link) parts.push(`<p><a href="${escHTML(link)}">open</a></p>`);
|
|
802
|
+
if (inline.length) parts.push(`<p>${escHTML(inline.join(' | '))}</p>`);
|
|
803
|
+
parts.push(...long);
|
|
804
|
+
for (const body of bodies) parts.push(`<div>${body}</div>`);
|
|
805
|
+
parts.push('</article>');
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
// What the rows no longer carry, said once. Empty-everywhere fields are
|
|
809
|
+
// named but not valued, because their value is the fact that there isn't one.
|
|
810
|
+
const shown = [...constants].filter(([, v]) => v !== '');
|
|
811
|
+
const empty = [...constants].filter(([, v]) => v === '').map(([k]) => k);
|
|
812
|
+
const clipped = (list, render) => {
|
|
813
|
+
const head = list.slice(0, CONST_LISTED).map(render).join(', ');
|
|
814
|
+
return list.length > CONST_LISTED ? `${head}, +${list.length - CONST_LISTED} more` : head;
|
|
815
|
+
};
|
|
816
|
+
if (shown.length) {
|
|
817
|
+
parts.push(`<p>same on every item: ${escHTML(clipped(shown, ([k, v]) => `${k}=${v.length > 60 ? `${v.slice(0, 60)}...` : v}`))}</p>`);
|
|
818
|
+
}
|
|
819
|
+
if (empty.length) parts.push(`<p>empty on every item: ${escHTML(clipped(empty, (k) => k))}</p>`);
|
|
820
|
+
if (dropped.length) {
|
|
821
|
+
const names = dropped.slice(0, DROPPED_LISTED).join(', ');
|
|
822
|
+
const more = dropped.length > DROPPED_LISTED ? `, +${dropped.length - DROPPED_LISTED} more` : '';
|
|
823
|
+
parts.push(`<p>${dropped.length} fields per item not shown (${escHTML(names + more)}), 'oc raw' has them</p>`);
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
const metaBits = [];
|
|
827
|
+
for (const [key, value] of Object.entries(meta)) {
|
|
828
|
+
if (value === null || typeof value === 'object') continue;
|
|
829
|
+
const rendered = renderValue(key, value);
|
|
830
|
+
if (rendered) metaBits.push(`${key}=${rendered}`);
|
|
831
|
+
}
|
|
832
|
+
if (metaBits.length) parts.push(`<p>response: ${escHTML(metaBits.join(', '))}</p>`);
|
|
833
|
+
|
|
834
|
+
const count = `${items.length} ${items.length === 1 ? 'item' : 'items'}`;
|
|
835
|
+
return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
/**
|
|
839
|
+
* An API response has no title of its own, so the endpoint becomes one. The
|
|
840
|
+
* query parameter is worth the tokens it costs: it is the only part of a
|
|
841
|
+
* search URL that says what the page is, and without it every search a session
|
|
842
|
+
* runs is titled the same.
|
|
843
|
+
* @param {string} url
|
|
844
|
+
* @param {string} count
|
|
845
|
+
* @returns {string}
|
|
846
|
+
*/
|
|
847
|
+
function jsonTitle(url, count) {
|
|
848
|
+
try {
|
|
849
|
+
const u = new URL(url);
|
|
850
|
+
const query = ['q', 'query', 'search', 'terms', 'keywords', 'text']
|
|
851
|
+
.map((k) => u.searchParams.get(k))
|
|
852
|
+
.find((v) => v);
|
|
853
|
+
const base = `${u.host}${u.pathname}`.replace(/\/+$/, '');
|
|
854
|
+
return query ? `${base}: "${query}" (${count})` : `${base} (${count})`;
|
|
855
|
+
} catch {
|
|
856
|
+
return `JSON (${count})`;
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
|
|
148
860
|
/**
|
|
149
861
|
* Adjacent text nodes arrive fragmented (one per inline element boundary).
|
|
150
862
|
* Merging them is what turns DOM noise into readable lines.
|
|
@@ -155,13 +867,27 @@ function mergeText(blocks) {
|
|
|
155
867
|
/** @type {Block[]} */
|
|
156
868
|
const out = [];
|
|
157
869
|
for (const b of blocks) {
|
|
870
|
+
if (b.type === 'break') {
|
|
871
|
+
// A boundary only has to stop the merge, and it does that by being the
|
|
872
|
+
// last thing in the list when the next text block arrives.
|
|
873
|
+
if (out[out.length - 1]?.type === 'text') out.push(b);
|
|
874
|
+
continue;
|
|
875
|
+
}
|
|
158
876
|
const prev = out[out.length - 1];
|
|
159
877
|
if (b.type === 'text' && prev?.type === 'text') {
|
|
160
|
-
|
|
878
|
+
// Two fragments of one element with no whitespace between them were one
|
|
879
|
+
// word before the parser split them. Anything else was separated on the
|
|
880
|
+
// page and stays separated here.
|
|
881
|
+
const glued = prev.host && prev.host === b.host && !prev.post && !b.pre;
|
|
882
|
+
prev.text = glued ? `${prev.text}${b.text}` : `${prev.text} ${b.text}`;
|
|
883
|
+
prev.post = b.post;
|
|
161
884
|
} else {
|
|
162
885
|
out.push(b);
|
|
163
886
|
}
|
|
164
887
|
}
|
|
165
888
|
// Single stray characters (list bullets, separators) cost tokens and say nothing.
|
|
166
|
-
return out
|
|
889
|
+
return out
|
|
890
|
+
.filter((b) => b.type !== 'break')
|
|
891
|
+
.filter((b) => b.type !== 'text' || b.text.length > 1)
|
|
892
|
+
.map(({ host, pre, post, ...block }) => block);
|
|
167
893
|
}
|