@only-cli/oc 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -26
- package/clis/finance.yahoo.com.json +13 -0
- package/clis/github.com.json +10 -0
- package/clis/linkedin.com.json +8 -0
- package/clis/stackoverflow.com.json +9 -0
- package/clis/x.com.json +7 -0
- package/clis/youtube.com.json +7 -0
- package/package.json +1 -1
- package/skills/only-cli/SKILL.md +49 -7
- package/src/act.js +235 -17
- package/src/cli.js +65 -15
- package/src/distill.js +380 -22
- package/src/fetch.js +151 -14
- package/src/render.js +75 -24
- package/src/session.js +143 -5
package/src/cli.js
CHANGED
|
@@ -4,30 +4,55 @@ import { fetchPage } from './fetch.js';
|
|
|
4
4
|
import { distill, toMarkdown, toHTML } from './distill.js';
|
|
5
5
|
import { render, estimateTokens } from './render.js';
|
|
6
6
|
import * as act from './act.js';
|
|
7
|
+
import { DEFAULT_SESSION, loadSession, saveSession, sessionFromPage } from './session.js';
|
|
7
8
|
|
|
8
9
|
const HELP = `only-cli: the web as a compact terminal, built for AI agents.
|
|
9
10
|
|
|
10
11
|
usage: oc <command> [args] [flags]
|
|
11
12
|
|
|
12
13
|
open <url> fetch and render a page with numbered actions
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
find <query> where a string appears on the page already open
|
|
15
|
+
next the next budget worth of the page already open
|
|
16
|
+
read <n> full text of the region at [n], up to 2000 tokens
|
|
17
|
+
raw [url] distilled markdown of the whole page
|
|
18
|
+
do <n> follow the numbered link [n], or read [n] if it is text
|
|
15
19
|
fill <n> <text> type into a numbered input (v0.2)
|
|
16
20
|
submit [n] submit a form (v0.2)
|
|
17
|
-
|
|
18
|
-
find <query> search visible text on the current page (v0.2)
|
|
19
|
-
back | next history and pagination (v0.2)
|
|
21
|
+
back return to the previous page (v0.2)
|
|
20
22
|
session ls|rm manage saved sessions (v0.2)
|
|
21
23
|
|
|
22
24
|
flags:
|
|
23
|
-
--budget <tokens> tighten or loosen the render budget (default 500
|
|
25
|
+
--budget <tokens> tighten or loosen the render budget (default 500,
|
|
26
|
+
2000 for read). It is a target, not a ceiling: a page
|
|
27
|
+
that would finish within about four times it is printed
|
|
28
|
+
whole rather than costing you a second command
|
|
24
29
|
--json machine-stable JSON output
|
|
25
30
|
--html raw only: cleaned HTML instead of markdown
|
|
26
31
|
--verbose, -v metrics on stderr: tokens saved vs the page HTML, HTTP
|
|
27
32
|
status and client identity, timing, transfer size, and
|
|
28
33
|
memory. --stats is an alias; OC_VERBOSE=1 turns it on
|
|
29
34
|
globally. Off by default because metrics cost tokens too.
|
|
30
|
-
--session <name>
|
|
35
|
+
--session <name> keep separate page state under a name (default: default)
|
|
36
|
+
|
|
37
|
+
'oc open' remembers the page it printed, so 'oc do 3' follows link [3] without
|
|
38
|
+
you ever handling its URL, and 'oc next' or 'oc read 12' picks up what the
|
|
39
|
+
budget left behind without fetching it again. State lives in ~/.only-cli
|
|
40
|
+
(override with OC_HOME).`;
|
|
41
|
+
|
|
42
|
+
// A rendered page has to be remembered or its [3] means nothing to the next
|
|
43
|
+
// command. Saving state must never break a render, so a home directory that
|
|
44
|
+
// cannot be written costs the agent `do`, `read`, and `next`, and nothing
|
|
45
|
+
// else, but silently: a sandbox that blocks the write leaves `do` resolving
|
|
46
|
+
// against whatever session last saved successfully, possibly from an
|
|
47
|
+
// unrelated page, with no sign anything is wrong. So the failure still prints,
|
|
48
|
+
// on stderr where it costs nothing until something breaks.
|
|
49
|
+
const remember = (page, name, cursor) => {
|
|
50
|
+
try {
|
|
51
|
+
saveSession(name, sessionFromPage(page, loadSession(name), { cursor }));
|
|
52
|
+
} catch (err) {
|
|
53
|
+
console.error(`oc: warning: could not save session '${name}' (${err.message}), 'do'/'read'/'next' may act on stale state`);
|
|
54
|
+
}
|
|
55
|
+
};
|
|
31
56
|
|
|
32
57
|
// What browsing costs without this tool is the raw page HTML in context.
|
|
33
58
|
const savings = (out, raw) =>
|
|
@@ -57,13 +82,36 @@ async function main() {
|
|
|
57
82
|
return;
|
|
58
83
|
}
|
|
59
84
|
|
|
85
|
+
const sessionName = values.session || DEFAULT_SESSION;
|
|
86
|
+
// Zero means "whatever this command's default is", which differs: the
|
|
87
|
+
// compact view targets 500 tokens, read targets 2000.
|
|
88
|
+
const asked = values.budget ? Number(values.budget) : 0;
|
|
89
|
+
if (values.budget && (!Number.isFinite(asked) || asked <= 0)) {
|
|
90
|
+
throw new Error('--budget must be a positive number');
|
|
91
|
+
}
|
|
92
|
+
|
|
60
93
|
switch (command) {
|
|
61
94
|
case 'open':
|
|
95
|
+
case 'do':
|
|
62
96
|
case 'raw': {
|
|
63
|
-
|
|
97
|
+
// `do` is `open` with the URL looked up from the last render instead of
|
|
98
|
+
// typed, so both commands share one fetch, render, and save path. `raw`
|
|
99
|
+
// with no URL means the page already open, which is what the compact
|
|
100
|
+
// view's footer offers when it has cut something.
|
|
101
|
+
let url;
|
|
102
|
+
if (command === 'do') {
|
|
103
|
+
const target = act.activate(Number(args[0]), { session: sessionName });
|
|
104
|
+
// A number that points at text has no page behind it, so `do` reads it
|
|
105
|
+
// rather than making the agent pay for a second command to be told.
|
|
106
|
+
if (target.read != null) {
|
|
107
|
+
return console.log(act.read(target.read, { session: sessionName, budget: asked || 2000 }));
|
|
108
|
+
}
|
|
109
|
+
url = target.url;
|
|
110
|
+
} else {
|
|
111
|
+
url = args[0] ?? (command === 'raw' ? loadSession(sessionName)?.url : undefined);
|
|
112
|
+
}
|
|
64
113
|
if (!url) throw new Error(`usage: oc ${command} <url>`);
|
|
65
|
-
const budget =
|
|
66
|
-
if (!Number.isFinite(budget) || budget <= 0) throw new Error('--budget must be a positive number');
|
|
114
|
+
const budget = asked || 500;
|
|
67
115
|
const t0 = performance.now();
|
|
68
116
|
const { url: finalUrl, html, status, via } = await fetchPage(url);
|
|
69
117
|
const fetchMs = performance.now() - t0;
|
|
@@ -74,7 +122,9 @@ async function main() {
|
|
|
74
122
|
+ `${Math.round(html.length / 1024)}KB transferred, ${Math.round(rss / 1048576)}MB memory`;
|
|
75
123
|
};
|
|
76
124
|
if (values.json) {
|
|
77
|
-
|
|
125
|
+
const page = distill(html, finalUrl);
|
|
126
|
+
remember(page, sessionName);
|
|
127
|
+
console.log(JSON.stringify(page));
|
|
78
128
|
if (verbose) console.error(resources());
|
|
79
129
|
return;
|
|
80
130
|
}
|
|
@@ -87,19 +137,19 @@ async function main() {
|
|
|
87
137
|
}
|
|
88
138
|
const page = distill(html, finalUrl);
|
|
89
139
|
const { text, stats } = render(page, { budget });
|
|
140
|
+
remember(page, sessionName, stats.next);
|
|
90
141
|
console.log(text);
|
|
91
142
|
if (verbose) {
|
|
92
143
|
console.error(`~${stats.tokens} tokens, ${stats.rendered}/${stats.blocks} blocks rendered, ${savings(stats.tokens, htmlTokens)}; ${resources()}`);
|
|
93
144
|
}
|
|
94
145
|
return;
|
|
95
146
|
}
|
|
96
|
-
case '
|
|
147
|
+
case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 }));
|
|
148
|
+
case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 }));
|
|
149
|
+
case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 }));
|
|
97
150
|
case 'fill': return act.fill(Number(args[0]), args.slice(1).join(' '));
|
|
98
151
|
case 'submit': return act.submit(args[0] ? Number(args[0]) : undefined);
|
|
99
|
-
case 'read': return act.read(args[0] ? Number(args[0]) : undefined);
|
|
100
|
-
case 'find': return act.find(args.join(' '));
|
|
101
152
|
case 'back': return act.back();
|
|
102
|
-
case 'next': return act.next();
|
|
103
153
|
case 'session': throw new act.NotImplemented('session');
|
|
104
154
|
default:
|
|
105
155
|
throw new Error(`unknown command '${command}', run oc --help`);
|
package/src/distill.js
CHANGED
|
@@ -3,9 +3,9 @@ import TurndownService from 'turndown';
|
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
5
|
* @typedef {Object} Block
|
|
6
|
-
* @property {'heading'|'text'|'link'|'input'|'button'} type
|
|
6
|
+
* @property {'heading'|'text'|'link'|'input'|'button'|'divider'} type
|
|
7
7
|
* @property {string} text
|
|
8
|
-
* @property {number} [n] - action handle
|
|
8
|
+
* @property {number} [n] - action handle
|
|
9
9
|
* @property {number} [level] - heading level 1..6
|
|
10
10
|
* @property {string} [href] - links only
|
|
11
11
|
* @property {string} [name] - inputs only
|
|
@@ -16,6 +16,10 @@ import TurndownService from 'turndown';
|
|
|
16
16
|
* @property {Block[]} blocks
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
+
// Where the compact view cuts a text block. It lives here because numbering
|
|
20
|
+
// depends on it: a block long enough to be cut is a block that needs a handle.
|
|
21
|
+
export const TEXT_CAP = 200;
|
|
22
|
+
|
|
19
23
|
// Dropped wholesale, subtree included. Nav and footer stay in v0.1: on many
|
|
20
24
|
// sites they carry the only working links, and the budget in render.js is
|
|
21
25
|
// what keeps them from costing anything.
|
|
@@ -24,35 +28,87 @@ const DROP = new Set([
|
|
|
24
28
|
'link', 'meta', 'head', 'canvas', 'video', 'audio', 'object',
|
|
25
29
|
]);
|
|
26
30
|
|
|
31
|
+
// Elements that are furniture by definition. The main content is never one of
|
|
32
|
+
// them, so the search below refuses to descend into them.
|
|
33
|
+
const FURNITURE = new Set(['nav', 'header', 'footer', 'aside']);
|
|
34
|
+
|
|
35
|
+
// Elements that end a line on the page and so must end one here. Without this
|
|
36
|
+
// the text of six separate posts merges into a single block, because nothing
|
|
37
|
+
// between them survives distillation to keep them apart.
|
|
38
|
+
const BLOCKY = new Set([
|
|
39
|
+
'p', 'div', 'article', 'section', 'li', 'ul', 'ol', 'tr', 'td', 'th',
|
|
40
|
+
'blockquote', 'pre', 'figure', 'figcaption', 'dt', 'dd', 'form', 'br', 'hr',
|
|
41
|
+
'main', 'nav', 'header', 'footer', 'aside',
|
|
42
|
+
]);
|
|
43
|
+
|
|
44
|
+
// Selectors a page uses to say where its content is. One match is a claim
|
|
45
|
+
// worth believing; several <article> elements mean a listing, where the list
|
|
46
|
+
// is the content and taking the first would throw the rest away.
|
|
47
|
+
const MAIN_SELECTORS = ['main', '[role="main"]', 'article'];
|
|
48
|
+
|
|
49
|
+
// Reordering only matters on a page that cannot be printed whole: if the
|
|
50
|
+
// budget covers everything, what leads is a question with no consequences.
|
|
51
|
+
// Below this much text the page is left in document order.
|
|
52
|
+
const MIN_PAGE = 3000;
|
|
53
|
+
// A candidate has to hold this share of the page's prose to be believed.
|
|
54
|
+
const MIN_SHARE = 0.25;
|
|
55
|
+
// How much of its parent's prose a child must hold before the search moves
|
|
56
|
+
// down into it, and how much of the page's it must keep to be worth choosing.
|
|
57
|
+
const DESCEND_SHARE = 0.6;
|
|
58
|
+
const KEEP_SHARE = 0.5;
|
|
59
|
+
const MAX_DEPTH = 12;
|
|
60
|
+
|
|
61
|
+
// A short link or button label repeated this many times down a page is
|
|
62
|
+
// per-item furniture (permalink, save, reply, like, repost) rather than
|
|
63
|
+
// content. On a forum thread or a social timeline there is one set per item,
|
|
64
|
+
// which costs more than the items do.
|
|
65
|
+
// A link label can be content, so it gets the benefit of the doubt for longer
|
|
66
|
+
// than a button label, which never is.
|
|
67
|
+
const REPEAT_LIMIT = 5;
|
|
68
|
+
const REPEAT_LIMIT_BUTTON = 3;
|
|
69
|
+
const REPEAT_MAX_LEN = 25;
|
|
70
|
+
|
|
27
71
|
const clean = (s) => s.replace(/\s+/g, ' ').trim();
|
|
28
72
|
|
|
29
73
|
/**
|
|
30
74
|
* Reduce raw HTML to an interaction tree: readable text plus numbered
|
|
31
|
-
*
|
|
32
|
-
*
|
|
75
|
+
* elements, in document order. The walk is deterministic and numbering is a
|
|
76
|
+
* second pass over its result, so the same page always yields the same
|
|
77
|
+
* numbers.
|
|
33
78
|
* @param {string} html
|
|
34
79
|
* @param {string} url
|
|
35
80
|
* @returns {Page}
|
|
36
81
|
*/
|
|
37
82
|
export function distill(html, url = '') {
|
|
38
|
-
const { document } = parseHTML(html);
|
|
83
|
+
const { document } = parseHTML(youtubeToHTML(html) ?? transcriptToHTML(html) ?? feedToHTML(html) ?? html);
|
|
39
84
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
40
85
|
/** @type {Block[]} */
|
|
41
86
|
const blocks = [];
|
|
42
|
-
let handle = 0;
|
|
43
87
|
|
|
44
88
|
const hidden = (el) =>
|
|
45
89
|
el.getAttribute('hidden') !== null ||
|
|
46
90
|
el.getAttribute('aria-hidden') === 'true' ||
|
|
47
91
|
/display:\s*none/.test(el.getAttribute('style') ?? '');
|
|
48
92
|
|
|
93
|
+
/** Subtree already emitted, skipped when the rest of the page is walked. */
|
|
94
|
+
let done = null;
|
|
95
|
+
|
|
49
96
|
const walk = (node) => {
|
|
50
97
|
if (node.nodeType === 3) {
|
|
51
|
-
const
|
|
52
|
-
|
|
98
|
+
const raw = node.textContent ?? '';
|
|
99
|
+
const text = clean(raw);
|
|
100
|
+
// A parser is free to split one run of text into several nodes, and
|
|
101
|
+
// linkedom does it around apostrophes, so `what's` arrives as `what`,
|
|
102
|
+
// `'`, `s`. Remembering the parent and whether the fragment had space
|
|
103
|
+
// at its edges is what lets mergeText put it back without inventing a
|
|
104
|
+
// space that was never in the page.
|
|
105
|
+
if (text) {
|
|
106
|
+
blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
|
|
107
|
+
}
|
|
53
108
|
return;
|
|
54
109
|
}
|
|
55
110
|
if (node.nodeType !== 1) return;
|
|
111
|
+
if (node === done) return;
|
|
56
112
|
const tag = node.localName;
|
|
57
113
|
if (DROP.has(tag) || hidden(node)) return;
|
|
58
114
|
|
|
@@ -64,7 +120,7 @@ export function distill(html, url = '') {
|
|
|
64
120
|
if (tag === 'a' && node.getAttribute('href')) {
|
|
65
121
|
const text = clean(node.textContent);
|
|
66
122
|
if (text) {
|
|
67
|
-
blocks.push({ type: 'link',
|
|
123
|
+
blocks.push({ type: 'link', text, href: node.getAttribute('href') });
|
|
68
124
|
}
|
|
69
125
|
return;
|
|
70
126
|
}
|
|
@@ -72,24 +128,164 @@ export function distill(html, url = '') {
|
|
|
72
128
|
const kind = node.getAttribute('type') ?? 'text';
|
|
73
129
|
if (kind === 'hidden') return;
|
|
74
130
|
if (kind === 'submit' || kind === 'button') {
|
|
75
|
-
blocks.push({ type: 'button',
|
|
131
|
+
blocks.push({ type: 'button', text: node.getAttribute('value') ?? 'submit' });
|
|
76
132
|
return;
|
|
77
133
|
}
|
|
78
134
|
const name = node.getAttribute('name') ?? node.getAttribute('placeholder') ?? tag;
|
|
79
|
-
blocks.push({ type: 'input',
|
|
135
|
+
blocks.push({ type: 'input', text: kind, name });
|
|
80
136
|
return;
|
|
81
137
|
}
|
|
82
138
|
if (tag === 'button') {
|
|
83
|
-
|
|
84
|
-
|
|
139
|
+
// An icon button carries its name in aria-label or title, and a button
|
|
140
|
+
// with none of the three cannot be described or pressed, so printing it
|
|
141
|
+
// spends tokens to say nothing. X puts seven of them under every post.
|
|
142
|
+
const text = clean(node.textContent)
|
|
143
|
+
|| clean(node.getAttribute('aria-label') ?? '')
|
|
144
|
+
|| clean(node.getAttribute('title') ?? '');
|
|
145
|
+
if (text) blocks.push({ type: 'button', text });
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
if (BLOCKY.has(tag)) {
|
|
149
|
+
blocks.push({ type: 'break' });
|
|
150
|
+
for (const child of node.childNodes) walk(child);
|
|
151
|
+
blocks.push({ type: 'break' });
|
|
85
152
|
return;
|
|
86
153
|
}
|
|
87
154
|
for (const child of node.childNodes) walk(child);
|
|
88
155
|
};
|
|
89
156
|
|
|
90
157
|
const body = bodyOf(document);
|
|
91
|
-
|
|
92
|
-
|
|
158
|
+
const main = body ? mainOf(document, body) : null;
|
|
159
|
+
if (main) {
|
|
160
|
+
// Content first, everything else after it. The budget in render.js is
|
|
161
|
+
// spent top down, so whatever leads the list is what an agent gets for
|
|
162
|
+
// its first 500 tokens, and on most pages the top of the document is
|
|
163
|
+
// menus. Nothing is dropped: the chrome carries links `oc do` needs, it
|
|
164
|
+
// just stops being the thing that gets read.
|
|
165
|
+
walk(main);
|
|
166
|
+
done = main;
|
|
167
|
+
const contentBlocks = blocks.length;
|
|
168
|
+
if (body) walk(body);
|
|
169
|
+
if (blocks.length > contentBlocks) {
|
|
170
|
+
blocks.splice(contentBlocks, 0, { type: 'divider', text: '--- rest of page: navigation, sidebar, footer ---' });
|
|
171
|
+
}
|
|
172
|
+
} else if (body) {
|
|
173
|
+
walk(body);
|
|
174
|
+
}
|
|
175
|
+
return { url, title, blocks: number(mergeText(dropRepeats(blocks))) };
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Remove link labels that repeat down the page. They are the per-item controls
|
|
180
|
+
* a template stamps onto every row, and on a long thread they outweigh the
|
|
181
|
+
* content. What went is stated in place, because a view that quietly drops
|
|
182
|
+
* things is a view an agent cannot trust.
|
|
183
|
+
* @param {Block[]} blocks
|
|
184
|
+
* @returns {Block[]}
|
|
185
|
+
*/
|
|
186
|
+
function dropRepeats(blocks) {
|
|
187
|
+
const controls = (b) => b.type === 'link' || b.type === 'button';
|
|
188
|
+
/** @type {Map<string, {n: number, limit: number}>} */
|
|
189
|
+
const counts = new Map();
|
|
190
|
+
for (const b of blocks) {
|
|
191
|
+
if (!controls(b) || b.text.length > REPEAT_MAX_LEN) continue;
|
|
192
|
+
const key = b.text.toLowerCase();
|
|
193
|
+
const limit = b.type === 'button' ? REPEAT_LIMIT_BUTTON : REPEAT_LIMIT;
|
|
194
|
+
const seen = counts.get(key);
|
|
195
|
+
// A label worn by both a link and a button keeps the more patient limit.
|
|
196
|
+
counts.set(key, { n: (seen?.n ?? 0) + 1, limit: Math.max(seen?.limit ?? 0, limit) });
|
|
197
|
+
}
|
|
198
|
+
const repeated = new Set([...counts].filter(([, c]) => c.n >= c.limit).map(([k]) => k));
|
|
199
|
+
if (!repeated.size) return blocks;
|
|
200
|
+
const kept = blocks.filter((b) => !(controls(b) && repeated.has(b.text.toLowerCase())));
|
|
201
|
+
const gone = blocks.length - kept.length;
|
|
202
|
+
const names = [...repeated].slice(0, 3).join(', ');
|
|
203
|
+
const note = { type: 'divider', text: `--- ${gone} repeated controls hidden (${names}), 'oc raw' has them ---` };
|
|
204
|
+
// The note belongs with the content it was cut from, not after the chrome.
|
|
205
|
+
const boundary = kept.findIndex((b) => b.type === 'divider');
|
|
206
|
+
kept.splice(boundary === -1 ? kept.length : boundary, 0, note);
|
|
207
|
+
return kept;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/**
|
|
211
|
+
* Find the element holding the page's main content, or null when the page is
|
|
212
|
+
* too small or too flat for the question to have an answer.
|
|
213
|
+
* @param {any} document
|
|
214
|
+
* @param {any} body
|
|
215
|
+
* @returns {any}
|
|
216
|
+
*/
|
|
217
|
+
function mainOf(document, body) {
|
|
218
|
+
if (clean(body.textContent ?? '').length < MIN_PAGE) return null;
|
|
219
|
+
const total = prose(body);
|
|
220
|
+
for (const selector of MAIN_SELECTORS) {
|
|
221
|
+
const found = [...document.querySelectorAll(selector)];
|
|
222
|
+
if (found.length !== 1) continue;
|
|
223
|
+
const el = found[0];
|
|
224
|
+
if (el !== body && prose(el) >= total * MIN_SHARE) return el;
|
|
225
|
+
}
|
|
226
|
+
return densest(body, total);
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Walk down the tree while one child holds most of the prose of the element
|
|
231
|
+
* above it. That is what separates a content column from the page around it
|
|
232
|
+
* without knowing anything about the site.
|
|
233
|
+
* @param {any} body
|
|
234
|
+
* @param {number} total
|
|
235
|
+
* @returns {any}
|
|
236
|
+
*/
|
|
237
|
+
function densest(body, total) {
|
|
238
|
+
let node = body;
|
|
239
|
+
for (let depth = 0; depth < MAX_DEPTH; depth++) {
|
|
240
|
+
let best = null;
|
|
241
|
+
let bestProse = 0;
|
|
242
|
+
for (const child of node.children ?? []) {
|
|
243
|
+
if (FURNITURE.has(child.localName) || DROP.has(child.localName)) continue;
|
|
244
|
+
const len = prose(child);
|
|
245
|
+
if (len > bestProse) {
|
|
246
|
+
best = child;
|
|
247
|
+
bestProse = len;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
if (!best || bestProse < prose(node) * DESCEND_SHARE || bestProse < total * KEEP_SHARE) break;
|
|
251
|
+
node = best;
|
|
252
|
+
}
|
|
253
|
+
return node === body ? null : node;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Text length minus the text of controls, which is what a page is steered by
|
|
258
|
+
* rather than what it says.
|
|
259
|
+
* @param {any} el
|
|
260
|
+
* @returns {number}
|
|
261
|
+
*/
|
|
262
|
+
function prose(el) {
|
|
263
|
+
const text = clean(el.textContent ?? '').length;
|
|
264
|
+
let controls = 0;
|
|
265
|
+
// Link text is what navigation is made of. Option text is worse: a country
|
|
266
|
+
// dropdown is hundreds of characters that no link subtraction touches, and
|
|
267
|
+
// on a search results page it outscores results that are themselves links.
|
|
268
|
+
for (const node of el.querySelectorAll?.('a, select, button') ?? []) {
|
|
269
|
+
controls += clean(node.textContent ?? '').length;
|
|
270
|
+
}
|
|
271
|
+
return text - controls;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
/**
|
|
275
|
+
* Assign handles in document order. Interactive elements get one because they
|
|
276
|
+
* can be acted on, headings and long text blocks because they can be read:
|
|
277
|
+
* a text block over the cap is printed cut, and its number is what makes the
|
|
278
|
+
* rest of it reachable with `oc read <n>` instead of a second whole-page fetch.
|
|
279
|
+
* @param {Block[]} blocks
|
|
280
|
+
* @returns {Block[]}
|
|
281
|
+
*/
|
|
282
|
+
function number(blocks) {
|
|
283
|
+
let handle = 0;
|
|
284
|
+
for (const block of blocks) {
|
|
285
|
+
if (block.type === 'divider') continue;
|
|
286
|
+
if (block.type !== 'text' || block.text.length > TEXT_CAP) block.n = ++handle;
|
|
287
|
+
}
|
|
288
|
+
return blocks;
|
|
93
289
|
}
|
|
94
290
|
|
|
95
291
|
/**
|
|
@@ -107,7 +303,9 @@ const bodyOf = (document) => document.querySelector('body') ?? document.document
|
|
|
107
303
|
* @param {string} html
|
|
108
304
|
*/
|
|
109
305
|
function cleanDocument(html) {
|
|
110
|
-
const { document } = parseHTML(html);
|
|
306
|
+
const { document } = parseHTML(youtubeToHTML(html) ?? transcriptToHTML(html) ?? feedToHTML(html) ?? html);
|
|
307
|
+
// Read the title before the sweep below removes the head with it.
|
|
308
|
+
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
111
309
|
for (const tag of DROP) {
|
|
112
310
|
for (const el of [...document.querySelectorAll(tag)]) el.remove();
|
|
113
311
|
}
|
|
@@ -115,7 +313,7 @@ function cleanDocument(html) {
|
|
|
115
313
|
for (const el of [...document.querySelectorAll('[style]')]) {
|
|
116
314
|
if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
|
|
117
315
|
}
|
|
118
|
-
return document;
|
|
316
|
+
return { document, title };
|
|
119
317
|
}
|
|
120
318
|
|
|
121
319
|
/**
|
|
@@ -125,8 +323,7 @@ function cleanDocument(html) {
|
|
|
125
323
|
* @returns {string}
|
|
126
324
|
*/
|
|
127
325
|
export function toMarkdown(html) {
|
|
128
|
-
const document = cleanDocument(html);
|
|
129
|
-
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
326
|
+
const { document, title } = cleanDocument(html);
|
|
130
327
|
const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
|
|
131
328
|
const el = bodyOf(document);
|
|
132
329
|
const body = el ? turndown.turndown(el.innerHTML).trim() : '';
|
|
@@ -140,11 +337,158 @@ export function toMarkdown(html) {
|
|
|
140
337
|
* @returns {string}
|
|
141
338
|
*/
|
|
142
339
|
export function toHTML(html) {
|
|
143
|
-
const document = cleanDocument(html);
|
|
340
|
+
const { document } = cleanDocument(html);
|
|
144
341
|
const el = bodyOf(document);
|
|
145
342
|
return el ? el.innerHTML.trim() : '';
|
|
146
343
|
}
|
|
147
344
|
|
|
345
|
+
/**
|
|
346
|
+
* Sites behind hard bot challenges often leave their Atom or RSS feeds open:
|
|
347
|
+
* Stack Overflow challenges every HTML page but publishes full question and
|
|
348
|
+
* answer bodies under /feeds. A feed is XML with the real content escaped
|
|
349
|
+
* inside each entry, so this converts one into a plain HTML document and
|
|
350
|
+
* everything downstream stays unchanged. Returns null for non-feed input.
|
|
351
|
+
* @param {string} text
|
|
352
|
+
* @returns {string | null}
|
|
353
|
+
*/
|
|
354
|
+
export function feedToHTML(text) {
|
|
355
|
+
const head = text.slice(0, 2000);
|
|
356
|
+
if (!/<(feed|rss)[\s>]/i.test(head) || /<(html|body)[\s>]/i.test(head)) return null;
|
|
357
|
+
// RSS wraps bodies in CDATA, which an HTML parser reads as a comment and
|
|
358
|
+
// drops. Escaping the section turns it into ordinary text, the same shape
|
|
359
|
+
// Atom feeds already use.
|
|
360
|
+
const xml = text.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, (_, inner) =>
|
|
361
|
+
inner.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'));
|
|
362
|
+
const { document } = parseHTML(xml);
|
|
363
|
+
const root = document.querySelector('feed, rss');
|
|
364
|
+
if (!root) return null;
|
|
365
|
+
const esc = (s) => s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
366
|
+
// Feed elements are unknown to the HTML parser, so self-closed ones like
|
|
367
|
+
// <category /> stay open and swallow their siblings. Descendant queries
|
|
368
|
+
// still land, because a closing </entry> or </item> pops the whole pile.
|
|
369
|
+
const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
|
|
370
|
+
const feedTitle = field(root, 'title');
|
|
371
|
+
const parts = [];
|
|
372
|
+
for (const entry of root.querySelectorAll('entry, item')) {
|
|
373
|
+
const title = field(entry, 'title');
|
|
374
|
+
const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
|
|
375
|
+
?? entry.querySelector('link[href]')?.getAttribute('href')
|
|
376
|
+
?? field(entry, 'guid');
|
|
377
|
+
const author = field(entry, 'author name') || field(entry, 'author');
|
|
378
|
+
const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
|
|
379
|
+
const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
|
|
380
|
+
// Atom escapes the entry body, so textContent of content/summary is the
|
|
381
|
+
// HTML itself, ready to be embedded and parsed like any page.
|
|
382
|
+
const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? '';
|
|
383
|
+
parts.push('<article>');
|
|
384
|
+
if (title) parts.push(`<h2>${esc(title)}</h2>`);
|
|
385
|
+
if (byline || href) {
|
|
386
|
+
parts.push(`<p>${esc(byline)}${href ? ` <a href="${esc(href)}">open</a>` : ''}</p>`);
|
|
387
|
+
}
|
|
388
|
+
parts.push(body, '</article>');
|
|
389
|
+
}
|
|
390
|
+
// A full skeleton, because linkedom treats the first element of a bare
|
|
391
|
+
// multi-rooted fragment as the whole document and drops its siblings.
|
|
392
|
+
return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const escHTML = (s) => String(s ?? '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
396
|
+
|
|
397
|
+
/**
|
|
398
|
+
* Pull one embedded JSON object out of a page by its variable name, scanning
|
|
399
|
+
* forward from the first `{` and tracking string/escape state so a brace
|
|
400
|
+
* inside a quoted value never ends the object early. A regex can't do this
|
|
401
|
+
* safely because the JSON itself contains braces.
|
|
402
|
+
* @param {string} text
|
|
403
|
+
* @param {string} marker
|
|
404
|
+
* @returns {any}
|
|
405
|
+
*/
|
|
406
|
+
function extractJSON(text, marker) {
|
|
407
|
+
const at = text.indexOf(marker);
|
|
408
|
+
if (at === -1) return null;
|
|
409
|
+
const start = text.indexOf('{', at);
|
|
410
|
+
if (start === -1) return null;
|
|
411
|
+
let depth = 0, inString = false, escaped = false;
|
|
412
|
+
for (let i = start; i < text.length; i++) {
|
|
413
|
+
const c = text[i];
|
|
414
|
+
if (inString) {
|
|
415
|
+
if (escaped) escaped = false;
|
|
416
|
+
else if (c === '\\') escaped = true;
|
|
417
|
+
else if (c === '"') inString = false;
|
|
418
|
+
continue;
|
|
419
|
+
}
|
|
420
|
+
if (c === '"') inString = true;
|
|
421
|
+
else if (c === '{') depth++;
|
|
422
|
+
else if (c === '}' && --depth === 0) {
|
|
423
|
+
try {
|
|
424
|
+
return JSON.parse(text.slice(start, i + 1));
|
|
425
|
+
} catch {
|
|
426
|
+
return null;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
return null;
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* A YouTube watch page needs client JS to become interactive, but the initial
|
|
435
|
+
* HTML response already carries the title, description, view count, and
|
|
436
|
+
* caption tracks inline as `ytInitialPlayerResponse`, so none of that has to
|
|
437
|
+
* wait on the v0.3 headless fallback. Each caption track becomes a link to
|
|
438
|
+
* its timedtext URL, which `oc do` follows and transcriptToHTML turns into
|
|
439
|
+
* readable text. Returns null for anything that isn't a watch page.
|
|
440
|
+
* @param {string} text
|
|
441
|
+
* @returns {string | null}
|
|
442
|
+
*/
|
|
443
|
+
export function youtubeToHTML(text) {
|
|
444
|
+
const data = extractJSON(text, 'ytInitialPlayerResponse');
|
|
445
|
+
const details = data?.videoDetails;
|
|
446
|
+
if (!details?.title) return null;
|
|
447
|
+
const micro = data.microformat?.playerMicroformatRenderer;
|
|
448
|
+
const tracks = data.captions?.playerCaptionsTracklistRenderer?.captionTracks ?? [];
|
|
449
|
+
const views = details.viewCount ? `${Number(details.viewCount).toLocaleString('en-US')} views` : '';
|
|
450
|
+
const byline = [details.author && `by ${details.author}`, views, micro?.publishDate].filter(Boolean).join(', ');
|
|
451
|
+
const channelHref = details.channelId ? `https://www.youtube.com/channel/${details.channelId}` : '';
|
|
452
|
+
const parts = [`<h1>${escHTML(details.title)}</h1>`];
|
|
453
|
+
if (byline || channelHref) {
|
|
454
|
+
parts.push(`<p>${escHTML(byline)}${channelHref ? ` <a href="${escHTML(channelHref)}">channel</a>` : ''}</p>`);
|
|
455
|
+
}
|
|
456
|
+
const description = details.shortDescription ?? micro?.description?.simpleText ?? '';
|
|
457
|
+
for (const para of description.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean)) {
|
|
458
|
+
parts.push(`<p>${escHTML(para)}</p>`);
|
|
459
|
+
}
|
|
460
|
+
for (const track of tracks) {
|
|
461
|
+
const label = track.name?.simpleText || track.languageCode || 'transcript';
|
|
462
|
+
const kind = track.kind === 'asr' ? ' (auto-generated)' : '';
|
|
463
|
+
parts.push(`<p><a href="${escHTML(track.baseUrl)}">transcript: ${escHTML(label)}${kind}</a></p>`);
|
|
464
|
+
}
|
|
465
|
+
return `<html><head><title>${escHTML(details.title)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* YouTube's timedtext endpoint answers plain XML, one `<text>` cue per
|
|
470
|
+
* caption line. Concatenated into a single block rather than one paragraph
|
|
471
|
+
* per cue, so a whole transcript pages through `oc next`/`oc read` like any
|
|
472
|
+
* other long document instead of costing one numbered block per few words.
|
|
473
|
+
* Consecutive duplicate cues (auto captions repeat words across overlapping
|
|
474
|
+
* segments) are dropped. Returns null for anything that isn't a transcript.
|
|
475
|
+
* @param {string} text
|
|
476
|
+
* @returns {string | null}
|
|
477
|
+
*/
|
|
478
|
+
export function transcriptToHTML(text) {
|
|
479
|
+
if (!/<transcript[\s>]/i.test(text.slice(0, 200))) return null;
|
|
480
|
+
const { document } = parseHTML(text);
|
|
481
|
+
const cues = [...document.querySelectorAll('text')];
|
|
482
|
+
if (!cues.length) return null;
|
|
483
|
+
const lines = [];
|
|
484
|
+
for (const cue of cues) {
|
|
485
|
+
const line = clean(cue.textContent ?? '');
|
|
486
|
+
if (line && line !== lines[lines.length - 1]) lines.push(line);
|
|
487
|
+
}
|
|
488
|
+
if (!lines.length) return null;
|
|
489
|
+
return `<html><head><title>Transcript</title></head><body>\n<p>${escHTML(lines.join(' '))}</p>\n</body></html>`;
|
|
490
|
+
}
|
|
491
|
+
|
|
148
492
|
/**
|
|
149
493
|
* Adjacent text nodes arrive fragmented (one per inline element boundary).
|
|
150
494
|
* Merging them is what turns DOM noise into readable lines.
|
|
@@ -155,13 +499,27 @@ function mergeText(blocks) {
|
|
|
155
499
|
/** @type {Block[]} */
|
|
156
500
|
const out = [];
|
|
157
501
|
for (const b of blocks) {
|
|
502
|
+
if (b.type === 'break') {
|
|
503
|
+
// A boundary only has to stop the merge, and it does that by being the
|
|
504
|
+
// last thing in the list when the next text block arrives.
|
|
505
|
+
if (out[out.length - 1]?.type === 'text') out.push(b);
|
|
506
|
+
continue;
|
|
507
|
+
}
|
|
158
508
|
const prev = out[out.length - 1];
|
|
159
509
|
if (b.type === 'text' && prev?.type === 'text') {
|
|
160
|
-
|
|
510
|
+
// Two fragments of one element with no whitespace between them were one
|
|
511
|
+
// word before the parser split them. Anything else was separated on the
|
|
512
|
+
// page and stays separated here.
|
|
513
|
+
const glued = prev.host && prev.host === b.host && !prev.post && !b.pre;
|
|
514
|
+
prev.text = glued ? `${prev.text}${b.text}` : `${prev.text} ${b.text}`;
|
|
515
|
+
prev.post = b.post;
|
|
161
516
|
} else {
|
|
162
517
|
out.push(b);
|
|
163
518
|
}
|
|
164
519
|
}
|
|
165
520
|
// Single stray characters (list bullets, separators) cost tokens and say nothing.
|
|
166
|
-
return out
|
|
521
|
+
return out
|
|
522
|
+
.filter((b) => b.type !== 'break')
|
|
523
|
+
.filter((b) => b.type !== 'text' || b.text.length > 1)
|
|
524
|
+
.map(({ host, pre, post, ...block }) => block);
|
|
167
525
|
}
|