@only-cli/oc 0.5.7 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/clis/caniuse.com.json +7 -0
- package/clis/codeberg.org.json +13 -0
- package/clis/crates.io.json +9 -0
- package/clis/discourse.nixos.org.json +8 -0
- package/clis/docs.astral.sh.json +8 -0
- package/clis/docs.pola.rs.json +6 -0
- package/clis/endoflife.date.json +8 -0
- package/clis/forums.swift.org.json +8 -0
- package/clis/github.com.json +6 -1
- package/clis/gitlab.com.json +13 -0
- package/clis/hub.docker.com.json +9 -0
- package/clis/man7.org.json +7 -0
- package/clis/npmjs.com.json +9 -0
- package/clis/osv.dev.json +6 -0
- package/clis/pypi.org.json +8 -0
- package/clis/rfc-editor.org.json +7 -0
- package/clis/rubygems.org.json +8 -0
- package/clis/squidfunk.github.io.json +6 -0
- package/clis/tldr.sh.json +9 -0
- package/clis/users.rust-lang.org.json +8 -0
- package/clis/www.mkdocs.org.json +6 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +11 -11
- package/src/act.js +14 -6
- package/src/apisearch.js +20 -10
- package/src/cache.js +9 -4
- package/src/cli.js +25 -8
- package/src/distill.js +382 -26
- package/src/fetch.js +34 -17
- package/src/mkdocs.js +190 -0
- package/src/nodedocs.js +15 -7
- package/src/render.js +7 -3
- package/src/sites.js +31 -11
package/src/cli.js
CHANGED
|
@@ -8,6 +8,7 @@ import { resolveSite, listSites } from './sites.js';
|
|
|
8
8
|
import { sphinxSearch } from './sphinx.js';
|
|
9
9
|
import { nodeSearch } from './nodedocs.js';
|
|
10
10
|
import { rdocSearch } from './rdoc.js';
|
|
11
|
+
import { mkdocsSearch } from './mkdocs.js';
|
|
11
12
|
import { apiSearch } from './apisearch.js';
|
|
12
13
|
import * as act from './act.js';
|
|
13
14
|
import { DEFAULT_SESSION, assertSafeName, clearSession, listSessions, loadSession, saveSession, sessionFromPage } from './session.js';
|
|
@@ -62,6 +63,7 @@ flags:
|
|
|
62
63
|
--domain <host> login only: the hostname those cookies belong to
|
|
63
64
|
--expires <dur> login only: how long the session lasts (default 1h)
|
|
64
65
|
--allow-http login only: let these cookies travel over plain http
|
|
66
|
+
--version, -V print the installed version and exit
|
|
65
67
|
|
|
66
68
|
Authenticated pages: run 'printf %s "session=..." | oc login --cookie - --domain
|
|
67
69
|
example.com' to seed cookies for a session (default lifetime 1h, override with
|
|
@@ -173,9 +175,18 @@ async function main() {
|
|
|
173
175
|
expires: { type: 'string' },
|
|
174
176
|
'allow-http': { type: 'boolean', default: false },
|
|
175
177
|
help: { type: 'boolean', short: 'h', default: false },
|
|
178
|
+
version: { type: 'boolean', short: 'V', default: false },
|
|
176
179
|
},
|
|
177
180
|
});
|
|
178
181
|
|
|
182
|
+
// Read at call time rather than baked in, so the number can never drift from
|
|
183
|
+
// the package that npm actually installed.
|
|
184
|
+
if (values.version) {
|
|
185
|
+
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
|
186
|
+
console.log(pkg.version);
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
|
|
179
190
|
// Metrics cost tokens too, so they are opt-in: agents pass --verbose only
|
|
180
191
|
// when their own verbose mode is on, or the user exports OC_VERBOSE=1.
|
|
181
192
|
const verbose = values.stats || values.verbose || process.env.OC_VERBOSE === '1';
|
|
@@ -190,6 +201,11 @@ async function main() {
|
|
|
190
201
|
// definition for, and a shortcut is only ever a URL, so it resolves to one
|
|
191
202
|
// here and the rest of this function never learns it was not typed.
|
|
192
203
|
let search = null;
|
|
204
|
+
// The fields a shortcut chose to show of a JSON response, if it chose any.
|
|
205
|
+
// A view is a deliberate cut, so a thin render of a heavy response is the
|
|
206
|
+
// choice working, not a gated page: the thin-content verdict stays out of
|
|
207
|
+
// it, and only a render with no text at all still fails.
|
|
208
|
+
let view = null;
|
|
193
209
|
if (!COMMANDS.has(command)) {
|
|
194
210
|
const site = resolveSite(command, args);
|
|
195
211
|
if (!site) throw new Error(`unknown command '${command}', run oc --help`);
|
|
@@ -199,6 +215,7 @@ async function main() {
|
|
|
199
215
|
command = 'search';
|
|
200
216
|
} else {
|
|
201
217
|
args = [site.url];
|
|
218
|
+
view = site.view ?? null;
|
|
202
219
|
command = 'open';
|
|
203
220
|
}
|
|
204
221
|
}
|
|
@@ -265,7 +282,7 @@ async function main() {
|
|
|
265
282
|
}
|
|
266
283
|
const jar = jarData ? createJarHandle(sessionName, jarData) : null;
|
|
267
284
|
const t0 = performance.now();
|
|
268
|
-
const { url: finalUrl, html, status, via } = await fetchPage(url, { jar: jar ?? undefined });
|
|
285
|
+
const { url: finalUrl, html, status, via, type } = await fetchPage(url, { jar: jar ?? undefined });
|
|
269
286
|
if (jar) saveCookieJar(sessionName, jar.toJSON());
|
|
270
287
|
const fetchMs = performance.now() - t0;
|
|
271
288
|
const resources = () => {
|
|
@@ -276,9 +293,9 @@ async function main() {
|
|
|
276
293
|
};
|
|
277
294
|
const htmlTokens = estimateTokens(html);
|
|
278
295
|
if (values.json) {
|
|
279
|
-
const page = distill(html, finalUrl);
|
|
296
|
+
const page = distill(html, finalUrl, { view, type });
|
|
280
297
|
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
281
|
-
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
|
298
|
+
const failure = auth ?? contentFailure(contentTokens(page), view ? 0 : htmlTokens);
|
|
282
299
|
// Always present, so a caller can branch on the field rather than on
|
|
283
300
|
// whether a field it was hoping for turned up.
|
|
284
301
|
console.log(JSON.stringify({ ...page, empty: failure != null }));
|
|
@@ -293,14 +310,14 @@ async function main() {
|
|
|
293
310
|
return;
|
|
294
311
|
}
|
|
295
312
|
if (command === 'raw') {
|
|
296
|
-
const page = distill(html, finalUrl);
|
|
313
|
+
const page = distill(html, finalUrl, { type });
|
|
297
314
|
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
298
315
|
if (auth) {
|
|
299
316
|
if (jar) clearCookieJar(sessionName);
|
|
300
317
|
noContent(finalUrl, auth);
|
|
301
318
|
return;
|
|
302
319
|
}
|
|
303
|
-
const out = values.html ? toHTML(html, finalUrl) : toMarkdown(html, finalUrl);
|
|
320
|
+
const out = values.html ? toHTML(html, finalUrl, { type }) : toMarkdown(html, finalUrl, { type });
|
|
304
321
|
const outTokens = estimateTokens(out);
|
|
305
322
|
console.log(out);
|
|
306
323
|
if (verbose) {
|
|
@@ -320,9 +337,9 @@ async function main() {
|
|
|
320
337
|
}
|
|
321
338
|
return;
|
|
322
339
|
}
|
|
323
|
-
const page = distill(html, finalUrl);
|
|
340
|
+
const page = distill(html, finalUrl, { view, type });
|
|
324
341
|
const auth = authFailure(page, finalUrl, { hadAuth });
|
|
325
|
-
const failure = auth ?? contentFailure(contentTokens(page), htmlTokens);
|
|
342
|
+
const failure = auth ?? contentFailure(contentTokens(page), view ? 0 : htmlTokens);
|
|
326
343
|
if (auth) {
|
|
327
344
|
if (jar) clearCookieJar(sessionName);
|
|
328
345
|
noContent(finalUrl, auth);
|
|
@@ -350,7 +367,7 @@ async function main() {
|
|
|
350
367
|
// follows a result. Only the list is ever printed; the index, corpus,
|
|
351
368
|
// and response stay out of context.
|
|
352
369
|
const t0 = performance.now();
|
|
353
|
-
const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch };
|
|
370
|
+
const local = { sphinx: sphinxSearch, nodedoc: nodeSearch, rdoc: rdocSearch, mkdocs: mkdocsSearch };
|
|
354
371
|
const kind = Object.keys(local).find((k) => search[k]);
|
|
355
372
|
const { url, html, via } = kind
|
|
356
373
|
? await local[kind](search[kind], search.query)
|
package/src/distill.js
CHANGED
|
@@ -15,6 +15,12 @@ import TurndownService from 'turndown';
|
|
|
15
15
|
* @property {string} title
|
|
16
16
|
* @property {Block[]} blocks
|
|
17
17
|
* @property {string} [base] - document base URL from <base href>, if the page set one
|
|
18
|
+
*
|
|
19
|
+
* @typedef {Object} View
|
|
20
|
+
* @property {string[]} keep - fields to show, in order: a dot path
|
|
21
|
+
* ('info.version') or a template joining several ('{crate_id} {req}')
|
|
22
|
+
* @property {string} [list] - dot path to the array the fields apply to each
|
|
23
|
+
* item of; absent means the response itself, whether one object or a list
|
|
18
24
|
*/
|
|
19
25
|
|
|
20
26
|
// Where the compact view cuts a text block. It lives here because numbering
|
|
@@ -107,7 +113,112 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim();
|
|
|
107
113
|
* @returns {string}
|
|
108
114
|
*/
|
|
109
115
|
const asHTML = (text, url = '', opts = {}) =>
|
|
110
|
-
jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text)
|
|
116
|
+
jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text)
|
|
117
|
+
?? textToHTML(text, url, opts) ?? text;
|
|
118
|
+
|
|
119
|
+
// What makes a body HTML when it does not open with a tag: one of the tags
|
|
120
|
+
// every page has somewhere near the top. A tag name alone is not enough,
|
|
121
|
+
// since `Record<string, T>` and `a <b` look like tags to a regex too.
|
|
122
|
+
const HTML_MARK = /<(!doctype|html|head|body|div|p|br|a href|span|script|style|meta|link|title|h[1-6]|table|ul|ol|li|img|article|main|section|nav|form|input|pre)[\s>/]/i;
|
|
123
|
+
|
|
124
|
+
const TEXT_PATH = /\.(md|markdown|txt|rst|adoc)$/i;
|
|
125
|
+
const pathOf = (url) => {
|
|
126
|
+
try {
|
|
127
|
+
return new URL(url).pathname;
|
|
128
|
+
} catch {
|
|
129
|
+
return '';
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
// One line of a Markdown link list: `- [title](url)`, then an optional
|
|
134
|
+
// description. llms.txt is made of these. A title may hold a bracketed
|
|
135
|
+
// word of its own, as Markdown allows: drizzle's reads `[Joins [SQL]]`.
|
|
136
|
+
const LINK_ITEM = /^[ \t]*[-*+][ \t]+\[((?:[^[\]\n]|\[[^[\]\n]*\])+)\]\(([^()\s]+)\)(.*)$/;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* A paragraph that is nothing but a Markdown link list, as a list of real
|
|
140
|
+
* links, so `do <n>` follows an entry of an llms.txt index or a README's
|
|
141
|
+
* table of links. Anything else in the paragraph keeps it verbatim.
|
|
142
|
+
* @param {string} p
|
|
143
|
+
* @param {string} base
|
|
144
|
+
* @returns {string | null}
|
|
145
|
+
*/
|
|
146
|
+
function linkList(p, base) {
|
|
147
|
+
const items = [];
|
|
148
|
+
for (const line of p.split('\n')) {
|
|
149
|
+
const m = line.match(LINK_ITEM);
|
|
150
|
+
if (!m) return null;
|
|
151
|
+
let href;
|
|
152
|
+
try {
|
|
153
|
+
href = new URL(m[2], base);
|
|
154
|
+
} catch {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
if (href.protocol !== 'https:' && href.protocol !== 'http:') return null;
|
|
158
|
+
// The description prints on a line of its own under the link, where the
|
|
159
|
+
// `: ` that joined it to the title reads as noise.
|
|
160
|
+
const text = m[3].replace(/^\s*[:-]\s*/, '').trim();
|
|
161
|
+
items.push(`<li><a href="${escHTML(href.href)}">${escHTML(m[1])}</a>${text ? ` ${escHTML(text)}` : ''}</li>`);
|
|
162
|
+
}
|
|
163
|
+
return `<ul>${items.join('')}</ul>`;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* A plain text body as a page: a source file, a manual page, a README
|
|
168
|
+
* fetched raw. The HTML parser eats everything from a `<` to the next `>`,
|
|
169
|
+
* so a JavaScript file lost its generics and comparisons and came out as
|
|
170
|
+
* one shuffled block. Each blank-line-separated paragraph becomes its own
|
|
171
|
+
* block, keeping its line breaks, and a Markdown heading becomes a heading,
|
|
172
|
+
* so `find` and `read` work on a tldr page or an RFC the way they do on a
|
|
173
|
+
* page. The raw modes keep the whole file as one block, verbatim.
|
|
174
|
+
* @param {string} text
|
|
175
|
+
* @param {string} url
|
|
176
|
+
* @param {{full?: boolean, type?: string}} [opts] - type is the declared Content-Type
|
|
177
|
+
* @returns {string | null}
|
|
178
|
+
*/
|
|
179
|
+
function textToHTML(text, url = '', { full = false, type = '' } = {}) {
|
|
180
|
+
const lead = text.replace(/^\uFEFF/, '').trimStart();
|
|
181
|
+
if (!lead || /^<(!doctype|html)[\s>]/i.test(lead)) return null;
|
|
182
|
+
// A README often opens with a centred logo in a <div>, and read as HTML
|
|
183
|
+
// everything after it was loose text the page reader dropped. A path that
|
|
184
|
+
// names a text format, or a server that says the body is text, is text
|
|
185
|
+
// unless it is a whole HTML document: a forum post quoting a <br> is not
|
|
186
|
+
// a page.
|
|
187
|
+
const declared = /^text\/(plain|markdown)\b/i.test(type) || TEXT_PATH.test(pathOf(url));
|
|
188
|
+
if (!declared && (lead.startsWith('<') || HTML_MARK.test(lead.slice(0, 4000)))) return null;
|
|
189
|
+
const body = lead.replace(/\r\n?/g, '\n').trimEnd();
|
|
190
|
+
const paragraphs = full ? [body] : body.split(/\n[ \t]*\n+/);
|
|
191
|
+
const heading = (p) => /^#{1,6} \S/.test(p) && !p.includes('\n') ? p.match(/^#+/)[0].length : 0;
|
|
192
|
+
const parts = paragraphs.map((p) => {
|
|
193
|
+
const level = heading(p);
|
|
194
|
+
if (level) return `<h${level}>${escHTML(p.slice(level + 1))}</h${level}>`;
|
|
195
|
+
if (full) return `<pre>${escHTML(p)}</pre>`;
|
|
196
|
+
// Expo's llms.txt writes each section heading on the line right above
|
|
197
|
+
// its list, with no blank line between, so the two arrive as one
|
|
198
|
+
// paragraph. The heading line is then a heading and the rest a list.
|
|
199
|
+
const cut = p.indexOf('\n');
|
|
200
|
+
const top = cut > 0 ? heading(p.slice(0, cut)) : 0;
|
|
201
|
+
const list = linkList(top ? p.slice(cut + 1) : p, url);
|
|
202
|
+
if (!list) return `<pre>${escHTML(p)}</pre>`;
|
|
203
|
+
return top ? `<h${top}>${escHTML(p.slice(top + 1, cut))}</h${top}>${list}` : list;
|
|
204
|
+
});
|
|
205
|
+
// Only a heading that opens the file names it. A README that starts with
|
|
206
|
+
// prose and has a `# More information` section further down is not titled
|
|
207
|
+
// "More information". A Setext heading (a line of = under the text) counts.
|
|
208
|
+
const opening = paragraphs[0] ?? '';
|
|
209
|
+
const setext = opening.match(/^([^\n]+)\n=+[ \t]*(?:\n|$)/);
|
|
210
|
+
let title = heading(opening) ? opening.replace(/^#+ /, '') : setext ? setext[1].trim() : '';
|
|
211
|
+
if (!title) {
|
|
212
|
+
try {
|
|
213
|
+
title = `${new URL(url).host}${new URL(url).pathname}`;
|
|
214
|
+
} catch {
|
|
215
|
+
title = 'text';
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
// The raw modes find the file again by this mark and hand it back whole.
|
|
219
|
+
const main = full ? '<main data-text>' : '<main>';
|
|
220
|
+
return `<html><head><title>${escHTML(title)}</title></head><body>${main}\n${parts.join('\n')}\n</main></body></html>`;
|
|
221
|
+
}
|
|
111
222
|
|
|
112
223
|
/**
|
|
113
224
|
* The link a heading is, if it is one. A search engine puts the result title
|
|
@@ -166,10 +277,14 @@ function isFollowableHref(href, base, pageUrl) {
|
|
|
166
277
|
* numbers.
|
|
167
278
|
* @param {string} html
|
|
168
279
|
* @param {string} url
|
|
280
|
+
* @param {{view?: View|null, type?: string}} [opts] - the Content-Type the
|
|
281
|
+
* server declared, and a site shortcut's choice of JSON fields,
|
|
282
|
+
* which only a JSON body reads
|
|
169
283
|
* @returns {Page}
|
|
170
284
|
*/
|
|
171
|
-
export function distill(html, url = '') {
|
|
172
|
-
const { document } = parseHTML(asHTML(html, url));
|
|
285
|
+
export function distill(html, url = '', { view = null, type = '' } = {}) {
|
|
286
|
+
const { document } = parseHTML(asHTML(html, url, { view, type }));
|
|
287
|
+
lowerAttributeNames(document);
|
|
173
288
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
174
289
|
const base = documentBase(document, url);
|
|
175
290
|
/** @type {Block[]} */
|
|
@@ -178,7 +293,7 @@ export function distill(html, url = '') {
|
|
|
178
293
|
const hidden = (el) =>
|
|
179
294
|
el.getAttribute('hidden') !== null ||
|
|
180
295
|
el.getAttribute('aria-hidden') === 'true' ||
|
|
181
|
-
|
|
296
|
+
DISPLAY_NONE.test(el.getAttribute('style') ?? '');
|
|
182
297
|
|
|
183
298
|
/** Subtree already emitted, skipped when the rest of the page is walked. */
|
|
184
299
|
let done = null;
|
|
@@ -233,6 +348,12 @@ export function distill(html, url = '') {
|
|
|
233
348
|
// space that was never in the page.
|
|
234
349
|
if (text) {
|
|
235
350
|
blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
|
|
351
|
+
} else if (raw && blocks.at(-1)?.type === 'text') {
|
|
352
|
+
// A node that is only a space still separates the words either side
|
|
353
|
+
// of it. Wikipedia writes every as a span holding nothing
|
|
354
|
+
// else, and the fragments around it share a parent, so without this
|
|
355
|
+
// an infobox read "Firstappeared" and "20February 1991".
|
|
356
|
+
blocks.at(-1).post = true;
|
|
236
357
|
}
|
|
237
358
|
return;
|
|
238
359
|
}
|
|
@@ -250,14 +371,13 @@ export function distill(html, url = '') {
|
|
|
250
371
|
return;
|
|
251
372
|
}
|
|
252
373
|
if (tag === 'a' && node.getAttribute('href')) {
|
|
253
|
-
const href = node.getAttribute('href');
|
|
254
374
|
// Same rule as headingHref: a fragment is this page, mailto: is not a
|
|
255
|
-
// page.
|
|
256
|
-
//
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
375
|
+
// page. The link keeps its number without the href, because a fragment
|
|
376
|
+
// is often the one handle on a place: rdoc's method signature links to
|
|
377
|
+
// its own id, and as plain text find borrowed the number above it,
|
|
378
|
+
// which read then opened on the method before.
|
|
379
|
+
const raw = node.getAttribute('href');
|
|
380
|
+
const href = isFollowableHref(raw, base, url) ? raw : null;
|
|
261
381
|
// An icon button already takes aria-label or title because it has no
|
|
262
382
|
// text. An anchor around an image is the same shape: img alt is the
|
|
263
383
|
// accessible name and is not a text node, so textContent is empty and
|
|
@@ -268,7 +388,7 @@ export function distill(html, url = '') {
|
|
|
268
388
|
|| clean(node.getAttribute('title') ?? '')
|
|
269
389
|
|| clean(node.querySelector('img')?.getAttribute('alt') ?? '');
|
|
270
390
|
if (text) {
|
|
271
|
-
blocks.push({ type: 'link', text, href });
|
|
391
|
+
blocks.push({ type: 'link', text, ...(href ? { href } : {}) });
|
|
272
392
|
}
|
|
273
393
|
return;
|
|
274
394
|
}
|
|
@@ -414,7 +534,14 @@ function mainOf(document, body) {
|
|
|
414
534
|
const found = [...document.querySelectorAll(selector)];
|
|
415
535
|
if (found.length !== 1) continue;
|
|
416
536
|
const el = found[0];
|
|
417
|
-
if (el
|
|
537
|
+
if (el === body || prose(el) < total * MIN_SHARE) continue;
|
|
538
|
+
// Material for MkDocs puts both sidebars, the page nav and the table of
|
|
539
|
+
// contents, inside <main>, so a docs page led with fifty nav links. A
|
|
540
|
+
// main that holds navigation and exactly one article holding most of its
|
|
541
|
+
// prose leads with the article; the rest still follows. Walking down to
|
|
542
|
+
// the densest column instead cut pkg.go.dev to its biggest type.
|
|
543
|
+
const articles = el.querySelector('nav') ? el.querySelectorAll('article') : [];
|
|
544
|
+
return articles.length === 1 && prose(articles[0]) >= prose(el) * KEEP_SHARE ? articles[0] : el;
|
|
418
545
|
}
|
|
419
546
|
return densest(body, total);
|
|
420
547
|
}
|
|
@@ -489,16 +616,44 @@ function number(blocks) {
|
|
|
489
616
|
*/
|
|
490
617
|
const bodyOf = (document) => document.querySelector('body') ?? document.documentElement;
|
|
491
618
|
|
|
619
|
+
/** CSS property names and keywords are case-insensitive: `DISPLAY : NONE` hides too. */
|
|
620
|
+
const DISPLAY_NONE = /display\s*:\s*none/i;
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* HTML attribute names are ASCII case-insensitive, but linkedom keeps them as
|
|
624
|
+
* written, so `<A HREF>` had no href, `<INPUT TYPE="hidden">` no type and
|
|
625
|
+
* `<P STYLE="display:none">` no style. Lowercase them once after parsing so
|
|
626
|
+
* every lookup below sees what a browser sees. As in a browser, the first of
|
|
627
|
+
* two spellings of one attribute wins.
|
|
628
|
+
* @param {Document} document
|
|
629
|
+
*/
|
|
630
|
+
function lowerAttributeNames(document) {
|
|
631
|
+
for (const el of document.querySelectorAll('*')) {
|
|
632
|
+
const attributes = [...el.attributes];
|
|
633
|
+
if (attributes.every(({ name }) => name === name.toLowerCase())) continue;
|
|
634
|
+
const first = new Map();
|
|
635
|
+
for (const { name, value } of attributes) {
|
|
636
|
+
const lower = name.toLowerCase();
|
|
637
|
+
if (!first.has(lower)) first.set(lower, value);
|
|
638
|
+
}
|
|
639
|
+
for (const { name } of attributes) el.removeAttribute(name);
|
|
640
|
+
for (const [name, value] of first) el.setAttribute(name, value);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
492
644
|
/**
|
|
493
645
|
* Shared cleanup for the raw modes: parse, then delete the same noise
|
|
494
646
|
* distill() skips, so neither raw output ever leaks scripts, styles, or
|
|
495
647
|
* hidden content.
|
|
496
648
|
* @param {string} html
|
|
649
|
+
* @param {string} [url]
|
|
650
|
+
* @param {string} [type] - the declared Content-Type
|
|
497
651
|
*/
|
|
498
|
-
function cleanDocument(html, url = '') {
|
|
652
|
+
function cleanDocument(html, url = '', type = '') {
|
|
499
653
|
// Raw is the mode an agent reaches for when the compact view left something
|
|
500
654
|
// out, so it is the one place a JSON response keeps every field.
|
|
501
|
-
const { document } = parseHTML(asHTML(html, url, { full: true }));
|
|
655
|
+
const { document } = parseHTML(asHTML(html, url, { full: true, type }));
|
|
656
|
+
lowerAttributeNames(document);
|
|
502
657
|
// Read the title before the sweep below removes the head with it.
|
|
503
658
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
504
659
|
for (const tag of DROP) {
|
|
@@ -509,7 +664,7 @@ function cleanDocument(html, url = '') {
|
|
|
509
664
|
if ((el.getAttribute('type') ?? '').toLowerCase() === 'hidden') el.remove();
|
|
510
665
|
}
|
|
511
666
|
for (const el of [...document.querySelectorAll('[style]')]) {
|
|
512
|
-
if (
|
|
667
|
+
if (DISPLAY_NONE.test(el.getAttribute('style') ?? '')) el.remove();
|
|
513
668
|
}
|
|
514
669
|
return { document, title };
|
|
515
670
|
}
|
|
@@ -520,10 +675,15 @@ function cleanDocument(html, url = '') {
|
|
|
520
675
|
* @param {string} html
|
|
521
676
|
* @param {string} url - only read when the body turns out to be JSON, whose
|
|
522
677
|
* title has to come from the endpoint because the payload has none
|
|
678
|
+
* @param {{type?: string}} [opts] - the declared Content-Type
|
|
523
679
|
* @returns {string}
|
|
524
680
|
*/
|
|
525
|
-
export function toMarkdown(html, url = '') {
|
|
526
|
-
const { document, title } = cleanDocument(html, url);
|
|
681
|
+
export function toMarkdown(html, url = '', { type = '' } = {}) {
|
|
682
|
+
const { document, title } = cleanDocument(html, url, type);
|
|
683
|
+
// A plain text file is already the markdown of itself. Run through the
|
|
684
|
+
// converter it came back with every `#` and backtick escaped.
|
|
685
|
+
const file = document.querySelector('main[data-text] > pre');
|
|
686
|
+
if (file) return file.textContent;
|
|
527
687
|
const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
|
|
528
688
|
const el = bodyOf(document);
|
|
529
689
|
const body = el ? turndown.turndown(el.innerHTML).trim() : '';
|
|
@@ -535,10 +695,11 @@ export function toMarkdown(html, url = '') {
|
|
|
535
695
|
* work with markup than markdown. Same noise removal, no other rewriting.
|
|
536
696
|
* @param {string} html
|
|
537
697
|
* @param {string} url - see toMarkdown
|
|
698
|
+
* @param {{type?: string}} [opts] - see toMarkdown
|
|
538
699
|
* @returns {string}
|
|
539
700
|
*/
|
|
540
|
-
export function toHTML(html, url = '') {
|
|
541
|
-
const { document } = cleanDocument(html, url);
|
|
701
|
+
export function toHTML(html, url = '', { type = '' } = {}) {
|
|
702
|
+
const { document } = cleanDocument(html, url, type);
|
|
542
703
|
const el = bodyOf(document);
|
|
543
704
|
return el ? el.innerHTML.trim() : '';
|
|
544
705
|
}
|
|
@@ -596,7 +757,7 @@ export function feedToHTML(text) {
|
|
|
596
757
|
|| rssLink(entry)
|
|
597
758
|
|| (guidIsPermalink ? field(entry, 'guid') : '');
|
|
598
759
|
const author = field(entry, 'author name') || field(entry, 'author');
|
|
599
|
-
const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate'))
|
|
760
|
+
const date = feedDate(field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate'));
|
|
600
761
|
const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
|
|
601
762
|
// Atom escapes the entry body, so textContent of content/summary is the
|
|
602
763
|
// HTML itself, ready to be embedded and parsed like any page.
|
|
@@ -624,6 +785,27 @@ export function feedToHTML(text) {
|
|
|
624
785
|
return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
625
786
|
}
|
|
626
787
|
|
|
788
|
+
const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
|
|
789
|
+
const RFC822_DATE = /^(?:[a-z]{3},\s*)?(\d{1,2})\s+([a-z]{3})\s+(\d{4})\b/i;
|
|
790
|
+
|
|
791
|
+
/**
|
|
792
|
+
* An entry's date as YYYY-MM-DD. Atom writes ISO 8601, so the day is its
|
|
793
|
+
* first ten characters. RSS 2.0 writes RFC 822 ('Mon, 17 Aug 2026 00:00:00
|
|
794
|
+
* GMT'), and ten characters of that are 'Mon, 17 Au'. Its day, month, and
|
|
795
|
+
* year are read as written rather than through Date, which would move a late
|
|
796
|
+
* evening post to the next day in UTC; the Atom slice keeps the feed's own
|
|
797
|
+
* day too. Anything else is printed as the feed wrote it, not cut.
|
|
798
|
+
* @param {string} s
|
|
799
|
+
* @returns {string}
|
|
800
|
+
*/
|
|
801
|
+
function feedDate(s) {
|
|
802
|
+
if (/^\d{4}-\d\d-\d\d/.test(s)) return s.slice(0, 10);
|
|
803
|
+
const m = s.match(RFC822_DATE);
|
|
804
|
+
const month = m ? MONTHS.indexOf(m[2].toLowerCase()) + 1 : 0;
|
|
805
|
+
if (!month) return s;
|
|
806
|
+
return `${m[3]}-${String(month).padStart(2, '0')}-${m[1].padStart(2, '0')}`;
|
|
807
|
+
}
|
|
808
|
+
|
|
627
809
|
const escHTML = (s) => String(s ?? '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
628
810
|
|
|
629
811
|
/**
|
|
@@ -777,6 +959,7 @@ const BODY_CAP = 4000;
|
|
|
777
959
|
const EPOCH_S = [1e9, 4e9];
|
|
778
960
|
const EPOCH_MS = [1e12, 4e12];
|
|
779
961
|
const DATE_KEY = /(^|_)(date|at|time|timestamp|created|updated|published|modified)$/i;
|
|
962
|
+
const ISO_DATE = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(Z|[+-]\d\d:?\d\d)$/;
|
|
780
963
|
|
|
781
964
|
// Named entities worth knowing without a table: the five XML ones plus the
|
|
782
965
|
// space. Everything else arrives numeric.
|
|
@@ -855,6 +1038,11 @@ function renderValue(key, value) {
|
|
|
855
1038
|
: null;
|
|
856
1039
|
if (ms !== null) return new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
|
|
857
1040
|
}
|
|
1041
|
+
// An ISO timestamp under the same keys reads the same way: the fraction
|
|
1042
|
+
// and zone suffix are a third of its characters and never the point.
|
|
1043
|
+
if (typeof value === 'string' && DATE_KEY.test(key) && ISO_DATE.test(value)) {
|
|
1044
|
+
return value.slice(0, 19).replace('T', ' ');
|
|
1045
|
+
}
|
|
858
1046
|
return typeof value === 'string' ? decodeEntities(value) : String(value);
|
|
859
1047
|
}
|
|
860
1048
|
|
|
@@ -966,11 +1154,12 @@ function constantFields(rows) {
|
|
|
966
1154
|
* Returns null for anything that is not JSON.
|
|
967
1155
|
* @param {string} text
|
|
968
1156
|
* @param {string} url
|
|
969
|
-
* @param {{full?: boolean}} [opts] - full keeps every
|
|
970
|
-
* raw modes are for; the compact view keeps the
|
|
1157
|
+
* @param {{full?: boolean, view?: View|null}} [opts] - full keeps every
|
|
1158
|
+
* field, which is what the raw modes are for; the compact view keeps the
|
|
1159
|
+
* ones that earn their tokens, or the ones a site shortcut's view names
|
|
971
1160
|
* @returns {string | null}
|
|
972
1161
|
*/
|
|
973
|
-
export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
1162
|
+
export function jsonToHTML(text, url = '', { full = false, view = null } = {}) {
|
|
974
1163
|
if (!/^\s*[[{]/.test(text.slice(0, 200))) return null;
|
|
975
1164
|
let data;
|
|
976
1165
|
try {
|
|
@@ -980,6 +1169,14 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
|
980
1169
|
}
|
|
981
1170
|
if (!data || typeof data !== 'object') return null;
|
|
982
1171
|
|
|
1172
|
+
// A shortcut that names its fields knows the response better than the
|
|
1173
|
+
// scoring below can guess, so its choice wins, except in the raw modes,
|
|
1174
|
+
// which exist to show what a choice left out.
|
|
1175
|
+
if (!full && view?.keep?.length) {
|
|
1176
|
+
const chosen = chosenToHTML(data, url, view);
|
|
1177
|
+
if (chosen) return chosen;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
983
1180
|
const { items, meta } = mainArray(data);
|
|
984
1181
|
const flats = items.map((item) => (isPlain(item) ? flattenItem(item) : null));
|
|
985
1182
|
const rows = flats.map((flat) => {
|
|
@@ -1095,6 +1292,163 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
|
1095
1292
|
return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
1096
1293
|
}
|
|
1097
1294
|
|
|
1295
|
+
/**
|
|
1296
|
+
* Read a dot path out of a response ('info.version', 'versions.0.license').
|
|
1297
|
+
* The empty path is the response itself, for an API whose answer is the
|
|
1298
|
+
* bare list.
|
|
1299
|
+
* @param {any} obj
|
|
1300
|
+
* @param {string} path
|
|
1301
|
+
* @returns {any}
|
|
1302
|
+
*/
|
|
1303
|
+
export const pick = (obj, path) =>
|
|
1304
|
+
path === '' ? obj : String(path).split('.').reduce((o, key) => (o == null ? undefined : o[key]), obj);
|
|
1305
|
+
|
|
1306
|
+
const SLOT = /\{([^{}]+)\}/g;
|
|
1307
|
+
|
|
1308
|
+
/**
|
|
1309
|
+
* One chosen field as a string. Booleans read as flags: a true one prints the
|
|
1310
|
+
* field's own name and a false one nothing, so a dependency row reads
|
|
1311
|
+
* `bytes ^1 normal optional` instead of repeating `optional: false` on every
|
|
1312
|
+
* line. An object of scalars (npm's `dependencies`, PyPI's `project_urls`)
|
|
1313
|
+
* prints as `key=value` pairs, since naming the field was the request to see
|
|
1314
|
+
* inside it.
|
|
1315
|
+
* @param {string} path
|
|
1316
|
+
* @param {any} value
|
|
1317
|
+
* @returns {string}
|
|
1318
|
+
*/
|
|
1319
|
+
function renderField(path, value) {
|
|
1320
|
+
const key = path.split('.').pop() ?? path;
|
|
1321
|
+
if (typeof value === 'boolean') return value ? key : '';
|
|
1322
|
+
if (isPlain(value)) {
|
|
1323
|
+
return Object.entries(value)
|
|
1324
|
+
.filter(([, v]) => v !== null && typeof v !== 'object')
|
|
1325
|
+
.map(([k, v]) => `${k}=${renderValue(k, v)}`)
|
|
1326
|
+
.join(', ');
|
|
1327
|
+
}
|
|
1328
|
+
return renderValue(key, value);
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/**
|
|
1332
|
+
* Fill the `{path}` slots of a template from one item. A template whose
|
|
1333
|
+
* every slot came up empty is empty itself rather than its own literal text,
|
|
1334
|
+
* so a changed API reads as missing fields and not as a page of bare
|
|
1335
|
+
* punctuation.
|
|
1336
|
+
* @param {string} template
|
|
1337
|
+
* @param {any} item
|
|
1338
|
+
* @returns {string}
|
|
1339
|
+
*/
|
|
1340
|
+
export function fillTemplate(template, item) {
|
|
1341
|
+
let filled = 0;
|
|
1342
|
+
const text = template.replace(SLOT, (_, path) => {
|
|
1343
|
+
const value = renderField(path, pick(item, path));
|
|
1344
|
+
if (value) filled++;
|
|
1345
|
+
return value;
|
|
1346
|
+
});
|
|
1347
|
+
return filled ? clean(text) : '';
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
/**
|
|
1351
|
+
* The absolute URL a filled template names, when the template itself starts
|
|
1352
|
+
* as one: a site path ("/raw/{id}") or an https URL. Only the template's own
|
|
1353
|
+
* prefix counts, so a field value can never turn plain text into a link or
|
|
1354
|
+
* point one at another host.
|
|
1355
|
+
* @param {string} template
|
|
1356
|
+
* @param {string} text - the filled template
|
|
1357
|
+
* @param {string} base - the endpoint URL, to resolve a site path
|
|
1358
|
+
* @returns {string}
|
|
1359
|
+
*/
|
|
1360
|
+
function templateLink(template, text, base) {
|
|
1361
|
+
if (!/^(\/[^/{]|https:\/\/[^/{])/.test(template) || /\s/.test(text)) return '';
|
|
1362
|
+
try {
|
|
1363
|
+
const href = new URL(text, base);
|
|
1364
|
+
return href.protocol === 'https:' || href.protocol === 'http:' ? href.href : '';
|
|
1365
|
+
} catch {
|
|
1366
|
+
return '';
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
/**
|
|
1371
|
+
* The view a site shortcut asked for: exactly the fields it named, in the
|
|
1372
|
+
* order it named them, for the one object the response is or for each item
|
|
1373
|
+
* of the list it points at. This exists because the generic view below has
|
|
1374
|
+
* to guess, and on registry data it guesses wrong: PyPI's package endpoint
|
|
1375
|
+
* rendered the wheel files and their hashes instead of the version, and a
|
|
1376
|
+
* crates.io dependency list showed row ids and dropped the field that says
|
|
1377
|
+
* whether a dependency is dev-only.
|
|
1378
|
+
*
|
|
1379
|
+
* A field holding a URL becomes a link of its own, so `oc do` follows it. A
|
|
1380
|
+
* name-like first field is the item's title and drops its label. Short
|
|
1381
|
+
* fields that follow each other share a line, the way the generic view
|
|
1382
|
+
* joins them. Returns null when no item held any of the named fields (an
|
|
1383
|
+
* error body, an API that changed shape), so the response renders untouched
|
|
1384
|
+
* and the real reason stays visible.
|
|
1385
|
+
* @param {any} data
|
|
1386
|
+
* @param {string} url
|
|
1387
|
+
* @param {View} view
|
|
1388
|
+
* @returns {string | null}
|
|
1389
|
+
*/
|
|
1390
|
+
function chosenToHTML(data, url, view) {
|
|
1391
|
+
const list = view.list == null ? (Array.isArray(data) ? data : [data]) : pick(data, view.list);
|
|
1392
|
+
if (!Array.isArray(list)) return null;
|
|
1393
|
+
const parts = [];
|
|
1394
|
+
for (const item of list) {
|
|
1395
|
+
if (!isPlain(item)) continue;
|
|
1396
|
+
/** @type {Array<{text: string, label?: string, link?: string}>} */
|
|
1397
|
+
const entries = [];
|
|
1398
|
+
for (const field of view.keep) {
|
|
1399
|
+
if (field.includes('{')) {
|
|
1400
|
+
const text = fillTemplate(field, item);
|
|
1401
|
+
// A template that is a URL on the site ("/raw/{id}") is the item's
|
|
1402
|
+
// link: a forum's topic list names no URL of its own to follow.
|
|
1403
|
+
const link = text && templateLink(field, text, url);
|
|
1404
|
+
if (link) entries.push({ text: link, link });
|
|
1405
|
+
else if (text) entries.push({ text });
|
|
1406
|
+
continue;
|
|
1407
|
+
}
|
|
1408
|
+
const value = pick(item, field);
|
|
1409
|
+
const text = renderField(field, value);
|
|
1410
|
+
if (!text) continue;
|
|
1411
|
+
const label = field.split('.').pop() ?? field;
|
|
1412
|
+
if (isURL(text)) entries.push({ text, label, link: text });
|
|
1413
|
+
else entries.push({ text, label: typeof value === 'boolean' ? undefined : label });
|
|
1414
|
+
}
|
|
1415
|
+
if (!entries.length) continue;
|
|
1416
|
+
parts.push('<article>');
|
|
1417
|
+
const [first, ...rest] = entries;
|
|
1418
|
+
// A name-like first field is the item's title: bare, on a line of its own.
|
|
1419
|
+
const titled = first.label && !first.link && TITLE_KEYS.includes(first.label) && first.text.length <= TITLE_MAX;
|
|
1420
|
+
if (titled) parts.push(`<p>${escHTML(first.text)}</p>`);
|
|
1421
|
+
const inline = [];
|
|
1422
|
+
let width = 0;
|
|
1423
|
+
const flush = () => {
|
|
1424
|
+
if (inline.length) parts.push(`<p>${escHTML(inline.join(' | '))}</p>`);
|
|
1425
|
+
inline.length = 0;
|
|
1426
|
+
width = 0;
|
|
1427
|
+
};
|
|
1428
|
+
for (const entry of titled ? rest : entries) {
|
|
1429
|
+
const line = entry.label ? `${entry.label}: ${entry.text}` : entry.text;
|
|
1430
|
+
if (entry.link) {
|
|
1431
|
+
flush();
|
|
1432
|
+
parts.push(`<p><a href="${escHTML(entry.link)}">${escHTML(line)}</a></p>`);
|
|
1433
|
+
} else if (line.length > TEXT_CAP) {
|
|
1434
|
+
flush();
|
|
1435
|
+
parts.push(`<p>${escHTML(line)}</p>`);
|
|
1436
|
+
} else {
|
|
1437
|
+
// A shared line that outgrew the cap would be cut, and these fields
|
|
1438
|
+
// were asked for by name, so the line wraps instead.
|
|
1439
|
+
if (inline.length && width + line.length + 3 > TEXT_CAP) flush();
|
|
1440
|
+
width = inline.length ? width + line.length + 3 : line.length;
|
|
1441
|
+
inline.push(line);
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
flush();
|
|
1445
|
+
parts.push('</article>');
|
|
1446
|
+
}
|
|
1447
|
+
if (!parts.length) return null;
|
|
1448
|
+
const count = `${list.length} ${list.length === 1 ? 'item' : 'items'}`;
|
|
1449
|
+
return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1098
1452
|
/**
|
|
1099
1453
|
* An API response has no title of its own, so the endpoint becomes one. The
|
|
1100
1454
|
* query parameter is worth the tokens it costs: it is the only part of a
|
|
@@ -1145,9 +1499,11 @@ function mergeText(blocks) {
|
|
|
1145
1499
|
out.push(b);
|
|
1146
1500
|
}
|
|
1147
1501
|
}
|
|
1148
|
-
// Single stray characters (list bullets, separators) cost tokens and say
|
|
1502
|
+
// Single stray characters (list bullets, separators) cost tokens and say
|
|
1503
|
+
// nothing. A single letter or digit is a value, though: a table cell holds
|
|
1504
|
+
// one as a block of its own, and an infobox read "Children" with no number.
|
|
1149
1505
|
return out
|
|
1150
1506
|
.filter((b) => b.type !== 'break')
|
|
1151
|
-
.filter((b) => b.type !== 'text' || b.text.length > 1)
|
|
1507
|
+
.filter((b) => b.type !== 'text' || b.text.length > 1 || /[\p{L}\p{N}]/u.test(b.text))
|
|
1152
1508
|
.map(({ host, pre, post, ...block }) => block);
|
|
1153
1509
|
}
|