@only-cli/oc 0.5.8 → 0.5.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/clis/caniuse.com.json +7 -0
- package/clis/codeberg.org.json +13 -0
- package/clis/crates.io.json +9 -0
- package/clis/discourse.nixos.org.json +8 -0
- package/clis/docs.astral.sh.json +8 -0
- package/clis/docs.pola.rs.json +6 -0
- package/clis/endoflife.date.json +8 -0
- package/clis/forums.swift.org.json +8 -0
- package/clis/github.com.json +6 -1
- package/clis/gitlab.com.json +13 -0
- package/clis/hub.docker.com.json +9 -0
- package/clis/man7.org.json +7 -0
- package/clis/npmjs.com.json +9 -0
- package/clis/osv.dev.json +6 -0
- package/clis/pypi.org.json +8 -0
- package/clis/rfc-editor.org.json +7 -0
- package/clis/rubygems.org.json +8 -0
- package/clis/squidfunk.github.io.json +6 -0
- package/clis/tldr.sh.json +9 -0
- package/clis/users.rust-lang.org.json +8 -0
- package/clis/www.mkdocs.org.json +6 -0
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +11 -11
- package/src/act.js +10 -3
- package/src/apisearch.js +20 -10
- package/src/cache.js +9 -4
- package/src/cli.js +25 -8
- package/src/distill.js +375 -18
- package/src/fetch.js +34 -17
- package/src/mkdocs.js +190 -0
- package/src/nodedocs.js +15 -7
- package/src/render.js +4 -1
- package/src/sites.js +31 -11
package/src/distill.js
CHANGED
|
@@ -15,6 +15,12 @@ import TurndownService from 'turndown';
|
|
|
15
15
|
* @property {string} title
|
|
16
16
|
* @property {Block[]} blocks
|
|
17
17
|
* @property {string} [base] - document base URL from <base href>, if the page set one
|
|
18
|
+
*
|
|
19
|
+
* @typedef {Object} View
|
|
20
|
+
* @property {string[]} keep - fields to show, in order: a dot path
|
|
21
|
+
* ('info.version') or a template joining several ('{crate_id} {req}')
|
|
22
|
+
* @property {string} [list] - dot path to the array the fields apply to each
|
|
23
|
+
* item of; absent means the response itself, whether one object or a list
|
|
18
24
|
*/
|
|
19
25
|
|
|
20
26
|
// Where the compact view cuts a text block. It lives here because numbering
|
|
@@ -107,7 +113,112 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim();
|
|
|
107
113
|
* @returns {string}
|
|
108
114
|
*/
|
|
109
115
|
const asHTML = (text, url = '', opts = {}) =>
|
|
110
|
-
jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text)
|
|
116
|
+
jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text)
|
|
117
|
+
?? textToHTML(text, url, opts) ?? text;
|
|
118
|
+
|
|
119
|
+
// What makes a body HTML when it does not open with a tag: one of the tags
|
|
120
|
+
// every page has somewhere near the top. A tag name alone is not enough,
|
|
121
|
+
// since `Record<string, T>` and `a <b` look like tags to a regex too.
|
|
122
|
+
const HTML_MARK = /<(!doctype|html|head|body|div|p|br|a href|span|script|style|meta|link|title|h[1-6]|table|ul|ol|li|img|article|main|section|nav|form|input|pre)[\s>/]/i;
|
|
123
|
+
|
|
124
|
+
const TEXT_PATH = /\.(md|markdown|txt|rst|adoc)$/i;
|
|
125
|
+
const pathOf = (url) => {
|
|
126
|
+
try {
|
|
127
|
+
return new URL(url).pathname;
|
|
128
|
+
} catch {
|
|
129
|
+
return '';
|
|
130
|
+
}
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
// One line of a Markdown link list: `- [title](url)`, then an optional
|
|
134
|
+
// description. llms.txt is made of these. A title may hold a bracketed
|
|
135
|
+
// word of its own, as Markdown allows: drizzle's reads `[Joins [SQL]]`.
|
|
136
|
+
const LINK_ITEM = /^[ \t]*[-*+][ \t]+\[((?:[^[\]\n]|\[[^[\]\n]*\])+)\]\(([^()\s]+)\)(.*)$/;
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* A paragraph that is nothing but a Markdown link list, as a list of real
|
|
140
|
+
* links, so `do <n>` follows an entry of an llms.txt index or a README's
|
|
141
|
+
* table of links. Anything else in the paragraph keeps it verbatim.
|
|
142
|
+
* @param {string} p
|
|
143
|
+
* @param {string} base
|
|
144
|
+
* @returns {string | null}
|
|
145
|
+
*/
|
|
146
|
+
function linkList(p, base) {
|
|
147
|
+
const items = [];
|
|
148
|
+
for (const line of p.split('\n')) {
|
|
149
|
+
const m = line.match(LINK_ITEM);
|
|
150
|
+
if (!m) return null;
|
|
151
|
+
let href;
|
|
152
|
+
try {
|
|
153
|
+
href = new URL(m[2], base);
|
|
154
|
+
} catch {
|
|
155
|
+
return null;
|
|
156
|
+
}
|
|
157
|
+
if (href.protocol !== 'https:' && href.protocol !== 'http:') return null;
|
|
158
|
+
// The description prints on a line of its own under the link, where the
|
|
159
|
+
// `: ` that joined it to the title reads as noise.
|
|
160
|
+
const text = m[3].replace(/^\s*[:-]\s*/, '').trim();
|
|
161
|
+
items.push(`<li><a href="${escHTML(href.href)}">${escHTML(m[1])}</a>${text ? ` ${escHTML(text)}` : ''}</li>`);
|
|
162
|
+
}
|
|
163
|
+
return `<ul>${items.join('')}</ul>`;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
/**
|
|
167
|
+
* A plain text body as a page: a source file, a manual page, a README
|
|
168
|
+
* fetched raw. The HTML parser eats everything from a `<` to the next `>`,
|
|
169
|
+
* so a JavaScript file lost its generics and comparisons and came out as
|
|
170
|
+
* one shuffled block. Each blank-line-separated paragraph becomes its own
|
|
171
|
+
* block, keeping its line breaks, and a Markdown heading becomes a heading,
|
|
172
|
+
* so `find` and `read` work on a tldr page or an RFC the way they do on a
|
|
173
|
+
* page. The raw modes keep the whole file as one block, verbatim.
|
|
174
|
+
* @param {string} text
|
|
175
|
+
* @param {string} url
|
|
176
|
+
* @param {{full?: boolean, type?: string}} [opts] - type is the declared Content-Type
|
|
177
|
+
* @returns {string | null}
|
|
178
|
+
*/
|
|
179
|
+
function textToHTML(text, url = '', { full = false, type = '' } = {}) {
|
|
180
|
+
const lead = text.replace(/^\uFEFF/, '').trimStart();
|
|
181
|
+
if (!lead || /^<(!doctype|html)[\s>]/i.test(lead)) return null;
|
|
182
|
+
// A README often opens with a centred logo in a <div>, and read as HTML
|
|
183
|
+
// everything after it was loose text the page reader dropped. A path that
|
|
184
|
+
// names a text format, or a server that says the body is text, is text
|
|
185
|
+
// unless it is a whole HTML document: a forum post quoting a <br> is not
|
|
186
|
+
// a page.
|
|
187
|
+
const declared = /^text\/(plain|markdown)\b/i.test(type) || TEXT_PATH.test(pathOf(url));
|
|
188
|
+
if (!declared && (lead.startsWith('<') || HTML_MARK.test(lead.slice(0, 4000)))) return null;
|
|
189
|
+
const body = lead.replace(/\r\n?/g, '\n').trimEnd();
|
|
190
|
+
const paragraphs = full ? [body] : body.split(/\n[ \t]*\n+/);
|
|
191
|
+
const heading = (p) => /^#{1,6} \S/.test(p) && !p.includes('\n') ? p.match(/^#+/)[0].length : 0;
|
|
192
|
+
const parts = paragraphs.map((p) => {
|
|
193
|
+
const level = heading(p);
|
|
194
|
+
if (level) return `<h${level}>${escHTML(p.slice(level + 1))}</h${level}>`;
|
|
195
|
+
if (full) return `<pre>${escHTML(p)}</pre>`;
|
|
196
|
+
// Expo's llms.txt writes each section heading on the line right above
|
|
197
|
+
// its list, with no blank line between, so the two arrive as one
|
|
198
|
+
// paragraph. The heading line is then a heading and the rest a list.
|
|
199
|
+
const cut = p.indexOf('\n');
|
|
200
|
+
const top = cut > 0 ? heading(p.slice(0, cut)) : 0;
|
|
201
|
+
const list = linkList(top ? p.slice(cut + 1) : p, url);
|
|
202
|
+
if (!list) return `<pre>${escHTML(p)}</pre>`;
|
|
203
|
+
return top ? `<h${top}>${escHTML(p.slice(top + 1, cut))}</h${top}>${list}` : list;
|
|
204
|
+
});
|
|
205
|
+
// Only a heading that opens the file names it. A README that starts with
|
|
206
|
+
// prose and has a `# More information` section further down is not titled
|
|
207
|
+
// "More information". A Setext heading (a line of = under the text) counts.
|
|
208
|
+
const opening = paragraphs[0] ?? '';
|
|
209
|
+
const setext = opening.match(/^([^\n]+)\n=+[ \t]*(?:\n|$)/);
|
|
210
|
+
let title = heading(opening) ? opening.replace(/^#+ /, '') : setext ? setext[1].trim() : '';
|
|
211
|
+
if (!title) {
|
|
212
|
+
try {
|
|
213
|
+
title = `${new URL(url).host}${new URL(url).pathname}`;
|
|
214
|
+
} catch {
|
|
215
|
+
title = 'text';
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
// The raw modes find the file again by this mark and hand it back whole.
|
|
219
|
+
const main = full ? '<main data-text>' : '<main>';
|
|
220
|
+
return `<html><head><title>${escHTML(title)}</title></head><body>${main}\n${parts.join('\n')}\n</main></body></html>`;
|
|
221
|
+
}
|
|
111
222
|
|
|
112
223
|
/**
|
|
113
224
|
* The link a heading is, if it is one. A search engine puts the result title
|
|
@@ -166,10 +277,14 @@ function isFollowableHref(href, base, pageUrl) {
|
|
|
166
277
|
* numbers.
|
|
167
278
|
* @param {string} html
|
|
168
279
|
* @param {string} url
|
|
280
|
+
* @param {{view?: View|null, type?: string}} [opts] - the Content-Type the
|
|
281
|
+
* server declared, and a site shortcut's choice of JSON fields,
|
|
282
|
+
* which only a JSON body reads
|
|
169
283
|
* @returns {Page}
|
|
170
284
|
*/
|
|
171
|
-
export function distill(html, url = '') {
|
|
172
|
-
const { document } = parseHTML(asHTML(html, url));
|
|
285
|
+
export function distill(html, url = '', { view = null, type = '' } = {}) {
|
|
286
|
+
const { document } = parseHTML(asHTML(html, url, { view, type }));
|
|
287
|
+
lowerAttributeNames(document);
|
|
173
288
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
174
289
|
const base = documentBase(document, url);
|
|
175
290
|
/** @type {Block[]} */
|
|
@@ -178,7 +293,7 @@ export function distill(html, url = '') {
|
|
|
178
293
|
const hidden = (el) =>
|
|
179
294
|
el.getAttribute('hidden') !== null ||
|
|
180
295
|
el.getAttribute('aria-hidden') === 'true' ||
|
|
181
|
-
|
|
296
|
+
DISPLAY_NONE.test(el.getAttribute('style') ?? '');
|
|
182
297
|
|
|
183
298
|
/** Subtree already emitted, skipped when the rest of the page is walked. */
|
|
184
299
|
let done = null;
|
|
@@ -233,6 +348,12 @@ export function distill(html, url = '') {
|
|
|
233
348
|
// space that was never in the page.
|
|
234
349
|
if (text) {
|
|
235
350
|
blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
|
|
351
|
+
} else if (raw && blocks.at(-1)?.type === 'text') {
|
|
352
|
+
// A node that is only a space still separates the words either side
|
|
353
|
+
// of it. Wikipedia writes every as a span holding nothing
|
|
354
|
+
// else, and the fragments around it share a parent, so without this
|
|
355
|
+
// an infobox read "Firstappeared" and "20February 1991".
|
|
356
|
+
blocks.at(-1).post = true;
|
|
236
357
|
}
|
|
237
358
|
return;
|
|
238
359
|
}
|
|
@@ -413,7 +534,14 @@ function mainOf(document, body) {
|
|
|
413
534
|
const found = [...document.querySelectorAll(selector)];
|
|
414
535
|
if (found.length !== 1) continue;
|
|
415
536
|
const el = found[0];
|
|
416
|
-
if (el
|
|
537
|
+
if (el === body || prose(el) < total * MIN_SHARE) continue;
|
|
538
|
+
// Material for MkDocs puts both sidebars, the page nav and the table of
|
|
539
|
+
// contents, inside <main>, so a docs page led with fifty nav links. A
|
|
540
|
+
// main that holds navigation and exactly one article holding most of its
|
|
541
|
+
// prose leads with the article; the rest still follows. Walking down to
|
|
542
|
+
// the densest column instead cut pkg.go.dev to its biggest type.
|
|
543
|
+
const articles = el.querySelector('nav') ? el.querySelectorAll('article') : [];
|
|
544
|
+
return articles.length === 1 && prose(articles[0]) >= prose(el) * KEEP_SHARE ? articles[0] : el;
|
|
417
545
|
}
|
|
418
546
|
return densest(body, total);
|
|
419
547
|
}
|
|
@@ -488,16 +616,44 @@ function number(blocks) {
|
|
|
488
616
|
*/
|
|
489
617
|
const bodyOf = (document) => document.querySelector('body') ?? document.documentElement;
|
|
490
618
|
|
|
619
|
+
/** CSS property names and keywords are case-insensitive: `DISPLAY : NONE` hides too. */
|
|
620
|
+
const DISPLAY_NONE = /display\s*:\s*none/i;
|
|
621
|
+
|
|
622
|
+
/**
|
|
623
|
+
* HTML attribute names are ASCII case-insensitive, but linkedom keeps them as
|
|
624
|
+
* written, so `<A HREF>` had no href, `<INPUT TYPE="hidden">` no type and
|
|
625
|
+
* `<P STYLE="display:none">` no style. Lowercase them once after parsing so
|
|
626
|
+
* every lookup below sees what a browser sees. As in a browser, the first of
|
|
627
|
+
* two spellings of one attribute wins.
|
|
628
|
+
* @param {Document} document
|
|
629
|
+
*/
|
|
630
|
+
function lowerAttributeNames(document) {
|
|
631
|
+
for (const el of document.querySelectorAll('*')) {
|
|
632
|
+
const attributes = [...el.attributes];
|
|
633
|
+
if (attributes.every(({ name }) => name === name.toLowerCase())) continue;
|
|
634
|
+
const first = new Map();
|
|
635
|
+
for (const { name, value } of attributes) {
|
|
636
|
+
const lower = name.toLowerCase();
|
|
637
|
+
if (!first.has(lower)) first.set(lower, value);
|
|
638
|
+
}
|
|
639
|
+
for (const { name } of attributes) el.removeAttribute(name);
|
|
640
|
+
for (const [name, value] of first) el.setAttribute(name, value);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
491
644
|
/**
|
|
492
645
|
* Shared cleanup for the raw modes: parse, then delete the same noise
|
|
493
646
|
* distill() skips, so neither raw output ever leaks scripts, styles, or
|
|
494
647
|
* hidden content.
|
|
495
648
|
* @param {string} html
|
|
649
|
+
* @param {string} [url]
|
|
650
|
+
* @param {string} [type] - the declared Content-Type
|
|
496
651
|
*/
|
|
497
|
-
function cleanDocument(html, url = '') {
|
|
652
|
+
function cleanDocument(html, url = '', type = '') {
|
|
498
653
|
// Raw is the mode an agent reaches for when the compact view left something
|
|
499
654
|
// out, so it is the one place a JSON response keeps every field.
|
|
500
|
-
const { document } = parseHTML(asHTML(html, url, { full: true }));
|
|
655
|
+
const { document } = parseHTML(asHTML(html, url, { full: true, type }));
|
|
656
|
+
lowerAttributeNames(document);
|
|
501
657
|
// Read the title before the sweep below removes the head with it.
|
|
502
658
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
503
659
|
for (const tag of DROP) {
|
|
@@ -508,7 +664,7 @@ function cleanDocument(html, url = '') {
|
|
|
508
664
|
if ((el.getAttribute('type') ?? '').toLowerCase() === 'hidden') el.remove();
|
|
509
665
|
}
|
|
510
666
|
for (const el of [...document.querySelectorAll('[style]')]) {
|
|
511
|
-
if (
|
|
667
|
+
if (DISPLAY_NONE.test(el.getAttribute('style') ?? '')) el.remove();
|
|
512
668
|
}
|
|
513
669
|
return { document, title };
|
|
514
670
|
}
|
|
@@ -519,10 +675,15 @@ function cleanDocument(html, url = '') {
|
|
|
519
675
|
* @param {string} html
|
|
520
676
|
* @param {string} url - only read when the body turns out to be JSON, whose
|
|
521
677
|
* title has to come from the endpoint because the payload has none
|
|
678
|
+
* @param {{type?: string}} [opts] - the declared Content-Type
|
|
522
679
|
* @returns {string}
|
|
523
680
|
*/
|
|
524
|
-
export function toMarkdown(html, url = '') {
|
|
525
|
-
const { document, title } = cleanDocument(html, url);
|
|
681
|
+
export function toMarkdown(html, url = '', { type = '' } = {}) {
|
|
682
|
+
const { document, title } = cleanDocument(html, url, type);
|
|
683
|
+
// A plain text file is already the markdown of itself. Run through the
|
|
684
|
+
// converter it came back with every `#` and backtick escaped.
|
|
685
|
+
const file = document.querySelector('main[data-text] > pre');
|
|
686
|
+
if (file) return file.textContent;
|
|
526
687
|
const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
|
|
527
688
|
const el = bodyOf(document);
|
|
528
689
|
const body = el ? turndown.turndown(el.innerHTML).trim() : '';
|
|
@@ -534,10 +695,11 @@ export function toMarkdown(html, url = '') {
|
|
|
534
695
|
* work with markup than markdown. Same noise removal, no other rewriting.
|
|
535
696
|
* @param {string} html
|
|
536
697
|
* @param {string} url - see toMarkdown
|
|
698
|
+
* @param {{type?: string}} [opts] - see toMarkdown
|
|
537
699
|
* @returns {string}
|
|
538
700
|
*/
|
|
539
|
-
export function toHTML(html, url = '') {
|
|
540
|
-
const { document } = cleanDocument(html, url);
|
|
701
|
+
export function toHTML(html, url = '', { type = '' } = {}) {
|
|
702
|
+
const { document } = cleanDocument(html, url, type);
|
|
541
703
|
const el = bodyOf(document);
|
|
542
704
|
return el ? el.innerHTML.trim() : '';
|
|
543
705
|
}
|
|
@@ -595,7 +757,7 @@ export function feedToHTML(text) {
|
|
|
595
757
|
|| rssLink(entry)
|
|
596
758
|
|| (guidIsPermalink ? field(entry, 'guid') : '');
|
|
597
759
|
const author = field(entry, 'author name') || field(entry, 'author');
|
|
598
|
-
const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate'))
|
|
760
|
+
const date = feedDate(field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate'));
|
|
599
761
|
const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
|
|
600
762
|
// Atom escapes the entry body, so textContent of content/summary is the
|
|
601
763
|
// HTML itself, ready to be embedded and parsed like any page.
|
|
@@ -623,6 +785,27 @@ export function feedToHTML(text) {
|
|
|
623
785
|
return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
624
786
|
}
|
|
625
787
|
|
|
788
|
+
const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
|
|
789
|
+
const RFC822_DATE = /^(?:[a-z]{3},\s*)?(\d{1,2})\s+([a-z]{3})\s+(\d{4})\b/i;
|
|
790
|
+
|
|
791
|
+
/**
|
|
792
|
+
* An entry's date as YYYY-MM-DD. Atom writes ISO 8601, so the day is its
|
|
793
|
+
* first ten characters. RSS 2.0 writes RFC 822 ('Mon, 17 Aug 2026 00:00:00
|
|
794
|
+
* GMT'), and ten characters of that are 'Mon, 17 Au'. Its day, month, and
|
|
795
|
+
* year are read as written rather than through Date, which would move a late
|
|
796
|
+
* evening post to the next day in UTC; the Atom slice keeps the feed's own
|
|
797
|
+
* day too. Anything else is printed as the feed wrote it, not cut.
|
|
798
|
+
* @param {string} s
|
|
799
|
+
* @returns {string}
|
|
800
|
+
*/
|
|
801
|
+
function feedDate(s) {
|
|
802
|
+
if (/^\d{4}-\d\d-\d\d/.test(s)) return s.slice(0, 10);
|
|
803
|
+
const m = s.match(RFC822_DATE);
|
|
804
|
+
const month = m ? MONTHS.indexOf(m[2].toLowerCase()) + 1 : 0;
|
|
805
|
+
if (!month) return s;
|
|
806
|
+
return `${m[3]}-${String(month).padStart(2, '0')}-${m[1].padStart(2, '0')}`;
|
|
807
|
+
}
|
|
808
|
+
|
|
626
809
|
const escHTML = (s) => String(s ?? '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
|
627
810
|
|
|
628
811
|
/**
|
|
@@ -776,6 +959,7 @@ const BODY_CAP = 4000;
|
|
|
776
959
|
const EPOCH_S = [1e9, 4e9];
|
|
777
960
|
const EPOCH_MS = [1e12, 4e12];
|
|
778
961
|
const DATE_KEY = /(^|_)(date|at|time|timestamp|created|updated|published|modified)$/i;
|
|
962
|
+
const ISO_DATE = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(Z|[+-]\d\d:?\d\d)$/;
|
|
779
963
|
|
|
780
964
|
// Named entities worth knowing without a table: the five XML ones plus the
|
|
781
965
|
// space. Everything else arrives numeric.
|
|
@@ -854,6 +1038,11 @@ function renderValue(key, value) {
|
|
|
854
1038
|
: null;
|
|
855
1039
|
if (ms !== null) return new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
|
|
856
1040
|
}
|
|
1041
|
+
// An ISO timestamp under the same keys reads the same way: the fraction
|
|
1042
|
+
// and zone suffix are a third of its characters and never the point.
|
|
1043
|
+
if (typeof value === 'string' && DATE_KEY.test(key) && ISO_DATE.test(value)) {
|
|
1044
|
+
return value.slice(0, 19).replace('T', ' ');
|
|
1045
|
+
}
|
|
857
1046
|
return typeof value === 'string' ? decodeEntities(value) : String(value);
|
|
858
1047
|
}
|
|
859
1048
|
|
|
@@ -965,11 +1154,12 @@ function constantFields(rows) {
|
|
|
965
1154
|
* Returns null for anything that is not JSON.
|
|
966
1155
|
* @param {string} text
|
|
967
1156
|
* @param {string} url
|
|
968
|
-
* @param {{full?: boolean}} [opts] - full keeps every
|
|
969
|
-
* raw modes are for; the compact view keeps the
|
|
1157
|
+
* @param {{full?: boolean, view?: View|null}} [opts] - full keeps every
|
|
1158
|
+
* field, which is what the raw modes are for; the compact view keeps the
|
|
1159
|
+
* ones that earn their tokens, or the ones a site shortcut's view names
|
|
970
1160
|
* @returns {string | null}
|
|
971
1161
|
*/
|
|
972
|
-
export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
1162
|
+
export function jsonToHTML(text, url = '', { full = false, view = null } = {}) {
|
|
973
1163
|
if (!/^\s*[[{]/.test(text.slice(0, 200))) return null;
|
|
974
1164
|
let data;
|
|
975
1165
|
try {
|
|
@@ -979,6 +1169,14 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
|
979
1169
|
}
|
|
980
1170
|
if (!data || typeof data !== 'object') return null;
|
|
981
1171
|
|
|
1172
|
+
// A shortcut that names its fields knows the response better than the
|
|
1173
|
+
// scoring below can guess, so its choice wins, except in the raw modes,
|
|
1174
|
+
// which exist to show what a choice left out.
|
|
1175
|
+
if (!full && view?.keep?.length) {
|
|
1176
|
+
const chosen = chosenToHTML(data, url, view);
|
|
1177
|
+
if (chosen) return chosen;
|
|
1178
|
+
}
|
|
1179
|
+
|
|
982
1180
|
const { items, meta } = mainArray(data);
|
|
983
1181
|
const flats = items.map((item) => (isPlain(item) ? flattenItem(item) : null));
|
|
984
1182
|
const rows = flats.map((flat) => {
|
|
@@ -1094,6 +1292,163 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
|
1094
1292
|
return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
1095
1293
|
}
|
|
1096
1294
|
|
|
1295
|
+
/**
|
|
1296
|
+
* Read a dot path out of a response ('info.version', 'versions.0.license').
|
|
1297
|
+
* The empty path is the response itself, for an API whose answer is the
|
|
1298
|
+
* bare list.
|
|
1299
|
+
* @param {any} obj
|
|
1300
|
+
* @param {string} path
|
|
1301
|
+
* @returns {any}
|
|
1302
|
+
*/
|
|
1303
|
+
export const pick = (obj, path) =>
|
|
1304
|
+
path === '' ? obj : String(path).split('.').reduce((o, key) => (o == null ? undefined : o[key]), obj);
|
|
1305
|
+
|
|
1306
|
+
const SLOT = /\{([^{}]+)\}/g;
|
|
1307
|
+
|
|
1308
|
+
/**
|
|
1309
|
+
* One chosen field as a string. Booleans read as flags: a true one prints the
|
|
1310
|
+
* field's own name and a false one nothing, so a dependency row reads
|
|
1311
|
+
* `bytes ^1 normal optional` instead of repeating `optional: false` on every
|
|
1312
|
+
* line. An object of scalars (npm's `dependencies`, PyPI's `project_urls`)
|
|
1313
|
+
* prints as `key=value` pairs, since naming the field was the request to see
|
|
1314
|
+
* inside it.
|
|
1315
|
+
* @param {string} path
|
|
1316
|
+
* @param {any} value
|
|
1317
|
+
* @returns {string}
|
|
1318
|
+
*/
|
|
1319
|
+
function renderField(path, value) {
|
|
1320
|
+
const key = path.split('.').pop() ?? path;
|
|
1321
|
+
if (typeof value === 'boolean') return value ? key : '';
|
|
1322
|
+
if (isPlain(value)) {
|
|
1323
|
+
return Object.entries(value)
|
|
1324
|
+
.filter(([, v]) => v !== null && typeof v !== 'object')
|
|
1325
|
+
.map(([k, v]) => `${k}=${renderValue(k, v)}`)
|
|
1326
|
+
.join(', ');
|
|
1327
|
+
}
|
|
1328
|
+
return renderValue(key, value);
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
/**
|
|
1332
|
+
* Fill the `{path}` slots of a template from one item. A template whose
|
|
1333
|
+
* every slot came up empty is empty itself rather than its own literal text,
|
|
1334
|
+
* so a changed API reads as missing fields and not as a page of bare
|
|
1335
|
+
* punctuation.
|
|
1336
|
+
* @param {string} template
|
|
1337
|
+
* @param {any} item
|
|
1338
|
+
* @returns {string}
|
|
1339
|
+
*/
|
|
1340
|
+
export function fillTemplate(template, item) {
|
|
1341
|
+
let filled = 0;
|
|
1342
|
+
const text = template.replace(SLOT, (_, path) => {
|
|
1343
|
+
const value = renderField(path, pick(item, path));
|
|
1344
|
+
if (value) filled++;
|
|
1345
|
+
return value;
|
|
1346
|
+
});
|
|
1347
|
+
return filled ? clean(text) : '';
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
/**
|
|
1351
|
+
* The absolute URL a filled template names, when the template itself starts
|
|
1352
|
+
* as one: a site path ("/raw/{id}") or an https URL. Only the template's own
|
|
1353
|
+
* prefix counts, so a field value can never turn plain text into a link or
|
|
1354
|
+
* point one at another host.
|
|
1355
|
+
* @param {string} template
|
|
1356
|
+
* @param {string} text - the filled template
|
|
1357
|
+
* @param {string} base - the endpoint URL, to resolve a site path
|
|
1358
|
+
* @returns {string}
|
|
1359
|
+
*/
|
|
1360
|
+
function templateLink(template, text, base) {
|
|
1361
|
+
if (!/^(\/[^/{]|https:\/\/[^/{])/.test(template) || /\s/.test(text)) return '';
|
|
1362
|
+
try {
|
|
1363
|
+
const href = new URL(text, base);
|
|
1364
|
+
return href.protocol === 'https:' || href.protocol === 'http:' ? href.href : '';
|
|
1365
|
+
} catch {
|
|
1366
|
+
return '';
|
|
1367
|
+
}
|
|
1368
|
+
}
|
|
1369
|
+
|
|
1370
|
+
/**
|
|
1371
|
+
* The view a site shortcut asked for: exactly the fields it named, in the
|
|
1372
|
+
* order it named them, for the one object the response is or for each item
|
|
1373
|
+
* of the list it points at. This exists because the generic view below has
|
|
1374
|
+
* to guess, and on registry data it guesses wrong: PyPI's package endpoint
|
|
1375
|
+
* rendered the wheel files and their hashes instead of the version, and a
|
|
1376
|
+
* crates.io dependency list showed row ids and dropped the field that says
|
|
1377
|
+
* whether a dependency is dev-only.
|
|
1378
|
+
*
|
|
1379
|
+
* A field holding a URL becomes a link of its own, so `oc do` follows it. A
|
|
1380
|
+
* name-like first field is the item's title and drops its label. Short
|
|
1381
|
+
* fields that follow each other share a line, the way the generic view
|
|
1382
|
+
* joins them. Returns null when no item held any of the named fields (an
|
|
1383
|
+
* error body, an API that changed shape), so the response renders untouched
|
|
1384
|
+
* and the real reason stays visible.
|
|
1385
|
+
* @param {any} data
|
|
1386
|
+
* @param {string} url
|
|
1387
|
+
* @param {View} view
|
|
1388
|
+
* @returns {string | null}
|
|
1389
|
+
*/
|
|
1390
|
+
function chosenToHTML(data, url, view) {
|
|
1391
|
+
const list = view.list == null ? (Array.isArray(data) ? data : [data]) : pick(data, view.list);
|
|
1392
|
+
if (!Array.isArray(list)) return null;
|
|
1393
|
+
const parts = [];
|
|
1394
|
+
for (const item of list) {
|
|
1395
|
+
if (!isPlain(item)) continue;
|
|
1396
|
+
/** @type {Array<{text: string, label?: string, link?: string}>} */
|
|
1397
|
+
const entries = [];
|
|
1398
|
+
for (const field of view.keep) {
|
|
1399
|
+
if (field.includes('{')) {
|
|
1400
|
+
const text = fillTemplate(field, item);
|
|
1401
|
+
// A template that is a URL on the site ("/raw/{id}") is the item's
|
|
1402
|
+
// link: a forum's topic list names no URL of its own to follow.
|
|
1403
|
+
const link = text && templateLink(field, text, url);
|
|
1404
|
+
if (link) entries.push({ text: link, link });
|
|
1405
|
+
else if (text) entries.push({ text });
|
|
1406
|
+
continue;
|
|
1407
|
+
}
|
|
1408
|
+
const value = pick(item, field);
|
|
1409
|
+
const text = renderField(field, value);
|
|
1410
|
+
if (!text) continue;
|
|
1411
|
+
const label = field.split('.').pop() ?? field;
|
|
1412
|
+
if (isURL(text)) entries.push({ text, label, link: text });
|
|
1413
|
+
else entries.push({ text, label: typeof value === 'boolean' ? undefined : label });
|
|
1414
|
+
}
|
|
1415
|
+
if (!entries.length) continue;
|
|
1416
|
+
parts.push('<article>');
|
|
1417
|
+
const [first, ...rest] = entries;
|
|
1418
|
+
// A name-like first field is the item's title: bare, on a line of its own.
|
|
1419
|
+
const titled = first.label && !first.link && TITLE_KEYS.includes(first.label) && first.text.length <= TITLE_MAX;
|
|
1420
|
+
if (titled) parts.push(`<p>${escHTML(first.text)}</p>`);
|
|
1421
|
+
const inline = [];
|
|
1422
|
+
let width = 0;
|
|
1423
|
+
const flush = () => {
|
|
1424
|
+
if (inline.length) parts.push(`<p>${escHTML(inline.join(' | '))}</p>`);
|
|
1425
|
+
inline.length = 0;
|
|
1426
|
+
width = 0;
|
|
1427
|
+
};
|
|
1428
|
+
for (const entry of titled ? rest : entries) {
|
|
1429
|
+
const line = entry.label ? `${entry.label}: ${entry.text}` : entry.text;
|
|
1430
|
+
if (entry.link) {
|
|
1431
|
+
flush();
|
|
1432
|
+
parts.push(`<p><a href="${escHTML(entry.link)}">${escHTML(line)}</a></p>`);
|
|
1433
|
+
} else if (line.length > TEXT_CAP) {
|
|
1434
|
+
flush();
|
|
1435
|
+
parts.push(`<p>${escHTML(line)}</p>`);
|
|
1436
|
+
} else {
|
|
1437
|
+
// A shared line that outgrew the cap would be cut, and these fields
|
|
1438
|
+
// were asked for by name, so the line wraps instead.
|
|
1439
|
+
if (inline.length && width + line.length + 3 > TEXT_CAP) flush();
|
|
1440
|
+
width = inline.length ? width + line.length + 3 : line.length;
|
|
1441
|
+
inline.push(line);
|
|
1442
|
+
}
|
|
1443
|
+
}
|
|
1444
|
+
flush();
|
|
1445
|
+
parts.push('</article>');
|
|
1446
|
+
}
|
|
1447
|
+
if (!parts.length) return null;
|
|
1448
|
+
const count = `${list.length} ${list.length === 1 ? 'item' : 'items'}`;
|
|
1449
|
+
return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
|
1450
|
+
}
|
|
1451
|
+
|
|
1097
1452
|
/**
|
|
1098
1453
|
* An API response has no title of its own, so the endpoint becomes one. The
|
|
1099
1454
|
* query parameter is worth the tokens it costs: it is the only part of a
|
|
@@ -1144,9 +1499,11 @@ function mergeText(blocks) {
|
|
|
1144
1499
|
out.push(b);
|
|
1145
1500
|
}
|
|
1146
1501
|
}
|
|
1147
|
-
// Single stray characters (list bullets, separators) cost tokens and say
|
|
1502
|
+
// Single stray characters (list bullets, separators) cost tokens and say
|
|
1503
|
+
// nothing. A single letter or digit is a value, though: a table cell holds
|
|
1504
|
+
// one as a block of its own, and an infobox read "Children" with no number.
|
|
1148
1505
|
return out
|
|
1149
1506
|
.filter((b) => b.type !== 'break')
|
|
1150
|
-
.filter((b) => b.type !== 'text' || b.text.length > 1)
|
|
1507
|
+
.filter((b) => b.type !== 'text' || b.text.length > 1 || /[\p{L}\p{N}]/u.test(b.text))
|
|
1151
1508
|
.map(({ host, pre, post, ...block }) => block);
|
|
1152
1509
|
}
|
package/src/fetch.js
CHANGED
|
@@ -617,20 +617,30 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
|
|
|
617
617
|
* Fetch a page.
|
|
618
618
|
* @param {string} url - with or without a scheme, https is assumed
|
|
619
619
|
* @param {{ jar?: { cookieHeaderFor(url: string): string|undefined, storeFromResponse(url: string, headers: string[]): void } }} [opts]
|
|
620
|
-
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
|
621
|
-
* final URL after redirects, the body, the HTTP status,
|
|
622
|
-
* identity got the page (impers:chrome, impers:firefox, or fetch)
|
|
620
|
+
* @returns {Promise<{url: string, html: string, status: number, via: string, type: string}>}
|
|
621
|
+
* final URL after redirects, the body, the HTTP status, which client
|
|
622
|
+
* identity got the page (impers:chrome, impers:firefox, or fetch), and the
|
|
623
|
+
* Content-Type the server declared
|
|
623
624
|
*/
|
|
624
625
|
export async function fetchPage(url, { jar } = {}) {
|
|
625
|
-
|
|
626
|
-
// A reader with no cookies for reddit.com gets the feed where the page
|
|
627
|
-
// would be a login wall; one who logged in gets the page it asked for.
|
|
628
|
-
if (!jar?.cookieHeaderFor(target)) target = redditFeedURL(target) ?? target;
|
|
626
|
+
const target = readableURL(/^https?:\/\//i.test(url) ? url : `https://${url}`, jar);
|
|
629
627
|
await assertSafeTarget(target);
|
|
630
628
|
const impers = await loadImpers();
|
|
631
629
|
return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
|
|
632
630
|
}
|
|
633
631
|
|
|
632
|
+
/**
|
|
633
|
+
* The URL to ask for in place of this one. A reader with no cookies for
|
|
634
|
+
* reddit.com gets the feed where the page would be a login wall; one who
|
|
635
|
+
* logged in gets the page it asked for. Every redirect hop goes through this
|
|
636
|
+
* too: redd.it short links and the app's /r/<sub>/s/<code> share links answer
|
|
637
|
+
* with a 301 to the post page, the very wall the feed steps around.
|
|
638
|
+
* @param {string} url
|
|
639
|
+
* @param {{ cookieHeaderFor(url: string): string|undefined }} [jar]
|
|
640
|
+
* @returns {string}
|
|
641
|
+
*/
|
|
642
|
+
const readableURL = (url, jar) => (jar?.cookieHeaderFor(url) ? url : redditFeedURL(url) ?? url);
|
|
643
|
+
|
|
634
644
|
/**
|
|
635
645
|
* Follow redirects one hop at a time, validating each destination before the
|
|
636
646
|
* next request goes out.
|
|
@@ -643,9 +653,11 @@ export async function fetchPage(url, { jar } = {}) {
|
|
|
643
653
|
* process.
|
|
644
654
|
* @param {(url: string) => Promise<any>} get - one request, redirects not followed
|
|
645
655
|
* @param {string} start
|
|
656
|
+
* @param {{onResponse?: (url: string, res: any) => void, rewrite?: (url: string) => string}} [opts] -
|
|
657
|
+
* rewrite names the URL to ask for in place of a hop's Location
|
|
646
658
|
* @returns {Promise<{res: any, url: string}>} the first non-redirect response
|
|
647
659
|
*/
|
|
648
|
-
export async function followRedirects(get, start, { onResponse } = {}) {
|
|
660
|
+
export async function followRedirects(get, start, { onResponse, rewrite } = {}) {
|
|
649
661
|
let current = start;
|
|
650
662
|
for (let i = 0; ; i++) {
|
|
651
663
|
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
|
|
@@ -654,7 +666,8 @@ export async function followRedirects(get, start, { onResponse } = {}) {
|
|
|
654
666
|
const status = res.status ?? res.statusCode ?? 0;
|
|
655
667
|
const location = res.headers.get('location');
|
|
656
668
|
if (status >= 300 && status < 400 && location) {
|
|
657
|
-
|
|
669
|
+
const next = new URL(location, current).toString();
|
|
670
|
+
current = rewrite ? rewrite(next) : next;
|
|
658
671
|
await assertSafeTarget(current);
|
|
659
672
|
continue;
|
|
660
673
|
}
|
|
@@ -687,8 +700,9 @@ function captureSetCookie(jar, url, res) {
|
|
|
687
700
|
// letting firefox through. reddit.com started doing this in 2026 (#52), so
|
|
688
701
|
// starting with chrome there would turn every read into two requests against
|
|
689
702
|
// a per-address rate limit of about ten a minute. Subdomains inherit the
|
|
690
|
-
// entry.
|
|
691
|
-
|
|
703
|
+
// entry. A redd.it short link is a 301 to a reddit.com post, and the identity
|
|
704
|
+
// is picked once for the whole chain, so it starts the same way.
|
|
705
|
+
const FIREFOX_FIRST_HOSTS = ['reddit.com', 'redd.it'];
|
|
692
706
|
|
|
693
707
|
// Reddit has sent logged-out readers of its HTML pages to a login page since
|
|
694
708
|
// June 2026 (#52), while the Atom feed beside each of those pages still
|
|
@@ -757,7 +771,7 @@ const bufferedContent = (res) => {
|
|
|
757
771
|
* @param {any} impers - the impers module (or a stand-in with a get method)
|
|
758
772
|
* @param {string} target
|
|
759
773
|
* @param {object} [jar]
|
|
760
|
-
* @returns {Promise<{url: string, html: string, status: number, via: string}>}
|
|
774
|
+
* @returns {Promise<{url: string, html: string, status: number, via: string, type: string}>}
|
|
761
775
|
*/
|
|
762
776
|
export async function viaImpers(impers, target, jar) {
|
|
763
777
|
// A blocked first attempt gets one cheap retry with the other identity. An
|
|
@@ -777,9 +791,10 @@ export async function viaImpers(impers, target, jar) {
|
|
|
777
791
|
headers: jarHeaders(jar, url, {}),
|
|
778
792
|
});
|
|
779
793
|
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
794
|
+
const rewrite = (url) => readableURL(url, jar);
|
|
780
795
|
const attempt = async (impersonate) => {
|
|
781
796
|
try {
|
|
782
|
-
const { res } = await followRedirects(asking(impersonate), target, { onResponse });
|
|
797
|
+
const { res } = await followRedirects(asking(impersonate), target, { onResponse, rewrite });
|
|
783
798
|
return { res, status: res.status ?? res.statusCode ?? 0 };
|
|
784
799
|
} catch (err) {
|
|
785
800
|
if (err?.name !== 'ImpersonateError') throw err;
|
|
@@ -810,11 +825,11 @@ export async function viaImpers(impers, target, jar) {
|
|
|
810
825
|
// this is the only size this transport ever really checks. Refused before
|
|
811
826
|
// the decode so an oversized body is not allocated twice over.
|
|
812
827
|
assertBodySize(bytes.length, target);
|
|
813
|
-
return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via };
|
|
828
|
+
return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via, type: res.headers.get('content-type') ?? '' };
|
|
814
829
|
}
|
|
815
830
|
const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
|
|
816
831
|
assertBodySize(html.length, target);
|
|
817
|
-
return { url: res.url ?? target, html, status, via };
|
|
832
|
+
return { url: res.url ?? target, html, status, via, type: res.headers.get('content-type') ?? '' };
|
|
818
833
|
}
|
|
819
834
|
|
|
820
835
|
async function viaFetch(target, jar) {
|
|
@@ -826,13 +841,15 @@ async function viaFetch(target, jar) {
|
|
|
826
841
|
: fetch(url, { redirect: 'manual', headers });
|
|
827
842
|
};
|
|
828
843
|
const onResponse = (url, res) => captureSetCookie(jar, url, res);
|
|
829
|
-
const
|
|
844
|
+
const rewrite = (url) => readableURL(url, jar);
|
|
845
|
+
const { res, url: current } = await followRedirects(get, target, { onResponse, rewrite });
|
|
830
846
|
if (!res.ok) {
|
|
831
847
|
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
|
832
848
|
}
|
|
833
849
|
assertReadableType(res.headers.get('content-type'));
|
|
834
850
|
assertBodySize(Number(res.headers.get('content-length')) || 0, current);
|
|
835
|
-
return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch'
|
|
851
|
+
return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch',
|
|
852
|
+
type: res.headers.get('content-type') ?? '' };
|
|
836
853
|
}
|
|
837
854
|
|
|
838
855
|
/**
|