@only-cli/oc 0.5.8 → 0.5.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/distill.js CHANGED
@@ -15,6 +15,12 @@ import TurndownService from 'turndown';
15
15
  * @property {string} title
16
16
  * @property {Block[]} blocks
17
17
  * @property {string} [base] - document base URL from <base href>, if the page set one
18
+ *
19
+ * @typedef {Object} View
20
+ * @property {string[]} keep - fields to show, in order: a dot path
21
+ * ('info.version') or a template joining several ('{crate_id} {req}')
22
+ * @property {string} [list] - dot path to the array the fields apply to each
23
+ * item of; absent means the response itself, whether one object or a list
18
24
  */
19
25
 
20
26
  // Where the compact view cuts a text block. It lives here because numbering
@@ -107,7 +113,112 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim();
107
113
  * @returns {string}
108
114
  */
109
115
  const asHTML = (text, url = '', opts = {}) =>
110
- jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text) ?? text;
116
+ jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text)
117
+ ?? textToHTML(text, url, opts) ?? text;
118
+
119
+ // What makes a body HTML when it does not open with a tag: one of the tags
120
+ // every page has somewhere near the top. A tag name alone is not enough,
121
+ // since `Record<string, T>` and `a <b` look like tags to a regex too.
122
+ const HTML_MARK = /<(!doctype|html|head|body|div|p|br|a href|span|script|style|meta|link|title|h[1-6]|table|ul|ol|li|img|article|main|section|nav|form|input|pre)[\s>/]/i;
123
+
124
+ const TEXT_PATH = /\.(md|markdown|txt|rst|adoc)$/i;
125
+ const pathOf = (url) => {
126
+ try {
127
+ return new URL(url).pathname;
128
+ } catch {
129
+ return '';
130
+ }
131
+ };
132
+
133
+ // One line of a Markdown link list: `- [title](url)`, then an optional
134
+ // description. llms.txt is made of these. A title may hold a bracketed
135
+ // word of its own, as Markdown allows: drizzle's reads `[Joins [SQL]]`.
136
+ const LINK_ITEM = /^[ \t]*[-*+][ \t]+\[((?:[^[\]\n]|\[[^[\]\n]*\])+)\]\(([^()\s]+)\)(.*)$/;
137
+
138
+ /**
139
+ * A paragraph that is nothing but a Markdown link list, as a list of real
140
+ * links, so `do <n>` follows an entry of an llms.txt index or a README's
141
+ * table of links. Anything else in the paragraph keeps it verbatim.
142
+ * @param {string} p
143
+ * @param {string} base
144
+ * @returns {string | null}
145
+ */
146
+ function linkList(p, base) {
147
+ const items = [];
148
+ for (const line of p.split('\n')) {
149
+ const m = line.match(LINK_ITEM);
150
+ if (!m) return null;
151
+ let href;
152
+ try {
153
+ href = new URL(m[2], base);
154
+ } catch {
155
+ return null;
156
+ }
157
+ if (href.protocol !== 'https:' && href.protocol !== 'http:') return null;
158
+ // The description prints on a line of its own under the link, where the
159
+ // `: ` that joined it to the title reads as noise.
160
+ const text = m[3].replace(/^\s*[:-]\s*/, '').trim();
161
+ items.push(`<li><a href="${escHTML(href.href)}">${escHTML(m[1])}</a>${text ? ` ${escHTML(text)}` : ''}</li>`);
162
+ }
163
+ return `<ul>${items.join('')}</ul>`;
164
+ }
165
+
166
+ /**
167
+ * A plain text body as a page: a source file, a manual page, a README
168
+ * fetched raw. The HTML parser eats everything from a `<` to the next `>`,
169
+ * so a JavaScript file lost its generics and comparisons and came out as
170
+ * one shuffled block. Each blank-line-separated paragraph becomes its own
171
+ * block, keeping its line breaks, and a Markdown heading becomes a heading,
172
+ * so `find` and `read` work on a tldr page or an RFC the way they do on a
173
+ * page. The raw modes keep the whole file as one block, verbatim.
174
+ * @param {string} text
175
+ * @param {string} url
176
+ * @param {{full?: boolean, type?: string}} [opts] - type is the declared Content-Type
177
+ * @returns {string | null}
178
+ */
179
+ function textToHTML(text, url = '', { full = false, type = '' } = {}) {
180
+ const lead = text.replace(/^\uFEFF/, '').trimStart();
181
+ if (!lead || /^<(!doctype|html)[\s>]/i.test(lead)) return null;
182
+ // A README often opens with a centred logo in a <div>, and read as HTML
183
+ // everything after it was loose text the page reader dropped. A path that
184
+ // names a text format, or a server that says the body is text, is text
185
+ // unless it is a whole HTML document: a forum post quoting a <br> is not
186
+ // a page.
187
+ const declared = /^text\/(plain|markdown)\b/i.test(type) || TEXT_PATH.test(pathOf(url));
188
+ if (!declared && (lead.startsWith('<') || HTML_MARK.test(lead.slice(0, 4000)))) return null;
189
+ const body = lead.replace(/\r\n?/g, '\n').trimEnd();
190
+ const paragraphs = full ? [body] : body.split(/\n[ \t]*\n+/);
191
+ const heading = (p) => /^#{1,6} \S/.test(p) && !p.includes('\n') ? p.match(/^#+/)[0].length : 0;
192
+ const parts = paragraphs.map((p) => {
193
+ const level = heading(p);
194
+ if (level) return `<h${level}>${escHTML(p.slice(level + 1))}</h${level}>`;
195
+ if (full) return `<pre>${escHTML(p)}</pre>`;
196
+ // Expo's llms.txt writes each section heading on the line right above
197
+ // its list, with no blank line between, so the two arrive as one
198
+ // paragraph. The heading line is then a heading and the rest a list.
199
+ const cut = p.indexOf('\n');
200
+ const top = cut > 0 ? heading(p.slice(0, cut)) : 0;
201
+ const list = linkList(top ? p.slice(cut + 1) : p, url);
202
+ if (!list) return `<pre>${escHTML(p)}</pre>`;
203
+ return top ? `<h${top}>${escHTML(p.slice(top + 1, cut))}</h${top}>${list}` : list;
204
+ });
205
+ // Only a heading that opens the file names it. A README that starts with
206
+ // prose and has a `# More information` section further down is not titled
207
+ // "More information". A Setext heading (a line of = under the text) counts.
208
+ const opening = paragraphs[0] ?? '';
209
+ const setext = opening.match(/^([^\n]+)\n=+[ \t]*(?:\n|$)/);
210
+ let title = heading(opening) ? opening.replace(/^#+ /, '') : setext ? setext[1].trim() : '';
211
+ if (!title) {
212
+ try {
213
+ title = `${new URL(url).host}${new URL(url).pathname}`;
214
+ } catch {
215
+ title = 'text';
216
+ }
217
+ }
218
+ // The raw modes find the file again by this mark and hand it back whole.
219
+ const main = full ? '<main data-text>' : '<main>';
220
+ return `<html><head><title>${escHTML(title)}</title></head><body>${main}\n${parts.join('\n')}\n</main></body></html>`;
221
+ }
111
222
 
112
223
  /**
113
224
  * The link a heading is, if it is one. A search engine puts the result title
@@ -166,10 +277,14 @@ function isFollowableHref(href, base, pageUrl) {
166
277
  * numbers.
167
278
  * @param {string} html
168
279
  * @param {string} url
280
+ * @param {{view?: View|null, type?: string}} [opts] - the Content-Type the
281
+ * server declared, and a site shortcut's choice of JSON fields,
282
+ * which only a JSON body reads
169
283
  * @returns {Page}
170
284
  */
171
- export function distill(html, url = '') {
172
- const { document } = parseHTML(asHTML(html, url));
285
+ export function distill(html, url = '', { view = null, type = '' } = {}) {
286
+ const { document } = parseHTML(asHTML(html, url, { view, type }));
287
+ lowerAttributeNames(document);
173
288
  const title = clean(document.querySelector('title')?.textContent ?? '');
174
289
  const base = documentBase(document, url);
175
290
  /** @type {Block[]} */
@@ -178,7 +293,7 @@ export function distill(html, url = '') {
178
293
  const hidden = (el) =>
179
294
  el.getAttribute('hidden') !== null ||
180
295
  el.getAttribute('aria-hidden') === 'true' ||
181
- /display:\s*none/.test(el.getAttribute('style') ?? '');
296
+ DISPLAY_NONE.test(el.getAttribute('style') ?? '');
182
297
 
183
298
  /** Subtree already emitted, skipped when the rest of the page is walked. */
184
299
  let done = null;
@@ -233,6 +348,12 @@ export function distill(html, url = '') {
233
348
  // space that was never in the page.
234
349
  if (text) {
235
350
  blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
351
+ } else if (raw && blocks.at(-1)?.type === 'text') {
352
+ // A node that is only a space still separates the words either side
353
+ // of it. Wikipedia writes every &nbsp; as a span holding nothing
354
+ // else, and the fragments around it share a parent, so without this
355
+ // an infobox read "Firstappeared" and "20February 1991".
356
+ blocks.at(-1).post = true;
236
357
  }
237
358
  return;
238
359
  }
@@ -413,7 +534,14 @@ function mainOf(document, body) {
413
534
  const found = [...document.querySelectorAll(selector)];
414
535
  if (found.length !== 1) continue;
415
536
  const el = found[0];
416
- if (el !== body && prose(el) >= total * MIN_SHARE) return el;
537
+ if (el === body || prose(el) < total * MIN_SHARE) continue;
538
+ // Material for MkDocs puts both sidebars, the page nav and the table of
539
+ // contents, inside <main>, so a docs page led with fifty nav links. A
540
+ // main that holds navigation and exactly one article holding most of its
541
+ // prose leads with the article; the rest still follows. Walking down to
542
+ // the densest column instead cut pkg.go.dev to its biggest type.
543
+ const articles = el.querySelector('nav') ? el.querySelectorAll('article') : [];
544
+ return articles.length === 1 && prose(articles[0]) >= prose(el) * KEEP_SHARE ? articles[0] : el;
417
545
  }
418
546
  return densest(body, total);
419
547
  }
@@ -488,16 +616,44 @@ function number(blocks) {
488
616
  */
489
617
  const bodyOf = (document) => document.querySelector('body') ?? document.documentElement;
490
618
 
619
+ /** CSS property names and keywords are case-insensitive: `DISPLAY : NONE` hides too. */
620
+ const DISPLAY_NONE = /display\s*:\s*none/i;
621
+
622
+ /**
623
+ * HTML attribute names are ASCII case-insensitive, but linkedom keeps them as
624
+ * written, so `<A HREF>` had no href, `<INPUT TYPE="hidden">` no type and
625
+ * `<P STYLE="display:none">` no style. Lowercase them once after parsing so
626
+ * every lookup below sees what a browser sees. As in a browser, the first of
627
+ * two spellings of one attribute wins.
628
+ * @param {Document} document
629
+ */
630
+ function lowerAttributeNames(document) {
631
+ for (const el of document.querySelectorAll('*')) {
632
+ const attributes = [...el.attributes];
633
+ if (attributes.every(({ name }) => name === name.toLowerCase())) continue;
634
+ const first = new Map();
635
+ for (const { name, value } of attributes) {
636
+ const lower = name.toLowerCase();
637
+ if (!first.has(lower)) first.set(lower, value);
638
+ }
639
+ for (const { name } of attributes) el.removeAttribute(name);
640
+ for (const [name, value] of first) el.setAttribute(name, value);
641
+ }
642
+ }
643
+
491
644
  /**
492
645
  * Shared cleanup for the raw modes: parse, then delete the same noise
493
646
  * distill() skips, so neither raw output ever leaks scripts, styles, or
494
647
  * hidden content.
495
648
  * @param {string} html
649
+ * @param {string} [url]
650
+ * @param {string} [type] - the declared Content-Type
496
651
  */
497
- function cleanDocument(html, url = '') {
652
+ function cleanDocument(html, url = '', type = '') {
498
653
  // Raw is the mode an agent reaches for when the compact view left something
499
654
  // out, so it is the one place a JSON response keeps every field.
500
- const { document } = parseHTML(asHTML(html, url, { full: true }));
655
+ const { document } = parseHTML(asHTML(html, url, { full: true, type }));
656
+ lowerAttributeNames(document);
501
657
  // Read the title before the sweep below removes the head with it.
502
658
  const title = clean(document.querySelector('title')?.textContent ?? '');
503
659
  for (const tag of DROP) {
@@ -508,7 +664,7 @@ function cleanDocument(html, url = '') {
508
664
  if ((el.getAttribute('type') ?? '').toLowerCase() === 'hidden') el.remove();
509
665
  }
510
666
  for (const el of [...document.querySelectorAll('[style]')]) {
511
- if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
667
+ if (DISPLAY_NONE.test(el.getAttribute('style') ?? '')) el.remove();
512
668
  }
513
669
  return { document, title };
514
670
  }
@@ -519,10 +675,15 @@ function cleanDocument(html, url = '') {
519
675
  * @param {string} html
520
676
  * @param {string} url - only read when the body turns out to be JSON, whose
521
677
  * title has to come from the endpoint because the payload has none
678
+ * @param {{type?: string}} [opts] - the declared Content-Type
522
679
  * @returns {string}
523
680
  */
524
- export function toMarkdown(html, url = '') {
525
- const { document, title } = cleanDocument(html, url);
681
+ export function toMarkdown(html, url = '', { type = '' } = {}) {
682
+ const { document, title } = cleanDocument(html, url, type);
683
+ // A plain text file is already the markdown of itself. Run through the
684
+ // converter it came back with every `#` and backtick escaped.
685
+ const file = document.querySelector('main[data-text] > pre');
686
+ if (file) return file.textContent;
526
687
  const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
527
688
  const el = bodyOf(document);
528
689
  const body = el ? turndown.turndown(el.innerHTML).trim() : '';
@@ -534,10 +695,11 @@ export function toMarkdown(html, url = '') {
534
695
  * work with markup than markdown. Same noise removal, no other rewriting.
535
696
  * @param {string} html
536
697
  * @param {string} url - see toMarkdown
698
+ * @param {{type?: string}} [opts] - see toMarkdown
537
699
  * @returns {string}
538
700
  */
539
- export function toHTML(html, url = '') {
540
- const { document } = cleanDocument(html, url);
701
+ export function toHTML(html, url = '', { type = '' } = {}) {
702
+ const { document } = cleanDocument(html, url, type);
541
703
  const el = bodyOf(document);
542
704
  return el ? el.innerHTML.trim() : '';
543
705
  }
@@ -595,7 +757,7 @@ export function feedToHTML(text) {
595
757
  || rssLink(entry)
596
758
  || (guidIsPermalink ? field(entry, 'guid') : '');
597
759
  const author = field(entry, 'author name') || field(entry, 'author');
598
- const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
760
+ const date = feedDate(field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate'));
599
761
  const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
600
762
  // Atom escapes the entry body, so textContent of content/summary is the
601
763
  // HTML itself, ready to be embedded and parsed like any page.
@@ -623,6 +785,27 @@ export function feedToHTML(text) {
623
785
  return `<html><head><title>${esc(feedTitle)}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
624
786
  }
625
787
 
788
+ const MONTHS = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'];
789
+ const RFC822_DATE = /^(?:[a-z]{3},\s*)?(\d{1,2})\s+([a-z]{3})\s+(\d{4})\b/i;
790
+
791
+ /**
792
+ * An entry's date as YYYY-MM-DD. Atom writes ISO 8601, so the day is its
793
+ * first ten characters. RSS 2.0 writes RFC 822 ('Mon, 17 Aug 2026 00:00:00
794
+ * GMT'), and ten characters of that are 'Mon, 17 Au'. Its day, month, and
795
+ * year are read as written rather than through Date, which would move a late
796
+ * evening post to the next day in UTC; the Atom slice keeps the feed's own
797
+ * day too. Anything else is printed as the feed wrote it, not cut.
798
+ * @param {string} s
799
+ * @returns {string}
800
+ */
801
+ function feedDate(s) {
802
+ if (/^\d{4}-\d\d-\d\d/.test(s)) return s.slice(0, 10);
803
+ const m = s.match(RFC822_DATE);
804
+ const month = m ? MONTHS.indexOf(m[2].toLowerCase()) + 1 : 0;
805
+ if (!month) return s;
806
+ return `${m[3]}-${String(month).padStart(2, '0')}-${m[1].padStart(2, '0')}`;
807
+ }
808
+
626
809
  const escHTML = (s) => String(s ?? '').replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;').replace(/"/g, '&quot;');
627
810
 
628
811
  /**
@@ -776,6 +959,7 @@ const BODY_CAP = 4000;
776
959
  const EPOCH_S = [1e9, 4e9];
777
960
  const EPOCH_MS = [1e12, 4e12];
778
961
  const DATE_KEY = /(^|_)(date|at|time|timestamp|created|updated|published|modified)$/i;
962
+ const ISO_DATE = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d+)?(Z|[+-]\d\d:?\d\d)$/;
779
963
 
780
964
  // Named entities worth knowing without a table: the five XML ones plus the
781
965
  // space. Everything else arrives numeric.
@@ -854,6 +1038,11 @@ function renderValue(key, value) {
854
1038
  : null;
855
1039
  if (ms !== null) return new Date(ms).toISOString().slice(0, 19).replace('T', ' ');
856
1040
  }
1041
+ // An ISO timestamp under the same keys reads the same way: the fraction
1042
+ // and zone suffix are a third of its characters and never the point.
1043
+ if (typeof value === 'string' && DATE_KEY.test(key) && ISO_DATE.test(value)) {
1044
+ return value.slice(0, 19).replace('T', ' ');
1045
+ }
857
1046
  return typeof value === 'string' ? decodeEntities(value) : String(value);
858
1047
  }
859
1048
 
@@ -965,11 +1154,12 @@ function constantFields(rows) {
965
1154
  * Returns null for anything that is not JSON.
966
1155
  * @param {string} text
967
1156
  * @param {string} url
968
- * @param {{full?: boolean}} [opts] - full keeps every field, which is what the
969
- * raw modes are for; the compact view keeps the ones that earn their tokens
1157
+ * @param {{full?: boolean, view?: View|null}} [opts] - full keeps every
1158
+ * field, which is what the raw modes are for; the compact view keeps the
1159
+ * ones that earn their tokens, or the ones a site shortcut's view names
970
1160
  * @returns {string | null}
971
1161
  */
972
- export function jsonToHTML(text, url = '', { full = false } = {}) {
1162
+ export function jsonToHTML(text, url = '', { full = false, view = null } = {}) {
973
1163
  if (!/^\s*[[{]/.test(text.slice(0, 200))) return null;
974
1164
  let data;
975
1165
  try {
@@ -979,6 +1169,14 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
979
1169
  }
980
1170
  if (!data || typeof data !== 'object') return null;
981
1171
 
1172
+ // A shortcut that names its fields knows the response better than the
1173
+ // scoring below can guess, so its choice wins, except in the raw modes,
1174
+ // which exist to show what a choice left out.
1175
+ if (!full && view?.keep?.length) {
1176
+ const chosen = chosenToHTML(data, url, view);
1177
+ if (chosen) return chosen;
1178
+ }
1179
+
982
1180
  const { items, meta } = mainArray(data);
983
1181
  const flats = items.map((item) => (isPlain(item) ? flattenItem(item) : null));
984
1182
  const rows = flats.map((flat) => {
@@ -1094,6 +1292,163 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
1094
1292
  return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
1095
1293
  }
1096
1294
 
1295
+ /**
1296
+ * Read a dot path out of a response ('info.version', 'versions.0.license').
1297
+ * The empty path is the response itself, for an API whose answer is the
1298
+ * bare list.
1299
+ * @param {any} obj
1300
+ * @param {string} path
1301
+ * @returns {any}
1302
+ */
1303
+ export const pick = (obj, path) =>
1304
+ path === '' ? obj : String(path).split('.').reduce((o, key) => (o == null ? undefined : o[key]), obj);
1305
+
1306
+ const SLOT = /\{([^{}]+)\}/g;
1307
+
1308
+ /**
1309
+ * One chosen field as a string. Booleans read as flags: a true one prints the
1310
+ * field's own name and a false one nothing, so a dependency row reads
1311
+ * `bytes ^1 normal optional` instead of repeating `optional: false` on every
1312
+ * line. An object of scalars (npm's `dependencies`, PyPI's `project_urls`)
1313
+ * prints as `key=value` pairs, since naming the field was the request to see
1314
+ * inside it.
1315
+ * @param {string} path
1316
+ * @param {any} value
1317
+ * @returns {string}
1318
+ */
1319
+ function renderField(path, value) {
1320
+ const key = path.split('.').pop() ?? path;
1321
+ if (typeof value === 'boolean') return value ? key : '';
1322
+ if (isPlain(value)) {
1323
+ return Object.entries(value)
1324
+ .filter(([, v]) => v !== null && typeof v !== 'object')
1325
+ .map(([k, v]) => `${k}=${renderValue(k, v)}`)
1326
+ .join(', ');
1327
+ }
1328
+ return renderValue(key, value);
1329
+ }
1330
+
1331
+ /**
1332
+ * Fill the `{path}` slots of a template from one item. A template whose
1333
+ * every slot came up empty is empty itself rather than its own literal text,
1334
+ * so a changed API reads as missing fields and not as a page of bare
1335
+ * punctuation.
1336
+ * @param {string} template
1337
+ * @param {any} item
1338
+ * @returns {string}
1339
+ */
1340
+ export function fillTemplate(template, item) {
1341
+ let filled = 0;
1342
+ const text = template.replace(SLOT, (_, path) => {
1343
+ const value = renderField(path, pick(item, path));
1344
+ if (value) filled++;
1345
+ return value;
1346
+ });
1347
+ return filled ? clean(text) : '';
1348
+ }
1349
+
1350
+ /**
1351
+ * The absolute URL a filled template names, when the template itself starts
1352
+ * as one: a site path ("/raw/{id}") or an https URL. Only the template's own
1353
+ * prefix counts, so a field value can never turn plain text into a link or
1354
+ * point one at another host.
1355
+ * @param {string} template
1356
+ * @param {string} text - the filled template
1357
+ * @param {string} base - the endpoint URL, to resolve a site path
1358
+ * @returns {string}
1359
+ */
1360
+ function templateLink(template, text, base) {
1361
+ if (!/^(\/[^/{]|https:\/\/[^/{])/.test(template) || /\s/.test(text)) return '';
1362
+ try {
1363
+ const href = new URL(text, base);
1364
+ return href.protocol === 'https:' || href.protocol === 'http:' ? href.href : '';
1365
+ } catch {
1366
+ return '';
1367
+ }
1368
+ }
1369
+
1370
+ /**
1371
+ * The view a site shortcut asked for: exactly the fields it named, in the
1372
+ * order it named them, for the one object the response is or for each item
1373
+ * of the list it points at. This exists because the generic view below has
1374
+ * to guess, and on registry data it guesses wrong: PyPI's package endpoint
1375
+ * rendered the wheel files and their hashes instead of the version, and a
1376
+ * crates.io dependency list showed row ids and dropped the field that says
1377
+ * whether a dependency is dev-only.
1378
+ *
1379
+ * A field holding a URL becomes a link of its own, so `oc do` follows it. A
1380
+ * name-like first field is the item's title and drops its label. Short
1381
+ * fields that follow each other share a line, the way the generic view
1382
+ * joins them. Returns null when no item held any of the named fields (an
1383
+ * error body, an API that changed shape), so the response renders untouched
1384
+ * and the real reason stays visible.
1385
+ * @param {any} data
1386
+ * @param {string} url
1387
+ * @param {View} view
1388
+ * @returns {string | null}
1389
+ */
1390
+ function chosenToHTML(data, url, view) {
1391
+ const list = view.list == null ? (Array.isArray(data) ? data : [data]) : pick(data, view.list);
1392
+ if (!Array.isArray(list)) return null;
1393
+ const parts = [];
1394
+ for (const item of list) {
1395
+ if (!isPlain(item)) continue;
1396
+ /** @type {Array<{text: string, label?: string, link?: string}>} */
1397
+ const entries = [];
1398
+ for (const field of view.keep) {
1399
+ if (field.includes('{')) {
1400
+ const text = fillTemplate(field, item);
1401
+ // A template that is a URL on the site ("/raw/{id}") is the item's
1402
+ // link: a forum's topic list names no URL of its own to follow.
1403
+ const link = text && templateLink(field, text, url);
1404
+ if (link) entries.push({ text: link, link });
1405
+ else if (text) entries.push({ text });
1406
+ continue;
1407
+ }
1408
+ const value = pick(item, field);
1409
+ const text = renderField(field, value);
1410
+ if (!text) continue;
1411
+ const label = field.split('.').pop() ?? field;
1412
+ if (isURL(text)) entries.push({ text, label, link: text });
1413
+ else entries.push({ text, label: typeof value === 'boolean' ? undefined : label });
1414
+ }
1415
+ if (!entries.length) continue;
1416
+ parts.push('<article>');
1417
+ const [first, ...rest] = entries;
1418
+ // A name-like first field is the item's title: bare, on a line of its own.
1419
+ const titled = first.label && !first.link && TITLE_KEYS.includes(first.label) && first.text.length <= TITLE_MAX;
1420
+ if (titled) parts.push(`<p>${escHTML(first.text)}</p>`);
1421
+ const inline = [];
1422
+ let width = 0;
1423
+ const flush = () => {
1424
+ if (inline.length) parts.push(`<p>${escHTML(inline.join(' | '))}</p>`);
1425
+ inline.length = 0;
1426
+ width = 0;
1427
+ };
1428
+ for (const entry of titled ? rest : entries) {
1429
+ const line = entry.label ? `${entry.label}: ${entry.text}` : entry.text;
1430
+ if (entry.link) {
1431
+ flush();
1432
+ parts.push(`<p><a href="${escHTML(entry.link)}">${escHTML(line)}</a></p>`);
1433
+ } else if (line.length > TEXT_CAP) {
1434
+ flush();
1435
+ parts.push(`<p>${escHTML(line)}</p>`);
1436
+ } else {
1437
+ // A shared line that outgrew the cap would be cut, and these fields
1438
+ // were asked for by name, so the line wraps instead.
1439
+ if (inline.length && width + line.length + 3 > TEXT_CAP) flush();
1440
+ width = inline.length ? width + line.length + 3 : line.length;
1441
+ inline.push(line);
1442
+ }
1443
+ }
1444
+ flush();
1445
+ parts.push('</article>');
1446
+ }
1447
+ if (!parts.length) return null;
1448
+ const count = `${list.length} ${list.length === 1 ? 'item' : 'items'}`;
1449
+ return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
1450
+ }
1451
+
1097
1452
  /**
1098
1453
  * An API response has no title of its own, so the endpoint becomes one. The
1099
1454
  * query parameter is worth the tokens it costs: it is the only part of a
@@ -1144,9 +1499,11 @@ function mergeText(blocks) {
1144
1499
  out.push(b);
1145
1500
  }
1146
1501
  }
1147
- // Single stray characters (list bullets, separators) cost tokens and say nothing.
1502
+ // Single stray characters (list bullets, separators) cost tokens and say
1503
+ // nothing. A single letter or digit is a value, though: a table cell holds
1504
+ // one as a block of its own, and an infobox read "Children" with no number.
1148
1505
  return out
1149
1506
  .filter((b) => b.type !== 'break')
1150
- .filter((b) => b.type !== 'text' || b.text.length > 1)
1507
+ .filter((b) => b.type !== 'text' || b.text.length > 1 || /[\p{L}\p{N}]/u.test(b.text))
1151
1508
  .map(({ host, pre, post, ...block }) => block);
1152
1509
  }
package/src/fetch.js CHANGED
@@ -617,20 +617,30 @@ export function proxyGet(url, proxy, headers = {}, tlsOpts = {}) {
617
617
  * Fetch a page.
618
618
  * @param {string} url - with or without a scheme, https is assumed
619
619
  * @param {{ jar?: { cookieHeaderFor(url: string): string|undefined, storeFromResponse(url: string, headers: string[]): void } }} [opts]
620
- * @returns {Promise<{url: string, html: string, status: number, via: string}>}
621
- * final URL after redirects, the body, the HTTP status, and which client
622
- * identity got the page (impers:chrome, impers:firefox, or fetch)
620
+ * @returns {Promise<{url: string, html: string, status: number, via: string, type: string}>}
621
+ * final URL after redirects, the body, the HTTP status, which client
622
+ * identity got the page (impers:chrome, impers:firefox, or fetch), and the
623
+ * Content-Type the server declared
623
624
  */
624
625
  export async function fetchPage(url, { jar } = {}) {
625
- let target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
626
- // A reader with no cookies for reddit.com gets the feed where the page
627
- // would be a login wall; one who logged in gets the page it asked for.
628
- if (!jar?.cookieHeaderFor(target)) target = redditFeedURL(target) ?? target;
626
+ const target = readableURL(/^https?:\/\//i.test(url) ? url : `https://${url}`, jar);
629
627
  await assertSafeTarget(target);
630
628
  const impers = await loadImpers();
631
629
  return impers ? viaImpers(impers, target, jar) : viaFetch(target, jar);
632
630
  }
633
631
 
632
+ /**
633
+ * The URL to ask for in place of this one. A reader with no cookies for
634
+ * reddit.com gets the feed where the page would be a login wall; one who
635
+ * logged in gets the page it asked for. Every redirect hop goes through this
636
+ * too: redd.it short links and the app's /r/<sub>/s/<code> share links answer
637
+ * with a 301 to the post page, the very wall the feed steps around.
638
+ * @param {string} url
639
+ * @param {{ cookieHeaderFor(url: string): string|undefined }} [jar]
640
+ * @returns {string}
641
+ */
642
+ const readableURL = (url, jar) => (jar?.cookieHeaderFor(url) ? url : redditFeedURL(url) ?? url);
643
+
634
644
  /**
635
645
  * Follow redirects one hop at a time, validating each destination before the
636
646
  * next request goes out.
@@ -643,9 +653,11 @@ export async function fetchPage(url, { jar } = {}) {
643
653
  * process.
644
654
  * @param {(url: string) => Promise<any>} get - one request, redirects not followed
645
655
  * @param {string} start
656
+ * @param {{onResponse?: (url: string, res: any) => void, rewrite?: (url: string) => string}} [opts] -
657
+ * rewrite names the URL to ask for in place of a hop's Location
646
658
  * @returns {Promise<{res: any, url: string}>} the first non-redirect response
647
659
  */
648
- export async function followRedirects(get, start, { onResponse } = {}) {
660
+ export async function followRedirects(get, start, { onResponse, rewrite } = {}) {
649
661
  let current = start;
650
662
  for (let i = 0; ; i++) {
651
663
  if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
@@ -654,7 +666,8 @@ export async function followRedirects(get, start, { onResponse } = {}) {
654
666
  const status = res.status ?? res.statusCode ?? 0;
655
667
  const location = res.headers.get('location');
656
668
  if (status >= 300 && status < 400 && location) {
657
- current = new URL(location, current).toString();
669
+ const next = new URL(location, current).toString();
670
+ current = rewrite ? rewrite(next) : next;
658
671
  await assertSafeTarget(current);
659
672
  continue;
660
673
  }
@@ -687,8 +700,9 @@ function captureSetCookie(jar, url, res) {
687
700
  // letting firefox through. reddit.com started doing this in 2026 (#52), so
688
701
  // starting with chrome there would turn every read into two requests against
689
702
  // a per-address rate limit of about ten a minute. Subdomains inherit the
690
- // entry.
691
- const FIREFOX_FIRST_HOSTS = ['reddit.com'];
703
+ // entry. A redd.it short link is a 301 to a reddit.com post, and the identity
704
+ // is picked once for the whole chain, so it starts the same way.
705
+ const FIREFOX_FIRST_HOSTS = ['reddit.com', 'redd.it'];
692
706
 
693
707
  // Reddit has sent logged-out readers of its HTML pages to a login page since
694
708
  // June 2026 (#52), while the Atom feed beside each of those pages still
@@ -757,7 +771,7 @@ const bufferedContent = (res) => {
757
771
  * @param {any} impers - the impers module (or a stand-in with a get method)
758
772
  * @param {string} target
759
773
  * @param {object} [jar]
760
- * @returns {Promise<{url: string, html: string, status: number, via: string}>}
774
+ * @returns {Promise<{url: string, html: string, status: number, via: string, type: string}>}
761
775
  */
762
776
  export async function viaImpers(impers, target, jar) {
763
777
  // A blocked first attempt gets one cheap retry with the other identity. An
@@ -777,9 +791,10 @@ export async function viaImpers(impers, target, jar) {
777
791
  headers: jarHeaders(jar, url, {}),
778
792
  });
779
793
  const onResponse = (url, res) => captureSetCookie(jar, url, res);
794
+ const rewrite = (url) => readableURL(url, jar);
780
795
  const attempt = async (impersonate) => {
781
796
  try {
782
- const { res } = await followRedirects(asking(impersonate), target, { onResponse });
797
+ const { res } = await followRedirects(asking(impersonate), target, { onResponse, rewrite });
783
798
  return { res, status: res.status ?? res.statusCode ?? 0 };
784
799
  } catch (err) {
785
800
  if (err?.name !== 'ImpersonateError') throw err;
@@ -810,11 +825,11 @@ export async function viaImpers(impers, target, jar) {
810
825
  // this is the only size this transport ever really checks. Refused before
811
826
  // the decode so an oversized body is not allocated twice over.
812
827
  assertBodySize(bytes.length, target);
813
- return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via };
828
+ return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via, type: res.headers.get('content-type') ?? '' };
814
829
  }
815
830
  const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
816
831
  assertBodySize(html.length, target);
817
- return { url: res.url ?? target, html, status, via };
832
+ return { url: res.url ?? target, html, status, via, type: res.headers.get('content-type') ?? '' };
818
833
  }
819
834
 
820
835
  async function viaFetch(target, jar) {
@@ -826,13 +841,15 @@ async function viaFetch(target, jar) {
826
841
  : fetch(url, { redirect: 'manual', headers });
827
842
  };
828
843
  const onResponse = (url, res) => captureSetCookie(jar, url, res);
829
- const { res, url: current } = await followRedirects(get, target, { onResponse });
844
+ const rewrite = (url) => readableURL(url, jar);
845
+ const { res, url: current } = await followRedirects(get, target, { onResponse, rewrite });
830
846
  if (!res.ok) {
831
847
  throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
832
848
  }
833
849
  assertReadableType(res.headers.get('content-type'));
834
850
  assertBodySize(Number(res.headers.get('content-length')) || 0, current);
835
- return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch' };
851
+ return { url: res.url || current, html: await readBody(res, current), status: res.status, via: 'fetch',
852
+ type: res.headers.get('content-type') ?? '' };
836
853
  }
837
854
 
838
855
  /**