@only-cli/oc 0.5.5 → 0.5.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -29,6 +29,8 @@ npm install -g @only-cli/oc
29
29
 
30
30
  Requires Node 20+. Requests impersonate Chrome via [impers](https://github.com/lexiforest/impers). When a site, or the local copy of libcurl-impersonate, refuses the Chrome identity, the request downgrades to Firefox, and to native fetch when impers is unavailable or refuses both.
31
31
 
32
+ Red Hat OpenShift also installs an `oc`, and both define `login` and `logout` with unrelated meanings, so whichever lands first on `PATH` shadows the other. If you use OpenShift, call this one with `npx @only-cli/oc` instead of installing it globally, or alias the global install to another name.
33
+
32
34
  ### Agent skill
33
35
 
34
36
  Install the [web-browsing-cli skill](https://www.skills.sh/only-cli/oc/web-browsing-cli) for Claude Code, Cursor, Codex, Copilot, and other compatible agents:
@@ -139,7 +141,7 @@ Copy the `Cookie` header from your browser's devtools (Application → Cookies,
139
141
 
140
142
  Seeded cookies are https-only. They almost always come from an https browser session, so `oc` marks them secure and never sends them over plain `http`, including on a hop an `https` page redirects into, where you never typed the downgrade. A site that really is http-only needs `--allow-http` at login. Cookies a site sets over https are pinned the same way.
141
143
 
142
- Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, owner-only (mode `0600`, or on Windows the ACL the directory hands down), not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
144
+ Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, owner-only (mode `0600` in a `0700` directory, or on Windows the ACL the directory hands down), not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
143
145
 
144
146
  `oc logout` forgets the whole session, not just its cookies: a page saved under that name can hold the distilled text of something only the login could reach, so the snapshot goes with the jar. `oc session rm [name]` forgets a session the same way (page plus cookies) without switching to it first, and `oc session ls` lists what is on disk (`[cookies]` marks a live jar). State lives in `~/.only-cli` (`%USERPROFILE%\.only-cli` on Windows, override with `OC_HOME`), one JSON per session plus search-index caches. Delete the directory to start over.
145
147
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@only-cli/oc",
3
- "version": "0.5.5",
3
+ "version": "0.5.7",
4
4
  "description": "Turn websites into a compact CLI so AI agents can browse without burning tokens.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -8,17 +8,17 @@ description: Token-efficient web browsing and web content extraction for AI agen
8
8
  Renders a web page as a compact, numbered terminal view instead of raw HTML. A typical page is under 500 tokens.
9
9
 
10
10
  ```
11
- npx --yes @only-cli/oc@0.5.5 open <url> compact view, numbered elements
12
- npx --yes @only-cli/oc@0.5.5 do <n> follow link [n], or read it if [n] is text
13
- npx --yes @only-cli/oc@0.5.5 find <query> where a string appears, or that place itself
11
+ npx --yes @only-cli/oc@0.5.7 open <url> compact view, numbered elements
12
+ npx --yes @only-cli/oc@0.5.7 do <n> follow link [n], or read it if [n] is text
13
+ npx --yes @only-cli/oc@0.5.7 find <query> where a string appears, or that place itself
14
14
  when only one matches
15
- npx --yes @only-cli/oc@0.5.5 next next ~500 tokens of the page already open
16
- npx --yes @only-cli/oc@0.5.5 read <n> full text of region [n]
17
- npx --yes @only-cli/oc@0.5.5 raw [url] whole page as markdown (--html for cleaned HTML)
18
- npx --yes @only-cli/oc@0.5.5 login seed cookies (--cookie, --domain, --expires)
19
- npx --yes @only-cli/oc@0.5.5 logout [session] forget a session: cookies and saved page
20
- npx --yes @only-cli/oc@0.5.5 session ls list saved sessions (name, url, title)
21
- npx --yes @only-cli/oc@0.5.5 session rm [name] forget a saved session: page and cookies
15
+ npx --yes @only-cli/oc@0.5.7 next next ~500 tokens of the page already open
16
+ npx --yes @only-cli/oc@0.5.7 read <n> full text of region [n]
17
+ npx --yes @only-cli/oc@0.5.7 raw [url] whole page as markdown (--html for cleaned HTML)
18
+ npx --yes @only-cli/oc@0.5.7 login seed cookies (--cookie, --domain, --expires)
19
+ npx --yes @only-cli/oc@0.5.7 logout [session] forget a session: cookies and saved page
20
+ npx --yes @only-cli/oc@0.5.7 session ls list saved sessions (name, url, title)
21
+ npx --yes @only-cli/oc@0.5.7 session rm [name] forget a saved session: page and cookies
22
22
  ```
23
23
 
24
24
  None of these except `open`/`do`/`raw <url>` fetch anything; they replay the page `open` already saved.
package/src/act.js CHANGED
@@ -42,17 +42,31 @@ function requireBlocks(session) {
42
42
  return state;
43
43
  }
44
44
 
45
+ /**
46
+ * The number a handle was printed as. The compact view writes `[2]`, and
47
+ * agents copy that form into `oc do [2]` / `oc read [2]`. Number('[2]') is
48
+ * NaN, so the printed form used to fail the usage check.
49
+ * @param {unknown} value
50
+ * @returns {number}
51
+ */
52
+ export function parseHandle(value) {
53
+ const s = String(value ?? '').trim();
54
+ const m = /^\[(\d+)\]$/.exec(s) || /^(\d+)$/.exec(s);
55
+ return m ? Number(m[1]) : NaN;
56
+ }
57
+
45
58
  /**
46
59
  * Resolve a numbered handle from the last render into something to open.
47
60
  * Returns the target URL; the caller fetches and renders it exactly as
48
61
  * `oc open` would, so `do` and `open` always agree on what a page looks like.
49
62
  * When the number is text rather than a link it returns `{read}` instead, and
50
63
  * the caller reads it.
51
- * @param {number} n
64
+ * @param {unknown} n
52
65
  * @param {{session?: string}} [opts]
53
66
  * @returns {{url?: string, read?: number, text: string}}
54
67
  */
55
68
  export function activate(n, { session = DEFAULT_SESSION } = {}) {
69
+ n = parseHandle(n);
56
70
  if (!Number.isInteger(n) || n < 1) {
57
71
  throw new Error('usage: oc do <n>, where <n> is a number from the last page');
58
72
  }
@@ -100,6 +114,7 @@ const TRAIL = 6;
100
114
  * @returns {string}
101
115
  */
102
116
  export function read(n, { session = DEFAULT_SESSION, budget = 2000 } = {}) {
117
+ n = parseHandle(n);
103
118
  if (!Number.isInteger(n) || n < 1) {
104
119
  throw new Error('usage: oc read <n>, where <n> is a number from the last page');
105
120
  }
@@ -230,7 +245,7 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
230
245
  // the example it names was in the block after it.
231
246
  if (hits.length === 1 && hits[0].n != null) {
232
247
  const only = hits[0];
233
- const follow = only.type === 'link' ? 'do <n> | ' : '';
248
+ const follow = only.type === 'link' || only.href ? 'do <n> | ' : '';
234
249
  return [
235
250
  `1 match for "${query}"${separately}, region [${only.n}]`,
236
251
  read(only.n, { session, budget: budget * FINISH }),
@@ -258,7 +273,7 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
258
273
  if (spent + cost > cap && shown) break;
259
274
  spent += cost;
260
275
  shown++;
261
- if (hit.type === 'link') hasLinks = true;
276
+ if (hit.type === 'link' || hit.href) hasLinks = true;
262
277
  lines.push(line);
263
278
  }
264
279
  if (shown < hits.length) {
@@ -298,7 +313,7 @@ function search(blocks, terms) {
298
313
  // are read rather than indexed: the whole-match mode above, and `read`.
299
314
  const window = block.text.slice(start, end).replace(/\n/g, ' ');
300
315
  const snippet = `${start > 0 ? '... ' : ''}${window}${end < block.text.length ? ' ...' : ''}`;
301
- out.push({ n, type: block.type, snippet, text: block.text });
316
+ out.push({ n, type: block.type, snippet, text: block.text, href: block.href });
302
317
  }
303
318
  return out;
304
319
  }
package/src/cli.js CHANGED
@@ -237,7 +237,7 @@ async function main() {
237
237
  // view's footer offers when it has cut something.
238
238
  let url;
239
239
  if (command === 'do') {
240
- const target = act.activate(Number(args[0]), { session: sessionName });
240
+ const target = act.activate(args[0], { session: sessionName });
241
241
  // A number that points at text has no page behind it, so `do` reads it
242
242
  // rather than making the agent pay for a second command to be told.
243
243
  if (target.read != null) {
@@ -369,7 +369,7 @@ async function main() {
369
369
  }
370
370
  return;
371
371
  }
372
- case 'read': return console.log(act.read(Number(args[0]), { session: sessionName, budget: asked || 2000 }));
372
+ case 'read': return console.log(act.read(args[0], { session: sessionName, budget: asked || 2000 }));
373
373
  case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 }));
374
374
  case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 }));
375
375
  case 'fill': return act.fill(Number(args[0]), args.slice(1).join(' '));
package/src/cookies.js CHANGED
@@ -325,6 +325,20 @@ function domainMatches(cookie, host) {
325
325
  return host === d || host.endsWith(`.${d}`);
326
326
  }
327
327
 
328
+ /**
329
+ * RFC 6265 stores one cookie per name, domain, and path. Domain matching is
330
+ * how a cookie is sent, not how it is replaced: a host-only cookie on
331
+ * www.example.com is a different cookie from one scoped to example.com, and
332
+ * Path=/app is a different cookie from Path=/.
333
+ * @param {Cookie} a
334
+ * @param {Cookie} b
335
+ */
336
+ function sameCookie(a, b) {
337
+ return a.name === b.name
338
+ && a.domain.toLowerCase().replace(/^\./, '') === b.domain.toLowerCase().replace(/^\./, '')
339
+ && (a.path || '/') === (b.path || '/');
340
+ }
341
+
328
342
  /**
329
343
  * @param {Cookie} cookie
330
344
  * @param {string} path
@@ -367,6 +381,11 @@ export function cookieHeaderFor(jar, urlStr) {
367
381
  return scopeMatches(c, jar, host, path);
368
382
  });
369
383
  if (!active.length) return undefined;
384
+ // RFC 6265 5.4 order: the longest path first, and for equal paths the
385
+ // cookie stored first. A server that reads one value for a name is meant to
386
+ // get the most specific cookie, which is what a browser sends it. The sort
387
+ // is stable, so jar order carries the tie the way creation time would.
388
+ active.sort((a, b) => (b.path || '/').length - (a.path || '/').length);
370
389
  return active.map((c) => `${c.name}=${c.value}`).join('; ');
371
390
  }
372
391
 
@@ -392,6 +411,38 @@ export function withheldForScheme(jar, urlStr) {
392
411
  return jar.cookies.some((c) => c.secure && scopeMatches(c, jar, host, path));
393
412
  }
394
413
 
414
+ /**
415
+ * An ISO timestamp, or undefined when the instant is outside what Date can
416
+ * name. A huge Max-Age used to throw RangeError from toISOString and abort
417
+ * the render of a page that was otherwise fine.
418
+ * @param {number} ms
419
+ * @returns {string | undefined}
420
+ */
421
+ function expiryISO(ms) {
422
+ try {
423
+ return new Date(ms).toISOString();
424
+ } catch {
425
+ return undefined;
426
+ }
427
+ }
428
+
429
+ /**
430
+ * Strip one pair of wrapping double quotes from a Path. ASP.NET and several
431
+ * Java containers send Path="/admin"; storing the quotes left pathMatches
432
+ * matching nothing, so the cookie was silently never sent again and the login
433
+ * looked gone on the page it was scoped to.
434
+ *
435
+ * Only Path gets this. A cookie value is stored and echoed back verbatim, as
436
+ * RFC 6265 5.2 and every browser have it, as jarFromCookieHeader already
437
+ * treats a seeded one, and because quoting is how a server protects a value
438
+ * holding a space or a comma.
439
+ * @param {string} s
440
+ * @returns {string}
441
+ */
442
+ function unquotePath(s) {
443
+ return s.length >= 2 && s.startsWith('"') && s.endsWith('"') ? s.slice(1, -1) : s;
444
+ }
445
+
395
446
  /**
396
447
  * Parse one Set-Cookie header value.
397
448
  * @param {string} header
@@ -425,6 +476,16 @@ export function parseSetCookie(header, requestUrl) {
425
476
  ...(url.protocol === 'https:' ? { secure: true } : {}),
426
477
  };
427
478
 
479
+ // Max-Age and Expires are collected separately: RFC 6265 5.3 says Max-Age
480
+ // wins even when Expires is written last. Express writes them in that order,
481
+ // and Expires is the server's clock, so a host whose clock is behind would
482
+ // otherwise look expired the moment it was set.
483
+ /** @type {string | undefined} */
484
+ let maxAgeExpires;
485
+ let sawMaxAge = false;
486
+ /** @type {string | undefined} */
487
+ let expiresAt;
488
+
428
489
  for (const attr of parts.slice(1)) {
429
490
  const sep = attr.indexOf('=');
430
491
  const key = (sep === -1 ? attr : attr.slice(0, sep)).trim().toLowerCase();
@@ -436,7 +497,7 @@ export function parseSetCookie(header, requestUrl) {
436
497
  // dependency this project will not take. User-seeded cookies still scope by
437
498
  // the --domain they pass, which normalizeDomain holds to the same floor.
438
499
  if (key === 'path') {
439
- cookie.path = val || '/';
500
+ cookie.path = unquotePath(val) || '/';
440
501
  } else if (key === 'secure') {
441
502
  cookie.secure = true;
442
503
  } else if (key === 'httponly') {
@@ -444,13 +505,19 @@ export function parseSetCookie(header, requestUrl) {
444
505
  } else if (key === 'max-age') {
445
506
  const age = Number(val);
446
507
  if (Number.isFinite(age)) {
447
- cookie.expires = new Date(Date.now() + age * 1000).toISOString();
508
+ sawMaxAge = true;
509
+ maxAgeExpires = age <= 0 ? new Date(0).toISOString() : expiryISO(Date.now() + age * 1000);
448
510
  }
449
511
  } else if (key === 'expires') {
450
512
  const exp = Date.parse(val);
451
- if (!Number.isNaN(exp)) cookie.expires = new Date(exp).toISOString();
513
+ if (!Number.isNaN(exp)) expiresAt = expiryISO(exp);
452
514
  }
453
515
  }
516
+ if (sawMaxAge) {
517
+ if (maxAgeExpires) cookie.expires = maxAgeExpires;
518
+ } else if (expiresAt) {
519
+ cookie.expires = expiresAt;
520
+ }
454
521
  return cookie;
455
522
  }
456
523
 
@@ -483,22 +550,26 @@ export function storeFromResponse(jar, url, setCookieHeaders) {
483
550
  for (const header of setCookieHeaders) {
484
551
  const parsed = parseSetCookie(header, url);
485
552
  if (!parsed) continue;
553
+ const at = cookies.findIndex((c) => sameCookie(c, parsed));
486
554
  // Max-Age=0 or Expires in the past deletes the cookie
487
555
  if (parsed.expires && Date.parse(parsed.expires) <= Date.now()) {
488
- cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
556
+ if (at >= 0) cookies.splice(at, 1);
489
557
  continue;
490
558
  }
491
- cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
492
559
  // Replacing a cookie the jar already holds is always allowed; growing past
493
560
  // the cap is not, so a page cannot bloat the sidecar with fresh names. The
494
561
  // cookies already there - the seeded login among them - are what survive.
495
- if (cookies.length >= MAX_COOKIES) continue;
562
+ if (at < 0 && cookies.length >= MAX_COOKIES) continue;
496
563
  if (parsed.expires) {
497
564
  const ceiling = Date.parse(jar.expiresAt);
498
565
  const exp = Date.parse(parsed.expires);
499
566
  if (exp > ceiling) parsed.expires = jar.expiresAt;
500
567
  }
501
- cookies.push(parsed);
568
+ // A refresh keeps the slot it already had. RFC 6265 carries the original
569
+ // creation time onto the new value, and jar order is what stands in for
570
+ // that when cookieHeaderFor breaks a tie between equal-length paths.
571
+ if (at >= 0) cookies[at] = parsed;
572
+ else cookies.push(parsed);
502
573
  }
503
574
  return { ...jar, cookies };
504
575
  }
package/src/distill.js CHANGED
@@ -14,6 +14,7 @@ import TurndownService from 'turndown';
14
14
  * @property {string} url
15
15
  * @property {string} title
16
16
  * @property {Block[]} blocks
17
+ * @property {string} [base] - document base URL from <base href>, if the page set one
17
18
  */
18
19
 
19
20
  // Where the compact view cuts a text block. It lives here because numbering
@@ -118,19 +119,46 @@ const asHTML = (text, url = '', opts = {}) =>
118
119
  * Every heading in the Rust book and every one on an AWS CLI reference page
119
120
  * carries a permalink to its own id, so following those would refetch the
120
121
  * page the agent is already reading, which is worse than the reading it
121
- * already gets. A bare fragment is never a destination.
122
+ * already gets. A bare fragment is never a destination, and neither is a
123
+ * mailto: or tel: href, which is not a page.
122
124
  * @param {any} node - the heading element
123
125
  * @param {string} text - its cleaned text
126
+ * @param {string} base - document base, to resolve the href
127
+ * @param {string} pageUrl - the page itself, for same-document checks
124
128
  * @returns {string|null}
125
129
  */
126
- function headingHref(node, text) {
130
+ function headingHref(node, text, base, pageUrl) {
127
131
  const anchors = node.querySelectorAll('a[href]');
128
132
  if (anchors.length !== 1) return null;
129
133
  const href = anchors[0].getAttribute('href') ?? '';
130
- if (!href || href.startsWith('#')) return null;
134
+ if (!href || !isFollowableHref(href, base, pageUrl)) return null;
131
135
  return clean(anchors[0].textContent) === text ? href : null;
132
136
  }
133
137
 
138
+ /**
139
+ * Whether this href is a page oc can fetch, other than the one already open.
140
+ * A bare fragment is the current page. mailto:/tel:/javascript: are not
141
+ * pages: fetchPage prefixes https:// onto a URL with no http(s) scheme, so
142
+ * mailto:hi@example.com became a GET of example.com with password "hi".
143
+ * The href resolves against <base href>, but "here" is the page URL: under
144
+ * <base href="/">, a link to / is the home page, not this one.
145
+ * @param {string} href
146
+ * @param {string} base
147
+ * @param {string} pageUrl
148
+ * @returns {boolean}
149
+ */
150
+ function isFollowableHref(href, base, pageUrl) {
151
+ try {
152
+ const dest = new URL(href, base || pageUrl || undefined);
153
+ if (!/^https?:$/i.test(dest.protocol)) return false;
154
+ if (!pageUrl) return true;
155
+ const here = new URL(pageUrl);
156
+ return dest.origin !== here.origin || dest.pathname !== here.pathname || dest.search !== here.search;
157
+ } catch {
158
+ return false;
159
+ }
160
+ }
161
+
134
162
  /**
135
163
  * Reduce raw HTML to an interaction tree: readable text plus numbered
136
164
  * elements, in document order. The walk is deterministic and numbering is a
@@ -143,6 +171,7 @@ function headingHref(node, text) {
143
171
  export function distill(html, url = '') {
144
172
  const { document } = parseHTML(asHTML(html, url));
145
173
  const title = clean(document.querySelector('title')?.textContent ?? '');
174
+ const base = documentBase(document, url);
146
175
  /** @type {Block[]} */
147
176
  const blocks = [];
148
177
 
@@ -215,20 +244,40 @@ export function distill(html, url = '') {
215
244
  if (/^h[1-6]$/.test(tag)) {
216
245
  const text = clean(node.textContent);
217
246
  if (text) {
218
- const href = headingHref(node, text);
247
+ const href = headingHref(node, text, base, url);
219
248
  blocks.push({ type: 'heading', level: Number(tag[1]), text, ...(href ? { href } : {}) });
220
249
  }
221
250
  return;
222
251
  }
223
252
  if (tag === 'a' && node.getAttribute('href')) {
224
- const text = clean(node.textContent);
253
+ const href = node.getAttribute('href');
254
+ // Same rule as headingHref: a fragment is this page, mailto: is not a
255
+ // page. Walk children so the label stays readable text instead of a
256
+ // numbered handle that do cannot fetch.
257
+ if (!isFollowableHref(href, base, url)) {
258
+ for (const child of node.childNodes) walk(child);
259
+ return;
260
+ }
261
+ // An icon button already takes aria-label or title because it has no
262
+ // text. An anchor around an image is the same shape: img alt is the
263
+ // accessible name and is not a text node, so textContent is empty and
264
+ // the link used to be dropped. Visible text still wins when there is
265
+ // any, because that is what the page wrote for a reader.
266
+ const text = clean(node.textContent)
267
+ || clean(node.getAttribute('aria-label') ?? '')
268
+ || clean(node.getAttribute('title') ?? '')
269
+ || clean(node.querySelector('img')?.getAttribute('alt') ?? '');
225
270
  if (text) {
226
- blocks.push({ type: 'link', text, href: node.getAttribute('href') });
271
+ blocks.push({ type: 'link', text, href });
227
272
  }
228
273
  return;
229
274
  }
230
275
  if (tag === 'input' || tag === 'textarea' || tag === 'select') {
231
- const kind = node.getAttribute('type') ?? 'text';
276
+ // HTML's type attribute is ASCII case-insensitive. type=HIDDEN used to
277
+ // appear as a numbered input, type=SUBMIT as an input rather than a
278
+ // button, and type=PASSWORD as a field named PASSWORD, so a login wall
279
+ // written that way was not detected.
280
+ const kind = (node.getAttribute('type') ?? 'text').toLowerCase();
232
281
  if (kind === 'hidden') return;
233
282
  if (kind === 'submit' || kind === 'button') {
234
283
  blocks.push({ type: 'button', text: node.getAttribute('value') ?? 'submit' });
@@ -290,7 +339,33 @@ export function distill(html, url = '') {
290
339
  } else if (body) {
291
340
  walk(body);
292
341
  }
293
- return { url, title, blocks: number(mergeText(dropRepeats(blocks))) };
342
+ return {
343
+ url,
344
+ title,
345
+ blocks: number(mergeText(dropRepeats(blocks))),
346
+ ...(base !== url ? { base } : {}),
347
+ };
348
+ }
349
+
350
+ /**
351
+ * Document base URL from the first <base href>, resolved against the page
352
+ * URL. Relative links on the page are relative to this, not to the request
353
+ * URL: a mirror, a CDN, and many docs generators put one in the head.
354
+ * Head is dropped as chrome, so the href is read here and carried on the
355
+ * page for sessionFromPage, which is what `oc do` resolves against. An
356
+ * unusable href leaves the page URL in charge.
357
+ * @param {any} document
358
+ * @param {string} url
359
+ * @returns {string}
360
+ */
361
+ function documentBase(document, url) {
362
+ const href = clean(document.querySelector('base[href]')?.getAttribute('href') ?? '');
363
+ if (!href) return url;
364
+ try {
365
+ return new URL(href, url || undefined).href;
366
+ } catch {
367
+ return url;
368
+ }
294
369
  }
295
370
 
296
371
  /**
@@ -429,7 +504,10 @@ function cleanDocument(html, url = '') {
429
504
  for (const tag of DROP) {
430
505
  for (const el of [...document.querySelectorAll(tag)]) el.remove();
431
506
  }
432
- for (const el of [...document.querySelectorAll('[hidden], [aria-hidden="true"], input[type="hidden"]')]) el.remove();
507
+ for (const el of [...document.querySelectorAll('[hidden], [aria-hidden="true"]')]) el.remove();
508
+ for (const el of [...document.querySelectorAll('input')]) {
509
+ if ((el.getAttribute('type') ?? '').toLowerCase() === 'hidden') el.remove();
510
+ }
433
511
  for (const el of [...document.querySelectorAll('[style]')]) {
434
512
  if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
435
513
  }
@@ -490,19 +568,44 @@ export function feedToHTML(text) {
490
568
  // <category /> stay open and swallow their siblings. Descendant queries
491
569
  // still land, because a closing </entry> or </item> pops the whole pile.
492
570
  const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
571
+ // HTML treats <link> as void, so an RSS 2.0 URL written as the element's
572
+ // text is not in textContent: the parser closes the tag immediately and the
573
+ // URL sits in the following text node. Skipping that node made `do` on a
574
+ // WordPress or HN item follow the guid instead, which is often an internal
575
+ // id, or follow nothing when the item has no guid at all.
576
+ const rssLink = (entry) => {
577
+ const link = entry.querySelector('link');
578
+ if (!link || link.getAttribute('href')) return '';
579
+ let n = link.nextSibling;
580
+ while (n && n.nodeType === 3 && !/\S/.test(n.textContent ?? '')) n = n.nextSibling;
581
+ const text = n?.nodeType === 3 ? clean(n.textContent ?? '') : '';
582
+ return text && !/\s/.test(text) ? text : '';
583
+ };
493
584
  const feedTitle = field(root, 'title');
494
585
  const parts = [];
495
586
  for (const entry of root.querySelectorAll('entry, item')) {
496
587
  const title = field(entry, 'title');
588
+ // Atom puts the URL on link href. RSS 2.0 writes it as <link> text, which
589
+ // rssLink reads. guid is only a permalink when it says so (the default),
590
+ // and isPermaLink="false" means the value is an id, not a URL.
591
+ const guid = entry.querySelector('guid');
592
+ const guidIsPermalink = !guid
593
+ || (guid.getAttribute('ispermalink') ?? 'true').toLowerCase() !== 'false';
497
594
  const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
498
- ?? entry.querySelector('link[href]')?.getAttribute('href')
499
- ?? field(entry, 'guid');
595
+ || entry.querySelector('link[href]')?.getAttribute('href')
596
+ || rssLink(entry)
597
+ || (guidIsPermalink ? field(entry, 'guid') : '');
500
598
  const author = field(entry, 'author name') || field(entry, 'author');
501
599
  const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
502
600
  const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
503
601
  // Atom escapes the entry body, so textContent of content/summary is the
504
602
  // HTML itself, ready to be embedded and parsed like any page.
505
- const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? '';
603
+ // RSS 2.0 keeps the full post in content:encoded and the excerpt in
604
+ // description, so the encoded body has to win or a feed is only ledes.
605
+ const body = (entry.querySelector('content\\:encoded')
606
+ ?? entry.querySelector('content')
607
+ ?? entry.querySelector('summary')
608
+ ?? entry.querySelector('description'))?.textContent ?? '';
506
609
  parts.push('<article>');
507
610
  // The title is the link. It used to sit beside the byline as an anchor
508
611
  // labelled "open", the same label on every entry, and the repeated-controls
package/src/fetch.js CHANGED
@@ -12,6 +12,7 @@ import http from 'node:http';
12
12
  import https from 'node:https';
13
13
  import net from 'node:net';
14
14
  import tls from 'node:tls';
15
+ import zlib from 'node:zlib';
15
16
 
16
17
  import { getSetCookieHeaders } from './cookies.js';
17
18
 
@@ -422,6 +423,26 @@ function authority(target) {
422
423
  return `${host}:${port}`;
423
424
  }
424
425
 
426
+ // Native fetch (undici) already decodes Content-Encoding. Node's http parser
427
+ // does not, so the proxy transport used to charset-decode gzip bytes as the
428
+ // page: binary noise an agent pays for. Only gzip is handled here; deflate
429
+ // and br are left as they arrive, because this path does not ask for them.
430
+ // maxOutputLength is what stops a tiny gzip from expanding past the cap,
431
+ // which the compressed Content-Length cannot see.
432
+ function decompressBody(bytes, encoding, url) {
433
+ const enc = encoding?.split(',')[0]?.trim().toLowerCase();
434
+ if (!bytes.length || (enc !== 'gzip' && enc !== 'x-gzip')) return bytes;
435
+ let out;
436
+ try {
437
+ out = zlib.gunzipSync(bytes, { maxOutputLength: MAX_BODY });
438
+ } catch (err) {
439
+ if (err.code === 'ERR_BUFFER_TOO_LARGE') assertBodySize(MAX_BODY + 1, url);
440
+ throw new Error(`failed to decode gzip body for ${url}: ${err.message}`);
441
+ }
442
+ assertBodySize(out.length, url);
443
+ return out;
444
+ }
445
+
425
446
  function wrapNodeResponse(res, url) {
426
447
  const headers = {
427
448
  get(name) {
@@ -452,7 +473,14 @@ function wrapNodeResponse(res, url) {
452
473
  }
453
474
  chunks.push(c);
454
475
  });
455
- res.on('end', () => resolve(decodeBody(Buffer.concat(chunks), headers.get('content-type'))));
476
+ res.on('end', () => {
477
+ try {
478
+ const bytes = decompressBody(Buffer.concat(chunks), headers.get('content-encoding'), url);
479
+ resolve(decodeBody(bytes, headers.get('content-type')));
480
+ } catch (err) {
481
+ reject(err);
482
+ }
483
+ });
456
484
  res.on('error', reject);
457
485
  });
458
486
  const status = res.statusCode ?? 0;
package/src/render.js CHANGED
@@ -127,7 +127,9 @@ export function render(page, { budget = 500, from = 0 } = {}) {
127
127
  if (spent + cost > limit && lines.length > head.length) break;
128
128
  spent += cost;
129
129
  lines.push(line);
130
- if (block.type === 'link' || block.type === 'button') hasLinks = true;
130
+ // A heading that is a link (search result titles) is what do follows, so
131
+ // it has to count here or the footer omits do on a page of only those.
132
+ if (block.type === 'link' || block.type === 'button' || block.href) hasLinks = true;
131
133
  }
132
134
 
133
135
  const rest = blocks.slice(i);
package/src/session.js CHANGED
@@ -104,16 +104,29 @@ function unwrapRedirect(url) {
104
104
 
105
105
  /**
106
106
  * Absolute URL for a handle, or null when the link is not followable
107
- * (javascript: handlers, malformed hrefs).
107
+ * (javascript: handlers, mailto:/tel:, same-document fragments, malformed
108
+ * hrefs). fetchPage prefixes https:// onto anything that is not already
109
+ * http(s), so mailto:hi@example.com used to become a GET of example.com
110
+ * with password "hi". A bare fragment is the page already open.
108
111
  * @param {string} href
109
- * @param {string} base
112
+ * @param {string} base - <base href>, or the page URL when it set none
113
+ * @param {string} [pageUrl] - the page itself, for same-document checks
110
114
  * @returns {string|null}
111
115
  */
112
- export function resolveHref(href, base) {
113
- if (!href || /^(javascript|about):/i.test(href)) return null;
116
+ export function resolveHref(href, base, pageUrl = base) {
117
+ if (!href) return null;
114
118
  try {
115
119
  const url = new URL(href, base || undefined);
116
- return unwrapRedirect(url) ?? url.href;
120
+ if (!/^https?:$/i.test(url.protocol)) return null;
121
+ const dest = unwrapRedirect(url) ?? url.href;
122
+ if (pageUrl) {
123
+ const here = new URL(pageUrl);
124
+ const there = new URL(dest);
125
+ if (here.origin === there.origin && here.pathname === there.pathname && here.search === there.search) {
126
+ return null;
127
+ }
128
+ }
129
+ return dest;
117
130
  } catch {
118
131
  return null;
119
132
  }
@@ -144,7 +157,8 @@ export function sessionFromPage(page, previous, { cursor = 0 } = {}) {
144
157
  dropped++;
145
158
  continue;
146
159
  }
147
- const href = block.href ? resolveHref(block.href, page.url) : null;
160
+ // Relative hrefs are against <base href> when the page set one.
161
+ const href = block.href ? resolveHref(block.href, page.base || page.url, page.url) : null;
148
162
  blocks.push({
149
163
  type: block.type,
150
164
  text: block.text,
package/src/sites.js CHANGED
@@ -145,7 +145,12 @@ export function resolveSite(name, args) {
145
145
  */
146
146
  function encode(value, template, arg) {
147
147
  const query = template.includes('?') && template.indexOf(`{${arg}}`) > template.indexOf('?');
148
- const encoded = encodeURIComponent(value);
148
+ // A path slot that starts with @ is a copied handle (@name, @channel).
149
+ // encodeURIComponent would turn it into %40 and miss the user, and YouTube's
150
+ // template already has the @ in the URL. Search queries keep it, so a
151
+ // mention in a query is still the thing being searched for.
152
+ const raw = !query && value.startsWith('@') && value.length > 1 ? value.slice(1) : value;
153
+ const encoded = encodeURIComponent(raw);
149
154
  return query ? encoded : encoded.replaceAll('%2F', '/');
150
155
  }
151
156
 
package/src/sphinx.js CHANGED
@@ -84,11 +84,17 @@ function objectHits(index, query) {
84
84
  for (const [doc, typeIdx, priority, anchor, name] of entries) {
85
85
  const full = prefix ? `${prefix}.${name}` : name;
86
86
  if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
87
+ // Empty string: the object's own name is the HTML id. The sentinel '-'
88
+ // means Sphinx built '{objtype}-{fullname}' instead, which is how
89
+ // docs.python.org names every module heading (#module-json).
90
+ const id = anchor === '' ? full
91
+ : anchor === '-' ? `${index.objnames?.[typeIdx]?.[1] ?? ''}-${full}`
92
+ : anchor;
87
93
  hits.push({
88
94
  name: full,
89
95
  type: index.objnames?.[typeIdx]?.[2] ?? '',
90
96
  doc,
91
- anchor: anchor === '' ? full : anchor,
97
+ anchor: id,
92
98
  priority,
93
99
  });
94
100
  }