@only-cli/oc 0.5.5 → 0.5.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@only-cli/oc",
3
- "version": "0.5.5",
3
+ "version": "0.5.6",
4
4
  "description": "Turn websites into a compact CLI so AI agents can browse without burning tokens.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -8,17 +8,17 @@ description: Token-efficient web browsing and web content extraction for AI agen
8
8
  Renders a web page as a compact, numbered terminal view instead of raw HTML. A typical page is under 500 tokens.
9
9
 
10
10
  ```
11
- npx --yes @only-cli/oc@0.5.5 open <url> compact view, numbered elements
12
- npx --yes @only-cli/oc@0.5.5 do <n> follow link [n], or read it if [n] is text
13
- npx --yes @only-cli/oc@0.5.5 find <query> where a string appears, or that place itself
11
+ npx --yes @only-cli/oc@0.5.6 open <url> compact view, numbered elements
12
+ npx --yes @only-cli/oc@0.5.6 do <n> follow link [n], or read it if [n] is text
13
+ npx --yes @only-cli/oc@0.5.6 find <query> where a string appears, or that place itself
14
14
  when only one matches
15
- npx --yes @only-cli/oc@0.5.5 next next ~500 tokens of the page already open
16
- npx --yes @only-cli/oc@0.5.5 read <n> full text of region [n]
17
- npx --yes @only-cli/oc@0.5.5 raw [url] whole page as markdown (--html for cleaned HTML)
18
- npx --yes @only-cli/oc@0.5.5 login seed cookies (--cookie, --domain, --expires)
19
- npx --yes @only-cli/oc@0.5.5 logout [session] forget a session: cookies and saved page
20
- npx --yes @only-cli/oc@0.5.5 session ls list saved sessions (name, url, title)
21
- npx --yes @only-cli/oc@0.5.5 session rm [name] forget a saved session: page and cookies
15
+ npx --yes @only-cli/oc@0.5.6 next next ~500 tokens of the page already open
16
+ npx --yes @only-cli/oc@0.5.6 read <n> full text of region [n]
17
+ npx --yes @only-cli/oc@0.5.6 raw [url] whole page as markdown (--html for cleaned HTML)
18
+ npx --yes @only-cli/oc@0.5.6 login seed cookies (--cookie, --domain, --expires)
19
+ npx --yes @only-cli/oc@0.5.6 logout [session] forget a session: cookies and saved page
20
+ npx --yes @only-cli/oc@0.5.6 session ls list saved sessions (name, url, title)
21
+ npx --yes @only-cli/oc@0.5.6 session rm [name] forget a saved session: page and cookies
22
22
  ```
23
23
 
24
24
  None of these except `open`/`do`/`raw <url>` fetch anything; they replay the page `open` already saved.
package/src/cookies.js CHANGED
@@ -325,6 +325,20 @@ function domainMatches(cookie, host) {
325
325
  return host === d || host.endsWith(`.${d}`);
326
326
  }
327
327
 
328
+ /**
329
+ * RFC 6265 stores one cookie per name, domain, and path. Domain matching is
330
+ * how a cookie is sent, not how it is replaced: a host-only cookie on
331
+ * www.example.com is a different cookie from one scoped to example.com, and
332
+ * Path=/app is a different cookie from Path=/.
333
+ * @param {Cookie} a
334
+ * @param {Cookie} b
335
+ */
336
+ function sameCookie(a, b) {
337
+ return a.name === b.name
338
+ && a.domain.toLowerCase().replace(/^\./, '') === b.domain.toLowerCase().replace(/^\./, '')
339
+ && (a.path || '/') === (b.path || '/');
340
+ }
341
+
328
342
  /**
329
343
  * @param {Cookie} cookie
330
344
  * @param {string} path
@@ -367,6 +381,11 @@ export function cookieHeaderFor(jar, urlStr) {
367
381
  return scopeMatches(c, jar, host, path);
368
382
  });
369
383
  if (!active.length) return undefined;
384
+ // RFC 6265 5.4 order: the longest path first, and for equal paths the
385
+ // cookie stored first. A server that reads one value for a name is meant to
386
+ // get the most specific cookie, which is what a browser sends it. The sort
387
+ // is stable, so jar order carries the tie the way creation time would.
388
+ active.sort((a, b) => (b.path || '/').length - (a.path || '/').length);
370
389
  return active.map((c) => `${c.name}=${c.value}`).join('; ');
371
390
  }
372
391
 
@@ -483,22 +502,26 @@ export function storeFromResponse(jar, url, setCookieHeaders) {
483
502
  for (const header of setCookieHeaders) {
484
503
  const parsed = parseSetCookie(header, url);
485
504
  if (!parsed) continue;
505
+ const at = cookies.findIndex((c) => sameCookie(c, parsed));
486
506
  // Max-Age=0 or Expires in the past deletes the cookie
487
507
  if (parsed.expires && Date.parse(parsed.expires) <= Date.now()) {
488
- cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
508
+ if (at >= 0) cookies.splice(at, 1);
489
509
  continue;
490
510
  }
491
- cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
492
511
  // Replacing a cookie the jar already holds is always allowed; growing past
493
512
  // the cap is not, so a page cannot bloat the sidecar with fresh names. The
494
513
  // cookies already there - the seeded login among them - are what survive.
495
- if (cookies.length >= MAX_COOKIES) continue;
514
+ if (at < 0 && cookies.length >= MAX_COOKIES) continue;
496
515
  if (parsed.expires) {
497
516
  const ceiling = Date.parse(jar.expiresAt);
498
517
  const exp = Date.parse(parsed.expires);
499
518
  if (exp > ceiling) parsed.expires = jar.expiresAt;
500
519
  }
501
- cookies.push(parsed);
520
+ // A refresh keeps the slot it already had. RFC 6265 carries the original
521
+ // creation time onto the new value, and jar order is what stands in for
522
+ // that when cookieHeaderFor breaks a tie between equal-length paths.
523
+ if (at >= 0) cookies[at] = parsed;
524
+ else cookies.push(parsed);
502
525
  }
503
526
  return { ...jar, cookies };
504
527
  }
package/src/distill.js CHANGED
@@ -14,6 +14,7 @@ import TurndownService from 'turndown';
14
14
  * @property {string} url
15
15
  * @property {string} title
16
16
  * @property {Block[]} blocks
17
+ * @property {string} [base] - document base URL from <base href>, if the page set one
17
18
  */
18
19
 
19
20
  // Where the compact view cuts a text block. It lives here because numbering
@@ -143,6 +144,7 @@ function headingHref(node, text) {
143
144
  export function distill(html, url = '') {
144
145
  const { document } = parseHTML(asHTML(html, url));
145
146
  const title = clean(document.querySelector('title')?.textContent ?? '');
147
+ const base = documentBase(document, url);
146
148
  /** @type {Block[]} */
147
149
  const blocks = [];
148
150
 
@@ -221,7 +223,15 @@ export function distill(html, url = '') {
221
223
  return;
222
224
  }
223
225
  if (tag === 'a' && node.getAttribute('href')) {
224
- const text = clean(node.textContent);
226
+ // An icon button already takes aria-label or title because it has no
227
+ // text. An anchor around an image is the same shape: img alt is the
228
+ // accessible name and is not a text node, so textContent is empty and
229
+ // the link used to be dropped. Visible text still wins when there is
230
+ // any, because that is what the page wrote for a reader.
231
+ const text = clean(node.textContent)
232
+ || clean(node.getAttribute('aria-label') ?? '')
233
+ || clean(node.getAttribute('title') ?? '')
234
+ || clean(node.querySelector('img')?.getAttribute('alt') ?? '');
225
235
  if (text) {
226
236
  blocks.push({ type: 'link', text, href: node.getAttribute('href') });
227
237
  }
@@ -290,7 +300,33 @@ export function distill(html, url = '') {
290
300
  } else if (body) {
291
301
  walk(body);
292
302
  }
293
- return { url, title, blocks: number(mergeText(dropRepeats(blocks))) };
303
+ return {
304
+ url,
305
+ title,
306
+ blocks: number(mergeText(dropRepeats(blocks))),
307
+ ...(base !== url ? { base } : {}),
308
+ };
309
+ }
310
+
311
+ /**
312
+ * Document base URL from the first <base href>, resolved against the page
313
+ * URL. Relative links on the page are relative to this, not to the request
314
+ * URL: a mirror, a CDN, and many docs generators put one in the head.
315
+ * Head is dropped as chrome, so the href is read here and carried on the
316
+ * page for sessionFromPage, which is what `oc do` resolves against. An
317
+ * unusable href leaves the page URL in charge.
318
+ * @param {any} document
319
+ * @param {string} url
320
+ * @returns {string}
321
+ */
322
+ function documentBase(document, url) {
323
+ const href = clean(document.querySelector('base[href]')?.getAttribute('href') ?? '');
324
+ if (!href) return url;
325
+ try {
326
+ return new URL(href, url || undefined).href;
327
+ } catch {
328
+ return url;
329
+ }
294
330
  }
295
331
 
296
332
  /**
@@ -490,19 +526,44 @@ export function feedToHTML(text) {
490
526
  // <category /> stay open and swallow their siblings. Descendant queries
491
527
  // still land, because a closing </entry> or </item> pops the whole pile.
492
528
  const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
529
+ // HTML treats <link> as void, so an RSS 2.0 URL written as the element's
530
+ // text is not in textContent: the parser closes the tag immediately and the
531
+ // URL sits in the following text node. Skipping that node made `do` on a
532
+ // WordPress or HN item follow the guid instead, which is often an internal
533
+ // id, or follow nothing when the item has no guid at all.
534
+ const rssLink = (entry) => {
535
+ const link = entry.querySelector('link');
536
+ if (!link || link.getAttribute('href')) return '';
537
+ let n = link.nextSibling;
538
+ while (n && n.nodeType === 3 && !/\S/.test(n.textContent ?? '')) n = n.nextSibling;
539
+ const text = n?.nodeType === 3 ? clean(n.textContent ?? '') : '';
540
+ return text && !/\s/.test(text) ? text : '';
541
+ };
493
542
  const feedTitle = field(root, 'title');
494
543
  const parts = [];
495
544
  for (const entry of root.querySelectorAll('entry, item')) {
496
545
  const title = field(entry, 'title');
546
+ // Atom puts the URL on link href. RSS 2.0 writes it as <link> text, which
547
+ // rssLink reads. guid is only a permalink when it says so (the default),
548
+ // and isPermaLink="false" means the value is an id, not a URL.
549
+ const guid = entry.querySelector('guid');
550
+ const guidIsPermalink = !guid
551
+ || (guid.getAttribute('ispermalink') ?? 'true').toLowerCase() !== 'false';
497
552
  const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
498
- ?? entry.querySelector('link[href]')?.getAttribute('href')
499
- ?? field(entry, 'guid');
553
+ || entry.querySelector('link[href]')?.getAttribute('href')
554
+ || rssLink(entry)
555
+ || (guidIsPermalink ? field(entry, 'guid') : '');
500
556
  const author = field(entry, 'author name') || field(entry, 'author');
501
557
  const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
502
558
  const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
503
559
  // Atom escapes the entry body, so textContent of content/summary is the
504
560
  // HTML itself, ready to be embedded and parsed like any page.
505
- const body = (entry.querySelector('content') ?? entry.querySelector('summary') ?? entry.querySelector('description'))?.textContent ?? '';
561
+ // RSS 2.0 keeps the full post in content:encoded and the excerpt in
562
+ // description, so the encoded body has to win or a feed is only ledes.
563
+ const body = (entry.querySelector('content\\:encoded')
564
+ ?? entry.querySelector('content')
565
+ ?? entry.querySelector('summary')
566
+ ?? entry.querySelector('description'))?.textContent ?? '';
506
567
  parts.push('<article>');
507
568
  // The title is the link. It used to sit beside the byline as an anchor
508
569
  // labelled "open", the same label on every entry, and the repeated-controls
package/src/fetch.js CHANGED
@@ -12,6 +12,7 @@ import http from 'node:http';
12
12
  import https from 'node:https';
13
13
  import net from 'node:net';
14
14
  import tls from 'node:tls';
15
+ import zlib from 'node:zlib';
15
16
 
16
17
  import { getSetCookieHeaders } from './cookies.js';
17
18
 
@@ -422,6 +423,26 @@ function authority(target) {
422
423
  return `${host}:${port}`;
423
424
  }
424
425
 
426
+ // Native fetch (undici) already decodes Content-Encoding. Node's http parser
427
+ // does not, so the proxy transport used to charset-decode gzip bytes as the
428
+ // page: binary noise an agent pays for. Only gzip is handled here; deflate
429
+ // and br are left as they arrive, because this path does not ask for them.
430
+ // maxOutputLength is what stops a tiny gzip from expanding past the cap,
431
+ // which the compressed Content-Length cannot see.
432
+ function decompressBody(bytes, encoding, url) {
433
+ const enc = encoding?.split(',')[0]?.trim().toLowerCase();
434
+ if (!bytes.length || (enc !== 'gzip' && enc !== 'x-gzip')) return bytes;
435
+ let out;
436
+ try {
437
+ out = zlib.gunzipSync(bytes, { maxOutputLength: MAX_BODY });
438
+ } catch (err) {
439
+ if (err.code === 'ERR_BUFFER_TOO_LARGE') assertBodySize(MAX_BODY + 1, url);
440
+ throw new Error(`failed to decode gzip body for ${url}: ${err.message}`);
441
+ }
442
+ assertBodySize(out.length, url);
443
+ return out;
444
+ }
445
+
425
446
  function wrapNodeResponse(res, url) {
426
447
  const headers = {
427
448
  get(name) {
@@ -452,7 +473,14 @@ function wrapNodeResponse(res, url) {
452
473
  }
453
474
  chunks.push(c);
454
475
  });
455
- res.on('end', () => resolve(decodeBody(Buffer.concat(chunks), headers.get('content-type'))));
476
+ res.on('end', () => {
477
+ try {
478
+ const bytes = decompressBody(Buffer.concat(chunks), headers.get('content-encoding'), url);
479
+ resolve(decodeBody(bytes, headers.get('content-type')));
480
+ } catch (err) {
481
+ reject(err);
482
+ }
483
+ });
456
484
  res.on('error', reject);
457
485
  });
458
486
  const status = res.statusCode ?? 0;
package/src/session.js CHANGED
@@ -144,7 +144,8 @@ export function sessionFromPage(page, previous, { cursor = 0 } = {}) {
144
144
  dropped++;
145
145
  continue;
146
146
  }
147
- const href = block.href ? resolveHref(block.href, page.url) : null;
147
+ // Relative hrefs are against <base href> when the page set one.
148
+ const href = block.href ? resolveHref(block.href, page.base || page.url) : null;
148
149
  blocks.push({
149
150
  type: block.type,
150
151
  text: block.text,
package/src/sites.js CHANGED
@@ -145,7 +145,12 @@ export function resolveSite(name, args) {
145
145
  */
146
146
  function encode(value, template, arg) {
147
147
  const query = template.includes('?') && template.indexOf(`{${arg}}`) > template.indexOf('?');
148
- const encoded = encodeURIComponent(value);
148
+ // A path slot that starts with @ is a copied handle (@name, @channel).
149
+ // encodeURIComponent would turn it into %40 and miss the user, and YouTube's
150
+ // template already has the @ in the URL. Search queries keep it, so a
151
+ // mention in a query is still the thing being searched for.
152
+ const raw = !query && value.startsWith('@') && value.length > 1 ? value.slice(1) : value;
153
+ const encoded = encodeURIComponent(raw);
149
154
  return query ? encoded : encoded.replaceAll('%2F', '/');
150
155
  }
151
156
 
package/src/sphinx.js CHANGED
@@ -84,11 +84,17 @@ function objectHits(index, query) {
84
84
  for (const [doc, typeIdx, priority, anchor, name] of entries) {
85
85
  const full = prefix ? `${prefix}.${name}` : name;
86
86
  if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
87
+ // Empty string: the object's own name is the HTML id. The sentinel '-'
88
+ // means Sphinx built '{objtype}-{fullname}' instead, which is how
89
+ // docs.python.org names every module heading (#module-json).
90
+ const id = anchor === '' ? full
91
+ : anchor === '-' ? `${index.objnames?.[typeIdx]?.[1] ?? ''}-${full}`
92
+ : anchor;
87
93
  hits.push({
88
94
  name: full,
89
95
  type: index.objnames?.[typeIdx]?.[2] ?? '',
90
96
  doc,
91
- anchor: anchor === '' ? full : anchor,
97
+ anchor: id,
92
98
  priority,
93
99
  });
94
100
  }