@only-cli/oc 0.5.4 → 0.5.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +10 -10
- package/src/cli.js +1 -1
- package/src/cookies.js +30 -7
- package/src/distill.js +66 -5
- package/src/fetch.js +150 -3
- package/src/session.js +34 -4
- package/src/sites.js +6 -1
- package/src/sphinx.js +7 -1
package/README.md
CHANGED
|
@@ -139,7 +139,7 @@ Copy the `Cookie` header from your browser's devtools (Application → Cookies,
|
|
|
139
139
|
|
|
140
140
|
Seeded cookies are https-only. They almost always come from an https browser session, so `oc` marks them secure and never sends them over plain `http`, including on a hop an `https` page redirects into, where you never typed the downgrade. A site that really is http-only needs `--allow-http` at login. Cookies a site sets over https are pinned the same way.
|
|
141
141
|
|
|
142
|
-
Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, mode `0600`, not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
|
|
142
|
+
Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, owner-only (mode `0600`, or on Windows the ACL the directory hands down), not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
|
|
143
143
|
|
|
144
144
|
`oc logout` forgets the whole session, not just its cookies: a page saved under that name can hold the distilled text of something only the login could reach, so the snapshot goes with the jar. `oc session rm [name]` forgets a session the same way (page plus cookies) without switching to it first, and `oc session ls` lists what is on disk (`[cookies]` marks a live jar). State lives in `~/.only-cli` (`%USERPROFILE%\.only-cli` on Windows, override with `OC_HOME`), one JSON per session plus search-index caches. Delete the directory to start over.
|
|
145
145
|
|
package/package.json
CHANGED
|
@@ -8,17 +8,17 @@ description: Token-efficient web browsing and web content extraction for AI agen
|
|
|
8
8
|
Renders a web page as a compact, numbered terminal view instead of raw HTML. A typical page is under 500 tokens.
|
|
9
9
|
|
|
10
10
|
```
|
|
11
|
-
npx --yes @only-cli/oc@0.5.
|
|
12
|
-
npx --yes @only-cli/oc@0.5.
|
|
13
|
-
npx --yes @only-cli/oc@0.5.
|
|
11
|
+
npx --yes @only-cli/oc@0.5.6 open <url> compact view, numbered elements
|
|
12
|
+
npx --yes @only-cli/oc@0.5.6 do <n> follow link [n], or read it if [n] is text
|
|
13
|
+
npx --yes @only-cli/oc@0.5.6 find <query> where a string appears, or that place itself
|
|
14
14
|
when only one matches
|
|
15
|
-
npx --yes @only-cli/oc@0.5.
|
|
16
|
-
npx --yes @only-cli/oc@0.5.
|
|
17
|
-
npx --yes @only-cli/oc@0.5.
|
|
18
|
-
npx --yes @only-cli/oc@0.5.
|
|
19
|
-
npx --yes @only-cli/oc@0.5.
|
|
20
|
-
npx --yes @only-cli/oc@0.5.
|
|
21
|
-
npx --yes @only-cli/oc@0.5.
|
|
15
|
+
npx --yes @only-cli/oc@0.5.6 next next ~500 tokens of the page already open
|
|
16
|
+
npx --yes @only-cli/oc@0.5.6 read <n> full text of region [n]
|
|
17
|
+
npx --yes @only-cli/oc@0.5.6 raw [url] whole page as markdown (--html for cleaned HTML)
|
|
18
|
+
npx --yes @only-cli/oc@0.5.6 login seed cookies (--cookie, --domain, --expires)
|
|
19
|
+
npx --yes @only-cli/oc@0.5.6 logout [session] forget a session: cookies and saved page
|
|
20
|
+
npx --yes @only-cli/oc@0.5.6 session ls list saved sessions (name, url, title)
|
|
21
|
+
npx --yes @only-cli/oc@0.5.6 session rm [name] forget a saved session: page and cookies
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
None of these except `open`/`do`/`raw <url>` fetch anything; they replay the page `open` already saved.
|
package/src/cli.js
CHANGED
|
@@ -272,7 +272,7 @@ async function main() {
|
|
|
272
272
|
const processMs = performance.now() - t0 - fetchMs;
|
|
273
273
|
const rss = process.memoryUsage().rss;
|
|
274
274
|
return `HTTP ${status} via ${via}, fetch ${Math.round(fetchMs)}ms, process ${Math.round(processMs)}ms, `
|
|
275
|
-
+ `${Math.round(html
|
|
275
|
+
+ `${Math.round(Buffer.byteLength(html) / 1024)}KB transferred, ${Math.round(rss / 1048576)}MB memory`;
|
|
276
276
|
};
|
|
277
277
|
const htmlTokens = estimateTokens(html);
|
|
278
278
|
if (values.json) {
|
package/src/cookies.js
CHANGED
|
@@ -5,9 +5,9 @@
|
|
|
5
5
|
|
|
6
6
|
import net from 'node:net';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
|
-
import {
|
|
8
|
+
import { readFileSync, writeFileSync, unlinkSync, readdirSync, chmodSync } from 'node:fs';
|
|
9
9
|
|
|
10
|
-
import { sessionDir, assertSafeName, COOKIE_JAR_SUFFIX } from './session.js';
|
|
10
|
+
import { sessionDir, ensureSessionDir, assertSafeName, COOKIE_JAR_SUFFIX } from './session.js';
|
|
11
11
|
|
|
12
12
|
const DEFAULT_EXPIRES_MS = 60 * 60 * 1000; // 1h
|
|
13
13
|
export { DEFAULT_EXPIRES_MS };
|
|
@@ -256,7 +256,7 @@ export function loadCookieJar(name) {
|
|
|
256
256
|
* @param {CookieJar} jar
|
|
257
257
|
*/
|
|
258
258
|
export function saveCookieJar(name, jar) {
|
|
259
|
-
|
|
259
|
+
ensureSessionDir();
|
|
260
260
|
// writeFileSync only sets the mode on create, so an existing sidecar has its
|
|
261
261
|
// owner-only mode reasserted on every save.
|
|
262
262
|
const path = cookieJarPath(name);
|
|
@@ -325,6 +325,20 @@ function domainMatches(cookie, host) {
|
|
|
325
325
|
return host === d || host.endsWith(`.${d}`);
|
|
326
326
|
}
|
|
327
327
|
|
|
328
|
+
/**
|
|
329
|
+
* RFC 6265 stores one cookie per name, domain, and path. Domain matching is
|
|
330
|
+
* how a cookie is sent, not how it is replaced: a host-only cookie on
|
|
331
|
+
* www.example.com is a different cookie from one scoped to example.com, and
|
|
332
|
+
* Path=/app is a different cookie from Path=/.
|
|
333
|
+
* @param {Cookie} a
|
|
334
|
+
* @param {Cookie} b
|
|
335
|
+
*/
|
|
336
|
+
function sameCookie(a, b) {
|
|
337
|
+
return a.name === b.name
|
|
338
|
+
&& a.domain.toLowerCase().replace(/^\./, '') === b.domain.toLowerCase().replace(/^\./, '')
|
|
339
|
+
&& (a.path || '/') === (b.path || '/');
|
|
340
|
+
}
|
|
341
|
+
|
|
328
342
|
/**
|
|
329
343
|
* @param {Cookie} cookie
|
|
330
344
|
* @param {string} path
|
|
@@ -367,6 +381,11 @@ export function cookieHeaderFor(jar, urlStr) {
|
|
|
367
381
|
return scopeMatches(c, jar, host, path);
|
|
368
382
|
});
|
|
369
383
|
if (!active.length) return undefined;
|
|
384
|
+
// RFC 6265 5.4 order: the longest path first, and for equal paths the
|
|
385
|
+
// cookie stored first. A server that reads one value for a name is meant to
|
|
386
|
+
// get the most specific cookie, which is what a browser sends it. The sort
|
|
387
|
+
// is stable, so jar order carries the tie the way creation time would.
|
|
388
|
+
active.sort((a, b) => (b.path || '/').length - (a.path || '/').length);
|
|
370
389
|
return active.map((c) => `${c.name}=${c.value}`).join('; ');
|
|
371
390
|
}
|
|
372
391
|
|
|
@@ -483,22 +502,26 @@ export function storeFromResponse(jar, url, setCookieHeaders) {
|
|
|
483
502
|
for (const header of setCookieHeaders) {
|
|
484
503
|
const parsed = parseSetCookie(header, url);
|
|
485
504
|
if (!parsed) continue;
|
|
505
|
+
const at = cookies.findIndex((c) => sameCookie(c, parsed));
|
|
486
506
|
// Max-Age=0 or Expires in the past deletes the cookie
|
|
487
507
|
if (parsed.expires && Date.parse(parsed.expires) <= Date.now()) {
|
|
488
|
-
|
|
508
|
+
if (at >= 0) cookies.splice(at, 1);
|
|
489
509
|
continue;
|
|
490
510
|
}
|
|
491
|
-
cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
|
|
492
511
|
// Replacing a cookie the jar already holds is always allowed; growing past
|
|
493
512
|
// the cap is not, so a page cannot bloat the sidecar with fresh names. The
|
|
494
513
|
// cookies already there - the seeded login among them - are what survive.
|
|
495
|
-
if (cookies.length >= MAX_COOKIES) continue;
|
|
514
|
+
if (at < 0 && cookies.length >= MAX_COOKIES) continue;
|
|
496
515
|
if (parsed.expires) {
|
|
497
516
|
const ceiling = Date.parse(jar.expiresAt);
|
|
498
517
|
const exp = Date.parse(parsed.expires);
|
|
499
518
|
if (exp > ceiling) parsed.expires = jar.expiresAt;
|
|
500
519
|
}
|
|
501
|
-
|
|
520
|
+
// A refresh keeps the slot it already had. RFC 6265 carries the original
|
|
521
|
+
// creation time onto the new value, and jar order is what stands in for
|
|
522
|
+
// that when cookieHeaderFor breaks a tie between equal-length paths.
|
|
523
|
+
if (at >= 0) cookies[at] = parsed;
|
|
524
|
+
else cookies.push(parsed);
|
|
502
525
|
}
|
|
503
526
|
return { ...jar, cookies };
|
|
504
527
|
}
|
package/src/distill.js
CHANGED
|
@@ -14,6 +14,7 @@ import TurndownService from 'turndown';
|
|
|
14
14
|
* @property {string} url
|
|
15
15
|
* @property {string} title
|
|
16
16
|
* @property {Block[]} blocks
|
|
17
|
+
* @property {string} [base] - document base URL from <base href>, if the page set one
|
|
17
18
|
*/
|
|
18
19
|
|
|
19
20
|
// Where the compact view cuts a text block. It lives here because numbering
|
|
@@ -143,6 +144,7 @@ function headingHref(node, text) {
|
|
|
143
144
|
export function distill(html, url = '') {
|
|
144
145
|
const { document } = parseHTML(asHTML(html, url));
|
|
145
146
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
147
|
+
const base = documentBase(document, url);
|
|
146
148
|
/** @type {Block[]} */
|
|
147
149
|
const blocks = [];
|
|
148
150
|
|
|
@@ -221,7 +223,15 @@ export function distill(html, url = '') {
|
|
|
221
223
|
return;
|
|
222
224
|
}
|
|
223
225
|
if (tag === 'a' && node.getAttribute('href')) {
|
|
224
|
-
|
|
226
|
+
// An icon button already takes aria-label or title because it has no
|
|
227
|
+
// text. An anchor around an image is the same shape: img alt is the
|
|
228
|
+
// accessible name and is not a text node, so textContent is empty and
|
|
229
|
+
// the link used to be dropped. Visible text still wins when there is
|
|
230
|
+
// any, because that is what the page wrote for a reader.
|
|
231
|
+
const text = clean(node.textContent)
|
|
232
|
+
|| clean(node.getAttribute('aria-label') ?? '')
|
|
233
|
+
|| clean(node.getAttribute('title') ?? '')
|
|
234
|
+
|| clean(node.querySelector('img')?.getAttribute('alt') ?? '');
|
|
225
235
|
if (text) {
|
|
226
236
|
blocks.push({ type: 'link', text, href: node.getAttribute('href') });
|
|
227
237
|
}
|
|
@@ -290,7 +300,33 @@ export function distill(html, url = '') {
|
|
|
290
300
|
} else if (body) {
|
|
291
301
|
walk(body);
|
|
292
302
|
}
|
|
293
|
-
return {
|
|
303
|
+
return {
|
|
304
|
+
url,
|
|
305
|
+
title,
|
|
306
|
+
blocks: number(mergeText(dropRepeats(blocks))),
|
|
307
|
+
...(base !== url ? { base } : {}),
|
|
308
|
+
};
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Document base URL from the first <base href>, resolved against the page
|
|
313
|
+
* URL. Relative links on the page are relative to this, not to the request
|
|
314
|
+
* URL: a mirror, a CDN, and many docs generators put one in the head.
|
|
315
|
+
* Head is dropped as chrome, so the href is read here and carried on the
|
|
316
|
+
* page for sessionFromPage, which is what `oc do` resolves against. An
|
|
317
|
+
* unusable href leaves the page URL in charge.
|
|
318
|
+
* @param {any} document
|
|
319
|
+
* @param {string} url
|
|
320
|
+
* @returns {string}
|
|
321
|
+
*/
|
|
322
|
+
function documentBase(document, url) {
|
|
323
|
+
const href = clean(document.querySelector('base[href]')?.getAttribute('href') ?? '');
|
|
324
|
+
if (!href) return url;
|
|
325
|
+
try {
|
|
326
|
+
return new URL(href, url || undefined).href;
|
|
327
|
+
} catch {
|
|
328
|
+
return url;
|
|
329
|
+
}
|
|
294
330
|
}
|
|
295
331
|
|
|
296
332
|
/**
|
|
@@ -490,19 +526,44 @@ export function feedToHTML(text) {
|
|
|
490
526
|
// <category /> stay open and swallow their siblings. Descendant queries
|
|
491
527
|
// still land, because a closing </entry> or </item> pops the whole pile.
|
|
492
528
|
const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
|
|
529
|
+
// HTML treats <link> as void, so an RSS 2.0 URL written as the element's
|
|
530
|
+
// text is not in textContent: the parser closes the tag immediately and the
|
|
531
|
+
// URL sits in the following text node. Skipping that node made `do` on a
|
|
532
|
+
// WordPress or HN item follow the guid instead, which is often an internal
|
|
533
|
+
// id, or follow nothing when the item has no guid at all.
|
|
534
|
+
const rssLink = (entry) => {
|
|
535
|
+
const link = entry.querySelector('link');
|
|
536
|
+
if (!link || link.getAttribute('href')) return '';
|
|
537
|
+
let n = link.nextSibling;
|
|
538
|
+
while (n && n.nodeType === 3 && !/\S/.test(n.textContent ?? '')) n = n.nextSibling;
|
|
539
|
+
const text = n?.nodeType === 3 ? clean(n.textContent ?? '') : '';
|
|
540
|
+
return text && !/\s/.test(text) ? text : '';
|
|
541
|
+
};
|
|
493
542
|
const feedTitle = field(root, 'title');
|
|
494
543
|
const parts = [];
|
|
495
544
|
for (const entry of root.querySelectorAll('entry, item')) {
|
|
496
545
|
const title = field(entry, 'title');
|
|
546
|
+
// Atom puts the URL on link href. RSS 2.0 writes it as <link> text, which
|
|
547
|
+
// rssLink reads. guid is only a permalink when it says so (the default),
|
|
548
|
+
// and isPermaLink="false" means the value is an id, not a URL.
|
|
549
|
+
const guid = entry.querySelector('guid');
|
|
550
|
+
const guidIsPermalink = !guid
|
|
551
|
+
|| (guid.getAttribute('ispermalink') ?? 'true').toLowerCase() !== 'false';
|
|
497
552
|
const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
|
|
498
|
-
|
|
499
|
-
|
|
553
|
+
|| entry.querySelector('link[href]')?.getAttribute('href')
|
|
554
|
+
|| rssLink(entry)
|
|
555
|
+
|| (guidIsPermalink ? field(entry, 'guid') : '');
|
|
500
556
|
const author = field(entry, 'author name') || field(entry, 'author');
|
|
501
557
|
const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
|
|
502
558
|
const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
|
|
503
559
|
// Atom escapes the entry body, so textContent of content/summary is the
|
|
504
560
|
// HTML itself, ready to be embedded and parsed like any page.
|
|
505
|
-
|
|
561
|
+
// RSS 2.0 keeps the full post in content:encoded and the excerpt in
|
|
562
|
+
// description, so the encoded body has to win or a feed is only ledes.
|
|
563
|
+
const body = (entry.querySelector('content\\:encoded')
|
|
564
|
+
?? entry.querySelector('content')
|
|
565
|
+
?? entry.querySelector('summary')
|
|
566
|
+
?? entry.querySelector('description'))?.textContent ?? '';
|
|
506
567
|
parts.push('<article>');
|
|
507
568
|
// The title is the link. It used to sit beside the byline as an anchor
|
|
508
569
|
// labelled "open", the same label on every entry, and the repeated-controls
|
package/src/fetch.js
CHANGED
|
@@ -12,6 +12,7 @@ import http from 'node:http';
|
|
|
12
12
|
import https from 'node:https';
|
|
13
13
|
import net from 'node:net';
|
|
14
14
|
import tls from 'node:tls';
|
|
15
|
+
import zlib from 'node:zlib';
|
|
15
16
|
|
|
16
17
|
import { getSetCookieHeaders } from './cookies.js';
|
|
17
18
|
|
|
@@ -73,6 +74,103 @@ export function assertReadableType(type) {
|
|
|
73
74
|
throw new Error(`not a page oc can read (${type.split(';')[0].trim()}), it renders HTML, XML feeds, JSON, and plain text`);
|
|
74
75
|
}
|
|
75
76
|
|
|
77
|
+
// A body read as UTF-8 whatever it declares turns each non-ASCII character of
|
|
78
|
+
// a Shift_JIS, EUC-KR, GBK, or windows-1252 page into U+FFFD, noise an agent
|
|
79
|
+
// pays for and cannot read. So the encoding is looked for where a browser
|
|
80
|
+
// looks: the Content-Type header, then a <meta> tag, which the HTML spec
|
|
81
|
+
// requires within the first 1024 bytes, or a feed's XML declaration.
|
|
82
|
+
// The comma is excluded because a duplicated Content-Type header arrives from
|
|
83
|
+
// headers.get as one value joined by ", ", and a label with a comma stuck to
|
|
84
|
+
// its end is one TextDecoder refuses.
|
|
85
|
+
const CHARSET_PARAM = /;\s*charset\s*=\s*["']?([^"';,\s]+)/i;
|
|
86
|
+
const XML_ENCODING = /^\s*<\?xml\b[^>]*?\bencoding\s*=\s*["']([^"']+)/;
|
|
87
|
+
// One tag's attributes, quoted values kept whole, so a charset written inside
|
|
88
|
+
// another attribute's value (an og:url carrying a query string, say) stays
|
|
89
|
+
// part of that value instead of reading as a declaration.
|
|
90
|
+
const ATTRS = /([^\s=/>]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]*))?/g;
|
|
91
|
+
// Only markup declares an encoding in its own text, so only markup is read for
|
|
92
|
+
// one. JSON is UTF-8 by definition (RFC 8259), which is why a browser ignores
|
|
93
|
+
// a charset on it, and plain text or JavaScript carries no declaration a
|
|
94
|
+
// scan could find, only strings that look like one.
|
|
95
|
+
const MARKUP_TYPE = /^\s*(?:text\/(?:html|xml)|application\/(?:xml|[\w.+-]*\+xml))/i;
|
|
96
|
+
const JSON_TYPE = /^\s*(?:application\/(?:json|x-ndjson|[\w.+-]*\+json)|text\/json)/i;
|
|
97
|
+
|
|
98
|
+
// TextDecoder knows the labels browsers do, so no dependency is needed, and it
|
|
99
|
+
// refuses the ones they do not, which then fall through to the next place a
|
|
100
|
+
// label can come from.
|
|
101
|
+
const UTF8 = new TextDecoder();
|
|
102
|
+
const decoderFor = (label) => {
|
|
103
|
+
try {
|
|
104
|
+
return label ? new TextDecoder(label) : null;
|
|
105
|
+
} catch {
|
|
106
|
+
return null;
|
|
107
|
+
}
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* The charset one <meta> tag declares: the attribute browsers read first, then
|
|
112
|
+
* the older http-equiv form, which only counts on a content-type, not on any
|
|
113
|
+
* content attribute that happens to mention a charset.
|
|
114
|
+
* @param {string} tag
|
|
115
|
+
* @returns {string | null}
|
|
116
|
+
*/
|
|
117
|
+
function metaLabel(tag) {
|
|
118
|
+
const attrs = new Map();
|
|
119
|
+
for (const [, name, value = ''] of tag.slice('<meta'.length).matchAll(ATTRS)) {
|
|
120
|
+
attrs.set(name.toLowerCase(), value.replace(/^(["'])([\s\S]*)\1$/, '$2').trim());
|
|
121
|
+
}
|
|
122
|
+
const charset = attrs.get('charset');
|
|
123
|
+
if (charset) return charset;
|
|
124
|
+
if (attrs.get('http-equiv')?.toLowerCase() !== 'content-type') return null;
|
|
125
|
+
return attrs.get('content')?.match(CHARSET_PARAM)?.[1] ?? null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* The decoder a page asks for, read the way a browser's prescan reads it
|
|
130
|
+
* rather than with one regex over the head: a label written inside a comment,
|
|
131
|
+
* a <script> string, or another tag's attribute is not a declaration, and
|
|
132
|
+
* decoding by one of those is how a page that was already readable turns to
|
|
133
|
+
* mojibake. A label TextDecoder refuses is not the end of the search, so a
|
|
134
|
+
* feed declaring `cp949` still gets the <meta> tag underneath it.
|
|
135
|
+
* @param {string} head - the first 1024 bytes as latin1
|
|
136
|
+
* @returns {TextDecoder | null}
|
|
137
|
+
*/
|
|
138
|
+
function decoderInPage(head) {
|
|
139
|
+
const xml = decoderFor(head.match(XML_ENCODING)?.[1]);
|
|
140
|
+
if (xml) return xml;
|
|
141
|
+
const markup = head
|
|
142
|
+
.replace(/<!--[\s\S]*?(?:-->|$)/g, ' ')
|
|
143
|
+
.replace(/<script\b[\s\S]*?(?:<\/script>|$)/gi, ' ');
|
|
144
|
+
// Every tag is consumed whole, so a <meta> written inside a quoted attribute
|
|
145
|
+
// value belongs to the tag quoting it and is never read as a tag itself.
|
|
146
|
+
for (const [tag] of markup.matchAll(/<[a-zA-Z][^>]*>?/g)) {
|
|
147
|
+
const decoder = /^<meta\b/i.test(tag) ? decoderFor(metaLabel(tag)) : null;
|
|
148
|
+
if (decoder) return decoder;
|
|
149
|
+
}
|
|
150
|
+
return null;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* A body as text in the encoding it declares, header first, then the page.
|
|
155
|
+
* With neither, or only labels no browser knows, it is UTF-8, which is what
|
|
156
|
+
* oc assumed for every page before.
|
|
157
|
+
* @param {Buffer} bytes
|
|
158
|
+
* @param {string | null | undefined} type - the content-type header
|
|
159
|
+
* @returns {string}
|
|
160
|
+
*/
|
|
161
|
+
export function decodeBody(bytes, type) {
|
|
162
|
+
const declared = JSON_TYPE.test(type ?? '') ? null : decoderFor(type?.match(CHARSET_PARAM)?.[1]);
|
|
163
|
+
if (declared) return declared.decode(bytes);
|
|
164
|
+
// A missing header is not a refusal anywhere else in oc and is common on
|
|
165
|
+
// small servers, so a body that names no type is still read for one.
|
|
166
|
+
if (type && !MARKUP_TYPE.test(type)) return UTF8.decode(bytes);
|
|
167
|
+
const inPage = decoderInPage(bytes.toString('latin1', 0, 1024));
|
|
168
|
+
// A label found by reading the bytes as ASCII cannot be UTF-16, whose ASCII
|
|
169
|
+
// is interleaved with zero bytes, so the HTML spec reads that claim as UTF-8.
|
|
170
|
+
if (!inPage || inPage.encoding.startsWith('utf-16')) return UTF8.decode(bytes);
|
|
171
|
+
return inPage.decode(bytes);
|
|
172
|
+
}
|
|
173
|
+
|
|
76
174
|
// IPv4 ranges with no business receiving a server-initiated fetch: loopback,
|
|
77
175
|
// link-local, the three RFC 1918 private blocks, carrier-grade NAT, the
|
|
78
176
|
// unspecified/broadcast addresses, and the documentation/benchmark ranges.
|
|
@@ -325,6 +423,26 @@ function authority(target) {
|
|
|
325
423
|
return `${host}:${port}`;
|
|
326
424
|
}
|
|
327
425
|
|
|
426
|
+
// Native fetch (undici) already decodes Content-Encoding. Node's http parser
|
|
427
|
+
// does not, so the proxy transport used to charset-decode gzip bytes as the
|
|
428
|
+
// page: binary noise an agent pays for. Only gzip is handled here; deflate
|
|
429
|
+
// and br are left as they arrive, because this path does not ask for them.
|
|
430
|
+
// maxOutputLength is what stops a tiny gzip from expanding past the cap,
|
|
431
|
+
// which the compressed Content-Length cannot see.
|
|
432
|
+
function decompressBody(bytes, encoding, url) {
|
|
433
|
+
const enc = encoding?.split(',')[0]?.trim().toLowerCase();
|
|
434
|
+
if (!bytes.length || (enc !== 'gzip' && enc !== 'x-gzip')) return bytes;
|
|
435
|
+
let out;
|
|
436
|
+
try {
|
|
437
|
+
out = zlib.gunzipSync(bytes, { maxOutputLength: MAX_BODY });
|
|
438
|
+
} catch (err) {
|
|
439
|
+
if (err.code === 'ERR_BUFFER_TOO_LARGE') assertBodySize(MAX_BODY + 1, url);
|
|
440
|
+
throw new Error(`failed to decode gzip body for ${url}: ${err.message}`);
|
|
441
|
+
}
|
|
442
|
+
assertBodySize(out.length, url);
|
|
443
|
+
return out;
|
|
444
|
+
}
|
|
445
|
+
|
|
328
446
|
function wrapNodeResponse(res, url) {
|
|
329
447
|
const headers = {
|
|
330
448
|
get(name) {
|
|
@@ -355,7 +473,14 @@ function wrapNodeResponse(res, url) {
|
|
|
355
473
|
}
|
|
356
474
|
chunks.push(c);
|
|
357
475
|
});
|
|
358
|
-
res.on('end', () =>
|
|
476
|
+
res.on('end', () => {
|
|
477
|
+
try {
|
|
478
|
+
const bytes = decompressBody(Buffer.concat(chunks), headers.get('content-encoding'), url);
|
|
479
|
+
resolve(decodeBody(bytes, headers.get('content-type')));
|
|
480
|
+
} catch (err) {
|
|
481
|
+
reject(err);
|
|
482
|
+
}
|
|
483
|
+
});
|
|
359
484
|
res.on('error', reject);
|
|
360
485
|
});
|
|
361
486
|
const status = res.statusCode ?? 0;
|
|
@@ -614,6 +739,17 @@ export function identityOrder(target) {
|
|
|
614
739
|
return firefoxFirst ? ['firefox', 'chrome'] : ['chrome', 'firefox'];
|
|
615
740
|
}
|
|
616
741
|
|
|
742
|
+
// impers throws out of its own `content` getter when a response is streaming
|
|
743
|
+
// instead of buffered, so the question has to be asked inside a try: the answer
|
|
744
|
+
// is the bytes it holds, or nothing, never an error from a getter.
|
|
745
|
+
const bufferedContent = (res) => {
|
|
746
|
+
try {
|
|
747
|
+
return Buffer.isBuffer(res.content) ? res.content : null;
|
|
748
|
+
} catch {
|
|
749
|
+
return null;
|
|
750
|
+
}
|
|
751
|
+
};
|
|
752
|
+
|
|
617
753
|
/**
|
|
618
754
|
* Fetch a page through impers, downgrading identity when one is refused.
|
|
619
755
|
* Exported so the downgrade chain can be proven against a fake impers; the
|
|
@@ -664,7 +800,18 @@ export async function viaImpers(impers, target, jar) {
|
|
|
664
800
|
assertBodySize(Number(res.headers.get('content-length')) || 0, target);
|
|
665
801
|
// impers buffers inside its own binding, so the size of what it already
|
|
666
802
|
// holds is all there is to check; the bound still stops an oversized body
|
|
667
|
-
// from travelling any further.
|
|
803
|
+
// from travelling any further. impers decodes a few Western charsets itself
|
|
804
|
+
// and reads every other one as UTF-8, so the bytes it keeps in `content` are
|
|
805
|
+
// decoded here instead, the way the fetch transport's are.
|
|
806
|
+
const bytes = bufferedContent(res);
|
|
807
|
+
if (bytes) {
|
|
808
|
+
// What arrived, not what it decodes to: a CJK page is about half as many
|
|
809
|
+
// characters as it is bytes, and with the Content-Length above optional
|
|
810
|
+
// this is the only size this transport ever really checks. Refused before
|
|
811
|
+
// the decode so an oversized body is not allocated twice over.
|
|
812
|
+
assertBodySize(bytes.length, target);
|
|
813
|
+
return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via };
|
|
814
|
+
}
|
|
668
815
|
const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
|
|
669
816
|
assertBodySize(html.length, target);
|
|
670
817
|
return { url: res.url ?? target, html, status, via };
|
|
@@ -706,5 +853,5 @@ export async function readBody(res, url) {
|
|
|
706
853
|
assertBodySize(size, url);
|
|
707
854
|
chunks.push(chunk);
|
|
708
855
|
}
|
|
709
|
-
return Buffer.concat(chunks).
|
|
856
|
+
return decodeBody(Buffer.concat(chunks), res.headers.get('content-type'));
|
|
710
857
|
}
|
package/src/session.js
CHANGED
|
@@ -25,17 +25,46 @@ export const COOKIE_JAR_SUFFIX = '.cookies.json';
|
|
|
25
25
|
// OC_HOME relocates the whole state directory, for sandboxes, CI, and tests.
|
|
26
26
|
export const sessionDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sessions');
|
|
27
27
|
|
|
28
|
+
/**
|
|
29
|
+
* The store directory is what decides who can reach a session: an owner-only
|
|
30
|
+
* mode on a cookie jar is worth nothing inside a directory another local
|
|
31
|
+
* account can list, write to, or unlink from, which is what a permissive umask
|
|
32
|
+
* (0 or 002, the default in some container and CI images) leaves behind. mkdir
|
|
33
|
+
* applies its mode only to a directory it creates, so one already on disk is
|
|
34
|
+
* tightened too, the way the files in it are on every save.
|
|
35
|
+
* @returns {string} the store directory, now present and owner-only
|
|
36
|
+
*/
|
|
37
|
+
export function ensureSessionDir() {
|
|
38
|
+
const dir = sessionDir();
|
|
39
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
40
|
+
try {
|
|
41
|
+
chmodSync(dir, 0o700);
|
|
42
|
+
} catch {
|
|
43
|
+
// A shared OC_HOME that oc does not own keeps the mode it has rather than
|
|
44
|
+
// failing the save; the files written into it are still owner-only.
|
|
45
|
+
}
|
|
46
|
+
return dir;
|
|
47
|
+
}
|
|
48
|
+
|
|
28
49
|
// A session name is interpolated straight into a filename, and the cookie
|
|
29
50
|
// sidecar it names now holds real credentials, so a name that is a path
|
|
30
51
|
// (absolute, or with a separator or '..') could write or delete a file outside
|
|
31
52
|
// the store. Names are user-facing labels, so this charset loses nothing real.
|
|
32
53
|
const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
|
|
33
54
|
|
|
55
|
+
// Win32 reads a path component as a device when the part before its first
|
|
56
|
+
// period is a reserved name, which would put session 'nul''s cookie jar on the
|
|
57
|
+
// NUL device rather than in a file. Node 24 on Windows 11 writes a real file
|
|
58
|
+
// there (libuv's path handling sidesteps the rule, tested both ways), so this
|
|
59
|
+
// guards the name rather than fixing a failure oc has today: an older Windows,
|
|
60
|
+
// another runtime, or a path that reaches a shell still reads them as devices.
|
|
61
|
+
const DEVICE_NAME = /^(?:con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
|
|
62
|
+
|
|
34
63
|
// A name ending in '.cookies' would save its page at `<x>.cookies.json`, the
|
|
35
64
|
// path of session `<x>`'s cookie jar, so `oc logout x` would delete it and
|
|
36
65
|
// `oc session ls` would hide it.
|
|
37
66
|
const isSafeName = (name) => typeof name === 'string' && name !== '.' && name !== '..'
|
|
38
|
-
&& !name.endsWith('.cookies') && SAFE_NAME.test(name);
|
|
67
|
+
&& !name.endsWith('.cookies') && !DEVICE_NAME.test(name.split('.')[0]) && SAFE_NAME.test(name);
|
|
39
68
|
|
|
40
69
|
/**
|
|
41
70
|
* @param {string} name
|
|
@@ -43,7 +72,7 @@ const isSafeName = (name) => typeof name === 'string' && name !== '.' && name !=
|
|
|
43
72
|
*/
|
|
44
73
|
export function assertSafeName(name) {
|
|
45
74
|
if (!isSafeName(name)) {
|
|
46
|
-
throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_' (not ending in '.cookies')`);
|
|
75
|
+
throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_' (not ending in '.cookies', not a device name like 'nul')`);
|
|
47
76
|
}
|
|
48
77
|
return name;
|
|
49
78
|
}
|
|
@@ -115,7 +144,8 @@ export function sessionFromPage(page, previous, { cursor = 0 } = {}) {
|
|
|
115
144
|
dropped++;
|
|
116
145
|
continue;
|
|
117
146
|
}
|
|
118
|
-
|
|
147
|
+
// Relative hrefs are against <base href> when the page set one.
|
|
148
|
+
const href = block.href ? resolveHref(block.href, page.base || page.url) : null;
|
|
119
149
|
blocks.push({
|
|
120
150
|
type: block.type,
|
|
121
151
|
text: block.text,
|
|
@@ -165,7 +195,7 @@ export function handleNumbers(state) {
|
|
|
165
195
|
* @param {object} state
|
|
166
196
|
*/
|
|
167
197
|
export function saveSession(name, state) {
|
|
168
|
-
|
|
198
|
+
ensureSessionDir();
|
|
169
199
|
// A snapshot of an authenticated page holds that page's text, so it gets the
|
|
170
200
|
// same owner-only mode as the cookie sidecar. writeFileSync only sets the
|
|
171
201
|
// mode on create, so a snapshot left world-readable by an older version is
|
package/src/sites.js
CHANGED
|
@@ -145,7 +145,12 @@ export function resolveSite(name, args) {
|
|
|
145
145
|
*/
|
|
146
146
|
function encode(value, template, arg) {
|
|
147
147
|
const query = template.includes('?') && template.indexOf(`{${arg}}`) > template.indexOf('?');
|
|
148
|
-
|
|
148
|
+
// A path slot that starts with @ is a copied handle (@name, @channel).
|
|
149
|
+
// encodeURIComponent would turn it into %40 and miss the user, and YouTube's
|
|
150
|
+
// template already has the @ in the URL. Search queries keep it, so a
|
|
151
|
+
// mention in a query is still the thing being searched for.
|
|
152
|
+
const raw = !query && value.startsWith('@') && value.length > 1 ? value.slice(1) : value;
|
|
153
|
+
const encoded = encodeURIComponent(raw);
|
|
149
154
|
return query ? encoded : encoded.replaceAll('%2F', '/');
|
|
150
155
|
}
|
|
151
156
|
|
package/src/sphinx.js
CHANGED
|
@@ -84,11 +84,17 @@ function objectHits(index, query) {
|
|
|
84
84
|
for (const [doc, typeIdx, priority, anchor, name] of entries) {
|
|
85
85
|
const full = prefix ? `${prefix}.${name}` : name;
|
|
86
86
|
if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
|
|
87
|
+
// Empty string: the object's own name is the HTML id. The sentinel '-'
|
|
88
|
+
// means Sphinx built '{objtype}-{fullname}' instead, which is how
|
|
89
|
+
// docs.python.org names every module heading (#module-json).
|
|
90
|
+
const id = anchor === '' ? full
|
|
91
|
+
: anchor === '-' ? `${index.objnames?.[typeIdx]?.[1] ?? ''}-${full}`
|
|
92
|
+
: anchor;
|
|
87
93
|
hits.push({
|
|
88
94
|
name: full,
|
|
89
95
|
type: index.objnames?.[typeIdx]?.[2] ?? '',
|
|
90
96
|
doc,
|
|
91
|
-
anchor:
|
|
97
|
+
anchor: id,
|
|
92
98
|
priority,
|
|
93
99
|
});
|
|
94
100
|
}
|