@only-cli/oc 0.5.4 → 0.5.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -139,7 +139,7 @@ Copy the `Cookie` header from your browser's devtools (Application → Cookies,
139
139
 
140
140
  Seeded cookies are https-only. They almost always come from an https browser session, so `oc` marks them secure and never sends them over plain `http`, including on a hop an `https` page redirects into, where you never typed the downgrade. A site that really is http-only needs `--allow-http` at login. Cookies a site sets over https are pinned the same way.
141
141
 
142
- Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, mode `0600`, not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
142
+ Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, owner-only (mode `0600`, or on Windows the ACL the directory hands down), not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
143
143
 
144
144
  `oc logout` forgets the whole session, not just its cookies: a page saved under that name can hold the distilled text of something only the login could reach, so the snapshot goes with the jar. `oc session rm [name]` forgets a session the same way (page plus cookies) without switching to it first, and `oc session ls` lists what is on disk (`[cookies]` marks a live jar). State lives in `~/.only-cli` (`%USERPROFILE%\.only-cli` on Windows, override with `OC_HOME`), one JSON per session plus search-index caches. Delete the directory to start over.
145
145
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@only-cli/oc",
3
- "version": "0.5.4",
3
+ "version": "0.5.5",
4
4
  "description": "Turn websites into a compact CLI so AI agents can browse without burning tokens.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -8,17 +8,17 @@ description: Token-efficient web browsing and web content extraction for AI agen
8
8
  Renders a web page as a compact, numbered terminal view instead of raw HTML. A typical page is under 500 tokens.
9
9
 
10
10
  ```
11
- npx --yes @only-cli/oc@0.5.4 open <url> compact view, numbered elements
12
- npx --yes @only-cli/oc@0.5.4 do <n> follow link [n], or read it if [n] is text
13
- npx --yes @only-cli/oc@0.5.4 find <query> where a string appears, or that place itself
11
+ npx --yes @only-cli/oc@0.5.5 open <url> compact view, numbered elements
12
+ npx --yes @only-cli/oc@0.5.5 do <n> follow link [n], or read it if [n] is text
13
+ npx --yes @only-cli/oc@0.5.5 find <query> where a string appears, or that place itself
14
14
  when only one matches
15
- npx --yes @only-cli/oc@0.5.4 next next ~500 tokens of the page already open
16
- npx --yes @only-cli/oc@0.5.4 read <n> full text of region [n]
17
- npx --yes @only-cli/oc@0.5.4 raw [url] whole page as markdown (--html for cleaned HTML)
18
- npx --yes @only-cli/oc@0.5.4 login seed cookies (--cookie, --domain, --expires)
19
- npx --yes @only-cli/oc@0.5.4 logout [session] forget a session: cookies and saved page
20
- npx --yes @only-cli/oc@0.5.4 session ls list saved sessions (name, url, title)
21
- npx --yes @only-cli/oc@0.5.4 session rm [name] forget a saved session: page and cookies
15
+ npx --yes @only-cli/oc@0.5.5 next next ~500 tokens of the page already open
16
+ npx --yes @only-cli/oc@0.5.5 read <n> full text of region [n]
17
+ npx --yes @only-cli/oc@0.5.5 raw [url] whole page as markdown (--html for cleaned HTML)
18
+ npx --yes @only-cli/oc@0.5.5 login seed cookies (--cookie, --domain, --expires)
19
+ npx --yes @only-cli/oc@0.5.5 logout [session] forget a session: cookies and saved page
20
+ npx --yes @only-cli/oc@0.5.5 session ls list saved sessions (name, url, title)
21
+ npx --yes @only-cli/oc@0.5.5 session rm [name] forget a saved session: page and cookies
22
22
  ```
23
23
 
24
24
  None of these except `open`/`do`/`raw <url>` fetch anything; they replay the page `open` already saved.
package/src/cli.js CHANGED
@@ -272,7 +272,7 @@ async function main() {
272
272
  const processMs = performance.now() - t0 - fetchMs;
273
273
  const rss = process.memoryUsage().rss;
274
274
  return `HTTP ${status} via ${via}, fetch ${Math.round(fetchMs)}ms, process ${Math.round(processMs)}ms, `
275
- + `${Math.round(html.length / 1024)}KB transferred, ${Math.round(rss / 1048576)}MB memory`;
275
+ + `${Math.round(Buffer.byteLength(html) / 1024)}KB transferred, ${Math.round(rss / 1048576)}MB memory`;
276
276
  };
277
277
  const htmlTokens = estimateTokens(html);
278
278
  if (values.json) {
package/src/cookies.js CHANGED
@@ -5,9 +5,9 @@
5
5
 
6
6
  import net from 'node:net';
7
7
  import { join } from 'node:path';
8
- import { mkdirSync, readFileSync, writeFileSync, unlinkSync, readdirSync, chmodSync } from 'node:fs';
8
+ import { readFileSync, writeFileSync, unlinkSync, readdirSync, chmodSync } from 'node:fs';
9
9
 
10
- import { sessionDir, assertSafeName, COOKIE_JAR_SUFFIX } from './session.js';
10
+ import { sessionDir, ensureSessionDir, assertSafeName, COOKIE_JAR_SUFFIX } from './session.js';
11
11
 
12
12
  const DEFAULT_EXPIRES_MS = 60 * 60 * 1000; // 1h
13
13
  export { DEFAULT_EXPIRES_MS };
@@ -256,7 +256,7 @@ export function loadCookieJar(name) {
256
256
  * @param {CookieJar} jar
257
257
  */
258
258
  export function saveCookieJar(name, jar) {
259
- mkdirSync(sessionDir(), { recursive: true });
259
+ ensureSessionDir();
260
260
  // writeFileSync only sets the mode on create, so an existing sidecar has its
261
261
  // owner-only mode reasserted on every save.
262
262
  const path = cookieJarPath(name);
package/src/fetch.js CHANGED
@@ -73,6 +73,103 @@ export function assertReadableType(type) {
73
73
  throw new Error(`not a page oc can read (${type.split(';')[0].trim()}), it renders HTML, XML feeds, JSON, and plain text`);
74
74
  }
75
75
 
76
+ // A body read as UTF-8 whatever it declares turns each non-ASCII character of
77
+ // a Shift_JIS, EUC-KR, GBK, or windows-1252 page into U+FFFD, noise an agent
78
+ // pays for and cannot read. So the encoding is looked for where a browser
79
+ // looks: the Content-Type header, then a <meta> tag, which the HTML spec
80
+ // requires within the first 1024 bytes, or a feed's XML declaration.
81
+ // The comma is excluded because a duplicated Content-Type header arrives from
82
+ // headers.get as one value joined by ", ", and a label with a comma stuck to
83
+ // its end is one TextDecoder refuses.
84
+ const CHARSET_PARAM = /;\s*charset\s*=\s*["']?([^"';,\s]+)/i;
85
+ const XML_ENCODING = /^\s*<\?xml\b[^>]*?\bencoding\s*=\s*["']([^"']+)/;
86
+ // One tag's attributes, quoted values kept whole, so a charset written inside
87
+ // another attribute's value (an og:url carrying a query string, say) stays
88
+ // part of that value instead of reading as a declaration.
89
+ const ATTRS = /([^\s=/>]+)(?:\s*=\s*("[^"]*"|'[^']*'|[^\s>]*))?/g;
90
+ // Only markup declares an encoding in its own text, so only markup is read for
91
+ // one. JSON is UTF-8 by definition (RFC 8259), which is why a browser ignores
92
+ // a charset on it, and plain text or JavaScript carries no declaration a
93
+ // scan could find, only strings that look like one.
94
+ const MARKUP_TYPE = /^\s*(?:text\/(?:html|xml)|application\/(?:xml|[\w.+-]*\+xml))/i;
95
+ const JSON_TYPE = /^\s*(?:application\/(?:json|x-ndjson|[\w.+-]*\+json)|text\/json)/i;
96
+
97
+ // TextDecoder knows the labels browsers do, so no dependency is needed, and it
98
+ // refuses the ones they do not, which then fall through to the next place a
99
+ // label can come from.
100
+ const UTF8 = new TextDecoder();
101
+ const decoderFor = (label) => {
102
+ try {
103
+ return label ? new TextDecoder(label) : null;
104
+ } catch {
105
+ return null;
106
+ }
107
+ };
108
+
109
+ /**
110
+ * The charset one <meta> tag declares: the attribute browsers read first, then
111
+ * the older http-equiv form, which only counts on a content-type, not on any
112
+ * content attribute that happens to mention a charset.
113
+ * @param {string} tag
114
+ * @returns {string | null}
115
+ */
116
+ function metaLabel(tag) {
117
+ const attrs = new Map();
118
+ for (const [, name, value = ''] of tag.slice('<meta'.length).matchAll(ATTRS)) {
119
+ attrs.set(name.toLowerCase(), value.replace(/^(["'])([\s\S]*)\1$/, '$2').trim());
120
+ }
121
+ const charset = attrs.get('charset');
122
+ if (charset) return charset;
123
+ if (attrs.get('http-equiv')?.toLowerCase() !== 'content-type') return null;
124
+ return attrs.get('content')?.match(CHARSET_PARAM)?.[1] ?? null;
125
+ }
126
+
127
+ /**
128
+ * The decoder a page asks for, read the way a browser's prescan reads it
129
+ * rather than with one regex over the head: a label written inside a comment,
130
+ * a <script> string, or another tag's attribute is not a declaration, and
131
+ * decoding by one of those is how a page that was already readable turns to
132
+ * mojibake. A label TextDecoder refuses is not the end of the search, so a
133
+ * feed declaring `cp949` still gets the <meta> tag underneath it.
134
+ * @param {string} head - the first 1024 bytes as latin1
135
+ * @returns {TextDecoder | null}
136
+ */
137
+ function decoderInPage(head) {
138
+ const xml = decoderFor(head.match(XML_ENCODING)?.[1]);
139
+ if (xml) return xml;
140
+ const markup = head
141
+ .replace(/<!--[\s\S]*?(?:-->|$)/g, ' ')
142
+ .replace(/<script\b[\s\S]*?(?:<\/script>|$)/gi, ' ');
143
+ // Every tag is consumed whole, so a <meta> written inside a quoted attribute
144
+ // value belongs to the tag quoting it and is never read as a tag itself.
145
+ for (const [tag] of markup.matchAll(/<[a-zA-Z][^>]*>?/g)) {
146
+ const decoder = /^<meta\b/i.test(tag) ? decoderFor(metaLabel(tag)) : null;
147
+ if (decoder) return decoder;
148
+ }
149
+ return null;
150
+ }
151
+
152
+ /**
153
+ * A body as text in the encoding it declares, header first, then the page.
154
+ * With neither, or only labels no browser knows, it is UTF-8, which is what
155
+ * oc assumed for every page before.
156
+ * @param {Buffer} bytes
157
+ * @param {string | null | undefined} type - the content-type header
158
+ * @returns {string}
159
+ */
160
+ export function decodeBody(bytes, type) {
161
+ const declared = JSON_TYPE.test(type ?? '') ? null : decoderFor(type?.match(CHARSET_PARAM)?.[1]);
162
+ if (declared) return declared.decode(bytes);
163
+ // A missing header is not a refusal anywhere else in oc and is common on
164
+ // small servers, so a body that names no type is still read for one.
165
+ if (type && !MARKUP_TYPE.test(type)) return UTF8.decode(bytes);
166
+ const inPage = decoderInPage(bytes.toString('latin1', 0, 1024));
167
+ // A label found by reading the bytes as ASCII cannot be UTF-16, whose ASCII
168
+ // is interleaved with zero bytes, so the HTML spec reads that claim as UTF-8.
169
+ if (!inPage || inPage.encoding.startsWith('utf-16')) return UTF8.decode(bytes);
170
+ return inPage.decode(bytes);
171
+ }
172
+
76
173
  // IPv4 ranges with no business receiving a server-initiated fetch: loopback,
77
174
  // link-local, the three RFC 1918 private blocks, carrier-grade NAT, the
78
175
  // unspecified/broadcast addresses, and the documentation/benchmark ranges.
@@ -355,7 +452,7 @@ function wrapNodeResponse(res, url) {
355
452
  }
356
453
  chunks.push(c);
357
454
  });
358
- res.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
455
+ res.on('end', () => resolve(decodeBody(Buffer.concat(chunks), headers.get('content-type'))));
359
456
  res.on('error', reject);
360
457
  });
361
458
  const status = res.statusCode ?? 0;
@@ -614,6 +711,17 @@ export function identityOrder(target) {
614
711
  return firefoxFirst ? ['firefox', 'chrome'] : ['chrome', 'firefox'];
615
712
  }
616
713
 
714
+ // impers throws out of its own `content` getter when a response is streaming
715
+ // instead of buffered, so the question has to be asked inside a try: the answer
716
+ // is the bytes it holds, or nothing, never an error from a getter.
717
+ const bufferedContent = (res) => {
718
+ try {
719
+ return Buffer.isBuffer(res.content) ? res.content : null;
720
+ } catch {
721
+ return null;
722
+ }
723
+ };
724
+
617
725
  /**
618
726
  * Fetch a page through impers, downgrading identity when one is refused.
619
727
  * Exported so the downgrade chain can be proven against a fake impers; the
@@ -664,7 +772,18 @@ export async function viaImpers(impers, target, jar) {
664
772
  assertBodySize(Number(res.headers.get('content-length')) || 0, target);
665
773
  // impers buffers inside its own binding, so the size of what it already
666
774
  // holds is all there is to check; the bound still stops an oversized body
667
- // from travelling any further.
775
+ // from travelling any further. impers decodes a few Western charsets itself
776
+ // and reads every other one as UTF-8, so the bytes it keeps in `content` are
777
+ // decoded here instead, the way the fetch transport's are.
778
+ const bytes = bufferedContent(res);
779
+ if (bytes) {
780
+ // What arrived, not what it decodes to: a CJK page is about half as many
781
+ // characters as it is bytes, and with the Content-Length above optional
782
+ // this is the only size this transport ever really checks. Refused before
783
+ // the decode so an oversized body is not allocated twice over.
784
+ assertBodySize(bytes.length, target);
785
+ return { url: res.url ?? target, html: decodeBody(bytes, res.headers.get('content-type')), status, via };
786
+ }
668
787
  const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
669
788
  assertBodySize(html.length, target);
670
789
  return { url: res.url ?? target, html, status, via };
@@ -706,5 +825,5 @@ export async function readBody(res, url) {
706
825
  assertBodySize(size, url);
707
826
  chunks.push(chunk);
708
827
  }
709
- return Buffer.concat(chunks).toString('utf8');
828
+ return decodeBody(Buffer.concat(chunks), res.headers.get('content-type'));
710
829
  }
package/src/session.js CHANGED
@@ -25,17 +25,46 @@ export const COOKIE_JAR_SUFFIX = '.cookies.json';
25
25
  // OC_HOME relocates the whole state directory, for sandboxes, CI, and tests.
26
26
  export const sessionDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sessions');
27
27
 
28
+ /**
29
+ * The store directory is what decides who can reach a session: an owner-only
30
+ * mode on a cookie jar is worth nothing inside a directory another local
31
+ * account can list, write to, or unlink from, which is what a permissive umask
32
+ * (0 or 002, the default in some container and CI images) leaves behind. mkdir
33
+ * applies its mode only to a directory it creates, so one already on disk is
34
+ * tightened too, the way the files in it are on every save.
35
+ * @returns {string} the store directory, now present and owner-only
36
+ */
37
+ export function ensureSessionDir() {
38
+ const dir = sessionDir();
39
+ mkdirSync(dir, { recursive: true, mode: 0o700 });
40
+ try {
41
+ chmodSync(dir, 0o700);
42
+ } catch {
43
+ // A shared OC_HOME that oc does not own keeps the mode it has rather than
44
+ // failing the save; the files written into it are still owner-only.
45
+ }
46
+ return dir;
47
+ }
48
+
28
49
  // A session name is interpolated straight into a filename, and the cookie
29
50
  // sidecar it names now holds real credentials, so a name that is a path
30
51
  // (absolute, or with a separator or '..') could write or delete a file outside
31
52
  // the store. Names are user-facing labels, so this charset loses nothing real.
32
53
  const SAFE_NAME = /^[A-Za-z0-9._-]+$/;
33
54
 
55
+ // Win32 reads a path component as a device when the part before its first
56
+ // period is a reserved name, which would put session 'nul''s cookie jar on the
57
+ // NUL device rather than in a file. Node 24 on Windows 11 writes a real file
58
+ // there (libuv's path handling sidesteps the rule, tested both ways), so this
59
+ // guards the name rather than fixing a failure oc has today: an older Windows,
60
+ // another runtime, or a path that reaches a shell still reads them as devices.
61
+ const DEVICE_NAME = /^(?:con|prn|aux|nul|com[0-9]|lpt[0-9])$/i;
62
+
34
63
  // A name ending in '.cookies' would save its page at `<x>.cookies.json`, the
35
64
  // path of session `<x>`'s cookie jar, so `oc logout x` would delete it and
36
65
  // `oc session ls` would hide it.
37
66
  const isSafeName = (name) => typeof name === 'string' && name !== '.' && name !== '..'
38
- && !name.endsWith('.cookies') && SAFE_NAME.test(name);
67
+ && !name.endsWith('.cookies') && !DEVICE_NAME.test(name.split('.')[0]) && SAFE_NAME.test(name);
39
68
 
40
69
  /**
41
70
  * @param {string} name
@@ -43,7 +72,7 @@ const isSafeName = (name) => typeof name === 'string' && name !== '.' && name !=
43
72
  */
44
73
  export function assertSafeName(name) {
45
74
  if (!isSafeName(name)) {
46
- throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_' (not ending in '.cookies')`);
75
+ throw new Error(`invalid session name '${name}', use letters, numbers, '.', '-', or '_' (not ending in '.cookies', not a device name like 'nul')`);
47
76
  }
48
77
  return name;
49
78
  }
@@ -165,7 +194,7 @@ export function handleNumbers(state) {
165
194
  * @param {object} state
166
195
  */
167
196
  export function saveSession(name, state) {
168
- mkdirSync(sessionDir(), { recursive: true });
197
+ ensureSessionDir();
169
198
  // A snapshot of an authenticated page holds that page's text, so it gets the
170
199
  // same owner-only mode as the cookie sidecar. writeFileSync only sets the
171
200
  // mode on create, so a snapshot left world-readable by an older version is