@only-cli/oc 0.2.0-beta.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -26
- package/clis/finance.yahoo.com.json +13 -0
- package/clis/stackoverflow.com.json +9 -0
- package/clis/x.com.json +7 -0
- package/clis/youtube.com.json +7 -0
- package/package.json +1 -1
- package/skills/only-cli/SKILL.md +49 -7
- package/src/act.js +235 -17
- package/src/cli.js +65 -15
- package/src/distill.js +380 -22
- package/src/fetch.js +151 -14
- package/src/render.js +75 -24
- package/src/session.js +143 -5
package/src/fetch.js
CHANGED
|
@@ -7,6 +7,9 @@
|
|
|
7
7
|
* never import a browser here.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
|
+
import dns from 'node:dns/promises';
|
|
11
|
+
import net from 'node:net';
|
|
12
|
+
|
|
10
13
|
// The fetch fallback can't fake a TLS fingerprint like impers does, but it
|
|
11
14
|
// should at least send the same Chrome identity in its headers.
|
|
12
15
|
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36';
|
|
@@ -18,6 +21,112 @@ const loadImpers = () => {
|
|
|
18
21
|
return impersPromise;
|
|
19
22
|
};
|
|
20
23
|
|
|
24
|
+
const BLOCKED_MESSAGE = 'blocked: private or internal URL';
|
|
25
|
+
const MAX_REDIRECTS = 20;
|
|
26
|
+
|
|
27
|
+
// IPv4 ranges with no business receiving a server-initiated fetch: loopback,
|
|
28
|
+
// link-local, the three RFC 1918 private blocks, carrier-grade NAT, the
|
|
29
|
+
// unspecified/broadcast addresses, and the documentation/benchmark ranges.
|
|
30
|
+
const IPV4_BLOCKED_RANGES = [
|
|
31
|
+
['0.0.0.0', 8],
|
|
32
|
+
['10.0.0.0', 8],
|
|
33
|
+
['100.64.0.0', 10],
|
|
34
|
+
['127.0.0.0', 8],
|
|
35
|
+
['169.254.0.0', 16],
|
|
36
|
+
['172.16.0.0', 12],
|
|
37
|
+
['192.0.0.0', 24],
|
|
38
|
+
['192.0.2.0', 24],
|
|
39
|
+
['192.168.0.0', 16],
|
|
40
|
+
['198.18.0.0', 15],
|
|
41
|
+
['198.51.100.0', 24],
|
|
42
|
+
['203.0.113.0', 24],
|
|
43
|
+
['224.0.0.0', 4],
|
|
44
|
+
['240.0.0.0', 4],
|
|
45
|
+
['255.255.255.255', 32],
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
function ipv4ToInt(ip) {
|
|
49
|
+
const parts = ip.split('.').map(Number);
|
|
50
|
+
return ((parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]) >>> 0;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function isBlockedIPv4Int(addr) {
|
|
54
|
+
return IPV4_BLOCKED_RANGES.some(([base, bits]) => {
|
|
55
|
+
const mask = bits === 0 ? 0 : (0xffffffff << (32 - bits)) >>> 0;
|
|
56
|
+
return (addr & mask) === (ipv4ToInt(base) & mask);
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function isBlockedIPv4(ip) {
|
|
61
|
+
return isBlockedIPv4Int(ipv4ToInt(ip));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
// Expand a parsed IPv6 literal (as given by URL.hostname or dns.lookup, so
|
|
65
|
+
// already bracket-free and lowercase) into its 8 16-bit groups.
|
|
66
|
+
function expandIPv6(ip) {
|
|
67
|
+
const sides = ip.split('::');
|
|
68
|
+
if (sides.length > 2) return null;
|
|
69
|
+
const head = sides[0] ? sides[0].split(':') : [];
|
|
70
|
+
const tail = sides.length === 2 && sides[1] ? sides[1].split(':') : [];
|
|
71
|
+
const missing = 8 - head.length - tail.length;
|
|
72
|
+
if (missing < 0) return null;
|
|
73
|
+
const groups = [...head, ...Array(missing).fill('0'), ...tail];
|
|
74
|
+
if (groups.length !== 8) return null;
|
|
75
|
+
return groups.map((g) => parseInt(g, 16));
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function isBlockedIPv6(ip) {
|
|
79
|
+
const g = expandIPv6(ip);
|
|
80
|
+
if (!g) return true; // unparsable - fail closed
|
|
81
|
+
if (g.every((x) => x === 0)) return true; // :: (unspecified)
|
|
82
|
+
if (g[0] === 0 && g[1] === 0 && g[2] === 0 && g[3] === 0 && g[4] === 0 && g[5] === 0 && g[6] === 0 && g[7] === 1) {
|
|
83
|
+
return true; // ::1 (loopback)
|
|
84
|
+
}
|
|
85
|
+
if (g[0] === 0 && g[1] === 0 && g[2] === 0 && g[3] === 0 && g[4] === 0 && (g[5] === 0xffff || g[5] === 0)) {
|
|
86
|
+
// ::ffff:a.b.c.d (IPv4-mapped) or the deprecated ::a.b.c.d (IPv4-compatible)
|
|
87
|
+
return isBlockedIPv4Int(((g[6] << 16) | g[7]) >>> 0);
|
|
88
|
+
}
|
|
89
|
+
if ((g[0] & 0xfe00) === 0xfc00) return true; // fc00::/7 (unique local)
|
|
90
|
+
if ((g[0] & 0xffc0) === 0xfe80) return true; // fe80::/10 (link-local)
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// Validates the target a socket is actually about to connect to: the parsed
|
|
95
|
+
// IP if the URL is a literal, or every address the hostname resolves to
|
|
96
|
+
// otherwise. Resolving before connecting (rather than pattern-matching the
|
|
97
|
+
// hostname string) is what closes off IPv4-mapped IPv6 loopback, 0.0.0.0,
|
|
98
|
+
// and DNS rebinding through an attacker-controlled domain. It runs again on
|
|
99
|
+
// every redirect hop, since a public URL that later 302s to an internal
|
|
100
|
+
// address is the same attack one step removed.
|
|
101
|
+
//
|
|
102
|
+
// It does not pin the resolved address for the connection itself - neither
|
|
103
|
+
// impers (curl) nor Node's fetch expose that here - so a name that
|
|
104
|
+
// re-resolves to a different address between this check and the actual
|
|
105
|
+
// connect is a known residual gap, not one this guard can close without
|
|
106
|
+
// deeper transport changes.
|
|
107
|
+
async function assertSafeTarget(urlStr) {
|
|
108
|
+
const u = new URL(urlStr);
|
|
109
|
+
if (!/^https?:$/.test(u.protocol)) throw new Error(BLOCKED_MESSAGE);
|
|
110
|
+
// URL.hostname keeps the brackets around an IPv6 literal (e.g. "[::1]");
|
|
111
|
+
// net.isIP and the group-based checks below expect the bare address.
|
|
112
|
+
const hostname = u.hostname.toLowerCase().replace(/^\[|\]$/g, '');
|
|
113
|
+
const ipType = net.isIP(hostname);
|
|
114
|
+
if (ipType === 4) {
|
|
115
|
+
if (isBlockedIPv4(hostname)) throw new Error(BLOCKED_MESSAGE);
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
if (ipType === 6) {
|
|
119
|
+
if (isBlockedIPv6(hostname)) throw new Error(BLOCKED_MESSAGE);
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
if (hostname === 'localhost') throw new Error(BLOCKED_MESSAGE);
|
|
123
|
+
const addresses = await dns.lookup(hostname, { all: true }).catch(() => []);
|
|
124
|
+
for (const { address, family } of addresses) {
|
|
125
|
+
if (family === 4 && isBlockedIPv4(address)) throw new Error(BLOCKED_MESSAGE);
|
|
126
|
+
if (family === 6 && isBlockedIPv6(address)) throw new Error(BLOCKED_MESSAGE);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
21
130
|
/**
|
|
22
131
|
* Fetch a page.
|
|
23
132
|
* @param {string} url - with or without a scheme, https is assumed
|
|
@@ -27,20 +136,36 @@ const loadImpers = () => {
|
|
|
27
136
|
*/
|
|
28
137
|
export async function fetchPage(url) {
|
|
29
138
|
const target = /^https?:\/\//i.test(url) ? url : `https://${url}`;
|
|
139
|
+
await assertSafeTarget(target);
|
|
30
140
|
const impers = await loadImpers();
|
|
31
141
|
return impers ? viaImpers(impers, target) : viaFetch(target);
|
|
32
142
|
}
|
|
33
143
|
|
|
144
|
+
async function followImpersRedirects(impers, startUrl, impersonate) {
|
|
145
|
+
let current = startUrl;
|
|
146
|
+
for (let i = 0; ; i++) {
|
|
147
|
+
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${startUrl}`);
|
|
148
|
+
const res = await impers.get(current, { impersonate, allowRedirects: false });
|
|
149
|
+
const status = res.status ?? res.statusCode ?? 0;
|
|
150
|
+
const location = res.headers.get('location');
|
|
151
|
+
if (status >= 300 && status < 400 && location) {
|
|
152
|
+
current = new URL(location, current).toString();
|
|
153
|
+
await assertSafeTarget(current);
|
|
154
|
+
continue;
|
|
155
|
+
}
|
|
156
|
+
return res;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
34
160
|
async function viaImpers(impers, target) {
|
|
35
161
|
// Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a
|
|
36
162
|
// blocked first attempt gets one cheap retry with a second identity.
|
|
37
163
|
let via = 'impers:chrome';
|
|
38
|
-
let res = await impers
|
|
39
|
-
// impers mirrors the curl_cffi response shape, not the WHATWG one.
|
|
164
|
+
let res = await followImpersRedirects(impers, target, 'chrome');
|
|
40
165
|
let status = res.status ?? res.statusCode ?? 0;
|
|
41
166
|
if (status >= 400) {
|
|
42
167
|
via = 'impers:firefox';
|
|
43
|
-
res = await impers
|
|
168
|
+
res = await followImpersRedirects(impers, target, 'firefox');
|
|
44
169
|
status = res.status ?? res.statusCode ?? 0;
|
|
45
170
|
}
|
|
46
171
|
if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
|
|
@@ -49,20 +174,32 @@ async function viaImpers(impers, target) {
|
|
|
49
174
|
}
|
|
50
175
|
|
|
51
176
|
async function viaFetch(target) {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
177
|
+
let current = target;
|
|
178
|
+
let res;
|
|
179
|
+
for (let i = 0; ; i++) {
|
|
180
|
+
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${target}`);
|
|
181
|
+
res = await fetch(current, {
|
|
182
|
+
redirect: 'manual',
|
|
183
|
+
headers: {
|
|
184
|
+
'user-agent': UA,
|
|
185
|
+
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
186
|
+
'accept-language': 'en-US,en;q=0.9',
|
|
187
|
+
},
|
|
188
|
+
});
|
|
189
|
+
const location = res.headers.get('location');
|
|
190
|
+
if (res.status >= 300 && res.status < 400 && location) {
|
|
191
|
+
current = new URL(location, current).toString();
|
|
192
|
+
await assertSafeTarget(current);
|
|
193
|
+
continue;
|
|
194
|
+
}
|
|
195
|
+
break;
|
|
196
|
+
}
|
|
60
197
|
if (!res.ok) {
|
|
61
|
-
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${
|
|
198
|
+
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
|
62
199
|
}
|
|
63
200
|
const type = res.headers.get('content-type') ?? '';
|
|
64
|
-
if (type && !type.includes('html')) {
|
|
201
|
+
if (type && !type.includes('html') && !type.includes('xml')) {
|
|
65
202
|
throw new Error(`not an HTML page (${type.split(';')[0]}), nothing to distill`);
|
|
66
203
|
}
|
|
67
|
-
return { url: res.url, html: await res.text(), status: res.status, via: 'fetch' };
|
|
204
|
+
return { url: res.url || current, html: await res.text(), status: res.status, via: 'fetch' };
|
|
68
205
|
}
|
package/src/render.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Rendering is where the token budget is enforced. Everything printed here
|
|
3
|
-
* gets read by a paying model, so the default view is dense
|
|
4
|
-
*
|
|
3
|
+
* gets read by a paying model, so the default view is dense, anything cut says
|
|
4
|
+
* how much was cut, and the footer names the cheapest command that gets it.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
|
|
7
|
+
import { TEXT_CAP } from './distill.js';
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
10
|
* Rough but stable token estimate. Close enough for budgets; the point is
|
|
@@ -14,49 +14,92 @@ const TEXT_CAP = 200;
|
|
|
14
14
|
*/
|
|
15
15
|
export const estimateTokens = (s) => Math.ceil(s.length / 4);
|
|
16
16
|
|
|
17
|
+
const num = (v) => v.toLocaleString('en-US');
|
|
18
|
+
|
|
19
|
+
// How far past the budget a page may run and still be printed whole. Cutting a
|
|
20
|
+
// page that was nearly done costs the agent a second command, and a command is
|
|
21
|
+
// dear: measured inside Claude Code, one tool call is 23,000 to 33,000 tokens of
|
|
22
|
+
// session overhead no matter what it prints. Against that, break-even sits near
|
|
23
|
+
// fifty times the budget. The number is far lower than break-even because the
|
|
24
|
+
// saving is only collected when the agent would have paged at all, while the
|
|
25
|
+
// overspend is paid on every page that runs a little long, including the ones
|
|
26
|
+
// answered by their first few lines. Four caps that overspend near 1,500 tokens.
|
|
27
|
+
const FINISH = 4;
|
|
28
|
+
|
|
17
29
|
/**
|
|
18
|
-
* Budget-aware compact view of a distilled page.
|
|
30
|
+
* Budget-aware compact view of a distilled page. `from` is a position in the
|
|
31
|
+
* collapsed block list, which is how `oc next` resumes a page where the last
|
|
32
|
+
* render stopped.
|
|
19
33
|
* @param {import('./distill.js').Page} page
|
|
20
|
-
* @param {{budget?: number}} [opts]
|
|
21
|
-
* @returns {{text: string, stats: {tokens: number, blocks: number, rendered: number}}}
|
|
34
|
+
* @param {{budget?: number, from?: number}} [opts]
|
|
35
|
+
* @returns {{text: string, stats: {tokens: number, blocks: number, rendered: number, next: number|null, left: number, leftTokens: number}}}
|
|
22
36
|
*/
|
|
23
|
-
export function render(page, { budget = 500 } = {}) {
|
|
24
|
-
const
|
|
37
|
+
export function render(page, { budget = 500, from = 0 } = {}) {
|
|
38
|
+
const blocks = collapseRuns(page.blocks);
|
|
39
|
+
const head = page.title ? [from > 0 ? `# ${page.title} (continued)` : `# ${page.title}`] : [];
|
|
40
|
+
const lines = [...head];
|
|
25
41
|
let spent = estimateTokens(lines.join('\n'));
|
|
26
|
-
let skipped = 0;
|
|
27
42
|
let hasLinks = false;
|
|
28
43
|
let hasInputs = false;
|
|
44
|
+
let i = Math.max(0, from);
|
|
45
|
+
|
|
46
|
+
// What the rest of the page would cost if it were all printed. When that is
|
|
47
|
+
// within reach the budget stands aside, because stopping here would only
|
|
48
|
+
// move those tokens into a second command and add a turn's overhead on top.
|
|
49
|
+
const whole = blocks.slice(i).reduce((sum, b) => {
|
|
50
|
+
const line = formatBlock(b);
|
|
51
|
+
return line ? sum + estimateTokens(line) + 1 : sum;
|
|
52
|
+
}, spent);
|
|
53
|
+
const limit = whole <= budget * FINISH ? Infinity : budget;
|
|
29
54
|
|
|
30
|
-
for (
|
|
55
|
+
for (; i < blocks.length; i++) {
|
|
56
|
+
const block = blocks[i];
|
|
31
57
|
// Never print the same content twice. Pages often repeat the title as
|
|
32
58
|
// their first heading.
|
|
33
59
|
if (block.type === 'heading' && block.text === page.title) continue;
|
|
34
60
|
const line = formatBlock(block);
|
|
35
61
|
if (!line) continue;
|
|
36
62
|
const cost = estimateTokens(line) + 1;
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
63
|
+
// Stop at the first block that does not fit instead of skipping past it:
|
|
64
|
+
// what is left has to stay one contiguous run for `oc next` to continue.
|
|
65
|
+
// The exception is a first block bigger than the whole budget, which is
|
|
66
|
+
// printed anyway so the cursor always moves.
|
|
67
|
+
if (spent + cost > limit && lines.length > head.length) break;
|
|
41
68
|
spent += cost;
|
|
42
69
|
lines.push(line);
|
|
43
70
|
if (block.type === 'link' || block.type === 'button') hasLinks = true;
|
|
44
71
|
if (block.type === 'input') hasInputs = true;
|
|
45
72
|
}
|
|
46
73
|
|
|
47
|
-
|
|
74
|
+
const rest = blocks.slice(i);
|
|
75
|
+
const leftTokens = rest.reduce((sum, b) => {
|
|
76
|
+
const line = formatBlock(b);
|
|
77
|
+
return line ? sum + estimateTokens(line) + 1 : sum;
|
|
78
|
+
}, 0);
|
|
79
|
+
if (rest.length) {
|
|
80
|
+
lines.push(`... ${num(rest.length)} more blocks (~${num(leftTokens)} tokens): 'oc next' for the next ~${num(budget)}, 'oc raw' for all`);
|
|
81
|
+
}
|
|
48
82
|
const actions = [
|
|
49
83
|
hasLinks && 'do <n>',
|
|
50
84
|
hasInputs && 'fill <n> <text>',
|
|
51
85
|
hasInputs && 'submit',
|
|
52
|
-
'
|
|
86
|
+
'read <n>',
|
|
87
|
+
rest.length && 'next',
|
|
88
|
+
'raw',
|
|
53
89
|
].filter(Boolean);
|
|
54
90
|
lines.push(`actions: ${actions.join(' | ')}`);
|
|
55
91
|
|
|
56
92
|
const text = lines.join('\n');
|
|
57
93
|
return {
|
|
58
94
|
text,
|
|
59
|
-
stats: {
|
|
95
|
+
stats: {
|
|
96
|
+
tokens: estimateTokens(text),
|
|
97
|
+
blocks: blocks.length,
|
|
98
|
+
rendered: i - Math.max(0, from),
|
|
99
|
+
next: rest.length ? i : null,
|
|
100
|
+
left: rest.length,
|
|
101
|
+
leftTokens,
|
|
102
|
+
},
|
|
60
103
|
};
|
|
61
104
|
}
|
|
62
105
|
|
|
@@ -94,22 +137,30 @@ function collapseRuns(blocks) {
|
|
|
94
137
|
}
|
|
95
138
|
|
|
96
139
|
/**
|
|
140
|
+
* One line per block. `full` keeps the whole text, which is what `oc read`
|
|
141
|
+
* prints; the compact view cuts at TEXT_CAP and says how many characters went
|
|
142
|
+
* with the cut, so the agent can price the rest before asking for it.
|
|
97
143
|
* @param {import('./distill.js').Block} b
|
|
144
|
+
* @param {{full?: boolean}} [opts]
|
|
98
145
|
* @returns {string}
|
|
99
146
|
*/
|
|
100
|
-
function formatBlock(b) {
|
|
147
|
+
export function formatBlock(b, { full = false } = {}) {
|
|
148
|
+
const tag = b.n == null ? '' : `[${b.n}] `;
|
|
101
149
|
switch (b.type) {
|
|
102
150
|
case 'heading':
|
|
103
|
-
return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${b.text}`;
|
|
151
|
+
return `${'#'.repeat(Math.min(b.level ?? 2, 3))} ${tag}${b.text}`;
|
|
104
152
|
case 'link':
|
|
105
|
-
return
|
|
153
|
+
return `${tag}${full ? b.text : truncate(b.text)}`;
|
|
106
154
|
case 'button':
|
|
107
|
-
return
|
|
155
|
+
return `${tag}button "${full ? b.text : truncate(b.text)}"`;
|
|
108
156
|
case 'input':
|
|
109
|
-
return
|
|
157
|
+
return `${tag}input ${b.name} (${b.text})`;
|
|
158
|
+
case 'divider':
|
|
159
|
+
return b.text;
|
|
110
160
|
default:
|
|
111
|
-
return truncate(b.text)
|
|
161
|
+
return full ? `${tag}${b.text}` : `${tag}${truncate(b.text)}`;
|
|
112
162
|
}
|
|
113
163
|
}
|
|
114
164
|
|
|
115
|
-
const truncate = (s) =>
|
|
165
|
+
const truncate = (s) =>
|
|
166
|
+
s.length > TEXT_CAP ? `${s.slice(0, TEXT_CAP)} ... +${num(s.length - TEXT_CAP)} chars` : s;
|
package/src/session.js
CHANGED
|
@@ -1,16 +1,154 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Sessions are plain JSON files on disk, one per name: current URL,
|
|
3
|
-
*
|
|
4
|
-
* daemon, no background process
|
|
2
|
+
* Sessions are plain JSON files on disk, one per name: the current URL, the
|
|
3
|
+
* distilled blocks of the page it holds, how far the last render got through
|
|
4
|
+
* them, and a short history. No daemon, no background process, no cookies yet.
|
|
5
|
+
*
|
|
6
|
+
* The file exists so `oc do <n>` can follow a link the compact view never
|
|
7
|
+
* printed the URL of. Hiding URLs is what makes `oc open` cheap; this is what
|
|
8
|
+
* makes hiding them free. Keeping the blocks costs disk, not tokens, and it is
|
|
9
|
+
* what lets `oc read` and `oc next` answer without fetching the page again.
|
|
5
10
|
*/
|
|
6
11
|
|
|
7
12
|
import { homedir } from 'node:os';
|
|
8
13
|
import { join } from 'node:path';
|
|
14
|
+
import { mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
9
15
|
|
|
10
|
-
export const
|
|
16
|
+
export const DEFAULT_SESSION = 'default';
|
|
17
|
+
|
|
18
|
+
// OC_HOME relocates the whole state directory, for sandboxes, CI, and tests.
|
|
19
|
+
export const sessionDir = () => join(process.env.OC_HOME ?? join(homedir(), '.only-cli'), 'sessions');
|
|
11
20
|
|
|
12
21
|
/**
|
|
13
22
|
* @param {string} name
|
|
14
23
|
* @returns {string}
|
|
15
24
|
*/
|
|
16
|
-
export const sessionPath = (name) => join(
|
|
25
|
+
export const sessionPath = (name) => join(sessionDir(), `${name}.json`);
|
|
26
|
+
|
|
27
|
+
// Search engines and link aggregators wrap outbound links in a tracking
|
|
28
|
+
// redirector whose landing page is a script, not content, so following one
|
|
29
|
+
// verbatim renders nothing. The target is sitting in the query string.
|
|
30
|
+
const REDIRECT_PATH = /^\/(l|l\.php|url|out|redirect|away|link)\/?$/i;
|
|
31
|
+
const REDIRECT_PARAMS = ['uddg', 'url', 'u', 'q', 'target', 'to', 'dest'];
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* @param {URL} url
|
|
35
|
+
* @returns {string|null} the wrapped destination, or null if this is a normal link
|
|
36
|
+
*/
|
|
37
|
+
function unwrapRedirect(url) {
|
|
38
|
+
if (!REDIRECT_PATH.test(url.pathname)) return null;
|
|
39
|
+
for (const param of REDIRECT_PARAMS) {
|
|
40
|
+
const value = url.searchParams.get(param);
|
|
41
|
+
if (value && /^https?:\/\//i.test(value)) return value;
|
|
42
|
+
}
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Absolute URL for a handle, or null when the link is not followable
|
|
48
|
+
* (javascript: handlers, malformed hrefs).
|
|
49
|
+
* @param {string} href
|
|
50
|
+
* @param {string} base
|
|
51
|
+
* @returns {string|null}
|
|
52
|
+
*/
|
|
53
|
+
export function resolveHref(href, base) {
|
|
54
|
+
if (!href || /^(javascript|about):/i.test(href)) return null;
|
|
55
|
+
try {
|
|
56
|
+
const url = new URL(href, base || undefined);
|
|
57
|
+
return unwrapRedirect(url) ?? url.href;
|
|
58
|
+
} catch {
|
|
59
|
+
return null;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const HISTORY_LIMIT = 20;
|
|
64
|
+
|
|
65
|
+
// A ceiling on what one page may leave on disk. Nothing real comes close;
|
|
66
|
+
// it is here so a runaway page cannot fill a home directory.
|
|
67
|
+
const SNAPSHOT_CHARS = 500_000;
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Session state for a freshly rendered page. Every block is kept, including
|
|
71
|
+
* the ones the budget stopped short of, because the part an agent wants next
|
|
72
|
+
* is by definition the part that did not fit.
|
|
73
|
+
* @param {import('./distill.js').Page} page
|
|
74
|
+
* @param {{history?: string[]}} [previous]
|
|
75
|
+
* @param {{cursor?: number|null}} [opts] - where the render stopped, null when it finished the page
|
|
76
|
+
*/
|
|
77
|
+
export function sessionFromPage(page, previous, { cursor = 0 } = {}) {
|
|
78
|
+
/** @type {import('./distill.js').Block[]} */
|
|
79
|
+
const blocks = [];
|
|
80
|
+
let chars = 0;
|
|
81
|
+
let dropped = 0;
|
|
82
|
+
for (const block of page.blocks) {
|
|
83
|
+
chars += block.text.length;
|
|
84
|
+
if (chars > SNAPSHOT_CHARS) {
|
|
85
|
+
dropped++;
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
const href = block.href ? resolveHref(block.href, page.url) : null;
|
|
89
|
+
blocks.push({
|
|
90
|
+
type: block.type,
|
|
91
|
+
text: block.text,
|
|
92
|
+
...(block.n == null ? {} : { n: block.n }),
|
|
93
|
+
...(block.level == null ? {} : { level: block.level }),
|
|
94
|
+
...(href ? { href } : {}),
|
|
95
|
+
...(block.name ? { name: block.name } : {}),
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
const history = [...(previous?.history ?? []), page.url].slice(-HISTORY_LIMIT);
|
|
99
|
+
return {
|
|
100
|
+
url: page.url,
|
|
101
|
+
title: page.title,
|
|
102
|
+
savedAt: new Date().toISOString(),
|
|
103
|
+
blocks,
|
|
104
|
+
cursor,
|
|
105
|
+
...(dropped ? { dropped } : {}),
|
|
106
|
+
history,
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Handle lookup for a saved page. Sessions written by earlier versions hold a
|
|
112
|
+
* handles map instead of blocks, so `oc do` keeps working across an upgrade
|
|
113
|
+
* even though `oc read` and `oc next` need the page reopened.
|
|
114
|
+
* @param {any} state
|
|
115
|
+
* @param {number} n
|
|
116
|
+
*/
|
|
117
|
+
export function handleFor(state, n) {
|
|
118
|
+
if (state?.blocks) return state.blocks.find((b) => b.n === n) ?? null;
|
|
119
|
+
return state?.handles?.[n] ?? null;
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* The numbers a saved page offers, for error messages that tell an agent what
|
|
124
|
+
* it could have asked for.
|
|
125
|
+
* @param {any} state
|
|
126
|
+
* @returns {number[]}
|
|
127
|
+
*/
|
|
128
|
+
export function handleNumbers(state) {
|
|
129
|
+
if (state?.blocks) return state.blocks.filter((b) => b.n != null).map((b) => b.n);
|
|
130
|
+
return Object.keys(state?.handles ?? {}).map(Number);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* @param {string} name
|
|
135
|
+
* @param {object} state
|
|
136
|
+
*/
|
|
137
|
+
export function saveSession(name, state) {
|
|
138
|
+
mkdirSync(sessionDir(), { recursive: true });
|
|
139
|
+
writeFileSync(sessionPath(name), JSON.stringify(state));
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Missing or unreadable state is not an error: it means nothing is open yet,
|
|
144
|
+
* and the caller says so in a sentence that names the next command.
|
|
145
|
+
* @param {string} name
|
|
146
|
+
* @returns {any|null}
|
|
147
|
+
*/
|
|
148
|
+
export function loadSession(name) {
|
|
149
|
+
try {
|
|
150
|
+
return JSON.parse(readFileSync(sessionPath(name), 'utf8'));
|
|
151
|
+
} catch {
|
|
152
|
+
return null;
|
|
153
|
+
}
|
|
154
|
+
}
|