@only-cli/oc 0.3.0-beta.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +11 -9
- package/src/act.js +5 -1
- package/src/distill.js +80 -0
- package/src/fetch.js +30 -27
- package/src/render.js +6 -0
package/package.json
CHANGED
|
@@ -8,12 +8,13 @@ description: Token-efficient web browsing and web content extraction for AI agen
|
|
|
8
8
|
Renders a web page as a compact, numbered terminal view instead of raw HTML. A typical page is under 500 tokens.
|
|
9
9
|
|
|
10
10
|
```
|
|
11
|
-
npx --yes @only-cli/oc@0.
|
|
12
|
-
npx --yes @only-cli/oc@0.
|
|
13
|
-
npx --yes @only-cli/oc@0.
|
|
14
|
-
|
|
15
|
-
npx --yes @only-cli/oc@0.
|
|
16
|
-
npx --yes @only-cli/oc@0.
|
|
11
|
+
npx --yes @only-cli/oc@0.3.0 open <url> compact view, numbered elements
|
|
12
|
+
npx --yes @only-cli/oc@0.3.0 do <n> follow link [n], or read it if [n] is text
|
|
13
|
+
npx --yes @only-cli/oc@0.3.0 find <query> where a string appears, or that place itself
|
|
14
|
+
when only one matches
|
|
15
|
+
npx --yes @only-cli/oc@0.3.0 next next ~500 tokens of the page already open
|
|
16
|
+
npx --yes @only-cli/oc@0.3.0 read <n> full text of region [n]
|
|
17
|
+
npx --yes @only-cli/oc@0.3.0 raw [url] whole page as markdown (--html for cleaned HTML)
|
|
17
18
|
```
|
|
18
19
|
|
|
19
20
|
None of these except `open`/`do`/`raw <url>` fetch anything — they replay the page `open` already saved.
|
|
@@ -23,13 +24,14 @@ None of these except `open`/`do`/`raw <url>` fetch anything — they replay the
|
|
|
23
24
|
- Line 1 is the title, then main content (article/thread/results); nav/sidebar/footer follow after `--- rest of page ---`, still numbered.
|
|
24
25
|
- `--- repeated controls hidden ---`: per-item chrome (save/report/reply) dropped as repetitive; `raw` keeps it.
|
|
25
26
|
- `[n]` marks a link, button, input, heading, or a text block long enough to be cut.
|
|
26
|
-
-
|
|
27
|
+
- Code blocks arrive as the page wrote them, lines and indentation intact, so a command in one can be run as printed.
|
|
28
|
+
- `... +820 chars`: block was cut there; `read <n>` prints it whole. The cut lands on the end of a sentence, or of a line in code, so what is shown is never half of one.
|
|
27
29
|
- `... 164 more blocks (~7,100 tokens)`: rest of page past budget — a cost estimate, not a fetch. Omitted when the page would finish only a little over budget; then it's printed whole instead.
|
|
28
30
|
- `actions:` footer lists valid next commands.
|
|
29
31
|
|
|
30
32
|
## Going further, cheapest first
|
|
31
33
|
|
|
32
|
-
- `find <query>` — every place a string appears, one line + number each. Matches as a phrase (case-insensitive), falling back to separate words; reports how many matches didn't fit.
|
|
34
|
+
- `find <query>` — every place a string appears, one line + number each. Matches as a phrase (case-insensitive), falling back to separate words; reports how many matches didn't fit. When one place matches, or when the matches all fit, it prints them in full: no `read <n>` afterwards.
|
|
33
35
|
- `read <n>` — one region in full: the block at `[n]` plus a little context, or the whole section for a heading.
|
|
34
36
|
- `next` — continues the same page from where the budget stopped.
|
|
35
37
|
- `raw` — everything, ~10x the cost. Use only when you need the whole page, not to hunt for a link's URL (use `do` for that).
|
|
@@ -42,7 +44,7 @@ None of these except `open`/`do`/`raw <url>` fetch anything — they replay the
|
|
|
42
44
|
- `[6-9] 4 similar links` markers still work despite the collapsed text.
|
|
43
45
|
- Search result links resolve to the destination, not the tracking redirect.
|
|
44
46
|
- `do` on an input/button reports that instead (typing/submitting not yet supported).
|
|
45
|
-
- `do` on a heading/text block prints the read instead of refusing, since there's nothing to follow.
|
|
47
|
+
- `do` on a heading/text block prints the read instead of refusing, since there's nothing to follow. A heading that is itself a link, which is what a search result title is, opens instead.
|
|
46
48
|
- `--session <name>` keeps separate page state, for working on two sites at once.
|
|
47
49
|
|
|
48
50
|
## Flags
|
package/src/act.js
CHANGED
|
@@ -282,7 +282,11 @@ function search(blocks, terms) {
|
|
|
282
282
|
if (out.length && out.at(-1).n === n) continue;
|
|
283
283
|
const start = Math.max(0, Math.min(...found) - BEFORE);
|
|
284
284
|
const end = Math.min(block.text.length, start + SNIPPET);
|
|
285
|
-
|
|
285
|
+
// One line per match is the promise the list makes, and a code block is the
|
|
286
|
+
// one kind of block that carries lines of its own. They survive where they
|
|
287
|
+
// are read rather than indexed: the whole-match mode above, and `read`.
|
|
288
|
+
const window = block.text.slice(start, end).replace(/\n/g, ' ');
|
|
289
|
+
const snippet = `${start > 0 ? '... ' : ''}${window}${end < block.text.length ? ' ...' : ''}`;
|
|
286
290
|
out.push({ n, type: block.type, snippet, text: block.text });
|
|
287
291
|
}
|
|
288
292
|
return out;
|
package/src/distill.js
CHANGED
|
@@ -32,6 +32,32 @@ const DROP = new Set([
|
|
|
32
32
|
// them, so the search below refuses to descend into them.
|
|
33
33
|
const FURNITURE = new Set(['nav', 'header', 'footer', 'aside']);
|
|
34
34
|
|
|
35
|
+
// Controls a page puts inside its own code samples, and the selector that
|
|
36
|
+
// finds one. Node's API docs give every code block a copy button, a module
|
|
37
|
+
// toggle and a language label, so the text of the sample ends in
|
|
38
|
+
// `javascriptcopy` unless the toolbar holding them is left out of it.
|
|
39
|
+
const CODE_CONTROLS = new Set(['button', 'input', 'select', 'textarea', 'label']);
|
|
40
|
+
const CONTROL = [...CODE_CONTROLS].join(', ');
|
|
41
|
+
|
|
42
|
+
// Lines in a code block are kept, where every other kind of text has its
|
|
43
|
+
// whitespace collapsed. Joining them would put `// comment` in front of the
|
|
44
|
+
// statements that followed it, so a sample an agent could run would arrive
|
|
45
|
+
// commented out, and the shape of the wreck is invisible on one line. Blank
|
|
46
|
+
// runs and the indentation the whole block shares are the parts that carry no
|
|
47
|
+
// meaning, so those go.
|
|
48
|
+
const codeText = (raw) => {
|
|
49
|
+
const lines = raw.replace(/\r\n?/g, '\n').replace(/[^\S\n]+$/gm, '').split('\n');
|
|
50
|
+
while (lines.length && !lines[0].trim()) lines.shift();
|
|
51
|
+
while (lines.length && !lines[lines.length - 1].trim()) lines.pop();
|
|
52
|
+
const indent = lines
|
|
53
|
+
.filter((l) => l.trim())
|
|
54
|
+
.reduce((least, l) => Math.min(least, l.length - l.trimStart().length), Infinity);
|
|
55
|
+
return lines
|
|
56
|
+
.map((l) => (Number.isFinite(indent) ? l.slice(indent) : l).trimEnd())
|
|
57
|
+
.join('\n')
|
|
58
|
+
.replace(/\n{3,}/g, '\n\n');
|
|
59
|
+
};
|
|
60
|
+
|
|
35
61
|
// Elements that end a line on the page and so must end one here. Without this
|
|
36
62
|
// the text of six separate posts merges into a single block, because nothing
|
|
37
63
|
// between them survives distillation to keep them apart.
|
|
@@ -128,6 +154,45 @@ export function distill(html, url = '') {
|
|
|
128
154
|
/** Subtree already emitted, skipped when the rest of the page is walked. */
|
|
129
155
|
let done = null;
|
|
130
156
|
|
|
157
|
+
/**
|
|
158
|
+
* The text of a code subtree, read as one string.
|
|
159
|
+
*
|
|
160
|
+
* A syntax highlighter gives every token its own element, so `s3://bucket/`
|
|
161
|
+
* reaches the walk as `s3`, `:`, `//`, `bucket`, `/`, and the rule that puts
|
|
162
|
+
* fragments back together only glues the ones that share a parent. The rest
|
|
163
|
+
* are space-joined, which turned an AWS example into
|
|
164
|
+
* `aws s3 cp s3 : // bucket / -- recursive`, a command an agent cannot run.
|
|
165
|
+
* Reading the subtree whole is what fixes that, and it discards nothing: over
|
|
166
|
+
* 172 pre elements on the AWS CLI reference, the Rust book, the Node API docs
|
|
167
|
+
* and the Python library docs, 159 are split this way and not one of them
|
|
168
|
+
* contains a link.
|
|
169
|
+
*
|
|
170
|
+
* textContent would be enough if pages put only code in their code blocks.
|
|
171
|
+
* A control inside one is chrome, and so is the block-level element holding
|
|
172
|
+
* it: that is how the toolbar's `javascript` label leaves with the copy
|
|
173
|
+
* button it sits beside. The test stays on block-level wrappers because a
|
|
174
|
+
* highlighter's own elements are inline, so a stray control can never take a
|
|
175
|
+
* line of code out with it.
|
|
176
|
+
* @param {any} node
|
|
177
|
+
* @returns {string}
|
|
178
|
+
*/
|
|
179
|
+
const verbatim = (node) => {
|
|
180
|
+
let out = '';
|
|
181
|
+
const gather = (n) => {
|
|
182
|
+
if (n.nodeType === 3) {
|
|
183
|
+
out += n.textContent ?? '';
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
if (n.nodeType !== 1) return;
|
|
187
|
+
const tag = n.localName;
|
|
188
|
+
if (DROP.has(tag) || CODE_CONTROLS.has(tag) || hidden(n)) return;
|
|
189
|
+
if (n !== node && BLOCKY.has(tag) && n.querySelector(CONTROL)) return;
|
|
190
|
+
for (const child of n.childNodes) gather(child);
|
|
191
|
+
};
|
|
192
|
+
gather(node);
|
|
193
|
+
return out;
|
|
194
|
+
};
|
|
195
|
+
|
|
131
196
|
const walk = (node) => {
|
|
132
197
|
if (node.nodeType === 3) {
|
|
133
198
|
const raw = node.textContent ?? '';
|
|
@@ -183,6 +248,21 @@ export function distill(html, url = '') {
|
|
|
183
248
|
if (text) blocks.push({ type: 'button', text });
|
|
184
249
|
return;
|
|
185
250
|
}
|
|
251
|
+
if (tag === 'pre' || tag === 'code') {
|
|
252
|
+
const raw = verbatim(node);
|
|
253
|
+
const text = tag === 'pre' ? codeText(raw) : clean(raw);
|
|
254
|
+
// A pre is its own line, which is what BLOCKY gave it before this branch
|
|
255
|
+
// started claiming it first. Inline code belongs to the sentence around
|
|
256
|
+
// it, so it carries the parent and the edge spacing a text node would and
|
|
257
|
+
// merges back into that sentence the same way.
|
|
258
|
+
const own = tag === 'pre';
|
|
259
|
+
if (own) blocks.push({ type: 'break' });
|
|
260
|
+
if (text) {
|
|
261
|
+
blocks.push({ type: 'text', text, host: node.parentNode, pre: /^\s/.test(raw), post: /\s$/.test(raw) });
|
|
262
|
+
}
|
|
263
|
+
if (own) blocks.push({ type: 'break' });
|
|
264
|
+
return;
|
|
265
|
+
}
|
|
186
266
|
if (BLOCKY.has(tag)) {
|
|
187
267
|
blocks.push({ type: 'break' });
|
|
188
268
|
for (const child of node.childNodes) walk(child);
|
package/src/fetch.js
CHANGED
|
@@ -160,11 +160,25 @@ export async function fetchPage(url) {
|
|
|
160
160
|
return impers ? viaImpers(impers, target) : viaFetch(target);
|
|
161
161
|
}
|
|
162
162
|
|
|
163
|
-
|
|
164
|
-
|
|
163
|
+
/**
|
|
164
|
+
* Follow redirects one hop at a time, validating each destination before the
|
|
165
|
+
* next request goes out.
|
|
166
|
+
*
|
|
167
|
+
* Both transports share this loop. They used to carry one each, which made the
|
|
168
|
+
* check that matters something a change could fix in one place and leave broken
|
|
169
|
+
* in the other, and made the guarantee testable only through a third party
|
|
170
|
+
* willing to 302 wherever it was told. Taking the request as a callback is what
|
|
171
|
+
* lets the hop check be proven against a transport that never leaves the
|
|
172
|
+
* process.
|
|
173
|
+
* @param {(url: string) => Promise<any>} get - one request, redirects not followed
|
|
174
|
+
* @param {string} start
|
|
175
|
+
* @returns {Promise<{res: any, url: string}>} the first non-redirect response
|
|
176
|
+
*/
|
|
177
|
+
export async function followRedirects(get, start) {
|
|
178
|
+
let current = start;
|
|
165
179
|
for (let i = 0; ; i++) {
|
|
166
|
-
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${
|
|
167
|
-
const res = await
|
|
180
|
+
if (i > MAX_REDIRECTS) throw new Error(`too many redirects for ${start}`);
|
|
181
|
+
const res = await get(current);
|
|
168
182
|
const status = res.status ?? res.statusCode ?? 0;
|
|
169
183
|
const location = res.headers.get('location');
|
|
170
184
|
if (status >= 300 && status < 400 && location) {
|
|
@@ -172,19 +186,20 @@ async function followImpersRedirects(impers, startUrl, impersonate) {
|
|
|
172
186
|
await assertSafeTarget(current);
|
|
173
187
|
continue;
|
|
174
188
|
}
|
|
175
|
-
return res;
|
|
189
|
+
return { res, url: current };
|
|
176
190
|
}
|
|
177
191
|
}
|
|
178
192
|
|
|
179
193
|
async function viaImpers(impers, target) {
|
|
180
194
|
// Some sites (Reddit) 403 the chrome fingerprint but accept firefox, so a
|
|
181
195
|
// blocked first attempt gets one cheap retry with a second identity.
|
|
196
|
+
const asking = (impersonate) => (url) => impers.get(url, { impersonate, allowRedirects: false });
|
|
182
197
|
let via = 'impers:chrome';
|
|
183
|
-
let res = await
|
|
198
|
+
let { res } = await followRedirects(asking('chrome'), target);
|
|
184
199
|
let status = res.status ?? res.statusCode ?? 0;
|
|
185
200
|
if (status >= 400) {
|
|
186
201
|
via = 'impers:firefox';
|
|
187
|
-
res = await
|
|
202
|
+
({ res } = await followRedirects(asking('firefox'), target));
|
|
188
203
|
status = res.status ?? res.statusCode ?? 0;
|
|
189
204
|
}
|
|
190
205
|
if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
|
|
@@ -194,26 +209,14 @@ async function viaImpers(impers, target) {
|
|
|
194
209
|
}
|
|
195
210
|
|
|
196
211
|
async function viaFetch(target) {
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
206
|
-
'accept-language': 'en-US,en;q=0.9',
|
|
207
|
-
},
|
|
208
|
-
});
|
|
209
|
-
const location = res.headers.get('location');
|
|
210
|
-
if (res.status >= 300 && res.status < 400 && location) {
|
|
211
|
-
current = new URL(location, current).toString();
|
|
212
|
-
await assertSafeTarget(current);
|
|
213
|
-
continue;
|
|
214
|
-
}
|
|
215
|
-
break;
|
|
216
|
-
}
|
|
212
|
+
const { res, url: current } = await followRedirects((url) => fetch(url, {
|
|
213
|
+
redirect: 'manual',
|
|
214
|
+
headers: {
|
|
215
|
+
'user-agent': UA,
|
|
216
|
+
accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
217
|
+
'accept-language': 'en-US,en;q=0.9',
|
|
218
|
+
},
|
|
219
|
+
}), target);
|
|
217
220
|
if (!res.ok) {
|
|
218
221
|
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
|
219
222
|
}
|
package/src/render.js
CHANGED
|
@@ -174,6 +174,12 @@ const SENTENCE_FLOOR = 0.7;
|
|
|
174
174
|
|
|
175
175
|
const truncate = (s) => {
|
|
176
176
|
if (s.length <= TEXT_CAP) return s;
|
|
177
|
+
// A line is to code what a sentence is to prose, and a code block is the only
|
|
178
|
+
// text that keeps its newlines, so the same courtesy applies: cut where a
|
|
179
|
+
// line ended. A period in code ends nothing, which is why this returns
|
|
180
|
+
// instead of falling through to the sentence rule below.
|
|
181
|
+
const line = s.slice(0, TEXT_CAP).lastIndexOf('\n');
|
|
182
|
+
if (line >= TEXT_CAP * SENTENCE_FLOOR) return `${s.slice(0, line)} ... +${num(s.length - line)} chars`;
|
|
177
183
|
let cut = TEXT_CAP;
|
|
178
184
|
for (const m of s.slice(0, TEXT_CAP).matchAll(SENTENCE_END)) {
|
|
179
185
|
const end = (m.index ?? 0) + m[0].length;
|