@only-cli/oc 0.2.0-beta.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -26
- package/clis/finance.yahoo.com.json +13 -0
- package/clis/stackoverflow.com.json +9 -0
- package/clis/x.com.json +7 -0
- package/clis/youtube.com.json +7 -0
- package/package.json +1 -1
- package/skills/only-cli/SKILL.md +49 -7
- package/src/act.js +235 -17
- package/src/cli.js +65 -15
- package/src/distill.js +380 -22
- package/src/fetch.js +151 -14
- package/src/render.js +75 -24
- package/src/session.js +143 -5
package/README.md
CHANGED
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# only-cli
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+

|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
[](https://www.npmjs.com/package/@only-cli/oc) [](https://nodejs.org) [](#license)
|
|
6
|
+
|
|
7
|
+
Turns websites into a command line interface for AI agents. `oc open <url>` fetches a page and hands back a compact, numbered view instead of raw HTML or a screenshot, so agents like Claude Code, Codex, and Antigravity can browse without burning tokens. It also gets past blocks that stop naive fetchers on some sites, by talking to the page the way a real browser would.
|
|
6
8
|
|
|
7
9
|
```
|
|
8
10
|
$ oc open news.ycombinator.com
|
|
@@ -10,12 +12,14 @@ $ oc open news.ycombinator.com
|
|
|
10
12
|
[1] Show HN: I built a tiny CSV toolkit
|
|
11
13
|
[2] 312 comments
|
|
12
14
|
...
|
|
13
|
-
actions: do <n> |
|
|
15
|
+
actions: do <n> | read <n> | next | raw
|
|
14
16
|
|
|
15
17
|
$ oc do 1
|
|
16
18
|
```
|
|
17
19
|
|
|
18
|
-
|
|
20
|
+
A typical page is tens of thousands of tokens of markup; the view above fits in a few hundred. No per-site adapters required, no browser extension, no daemon.
|
|
21
|
+
|
|
22
|
+
If you are an LLM reading this repository, [llms.txt](llms.txt) is the short version.
|
|
19
23
|
|
|
20
24
|
## Install
|
|
21
25
|
|
|
@@ -23,64 +27,69 @@ No per-site adapters required, no browser extension, no daemon. One generic dist
|
|
|
23
27
|
npm install -g @only-cli/oc
|
|
24
28
|
```
|
|
25
29
|
|
|
26
|
-
Requires Node 20+. Requests
|
|
30
|
+
Requires Node 20+. Requests impersonate Chrome via [impers](https://github.com/lexiforest/impers); falls back to native fetch if impers is unavailable.
|
|
27
31
|
|
|
28
32
|
## For AI agents
|
|
29
33
|
|
|
30
|
-
|
|
34
|
+
Add one line to your agent's instructions file (CLAUDE.md, AGENTS.md, or equivalent):
|
|
31
35
|
|
|
32
36
|
> When you need content from a web page, run `npx @only-cli/oc open <url>` instead of fetching raw HTML. Run `npx @only-cli/oc --help` once to learn the commands.
|
|
33
37
|
|
|
34
|
-
Claude Code users can install the skill instead: copy `skills/only-cli/` into
|
|
35
|
-
|
|
36
|
-
```
|
|
37
|
-
npx skills add only-cli/oc
|
|
38
|
-
```
|
|
38
|
+
Claude Code users can install the skill instead: copy `skills/only-cli/` into `.claude/skills/`, or run `npx skills add only-cli/oc`, which also works in Cursor, Codex, Copilot, and others via [skills.sh](https://skills.sh).
|
|
39
39
|
|
|
40
|
-
No setup at all also works: `npx @only-cli/oc` runs without a global install, and
|
|
40
|
+
No setup at all also works: `npx @only-cli/oc` runs without a global install, and teaches its own commands through `--help` and the `actions:` line on every render.
|
|
41
41
|
|
|
42
42
|
## Commands
|
|
43
43
|
|
|
44
44
|
```
|
|
45
45
|
oc open <url> fetch and render a page with numbered actions
|
|
46
|
-
oc
|
|
47
|
-
oc
|
|
46
|
+
oc do <n> follow the numbered link [n], or read [n] if it is text
|
|
47
|
+
oc find <query> where a string appears on the page already open
|
|
48
|
+
oc read <n> full text of the region at [n], up to 2000 tokens
|
|
49
|
+
oc next the next budget worth of the page already open
|
|
50
|
+
oc raw [url] distilled markdown of the whole page
|
|
48
51
|
oc fill <n> <text> type into a numbered input (v0.2)
|
|
49
52
|
oc submit [n] submit a form (v0.2)
|
|
50
53
|
```
|
|
51
54
|
|
|
52
|
-
Flags: `--budget <tokens>` (default 500), `--json`, `--html` (raw as cleaned HTML
|
|
55
|
+
Flags: `--budget <tokens>` (default 500), `--json`, `--html` (raw as cleaned HTML), `--session <name>`, `--verbose`/`-v` (metrics on stderr, or export `OC_VERBOSE=1`).
|
|
56
|
+
|
|
57
|
+
`oc open` remembers the page it rendered in a JSON file per session under `~/.only-cli` (override with `OC_HOME`), so `oc do 3` follows `[3]` without the agent ever handling a URL. Pages longer than the budget say what they left out; `oc find`, `oc read <n>`, and `oc next` read the rest without refetching the page. The budget is a target rather than a hard cap: a page that would only run a little long is printed whole rather than cut, since one extra tool call costs far more than the tokens it would have saved.
|
|
53
58
|
|
|
54
59
|
## Supported websites
|
|
55
60
|
|
|
56
|
-
|
|
61
|
+
Works on any mostly-static site with no per-site setup: news sites, blogs, documentation, forums, search engines. On top of that, `clis/` ships tuned shortcuts for:
|
|
57
62
|
|
|
58
63
|
| website | domain | shortcuts |
|
|
59
64
|
| --- | --- | --- |
|
|
60
65
|
| Hacker News | news.ycombinator.com | `top`, `new`, `item <id>`, `user <name>` |
|
|
61
66
|
| Reddit | reddit.com (via old.reddit.com) | `sub <name>`, `post <id>`, `user <name>`, `search <query>` |
|
|
62
67
|
| GitHub | github.com | `repo <owner> <name>`, `user <name>`, `search <query>`, `trending`, `issues <owner> <name>` |
|
|
68
|
+
| X | x.com | `user <name>`, `post <id>` |
|
|
63
69
|
| LinkedIn | linkedin.com | `profile <name>`, `company <name>`, `jobs <query>` (public guest views) |
|
|
64
70
|
| DuckDuckGo | duckduckgo.com | `search <query>`, `lite <query>` |
|
|
65
71
|
| Bing | bing.com | `search <query>`, `news <query>` |
|
|
72
|
+
| Stack Overflow | stackoverflow.com (via Atom feeds) | `question <id>`, `tag <name>`, `user <id>`, `recent` |
|
|
73
|
+
| Yahoo Finance | finance.yahoo.com | `quote <symbol>`, `news <symbol>`, `history <symbol>`, `lookup <query>`, `markets`, `gainers`, `losers`, `trending` |
|
|
74
|
+
| YouTube | youtube.com | `video <id>`, `channel <name>` |
|
|
66
75
|
|
|
67
|
-
|
|
76
|
+
A few of these (X, Stack Overflow, YouTube) read pages that look login-gated or JS-only from the outside, by finding the server-rendered HTML, feed, or inline data the page already ships without a login. Not supported yet: pages that only render with JavaScript, sites behind logins, and sites with hard bot challenges that expose no feed.
|
|
68
77
|
|
|
69
|
-
|
|
78
|
+
Want a website on that list? Open a pull request, or an issue naming the site — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
70
79
|
|
|
71
|
-
|
|
80
|
+
## Benchmarks
|
|
72
81
|
|
|
73
|
-
|
|
74
|
-
| --- | ---: | ---: | ---: |
|
|
75
|
-
| `oc open` | 4/4 | 1,444 | 591 |
|
|
76
|
-
| `oc raw` | 4/4 | 15,170 | 659 |
|
|
77
|
-
| raw HTML fetch | 4/4 | 65,372 | 247 |
|
|
82
|
+
Full methodology, per-task numbers, and other agents/models live in [only-cli/benchmarks](https://github.com/only-cli/benchmarks). The short version, measured against live sites across a news front page, a Reddit discussion, a search results page, and more:
|
|
78
83
|
|
|
79
|
-
|
|
84
|
+
| method | tokens for 6 real pages | notes |
|
|
85
|
+
| --- | ---: | --- |
|
|
86
|
+
| `oc open` | 1,936 | only method that returned real content on every page |
|
|
87
|
+
| Jina Reader | 16,402 | blocked on the Reddit page |
|
|
88
|
+
| raw HTML fetch | 177,685 | blocked on the search page |
|
|
80
89
|
|
|
81
90
|
## Status
|
|
82
91
|
|
|
83
|
-
Early. v0.1 covers static pages, budget-aware rendering, and offline tests. Sessions and actions land in v0.2, a lazy headless fallback for script-heavy pages in v0.3.
|
|
92
|
+
Early. v0.1 covers static pages, budget-aware rendering, and offline tests. Sessions, `oc do <n>`, `oc find <query>`, `oc read <n>`, and `oc next` are in, the rest of the actions (`fill`, `submit`, `back`) land in v0.2, and a lazy headless fallback for script-heavy pages in v0.3.
|
|
84
93
|
|
|
85
94
|
Known limits, honestly: no JavaScript rendering yet, no sites behind logins yet, and pages behind hard bot challenges may still refuse the tool.
|
|
86
95
|
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
{
|
|
2
|
+
"domain": "finance.yahoo.com",
|
|
3
|
+
"commands": {
|
|
4
|
+
"quote": { "open": "https://finance.yahoo.com/quote/{symbol}", "args": ["symbol"] },
|
|
5
|
+
"news": { "open": "https://finance.yahoo.com/quote/{symbol}/news", "args": ["symbol"] },
|
|
6
|
+
"history": { "open": "https://finance.yahoo.com/quote/{symbol}/history", "args": ["symbol"] },
|
|
7
|
+
"lookup": { "open": "https://finance.yahoo.com/lookup?s={query}", "args": ["query"] },
|
|
8
|
+
"markets": { "open": "https://finance.yahoo.com/markets" },
|
|
9
|
+
"gainers": { "open": "https://finance.yahoo.com/markets/stocks/gainers/" },
|
|
10
|
+
"losers": { "open": "https://finance.yahoo.com/markets/stocks/losers/" },
|
|
11
|
+
"trending": { "open": "https://finance.yahoo.com/markets/stocks/trending/" }
|
|
12
|
+
}
|
|
13
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
{
|
|
2
|
+
"domain": "stackoverflow.com",
|
|
3
|
+
"commands": {
|
|
4
|
+
"question": { "open": "https://stackoverflow.com/feeds/question/{id}", "args": ["id"] },
|
|
5
|
+
"tag": { "open": "https://stackoverflow.com/feeds/tag?tagnames={name}&sort=newest", "args": ["name"] },
|
|
6
|
+
"user": { "open": "https://stackoverflow.com/feeds/user/{id}", "args": ["id"] },
|
|
7
|
+
"recent": { "open": "https://stackoverflow.com/feeds" }
|
|
8
|
+
}
|
|
9
|
+
}
|
package/clis/x.com.json
ADDED
package/package.json
CHANGED
package/skills/only-cli/SKILL.md
CHANGED
|
@@ -11,23 +11,65 @@ No install needed, run it with npx:
|
|
|
11
11
|
|
|
12
12
|
```
|
|
13
13
|
npx @only-cli/oc open <url> compact view with numbered elements
|
|
14
|
-
npx @only-cli/oc
|
|
14
|
+
npx @only-cli/oc do <n> follow numbered link [n] from the last page
|
|
15
|
+
npx @only-cli/oc find <query> where a string appears on the page already open
|
|
16
|
+
npx @only-cli/oc next the next ~500 tokens of the page already open
|
|
17
|
+
npx @only-cli/oc read <n> full text of the region at [n]
|
|
18
|
+
npx @only-cli/oc raw [url] whole page as markdown (add --html for cleaned HTML)
|
|
15
19
|
```
|
|
16
20
|
|
|
17
21
|
## Reading the output
|
|
18
22
|
|
|
19
|
-
- The first line is the page title, then
|
|
20
|
-
-
|
|
23
|
+
- The first line is the page title, then the page's main content: the article, the comment thread, the results. Navigation, sidebar, and footer come after it, under a `--- rest of page ---` line, still numbered and still followable with `do <n>`.
|
|
24
|
+
- A `--- repeated controls hidden ---` line means per item chrome (save, report, reply, like) was removed because it repeated down the page. `oc raw` still has it.
|
|
25
|
+
- `[n]` marks a link, button, input, heading, or a text block long enough to be cut.
|
|
26
|
+
- `... +820 chars` at the end of a line means that block was cut there. `read <n>` prints it whole.
|
|
27
|
+
- `... 164 more blocks (~7,100 tokens)` means the page ran past the budget. That is the price of the rest, so you can decide before paying. A page that would have finished a little past the budget has no such line: it is printed whole, because a second command costs more than the lines it would have saved.
|
|
21
28
|
- The `actions:` line at the bottom lists valid next commands.
|
|
22
|
-
- `... N more blocks over budget` means content was cut to stay cheap. Rerun with `--budget 1500` if you need more, or use `raw` for everything.
|
|
23
29
|
|
|
24
|
-
##
|
|
30
|
+
## Reading more of a page
|
|
25
31
|
|
|
26
|
-
|
|
32
|
+
Four ways to go past the first view, cheapest first. Pick by what you need, not by habit.
|
|
33
|
+
|
|
34
|
+
- `oc find <query>` prints every place a string appears on the page, one line each with the number to read it by. When you know what you are looking for, this is the whole job in one command.
|
|
35
|
+
- `oc read <n>` prints one region in full: the block at `[n]` with a little context, or the whole section when `[n]` is a heading. Use it when the view or a `find` hit shows you exactly the block you want.
|
|
36
|
+
- `oc next` prints the next budget worth of the same page and remembers where it stopped, so calling it again continues. Use it when you are reading rather than looking something up.
|
|
37
|
+
- `oc raw` (no URL needed once a page is open) prints everything. It costs an order of magnitude more, so use it when you genuinely need the whole page.
|
|
38
|
+
|
|
39
|
+
Measured on one Reddit thread: `open` 436 tokens, one `find` 142, one `read` 143, each `next` about 450, `raw` 9,636. None of them fetches anything; they all work from the page `open` already saved.
|
|
40
|
+
|
|
41
|
+
```
|
|
42
|
+
oc open https://old.reddit.com/r/linuxquestions/comments/xpznb1/best_terminal_web_browser/
|
|
43
|
+
oc find w3m -> 7 matches with their numbers, 142 tokens
|
|
44
|
+
oc read 23 -> that comment in full, 143 tokens
|
|
45
|
+
oc next -> keep reading, 450 tokens at a time
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
`find` matches the query as a phrase, case insensitive, and falls back to matching the words separately when the phrase is not there. It says how many matches it held back if they did not fit the budget.
|
|
49
|
+
|
|
50
|
+
## Following links
|
|
51
|
+
|
|
52
|
+
Use `do <n>`. The compact view leaves link URLs out because they cost tokens and you do not need them, so to open `[15] 41 comments` run `oc do 15`. It renders the new page exactly like `open` does, and the numbers then refer to that new page.
|
|
53
|
+
|
|
54
|
+
```
|
|
55
|
+
oc open news.ycombinator.com -> [15] 41 comments
|
|
56
|
+
oc do 15 -> the comment thread, renumbered
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Notes that save a round trip:
|
|
60
|
+
|
|
61
|
+
- Numbers come from the most recent render, so re-read the newest output before choosing one. Any command that renders a page renumbers.
|
|
62
|
+
- Handles hidden behind a `[6-9] 4 similar links` marker still work, even though their text was collapsed.
|
|
63
|
+
- Search result links resolve to the destination, not the search engine's tracking redirect.
|
|
64
|
+
- `do` on an input or a button says so; typing and submitting are not available yet.
|
|
65
|
+
- `do` on a heading or a text block has nothing to follow, so it prints the read instead of refusing.
|
|
66
|
+
- `--session <name>` keeps separate page state, for working on two sites at once.
|
|
67
|
+
|
|
68
|
+
Reach for `raw <url>` when you need the whole page text, not to hunt for a URL.
|
|
27
69
|
|
|
28
70
|
## Flags
|
|
29
71
|
|
|
30
|
-
- `--budget <tokens>` raise or lower the render budget (default 500)
|
|
72
|
+
- `--budget <tokens>` raise or lower the render budget (default 500, 2000 for `read`). It is a target rather than a hard cap: a page that would finish within about four times it comes out whole instead of being cut.
|
|
31
73
|
- `--json` machine-stable JSON of the distilled page
|
|
32
74
|
- `--html` with raw: cleaned HTML instead of markdown, if markup suits your task better
|
|
33
75
|
- `--verbose` (`-v`, alias `--stats`) metrics on stderr: tokens saved vs the page HTML, HTTP status and which client identity got the page, fetch and processing time, bytes transferred, memory use. Only pass this when you are running in verbose mode or diagnosing a problem; the metrics line costs tokens like everything else. Users can export `OC_VERBOSE=1` to turn it on globally.
|
package/src/act.js
CHANGED
|
@@ -1,18 +1,169 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Actions against a
|
|
3
|
-
*
|
|
4
|
-
*
|
|
2
|
+
* Actions against a saved session: do, read, and next now, fill, submit, find,
|
|
3
|
+
* and back still ahead. Errors here are read by agents, so every one of them
|
|
4
|
+
* names the command to run next.
|
|
5
|
+
*
|
|
6
|
+
* Nothing in this file touches the network. The page an agent is working on is
|
|
7
|
+
* already on disk from the last `oc open`, so continuing to read it costs one
|
|
8
|
+
* file read and whatever tokens the caller asked for.
|
|
5
9
|
*/
|
|
6
10
|
|
|
11
|
+
import { DEFAULT_SESSION, handleFor, handleNumbers, loadSession, saveSession } from './session.js';
|
|
12
|
+
import { estimateTokens, formatBlock, render } from './render.js';
|
|
13
|
+
|
|
7
14
|
export class NotImplemented extends Error {
|
|
8
15
|
constructor(command) {
|
|
9
|
-
super(`'oc ${command}'
|
|
16
|
+
super(`'oc ${command}' is not available yet. Until then use 'oc open' and 'oc raw'.`);
|
|
10
17
|
}
|
|
11
18
|
}
|
|
12
19
|
|
|
13
|
-
/**
|
|
14
|
-
|
|
15
|
-
|
|
20
|
+
/**
|
|
21
|
+
* @param {string} session
|
|
22
|
+
* @returns {any}
|
|
23
|
+
*/
|
|
24
|
+
function requireSession(session) {
|
|
25
|
+
const state = loadSession(session);
|
|
26
|
+
if (!state) {
|
|
27
|
+
throw new Error("nothing open in this session yet, run 'oc open <url>' first");
|
|
28
|
+
}
|
|
29
|
+
return state;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* `read` and `next` work from the saved blocks, which older sessions do not
|
|
34
|
+
* have. One `oc open` fixes it, so say that instead of failing blankly.
|
|
35
|
+
* @param {string} session
|
|
36
|
+
*/
|
|
37
|
+
function requireBlocks(session) {
|
|
38
|
+
const state = requireSession(session);
|
|
39
|
+
if (!Array.isArray(state.blocks)) {
|
|
40
|
+
throw new Error(`this session was saved by an older oc, run 'oc open ${state.url}' again`);
|
|
41
|
+
}
|
|
42
|
+
return state;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Resolve a numbered handle from the last render into something to open.
|
|
47
|
+
* Returns the target URL; the caller fetches and renders it exactly as
|
|
48
|
+
* `oc open` would, so `do` and `open` always agree on what a page looks like.
|
|
49
|
+
* When the number is text rather than a link it returns `{read}` instead, and
|
|
50
|
+
* the caller reads it.
|
|
51
|
+
* @param {number} n
|
|
52
|
+
* @param {{session?: string}} [opts]
|
|
53
|
+
* @returns {{url?: string, read?: number, text: string}}
|
|
54
|
+
*/
|
|
55
|
+
export function activate(n, { session = DEFAULT_SESSION } = {}) {
|
|
56
|
+
if (!Number.isInteger(n) || n < 1) {
|
|
57
|
+
throw new Error('usage: oc do <n>, where <n> is a number from the last page');
|
|
58
|
+
}
|
|
59
|
+
const state = requireSession(session);
|
|
60
|
+
const handle = handleFor(state, n);
|
|
61
|
+
if (!handle) {
|
|
62
|
+
const nums = handleNumbers(state);
|
|
63
|
+
const range = nums.length ? `1-${Math.max(...nums)}` : 'none';
|
|
64
|
+
throw new Error(`no [${n}] on ${state.url} (handles ${range}), run 'oc open <url>' again to renumber`);
|
|
65
|
+
}
|
|
66
|
+
if (handle.type === 'text' || handle.type === 'heading') {
|
|
67
|
+
// There is nothing to follow, but the agent asked to see what is at [n],
|
|
68
|
+
// and that is what read prints. Refusing would spend a whole turn to name
|
|
69
|
+
// the command that should have run, and a turn costs more than the page.
|
|
70
|
+
return { read: n, text: handle.text };
|
|
71
|
+
}
|
|
72
|
+
if (handle.type === 'input') {
|
|
73
|
+
throw new Error(`[${n}] is an input (${handle.name ?? 'text'}), typing needs 'oc fill', which is not available yet`);
|
|
74
|
+
}
|
|
75
|
+
if (handle.type === 'button' || !handle.href) {
|
|
76
|
+
throw new Error(`[${n}] has no link to follow, it is a ${handle.type} the page handles itself`);
|
|
77
|
+
}
|
|
78
|
+
return { url: handle.href, text: handle.text };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// How many blocks of run-up `read` prints before the one it was asked for, so
|
|
82
|
+
// a comment or paragraph arrives with the byline that introduces it, and how
|
|
83
|
+
// many it prints after. The trailing window only applies when the target is
|
|
84
|
+
// not a heading: a heading owns its whole section, anything else is one thing
|
|
85
|
+
// the agent asked to see in full and a little of what follows it.
|
|
86
|
+
const LEAD = 2;
|
|
87
|
+
const TRAIL = 6;
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Full text of one region of the current page: the block at [n], the couple of
|
|
91
|
+
* blocks that lead into it, and either the rest of its section when [n] is a
|
|
92
|
+
* heading or a short run after it when it is not. This is the middle setting
|
|
93
|
+
* between the 500 token view and the whole page.
|
|
94
|
+
* @param {number} n
|
|
95
|
+
* @param {{session?: string, budget?: number}} [opts]
|
|
96
|
+
* @returns {string}
|
|
97
|
+
*/
|
|
98
|
+
export function read(n, { session = DEFAULT_SESSION, budget = 2000 } = {}) {
|
|
99
|
+
if (!Number.isInteger(n) || n < 1) {
|
|
100
|
+
throw new Error('usage: oc read <n>, where <n> is a number from the last page');
|
|
101
|
+
}
|
|
102
|
+
const state = requireBlocks(session);
|
|
103
|
+
const blocks = state.blocks;
|
|
104
|
+
const at = blocks.findIndex((b) => b.n === n);
|
|
105
|
+
if (at < 0) {
|
|
106
|
+
const nums = handleNumbers(state);
|
|
107
|
+
const range = nums.length ? `1-${Math.max(...nums)}` : 'none';
|
|
108
|
+
throw new Error(`no [${n}] on ${state.url} (handles ${range}), run 'oc open <url>' again to renumber`);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const isHeading = blocks[at].type === 'heading';
|
|
112
|
+
let start = isHeading ? at : Math.max(0, at - LEAD);
|
|
113
|
+
// Never open with the tail of the section before this one.
|
|
114
|
+
for (let i = at - 1; i > start; i--) {
|
|
115
|
+
if (blocks[i].type === 'heading') {
|
|
116
|
+
start = i;
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
// A heading owns everything down to the next heading of its level or above;
|
|
121
|
+
// anything else runs to the next heading of any level.
|
|
122
|
+
const stopLevel = blocks[start].type === 'heading' ? (blocks[start].level ?? 2) : 6;
|
|
123
|
+
const end = isHeading ? blocks.length : Math.min(blocks.length, at + TRAIL + 1);
|
|
124
|
+
|
|
125
|
+
const lines = [];
|
|
126
|
+
let spent = 0;
|
|
127
|
+
let i = start;
|
|
128
|
+
for (; i < end; i++) {
|
|
129
|
+
const block = blocks[i];
|
|
130
|
+
if (i > start && block.type === 'heading' && (block.level ?? 2) <= stopLevel) break;
|
|
131
|
+
const line = formatBlock(block, { full: true });
|
|
132
|
+
if (!line) continue;
|
|
133
|
+
const cost = estimateTokens(line) + 1;
|
|
134
|
+
if (spent + cost > budget && lines.length) break;
|
|
135
|
+
spent += cost;
|
|
136
|
+
lines.push(line);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const stoppedEarly = i < end && !(blocks[i].type === 'heading' && (blocks[i].level ?? 2) <= stopLevel);
|
|
140
|
+
if (stoppedEarly) {
|
|
141
|
+
const resume = blocks.slice(i).find((b) => b.n != null)?.n;
|
|
142
|
+
const how = resume ? `continue with 'oc read ${resume}'` : "use 'oc raw' for the rest";
|
|
143
|
+
lines.push(`... region cut at ~${budget} tokens, ${how} or raise --budget`);
|
|
144
|
+
}
|
|
145
|
+
return lines.join('\n');
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The next budget worth of the page the session already holds. `oc open` says
|
|
150
|
+
* how many blocks it left behind; this is how an agent takes them a screenful
|
|
151
|
+
* at a time instead of paying for the whole page to get one more paragraph.
|
|
152
|
+
* @param {{session?: string, budget?: number}} [opts]
|
|
153
|
+
* @returns {string}
|
|
154
|
+
*/
|
|
155
|
+
export function next({ session = DEFAULT_SESSION, budget = 500 } = {}) {
|
|
156
|
+
const state = requireBlocks(session);
|
|
157
|
+
const from = state.cursor;
|
|
158
|
+
if (from == null) {
|
|
159
|
+
return `end of ${state.url}, nothing left to render. 'oc open <url>' to reload it, 'oc raw' for the whole page.`;
|
|
160
|
+
}
|
|
161
|
+
const page = { url: state.url, title: state.title, blocks: state.blocks };
|
|
162
|
+
const { text, stats } = render(page, { budget, from });
|
|
163
|
+
try {
|
|
164
|
+
saveSession(session, { ...state, cursor: stats.next });
|
|
165
|
+
} catch {}
|
|
166
|
+
return text;
|
|
16
167
|
}
|
|
17
168
|
|
|
18
169
|
/** @param {number} n @param {string} text */
|
|
@@ -25,20 +176,87 @@ export function submit(n) {
|
|
|
25
176
|
throw new NotImplemented('submit');
|
|
26
177
|
}
|
|
27
178
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
179
|
+
// How much of a matching block to print around the hit. Wide enough to judge
|
|
180
|
+
// whether the match is the one you wanted, narrow enough that twenty hits
|
|
181
|
+
// still fit in a screenful.
|
|
182
|
+
const BEFORE = 60;
|
|
183
|
+
const SNIPPET = 200;
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Where a string appears on the current page. This is the answer to "the page
|
|
187
|
+
* is long and I only care about one thing in it": one command, no fetch, and
|
|
188
|
+
* a number to read the region it landed in.
|
|
189
|
+
* @param {string} query
|
|
190
|
+
* @param {{session?: string, budget?: number}} [opts]
|
|
191
|
+
* @returns {string}
|
|
192
|
+
*/
|
|
193
|
+
export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
|
|
194
|
+
const q = (query ?? '').trim().toLowerCase();
|
|
195
|
+
if (!q) throw new Error('usage: oc find <query>, searching the page already open');
|
|
196
|
+
const state = requireBlocks(session);
|
|
197
|
+
const blocks = state.blocks;
|
|
198
|
+
|
|
199
|
+
let hits = search(blocks, [q]);
|
|
200
|
+
// A phrase that matches nothing is usually word order, not absence, so try
|
|
201
|
+
// the words separately rather than making the agent guess again.
|
|
202
|
+
const terms = q.split(/\s+/);
|
|
203
|
+
const loose = !hits.length && terms.length > 1;
|
|
204
|
+
if (loose) hits = search(blocks, terms);
|
|
205
|
+
|
|
206
|
+
if (!hits.length) {
|
|
207
|
+
const tried = terms.length > 1 ? ', as a phrase or as separate words' : '';
|
|
208
|
+
return `no match for "${query}"${tried} in ${blocks.length} blocks on ${state.url}, try fewer words or 'oc raw' for the full text`;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const lines = [`${hits.length} ${hits.length === 1 ? 'match' : 'matches'} for "${query}"${loose ? ', matching the words separately' : ''}`];
|
|
212
|
+
let spent = estimateTokens(lines[0]);
|
|
213
|
+
let shown = 0;
|
|
214
|
+
let hasLinks = false;
|
|
215
|
+
for (const hit of hits) {
|
|
216
|
+
const line = `[${hit.n ?? '?'}] ${hit.snippet}`;
|
|
217
|
+
const cost = estimateTokens(line) + 1;
|
|
218
|
+
if (spent + cost > budget && shown) break;
|
|
219
|
+
spent += cost;
|
|
220
|
+
shown++;
|
|
221
|
+
if (hit.type === 'link') hasLinks = true;
|
|
222
|
+
lines.push(line);
|
|
223
|
+
}
|
|
224
|
+
if (shown < hits.length) {
|
|
225
|
+
lines.push(`... ${hits.length - shown} more matches, narrow the query or raise --budget`);
|
|
226
|
+
}
|
|
227
|
+
lines.push(`actions: ${[hasLinks && 'do <n>', 'read <n>', 'next', 'raw'].filter(Boolean).join(' | ')}`);
|
|
228
|
+
return lines.join('\n');
|
|
31
229
|
}
|
|
32
230
|
|
|
33
|
-
/**
|
|
34
|
-
|
|
35
|
-
|
|
231
|
+
/**
|
|
232
|
+
* Blocks containing every term, each with the piece of text around the first
|
|
233
|
+
* one and a number to read it with. A phrase search is the same thing with a
|
|
234
|
+
* single term. Short blocks carry no handle of their own, so they borrow the
|
|
235
|
+
* nearest one above them, which is what `oc read` needs to put the match back
|
|
236
|
+
* in context.
|
|
237
|
+
* @param {import('./distill.js').Block[]} blocks
|
|
238
|
+
* @param {string[]} terms - lower case, all of them must appear
|
|
239
|
+
*/
|
|
240
|
+
function search(blocks, terms) {
|
|
241
|
+
const out = [];
|
|
242
|
+
let anchor = null;
|
|
243
|
+
for (const block of blocks) {
|
|
244
|
+
if (block.n != null) anchor = block.n;
|
|
245
|
+
const text = block.text.toLowerCase();
|
|
246
|
+
const found = terms.map((t) => text.indexOf(t));
|
|
247
|
+
if (found.some((i) => i < 0)) continue;
|
|
248
|
+
const n = block.n ?? anchor;
|
|
249
|
+
// One number, one line: a run of short blocks under the same handle would
|
|
250
|
+
// otherwise report the same place several times.
|
|
251
|
+
if (out.length && out.at(-1).n === n) continue;
|
|
252
|
+
const start = Math.max(0, Math.min(...found) - BEFORE);
|
|
253
|
+
const end = Math.min(block.text.length, start + SNIPPET);
|
|
254
|
+
const snippet = `${start > 0 ? '... ' : ''}${block.text.slice(start, end)}${end < block.text.length ? ' ...' : ''}`;
|
|
255
|
+
out.push({ n, type: block.type, snippet });
|
|
256
|
+
}
|
|
257
|
+
return out;
|
|
36
258
|
}
|
|
37
259
|
|
|
38
260
|
export function back() {
|
|
39
261
|
throw new NotImplemented('back');
|
|
40
262
|
}
|
|
41
|
-
|
|
42
|
-
export function next() {
|
|
43
|
-
throw new NotImplemented('next');
|
|
44
|
-
}
|