@only-cli/oc 0.3.0-beta.1 → 0.3.0-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -2
- package/package.json +1 -1
- package/src/act.js +37 -6
- package/src/cli.js +4 -2
- package/src/distill.js +90 -19
- package/src/fetch.js +21 -7
- package/src/render.js +20 -3
package/README.md
CHANGED
|
@@ -59,7 +59,8 @@ No setup at all also works: `npx @only-cli/oc` runs without a global install, an
|
|
|
59
59
|
```
|
|
60
60
|
oc open <url> fetch and render a page with numbered actions
|
|
61
61
|
oc do <n> follow the numbered link [n], or read [n] if it is text
|
|
62
|
-
oc find <query> where a string appears on the page already open
|
|
62
|
+
oc find <query> where a string appears on the page already open, or
|
|
63
|
+
the region itself when only one place matches
|
|
63
64
|
oc read <n> full text of the region at [n], up to 2000 tokens
|
|
64
65
|
oc next the next budget worth of the page already open
|
|
65
66
|
oc raw [url] distilled markdown of the whole page
|
|
@@ -69,7 +70,7 @@ oc submit [n] submit a form (planned)
|
|
|
69
70
|
|
|
70
71
|
Flags: `--budget <tokens>` (default 500), `--json`, `--html` (raw as cleaned HTML), `--session <name>`, `--verbose`/`-v` (metrics on stderr, or export `OC_VERBOSE=1`).
|
|
71
72
|
|
|
72
|
-
`oc open` remembers the page it rendered in a JSON file per session under `~/.only-cli` (override with `OC_HOME`), so `oc do 3` follows `[3]` without the agent ever handling a URL. Pages longer than the budget say what they left out; `oc find`, `oc read <n>`, and `oc next` read the rest without refetching the page. The budget is a target rather than a hard cap: a page that would only run a little long is printed whole rather than cut, since one extra tool call costs far more than the tokens it would have saved.
|
|
73
|
+
`oc open` remembers the page it rendered in a JSON file per session under `~/.only-cli` (override with `OC_HOME`), so `oc do 3` follows `[3]` without the agent ever handling a URL. A result title on a search page is a link, so `oc do` on it opens the result rather than repeating the title. Pages longer than the budget say what they left out; `oc find`, `oc read <n>`, and `oc next` read the rest without refetching the page, and a `find` with a single match prints that region instead of the number to read it with. The budget is a target rather than a hard cap: a page that would only run a little long is printed whole rather than cut, since one extra tool call costs far more than the tokens it would have saved.
|
|
73
74
|
|
|
74
75
|
## Supported websites
|
|
75
76
|
|
package/package.json
CHANGED
package/src/act.js
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
import { DEFAULT_SESSION, handleFor, handleNumbers, loadSession, saveSession } from './session.js';
|
|
12
|
-
import { estimateTokens, formatBlock, render } from './render.js';
|
|
12
|
+
import { FINISH, estimateTokens, formatBlock, render } from './render.js';
|
|
13
13
|
|
|
14
14
|
export class NotImplemented extends Error {
|
|
15
15
|
constructor(command) {
|
|
@@ -63,7 +63,11 @@ export function activate(n, { session = DEFAULT_SESSION } = {}) {
|
|
|
63
63
|
const range = nums.length ? `1-${Math.max(...nums)}` : 'none';
|
|
64
64
|
throw new Error(`no [${n}] on ${state.url} (handles ${range}), run 'oc open <url>' again to renumber`);
|
|
65
65
|
}
|
|
66
|
-
|
|
66
|
+
// A heading can be a link: on a search results page the result title is one,
|
|
67
|
+
// and opening it is what `do` was asked for. Reading the title back instead
|
|
68
|
+
// cost a turn, and then another to find the number that does navigate, so a
|
|
69
|
+
// heading that has an href falls through to the link below.
|
|
70
|
+
if ((handle.type === 'text' || handle.type === 'heading') && !handle.href) {
|
|
67
71
|
// There is nothing to follow, but the agent asked to see what is at [n],
|
|
68
72
|
// and that is what read prints. Refusing would spend a whole turn to name
|
|
69
73
|
// the command that should have run, and a turn costs more than the page.
|
|
@@ -208,14 +212,41 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
|
|
|
208
212
|
return `no match for "${query}"${tried} in ${blocks.length} blocks on ${state.url}, try fewer words or 'oc raw' for the full text`;
|
|
209
213
|
}
|
|
210
214
|
|
|
211
|
-
const
|
|
215
|
+
const separately = loose ? ', matching the words separately' : '';
|
|
216
|
+
|
|
217
|
+
// One match is not an index, it is the answer. Naming the number and
|
|
218
|
+
// stopping spends a turn to say where to look, and the agent's next command
|
|
219
|
+
// is always the `read` that looks, so find does the reading. Measured on an
|
|
220
|
+
// AWS CLI reference page, `find "Example 7"` printed a 24 token heading and
|
|
221
|
+
// the example it names was in the block after it.
|
|
222
|
+
if (hits.length === 1 && hits[0].n != null) {
|
|
223
|
+
const only = hits[0];
|
|
224
|
+
const follow = only.type === 'link' ? 'do <n> | ' : '';
|
|
225
|
+
return [
|
|
226
|
+
`1 match for "${query}"${separately}, region [${only.n}]`,
|
|
227
|
+
read(only.n, { session, budget: budget * FINISH }),
|
|
228
|
+
`actions: ${follow}read <n> | next | raw`,
|
|
229
|
+
].join('\n');
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
const lines = [`${hits.length} ${hits.length === 1 ? 'match' : 'matches'} for "${query}"${separately}`];
|
|
233
|
+
// A snippet is short because many matches have to share one screen. When
|
|
234
|
+
// every match would fit whole inside the allowance a page already gets for
|
|
235
|
+
// finishing (render's FINISH), showing them whole makes this command the
|
|
236
|
+
// answer instead of an index into a `read <n>` that has to run next. The
|
|
237
|
+
// trade is the same one FINISH documents, and it is not close: a few
|
|
238
|
+
// hundred tokens against a turn.
|
|
239
|
+
const label = (hit, text) => `[${hit.n ?? '?'}] ${text}`;
|
|
240
|
+
const whole = hits.reduce((n, h) => n + estimateTokens(label(h, h.text)) + 1, estimateTokens(lines[0]));
|
|
241
|
+
const full = whole <= budget * FINISH;
|
|
242
|
+
const cap = full ? budget * FINISH : budget;
|
|
212
243
|
let spent = estimateTokens(lines[0]);
|
|
213
244
|
let shown = 0;
|
|
214
245
|
let hasLinks = false;
|
|
215
246
|
for (const hit of hits) {
|
|
216
|
-
const line =
|
|
247
|
+
const line = label(hit, full ? hit.text : hit.snippet);
|
|
217
248
|
const cost = estimateTokens(line) + 1;
|
|
218
|
-
if (spent + cost >
|
|
249
|
+
if (spent + cost > cap && shown) break;
|
|
219
250
|
spent += cost;
|
|
220
251
|
shown++;
|
|
221
252
|
if (hit.type === 'link') hasLinks = true;
|
|
@@ -252,7 +283,7 @@ function search(blocks, terms) {
|
|
|
252
283
|
const start = Math.max(0, Math.min(...found) - BEFORE);
|
|
253
284
|
const end = Math.min(block.text.length, start + SNIPPET);
|
|
254
285
|
const snippet = `${start > 0 ? '... ' : ''}${block.text.slice(start, end)}${end < block.text.length ? ' ...' : ''}`;
|
|
255
|
-
out.push({ n, type: block.type, snippet });
|
|
286
|
+
out.push({ n, type: block.type, snippet, text: block.text });
|
|
256
287
|
}
|
|
257
288
|
return out;
|
|
258
289
|
}
|
package/src/cli.js
CHANGED
|
@@ -11,7 +11,8 @@ const HELP = `only-cli: the web as a compact terminal, built for AI agents.
|
|
|
11
11
|
usage: oc <command> [args] [flags]
|
|
12
12
|
|
|
13
13
|
open <url> fetch and render a page with numbered actions
|
|
14
|
-
find <query> where a string appears on the page already open
|
|
14
|
+
find <query> where a string appears on the page already open, or
|
|
15
|
+
the region itself when only one place matches
|
|
15
16
|
next the next budget worth of the page already open
|
|
16
17
|
read <n> full text of the region at [n], up to 2000 tokens
|
|
17
18
|
raw [url] distilled markdown of the whole page
|
|
@@ -36,7 +37,8 @@ flags:
|
|
|
36
37
|
|
|
37
38
|
'oc open' remembers the page it printed, so 'oc do 3' follows link [3] without
|
|
38
39
|
you ever handling its URL, and 'oc next' or 'oc read 12' picks up what the
|
|
39
|
-
budget left behind without fetching it again.
|
|
40
|
+
budget left behind without fetching it again. 'oc do' on a search result title
|
|
41
|
+
opens the result, because that title is a link. State lives in ~/.only-cli
|
|
40
42
|
(override with OC_HOME).`;
|
|
41
43
|
|
|
42
44
|
// A rendered page has to be remembered or its [3] means nothing to the next
|
package/src/distill.js
CHANGED
|
@@ -7,7 +7,7 @@ import TurndownService from 'turndown';
|
|
|
7
7
|
* @property {string} text
|
|
8
8
|
* @property {number} [n] - action handle
|
|
9
9
|
* @property {number} [level] - heading level 1..6
|
|
10
|
-
* @property {string} [href] - links
|
|
10
|
+
* @property {string} [href] - links, and a heading that is one
|
|
11
11
|
* @property {string} [name] - inputs only
|
|
12
12
|
*
|
|
13
13
|
* @typedef {Object} Page
|
|
@@ -82,6 +82,29 @@ const clean = (s) => s.replace(/\s+/g, ' ').trim();
|
|
|
82
82
|
const asHTML = (text, url = '', opts = {}) =>
|
|
83
83
|
jsonToHTML(text, url, opts) ?? youtubeToHTML(text) ?? transcriptToHTML(text) ?? feedToHTML(text) ?? text;
|
|
84
84
|
|
|
85
|
+
/**
|
|
86
|
+
* The link a heading is, if it is one. A search engine puts the result title
|
|
87
|
+
* in an anchor inside an <h2>, so a heading can be the most followable thing
|
|
88
|
+
* on the page, and taking only its text threw that away.
|
|
89
|
+
*
|
|
90
|
+
* The heading has to BE the link, not merely contain one: exactly one anchor,
|
|
91
|
+
* labelled with the whole heading. Documentation fails that test on purpose.
|
|
92
|
+
* Every heading in the Rust book and every one on an AWS CLI reference page
|
|
93
|
+
* carries a permalink to its own id, so following those would refetch the
|
|
94
|
+
* page the agent is already reading, which is worse than the reading it
|
|
95
|
+
* already gets. A bare fragment is never a destination.
|
|
96
|
+
* @param {any} node - the heading element
|
|
97
|
+
* @param {string} text - its cleaned text
|
|
98
|
+
* @returns {string|null}
|
|
99
|
+
*/
|
|
100
|
+
function headingHref(node, text) {
|
|
101
|
+
const anchors = node.querySelectorAll('a[href]');
|
|
102
|
+
if (anchors.length !== 1) return null;
|
|
103
|
+
const href = anchors[0].getAttribute('href') ?? '';
|
|
104
|
+
if (!href || href.startsWith('#')) return null;
|
|
105
|
+
return clean(anchors[0].textContent) === text ? href : null;
|
|
106
|
+
}
|
|
107
|
+
|
|
85
108
|
/**
|
|
86
109
|
* Reduce raw HTML to an interaction tree: readable text plus numbered
|
|
87
110
|
* elements, in document order. The walk is deterministic and numbering is a
|
|
@@ -126,7 +149,10 @@ export function distill(html, url = '') {
|
|
|
126
149
|
|
|
127
150
|
if (/^h[1-6]$/.test(tag)) {
|
|
128
151
|
const text = clean(node.textContent);
|
|
129
|
-
if (text)
|
|
152
|
+
if (text) {
|
|
153
|
+
const href = headingHref(node, text);
|
|
154
|
+
blocks.push({ type: 'heading', level: Number(tag[1]), text, ...(href ? { href } : {}) });
|
|
155
|
+
}
|
|
130
156
|
return;
|
|
131
157
|
}
|
|
132
158
|
if (tag === 'a' && node.getAttribute('href')) {
|
|
@@ -513,6 +539,15 @@ const TITLE_KEYS = [
|
|
|
513
539
|
'full_name', 'summary', 'question', 'message',
|
|
514
540
|
];
|
|
515
541
|
|
|
542
|
+
// Keys an API is likely to put its list of results under. A response using one
|
|
543
|
+
// of these is a list whatever else it carries, so the name settles it before
|
|
544
|
+
// shape does: a sideloaded `included` array can outnumber the `items` the
|
|
545
|
+
// request was for without being what the request was for.
|
|
546
|
+
const CONTAINER_KEYS = [
|
|
547
|
+
'items', 'data', 'results', 'hits', 'records', 'rows', 'entries',
|
|
548
|
+
'nodes', 'edges', 'docs', 'list', 'children', 'values',
|
|
549
|
+
];
|
|
550
|
+
|
|
516
551
|
// Keys holding the item's own page. A URL under any other name is still found,
|
|
517
552
|
// by looking at values rather than names, but these win when several qualify.
|
|
518
553
|
const LINK_KEYS = ['link', 'url', 'html_url', 'web_url', 'permalink', 'href'];
|
|
@@ -540,6 +575,14 @@ const NESTED_PENALTY = 0.25;
|
|
|
540
575
|
// How many names the footer lists when it says which fields it left out.
|
|
541
576
|
const DROPPED_LISTED = 5;
|
|
542
577
|
|
|
578
|
+
// Characters of markup a field may carry before the compact view stops
|
|
579
|
+
// treating it as a document and starts treating it as somewhere to go. A
|
|
580
|
+
// question body arrives well under this and is worth rendering in place, links
|
|
581
|
+
// and code and all. A package readme arrives at four figures and distils into
|
|
582
|
+
// more blocks than the resource it is attached to has fields, which buries the
|
|
583
|
+
// resource the response was fetched for. `oc raw` renders either one in full.
|
|
584
|
+
const BODY_CAP = 4000;
|
|
585
|
+
|
|
543
586
|
// Seconds and milliseconds since the epoch, bounded either side so an ordinary
|
|
544
587
|
// count (a score, a byte size) is never mistaken for a date.
|
|
545
588
|
const EPOCH_S = [1e9, 4e9];
|
|
@@ -572,6 +615,10 @@ const decodeEntities = (s) =>
|
|
|
572
615
|
const isPlain = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
573
616
|
const isURL = (v) => typeof v === 'string' && /^https?:\/\/\S+$/.test(v);
|
|
574
617
|
const looksHTML = (v) => typeof v === 'string' && /<\/?(p|div|pre|code|br|ul|ol|li|h[1-6]|blockquote|table|img|a|em|strong)\b[^>]*>/i.test(v);
|
|
618
|
+
// Markup as one line of prose, for a body the compact view is pointing at
|
|
619
|
+
// rather than rendering. The distiller is what reads markup properly; this
|
|
620
|
+
// only has to make a line an agent can tell one body from another by.
|
|
621
|
+
const stripTags = (v) => clean(decodeEntities(String(v).replace(/<[^>]*>/g, ' ')));
|
|
575
622
|
|
|
576
623
|
/**
|
|
577
624
|
* One level of flattening, so `owner: {display_name}` becomes an
|
|
@@ -662,32 +709,40 @@ function chooseFields(rows, skip) {
|
|
|
662
709
|
}
|
|
663
710
|
|
|
664
711
|
/**
|
|
665
|
-
* Pick
|
|
666
|
-
* otherwise the
|
|
667
|
-
*
|
|
668
|
-
*
|
|
712
|
+
* Pick what the response is actually about: the root when it is an array or a
|
|
713
|
+
* single resource, otherwise the array of objects at the top level that holds
|
|
714
|
+
* the results. Everything beside it is metadata about the request rather than
|
|
715
|
+
* content.
|
|
669
716
|
* @param {any} data
|
|
670
717
|
* @returns {{items: any[], meta: Record<string, any>}}
|
|
671
718
|
*/
|
|
672
719
|
function mainArray(data) {
|
|
673
720
|
if (Array.isArray(data)) return { items: data, meta: {} };
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
let items = null;
|
|
721
|
+
/** @type {Map<string, any[]>} */
|
|
722
|
+
const arrays = new Map();
|
|
677
723
|
for (const [k, v] of Object.entries(data)) {
|
|
678
724
|
if (!Array.isArray(v) || !v.length) continue;
|
|
679
725
|
if (!v.some(isPlain)) continue;
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
726
|
+
arrays.set(k, v);
|
|
727
|
+
}
|
|
728
|
+
let key = CONTAINER_KEYS.find((k) => arrays.has(k)) ?? '';
|
|
729
|
+
// A root carrying its own name is the resource, and an array hanging off it
|
|
730
|
+
// describes that resource rather than being the subject in its place. Taking
|
|
731
|
+
// the longest array regardless titled the npm registry's package endpoint
|
|
732
|
+
// after its two maintainers and demoted the package to the metadata line,
|
|
733
|
+
// where a 9KB readme then cost more than the rest of the page put together.
|
|
734
|
+
const named = TITLE_KEYS.some((k) => typeof data[k] === 'string' && data[k].trim() !== '');
|
|
735
|
+
if (!key && !named) {
|
|
736
|
+
for (const [k, v] of arrays) {
|
|
737
|
+
if (!key || v.length > (arrays.get(key)?.length ?? 0)) key = k;
|
|
683
738
|
}
|
|
684
739
|
}
|
|
685
|
-
// A response with no array is a single resource, which
|
|
686
|
-
// rather than as a special case.
|
|
687
|
-
if (!
|
|
740
|
+
// A response with no results array of its own is a single resource, which
|
|
741
|
+
// renders as one item rather than as a special case.
|
|
742
|
+
if (!key) return { items: [data], meta: {} };
|
|
688
743
|
const meta = { ...data };
|
|
689
744
|
delete meta[key];
|
|
690
|
-
return { items, meta };
|
|
745
|
+
return { items: arrays.get(key) ?? [data], meta };
|
|
691
746
|
}
|
|
692
747
|
|
|
693
748
|
/**
|
|
@@ -784,9 +839,16 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
|
784
839
|
if (kept && !kept.has(key) && !looksHTML(value)) continue;
|
|
785
840
|
// A field carrying HTML is a page in itself: `filter=withbody` on the
|
|
786
841
|
// Stack Exchange API puts a whole question in one. It goes through the
|
|
787
|
-
// distiller like any other markup instead of into a cell
|
|
842
|
+
// distiller like any other markup instead of into a cell, unless it is
|
|
843
|
+
// longer than the compact view can afford, in which case it becomes one
|
|
844
|
+
// numbered line rather than a dozen blocks that bury the item it hangs
|
|
845
|
+
// off. `oc read <n>` opens it at a budget that fits it, `oc raw` always.
|
|
788
846
|
if (looksHTML(value)) {
|
|
789
|
-
|
|
847
|
+
if (full || String(value).length <= BODY_CAP) {
|
|
848
|
+
bodies.push(String(value));
|
|
849
|
+
continue;
|
|
850
|
+
}
|
|
851
|
+
long.push(`<p>${escHTML(`${key}: ${stripTags(value)}`)}</p>`);
|
|
790
852
|
continue;
|
|
791
853
|
}
|
|
792
854
|
const rendered = row.get(key);
|
|
@@ -824,12 +886,21 @@ export function jsonToHTML(text, url = '', { full = false } = {}) {
|
|
|
824
886
|
}
|
|
825
887
|
|
|
826
888
|
const metaBits = [];
|
|
889
|
+
const metaLong = [];
|
|
827
890
|
for (const [key, value] of Object.entries(meta)) {
|
|
828
891
|
if (value === null || typeof value === 'object') continue;
|
|
829
892
|
const rendered = renderValue(key, value);
|
|
830
|
-
if (rendered)
|
|
893
|
+
if (!rendered) continue;
|
|
894
|
+
// A summary line has to stay a line. One long scalar at the root, a
|
|
895
|
+
// package readme or an endpoint description, would otherwise spend the
|
|
896
|
+
// whole page budget here, so it becomes a block of its own instead. The
|
|
897
|
+
// block is numbered, so `oc read <n>` opens it when it fits that budget
|
|
898
|
+
// and `oc raw` has it whatever its size.
|
|
899
|
+
if (rendered.length > TEXT_CAP) metaLong.push(`<p>${escHTML(`${key}: ${rendered}`)}</p>`);
|
|
900
|
+
else metaBits.push(`${key}=${rendered}`);
|
|
831
901
|
}
|
|
832
902
|
if (metaBits.length) parts.push(`<p>response: ${escHTML(metaBits.join(', '))}</p>`);
|
|
903
|
+
parts.push(...metaLong);
|
|
833
904
|
|
|
834
905
|
const count = `${items.length} ${items.length === 1 ? 'item' : 'items'}`;
|
|
835
906
|
return `<html><head><title>${escHTML(jsonTitle(url, count))}</title></head><body>\n${parts.join('\n')}\n</body></html>`;
|
package/src/fetch.js
CHANGED
|
@@ -24,6 +24,25 @@ const loadImpers = () => {
|
|
|
24
24
|
const BLOCKED_MESSAGE = 'blocked: private or internal URL';
|
|
25
25
|
const MAX_REDIRECTS = 20;
|
|
26
26
|
|
|
27
|
+
// What oc can turn into text: any text/* type, plus the application/* types
|
|
28
|
+
// that are really text (json, xml, and the +json / +xml families a feed or an
|
|
29
|
+
// API answers with). A PNG matches none of these, and rendering one produces
|
|
30
|
+
// pages of mojibake an agent then pays for, so it is refused by name instead.
|
|
31
|
+
const READABLE_TYPE = /^\s*(?:text\/|application\/(?:json|xml|javascript|x-ndjson|[\w.+-]*\+(?:json|xml)))/i;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Refuse a response oc cannot read as text. Both transports call this: the
|
|
35
|
+
* gate has to live on whichever client got the page, or the same URL renders
|
|
36
|
+
* as an error through fetch and as binary noise through impers.
|
|
37
|
+
* @param {string | null | undefined} type - the content-type header
|
|
38
|
+
*/
|
|
39
|
+
export function assertReadableType(type) {
|
|
40
|
+
// No header at all is not a refusal: plenty of small servers omit it, and
|
|
41
|
+
// the distiller handles whatever comes back.
|
|
42
|
+
if (!type || READABLE_TYPE.test(type)) return;
|
|
43
|
+
throw new Error(`not a page oc can read (${type.split(';')[0].trim()}), it renders HTML, XML feeds, JSON, and plain text`);
|
|
44
|
+
}
|
|
45
|
+
|
|
27
46
|
// IPv4 ranges with no business receiving a server-initiated fetch: loopback,
|
|
28
47
|
// link-local, the three RFC 1918 private blocks, carrier-grade NAT, the
|
|
29
48
|
// unspecified/broadcast addresses, and the documentation/benchmark ranges.
|
|
@@ -169,6 +188,7 @@ async function viaImpers(impers, target) {
|
|
|
169
188
|
status = res.status ?? res.statusCode ?? 0;
|
|
170
189
|
}
|
|
171
190
|
if (status >= 400) throw new Error(`fetch failed: ${status} for ${target}`);
|
|
191
|
+
assertReadableType(res.headers.get('content-type'));
|
|
172
192
|
const html = typeof res.text === 'function' ? await res.text() : String(res.text ?? res.body ?? '');
|
|
173
193
|
return { url: res.url ?? target, html, status, via };
|
|
174
194
|
}
|
|
@@ -197,12 +217,6 @@ async function viaFetch(target) {
|
|
|
197
217
|
if (!res.ok) {
|
|
198
218
|
throw new Error(`fetch failed: ${res.status} ${res.statusText} for ${current}`);
|
|
199
219
|
}
|
|
200
|
-
|
|
201
|
-
// The impers path never checked the type at all, so this is also what keeps
|
|
202
|
-
// the two transports rendering the same URL the same way.
|
|
203
|
-
const type = res.headers.get('content-type') ?? '';
|
|
204
|
-
if (type && !/html|xml|json/.test(type)) {
|
|
205
|
-
throw new Error(`not a page oc can read (${type.split(';')[0]}), it renders HTML, XML feeds, and JSON`);
|
|
206
|
-
}
|
|
220
|
+
assertReadableType(res.headers.get('content-type'));
|
|
207
221
|
return { url: res.url || current, html: await res.text(), status: res.status, via: 'fetch' };
|
|
208
222
|
}
|
package/src/render.js
CHANGED
|
@@ -24,7 +24,7 @@ const num = (v) => v.toLocaleString('en-US');
|
|
|
24
24
|
// saving is only collected when the agent would have paged at all, while the
|
|
25
25
|
// overspend is paid on every page that runs a little long, including the ones
|
|
26
26
|
// answered by their first few lines. Four caps that overspend near 1,500 tokens.
|
|
27
|
-
const FINISH = 4;
|
|
27
|
+
export const FINISH = 4;
|
|
28
28
|
|
|
29
29
|
/**
|
|
30
30
|
* Budget-aware compact view of a distilled page. `from` is a position in the
|
|
@@ -162,5 +162,22 @@ export function formatBlock(b, { full = false } = {}) {
|
|
|
162
162
|
}
|
|
163
163
|
}
|
|
164
164
|
|
|
165
|
-
|
|
166
|
-
|
|
165
|
+
// A cut inside a sentence makes the half that is shown untrustworthy. Asked
|
|
166
|
+
// for the first sentence of a page, an agent was given it in full, followed by
|
|
167
|
+
// a truncation marker, and spent a turn on `read` to find out whether the
|
|
168
|
+
// sentence carried on. Ending on the last sentence that finished inside the cap
|
|
169
|
+
// answers that in the view itself. The floor bounds what the courtesy costs: a
|
|
170
|
+
// block whose only sentence end is early keeps the plain cut instead of
|
|
171
|
+
// throwing away a third of the window.
|
|
172
|
+
const SENTENCE_END = /[.!?]["')\]]*(?=\s)/g;
|
|
173
|
+
const SENTENCE_FLOOR = 0.7;
|
|
174
|
+
|
|
175
|
+
const truncate = (s) => {
|
|
176
|
+
if (s.length <= TEXT_CAP) return s;
|
|
177
|
+
let cut = TEXT_CAP;
|
|
178
|
+
for (const m of s.slice(0, TEXT_CAP).matchAll(SENTENCE_END)) {
|
|
179
|
+
const end = (m.index ?? 0) + m[0].length;
|
|
180
|
+
if (end >= TEXT_CAP * SENTENCE_FLOOR) cut = end;
|
|
181
|
+
}
|
|
182
|
+
return `${s.slice(0, cut).trimEnd()} ... +${num(s.length - cut)} chars`;
|
|
183
|
+
};
|