@only-cli/oc 0.5.5 → 0.5.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +1 -1
- package/skills/web-browsing-cli/SKILL.md +10 -10
- package/src/act.js +19 -4
- package/src/cli.js +2 -2
- package/src/cookies.js +78 -7
- package/src/distill.js +115 -12
- package/src/fetch.js +29 -1
- package/src/render.js +3 -1
- package/src/session.js +20 -6
- package/src/sites.js +6 -1
- package/src/sphinx.js +7 -1
package/README.md
CHANGED
|
@@ -29,6 +29,8 @@ npm install -g @only-cli/oc
|
|
|
29
29
|
|
|
30
30
|
Requires Node 20+. Requests impersonate Chrome via [impers](https://github.com/lexiforest/impers). When a site, or the local copy of libcurl-impersonate, refuses the Chrome identity, the request downgrades to Firefox, and to native fetch when impers is unavailable or refuses both.
|
|
31
31
|
|
|
32
|
+
Red Hat OpenShift also installs an `oc`, and both define `login` and `logout` with unrelated meanings, so whichever lands first on `PATH` shadows the other. If you use OpenShift, call this one with `npx @only-cli/oc` instead of installing it globally, or alias the global install to another name.
|
|
33
|
+
|
|
32
34
|
### Agent skill
|
|
33
35
|
|
|
34
36
|
Install the [web-browsing-cli skill](https://www.skills.sh/only-cli/oc/web-browsing-cli) for Claude Code, Cursor, Codex, Copilot, and other compatible agents:
|
|
@@ -139,7 +141,7 @@ Copy the `Cookie` header from your browser's devtools (Application → Cookies,
|
|
|
139
141
|
|
|
140
142
|
Seeded cookies are https-only. They almost always come from an https browser session, so `oc` marks them secure and never sends them over plain `http`, including on a hop an `https` page redirects into, where you never typed the downgrade. A site that really is http-only needs `--allow-http` at login. Cookies a site sets over https are pinned the same way.
|
|
141
143
|
|
|
142
|
-
Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, owner-only (mode `0600
|
|
144
|
+
Cookies live in a separate sidecar file (`<session>.cookies.json`) under `~/.only-cli/sessions/`, owner-only (mode `0600` in a `0700` directory, or on Windows the ACL the directory hands down), not in the page-state JSON and never in `--json` output. The default lifetime is one hour (`--expires 1h`), and a jar holds at most 50 cookies so a page cannot bloat it. When cookies expire or the site returns a login page, `oc` says so plainly (exit 2) instead of distilling the login form as content.
|
|
143
145
|
|
|
144
146
|
`oc logout` forgets the whole session, not just its cookies: a page saved under that name can hold the distilled text of something only the login could reach, so the snapshot goes with the jar. `oc session rm [name]` forgets a session the same way (page plus cookies) without switching to it first, and `oc session ls` lists what is on disk (`[cookies]` marks a live jar). State lives in `~/.only-cli` (`%USERPROFILE%\.only-cli` on Windows, override with `OC_HOME`), one JSON per session plus search-index caches. Delete the directory to start over.
|
|
145
147
|
|
package/package.json
CHANGED
|
@@ -8,17 +8,17 @@ description: Token-efficient web browsing and web content extraction for AI agen
|
|
|
8
8
|
Renders a web page as a compact, numbered terminal view instead of raw HTML. A typical page is under 500 tokens.
|
|
9
9
|
|
|
10
10
|
```
|
|
11
|
-
npx --yes @only-cli/oc@0.5.
|
|
12
|
-
npx --yes @only-cli/oc@0.5.
|
|
13
|
-
npx --yes @only-cli/oc@0.5.
|
|
11
|
+
npx --yes @only-cli/oc@0.5.7 open <url> compact view, numbered elements
|
|
12
|
+
npx --yes @only-cli/oc@0.5.7 do <n> follow link [n], or read it if [n] is text
|
|
13
|
+
npx --yes @only-cli/oc@0.5.7 find <query> where a string appears, or that place itself
|
|
14
14
|
when only one matches
|
|
15
|
-
npx --yes @only-cli/oc@0.5.
|
|
16
|
-
npx --yes @only-cli/oc@0.5.
|
|
17
|
-
npx --yes @only-cli/oc@0.5.
|
|
18
|
-
npx --yes @only-cli/oc@0.5.
|
|
19
|
-
npx --yes @only-cli/oc@0.5.
|
|
20
|
-
npx --yes @only-cli/oc@0.5.
|
|
21
|
-
npx --yes @only-cli/oc@0.5.
|
|
15
|
+
npx --yes @only-cli/oc@0.5.7 next next ~500 tokens of the page already open
|
|
16
|
+
npx --yes @only-cli/oc@0.5.7 read <n> full text of region [n]
|
|
17
|
+
npx --yes @only-cli/oc@0.5.7 raw [url] whole page as markdown (--html for cleaned HTML)
|
|
18
|
+
npx --yes @only-cli/oc@0.5.7 login seed cookies (--cookie, --domain, --expires)
|
|
19
|
+
npx --yes @only-cli/oc@0.5.7 logout [session] forget a session: cookies and saved page
|
|
20
|
+
npx --yes @only-cli/oc@0.5.7 session ls list saved sessions (name, url, title)
|
|
21
|
+
npx --yes @only-cli/oc@0.5.7 session rm [name] forget a saved session: page and cookies
|
|
22
22
|
```
|
|
23
23
|
|
|
24
24
|
None of these except `open`/`do`/`raw <url>` fetch anything; they replay the page `open` already saved.
|
package/src/act.js
CHANGED
|
@@ -42,17 +42,31 @@ function requireBlocks(session) {
|
|
|
42
42
|
return state;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
|
+
/**
|
|
46
|
+
* The number a handle was printed as. The compact view writes `[2]`, and
|
|
47
|
+
* agents copy that form into `oc do [2]` / `oc read [2]`. Number('[2]') is
|
|
48
|
+
* NaN, so the printed form used to fail the usage check.
|
|
49
|
+
* @param {unknown} value
|
|
50
|
+
* @returns {number}
|
|
51
|
+
*/
|
|
52
|
+
export function parseHandle(value) {
|
|
53
|
+
const s = String(value ?? '').trim();
|
|
54
|
+
const m = /^\[(\d+)\]$/.exec(s) || /^(\d+)$/.exec(s);
|
|
55
|
+
return m ? Number(m[1]) : NaN;
|
|
56
|
+
}
|
|
57
|
+
|
|
45
58
|
/**
|
|
46
59
|
* Resolve a numbered handle from the last render into something to open.
|
|
47
60
|
* Returns the target URL; the caller fetches and renders it exactly as
|
|
48
61
|
* `oc open` would, so `do` and `open` always agree on what a page looks like.
|
|
49
62
|
* When the number is text rather than a link it returns `{read}` instead, and
|
|
50
63
|
* the caller reads it.
|
|
51
|
-
* @param {
|
|
64
|
+
* @param {unknown} n
|
|
52
65
|
* @param {{session?: string}} [opts]
|
|
53
66
|
* @returns {{url?: string, read?: number, text: string}}
|
|
54
67
|
*/
|
|
55
68
|
export function activate(n, { session = DEFAULT_SESSION } = {}) {
|
|
69
|
+
n = parseHandle(n);
|
|
56
70
|
if (!Number.isInteger(n) || n < 1) {
|
|
57
71
|
throw new Error('usage: oc do <n>, where <n> is a number from the last page');
|
|
58
72
|
}
|
|
@@ -100,6 +114,7 @@ const TRAIL = 6;
|
|
|
100
114
|
* @returns {string}
|
|
101
115
|
*/
|
|
102
116
|
export function read(n, { session = DEFAULT_SESSION, budget = 2000 } = {}) {
|
|
117
|
+
n = parseHandle(n);
|
|
103
118
|
if (!Number.isInteger(n) || n < 1) {
|
|
104
119
|
throw new Error('usage: oc read <n>, where <n> is a number from the last page');
|
|
105
120
|
}
|
|
@@ -230,7 +245,7 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
|
|
|
230
245
|
// the example it names was in the block after it.
|
|
231
246
|
if (hits.length === 1 && hits[0].n != null) {
|
|
232
247
|
const only = hits[0];
|
|
233
|
-
const follow = only.type === 'link' ? 'do <n> | ' : '';
|
|
248
|
+
const follow = only.type === 'link' || only.href ? 'do <n> | ' : '';
|
|
234
249
|
return [
|
|
235
250
|
`1 match for "${query}"${separately}, region [${only.n}]`,
|
|
236
251
|
read(only.n, { session, budget: budget * FINISH }),
|
|
@@ -258,7 +273,7 @@ export function find(query, { session = DEFAULT_SESSION, budget = 500 } = {}) {
|
|
|
258
273
|
if (spent + cost > cap && shown) break;
|
|
259
274
|
spent += cost;
|
|
260
275
|
shown++;
|
|
261
|
-
if (hit.type === 'link') hasLinks = true;
|
|
276
|
+
if (hit.type === 'link' || hit.href) hasLinks = true;
|
|
262
277
|
lines.push(line);
|
|
263
278
|
}
|
|
264
279
|
if (shown < hits.length) {
|
|
@@ -298,7 +313,7 @@ function search(blocks, terms) {
|
|
|
298
313
|
// are read rather than indexed: the whole-match mode above, and `read`.
|
|
299
314
|
const window = block.text.slice(start, end).replace(/\n/g, ' ');
|
|
300
315
|
const snippet = `${start > 0 ? '... ' : ''}${window}${end < block.text.length ? ' ...' : ''}`;
|
|
301
|
-
out.push({ n, type: block.type, snippet, text: block.text });
|
|
316
|
+
out.push({ n, type: block.type, snippet, text: block.text, href: block.href });
|
|
302
317
|
}
|
|
303
318
|
return out;
|
|
304
319
|
}
|
package/src/cli.js
CHANGED
|
@@ -237,7 +237,7 @@ async function main() {
|
|
|
237
237
|
// view's footer offers when it has cut something.
|
|
238
238
|
let url;
|
|
239
239
|
if (command === 'do') {
|
|
240
|
-
const target = act.activate(
|
|
240
|
+
const target = act.activate(args[0], { session: sessionName });
|
|
241
241
|
// A number that points at text has no page behind it, so `do` reads it
|
|
242
242
|
// rather than making the agent pay for a second command to be told.
|
|
243
243
|
if (target.read != null) {
|
|
@@ -369,7 +369,7 @@ async function main() {
|
|
|
369
369
|
}
|
|
370
370
|
return;
|
|
371
371
|
}
|
|
372
|
-
case 'read': return console.log(act.read(
|
|
372
|
+
case 'read': return console.log(act.read(args[0], { session: sessionName, budget: asked || 2000 }));
|
|
373
373
|
case 'next': return console.log(act.next({ session: sessionName, budget: asked || 500 }));
|
|
374
374
|
case 'find': return console.log(act.find(args.join(' '), { session: sessionName, budget: asked || 500 }));
|
|
375
375
|
case 'fill': return act.fill(Number(args[0]), args.slice(1).join(' '));
|
package/src/cookies.js
CHANGED
|
@@ -325,6 +325,20 @@ function domainMatches(cookie, host) {
|
|
|
325
325
|
return host === d || host.endsWith(`.${d}`);
|
|
326
326
|
}
|
|
327
327
|
|
|
328
|
+
/**
|
|
329
|
+
* RFC 6265 stores one cookie per name, domain, and path. Domain matching is
|
|
330
|
+
* how a cookie is sent, not how it is replaced: a host-only cookie on
|
|
331
|
+
* www.example.com is a different cookie from one scoped to example.com, and
|
|
332
|
+
* Path=/app is a different cookie from Path=/.
|
|
333
|
+
* @param {Cookie} a
|
|
334
|
+
* @param {Cookie} b
|
|
335
|
+
*/
|
|
336
|
+
function sameCookie(a, b) {
|
|
337
|
+
return a.name === b.name
|
|
338
|
+
&& a.domain.toLowerCase().replace(/^\./, '') === b.domain.toLowerCase().replace(/^\./, '')
|
|
339
|
+
&& (a.path || '/') === (b.path || '/');
|
|
340
|
+
}
|
|
341
|
+
|
|
328
342
|
/**
|
|
329
343
|
* @param {Cookie} cookie
|
|
330
344
|
* @param {string} path
|
|
@@ -367,6 +381,11 @@ export function cookieHeaderFor(jar, urlStr) {
|
|
|
367
381
|
return scopeMatches(c, jar, host, path);
|
|
368
382
|
});
|
|
369
383
|
if (!active.length) return undefined;
|
|
384
|
+
// RFC 6265 5.4 order: the longest path first, and for equal paths the
|
|
385
|
+
// cookie stored first. A server that reads one value for a name is meant to
|
|
386
|
+
// get the most specific cookie, which is what a browser sends it. The sort
|
|
387
|
+
// is stable, so jar order carries the tie the way creation time would.
|
|
388
|
+
active.sort((a, b) => (b.path || '/').length - (a.path || '/').length);
|
|
370
389
|
return active.map((c) => `${c.name}=${c.value}`).join('; ');
|
|
371
390
|
}
|
|
372
391
|
|
|
@@ -392,6 +411,38 @@ export function withheldForScheme(jar, urlStr) {
|
|
|
392
411
|
return jar.cookies.some((c) => c.secure && scopeMatches(c, jar, host, path));
|
|
393
412
|
}
|
|
394
413
|
|
|
414
|
+
/**
|
|
415
|
+
* An ISO timestamp, or undefined when the instant is outside what Date can
|
|
416
|
+
* name. A huge Max-Age used to throw RangeError from toISOString and abort
|
|
417
|
+
* the render of a page that was otherwise fine.
|
|
418
|
+
* @param {number} ms
|
|
419
|
+
* @returns {string | undefined}
|
|
420
|
+
*/
|
|
421
|
+
function expiryISO(ms) {
|
|
422
|
+
try {
|
|
423
|
+
return new Date(ms).toISOString();
|
|
424
|
+
} catch {
|
|
425
|
+
return undefined;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
/**
|
|
430
|
+
* Strip one pair of wrapping double quotes from a Path. ASP.NET and several
|
|
431
|
+
* Java containers send Path="/admin"; storing the quotes left pathMatches
|
|
432
|
+
* matching nothing, so the cookie was silently never sent again and the login
|
|
433
|
+
* looked gone on the page it was scoped to.
|
|
434
|
+
*
|
|
435
|
+
* Only Path gets this. A cookie value is stored and echoed back verbatim, as
|
|
436
|
+
* RFC 6265 5.2 and every browser have it, as jarFromCookieHeader already
|
|
437
|
+
* treats a seeded one, and because quoting is how a server protects a value
|
|
438
|
+
* holding a space or a comma.
|
|
439
|
+
* @param {string} s
|
|
440
|
+
* @returns {string}
|
|
441
|
+
*/
|
|
442
|
+
function unquotePath(s) {
|
|
443
|
+
return s.length >= 2 && s.startsWith('"') && s.endsWith('"') ? s.slice(1, -1) : s;
|
|
444
|
+
}
|
|
445
|
+
|
|
395
446
|
/**
|
|
396
447
|
* Parse one Set-Cookie header value.
|
|
397
448
|
* @param {string} header
|
|
@@ -425,6 +476,16 @@ export function parseSetCookie(header, requestUrl) {
|
|
|
425
476
|
...(url.protocol === 'https:' ? { secure: true } : {}),
|
|
426
477
|
};
|
|
427
478
|
|
|
479
|
+
// Max-Age and Expires are collected separately: RFC 6265 5.3 says Max-Age
|
|
480
|
+
// wins even when Expires is written last. Express writes them in that order,
|
|
481
|
+
// and Expires is the server's clock, so a host whose clock is behind would
|
|
482
|
+
// otherwise look expired the moment it was set.
|
|
483
|
+
/** @type {string | undefined} */
|
|
484
|
+
let maxAgeExpires;
|
|
485
|
+
let sawMaxAge = false;
|
|
486
|
+
/** @type {string | undefined} */
|
|
487
|
+
let expiresAt;
|
|
488
|
+
|
|
428
489
|
for (const attr of parts.slice(1)) {
|
|
429
490
|
const sep = attr.indexOf('=');
|
|
430
491
|
const key = (sep === -1 ? attr : attr.slice(0, sep)).trim().toLowerCase();
|
|
@@ -436,7 +497,7 @@ export function parseSetCookie(header, requestUrl) {
|
|
|
436
497
|
// dependency this project will not take. User-seeded cookies still scope by
|
|
437
498
|
// the --domain they pass, which normalizeDomain holds to the same floor.
|
|
438
499
|
if (key === 'path') {
|
|
439
|
-
cookie.path = val || '/';
|
|
500
|
+
cookie.path = unquotePath(val) || '/';
|
|
440
501
|
} else if (key === 'secure') {
|
|
441
502
|
cookie.secure = true;
|
|
442
503
|
} else if (key === 'httponly') {
|
|
@@ -444,13 +505,19 @@ export function parseSetCookie(header, requestUrl) {
|
|
|
444
505
|
} else if (key === 'max-age') {
|
|
445
506
|
const age = Number(val);
|
|
446
507
|
if (Number.isFinite(age)) {
|
|
447
|
-
|
|
508
|
+
sawMaxAge = true;
|
|
509
|
+
maxAgeExpires = age <= 0 ? new Date(0).toISOString() : expiryISO(Date.now() + age * 1000);
|
|
448
510
|
}
|
|
449
511
|
} else if (key === 'expires') {
|
|
450
512
|
const exp = Date.parse(val);
|
|
451
|
-
if (!Number.isNaN(exp))
|
|
513
|
+
if (!Number.isNaN(exp)) expiresAt = expiryISO(exp);
|
|
452
514
|
}
|
|
453
515
|
}
|
|
516
|
+
if (sawMaxAge) {
|
|
517
|
+
if (maxAgeExpires) cookie.expires = maxAgeExpires;
|
|
518
|
+
} else if (expiresAt) {
|
|
519
|
+
cookie.expires = expiresAt;
|
|
520
|
+
}
|
|
454
521
|
return cookie;
|
|
455
522
|
}
|
|
456
523
|
|
|
@@ -483,22 +550,26 @@ export function storeFromResponse(jar, url, setCookieHeaders) {
|
|
|
483
550
|
for (const header of setCookieHeaders) {
|
|
484
551
|
const parsed = parseSetCookie(header, url);
|
|
485
552
|
if (!parsed) continue;
|
|
553
|
+
const at = cookies.findIndex((c) => sameCookie(c, parsed));
|
|
486
554
|
// Max-Age=0 or Expires in the past deletes the cookie
|
|
487
555
|
if (parsed.expires && Date.parse(parsed.expires) <= Date.now()) {
|
|
488
|
-
|
|
556
|
+
if (at >= 0) cookies.splice(at, 1);
|
|
489
557
|
continue;
|
|
490
558
|
}
|
|
491
|
-
cookies = cookies.filter((c) => !(c.name === parsed.name && domainMatches(c, parsed.domain)));
|
|
492
559
|
// Replacing a cookie the jar already holds is always allowed; growing past
|
|
493
560
|
// the cap is not, so a page cannot bloat the sidecar with fresh names. The
|
|
494
561
|
// cookies already there - the seeded login among them - are what survive.
|
|
495
|
-
if (cookies.length >= MAX_COOKIES) continue;
|
|
562
|
+
if (at < 0 && cookies.length >= MAX_COOKIES) continue;
|
|
496
563
|
if (parsed.expires) {
|
|
497
564
|
const ceiling = Date.parse(jar.expiresAt);
|
|
498
565
|
const exp = Date.parse(parsed.expires);
|
|
499
566
|
if (exp > ceiling) parsed.expires = jar.expiresAt;
|
|
500
567
|
}
|
|
501
|
-
|
|
568
|
+
// A refresh keeps the slot it already had. RFC 6265 carries the original
|
|
569
|
+
// creation time onto the new value, and jar order is what stands in for
|
|
570
|
+
// that when cookieHeaderFor breaks a tie between equal-length paths.
|
|
571
|
+
if (at >= 0) cookies[at] = parsed;
|
|
572
|
+
else cookies.push(parsed);
|
|
502
573
|
}
|
|
503
574
|
return { ...jar, cookies };
|
|
504
575
|
}
|
package/src/distill.js
CHANGED
|
@@ -14,6 +14,7 @@ import TurndownService from 'turndown';
|
|
|
14
14
|
* @property {string} url
|
|
15
15
|
* @property {string} title
|
|
16
16
|
* @property {Block[]} blocks
|
|
17
|
+
* @property {string} [base] - document base URL from <base href>, if the page set one
|
|
17
18
|
*/
|
|
18
19
|
|
|
19
20
|
// Where the compact view cuts a text block. It lives here because numbering
|
|
@@ -118,19 +119,46 @@ const asHTML = (text, url = '', opts = {}) =>
|
|
|
118
119
|
* Every heading in the Rust book and every one on an AWS CLI reference page
|
|
119
120
|
* carries a permalink to its own id, so following those would refetch the
|
|
120
121
|
* page the agent is already reading, which is worse than the reading it
|
|
121
|
-
* already gets. A bare fragment is never a destination
|
|
122
|
+
* already gets. A bare fragment is never a destination, and neither is a
|
|
123
|
+
* mailto: or tel: href, which is not a page.
|
|
122
124
|
* @param {any} node - the heading element
|
|
123
125
|
* @param {string} text - its cleaned text
|
|
126
|
+
* @param {string} base - document base, to resolve the href
|
|
127
|
+
* @param {string} pageUrl - the page itself, for same-document checks
|
|
124
128
|
* @returns {string|null}
|
|
125
129
|
*/
|
|
126
|
-
function headingHref(node, text) {
|
|
130
|
+
function headingHref(node, text, base, pageUrl) {
|
|
127
131
|
const anchors = node.querySelectorAll('a[href]');
|
|
128
132
|
if (anchors.length !== 1) return null;
|
|
129
133
|
const href = anchors[0].getAttribute('href') ?? '';
|
|
130
|
-
if (!href || href
|
|
134
|
+
if (!href || !isFollowableHref(href, base, pageUrl)) return null;
|
|
131
135
|
return clean(anchors[0].textContent) === text ? href : null;
|
|
132
136
|
}
|
|
133
137
|
|
|
138
|
+
/**
|
|
139
|
+
* Whether this href is a page oc can fetch, other than the one already open.
|
|
140
|
+
* A bare fragment is the current page. mailto:/tel:/javascript: are not
|
|
141
|
+
* pages: fetchPage prefixes https:// onto a URL with no http(s) scheme, so
|
|
142
|
+
* mailto:hi@example.com became a GET of example.com with password "hi".
|
|
143
|
+
* The href resolves against <base href>, but "here" is the page URL: under
|
|
144
|
+
* <base href="/">, a link to / is the home page, not this one.
|
|
145
|
+
* @param {string} href
|
|
146
|
+
* @param {string} base
|
|
147
|
+
* @param {string} pageUrl
|
|
148
|
+
* @returns {boolean}
|
|
149
|
+
*/
|
|
150
|
+
function isFollowableHref(href, base, pageUrl) {
|
|
151
|
+
try {
|
|
152
|
+
const dest = new URL(href, base || pageUrl || undefined);
|
|
153
|
+
if (!/^https?:$/i.test(dest.protocol)) return false;
|
|
154
|
+
if (!pageUrl) return true;
|
|
155
|
+
const here = new URL(pageUrl);
|
|
156
|
+
return dest.origin !== here.origin || dest.pathname !== here.pathname || dest.search !== here.search;
|
|
157
|
+
} catch {
|
|
158
|
+
return false;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
134
162
|
/**
|
|
135
163
|
* Reduce raw HTML to an interaction tree: readable text plus numbered
|
|
136
164
|
* elements, in document order. The walk is deterministic and numbering is a
|
|
@@ -143,6 +171,7 @@ function headingHref(node, text) {
|
|
|
143
171
|
export function distill(html, url = '') {
|
|
144
172
|
const { document } = parseHTML(asHTML(html, url));
|
|
145
173
|
const title = clean(document.querySelector('title')?.textContent ?? '');
|
|
174
|
+
const base = documentBase(document, url);
|
|
146
175
|
/** @type {Block[]} */
|
|
147
176
|
const blocks = [];
|
|
148
177
|
|
|
@@ -215,20 +244,40 @@ export function distill(html, url = '') {
|
|
|
215
244
|
if (/^h[1-6]$/.test(tag)) {
|
|
216
245
|
const text = clean(node.textContent);
|
|
217
246
|
if (text) {
|
|
218
|
-
const href = headingHref(node, text);
|
|
247
|
+
const href = headingHref(node, text, base, url);
|
|
219
248
|
blocks.push({ type: 'heading', level: Number(tag[1]), text, ...(href ? { href } : {}) });
|
|
220
249
|
}
|
|
221
250
|
return;
|
|
222
251
|
}
|
|
223
252
|
if (tag === 'a' && node.getAttribute('href')) {
|
|
224
|
-
const
|
|
253
|
+
const href = node.getAttribute('href');
|
|
254
|
+
// Same rule as headingHref: a fragment is this page, mailto: is not a
|
|
255
|
+
// page. Walk children so the label stays readable text instead of a
|
|
256
|
+
// numbered handle that do cannot fetch.
|
|
257
|
+
if (!isFollowableHref(href, base, url)) {
|
|
258
|
+
for (const child of node.childNodes) walk(child);
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
// An icon button already takes aria-label or title because it has no
|
|
262
|
+
// text. An anchor around an image is the same shape: img alt is the
|
|
263
|
+
// accessible name and is not a text node, so textContent is empty and
|
|
264
|
+
// the link used to be dropped. Visible text still wins when there is
|
|
265
|
+
// any, because that is what the page wrote for a reader.
|
|
266
|
+
const text = clean(node.textContent)
|
|
267
|
+
|| clean(node.getAttribute('aria-label') ?? '')
|
|
268
|
+
|| clean(node.getAttribute('title') ?? '')
|
|
269
|
+
|| clean(node.querySelector('img')?.getAttribute('alt') ?? '');
|
|
225
270
|
if (text) {
|
|
226
|
-
blocks.push({ type: 'link', text, href
|
|
271
|
+
blocks.push({ type: 'link', text, href });
|
|
227
272
|
}
|
|
228
273
|
return;
|
|
229
274
|
}
|
|
230
275
|
if (tag === 'input' || tag === 'textarea' || tag === 'select') {
|
|
231
|
-
|
|
276
|
+
// HTML's type attribute is ASCII case-insensitive. type=HIDDEN used to
|
|
277
|
+
// appear as a numbered input, type=SUBMIT as an input rather than a
|
|
278
|
+
// button, and type=PASSWORD as a field named PASSWORD, so a login wall
|
|
279
|
+
// written that way was not detected.
|
|
280
|
+
const kind = (node.getAttribute('type') ?? 'text').toLowerCase();
|
|
232
281
|
if (kind === 'hidden') return;
|
|
233
282
|
if (kind === 'submit' || kind === 'button') {
|
|
234
283
|
blocks.push({ type: 'button', text: node.getAttribute('value') ?? 'submit' });
|
|
@@ -290,7 +339,33 @@ export function distill(html, url = '') {
|
|
|
290
339
|
} else if (body) {
|
|
291
340
|
walk(body);
|
|
292
341
|
}
|
|
293
|
-
return {
|
|
342
|
+
return {
|
|
343
|
+
url,
|
|
344
|
+
title,
|
|
345
|
+
blocks: number(mergeText(dropRepeats(blocks))),
|
|
346
|
+
...(base !== url ? { base } : {}),
|
|
347
|
+
};
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
/**
|
|
351
|
+
* Document base URL from the first <base href>, resolved against the page
|
|
352
|
+
* URL. Relative links on the page are relative to this, not to the request
|
|
353
|
+
* URL: a mirror, a CDN, and many docs generators put one in the head.
|
|
354
|
+
* Head is dropped as chrome, so the href is read here and carried on the
|
|
355
|
+
* page for sessionFromPage, which is what `oc do` resolves against. An
|
|
356
|
+
* unusable href leaves the page URL in charge.
|
|
357
|
+
* @param {any} document
|
|
358
|
+
* @param {string} url
|
|
359
|
+
* @returns {string}
|
|
360
|
+
*/
|
|
361
|
+
function documentBase(document, url) {
|
|
362
|
+
const href = clean(document.querySelector('base[href]')?.getAttribute('href') ?? '');
|
|
363
|
+
if (!href) return url;
|
|
364
|
+
try {
|
|
365
|
+
return new URL(href, url || undefined).href;
|
|
366
|
+
} catch {
|
|
367
|
+
return url;
|
|
368
|
+
}
|
|
294
369
|
}
|
|
295
370
|
|
|
296
371
|
/**
|
|
@@ -429,7 +504,10 @@ function cleanDocument(html, url = '') {
|
|
|
429
504
|
for (const tag of DROP) {
|
|
430
505
|
for (const el of [...document.querySelectorAll(tag)]) el.remove();
|
|
431
506
|
}
|
|
432
|
-
for (const el of [...document.querySelectorAll('[hidden], [aria-hidden="true"]
|
|
507
|
+
for (const el of [...document.querySelectorAll('[hidden], [aria-hidden="true"]')]) el.remove();
|
|
508
|
+
for (const el of [...document.querySelectorAll('input')]) {
|
|
509
|
+
if ((el.getAttribute('type') ?? '').toLowerCase() === 'hidden') el.remove();
|
|
510
|
+
}
|
|
433
511
|
for (const el of [...document.querySelectorAll('[style]')]) {
|
|
434
512
|
if (/display:\s*none/.test(el.getAttribute('style') ?? '')) el.remove();
|
|
435
513
|
}
|
|
@@ -490,19 +568,44 @@ export function feedToHTML(text) {
|
|
|
490
568
|
// <category /> stay open and swallow their siblings. Descendant queries
|
|
491
569
|
// still land, because a closing </entry> or </item> pops the whole pile.
|
|
492
570
|
const field = (el, sel) => clean(el.querySelector(sel)?.textContent ?? '');
|
|
571
|
+
// HTML treats <link> as void, so an RSS 2.0 URL written as the element's
|
|
572
|
+
// text is not in textContent: the parser closes the tag immediately and the
|
|
573
|
+
// URL sits in the following text node. Skipping that node made `do` on a
|
|
574
|
+
// WordPress or HN item follow the guid instead, which is often an internal
|
|
575
|
+
// id, or follow nothing when the item has no guid at all.
|
|
576
|
+
const rssLink = (entry) => {
|
|
577
|
+
const link = entry.querySelector('link');
|
|
578
|
+
if (!link || link.getAttribute('href')) return '';
|
|
579
|
+
let n = link.nextSibling;
|
|
580
|
+
while (n && n.nodeType === 3 && !/\S/.test(n.textContent ?? '')) n = n.nextSibling;
|
|
581
|
+
const text = n?.nodeType === 3 ? clean(n.textContent ?? '') : '';
|
|
582
|
+
return text && !/\s/.test(text) ? text : '';
|
|
583
|
+
};
|
|
493
584
|
const feedTitle = field(root, 'title');
|
|
494
585
|
const parts = [];
|
|
495
586
|
for (const entry of root.querySelectorAll('entry, item')) {
|
|
496
587
|
const title = field(entry, 'title');
|
|
588
|
+
// Atom puts the URL on link href. RSS 2.0 writes it as <link> text, which
|
|
589
|
+
// rssLink reads. guid is only a permalink when it says so (the default),
|
|
590
|
+
// and isPermaLink="false" means the value is an id, not a URL.
|
|
591
|
+
const guid = entry.querySelector('guid');
|
|
592
|
+
const guidIsPermalink = !guid
|
|
593
|
+
|| (guid.getAttribute('ispermalink') ?? 'true').toLowerCase() !== 'false';
|
|
497
594
|
const href = entry.querySelector('link[rel="alternate"]')?.getAttribute('href')
|
|
498
|
-
|
|
499
|
-
|
|
595
|
+
|| entry.querySelector('link[href]')?.getAttribute('href')
|
|
596
|
+
|| rssLink(entry)
|
|
597
|
+
|| (guidIsPermalink ? field(entry, 'guid') : '');
|
|
500
598
|
const author = field(entry, 'author name') || field(entry, 'author');
|
|
501
599
|
const date = (field(entry, 'updated') || field(entry, 'published') || field(entry, 'pubdate')).slice(0, 10);
|
|
502
600
|
const byline = [author && `by ${author}`, date].filter(Boolean).join(', ');
|
|
503
601
|
// Atom escapes the entry body, so textContent of content/summary is the
|
|
504
602
|
// HTML itself, ready to be embedded and parsed like any page.
|
|
505
|
-
|
|
603
|
+
// RSS 2.0 keeps the full post in content:encoded and the excerpt in
|
|
604
|
+
// description, so the encoded body has to win or a feed is only ledes.
|
|
605
|
+
const body = (entry.querySelector('content\\:encoded')
|
|
606
|
+
?? entry.querySelector('content')
|
|
607
|
+
?? entry.querySelector('summary')
|
|
608
|
+
?? entry.querySelector('description'))?.textContent ?? '';
|
|
506
609
|
parts.push('<article>');
|
|
507
610
|
// The title is the link. It used to sit beside the byline as an anchor
|
|
508
611
|
// labelled "open", the same label on every entry, and the repeated-controls
|
package/src/fetch.js
CHANGED
|
@@ -12,6 +12,7 @@ import http from 'node:http';
|
|
|
12
12
|
import https from 'node:https';
|
|
13
13
|
import net from 'node:net';
|
|
14
14
|
import tls from 'node:tls';
|
|
15
|
+
import zlib from 'node:zlib';
|
|
15
16
|
|
|
16
17
|
import { getSetCookieHeaders } from './cookies.js';
|
|
17
18
|
|
|
@@ -422,6 +423,26 @@ function authority(target) {
|
|
|
422
423
|
return `${host}:${port}`;
|
|
423
424
|
}
|
|
424
425
|
|
|
426
|
+
// Native fetch (undici) already decodes Content-Encoding. Node's http parser
|
|
427
|
+
// does not, so the proxy transport used to charset-decode gzip bytes as the
|
|
428
|
+
// page: binary noise an agent pays for. Only gzip is handled here; deflate
|
|
429
|
+
// and br are left as they arrive, because this path does not ask for them.
|
|
430
|
+
// maxOutputLength is what stops a tiny gzip from expanding past the cap,
|
|
431
|
+
// which the compressed Content-Length cannot see.
|
|
432
|
+
function decompressBody(bytes, encoding, url) {
|
|
433
|
+
const enc = encoding?.split(',')[0]?.trim().toLowerCase();
|
|
434
|
+
if (!bytes.length || (enc !== 'gzip' && enc !== 'x-gzip')) return bytes;
|
|
435
|
+
let out;
|
|
436
|
+
try {
|
|
437
|
+
out = zlib.gunzipSync(bytes, { maxOutputLength: MAX_BODY });
|
|
438
|
+
} catch (err) {
|
|
439
|
+
if (err.code === 'ERR_BUFFER_TOO_LARGE') assertBodySize(MAX_BODY + 1, url);
|
|
440
|
+
throw new Error(`failed to decode gzip body for ${url}: ${err.message}`);
|
|
441
|
+
}
|
|
442
|
+
assertBodySize(out.length, url);
|
|
443
|
+
return out;
|
|
444
|
+
}
|
|
445
|
+
|
|
425
446
|
function wrapNodeResponse(res, url) {
|
|
426
447
|
const headers = {
|
|
427
448
|
get(name) {
|
|
@@ -452,7 +473,14 @@ function wrapNodeResponse(res, url) {
|
|
|
452
473
|
}
|
|
453
474
|
chunks.push(c);
|
|
454
475
|
});
|
|
455
|
-
res.on('end', () =>
|
|
476
|
+
res.on('end', () => {
|
|
477
|
+
try {
|
|
478
|
+
const bytes = decompressBody(Buffer.concat(chunks), headers.get('content-encoding'), url);
|
|
479
|
+
resolve(decodeBody(bytes, headers.get('content-type')));
|
|
480
|
+
} catch (err) {
|
|
481
|
+
reject(err);
|
|
482
|
+
}
|
|
483
|
+
});
|
|
456
484
|
res.on('error', reject);
|
|
457
485
|
});
|
|
458
486
|
const status = res.statusCode ?? 0;
|
package/src/render.js
CHANGED
|
@@ -127,7 +127,9 @@ export function render(page, { budget = 500, from = 0 } = {}) {
|
|
|
127
127
|
if (spent + cost > limit && lines.length > head.length) break;
|
|
128
128
|
spent += cost;
|
|
129
129
|
lines.push(line);
|
|
130
|
-
|
|
130
|
+
// A heading that is a link (search result titles) is what do follows, so
|
|
131
|
+
// it has to count here or the footer omits do on a page of only those.
|
|
132
|
+
if (block.type === 'link' || block.type === 'button' || block.href) hasLinks = true;
|
|
131
133
|
}
|
|
132
134
|
|
|
133
135
|
const rest = blocks.slice(i);
|
package/src/session.js
CHANGED
|
@@ -104,16 +104,29 @@ function unwrapRedirect(url) {
|
|
|
104
104
|
|
|
105
105
|
/**
|
|
106
106
|
* Absolute URL for a handle, or null when the link is not followable
|
|
107
|
-
* (javascript: handlers, malformed
|
|
107
|
+
* (javascript: handlers, mailto:/tel:, same-document fragments, malformed
|
|
108
|
+
* hrefs). fetchPage prefixes https:// onto anything that is not already
|
|
109
|
+
* http(s), so mailto:hi@example.com used to become a GET of example.com
|
|
110
|
+
* with password "hi". A bare fragment is the page already open.
|
|
108
111
|
* @param {string} href
|
|
109
|
-
* @param {string} base
|
|
112
|
+
* @param {string} base - <base href>, or the page URL when it set none
|
|
113
|
+
* @param {string} [pageUrl] - the page itself, for same-document checks
|
|
110
114
|
* @returns {string|null}
|
|
111
115
|
*/
|
|
112
|
-
export function resolveHref(href, base) {
|
|
113
|
-
if (!href
|
|
116
|
+
export function resolveHref(href, base, pageUrl = base) {
|
|
117
|
+
if (!href) return null;
|
|
114
118
|
try {
|
|
115
119
|
const url = new URL(href, base || undefined);
|
|
116
|
-
|
|
120
|
+
if (!/^https?:$/i.test(url.protocol)) return null;
|
|
121
|
+
const dest = unwrapRedirect(url) ?? url.href;
|
|
122
|
+
if (pageUrl) {
|
|
123
|
+
const here = new URL(pageUrl);
|
|
124
|
+
const there = new URL(dest);
|
|
125
|
+
if (here.origin === there.origin && here.pathname === there.pathname && here.search === there.search) {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return dest;
|
|
117
130
|
} catch {
|
|
118
131
|
return null;
|
|
119
132
|
}
|
|
@@ -144,7 +157,8 @@ export function sessionFromPage(page, previous, { cursor = 0 } = {}) {
|
|
|
144
157
|
dropped++;
|
|
145
158
|
continue;
|
|
146
159
|
}
|
|
147
|
-
|
|
160
|
+
// Relative hrefs are against <base href> when the page set one.
|
|
161
|
+
const href = block.href ? resolveHref(block.href, page.base || page.url, page.url) : null;
|
|
148
162
|
blocks.push({
|
|
149
163
|
type: block.type,
|
|
150
164
|
text: block.text,
|
package/src/sites.js
CHANGED
|
@@ -145,7 +145,12 @@ export function resolveSite(name, args) {
|
|
|
145
145
|
*/
|
|
146
146
|
function encode(value, template, arg) {
|
|
147
147
|
const query = template.includes('?') && template.indexOf(`{${arg}}`) > template.indexOf('?');
|
|
148
|
-
|
|
148
|
+
// A path slot that starts with @ is a copied handle (@name, @channel).
|
|
149
|
+
// encodeURIComponent would turn it into %40 and miss the user, and YouTube's
|
|
150
|
+
// template already has the @ in the URL. Search queries keep it, so a
|
|
151
|
+
// mention in a query is still the thing being searched for.
|
|
152
|
+
const raw = !query && value.startsWith('@') && value.length > 1 ? value.slice(1) : value;
|
|
153
|
+
const encoded = encodeURIComponent(raw);
|
|
149
154
|
return query ? encoded : encoded.replaceAll('%2F', '/');
|
|
150
155
|
}
|
|
151
156
|
|
package/src/sphinx.js
CHANGED
|
@@ -84,11 +84,17 @@ function objectHits(index, query) {
|
|
|
84
84
|
for (const [doc, typeIdx, priority, anchor, name] of entries) {
|
|
85
85
|
const full = prefix ? `${prefix}.${name}` : name;
|
|
86
86
|
if (full.toLowerCase() !== q && name.toLowerCase() !== q) continue;
|
|
87
|
+
// Empty string: the object's own name is the HTML id. The sentinel '-'
|
|
88
|
+
// means Sphinx built '{objtype}-{fullname}' instead, which is how
|
|
89
|
+
// docs.python.org names every module heading (#module-json).
|
|
90
|
+
const id = anchor === '' ? full
|
|
91
|
+
: anchor === '-' ? `${index.objnames?.[typeIdx]?.[1] ?? ''}-${full}`
|
|
92
|
+
: anchor;
|
|
87
93
|
hits.push({
|
|
88
94
|
name: full,
|
|
89
95
|
type: index.objnames?.[typeIdx]?.[2] ?? '',
|
|
90
96
|
doc,
|
|
91
|
-
anchor:
|
|
97
|
+
anchor: id,
|
|
92
98
|
priority,
|
|
93
99
|
});
|
|
94
100
|
}
|