getbotons 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +505 -2
- package/installer/admin.mjs +6 -0
- package/installer/botons.mjs +430 -0
- package/installer/brain-clis.mjs +28 -0
- package/installer/linger.mjs +39 -0
- package/installer/output.mjs +27 -0
- package/installer/revision.mjs +30 -0
- package/installer/shell-path.mjs +66 -0
- package/installer/source-lock.json +2891 -0
- package/package.json +26 -4
- package/payload/LICENSE +21 -0
- package/payload/SECURITY.md +33 -0
- package/payload/installer/admin.mjs +6 -0
- package/payload/installer/botons.mjs +430 -0
- package/payload/installer/brain-clis.mjs +28 -0
- package/payload/installer/linger.mjs +39 -0
- package/payload/installer/output.mjs +27 -0
- package/payload/installer/revision.mjs +30 -0
- package/payload/installer/shell-path.mjs +66 -0
- package/payload/installer/source-lock.json +2891 -0
- package/payload/npm-shrinkwrap.json +51 -0
- package/payload/package.json +13 -0
- package/payload/scripts/backup.ts +47 -0
- package/payload/scripts/botons-backup.service +15 -0
- package/payload/scripts/botons-backup.timer +11 -0
- package/payload/scripts/botons.service +36 -0
- package/payload/scripts/check-browser.ts +90 -0
- package/payload/scripts/doctor.ts +790 -0
- package/payload/scripts/install-backup-timer.ts +52 -0
- package/payload/scripts/install-service.ts +206 -0
- package/payload/scripts/login.ts +17 -0
- package/payload/scripts/migrate-names.ts +47 -0
- package/payload/scripts/pair.ts +163 -0
- package/payload/scripts/restore.ts +108 -0
- package/payload/server/bin/botons +4 -0
- package/payload/server/bin/legacy/minion +4 -0
- package/payload/server/bin/legacy/tots +4 -0
- package/payload/server/src/acting.ts +16 -0
- package/payload/server/src/api-keys.ts +544 -0
- package/payload/server/src/api.ts +1484 -0
- package/payload/server/src/app.ts +546 -0
- package/payload/server/src/approvals.ts +873 -0
- package/payload/server/src/ask-rules.ts +181 -0
- package/payload/server/src/audit.ts +158 -0
- package/payload/server/src/auth.ts +497 -0
- package/payload/server/src/backup.ts +382 -0
- package/payload/server/src/brain-login.ts +75 -0
- package/payload/server/src/browser-cert.ts +67 -0
- package/payload/server/src/browser-handoff.ts +119 -0
- package/payload/server/src/browser-handover.ts +175 -0
- package/payload/server/src/browser-policy.ts +210 -0
- package/payload/server/src/browser-proxy.ts +580 -0
- package/payload/server/src/browser-request.ts +335 -0
- package/payload/server/src/browser-screenshot.ts +56 -0
- package/payload/server/src/browser-send.ts +155 -0
- package/payload/server/src/browser.ts +2164 -0
- package/payload/server/src/callback.ts +218 -0
- package/payload/server/src/capabilities.ts +90 -0
- package/payload/server/src/cards.ts +243 -0
- package/payload/server/src/cli/approve-mcp.ts +119 -0
- package/payload/server/src/cli/botons.ts +629 -0
- package/payload/server/src/cli/browser-mcp.ts +168 -0
- package/payload/server/src/cli/workspace-file.ts +78 -0
- package/payload/server/src/cli-homes.ts +38 -0
- package/payload/server/src/cli-install.ts +104 -0
- package/payload/server/src/cli-updates.ts +112 -0
- package/payload/server/src/codex-config-repair.ts +80 -0
- package/payload/server/src/config.ts +473 -0
- package/payload/server/src/connector-access.ts +158 -0
- package/payload/server/src/connector-launch.ts +203 -0
- package/payload/server/src/connectors.ts +947 -0
- package/payload/server/src/disk-usage.ts +62 -0
- package/payload/server/src/engine.ts +2328 -0
- package/payload/server/src/env-init.ts +5 -0
- package/payload/server/src/events.ts +46 -0
- package/payload/server/src/files.ts +528 -0
- package/payload/server/src/github-access.ts +52 -0
- package/payload/server/src/health.ts +117 -0
- package/payload/server/src/held-dir.ts +96 -0
- package/payload/server/src/hooks.ts +141 -0
- package/payload/server/src/http.ts +207 -0
- package/payload/server/src/index.ts +139 -0
- package/payload/server/src/install-type.ts +11 -0
- package/payload/server/src/instructions.ts +204 -0
- package/payload/server/src/isolation.ts +234 -0
- package/payload/server/src/legacy-upgrade.ts +668 -0
- package/payload/server/src/limits.ts +44 -0
- package/payload/server/src/lock.ts +75 -0
- package/payload/server/src/log.ts +137 -0
- package/payload/server/src/managed-clis.ts +65 -0
- package/payload/server/src/mcp-client.ts +229 -0
- package/payload/server/src/notify.ts +193 -0
- package/payload/server/src/oauth-connections.ts +650 -0
- package/payload/server/src/oauth.ts +311 -0
- package/payload/server/src/outbound.ts +142 -0
- package/payload/server/src/portable.ts +1669 -0
- package/payload/server/src/prompt.ts +676 -0
- package/payload/server/src/providers/api-brain.ts +612 -0
- package/payload/server/src/providers/api-history.ts +131 -0
- package/payload/server/src/providers/api-tools.ts +481 -0
- package/payload/server/src/providers/api-wire.ts +690 -0
- package/payload/server/src/providers/claude-cli.ts +620 -0
- package/payload/server/src/providers/cli-check.ts +265 -0
- package/payload/server/src/providers/cli-turn.ts +20 -0
- package/payload/server/src/providers/codex-cli.ts +552 -0
- package/payload/server/src/providers/codex-workspace.ts +61 -0
- package/payload/server/src/providers/failures.ts +143 -0
- package/payload/server/src/providers/index.ts +131 -0
- package/payload/server/src/providers/mcp-stdio.ts +147 -0
- package/payload/server/src/providers/mock.ts +94 -0
- package/payload/server/src/providers/process.ts +268 -0
- package/payload/server/src/providers/readiness.ts +43 -0
- package/payload/server/src/providers/sse.ts +73 -0
- package/payload/server/src/providers/types.ts +173 -0
- package/payload/server/src/rates.ts +128 -0
- package/payload/server/src/reactions.ts +266 -0
- package/payload/server/src/recent-logs.ts +489 -0
- package/payload/server/src/rehearsal.ts +593 -0
- package/payload/server/src/rehearsals.ts +303 -0
- package/payload/server/src/replay.ts +165 -0
- package/payload/server/src/sandbox.ts +767 -0
- package/payload/server/src/schedule.ts +214 -0
- package/payload/server/src/scheduler.ts +24 -0
- package/payload/server/src/schema.ts +882 -0
- package/payload/server/src/secrets.ts +46 -0
- package/payload/server/src/setup.ts +356 -0
- package/payload/server/src/skill-packages.ts +363 -0
- package/payload/server/src/skills.ts +527 -0
- package/payload/server/src/spend.ts +255 -0
- package/payload/server/src/ssrf.ts +126 -0
- package/payload/server/src/starter-gate.ts +168 -0
- package/payload/server/src/starters.ts +610 -0
- package/payload/server/src/storage.ts +32 -0
- package/payload/server/src/store.ts +3087 -0
- package/payload/server/src/tailnet.ts +132 -0
- package/payload/server/src/templates.ts +454 -0
- package/payload/server/src/trash.ts +772 -0
- package/payload/server/src/util.ts +97 -0
- package/payload/server/src/webhooks.ts +216 -0
- package/payload/server/src/work-feed.ts +473 -0
- package/payload/shared/app-version.d.mts +1 -0
- package/payload/shared/app-version.mjs +3 -0
- package/payload/shared/boton.ts +323 -0
- package/payload/shared/brand.ts +107 -0
- package/payload/shared/cards.ts +427 -0
- package/payload/shared/effort.ts +91 -0
- package/payload/shared/files.ts +104 -0
- package/payload/shared/handoff.ts +97 -0
- package/payload/shared/install-commands.ts +7 -0
- package/payload/shared/legacy-names.d.mts +35 -0
- package/payload/shared/legacy-names.mjs +244 -0
- package/payload/shared/live-view.ts +94 -0
- package/payload/shared/look-critter.ts +189 -0
- package/payload/shared/look-tater.ts +127 -0
- package/payload/shared/look.ts +200 -0
- package/payload/shared/reactions.ts +43 -0
- package/payload/shared/seeded.ts +34 -0
- package/payload/shared/services.ts +90 -0
- package/payload/shared/types.ts +3016 -0
- package/payload/shared/util.ts +68 -0
- package/payload/shared/versions.ts +44 -0
- package/payload/shared/wake.ts +164 -0
- package/payload/web/dist/apple-touch-icon.png +0 -0
- package/payload/web/dist/assets/Icons-nytVAa9g.js +1 -0
- package/payload/web/dist/assets/NewBotDialog-DXr2XhDP.js +1 -0
- package/payload/web/dist/assets/PairScreen-CMcJO7cQ.js +1 -0
- package/payload/web/dist/assets/Panel-Bvqz3I3m.js +4 -0
- package/payload/web/dist/assets/ReactionPicker-B0SKsjiy.js +1 -0
- package/payload/web/dist/assets/SettingRows-CvLomtuo.js +4 -0
- package/payload/web/dist/assets/SettingsDialog-CQ41LV-P.js +8 -0
- package/payload/web/dist/assets/Setup-C-3P7Mtn.js +1 -0
- package/payload/web/dist/assets/Skills-DXMH69kH.js +1 -0
- package/payload/web/dist/assets/StarterSetup-Ce2V1_Cs.js +5 -0
- package/payload/web/dist/assets/Steps-CKtB3GAM.js +1 -0
- package/payload/web/dist/assets/emoji-catalog-DfpPz8qw.js +540 -0
- package/payload/web/dist/assets/index-BEU7FDxV.css +1 -0
- package/payload/web/dist/assets/index-Dhwn-rf0.js +3 -0
- package/payload/web/dist/assets/react-C1iE8mnY.js +9 -0
- package/payload/web/dist/assets/reactions-rw0YeCmA.js +1 -0
- package/payload/web/dist/assets/rolldown-runtime-XKyaHtl9.js +1 -0
- package/payload/web/dist/assets/util-Cx5feSJu.js +1 -0
- package/payload/web/dist/assets/vendor-B3qdcjGo.js +14 -0
- package/payload/web/dist/favicon.svg +1 -0
- package/payload/web/dist/icon-192.png +0 -0
- package/payload/web/dist/icon-512.png +0 -0
- package/payload/web/dist/icon.svg +1 -0
- package/payload/web/dist/index.html +67 -0
- package/payload/web/dist/manifest.webmanifest +12 -0
- package/shared/legacy-names.d.mts +35 -0
- package/shared/legacy-names.mjs +244 -0
|
@@ -0,0 +1,2164 @@
|
|
|
1
|
+
import { randomBytes } from 'node:crypto';
|
|
2
|
+
import { brotliDecompressSync, gunzipSync, inflateSync, zstdDecompressSync } from 'node:zlib';
|
|
3
|
+
import { appendFileSync, chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { basename, dirname, join, resolve } from 'node:path';
|
|
5
|
+
import type {
|
|
6
|
+
AccessLevel,
|
|
7
|
+
BrowserActionInput,
|
|
8
|
+
BrowserActionName,
|
|
9
|
+
BrowserActionResult,
|
|
10
|
+
BrowserControl,
|
|
11
|
+
BrowserHandover,
|
|
12
|
+
BrowserHandoverInput,
|
|
13
|
+
BrowserHandoverResult,
|
|
14
|
+
BrowserHandoverStart,
|
|
15
|
+
HandoverNotice,
|
|
16
|
+
BrowserJournalEntry,
|
|
17
|
+
BrowserPolicy,
|
|
18
|
+
BrowserPolicyInput,
|
|
19
|
+
BrowserStatus,
|
|
20
|
+
} from '../../shared/types.ts';
|
|
21
|
+
import { BROWSER_CONNECTOR, BROWSER_SERVER } from '../../shared/util.ts';
|
|
22
|
+
import type { Approvals, BrowserHolds } from './approvals.ts';
|
|
23
|
+
import { type Audit, SYSTEM_ACTOR } from './audit.ts';
|
|
24
|
+
import type { Principal } from './auth.ts';
|
|
25
|
+
import { interceptCert } from './browser-cert.ts';
|
|
26
|
+
import { fitViewport, FRAME_MIN_MS, IDLE_MS, INPUT_WINDOW_MS, INTENT_MS, MAX_COPY, MAX_HANDOVER_MS, MAX_INPUTS, parseOwnerInput, pressed, RateWindow, VIEWPORT } from './browser-handover.ts';
|
|
27
|
+
import type { BrowserHandoffs } from './browser-handoff.ts';
|
|
28
|
+
import { applyPolicy, displayUrl, effectiveSites, judgeRequest, parsePolicy, portOf } from './browser-policy.ts';
|
|
29
|
+
import { type BrowserProxy, type ProxyPolicy, startBrowserProxy } from './browser-proxy.ts';
|
|
30
|
+
import { type Binding, describeRequest, type FieldInfo, fingerprint, type HeldRequest, heldCookies, sameRequest, shown, staleCookies, typingProblem, wireHeaders } from './browser-request.ts';
|
|
31
|
+
import { screenshotToChat, type ShareFile } from './browser-screenshot.ts';
|
|
32
|
+
import { sendHeld } from './browser-send.ts';
|
|
33
|
+
import type { Config } from './config.ts';
|
|
34
|
+
import { log } from './log.ts';
|
|
35
|
+
import type { TurnMcpServer } from './providers/types.ts';
|
|
36
|
+
import { HttpError, type Store } from './store.ts';
|
|
37
|
+
|
|
38
|
+
/*
|
|
39
|
+
* The managed browser: one Chromium profile per Boton, run by the server, never by the Boton. It reaches the
|
|
40
|
+
* Boton's turns as a Botons-managed connector (BROWSER_CONNECTOR, `botons browser-mcp`, the same for Claude and
|
|
41
|
+
* Codex): the owner turns it on by giving the Boton that connector, and lists the sites it may load.
|
|
42
|
+
*
|
|
43
|
+
* Opening and reading a page loads what the page loads: GET and HEAD requests to the allowed sites, as any browser
|
|
44
|
+
* would (a site can act on a GET too, so that is all it is: site-bound, not proof that nothing changes). Acting on a
|
|
45
|
+
* page is what the owner's cards are for. The Boton clicks and types into the controls a read listed (never into a
|
|
46
|
+
* password, code or card field), and nothing it does runs its own script in a page or reads cookies or files. A click
|
|
47
|
+
* on a link to an allowed site opens that address, as open would, without running the page's script. Any other click
|
|
48
|
+
* or typing marks the page as acted on, and from then on nothing it sends leaves on its own, whatever the method:
|
|
49
|
+
* while the step runs, the first request it makes (a form before a script's request) waits on a card that shows that
|
|
50
|
+
* exact request (method, full address, every field, its cookies by name) and is bound to its fingerprint, the turn,
|
|
51
|
+
* this browser and the page's document. Its cookies must be ones the browser held for that address while the step ran
|
|
52
|
+
* (a site's own cookies change as it works, and that is fine; one the browser never held means no card), and it goes
|
|
53
|
+
* with exactly those, as held; every other request it makes, then or later (a
|
|
54
|
+
* timer, a poll, a beacon as it unloads), is refused. Allowed, the card's request is sent once by browser-send.ts,
|
|
55
|
+
* exactly as held, with no cookie added, through the proxy, with no redirect that would send it again; denied,
|
|
56
|
+
* withdrawn, or not decided while the step waits, it never leaves. Opening an address or going back from a page
|
|
57
|
+
* acted on ends that whole browser (its Chromium, and its proxy, sealed until it closes) and starts another on the
|
|
58
|
+
* same profile with the same cookies, so nothing the old page had under way (an unload's beacon, a keepalive fetch)
|
|
59
|
+
* can go: not on a timer's guess that it has arrived, but because the process that held it and the proxy it could
|
|
60
|
+
* reach are gone. Labels are the site's text, never a reason.
|
|
61
|
+
*
|
|
62
|
+
* What holds whatever the Boton or a page does:
|
|
63
|
+
* - It runs only inside a turn that was given it, for that turn's token, and closes when the turn ends; taking the
|
|
64
|
+
* connector away (Stop, or unchecking it) closes it at once. Webhook runs get no connectors, so no browser.
|
|
65
|
+
* - Every request goes through the server's proxy (browser-proxy.ts), which allows only the owner's sites and, for a
|
|
66
|
+
* name, only public addresses it resolves to; routing refuses anything but GET and HEAD before it leaves Chromium,
|
|
67
|
+
* and everything from a page acted on (but for the one request a card holds), and WebSockets never connect. Changing the sites cuts tunnels, and plain HTTP responses still arriving, to a site
|
|
68
|
+
* no longer listed.
|
|
69
|
+
* - Chromium's own sandbox is on (in Docker, compose.yaml's seccomp profile lets it start; BOTONS_BROWSER_SANDBOX=0
|
|
70
|
+
* is the owner's last resort where a host still can't). Downloads and file pickers are refused, other tabs
|
|
71
|
+
* closed, dialogs dismissed, service workers blocked.
|
|
72
|
+
* - What a page says comes back marked as the site's text, never as the owner's words.
|
|
73
|
+
* - The owner can take the browser over (takeOver, to sign it in, say): from then until the owner finishes, the
|
|
74
|
+
* Boton's tools are refused and its browser is closed, and another Chromium runs for the owner alone, driven only
|
|
75
|
+
* by fixed inputs from the one paired device that took it over (browser-handover.ts says what that browser may
|
|
76
|
+
* send). It runs on a profile of its own, made for the handover with the Boton's cookies and deleted when it ends
|
|
77
|
+
* (and at start, if the server stopped without ending it), so nothing Chromium keeps of what the owner typed (a
|
|
78
|
+
* session's form values, its history) outlives the handover or reaches a backup. Only the cookies come back: into
|
|
79
|
+
* the Boton's profile, and in memory (#carry, until the server stops) for session ones, so the Boton's next
|
|
80
|
+
* browser is signed in as the owner left it.
|
|
81
|
+
* - Each step goes to the Boton's browser journal (origin and path only), which the owner can read. A step that
|
|
82
|
+
* fails says so in Botons' own words (failureOf): Playwright's and Chromium's messages carry the full address, query
|
|
83
|
+
* included, and whatever else the site put there, so none of their text reaches a result, the journal or the log.
|
|
84
|
+
*/
|
|
85
|
+
|
|
86
|
+
const ACTIONS: BrowserActionName[] = ['open', 'read', 'back', 'screenshot', 'click', 'type', 'handoff'];
|
|
87
|
+
const MAX_TEXT = 20_000;
|
|
88
|
+
const MAX_LINKS = 100;
|
|
89
|
+
const MAX_CONTROLS = 150;
|
|
90
|
+
const MAX_TYPED = 2000;
|
|
91
|
+
/** After a click or typing, how long a request the page sends because of it is still held for a card. */
|
|
92
|
+
const SETTLE_MS = 1500;
|
|
93
|
+
/** Once a script's request is held, how much longer a form the same step submits may take its place. */
|
|
94
|
+
const FORM_GRACE_MS = 300;
|
|
95
|
+
/** What read lists as controls, and click and type find again by their place in this list. */
|
|
96
|
+
const CONTROLS = 'a[href], button, input:not([type="hidden"]), select, textarea, [role="button"], [role="link"], [role="checkbox"]';
|
|
97
|
+
const ACTION_TIMEOUT_MS = 30_000;
|
|
98
|
+
/** Why a handover ended, for the journal, the log and the audit. */
|
|
99
|
+
export type HandoverEnd = 'finished' | 'stopped' | 'connector' | 'idle' | 'revoked' | 'closed' | 'failed' | 'shutdown' | 'deleted';
|
|
100
|
+
const HANDOVER_ENDS: Record<HandoverEnd, string> = {
|
|
101
|
+
finished: 'finished',
|
|
102
|
+
stopped: 'stopped with the kill switch',
|
|
103
|
+
connector: 'the browser connector was taken away',
|
|
104
|
+
idle: 'nothing was sent for too long, or it reached its time limit',
|
|
105
|
+
revoked: 'the device that drove it was revoked',
|
|
106
|
+
closed: 'its browser closed',
|
|
107
|
+
failed: 'its browser did not start',
|
|
108
|
+
shutdown: 'the server is stopping',
|
|
109
|
+
deleted: 'the Boton was deleted',
|
|
110
|
+
};
|
|
111
|
+
/** The most of an allowed request's answer that is passed to the page. */
|
|
112
|
+
const ANSWER_MAX = 20 * 1024 * 1024;
|
|
113
|
+
const JOURNAL_KEEP = 2000;
|
|
114
|
+
const DEFAULT_PATHS = ['/usr/bin/chromium', '/usr/bin/chromium-browser', '/usr/bin/google-chrome', '/usr/bin/google-chrome-stable', '/opt/google/chrome/chrome'];
|
|
115
|
+
export const CAPABILITY =
|
|
116
|
+
"Opens pages on the listed sites (loading them as any browser does, with GET requests), reads their text, links and controls, goes back, takes screenshots, clicks and types (never into password, code or card fields). Once it clicks or types on a page, nothing that page sends goes without your card: the request it makes (a form, a POST, or a GET) waits on a card that shows exactly that request, sent once only if you allow it while the Boton's step waits, and anything else the page sends is refused. It never signs in itself: when a step is yours (a sign-in, a code, a captcha), it asks in the chat and you tap Take over to do it in its browser (it can't use the browser until you hand it back, and keeps the sign-in after). It does not download or upload.";
|
|
117
|
+
const DESKTOP_WHY =
|
|
118
|
+
"Botons doesn't control this computer's screen, mouse or keyboard: Botons work in their own managed browser instead, which needs no display.";
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Why a step failed, from a fixed set of Botons' own words: never the error's text, which names the full address
|
|
122
|
+
* (with its query) and can carry what a site sent. Chromium's net:: codes are matched only to pick the words.
|
|
123
|
+
*/
|
|
124
|
+
export function failureOf(err: unknown): string {
|
|
125
|
+
const e = err as { name?: unknown; message?: unknown } | null;
|
|
126
|
+
const name = typeof e?.name === 'string' ? e.name : '';
|
|
127
|
+
const text = typeof e?.message === 'string' ? e.message : '';
|
|
128
|
+
if (name === 'TimeoutError' || /\bTimeout \d+ms exceeded\b/.test(text)) return `it took longer than ${ACTION_TIMEOUT_MS / 1000} seconds`;
|
|
129
|
+
const code = /\bnet::(ERR_[A-Z0-9_]+)\b/.exec(text)?.[1] ?? '';
|
|
130
|
+
if (/^[^\n]*\bDownload is starting\b/.test(text)) return 'it is a download, which the browser refuses';
|
|
131
|
+
if (code === 'ERR_BLOCKED_BY_CLIENT') return 'it led to an address the browser may not load (a site not listed, or not a read)';
|
|
132
|
+
if (code === 'ERR_NAME_NOT_RESOLVED') return "the site's name could not be found";
|
|
133
|
+
if (code === 'ERR_ABORTED') return 'loading stopped before the page arrived (a download, or another page opened instead)';
|
|
134
|
+
if (code === 'ERR_TOO_MANY_REDIRECTS') return 'the site redirected too many times';
|
|
135
|
+
if (code.startsWith('ERR_CERT_') || code.startsWith('ERR_SSL_')) return "the site's certificate was not accepted";
|
|
136
|
+
if (/^ERR_(CONNECTION|TUNNEL|PROXY|ADDRESS|NETWORK|INTERNET|EMPTY_RESPONSE|TIMED_OUT)/.test(code)) return 'the site could not be reached';
|
|
137
|
+
if (/Target (page, context or browser )?(has been )?closed|Browser has been closed/i.test(text)) return 'the browser closed';
|
|
138
|
+
return 'the browser could not finish it';
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const hostOfUrl = (url: string) => {
|
|
142
|
+
try {
|
|
143
|
+
return new URL(url).hostname.replace(/^\[|\]$/g, '').toLowerCase();
|
|
144
|
+
} catch {
|
|
145
|
+
return '';
|
|
146
|
+
}
|
|
147
|
+
};
|
|
148
|
+
|
|
149
|
+
const originOf = (url: string) => {
|
|
150
|
+
try {
|
|
151
|
+
return new URL(url).origin;
|
|
152
|
+
} catch {
|
|
153
|
+
return '';
|
|
154
|
+
}
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
/** The page's main navigation as it ended (after any redirect): its last request, and how it ended. */
|
|
158
|
+
export interface Navigation {
|
|
159
|
+
url: string;
|
|
160
|
+
/** It failed in Chromium (an error, no response). */
|
|
161
|
+
failed: boolean;
|
|
162
|
+
/** The response's x-botons-proxy header, if it had one. */
|
|
163
|
+
mark: string | null;
|
|
164
|
+
/** That last request: a block by the browser's routing is matched to it exactly. */
|
|
165
|
+
request: object | null;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* What Botons said of the page's own navigation, in the words the Boton is shown: its block ("Blocked by Botons: ..."), or
|
|
170
|
+
* its site not found or not reached (never called a block). Null when it was the site's own answer, or when Botons said
|
|
171
|
+
* nothing of it. Matched to where the navigation ended, so a redirect is judged by its destination. Provenance never
|
|
172
|
+
* rests on what a site can send: a block by the routing is that request's own; the proxy's mark counts on HTTP and
|
|
173
|
+
* HTTPS alike, since every request of this browser is answered by its proxy (its HTTPS ends there too), which takes
|
|
174
|
+
* the mark off every site's response; and a block of another request (a POST the page made) never stands for the
|
|
175
|
+
* navigation.
|
|
176
|
+
*/
|
|
177
|
+
export function navigationAnswer(nav: Navigation, s: Pick<Session, 'blocked' | 'unreachable'>): string | null {
|
|
178
|
+
const routed = nav.request ? s.blocked.findLast((b) => b.request === nav.request) : undefined;
|
|
179
|
+
if (routed) return `Blocked by Botons: ${routed.why}.`;
|
|
180
|
+
const kind = nav.failed ? 'error' : /^https?:/.test(originOf(nav.url)) ? nav.mark : null;
|
|
181
|
+
if (kind !== 'error' && kind !== 'blocked' && kind !== 'unreachable') return null;
|
|
182
|
+
const origin = originOf(nav.url);
|
|
183
|
+
const failed = kind !== 'blocked' && s.unreachable.findLast((u) => originOf(u.url) === origin);
|
|
184
|
+
if (failed) return failed.message;
|
|
185
|
+
const blocked = kind !== 'unreachable' && s.blocked.findLast((b) => !b.request && originOf(b.url) === origin);
|
|
186
|
+
if (blocked) return `Blocked by Botons: ${blocked.why}.`;
|
|
187
|
+
// The proxy's own answer, its record of it already full: said as what it was.
|
|
188
|
+
if (kind === 'blocked') return 'Blocked by Botons.';
|
|
189
|
+
if (kind === 'unreachable') return `Couldn\u2019t connect to ${hostOfUrl(nav.url)}.`;
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* The browser's own user agent with only its headless mark taken out (HeadlessChrome/... becomes Chrome/...): the real
|
|
195
|
+
* version and platform stay as the running Chromium reports them. Some sites refuse any agent that says it is headless.
|
|
196
|
+
*/
|
|
197
|
+
export function withoutHeadless(userAgent: string): string {
|
|
198
|
+
return userAgent.replace(/\bHeadlessChrome\//, 'Chrome/');
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
export type Playwright = typeof import('playwright-core');
|
|
202
|
+
type Context = import('playwright-core').BrowserContext;
|
|
203
|
+
type Page = import('playwright-core').Page;
|
|
204
|
+
type Cookie = Awaited<ReturnType<Context['cookies']>>[number];
|
|
205
|
+
|
|
206
|
+
type Route = import('playwright-core').Route;
|
|
207
|
+
type Request = import('playwright-core').Request;
|
|
208
|
+
|
|
209
|
+
/** A control a read listed: what the page said of it (its label is the site's text), found again by `index`. */
|
|
210
|
+
interface Control extends FieldInfo {
|
|
211
|
+
index: number;
|
|
212
|
+
kind: BrowserControl['kind'];
|
|
213
|
+
/** A link's address (else empty). */
|
|
214
|
+
href: string;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/** A click or typing in progress: while `open`, the first request of the page (to an allowed site) is held here, not refused. */
|
|
218
|
+
interface Op {
|
|
219
|
+
open: boolean;
|
|
220
|
+
/** The whole cookie jar as the step began (null if it could not be read): the one every later read must equal. */
|
|
221
|
+
cookies: Cookie[] | null;
|
|
222
|
+
held: { route: Route; request: Request; rank: number } | null;
|
|
223
|
+
onHeld: () => void;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Which request a card is for when a step makes several: a navigation (a form), then a script's, then anything else. */
|
|
227
|
+
const rankOf = (r: Request) => (r.isNavigationRequest() ? 3 : ['fetch', 'xhr', 'eventsource', 'ping'].includes(r.resourceType()) ? 2 : 1);
|
|
228
|
+
|
|
229
|
+
interface Session {
|
|
230
|
+
context: Context;
|
|
231
|
+
page: Page;
|
|
232
|
+
proxy: BrowserProxy;
|
|
233
|
+
/** What was refused in this step: by the proxy, or by the browser's routing (with that request). */
|
|
234
|
+
blocked: { url: string; why: string; request?: object }[];
|
|
235
|
+
/** Sites the proxy could not reach in this step (not found, no address answered): never Botons refusing them. */
|
|
236
|
+
unreachable: { url: string; host: string; message: string }[];
|
|
237
|
+
/** New each time Chromium starts: a card made for one browser never applies to the next. */
|
|
238
|
+
id: string;
|
|
239
|
+
/** The page's document: one more each time its main frame navigates, so refs and cards from before no longer apply. */
|
|
240
|
+
revision: number;
|
|
241
|
+
/** The controls the last read listed, and for which document. */
|
|
242
|
+
controls: { revision: number; list: Control[] } | null;
|
|
243
|
+
op: Op | null;
|
|
244
|
+
/** The page was clicked or typed on: nothing it sends leaves without a card any more (a new browser ends that). */
|
|
245
|
+
actedOn: boolean;
|
|
246
|
+
/** Some page of this browser was acted on: a request no page can be named for (a worker's) is refused from then on. */
|
|
247
|
+
everActed: boolean;
|
|
248
|
+
/** The main frame's addresses, for back from a page acted on (which opens the one before in a new browser). */
|
|
249
|
+
trail: string[];
|
|
250
|
+
/** Ends the wait of the card holding a request (the browser is closing). */
|
|
251
|
+
endWait: (() => void) | null;
|
|
252
|
+
/** Cuts the send of an allowed request (the browser is closing). */
|
|
253
|
+
sending: AbortController | null;
|
|
254
|
+
closed: boolean;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/** The owner's browser in a handover: run for the owner alone, never reached by the Boton. */
|
|
258
|
+
interface OwnerSession {
|
|
259
|
+
context: Context;
|
|
260
|
+
page: Page;
|
|
261
|
+
proxy: BrowserProxy;
|
|
262
|
+
/** The server's own DevTools session on that page, for the live view and a copy: never the owner's or the page's. */
|
|
263
|
+
cdp: import('playwright-core').CDPSession | null;
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
const sameHost = (a: string, b: string) => {
|
|
267
|
+
try {
|
|
268
|
+
return new URL(a).host === new URL(b).host;
|
|
269
|
+
} catch {
|
|
270
|
+
return false;
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
/** Where the live view goes: the driving device's open stream. */
|
|
275
|
+
export interface HandoverSink {
|
|
276
|
+
/** One frame (JPEG) of the page at `width` × `height` CSS pixels; resolves once it has gone, so frames never pile up. */
|
|
277
|
+
frame(jpeg: Buffer, width: number, height: number): Promise<void>;
|
|
278
|
+
notice(n: HandoverNotice): void;
|
|
279
|
+
end(): void;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/** In a world of the server's own (the page's script can't reach it): the owner's selection, never a password field's. */
|
|
283
|
+
const COPY_SELECTION = `(() => {
|
|
284
|
+
const a = document.activeElement;
|
|
285
|
+
if (a instanceof HTMLInputElement || a instanceof HTMLTextAreaElement) {
|
|
286
|
+
if (a instanceof HTMLInputElement && a.type === 'password') return '';
|
|
287
|
+
try { return a.value.slice(a.selectionStart ?? 0, a.selectionEnd ?? 0); } catch { return ''; }
|
|
288
|
+
}
|
|
289
|
+
return String(document.getSelection() ?? '');
|
|
290
|
+
})()`;
|
|
291
|
+
/** In the same world: whether what has focus takes text. Only true or false. */
|
|
292
|
+
const FOCUS_TAKES_TEXT = `(() => {
|
|
293
|
+
const a = document.activeElement;
|
|
294
|
+
if (!a) return false;
|
|
295
|
+
if (a instanceof HTMLIFrameElement) return true;
|
|
296
|
+
if (a instanceof HTMLElement && a.isContentEditable) return true;
|
|
297
|
+
if (a instanceof HTMLTextAreaElement) return !a.readOnly && !a.disabled;
|
|
298
|
+
if (a instanceof HTMLInputElement) return !a.readOnly && !a.disabled && !['button', 'submit', 'reset', 'checkbox', 'radio', 'file', 'image', 'range', 'color', 'hidden'].includes(a.type);
|
|
299
|
+
return false;
|
|
300
|
+
})()`;
|
|
301
|
+
|
|
302
|
+
/** A handover: from takeOver until giveBack, the Boton's browser is the owner's. */
|
|
303
|
+
interface Handover {
|
|
304
|
+
id: string;
|
|
305
|
+
/** The device that took it over: the only one that gets frames and sends input. */
|
|
306
|
+
deviceId: string | null;
|
|
307
|
+
deviceLabel: string;
|
|
308
|
+
startedAt: number;
|
|
309
|
+
/** The owner's last input or frame (for the idle limit). */
|
|
310
|
+
lastActive: number;
|
|
311
|
+
/** The owner's last click, key or typing: what the page sends within INTENT_MS of it may go. */
|
|
312
|
+
lastIntent: number;
|
|
313
|
+
session: Promise<OwnerSession>;
|
|
314
|
+
/** Its page once open, and that page's title as last read. */
|
|
315
|
+
view: Page | null;
|
|
316
|
+
title: string;
|
|
317
|
+
ended: boolean;
|
|
318
|
+
ending: Promise<void> | null;
|
|
319
|
+
timer: NodeJS.Timeout | null;
|
|
320
|
+
/** The owner's inputs, one at a time and in order. */
|
|
321
|
+
queue: Promise<unknown>;
|
|
322
|
+
framing: boolean;
|
|
323
|
+
lastFrame: number;
|
|
324
|
+
inputs: RateWindow;
|
|
325
|
+
/** The page's size (CSS pixels), following the owner's view. */
|
|
326
|
+
viewport: { width: number; height: number };
|
|
327
|
+
/** The live view's stream, while one is open (one at a time, the driving device's). */
|
|
328
|
+
sink: HandoverSink | null;
|
|
329
|
+
/** Stops the screencast to `sink`. */
|
|
330
|
+
stopCast: (() => Promise<void>) | null;
|
|
331
|
+
/** The Boton's page to go back to (in full), if it handed over from one: opened only when the owner asks (resume). */
|
|
332
|
+
resume: string | null;
|
|
333
|
+
/** Run once it has ended (stops watching the device for revocation). */
|
|
334
|
+
onEnd: (() => void)[];
|
|
335
|
+
/** Its browser's cookie jar as of the owner's last input: what comes back if that browser dies before it can be read. */
|
|
336
|
+
jar: Cookie[] | null;
|
|
337
|
+
/** It started with the Boton's whole jar: so its jar at the end is the whole jar, cookies it dropped included. */
|
|
338
|
+
seeded: boolean;
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
export interface BrowserDeps {
|
|
342
|
+
cfg: Config;
|
|
343
|
+
store: Store;
|
|
344
|
+
approvals: Approvals;
|
|
345
|
+
/** Whether a brain can run the browser (drivesBrowser: a CLI provider). */
|
|
346
|
+
usesConnectors: (provider: string) => boolean;
|
|
347
|
+
/** The Boton's access in its running turn, if any. */
|
|
348
|
+
turnAccess: (botId: string) => AccessLevel | null;
|
|
349
|
+
share: ShareFile;
|
|
350
|
+
/** The audit log, for a handover's start and end. */
|
|
351
|
+
audit?: Audit;
|
|
352
|
+
/** Calls `close` when the principal's device is revoked; returns how to stop watching. */
|
|
353
|
+
trackDevice?: (principal: Principal | null, close: () => void) => () => void;
|
|
354
|
+
/** The handoff cards in the chat (browser-handoff.ts): told when the owner takes the browser over and when that ends. */
|
|
355
|
+
handoffs?: BrowserHandoffs;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
export interface Availability {
|
|
359
|
+
available: boolean;
|
|
360
|
+
why: string | null;
|
|
361
|
+
executable: string | null;
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
export class Browser implements BrowserHolds {
|
|
365
|
+
#deps: BrowserDeps;
|
|
366
|
+
/** Tests only: the certificate of their own HTTPS site, trusted by the browser's proxy and by the send of an allowed request. */
|
|
367
|
+
testTrust: { cert: string; spki: string } | null = null;
|
|
368
|
+
/** Tests only: the Boton's browser's cookies for `url` now (none without a browser). */
|
|
369
|
+
async testCookies(botId: string, url: string): Promise<{ name: string; value: string }[]> {
|
|
370
|
+
const s = await this.#sessions.get(botId)?.catch(() => null);
|
|
371
|
+
return s ? s.context.cookies(url) : [];
|
|
372
|
+
}
|
|
373
|
+
/** Tests only: the Boton's browser's cookie jar, to change between the gates below (nothing outside tests sets this). */
|
|
374
|
+
async testJar(botId: string): Promise<Context | null> {
|
|
375
|
+
return (await this.#sessions.get(botId)?.catch(() => null))?.context ?? null;
|
|
376
|
+
}
|
|
377
|
+
/**
|
|
378
|
+
* Tests only: awaited at fixed points of a click or typing, so a test can change the jar exactly there: `acting` once
|
|
379
|
+
* the step's jar is read and before the page acts (so before Chromium writes any Cookie header), `held` once a held
|
|
380
|
+
* request's headers are read and before the jar is read again, `card` once its card exists and waits.
|
|
381
|
+
*/
|
|
382
|
+
testGate: ((at: 'acting' | 'held' | 'card', botId: string) => Promise<void>) | null = null;
|
|
383
|
+
/** Tests only: shorter handover limits than INTENT_MS and IDLE_MS. */
|
|
384
|
+
testHandover: { intentMs?: number; idleMs?: number } | null = null;
|
|
385
|
+
/** Tests only: the owner's handover browser's cookie jar (null without one). */
|
|
386
|
+
async testOwnerJar(botId: string): Promise<Context | null> {
|
|
387
|
+
return (await this.#handovers.get(botId)?.session.catch(() => null))?.context ?? null;
|
|
388
|
+
}
|
|
389
|
+
/** Handovers under way, by Boton: while one is here, the Boton's browser is the owner's. */
|
|
390
|
+
#handovers = new Map<string, Handover>();
|
|
391
|
+
/** Each Boton's cookie jar as its last browser closed, put back when the next one opens (in memory only). */
|
|
392
|
+
#carry = new Map<string, Cookie[]>();
|
|
393
|
+
/**
|
|
394
|
+
* Where a Boton's browser was when it handed over (browser_handoff), in full (its query too): in memory only, never in
|
|
395
|
+
* a card, the journal or a log, so the owner's Open goes back to exactly that page (a sign-in's challenge lives in its
|
|
396
|
+
* address). Gone once a handover ends.
|
|
397
|
+
*/
|
|
398
|
+
#handoffPage = new Map<string, string>();
|
|
399
|
+
/** Cards waiting on a request the browser holds, by approval id: how to wake the step that waits. */
|
|
400
|
+
#waits = new Map<string, () => void>();
|
|
401
|
+
readonly dir: string;
|
|
402
|
+
#sessions = new Map<string, Promise<Session>>();
|
|
403
|
+
/** The token of the turn that was given the browser, by Boton: only it may drive it. */
|
|
404
|
+
#turns = new Map<string, string>();
|
|
405
|
+
/** One action at a time per Boton. */
|
|
406
|
+
#queues = new Map<string, Promise<unknown>>();
|
|
407
|
+
#availability: Promise<Availability> | null = null;
|
|
408
|
+
#probe: Promise<Availability> | null = null;
|
|
409
|
+
#known: Availability | null = null;
|
|
410
|
+
#closed = false;
|
|
411
|
+
|
|
412
|
+
constructor(deps: BrowserDeps) {
|
|
413
|
+
this.#deps = deps;
|
|
414
|
+
this.dir = join(deps.cfg.home, 'browser');
|
|
415
|
+
// Before any turn: privatePaths() lists the data folder's entries, so a Workspace Boton can't read the profiles.
|
|
416
|
+
mkdirSync(this.dir, { recursive: true, mode: 0o700 });
|
|
417
|
+
chmodSync(this.dir, 0o700);
|
|
418
|
+
// A handover's profile is never left behind: one still here is from a server that stopped without ending it.
|
|
419
|
+
for (const name of readdirSync(this.dir)) this.#dropOwnerProfile(name);
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/** Whether a step still waits on this card, holding its request (approvals.ts asks before taking a decision). */
|
|
423
|
+
holds(approvalId: string): boolean {
|
|
424
|
+
return this.#waits.has(approvalId);
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
/** The card was decided or withdrawn: the step that waits on it goes on. */
|
|
428
|
+
decided(approvalId: string): void {
|
|
429
|
+
this.#waits.get(approvalId)?.();
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// ------------------------------------------------------------- settings
|
|
433
|
+
|
|
434
|
+
#key = (botId: string) => `browser:${botId}`;
|
|
435
|
+
|
|
436
|
+
policy(botId: string): BrowserPolicy {
|
|
437
|
+
const row = this.#deps.store.db.prepare('select value from settings where key = ?').get(this.#key(botId)) as { value: string } | undefined;
|
|
438
|
+
return parsePolicy(row?.value);
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
#access(botId: string): AccessLevel {
|
|
442
|
+
const bot = this.#deps.store.getBot(botId);
|
|
443
|
+
const turn = this.#deps.turnAccess(botId);
|
|
444
|
+
return turn === 'workspace' || bot.access === 'workspace' ? 'workspace' : 'host';
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/** The owner gave it the browser connector, and its brain can use connectors. */
|
|
448
|
+
enabled(botId: string): boolean {
|
|
449
|
+
if (this.#closed || !this.#deps.store.hasBot(botId)) return false;
|
|
450
|
+
const bot = this.#deps.store.getBot(botId);
|
|
451
|
+
return bot.connections.includes(BROWSER_CONNECTOR) && this.#deps.usesConnectors(bot.provider);
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/** The sites enforced now: the saved ones within the Boton's access. */
|
|
455
|
+
sites(botId: string): string[] {
|
|
456
|
+
if (!this.#deps.store.hasBot(botId)) return [];
|
|
457
|
+
return effectiveSites(this.policy(botId).sites, this.#access(botId), this.#deps.cfg.browser.allowPrivate);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
/** What the proxy lets through: nothing unless it is on and a turn holds it. */
|
|
461
|
+
#proxyPolicy(botId: string): ProxyPolicy {
|
|
462
|
+
if (!this.enabled(botId) || !this.#turns.has(botId)) return { enabled: false, sites: [] };
|
|
463
|
+
return { enabled: true, sites: this.sites(botId) };
|
|
464
|
+
}
|
|
465
|
+
|
|
466
|
+
async setPolicy(botId: string, input: BrowserPolicyInput): Promise<BrowserStatus> {
|
|
467
|
+
const bot = this.#deps.store.getBot(botId);
|
|
468
|
+
const next = applyPolicy(this.policy(botId), input, bot.access, this.#deps.cfg.browser.allowPrivate);
|
|
469
|
+
this.#deps.store.db.prepare('insert into settings(key, value) values (?, ?) on conflict(key) do update set value = excluded.value').run(this.#key(botId), JSON.stringify(next));
|
|
470
|
+
log.info('browser sites changed', { botId, sites: next.sites.length });
|
|
471
|
+
await this.refresh(botId);
|
|
472
|
+
return this.status(botId);
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
/** A new Boton's sites from a configuration file (portable.ts), checked like the owner's, inside the caller's transaction. */
|
|
476
|
+
importSites(botId: string, sites: string[], access: AccessLevel): void {
|
|
477
|
+
if (!sites.length) return;
|
|
478
|
+
const next = applyPolicy({ sites: [], updatedAt: null }, { sites }, access, this.#deps.cfg.browser.allowPrivate);
|
|
479
|
+
this.#deps.store.db.prepare('insert into settings(key, value) values (?, ?) on conflict(key) do update set value = excluded.value').run(this.#key(botId), JSON.stringify(next));
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* After anything that changes what it may do (its connectors, access, brain or sites): closed if it lost the
|
|
484
|
+
* browser, else tunnels to sites it may no longer reach are cut.
|
|
485
|
+
*/
|
|
486
|
+
async refresh(botId: string, why = 'The browser connector was taken away: the browser is closed.', end: HandoverEnd = 'connector', by: Principal | null = null): Promise<void> {
|
|
487
|
+
const h = this.#handovers.get(botId);
|
|
488
|
+
if (!this.enabled(botId)) this.#deps.handoffs?.closeWaiting(botId, 'its browser was turned off');
|
|
489
|
+
if (h && !this.enabled(botId)) await this.giveBack(botId, null, end, by);
|
|
490
|
+
else if (h) (await h.session.catch(() => null))?.proxy.recheck();
|
|
491
|
+
if (!this.#sessions.has(botId)) return;
|
|
492
|
+
if (!this.enabled(botId)) {
|
|
493
|
+
await this.#close(botId);
|
|
494
|
+
this.#journal(botId, { action: 'stopped', url: '', outcome: 'done', detail: why });
|
|
495
|
+
return;
|
|
496
|
+
}
|
|
497
|
+
const s = await this.#sessions.get(botId)!.catch(() => null);
|
|
498
|
+
s?.proxy.recheck();
|
|
499
|
+
}
|
|
500
|
+
|
|
501
|
+
/** The Boton is gone: its browser, profile, journal and settings too. */
|
|
502
|
+
async forget(botId: string): Promise<void> {
|
|
503
|
+
this.#turns.delete(botId);
|
|
504
|
+
await this.giveBack(botId, null, 'deleted');
|
|
505
|
+
await this.#close(botId);
|
|
506
|
+
this.#carry.delete(botId);
|
|
507
|
+
this.#handoffPage.delete(botId);
|
|
508
|
+
this.#deps.store.db.prepare('delete from settings where key = ?').run(this.#key(botId));
|
|
509
|
+
rmSync(this.#botDir(botId), { recursive: true, force: true });
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/**
|
|
513
|
+
* Whether this server can run the browser: its library and a Chromium (cheap, at start), and with `probe`, that the
|
|
514
|
+
* Chromium really starts the way turns start it (once; a container that can't start its sandbox fails here).
|
|
515
|
+
*/
|
|
516
|
+
availability(probe = false): Promise<Availability> {
|
|
517
|
+
if (probe) {
|
|
518
|
+
this.#probe ??= this.availability().then(async (a) => {
|
|
519
|
+
if (!a.available) return a;
|
|
520
|
+
const pw = (await this.#playwright())!;
|
|
521
|
+
try {
|
|
522
|
+
const b = await pw.chromium.launch({ executablePath: a.executable!, headless: true, chromiumSandbox: this.#deps.cfg.browser.sandbox, timeout: ACTION_TIMEOUT_MS });
|
|
523
|
+
await b.close();
|
|
524
|
+
return a;
|
|
525
|
+
} catch (err) {
|
|
526
|
+
const lines = (err as Error).message.split('\n').map((l) => l.trim()).filter(Boolean);
|
|
527
|
+
const first = lines.find((l) => /sandbox|namespace/i.test(l)) ?? lines[0] ?? '';
|
|
528
|
+
const why = `${a.executable} did not start (${first.slice(0, 200)})${/sandbox|namespace/i.test(first) ? ": Chromium's sandbox can't start here (in a container, see Browser in docs/DOCKER.md)" : ''}.`;
|
|
529
|
+
log.warn('the managed browser did not start', { why });
|
|
530
|
+
const out = { available: false, why, executable: a.executable };
|
|
531
|
+
this.#known = out;
|
|
532
|
+
return out;
|
|
533
|
+
}
|
|
534
|
+
});
|
|
535
|
+
return this.#probe;
|
|
536
|
+
}
|
|
537
|
+
this.#availability ??= findBrowser(this.#deps.cfg.browser.path).then((a) => {
|
|
538
|
+
this.#known ??= a;
|
|
539
|
+
return a;
|
|
540
|
+
});
|
|
541
|
+
return this.#availability;
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
/** What availability() last found, without waiting (null before it has looked). */
|
|
545
|
+
get known(): Availability | null {
|
|
546
|
+
return this.#known;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
#headed(): boolean {
|
|
550
|
+
return this.#deps.cfg.browser.headed && !!(process.env.DISPLAY || process.env.WAYLAND_DISPLAY);
|
|
551
|
+
}
|
|
552
|
+
|
|
553
|
+
async status(botId: string, deviceId: string | null = null): Promise<BrowserStatus> {
|
|
554
|
+
const bot = this.#deps.store.getBot(botId);
|
|
555
|
+
const a = await this.availability(true);
|
|
556
|
+
return {
|
|
557
|
+
botId,
|
|
558
|
+
connectorId: BROWSER_CONNECTOR,
|
|
559
|
+
enabled: this.enabled(botId),
|
|
560
|
+
policy: this.policy(botId),
|
|
561
|
+
available: a.available,
|
|
562
|
+
why: a.why,
|
|
563
|
+
executable: a.executable,
|
|
564
|
+
sandbox: this.#deps.cfg.browser.sandbox,
|
|
565
|
+
headed: this.#headed(),
|
|
566
|
+
running: this.#sessions.has(botId),
|
|
567
|
+
workspace: bot.access === 'workspace',
|
|
568
|
+
capability: CAPABILITY,
|
|
569
|
+
desktop: DESKTOP_WHY,
|
|
570
|
+
handover: this.handover(botId, deviceId),
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/** The connector's MCP server, which the engine gives the turn's API address and token. */
|
|
575
|
+
server(): TurnMcpServer {
|
|
576
|
+
return {
|
|
577
|
+
name: BROWSER_SERVER,
|
|
578
|
+
kind: 'stdio',
|
|
579
|
+
command: process.execPath,
|
|
580
|
+
args: ['--no-warnings', resolve(this.#deps.cfg.botonBinDir, '../src/cli/botons.ts'), 'browser-mcp'],
|
|
581
|
+
env: {},
|
|
582
|
+
// A click may wait for the owner's card: as long as an approval waits, and the send after.
|
|
583
|
+
toolTimeoutSec: Math.ceil(this.#deps.cfg.approvalWaitMs / 1000) + 2 * (ACTION_TIMEOUT_MS / 1000) + 30,
|
|
584
|
+
};
|
|
585
|
+
}
|
|
586
|
+
|
|
587
|
+
/** A turn that was given the browser starts: only its token drives it. */
|
|
588
|
+
beginTurn(botId: string, tokenId: string): void {
|
|
589
|
+
this.#turns.set(botId, tokenId);
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
/** That turn ended: its browser closes (the profile, with its cookies, stays). */
|
|
593
|
+
async endTurn(botId: string, tokenId: string): Promise<void> {
|
|
594
|
+
if (this.#turns.get(botId) !== tokenId) return;
|
|
595
|
+
this.#turns.delete(botId);
|
|
596
|
+
await this.#close(botId);
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
// ------------------------------------------------------------- journal
|
|
600
|
+
|
|
601
|
+
#botDir = (botId: string) => join(this.dir, botId.replace(/[^A-Za-z0-9_-]/g, '_'));
|
|
602
|
+
/** The owner's handover profile (and its Chromium's home), beside the Boton's: `browser/<boton>/handover`. */
|
|
603
|
+
#ownerDir = (botId: string) => join(this.#botDir(botId), 'handover');
|
|
604
|
+
|
|
605
|
+
/** Deletes a handover's profile, whole (its Chromium has exited). */
|
|
606
|
+
#dropOwnerProfile(dirName: string): void {
|
|
607
|
+
const dir = join(this.dir, dirName, 'handover');
|
|
608
|
+
try {
|
|
609
|
+
rmSync(dir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
|
|
610
|
+
} catch {}
|
|
611
|
+
if (existsSync(dir)) log.warn('could not delete a handover browser profile', { dir });
|
|
612
|
+
}
|
|
613
|
+
|
|
614
|
+
#journal(botId: string, e: Omit<BrowserJournalEntry, 'at'>): void {
|
|
615
|
+
try {
|
|
616
|
+
const dir = this.#botDir(botId);
|
|
617
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
618
|
+
const file = join(dir, 'journal.jsonl');
|
|
619
|
+
appendFileSync(file, `${JSON.stringify({ at: new Date().toISOString(), ...e, detail: e.detail.slice(0, 300) })}\n`, { mode: 0o600 });
|
|
620
|
+
const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean);
|
|
621
|
+
if (lines.length > JOURNAL_KEEP) writeFileSync(file, `${lines.slice(-JOURNAL_KEEP / 2).join('\n')}\n`, { mode: 0o600 });
|
|
622
|
+
} catch (err) {
|
|
623
|
+
log.warn('could not write to the browser journal', { botId, err });
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
journal(botId: string, limit = 200): BrowserJournalEntry[] {
|
|
628
|
+
this.#deps.store.getBot(botId);
|
|
629
|
+
const file = join(this.#botDir(botId), 'journal.jsonl');
|
|
630
|
+
if (!existsSync(file)) return [];
|
|
631
|
+
const out: BrowserJournalEntry[] = [];
|
|
632
|
+
for (const line of readFileSync(file, 'utf8').split('\n').filter(Boolean).slice(-limit)) {
|
|
633
|
+
try {
|
|
634
|
+
out.push(JSON.parse(line));
|
|
635
|
+
} catch {}
|
|
636
|
+
}
|
|
637
|
+
return out;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
// ------------------------------------------------------------- the browser process
|
|
641
|
+
|
|
642
|
+
#playwright(): Promise<Playwright | null> {
|
|
643
|
+
return loadPlaywright();
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
#session(botId: string): Promise<Session> {
|
|
647
|
+
return this.#sessions.get(botId) ?? this.#start(botId, this.#launch(botId));
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
#start(botId: string, starting: Promise<Session>): Promise<Session> {
|
|
651
|
+
this.#sessions.set(botId, starting);
|
|
652
|
+
const forget = () => {
|
|
653
|
+
if (this.#sessions.get(botId) === starting) this.#sessions.delete(botId);
|
|
654
|
+
};
|
|
655
|
+
// Chromium gone on its own (it crashed, or was killed): the next step starts it again.
|
|
656
|
+
starting.then((s) => s.context.on('close', forget), forget);
|
|
657
|
+
return starting;
|
|
658
|
+
}
|
|
659
|
+
|
|
660
|
+
/**
|
|
661
|
+
* In place of a browser whose page was acted on: that one ends (its Chromium exits, its proxy, still sealed, closes)
|
|
662
|
+
* before another starts on the same profile, with the cookies it had, the same address trail and the same list of
|
|
663
|
+
* what was blocked in this step. Whatever the old page still had on its way can go nowhere: the proxy it could
|
|
664
|
+
* reach refuses everything until it is gone, and nothing listens on its port after.
|
|
665
|
+
*/
|
|
666
|
+
#replace(botId: string, old: Session): Promise<Session> {
|
|
667
|
+
return this.#start(
|
|
668
|
+
botId,
|
|
669
|
+
(async () => {
|
|
670
|
+
const cookies = await old.context.cookies().catch(() => []);
|
|
671
|
+
old.closed = true;
|
|
672
|
+
old.endWait?.();
|
|
673
|
+
old.sending?.abort();
|
|
674
|
+
old.proxy.seal(true);
|
|
675
|
+
await old.context.close().catch(() => {});
|
|
676
|
+
await old.proxy.close();
|
|
677
|
+
return this.#launch(botId, { blocked: old.blocked, unreachable: old.unreachable, trail: old.trail, cookies });
|
|
678
|
+
})(),
|
|
679
|
+
);
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
async #launch(botId: string, carry?: { blocked: Session['blocked']; unreachable: Session['unreachable']; trail: string[]; cookies: Cookie[] }): Promise<Session> {
|
|
683
|
+
const pw = await this.#playwright();
|
|
684
|
+
const a = await this.availability();
|
|
685
|
+
if (!pw || !a.executable) throw new HttpError(503, a.why ?? 'No browser on this server');
|
|
686
|
+
const profile = join(this.#botDir(botId), 'profile');
|
|
687
|
+
mkdirSync(profile, { recursive: true, mode: 0o700 });
|
|
688
|
+
const blocked: Session['blocked'] = carry?.blocked ?? [];
|
|
689
|
+
const note = (url: string, why: string, request?: Request) => {
|
|
690
|
+
if (blocked.length < 50) blocked.push({ url: displayUrl(url), why, ...(request ? { request } : {}) });
|
|
691
|
+
this.#journal(botId, { action: 'blocked', url: displayUrl(url), outcome: 'refused', detail: why });
|
|
692
|
+
};
|
|
693
|
+
const unreachable: Session['unreachable'] = carry?.unreachable ?? [];
|
|
694
|
+
const noted = (url: string, message: string) => {
|
|
695
|
+
if (unreachable.length < 50) unreachable.push({ url: displayUrl(url), host: hostOfUrl(url), message });
|
|
696
|
+
this.#journal(botId, { action: 'unreachable', url: displayUrl(url), outcome: 'refused', detail: message });
|
|
697
|
+
};
|
|
698
|
+
let started: Session | null = null;
|
|
699
|
+
// The owner has it: no browser of the Boton's opens on the profile until they finish.
|
|
700
|
+
if (this.#handovers.has(botId)) throw new HttpError(409, `${this.#deps.store.getSettings().ownerName} has the browser now.`);
|
|
701
|
+
const { context, page, proxy } = await openGuarded(pw, profile, {
|
|
702
|
+
executable: a.executable,
|
|
703
|
+
sandbox: this.#deps.cfg.browser.sandbox,
|
|
704
|
+
headless: !this.#headed(),
|
|
705
|
+
policy: () => this.#proxyPolicy(botId),
|
|
706
|
+
note,
|
|
707
|
+
onUnreachable: noted,
|
|
708
|
+
// Its HTTPS ends at its proxy (openGuarded), which refuses every write it sees there (this policy never allows
|
|
709
|
+
// one): a request Playwright's routing never sees (a beacon or keepalive fetch as a document unloads) is judged
|
|
710
|
+
// too. Only the send of a card's request, by a pass, gets a tunnel to the site itself.
|
|
711
|
+
trustCa: this.testTrust ? [this.testTrust.cert] : undefined,
|
|
712
|
+
gate: (route, request, write) => {
|
|
713
|
+
const s = started;
|
|
714
|
+
if (!s || s.closed) return 'the browser is closing';
|
|
715
|
+
let page: Page | null = null;
|
|
716
|
+
try {
|
|
717
|
+
page = request.frame().page();
|
|
718
|
+
} catch {}
|
|
719
|
+
if (!page) return s.everActed ? 'no page of this browser made it (a worker), after a page was acted on' : write ? null : 'pass';
|
|
720
|
+
if (page !== s.page) return 'it came from a page the browser has left';
|
|
721
|
+
const op = s.op;
|
|
722
|
+
if (op?.open) {
|
|
723
|
+
// While a click or typing runs: one request for the card (a form before a script's, a script's before the rest).
|
|
724
|
+
const rank = rankOf(request);
|
|
725
|
+
if (op.held) {
|
|
726
|
+
if (rank <= op.held.rank) return 'the step had already made a request, which waits on the card';
|
|
727
|
+
const before = op.held;
|
|
728
|
+
note(before.request.url(), 'the step then made another request, which goes to the card instead', before.request);
|
|
729
|
+
void before.route.abort('blockedbyclient').catch(() => {});
|
|
730
|
+
}
|
|
731
|
+
op.held = { route, request, rank };
|
|
732
|
+
op.onHeld();
|
|
733
|
+
return 'held';
|
|
734
|
+
}
|
|
735
|
+
if (s.actedOn) return 'the page was clicked or typed on, so nothing more it sends leaves without a card (open an address to go on)';
|
|
736
|
+
return write ? null : 'pass';
|
|
737
|
+
},
|
|
738
|
+
onDownload: (url) => this.#journal(botId, { action: 'download', url: displayUrl(url), outcome: 'refused', detail: 'Downloads are refused.' }),
|
|
739
|
+
}).catch((err) => {
|
|
740
|
+
const first = (err as Error).message.split('\n').find((l) => l.trim()) ?? '';
|
|
741
|
+
log.error('the browser did not start', { botId, err });
|
|
742
|
+
throw new HttpError(503, `The browser did not start: ${first.slice(0, 200)}`);
|
|
743
|
+
});
|
|
744
|
+
const s: Session = {
|
|
745
|
+
context,
|
|
746
|
+
page,
|
|
747
|
+
proxy,
|
|
748
|
+
blocked,
|
|
749
|
+
unreachable,
|
|
750
|
+
id: randomBytes(12).toString('base64url'),
|
|
751
|
+
revision: 0,
|
|
752
|
+
controls: null,
|
|
753
|
+
op: null,
|
|
754
|
+
actedOn: false,
|
|
755
|
+
everActed: false,
|
|
756
|
+
trail: carry?.trail ?? [],
|
|
757
|
+
endWait: null,
|
|
758
|
+
sending: null,
|
|
759
|
+
closed: false,
|
|
760
|
+
};
|
|
761
|
+
// The cookies the browser before it had (session ones too, which a profile doesn't keep across a restart): the one
|
|
762
|
+
// it replaces, else the last one to close (the owner's, after a sign-in).
|
|
763
|
+
const cookies = carry?.cookies ?? this.#carry.get(botId) ?? [];
|
|
764
|
+
if (cookies.length) await context.addCookies(cookies).catch(() => {});
|
|
765
|
+
started = s;
|
|
766
|
+
watchPage(s, page);
|
|
767
|
+
context.on('close', () => {
|
|
768
|
+
s.closed = true;
|
|
769
|
+
s.endWait?.();
|
|
770
|
+
s.sending?.abort();
|
|
771
|
+
});
|
|
772
|
+
log.info('browser started', { botId, headless: !this.#headed(), sandbox: this.#deps.cfg.browser.sandbox });
|
|
773
|
+
return s;
|
|
774
|
+
}
|
|
775
|
+
|
|
776
|
+
async #close(botId: string): Promise<void> {
|
|
777
|
+
const open = this.#sessions.get(botId);
|
|
778
|
+
if (!open) return;
|
|
779
|
+
this.#sessions.delete(botId);
|
|
780
|
+
try {
|
|
781
|
+
const s = await open;
|
|
782
|
+
// A card waiting on a request of this browser is withdrawn, and the request never sent.
|
|
783
|
+
s.closed = true;
|
|
784
|
+
s.endWait?.();
|
|
785
|
+
s.sending?.abort();
|
|
786
|
+
s.proxy.seal(true);
|
|
787
|
+
await this.#keep(botId, s.context);
|
|
788
|
+
await s.context.close().catch(() => {});
|
|
789
|
+
await s.proxy.close();
|
|
790
|
+
log.info('browser closed', { botId });
|
|
791
|
+
} catch {}
|
|
792
|
+
}
|
|
793
|
+
|
|
794
|
+
/** The jar of a browser about to close, for the next one to open with (a profile keeps no session cookies). */
|
|
795
|
+
async #keep(botId: string, context: Context): Promise<void> {
|
|
796
|
+
const cookies = await context.cookies().catch(() => null);
|
|
797
|
+
if (cookies) this.#carry.set(botId, cookies);
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
/**
|
|
801
|
+
* The Boton's profile opened by itself, with nothing let out (its proxy is off) and no page loaded, for `fn` to read
|
|
802
|
+
* or set its cookie jar. Only while no browser of the Boton's is open: in a handover, which keeps any from opening.
|
|
803
|
+
*/
|
|
804
|
+
async #withProfile<T>(botId: string, fn: (context: Context) => Promise<T>): Promise<T> {
|
|
805
|
+
const pw = await this.#playwright();
|
|
806
|
+
const a = await this.availability();
|
|
807
|
+
if (!pw || !a.executable) throw new Error('no browser');
|
|
808
|
+
const profile = join(this.#botDir(botId), 'profile');
|
|
809
|
+
mkdirSync(profile, { recursive: true, mode: 0o700 });
|
|
810
|
+
const { context, proxy } = await openGuarded(pw, profile, {
|
|
811
|
+
executable: a.executable,
|
|
812
|
+
sandbox: this.#deps.cfg.browser.sandbox,
|
|
813
|
+
headless: true,
|
|
814
|
+
policy: () => ({ enabled: false, sites: [] }),
|
|
815
|
+
note: () => {},
|
|
816
|
+
});
|
|
817
|
+
try {
|
|
818
|
+
proxy.seal(true);
|
|
819
|
+
return await fn(context);
|
|
820
|
+
} finally {
|
|
821
|
+
await context.close().catch(() => {});
|
|
822
|
+
await proxy.close();
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
/** The Boton's whole jar, for a handover to start with: the last browser's, else its profile's (null if unread). */
|
|
827
|
+
async #jarFor(botId: string): Promise<Cookie[] | null> {
|
|
828
|
+
const kept = this.#carry.get(botId);
|
|
829
|
+
if (kept) return kept;
|
|
830
|
+
return this.#withProfile(botId, (c) => c.cookies()).catch((err) => {
|
|
831
|
+
log.warn("could not read the browser profile's cookies", { botId, err });
|
|
832
|
+
return null;
|
|
833
|
+
});
|
|
834
|
+
}
|
|
835
|
+
|
|
836
|
+
/**
|
|
837
|
+
* A handover's jar, back to the Boton: in memory for its next browser, and into its profile so a restart keeps the
|
|
838
|
+
* sign-in. `whole` (the handover started with the Boton's whole jar): cookies the site dropped go from the profile
|
|
839
|
+
* too; else they are only added, never removed.
|
|
840
|
+
*/
|
|
841
|
+
async #giveJar(botId: string, jar: Cookie[], whole: boolean): Promise<void> {
|
|
842
|
+
this.#carry.set(botId, jar);
|
|
843
|
+
await this.#withProfile(botId, async (c) => {
|
|
844
|
+
if (jar.length) await c.addCookies(jar);
|
|
845
|
+
if (!whole) return;
|
|
846
|
+
// Each by its whole identity, partition included (clearCookies can't name one): expired in place, as it does.
|
|
847
|
+
const stale = staleCookies(await c.cookies(), jar);
|
|
848
|
+
if (stale.length) await c.addCookies(stale.map((k) => ({ ...k, expires: 0 })));
|
|
849
|
+
}).catch((err) => log.warn("could not save the handover's cookies to the browser profile", { botId, err }));
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
async closeAll(): Promise<void> {
|
|
853
|
+
this.#closed = true;
|
|
854
|
+
this.#turns.clear();
|
|
855
|
+
await Promise.all([...this.#handovers.keys()].map((id) => this.giveBack(id, null, 'shutdown')));
|
|
856
|
+
await Promise.all([...this.#sessions.keys()].map((id) => this.#close(id)));
|
|
857
|
+
}
|
|
858
|
+
|
|
859
|
+
// ------------------------------------------------------------- the owner's handover
|
|
860
|
+
|
|
861
|
+
/** The handover under way, as `deviceId` sees it (null: none). */
|
|
862
|
+
handover(botId: string, deviceId: string | null = null): BrowserHandover | null {
|
|
863
|
+
const h = this.#handovers.get(botId);
|
|
864
|
+
if (!h || h.ended) return null;
|
|
865
|
+
return {
|
|
866
|
+
id: h.id,
|
|
867
|
+
startedAt: new Date(h.startedAt).toISOString(),
|
|
868
|
+
byThisDevice: h.deviceId === deviceId,
|
|
869
|
+
deviceLabel: h.deviceLabel,
|
|
870
|
+
page: h.view && /^https?:/.test(h.view.url()) ? { url: displayUrl(h.view.url()), title: h.title } : null,
|
|
871
|
+
idleEndsAt: new Date(Math.min(h.lastActive + this.#idleMs(), h.startedAt + MAX_HANDOVER_MS)).toISOString(),
|
|
872
|
+
viewport: { ...h.viewport },
|
|
873
|
+
resume: h.resume ? displayUrl(h.resume) : null,
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
#idleMs = () => this.testHandover?.idleMs ?? IDLE_MS;
|
|
878
|
+
#intentMs = () => this.testHandover?.intentMs ?? INTENT_MS;
|
|
879
|
+
|
|
880
|
+
/** What the proxy lets the owner's browser reach: nothing once the handover ends, the Boton's sites until then. */
|
|
881
|
+
#ownerPolicy(botId: string, h: Handover): ProxyPolicy {
|
|
882
|
+
if (h.ended || this.#handovers.get(botId) !== h || !this.enabled(botId)) return { enabled: false, sites: [] };
|
|
883
|
+
return { enabled: true, sites: this.sites(botId), writes: Date.now() - h.lastIntent < this.#intentMs() };
|
|
884
|
+
}
|
|
885
|
+
|
|
886
|
+
/** The handover `deviceId` drives, or why not (409: none, or not this one; 403: another device's). */
|
|
887
|
+
#driven(botId: string, deviceId: string | null, id: string | null): Handover {
|
|
888
|
+
this.#deps.store.getBot(botId);
|
|
889
|
+
const h = this.#handovers.get(botId);
|
|
890
|
+
if (!h || h.ended || (id !== null && h.id !== id)) throw new HttpError(409, 'That handover has ended: take the browser over again to go on.');
|
|
891
|
+
if (h.deviceId !== deviceId) throw new HttpError(403, `The browser is being driven from ${h.deviceLabel}: only that device sees and drives it (any device can end it).`);
|
|
892
|
+
return h;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
/** The owner did something: the idle limit starts again (never past the handover's time limit). */
|
|
896
|
+
#active(botId: string, h: Handover): void {
|
|
897
|
+
h.lastActive = Date.now();
|
|
898
|
+
if (h.timer) clearTimeout(h.timer);
|
|
899
|
+
const left = Math.min(this.#idleMs(), h.startedAt + MAX_HANDOVER_MS - Date.now());
|
|
900
|
+
h.timer = setTimeout(() => void this.giveBack(botId, h.id, 'idle'), Math.max(0, left));
|
|
901
|
+
h.timer.unref?.();
|
|
902
|
+
}
|
|
903
|
+
|
|
904
|
+
/**
|
|
905
|
+
* The owner (`by`, a paired device) takes the Boton's browser over. The Boton's browser closes first (a card it
|
|
906
|
+
* waits on is withdrawn, its request never sent), and while the handover lasts it opens none: its tools are refused.
|
|
907
|
+
* Then a browser of the owner's own opens on the same profile with the same cookies, on a blank page (nothing is
|
|
908
|
+
* opened for the owner: they choose the address). The same device asking again gets the same handover.
|
|
909
|
+
*/
|
|
910
|
+
async takeOver(botId: string, by: Principal | null, start?: BrowserHandoverStart | null): Promise<BrowserHandover> {
|
|
911
|
+
const owner = this.#deps.store.getSettings().ownerName;
|
|
912
|
+
const bot = this.#deps.store.getBot(botId);
|
|
913
|
+
if (by?.kind === 'boton') throw new HttpError(403, `Only ${owner}'s paired devices can take a Boton's browser over.`);
|
|
914
|
+
const deviceId = by?.kind === 'device' ? by.deviceId : null;
|
|
915
|
+
if (!this.enabled(botId)) throw new HttpError(409, `${bot.name}'s browser is off: give it the browser connector first.`);
|
|
916
|
+
if (!this.sites(botId).length) throw new HttpError(409, `List the sites ${bot.name}'s browser may use first: the handover reaches only those.`);
|
|
917
|
+
const a = await this.availability(true);
|
|
918
|
+
if (!a.available) throw new HttpError(503, a.why ?? 'No browser on this server');
|
|
919
|
+
const existing = this.#handovers.get(botId);
|
|
920
|
+
if (existing) {
|
|
921
|
+
if (existing.ended) await existing.ending;
|
|
922
|
+
else if (existing.deviceId === deviceId) {
|
|
923
|
+
await existing.session.catch(() => {});
|
|
924
|
+
const view = this.handover(botId, deviceId);
|
|
925
|
+
if (view) return view;
|
|
926
|
+
throw new HttpError(503, 'The browser did not start.');
|
|
927
|
+
} else throw new HttpError(409, `${bot.name}'s browser is already being driven from ${existing.deviceLabel}: finish there, or end that handover first.`);
|
|
928
|
+
}
|
|
929
|
+
// Checked again after every wait above: still on, still no handover.
|
|
930
|
+
if (!this.enabled(botId) || this.#handovers.has(botId)) throw new HttpError(409, `${bot.name}'s browser changed meanwhile: try again.`);
|
|
931
|
+
const deviceLabel = this.#deps.audit?.actorOf(by).label ?? 'a paired device';
|
|
932
|
+
const now = Date.now();
|
|
933
|
+
const h: Handover = {
|
|
934
|
+
id: `ho_${randomBytes(12).toString('base64url')}`,
|
|
935
|
+
deviceId,
|
|
936
|
+
deviceLabel,
|
|
937
|
+
startedAt: now,
|
|
938
|
+
lastActive: now,
|
|
939
|
+
lastIntent: 0,
|
|
940
|
+
session: null as unknown as Promise<OwnerSession>,
|
|
941
|
+
view: null,
|
|
942
|
+
title: '',
|
|
943
|
+
ended: false,
|
|
944
|
+
ending: null,
|
|
945
|
+
timer: null,
|
|
946
|
+
queue: Promise.resolve(),
|
|
947
|
+
framing: false,
|
|
948
|
+
lastFrame: 0,
|
|
949
|
+
viewport: start ? fitViewport(start.width ?? VIEWPORT.width, start.height ?? VIEWPORT.height) : { ...VIEWPORT },
|
|
950
|
+
sink: null,
|
|
951
|
+
stopCast: null,
|
|
952
|
+
resume: null,
|
|
953
|
+
inputs: new RateWindow(MAX_INPUTS, INPUT_WINDOW_MS),
|
|
954
|
+
onEnd: [],
|
|
955
|
+
jar: null,
|
|
956
|
+
seeded: false,
|
|
957
|
+
};
|
|
958
|
+
// From here on (synchronously, before any wait) the Boton's browser opens no more: #act refuses, #launch throws.
|
|
959
|
+
this.#handovers.set(botId, h);
|
|
960
|
+
const untrack = this.#deps.trackDevice?.(by, () => void this.giveBack(botId, h.id, 'revoked'));
|
|
961
|
+
if (untrack) h.onEnd.push(untrack);
|
|
962
|
+
h.session = (async () => {
|
|
963
|
+
// The Boton's page as it stands, or where it was when it asked for the handover.
|
|
964
|
+
const boton = await this.#sessions.get(botId)?.catch(() => null);
|
|
965
|
+
let at = '';
|
|
966
|
+
try {
|
|
967
|
+
at = boton && !boton.closed ? boton.page.url() : '';
|
|
968
|
+
} catch {}
|
|
969
|
+
h.resume = /^https?:/i.test(at) ? at : (this.#handoffPage.get(botId) ?? null);
|
|
970
|
+
await this.#close(botId);
|
|
971
|
+
const jar = await this.#jarFor(botId);
|
|
972
|
+
if (h.ended) throw new HttpError(409, 'The handover ended before its browser opened.');
|
|
973
|
+
h.jar = jar;
|
|
974
|
+
h.seeded = jar !== null;
|
|
975
|
+
return this.#launchOwner(botId, h, jar ?? []);
|
|
976
|
+
})();
|
|
977
|
+
try {
|
|
978
|
+
await h.session;
|
|
979
|
+
} catch (err) {
|
|
980
|
+
await this.giveBack(botId, h.id, 'failed');
|
|
981
|
+
if (err instanceof HttpError) throw err;
|
|
982
|
+
throw new HttpError(503, 'The browser did not start.');
|
|
983
|
+
}
|
|
984
|
+
if (h.ended) throw new HttpError(409, 'The handover ended before its browser opened.');
|
|
985
|
+
// The idle limit counts from when the owner can see it.
|
|
986
|
+
this.#active(botId, h);
|
|
987
|
+
this.#journal(botId, { action: 'handover', url: '', outcome: 'done', detail: `${owner} took the browser over (from ${deviceLabel}).` });
|
|
988
|
+
this.#deps.handoffs?.taken(botId, deviceLabel);
|
|
989
|
+
this.#deps.audit?.record(by, { action: 'browser.handover_started', target: { kind: 'boton', id: botId, label: bot.name } });
|
|
990
|
+
log.info('browser handover started', { botId });
|
|
991
|
+
return this.handover(botId, deviceId)!;
|
|
992
|
+
}
|
|
993
|
+
|
|
994
|
+
async #launchOwner(botId: string, h: Handover, cookies: Cookie[]): Promise<OwnerSession> {
|
|
995
|
+
const pw = await this.#playwright();
|
|
996
|
+
const a = await this.availability();
|
|
997
|
+
if (!pw || !a.executable) throw new HttpError(503, a.why ?? 'No browser on this server');
|
|
998
|
+
// A profile of its own, new for this handover (none is left from another: they are deleted as they end).
|
|
999
|
+
this.#dropOwnerProfile(basename(this.#botDir(botId)));
|
|
1000
|
+
const profile = join(this.#ownerDir(botId), 'profile');
|
|
1001
|
+
mkdirSync(profile, { recursive: true, mode: 0o700 });
|
|
1002
|
+
const owner = this.#deps.store.getSettings().ownerName;
|
|
1003
|
+
let ownerPage: Page | null = null;
|
|
1004
|
+
const opened = await openGuarded(pw, profile, {
|
|
1005
|
+
executable: a.executable,
|
|
1006
|
+
sandbox: this.#deps.cfg.browser.sandbox,
|
|
1007
|
+
headless: !this.#headed(),
|
|
1008
|
+
viewport: h.viewport,
|
|
1009
|
+
policy: () => this.#ownerPolicy(botId, h),
|
|
1010
|
+
note: (url, why) => {
|
|
1011
|
+
this.#journal(botId, { action: 'blocked', url: displayUrl(url), outcome: 'refused', detail: `In ${owner}'s handover: ${why}` });
|
|
1012
|
+
// Said in the view as Botons' own, apart from anything the site says: only the page's own site's requests
|
|
1013
|
+
// (Chromium's own calls and other sites' are refused as quietly as ever).
|
|
1014
|
+
if (sameHost(url, h.view?.url() ?? '')) h.sink?.notice({ type: 'paused', url: displayUrl(url), why });
|
|
1015
|
+
},
|
|
1016
|
+
onUnreachable: (url, message) => this.#journal(botId, { action: 'unreachable', url: displayUrl(url), outcome: 'refused', detail: `In ${owner}'s handover: ${message}` }),
|
|
1017
|
+
// Its HTTPS ends at its proxy, which sees and decides every request (a write only within the owner's moment) and
|
|
1018
|
+
// journals each write as it leaves: whatever in Chromium sent it, routing seen or not.
|
|
1019
|
+
trustCa: this.testTrust ? [this.testTrust.cert] : undefined,
|
|
1020
|
+
onWrite: (url, method) => this.#journal(botId, { action: 'handover', url: displayUrl(url), outcome: 'done', detail: `${method} sent by the page, after ${owner}'s input in the handover` }),
|
|
1021
|
+
gate: (_route, request, write) => {
|
|
1022
|
+
if (h.ended || this.#handovers.get(botId) !== h) return 'the handover has ended';
|
|
1023
|
+
let from: Page | null = null;
|
|
1024
|
+
try {
|
|
1025
|
+
from = request.frame().page();
|
|
1026
|
+
} catch {}
|
|
1027
|
+
if (!from) return write ? 'no page made it (a worker), and only the page the owner is on sends' : 'pass';
|
|
1028
|
+
if (from !== ownerPage) return 'it came from a page the browser has left';
|
|
1029
|
+
if (!write) return 'pass';
|
|
1030
|
+
if (Date.now() - h.lastIntent >= this.#intentMs()) {
|
|
1031
|
+
return `the page sent it on its own, not within ${Math.round(this.#intentMs() / 1000)} seconds of ${owner}'s click, key or typing, so it was refused`;
|
|
1032
|
+
}
|
|
1033
|
+
// The proxy decides it again as it leaves, and journals it then.
|
|
1034
|
+
return 'send';
|
|
1035
|
+
},
|
|
1036
|
+
onDownload: (url) => this.#journal(botId, { action: 'download', url: displayUrl(url), outcome: 'refused', detail: 'Downloads are refused.' }),
|
|
1037
|
+
}).catch((err) => {
|
|
1038
|
+
log.error('the browser did not start', { botId, err });
|
|
1039
|
+
throw new HttpError(503, 'The browser did not start.');
|
|
1040
|
+
});
|
|
1041
|
+
const { context, page, proxy } = opened;
|
|
1042
|
+
ownerPage = page;
|
|
1043
|
+
if (cookies.length) await context.addCookies(cookies).catch(() => {});
|
|
1044
|
+
h.view = page;
|
|
1045
|
+
// Where the page is, and the site's own error answers for it, for the live view (never called a block).
|
|
1046
|
+
page.on('framenavigated', (f) => {
|
|
1047
|
+
if (f !== page.mainFrame() || !/^https?:/i.test(f.url())) return;
|
|
1048
|
+
h.sink?.notice({ type: 'page', url: displayUrl(f.url()), title: h.title });
|
|
1049
|
+
});
|
|
1050
|
+
page.on('load', () => {
|
|
1051
|
+
void page
|
|
1052
|
+
.title()
|
|
1053
|
+
.then((t) => {
|
|
1054
|
+
h.title = t.slice(0, 200);
|
|
1055
|
+
if (/^https?:/i.test(page.url())) h.sink?.notice({ type: 'page', url: displayUrl(page.url()), title: h.title });
|
|
1056
|
+
})
|
|
1057
|
+
.catch(() => {});
|
|
1058
|
+
});
|
|
1059
|
+
page.on('response', (r) => {
|
|
1060
|
+
try {
|
|
1061
|
+
if (r.status() >= 400 && r.request().isNavigationRequest() && r.frame() === page.mainFrame()) h.sink?.notice({ type: 'site-error', url: displayUrl(r.url()), status: r.status() });
|
|
1062
|
+
} catch {}
|
|
1063
|
+
});
|
|
1064
|
+
// Chromium gone on its own (it crashed, or was killed): the handover is over.
|
|
1065
|
+
context.on('close', () => void this.giveBack(botId, h.id, 'closed'));
|
|
1066
|
+
log.info('browser started', { botId, headless: !this.#headed(), sandbox: this.#deps.cfg.browser.sandbox });
|
|
1067
|
+
return { context, page, proxy, cdp: null };
|
|
1068
|
+
}
|
|
1069
|
+
|
|
1070
|
+
/** The server's DevTools session on the owner's page (one, made when first needed). */
|
|
1071
|
+
async #cdp(s: OwnerSession): Promise<import('playwright-core').CDPSession> {
|
|
1072
|
+
s.cdp ??= await s.context.newCDPSession(s.page);
|
|
1073
|
+
return s.cdp;
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
/** `expression` in a world of the server's own on the page's main frame (the page's script can't reach or change it). */
|
|
1077
|
+
async #isolated(s: OwnerSession, expression: string): Promise<unknown> {
|
|
1078
|
+
const cdp = await this.#cdp(s);
|
|
1079
|
+
const { frameTree } = await cdp.send('Page.getFrameTree');
|
|
1080
|
+
const { executionContextId } = await cdp.send('Page.createIsolatedWorld', { frameId: frameTree.frame.id, worldName: 'botons-owner' });
|
|
1081
|
+
const r = await cdp.send('Runtime.evaluate', { expression, contextId: executionContextId, returnByValue: true });
|
|
1082
|
+
return r.result.value;
|
|
1083
|
+
}
|
|
1084
|
+
|
|
1085
|
+
/**
|
|
1086
|
+
* The live view: a screencast of the owner's page to the driving device's stream, each frame acknowledged only once
|
|
1087
|
+
* it has gone (so a slow connection gets fewer frames, never a backlog), with notices of where the page is, the
|
|
1088
|
+
* site's own errors and what Botons paused. One stream at a time: a new one ends the one before. While it is open
|
|
1089
|
+
* the handover counts as in use. Returns how to stop it.
|
|
1090
|
+
*/
|
|
1091
|
+
async stream(botId: string, deviceId: string | null, sink: HandoverSink): Promise<() => void> {
|
|
1092
|
+
const h = this.#driven(botId, deviceId, null);
|
|
1093
|
+
const s = await h.session;
|
|
1094
|
+
if (h.ended) throw new HttpError(409, 'That handover has ended.');
|
|
1095
|
+
const before = h.sink;
|
|
1096
|
+
if (before) {
|
|
1097
|
+
h.sink = null;
|
|
1098
|
+
await h.stopCast?.().catch(() => {});
|
|
1099
|
+
before.end();
|
|
1100
|
+
}
|
|
1101
|
+
h.sink = sink;
|
|
1102
|
+
this.#active(botId, h);
|
|
1103
|
+
const keep = setInterval(() => {
|
|
1104
|
+
if (h.sink === sink && !h.ended) this.#active(botId, h);
|
|
1105
|
+
}, 30_000);
|
|
1106
|
+
keep.unref?.();
|
|
1107
|
+
let stopped = false;
|
|
1108
|
+
const ended = () => {
|
|
1109
|
+
if (h.sink === sink) sink.notice({ type: 'ended' });
|
|
1110
|
+
stop();
|
|
1111
|
+
sink.end();
|
|
1112
|
+
};
|
|
1113
|
+
const stop = () => {
|
|
1114
|
+
if (stopped) return;
|
|
1115
|
+
stopped = true;
|
|
1116
|
+
clearInterval(keep);
|
|
1117
|
+
// This stream's own hook goes with it: a reconnect leaves nothing of the stream before it behind.
|
|
1118
|
+
const at = h.onEnd.indexOf(ended);
|
|
1119
|
+
if (at >= 0) h.onEnd.splice(at, 1);
|
|
1120
|
+
if (h.sink !== sink) return;
|
|
1121
|
+
h.sink = null;
|
|
1122
|
+
void h.stopCast?.().catch(() => {});
|
|
1123
|
+
h.stopCast = null;
|
|
1124
|
+
};
|
|
1125
|
+
h.onEnd.push(ended);
|
|
1126
|
+
try {
|
|
1127
|
+
if (/^https?:/i.test(s.page.url())) sink.notice({ type: 'page', url: displayUrl(s.page.url()), title: h.title });
|
|
1128
|
+
await this.#cast(h, s);
|
|
1129
|
+
} catch (err) {
|
|
1130
|
+
// The screencast didn't start: this stream ends (the device falls back to frames), the handover goes on.
|
|
1131
|
+
stop();
|
|
1132
|
+
throw err;
|
|
1133
|
+
}
|
|
1134
|
+
return stop;
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
/** (Re)starts the screencast to the open stream at the page's size now. */
|
|
1138
|
+
async #cast(h: Handover, s: OwnerSession): Promise<void> {
|
|
1139
|
+
await h.stopCast?.().catch(() => {});
|
|
1140
|
+
h.stopCast = null;
|
|
1141
|
+
const sink = h.sink;
|
|
1142
|
+
if (!sink || h.ended) return;
|
|
1143
|
+
const cdp = await this.#cdp(s);
|
|
1144
|
+
let live = true;
|
|
1145
|
+
const onFrame = (f: { data: string; sessionId: number; metadata: { deviceWidth?: number; deviceHeight?: number } }) => {
|
|
1146
|
+
if (!live) return;
|
|
1147
|
+
const width = Math.round(f.metadata.deviceWidth ?? h.viewport.width);
|
|
1148
|
+
const height = Math.round(f.metadata.deviceHeight ?? h.viewport.height);
|
|
1149
|
+
void sink
|
|
1150
|
+
.frame(Buffer.from(f.data, 'base64'), width, height)
|
|
1151
|
+
.catch(() => {})
|
|
1152
|
+
.then(() => (live ? cdp.send('Page.screencastFrameAck', { sessionId: f.sessionId }).catch(() => {}) : undefined));
|
|
1153
|
+
};
|
|
1154
|
+
cdp.on('Page.screencastFrame', onFrame);
|
|
1155
|
+
h.stopCast = async () => {
|
|
1156
|
+
live = false;
|
|
1157
|
+
cdp.off('Page.screencastFrame', onFrame);
|
|
1158
|
+
await cdp.send('Page.stopScreencast').catch(() => {});
|
|
1159
|
+
};
|
|
1160
|
+
await cdp.send('Page.startScreencast', {
|
|
1161
|
+
format: 'jpeg',
|
|
1162
|
+
quality: 75,
|
|
1163
|
+
// Headless Chromium casts at the page's CSS size, whatever the screen's pixel ratio.
|
|
1164
|
+
maxWidth: h.viewport.width,
|
|
1165
|
+
maxHeight: h.viewport.height,
|
|
1166
|
+
everyNthFrame: 1,
|
|
1167
|
+
});
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1170
|
+
/**
|
|
1171
|
+
* The handover ends (`id`: only that one; null: whichever is under way): nothing more of the owner's browser goes
|
|
1172
|
+
* (its proxy is sealed, then Chromium and the proxy close), its cookies are kept for the Boton's next browser, and
|
|
1173
|
+
* the Boton has its browser back. Safe to call any number of times, from anywhere: the first does it, the rest
|
|
1174
|
+
* wait for it. False if there was no such handover.
|
|
1175
|
+
*/
|
|
1176
|
+
async giveBack(botId: string, id: string | null, why: HandoverEnd, by: Principal | null = null): Promise<boolean> {
|
|
1177
|
+
const h = this.#handovers.get(botId);
|
|
1178
|
+
if (!h || (id !== null && h.id !== id)) return false;
|
|
1179
|
+
if (!h.ending) {
|
|
1180
|
+
h.ended = true;
|
|
1181
|
+
if (h.timer) clearTimeout(h.timer);
|
|
1182
|
+
h.ending = (async () => {
|
|
1183
|
+
const s = await h.session?.catch(() => null);
|
|
1184
|
+
let jar: Cookie[] | null = null;
|
|
1185
|
+
let page: string | null = null;
|
|
1186
|
+
if (s) {
|
|
1187
|
+
// Sealed first: nothing more leaves, not even what its pages send as they close.
|
|
1188
|
+
s.proxy.seal(true);
|
|
1189
|
+
// Where the owner left it, origin and path only (never its query), for the Boton's note.
|
|
1190
|
+
const at = (() => {
|
|
1191
|
+
try {
|
|
1192
|
+
return s.page.url();
|
|
1193
|
+
} catch {
|
|
1194
|
+
return '';
|
|
1195
|
+
}
|
|
1196
|
+
})();
|
|
1197
|
+
page = /^https?:/i.test(at) ? displayUrl(at) : null;
|
|
1198
|
+
jar = await s.context.cookies().catch(() => null);
|
|
1199
|
+
await s.context.close().catch(() => {});
|
|
1200
|
+
await s.proxy.close();
|
|
1201
|
+
}
|
|
1202
|
+
// Its Chromium has exited: its profile goes, with whatever it kept of the pages. Only the cookies come back
|
|
1203
|
+
// (if it died first, as of the owner's last input).
|
|
1204
|
+
this.#dropOwnerProfile(basename(this.#botDir(botId)));
|
|
1205
|
+
jar ??= h.jar;
|
|
1206
|
+
if (jar && why !== 'deleted' && why !== 'failed') await this.#giveJar(botId, jar, h.seeded);
|
|
1207
|
+
for (const f of h.onEnd.splice(0)) f();
|
|
1208
|
+
if (this.#handovers.get(botId) === h) this.#handovers.delete(botId);
|
|
1209
|
+
this.#journal(botId, { action: 'handover', url: '', outcome: 'done', detail: `The handover ended (${HANDOVER_ENDS[why]}): the browser is the Boton's again.` });
|
|
1210
|
+
this.#handoffPage.delete(botId);
|
|
1211
|
+
if (why !== 'deleted') this.#deps.handoffs?.handedBack(botId, page, why === 'finished' ? 'finished' : HANDOVER_ENDS[why]);
|
|
1212
|
+
log.info('browser handover ended', { botId, why });
|
|
1213
|
+
const label = this.#deps.store.hasBot(botId) ? this.#deps.store.getBot(botId).name : botId;
|
|
1214
|
+
if (why !== 'failed') {
|
|
1215
|
+
this.#deps.audit?.record(by ?? SYSTEM_ACTOR, {
|
|
1216
|
+
action: 'browser.handover_ended',
|
|
1217
|
+
target: { kind: 'boton', id: botId, label },
|
|
1218
|
+
details: { reason: why, minutes: Math.round((Date.now() - h.startedAt) / 60_000) },
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
})();
|
|
1222
|
+
}
|
|
1223
|
+
await h.ending;
|
|
1224
|
+
return true;
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
/** One of the owner's inputs, from the device that drives the handover, run after the ones before it. */
|
|
1228
|
+
ownerInput(botId: string, deviceId: string | null, raw: unknown): Promise<BrowserHandoverResult> {
|
|
1229
|
+
const input = parseOwnerInput(raw);
|
|
1230
|
+
const h = this.#driven(botId, deviceId, input.handover);
|
|
1231
|
+
if (!h.inputs.take()) throw new HttpError(429, `At most ${MAX_INPUTS} inputs in ${INPUT_WINDOW_MS / 1000} seconds: wait a moment.`);
|
|
1232
|
+
this.#active(botId, h);
|
|
1233
|
+
const next = h.queue.catch(() => {}).then(() => this.#ownerStep(botId, h, input));
|
|
1234
|
+
h.queue = next;
|
|
1235
|
+
return next;
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1238
|
+
async #ownerStep(botId: string, h: Handover, input: BrowserHandoverInput): Promise<BrowserHandoverResult> {
|
|
1239
|
+
const s = await h.session;
|
|
1240
|
+
try {
|
|
1241
|
+
return await this.#ownerDo(botId, h, s, input);
|
|
1242
|
+
} finally {
|
|
1243
|
+
if (!h.ended) h.jar = (await s.context.cookies().catch(() => null)) ?? h.jar;
|
|
1244
|
+
}
|
|
1245
|
+
}
|
|
1246
|
+
|
|
1247
|
+
async #ownerDo(botId: string, h: Handover, s: OwnerSession, input: BrowserHandoverInput): Promise<BrowserHandoverResult> {
|
|
1248
|
+
const page = s.page;
|
|
1249
|
+
const info = async () => {
|
|
1250
|
+
const title = (await page.title().catch(() => '')).slice(0, 200);
|
|
1251
|
+
h.title = title;
|
|
1252
|
+
return { url: displayUrl(page.url()), title };
|
|
1253
|
+
};
|
|
1254
|
+
if (h.ended) return { outcome: 'refused', message: 'The handover has ended.', page: null };
|
|
1255
|
+
const done = async (message: string): Promise<BrowserHandoverResult> => ({ outcome: 'done', message, page: await info() });
|
|
1256
|
+
// After a click or key: a moment for what it starts (a form's page) to commit, so the address said is the new one.
|
|
1257
|
+
const settle = async () => {
|
|
1258
|
+
await new Promise((r) => setTimeout(r, 400));
|
|
1259
|
+
await page.waitForLoadState('domcontentloaded', { timeout: 5000 }).catch(() => {});
|
|
1260
|
+
};
|
|
1261
|
+
const intent = () => (h.lastIntent = Date.now());
|
|
1262
|
+
// Open, back and reload are no consent to send: the moment of the owner's last input ends, and everything the
|
|
1263
|
+
// page has open or under way is cut, before the page goes (what it sends as it unloads is then refused).
|
|
1264
|
+
const leave = () => {
|
|
1265
|
+
h.lastIntent = 0;
|
|
1266
|
+
s.proxy.cut();
|
|
1267
|
+
};
|
|
1268
|
+
// Points inside the page as it is now (the view may be a moment behind a resize).
|
|
1269
|
+
const at = (p: { x: number; y: number }) => ({ x: Math.min(p.x, h.viewport.width - 1), y: Math.min(p.y, h.viewport.height - 1) });
|
|
1270
|
+
const takesText = async () => (await this.#isolated(s, FOCUS_TAKES_TEXT).catch(() => false)) === true;
|
|
1271
|
+
try {
|
|
1272
|
+
switch (input.action) {
|
|
1273
|
+
case 'open': {
|
|
1274
|
+
let u: URL;
|
|
1275
|
+
try {
|
|
1276
|
+
u = new URL(/^[a-z][a-z0-9+.-]*:/i.test(input.url) ? input.url : `https://${input.url}`);
|
|
1277
|
+
} catch {
|
|
1278
|
+
return { outcome: 'refused', message: 'That is not an address.', page: await info() };
|
|
1279
|
+
}
|
|
1280
|
+
if (u.protocol !== 'https:' && u.protocol !== 'http:') return { outcome: 'refused', message: 'Only http:// and https:// addresses open here.', page: await info() };
|
|
1281
|
+
const v = judgeRequest(u.href, 'GET', this.sites(botId));
|
|
1282
|
+
if (!v.ok) return { outcome: 'refused', message: `Not opened: ${v.why}. Add the site to the Boton's browser sites first.`, page: await info() };
|
|
1283
|
+
leave();
|
|
1284
|
+
this.#journal(botId, { action: 'handover', url: displayUrl(u.href), outcome: 'done', detail: 'Opened in the handover' });
|
|
1285
|
+
const res = await page.goto(u.href, { waitUntil: 'domcontentloaded', timeout: ACTION_TIMEOUT_MS }).catch((err: Error) => err);
|
|
1286
|
+
if (res instanceof Error) return { outcome: 'refused', message: `${displayUrl(u.href)} didn't load: ${failureOf(res)}.`, page: await info() };
|
|
1287
|
+
return done(`Opened (HTTP ${res?.status() ?? '—'}).`);
|
|
1288
|
+
}
|
|
1289
|
+
case 'resume': {
|
|
1290
|
+
// The Boton's page in full, as it handed over: the owner asked for it, so it opens like any address.
|
|
1291
|
+
const u = h.resume;
|
|
1292
|
+
if (!u) return { outcome: 'refused', message: 'There is no page to go back to: open an address instead.', page: await info() };
|
|
1293
|
+
const v = judgeRequest(u, 'GET', this.sites(botId));
|
|
1294
|
+
if (!v.ok) return { outcome: 'refused', message: `Not opened: ${v.why}.`, page: await info() };
|
|
1295
|
+
leave();
|
|
1296
|
+
this.#journal(botId, { action: 'handover', url: displayUrl(u), outcome: 'done', detail: "Opened the Boton's page in the handover" });
|
|
1297
|
+
const res = await page.goto(u, { waitUntil: 'domcontentloaded', timeout: ACTION_TIMEOUT_MS }).catch((err: Error) => err);
|
|
1298
|
+
if (res instanceof Error) return { outcome: 'refused', message: `${displayUrl(u)} didn't load: ${failureOf(res)}.`, page: await info() };
|
|
1299
|
+
return done(`Opened (HTTP ${res?.status() ?? '—'}).`);
|
|
1300
|
+
}
|
|
1301
|
+
case 'click': {
|
|
1302
|
+
intent();
|
|
1303
|
+
const p = at(input);
|
|
1304
|
+
await page.mouse.click(p.x, p.y, { clickCount: input.count ?? 1 });
|
|
1305
|
+
await settle();
|
|
1306
|
+
return { ...(await done('Clicked.')), editable: await takesText() };
|
|
1307
|
+
}
|
|
1308
|
+
case 'down': {
|
|
1309
|
+
intent();
|
|
1310
|
+
const p = at(input);
|
|
1311
|
+
await page.mouse.move(p.x, p.y);
|
|
1312
|
+
await page.mouse.down({ button: input.button, clickCount: input.count ?? 1 });
|
|
1313
|
+
return done('Pressed.');
|
|
1314
|
+
}
|
|
1315
|
+
case 'up': {
|
|
1316
|
+
intent();
|
|
1317
|
+
const p = at(input);
|
|
1318
|
+
await page.mouse.move(p.x, p.y);
|
|
1319
|
+
await page.mouse.up({ button: input.button, clickCount: input.count ?? 1 });
|
|
1320
|
+
return { ...(await done('Released.')), editable: await takesText() };
|
|
1321
|
+
}
|
|
1322
|
+
case 'move': {
|
|
1323
|
+
// Not consent to send: hovering is not a choice.
|
|
1324
|
+
const p = at(input);
|
|
1325
|
+
await page.mouse.move(p.x, p.y);
|
|
1326
|
+
return done('Moved.');
|
|
1327
|
+
}
|
|
1328
|
+
case 'wheel': {
|
|
1329
|
+
const p = at(input);
|
|
1330
|
+
await page.mouse.move(p.x, p.y);
|
|
1331
|
+
await page.mouse.wheel(input.dx, input.dy);
|
|
1332
|
+
return done('Scrolled.');
|
|
1333
|
+
}
|
|
1334
|
+
case 'type':
|
|
1335
|
+
intent();
|
|
1336
|
+
await page.keyboard.type(input.text);
|
|
1337
|
+
// Never the text: only that some was typed.
|
|
1338
|
+
return done('Typed.');
|
|
1339
|
+
case 'paste':
|
|
1340
|
+
intent();
|
|
1341
|
+
await page.keyboard.insertText(input.text);
|
|
1342
|
+
// Never the text: only that some was pasted.
|
|
1343
|
+
return done('Pasted.');
|
|
1344
|
+
case 'copy': {
|
|
1345
|
+
const text = await this.#isolated(s, COPY_SELECTION).catch(() => '');
|
|
1346
|
+
// For this device's clipboard only: never in the journal, the log or a message.
|
|
1347
|
+
return { ...(await done('Copied.')), copied: typeof text === 'string' ? text.slice(0, MAX_COPY) : '' };
|
|
1348
|
+
}
|
|
1349
|
+
case 'key':
|
|
1350
|
+
intent();
|
|
1351
|
+
await page.keyboard.press(pressed(input.key, input.mods));
|
|
1352
|
+
if (input.key === 'Enter') await settle();
|
|
1353
|
+
return done(`Pressed ${pressed(input.key, input.mods)}.`);
|
|
1354
|
+
case 'scroll':
|
|
1355
|
+
await page.mouse.wheel(0, input.dy);
|
|
1356
|
+
return done('Scrolled.');
|
|
1357
|
+
case 'resize': {
|
|
1358
|
+
const next = fitViewport(input.width, input.height);
|
|
1359
|
+
if (next.width !== h.viewport.width || next.height !== h.viewport.height) {
|
|
1360
|
+
h.viewport = next;
|
|
1361
|
+
await page.setViewportSize(next);
|
|
1362
|
+
await this.#cast(h, s);
|
|
1363
|
+
}
|
|
1364
|
+
return done('Resized.');
|
|
1365
|
+
}
|
|
1366
|
+
case 'back':
|
|
1367
|
+
leave();
|
|
1368
|
+
await page.goBack({ waitUntil: 'domcontentloaded', timeout: ACTION_TIMEOUT_MS });
|
|
1369
|
+
return done('Went back.');
|
|
1370
|
+
case 'forward':
|
|
1371
|
+
leave();
|
|
1372
|
+
await page.goForward({ waitUntil: 'domcontentloaded', timeout: ACTION_TIMEOUT_MS });
|
|
1373
|
+
return done('Went forward.');
|
|
1374
|
+
case 'reload':
|
|
1375
|
+
leave();
|
|
1376
|
+
await page.reload({ waitUntil: 'domcontentloaded', timeout: ACTION_TIMEOUT_MS });
|
|
1377
|
+
return done('Reloaded.');
|
|
1378
|
+
}
|
|
1379
|
+
} catch (err) {
|
|
1380
|
+
// Botons' own words only: Playwright's message can name the address, and for typing, what was typed.
|
|
1381
|
+
return { outcome: 'refused', message: `The browser couldn't do that: ${failureOf(err)}.`, page: await info().catch(() => null) };
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
/**
|
|
1386
|
+
* What the owner's browser shows now, as a JPEG, for the device that drives it: made on demand, kept nowhere (not in
|
|
1387
|
+
* the workspace, the chat, the journal or the audit), at most one at a time and one per FRAME_MIN_MS.
|
|
1388
|
+
*/
|
|
1389
|
+
async frame(botId: string, deviceId: string | null): Promise<Buffer> {
|
|
1390
|
+
const h = this.#driven(botId, deviceId, null);
|
|
1391
|
+
const now = Date.now();
|
|
1392
|
+
if (h.framing || now - h.lastFrame < FRAME_MIN_MS) throw new HttpError(429, `At most one frame every ${FRAME_MIN_MS} ms.`);
|
|
1393
|
+
h.framing = true;
|
|
1394
|
+
h.lastFrame = now;
|
|
1395
|
+
this.#active(botId, h);
|
|
1396
|
+
try {
|
|
1397
|
+
const s = await h.session;
|
|
1398
|
+
if (h.ended) throw new HttpError(409, 'That handover has ended.');
|
|
1399
|
+
return await s.page.screenshot({ type: 'jpeg', quality: 70, timeout: 10_000, caret: 'initial' });
|
|
1400
|
+
} catch (err) {
|
|
1401
|
+
if (err instanceof HttpError) throw err;
|
|
1402
|
+
throw new HttpError(503, `No frame: ${failureOf(err)}.`);
|
|
1403
|
+
} finally {
|
|
1404
|
+
h.framing = false;
|
|
1405
|
+
}
|
|
1406
|
+
}
|
|
1407
|
+
|
|
1408
|
+
// ------------------------------------------------------------- actions
|
|
1409
|
+
|
|
1410
|
+
/** One action from the Boton's turn, with that turn's token (`tokenId`). */
|
|
1411
|
+
act(botId: string, tokenId: string, input: BrowserActionInput): Promise<BrowserActionResult> {
|
|
1412
|
+
if (this.#turns.get(botId) !== tokenId) {
|
|
1413
|
+
throw new HttpError(403, "Only a turn that was given the browser drives it (through its browser tools), and only while it runs.");
|
|
1414
|
+
}
|
|
1415
|
+
const prev = this.#queues.get(botId) ?? Promise.resolve();
|
|
1416
|
+
const next = prev.catch(() => {}).then(() => this.#act(botId, tokenId, input));
|
|
1417
|
+
this.#queues.set(botId, next);
|
|
1418
|
+
void next
|
|
1419
|
+
.finally(() => {
|
|
1420
|
+
if (this.#queues.get(botId) === next) this.#queues.delete(botId);
|
|
1421
|
+
})
|
|
1422
|
+
.catch(() => {});
|
|
1423
|
+
return next;
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
async #act(botId: string, tokenId: string, input: BrowserActionInput): Promise<BrowserActionResult> {
|
|
1427
|
+
const action = input?.action;
|
|
1428
|
+
if (!action || !ACTIONS.includes(action)) throw new HttpError(400, `action must be one of ${ACTIONS.join(', ')}`);
|
|
1429
|
+
const owner = this.#deps.store.getSettings().ownerName;
|
|
1430
|
+
const refuse = (message: string, page: BrowserActionResult['page'] = null): BrowserActionResult => {
|
|
1431
|
+
this.#journal(botId, { action, url: page?.url ?? '', outcome: 'refused', detail: message });
|
|
1432
|
+
return { outcome: 'refused', message, page };
|
|
1433
|
+
};
|
|
1434
|
+
if (!this.enabled(botId) || this.#turns.get(botId) !== tokenId) return refuse(`Your browser is off: ${owner} took the browser connector away.`);
|
|
1435
|
+
if (action === 'handoff') return this.#handoff(botId, input, owner);
|
|
1436
|
+
if (this.#handovers.has(botId)) {
|
|
1437
|
+
return refuse(`${owner} has your browser now (to sign it in, or to look at a page): you can't use it until ${owner} finishes. Go on without it, or say what you need it for.`);
|
|
1438
|
+
}
|
|
1439
|
+
const sites = this.sites(botId);
|
|
1440
|
+
if (!sites.length) return refuse(`No site is allowed yet: ${owner} lists the sites your browser may use in your settings.`);
|
|
1441
|
+
const s = await this.#session(botId);
|
|
1442
|
+
s.blocked.length = 0;
|
|
1443
|
+
s.unreachable.length = 0;
|
|
1444
|
+
try {
|
|
1445
|
+
const result = await this.#run(botId, tokenId, s, action, input, sites);
|
|
1446
|
+
if (s.blocked.length) result.blocked = s.blocked.map((b) => `${b.url}: ${b.why}`);
|
|
1447
|
+
if (s.unreachable.length) result.unreachable = [...new Set(s.unreachable.map((u) => u.message))];
|
|
1448
|
+
this.#journal(botId, { action, url: result.page?.url ?? '', outcome: result.outcome, detail: result.message });
|
|
1449
|
+
return result;
|
|
1450
|
+
} catch (err) {
|
|
1451
|
+
if (err instanceof HttpError) throw err;
|
|
1452
|
+
return refuse(`The browser couldn't do that: ${failureOf(err)}.`, await this.#pageInfo(s).catch(() => null));
|
|
1453
|
+
}
|
|
1454
|
+
}
|
|
1455
|
+
|
|
1456
|
+
/**
|
|
1457
|
+
* browser_handoff: the Boton asks the owner to take its browser over (a sign-in, a captcha, a code, or a step only
|
|
1458
|
+
* the owner should do). It posts the card in the chat and a push; nothing else changes until the owner takes it over,
|
|
1459
|
+
* which is the handover with all its rules. Handing back wakes the Boton with a note of where the page is.
|
|
1460
|
+
*/
|
|
1461
|
+
async #handoff(botId: string, input: BrowserActionInput, owner: string): Promise<BrowserActionResult> {
|
|
1462
|
+
const handoffs = this.#deps.handoffs;
|
|
1463
|
+
if (!handoffs) return { outcome: 'refused', message: 'Handing the browser over is not available on this server.', page: null };
|
|
1464
|
+
if (!this.sites(botId).length) return { outcome: 'refused', message: `No site is allowed yet: ${owner} lists the sites your browser may use in your settings.`, page: null };
|
|
1465
|
+
const h = this.#handovers.get(botId);
|
|
1466
|
+
const s = h ? null : await this.#sessions.get(botId)?.catch(() => null);
|
|
1467
|
+
let url: string | null = null;
|
|
1468
|
+
try {
|
|
1469
|
+
const at = s && !s.closed ? s.page.url() : '';
|
|
1470
|
+
if (/^https?:/i.test(at)) {
|
|
1471
|
+
url = displayUrl(at);
|
|
1472
|
+
this.#handoffPage.set(botId, at);
|
|
1473
|
+
}
|
|
1474
|
+
} catch {}
|
|
1475
|
+
const { message, already } = handoffs.request(botId, input.reason, url, h && !h.ended ? h.deviceLabel : null);
|
|
1476
|
+
const page = url ? { url, title: '' } : null;
|
|
1477
|
+
this.#journal(botId, { action: 'handoff', url: url ?? '', outcome: 'done', detail: `Asked ${owner} to take the browser over (card ${message.id})` });
|
|
1478
|
+
if (already || (h && !h.ended)) {
|
|
1479
|
+
return { outcome: 'done', message: `${owner} has your browser now. End your turn: when ${owner} hands it back, you get a message saying where the page is.`, page, handoff: message.id };
|
|
1480
|
+
}
|
|
1481
|
+
return {
|
|
1482
|
+
outcome: 'done',
|
|
1483
|
+
message: `Asked ${owner} to take over: the chat shows a card with Take over, and ${owner} got a notification. Your browser is ${owner}'s until they hand it back, so end your turn now with one short line (no need to explain where to find anything). When ${owner} hands it back, you get a message saying where the page is.`,
|
|
1484
|
+
page,
|
|
1485
|
+
handoff: message.id,
|
|
1486
|
+
};
|
|
1487
|
+
}
|
|
1488
|
+
|
|
1489
|
+
/** The control `ref` names, found again on the page as the last read listed it, or why not. */
|
|
1490
|
+
async #control(s: Session, ref: unknown): Promise<{ control: Control; locator: import('playwright-core').Locator } | string> {
|
|
1491
|
+
if (typeof ref !== 'string' || !/^e\d{1,4}$/.test(ref)) throw new HttpError(400, 'click and type need the ref of a control, from read (such as e3)');
|
|
1492
|
+
if (!s.controls || s.controls.revision !== s.revision) return 'The page has changed since you last read it (or you have not read it): read it again, then use a ref from that read.';
|
|
1493
|
+
const index = Number(ref.slice(1));
|
|
1494
|
+
const control = s.controls.list.find((c) => c.index === index);
|
|
1495
|
+
if (!control) return `There is no control ${ref} on this page: read it again.`;
|
|
1496
|
+
const locator = s.page.locator(CONTROLS).nth(index);
|
|
1497
|
+
const now = await locator.evaluate(describeControl, null).catch(() => null);
|
|
1498
|
+
if (!now || s.revision !== s.controls.revision || sameControl(now) !== sameControl(control)) return `The control ${ref} has changed since you read the page: read it again.`;
|
|
1499
|
+
return { control, locator };
|
|
1500
|
+
}
|
|
1501
|
+
|
|
1502
|
+
/**
|
|
1503
|
+
* A click or typing, then the request it made the page send, if any: held for the owner's card. The page is acted on
|
|
1504
|
+
* from the moment the step starts: anything else it sends, during the step or after, is refused.
|
|
1505
|
+
*/
|
|
1506
|
+
async #operate(botId: string, tokenId: string, s: Session, what: string, step: () => Promise<void>, doneText: string): Promise<BrowserActionResult> {
|
|
1507
|
+
let wake = () => {};
|
|
1508
|
+
const heldNow = new Promise<void>((r) => (wake = r));
|
|
1509
|
+
const op: Op = { open: true, cookies: await s.context.cookies().catch(() => null), held: null, onHeld: () => wake() };
|
|
1510
|
+
// Acted on from here: the routing holds or refuses each request the page makes, and the proxy lets nothing go
|
|
1511
|
+
// (not even what Playwright can't name a frame for) but the one send of a card the owner allows.
|
|
1512
|
+
s.actedOn = true;
|
|
1513
|
+
s.everActed = true;
|
|
1514
|
+
s.proxy.seal(true);
|
|
1515
|
+
s.op = op;
|
|
1516
|
+
try {
|
|
1517
|
+
await this.testGate?.('acting', botId);
|
|
1518
|
+
await step();
|
|
1519
|
+
let timer: NodeJS.Timeout | undefined;
|
|
1520
|
+
await Promise.race([heldNow, new Promise<void>((r) => (timer = setTimeout(r, SETTLE_MS)))]);
|
|
1521
|
+
clearTimeout(timer);
|
|
1522
|
+
// A script's request was held first: the form the same step submits, if any, takes its place.
|
|
1523
|
+
if (op.held && op.held.rank < 3) await new Promise((r) => setTimeout(r, FORM_GRACE_MS));
|
|
1524
|
+
} catch (err) {
|
|
1525
|
+
op.open = false;
|
|
1526
|
+
s.op = null;
|
|
1527
|
+
if (op.held) await op.held.route.abort('blockedbyclient').catch(() => {});
|
|
1528
|
+
throw err;
|
|
1529
|
+
}
|
|
1530
|
+
op.open = false;
|
|
1531
|
+
s.op = null;
|
|
1532
|
+
if (!op.held) return { outcome: 'done', message: `${doneText} The page sent nothing. Use read to see the page.`, page: await this.#pageInfo(s) };
|
|
1533
|
+
return this.#approve(botId, tokenId, s, op.held, op.cookies, what);
|
|
1534
|
+
}
|
|
1535
|
+
|
|
1536
|
+
/**
|
|
1537
|
+
* What is bound for a held request: the request as Chromium would send it, with its headers read first and the whole
|
|
1538
|
+
* jar after, its cookies judged by heldCookies against the jar as the step began (`start`) and now, or why it can't
|
|
1539
|
+
* go (it carries a cookie the browser didn't hold for its address).
|
|
1540
|
+
*/
|
|
1541
|
+
async #readHeld(botId: string, s: Session, request: Request, start: Cookie[] | null, seen?: (jar: Cookie[]) => void): Promise<HeldRequest | { why: string }> {
|
|
1542
|
+
const headers = await request.allHeaders();
|
|
1543
|
+
await this.testGate?.('held', botId);
|
|
1544
|
+
const now = await s.context.cookies();
|
|
1545
|
+
seen?.(now);
|
|
1546
|
+
const cookies = heldCookies(headers.cookie, request.url(), start, now);
|
|
1547
|
+
if ('why' in cookies) return cookies;
|
|
1548
|
+
return { method: request.method().toUpperCase(), url: request.url(), headers, body: request.postDataBuffer(), changed: cookies.changed };
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
/** A held request: its card, the owner's decision, and, if allowed while still bound, the one send. */
|
|
1552
|
+
async #approve(botId: string, tokenId: string, s: Session, held: { route: Route; request: Request }, start: Cookie[] | null, what: string): Promise<BrowserActionResult> {
|
|
1553
|
+
const owner = this.#deps.store.getSettings().ownerName;
|
|
1554
|
+
const { route, request } = held;
|
|
1555
|
+
const abort = () => route.abort('blockedbyclient').catch(() => {});
|
|
1556
|
+
const info = async () => this.#pageInfo(s).catch(() => null);
|
|
1557
|
+
const site = (() => {
|
|
1558
|
+
try {
|
|
1559
|
+
return new URL(request.url()).host;
|
|
1560
|
+
} catch {
|
|
1561
|
+
return 'the site';
|
|
1562
|
+
}
|
|
1563
|
+
})();
|
|
1564
|
+
// Botons' own words for a request it didn't send, and the way on: the owner takes over and does that step.
|
|
1565
|
+
const paused = (what: string, why: string) =>
|
|
1566
|
+
`Botons paused a request ${site} tried to send (${what}): not sent, because ${why}. Nothing reached the site. If ${owner} can do this step in the page, call browser_handoff so they can take over.`;
|
|
1567
|
+
const revision = s.revision;
|
|
1568
|
+
const binding: Binding = { botId, tokenId, session: s.id, revision };
|
|
1569
|
+
let read: HeldRequest | { why: string };
|
|
1570
|
+
let heldJar: Cookie[] = [];
|
|
1571
|
+
try {
|
|
1572
|
+
read = await this.#readHeld(botId, s, request, start, (jar) => (heldJar = jar));
|
|
1573
|
+
} catch {
|
|
1574
|
+
await abort();
|
|
1575
|
+
return { outcome: 'refused', message: "The page's request could not be read, so it was not sent.", page: await info() };
|
|
1576
|
+
}
|
|
1577
|
+
if ('why' in read) {
|
|
1578
|
+
await abort();
|
|
1579
|
+
return { outcome: 'refused', message: paused(`${request.method().toUpperCase()} ${displayUrl(request.url())}`, read.why), page: await info() };
|
|
1580
|
+
}
|
|
1581
|
+
const req = read;
|
|
1582
|
+
const where = `${req.method} ${displayUrl(req.url)}`;
|
|
1583
|
+
const key = fingerprint(binding, req);
|
|
1584
|
+
const card = describeRequest(req, `the ${shown(what)}`, key);
|
|
1585
|
+
if ('why' in card) {
|
|
1586
|
+
await abort();
|
|
1587
|
+
return { outcome: 'refused', message: paused(where, card.why), page: await info() };
|
|
1588
|
+
}
|
|
1589
|
+
const waitUntil = new Date(Date.now() + this.#deps.cfg.approvalWaitMs).toISOString();
|
|
1590
|
+
let approvalId: string;
|
|
1591
|
+
try {
|
|
1592
|
+
approvalId = this.#deps.approvals.browserRequest(botId, { ...card, input: { method: req.method, url: displayUrl(req.url), request: key.slice(0, 16) }, key, waitUntil }).id;
|
|
1593
|
+
} catch (err) {
|
|
1594
|
+
await abort();
|
|
1595
|
+
return { outcome: 'refused', message: paused(where, `no card could be made (${(err as Error).message})`), page: await info() };
|
|
1596
|
+
}
|
|
1597
|
+
this.#journal(botId, { action: 'blocked', url: displayUrl(req.url), outcome: 'refused', detail: `${req.method} held for ${owner}'s card ${approvalId}` });
|
|
1598
|
+
|
|
1599
|
+
// The wait: the owner's decision, the card's time running out, or the browser closing (the turn's end, Stop).
|
|
1600
|
+
const waiting = new Promise<'decided' | 'time' | 'closed'>((resolve) => {
|
|
1601
|
+
const finish = (why: 'decided' | 'time' | 'closed') => {
|
|
1602
|
+
clearTimeout(timer);
|
|
1603
|
+
this.#waits.delete(approvalId);
|
|
1604
|
+
s.endWait = null;
|
|
1605
|
+
resolve(why);
|
|
1606
|
+
};
|
|
1607
|
+
const timer = setTimeout(() => finish('time'), Math.max(0, Date.parse(waitUntil) - Date.now()));
|
|
1608
|
+
this.#waits.set(approvalId, () => finish('decided'));
|
|
1609
|
+
s.endWait = () => finish('closed');
|
|
1610
|
+
if (s.closed) finish('closed');
|
|
1611
|
+
});
|
|
1612
|
+
await this.testGate?.('card', botId);
|
|
1613
|
+
const ended = await waiting;
|
|
1614
|
+
const result = (outcome: 'done' | 'refused', message: string, delivery: 'sent' | 'not-sent' | 'unknown', page: BrowserActionResult['page']): BrowserActionResult => {
|
|
1615
|
+
const a = this.#deps.approvals.find(approvalId);
|
|
1616
|
+
return { outcome, message, page, approval: { id: approvalId, status: a?.status ?? 'cancelled', delivery } };
|
|
1617
|
+
};
|
|
1618
|
+
if (ended !== 'decided') this.#deps.approvals.endBrowserWait(approvalId, ended);
|
|
1619
|
+
const a = this.#deps.approvals.get(approvalId);
|
|
1620
|
+
if (a.status !== 'allowed') {
|
|
1621
|
+
await abort();
|
|
1622
|
+
const note = a.note ? ` ${owner}'s note: ${a.note}` : '';
|
|
1623
|
+
const message =
|
|
1624
|
+
a.status === 'denied'
|
|
1625
|
+
? `${owner} denied sending ${where} (card ${approvalId}).${note} It was not sent. Don't send it another way.`
|
|
1626
|
+
: `Not sent: ${ended === 'time' ? `${owner} didn't decide in time` : ended === 'closed' ? 'the browser closed' : 'the card was withdrawn'} (card ${approvalId}).`;
|
|
1627
|
+
this.#journal(botId, { action: 'blocked', url: displayUrl(req.url), outcome: 'refused', detail: `${req.method} not sent: card ${approvalId} ${a.status}` });
|
|
1628
|
+
return result('refused', message, 'not-sent', await info());
|
|
1629
|
+
}
|
|
1630
|
+
|
|
1631
|
+
// Allowed: checked again right before it goes (this turn, this browser, this document, its site, the request, its
|
|
1632
|
+
// cookies still ones the browser held), then spent once. What goes is the card's request, byte for byte.
|
|
1633
|
+
const live = () =>
|
|
1634
|
+
this.#turns.get(botId) !== tokenId || !this.enabled(botId)
|
|
1635
|
+
? 'the browser was turned off or the turn ended'
|
|
1636
|
+
: s.closed || this.#sessions.get(botId) === undefined
|
|
1637
|
+
? 'the browser closed'
|
|
1638
|
+
: s.revision !== revision
|
|
1639
|
+
? 'the page changed while the card waited'
|
|
1640
|
+
: !judgeRequest(req.url, 'GET', this.sites(botId)).ok
|
|
1641
|
+
? `${displayUrl(req.url)} is no longer one of the allowed sites`
|
|
1642
|
+
: null;
|
|
1643
|
+
// Read again with the jar as it is now: a site's own cookie churn is fine, a different request is not.
|
|
1644
|
+
let again: HeldRequest | { why: string } | null = null;
|
|
1645
|
+
try {
|
|
1646
|
+
// Against what the browser held as the step began and as the card was made: a later change of the site's own
|
|
1647
|
+
// cookies doesn't take back what it held then.
|
|
1648
|
+
again = await this.#readHeld(botId, s, request, start && [...start, ...heldJar]);
|
|
1649
|
+
} catch {}
|
|
1650
|
+
const problem =
|
|
1651
|
+
live() ??
|
|
1652
|
+
(!again
|
|
1653
|
+
? 'the request could not be read again'
|
|
1654
|
+
: 'why' in again
|
|
1655
|
+
? `${again.why} any more (a sign-out, say): click again for a new card`
|
|
1656
|
+
: sameRequest(req, again));
|
|
1657
|
+
if (problem || !this.#deps.approvals.spendBrowser(approvalId, key)) {
|
|
1658
|
+
await abort();
|
|
1659
|
+
const why = problem ?? 'its card was already used';
|
|
1660
|
+
this.#journal(botId, { action: 'blocked', url: displayUrl(req.url), outcome: 'refused', detail: `${req.method} allowed but not sent: ${why}` });
|
|
1661
|
+
return result('refused', `${owner} allowed it, but it was not sent: ${why}.`, 'not-sent', await info());
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
// Sent from here, exactly as held (the headers and body it was bound to, no cookie added), through the browser's proxy.
|
|
1665
|
+
const sending = new AbortController();
|
|
1666
|
+
s.sending = sending;
|
|
1667
|
+
const target = new URL(req.url);
|
|
1668
|
+
const d = await sendHeld({
|
|
1669
|
+
proxy: { port: s.proxy.port, ...s.proxy.passOnce(target.hostname.replace(/^\[|\]$/g, ''), portOf(target)) },
|
|
1670
|
+
method: req.method,
|
|
1671
|
+
url: req.url,
|
|
1672
|
+
headers: wireHeaders(req),
|
|
1673
|
+
body: req.body,
|
|
1674
|
+
ready: live,
|
|
1675
|
+
signal: sending.signal,
|
|
1676
|
+
timeoutMs: ACTION_TIMEOUT_MS,
|
|
1677
|
+
maxBody: ANSWER_MAX,
|
|
1678
|
+
ca: this.testTrust?.cert,
|
|
1679
|
+
}).finally(() => {
|
|
1680
|
+
if (s.sending === sending) s.sending = null;
|
|
1681
|
+
});
|
|
1682
|
+
this.#deps.approvals.browserDelivered(approvalId, d.delivery);
|
|
1683
|
+
log.info('browser request sent', { botId, approvalId, delivery: d.delivery, status: d.delivery === 'sent' ? d.status : undefined });
|
|
1684
|
+
if (d.delivery === 'not-sent') {
|
|
1685
|
+
await abort();
|
|
1686
|
+
this.#journal(botId, { action: 'sent', url: displayUrl(req.url), outcome: 'refused', detail: `${req.method} allowed on card ${approvalId}; not sent: ${d.why}` });
|
|
1687
|
+
return result('refused', `${owner} allowed it, but it was not sent: ${d.why}. Nothing reached the site.`, 'not-sent', await info());
|
|
1688
|
+
}
|
|
1689
|
+
if (d.delivery === 'unknown') {
|
|
1690
|
+
await route.abort('failed').catch(() => {});
|
|
1691
|
+
this.#journal(botId, { action: 'sent', url: displayUrl(req.url), outcome: 'refused', detail: `${req.method} allowed on card ${approvalId}; delivery unknown: ${d.why}` });
|
|
1692
|
+
return result(
|
|
1693
|
+
'refused',
|
|
1694
|
+
`${owner} allowed it and it was sent, but ${d.why}: it may have reached ${new URL(req.url).host} and been acted on. Don't send it again or another way: tell ${owner}, who can check on the site.`,
|
|
1695
|
+
'unknown',
|
|
1696
|
+
await info(),
|
|
1697
|
+
);
|
|
1698
|
+
}
|
|
1699
|
+
const status = d.status;
|
|
1700
|
+
const location = d.headers.find(([k]) => k === 'location')?.[1];
|
|
1701
|
+
// A redirect is never followed from here: a 307 or 308 would send it again, and anything else would be a request
|
|
1702
|
+
// from the page acted on. Its address is said, to open if wanted (in a new browser, as a read).
|
|
1703
|
+
let next: string | null = null;
|
|
1704
|
+
if (status >= 300 && status < 400 && location) {
|
|
1705
|
+
try {
|
|
1706
|
+
next = new URL(location, req.url).href;
|
|
1707
|
+
} catch {}
|
|
1708
|
+
}
|
|
1709
|
+
const resends = !!next && (status === 307 || status === 308);
|
|
1710
|
+
let answered = false;
|
|
1711
|
+
if (!next && d.body) {
|
|
1712
|
+
try {
|
|
1713
|
+
const answer = answerOf(d.status, d.headers, d.body);
|
|
1714
|
+
await route.fulfill(answer);
|
|
1715
|
+
answered = true;
|
|
1716
|
+
} catch {}
|
|
1717
|
+
}
|
|
1718
|
+
if (!answered) await abort();
|
|
1719
|
+
await s.page.waitForLoadState('domcontentloaded', { timeout: 10_000 }).catch(() => {});
|
|
1720
|
+
this.#journal(botId, { action: 'sent', url: displayUrl(req.url), outcome: 'done', detail: `${req.method} sent once (HTTP ${status}), allowed on card ${approvalId}` });
|
|
1721
|
+
const onward = next && judgeRequest(next, 'GET', this.sites(botId)).ok ? ` It points to ${displayUrl(next)}: open that address to see where it led.` : '';
|
|
1722
|
+
const tail = resends
|
|
1723
|
+
? `The site answered with a redirect that would send it again, which the browser did not follow.${onward}`
|
|
1724
|
+
: next
|
|
1725
|
+
? `The site answered with a redirect, which the browser did not follow from the page it acted on.${onward}`
|
|
1726
|
+
: answered
|
|
1727
|
+
? 'Use read to see the page.'
|
|
1728
|
+
: 'Its answer was cut, too large or could not be read, so the page did not get it.';
|
|
1729
|
+
return result('done', `${owner} allowed it on a card: sent ${where} once (HTTP ${status}). ${tail}`, 'sent', await info());
|
|
1730
|
+
}
|
|
1731
|
+
|
|
1732
|
+
/** Loads an address in the page, or in a new browser if the page was acted on (so that page sends nothing more). */
|
|
1733
|
+
async #open(botId: string, s: Session, url: string, doneText: string): Promise<BrowserActionResult> {
|
|
1734
|
+
if (s.actedOn) {
|
|
1735
|
+
// Closed meanwhile (the turn ended, Stop): no new browser is started for it.
|
|
1736
|
+
if (s.closed || !this.#turns.has(botId) || !this.enabled(botId) || this.#handovers.has(botId)) return { outcome: 'refused', message: 'The browser closed.', page: null };
|
|
1737
|
+
const starting = this.#replace(botId, s);
|
|
1738
|
+
s = await starting;
|
|
1739
|
+
if (this.#sessions.get(botId) !== starting || s.closed) return { outcome: 'refused', message: 'The browser closed.', page: null };
|
|
1740
|
+
}
|
|
1741
|
+
// The main frame's last navigation request: where it ended after any redirect, failed or answered.
|
|
1742
|
+
const seen: { nav: Request | null } = { nav: null };
|
|
1743
|
+
const onRequest = (r: Request) => {
|
|
1744
|
+
try {
|
|
1745
|
+
if (r.isNavigationRequest() && r.frame() === s.page.mainFrame()) seen.nav = r;
|
|
1746
|
+
} catch {}
|
|
1747
|
+
};
|
|
1748
|
+
s.page.on('request', onRequest);
|
|
1749
|
+
const res = await s.page.goto(url, { waitUntil: 'domcontentloaded', timeout: ACTION_TIMEOUT_MS }).catch((err: Error) => err);
|
|
1750
|
+
s.page.off('request', onRequest);
|
|
1751
|
+
const last = res instanceof Error ? seen.nav : (res?.request() ?? seen.nav);
|
|
1752
|
+
// Botons answered, not the site: say what it said (a block by Botons, or a site it couldn't reach).
|
|
1753
|
+
const why = navigationAnswer(
|
|
1754
|
+
{ url: last?.url() ?? url, failed: res instanceof Error, mark: res instanceof Error ? null : (res?.headers()['x-botons-proxy'] ?? null), request: last },
|
|
1755
|
+
s,
|
|
1756
|
+
);
|
|
1757
|
+
if (why) return { outcome: 'refused', message: `${displayUrl(url)} didn't load: ${why}`, page: await this.#pageInfo(s).catch(() => null) };
|
|
1758
|
+
if (res instanceof Error) return { outcome: 'refused', message: `${displayUrl(url)} didn't load: ${failureOf(res)}.`, page: await this.#pageInfo(s).catch(() => null) };
|
|
1759
|
+
const status = res?.status();
|
|
1760
|
+
// The site's own answer, error pages included: a 403 here is the site's, never a block by Botons.
|
|
1761
|
+
if (status && status >= 400) return { outcome: 'done', message: `${doneText}, but the site itself answered HTTP ${status} (its own error page, not a block by Botons). Use read to see the page.`, page: await this.#pageInfo(s) };
|
|
1762
|
+
return { outcome: 'done', message: `${doneText} (HTTP ${status ?? '—'}). Use read to see the page.`, page: await this.#pageInfo(s) };
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
async #pageInfo(s: Session): Promise<{ url: string; title: string }> {
|
|
1766
|
+
return { url: displayUrl(s.page.url()), title: (await s.page.title().catch(() => '')).slice(0, 200) };
|
|
1767
|
+
}
|
|
1768
|
+
|
|
1769
|
+
async #run(botId: string, tokenId: string, s: Session, action: BrowserActionName, input: BrowserActionInput, sites: string[]): Promise<BrowserActionResult> {
|
|
1770
|
+
const owner = this.#deps.store.getSettings().ownerName;
|
|
1771
|
+
const done = async (message: string): Promise<BrowserActionResult> => ({ outcome: 'done', message, page: await this.#pageInfo(s) });
|
|
1772
|
+
switch (action) {
|
|
1773
|
+
case 'open': {
|
|
1774
|
+
if (typeof input.url !== 'string' || !input.url.trim()) throw new HttpError(400, 'open needs a url');
|
|
1775
|
+
let u: URL;
|
|
1776
|
+
try {
|
|
1777
|
+
u = new URL(input.url.trim());
|
|
1778
|
+
} catch {
|
|
1779
|
+
return { outcome: 'refused', message: 'That is not an address (use https://…).', page: null };
|
|
1780
|
+
}
|
|
1781
|
+
if (u.protocol !== 'https:' && u.protocol !== 'http:') return { outcome: 'refused', message: 'The browser opens only http:// and https:// addresses.', page: null };
|
|
1782
|
+
const v = judgeRequest(u.href, 'GET', sites);
|
|
1783
|
+
if (!v.ok) return { outcome: 'refused', message: `Not opened: ${v.why}. ${owner} chooses the sites in your settings.`, page: null };
|
|
1784
|
+
return this.#open(botId, s, u.href, 'Opened');
|
|
1785
|
+
}
|
|
1786
|
+
case 'back': {
|
|
1787
|
+
if (!s.actedOn) {
|
|
1788
|
+
await s.page.goBack({ waitUntil: 'domcontentloaded', timeout: ACTION_TIMEOUT_MS });
|
|
1789
|
+
return done('Went back. Use read to see the page.');
|
|
1790
|
+
}
|
|
1791
|
+
// From a page acted on, the address before it, in a new browser: the old page's script never runs again.
|
|
1792
|
+
s.trail.pop();
|
|
1793
|
+
const before = s.trail.pop();
|
|
1794
|
+
if (!before || !judgeRequest(before, 'GET', sites).ok) return { outcome: 'refused', message: 'There is no page to go back to: open an address instead.', page: await this.#pageInfo(s) };
|
|
1795
|
+
return this.#open(botId, s, before, 'Went back to the page before, in a new browser');
|
|
1796
|
+
}
|
|
1797
|
+
case 'read': {
|
|
1798
|
+
const text = ((await s.page.locator('body').innerText({ timeout: 10_000 }).catch(() => '')) ?? '').replace(/\n{3,}/g, '\n\n');
|
|
1799
|
+
// Links it may open next: only to the listed sites, by their full address (open takes it as it is).
|
|
1800
|
+
const all = await s.page
|
|
1801
|
+
.locator('a[href]')
|
|
1802
|
+
.evaluateAll((els) => els.slice(0, 2000).map((el) => ({ text: ((el as HTMLElement).innerText || el.getAttribute('aria-label') || '').replace(/\s+/g, ' ').trim().slice(0, 200), url: (el as HTMLAnchorElement).href })))
|
|
1803
|
+
.catch(() => [] as { text: string; url: string }[]);
|
|
1804
|
+
const links = all.filter((l) => /^https?:/.test(l.url) && judgeRequest(l.url, 'GET', sites).ok).slice(0, MAX_LINKS);
|
|
1805
|
+
const truncated = text.length > MAX_TEXT;
|
|
1806
|
+
// Its controls, by ref, for click and type: valid until the page changes (the document they were read on).
|
|
1807
|
+
const revision = s.revision;
|
|
1808
|
+
const list = await listControls(s.page);
|
|
1809
|
+
s.controls = { revision, list };
|
|
1810
|
+
const controls: BrowserControl[] = list
|
|
1811
|
+
.filter((c) => c.kind !== 'link' || judgeRequest(c.href, 'GET', sites).ok)
|
|
1812
|
+
.map((c) => ({ ref: `e${c.index}`, kind: c.kind, label: c.label, noTyping: c.kind === 'field' ? typingProblem(c) : null }));
|
|
1813
|
+
return {
|
|
1814
|
+
...(await done(`Read the page: ${links.length} links to allowed sites, ${controls.length} controls.`)),
|
|
1815
|
+
content: { text: text.slice(0, MAX_TEXT), links, controls, truncated },
|
|
1816
|
+
};
|
|
1817
|
+
}
|
|
1818
|
+
case 'click':
|
|
1819
|
+
case 'type': {
|
|
1820
|
+
const found = await this.#control(s, input.ref);
|
|
1821
|
+
if (typeof found === 'string') return { outcome: 'refused', message: found, page: await this.#pageInfo(s) };
|
|
1822
|
+
const { control, locator } = found;
|
|
1823
|
+
const what = `${control.kind} ${JSON.stringify(control.label || control.name || '(no label)')}`;
|
|
1824
|
+
// A link to an allowed site: its address is opened, as open would, without the page's script for the click.
|
|
1825
|
+
if (action === 'click' && control.tag === 'a' && /^https?:/.test(control.href) && judgeRequest(control.href, 'GET', sites).ok) {
|
|
1826
|
+
return this.#open(botId, s, control.href, `Opened the address of the ${what} (the browser opens a link's address; it doesn't run the page's script for the click)`);
|
|
1827
|
+
}
|
|
1828
|
+
if (action === 'type') {
|
|
1829
|
+
const problem = typingProblem(control);
|
|
1830
|
+
if (problem) return { outcome: 'refused', message: `Not typed: ${problem}. Botons never types passwords, codes, card numbers or other secrets; ${owner} signs in for you.`, page: await this.#pageInfo(s) };
|
|
1831
|
+
const text = input.text;
|
|
1832
|
+
if (typeof text !== 'string') throw new HttpError(400, 'type needs text');
|
|
1833
|
+
if (text.length > MAX_TYPED) return { outcome: 'refused', message: `Not typed: at most ${MAX_TYPED} characters.`, page: await this.#pageInfo(s) };
|
|
1834
|
+
if (/[\u0000-\u0008\u000b-\u001f\u007f]/.test(text) || (control.tag !== 'textarea' && /[\n\r]/.test(text))) {
|
|
1835
|
+
return { outcome: 'refused', message: 'Not typed: the text has control characters (or a new line, in a one-line field).', page: await this.#pageInfo(s) };
|
|
1836
|
+
}
|
|
1837
|
+
// fill() puts the text in and presses nothing: no Enter, so typing alone never submits a form.
|
|
1838
|
+
return this.#operate(botId, tokenId, s, what, () => locator.fill(text, { timeout: ACTION_TIMEOUT_MS }), `Typed into the ${what}.`);
|
|
1839
|
+
}
|
|
1840
|
+
return this.#operate(botId, tokenId, s, what, () => locator.click({ timeout: ACTION_TIMEOUT_MS, noWaitAfter: true }), `Clicked the ${what}.`);
|
|
1841
|
+
}
|
|
1842
|
+
case 'screenshot': {
|
|
1843
|
+
const bot = this.#deps.store.getBot(botId);
|
|
1844
|
+
const bytes = await s.page.screenshot({ type: 'png', timeout: ACTION_TIMEOUT_MS });
|
|
1845
|
+
return screenshotToChat(this.#deps.share, bot, bytes, await this.#pageInfo(s));
|
|
1846
|
+
}
|
|
1847
|
+
case 'handoff':
|
|
1848
|
+
// Answered in #act, before any browser opens.
|
|
1849
|
+
throw new HttpError(400, 'handoff is not a step of the page');
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
|
|
1854
|
+
/** In the page: what a control says of itself (all of it the site's own text). */
|
|
1855
|
+
function describeControl(el: Element): Omit<Control, 'index'> {
|
|
1856
|
+
const e = el as HTMLInputElement;
|
|
1857
|
+
const tag = e.tagName.toLowerCase();
|
|
1858
|
+
const type = (e.getAttribute('type') ?? '').toLowerCase();
|
|
1859
|
+
const role = (e.getAttribute('role') ?? '').toLowerCase();
|
|
1860
|
+
const labels = e.labels ? Array.from(e.labels).map((l) => (l as HTMLElement).innerText).join(' ') : '';
|
|
1861
|
+
const button = tag === 'input' && ['submit', 'button', 'reset', 'image'].includes(type);
|
|
1862
|
+
const label = (e.getAttribute('aria-label') || labels || e.getAttribute('placeholder') || (tag === 'input' ? '' : (e as unknown as HTMLElement).innerText) || (button ? e.value : '') || e.getAttribute('title') || '')
|
|
1863
|
+
.replace(/\s+/g, ' ')
|
|
1864
|
+
.trim()
|
|
1865
|
+
.slice(0, 120);
|
|
1866
|
+
const kind =
|
|
1867
|
+
tag === 'a' || role === 'link'
|
|
1868
|
+
? 'link'
|
|
1869
|
+
: tag === 'button' || role === 'button' || button
|
|
1870
|
+
? 'button'
|
|
1871
|
+
: (tag === 'input' && (type === 'checkbox' || type === 'radio')) || role === 'checkbox'
|
|
1872
|
+
? 'checkbox'
|
|
1873
|
+
: tag === 'textarea' || (tag === 'input' && ['', 'text', 'search', 'email', 'url', 'tel', 'number', 'password'].includes(type))
|
|
1874
|
+
? 'field'
|
|
1875
|
+
: 'other';
|
|
1876
|
+
return { tag, type, name: e.getAttribute('name') ?? '', id: e.id ?? '', autocomplete: e.getAttribute('autocomplete') ?? '', label, kind, href: tag === 'a' ? (e as unknown as HTMLAnchorElement).href : '' };
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
/** What must stay the same for a ref to still name its control. */
|
|
1880
|
+
const sameControl = (c: Omit<Control, 'index'>) => JSON.stringify([c.tag, c.type, c.name, c.id, c.autocomplete, c.label, c.kind, c.href]);
|
|
1881
|
+
|
|
1882
|
+
/** Each new document of the page's main frame: refs and cards from before no longer apply, and its address is kept for back. */
|
|
1883
|
+
function watchPage(s: Session, page: Page): void {
|
|
1884
|
+
page.on('framenavigated', (f) => {
|
|
1885
|
+
if (s.page !== page || f !== page.mainFrame()) return;
|
|
1886
|
+
s.revision++;
|
|
1887
|
+
const url = f.url();
|
|
1888
|
+
if (/^https?:/.test(url) && s.trail.at(-1) !== url) s.trail.push(url);
|
|
1889
|
+
if (s.trail.length > 50) s.trail.shift();
|
|
1890
|
+
});
|
|
1891
|
+
}
|
|
1892
|
+
|
|
1893
|
+
/** An allowed request's answer, for the page: decoded (it was asked for as Chromium asked), its framing headers dropped. */
|
|
1894
|
+
function answerOf(status: number, raw: [string, string][], body: Buffer): { status: number; headers: Record<string, string>; body: Buffer } {
|
|
1895
|
+
const headers: Record<string, string> = {};
|
|
1896
|
+
for (const [k, v] of raw) headers[k] = k in headers ? `${headers[k]}\n${v}` : v;
|
|
1897
|
+
let out = body;
|
|
1898
|
+
const enc = (headers['content-encoding'] ?? '').trim().toLowerCase();
|
|
1899
|
+
const decode: Record<string, (b: Buffer) => Buffer> = {
|
|
1900
|
+
gzip: (b) => gunzipSync(b, { maxOutputLength: ANSWER_MAX }),
|
|
1901
|
+
'x-gzip': (b) => gunzipSync(b, { maxOutputLength: ANSWER_MAX }),
|
|
1902
|
+
deflate: (b) => inflateSync(b, { maxOutputLength: ANSWER_MAX }),
|
|
1903
|
+
br: (b) => brotliDecompressSync(b, { maxOutputLength: ANSWER_MAX }),
|
|
1904
|
+
zstd: (b) => zstdDecompressSync(b, { maxOutputLength: ANSWER_MAX }),
|
|
1905
|
+
};
|
|
1906
|
+
if (enc && enc !== 'identity') {
|
|
1907
|
+
const d = decode[enc];
|
|
1908
|
+
if (!d) throw new Error(`an answer in ${enc}`);
|
|
1909
|
+
out = d(body);
|
|
1910
|
+
}
|
|
1911
|
+
for (const k of ['content-encoding', 'content-length', 'transfer-encoding', 'connection', 'keep-alive']) delete headers[k];
|
|
1912
|
+
return { status, headers, body: out };
|
|
1913
|
+
}
|
|
1914
|
+
|
|
1915
|
+
/** The controls of the page's main document, as read lists them: shown and enabled ones, at most MAX_CONTROLS. */
|
|
1916
|
+
async function listControls(page: Page): Promise<Control[]> {
|
|
1917
|
+
const all = await page
|
|
1918
|
+
.locator(CONTROLS)
|
|
1919
|
+
.evaluateAll(
|
|
1920
|
+
(els, max) =>
|
|
1921
|
+
els.slice(0, 2000).flatMap((el, index) => {
|
|
1922
|
+
const e = el as HTMLElement & { disabled?: boolean };
|
|
1923
|
+
const shown = !!(e.offsetWidth || e.offsetHeight || e.getClientRects().length) && !e.disabled;
|
|
1924
|
+
return shown && index < max ? [index] : [];
|
|
1925
|
+
}),
|
|
1926
|
+
MAX_CONTROLS * 4,
|
|
1927
|
+
)
|
|
1928
|
+
.catch(() => [] as number[]);
|
|
1929
|
+
const out: Control[] = [];
|
|
1930
|
+
for (const index of all.slice(0, MAX_CONTROLS)) {
|
|
1931
|
+
const d = await page.locator(CONTROLS).nth(index).evaluate(describeControl, null).catch(() => null);
|
|
1932
|
+
if (d) out.push({ ...d, index });
|
|
1933
|
+
}
|
|
1934
|
+
return out;
|
|
1935
|
+
}
|
|
1936
|
+
|
|
1937
|
+
let playwright: Promise<Playwright | null> | null = null;
|
|
1938
|
+
|
|
1939
|
+
export function loadPlaywright(): Promise<Playwright | null> {
|
|
1940
|
+
playwright ??= import('playwright-core').then(
|
|
1941
|
+
(m) => (m as unknown as { default?: Playwright }).default ?? (m as unknown as Playwright),
|
|
1942
|
+
() => null,
|
|
1943
|
+
);
|
|
1944
|
+
return playwright;
|
|
1945
|
+
}
|
|
1946
|
+
|
|
1947
|
+
/** The library and the Chromium the browser would run: BOTONS_BROWSER_PATH (`path`), else Playwright's, else the system's. */
|
|
1948
|
+
export async function findBrowser(path: string): Promise<Availability> {
|
|
1949
|
+
const pw = await loadPlaywright();
|
|
1950
|
+
if (!pw) return { available: false, why: 'The browser library (playwright-core) is not installed with this server: run npm ci.', executable: null };
|
|
1951
|
+
let exe: string | null;
|
|
1952
|
+
if (path) exe = existsSync(path) ? path : null;
|
|
1953
|
+
else {
|
|
1954
|
+
let bundled = '';
|
|
1955
|
+
try {
|
|
1956
|
+
bundled = pw.chromium.executablePath();
|
|
1957
|
+
} catch {}
|
|
1958
|
+
exe = [bundled, ...DEFAULT_PATHS].find((p) => p && existsSync(p)) ?? null;
|
|
1959
|
+
}
|
|
1960
|
+
if (!exe) {
|
|
1961
|
+
return {
|
|
1962
|
+
available: false,
|
|
1963
|
+
why: path
|
|
1964
|
+
? `BOTONS_BROWSER_PATH is ${path}, which is not there.`
|
|
1965
|
+
: 'No Chromium found on this computer: install one (the chromium package, or npx playwright-core install chromium) or set BOTONS_BROWSER_PATH.',
|
|
1966
|
+
executable: null,
|
|
1967
|
+
};
|
|
1968
|
+
}
|
|
1969
|
+
return { available: true, why: null, executable: exe };
|
|
1970
|
+
}
|
|
1971
|
+
|
|
1972
|
+
/**
|
|
1973
|
+
* Chromium's own password manager and autofill off in the profile, before it starts: a sign-in the owner types in a
|
|
1974
|
+
* handover leaves cookies, never a saved password (which, without a keyring, Chromium keeps under a fixed key) or
|
|
1975
|
+
* the fields' values for autofill, and Chromium never checks it against its leak service (which the proxy would
|
|
1976
|
+
* refuse anyway). Its other preferences are kept as they are.
|
|
1977
|
+
*/
|
|
1978
|
+
export function keepNoCredentials(profile: string): void {
|
|
1979
|
+
const file = join(profile, 'Default', 'Preferences');
|
|
1980
|
+
mkdirSync(dirname(file), { recursive: true, mode: 0o700 });
|
|
1981
|
+
let prefs: Record<string, any> = {};
|
|
1982
|
+
try {
|
|
1983
|
+
const v = JSON.parse(readFileSync(file, 'utf8'));
|
|
1984
|
+
if (v && typeof v === 'object' && !Array.isArray(v)) prefs = v;
|
|
1985
|
+
} catch {}
|
|
1986
|
+
const obj = (k: string) => (prefs[k] && typeof prefs[k] === 'object' && !Array.isArray(prefs[k]) ? prefs[k] : (prefs[k] = {}));
|
|
1987
|
+
prefs.credentials_enable_service = false;
|
|
1988
|
+
prefs.credentials_enable_autosignin = false;
|
|
1989
|
+
Object.assign(obj('profile'), { password_manager_enabled: false, password_manager_leak_detection: false });
|
|
1990
|
+
Object.assign(obj('autofill'), { enabled: false, profile_enabled: false, credit_card_enabled: false });
|
|
1991
|
+
writeFileSync(file, JSON.stringify(prefs), { mode: 0o600 });
|
|
1992
|
+
}
|
|
1993
|
+
|
|
1994
|
+
/** The user agent each Chromium (by its path and when it was installed) reports, its headless mark taken out. */
|
|
1995
|
+
const agents = new Map<string, Promise<string | null>>();
|
|
1996
|
+
|
|
1997
|
+
/**
|
|
1998
|
+
* The running Chromium's own user agent without its headless mark (withoutHeadless), asked of it once per executable
|
|
1999
|
+
* (started headless with no profile, opening no page, behind a proxy of its own that lets nothing out), or null if
|
|
2000
|
+
* it couldn't be asked. Its version and platform are the browser's own, never written here.
|
|
2001
|
+
*/
|
|
2002
|
+
function headfulAgent(pw: Playwright, executable: string, sandbox: boolean): Promise<string | null> {
|
|
2003
|
+
let stamp = '';
|
|
2004
|
+
try {
|
|
2005
|
+
stamp = String(statSync(executable).mtimeMs);
|
|
2006
|
+
} catch {}
|
|
2007
|
+
const key = `${executable}\0${stamp}\0${sandbox}`;
|
|
2008
|
+
let agent = agents.get(key);
|
|
2009
|
+
if (!agent) {
|
|
2010
|
+
agent = (async () => {
|
|
2011
|
+
const shut = await startBrowserProxy(() => ({ enabled: false, sites: [] }), () => {});
|
|
2012
|
+
try {
|
|
2013
|
+
const proxy = { server: `http://127.0.0.1:${shut.port}`, bypass: '<-loopback>', username: shut.username, password: shut.password };
|
|
2014
|
+
const browser = await pw.chromium.launch({ executablePath: executable, headless: true, chromiumSandbox: sandbox, proxy, args: ['--disable-background-networking', '--disable-component-update', '--no-first-run', '--disable-extensions'] });
|
|
2015
|
+
try {
|
|
2016
|
+
const cdp = await browser.newBrowserCDPSession();
|
|
2017
|
+
const v = (await cdp.send('Browser.getVersion')) as { userAgent?: string };
|
|
2018
|
+
return typeof v.userAgent === 'string' && v.userAgent ? withoutHeadless(v.userAgent) : null;
|
|
2019
|
+
} finally {
|
|
2020
|
+
await browser.close().catch(() => {});
|
|
2021
|
+
}
|
|
2022
|
+
} finally {
|
|
2023
|
+
await shut.close();
|
|
2024
|
+
}
|
|
2025
|
+
})().catch((err) => {
|
|
2026
|
+
log.warn('the browser did not say its user agent: it keeps its headless one', { err });
|
|
2027
|
+
agents.delete(key);
|
|
2028
|
+
return null;
|
|
2029
|
+
});
|
|
2030
|
+
agents.set(key, agent);
|
|
2031
|
+
}
|
|
2032
|
+
return agent;
|
|
2033
|
+
}
|
|
2034
|
+
|
|
2035
|
+
/**
|
|
2036
|
+
* Starts Chromium on `profile` with every guard the managed browser has: its own proxy (browser-proxy.ts), routing
|
|
2037
|
+
* that lets only reads to the policy's sites leave, no WebSockets, one tab, dialogs dismissed, downloads and file
|
|
2038
|
+
* pickers refused, service workers blocked, QUIC and direct WebRTC UDP off, and Chromium's sandbox unless the
|
|
2039
|
+
* operator turned it off. The server's turns and `npm run check:browser` both start it this way.
|
|
2040
|
+
*/
|
|
2041
|
+
export async function openGuarded(
|
|
2042
|
+
pw: Playwright,
|
|
2043
|
+
profile: string,
|
|
2044
|
+
o: {
|
|
2045
|
+
executable: string;
|
|
2046
|
+
sandbox: boolean;
|
|
2047
|
+
headless: boolean;
|
|
2048
|
+
policy: () => ProxyPolicy;
|
|
2049
|
+
/** A refusal: by the proxy, or by the routing (with the request it refused). */
|
|
2050
|
+
note: (url: string, why: string, request?: Request) => void;
|
|
2051
|
+
onDownload?: (url: string) => void;
|
|
2052
|
+
/**
|
|
2053
|
+
* Each request to an allowed site that is a read or a write a card could send (`write`): `pass` loads a read,
|
|
2054
|
+
* `held` keeps it for a card (neither sent nor refused yet), `send` lets a write go (only the owner's handover,
|
|
2055
|
+
* right after the owner's own input), a reason refuses it, null leaves it to the default (reads load, writes are
|
|
2056
|
+
* refused).
|
|
2057
|
+
*/
|
|
2058
|
+
gate?: (route: Route, request: Request, write: boolean) => 'pass' | 'held' | 'send' | string | null;
|
|
2059
|
+
/** The page's size (the owner's handover clicks in these coordinates). */
|
|
2060
|
+
viewport?: { width: number; height: number };
|
|
2061
|
+
/** Tests only: the certificate of their own HTTPS site, which the proxy then accepts (besides the system's roots). */
|
|
2062
|
+
trustCa?: string[];
|
|
2063
|
+
/** Each write as it leaves the proxy (only the owner's handover lets one go). */
|
|
2064
|
+
onWrite?: (url: string, method: string) => void;
|
|
2065
|
+
/** A site its proxy could not reach (not found, no address answered): never said as a block. */
|
|
2066
|
+
onUnreachable?: (url: string, message: string) => void;
|
|
2067
|
+
},
|
|
2068
|
+
): Promise<{ context: Context; page: Page; proxy: BrowserProxy }> {
|
|
2069
|
+
const note = o.note;
|
|
2070
|
+
// Its own home beside the profile, and only the environment it needs: none of the server's settings or secrets.
|
|
2071
|
+
const home = join(dirname(profile), 'home');
|
|
2072
|
+
mkdirSync(home, { recursive: true, mode: 0o700 });
|
|
2073
|
+
const env: Record<string, string> = { HOME: home, XDG_CONFIG_HOME: join(home, '.config'), XDG_CACHE_HOME: join(home, '.cache'), XDG_DATA_HOME: join(home, '.local', 'share') };
|
|
2074
|
+
for (const k of ['PATH', 'LANG', 'LC_ALL', 'TZ', 'TMPDIR', ...(o.headless ? [] : ['DISPLAY', 'WAYLAND_DISPLAY', 'XDG_RUNTIME_DIR'])]) {
|
|
2075
|
+
if (process.env[k]) env[k] = process.env[k]!;
|
|
2076
|
+
}
|
|
2077
|
+
keepNoCredentials(profile);
|
|
2078
|
+
// Its HTTPS always ends at its proxy, with a certificate made for this one browser, which this Chromium alone trusts
|
|
2079
|
+
// (and no other certificate error): so the proxy judges every request inside, whatever in Chromium sent it, routing
|
|
2080
|
+
// seen or not (browser-proxy.ts). No browser opened here gets a tunnel straight to a site.
|
|
2081
|
+
const own = interceptCert();
|
|
2082
|
+
const proxy = await startBrowserProxy(o.policy, note, { intercept: { key: own.key, cert: own.cert, ca: o.trustCa }, onWrite: o.onWrite, onUnreachable: o.onUnreachable });
|
|
2083
|
+
let context: Context;
|
|
2084
|
+
try {
|
|
2085
|
+
const proxySetting = { server: `http://127.0.0.1:${proxy.port}`, bypass: '<-loopback>', username: proxy.username, password: proxy.password };
|
|
2086
|
+
const userAgent = o.headless ? await headfulAgent(pw, o.executable, o.sandbox) : null;
|
|
2087
|
+
context = await pw.chromium.launchPersistentContext(profile, {
|
|
2088
|
+
...(userAgent ? { userAgent } : {}),
|
|
2089
|
+
executablePath: o.executable,
|
|
2090
|
+
headless: o.headless,
|
|
2091
|
+
env,
|
|
2092
|
+
// Playwright turns Chromium's sandbox off unless asked: a page's exploit would get the server's user.
|
|
2093
|
+
chromiumSandbox: o.sandbox,
|
|
2094
|
+
// Loopback too: Chromium would otherwise reach this computer's own services without the proxy.
|
|
2095
|
+
proxy: proxySetting,
|
|
2096
|
+
acceptDownloads: false,
|
|
2097
|
+
serviceWorkers: 'block',
|
|
2098
|
+
viewport: o.viewport ?? { width: 1280, height: 800 },
|
|
2099
|
+
args: [
|
|
2100
|
+
'--disable-quic',
|
|
2101
|
+
'--force-webrtc-ip-handling-policy=disable_non_proxied_udp',
|
|
2102
|
+
'--webrtc-ip-handling-policy=disable_non_proxied_udp',
|
|
2103
|
+
'--dns-prefetch-disable',
|
|
2104
|
+
'--disable-background-networking',
|
|
2105
|
+
'--disable-sync',
|
|
2106
|
+
'--disable-extensions',
|
|
2107
|
+
'--disable-component-update',
|
|
2108
|
+
'--no-first-run',
|
|
2109
|
+
'--no-default-browser-check',
|
|
2110
|
+
// Chromium's own calls home (including search preconnects and AI eligibility on startup/cookie changes) fill the journal.
|
|
2111
|
+
// No back-forward cache and no fetchLater: a page left behind is gone, and sends nothing after it is.
|
|
2112
|
+
'--disable-features=AutofillServerCommunication,OptimizationHints,AimEligibilityServiceIdentityImprovements,AimUrlNavigationFetchEnabled,AimServerRequestOnStartupEnabled,AimServerRequestOnIdentityChangeEnabled,PreconnectToSearch,PreconnectFromKeyedService,Translate,MediaRouter,BackForwardCache,FetchLaterAPI,PasswordLeakDetection',
|
|
2113
|
+
'--disable-client-side-phishing-detection',
|
|
2114
|
+
`--ignore-certificate-errors-spki-list=${own.spki}`,
|
|
2115
|
+
],
|
|
2116
|
+
});
|
|
2117
|
+
} catch (err) {
|
|
2118
|
+
await proxy.close();
|
|
2119
|
+
throw err;
|
|
2120
|
+
}
|
|
2121
|
+
try {
|
|
2122
|
+
await context.route('**/*', (route, request) => {
|
|
2123
|
+
const p = o.policy();
|
|
2124
|
+
const v = p.enabled ? judgeRequest(request.url(), request.method(), p.sites) : { ok: false as const, why: 'the browser is turned off' };
|
|
2125
|
+
const write = !v.ok && 'write' in v && !!v.write;
|
|
2126
|
+
const g = (v.ok || write) && o.gate ? o.gate(route, request, write) : null;
|
|
2127
|
+
if (g === 'held') return;
|
|
2128
|
+
if (write && g === 'send') return route.continue();
|
|
2129
|
+
if (v.ok && (g === null || g === 'pass')) return route.continue();
|
|
2130
|
+
note(request.url(), g && g !== 'pass' ? g : (v as { why: string }).why, request);
|
|
2131
|
+
return route.abort('blockedbyclient');
|
|
2132
|
+
});
|
|
2133
|
+
// A WebSocket sends whatever a page likes, so none connects: each is closed here, before the server sees it.
|
|
2134
|
+
await context.routeWebSocket(
|
|
2135
|
+
() => true,
|
|
2136
|
+
(ws) => {
|
|
2137
|
+
note(ws.url(), 'WebSockets are not for this browser: it only reads pages');
|
|
2138
|
+
void ws.close({ code: 1008, reason: 'Blocked by Botons' }).catch(() => {});
|
|
2139
|
+
},
|
|
2140
|
+
);
|
|
2141
|
+
const guard = (page: Page) => {
|
|
2142
|
+
page.on('dialog', (d) => void d.dismiss().catch(() => {}));
|
|
2143
|
+
page.on('download', (d) => {
|
|
2144
|
+
o.onDownload?.(d.url());
|
|
2145
|
+
void d.cancel().catch(() => {});
|
|
2146
|
+
});
|
|
2147
|
+
page.on('filechooser', () => note(page.url(), 'the page asked for a file to upload, which the browser never gives'));
|
|
2148
|
+
};
|
|
2149
|
+
const current = context.pages()[0] ?? (await context.newPage());
|
|
2150
|
+
context.on('page', (p) => {
|
|
2151
|
+
if (p === current) return;
|
|
2152
|
+
// One tab: another (a popup, target=_blank) is closed, and said so.
|
|
2153
|
+
note(p.url() || 'about:blank', 'the page opened another tab, which was closed (open its address instead)');
|
|
2154
|
+
void p.close().catch(() => {});
|
|
2155
|
+
});
|
|
2156
|
+
guard(current);
|
|
2157
|
+
context.on('close', () => void proxy.close());
|
|
2158
|
+
return { context, page: current, proxy };
|
|
2159
|
+
} catch (err) {
|
|
2160
|
+
await context.close().catch(() => {});
|
|
2161
|
+
await proxy.close();
|
|
2162
|
+
throw err;
|
|
2163
|
+
}
|
|
2164
|
+
}
|