@bivy/bivy 0.17.3-staging.29 → 0.17.3-staging.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/bivy.mjs +54 -3
- package/dist/apps/display-viewer.js +6 -3
- package/dist/apps/display.js +11 -1
- package/dist/apps/gateway.js +47 -7
- package/dist/apps/input.js +96 -0
- package/dist/apps/macos-display.js +169 -0
- package/dist/apps/macos-display.swift +726 -0
- package/dist/apps/preview-stream.js +17 -3
- package/dist/apps/rfb.js +69 -20
- package/dist/apps/service.js +32 -6
- package/dist/controllers/app-commands.js +1 -0
- package/dist/protocol/client-command-routes.js +1 -1
- package/dist/protocol/client-command-schemas.js +1 -0
- package/dist/server.js +2 -2
- package/package.json +1 -1
package/bin/bivy.mjs
CHANGED
|
@@ -2434,6 +2434,12 @@ async function cmdApp(args = []) {
|
|
|
2434
2434
|
bivy app present [view] [--path /page] [--note "…"] [--session <id>]
|
|
2435
2435
|
bivy app share [app-id] [--view <view>] [--session <id>]
|
|
2436
2436
|
bivy app run [--name <name>] [--restart-on-change] [--session <id>] -- <command> [args…]
|
|
2437
|
+
bivy app click <x> <y> [--right|--middle] [--double] [--app <app>]
|
|
2438
|
+
bivy app type <text> [--app <app>]
|
|
2439
|
+
bivy app key <keys> [--app <app>] e.g. enter, cmd+s, ctrl+shift+t
|
|
2440
|
+
bivy app scroll <x> <y> [--up|--down|--left|--right] [--steps <n>] [--app <app>]
|
|
2441
|
+
bivy app drag <x> <y> <to-x> <to-y> [--app <app>]
|
|
2442
|
+
bivy app move <x> <y> [--app <app>]
|
|
2437
2443
|
|
|
2438
2444
|
Manifest: {"version":1,"name":"My app","views":[
|
|
2439
2445
|
{"kind":"web","name":"Website","source":{"kind":"service","port":3000}},
|
|
@@ -2442,9 +2448,10 @@ Manifest: {"version":1,"name":"My app","views":[
|
|
|
2442
2448
|
|
|
2443
2449
|
Static web source: {"kind":"static","directory":"./dist"}
|
|
2444
2450
|
Server Bivy runs: {"kind":"service","port":5173,"start":{"command":"pnpm","args":["dev"]}}
|
|
2445
|
-
Desktop GUI app (Linux
|
|
2451
|
+
Desktop GUI app (Linux with TigerVNC's Xvnc, or macOS 13+):
|
|
2446
2452
|
{"kind":"display","name":"Editor","command":"cargo","args":["run"],"restartOnChange":true}
|
|
2447
|
-
It runs on a private display
|
|
2453
|
+
It runs on a private display (Linux) or shows its own windows (macOS),
|
|
2454
|
+
streamed into the preview and sized to the viewer.
|
|
2448
2455
|
"restartOnChange" restarts it after a turn that changed files. Shortcut with
|
|
2449
2456
|
no manifest: bivy app run -- cargo run
|
|
2450
2457
|
Publishing does not start commands. Open Apps in the session menu to view or run;
|
|
@@ -2455,6 +2462,13 @@ Terminal commands run with the node user's permissions, not in a new sandbox.
|
|
|
2455
2462
|
Chrome/Chromium (desktop apps: straight from their display) and prints the PNG paths — look at them before saying the UI is
|
|
2456
2463
|
done. It is off by default: turn it on in Bivy → Settings → this machine, or
|
|
2457
2464
|
run: bivy config set sessions.appScreenshots true
|
|
2465
|
+
click/type/key/scroll/drag/move use a desktop app like a person would, in the
|
|
2466
|
+
pixels of its screenshot (bivy app shot), and print a new screenshot after
|
|
2467
|
+
(when screenshots are on) — look at it to see what happened. --app picks the
|
|
2468
|
+
app (default: the one opened last). Keys: a character or enter, tab, escape,
|
|
2469
|
+
backspace, delete, arrows (up…), home, end, pageup, pagedown, f1–f12, with
|
|
2470
|
+
modifiers shift, ctrl, alt/option, cmd. On a Mac, the pointer and keyboard
|
|
2471
|
+
focus move to the app while it's used.
|
|
2458
2472
|
"present" says a change is ready for the user to look at: they get a card in
|
|
2459
2473
|
the chat with the app at phone width (and before/after), and it opens the live
|
|
2460
2474
|
preview. Use it when you finish a visible change, or when the user asks to see
|
|
@@ -2468,6 +2482,7 @@ who can read it may use the app.
|
|
|
2468
2482
|
Web previews require operator setup; see docs/apps.md.`);
|
|
2469
2483
|
return;
|
|
2470
2484
|
}
|
|
2485
|
+
if (INPUT_ACTIONS[action]) return appInput(action, rest);
|
|
2471
2486
|
if (!["publish", "list", "remove", "shot", "run", "present", "share"].includes(action)) throw new Error("Unknown app command. Run bivy app --help.");
|
|
2472
2487
|
// `run -- <command> [args…]`: publish a one-window desktop app without a manifest file.
|
|
2473
2488
|
let runManifest;
|
|
@@ -2528,10 +2543,14 @@ Web previews require operator setup; see docs/apps.md.`);
|
|
|
2528
2543
|
body.manifest = manifest;
|
|
2529
2544
|
} else if (action === "run") body.manifest = runManifest;
|
|
2530
2545
|
else if (action === "remove") body.appId = positional[0];
|
|
2546
|
+
await appRequest(action === "run" ? "publish" : action, body);
|
|
2547
|
+
}
|
|
2548
|
+
|
|
2549
|
+
async function appRequest(action, body) {
|
|
2531
2550
|
const config = loadConfig();
|
|
2532
2551
|
if (!(await ensureNodeRunning(config))) throw new Error("Could not reach the Bivy node.");
|
|
2533
2552
|
const token = await localDeviceToken(config);
|
|
2534
|
-
const response = await fetch(`${url(config)}/api/apps/${action
|
|
2553
|
+
const response = await fetch(`${url(config)}/api/apps/${action}`, {
|
|
2535
2554
|
method: "POST", headers: { authorization: `Bearer ${token}`, "content-type": "application/json" }, body: JSON.stringify(body),
|
|
2536
2555
|
});
|
|
2537
2556
|
const result = await response.json();
|
|
@@ -2545,6 +2564,38 @@ Web previews require operator setup; see docs/apps.md.`);
|
|
|
2545
2564
|
console.log(JSON.stringify(result, null, 2));
|
|
2546
2565
|
}
|
|
2547
2566
|
|
|
2567
|
+
// Computer use in desktop apps: each command's positional arguments, and the
|
|
2568
|
+
// flags that shape its action. The node validates the action.
|
|
2569
|
+
const INPUT_ACTIONS = {
|
|
2570
|
+
click: { args: ["x", "y"], flags: { "--right": { button: "right" }, "--middle": { button: "middle" }, "--double": { count: 2 } } },
|
|
2571
|
+
move: { args: ["x", "y"], flags: {} },
|
|
2572
|
+
drag: { args: ["x", "y", "toX", "toY"], flags: {} },
|
|
2573
|
+
scroll: { args: ["x", "y"], flags: { "--up": { direction: "up" }, "--down": { direction: "down" }, "--left": { direction: "left" }, "--right": { direction: "right" } }, defaults: { direction: "down" }, values: { "--steps": "steps" } },
|
|
2574
|
+
type: { args: ["text"], flags: {}, text: true },
|
|
2575
|
+
key: { args: ["keys"], flags: {}, text: true },
|
|
2576
|
+
};
|
|
2577
|
+
async function appInput(kind, rest) {
|
|
2578
|
+
const spec = INPUT_ACTIONS[kind];
|
|
2579
|
+
const action = { kind, ...spec.defaults };
|
|
2580
|
+
const options = {};
|
|
2581
|
+
const positional = [];
|
|
2582
|
+
for (let i = 0; i < rest.length; i++) {
|
|
2583
|
+
const arg = rest[i];
|
|
2584
|
+
if (spec.flags[arg]) Object.assign(action, spec.flags[arg]);
|
|
2585
|
+
else if (arg === "--app" || arg === "--session" || spec.values?.[arg]) {
|
|
2586
|
+
const value = rest[++i];
|
|
2587
|
+
if (value === undefined || value.startsWith("--")) throw new Error(`${arg} requires a value.`);
|
|
2588
|
+
if (spec.values?.[arg]) action[spec.values[arg]] = Number(value); else options[arg] = value;
|
|
2589
|
+
} else if (arg.startsWith("--")) throw new Error(`Unknown option ${arg}. Run bivy app --help.`);
|
|
2590
|
+
else positional.push(arg);
|
|
2591
|
+
}
|
|
2592
|
+
if (positional.length !== spec.args.length) throw new Error(`Usage: bivy app ${kind} ${spec.args.map((name) => `<${name}>`).join(" ")}. Run bivy app --help.`);
|
|
2593
|
+
spec.args.forEach((name, i) => { action[name] = spec.text ? positional[i] : Number(positional[i]); });
|
|
2594
|
+
const sessionId = resolveAttachSessionId({ sessionFlag: options["--session"], env: process.env });
|
|
2595
|
+
if (!sessionId) throw new Error("Set --session <id> or run inside a Bivy agent session.");
|
|
2596
|
+
await appRequest("input", { sessionId, action, ...(options["--app"] ? { target: options["--app"] } : {}) });
|
|
2597
|
+
}
|
|
2598
|
+
|
|
2548
2599
|
// Map a saved session's runtime id to the `bivy run` agent whose native CLI can
|
|
2549
2600
|
// resume it in a terminal. Only agents with a real native resume qualify; other
|
|
2550
2601
|
// runtimes (generic-cli, SDK-only) have no terminal resume and open in the web app.
|
|
@@ -3,12 +3,15 @@
|
|
|
3
3
|
export const DISPLAY_SOCKET_PATH = "/__bivy/display";
|
|
4
4
|
export const DISPLAY_STATS_PATH = "/__bivy/display-stats";
|
|
5
5
|
export const NOVNC_PATH = "/__bivy/novnc/";
|
|
6
|
+
/** The modifier apps paste with, as a keysym and noVNC key code: ⌘ on a Mac. */
|
|
7
|
+
const PASTE_MODIFIER = { darwin: [0xffeb, "MetaLeft"] };
|
|
6
8
|
/** The page a display view serves on its own origin: noVNC's client streaming
|
|
7
9
|
* the app's display. It asks the display to match its size in device pixels
|
|
8
10
|
* (`scale` is what the display was started at), scales down when the display
|
|
9
11
|
* is larger, reconnects while the app starts, carries the clipboard both ways
|
|
10
12
|
* on a tap, gives phones a keyboard, and reports stream latency to Bivy. */
|
|
11
|
-
export function displayViewer(nonce, name, scale) {
|
|
13
|
+
export function displayViewer(nonce, name, scale, platform = process.platform) {
|
|
14
|
+
const [modifier, modifierCode] = PASTE_MODIFIER[platform] ?? [0xffe3, "ControlLeft"];
|
|
12
15
|
const title = name.replace(/[<>&"]/g, (c) => `&#${c.charCodeAt(0)};`);
|
|
13
16
|
return `<!doctype html><html lang="en"><meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1, interactive-widget=resizes-content"><title>${title}</title>
|
|
14
17
|
<style nonce="${nonce}">:root{color-scheme:light dark;font-family:system-ui,sans-serif}html,body{margin:0;height:100%;overflow:hidden;background:Canvas;color:CanvasText}
|
|
@@ -62,8 +65,8 @@ function connect(){
|
|
|
62
65
|
connect();
|
|
63
66
|
|
|
64
67
|
$("copy").onclick=async()=>{try{await navigator.clipboard.writeText(remote);$("copy").textContent="Copied";setTimeout(()=>{$("copy").hidden=true;},1500);}catch{$("copy").textContent="Couldn’t copy";}};
|
|
65
|
-
// Paste: hand the text to the app's clipboard, then press Ctrl+V in it.
|
|
66
|
-
const paste=text=>{if(!text)return;rfb.clipboardPasteFrom(text);rfb.sendKey(
|
|
68
|
+
// Paste: hand the text to the app's clipboard, then press Ctrl+V (⌘V on a Mac) in it.
|
|
69
|
+
const paste=text=>{if(!text)return;rfb.clipboardPasteFrom(text);rfb.sendKey(${modifier},"${modifierCode}",true);rfb.sendKey(0x76,"KeyV");rfb.sendKey(${modifier},"${modifierCode}",false);rfb.focus();};
|
|
67
70
|
$("paste-open").onclick=async()=>{
|
|
68
71
|
try{const text=await navigator.clipboard.readText();if(text){paste(text);return;}}catch{}
|
|
69
72
|
// No clipboard access here: paste into a field instead.
|
package/dist/apps/display.js
CHANGED
|
@@ -5,6 +5,7 @@ import os from "node:os";
|
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import { spawn, execFileSync } from "node:child_process";
|
|
7
7
|
import { FitWindowManager, writeXauthority } from "./x11.js";
|
|
8
|
+
import { MacDisplayHost } from "./macos-display.js";
|
|
8
9
|
/** Environment for programs on a preview display. X11 for every toolkit, so an
|
|
9
10
|
* app never escapes to the machine's own Wayland/X session. */
|
|
10
11
|
const APP_ENV = {
|
|
@@ -48,7 +49,7 @@ export class DisplayHost {
|
|
|
48
49
|
/** Why desktop views can't run here, or undefined when they can. */
|
|
49
50
|
unavailable() {
|
|
50
51
|
if ((this.options.platform ?? process.platform) !== "linux")
|
|
51
|
-
return "Desktop app views need a Linux
|
|
52
|
+
return "Desktop app views need a Linux or macOS machine.";
|
|
52
53
|
if (!(this.options.xvnc ?? findXvnc)())
|
|
53
54
|
return "Desktop app views need TigerVNC's X server. Install it (Debian/Ubuntu: sudo apt install tigervnc-standalone-server), or set BIVY_XVNC to its path.";
|
|
54
55
|
return undefined;
|
|
@@ -133,3 +134,12 @@ function freeDisplay(x11Dir) {
|
|
|
133
134
|
}
|
|
134
135
|
throw new Error("No free X display number.");
|
|
135
136
|
}
|
|
137
|
+
/** How each platform gives a desktop app a preview display: a private X
|
|
138
|
+
* server on Linux, the app's own windows on a Mac. Both serve VNC on a
|
|
139
|
+
* private socket, so everything downstream is shared. */
|
|
140
|
+
const HOSTS = {
|
|
141
|
+
darwin: (dataDir) => new MacDisplayHost({ cacheDir: path.join(dataDir, "bin") }),
|
|
142
|
+
};
|
|
143
|
+
export function createDisplayHost(dataDir, platform = process.platform) {
|
|
144
|
+
return HOSTS[platform]?.(dataDir) ?? new DisplayHost({ platform });
|
|
145
|
+
}
|
package/dist/apps/gateway.js
CHANGED
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// Copyright (c) 2026 Petter André Sjulstad
|
|
3
3
|
import http from "node:http";
|
|
4
4
|
import net from "node:net";
|
|
5
|
-
import { randomBytes } from "node:crypto";
|
|
5
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
6
|
+
import { gzipSync } from "node:zlib";
|
|
6
7
|
import path from "node:path";
|
|
7
8
|
import { readFileSync } from "node:fs";
|
|
8
9
|
import { createRequire } from "node:module";
|
|
@@ -50,6 +51,47 @@ function allowInspector(policy, url) {
|
|
|
50
51
|
}
|
|
51
52
|
return directives.join("; ");
|
|
52
53
|
}
|
|
54
|
+
/** Text-like types that shrink well; images and fonts are compressed already. */
|
|
55
|
+
const COMPRESSIBLE = /^(?:text\/|application\/(?:json|javascript|xml|wasm)|image\/svg\+xml)/i;
|
|
56
|
+
/** Snapshot and noVNC buffers never change, so validators and gzip copies are made once. */
|
|
57
|
+
const encodings = new WeakMap();
|
|
58
|
+
function encoded(data, type) {
|
|
59
|
+
let entry = encodings.get(data);
|
|
60
|
+
if (!entry) {
|
|
61
|
+
// Weak: the identity and gzip bodies share it.
|
|
62
|
+
entry = { etag: `W/"${createHash("sha256").update(data).digest("base64url").slice(0, 27)}"` };
|
|
63
|
+
if (data.length >= 1024 && COMPRESSIBLE.test(type)) {
|
|
64
|
+
const gzip = gzipSync(data);
|
|
65
|
+
if (gzip.length < data.length)
|
|
66
|
+
entry.gzip = gzip;
|
|
67
|
+
}
|
|
68
|
+
encodings.set(data, entry);
|
|
69
|
+
}
|
|
70
|
+
return entry;
|
|
71
|
+
}
|
|
72
|
+
/** Serves fixed bytes with a validator (304 on a match) and gzip when the browser takes it. */
|
|
73
|
+
function sendBytes(req, res, status, headers, data) {
|
|
74
|
+
const file = encoded(data, String(headers["content-type"] ?? ""));
|
|
75
|
+
headers.etag = file.etag;
|
|
76
|
+
headers.vary = "Accept-Encoding";
|
|
77
|
+
if (status === 200 && (req.headers["if-none-match"] ?? "").split(",").some((tag) => tag.trim() === file.etag)) {
|
|
78
|
+
res.writeHead(304, headers);
|
|
79
|
+
res.end();
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const gzip = file.gzip && /\bgzip\b/i.test(req.headers["accept-encoding"] ?? "") ? file.gzip : undefined;
|
|
83
|
+
if (gzip)
|
|
84
|
+
headers["content-encoding"] = "gzip";
|
|
85
|
+
const body = gzip ?? data;
|
|
86
|
+
res.writeHead(status, { ...headers, "content-length": body.length });
|
|
87
|
+
res.end(req.method === "HEAD" ? undefined : body);
|
|
88
|
+
}
|
|
89
|
+
/** App content may sit in the browser's own cache, never a shared one, and is
|
|
90
|
+
* revalidated on every use whatever the app says: a preview must show what the
|
|
91
|
+
* agent just built. The app's validators still turn unchanged files into 304s. */
|
|
92
|
+
function privateCache(policy) {
|
|
93
|
+
return /\bno-store\b/i.test(policy ?? "") ? "no-store" : "private, no-cache";
|
|
94
|
+
}
|
|
53
95
|
const HOUR = 60 * 60_000;
|
|
54
96
|
/** A viewer that stops reading doesn't make the node buffer the display. */
|
|
55
97
|
const DISPLAY_BUFFER_BYTES = 4 * 1024 * 1024;
|
|
@@ -125,8 +167,8 @@ function responseHeaders(headers, ancestors, inspector) {
|
|
|
125
167
|
.map((cookie) => cookie.replace(/;\s*domain=[^;]*/ig, ""));
|
|
126
168
|
}
|
|
127
169
|
// Preview data must not become a shared proxy cache entry. Apps cannot frame
|
|
128
|
-
// Bivy
|
|
129
|
-
result["cache-control"] = "
|
|
170
|
+
// Bivy or retain an opener; service workers are refused (see handle).
|
|
171
|
+
result["cache-control"] = privateCache(headers["cache-control"]);
|
|
130
172
|
result["referrer-policy"] = "no-referrer";
|
|
131
173
|
result["cross-origin-opener-policy"] = "same-origin";
|
|
132
174
|
result["x-content-type-options"] = "nosniff";
|
|
@@ -474,8 +516,7 @@ fetch('${REDEEM_PATH}',{method:'POST',headers:{'Content-Type':'text/plain'},body
|
|
|
474
516
|
headers["content-type"] = MIME[path.extname(file).toLowerCase()] || "application/octet-stream";
|
|
475
517
|
if (inspect && path.extname(file).toLowerCase() === ".html" && data.length <= MAX_INJECT_BYTES)
|
|
476
518
|
data = withInspector(data);
|
|
477
|
-
res
|
|
478
|
-
res.end(req.method === "HEAD" ? undefined : data);
|
|
519
|
+
sendBytes(req, res, status, headers, data);
|
|
479
520
|
return;
|
|
480
521
|
}
|
|
481
522
|
if (entry.target.kind !== "service") {
|
|
@@ -573,8 +614,7 @@ fetch('${REDEEM_PATH}',{method:'POST',headers:{'Content-Type':'text/plain'},body
|
|
|
573
614
|
res.end();
|
|
574
615
|
return;
|
|
575
616
|
}
|
|
576
|
-
res
|
|
577
|
-
res.end(req.method === "HEAD" ? undefined : data);
|
|
617
|
+
sendBytes(req, res, 200, { ...responseHeaders({}, this.ancestors(id)), "content-type": "text/javascript; charset=utf-8" }, data);
|
|
578
618
|
return;
|
|
579
619
|
}
|
|
580
620
|
if (!isPageLoad(req)) {
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/** VNC button-mask bits: buttons, then the wheel's four directions. */
|
|
2
|
+
const BUTTONS = { left: 1, middle: 2, right: 4 };
|
|
3
|
+
const WHEEL = { up: 8, down: 16, left: 32, right: 64 };
|
|
4
|
+
/** X keysyms by name. Super is ⌘ on a Mac. */
|
|
5
|
+
const MODIFIERS = {
|
|
6
|
+
shift: 0xffe1, ctrl: 0xffe3, control: 0xffe3, alt: 0xffe9, option: 0xffe9, opt: 0xffe9,
|
|
7
|
+
cmd: 0xffeb, command: 0xffeb, super: 0xffeb, meta: 0xffeb, win: 0xffeb,
|
|
8
|
+
};
|
|
9
|
+
const KEYS = {
|
|
10
|
+
enter: 0xff0d, return: 0xff0d, tab: 0xff09, escape: 0xff1b, esc: 0xff1b, backspace: 0xff08, delete: 0xffff, insert: 0xff63,
|
|
11
|
+
home: 0xff50, end: 0xff57, pageup: 0xff55, pagedown: 0xff56, left: 0xff51, up: 0xff52, right: 0xff53, down: 0xff54,
|
|
12
|
+
space: 0x20, plus: 0x2b, ...Object.fromEntries(Array.from({ length: 12 }, (_, i) => [`f${i + 1}`, 0xffbe + i])),
|
|
13
|
+
};
|
|
14
|
+
/** Characters typed as keys rather than as text. */
|
|
15
|
+
const TYPED = { "\n": KEYS.enter, "\t": KEYS.tab };
|
|
16
|
+
const PAUSE = 30;
|
|
17
|
+
const keysym = (char) => { const cp = char.codePointAt(0); return cp <= 0xff ? cp : 0x0100_0000 + cp; };
|
|
18
|
+
const press = (sym) => [{ key: { keysym: sym, down: true } }, { key: { keysym: sym, down: false } }];
|
|
19
|
+
/** "cmd+shift+s", "ctrl+c", "enter": modifiers held around one key. */
|
|
20
|
+
function combo(keys) {
|
|
21
|
+
const parts = keys.toLowerCase().split("+").map((part) => part.trim());
|
|
22
|
+
const key = parts.pop();
|
|
23
|
+
const held = parts.map((part) => {
|
|
24
|
+
const sym = MODIFIERS[part];
|
|
25
|
+
if (sym === undefined)
|
|
26
|
+
throw new Error(`Unknown modifier "${part}". Use ${[...new Set(Object.keys(MODIFIERS))].join(", ")}.`);
|
|
27
|
+
return sym;
|
|
28
|
+
});
|
|
29
|
+
// A letter with Shift is its capital, as a keyboard sends it.
|
|
30
|
+
const char = [...key].length === 1 ? (held.includes(MODIFIERS.shift) ? key.toUpperCase() : key) : undefined;
|
|
31
|
+
const sym = KEYS[key] ?? MODIFIERS[key] ?? (char && keysym(char));
|
|
32
|
+
if (!sym)
|
|
33
|
+
throw new Error(`Unknown key "${key}". Use a character or ${Object.keys(KEYS).join(", ")}.`);
|
|
34
|
+
return [...held.map((s) => ({ key: { keysym: s, down: true } })), ...press(sym), ...held.reverse().map((s) => ({ key: { keysym: s, down: false } }))];
|
|
35
|
+
}
|
|
36
|
+
const point = (n) => Number.isInteger(n) && n >= 0 && n < 65536;
|
|
37
|
+
/** Checks an action from the command line or API. */
|
|
38
|
+
export function readAction(input) {
|
|
39
|
+
const a = (input ?? {});
|
|
40
|
+
const at = (...keys) => { if (!keys.every((k) => point(a[k])))
|
|
41
|
+
throw new Error("Coordinates must be whole pixels in the app's screenshot."); };
|
|
42
|
+
switch (a.kind) {
|
|
43
|
+
case "click":
|
|
44
|
+
at("x", "y");
|
|
45
|
+
if (a.button !== undefined && !(typeof a.button === "string" && a.button in BUTTONS))
|
|
46
|
+
throw new Error("Button must be left, middle or right.");
|
|
47
|
+
if (a.count !== undefined && ![1, 2, 3].includes(a.count))
|
|
48
|
+
throw new Error("Click count must be 1, 2 or 3.");
|
|
49
|
+
return a;
|
|
50
|
+
case "move":
|
|
51
|
+
at("x", "y");
|
|
52
|
+
return a;
|
|
53
|
+
case "drag":
|
|
54
|
+
at("x", "y", "toX", "toY");
|
|
55
|
+
return a;
|
|
56
|
+
case "scroll":
|
|
57
|
+
at("x", "y");
|
|
58
|
+
if (!(typeof a.direction === "string" && a.direction in WHEEL))
|
|
59
|
+
throw new Error("Scroll direction must be up, down, left or right.");
|
|
60
|
+
if (a.steps !== undefined && !(Number.isInteger(a.steps) && a.steps >= 1 && a.steps <= 50))
|
|
61
|
+
throw new Error("Scroll steps must be 1–50.");
|
|
62
|
+
return a;
|
|
63
|
+
case "type":
|
|
64
|
+
if (typeof a.text !== "string" || !a.text || a.text.length > 2000)
|
|
65
|
+
throw new Error("Text must be 1–2000 characters.");
|
|
66
|
+
return a;
|
|
67
|
+
case "key":
|
|
68
|
+
if (typeof a.keys !== "string" || !a.keys.trim() || a.keys.length > 100)
|
|
69
|
+
throw new Error("Keys must name a key, like enter or cmd+s.");
|
|
70
|
+
combo(a.keys);
|
|
71
|
+
return a;
|
|
72
|
+
default: throw new Error("Action must be click, move, drag, scroll, type or key.");
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
/** The pointer and key events that perform an action. */
|
|
76
|
+
export function inputEvents(action) {
|
|
77
|
+
const at = (x, y, buttons = 0) => ({ pointer: { x, y, buttons } });
|
|
78
|
+
switch (action.kind) {
|
|
79
|
+
case "move": return [at(action.x, action.y)];
|
|
80
|
+
case "click": {
|
|
81
|
+
const button = BUTTONS[action.button ?? "left"];
|
|
82
|
+
return [at(action.x, action.y), ...Array.from({ length: action.count ?? 1 }, () => [at(action.x, action.y, button), at(action.x, action.y)]).flat()];
|
|
83
|
+
}
|
|
84
|
+
case "drag": {
|
|
85
|
+
// Through a few points on the way, so apps see a drag, not a jump.
|
|
86
|
+
const path = Array.from({ length: 8 }, (_, i) => at(Math.round(action.x + (action.toX - action.x) * (i + 1) / 8), Math.round(action.y + (action.toY - action.y) * (i + 1) / 8), 1));
|
|
87
|
+
return [at(action.x, action.y), at(action.x, action.y, 1), ...path.flatMap((step) => [{ wait: PAUSE }, step]), at(action.toX, action.toY)];
|
|
88
|
+
}
|
|
89
|
+
case "scroll": {
|
|
90
|
+
const bit = WHEEL[action.direction];
|
|
91
|
+
return [at(action.x, action.y), ...Array.from({ length: action.steps ?? 3 }, () => [at(action.x, action.y, bit), at(action.x, action.y)]).flat()];
|
|
92
|
+
}
|
|
93
|
+
case "type": return [...action.text].flatMap((char) => [...press(TYPED[char] ?? keysym(char)), { wait: 5 }]);
|
|
94
|
+
case "key": return combo(action.keys);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
2
|
+
// Copyright (c) 2026 Petter André Sjulstad
|
|
3
|
+
import fs from "node:fs";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import { createHash, randomBytes } from "node:crypto";
|
|
7
|
+
import { createInterface } from "node:readline";
|
|
8
|
+
import { fileURLToPath } from "node:url";
|
|
9
|
+
import { spawn, execFile, execFileSync } from "node:child_process";
|
|
10
|
+
/** The helper's source ships next to this module; it is compiled on this Mac on first use. */
|
|
11
|
+
const SOURCE = fileURLToPath(new URL("./macos-display.swift", import.meta.url));
|
|
12
|
+
const START_MS = 10_000;
|
|
13
|
+
const BUILD_MS = 5 * 60_000;
|
|
14
|
+
/** Darwin 22 is macOS 13, the first with everything the helper uses. */
|
|
15
|
+
const MIN_DARWIN = 22;
|
|
16
|
+
/** What each missing permission is called in System Settings. */
|
|
17
|
+
const PERMISSIONS = { screenRecording: "Screen Recording", accessibility: "Accessibility" };
|
|
18
|
+
/** Desktop views on macOS. There are no private displays on a Mac, so each
|
|
19
|
+
* view gets a helper that serves its app's own windows as a VNC display on a
|
|
20
|
+
* private socket (see macos-display.swift): the same contract as an Xvnc
|
|
21
|
+
* display, so the viewer, relay, screenshots and input are shared. */
|
|
22
|
+
export class MacDisplayHost {
|
|
23
|
+
options;
|
|
24
|
+
displays = new Map();
|
|
25
|
+
build;
|
|
26
|
+
checked;
|
|
27
|
+
prompted = false;
|
|
28
|
+
constructor(options) {
|
|
29
|
+
this.options = options;
|
|
30
|
+
}
|
|
31
|
+
compiled;
|
|
32
|
+
get source() { return this.options.source ?? SOURCE; }
|
|
33
|
+
binary() {
|
|
34
|
+
this.compiled ??= path.join(this.options.cacheDir, `macos-display-${createHash("sha256").update(fs.readFileSync(this.source)).digest("hex").slice(0, 16)}`);
|
|
35
|
+
return this.compiled;
|
|
36
|
+
}
|
|
37
|
+
/** Why desktop views can't run here, or undefined when they can. */
|
|
38
|
+
unavailable() {
|
|
39
|
+
if ((this.options.platform ?? process.platform) !== "darwin")
|
|
40
|
+
return "This display host runs on macOS only.";
|
|
41
|
+
if (Number((this.options.release ?? os.release()).split(".")[0]) < MIN_DARWIN)
|
|
42
|
+
return "Desktop app views need macOS 13 or later.";
|
|
43
|
+
const binary = this.binary();
|
|
44
|
+
if (fs.existsSync(binary))
|
|
45
|
+
return this.missing(binary);
|
|
46
|
+
if (!(this.options.swiftc ?? hasSwiftc)())
|
|
47
|
+
return "Desktop app views on macOS need Apple's command line tools to build their helper. Install them with: xcode-select --install";
|
|
48
|
+
void this.helper().catch(() => { }); // build ahead of the first open
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
/** Permissions the helper lacks, as a message; checked at most every few seconds. */
|
|
52
|
+
missing(binary) {
|
|
53
|
+
if (this.checked && Date.now() - this.checked.at < 5_000)
|
|
54
|
+
return this.checked.missing;
|
|
55
|
+
let missing;
|
|
56
|
+
try {
|
|
57
|
+
const granted = JSON.parse(execFileSync(binary, ["check"], { encoding: "utf8", timeout: 5_000 }));
|
|
58
|
+
const lacking = Object.keys(PERMISSIONS).filter((key) => !granted[key]).map((key) => PERMISSIONS[key]);
|
|
59
|
+
if (lacking.length) {
|
|
60
|
+
missing = `Desktop app views on macOS need ${lacking.join(" and ")} permission. In System Settings → Privacy & Security, allow ${lacking.join(" and ")} for the program that runs Bivy (your terminal app, or ${process.execPath} when Bivy runs in the background), then restart Bivy.`;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
catch {
|
|
64
|
+
missing = "Couldn't run Bivy's macOS display helper.";
|
|
65
|
+
}
|
|
66
|
+
this.checked = { at: Date.now(), missing };
|
|
67
|
+
return missing;
|
|
68
|
+
}
|
|
69
|
+
/** The compiled helper: built once per version of its source, then reused. */
|
|
70
|
+
helper() {
|
|
71
|
+
const binary = this.binary();
|
|
72
|
+
if (fs.existsSync(binary))
|
|
73
|
+
return Promise.resolve(binary);
|
|
74
|
+
this.build ??= new Promise((resolve, reject) => {
|
|
75
|
+
fs.mkdirSync(this.options.cacheDir, { recursive: true, mode: 0o700 });
|
|
76
|
+
const temp = `${binary}.${process.pid}.tmp`;
|
|
77
|
+
execFile("xcrun", ["swiftc", "-O", "-swift-version", "5", this.source, "-o", temp], { timeout: BUILD_MS }, (error, _stdout, stderr) => {
|
|
78
|
+
if (error) {
|
|
79
|
+
fs.rmSync(temp, { force: true });
|
|
80
|
+
reject(new Error(`Couldn't build Bivy's macOS display helper: ${String(stderr).trim().split("\n").at(-1) ?? error.message}`));
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
fs.renameSync(temp, binary);
|
|
84
|
+
for (const old of fs.readdirSync(this.options.cacheDir))
|
|
85
|
+
if (old.startsWith("macos-display-") && path.join(this.options.cacheDir, old) !== binary)
|
|
86
|
+
fs.rmSync(path.join(this.options.cacheDir, old), { force: true });
|
|
87
|
+
resolve(binary);
|
|
88
|
+
});
|
|
89
|
+
}).finally(() => { this.build = undefined; });
|
|
90
|
+
return this.build;
|
|
91
|
+
}
|
|
92
|
+
running(id) { return this.displays.get(id)?.ready; }
|
|
93
|
+
/** `scale` (1 or 2) is the pixel density frames are captured at. */
|
|
94
|
+
ensure(id, _name, scale = 1) {
|
|
95
|
+
const existing = this.displays.get(id);
|
|
96
|
+
if (existing)
|
|
97
|
+
return existing.ready;
|
|
98
|
+
const state = { dir: fs.mkdtempSync(path.join(os.tmpdir(), "bivy-display-")) };
|
|
99
|
+
state.ready = this.start(state, scale === 2 ? 2 : 1).then((display) => {
|
|
100
|
+
state.child.once("exit", () => { if (this.displays.get(id) === state)
|
|
101
|
+
this.stop(id); });
|
|
102
|
+
return display;
|
|
103
|
+
}, (error) => { this.stop(id); throw error; });
|
|
104
|
+
this.displays.set(id, state);
|
|
105
|
+
return state.ready;
|
|
106
|
+
}
|
|
107
|
+
async start(state, scale) {
|
|
108
|
+
const reason = this.unavailable();
|
|
109
|
+
if (reason)
|
|
110
|
+
throw new Error(reason);
|
|
111
|
+
const binary = await this.helper();
|
|
112
|
+
const missing = this.missing(binary);
|
|
113
|
+
if (missing) {
|
|
114
|
+
// Once per run, let macOS ask: it lists the program in System Settings.
|
|
115
|
+
if (!this.prompted) {
|
|
116
|
+
this.prompted = true;
|
|
117
|
+
execFile(binary, ["check", "--prompt"], () => { });
|
|
118
|
+
}
|
|
119
|
+
throw new Error(missing);
|
|
120
|
+
}
|
|
121
|
+
const socket = path.join(state.dir, "vnc.sock");
|
|
122
|
+
const token = randomBytes(16).toString("hex");
|
|
123
|
+
const child = spawn(binary, ["serve", socket, token, String(scale)], { stdio: ["ignore", "pipe", "pipe"] });
|
|
124
|
+
state.child = child;
|
|
125
|
+
const wm = { count: 0 };
|
|
126
|
+
let errors = "";
|
|
127
|
+
child.stderr.on("data", (chunk) => { errors = (errors + chunk).slice(-2000); console.warn(`[apps] macOS display: ${String(chunk).trim()}`); });
|
|
128
|
+
const up = await new Promise((resolve) => {
|
|
129
|
+
const timer = setTimeout(() => { child.kill("SIGKILL"); resolve(false); }, START_MS);
|
|
130
|
+
child.once("exit", () => { clearTimeout(timer); resolve(false); });
|
|
131
|
+
child.once("error", () => { clearTimeout(timer); resolve(false); });
|
|
132
|
+
createInterface({ input: child.stdout }).on("line", (line) => {
|
|
133
|
+
if (line === "ready") {
|
|
134
|
+
clearTimeout(timer);
|
|
135
|
+
resolve(true);
|
|
136
|
+
}
|
|
137
|
+
const windows = /^windows (\d+)$/.exec(line);
|
|
138
|
+
if (windows)
|
|
139
|
+
wm.count = Number(windows[1]);
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
if (!up)
|
|
143
|
+
throw new Error(`Couldn't start a display for this app.${errors ? ` ${errors.trim().split("\n").at(-1)}` : ""}`);
|
|
144
|
+
return { socket, wm, scale, env: { BIVY_MAC_DISPLAY: token }, launch: [binary, "run", "--"] };
|
|
145
|
+
}
|
|
146
|
+
stop(id) {
|
|
147
|
+
const state = this.displays.get(id);
|
|
148
|
+
if (!state)
|
|
149
|
+
return;
|
|
150
|
+
this.displays.delete(id);
|
|
151
|
+
state.child?.kill("SIGTERM");
|
|
152
|
+
fs.rmSync(state.dir, { recursive: true, force: true });
|
|
153
|
+
}
|
|
154
|
+
stopAll() { for (const id of [...this.displays.keys()])
|
|
155
|
+
this.stop(id); }
|
|
156
|
+
}
|
|
157
|
+
let swiftc;
|
|
158
|
+
function hasSwiftc() {
|
|
159
|
+
if (swiftc === undefined) {
|
|
160
|
+
try {
|
|
161
|
+
execFileSync("xcrun", ["--find", "swiftc"], { stdio: "ignore", timeout: 10_000 });
|
|
162
|
+
swiftc = true;
|
|
163
|
+
}
|
|
164
|
+
catch {
|
|
165
|
+
swiftc = false;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
return swiftc;
|
|
169
|
+
}
|