@vashp2029/glovebox 0.0.0-stage → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,319 @@
1
+ #!/usr/bin/env node
2
+ // glovebox command line: MCP server, CLI sessions, setup, and approvals.
3
+ import fs from "node:fs";
4
+ import { parseArgs } from "node:util";
5
+ import { loadConfig, paths, readUserConfig, VERSION, writeUserConfig } from "../src/config.mjs";
6
+
7
+ const USAGE = `glovebox ${VERSION} — scriptable computer use for coding agents
8
+
9
+ Usage:
10
+ glovebox setup [--rebuild] Build the helper and ask macOS for permissions
11
+ glovebox signing [--create] Make permission grants survive helper rebuilds
12
+ glovebox doctor Check that everything is ready
13
+ glovebox mcp Run the stdio MCP server (tools: js, js_reset)
14
+ glovebox run [code] [-f file] Run a script in a persistent CLI session
15
+ [--session name] [--title t] [--timeout ms]
16
+ Reads code from stdin when no code or file is given.
17
+ Screenshots are saved to files and printed as paths.
18
+ glovebox reset [--session name] Drop a CLI session's bindings
19
+ glovebox stop Stop the CLI daemon and its sessions
20
+ glovebox smoke [--full] Batched, persistent smoke test on Calculator
21
+ (--full adds background click/drag/scroll in TextEdit)
22
+ glovebox allow <app>... Always allow an app (name or bundle id)
23
+ glovebox deny <app>... Never allow an app
24
+ glovebox revoke <app>... Remove an app from both lists
25
+ glovebox approval <ask|allowlist|allow-all>
26
+ glovebox config Print the effective configuration
27
+ glovebox log [-n 20] Show recent actions
28
+ `;
29
+
30
+ function die(message) {
31
+ process.stderr.write(`${message}\n`);
32
+ process.exit(1);
33
+ }
34
+
35
+ function printResult(result) {
36
+ for (const block of result.content ?? []) {
37
+ if (block.type === "text") process.stdout.write(`${block.text}\n`);
38
+ }
39
+ if (!result.ok) process.exitCode = 1;
40
+ }
41
+
42
+ async function readStdin() {
43
+ if (process.stdin.isTTY) return "";
44
+ let data = "";
45
+ for await (const chunk of process.stdin) data += chunk;
46
+ return data;
47
+ }
48
+
49
+ function editList(list, apps, add) {
50
+ const config = readUserConfig();
51
+ const lower = apps.map((a) => a.toLowerCase());
52
+ const next = { ...config };
53
+ for (const key of ["allow", "deny"]) {
54
+ next[key] = (config[key] ?? []).filter((entry) => !lower.includes(entry.toLowerCase()));
55
+ }
56
+ if (add) next[list] = [...next[list], ...apps];
57
+ writeUserConfig(next);
58
+ return next;
59
+ }
60
+
61
+ async function doctor() {
62
+ const lines = [];
63
+ let ok = true;
64
+ const check = (pass, label, fix) => {
65
+ lines.push(`${pass ? "ok " : "FAIL"} ${label}${!pass && fix ? `\n ${fix}` : ""}`);
66
+ if (!pass) ok = false;
67
+ };
68
+ const [major, minor] = process.versions.node.split(".").map(Number);
69
+ check(major > 20 || (major === 20 && minor >= 10), `Node ${process.versions.node} (need 20.10+)`);
70
+ let config;
71
+ try {
72
+ config = loadConfig();
73
+ check(true, `Config ${fs.existsSync(paths.config) ? paths.config : "(defaults)"}, approval: ${config.approval}`);
74
+ } catch (error) {
75
+ check(false, "Config", error.message);
76
+ }
77
+ if (process.platform === "darwin") {
78
+ const helper = await import("../src/macos-helper.mjs");
79
+ const built = helper.isBuilt();
80
+ check(built, `Helper built at ${helper.appPath()}`, "Run: glovebox setup");
81
+ if (built) {
82
+ const { MacBackend } = await import("../src/backends/macos.mjs");
83
+ const backend = new MacBackend({ client: "glovebox doctor" });
84
+ try {
85
+ const ping = await backend.request("ping");
86
+ check(true, `Helper running (pid ${ping.pid}, v${ping.version})`);
87
+ const bg = ping.background ?? {};
88
+ const bgOk = bg.postEventRecord && bg.setIntegerField && bg.processSerialNumber;
89
+ lines.push(
90
+ bgOk
91
+ ? "ok Background input available (apps stay in the background)"
92
+ : `warn Background input unavailable on this macOS; apps will be brought forward (${Object.entries(bg).filter(([, v]) => !v).map(([k]) => k).join(", ") || "old helper"})`,
93
+ );
94
+ const perms = await backend.request("permissions", { prompt: false });
95
+ check(perms.accessibility, "Accessibility granted to Glovebox Helper", "System Settings > Privacy & Security > Accessibility: enable Glovebox Helper");
96
+ check(perms.screenRecording, "Screen Recording granted to Glovebox Helper", "System Settings > Privacy & Security > Screen & System Audio Recording: enable Glovebox Helper, then run: glovebox setup --restart");
97
+ // In testing, Escape only reached the helper with Input Monitoring.
98
+ lines.push(
99
+ perms.inputMonitoring
100
+ ? "ok Input Monitoring granted: pressing Escape stops a running script"
101
+ : "info Input Monitoring not granted (optional): Escape will not stop scripts; switching to another app still does",
102
+ );
103
+ } catch (error) {
104
+ check(false, "Helper reachable", error.message);
105
+ } finally {
106
+ backend.close();
107
+ }
108
+ }
109
+ } else {
110
+ check(false, `Platform ${process.platform}`, "Only macOS has a backend today. See docs/roadmap.md.");
111
+ }
112
+ process.stdout.write(`${lines.join("\n")}\n`);
113
+ if (!ok) process.exitCode = 1;
114
+ }
115
+
116
+ // Polls the helper until `check(perms)` holds, or gives up after `minutes`.
117
+ async function waitFor(backend, check, minutes = 5) {
118
+ const deadline = Date.now() + minutes * 60_000;
119
+ while (Date.now() < deadline) {
120
+ if (check(await backend.request("permissions", { prompt: false }))) return true;
121
+ await new Promise((r) => setTimeout(r, 1500));
122
+ }
123
+ return false;
124
+ }
125
+
126
+ // Builds the helper if needed, then walks through the macOS permissions one
127
+ // at a time: macOS drops a second prompt that arrives while the first is open.
128
+ async function setup(flags) {
129
+ if (process.platform !== "darwin") die(`No backend for ${process.platform} yet. See docs/roadmap.md.`);
130
+ const helper = await import("../src/macos-helper.mjs");
131
+ const { MacBackend } = await import("../src/backends/macos.mjs");
132
+ const say = (text) => process.stdout.write(`${text}\n`);
133
+ const rebuilding = flags.rebuild || !helper.isBuilt();
134
+ const hadBuild = fs.existsSync(helper.appPath());
135
+ if (rebuilding || flags.restart) await helper.quit();
136
+ if (rebuilding) await helper.build({ log: say });
137
+ if (rebuilding && hadBuild && !(await helper.signingIdentity())) {
138
+ // An ad-hoc signed rebuild is a new app to macOS; stale grants keep showing as on.
139
+ say("The helper was rebuilt ad hoc, so macOS treats it as a new app. If old grants still show as");
140
+ say("on but this step never finishes, clear them in another terminal and run setup again:");
141
+ say(` tccutil reset Accessibility ${helper.BUNDLE_ID}`);
142
+ say(` tccutil reset ScreenCapture ${helper.BUNDLE_ID}`);
143
+ say(` tccutil reset ListenEvent ${helper.BUNDLE_ID}`);
144
+ say("To stop this happening on every rebuild: glovebox signing --create");
145
+ }
146
+
147
+ let backend = new MacBackend({ client: "glovebox setup" });
148
+ let perms = await backend.request("permissions", { prompt: false });
149
+ if (!perms.accessibility) {
150
+ say('\n1/2 Accessibility: macOS is asking now. Turn on "Glovebox Helper" in');
151
+ say(" System Settings > Privacy & Security > Accessibility. Waiting...");
152
+ await backend.request("permissions", { prompt: true });
153
+ if (!(await waitFor(backend, (p) => p.accessibility))) die("Timed out waiting for Accessibility. Run glovebox setup again.");
154
+ say(" Accessibility granted.");
155
+ }
156
+ perms = await backend.request("permissions", { prompt: false });
157
+ if (!perms.screenRecording) {
158
+ say('\n2/2 Screen Recording: macOS is asking now. Turn on "Glovebox Helper" in');
159
+ say(" System Settings > Privacy & Security > Screen & System Audio Recording.");
160
+ say(" If macOS offers to quit and reopen it, accept. If no prompt appears, use + and add");
161
+ say(` ${helper.appPath()} (press Cmd+Shift+G in the file picker). Waiting...`);
162
+ await backend.request("permissions", { prompt: true });
163
+ if (!(await waitFor(backend, (p) => p.screenRecording))) die("Timed out waiting for Screen Recording. Run glovebox setup again.");
164
+ say(" Screen Recording granted.");
165
+ // Relaunch so the key monitor and capture start with the new grants.
166
+ backend.close();
167
+ await helper.quit();
168
+ backend = new MacBackend({ client: "glovebox setup" });
169
+ }
170
+ perms = await backend.request("permissions", { prompt: false });
171
+ if (!perms.inputMonitoring) {
172
+ say("\nOptional: Input Monitoring lets you stop a running script by pressing Escape.");
173
+ say("Without it, switching to another app (or js_reset) still stops a script.");
174
+ say('To enable it, turn on "Glovebox Helper" in System Settings > Privacy & Security >');
175
+ say(`Input Monitoring (use + and ${helper.appPath()} if it is not listed), then run`);
176
+ say("`glovebox setup --restart`.");
177
+ await backend.request("permissions", { prompt: true, inputMonitoring: true });
178
+ }
179
+ backend.close();
180
+ const ready = perms.accessibility && perms.screenRecording;
181
+ say(ready ? "\nReady. Check with `glovebox doctor`, then try `glovebox smoke`." : "\nNot ready yet; run `glovebox doctor` to see what is missing.");
182
+ if (!ready) process.exitCode = 1;
183
+ }
184
+
185
+ async function main() {
186
+ const [command, ...rest] = process.argv.slice(2);
187
+ const { values: flags, positionals } = parseArgs({
188
+ args: rest,
189
+ allowPositionals: true,
190
+ options: {
191
+ file: { type: "string", short: "f" },
192
+ session: { type: "string", short: "s" },
193
+ title: { type: "string", short: "t" },
194
+ timeout: { type: "string" },
195
+ rebuild: { type: "boolean" },
196
+ restart: { type: "boolean" },
197
+ create: { type: "boolean" },
198
+ full: { type: "boolean" },
199
+ n: { type: "string", short: "n" },
200
+ help: { type: "boolean", short: "h" },
201
+ },
202
+ });
203
+
204
+ switch (command) {
205
+ case undefined:
206
+ case "help":
207
+ case "--help":
208
+ case "-h":
209
+ process.stdout.write(USAGE);
210
+ return;
211
+ case "--version":
212
+ case "-v":
213
+ case "version":
214
+ process.stdout.write(`${VERSION}\n`);
215
+ return;
216
+ case "mcp": {
217
+ const { serveMcp } = await import("../src/mcp.mjs");
218
+ serveMcp();
219
+ return;
220
+ }
221
+ case "daemon": {
222
+ const { runDaemon } = await import("../src/daemon.mjs");
223
+ runDaemon();
224
+ return;
225
+ }
226
+ case "run": {
227
+ const code = flags.file ? fs.readFileSync(flags.file, "utf8") : positionals.join(" ") || (await readStdin());
228
+ if (!code.trim()) die("glovebox run: no code. Pass it as an argument, with -f file, or on stdin.");
229
+ const { daemonRequest } = await import("../src/daemon.mjs");
230
+ printResult(
231
+ await daemonRequest({
232
+ op: "run",
233
+ code,
234
+ title: flags.title,
235
+ session: flags.session,
236
+ timeoutMs: flags.timeout ? Number(flags.timeout) : undefined,
237
+ client: process.env.GLOVEBOX_CLIENT,
238
+ }),
239
+ );
240
+ return;
241
+ }
242
+ case "reset":
243
+ case "stop": {
244
+ const { daemonRequest } = await import("../src/daemon.mjs");
245
+ try {
246
+ printResult(await daemonRequest({ op: command, session: flags.session }, { autostart: false }));
247
+ } catch {
248
+ process.stdout.write("No daemon running.\n");
249
+ }
250
+ return;
251
+ }
252
+ case "smoke": {
253
+ const { runSmoke } = await import("../src/smoke.mjs");
254
+ process.exitCode = (await runSmoke({ full: flags.full })) ? 0 : 1;
255
+ return;
256
+ }
257
+ case "setup":
258
+ await setup(flags);
259
+ return;
260
+ case "signing": {
261
+ if (process.platform !== "darwin") die("Signing applies to the macOS helper only.");
262
+ const helper = await import("../src/macos-helper.mjs");
263
+ let identity = await helper.signingIdentity();
264
+ if (!identity && flags.create) {
265
+ await helper.createSigningIdentity({ log: (m) => process.stdout.write(`${m}\n`) });
266
+ identity = await helper.signingIdentity();
267
+ }
268
+ if (identity) {
269
+ process.stdout.write(`Signing identity "${helper.SIGNING_NAME}" (${identity}) is ready.\n`);
270
+ process.stdout.write("Run `glovebox setup --rebuild` once to re-sign the helper with it, and grant permissions one last time.\n");
271
+ } else {
272
+ process.stdout.write(
273
+ [
274
+ "No local signing identity. The helper is signed ad hoc, so every rebuild resets its macOS permissions.",
275
+ `Create one (a self-signed certificate named "${helper.SIGNING_NAME}" in your login keychain):`,
276
+ " glovebox signing --create",
277
+ "",
278
+ ].join("\n"),
279
+ );
280
+ }
281
+ return;
282
+ }
283
+ case "doctor":
284
+ await doctor();
285
+ return;
286
+ case "allow":
287
+ case "deny":
288
+ case "revoke": {
289
+ if (positionals.length === 0) die(`glovebox ${command}: name at least one app`);
290
+ const next = editList(command, positionals, command !== "revoke");
291
+ process.stdout.write(`allow: ${JSON.stringify(next.allow)}\ndeny: ${JSON.stringify(next.deny)}\n`);
292
+ process.stdout.write("Running sessions keep approvals they already have; js_reset or glovebox reset to re-check.\n");
293
+ return;
294
+ }
295
+ case "approval": {
296
+ const mode = positionals[0];
297
+ if (!["ask", "allowlist", "allow-all"].includes(mode)) die("glovebox approval: use ask, allowlist, or allow-all");
298
+ writeUserConfig({ approval: mode });
299
+ process.stdout.write(`approval: ${mode}\n`);
300
+ if (mode === "allow-all") {
301
+ process.stdout.write("Warning: agents can now control any app without asking, except blocked apps (terminals, password managers, System Settings).\n");
302
+ }
303
+ return;
304
+ }
305
+ case "config":
306
+ process.stdout.write(`${JSON.stringify(loadConfig(), null, 2)}\n`);
307
+ return;
308
+ case "log": {
309
+ const n = Number(flags.n ?? 20);
310
+ const lines = fs.existsSync(paths.log) ? fs.readFileSync(paths.log, "utf8").trim().split("\n").slice(-n) : [];
311
+ process.stdout.write(lines.length ? `${lines.join("\n")}\n` : "No actions logged yet.\n");
312
+ return;
313
+ }
314
+ default:
315
+ die(`Unknown command "${command}".\n\n${USAGE}`);
316
+ }
317
+ }
318
+
319
+ main().catch((error) => die(error.message));
package/docs/agents.md ADDED
@@ -0,0 +1,124 @@
1
+ # Attaching glovebox to an agent
2
+
3
+ glovebox speaks MCP over stdio (`glovebox mcp`) and also has a CLI
4
+ (`glovebox run`) for agents that can only run shell commands. Both expose the
5
+ same persistent JavaScript session.
6
+
7
+ Install once, then attach to as many agents as you like:
8
+
9
+ ```bash
10
+ npm install -g github:vpatel9202/glovebox
11
+ glovebox setup # macOS: builds Glovebox Helper.app and walks through permissions
12
+ glovebox doctor # everything should say ok
13
+ glovebox smoke # optional: a real end-to-end check on Calculator
14
+ ```
15
+
16
+ If `glovebox` is not on your PATH (for example, under a version manager), use
17
+ the absolute path from `which glovebox`, or run it as
18
+ `node /path/to/glovebox/bin/glovebox.mjs mcp`.
19
+
20
+ The MCP server exposes two tools: `js` (run a script) and `js_reset` (drop the
21
+ session). Most agents name them `<server>__js` or `mcp__<server>__js`.
22
+
23
+ ## Claude Code
24
+
25
+ ```bash
26
+ claude mcp add --scope user glovebox -- glovebox mcp
27
+ ```
28
+
29
+ Check with `/mcp` in a session. To teach Claude the loop, copy the skill:
30
+
31
+ ```bash
32
+ mkdir -p ~/.claude/skills && cp -R "$(npm root -g)/@vashp2029/glovebox/skills/glovebox" ~/.claude/skills/
33
+ ```
34
+
35
+ ## Grok CLI
36
+
37
+ Add to `~/.grok/config.toml`, then start a new session:
38
+
39
+ ```toml
40
+ [mcp_servers.glovebox]
41
+ command = "glovebox"
42
+ args = ["mcp"]
43
+ enabled = true
44
+ ```
45
+
46
+ Verify with `grok mcp doctor glovebox`. This config, and the OpenCode and
47
+ Codex ones below, follow each client's documented MCP format but have not been
48
+ exercised end to end. Skill:
49
+
50
+ ```bash
51
+ mkdir -p ~/.grok/skills && cp -R "$(npm root -g)/@vashp2029/glovebox/skills/glovebox" ~/.grok/skills/
52
+ ```
53
+
54
+ ## OpenCode
55
+
56
+ Add to `opencode.json` (project) or `~/.config/opencode/opencode.json` (global):
57
+
58
+ ```json
59
+ {
60
+ "$schema": "https://opencode.ai/config.json",
61
+ "mcp": {
62
+ "glovebox": {
63
+ "type": "local",
64
+ "command": ["glovebox", "mcp"],
65
+ "enabled": true
66
+ }
67
+ }
68
+ }
69
+ ```
70
+
71
+ ## Codex CLI
72
+
73
+ Add to `~/.codex/config.toml`:
74
+
75
+ ```toml
76
+ [mcp_servers.glovebox]
77
+ command = "glovebox"
78
+ args = ["mcp"]
79
+ ```
80
+
81
+ ## Gemini CLI, Cursor, and other MCP clients
82
+
83
+ Most clients accept the common `mcpServers` JSON shape:
84
+
85
+ ```json
86
+ {
87
+ "mcpServers": {
88
+ "glovebox": { "command": "glovebox", "args": ["mcp"] }
89
+ }
90
+ }
91
+ ```
92
+
93
+ ## Agents without MCP: the CLI
94
+
95
+ `glovebox run` sends a script to a background daemon that keeps the session
96
+ alive between calls, so bindings persist exactly as they do over MCP.
97
+
98
+ ```bash
99
+ glovebox run 'const calc = await gb.getApp("Calculator"); await calc.typeText("2+2="); await calc.getAXState()'
100
+ glovebox run 'await calc.getAXState()' # calc is still bound
101
+ glovebox run -f step.js --title "fill form" # or from a file, or stdin
102
+ glovebox reset # drop the bindings
103
+ ```
104
+
105
+ Screenshots are written to `~/.glovebox/screenshots/` and the path is printed,
106
+ so an agent that can read image files can look at them. Use `--session name`
107
+ to keep separate sessions. The daemon exits after an hour idle
108
+ (`daemon.idleMinutes` in the config).
109
+
110
+ ## Approval prompts
111
+
112
+ The first time a session touches an app, glovebox shows a native dialog:
113
+ **Allow for This Session**, **Always Allow**, or **Deny**. "Session" means one
114
+ MCP connection or one CLI session. To pre-approve apps without dialogs:
115
+
116
+ ```bash
117
+ glovebox allow Calculator TextEdit
118
+ glovebox approval allowlist # never prompt; only allowlisted apps work
119
+ ```
120
+
121
+ `glovebox approval allow-all` turns prompts off for every app except the
122
+ blocked ones (terminals, password managers, and a few others). Those need an
123
+ explicit `glovebox allow`. The full list is in
124
+ [SECURITY.md](../SECURITY.md#blocked-apps).
package/docs/api.md ADDED
@@ -0,0 +1,150 @@
1
+ # glovebox API
2
+
3
+ Scripts run in one persistent JavaScript session. Top-level `const`, `let`,
4
+ `function`, `class`, and `await` persist until `js_reset` (MCP) or
5
+ `glovebox reset` (CLI). Redeclaring a name is allowed. `gb` is the entry
6
+ point; `cua` is an alias.
7
+
8
+ ```js
9
+ const calc = await gb.getApp("Calculator");
10
+ await calc.pressKey("Escape");
11
+ await calc.typeText("12*3=");
12
+ await calc.getAXState();
13
+ ```
14
+
15
+ ## gb
16
+
17
+ - `gb.getApp(query, {activate, foreground})` returns an App handle. `query` is
18
+ a display name, a bundle id, or `"pid:123"`. Launches the app if it is not
19
+ running, without bringing it to the front unless `activate: true`.
20
+ `foreground: true` makes every input to the handle use the foreground
21
+ route; it takes effect only on the call that first creates the handle.
22
+ The first action in each app may show an approval dialog to the user.
23
+ - `gb.listApps({installed, emit})` lists running apps (or installed ones too)
24
+ and returns `[{name, bundleId, pid, running, active}]`. It prints the list
25
+ unless `emit: false`.
26
+ - `gb.sleep(ms)` waits. Also available as `sleep(ms)`.
27
+ - `gb.permissions()` reports Accessibility and Screen Recording status.
28
+ - `help(topic)` prints this document, or the sections mentioning `topic`.
29
+ - `console.log(...)` and `nodeRepl.write(value)` add text to the result. The
30
+ value of the last expression is added too, unless a read already printed it.
31
+
32
+ ## Reading state
33
+
34
+ - `app.name`, `app.bundleId`, and `app.pid` identify the handle.
35
+ `app.view` is `{window, width, height, scale, ...}` for the window last read
36
+ (`null` before the first read); `scale` is image pixels per screen point.
37
+ - `app.getAXState({disableDiffing, frames, menus, window, maxNodes, maxDepth, emit})`
38
+ returns the accessibility tree as text. The first read of a window is the
39
+ full tree; later reads print only lines that changed (`~`), were added
40
+ (`+`), or a list of removed indices. `disableDiffing: true` prints the full
41
+ tree. `frames: true` appends `@x,y wxh` in image pixels. `menus: true` adds
42
+ the menu bar. `window` picks a window index from `app.windows()`.
43
+ Without `window`, reads and clicks use the focused window if it is a normal
44
+ one, else the window last used, else the main window; tooltips, popovers,
45
+ and rename fields never take over. Diffs compare against the last printed
46
+ read of the same window; `{emit: false}` reads don't change that.
47
+ `maxNodes` (default 800) and `maxDepth` (default 40) bound the tree; a
48
+ truncated tree says so.
49
+ - Line format: `<index> <role> "<label>" value="..." id=... help="..."
50
+ (focused, selected, disabled, settable) actions: ...`. Indentation is depth.
51
+ Indices are stable for the life of the session, so a diff means what it says.
52
+ - `app.getScreenshot({window, format, quality, emit})` returns
53
+ `{width, height, mimeType, data}` and adds the image to the result. Over the
54
+ CLI the image is saved to a file and the path is printed.
55
+ - `app.getAXStateAndScreenshot(options)` does both.
56
+ - `app.windows()` lists windows with their frames and image size.
57
+ - `app.frame(index)` and `app.center(index)` give an element's rectangle or
58
+ center in image pixels.
59
+ - Every read emits by default. Pass `{emit: false}` to only return the value.
60
+ - `app.waitForChange({timeout, window})` waits silently until the window
61
+ differs from the last printed read (pixels after a screenshot, the tree
62
+ after a tree read), then settles. Returns `{changed, ms}`; `changed: false`
63
+ at the timeout (default 10000 ms). Use it after an action in an app that
64
+ updates late. A blinking caret counts as a change in pixel mode.
65
+ - `app.settle({timeout, stableFor, window})` waits until two screenshots in a
66
+ row match (ignoring a blinking caret) and returns `{stable, ms}`. Nothing is
67
+ printed; read afterwards.
68
+
69
+ ## Coordinates
70
+
71
+ A target is either an element index (a number) or `[x, y]` in **image
72
+ pixels** of the window: `(0, 0)` is the window's top-left corner, and the
73
+ space is exactly the screenshot's pixels. The scale is fixed by the window
74
+ size and display, so `frames`, `center`, screenshots, and clicks always
75
+ agree, even if no screenshot was taken. The header of each read prints the
76
+ image size. If the window is resized, read again before clicking by pixels.
77
+
78
+ ## Actions
79
+
80
+ Input stays in the background: the app is not brought to the front, the
81
+ user's real cursor does not move, and a drawn agent cursor shows where each
82
+ action lands. If this macOS or an app can't take background input, glovebox
83
+ brings the app forward instead. `app.lastRoute` says how the last action got
84
+ there: `accessibility`, `background`, `background-skylight` (Chromium and
85
+ Electron), or `foreground`, with " (focus restored)" appended if the app took
86
+ focus and glovebox gave it back. Pass
87
+ `{foreground: true}` to `click`, `doubleClick`, `rightClick`, `pressKey`,
88
+ `typeText`, `scroll`, or `drag`, or `gb.getApp(name, {foreground: true})` for
89
+ every action, for apps that ignore background input (some Java, Tk, and game
90
+ windows).
91
+
92
+ - `app.click(target, {button, count, modifiers, raw, foreground})`. A plain
93
+ left click on a button, checkbox, link, or menu item is pressed through
94
+ accessibility. Anything else is a mouse click sent to the app's window.
95
+ `raw: true` uses the mouse, except on an open menu's items in the
96
+ background, which only accessibility reaches. A click on a menu item waits
97
+ for the menu to close. `modifiers` is a list such as `["shift"]`.
98
+ - `app.doubleClick(target)`, `app.rightClick(target)`.
99
+ - `app.pressKey(keys, {repeat})`. xdotool syntax: `Return`, `Tab`, `Escape`,
100
+ `BackSpace`, `Delete`, `Up`, `Page_Down`, `F5`, `ctrl+a`, `shift+Tab`,
101
+ `super+c` (super is Cmd on macOS). Space-separated keys press in sequence:
102
+ `"ctrl+a BackSpace"`. Keys go to the app's key window.
103
+ - `app.typeText(text)`. Types literal text; `\n` presses Return and `\t`
104
+ presses Tab.
105
+ - `app.scroll(target, dy = 3, {dx})` or `app.scroll(target, {dy, dx})`.
106
+ Positive `dy` scrolls down, in lines.
107
+ - `app.drag(from, to)` or `app.drag([[x, y], [x, y], ...])`.
108
+ - `app.setValue(index, value)` sets a field's value directly. Finder file
109
+ names refuse it (Finder would relabel without renaming); rename with a
110
+ click on the item, `pressKey("Return")`, `typeText(name)`,
111
+ `pressKey("Return")`, which works in the background.
112
+ - `app.focus(index)` gives an element keyboard focus.
113
+ - `app.selectText(index, {start, length})` selects text in a field.
114
+ - `app.listActions(index)` and `app.performSecondaryAction(index, name)`
115
+ run an accessibility action such as `"Show Menu"` or `"Increment"`.
116
+ - `app.activate()` brings the app to the front (rarely needed).
117
+
118
+ ## Approval and stopping
119
+
120
+ The first time a session touches an app, the user sees a dialog: Allow for
121
+ This Session, Always Allow, or Deny. Terminals, password managers, and System
122
+ Settings are blocked unless the user runs `glovebox allow <app>`. If a script
123
+ brought an app forward and the user switches away, the script stops with a
124
+ "Stopped: ..." message. Background scripts keep running while the user works in
125
+ other apps. Escape stops any script, but only when the user has granted Input
126
+ Monitoring. Do not continue GUI work until the user asks.
127
+ On the foreground route a script brings an app forward at most once per call.
128
+ Every action is appended to `~/.glovebox/actions.log` (typed text is logged
129
+ as a length only).
130
+
131
+ ## Known limits
132
+
133
+ - **Pop-up menus and your keyboard.** While a background app's pop-up menu is
134
+ open, macOS menu tracking is system-wide, so what you type goes to that app. A
135
+ word you type can land in its field. Keep the time the menu is open short: open
136
+ it and pick the item in one script, with no read in between. A dedicated
137
+ `app.select(popup, item)` is planned.
138
+ - **Window controls during background typing.** A target window can show active
139
+ window controls while it receives background keys, even though your app stays
140
+ frontmost and keeps your keyboard.
141
+ - **One-node trees.** Remote desktops such as Citrix expose a single accessibility
142
+ node, so work there runs on screenshots and pixel clicks.
143
+
144
+ ## Tips
145
+
146
+ - Bind each app once and reuse the handle; define helpers in the session.
147
+ - Batch a whole step into one call: several actions, then one read.
148
+ - Read the tree first. Use screenshots only when the tree has nothing useful
149
+ (remote desktops such as Citrix, canvases, games) or you need to see layout.
150
+ - When the tree is a single node, work in pixels from the latest screenshot.