@vashp2029/glovebox 0.0.0-stage → 0.3.0

package/src/axformat.mjs

@@ -0,0 +1,158 @@
+// Turns a backend's accessibility node list into compact text, and diffs it
+// against the previous read so repeat reads cost a few lines, not a tree.
+//
+// Node shape from a backend: {i, d, role, subrole?, title?, value?, desc?,
+// help?, placeholder?, id?, frame?: [x,y,w,h] screen points, actions?,
+// focused?, selected?, disabled?, settable?}
+
+const ROLE_NAMES = {
+  AXStaticText: "text",
+  AXTextField: "text field",
+  AXTextArea: "text area",
+  AXSecureTextField: "secure text field",
+  AXButton: "button",
+  AXCheckBox: "checkbox",
+  AXRadioButton: "radio button",
+  AXPopUpButton: "pop up button",
+  AXMenuButton: "menu button",
+  AXComboBox: "combo box",
+  AXScrollArea: "scroll area",
+  AXScrollBar: "scroll bar",
+  AXSplitGroup: "split group",
+  AXSplitter: "splitter",
+  AXGroup: "group",
+  AXWindow: "window",
+  AXSheet: "sheet",
+  AXToolbar: "toolbar",
+  AXTabGroup: "tab group",
+  AXTable: "table",
+  AXOutline: "outline",
+  AXRow: "row",
+  AXColumn: "column",
+  AXCell: "cell",
+  AXList: "list",
+  AXImage: "image",
+  AXLink: "link",
+  AXSlider: "slider",
+  AXMenuBar: "menu bar",
+  AXMenuBarItem: "menu bar item",
+  AXMenu: "menu",
+  AXMenuItem: "menu item",
+  AXWebArea: "web area",
+  AXHeading: "heading",
+  AXProgressIndicator: "progress indicator",
+  AXValueIndicator: "value indicator",
+  AXDisclosureTriangle: "disclosure triangle",
+  AXIncrementor: "stepper",
+  AXLayoutArea: "layout area",
+  AXUnknown: "element",
+};
+
+const SUBROLE_NAMES = {
+  AXCloseButton: "close button",
+  AXMinimizeButton: "minimize button",
+  AXZoomButton: "zoom button",
+  AXFullScreenButton: "full screen button",
+  AXSearchField: "search field",
+  AXStandardWindow: "window",
+  AXDialog: "dialog",
+  AXToggle: "toggle",
+  AXSwitch: "switch",
+  AXTabButton: "tab",
+};
+
+function roleName(node) {
+  if (node.subrole && SUBROLE_NAMES[node.subrole]) return SUBROLE_NAMES[node.subrole];
+  if (ROLE_NAMES[node.role]) return ROLE_NAMES[node.role];
+  return node.role
+    .replace(/^AX/, "")
+    .replace(/([a-z])([A-Z])/g, "$1 $2")
+    .toLowerCase();
+}
+
+function quote(text) {
+  // Scroll positions and sliders report floats like 0.1851851791143417.
+  if (/^-?\d+\.\d{4,}$/.test(text)) text = String(Number(Number(text).toFixed(3)));
+  return JSON.stringify(text.replace(/\s+/g, " ").trim());
+}
+
+// Actions that are implied by the role or covered by click()/scroll().
+const QUIET_ACTIONS = new Set([
+  "AXPress",
+  "AXRaise",
+  "AXCancel",
+  "AXConfirm",
+  "AXScrollLeftByPage",
+  "AXScrollRightByPage",
+  "AXScrollUpByPage",
+  "AXScrollDownByPage",
+]);
+
+function actionName(action) {
+  return action.replace(/^AX/, "").replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase();
+}
+
+// One node as one line, without indentation. `toImage` maps a screen-point
+// frame to image pixels when frames are requested.
+export function nodeLine(node, { toImage } = {}) {
+  const parts = [String(node.i), roleName(node)];
+  const label = node.title ?? node.desc;
+  if (label) parts.push(quote(label));
+  if (node.title && node.desc && node.desc !== node.title) parts.push(`desc=${quote(node.desc)}`);
+  if (node.value != null && node.value !== label) parts.push(`value=${quote(node.value)}`);
+  if (node.placeholder) parts.push(`placeholder=${quote(node.placeholder)}`);
+  if (node.id) parts.push(`id=${node.id}`);
+  if (node.help && node.help !== label) parts.push(`help=${quote(node.help)}`);
+  const flags = [];
+  if (node.focused) flags.push("focused");
+  if (node.selected) flags.push("selected");
+  if (node.disabled) flags.push("disabled");
+  if (node.settable && /text|combo|slider|search/.test(roleName(node))) flags.push("settable");
+  if (flags.length) parts.push(`(${flags.join(", ")})`);
+  const extra = (node.actions ?? []).filter((a) => !QUIET_ACTIONS.has(a) && !a.startsWith("Name:"));
+  if (extra.length) parts.push(`actions: ${extra.map(actionName).join(", ")}`);
+  if (toImage && node.frame) {
+    const [x, y, w, h] = toImage(node.frame).map((v) => Math.round(v));
+    parts.push(`@${x},${y} ${w}x${h}`);
+  }
+  return parts.join(" ");
+}
+
+export function formatTree(nodes, options = {}) {
+  return nodes.map((node) => `${"\t".repeat(node.d)}${nodeLine(node, options)}`);
+}
+
+// Diffs two formatted trees keyed by element index. Lines are compared
+// without indentation, so a change to any attribute shows as "~" but an
+// element that only moved deeper (wrapped in a new container) does not.
+export function diffTrees(previous, next) {
+  const key = (line) => line.trimStart().split(" ", 1)[0];
+  const before = new Map(previous.map((line) => [key(line), line.trimStart()]));
+  const nextKeys = new Set();
+  const out = [];
+  for (const line of next) {
+    const k = key(line);
+    nextKeys.add(k);
+    const old = before.get(k);
+    if (old === undefined) out.push(`+${line}`);
+    else if (old !== line.trimStart()) out.push(`~${line}`);
+  }
+  const removed = [...before.keys()].filter((k) => !nextKeys.has(k)).map(Number).sort((a, b) => a - b);
+  if (removed.length) out.push(`- removed: ${ranges(removed)}`);
+  return out;
+}
+
+function ranges(numbers) {
+  const parts = [];
+  let start = numbers[0];
+  let prev = start;
+  for (const n of numbers.slice(1).concat([NaN])) {
+    if (n === prev + 1) {
+      prev = n;
+      continue;
+    }
+    parts.push(start === prev ? `${start}` : `${start}-${prev}`);
+    start = prev = n;
+  }
+  return parts.join(", ");
+}

package/src/backends/index.mjs

@@ -0,0 +1,27 @@
+// Picks the backend for this OS. Each backend exposes request(method, params)
+// with the same methods and emits "escape" when the user presses Escape.
+// See docs/backend-protocol.md for the contract a new OS backend implements.
+import { MacBackend } from "./macos.mjs";
+
+export function createBackend(options) {
+  switch (process.platform) {
+    case "darwin":
+      return new MacBackend(options);
+    default:
+      return new UnsupportedBackend(process.platform);
+  }
+}
+
+class UnsupportedBackend {
+  constructor(platform) {
+    this.platform = platform;
+  }
+  on() {
+    return this;
+  }
+  async request() {
+    throw new Error(
+      `glovebox has no ${this.platform} backend yet. macOS is supported today; Windows and Linux are planned (see docs/roadmap.md).`,
+    );
+  }
+}

package/src/backends/macos.mjs

@@ -0,0 +1,106 @@
+// Client for Glovebox Helper.app over its Unix socket.
+import { EventEmitter } from "node:events";
+import fs from "node:fs";
+import net from "node:net";
+import * as helper from "../macos-helper.mjs";
+
+function connectOnce(path) {
+  return new Promise((resolve, reject) => {
+    const socket = net.connect(path);
+    socket.once("connect", () => resolve(socket));
+    socket.once("error", reject);
+  });
+}
+
+export class MacBackend extends EventEmitter {
+  platform = "macos";
+
+  constructor({ client = "An agent" } = {}) {
+    super();
+    this.client = client;
+    this.socket = null;
+    this.pending = new Map();
+    this.nextId = 1;
+    this.connecting = null;
+  }
+
+  async connect() {
+    if (this.socket) return;
+    this.connecting ??= this.#connect().finally(() => {
+      this.connecting = null;
+    });
+    await this.connecting;
+  }
+
+  async #connect() {
+    if (!fs.existsSync(helper.appPath()) || !helper.isBuilt()) {
+      throw new Error(
+        "Glovebox Helper is missing, or was built from different source than this glovebox. " +
+          "Run: glovebox setup. That rebuilds the helper; unless you ran `glovebox signing --create`, " +
+          "macOS will ask for its permissions again. If you have more than one install, check which `glovebox` this client runs.",
+      );
+    }
+    const path = helper.socketPath();
+    let socket;
+    try {
+      socket = await connectOnce(path);
+    } catch {
+      await helper.launch();
+      for (let attempt = 0; attempt < 50 && !socket; attempt++) {
+        await new Promise((r) => setTimeout(r, 100));
+        socket = await connectOnce(path).catch(() => null);
+      }
+      if (!socket) throw new Error(`Glovebox Helper did not start (no socket at ${path}). Run: glovebox doctor`);
+    }
+    this.socket = socket;
+    let buffer = "";
+    socket.setEncoding("utf8");
+    socket.on("data", (chunk) => {
+      buffer += chunk;
+      let newline;
+      while ((newline = buffer.indexOf("\n")) >= 0) {
+        const line = buffer.slice(0, newline);
+        buffer = buffer.slice(newline + 1);
+        if (line) this.#onMessage(JSON.parse(line));
+      }
+    });
+    socket.on("close", () => {
+      if (this.socket === socket) this.socket = null;
+      for (const { reject } of this.pending.values()) reject(new Error("Glovebox Helper disconnected"));
+      this.pending.clear();
+    });
+    socket.on("error", () => {});
+    await this.request("hello", { client: this.client });
+  }
+
+  #onMessage(message) {
+    if (message.event) {
+      this.emit(message.event, message);
+      return;
+    }
+    const waiter = this.pending.get(message.id);
+    if (!waiter) return;
+    this.pending.delete(message.id);
+    if (message.error) {
+      const error = new Error(message.error.message);
+      error.code = message.error.code;
+      waiter.reject(error);
+    } else {
+      waiter.resolve(message.result);
+    }
+  }
+
+  async request(method, params = {}) {
+    if (!this.socket) await this.connect();
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      this.pending.set(id, { resolve, reject });
+      this.socket.write(`${JSON.stringify({ id, method, params })}\n`);
+    });
+  }
+
+  close() {
+    this.socket?.destroy();
+    this.socket = null;
+  }
+}

package/src/config.mjs

@@ -0,0 +1,92 @@
+// Paths and user configuration. Everything lives under one directory so a user
+// can inspect or delete it: GLOVEBOX_HOME, else ~/.glovebox.
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+export const VERSION = "0.3.0";
+
+export function homeDir() {
+  return process.env.GLOVEBOX_HOME || path.join(os.homedir(), ".glovebox");
+}
+
+export const paths = {
+  get home() {
+    return homeDir();
+  },
+  get config() {
+    return path.join(homeDir(), "config.json");
+  },
+  get log() {
+    return path.join(homeDir(), "actions.log");
+  },
+  get bin() {
+    return path.join(homeDir(), "bin");
+  },
+  get shots() {
+    return path.join(homeDir(), "screenshots");
+  },
+  get socket() {
+    return process.platform === "win32"
+      ? `\\\\.\\pipe\\glovebox-${os.userInfo().username}`
+      : path.join(homeDir(), "daemon.sock");
+  },
+};
+
+export const DEFAULTS = {
+  // ask: native dialog on first use of each app. allowlist: only apps in `allow`.
+  // allow-all: no prompts (explicit opt-in; blocked apps still need `allow`).
+  approval: "ask",
+  allow: [],
+  deny: [],
+  // background: input reaches apps without focusing them (falls back to
+  // foreground when the app or macOS can't take it). foreground: always front.
+  input: "background",
+  // Apps that need foreground input, by name or bundle id.
+  foregroundApps: [],
+  // Draw the agent cursor where actions land.
+  cursor: true,
+  // Pause after each action so the app can update before the next one.
+  settleMs: 80,
+  screenshot: { maxLongSide: 1568, maxPixels: 1_150_000, format: "png", quality: 80 },
+  ax: { maxNodes: 800, maxDepth: 40, maxTextLength: 160 },
+  log: { enabled: true, text: false },
+  daemon: { idleMinutes: 60 },
+  timeoutMs: 120_000,
+};
+
+function merge(base, extra) {
+  const out = { ...base };
+  for (const [key, value] of Object.entries(extra ?? {})) {
+    out[key] =
+      value && typeof value === "object" && !Array.isArray(value) && base[key] && typeof base[key] === "object"
+        ? merge(base[key], value)
+        : value;
+  }
+  return out;
+}
+
+export function readUserConfig() {
+  try {
+    return JSON.parse(fs.readFileSync(paths.config, "utf8"));
+  } catch (error) {
+    if (error.code === "ENOENT") return {};
+    throw new Error(`Cannot read ${paths.config}: ${error.message}`);
+  }
+}
+
+export function loadConfig() {
+  const config = merge(DEFAULTS, readUserConfig());
+  if (process.env.GLOVEBOX_APPROVAL) config.approval = process.env.GLOVEBOX_APPROVAL;
+  if (!["ask", "allowlist", "allow-all"].includes(config.approval)) {
+    throw new Error(`approval must be ask, allowlist, or allow-all (got ${config.approval})`);
+  }
+  return config;
+}
+
+export function writeUserConfig(update) {
+  fs.mkdirSync(homeDir(), { recursive: true, mode: 0o700 });
+  const next = { ...readUserConfig(), ...update };
+  fs.writeFileSync(paths.config, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600 });
+  return next;
+}

package/src/daemon.mjs

@@ -0,0 +1,127 @@
+// Background process that keeps CLI sessions alive between `glovebox run` calls,
+// so shell-only agents get the same persistent bindings as MCP clients.
+// One JSON request per connection, one JSON reply.
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import net from "node:net";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { loadConfig, paths } from "./config.mjs";
+import { Session } from "./session.mjs";
+
+export function runDaemon() {
+  const sessions = new Map();
+  const config = loadConfig();
+  let lastActivity = Date.now();
+
+  const sessionFor = (name, client) => {
+    let session = sessions.get(name);
+    if (!session) {
+      session = new Session({ client: client || `glovebox CLI (${name})`, imageMode: "file" });
+      sessions.set(name, session);
+    }
+    return session;
+  };
+
+  async function handle(request) {
+    lastActivity = Date.now();
+    const name = request.session || "default";
+    switch (request.op) {
+      case "run":
+        return sessionFor(name, request.client).run(request.code, {
+          title: request.title,
+          timeoutMs: request.timeoutMs || config.timeoutMs,
+        });
+      case "reset":
+        sessions.get(name)?.reset();
+        sessions.delete(name);
+        return { ok: true, content: [{ type: "text", text: `Session "${name}" reset.` }] };
+      case "list":
+        return { ok: true, content: [{ type: "text", text: [...sessions.keys()].join("\n") || "(no sessions)" }] };
+      case "stop":
+        setTimeout(shutdown, 50);
+        return { ok: true, content: [{ type: "text", text: "Daemon stopped." }] };
+      default:
+        return { ok: false, content: [{ type: "text", text: `Unknown op ${request.op}` }] };
+    }
+  }
+
+  if (process.platform !== "win32") {
+    fs.mkdirSync(paths.home, { recursive: true, mode: 0o700 });
+    try {
+      fs.unlinkSync(paths.socket);
+    } catch {}
+  }
+  const server = net.createServer((socket) => {
+    let buffer = "";
+    socket.setEncoding("utf8");
+    socket.on("data", async (chunk) => {
+      buffer += chunk;
+      const newline = buffer.indexOf("\n");
+      if (newline < 0) return;
+      let reply;
+      try {
+        reply = await handle(JSON.parse(buffer.slice(0, newline)));
+      } catch (error) {
+        reply = { ok: false, content: [{ type: "text", text: error.message }] };
+      }
+      socket.end(`${JSON.stringify(reply)}\n`);
+    });
+    socket.on("error", () => {});
+  });
+  server.listen(paths.socket, () => {
+    if (process.platform !== "win32") fs.chmodSync(paths.socket, 0o600);
+  });
+
+  function shutdown() {
+    for (const session of sessions.values()) session.close();
+    server.close();
+    process.exit(0);
+  }
+  process.on("SIGTERM", shutdown);
+  process.on("SIGINT", shutdown);
+
+  const idleMs = (config.daemon.idleMinutes ?? 60) * 60_000;
+  if (idleMs > 0) {
+    setInterval(() => {
+      if (Date.now() - lastActivity > idleMs) shutdown();
+    }, 30_000).unref();
+  }
+}
+
+function send(request) {
+  return new Promise((resolve, reject) => {
+    const socket = net.connect(paths.socket);
+    let buffer = "";
+    socket.setEncoding("utf8");
+    socket.on("connect", () => socket.write(`${JSON.stringify(request)}\n`));
+    socket.on("data", (chunk) => (buffer += chunk));
+    socket.on("end", () => {
+      try {
+        resolve(JSON.parse(buffer));
+      } catch {
+        reject(new Error("Bad reply from glovebox daemon"));
+      }
+    });
+    socket.on("error", reject);
+  });
+}
+
+// Sends a request, starting the daemon first if nothing is listening.
+export async function daemonRequest(request, { autostart = true } = {}) {
+  try {
+    return await send(request);
+  } catch (error) {
+    if (!autostart || !["ENOENT", "ECONNREFUSED"].includes(error.code)) throw error;
+  }
+  const cli = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "bin", "glovebox.mjs");
+  const child = spawn(process.execPath, [cli, "daemon"], { detached: true, stdio: "ignore", env: process.env });
+  child.unref();
+  for (let attempt = 0; attempt < 50; attempt++) {
+    await new Promise((r) => setTimeout(r, 100));
+    try {
+      return await send(request);
+    } catch {}
+  }
+  throw new Error(`glovebox daemon did not start (socket ${paths.socket})`);
+}

package/src/geometry.mjs

@@ -0,0 +1,38 @@
+// The image space a model sees for a window, and the mapping back to screen points.
+//
+// Image pixels are window-local: (0, 0) is the window's top-left corner. One
+// view's scale `s` is image pixels per screen point. It depends only on the
+// window size, the display's backing scale, and the configured limits, so the
+// AX tree, screenshots, and clicks all agree even before any screenshot exists.
+
+export function viewFor(window, limits) {
+  const [x, y, w, h] = window.frame;
+  const backing = window.scale || 1;
+  const s = Math.min(
+    backing,
+    limits.maxLongSide / Math.max(w, h),
+    Math.sqrt(limits.maxPixels / Math.max(1, w * h)),
+  );
+  return {
+    window: window.index,
+    windowId: window.windowId,
+    title: window.title,
+    origin: [x, y],
+    size: [w, h],
+    scale: s,
+    width: Math.max(1, Math.round(w * s)),
+    height: Math.max(1, Math.round(h * s)),
+  };
+}
+
+export function toScreen(view, [ix, iy]) {
+  return [view.origin[0] + ix / view.scale, view.origin[1] + iy / view.scale];
+}
+
+export function toImage(view, [sx, sy, sw, sh]) {
+  return [(sx - view.origin[0]) * view.scale, (sy - view.origin[1]) * view.scale, sw * view.scale, sh * view.scale];
+}
+
+export function center([x, y, w, h]) {
+  return [x + w / 2, y + h / 2];
+}