@opencoreai/opencore 0.2.2

package/src/mac-controller.mjs

@@ -0,0 +1,614 @@
+import { execFile } from "node:child_process";
+import { promises as fs } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+
+const execFileAsync = promisify(execFile);
+const API_IMAGE_MAX_BYTES = 5 * 1024 * 1024;
+const API_IMAGE_TARGET_BYTES = 4_900_000;
+
+const SPECIAL_KEY_CODES = {
+  ARROWDOWN: 125,
+  ARROWLEFT: 123,
+  ARROWRIGHT: 124,
+  ARROWUP: 126,
+  BACKSPACE: 51,
+  DELETE: 117,
+  DOWN: 125,
+  DOWNARROW: 125,
+  END: 119,
+  ENTER: 36,
+  ESC: 53,
+  ESCAPE: 53,
+  HOME: 115,
+  LEFT: 123,
+  LEFTARROW: 123,
+  PAGEDN: 121,
+  PAGEDOWN: 121,
+  PAGEUP: 116,
+  PGDN: 121,
+  PGUP: 116,
+  RETURN: 36,
+  RIGHT: 124,
+  RIGHTARROW: 124,
+  SPACE: 49,
+  TAB: 48,
+  UP: 126,
+  UPARROW: 126,
+};
+
+const BUTTONS = {
+  left: {
+    cgButton: "$.kCGMouseButtonLeft",
+    down: "$.kCGEventLeftMouseDown",
+    up: "$.kCGEventLeftMouseUp",
+  },
+  right: {
+    cgButton: "$.kCGMouseButtonRight",
+    down: "$.kCGEventRightMouseDown",
+    up: "$.kCGEventRightMouseUp",
+  },
+  middle: {
+    cgButton: "$.kCGMouseButtonCenter",
+    down: "$.kCGEventOtherMouseDown",
+    up: "$.kCGEventOtherMouseUp",
+  },
+};
+
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function shellEscapeSingleQuotes(value) {
+  return String(value).replace(/\\/g, "\\\\").replace(/"/g, '\\"');
+}
+
+function normalizeKey(key) {
+  return String(key || "")
+    .trim()
+    .toUpperCase()
+    .replace(/[\s-]+/g, "")
+    .replace(/_/g, "");
+}
+
+function sanitizeCoordinate(value, fallback = 0) {
+  const num = Number(value);
+  if (!Number.isFinite(num)) return fallback;
+  return Math.max(0, Math.round(num));
+}
+
+function sanitizeButton(button) {
+  const raw = String(button || "left").toLowerCase();
+  if (raw === "center") return "middle";
+  return BUTTONS[raw] ? raw : "left";
+}
+
+function mapModifier(key) {
+  if (key === "CMDORCTRL") return "command down";
+  if (key === "CMD" || key === "COMMAND" || key === "META") return "command down";
+  if (key === "CTRL" || key === "CONTROL") return "control down";
+  if (key === "OPT") return "option down";
+  if (key === "ALT" || key === "OPTION") return "option down";
+  if (key === "SUPER" || key === "WINDOWS") return "command down";
+  if (key === "SHIFT") return "shift down";
+  return null;
+}
+
+function keyNameToCharacter(key) {
+  if (key === "SPACE") return " ";
+  if (key.length === 1) return key.toLowerCase();
+  return null;
+}
+
+async function runAppleScript(script) {
+  try {
+    await execFileAsync("osascript", ["-e", script]);
+  } catch (error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    throw new Error(`AppleScript execution failed on macOS: ${reason}`);
+  }
+}
+
+async function runJxa(script) {
+  const args = ["-l", "JavaScript", "-e", script];
+  let lastError = null;
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    try {
+      await execFileAsync("osascript", args);
+      return;
+    } catch (error) {
+      lastError = error;
+      await sleep(80 * (attempt + 1));
+    }
+  }
+
+  const scriptFile = path.join(
+    os.tmpdir(),
+    `opencore-jxa-${Date.now()}-${Math.random().toString(36).slice(2)}.js`,
+  );
+  try {
+    await fs.writeFile(scriptFile, `${script}\n`, "utf8");
+    await execFileAsync("osascript", ["-l", "JavaScript", scriptFile]);
+  } catch (error) {
+    const failed = error || lastError;
+    const message = failed instanceof Error ? failed.message : String(failed);
+    const stderr = String(failed?.stderr || "").trim();
+    const detail = stderr ? `${message} | stderr: ${stderr}` : message;
+    throw new Error(
+      "JXA execution failed on macOS. " +
+        "Ensure Accessibility and Input Monitoring are enabled for your terminal app in System Settings > Privacy & Security. " +
+        `Details: ${detail}`,
+    );
+  } finally {
+    fs.unlink(scriptFile).catch(() => {});
+  }
+}
+
+async function downscaleImageToLimit(sourceFile, targetBytes = API_IMAGE_TARGET_BYTES) {
+  const maxDims = [1800, 1600, 1400, 1200, 1024, 900, 768, 640];
+  const tempFiles = [];
+  try {
+    for (const dim of maxDims) {
+      const outFile = path.join(
+        os.tmpdir(),
+        `computer-use-scaled-${Date.now()}-${dim}-${Math.random().toString(36).slice(2)}.png`,
+      );
+      tempFiles.push(outFile);
+      await execFileAsync("sips", [sourceFile, "--resampleHeightWidthMax", String(dim), "--out", outFile]);
+      const scaled = await fs.readFile(outFile);
+      if (scaled.byteLength <= targetBytes) {
+        return scaled;
+      }
+    }
+    return null;
+  } finally {
+    for (const file of tempFiles) {
+      fs.unlink(file).catch(() => {});
+    }
+  }
+}
+
+async function convertToJpegToLimit(sourceFile, targetBytes = API_IMAGE_TARGET_BYTES) {
+  const maxDims = [1600, 1400, 1200, 1024, 900, 768, 640];
+  const qualities = [80, 70, 60, 50, 40];
+  const tempFiles = [];
+  try {
+    for (const dim of maxDims) {
+      for (const quality of qualities) {
+        const outFile = path.join(
+          os.tmpdir(),
+          `computer-use-jpeg-${Date.now()}-${dim}-${quality}-${Math.random().toString(36).slice(2)}.jpg`,
+        );
+        tempFiles.push(outFile);
+        await execFileAsync("sips", [
+          sourceFile,
+          "--resampleHeightWidthMax",
+          String(dim),
+          "--setProperty",
+          "format",
+          "jpeg",
+          "--setProperty",
+          "formatOptions",
+          String(quality),
+          "--out",
+          outFile,
+        ]);
+        const jpeg = await fs.readFile(outFile);
+        if (jpeg.byteLength <= targetBytes) {
+          return jpeg;
+        }
+      }
+    }
+    return null;
+  } finally {
+    for (const file of tempFiles) {
+      fs.unlink(file).catch(() => {});
+    }
+  }
+}
+
+export class MacController {
+  constructor({ displayWidth, displayHeight, screenshotDelayMs = 300 }) {
+    this.displayWidth = Number(displayWidth) || null;
+    this.displayHeight = Number(displayHeight) || null;
+    this.screenshotDelayMs = screenshotDelayMs;
+    this.cursorX = 0;
+    this.cursorY = 0;
+    this.lastScreenshotBase64 = null;
+    this.lastScreenshotBytes = 0;
+    this.lastScreenshotMediaType = "image/png";
+  }
+
+  async init() {
+    if (this.displayWidth && this.displayHeight) return;
+
+    const fallback = { width: 1440, height: 900 };
+    try {
+      const script = [
+        'ObjC.import("CoreGraphics");',
+        "const main = $.CGMainDisplayID();",
+        "console.log(`${$.CGDisplayPixelsWide(main)} ${$.CGDisplayPixelsHigh(main)}`);",
+      ].join(" ");
+      const { stdout } = await execFileAsync("osascript", ["-l", "JavaScript", "-e", script]);
+      const [w, h] = stdout.trim().split(/\s+/).map(Number);
+      this.displayWidth = Number.isFinite(w) && w > 0 ? w : fallback.width;
+      this.displayHeight = Number.isFinite(h) && h > 0 ? h : fallback.height;
+    } catch {
+      this.displayWidth = fallback.width;
+      this.displayHeight = fallback.height;
+    }
+    this.cursorX = Math.floor(this.displayWidth / 2);
+    this.cursorY = Math.floor(this.displayHeight / 2);
+  }
+
+  getToolConfig(environment = "mac") {
+    return {
+      type: "computer_use_preview",
+      display_width: this.displayWidth,
+      display_height: this.displayHeight,
+      environment,
+    };
+  }
+
+  async captureScreenshotBase64() {
+    const file = path.join(
+      os.tmpdir(),
+      `computer-use-${Date.now()}-${Math.random().toString(36).slice(2)}.png`,
+    );
+    try {
+      let captureError = null;
+      const attempts = [
+        ["-x", file],
+        ["-x", "-D", "1", file],
+      ];
+      for (const args of attempts) {
+        try {
+          await execFileAsync("screencapture", args);
+          captureError = null;
+          break;
+        } catch (error) {
+          captureError = error;
+          await sleep(120);
+        }
+      }
+      if (captureError) throw captureError;
+      let data = await fs.readFile(file);
+      let mediaType = "image/png";
+      if (data.byteLength > API_IMAGE_MAX_BYTES) {
+        const scaled = await downscaleImageToLimit(file);
+        if (scaled) data = scaled;
+      }
+      if (data.byteLength > API_IMAGE_MAX_BYTES) {
+        const jpeg = await convertToJpegToLimit(file);
+        if (jpeg) {
+          data = jpeg;
+          mediaType = "image/jpeg";
+        }
+      }
+      if (data.byteLength > API_IMAGE_MAX_BYTES) {
+        throw new Error(
+          `Screenshot too large for API after resize attempts (${data.byteLength} bytes > ${API_IMAGE_MAX_BYTES} bytes).`,
+        );
+      }
+      const encoded = data.toString("base64");
+      this.lastScreenshotBase64 = encoded;
+      this.lastScreenshotBytes = data.byteLength;
+      this.lastScreenshotMediaType = mediaType;
+      return encoded;
+    } catch (error) {
+      if (this.lastScreenshotBase64 && this.lastScreenshotBytes > 0 && this.lastScreenshotBytes <= API_IMAGE_MAX_BYTES) {
+        return this.lastScreenshotBase64;
+      }
+      const reason = error instanceof Error ? error.message : String(error);
+      throw new Error(
+        "Screenshot capture failed on macOS. " +
+          "Allow Screen Recording for your terminal app in System Settings > Privacy & Security > Screen Recording, " +
+          "then fully quit and reopen the terminal. Also ensure a display is active/unlocked. " +
+          `Original error: ${reason}`,
+      );
+    } finally {
+      fs.unlink(file).catch(() => {});
+    }
+  }
+
+  getLastScreenshotMediaType() {
+    return this.lastScreenshotMediaType || "image/png";
+  }
+
+  async executeAction(action = {}) {
+    const type = String(action.type || "").toLowerCase();
+
+    switch (type) {
+      case "click":
+        return this.click(action);
+      case "double_click":
+        return this.doubleClick(action);
+      case "move":
+        return this.move(action);
+      case "drag":
+        return this.drag(action);
+      case "scroll":
+        return this.scroll(action);
+      case "type":
+        return this.type(action);
+      case "keypress":
+        return this.keypress(action);
+      case "wait":
+        return sleep(Math.max(200, Number(action.ms || action.seconds * 1000 || 1000)));
+      case "screenshot":
+        return;
+      default:
+        throw new Error(`Unsupported action type: ${action.type ?? "unknown"}`);
+    }
+  }
+
+  async executeAnthropicAction(input = {}) {
+    const action = String(input.action || "").toLowerCase();
+    const [cx, cy] = Array.isArray(input.coordinate) ? input.coordinate : [null, null];
+    const withCoord = (base = {}) => ({ ...base, x: cx, y: cy });
+
+    switch (action) {
+      case "screenshot":
+      case "cursor_position":
+        return;
+      case "mouse_move":
+        return this.move(withCoord());
+      case "left_click":
+        return this.click(withCoord({ button: "left" }));
+      case "right_click":
+        return this.click(withCoord({ button: "right" }));
+      case "middle_click":
+        return this.click(withCoord({ button: "middle" }));
+      case "double_click":
+        return this.doubleClick(withCoord({ button: "left" }));
+      case "triple_click":
+        await this.doubleClick(withCoord({ button: "left" }));
+        return this.click(withCoord({ button: "left" }));
+      case "left_mouse_down":
+        return this.mouseDown(withCoord({ button: "left" }));
+      case "left_mouse_up":
+        return this.mouseUp(withCoord({ button: "left" }));
+      case "left_click_drag":
+        return this.dragAnthropic(input);
+      case "scroll":
+        return this.scrollAnthropic(input);
+      case "key":
+        return this.keyAnthropic(input);
+      case "hold_key":
+        return this.holdKeyAnthropic(input);
+      case "type":
+        return this.type({ text: input.text });
+      case "wait":
+        return sleep(Math.max(200, Number((input.duration_seconds || 1) * 1000)));
+      default:
+        throw new Error(`Unsupported anthropic action: ${input.action ?? "unknown"}`);
+    }
+  }
+
+  async move({ x, y }) {
+    const px = sanitizeCoordinate(x);
+    const py = sanitizeCoordinate(y);
+    const script = [
+      'ObjC.import("CoreGraphics");',
+      `var point = $.CGPointMake(${px}, ${py});`,
+      "var ev = $.CGEventCreateMouseEvent(null, $.kCGEventMouseMoved, point, $.kCGMouseButtonLeft);",
+      "$.CGEventPost($.kCGHIDEventTap, ev);",
+      "$.CFRelease(ev);",
+    ].join(" ");
+    await runJxa(script);
+    this.cursorX = px;
+    this.cursorY = py;
+  }
+
+  async click({ x, y, button }) {
+    const px = sanitizeCoordinate(x, this.cursorX);
+    const py = sanitizeCoordinate(y, this.cursorY);
+    const choice = BUTTONS[sanitizeButton(button)];
+    const script = [
+      'ObjC.import("CoreGraphics");',
+      `var point = $.CGPointMake(${px}, ${py});`,
+      `var down = $.CGEventCreateMouseEvent(null, ${choice.down}, point, ${choice.cgButton});`,
+      `var up = $.CGEventCreateMouseEvent(null, ${choice.up}, point, ${choice.cgButton});`,
+      "$.CGEventPost($.kCGHIDEventTap, down);",
+      "$.CGEventPost($.kCGHIDEventTap, up);",
+      "$.CFRelease(down);",
+      "$.CFRelease(up);",
+    ].join(" ");
+    await runJxa(script);
+    this.cursorX = px;
+    this.cursorY = py;
+  }
+
+  async doubleClick(action) {
+    await this.click(action);
+    await sleep(70);
+    await this.click(action);
+  }
+
+  async drag(action) {
+    const points = Array.isArray(action.path) ? action.path : [];
+    if (!points.length) {
+      const x = sanitizeCoordinate(action.x);
+      const y = sanitizeCoordinate(action.y);
+      const x2 = sanitizeCoordinate(action.x2 ?? action.to_x ?? x);
+      const y2 = sanitizeCoordinate(action.y2 ?? action.to_y ?? y);
+      points.push({ x, y }, { x: x2, y: y2 });
+    }
+    const safePoints = points
+      .map((point) => ({
+        x: sanitizeCoordinate(point.x),
+        y: sanitizeCoordinate(point.y),
+      }))
+      .filter((point) => Number.isFinite(point.x) && Number.isFinite(point.y));
+    if (safePoints.length < 2) return;
+
+    const start = safePoints[0];
+    const scriptLines = [
+      'ObjC.import("CoreGraphics");',
+      "function post(type, x, y) {",
+      "  var point = $.CGPointMake(x, y);",
+      "  var ev = $.CGEventCreateMouseEvent(null, type, point, $.kCGMouseButtonLeft);",
+      "  $.CGEventPost($.kCGHIDEventTap, ev);",
+      "  $.CFRelease(ev);",
+      "}",
+      `post($.kCGEventMouseMoved, ${start.x}, ${start.y});`,
+      `post($.kCGEventLeftMouseDown, ${start.x}, ${start.y});`,
+    ];
+
+    for (const point of safePoints.slice(1)) {
+      scriptLines.push(`post($.kCGEventLeftMouseDragged, ${point.x}, ${point.y});`);
+    }
+
+    const end = safePoints[safePoints.length - 1];
+    scriptLines.push(`post($.kCGEventLeftMouseUp, ${end.x}, ${end.y});`);
+    await runJxa(scriptLines.join(" "));
+    this.cursorX = end.x;
+    this.cursorY = end.y;
+  }
+
+  async scroll(action) {
+    const x = sanitizeCoordinate(action.x, Math.floor(this.displayWidth / 2));
+    const y = sanitizeCoordinate(action.y, Math.floor(this.displayHeight / 2));
+    const horizontal = Math.round(Number(action.scroll_x || 0));
+    const vertical = -Math.round(Number(action.scroll_y || 0));
+    const script = [
+      'ObjC.import("CoreGraphics");',
+      `var point = $.CGPointMake(${x}, ${y});`,
+      "var move = $.CGEventCreateMouseEvent(null, $.kCGEventMouseMoved, point, $.kCGMouseButtonLeft);",
+      "$.CGEventPost($.kCGHIDEventTap, move);",
+      "$.CFRelease(move);",
+      `var ev = $.CGEventCreateScrollWheelEvent(null, $.kCGScrollEventUnitLine, 2, ${vertical}, ${horizontal});`,
+      "$.CGEventPost($.kCGHIDEventTap, ev);",
+      "$.CFRelease(ev);",
+    ].join(" ");
+    await runJxa(script);
+  }
+
+  async dragAnthropic(input) {
+    const [toX, toY] = Array.isArray(input.coordinate) ? input.coordinate : [this.cursorX, this.cursorY];
+    const [fromX, fromY] = Array.isArray(input.from_coordinate)
+      ? input.from_coordinate
+      : [this.cursorX, this.cursorY];
+    return this.drag({
+      path: [
+        { x: fromX, y: fromY },
+        { x: toX, y: toY },
+      ],
+    });
+  }
+
+  async scrollAnthropic(input) {
+    const amount = Math.round(Number(input.scroll_amount || input.amount || 0));
+    const direction = String(input.scroll_direction || input.direction || "down").toLowerCase();
+    const modifiers = Array.isArray(input.modifiers) ? input.modifiers : [];
+    const payload = { x: this.cursorX, y: this.cursorY, scroll_x: 0, scroll_y: 0 };
+
+    if (direction === "up") payload.scroll_y = -amount;
+    else if (direction === "left") payload.scroll_x = -amount;
+    else if (direction === "right") payload.scroll_x = amount;
+    else payload.scroll_y = amount;
+
+    if (modifiers.length) {
+      await this.keypress({ keys: modifiers.map(String) });
+    }
+    await this.scroll(payload);
+  }
+
+  async keyAnthropic(input) {
+    const combo = String(input.text || input.key || "").trim();
+    if (!combo) return;
+    const keys = combo
+      .split("+")
+      .map((part) => part.trim())
+      .filter(Boolean);
+    return this.keypress({ keys });
+  }
+
+  async holdKeyAnthropic(input) {
+    await this.keyAnthropic(input);
+    return sleep(Math.max(200, Number((input.duration_seconds || 1) * 1000)));
+  }
+
+  async mouseDown({ x, y, button }) {
+    const px = sanitizeCoordinate(x, this.cursorX);
+    const py = sanitizeCoordinate(y, this.cursorY);
+    const choice = BUTTONS[sanitizeButton(button)];
+    const script = [
+      'ObjC.import("CoreGraphics");',
+      `var point = $.CGPointMake(${px}, ${py});`,
+      `var down = $.CGEventCreateMouseEvent(null, ${choice.down}, point, ${choice.cgButton});`,
+      "$.CGEventPost($.kCGHIDEventTap, down);",
+      "$.CFRelease(down);",
+    ].join(" ");
+    await runJxa(script);
+    this.cursorX = px;
+    this.cursorY = py;
+  }
+
+  async mouseUp({ x, y, button }) {
+    const px = sanitizeCoordinate(x, this.cursorX);
+    const py = sanitizeCoordinate(y, this.cursorY);
+    const choice = BUTTONS[sanitizeButton(button)];
+    const script = [
+      'ObjC.import("CoreGraphics");',
+      `var point = $.CGPointMake(${px}, ${py});`,
+      `var up = $.CGEventCreateMouseEvent(null, ${choice.up}, point, ${choice.cgButton});`,
+      "$.CGEventPost($.kCGHIDEventTap, up);",
+      "$.CFRelease(up);",
+    ].join(" ");
+    await runJxa(script);
+    this.cursorX = px;
+    this.cursorY = py;
+  }
+
+  getCursorPosition() {
+    return { x: this.cursorX, y: this.cursorY };
+  }
+
+  async type({ text }) {
+    const value = String(text ?? "");
+    if (!value.length) return;
+    const lines = value.split("\n");
+    for (let i = 0; i < lines.length; i += 1) {
+      const chunk = lines[i];
+      if (chunk.length) {
+        const safe = shellEscapeSingleQuotes(chunk);
+        await runAppleScript(`tell application "System Events" to keystroke "${safe}"`);
+      }
+      if (i < lines.length - 1) {
+        await runAppleScript('tell application "System Events" to key code 36');
+      }
+    }
+  }
+
+  async keypress({ keys }) {
+    const list = Array.isArray(keys) ? keys : [];
+    if (!list.length) return;
+
+    const normalized = list.map(normalizeKey).filter(Boolean);
+    const modifiers = normalized.map(mapModifier).filter(Boolean);
+    const nonModifierKeys = normalized.filter((key) => !mapModifier(key));
+    const main = nonModifierKeys[0];
+
+    if (!main && modifiers.length) return;
+    if (!main) return;
+
+    const usingClause = modifiers.length ? ` using {${modifiers.join(", ")}}` : "";
+    const keyCode = SPECIAL_KEY_CODES[main];
+
+    if (typeof keyCode === "number") {
+      await runAppleScript(`tell application "System Events" to key code ${keyCode}${usingClause}`);
+      return;
+    }
+
+    const char = keyNameToCharacter(main);
+    if (char !== null) {
+      const safe = shellEscapeSingleQuotes(char);
+      await runAppleScript(`tell application "System Events" to keystroke "${safe}"${usingClause}`);
+      return;
+    }
+
+    throw new Error(`Unsupported key in keypress action: ${main}`);
+  }
+}