@lardexx/arenacode 2.9.0 → 2.11.0

package/src/automation/mac.js

@@ -0,0 +1,657 @@
+'use strict';
+
+// macOS backend: CoreGraphics through koffi. Same idea as the Linux one — no helper
+// binaries, no `brew install cliclick`: the mouse is CGEventCreateMouseEvent + CGEventPost,
+// the keyboard is CGEventCreateKeyboardEvent (chords) and CGEventKeyboardSetUnicodeString
+// (any character, emoji included), windows come from CGWindowListCopyWindowInfo and the
+// screenshot from CGDisplayCreateImage. Only what CoreGraphics cannot do at all —
+// launching an app, bringing a window to the front, resizing it — still goes through
+// `osascript`, which ships with macOS.
+//
+// Permissions: posting events needs Accessibility for the terminal (System Settings →
+// Privacy & Security → Accessibility); window titles and pixels need Screen Recording.
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const { spawnSync } = require('node:child_process');
+const { fn, lib } = require('./ffi');
+const { encodePng } = require('./png');
+
+const CG = '/System/Library/Frameworks/CoreGraphics.framework/CoreGraphics';
+const CF = '/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation';
+
+const K_CG_HID_EVENT_TAP = 0;
+const K_CG_SESSION_EVENT_TAP = 1;
+
+const EVENT = {
+  leftDown: 1,
+  leftUp: 2,
+  rightDown: 3,
+  rightUp: 4,
+  mouseMoved: 5,
+  leftDragged: 6,
+  rightDragged: 7,
+  keyDown: 10,
+  keyUp: 11,
+  scrollWheel: 22,
+};
+const FIELD_CLICK_STATE = 1;
+
+const FLAG = {
+  shift: 0x20000,
+  control: 0x40000,
+  option: 0x80000,
+  command: 0x100000,
+};
+
+// kVK_* — fixed virtual keycodes; they do not depend on the keyboard layout.
+const VK = {
+  a: 0, s: 1, d: 2, f: 3, h: 4, g: 5, z: 6, x: 7, c: 8, v: 9,
+  b: 11, q: 12, w: 13, e: 14, r: 15, y: 16, t: 17,
+  1: 18, 2: 19, 3: 20, 4: 21, 6: 22, 5: 23, equal: 24, 9: 25, 7: 26, minus: 27,
+  8: 28, 0: 29, bracketright: 30, o: 31, u: 32, bracketleft: 33, i: 34, p: 35,
+  return: 36, l: 37, j: 38, quote: 39, k: 40, semicolon: 41, backslash: 42,
+  comma: 43, slash: 44, n: 45, m: 46, period: 47, tab: 48, space: 49, grave: 50,
+  backspace: 51, escape: 53, command: 55, shift: 56, capslock: 57, option: 58,
+  control: 59, rightshift: 60, rightoption: 61, rightcontrol: 62,
+  help: 114, home: 115, pageup: 116, delete: 117, end: 119, pagedown: 121,
+  left: 123, right: 124, down: 125, up: 126,
+  f1: 122, f2: 120, f3: 99, f4: 118, f5: 96, f6: 97, f7: 98, f8: 100,
+  f9: 101, f10: 109, f11: 103, f12: 111,
+  enter: 76,
+};
+
+const MODIFIERS = {
+  cmd: FLAG.command, command: FLAG.command, meta: FLAG.command, win: FLAG.command, super: FLAG.command,
+  ctrl: FLAG.control, control: FLAG.control,
+  alt: FLAG.option, option: FLAG.option, opt: FLAG.option,
+  shift: FLAG.shift,
+};
+
+// Names that AppleScript/linux-style tasks use, mapped to the kVK_* table.
+const ALIASES = {
+  esc: 'escape', delete: 'backspace', backspace: 'backspace', del: 'delete',
+  arrowup: 'up', arrowdown: 'down', arrowleft: 'left', arrowright: 'right',
+  up: 'up', down: 'down', left: 'left', right: 'right',
+  pgup: 'pageup', pgdn: 'pagedown', page_up: 'pageup', page_down: 'pagedown',
+  home: 'home', end: 'end', space: 'space', spacebar: 'space',
+  return: 'return', enter: 'return', tab: 'tab', capslock: 'capslock',
+  '[': 'bracketleft', ']': 'bracketright', ';': 'semicolon', '=': 'equal',
+  '-': 'minus', '`': 'grave', '\\': 'backslash', ',': 'comma', '.': 'period',
+  '/': 'slash', "'": 'quote',
+};
+
+let bound = null;
+
+function bindings() {
+  if (bound) return bound;
+  if (!lib(CG) || !lib(CF)) throw new Error('CoreGraphics not available (not macOS?)');
+  bound = {
+    CGMainDisplayID: fn(CG, 'CGMainDisplayID', 'unsigned int CGMainDisplayID(void)'),
+    CGDisplayPixelsWide: fn(CG, 'CGDisplayPixelsWide', 'unsigned long CGDisplayPixelsWide(unsigned int display)'),
+    CGDisplayPixelsHigh: fn(CG, 'CGDisplayPixelsHigh', 'unsigned long CGDisplayPixelsHigh(unsigned int display)'),
+    CGEventCreateMouseEvent: fn(
+      CG,
+      'CGEventCreateMouseEvent',
+      'void* CGEventCreateMouseEvent(void* source, int mouseType, double x, double y, int mouseButton)'
+    ),
+    CGEventCreateKeyboardEvent: fn(
+      CG,
+      'CGEventCreateKeyboardEvent',
+      'void* CGEventCreateKeyboardEvent(void* source, unsigned short virtualKey, bool keyDown)'
+    ),
+    CGEventCreateScrollWheelEvent2: fn(
+      CG,
+      'CGEventCreateScrollWheelEvent2',
+      'void* CGEventCreateScrollWheelEvent2(void* source, int units, unsigned int wheelCount, int wheel1, int wheel2, int wheel3)'
+    ),
+    CGEventPost: fn(CG, 'CGEventPost', 'void CGEventPost(int tap, void* event)'),
+    CGEventSetFlags: fn(CG, 'CGEventSetFlags', 'void CGEventSetFlags(void* event, unsigned long long flags)'),
+    CGEventSetIntegerValueField: fn(
+      CG,
+      'CGEventSetIntegerValueField',
+      'void CGEventSetIntegerValueField(void* event, int field, long long value)'
+    ),
+    CGEventKeyboardSetUnicodeString: fn(
+      CG,
+      'CGEventKeyboardSetUnicodeString',
+      'void CGEventKeyboardSetUnicodeString(void* event, unsigned long length, void* unicodeString)'
+    ),
+    CGWindowListCopyWindowInfo: fn(
+      CG,
+      'CGWindowListCopyWindowInfo',
+      'void* CGWindowListCopyWindowInfo(unsigned int option, unsigned int relativeToWindow)'
+    ),
+    CGDisplayCreateImage: fn(CG, 'CGDisplayCreateImage', 'void* CGDisplayCreateImage(unsigned int display)'),
+    CGImageGetWidth: fn(CG, 'CGImageGetWidth', 'unsigned long CGImageGetWidth(void* image)'),
+    CGImageGetHeight: fn(CG, 'CGImageGetHeight', 'unsigned long CGImageGetHeight(void* image)'),
+    CGImageGetBytesPerRow: fn(CG, 'CGImageGetBytesPerRow', 'unsigned long CGImageGetBytesPerRow(void* image)'),
+    CGImageGetBitsPerPixel: fn(CG, 'CGImageGetBitsPerPixel', 'unsigned long CGImageGetBitsPerPixel(void* image)'),
+    CGImageGetDataProvider: fn(CG, 'CGImageGetDataProvider', 'void* CGImageGetDataProvider(void* image)'),
+    CFDataGetBytePtr: fn(CF, 'CFDataGetBytePtr', 'const unsigned char* CFDataGetBytePtr(void* data)'),
+    CFDataGetLength: fn(CF, 'CFDataGetLength', 'long CFDataGetLength(void* data)'),
+    CFDataGetBytes: fn(CF, 'CFDataGetBytes', 'void CFDataGetBytes(void* data, void* buffer, unsigned long length)'),
+    CGDataProviderCopyData: fn(CF, 'CGDataProviderCopyData', 'void* CGDataProviderCopyData(void* provider)'),
+    CFRelease: fn(CF, 'CFRelease', 'void CFRelease(void* ref)'),
+    // CF containers are only aliases of the CoreFoundation types we read windows with
+    CFArrayGetCount: fn(CF, 'CFArrayGetCount', 'long CFArrayGetCount(void* array)'),
+    CFArrayGetValueAtIndex: fn(CF, 'CFArrayGetValueAtIndex', 'void* CFArrayGetValueAtIndex(void* array, long index)'),
+    CFDictionaryGetValue: fn(CF, 'CFDictionaryGetValue', 'void* CFDictionaryGetValue(void* dict, void* key)'),
+    CFStringCreateWithCString: fn(
+      CF,
+      'CFStringCreateWithCString',
+      'void* CFStringCreateWithCString(void* allocator, const char* cString, unsigned int encoding)'
+    ),
+    CFStringGetCString: fn(
+      CF,
+      'CFStringGetCString',
+      'bool CFStringGetCString(void* string, void* buffer, long bufferSize, unsigned int encoding)'
+    ),
+    CFStringGetLength: fn(CF, 'CFStringGetLength', 'long CFStringGetLength(void* string)'),
+    CFNumberGetValue: fn(CF, 'CFNumberGetValue', 'bool CFNumberGetValue(void* number, int type, void* valuePtr)'),
+    CFBooleanGetValue: fn(CF, 'CFBooleanGetValue', 'bool CFBooleanGetValue(void* boolean)'),
+  };
+  return bound;
+}
+
+function requireCG() {
+  if (process.platform !== 'darwin') throw new Error('the macOS backend runs on macOS only');
+  return bindings();
+}
+
+const K_CF_STRING_ENCODING_UTF8 = 0x08000100;
+
+function cfString(text) {
+  const b = bindings();
+  return b.CFStringCreateWithCString(null, String(text), K_CF_STRING_ENCODING_UTF8);
+}
+
+function cfToString(ref) {
+  if (!ref) return '';
+  const koffi = require('koffi');
+  const b = bindings();
+  const length = Number(b.CFStringGetLength(ref));
+  if (!Number.isFinite(length) || length < 0) return '';
+  const size = Math.max(16, length * 4 + 1);
+  const buffer = koffi.alloc('uint8_t', size);
+  if (!b.CFStringGetCString(ref, buffer, size, K_CF_STRING_ENCODING_UTF8)) return '';
+  return String(koffi.decode(buffer, 'uint8_t', size)).split('\0')[0];
+}
+
+function cfNumber(ref) {
+  if (!ref) return 0;
+  const koffi = require('koffi');
+  const out = koffi.alloc('double', 1);
+  const b = bindings();
+  // kCFNumberDoubleType = 13, kCFNumberSInt64Type = 4
+  if (b.CFNumberGetValue(ref, 13, out)) return Number(koffi.decode(out, 'double', 1));
+  return 0;
+}
+
+/** Reads a key out of the CGWindowList dictionaries. */
+function dictValue(dict, key, stringKey) {
+  const b = bindings();
+  const name = cfString(key);
+  let raw = null;
+  try {
+    raw = b.CFDictionaryGetValue(dict, name);
+    if (!raw) return stringKey ? '' : 0;
+    return stringKey ? cfToString(raw) : cfNumber(raw);
+  } finally {
+    if (name) b.CFRelease(name);
+  }
+}
+
+const sleep = (ms) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+
+// ------------------------------------------------------------------- input --
+
+function post(event, tap = K_CG_HID_EVENT_TAP) {
+  const b = bindings();
+  b.CGEventPost(tap, event);
+  if (event) b.CFRelease(event);
+  // let the window server digest the event before the next one
+  sleep(2);
+}
+
+function moveMouse(x, y) {
+  const b = requireCG();
+  const event = b.CGEventCreateMouseEvent(null, EVENT.mouseMoved, Number(x), Number(y), 0);
+  if (!event) return { ok: false, error: 'CGEventCreateMouseEvent failed' };
+  post(event);
+  return { ok: true, x, y };
+}
+
+function buttonEvent(type, x, y, button = 'left') {
+  const b = bindings();
+  const which = button === 'right' ? 1 : button === 'middle' ? 2 : 0;
+  return b.CGEventCreateMouseEvent(null, type, Number(x), Number(y), which);
+}
+
+function click(x, y, { button = 'left', count = 1 } = {}) {
+  const b = requireCG();
+  const clicks = Math.max(1, Math.trunc(count) || 1);
+  const down = button === 'right' ? EVENT.rightDown : EVENT.leftDown;
+  const up = button === 'right' ? EVENT.rightUp : EVENT.leftUp;
+  moveMouse(x, y);
+  for (let i = 1; i <= Math.min(3, clicks); i += 1) {
+    const press = buttonEvent(down, x, y, button);
+    if (press) {
+      b.CGEventSetIntegerValueField(press, FIELD_CLICK_STATE, i);
+      post(press);
+    }
+    const release = buttonEvent(up, x, y, button);
+    if (release) {
+      b.CGEventSetIntegerValueField(release, FIELD_CLICK_STATE, i);
+      post(release);
+    }
+  }
+  return { ok: true, x, y, button, count: clicks };
+}
+
+function mouseButton(x, y, button, down) {
+  const b = requireCG();
+  moveMouse(x, y);
+  const type = button === 'right' ? (down ? EVENT.rightDown : EVENT.rightUp) : down ? EVENT.leftDown : EVENT.leftUp;
+  const event = buttonEvent(type, x, y, button);
+  if (!event) return { ok: false, error: 'CGEventCreateMouseEvent failed' };
+  post(event);
+  return { ok: true, x, y, button };
+}
+
+function drag(x, y, x2, y2) {
+  const b = requireCG();
+  const downEvent = buttonEvent(EVENT.leftDown, x, y, 'left');
+  if (!downEvent) return { ok: false, error: 'CGEventCreateMouseEvent failed' };
+  post(downEvent);
+  const steps = 12;
+  for (let i = 1; i <= steps; i += 1) {
+    const px = Math.round(x + ((x2 - x) * i) / steps);
+    const py = Math.round(y + ((y2 - y) * i) / steps);
+    const moved = b.CGEventCreateMouseEvent(null, EVENT.leftDragged, px, py, 0);
+    if (moved) post(moved);
+  }
+  const upEvent = buttonEvent(EVENT.leftUp, x2, y2, 'left');
+  if (upEvent) post(upEvent);
+  const released = b.CGEventCreateMouseEvent(null, EVENT.mouseMoved, x2, y2, 0);
+  if (released) post(released);
+  return { ok: true, from: [x, y], to: [x2, y2] };
+}
+
+function scroll(amount, x = null, y = null) {
+  const b = requireCG();
+  if (x !== null && y !== null) moveMouse(x, y);
+  const lines = Math.max(1, Math.abs(Math.trunc(amount) || 1));
+  const signed = (Math.trunc(amount) || 1) > 0 ? lines : -lines;
+  const event = b.CGEventCreateScrollWheelEvent2(null, 1 /* line */, 1, signed, 0, 0);
+  if (!event) return { ok: false, error: 'CGEventCreateScrollWheelEvent2 failed' };
+  post(event);
+  return { ok: true, amount };
+}
+
+/** Types text as unicode: layout-independent, works for é, ü, 中文, emoji. */
+function typeText(text, x = null, y = null) {
+  const b = requireCG();
+  if (x !== null && y !== null) click(x, y);
+  const value = String(text || '');
+  let typed = 0;
+  for (const ch of value) {
+    const utf16 = Buffer.from(ch, 'utf16le');
+    const length = utf16.length / 2;
+    const down = b.CGEventCreateKeyboardEvent(null, 0, true);
+    const up = b.CGEventCreateKeyboardEvent(null, 0, false);
+    if (!down || !up) return { ok: false, error: 'CGEventCreateKeyboardEvent failed' };
+    const koffi = require('koffi');
+    const buffer = koffi.alloc('uint16_t', length);
+    koffi.encode(buffer, koffi.array('uint16_t', length), [...Array(length).keys()].map((i) => utf16.readUInt16LE(i * 2)));
+    b.CGEventKeyboardSetUnicodeString(down, length, buffer);
+    b.CGEventKeyboardSetUnicodeString(up, length, buffer);
+    post(down);
+    post(up);
+    typed += 1;
+  }
+  return { ok: true, chars: value.length, typed };
+}
+
+function keycodeFor(name) {
+  const key = String(name || '').toLowerCase();
+  const alias = ALIASES[key] || key;
+  if (alias in VK) return VK[alias];
+  if (key.length === 1) {
+    const lower = key.toLowerCase();
+    if (lower in VK) return VK[lower];
+  }
+  return null;
+}
+
+/**
+ * `ctrl+shift+t`, `cmd+space`, `alt+f4 x2` → { flags, keys, repeat }. Pure, so it is
+ * checked on every platform by test/native.js.
+ */
+function parseChord(combo) {
+  const parts = String(combo || '')
+    .toLowerCase()
+    .split(/[\s+]+/)
+    .filter(Boolean);
+  if (!parts.length) return { ok: false, error: `unreadable key combination: ${combo}` };
+  let repeat = 1;
+  let flags = 0;
+  const keys = [];
+  const modifiers = [];
+  for (const part of parts) {
+    if (part === 'repeat') {
+      repeat = 2;
+      continue;
+    }
+    if (/^x\d+$/.test(part)) {
+      repeat = Number(part.slice(1));
+      continue;
+    }
+    if (MODIFIERS[part] !== undefined) {
+      flags |= MODIFIERS[part];
+      modifiers.push(part);
+      continue;
+    }
+    const code = keycodeFor(part);
+    if (code !== null) keys.push(code);
+  }
+  if (!keys.length && !modifiers.length) return { ok: false, error: `unreadable key combination: ${combo}` };
+  if (!keys.length) keys.push(VK.space); // a bare modifier: press it
+  return { ok: true, flags, keys, modifiers, repeat: Math.min(50, Math.max(1, repeat)) };
+}
+
+function pressKeys(combo) {
+  const b = requireCG();
+  const chord = parseChord(combo);
+  if (!chord.ok) return chord;
+  const { flags, keys, repeat } = chord;
+  for (let i = 0; i < repeat; i += 1) {
+    for (const code of keys) {
+      const down = b.CGEventCreateKeyboardEvent(null, code, true);
+      const up = b.CGEventCreateKeyboardEvent(null, code, false);
+      if (!down || !up) return { ok: false, error: 'CGEventCreateKeyboardEvent failed' };
+      if (flags) {
+        b.CGEventSetFlags(down, flags);
+        b.CGEventSetFlags(up, flags);
+      }
+      post(down);
+      post(up);
+    }
+  }
+  return { ok: true, keys: combo };
+}
+
+// ------------------------------------------------------------------ screen --
+
+function screenSize() {
+  const b = requireCG();
+  const display = b.CGMainDisplayID();
+  return { ok: true, width: Number(b.CGDisplayPixelsWide(display)), height: Number(b.CGDisplayPixelsHigh(display)) };
+}
+
+function windows() {
+  const b = requireCG();
+  const options = 1 | 16; // kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements
+  const list = b.CGWindowListCopyWindowInfo(options, 0);
+  if (!list) return { ok: true, windows: [] };
+  const out = [];
+  try {
+    const count = Number(b.CFArrayGetCount(list));
+    for (let i = 0; i < count; i += 1) {
+      const dict = b.CFArrayGetValueAtIndex(list, i);
+      if (!dict) continue;
+      const name = dictValue(dict, 'kCGWindowName', true) || dictValue(dict, 'kCGWindowOwnerName', true);
+      if (!name) continue;
+      const layer = dictValue(dict, 'kCGWindowLayer', false);
+      if (layer !== 0) continue; // menu bar, dock, overlays
+      // kCGWindowBounds is a nested dictionary with X/Y/Width/Height numbers
+      const boundsRef = (() => {
+        const key = cfString('kCGWindowBounds');
+        try {
+          return b.CFDictionaryGetValue(dict, key);
+        } finally {
+          if (key) b.CFRelease(key);
+        }
+      })();
+      const x = boundsRef ? dictValue(boundsRef, 'X', false) : 0;
+      const y = boundsRef ? dictValue(boundsRef, 'Y', false) : 0;
+      const width = boundsRef ? dictValue(boundsRef, 'Width', false) : 0;
+      const height = boundsRef ? dictValue(boundsRef, 'Height', false) : 0;
+      if (!width || !height) continue;
+      out.push({
+        name,
+        handle: dictValue(dict, 'kCGWindowNumber', false),
+        hwnd: dictValue(dict, 'kCGWindowNumber', false),
+        x: Math.round(x),
+        y: Math.round(y),
+        width: Math.round(width),
+        height: Math.round(height),
+        process_id: dictValue(dict, 'kCGWindowOwnerPID', false),
+        status: 'normal',
+      });
+    }
+  } finally {
+    b.CFRelease(list);
+  }
+  return { ok: true, windows: out };
+}
+
+function activeWindow() {
+  const res = osascript('tell application "System Events" to get name of first application process whose frontmost is true');
+  if (!res.ok || !res.out) return { ok: true, name: '', handle: null };
+  const hit = windows().windows.find((w) => w.name === res.out) || null;
+  return { ok: true, name: res.out, handle: hit ? hit.handle : null };
+}
+
+/** CoreGraphics hands out BGRA (little endian, alpha first) with a padded row stride. */
+function bgraToRgb(raw, width, height, bytesPerRow, bitsPerPixel = 32) {
+  const rgb = Buffer.alloc(width * height * 3);
+  const bpp = Math.max(4, Math.floor(bitsPerPixel / 8));
+  for (let y = 0; y < height; y += 1) {
+    for (let x = 0; x < width; x += 1) {
+      const src = y * bytesPerRow + x * bpp;
+      const dst = (y * width + x) * 3;
+      rgb[dst] = raw[src + 2];
+      rgb[dst + 1] = raw[src + 1];
+      rgb[dst + 2] = raw[src];
+    }
+  }
+  return rgb;
+}
+
+function screenshot(dest) {
+  const b = requireCG();
+  const display = b.CGMainDisplayID();
+  const image = b.CGDisplayCreateImage(display);
+  if (!image) return { ok: false, error: 'CGDisplayCreateImage failed (grant Screen Recording to the terminal)' };
+  try {
+    const width = Number(b.CGImageGetWidth(image));
+    const height = Number(b.CGImageGetHeight(image));
+    const bytesPerRow = Number(b.CGImageGetBytesPerRow(image));
+    const bits = Number(b.CGImageGetBitsPerPixel(image));
+    const provider = b.CGImageGetDataProvider(image);
+    const data = provider ? b.CGDataProviderCopyData(provider) : null;
+    if (!data) return { ok: false, error: 'CGDataProviderCopyData failed' };
+    const koffi = require('koffi');
+    try {
+      const length = Number(b.CFDataGetLength(data));
+      const ptr = b.CFDataGetBytePtr(data);
+      if (!ptr) return { ok: false, error: 'CFDataGetBytePtr failed' };
+      const raw = koffi.decode(ptr, 'uint8_t', length);
+      const rgb = bgraToRgb(raw, width, height, bytesPerRow, bits);
+      fs.mkdirSync(path.dirname(dest), { recursive: true });
+      fs.writeFileSync(dest, encodePng(rgb, width, height));
+      return { ok: true, file: dest, width, height, mime: 'image/png' };
+    } finally {
+      b.CFRelease(data);
+    }
+  } finally {
+    b.CFRelease(image);
+  }
+}
+
+// ----------------------------------------------------- what CoreGraphics cannot do --
+
+function osascript(script) {
+  const res = spawnSync('osascript', ['-e', script], { encoding: 'utf8', timeout: 20000 });
+  return { ok: res.status === 0, out: String(res.stdout || '').trim(), error: String(res.stderr || '').trim() };
+}
+
+function launchApp(name) {
+  const res = spawnSync('open', ['-a', String(name)], { encoding: 'utf8', timeout: 30000 });
+  return res.status === 0 ? { ok: true, app: name } : { ok: false, error: String(res.stderr || 'open -a failed').trim() };
+}
+
+function focusWindow(name) {
+  const res = osascript(
+    `tell application "System Events" to set frontmost of (first application process whose name is ${JSON.stringify(
+      String(name)
+    )}) to true`
+  );
+  return res.ok ? { ok: true, name } : { ok: false, error: res.error || `window not found: ${name}` };
+}
+
+function resizeWindow(name, x, y, width, height) {
+  const res = osascript(
+    `tell application "System Events" to tell (first application process whose name is ${JSON.stringify(
+      String(name)
+    )}) to set {position, size} of window 1 to {{${x}, ${y}}, {${width}, ${height}}}`
+  );
+  return res.ok ? { ok: true, name, x, y, width, height } : { ok: false, error: res.error || 'resize failed' };
+}
+
+function listApps() {
+  const out = [];
+  for (const dir of ['/Applications', '/System/Applications', path.join(os.homedir(), 'Applications')]) {
+    let entries = [];
+    try {
+      entries = fs.readdirSync(dir);
+    } catch {
+      continue;
+    }
+    for (const entry of entries) {
+      if (!entry.endsWith('.app')) continue;
+      out.push({ name: entry.replace(/\.app$/, ''), app_id: path.join(dir, entry) });
+    }
+  }
+  return { ok: true, apps: out };
+}
+
+function state(handle = null) {
+  const wins = windows();
+  const size = screenSize();
+  const active = activeWindow();
+  const list = wins.ok ? wins.windows : [];
+  const focused = handle || active.handle;
+  const elements = list
+    .filter((w) => !focused || w.handle === focused)
+    .map((w) => ({
+      name: w.name,
+      control_type: 'Window',
+      kind: 'Window',
+      bounding_box: { left: w.x, top: w.y, right: w.x + w.width, bottom: w.y + w.height },
+      center: { x: w.x + Math.floor(w.width / 2), y: w.y + Math.floor(w.height / 2) },
+      value: '',
+      enabled: true,
+      automation_id: '',
+    }));
+  return {
+    ok: true,
+    windows: list,
+    active_window: active.handle ? list.find((w) => w.handle === active.handle) || { name: active.name, handle: active.handle } : null,
+    interactive_elements: elements,
+    scrollable_elements: [],
+    interactive_elements_text: elements
+      .map((e, i) => `${i + 1}. ${e.control_type} '${e.name}' at (${e.center.x}, ${e.center.y})`)
+      .join('\n'),
+    scrollable_elements_text: '',
+    tree_status: 'coregraphics',
+    system: {
+      windows_version: `darwin ${os.release()}`,
+      os_version: os.release(),
+      screen_size: { width: size.width, height: size.height },
+      user_account_type: 'Local Account',
+      default_language: process.env.LANG || 'en',
+      dpi_scaling: 1,
+    },
+    metadata: { depth: 'top-level', kind: 'windows-list', all_desktops: false },
+  };
+}
+
+function available() {
+  if (process.platform !== 'darwin') return { ok: false, backend: 'macos', error: 'macOS only' };
+  try {
+    const size = screenSize();
+    if (!size.width || !size.height) throw new Error('no display');
+    return { ok: true, backend: 'macos', detail: 'CoreGraphics via FFI (no external tools)', screen: size };
+  } catch (err) {
+    return { ok: false, backend: 'macos', error: err.message };
+  }
+}
+
+function sysinfo() {
+  const size = screenSize();
+  return {
+    ok: true,
+    system: {
+      windows_version: `darwin ${os.release()}`,
+      os_version: os.release(),
+      screen_size: { width: size.width, height: size.height },
+      user_account_type: 'Local Account',
+      default_language: process.env.LANG || 'en',
+      dpi_scaling: 1,
+    },
+  };
+}
+
+const backend = {
+  name: 'coregraphics-ffi',
+  available,
+  screenSize,
+  windows,
+  activeWindow,
+  state,
+  moveMouse,
+  click,
+  mouseButton,
+  drag,
+  scroll,
+  typeText,
+  pressKeys,
+  focusWindow,
+  resizeWindow,
+  screenshot,
+  launchApp,
+  listApps,
+  sysinfo,
+};
+
+module.exports = {
+  backend,
+  available,
+  screenSize,
+  windows,
+  activeWindow,
+  state,
+  moveMouse,
+  click,
+  mouseButton,
+  drag,
+  scroll,
+  typeText,
+  pressKeys,
+  focusWindow,
+  resizeWindow,
+  screenshot,
+  launchApp,
+  listApps,
+  sysinfo,
+  VK,
+  FLAG,
+  MODIFIERS,
+  EVENT,
+  keycodeFor,
+  parseChord,
+  bgraToRgb,
+};

package/src/automation/png.js

@@ -0,0 +1,47 @@
+'use strict';
+
+// PNG encoding without dependencies: the screenshots of the Linux and macOS backends
+// are RGB rasters, wrapped here in IHDR/IDAT/IEND with zlib.
+
+const zlib = require('node:zlib');
+
+function crc32(buf) {
+  let c = 0xffffffff;
+  for (let i = 0; i < buf.length; i += 1) {
+    c ^= buf[i];
+    for (let k = 0; k < 8; k += 1) c = c & 1 ? 0xedb88320 ^ (c >>> 1) : c >>> 1;
+  }
+  return (c ^ 0xffffffff) >>> 0;
+}
+
+function pngChunk(type, data) {
+  const len = Buffer.alloc(4);
+  len.writeUInt32BE(data.length);
+  const body = Buffer.concat([Buffer.from(type, 'ascii'), data]);
+  const crc = Buffer.alloc(4);
+  crc.writeUInt32BE(crc32(body));
+  return Buffer.concat([len, body, crc]);
+}
+
+/** rgb: width*height*3 bytes, top-down, no padding. */
+function encodePng(rgb, width, height) {
+  const stride = width * 3;
+  const raw = Buffer.alloc((stride + 1) * height);
+  for (let y = 0; y < height; y += 1) {
+    raw[y * (stride + 1)] = 0; // filter: none
+    rgb.copy(raw, y * (stride + 1) + 1, y * stride, (y + 1) * stride);
+  }
+  const ihdr = Buffer.alloc(13);
+  ihdr.writeUInt32BE(width, 0);
+  ihdr.writeUInt32BE(height, 4);
+  ihdr[8] = 8; // bit depth
+  ihdr[9] = 2; // truecolour
+  return Buffer.concat([
+    Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]),
+    pngChunk('IHDR', ihdr),
+    pngChunk('IDAT', zlib.deflateSync(raw, { level: 6 })),
+    pngChunk('IEND', Buffer.alloc(0)),
+  ]);
+}
+
+module.exports = { encodePng, crc32 };