@vox-ai-app/integrations 1.0.0

package/src/screen/control/mac/python-keyboard.js

@@ -0,0 +1,66 @@
+const MOD_FLAGS_PY = `MOD_FLAGS = {
+    'command': Quartz.kCGEventFlagMaskCommand,
+    'cmd': Quartz.kCGEventFlagMaskCommand,
+    'shift': Quartz.kCGEventFlagMaskShift,
+    'option': Quartz.kCGEventFlagMaskAlternate,
+    'alt': Quartz.kCGEventFlagMaskAlternate,
+    'control': Quartz.kCGEventFlagMaskControl,
+    'ctrl': Quartz.kCGEventFlagMaskControl,
+}`
+
+export const pyTypeText = (text) => {
+  const b64 = Buffer.from(text, 'utf8').toString('base64')
+  return `
+import Quartz, time, base64
+text = base64.b64decode('${b64}').decode('utf-8')
+src = Quartz.CGEventSourceCreate(Quartz.kCGEventSourceStateHIDSystemState)
+for char in text:
+    e = Quartz.CGEventCreateKeyboardEvent(src, 0, True)
+    Quartz.CGEventKeyboardSetUnicodeString(e, 1, char)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+    time.sleep(0.01)
+    e = Quartz.CGEventCreateKeyboardEvent(src, 0, False)
+    Quartz.CGEventKeyboardSetUnicodeString(e, 1, char)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+    time.sleep(0.02)
+`
+}
+
+export const pyKeyCode = (keyCode, mods) => `
+import Quartz, time
+${MOD_FLAGS_PY}
+flags = 0
+for m in ${JSON.stringify(mods)}:
+    flags |= MOD_FLAGS.get(m.lower(), 0)
+src = Quartz.CGEventSourceCreate(Quartz.kCGEventSourceStateHIDSystemState)
+e = Quartz.CGEventCreateKeyboardEvent(src, ${keyCode}, True)
+if flags:
+    Quartz.CGEventSetFlags(e, flags)
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+time.sleep(0.05)
+e = Quartz.CGEventCreateKeyboardEvent(src, ${keyCode}, False)
+if flags:
+    Quartz.CGEventSetFlags(e, flags)
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+`
+
+export const pyCharKey = (b64, mods) => `
+import Quartz, time, base64
+${MOD_FLAGS_PY}
+flags = 0
+for m in ${JSON.stringify(mods)}:
+    flags |= MOD_FLAGS.get(m.lower(), 0)
+text = base64.b64decode('${b64}').decode('utf-8')
+src = Quartz.CGEventSourceCreate(Quartz.kCGEventSourceStateHIDSystemState)
+for char in text:
+    e = Quartz.CGEventCreateKeyboardEvent(src, 0, True)
+    Quartz.CGEventKeyboardSetUnicodeString(e, 1, char)
+    if flags:
+        Quartz.CGEventSetFlags(e, flags)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+    time.sleep(0.01)
+    e = Quartz.CGEventCreateKeyboardEvent(src, 0, False)
+    Quartz.CGEventKeyboardSetUnicodeString(e, 1, char)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+    time.sleep(0.02)
+`

package/src/screen/control/mac/python-mouse.js

@@ -0,0 +1,66 @@
+export const pyClick = (x, y, button = 'left', count = 1) => `
+import sys, time
+try:
+    import Quartz
+except ImportError:
+    sys.exit(1)
+
+x, y = ${x}, ${y}
+count = ${count}
+is_right = ${button === 'right' ? 'True' : 'False'}
+
+btn  = Quartz.kCGMouseButtonRight   if is_right else Quartz.kCGMouseButtonLeft
+down = Quartz.kCGEventRightMouseDown if is_right else Quartz.kCGEventLeftMouseDown
+up   = Quartz.kCGEventRightMouseUp   if is_right else Quartz.kCGEventLeftMouseUp
+pt   = Quartz.CGPoint(x, y)
+
+for i in range(count):
+    n = i + 1
+    e = Quartz.CGEventCreateMouseEvent(None, down, pt, btn)
+    Quartz.CGEventSetIntegerValueField(e, Quartz.kCGMouseEventClickState, n)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+    time.sleep(0.05)
+    e = Quartz.CGEventCreateMouseEvent(None, up, pt, btn)
+    Quartz.CGEventSetIntegerValueField(e, Quartz.kCGMouseEventClickState, n)
+    Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+    if i < count - 1:
+        time.sleep(0.12)
+`
+
+export const pyMove = (x, y) => `
+import Quartz
+pt = Quartz.CGPoint(${x}, ${y})
+e = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventMouseMoved, pt, Quartz.kCGMouseButtonLeft)
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+`
+
+export const pyScroll = (x, y, dx, dy) => `
+import Quartz, time
+pt = Quartz.CGPoint(${x}, ${y})
+e = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventMouseMoved, pt, Quartz.kCGMouseButtonLeft)
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+time.sleep(0.05)
+e = Quartz.CGEventCreateScrollWheelEvent(None, Quartz.kCGScrollEventUnitLine, 2, ${dy}, ${dx})
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+`
+
+export const pyDrag = (x1, y1, x2, y2) => `
+import Quartz, time
+p1 = Quartz.CGPoint(${x1}, ${y1})
+p2 = Quartz.CGPoint(${x2}, ${y2})
+e = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventLeftMouseDown, p1, Quartz.kCGMouseButtonLeft)
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+time.sleep(0.08)
+e = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventLeftMouseDragged, p2, Quartz.kCGMouseButtonLeft)
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+time.sleep(0.05)
+e = Quartz.CGEventCreateMouseEvent(None, Quartz.kCGEventLeftMouseUp, p2, Quartz.kCGMouseButtonLeft)
+Quartz.CGEventPost(Quartz.kCGHIDEventTap, e)
+`
+
+export const pyGetMousePos = () => `
+import Quartz
+event = Quartz.CGEventCreate(None)
+pos = Quartz.CGEventGetLocation(event)
+print(f"{int(pos.x)},{int(pos.y)}")
+`

package/src/screen/control/mac/python.js

@@ -0,0 +1,2 @@
+export * from './python-mouse.js'
+export * from './python-keyboard.js'

package/src/screen/control/mac/ui-scan.js

@@ -0,0 +1,45 @@
+export const LONG_TIMEOUT = 60_000
+
+export const UI_ELEMENTS_SCRIPT = `
+ObjC.import('Foundation');
+function collect(elem, depth, out) {
+  if (depth > 5) return;
+  var children;
+  try { children = elem.entireContents(); } catch(e) { return; }
+  for (var i = 0; i < children.length; i++) {
+    var el = children[i];
+    try {
+      var role = '';
+      try { role = el.role(); } catch(e) { continue; }
+      var label = '';
+      try { label = el.title(); } catch(e) {}
+      if (!label) try { label = el.description(); } catch(e) {}
+      if (!label) try { var v = el.value(); if (typeof v === 'string') label = v; } catch(e) {}
+      var enabled = true;
+      try { enabled = el.enabled(); } catch(e) {}
+      var pos = { x: 0, y: 0 };
+      try { var p = el.position(); pos = { x: p.x, y: p.y }; } catch(e) {}
+      var size = { w: 0, h: 0 };
+      try { var s = el.size(); size = { w: s.width, h: s.height }; } catch(e) {}
+      var INTERACTIVE = ['AXButton','AXTextField','AXTextArea','AXCheckBox','AXRadioButton',
+        'AXPopUpButton','AXComboBox','AXSlider','AXLink','AXMenuItem','AXMenu',
+        'AXTab','AXCell','AXRow','AXStaticText'];
+      if (label || INTERACTIVE.indexOf(role) !== -1) {
+        out.push({ role: role, label: label, enabled: enabled,
+          x: Math.round(pos.x), y: Math.round(pos.y),
+          w: Math.round(size.w), h: Math.round(size.h) });
+      }
+      collect(el, depth + 1, out);
+    } catch(e) {}
+  }
+}
+var se = Application('System Events');
+var proc = se.processes.whose({ frontmost: true })[0];
+var appName = proc.name();
+var wins = proc.windows();
+var out = [];
+for (var w = 0; w < Math.min(wins.length, 3); w++) {
+  collect(wins[w], 0, out);
+}
+JSON.stringify({ app: appName, count: out.length, elements: out });
+`

package/src/screen/def.js

@@ -0,0 +1,304 @@
+export const SCREEN_TOOL_DEFINITIONS = [
+  {
+    name: 'capture_full_screen',
+    description:
+      "Capture a screenshot of the user's entire screen. Use this whenever the user asks you to look at, read, or help with something on their screen — including emails, documents, web pages, code, chat messages, forms, or any other visible content. The returned image contains the full screen rendered at the time of capture. You must read and analyze ALL visible text and UI elements in the image in detail, including the content of open applications, browser tabs, emails, and documents. Never say you cannot read text from an image — you have full vision capabilities and must extract and reason over all on-screen content.",
+    parameters: {
+      type: 'object',
+      properties: {}
+    }
+  },
+  {
+    name: 'capture_region',
+    description:
+      'Capture a screenshot of a specific rectangular region of the screen. Use this instead of capture_full_screen when you already know the area of interest — it is faster and returns a smaller, more focused image. Coordinates must be in logical screen points, matching the values from click_at.',
+    parameters: {
+      type: 'object',
+      properties: {
+        x: {
+          type: 'number',
+          description: 'Left edge of the region in screen points.'
+        },
+        y: {
+          type: 'number',
+          description: 'Top edge of the region in screen points.'
+        },
+        width: {
+          type: 'number',
+          description: 'Width of the region in screen points.'
+        },
+        height: {
+          type: 'number',
+          description: 'Height of the region in screen points.'
+        }
+      },
+      required: ['x', 'y', 'width', 'height']
+    }
+  },
+  {
+    name: 'click_at',
+    description:
+      "Click the mouse at specific screen coordinates on the user's Mac. Use after capture_full_screen to interact with UI elements. Supports left click, right click, and double click.",
+    parameters: {
+      type: 'object',
+      properties: {
+        x: {
+          type: 'number',
+          description: 'Horizontal screen coordinate in pixels.'
+        },
+        y: {
+          type: 'number',
+          description: 'Vertical screen coordinate in pixels.'
+        },
+        button: {
+          type: 'string',
+          description: 'Mouse button: "left" (default) or "right".'
+        },
+        count: {
+          type: 'integer',
+          description: 'Number of clicks: 1 (default) or 2 for double-click.'
+        }
+      },
+      required: ['x', 'y']
+    }
+  },
+  {
+    name: 'move_mouse',
+    description: 'Move the mouse cursor to specific screen coordinates without clicking.',
+    parameters: {
+      type: 'object',
+      properties: {
+        x: {
+          type: 'number',
+          description: 'Horizontal screen coordinate in pixels.'
+        },
+        y: {
+          type: 'number',
+          description: 'Vertical screen coordinate in pixels.'
+        }
+      },
+      required: ['x', 'y']
+    }
+  },
+  {
+    name: 'type_text',
+    description:
+      'Type text at the current cursor position as if typed on a keyboard. Use after clicking the target input field.',
+    parameters: {
+      type: 'object',
+      properties: {
+        text: {
+          type: 'string',
+          description: 'The text to type.'
+        }
+      },
+      required: ['text']
+    }
+  },
+  {
+    name: 'key_press',
+    description:
+      'Press a keyboard key, optionally with modifier keys (command, shift, option, control). Use for shortcuts like Cmd+C, Cmd+V, Enter, Escape, arrow keys, etc.',
+    parameters: {
+      type: 'object',
+      properties: {
+        key: {
+          type: 'string',
+          description:
+            'Key to press. Named keys: return, tab, space, delete, escape, left, right, up, down, f1-f12, home, end, pageup, pagedown. Or a single character like "a", "c", "v".'
+        },
+        modifiers: {
+          type: 'array',
+          items: {
+            type: 'string'
+          },
+          description:
+            'Modifier keys to hold: "command" (or "cmd"), "shift", "option" (or "alt"), "control" (or "ctrl").'
+        }
+      },
+      required: ['key']
+    }
+  },
+  {
+    name: 'scroll',
+    description:
+      'Scroll at specific screen coordinates. Positive deltaY scrolls up, negative scrolls down.',
+    parameters: {
+      type: 'object',
+      properties: {
+        x: {
+          type: 'number',
+          description: 'Horizontal screen coordinate.'
+        },
+        y: {
+          type: 'number',
+          description: 'Vertical screen coordinate.'
+        },
+        deltaX: {
+          type: 'number',
+          description: 'Horizontal scroll amount (default 0).'
+        },
+        deltaY: {
+          type: 'number',
+          description:
+            'Vertical scroll amount. Negative = scroll down, positive = scroll up. Default -3.'
+        }
+      },
+      required: ['x', 'y']
+    }
+  },
+  {
+    name: 'drag',
+    description: 'Click and drag from one screen coordinate to another.',
+    parameters: {
+      type: 'object',
+      properties: {
+        fromX: {
+          type: 'number',
+          description: 'Start X coordinate.'
+        },
+        fromY: {
+          type: 'number',
+          description: 'Start Y coordinate.'
+        },
+        toX: {
+          type: 'number',
+          description: 'End X coordinate.'
+        },
+        toY: {
+          type: 'number',
+          description: 'End Y coordinate.'
+        }
+      },
+      required: ['fromX', 'fromY', 'toX', 'toY']
+    }
+  },
+  {
+    name: 'get_mouse_position',
+    description: 'Get the current mouse cursor position on screen.',
+    parameters: {
+      type: 'object',
+      properties: {}
+    }
+  },
+  {
+    name: 'get_ui_elements',
+    description:
+      'Inspect the macOS Accessibility tree of the frontmost app (or a named app) and return every visible UI element with its role, label, position (x, y), and size (w, h). Use this INSTEAD of screenshot-based coordinate guessing when you need to click a specific button, text field, menu item, or link — find the element by its label, then pass its center coordinates (x + w/2, y + h/2) to click_at. This is faster and more reliable than clicking by pixel.',
+    parameters: {
+      type: 'object',
+      properties: {
+        app: {
+          type: 'string',
+          description:
+            'Name of the macOS app process to inspect (e.g. "Safari", "Mail", "Finder"). If omitted, inspects whichever app is currently in the foreground.'
+        },
+        maxElements: {
+          type: 'integer',
+          description: 'Maximum number of UI elements to return (default 200, max 1000).'
+        }
+      }
+    }
+  },
+  {
+    name: 'clipboard_read',
+    description: 'Read the current text content of the system clipboard.',
+    parameters: {
+      type: 'object',
+      properties: {}
+    }
+  },
+  {
+    name: 'clipboard_write',
+    description: 'Write text to the system clipboard, replacing its current content.',
+    parameters: {
+      type: 'object',
+      properties: {
+        text: {
+          type: 'string',
+          description: 'Text to place on the clipboard.'
+        }
+      },
+      required: ['text']
+    }
+  },
+  {
+    name: 'focus_app',
+    description:
+      'Bring a macOS application to the foreground. Use before interacting with an app that may be in the background.',
+    parameters: {
+      type: 'object',
+      properties: {
+        app: {
+          type: 'string',
+          description:
+            'Application name as it appears in /Applications (e.g. "Safari", "Finder", "Visual Studio Code").'
+        }
+      },
+      required: ['app']
+    }
+  },
+  {
+    name: 'launch_app',
+    description: 'Launch a macOS application, opening it if not already running.',
+    parameters: {
+      type: 'object',
+      properties: {
+        app: {
+          type: 'string',
+          description: 'Application name (e.g. "Terminal", "Xcode").'
+        },
+        args: {
+          type: 'array',
+          items: {
+            type: 'string'
+          },
+          description: 'Optional arguments to pass to the application.'
+        }
+      },
+      required: ['app']
+    }
+  },
+  {
+    name: 'list_apps',
+    description: "List all installed applications in /Applications on the user's Mac.",
+    parameters: {
+      type: 'object',
+      properties: {}
+    }
+  },
+  {
+    name: 'acquire_screen',
+    description:
+      'Claim exclusive screen control for the current agent session. Call this before starting a multi-step screen automation task to prevent other agents from interfering. The lock auto-expires after 30s of screen inactivity. Always pair with release_screen when done. Use force=true to break a stuck lock from a dead session.',
+    parameters: {
+      type: 'object',
+      properties: {
+        sessionId: {
+          type: 'string',
+          description: 'Unique identifier for this agent session.'
+        },
+        force: {
+          type: 'boolean',
+          description:
+            'Override an existing lock held by another session. Use only when a previous session is known to be dead.'
+        }
+      },
+      required: ['sessionId']
+    }
+  },
+  {
+    name: 'release_screen',
+    description: 'Release the screen control lock acquired with acquire_screen.',
+    parameters: {
+      type: 'object',
+      properties: {
+        sessionId: {
+          type: 'string',
+          description: 'The session ID used in acquire_screen.'
+        }
+      },
+      required: ['sessionId']
+    }
+  }
+]

package/src/screen/index.js

@@ -0,0 +1,17 @@
+export { captureFullScreen, captureRegion, waitForScreenPermission } from './capture/index.js'
+export {
+  clickAt,
+  moveMouse,
+  typeText,
+  keyPress,
+  scroll,
+  drag,
+  getMousePosition,
+  getUiElements,
+  clipboardRead,
+  clipboardWrite,
+  focusApp,
+  launchApp,
+  listApps
+} from './control/index.js'
+export { enqueueScreen, acquireScreen, releaseScreen, getScreenSession } from './queue.js'

package/src/screen/queue.js

@@ -0,0 +1,54 @@
+const LOCK_TTL_MS = 30_000
+let _tail = Promise.resolve()
+let _sessionId = null
+let _lockExpiry = null
+let _lockTimer = null
+const _clearLock = () => {
+  _sessionId = null
+  _lockExpiry = null
+  if (_lockTimer) {
+    clearTimeout(_lockTimer)
+    _lockTimer = null
+  }
+}
+const _renewLock = () => {
+  if (!_sessionId) return
+  if (_lockTimer) clearTimeout(_lockTimer)
+  _lockExpiry = Date.now() + LOCK_TTL_MS
+  _lockTimer = setTimeout(_clearLock, LOCK_TTL_MS)
+}
+const _isLockExpired = () => _lockExpiry !== null && Date.now() > _lockExpiry
+export const enqueueScreen = (fn) => {
+  _renewLock()
+  const task = _tail.then(fn)
+  _tail = task.catch(() => {})
+  return task
+}
+export const acquireScreen = ({ sessionId, force = false }) => {
+  if (_sessionId && _sessionId !== sessionId && !_isLockExpired() && !force) {
+    throw Object.assign(
+      new Error(`Screen is in use by session "${_sessionId}". Release it first or wait.`),
+      {
+        code: 'SCREEN_LOCKED',
+        holder: _sessionId
+      }
+    )
+  }
+  _clearLock()
+  _sessionId = sessionId
+  _renewLock()
+  return {
+    sessionId
+  }
+}
+export const releaseScreen = ({ sessionId }) => {
+  if (_sessionId === sessionId) _clearLock()
+  return {
+    ok: true
+  }
+}
+export const getScreenSession = () => ({
+  sessionId: _sessionId,
+  expiresAt: _lockExpiry,
+  ttlMs: _lockExpiry ? Math.max(0, _lockExpiry - Date.now()) : null
+})

package/src/screen/tools.js

@@ -0,0 +1,50 @@
+import { SCREEN_TOOL_DEFINITIONS } from './def.js'
+import { captureFullScreen, captureRegion } from './capture/index.js'
+import {
+  clickAt,
+  moveMouse,
+  typeText,
+  keyPress,
+  scroll,
+  drag,
+  getMousePosition,
+  getUiElements,
+  clipboardRead,
+  clipboardWrite,
+  focusApp,
+  launchApp,
+  listApps
+} from './control/index.js'
+import { acquireScreen, releaseScreen } from './queue.js'
+
+const DARWIN_ONLY = () => {
+  throw new Error('Screen tools are only available on macOS.')
+}
+
+const isDarwin = process.platform === 'darwin'
+const guard = (fn) => (isDarwin ? fn : DARWIN_ONLY)
+
+const executors = {
+  capture_full_screen: (_ctx) => guard(captureFullScreen),
+  capture_region: (_ctx) => guard(captureRegion),
+  click_at: (_ctx) => guard(clickAt),
+  move_mouse: (_ctx) => guard(moveMouse),
+  type_text: (_ctx) => guard(typeText),
+  key_press: (_ctx) => guard(keyPress),
+  scroll: (_ctx) => guard(scroll),
+  drag: (_ctx) => guard(drag),
+  get_mouse_position: (_ctx) => guard(getMousePosition),
+  get_ui_elements: (_ctx) => guard(getUiElements),
+  clipboard_read: (_ctx) => clipboardRead,
+  clipboard_write: (_ctx) => clipboardWrite,
+  focus_app: (_ctx) => guard(focusApp),
+  launch_app: (_ctx) => guard(launchApp),
+  list_apps: (_ctx) => guard(listApps),
+  acquire_screen: (_ctx) => acquireScreen,
+  release_screen: (_ctx) => releaseScreen
+}
+
+export const SCREEN_TOOLS = SCREEN_TOOL_DEFINITIONS.map((def) => ({
+  definition: def,
+  execute: executors[def.name]
+}))

package/src/tools.js

@@ -0,0 +1,6 @@
+import { SCREEN_TOOLS } from './screen/tools.js'
+import { MAIL_TOOLS } from './mail/tools.js'
+import { IMESSAGE_TOOLS } from './imessage/tools.js'
+
+export const ALL_INTEGRATION_TOOLS = [...SCREEN_TOOLS, ...MAIL_TOOLS, ...IMESSAGE_TOOLS]
+export { SCREEN_TOOLS, MAIL_TOOLS, IMESSAGE_TOOLS }