@vox-ai-app/integrations 1.0.0

package/src/mail/send/mac/index.js

@@ -0,0 +1,93 @@
+import { resolveLocalPath } from '@vox-ai-app/tools'
+import {
+  execAbortable,
+  esc,
+  EXEC_TIMEOUT,
+  writeTempScript,
+  cleanupTemp,
+  parseTabSeparated
+} from '@vox-ai-app/tools/exec'
+import { ensureAppleMailConfigured } from '../../shared/index.js'
+export const sendEmailMac = async (
+  { to, cc, bcc, subject, body, attachments, account },
+  { signal } = {}
+) => {
+  await ensureAppleMailConfigured(signal)
+  const bodyEsc = esc(body).replace(/\n/g, '\\n')
+  const lines = ['tell application "Mail"']
+  if (account) {
+    lines.push(
+      `  set acct to first account whose name contains "${esc(account)}"`,
+      '  set addressesList to email addresses of acct',
+      '  set senderAddr to item 1 of addressesList',
+      `  set msg to make new outgoing message with properties {subject:"${esc(subject)}", content:"${bodyEsc}", visible:false, sender:senderAddr}`
+    )
+  } else {
+    lines.push(
+      `  set msg to make new outgoing message with properties {subject:"${esc(subject)}", content:"${bodyEsc}", visible:false}`
+    )
+  }
+  lines.push('  tell msg')
+  for (const addr of to) {
+    lines.push(`    make new to recipient with properties {address:"${esc(addr)}"}`)
+  }
+  for (const addr of cc) {
+    lines.push(`    make new cc recipient with properties {address:"${esc(addr)}"}`)
+  }
+  for (const addr of bcc) {
+    lines.push(`    make new bcc recipient with properties {address:"${esc(addr)}"}`)
+  }
+  for (const p of attachments) {
+    const abs = resolveLocalPath(p)
+    lines.push(
+      `    make new attachment with properties {file name:(POSIX file "${esc(abs)}")} at after the last paragraph`
+    )
+  }
+  lines.push('  end tell', '  send msg', 'end tell')
+  const scriptFile = await writeTempScript(lines.join('\n'), 'scpt')
+  try {
+    await execAbortable(
+      `osascript "${scriptFile}"`,
+      {
+        timeout: EXEC_TIMEOUT
+      },
+      signal
+    )
+    return {
+      status: 'sent',
+      to,
+      subject
+    }
+  } finally {
+    await cleanupTemp(scriptFile)
+  }
+}
+export const searchContactsMac = async (query, { signal } = {}) => {
+  const script = [
+    `set Q to "${esc(query)}"`,
+    'set output to ""',
+    'tell application "Contacts"',
+    '  set matched to every person whose name contains Q',
+    '  repeat with p in matched',
+    '    set pName to name of p',
+    '    repeat with e in emails of p',
+    '      set output to output & pName & "\\t" & (value of e) & "\\n"',
+    '    end repeat',
+    '  end repeat',
+    'end tell',
+    'return output'
+  ].join('\n')
+  const scriptFile = await writeTempScript(script, 'scpt')
+  try {
+    const { stdout } = await execAbortable(
+      `osascript "${scriptFile}"`,
+      {
+        timeout: EXEC_TIMEOUT
+      },
+      signal
+    )
+    return parseTabSeparated(stdout)
+  } finally {
+    await cleanupTemp(scriptFile)
+  }
+}

package/src/mail/shared/index.js

@@ -0,0 +1,6 @@
+export {
+  openMailAutomationSettings,
+  getAppleMailAccounts,
+  ensureAppleMailConfigured,
+  APPLE_MAIL_SETUP_MESSAGE
+} from './mac/index.js'

package/src/mail/shared/mac/index.js

@@ -0,0 +1,48 @@
+import { shell } from 'electron'
+import { execAbortable } from '@vox-ai-app/tools/exec'
+const AUTOMATION_PERMISSION_MESSAGE =
+  'Vox needs permission to control Mail. Please grant it in System Settings → Privacy & Security → Automation → Vox → Mail.'
+const APPLE_MAIL_SETUP_MESSAGE =
+  'macOS mail actions in Vox require Apple Mail with at least one configured account. If you use Gmail or Outlook elsewhere, add the account in Apple Mail first. If Mail is already set up, check macOS Automation permissions for Vox and try again.'
+const isAutomationDeniedError = (err) => {
+  const msg = String(err?.message || err?.stderr || '').toLowerCase()
+  return (
+    msg.includes('not allowed to send apple events') ||
+    msg.includes('apple event handler failed') ||
+    msg.includes('-1743') ||
+    msg.includes('access not allowed')
+  )
+}
+export const openMailAutomationSettings = () => {
+  shell.openExternal('x-apple.systempreferences:com.apple.preference.security?Privacy_Automation')
+}
+export const getAppleMailAccounts = async (signal) => {
+  try {
+    const { stdout } = await execAbortable(
+      'osascript -e \'tell application "Mail" to return name of every account\'',
+      {
+        timeout: 15_000
+      },
+      signal
+    )
+    return String(stdout)
+      .trim()
+      .split(',')
+      .map((name) => name.trim())
+      .filter(Boolean)
+  } catch (err) {
+    if (isAutomationDeniedError(err)) {
+      openMailAutomationSettings()
+      throw Object.assign(new Error(AUTOMATION_PERMISSION_MESSAGE), {
+        code: 'MAIL_AUTOMATION_REQUIRED'
+      })
+    }
+    throw new Error(APPLE_MAIL_SETUP_MESSAGE)
+  }
+}
+export const ensureAppleMailConfigured = async (signal) => {
+  const accounts = await getAppleMailAccounts(signal)
+  if (!accounts.length) throw new Error(APPLE_MAIL_SETUP_MESSAGE)
+  return accounts
+}
+export { APPLE_MAIL_SETUP_MESSAGE }

package/src/mail/tools.js

@@ -0,0 +1,41 @@
+import { MAIL_TOOL_DEFINITIONS } from './def.js'
+import {
+  sendEmail,
+  readEmails,
+  searchContacts,
+  getEmailBody,
+  replyToEmail,
+  forwardEmail,
+  markEmailRead,
+  flagEmail,
+  deleteEmail,
+  moveEmail,
+  createDraft,
+  saveAttachment
+} from './index.js'
+
+const DARWIN_ONLY = () => {
+  throw new Error('Mail tools are only available on macOS.')
+}
+
+const isDarwin = process.platform === 'darwin'
+
+const executors = {
+  read_emails: (_ctx) => (isDarwin ? readEmails : DARWIN_ONLY),
+  search_contacts: (_ctx) => (isDarwin ? searchContacts : DARWIN_ONLY),
+  send_email: (_ctx) => (isDarwin ? sendEmail : DARWIN_ONLY),
+  get_email_body: (_ctx) => (isDarwin ? getEmailBody : DARWIN_ONLY),
+  reply_to_email: (_ctx) => (isDarwin ? replyToEmail : DARWIN_ONLY),
+  forward_email: (_ctx) => (isDarwin ? forwardEmail : DARWIN_ONLY),
+  mark_email_read: (_ctx) => (isDarwin ? markEmailRead : DARWIN_ONLY),
+  flag_email: (_ctx) => (isDarwin ? flagEmail : DARWIN_ONLY),
+  delete_email: (_ctx) => (isDarwin ? deleteEmail : DARWIN_ONLY),
+  move_email: (_ctx) => (isDarwin ? moveEmail : DARWIN_ONLY),
+  create_draft: (_ctx) => (isDarwin ? createDraft : DARWIN_ONLY),
+  save_attachment: (_ctx) => (isDarwin ? saveAttachment : DARWIN_ONLY)
+}
+
+export const MAIL_TOOLS = MAIL_TOOL_DEFINITIONS.map((def) => ({
+  definition: def,
+  execute: executors[def.name]
+}))

package/src/screen/capture/index.js

@@ -0,0 +1 @@
+export { captureFullScreen, captureRegion, waitForScreenPermission } from './mac/index.js'

package/src/screen/capture/mac/index.js

@@ -0,0 +1,109 @@
+import { desktopCapturer, screen, shell, systemPreferences } from 'electron'
+import { exec } from 'child_process'
+import fs from 'fs/promises'
+import os from 'os'
+import path from 'path'
+const execAbortable = (command, options = {}, signal) => {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(new Error('Aborted'))
+      return
+    }
+    const child = exec(command, options, (error, stdout, stderr) => {
+      if (signal) signal.removeEventListener('abort', onAbort)
+      if (error)
+        reject(
+          Object.assign(error, {
+            stderr
+          })
+        )
+      else
+        resolve({
+          stdout,
+          stderr
+        })
+    })
+    const onAbort = () => {
+      try {
+        child.kill('SIGTERM')
+      } catch {
+        void 0
+      }
+    }
+    if (signal)
+      signal.addEventListener('abort', onAbort, {
+        once: true
+      })
+  })
+}
+export const waitForScreenPermission = async (signal) => {
+  const initial = systemPreferences.getMediaAccessStatus('screen')
+  if (initial === 'granted') return
+  await desktopCapturer
+    .getSources({
+      types: ['screen'],
+      thumbnailSize: {
+        width: 1,
+        height: 1
+      }
+    })
+    .catch(() => {})
+  await shell.openExternal(
+    'x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture'
+  )
+  const deadline = Date.now() + 60_000
+  while (Date.now() < deadline) {
+    if (signal?.aborted) throw new Error('Aborted')
+    await new Promise((r) => setTimeout(r, 500))
+    if (systemPreferences.getMediaAccessStatus('screen') === 'granted') return
+  }
+  throw new Error(
+    'Screen recording permission was not granted. Please allow access in System Settings → Privacy & Security → Screen Recording.'
+  )
+}
+export const captureFullScreen = async (_, { signal } = {}) => {
+  await waitForScreenPermission(signal)
+  const primaryDisplay = screen.getPrimaryDisplay()
+  const { width, height } = primaryDisplay.size
+  const sources = await desktopCapturer.getSources({
+    types: ['screen'],
+    thumbnailSize: {
+      width,
+      height
+    }
+  })
+  const primary = sources[0]
+  if (!primary) throw new Error('No screen source available for capture.')
+  const captureSize = primary.thumbnail.getSize()
+  const base64Image = primary.thumbnail.toJPEG(75).toString('base64')
+  return {
+    text: `Captured full screen (${captureSize.width}x${captureSize.height}). Coordinates in this image map directly to click_at screen coordinates.`,
+    imageBase64: base64Image,
+    mimeType: 'image/jpeg'
+  }
+}
+export const captureRegion = async ({ x, y, width, height }, { signal } = {}) => {
+  await waitForScreenPermission(signal)
+  const xi = Math.round(Number(x))
+  const yi = Math.round(Number(y))
+  const wi = Math.round(Number(width))
+  const hi = Math.round(Number(height))
+  const tmpFile = path.join(os.tmpdir(), `vox_region_${Date.now()}.jpg`)
+  try {
+    await execAbortable(
+      `screencapture -R ${xi},${yi},${wi},${hi} -t jpg "${tmpFile}"`,
+      {
+        timeout: 10_000
+      },
+      signal
+    )
+    const buffer = await fs.readFile(tmpFile)
+    return {
+      text: `Captured region (${wi}x${hi}) at (${xi},${yi}). Coordinates map directly to click_at screen coordinates.`,
+      imageBase64: buffer.toString('base64'),
+      mimeType: 'image/jpeg'
+    }
+  } finally {
+    await fs.unlink(tmpFile).catch(() => {})
+  }
+}

package/src/screen/control/index.js

@@ -0,0 +1,15 @@
+export {
+  clickAt,
+  moveMouse,
+  typeText,
+  keyPress,
+  scroll,
+  drag,
+  getMousePosition,
+  getUiElements,
+  clipboardRead,
+  clipboardWrite,
+  focusApp,
+  launchApp,
+  listApps
+} from './mac/index.js'

package/src/screen/control/mac/accessibility.js

@@ -0,0 +1,25 @@
+import { systemPreferences, shell } from 'electron'
+
+const openAccessibilitySettings = () => {
+  shell.openExternal(
+    'x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility'
+  )
+}
+
+export const ensureAccessibility = () => {
+  const trusted = systemPreferences.isTrustedAccessibilityClient(false)
+  if (!trusted) {
+    const prompted = systemPreferences.isTrustedAccessibilityClient(true)
+    if (!prompted) {
+      openAccessibilitySettings()
+      throw Object.assign(
+        new Error(
+          'Vox needs Accessibility permission to control the screen. Opening System Settings → Privacy & Security → Accessibility — please enable it for Vox and try again.'
+        ),
+        {
+          code: 'ACCESSIBILITY_REQUIRED'
+        }
+      )
+    }
+  }
+}

package/src/screen/control/mac/apps.js

@@ -0,0 +1,62 @@
+import { enqueueScreen } from '../../queue.js'
+import {
+  ensureAccessibility,
+  UI_ELEMENTS_SCRIPT,
+  LONG_TIMEOUT,
+  execAbortable,
+  writeTmp,
+  cleanTmp
+} from './helpers.js'
+
+export const getUiElements = ({ app, maxElements } = {}, { signal } = {}) =>
+  enqueueScreen(async () => {
+    ensureAccessibility()
+    const limit = Math.max(1, Math.min(1000, Number(maxElements) || 200))
+    let script = UI_ELEMENTS_SCRIPT
+    if (app) {
+      script = UI_ELEMENTS_SCRIPT.replace(
+        'var proc = se.processes.whose({ frontmost: true })[0];',
+        `var proc = se.processes.whose({ name: "${String(app).replace(/"/g, '\\"')}" })[0];`
+      )
+    }
+    const file = await writeTmp(script, 'js')
+    try {
+      const { stdout } = await execAbortable(
+        `osascript -l JavaScript "${file}"`,
+        { timeout: LONG_TIMEOUT },
+        signal
+      )
+      const all = JSON.parse(stdout.trim())
+      const elements = Array.isArray(all) ? all : (all?.elements ?? [])
+      const total = elements.length
+      return { elements: elements.slice(0, limit), total, truncated: total > limit }
+    } catch (err) {
+      throw new Error(`UI element inspection failed: ${err?.message || err}`)
+    } finally {
+      await cleanTmp(file)
+    }
+  })
+
+export const focusApp = async ({ app }, { signal } = {}) => {
+  await execAbortable(`open -a ${JSON.stringify(app)}`, { timeout: 10_000 }, signal)
+  return { action: 'focus_app', app }
+}
+
+export const launchApp = async ({ app, args = [] }, { signal } = {}) => {
+  const argStr =
+    Array.isArray(args) && args.length
+      ? ` --args ${args.map((a) => JSON.stringify(a)).join(' ')}`
+      : ''
+  await execAbortable(`open -a ${JSON.stringify(app)}${argStr}`, { timeout: 15_000 }, signal)
+  return { action: 'launch_app', app }
+}
+
+export const listApps = async (_, { signal } = {}) => {
+  const { stdout } = await execAbortable('ls /Applications/', { timeout: 10_000 }, signal)
+  const apps = stdout
+    .trim()
+    .split('\n')
+    .filter((a) => a.endsWith('.app'))
+    .map((a) => a.replace(/\.app$/, ''))
+  return { apps }
+}

package/src/screen/control/mac/exec.js

@@ -0,0 +1,66 @@
+import { exec } from 'child_process'
+import fs from 'fs/promises'
+import os from 'os'
+import path from 'path'
+
+const EXEC_TIMEOUT = 30_000
+
+export const execAbortable = (command, options = {}, signal) => {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(new Error('Aborted'))
+      return
+    }
+    const child = exec(command, options, (error, stdout, stderr) => {
+      if (signal) signal.removeEventListener('abort', onAbort)
+      if (error)
+        reject(
+          Object.assign(error, {
+            stderr
+          })
+        )
+      else
+        resolve({
+          stdout,
+          stderr
+        })
+    })
+    const onAbort = () => {
+      try {
+        child.kill('SIGTERM')
+      } catch {
+        void 0
+      }
+    }
+    if (signal)
+      signal.addEventListener('abort', onAbort, {
+        once: true
+      })
+  })
+}
+
+export const writeTmp = async (content, ext) => {
+  const file = path.join(
+    os.tmpdir(),
+    `vox_ctrl_${Date.now()}_${Math.random().toString(36).slice(2)}.${ext}`
+  )
+  await fs.writeFile(file, content, 'utf8')
+  return file
+}
+
+export const cleanTmp = (file) => fs.unlink(file).catch(() => {})
+
+export const runPy = async (script, signal, timeout = EXEC_TIMEOUT) => {
+  const file = await writeTmp(script, 'py')
+  try {
+    return await execAbortable(
+      `python3 "${file}"`,
+      {
+        timeout
+      },
+      signal
+    )
+  } finally {
+    await cleanTmp(file)
+  }
+}

package/src/screen/control/mac/helpers.js

@@ -0,0 +1,5 @@
+export * from './exec.js'
+export * from './accessibility.js'
+export * from './python.js'
+export * from './keycodes.js'
+export * from './ui-scan.js'

package/src/screen/control/mac/index.js

@@ -0,0 +1,10 @@
+import { clipboard } from 'electron'
+export * from './mouse.js'
+export * from './keyboard.js'
+export * from './apps.js'
+
+export const clipboardRead = () => ({ text: clipboard.readText() })
+export const clipboardWrite = ({ text }) => {
+  clipboard.writeText(String(text || ''))
+  return { ok: true }
+}

package/src/screen/control/mac/keyboard.js

@@ -0,0 +1,34 @@
+import { enqueueScreen } from '../../queue.js'
+import {
+  ensureAccessibility,
+  CHAR_CODES,
+  KEY_CODES,
+  pyTypeText,
+  pyKeyCode,
+  pyCharKey,
+  runPy
+} from './helpers.js'
+
+export const typeText = ({ text }, { signal } = {}) =>
+  enqueueScreen(async () => {
+    ensureAccessibility()
+    if (!text) throw new Error('"text" is required.')
+    await runPy(pyTypeText(text), signal)
+    return { action: 'type', length: text.length }
+  })
+
+export const keyPress = ({ key, modifiers = [] }, { signal } = {}) =>
+  enqueueScreen(async () => {
+    ensureAccessibility()
+    if (!key) throw new Error('"key" is required.')
+    const keyLower = String(key).toLowerCase().trim()
+    const mods = (Array.isArray(modifiers) ? modifiers : [modifiers]).filter(Boolean)
+    const keyCode = KEY_CODES[keyLower] ?? CHAR_CODES[keyLower]
+    if (keyCode !== undefined) {
+      await runPy(pyKeyCode(keyCode, mods), signal)
+    } else {
+      const b64 = Buffer.from(keyLower, 'utf8').toString('base64')
+      await runPy(pyCharKey(b64, mods), signal)
+    }
+    return { action: 'key_press', key, modifiers: mods }
+  })

package/src/screen/control/mac/keycodes.js

@@ -0,0 +1,87 @@
+export const KEY_CODES = {
+  return: 36,
+  enter: 36,
+  tab: 48,
+  space: 49,
+  delete: 51,
+  backspace: 51,
+  escape: 53,
+  esc: 53,
+  command: 55,
+  cmd: 55,
+  shift: 56,
+  option: 58,
+  alt: 58,
+  control: 59,
+  ctrl: 59,
+  left: 123,
+  right: 124,
+  down: 125,
+  up: 126,
+  f1: 122,
+  f2: 120,
+  f3: 99,
+  f4: 118,
+  f5: 96,
+  f6: 97,
+  f7: 98,
+  f8: 100,
+  f9: 101,
+  f10: 109,
+  f11: 103,
+  f12: 111,
+  home: 115,
+  end: 119,
+  pageup: 116,
+  pagedown: 121
+}
+
+export const CHAR_CODES = {
+  a: 0,
+  s: 1,
+  d: 2,
+  f: 3,
+  h: 4,
+  g: 5,
+  z: 6,
+  x: 7,
+  c: 8,
+  v: 9,
+  b: 11,
+  q: 12,
+  w: 13,
+  e: 14,
+  r: 15,
+  y: 16,
+  t: 17,
+  1: 18,
+  2: 19,
+  3: 20,
+  4: 21,
+  6: 22,
+  5: 23,
+  9: 25,
+  7: 26,
+  8: 28,
+  0: 29,
+  o: 31,
+  u: 32,
+  i: 34,
+  p: 35,
+  l: 37,
+  j: 38,
+  k: 40,
+  n: 45,
+  m: 46,
+  ',': 43,
+  '.': 47,
+  '/': 44,
+  ';': 41,
+  "'": 39,
+  '[': 33,
+  ']': 30,
+  '\\': 42,
+  '-': 27,
+  '=': 24,
+  '`': 50
+}

package/src/screen/control/mac/mouse.js

@@ -0,0 +1,59 @@
+import { enqueueScreen } from '../../queue.js'
+import {
+  ensureAccessibility,
+  pyClick,
+  pyMove,
+  pyDrag,
+  pyScroll,
+  pyGetMousePos,
+  runPy
+} from './helpers.js'
+
+export const clickAt = ({ x, y, button = 'left', count = 1 }, { signal } = {}) =>
+  enqueueScreen(async () => {
+    ensureAccessibility()
+    const xInt = Math.round(Number(x))
+    const yInt = Math.round(Number(y))
+    const btn = button === 'right' ? 'right' : 'left'
+    const clicks = Math.max(1, Math.min(3, Number(count)))
+    await runPy(pyClick(xInt, yInt, btn, clicks), signal)
+    return { action: 'click', x: xInt, y: yInt, button: btn, count: clicks }
+  })
+
+export const moveMouse = ({ x, y }, { signal } = {}) =>
+  enqueueScreen(async () => {
+    ensureAccessibility()
+    const xInt = Math.round(Number(x))
+    const yInt = Math.round(Number(y))
+    await runPy(pyMove(xInt, yInt), signal)
+    return { action: 'move', x: xInt, y: yInt }
+  })
+
+export const scroll = ({ x, y, deltaX = 0, deltaY = -3 }, { signal } = {}) =>
+  enqueueScreen(async () => {
+    ensureAccessibility()
+    const xInt = Math.round(Number(x))
+    const yInt = Math.round(Number(y))
+    const dx = Math.round(Number(deltaX))
+    const dy = Math.round(Number(deltaY))
+    await runPy(pyScroll(xInt, yInt, dx, dy), signal)
+    return { action: 'scroll', x: xInt, y: yInt, deltaX: dx, deltaY: dy }
+  })
+
+export const drag = ({ fromX, fromY, toX, toY }, { signal } = {}) =>
+  enqueueScreen(async () => {
+    ensureAccessibility()
+    const x1 = Math.round(Number(fromX))
+    const y1 = Math.round(Number(fromY))
+    const x2 = Math.round(Number(toX))
+    const y2 = Math.round(Number(toY))
+    await runPy(pyDrag(x1, y1, x2, y2), signal)
+    return { action: 'drag', from: { x: x1, y: y1 }, to: { x: x2, y: y2 } }
+  })
+
+export const getMousePosition = (_, { signal } = {}) =>
+  enqueueScreen(async () => {
+    const { stdout } = await runPy(pyGetMousePos(), signal)
+    const [x, y] = stdout.trim().split(',').map(Number)
+    return { x: x ?? 0, y: y ?? 0 }
+  })