@vox-ai-app/integrations 1.0.0

package/README.md

@@ -0,0 +1,125 @@
+# @vox-ai-app/vox-integrations
+
+macOS system integrations for Vox: Apple Mail, Screen control, and iMessage. Each integration ships with tool implementations and LLM tool definitions.
+
+Requires macOS. Each integration needs specific system permissions granted by the user.
+
+## Install
+
+```sh
+npm install @vox-ai-app/vox-integrations
+```
+
+Peer dependency: `electron >= 28`
+
+## Exports
+
+| Export                                        | Contents                      |
+| --------------------------------------------- | ----------------------------- |
+| `@vox-ai-app/vox-integrations`                | All exports                   |
+| `@vox-ai-app/vox-integrations/defs`           | All tool definitions          |
+| `@vox-ai-app/vox-integrations/mail`           | Mail functions                |
+| `@vox-ai-app/vox-integrations/screen`         | Screen capture + control      |
+| `@vox-ai-app/vox-integrations/screen/capture` | Capture only                  |
+| `@vox-ai-app/vox-integrations/screen/control` | Control only                  |
+| `@vox-ai-app/vox-integrations/screen/queue`   | Session acquire/release       |
+| `@vox-ai-app/vox-integrations/imessage`       | iMessage data, reply, service |
+
+## Mail
+
+Requires **Automation permission** for Mail (System Settings → Privacy & Security → Automation).
+
+```js
+import {
+  sendEmail,
+  readEmails,
+  searchContacts,
+  replyToEmail
+} from '@vox-ai-app/vox-integrations/mail'
+
+const emails = await readEmails({ account: 'Work', folder: 'INBOX', limit: 20 })
+await sendEmail({ to: 'user@example.com', subject: 'Hi', body: 'Hello.' })
+await replyToEmail({ messageId: '...', body: 'Thanks!' })
+```
+
+Tool definitions:
+
+```js
+import { MAIL_TOOL_DEFINITIONS } from '@vox-ai-app/vox-integrations/defs'
+```
+
+## Screen
+
+Requires **Accessibility permission** (System Settings → Privacy & Security → Accessibility).
+
+```js
+import {
+  captureFullScreen,
+  clickAt,
+  typeText,
+  getUiElements
+} from '@vox-ai-app/vox-integrations/screen'
+import { acquireScreen, releaseScreen } from '@vox-ai-app/vox-integrations/screen/queue'
+
+const session = await acquireScreen()
+try {
+  const img = await captureFullScreen()
+  await clickAt({ x: 100, y: 200 })
+  await typeText({ text: 'Hello' })
+} finally {
+  await releaseScreen(session)
+}
+```
+
+Tool definitions:
+
+```js
+import { SCREEN_TOOL_DEFINITIONS } from '@vox-ai-app/vox-integrations/defs'
+```
+
+## iMessage
+
+Requires **Full Disk Access** (System Settings → Privacy & Security → Full Disk Access).
+
+### Tool use (read conversations, send messages)
+
+```js
+import { listConversations, listContacts, sendReply } from '@vox-ai-app/vox-integrations/imessage'
+
+const conversations = listConversations()
+const contacts = listContacts()
+await sendReply('+15551234567', 'Hello from Vox!')
+```
+
+### Gateway service (AI replies to incoming iMessages)
+
+```js
+import { createIMessageService } from '@vox-ai-app/vox-integrations/imessage'
+
+const svc = createIMessageService({
+  logger,
+  onTranscript: (text, handle) => {
+    /* emit to UI */
+  },
+  onOpenSettings: () => shell.openExternal('x-apple.systempreferences:...'),
+  onMessage: async (text, handle) => {
+    // call your AI here, return the reply string
+    return await askAI(text)
+  }
+})
+
+svc.start('my-passphrase')
+// user sends "my-passphrase\nWhat's the weather?" → AI replies back
+```
+
+`onMessage` must return a `Promise<string | null>`. Returning `null` skips the reply.
+
+Tool definitions:
+
+```js
+import { IMESSAGE_TOOL_DEFINITIONS } from '@vox-ai-app/vox-integrations/defs'
+```
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@vox-ai-app/integrations",
+  "version": "1.0.0",
+  "type": "module",
+  "license": "MIT",
+  "description": "macOS integrations (Mail, Screen, iMessage) for Vox",
+  "main": "./src/index.js",
+  "private": false,
+  "exports": {
+    ".": "./src/index.js",
+    "./defs": "./src/defs/index.js",
+    "./defs/mail": "./src/defs/mail.js",
+    "./defs/screen": "./src/defs/screen.js",
+    "./defs/imessage": "./src/defs/imessage.js",
+    "./mail": "./src/mail/index.js",
+    "./screen": "./src/screen/index.js",
+    "./screen/capture": "./src/screen/capture/index.js",
+    "./screen/control": "./src/screen/control/index.js",
+    "./screen/queue": "./src/screen/queue.js",
+    "./imessage": "./src/imessage/index.js"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/vox-ai-app/vox"
+  },
+  "files": [
+    "src",
+    "LICENSE",
+    "README.md"
+  ],
+  "peerDependencies": {
+    "electron": ">=28.0.0"
+  },
+  "dependencies": {
+    "@vox-ai-app/tools": "^1.0.0",
+    "better-sqlite3": "^12.0.0"
+  }
+}

package/src/imessage/def.js

@@ -0,0 +1,41 @@
+export const IMESSAGE_TOOL_DEFINITIONS = [
+  {
+    name: 'list_imessage_conversations',
+    description:
+      "List recent iMessage/SMS conversations from the user's macOS Messages app. Returns the 50 most recent threads with contact handle and a short snippet. Requires Full Disk Access in System Settings → Privacy & Security.",
+    parameters: {
+      type: 'object',
+      properties: {},
+      required: []
+    }
+  },
+  {
+    name: 'list_imessage_contacts',
+    description:
+      "List contacts from the user's macOS AddressBook with their iMessage handles (phone numbers and email addresses). Useful for looking up who to message.",
+    parameters: {
+      type: 'object',
+      properties: {},
+      required: []
+    }
+  },
+  {
+    name: 'send_imessage',
+    description:
+      'Send an iMessage or SMS to a contact via the macOS Messages app using AppleScript. The handle must be a phone number (e.g. +14155551234) or email address registered with iMessage.',
+    parameters: {
+      type: 'object',
+      properties: {
+        handle: {
+          type: 'string',
+          description: 'Phone number or email address of the recipient.'
+        },
+        text: {
+          type: 'string',
+          description: 'Message text to send.'
+        }
+      },
+      required: ['handle', 'text']
+    }
+  }
+]

package/src/imessage/index.js

@@ -0,0 +1,9 @@
+export {
+  canReadDb,
+  queryNewMessages,
+  getMaxRowId,
+  listConversations,
+  listContacts
+} from './mac/data.js'
+export { sendReply } from './mac/reply.js'
+export { createIMessageService } from './mac/service.js'

package/src/imessage/mac/data.js

@@ -0,0 +1,144 @@
+import { existsSync, readdirSync } from 'fs'
+import { join } from 'path'
+import Database from 'better-sqlite3'
+
+const HOME = process.env.HOME
+const MESSAGES_DB = `${HOME}/Library/Messages/chat.db`
+const AB_DIR = join(HOME, 'Library/Application Support/AddressBook')
+
+export const canReadDb = () => {
+  try {
+    const db = new Database(MESSAGES_DB, { readonly: true, fileMustExist: true })
+    db.close()
+    return true
+  } catch {
+    return false
+  }
+}
+
+export const queryNewMessages = (afterRowId) => {
+  const db = new Database(MESSAGES_DB, { readonly: true, fileMustExist: true })
+  try {
+    return db
+      .prepare(
+        `SELECT m.ROWID, m.text,
+                CASE
+                  WHEN m.is_from_me = 0 THEN h.id
+                  ELSE COALESCE(
+                    (
+                      SELECT h2.id FROM chat_message_join cmj
+                      JOIN chat_handle_join chj ON chj.chat_id = cmj.chat_id
+                      JOIN handle h2 ON h2.ROWID = chj.handle_id
+                      WHERE cmj.message_id = m.ROWID
+                      LIMIT 1
+                    ),
+                    (
+                      SELECT REPLACE(REPLACE(REPLACE(c.chat_identifier, 'iMessage;-;', ''), 'SMS;-;', ''), 'tel:', '')
+                      FROM chat_message_join cmj2
+                      JOIN chat c ON c.ROWID = cmj2.chat_id
+                      WHERE cmj2.message_id = m.ROWID
+                      LIMIT 1
+                    )
+                  )
+                END AS reply_handle
+         FROM message m
+         LEFT JOIN handle h ON m.handle_id = h.ROWID
+         WHERE m.ROWID > ?
+           AND m.text IS NOT NULL AND m.text != ''
+         ORDER BY m.ROWID ASC`
+      )
+      .all(afterRowId)
+  } finally {
+    db.close()
+  }
+}
+
+export const getMaxRowId = () => {
+  const db = new Database(MESSAGES_DB, { readonly: true, fileMustExist: true })
+  try {
+    const row = db.prepare(`SELECT MAX(ROWID) AS maxId FROM message`).get()
+    return row?.maxId ?? 0
+  } finally {
+    db.close()
+  }
+}
+
+export const listConversations = () => {
+  const db = new Database(MESSAGES_DB, { readonly: true, fileMustExist: true })
+  try {
+    const rows = db
+      .prepare(
+        `SELECT h.id AS handle_id,
+                MAX(m.date) AS last_date,
+                (SELECT text FROM message WHERE handle_id = h.ROWID AND text IS NOT NULL AND text != '' ORDER BY date DESC LIMIT 1) AS snippet
+         FROM handle h
+         JOIN message m ON m.handle_id = h.ROWID
+         WHERE m.text IS NOT NULL
+         GROUP BY h.id
+         ORDER BY last_date DESC
+         LIMIT 50`
+      )
+      .all()
+    return rows.map((r) => ({
+      handle: r.handle_id,
+      snippet: r.snippet ? String(r.snippet).slice(0, 80) : ''
+    }))
+  } finally {
+    db.close()
+  }
+}
+
+const findAddressBookDbs = () => {
+  const dbs = []
+  const main = join(AB_DIR, 'AddressBook-v22.abcddb')
+  if (existsSync(main)) dbs.push(main)
+  try {
+    const sources = join(AB_DIR, 'Sources')
+    for (const entry of readdirSync(sources)) {
+      const p = join(sources, entry, 'AddressBook-v22.abcddb')
+      if (existsSync(p)) dbs.push(p)
+    }
+  } catch {
+    void 0
+  }
+  return dbs
+}
+
+const CONTACTS_QUERY = `
+  SELECT COALESCE(r.ZNAME, TRIM(COALESCE(r.ZFIRSTNAME,'') || ' ' || COALESCE(r.ZLASTNAME,''))) AS name,
+         p.ZFULLNUMBER AS handle
+  FROM ZABCDRECORD r
+  JOIN ZABCDPHONENUMBER p ON p.ZOWNER = r.Z_PK
+  WHERE name != '' AND handle IS NOT NULL AND handle != ''
+  UNION ALL
+  SELECT COALESCE(r.ZNAME, TRIM(COALESCE(r.ZFIRSTNAME,'') || ' ' || COALESCE(r.ZLASTNAME,''))) AS name,
+         e.ZADDRESS AS handle
+  FROM ZABCDRECORD r
+  JOIN ZABCDEMAILADDRESS e ON e.ZOWNER = r.Z_PK
+  WHERE name != '' AND handle IS NOT NULL AND handle != ''
+  ORDER BY name ASC
+  LIMIT 500`
+
+export const listContacts = (onError) => {
+  const seen = new Set()
+  const results = []
+  for (const dbPath of findAddressBookDbs()) {
+    try {
+      const db = new Database(dbPath, { readonly: true, fileMustExist: true })
+      try {
+        for (const row of db.prepare(CONTACTS_QUERY).all()) {
+          const key = `${row.name}|${row.handle}`
+          if (!seen.has(key)) {
+            seen.add(key)
+            results.push({ name: row.name, handle: row.handle })
+          }
+        }
+      } finally {
+        db.close()
+      }
+    } catch (err) {
+      onError?.(err)
+    }
+  }
+  return results
+}

package/src/imessage/mac/reply.js

@@ -0,0 +1,68 @@
+import fs from 'fs/promises'
+import os from 'os'
+import path from 'path'
+import { promisify } from 'util'
+import { exec } from 'child_process'
+
+const execAsync = promisify(exec)
+
+const writeTmp = async (content, ext) => {
+  const file = path.join(os.tmpdir(), `vox_ims_${Date.now()}.${ext}`)
+  await fs.writeFile(file, content, 'utf8')
+  return file
+}
+
+const escapeAppleScript = (s) => String(s).replace(/\\/g, '\\\\').replace(/"/g, '\\"')
+
+const toAppleScriptString = (s) => {
+  const lines = escapeAppleScript(s).split('\n')
+  if (lines.length === 1) return `"${lines[0]}"`
+  return lines.map((l) => `"${l}"`).join(' & return & ')
+}
+
+export const sendReply = async (handle, text, filePaths = []) => {
+  const handleEsc = escapeAppleScript(handle)
+  const textExpr = toAppleScriptString(text)
+
+  const sendText = text
+    ? `
+  try
+    send ${textExpr} to buddy "${handleEsc}" of service "iMessage"
+    set sent to true
+  end try
+  if not sent then
+    try
+      send ${textExpr} to buddy "${handleEsc}" of (first service)
+      set sent to true
+    end try
+  end if`
+    : ''
+
+  const sendFiles = filePaths
+    .map((p) => {
+      const pEsc = escapeAppleScript(p)
+      return `
+  try
+    send POSIX file "${pEsc}" to buddy "${handleEsc}" of service "iMessage"
+    set sent to true
+  end try`
+    })
+    .join('')
+
+  const script = `tell application "Messages"
+  set sent to false
+${sendText}
+${sendFiles}
+  if not sent then
+    error "Could not send reply to ${handleEsc}"
+  end if
+end tell`
+
+  const scriptFile = await writeTmp(script, 'scpt')
+  try {
+    await execAsync(`osascript "${scriptFile}"`, { timeout: 15_000 })
+    console.info('[imessage] Reply sent to', handle, 'files:', filePaths.length)
+  } finally {
+    await fs.unlink(scriptFile).catch(() => {})
+  }
+}

package/src/imessage/mac/service.js

@@ -0,0 +1,141 @@
+import {
+  canReadDb,
+  getMaxRowId,
+  queryNewMessages,
+  listConversations as listConversationsFromDb,
+  listContacts as listContactsFromDb
+} from './data.js'
+import { sendReply } from './reply.js'
+
+const POLL_INTERVAL_MS = 3_000
+const REPLY_TIMEOUT_MS = 90_000
+const FDA_ERROR_MESSAGE =
+  'Vox needs Full Disk Access to read Messages. Opening System Settings → Privacy & Security → Full Disk Access — please enable it for Vox and try again.'
+
+/**
+ * Creates a self-contained iMessage watcher service.
+ *
+ * @param {object} opts
+ * @param {(text: string, handle: string) => Promise<string|null>} opts.onMessage
+ *   Called when a passphrase-matched message arrives. Must return the AI reply text (or null to skip reply).
+ * @param {(text: string, handle: string) => void} [opts.onTranscript]
+ *   Optional: called before onMessage, useful for emitting transcript events upstream.
+ * @param {() => void} [opts.onOpenSettings]
+ *   Optional: called when Full Disk Access is required. Default opens nothing.
+ * @param {{ info: Function, warn: Function, error: Function }} [opts.logger]
+ *   Optional: defaults to console.
+ * @param {number} [opts.pollIntervalMs]
+ *   Poll interval in ms. Default 3000.
+ * @returns {{ start, stop, getPassphrase, listConversations, listContacts, openSettings }}
+ */
+export const createIMessageService = ({
+  onMessage,
+  onTranscript,
+  onOpenSettings,
+  logger,
+  pollIntervalMs = POLL_INTERVAL_MS
+} = {}) => {
+  const _log = logger ?? console
+
+  let _pollTimer = null
+  let _passphrase = null
+  let _lastSeenRowId = 0
+
+  const openSettings = () => {
+    onOpenSettings?.()
+  }
+
+  const handleIncomingMessage = async (row) => {
+    if (!_passphrase) return
+    const raw = String(row.text || '')
+    const newline = raw.indexOf('\n')
+    if (newline === -1) return
+    const firstLine = raw.slice(0, newline).trim()
+    if (firstLine !== _passphrase) return
+    const text = raw.slice(newline + 1).trim()
+    if (!text) return
+
+    const sender = row.reply_handle
+    _log.info('[imessage] Passphrase match, reply_handle:', sender, 'text:', text.slice(0, 60))
+    if (!sender) return
+
+    onTranscript?.(text, sender)
+
+    let aiText = null
+    try {
+      aiText = await Promise.race([
+        onMessage(text, sender),
+        new Promise((resolve) => setTimeout(() => resolve(null), REPLY_TIMEOUT_MS))
+      ])
+    } catch (err) {
+      _log.error('[imessage] onMessage error:', err?.message)
+      return
+    }
+
+    if (!aiText) {
+      _log.warn('[imessage] No AI response received')
+      return
+    }
+
+    try {
+      await sendReply(sender, aiText)
+    } catch (err) {
+      _log.error('[imessage] Reply failed:', err?.message)
+    }
+  }
+
+  const poll = async () => {
+    if (!_passphrase || !canReadDb()) return
+    try {
+      const rows = queryNewMessages(_lastSeenRowId)
+      for (const row of rows) {
+        if (row.ROWID > _lastSeenRowId) _lastSeenRowId = row.ROWID
+        await handleIncomingMessage(row)
+      }
+    } catch (err) {
+      _log.error('[imessage] Poll error:', err?.message)
+    }
+  }
+
+  const start = (passphrase) => {
+    if (!canReadDb()) {
+      openSettings()
+      throw Object.assign(new Error(FDA_ERROR_MESSAGE), { code: 'IMESSAGE_FDA_REQUIRED' })
+    }
+    if (!passphrase?.trim()) throw new Error('Passphrase cannot be empty.')
+
+    stop()
+    _passphrase = passphrase.trim()
+    _lastSeenRowId = getMaxRowId()
+    _pollTimer = setInterval(poll, pollIntervalMs)
+    _log.info('[imessage] Watching all messages with passphrase')
+    return { passphrase: _passphrase }
+  }
+
+  const stop = () => {
+    if (_pollTimer) {
+      clearInterval(_pollTimer)
+      _pollTimer = null
+    }
+    _passphrase = null
+    _lastSeenRowId = 0
+  }
+
+  const getPassphrase = () => _passphrase
+
+  const listConversations = () => {
+    if (!canReadDb()) {
+      openSettings()
+      throw Object.assign(new Error(FDA_ERROR_MESSAGE), { code: 'IMESSAGE_FDA_REQUIRED' })
+    }
+    return listConversationsFromDb()
+  }
+
+  const listContacts = () => {
+    return listContactsFromDb((err) => {
+      _log.warn('[imessage] Skipping AddressBook DB:', err?.message)
+    })
+  }
+
+  return { start, stop, getPassphrase, listConversations, listContacts, openSettings }
+}

package/src/imessage/tools.js

@@ -0,0 +1,44 @@
+import { IMESSAGE_TOOL_DEFINITIONS } from './def.js'
+import { listConversations, listContacts } from './mac/data.js'
+import { sendReply } from './mac/reply.js'
+
+const DARWIN_ONLY = () => {
+  throw new Error('iMessage is only available on macOS.')
+}
+
+const listImessageConversations = async () => {
+  const conversations = listConversations()
+  return { conversations }
+}
+
+const listImessageContacts = async () => {
+  const contacts = listContacts()
+  return { contacts }
+}
+
+const sendImessage = async ({ handle, text }) => {
+  if (!handle) throw new Error('"handle" is required.')
+  if (!text) throw new Error('"text" is required.')
+  await sendReply(handle, String(text))
+  return { ok: true, handle }
+}
+
+const macExecutors = {
+  list_imessage_conversations: (_ctx) => listImessageConversations,
+  list_imessage_contacts: (_ctx) => listImessageContacts,
+  send_imessage: (_ctx) => sendImessage
+}
+
+const executors =
+  process.platform === 'darwin'
+    ? macExecutors
+    : {
+        list_imessage_conversations: (_ctx) => DARWIN_ONLY,
+        list_imessage_contacts: (_ctx) => DARWIN_ONLY,
+        send_imessage: (_ctx) => DARWIN_ONLY
+      }
+
+export const IMESSAGE_TOOLS = IMESSAGE_TOOL_DEFINITIONS.map((def) => ({
+  definition: def,
+  execute: executors[def.name]
+}))

package/src/index.js

@@ -0,0 +1,7 @@
+export { MAIL_TOOL_DEFINITIONS } from './mail/def.js'
+export { SCREEN_TOOL_DEFINITIONS } from './screen/def.js'
+export { IMESSAGE_TOOL_DEFINITIONS } from './imessage/def.js'
+export * from './mail/index.js'
+export * from './screen/index.js'
+export * from './imessage/index.js'
+export { ALL_INTEGRATION_TOOLS, SCREEN_TOOLS, MAIL_TOOLS, IMESSAGE_TOOLS } from './tools.js'