@cohortapp/agent-sdk 2.3.0

package/desktop-control/README.md

@@ -0,0 +1,56 @@
+# Desktop Control Layer
+
+This directory contains the architecture and tooling for controlling real macOS applications on the dedicated Mac mini. The system operates as Robin Hayes's workstation — her desk, terminal, inbox, and execution cockpit.
+
+## Architecture
+
+The desktop control layer uses a combination of:
+
+- **Claude Computer Use** — Screen capture, UI element identification, click/type/scroll actions
+- **AppleScript / JXA** — Native macOS automation for app control, window management, notifications
+- **Accessibility APIs** — UI element inspection and programmatic interaction
+- **Shell commands** — `open`, `osascript`, process management via `launchctl`
+
+## Control Flow
+
+```
+Agent Request → Desktop Operator → Safety Check → Action Execution → State Verification → Result
+```
+
+Every desktop action follows this sequence:
+
+1. **Intent**: An agent requests a desktop action (e.g., "read latest messages in #general on Slack")
+2. **Safety Check**: Action is classified by risk level and checked against policies
+3. **State Capture**: Screenshot taken to verify current app/window state
+4. **Execution**: Action performed via appropriate control method
+5. **Verification**: Screenshot taken to verify action completed correctly
+6. **Logging**: Full action log written with before/after screenshots
+7. **Result**: Extracted information or confirmation returned to requesting agent
+
+## Application Support
+
+| Application             | Control Method             | Risk Level  | Primary Use                               |
+| ----------------------- | -------------------------- | ----------- | ----------------------------------------- |
+| Slack                   | Computer Use + AppleScript | Medium-High | Channel monitoring, messaging, follow-ups |
+| Gmail (Browser)         | Computer Use               | Medium-High | Inbox triage, drafting, sending           |
+| WhatsApp Desktop        | Computer Use               | High        | Chat monitoring, reply drafting           |
+| Browser (Safari/Chrome) | Computer Use + AppleScript | Medium      | Calendar, dashboards, portals             |
+| Calendar                | Computer Use + AppleScript | Low-Medium  | Schedule viewing, event management        |
+| Finder                  | AppleScript                | Low         | File management                           |
+| Terminal                | Shell + AppleScript        | Medium      | Command execution                         |
+
+## Safety Principles
+
+1. **Observe before acting** — Always capture state before performing any action
+2. **Verify after acting** — Always confirm the action had the intended effect
+3. **Log everything** — Every action, screenshot, and result is logged
+4. **Respect approval gates** — Sending actions require appropriate authorisation
+5. **Rate limit** — No more than 30 actions per minute, 5-second minimum between send operations
+6. **Kill switch** — `scripts/emergency-stop.sh` halts all desktop operations immediately
+
+## Files
+
+- `app-profiles/` — Per-application control profiles and navigation patterns
+- `safety-policies/` — Action-level safety rules per application
+- `scripts/` — AppleScript and shell utilities for app control
+- `state/` — Current application state cache

package/desktop-control/app-profiles/gmail.yaml

@@ -0,0 +1,120 @@
+# Gmail App Control Profile (Browser-based)
+# Controls Gmail via browser on the Mac mini
+# Account: robin@example.com
+
+app_name: Gmail
+access_method: browser # Accessed via Safari or Chrome
+url: https://mail.google.com
+control_methods: [computer-use]
+risk_level: medium-high
+
+# Launch and focus
+launch:
+  method: computer-use
+  steps:
+    - action: open_browser
+    - action: navigate_to
+      url: https://mail.google.com
+    - action: verify_state
+      check: gmail_inbox_loaded
+
+# Read operations (always permitted)
+read_operations:
+  scan_inbox:
+    risk: low
+    approval: none
+    steps:
+      - action: screenshot
+      - action: extract_email_list
+    output: inbox_summary
+
+  read_email:
+    risk: low
+    approval: none
+    steps:
+      - action: click_email
+      - action: screenshot
+      - action: extract_email_content
+    output: email_content
+
+  search_emails:
+    risk: low
+    approval: none
+    steps:
+      - action: click_search
+      - action: type_text
+        text: "{query}"
+      - action: press_key
+        key: enter
+      - action: screenshot
+    output: search_results
+
+# Write operations
+write_operations:
+  draft_reply:
+    risk: low
+    approval: none # drafting is safe
+    steps:
+      - action: click_reply
+      - action: type_text
+        text: "{reply_content}"
+      # DO NOT click send
+    output: draft_created
+
+  draft_compose:
+    risk: low
+    approval: none
+    steps:
+      - action: click_compose
+      - action: fill_to
+        text: "{recipient}"
+      - action: fill_subject
+        text: "{subject}"
+      - action: fill_body
+        text: "{body}"
+      # DO NOT click send
+    output: draft_created
+
+  send_email_internal:
+    risk: medium
+    approval: pre-approved
+    requires:
+      - recipient_domain: northwind.ai
+      - message_category: [routine-followup, status-request, acknowledgement]
+    steps:
+      - action: verify_recipient_is_internal
+      - action: verify_content_safe
+      - action: click_send
+      - action: verify_state
+        check: email_sent_confirmation
+    logging: full
+
+  send_email_external:
+    risk: high
+    approval: explicit
+    requires:
+      - approval_id: "{approval_reference}"
+    steps:
+      - action: verify_approval_granted
+      - action: click_send
+      - action: verify_state
+        check: email_sent_confirmation
+    logging: full
+
+# Rate limits
+rate_limits:
+  max_actions_per_minute: 15
+  min_delay_between_sends: 10000 # 10 seconds
+  max_sends_per_hour: 20
+  max_sends_per_day: 100
+
+# Forbidden actions
+forbidden:
+  - delete_email
+  - empty_trash
+  - change_account_settings
+  - add_forwarding_rules
+  - modify_filters
+  - grant_account_access
+  - change_password
+  - access_other_accounts

package/desktop-control/app-profiles/slack.yaml

@@ -0,0 +1,315 @@
+# Slack Desktop App Control Profile
+# Defines how Robin controls the Slack application on the Mac mini
+
+app_name: Slack
+bundle_id: com.tinyspeck.slackmacgap
+control_methods: [computer-use, applescript]
+risk_level: medium-high
+
+# Launch and focus
+launch:
+  command: "open -a Slack"
+  applescript: 'tell application "Slack" to activate'
+  verify: screenshot # confirm Slack is in foreground
+
+# Navigation patterns
+navigation:
+  switch_workspace:
+    method: computer-use
+    steps:
+      - action: keyboard_shortcut
+        keys: ["cmd", "1-9"] # workspace number
+      - action: verify_state
+        check: workspace_name_visible
+    risk: low
+
+  open_channel:
+    method: computer-use
+    steps:
+      - action: keyboard_shortcut
+        keys: ["cmd", "k"] # quick switcher
+      - action: type_text
+        text: "{channel_name}"
+      - action: press_key
+        key: enter
+      - action: verify_state
+        check: channel_header_matches
+    risk: low
+
+  open_dm:
+    method: computer-use
+    steps:
+      - action: keyboard_shortcut
+        keys: ["cmd", "k"]
+      - action: type_text
+        text: "{person_name}"
+      - action: press_key
+        key: enter
+      - action: verify_state
+        check: dm_header_matches
+    risk: low
+
+  search:
+    method: computer-use
+    steps:
+      - action: keyboard_shortcut
+        keys: ["cmd", "f"]
+      - action: type_text
+        text: "{search_query}"
+      - action: press_key
+        key: enter
+    risk: low
+
+  scroll_history:
+    method: computer-use
+    steps:
+      - action: scroll
+        direction: up
+        amount: "{pages}"
+    risk: low
+
+# Read operations (always permitted)
+read_operations:
+  read_channel_messages:
+    risk: low
+    approval: none
+    method: computer-use
+    steps:
+      - navigate: open_channel
+      - action: screenshot
+      - action: extract_text
+    output: message_list
+
+  read_thread:
+    risk: low
+    approval: none
+    method: computer-use
+    steps:
+      - action: click_thread
+      - action: screenshot
+      - action: extract_text
+    output: thread_content
+
+  read_dm:
+    risk: low
+    approval: none
+    method: computer-use
+    steps:
+      - navigate: open_dm
+      - action: screenshot
+      - action: extract_text
+    output: dm_content
+
+  search_messages:
+    risk: low
+    approval: none
+    method: computer-use
+    steps:
+      - navigate: search
+      - action: screenshot
+      - action: extract_text
+    output: search_results
+
+# Write operations (require varying approval)
+write_operations:
+  draft_message:
+    risk: low
+    approval: none # drafting is always safe
+    method: computer-use
+    steps:
+      - navigate: "{channel_or_dm}"
+      - action: click_message_input
+      - action: type_text
+        text: "{message_content}"
+      # DO NOT press enter — message stays in draft
+    output: draft_ready
+
+  send_message_internal:
+    risk: medium
+    approval: pre-approved # internal team messages
+    method: computer-use
+    steps:
+      - action: verify_recipient_is_internal
+      - action: verify_message_content_safe
+      - navigate: "{channel_or_dm}"
+      - action: click_message_input
+      - action: type_text
+        text: "{message_content}"
+      - action: press_key
+        key: enter
+      - action: verify_state
+        check: message_sent_confirmation
+    requires:
+      - recipient_is_internal_team: true
+      - message_category: [routine-followup, status-request, acknowledgement]
+    logging: full # log message content, recipient, timestamp
+
+  send_message_external:
+    risk: high
+    approval: explicit # requires Alex's approval
+    method: computer-use
+    steps:
+      - action: verify_approval_granted
+      - navigate: "{channel_or_dm}"
+      - action: click_message_input
+      - action: type_text
+        text: "{message_content}"
+      - action: press_key
+        key: enter
+      - action: verify_state
+        check: message_sent_confirmation
+    requires:
+      - approval_id: "{approval_reference}"
+    logging: full
+
+  reply_in_thread:
+    risk: medium
+    approval: varies # depends on thread context
+    method: computer-use
+    steps:
+      - action: click_thread
+      - action: click_reply_input
+      - action: type_text
+        text: "{reply_content}"
+      - action: verify_content_before_send
+      # Send only if approved
+    logging: full
+
+# Monitoring workflows
+monitoring:
+  channel_watch:
+    description: Monitor specific channels for new messages
+    interval: 300 # every 5 minutes
+    method: computer-use
+    channels: [] # configured in config/slack-channels.yaml
+    actions:
+      - screenshot_channel
+      - extract_new_messages
+      - classify_urgency
+      - log_activity
+
+  mention_watch:
+    description: Monitor for @mentions of Robin or Alex
+    interval: 120 # every 2 minutes
+    method: computer-use
+    steps:
+      - action: check_notifications
+      - action: classify_urgency
+      - action: route_to_appropriate_queue
+
+# Rate limits
+rate_limits:
+  max_actions_per_minute: 20
+  min_delay_between_sends: 5000 # 5 seconds
+  max_sends_per_hour: 30
+  max_sends_per_day: 200
+
+# ---------------------------------------------------------------------------
+# Huddle operations (voice)
+# ---------------------------------------------------------------------------
+
+huddle_operations:
+  launch_with_cdp:
+    description: Launch or relaunch Slack with Chrome DevTools Protocol enabled
+    method: shell
+    command: "scripts/huddle/launch-slack.sh"
+    risk: low
+    approval: none
+
+  join_huddle:
+    description: Join an active huddle in a channel or DM
+    method: keyboard-shortcut
+    steps:
+      - navigate: open_channel  # or open_dm
+      - action: keyboard_shortcut
+        keys: ["cmd", "shift", "h"]
+      - action: verify_state
+        check: huddle_active_indicator
+    risk: medium
+    approval: pre-approved
+    requires:
+      - audio_bridge_ready: true
+    logging: full
+
+  initiate_huddle:
+    description: Start a new huddle in a channel or DM
+    method: keyboard-shortcut
+    steps:
+      - navigate: "{channel_or_dm}"
+      - action: keyboard_shortcut
+        keys: ["cmd", "shift", "h"]
+      - action: verify_state
+        check: huddle_active_indicator
+      - action: wait_for_participant
+        timeout: 120  # 2 minutes
+    risk: medium
+    approval: pre-approved
+    logging: full
+
+  leave_huddle:
+    description: Leave the current huddle
+    method: keyboard-shortcut
+    steps:
+      - action: keyboard_shortcut
+        keys: ["cmd", "shift", "h"]
+      - action: verify_state
+        check: huddle_not_active
+    fallback:
+      - action: click_button
+        target: '[aria-label*="Leave huddle"]'
+    risk: low
+    approval: none
+    logging: full
+
+  accept_huddle_invitation:
+    description: Accept an incoming huddle invitation
+    method: dom-interaction
+    steps:
+      - action: detect_element
+        selector: '[data-qa="huddle-invitation"] button, button[aria-label*="Join huddle"]'
+      - action: click_element
+      - action: verify_state
+        check: huddle_active_indicator
+    risk: medium
+    approval: pre-approved
+    logging: full
+
+  detect_huddle_invitation:
+    description: Poll for incoming huddle invitation popups
+    method: cdp-dom-query
+    interval: 2000  # 2 seconds
+    selectors:
+      - '[class*="huddle"][class*="invitation"]'
+      - '[data-qa="huddle-invitation"]'
+      - '[class*="huddle"][class*="incoming"]'
+    risk: low
+    approval: none
+
+# Audio routing for huddles
+huddle_audio:
+  capture_device: "BlackHole 2ch"     # Slack speaker output → Robin listens
+  playback_device: "BlackHole 16ch"   # Robin speaks → Slack mic input
+  slack_speaker: "BlackHole 2ch"      # Set in Slack Preferences > Audio & Video
+  slack_microphone: "BlackHole 16ch"  # Set in Slack Preferences > Audio & Video
+  sample_rate: 16000
+  channels: 1
+  bit_depth: 16
+
+# Huddle rate limits
+huddle_rate_limits:
+  max_concurrent_huddles: 1
+  max_huddles_per_hour: 10
+  min_gap_between_huddles: 60  # seconds
+  max_huddle_duration: 3600    # 1 hour auto-leave
+
+# Forbidden actions
+forbidden:
+  - leave_workspace
+  - leave_channel
+  - delete_message
+  - edit_sent_message # unless correcting an error within 60 seconds
+  - change_workspace_settings
+  - invite_external_users
+  - create_public_channels
+  - modify_channel_settings
+  - install_apps_or_integrations

package/desktop-control/app-profiles/whatsapp.yaml

@@ -0,0 +1,107 @@
+# WhatsApp Desktop App Control Profile
+# Controls WhatsApp Desktop on the Mac mini
+
+app_name: WhatsApp
+bundle_id: net.whatsapp.WhatsApp
+control_methods: [computer-use]
+risk_level: high # Personal messaging app — extra caution
+
+# Launch and focus
+launch:
+  command: "open -a WhatsApp"
+  verify: screenshot
+
+# Read operations
+read_operations:
+  scan_chats:
+    risk: low
+    approval: none
+    steps:
+      - action: screenshot
+      - action: extract_chat_list
+    output: chat_summary
+
+  read_chat:
+    risk: low
+    approval: none
+    steps:
+      - action: click_chat
+        target: "{contact_name}"
+      - action: screenshot
+      - action: extract_messages
+    output: chat_content
+
+  search_chats:
+    risk: low
+    approval: none
+    steps:
+      - action: click_search
+      - action: type_text
+        text: "{query}"
+      - action: screenshot
+    output: search_results
+
+# Write operations — ALL require explicit approval
+write_operations:
+  draft_message:
+    risk: medium
+    approval: none # drafting is safe
+    steps:
+      - action: click_chat
+        target: "{contact_name}"
+      - action: click_message_input
+      - action: type_text
+        text: "{message_content}"
+      # DO NOT press enter
+    output: draft_ready
+
+  send_message:
+    risk: high
+    approval: explicit # ALL WhatsApp sends require approval
+    requires:
+      - approval_id: "{approval_reference}"
+      - context: business-critical # must be flagged as business-relevant
+    steps:
+      - action: verify_approval_granted
+      - action: verify_correct_recipient
+      - action: click_message_input
+      - action: type_text
+        text: "{message_content}"
+      - action: press_key
+        key: enter
+      - action: verify_state
+        check: message_sent
+    logging: full
+
+# Context classification
+context_classification:
+  business_critical:
+    - contacts with @example.com in notes
+    - contacts tagged as investor/advisor/partner/board
+    - group chats tagged as business
+  personal:
+    - all other contacts
+  rules:
+    - NEVER send to personal contacts without explicit instruction
+    - Business contacts require standard approval flow
+    - Group chat messages require extra scrutiny
+
+# Rate limits (more restrictive than other channels)
+rate_limits:
+  max_actions_per_minute: 10
+  min_delay_between_sends: 30000 # 30 seconds
+  max_sends_per_hour: 10
+  max_sends_per_day: 30
+
+# Forbidden actions
+forbidden:
+  - delete_messages
+  - delete_chats
+  - leave_groups
+  - create_groups
+  - change_settings
+  - change_profile
+  - add_contacts
+  - block_contacts
+  - access_media_unless_instructed
+  - forward_messages_between_chats