desktop-pilot-mcp 1.0.0

package/Sources/DesktopPilot/MCP/Types.swift

@@ -0,0 +1,107 @@
+import Foundation
+
+// MARK: - UI Element Types
+
+/// A UI element in the accessibility tree.
+struct PilotElement: Codable, Sendable {
+    let ref: String
+    let role: String
+    let title: String?
+    let value: String?
+    let description: String?
+    let enabled: Bool
+    let focused: Bool
+    let bounds: ElementBounds?
+    let children: [PilotElement]?
+}
+
+/// Screen position and size of a UI element.
+struct ElementBounds: Codable, Sendable {
+    let x: Double
+    let y: Double
+    let width: Double
+    let height: Double
+}
+
+// MARK: - App Snapshot
+
+/// Snapshot of an app's full UI tree at a point in time.
+struct AppSnapshot: Codable, Sendable {
+    let app: String
+    let bundleID: String?
+    let pid: Int32
+    let timestamp: String
+    let elementCount: Int
+    let elements: [PilotElement]
+}
+
+// MARK: - Action Result
+
+/// Result of performing an action on a UI element.
+struct ActionResult: Codable, Sendable {
+    let success: Bool
+    let message: String
+    let ref: String?
+}
+
+// MARK: - App Info
+
+/// Summary info about a running application.
+struct AppInfo: Codable, Sendable {
+    let name: String
+    let bundleID: String?
+    let pid: Int32
+    let isScriptable: Bool
+    let windowCount: Int
+}
+
+// MARK: - Tool Input Types
+
+struct SnapshotInput: Codable, Sendable {
+    let app: String?
+    let maxDepth: Int?
+}
+
+struct ClickInput: Codable, Sendable {
+    let ref: String
+}
+
+struct TypeInput: Codable, Sendable {
+    let ref: String
+    let text: String
+}
+
+struct ReadInput: Codable, Sendable {
+    let ref: String
+}
+
+struct FindInput: Codable, Sendable {
+    let role: String?
+    let title: String?
+    let value: String?
+    let app: String?
+}
+
+struct MenuInput: Codable, Sendable {
+    let path: String
+    let app: String?
+}
+
+struct ScriptInput: Codable, Sendable {
+    let app: String
+    let code: String
+    let language: String?
+}
+
+struct ScreenshotInput: Codable, Sendable {
+    let ref: String?
+}
+
+struct BatchAction: Codable, Sendable {
+    let tool: String
+    let params: [String: String]
+}
+
+struct BatchInput: Codable, Sendable {
+    let actions: [BatchAction]
+}

package/Sources/DesktopPilot/Platform/PlatformProtocol.swift

@@ -0,0 +1,49 @@
+import Foundation
+
+// MARK: - Platform Errors
+
+/// Errors that can occur during platform operations.
+enum PlatformError: Error, Sendable {
+    case permissionDenied
+    case elementNotFound(ref: String)
+    case actionFailed(action: String, reason: String)
+    case attributeReadFailed(attribute: String)
+    case appNotFound(name: String)
+    case menuNavigationFailed(path: [String])
+    case unsupported(description: String)
+}
+
+// MARK: - Platform Bridge Protocol
+
+/// Protocol for platform-specific UI automation.
+/// macOS implementation uses AXUIElement; future Windows implementation
+/// would use UI Automation.
+protocol PlatformBridge: Sendable {
+
+    /// Check if accessibility permissions are granted.
+    func checkPermissions() -> Bool
+
+    /// Request accessibility permissions (shows system dialog).
+    func requestPermissions() -> Bool
+
+    /// Get the frontmost application info.
+    func getFrontmostApp() -> (name: String, bundleID: String?, pid: Int32)?
+
+    /// List all running applications with windows.
+    func listApps() -> [(name: String, bundleID: String?, pid: Int32, windowCount: Int)]
+
+    /// Get the accessibility tree for an app by PID.
+    func getAccessibilityTree(pid: Int32, maxDepth: Int) -> [PilotElement]
+
+    /// Perform click on element.
+    func clickElement(ref: String) throws
+
+    /// Type text into element.
+    func typeIntoElement(ref: String, text: String) throws
+
+    /// Read value from element.
+    func readElement(ref: String) throws -> String?
+
+    /// Navigate menu bar.
+    func navigateMenu(appPID: Int32, path: [String]) throws
+}

package/Sources/DesktopPilot/Platform/macOS/AXBridge.swift

@@ -0,0 +1,232 @@
+import ApplicationServices
+import AppKit
+import Foundation
+
+// MARK: - Sendable Wrapper
+
+/// Wraps an AXUIElement so it can cross concurrency boundaries.
+/// AXUIElement is a CFTypeRef (thread-safe by Apple's AX implementation)
+/// but Swift 6 does not know that, so we use @unchecked Sendable.
+public final class AXElementWrapper: @unchecked Sendable {
+    public let element: AXUIElement
+
+    public init(_ element: AXUIElement) {
+        self.element = element
+    }
+}
+
+// MARK: - AXBridge
+
+/// Low-level wrapper around the macOS AXUIElement C API.
+/// Every public method is pure — no shared mutable state — so the
+/// type is safe to use from any isolation context.
+public final class AXBridge: @unchecked Sendable {
+
+    public init() {}
+
+    // MARK: - Element Creation
+
+    /// Create the root accessibility element for a running app.
+    func appElement(pid: pid_t) -> AXUIElement {
+        return AXUIElementCreateApplication(pid)
+    }
+
+    // MARK: - Attribute Reading
+
+    /// Read a single attribute from an element.
+    func getAttribute(_ element: AXUIElement, _ attribute: String) -> CFTypeRef? {
+        var value: CFTypeRef?
+        let error = AXUIElementCopyAttributeValue(element, attribute as CFString, &value)
+        guard error == .success else { return nil }
+        return value
+    }
+
+    /// Read multiple attributes in one call (faster than individual reads).
+    func getAttributes(_ element: AXUIElement, _ attributes: [String]) -> [CFTypeRef?] {
+        let cfAttributes = attributes.map { $0 as CFString } as CFArray
+        var values: CFArray?
+        let error = AXUIElementCopyMultipleAttributeValues(
+            element,
+            cfAttributes,
+            AXCopyMultipleAttributeOptions(rawValue: 0),
+            &values
+        )
+        guard error == .success, let cfArray = values else {
+            return Array(repeating: nil, count: attributes.count)
+        }
+        let count = CFArrayGetCount(cfArray)
+        var result: [CFTypeRef?] = []
+        for i in 0..<count {
+            let raw = CFArrayGetValueAtIndex(cfArray, i)
+            if let raw = raw {
+                let ref = Unmanaged<CFTypeRef>.fromOpaque(raw).takeUnretainedValue()
+                // AXError sentinel values come back as kCFNull
+                if CFGetTypeID(ref) == CFNullGetTypeID() {
+                    result.append(nil)
+                } else {
+                    result.append(ref)
+                }
+            } else {
+                result.append(nil)
+            }
+        }
+        return result
+    }
+
+    // MARK: - Attribute Writing
+
+    /// Set an attribute value on an element.
+    func setAttribute(_ element: AXUIElement, _ attribute: String, _ value: CFTypeRef) -> Bool {
+        let error = AXUIElementSetAttributeValue(element, attribute as CFString, value)
+        return error == .success
+    }
+
+    // MARK: - Actions
+
+    /// Perform a named action on an element (e.g. kAXPressAction).
+    func performAction(_ element: AXUIElement, _ action: String) -> Bool {
+        let error = AXUIElementPerformAction(element, action as CFString)
+        return error == .success
+    }
+
+    // MARK: - Convenience Readers
+
+    /// Get children of an element.
+    func getChildren(_ element: AXUIElement) -> [AXUIElement] {
+        guard let value = getAttribute(element, kAXChildrenAttribute) else { return [] }
+        guard let children = value as? [AXUIElement] else { return [] }
+        return children
+    }
+
+    /// Get the role of an element (e.g. "AXButton").
+    func getRole(_ element: AXUIElement) -> String? {
+        guard let value = getAttribute(element, kAXRoleAttribute) else { return nil }
+        return value as? String
+    }
+
+    /// Get the title of an element.
+    func getTitle(_ element: AXUIElement) -> String? {
+        guard let value = getAttribute(element, kAXTitleAttribute) else { return nil }
+        return value as? String
+    }
+
+    /// Get the value of an element as a string.
+    func getValue(_ element: AXUIElement) -> String? {
+        guard let value = getAttribute(element, kAXValueAttribute) else { return nil }
+        if let str = value as? String { return str }
+        if let num = value as? NSNumber { return num.stringValue }
+        return String(describing: value)
+    }
+
+    /// Get the accessibility description of an element.
+    func getDescription(_ element: AXUIElement) -> String? {
+        guard let value = getAttribute(element, kAXDescriptionAttribute) else { return nil }
+        return value as? String
+    }
+
+    /// Check if element is enabled.
+    func isEnabled(_ element: AXUIElement) -> Bool {
+        guard let value = getAttribute(element, kAXEnabledAttribute) else { return true }
+        return (value as? Bool) ?? true
+    }
+
+    /// Check if element has keyboard focus.
+    func isFocused(_ element: AXUIElement) -> Bool {
+        guard let value = getAttribute(element, kAXFocusedAttribute) else { return false }
+        return (value as? Bool) ?? false
+    }
+
+    /// Get the bounding rectangle of an element in screen coordinates.
+    func getBounds(_ element: AXUIElement) -> ElementBounds? {
+        guard let posValue = getAttribute(element, kAXPositionAttribute) else { return nil }
+        var point = CGPoint.zero
+        // swiftlint:disable force_cast
+        guard AXValueGetValue(posValue as! AXValue, .cgPoint, &point) else { return nil }
+
+        guard let sizeValue = getAttribute(element, kAXSizeAttribute) else { return nil }
+        var size = CGSize.zero
+        guard AXValueGetValue(sizeValue as! AXValue, .cgSize, &size) else { return nil }
+        // swiftlint:enable force_cast
+
+        return ElementBounds(
+            x: Double(point.x),
+            y: Double(point.y),
+            width: Double(size.width),
+            height: Double(size.height)
+        )
+    }
+
+    // MARK: - App-level Queries
+
+    /// Get the currently focused UI element inside an app.
+    func getFocusedElement(_ appElement: AXUIElement) -> AXUIElement? {
+        guard let value = getAttribute(appElement, kAXFocusedUIElementAttribute) else { return nil }
+        // value is guaranteed to be AXUIElement when the attribute exists
+        return (value as! AXUIElement) // swiftlint:disable:this force_cast
+    }
+
+    /// Get all windows belonging to an app.
+    func getWindows(_ appElement: AXUIElement) -> [AXUIElement] {
+        guard let value = getAttribute(appElement, kAXWindowsAttribute) else { return [] }
+        guard let windows = value as? [AXUIElement] else { return [] }
+        return windows
+    }
+
+    /// Get the menu bar element for an app.
+    func getMenuBar(_ appElement: AXUIElement) -> AXUIElement? {
+        guard let value = getAttribute(appElement, kAXMenuBarAttribute) else { return nil }
+        return (value as! AXUIElement) // swiftlint:disable:this force_cast
+    }
+
+    // MARK: - Menu Navigation
+
+    /// Walk a menu path like ["File", "Save As..."] and press the final item.
+    func navigateMenu(_ appElement: AXUIElement, path: [String]) -> Bool {
+        guard let menuBar = getMenuBar(appElement), !path.isEmpty else { return false }
+
+        var current: AXUIElement = menuBar
+
+        for (index, menuName) in path.enumerated() {
+            let children = getChildren(current)
+            var matched = false
+
+            for child in children {
+                let title = getTitle(child)
+                guard title == menuName else { continue }
+
+                let isLast = index == path.count - 1
+                if isLast {
+                    return performAction(child, kAXPressAction)
+                }
+
+                // Intermediate menu — drill into its submenu children
+                let subChildren = getChildren(child)
+                if let submenu = subChildren.first {
+                    current = submenu
+                    matched = true
+                    break
+                }
+            }
+
+            if !matched {
+                return false
+            }
+        }
+
+        return false
+    }
+
+    // MARK: - Permissions
+
+    /// Check if the current process is trusted for accessibility.
+    public func isAccessibilityEnabled() -> Bool {
+        return AXIsProcessTrusted()
+    }
+
+    /// Prompt the user to grant accessibility access if not already trusted.
+    public func promptForAccessibility() -> Bool {
+        let key = "AXTrustedCheckOptionPrompt" as CFString
+        let options = [key: true] as CFDictionary
+        return AXIsProcessTrustedWithOptions(options)
+    }
+}

package/Sources/DesktopPilot/Platform/macOS/Permissions.swift

@@ -0,0 +1,34 @@
+import ApplicationServices
+import Foundation
+
+// MARK: - Permission Manager
+
+/// Handles accessibility permission checking and requesting.
+struct PermissionManager: Sendable {
+
+    /// Check if the current process has accessibility permissions.
+    static func isGranted() -> Bool {
+        return AXIsProcessTrusted()
+    }
+
+    /// Check permissions and optionally prompt the user via the system dialog.
+    /// Returns `true` if already granted; if not, shows the dialog and returns
+    /// the (likely still-false) current state — the user must toggle the setting
+    /// in System Settings and restart the process.
+    static func checkOrPrompt() -> Bool {
+        let key = "AXTrustedCheckOptionPrompt" as CFString
+        let options = [key: true] as CFDictionary
+        return AXIsProcessTrustedWithOptions(options)
+    }
+
+    /// Write a human-readable permission status to stderr.
+    static func logPermissionStatus() {
+        let granted = isGranted()
+        let message = granted
+            ? "[DesktopPilot] Accessibility permissions: GRANTED"
+            : "[DesktopPilot] Accessibility permissions: NOT GRANTED — open System Settings > Privacy & Security > Accessibility"
+        if let data = (message + "\n").data(using: .utf8) {
+            FileHandle.standardError.write(data)
+        }
+    }
+}