desktop-pilot-mcp 1.0.0

package/Sources/DesktopPilot/Core/Router.swift

@@ -0,0 +1,242 @@
+import AppKit
+import Foundation
+
+// MARK: - Interaction Method
+
+/// The layer used to interact with a macOS application.
+enum InteractionMethod: Sendable, Equatable {
+    case accessibility
+    case applescript
+    case cgevent
+    case screenshot
+}
+
+// MARK: - App Category
+
+/// Classification of an app based on its technology stack and scripting support.
+enum AppCategory: Sendable, Equatable {
+    /// Has an AppleScript dictionary (scriptable via `sdef`).
+    case scriptable
+    /// Chromium-based (Electron). Limited Accessibility support, no AppleScript.
+    case electron
+    /// Standard native macOS app with good Accessibility support.
+    case nativeStandard
+    /// Unknown technology stack.
+    case unknown
+}
+
+// MARK: - Router
+
+/// Routes operations to the best available interaction layer for a given
+/// app and action.
+///
+/// Phase 2 implementation selects the optimal method based on:
+/// - The action being performed (snapshot, click, type, menu, script, find)
+/// - The target app's category (scriptable, Electron, native, unknown)
+/// - Known bundle-ID lists for scriptable and Electron apps
+final class Router: Sendable {
+
+    // MARK: - Known Bundle IDs
+
+    /// macOS apps that expose an AppleScript dictionary.
+    private static let scriptableBundleIDs: Set<String> = [
+        "com.apple.finder",
+        "com.apple.Safari",
+        "com.apple.mail",
+        "com.apple.Notes",
+        "com.apple.iWork.Keynote",
+        "com.apple.iWork.Numbers",
+        "com.apple.iWork.Pages",
+        "com.apple.dt.Xcode",
+        "com.apple.iMovie",
+        "com.apple.garageband",
+        "com.apple.MobileSMS",
+        "com.apple.ical",
+        "com.apple.reminders",
+        "com.apple.Music",
+    ]
+
+    /// Electron / Chromium-based apps with limited AX and no AppleScript.
+    private static let electronBundleIDs: Set<String> = [
+        "com.hnc.Discord",
+        "com.microsoft.VSCode",
+        "com.tinyspeck.slackmacgap",
+        "org.whispersystems.signal-desktop",
+        "com.spotify.client",
+    ]
+
+    // MARK: - App Categorization
+
+    /// Classify an app by its bundle ID, with optional `sdef` detection fallback.
+    ///
+    /// The lookup order is:
+    /// 1. Check the static Electron set (these must never use AppleScript).
+    /// 2. Check the static scriptable set.
+    /// 3. Attempt dynamic `sdef` detection for unknown bundle IDs.
+    /// 4. Fall back to `.nativeStandard` for Apple first-party apps,
+    ///    `.unknown` otherwise.
+    ///
+    /// - Parameters:
+    ///   - bundleID: The bundle identifier, if known.
+    ///   - appName: The display name (used as a heuristic when bundle ID is nil).
+    /// - Returns: The inferred `AppCategory`.
+    func categorize(bundleID: String?, appName: String) -> AppCategory {
+        guard let id = bundleID else {
+            return .unknown
+        }
+
+        if Self.electronBundleIDs.contains(id) {
+            return .electron
+        }
+
+        if Self.scriptableBundleIDs.contains(id) {
+            return .scriptable
+        }
+
+        if hasSdefDictionary(bundleID: id) {
+            return .scriptable
+        }
+
+        if id.hasPrefix("com.apple.") {
+            return .nativeStandard
+        }
+
+        return .unknown
+    }
+
+    // MARK: - App-level Routing
+
+    /// Determine the best general interaction method for an app.
+    ///
+    /// - Parameters:
+    ///   - appName: The display name of the target application.
+    ///   - bundleID: The bundle identifier, if known.
+    /// - Returns: The recommended interaction method.
+    func bestMethod(appName: String, bundleID: String?) -> InteractionMethod {
+        let category = categorize(bundleID: bundleID, appName: appName)
+
+        switch category {
+        case .scriptable:
+            return .applescript
+        case .electron:
+            return .accessibility
+        case .nativeStandard:
+            return .accessibility
+        case .unknown:
+            return .accessibility
+        }
+    }
+
+    // MARK: - Action-level Routing
+
+    /// Determine the best method for a specific operation on an app.
+    ///
+    /// Routing rules by action:
+    /// - `snapshot` / `read` / `find` -- Accessibility (only method that reads UI state)
+    /// - `click` -- Accessibility (AXPress is more precise than coordinate clicking)
+    /// - `type` -- CGEvent for most apps (more reliable), Accessibility for Electron
+    /// - `menu` -- Accessibility (menu bar traversal)
+    /// - `script` -- AppleScript for scriptable apps, Accessibility otherwise
+    ///
+    /// - Parameters:
+    ///   - action: The action being performed (e.g. "click", "type", "snapshot").
+    ///   - appName: The display name of the target application.
+    ///   - bundleID: The bundle identifier, if known.
+    /// - Returns: The recommended interaction method.
+    func bestMethodForAction(
+        action: String,
+        appName: String,
+        bundleID: String?
+    ) -> InteractionMethod {
+        let normalized = action.lowercased()
+        let category = categorize(bundleID: bundleID, appName: appName)
+
+        switch normalized {
+        case "snapshot", "read", "find":
+            return .accessibility
+
+        case "click":
+            return .accessibility
+
+        case "type":
+            return routeTyping(category: category)
+
+        case "menu":
+            return .accessibility
+
+        case "script":
+            return routeScripting(category: category)
+
+        default:
+            return bestMethod(appName: appName, bundleID: bundleID)
+        }
+    }
+
+    // MARK: - Capability Check
+
+    /// Check whether a given method is available on this system.
+    ///
+    /// All methods are available in Phase 2.
+    ///
+    /// - Parameter method: The interaction method to check.
+    /// - Returns: `true` if the method can be used.
+    func isAvailable(_ method: InteractionMethod) -> Bool {
+        return true
+    }
+
+    // MARK: - Private Helpers
+
+    /// Pick the best method for typing text into the given app category.
+    ///
+    /// CGEvent is generally more reliable for keystroke injection, but
+    /// Electron apps sometimes swallow raw key events, so Accessibility
+    /// (AXSetValue) is safer there.
+    private func routeTyping(category: AppCategory) -> InteractionMethod {
+        switch category {
+        case .electron:
+            return .accessibility
+        case .scriptable, .nativeStandard, .unknown:
+            return .cgevent
+        }
+    }
+
+    /// Pick the best method for executing a script against the given app category.
+    ///
+    /// Only apps with an AppleScript dictionary benefit from `.applescript`;
+    /// everything else falls back to Accessibility.
+    private func routeScripting(category: AppCategory) -> InteractionMethod {
+        switch category {
+        case .scriptable:
+            return .applescript
+        case .electron, .nativeStandard, .unknown:
+            return .accessibility
+        }
+    }
+
+    /// Detect whether an app has an AppleScript dictionary via `sdef`.
+    ///
+    /// Shells out to `/usr/bin/sdef` with the app's bundle path resolved
+    /// through `NSWorkspace`. Returns `false` on any error or if the app
+    /// cannot be found.
+    private func hasSdefDictionary(bundleID: String) -> Bool {
+        guard let url = NSWorkspace.shared.urlForApplication(
+            withBundleIdentifier: bundleID
+        ) else {
+            return false
+        }
+
+        let process = Process()
+        process.executableURL = URL(fileURLWithPath: "/usr/bin/sdef")
+        process.arguments = [url.path]
+        process.standardOutput = FileHandle.nullDevice
+        process.standardError = FileHandle.nullDevice
+
+        do {
+            try process.run()
+            process.waitUntilExit()
+            return process.terminationStatus == 0
+        } catch {
+            return false
+        }
+    }
+}

package/Sources/DesktopPilot/Core/Snapshot.swift

@@ -0,0 +1,192 @@
+import ApplicationServices
+import Foundation
+
+// MARK: - Snapshot Builder
+
+/// Builds a `PilotElement` tree from a running app's accessibility tree.
+///
+/// Walks the AX hierarchy starting from the app element, reads attributes
+/// via `AXBridge`, and registers every meaningful element in the
+/// `ElementStore` so it can be referenced later by its opaque ref.
+struct SnapshotBuilder: Sendable {
+    let bridge: AXBridge
+
+    /// Attributes fetched in a single batch call per element for performance.
+    private static let batchAttributes: [String] = [
+        kAXRoleAttribute,
+        kAXTitleAttribute,
+        kAXValueAttribute,
+        kAXDescriptionAttribute,
+        kAXEnabledAttribute,
+        kAXFocusedAttribute,
+    ]
+
+    // MARK: - Public API
+
+    /// Build a complete snapshot of an app's UI tree.
+    ///
+    /// - Parameters:
+    ///   - appElement: The root AXUIElement for the app (from `AXBridge.appElement`).
+    ///   - appName: Display name of the application.
+    ///   - bundleID: Bundle identifier (e.g. "com.apple.Safari"), if available.
+    ///   - pid: Process identifier.
+    ///   - store: The `ElementStore` actor that will hold ref-to-element mappings.
+    ///   - maxDepth: Maximum recursion depth to prevent runaway traversal (default 10).
+    /// - Returns: A fully populated `AppSnapshot`.
+    func buildSnapshot(
+        appElement: AXUIElement,
+        appName: String,
+        bundleID: String?,
+        pid: Int32,
+        store: ElementStore,
+        maxDepth: Int = 10
+    ) async -> AppSnapshot {
+        await store.reset()
+
+        let windows = bridge.getWindows(appElement)
+        var topLevelElements: [PilotElement] = []
+
+        for window in windows {
+            let element = await buildElement(
+                from: window,
+                store: store,
+                depth: 0,
+                maxDepth: maxDepth
+            )
+            if let element {
+                topLevelElements.append(element)
+            }
+        }
+
+        // If no windows, try direct children of the app element
+        if topLevelElements.isEmpty {
+            let children = bridge.getChildren(appElement)
+            for child in children {
+                let element = await buildElement(
+                    from: child,
+                    store: store,
+                    depth: 0,
+                    maxDepth: maxDepth
+                )
+                if let element {
+                    topLevelElements.append(element)
+                }
+            }
+        }
+
+        let count = await store.count()
+        let formatter = ISO8601DateFormatter()
+
+        return AppSnapshot(
+            app: appName,
+            bundleID: bundleID,
+            pid: pid,
+            timestamp: formatter.string(from: Date()),
+            elementCount: count,
+            elements: topLevelElements
+        )
+    }
+
+    // MARK: - Recursive Tree Building
+
+    /// Build a single `PilotElement` from an `AXUIElement`, recursing into children.
+    ///
+    /// Returns `nil` if the element has no useful information (no role, title, or value).
+    private func buildElement(
+        from axElement: AXUIElement,
+        store: ElementStore,
+        depth: Int,
+        maxDepth: Int
+    ) async -> PilotElement? {
+        let attrs = readBatchAttributes(axElement)
+
+        let role = attrs.role
+
+        // Skip elements with unknown or missing roles that carry no info
+        if role == nil && attrs.title == nil && attrs.value == nil && attrs.description == nil {
+            return nil
+        }
+
+        // Skip explicitly unknown roles
+        if role == "AXUnknown" {
+            return nil
+        }
+
+        let wrapper = AXElementWrapper(axElement)
+        let ref = await store.register(wrapper)
+        let bounds = bridge.getBounds(axElement)
+
+        // Recurse into children if within depth limit
+        var childElements: [PilotElement]?
+        if depth < maxDepth {
+            let axChildren = bridge.getChildren(axElement)
+            if !axChildren.isEmpty {
+                var built: [PilotElement] = []
+                for child in axChildren {
+                    if let childElement = await buildElement(
+                        from: child,
+                        store: store,
+                        depth: depth + 1,
+                        maxDepth: maxDepth
+                    ) {
+                        built.append(childElement)
+                    }
+                }
+                childElements = built.isEmpty ? nil : built
+            }
+        }
+
+        return PilotElement(
+            ref: ref,
+            role: role ?? "AXUnknown",
+            title: attrs.title,
+            value: attrs.value,
+            description: attrs.description,
+            enabled: attrs.enabled,
+            focused: attrs.focused,
+            bounds: bounds,
+            children: childElements
+        )
+    }
+
+    // MARK: - Batch Attribute Reading
+
+    /// Holds the parsed result of a batch attribute read.
+    private struct BatchResult {
+        let role: String?
+        let title: String?
+        let value: String?
+        let description: String?
+        let enabled: Bool
+        let focused: Bool
+    }
+
+    /// Read all standard attributes in one batch call for performance.
+    private func readBatchAttributes(_ element: AXUIElement) -> BatchResult {
+        let values = bridge.getAttributes(element, Self.batchAttributes)
+
+        let role = values[0] as? String
+        let title = values[1] as? String
+
+        // Value needs special handling: could be String, NSNumber, etc.
+        let value: String? = {
+            guard let raw = values[2] else { return nil }
+            if let str = raw as? String { return str }
+            if let num = raw as? NSNumber { return num.stringValue }
+            return String(describing: raw)
+        }()
+
+        let description = values[3] as? String
+        let enabled = (values[4] as? Bool) ?? true
+        let focused = (values[5] as? Bool) ?? false
+
+        return BatchResult(
+            role: role,
+            title: title,
+            value: value,
+            description: description,
+            enabled: enabled,
+            focused: focused
+        )
+    }
+}

package/Sources/DesktopPilot/Layers/AccessibilityLayer.swift

@@ -0,0 +1,190 @@
+import ApplicationServices
+import AppKit
+import Foundation
+
+// MARK: - Layer-local Element Cache
+
+/// Lock-based element cache used by AccessibilityLayer.
+/// Unlike the actor-based `ElementStore` in Core, this uses NSLock so it
+/// can be called from synchronous `InteractionLayer` protocol methods
+/// without crossing isolation boundaries.
+private final class LayerElementCache: @unchecked Sendable {
+
+    private let lock = NSLock()
+    private var elements: [String: AXElementWrapper] = [:]
+    private var counter: Int = 0
+
+    /// Register an element and return its sequential ref.
+    func register(_ element: AXUIElement) -> String {
+        lock.lock()
+        defer { lock.unlock() }
+        counter += 1
+        let ref = "e\(counter)"
+        elements[ref] = AXElementWrapper(element)
+        return ref
+    }
+
+    /// Look up a previously stored element by ref.
+    func lookup(_ ref: String) -> AXElementWrapper? {
+        lock.lock()
+        defer { lock.unlock() }
+        return elements[ref]
+    }
+
+    /// Remove all stored elements (call before a fresh snapshot).
+    func clear() {
+        lock.lock()
+        defer { lock.unlock() }
+        elements.removeAll()
+        counter = 0
+    }
+
+    /// Current number of stored elements.
+    func count() -> Int {
+        lock.lock()
+        defer { lock.unlock() }
+        return elements.count
+    }
+}
+
+// MARK: - Accessibility Layer
+
+/// Primary interaction layer using the macOS Accessibility API.
+/// Walks AXUIElement trees via `AXBridge`, builds `PilotElement`
+/// snapshots, and performs actions (click, type, read).
+final class AccessibilityLayer: @unchecked Sendable, InteractionLayer {
+
+    let name: String = "Accessibility"
+    let priority: Int = 0
+
+    private let bridge: AXBridge
+    private let cache: LayerElementCache
+
+    init(bridge: AXBridge = AXBridge()) {
+        self.bridge = bridge
+        self.cache = LayerElementCache()
+    }
+
+    // MARK: - InteractionLayer Conformance
+
+    func canHandle(bundleID: String?, appName: String) -> Bool {
+        // The accessibility layer can handle any app that exposes an AX tree.
+        // We optimistically return true; individual operations will fail
+        // gracefully if the app doesn't cooperate.
+        return true
+    }
+
+    func snapshot(pid: Int32, maxDepth: Int) throws -> [PilotElement] {
+        guard bridge.isAccessibilityEnabled() else {
+            throw PlatformError.permissionDenied
+        }
+
+        cache.clear()
+
+        let appEl = bridge.appElement(pid: pid)
+        let windows = bridge.getWindows(appEl)
+        let sources = windows.isEmpty ? bridge.getChildren(appEl) : windows
+
+        if sources.isEmpty {
+            throw LayerError.snapshotFailed(
+                pid: pid,
+                reason: "App has no windows or accessible children"
+            )
+        }
+
+        return sources.compactMap { buildElement($0, depth: 0, maxDepth: maxDepth) }
+    }
+
+    func click(ref: String) throws {
+        let wrapper = try resolveRef(ref)
+        let pressed = bridge.performAction(wrapper.element, kAXPressAction)
+        if !pressed {
+            throw PlatformError.actionFailed(
+                action: "press",
+                reason: "AXPress action failed for ref '\(ref)'"
+            )
+        }
+    }
+
+    func typeText(ref: String, text: String) throws {
+        let wrapper = try resolveRef(ref)
+        let element = wrapper.element
+
+        // Focus the element first
+        _ = bridge.setAttribute(element, kAXFocusedAttribute, kCFBooleanTrue)
+
+        // Try setting the value directly
+        let success = bridge.setAttribute(element, kAXValueAttribute, text as CFTypeRef)
+        if !success {
+            throw LayerError.typingFailed(
+                ref: ref,
+                reason: "Could not set AXValue on element -- it may not be an editable text field"
+            )
+        }
+    }
+
+    func readValue(ref: String) throws -> String? {
+        let wrapper = try resolveRef(ref)
+        return bridge.getValue(wrapper.element)
+    }
+
+    // MARK: - Public Helpers
+
+    /// Look up a stored element wrapper by ref.
+    func findElement(ref: String) -> AXElementWrapper? {
+        return cache.lookup(ref)
+    }
+
+    // MARK: - Private Helpers
+
+    /// Resolve a ref string to an AXElementWrapper, throwing if not found.
+    private func resolveRef(_ ref: String) throws -> AXElementWrapper {
+        guard let wrapper = cache.lookup(ref) else {
+            throw PlatformError.elementNotFound(ref: ref)
+        }
+        return wrapper
+    }
+
+    /// Recursively build a PilotElement tree from an AXUIElement.
+    private func buildElement(
+        _ element: AXUIElement,
+        depth: Int,
+        maxDepth: Int
+    ) -> PilotElement? {
+        guard let role = bridge.getRole(element) else { return nil }
+
+        // Skip explicitly unknown roles
+        if role == "AXUnknown" { return nil }
+
+        let ref = cache.register(element)
+        let title = bridge.getTitle(element)
+        let value = bridge.getValue(element)
+        let description = bridge.getDescription(element)
+        let enabled = bridge.isEnabled(element)
+        let focused = bridge.isFocused(element)
+        let bounds = bridge.getBounds(element)
+
+        var children: [PilotElement]?
+        if depth < maxDepth {
+            let axChildren = bridge.getChildren(element)
+            if !axChildren.isEmpty {
+                let built = axChildren.compactMap { child in
+                    buildElement(child, depth: depth + 1, maxDepth: maxDepth)
+                }
+                children = built.isEmpty ? nil : built
+            }
+        }
+
+        return PilotElement(
+            ref: ref,
+            role: role,
+            title: title,
+            value: value,
+            description: description,
+            enabled: enabled,
+            focused: focused,
+            bounds: bounds,
+            children: children
+        )
+    }
+}