desktop-pilot-mcp 1.0.0

package/Sources/DesktopPilot/Platform/macOS/SystemEvents.swift

@@ -0,0 +1,323 @@
+import Foundation
+
+// MARK: - System Events Error
+
+/// Errors from AppleScript/System Events execution.
+enum SystemEventsError: Error, LocalizedError, Sendable {
+    case scriptFailed(String)
+    case appNotFound(String)
+    case elementNotFound(String)
+
+    var errorDescription: String? {
+        switch self {
+        case .scriptFailed(let msg):
+            return "AppleScript error: \(msg)"
+        case .appNotFound(let name):
+            return "App not found: \(name)"
+        case .elementNotFound(let desc):
+            return "Element not found: \(desc)"
+        }
+    }
+}
+
+// MARK: - System Events Helper
+
+/// Helper for executing AppleScript/JXA and System Events commands.
+///
+/// System Events is the universal UI automation bridge -- it works
+/// with ANY app, not just scriptable ones. This helper wraps common
+/// operations: running scripts, clicking elements, typing text,
+/// pressing keys, and inspecting UI trees.
+struct SystemEventsHelper: Sendable {
+
+    // MARK: - Script Execution
+
+    /// Execute AppleScript code and return the result.
+    func runAppleScript(_ code: String) -> Result<String, Error> {
+        let process = Process()
+        let stdout = Pipe()
+        let stderr = Pipe()
+
+        process.executableURL = URL(fileURLWithPath: "/usr/bin/osascript")
+        process.arguments = ["-e", code]
+        process.standardOutput = stdout
+        process.standardError = stderr
+
+        do {
+            try process.run()
+            process.waitUntilExit()
+
+            let outData = stdout.fileHandleForReading.readDataToEndOfFile()
+            let errData = stderr.fileHandleForReading.readDataToEndOfFile()
+            let output = String(data: outData, encoding: .utf8)?
+                .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+            let errorOutput = String(data: errData, encoding: .utf8)?
+                .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+
+            if process.terminationStatus == 0 {
+                return .success(output)
+            } else {
+                return .failure(SystemEventsError.scriptFailed(errorOutput))
+            }
+        } catch {
+            return .failure(error)
+        }
+    }
+
+    /// Execute JXA (JavaScript for Automation) code.
+    func runJXA(_ code: String) -> Result<String, Error> {
+        let process = Process()
+        let stdout = Pipe()
+        let stderr = Pipe()
+
+        process.executableURL = URL(fileURLWithPath: "/usr/bin/osascript")
+        process.arguments = ["-l", "JavaScript", "-e", code]
+        process.standardOutput = stdout
+        process.standardError = stderr
+
+        do {
+            try process.run()
+            process.waitUntilExit()
+
+            let outData = stdout.fileHandleForReading.readDataToEndOfFile()
+            let errData = stderr.fileHandleForReading.readDataToEndOfFile()
+            let output = String(data: outData, encoding: .utf8)?
+                .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+            let errorOutput = String(data: errData, encoding: .utf8)?
+                .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+
+            if process.terminationStatus == 0 {
+                return .success(output)
+            } else {
+                return .failure(SystemEventsError.scriptFailed(errorOutput))
+            }
+        } catch {
+            return .failure(error)
+        }
+    }
+
+    // MARK: - Scriptability Detection
+
+    /// Check if an app has an AppleScript dictionary (is natively scriptable).
+    ///
+    /// Uses `sdef` to probe for a scripting definition file. Apps with an sdef
+    /// support direct `tell application` commands beyond System Events.
+    func isScriptable(appName: String) -> Bool {
+        guard let appPath = findAppPath(appName) else { return false }
+
+        let process = Process()
+        process.executableURL = URL(fileURLWithPath: "/usr/bin/sdef")
+        process.arguments = [appPath]
+        process.standardOutput = Pipe()
+        process.standardError = Pipe()
+
+        do {
+            try process.run()
+            process.waitUntilExit()
+            return process.terminationStatus == 0
+        } catch {
+            return false
+        }
+    }
+
+    /// Find the filesystem path to an app by its display name.
+    ///
+    /// Uses Spotlight (`mdfind`) to locate the .app bundle.
+    func findAppPath(_ appName: String) -> String? {
+        let process = Process()
+        let pipe = Pipe()
+
+        process.executableURL = URL(fileURLWithPath: "/usr/bin/mdfind")
+        process.arguments = [
+            "kMDItemDisplayName == '\(appName)' && kMDItemContentType == 'com.apple.application-bundle'"
+        ]
+        process.standardOutput = pipe
+        process.standardError = Pipe()
+
+        do {
+            try process.run()
+            process.waitUntilExit()
+            let data = pipe.fileHandleForReading.readDataToEndOfFile()
+            let output = String(data: data, encoding: .utf8)?
+                .trimmingCharacters(in: .whitespacesAndNewlines) ?? ""
+            return output.components(separatedBy: "\n").first(where: { !$0.isEmpty })
+        } catch {
+            return nil
+        }
+    }
+
+    // MARK: - System Events UI Actions
+
+    /// Click a UI element via System Events using its description/path.
+    ///
+    /// - Parameters:
+    ///   - appName: The process name of the target application.
+    ///   - elementDescription: AppleScript element reference, e.g.
+    ///     `button "OK" of window 1`.
+    func clickElement(
+        appName: String,
+        elementDescription: String
+    ) -> Result<String, Error> {
+        let script = """
+            tell application "System Events"
+                tell process "\(escapeForAppleScript(appName))"
+                    click \(elementDescription)
+                end tell
+            end tell
+            """
+        return runAppleScript(script)
+    }
+
+    /// Get the UI element tree for an app via System Events.
+    ///
+    /// Returns a text listing of every window and its contents
+    /// (class, name, description) suitable for LLM consumption.
+    func getUIElements(appName: String) -> Result<String, Error> {
+        let escaped = escapeForAppleScript(appName)
+        let script = """
+            tell application "System Events"
+                tell process "\(escaped)"
+                    set windowList to every window
+                    set resultText to ""
+                    repeat with w in windowList
+                        set resultText to resultText & "Window: " & (name of w) & linefeed
+                        try
+                            set uiElements to entire contents of w
+                            repeat with elem in uiElements
+                                set elemClass to class of elem as text
+                                set elemName to ""
+                                try
+                                    set elemName to name of elem
+                                end try
+                                set elemDesc to ""
+                                try
+                                    set elemDesc to description of elem
+                                end try
+                                set resultText to resultText & "  " & elemClass & ": " & elemName & " (" & elemDesc & ")" & linefeed
+                            end repeat
+                        end try
+                    end repeat
+                    return resultText
+                end tell
+            end tell
+            """
+        return runAppleScript(script)
+    }
+
+    /// Type text into the focused field of an app via System Events.
+    ///
+    /// Uses `keystroke` which simulates actual key presses, making it
+    /// more reliable than `AXSetValue` for some apps (e.g. Electron apps).
+    func typeText(
+        appName: String,
+        text: String
+    ) -> Result<String, Error> {
+        let escaped = escapeForAppleScript(text)
+        let script = """
+            tell application "System Events"
+                tell process "\(escapeForAppleScript(appName))"
+                    keystroke "\(escaped)"
+                end tell
+            end tell
+            """
+        return runAppleScript(script)
+    }
+
+    /// Press a keyboard shortcut via System Events.
+    ///
+    /// - Parameters:
+    ///   - appName: The process name of the target application.
+    ///   - key: The key code (numeric) or key character to press.
+    ///   - modifiers: Modifier names, e.g. `["command down", "shift down"]`.
+    func keyPress(
+        appName: String,
+        key: String,
+        modifiers: [String]
+    ) -> Result<String, Error> {
+        let modifierStr = modifiers.isEmpty
+            ? ""
+            : " using {\(modifiers.joined(separator: ", "))}"
+        let script = """
+            tell application "System Events"
+                tell process "\(escapeForAppleScript(appName))"
+                    key code \(key)\(modifierStr)
+                end tell
+            end tell
+            """
+        return runAppleScript(script)
+    }
+
+    /// Press a keystroke with modifiers (character-based, not key code).
+    ///
+    /// Use this for shortcuts like Cmd+C, Cmd+V where you know the character.
+    func keystrokeWithModifiers(
+        appName: String,
+        character: String,
+        modifiers: [String]
+    ) -> Result<String, Error> {
+        let modifierStr = modifiers.isEmpty
+            ? ""
+            : " using {\(modifiers.joined(separator: ", "))}"
+        let script = """
+            tell application "System Events"
+                tell process "\(escapeForAppleScript(appName))"
+                    keystroke "\(escapeForAppleScript(character))"\(modifierStr)
+                end tell
+            end tell
+            """
+        return runAppleScript(script)
+    }
+
+    // MARK: - App Queries
+
+    /// Check if an app process is currently running via System Events.
+    func isRunning(appName: String) -> Bool {
+        let script = """
+            tell application "System Events"
+                return (name of every process) contains "\(escapeForAppleScript(appName))"
+            end tell
+            """
+        switch runAppleScript(script) {
+        case .success(let result):
+            return result.lowercased() == "true"
+        case .failure:
+            return false
+        }
+    }
+
+    /// Get the frontmost status of an app via System Events.
+    func isFrontmost(appName: String) -> Bool {
+        let script = """
+            tell application "System Events"
+                tell process "\(escapeForAppleScript(appName))"
+                    return frontmost
+                end tell
+            end tell
+            """
+        switch runAppleScript(script) {
+        case .success(let result):
+            return result.lowercased() == "true"
+        case .failure:
+            return false
+        }
+    }
+
+    /// Bring an app to the front via System Events.
+    func activate(appName: String) -> Result<String, Error> {
+        let script = """
+            tell application "System Events"
+                set frontmost of process "\(escapeForAppleScript(appName))" to true
+            end tell
+            """
+        return runAppleScript(script)
+    }
+
+    // MARK: - Private Helpers
+
+    /// Escape special characters for safe embedding in AppleScript strings.
+    private func escapeForAppleScript(_ input: String) -> String {
+        return input
+            .replacingOccurrences(of: "\\", with: "\\\\")
+            .replacingOccurrences(of: "\"", with: "\\\"")
+    }
+}

package/Sources/DesktopPilot/main.swift

@@ -0,0 +1,19 @@
+import Foundation
+
+let bridge = AXBridge()
+let store = ElementStore()
+
+// Check accessibility permissions on startup
+if !bridge.isAccessibilityEnabled() {
+    Log.error(
+        "Accessibility permission not granted. "
+        + "Go to System Settings > Privacy & Security > Accessibility "
+        + "and add this application."
+    )
+    _ = bridge.promptForAccessibility()
+}
+
+let toolHandler = PilotToolHandler(bridge: bridge, store: store)
+let server = MCPServer(toolHandler: toolHandler)
+
+await server.run()

package/Sources/DesktopPilotCLI/main.swift

@@ -0,0 +1,19 @@
+import DesktopPilot
+
+let bridge = AXBridge()
+let store = ElementStore()
+
+// Check accessibility permissions on startup
+if !bridge.isAccessibilityEnabled() {
+    Log.error(
+        "Accessibility permission not granted. "
+        + "Go to System Settings > Privacy & Security > Accessibility "
+        + "and add this application."
+    )
+    _ = bridge.promptForAccessibility()
+}
+
+let toolHandler = PilotToolHandler(bridge: bridge, store: store)
+let server = MCPServer(toolHandler: toolHandler)
+
+await server.run()

package/Tests/DesktopPilotTests/DesktopPilotTests.swift

@@ -0,0 +1,290 @@
+import XCTest
+@testable import DesktopPilot
+
+final class DesktopPilotTests: XCTestCase {
+
+    // MARK: - Types Tests
+
+    func testPilotElementEncoding() throws {
+        let element = PilotElement(
+            ref: "e1",
+            role: "AXButton",
+            title: "OK",
+            value: nil,
+            description: "Confirm button",
+            enabled: true,
+            focused: false,
+            bounds: ElementBounds(x: 100, y: 200, width: 80, height: 30),
+            children: nil
+        )
+
+        let encoder = JSONEncoder()
+        let data = try encoder.encode(element)
+        let decoded = try JSONDecoder().decode(PilotElement.self, from: data)
+
+        XCTAssertEqual(decoded.ref, "e1")
+        XCTAssertEqual(decoded.role, "AXButton")
+        XCTAssertEqual(decoded.title, "OK")
+        XCTAssertNil(decoded.value)
+        XCTAssertTrue(decoded.enabled)
+        XCTAssertFalse(decoded.focused)
+        XCTAssertEqual(decoded.bounds?.x, 100)
+    }
+
+    func testAppSnapshotEncoding() throws {
+        let snapshot = AppSnapshot(
+            app: "Finder",
+            bundleID: "com.apple.finder",
+            pid: 1234,
+            timestamp: "2024-01-01T00:00:00Z",
+            elementCount: 5,
+            elements: []
+        )
+
+        let data = try JSONEncoder().encode(snapshot)
+        let decoded = try JSONDecoder().decode(AppSnapshot.self, from: data)
+
+        XCTAssertEqual(decoded.app, "Finder")
+        XCTAssertEqual(decoded.bundleID, "com.apple.finder")
+        XCTAssertEqual(decoded.pid, 1234)
+        XCTAssertEqual(decoded.elementCount, 5)
+    }
+
+    func testAppInfoEncoding() throws {
+        let info = AppInfo(
+            name: "Safari",
+            bundleID: "com.apple.Safari",
+            pid: 5678,
+            isScriptable: true,
+            windowCount: 3
+        )
+
+        let data = try JSONEncoder().encode(info)
+        let decoded = try JSONDecoder().decode(AppInfo.self, from: data)
+
+        XCTAssertEqual(decoded.name, "Safari")
+        XCTAssertTrue(decoded.isScriptable)
+        XCTAssertEqual(decoded.windowCount, 3)
+    }
+
+    // MARK: - JSON Value Tests
+
+    func testJSONValueStringExtraction() {
+        let json: JSONValue = .object([
+            "name": .string("test"),
+            "count": .number(42)
+        ])
+
+        XCTAssertEqual(json.stringValue(forKey: "name"), "test")
+        XCTAssertNil(json.stringValue(forKey: "missing"))
+        XCTAssertEqual(json.intValue(forKey: "count"), 42)
+    }
+
+    func testJSONValueCodingRoundTrip() throws {
+        let original: JSONValue = .object([
+            "string": .string("hello"),
+            "number": .number(3.14),
+            "bool": .bool(true),
+            "null": .null,
+            "array": .array([.string("a"), .number(1)]),
+            "nested": .object(["key": .string("value")])
+        ])
+
+        let data = try JSONEncoder().encode(original)
+        let decoded = try JSONDecoder().decode(JSONValue.self, from: data)
+
+        XCTAssertEqual(decoded, original)
+    }
+
+    // MARK: - Element Store Tests
+
+    func testElementStoreBasics() async {
+        let store = ElementStore()
+
+        // Initially empty
+        let count = await store.count()
+        XCTAssertEqual(count, 0)
+
+        // Resolve unknown ref returns nil
+        let result = await store.resolve("e1")
+        XCTAssertNil(result)
+    }
+
+    func testElementStoreReset() async {
+        let store = ElementStore()
+        let count = await store.count()
+        XCTAssertEqual(count, 0)
+
+        await store.reset()
+        let countAfter = await store.count()
+        XCTAssertEqual(countAfter, 0)
+    }
+
+    // MARK: - Router Tests
+
+    func testRouterCategorization() {
+        let router = Router()
+
+        // Known scriptable
+        let finder = router.categorize(bundleID: "com.apple.finder", appName: "Finder")
+        XCTAssertEqual(finder, .scriptable)
+
+        // Known Electron
+        let discord = router.categorize(bundleID: "com.hnc.Discord", appName: "Discord")
+        XCTAssertEqual(discord, .electron)
+
+        // Unknown app
+        let unknown = router.categorize(bundleID: "com.example.unknown", appName: "SomeApp")
+        XCTAssertTrue(unknown == .unknown || unknown == .nativeStandard)
+    }
+
+    func testRouterSnapshotAlwaysAccessibility() {
+        let router = Router()
+
+        let method = router.bestMethodForAction(
+            action: "snapshot",
+            appName: "Finder",
+            bundleID: "com.apple.finder"
+        )
+        XCTAssertEqual(method, .accessibility)
+    }
+
+    func testRouterReadAlwaysAccessibility() {
+        let router = Router()
+
+        let method = router.bestMethodForAction(
+            action: "read",
+            appName: "Safari",
+            bundleID: "com.apple.Safari"
+        )
+        XCTAssertEqual(method, .accessibility)
+    }
+
+    func testRouterTypingPrefersCGEvent() {
+        let router = Router()
+
+        let method = router.bestMethodForAction(
+            action: "type",
+            appName: "Notes",
+            bundleID: "com.apple.Notes"
+        )
+        XCTAssertEqual(method, .cgevent)
+    }
+
+    func testRouterAllMethodsAvailable() {
+        let router = Router()
+
+        XCTAssertTrue(router.isAvailable(.accessibility))
+        XCTAssertTrue(router.isAvailable(.applescript))
+        XCTAssertTrue(router.isAvailable(.cgevent))
+        XCTAssertTrue(router.isAvailable(.screenshot))
+    }
+
+    // MARK: - App Registry Tests
+
+    func testAppRegistryListApps() {
+        let registry = AppRegistry()
+        let apps = registry.listApps()
+
+        // Should always find at least a few running apps
+        XCTAssertGreaterThan(apps.count, 0)
+
+        // Each app should have a name
+        for app in apps {
+            XCTAssertFalse(app.name.isEmpty)
+            XCTAssertGreaterThan(app.pid, 0)
+        }
+    }
+
+    func testAppRegistryFrontmostApp() {
+        let registry = AppRegistry()
+        let frontmost = registry.frontmostApp()
+
+        // Should always have a frontmost app
+        XCTAssertNotNil(frontmost)
+        XCTAssertFalse(frontmost!.name.isEmpty)
+    }
+
+    // MARK: - MCP Protocol Tests
+
+    func testMCPToolResultSuccess() {
+        let result = MCPToolResult.success("hello")
+        XCTAssertFalse(result.isError)
+        XCTAssertEqual(result.content.count, 1)
+        XCTAssertEqual(result.content[0].text, "hello")
+    }
+
+    func testMCPToolResultError() {
+        let result = MCPToolResult.error("something failed")
+        XCTAssertTrue(result.isError)
+        XCTAssertEqual(result.content[0].text, "something failed")
+    }
+
+    func testMCPContentText() {
+        let content = MCPContent.text("test")
+        XCTAssertEqual(content.type, "text")
+        XCTAssertEqual(content.text, "test")
+        XCTAssertNil(content.data)
+    }
+
+    func testMCPContentImage() {
+        let content = MCPContent.image(base64: "abc123", mimeType: "image/png")
+        XCTAssertEqual(content.type, "image")
+        XCTAssertNil(content.text)
+        XCTAssertEqual(content.data, "abc123")
+        XCTAssertEqual(content.mimeType, "image/png")
+    }
+
+    // MARK: - Tool Handler Tests
+
+    func testToolHandlerListsAllTools() {
+        let bridge = AXBridge()
+        let store = ElementStore()
+        let handler = PilotToolHandler(bridge: bridge, store: store)
+
+        let tools = handler.listTools()
+        XCTAssertEqual(tools.count, 10)
+
+        let names = tools.map { $0.name }
+        XCTAssertTrue(names.contains("pilot_snapshot"))
+        XCTAssertTrue(names.contains("pilot_click"))
+        XCTAssertTrue(names.contains("pilot_type"))
+        XCTAssertTrue(names.contains("pilot_read"))
+        XCTAssertTrue(names.contains("pilot_find"))
+        XCTAssertTrue(names.contains("pilot_list_apps"))
+        XCTAssertTrue(names.contains("pilot_menu"))
+        XCTAssertTrue(names.contains("pilot_script"))
+        XCTAssertTrue(names.contains("pilot_screenshot"))
+        XCTAssertTrue(names.contains("pilot_batch"))
+    }
+
+    func testToolHandlerUnknownTool() async throws {
+        let bridge = AXBridge()
+        let store = ElementStore()
+        let handler = PilotToolHandler(bridge: bridge, store: store)
+
+        let result = try await handler.callTool(name: "nonexistent", arguments: nil)
+        XCTAssertTrue(result.isError)
+    }
+
+    func testToolHandlerClickMissingRef() async throws {
+        let bridge = AXBridge()
+        let store = ElementStore()
+        let handler = PilotToolHandler(bridge: bridge, store: store)
+
+        let result = try await handler.callTool(name: "pilot_click", arguments: nil)
+        XCTAssertTrue(result.isError)
+        XCTAssertTrue(result.content[0].text?.contains("Missing") ?? false)
+    }
+
+    func testToolHandlerClickUnknownRef() async throws {
+        let bridge = AXBridge()
+        let store = ElementStore()
+        let handler = PilotToolHandler(bridge: bridge, store: store)
+
+        let args: JSONValue = .object(["ref": .string("e999")])
+        let result = try await handler.callTool(name: "pilot_click", arguments: args)
+        XCTAssertTrue(result.isError)
+        XCTAssertTrue(result.content[0].text?.contains("Unknown ref") ?? false)
+    }
+}