mcp-server-macos-use 0.1.7 → 0.1.10

package/Package.resolved

@@ -15,7 +15,7 @@
       "location" : "https://github.com/mediar-ai/MacosUseSDK.git",
       "state" : {
         "branch" : "main",
-        "revision" : "df6989da30b580c203828fa24e9e25a318c2d113"
+        "revision" : "aef5bfc4c0b5ab13e4981909a314d772d8a02ca2"
       }
     },
     {

package/Sources/main.swift

@@ -201,6 +201,55 @@ struct ToolResponse: Codable {
     var diff: EnrichedTraversalDiff?           // for click/type/press: what changed
     var primaryActionError: String?
     var traversalError: String?
+
+    // Cross-app handoff: populated when a different app became frontmost after the action
+    var appSwitchPid: pid_t?
+    var appSwitchAppName: String?
+    var appSwitchTraversal: EnrichedResponseData?
+
+    // Sheet/dialog detection
+    var sheetDetected: Bool?
+}
+
+// --- Sheet Detection ---
+
+/// Check if the app has an AXSheet child (file dialogs, save sheets, etc.)
+/// and return its bounds for viewport scoping.
+func findSheetBounds(pid: pid_t) -> CGRect? {
+    let appElement = AXUIElementCreateApplication(pid)
+    AXUIElementSetMessagingTimeout(appElement, 5.0)
+
+    // Check all windows for AXSheet children
+    var windowsRef: CFTypeRef?
+    guard AXUIElementCopyAttributeValue(appElement, "AXWindows" as CFString, &windowsRef) == .success,
+          let windows = windowsRef as? [AXUIElement] else { return nil }
+
+    for window in windows {
+        AXUIElementSetMessagingTimeout(window, 5.0)
+        // Look for AXSheet role among children
+        var childCount: CFIndex = 0
+        guard AXUIElementGetAttributeValueCount(window, kAXChildrenAttribute as CFString, &childCount) == .success,
+              childCount > 0 else { continue }
+        let fetchCount = min(CFIndex(50), childCount)
+        var childrenRef: CFArray?
+        guard AXUIElementCopyAttributeValues(window, kAXChildrenAttribute as CFString, 0, fetchCount, &childrenRef) == .success,
+              let cfArray = childrenRef else { continue }
+        let children = cfArray as [AnyObject]
+        for child in children {
+            let childElement = child as! AXUIElement
+            AXUIElementSetMessagingTimeout(childElement, 5.0)
+            var roleRef: CFTypeRef?
+            guard AXUIElementCopyAttributeValue(childElement, kAXRoleAttribute as CFString, &roleRef) == .success,
+                  let role = roleRef as? String else { continue }
+            if role == "AXSheet" {
+                if let frame = getAXElementFrame(childElement) {
+                    fputs("log: findSheetBounds: found AXSheet at \(frame)\n", stderr)
+                    return frame
+                }
+            }
+        }
+    }
+    return nil
 }
 
 // --- Viewport Detection Helpers ---
@@ -225,6 +274,7 @@ func getWindowContainingPoint(appElement: AXUIElement, point: CGPoint) -> (eleme
     if AXUIElementCopyAttributeValue(appElement, "AXWindows" as CFString, &windowsRef) == .success,
        let windows = windowsRef as? [AXUIElement] {
         for window in windows {
+            AXUIElementSetMessagingTimeout(window, 5.0)
             guard let frame = getAXElementFrame(window) else { continue }
             if frame.contains(point) {
                 fputs("log: getWindowContainingPoint: matched window \(frame) for point \(point)\n", stderr)
@@ -236,6 +286,7 @@ func getWindowContainingPoint(appElement: AXUIElement, point: CGPoint) -> (eleme
     var winRef: CFTypeRef?
     guard AXUIElementCopyAttributeValue(appElement, "AXMainWindow" as CFString, &winRef) == .success else { return nil }
     let win = winRef as! AXUIElement
+    AXUIElementSetMessagingTimeout(win, 5.0)
     guard let frame = getAXElementFrame(win) else { return nil }
     fputs("log: getWindowContainingPoint: no window contains \(point), falling back to main window \(frame)\n", stderr)
     return (win, frame)
@@ -244,6 +295,7 @@ func getWindowContainingPoint(appElement: AXUIElement, point: CGPoint) -> (eleme
 /// Get window bounds directly from the accessibility API
 func getWindowBoundsFromAPI(pid: pid_t) -> CGRect? {
     let appElement = AXUIElementCreateApplication(pid)
+    AXUIElementSetMessagingTimeout(appElement, 5.0)
 
     var windowValue: CFTypeRef?
     guard AXUIElementCopyAttributeValue(appElement, "AXMainWindow" as CFString, &windowValue) == .success else {
@@ -271,6 +323,133 @@ func getWindowBoundsFromAPI(pid: pid_t) -> CGRect? {
     return CGRect(origin: position, size: size)
 }
 
+/// Capture a screenshot of the window(s) belonging to a given PID and save as PNG.
+/// If `clickPoint` is provided (screen coordinates), draws a red crosshair at that location.
+/// Returns the file path on success, nil on failure.
+func captureWindowScreenshot(pid: pid_t, outputPath: String, clickPoint: CGPoint? = nil) -> String? {
+    // Get the list of windows for this PID
+    guard let windowList = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else {
+        fputs("warning: captureWindowScreenshot: could not get window list\n", stderr)
+        return nil
+    }
+
+    // Find the main window ID for this PID
+    var targetWindowID: CGWindowID? = nil
+    var windowBoundsDict: CFDictionary? = nil
+    for window in windowList {
+        guard let ownerPID = window[kCGWindowOwnerPID as String] as? pid_t,
+              ownerPID == pid,
+              let layer = window[kCGWindowLayer as String] as? Int,
+              layer == 0, // normal window layer
+              let windowID = window[kCGWindowNumber as String] as? CGWindowID else {
+            continue
+        }
+        targetWindowID = windowID
+        windowBoundsDict = window[kCGWindowBounds as String] as! CFDictionary?
+        fputs("log: captureWindowScreenshot: found window \(windowID), boundsDict=\(windowBoundsDict != nil ? "yes" : "nil")\n", stderr)
+        break
+    }
+
+    guard let windowID = targetWindowID else {
+        fputs("warning: captureWindowScreenshot: no on-screen window found for PID \(pid)\n", stderr)
+        return nil
+    }
+
+    // Capture just this window — use a short timeout approach:
+    // Try the capture in a detached thread and bail if it takes too long
+    // (CGWindowListCreateImage can block indefinitely without screen recording permission)
+    fputs("log: captureWindowScreenshot: starting image capture for window \(windowID)...\n", stderr)
+    var capturedImage: CGImage? = nil
+    let captureGroup = DispatchGroup()
+    captureGroup.enter()
+    DispatchQueue.global(qos: .userInitiated).async {
+        capturedImage = CGWindowListCreateImage(.null, .optionIncludingWindow, windowID, [.boundsIgnoreFraming, .bestResolution])
+        captureGroup.leave()
+    }
+    let captureResult = captureGroup.wait(timeout: .now() + 3.0)
+    if captureResult == .timedOut {
+        fputs("warning: captureWindowScreenshot: CGWindowListCreateImage timed out (3s) — likely missing screen recording permission\n", stderr)
+        return nil
+    }
+    guard let image = capturedImage else {
+        fputs("warning: captureWindowScreenshot: CGWindowListCreateImage failed for window \(windowID)\n", stderr)
+        return nil
+    }
+
+    // Draw click point crosshair if provided
+    var finalImage = image
+    fputs("log: captureWindowScreenshot: clickPoint=\(clickPoint?.debugDescription ?? "nil"), boundsDict=\(windowBoundsDict != nil ? "present" : "nil")\n", stderr)
+    if let clickPoint = clickPoint, let boundsDict = windowBoundsDict {
+        var windowRect = CGRect.zero
+        CGRectMakeWithDictionaryRepresentation(boundsDict, &windowRect)
+
+        // Convert screen coordinates to image coordinates
+        // CGWindowListCreateImage with .boundsIgnoreFraming may include shadow, so we
+        // compute the offset from the window's screen origin to the image pixel space.
+        let imageWidth = CGFloat(image.width)
+        let imageHeight = CGFloat(image.height)
+        // Scale factor: image pixels may differ from window points (Retina)
+        let scaleX = imageWidth / windowRect.width
+        let scaleY = imageHeight / windowRect.height
+        let localX = (clickPoint.x - windowRect.origin.x) * scaleX
+        let localY = (clickPoint.y - windowRect.origin.y) * scaleY
+
+        fputs("log: captureWindowScreenshot: drawing crosshair at screen(\(clickPoint.x),\(clickPoint.y)) → image(\(localX),\(localY)) windowOrigin(\(windowRect.origin.x),\(windowRect.origin.y)) scale(\(scaleX),\(scaleY))\n", stderr)
+
+        // Draw crosshair on the image
+        let colorSpace = CGColorSpaceCreateDeviceRGB()
+        if let ctx = CGContext(data: nil, width: image.width, height: image.height,
+                               bitsPerComponent: 8, bytesPerRow: 0, space: colorSpace,
+                               bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) {
+            // Draw original image
+            ctx.draw(image, in: CGRect(x: 0, y: 0, width: imageWidth, height: imageHeight))
+
+            // Flip Y for CoreGraphics drawing (origin is bottom-left)
+            let drawX = localX
+            let drawY = imageHeight - localY
+
+            // Red crosshair
+            ctx.setStrokeColor(CGColor(red: 1, green: 0, blue: 0, alpha: 1))
+            ctx.setLineWidth(2.0 * max(scaleX, scaleY))
+
+            let armLength: CGFloat = 15 * max(scaleX, scaleY)
+            // Horizontal line
+            ctx.move(to: CGPoint(x: drawX - armLength, y: drawY))
+            ctx.addLine(to: CGPoint(x: drawX + armLength, y: drawY))
+            // Vertical line
+            ctx.move(to: CGPoint(x: drawX, y: drawY - armLength))
+            ctx.addLine(to: CGPoint(x: drawX, y: drawY + armLength))
+            ctx.strokePath()
+
+            // Circle around crosshair
+            ctx.setLineWidth(1.5 * max(scaleX, scaleY))
+            let radius: CGFloat = 10 * max(scaleX, scaleY)
+            ctx.addEllipse(in: CGRect(x: drawX - radius, y: drawY - radius, width: radius * 2, height: radius * 2))
+            ctx.strokePath()
+
+            if let annotatedImage = ctx.makeImage() {
+                finalImage = annotatedImage
+            }
+        }
+    }
+
+    // Convert to PNG data and write to file
+    let bitmapRep = NSBitmapImageRep(cgImage: finalImage)
+    guard let pngData = bitmapRep.representation(using: .png, properties: [:]) else {
+        fputs("warning: captureWindowScreenshot: failed to create PNG data\n", stderr)
+        return nil
+    }
+
+    do {
+        try pngData.write(to: URL(fileURLWithPath: outputPath))
+        fputs("log: captureWindowScreenshot: saved screenshot to \(outputPath) (\(pngData.count) bytes)\n", stderr)
+        return outputPath
+    } catch {
+        fputs("warning: captureWindowScreenshot: failed to write screenshot: \(error)\n", stderr)
+        return nil
+    }
+}
+
 /// Enrich a ResponseData with in_viewport metadata for each element
 func enrichResponseData(_ response: ResponseData, windowBounds: CGRect?) -> EnrichedResponseData {
     let enrichedElements = response.elements.map { element -> EnrichedElementData in
@@ -370,11 +549,21 @@ func buildToolResponse(_ result: ActionResult, hasDiff: Bool) -> ToolResponse {
         }
     }
 
+    // Check for AXSheet (file dialogs, save sheets) — use sheet bounds for viewport
+    var sheetDetected = false
+    if let pid = result.traversalPid ?? result.openResult?.pid {
+        if let sheetBounds = findSheetBounds(pid: pid) {
+            windowBounds = sheetBounds
+            sheetDetected = true
+        }
+    }
+
     var response = ToolResponse()
     response.openResult = result.openResult
     response.traversalPid = result.traversalPid
     response.primaryActionError = result.primaryActionError
     response.traversalError = result.traversalAfterError ?? result.traversalBeforeError
+    response.sheetDetected = sheetDetected ? true : nil
 
     if hasDiff, let rawDiff = result.traversalDiff {
         let coordinateAttrs: Set<String> = ["x", "y", "width", "height"]
@@ -459,7 +648,7 @@ func buildToolResponse(_ result: ActionResult, hasDiff: Bool) -> ToolResponse {
 
 /// Build a concise text summary for the MCP response instead of returning the full JSON.
 /// The full JSON is written to a file; this summary contains just the key info + file path.
-func buildCompactSummary(toolName: String, params: CallTool.Parameters, toolResponse: ToolResponse, filepath: String) -> String {
+func buildCompactSummary(toolName: String, params: CallTool.Parameters, toolResponse: ToolResponse, filepath: String, fileSize: Int, screenshotPath: String? = nil) -> String {
     var lines: [String] = []
 
     // Status line
@@ -473,9 +662,26 @@ func buildCompactSummary(toolName: String, params: CallTool.Parameters, toolResp
     if let appName = toolResponse.traversal?.app_name ?? toolResponse.openResult?.appName {
         lines.append("app: \(appName)")
     }
+    if toolResponse.sheetDetected == true {
+        lines.append("dialog: AXSheet detected (viewport scoped to sheet bounds)")
+    }
 
-    // File path
+    // File path + metadata
     lines.append("file: \(filepath)")
+    let elementCount: Int
+    if let traversal = toolResponse.traversal {
+        elementCount = traversal.elements.count
+    } else {
+        let added = toolResponse.diff?.added.count ?? 0
+        let removed = toolResponse.diff?.removed.count ?? 0
+        let modified = toolResponse.diff?.modified.count ?? 0
+        elementCount = added + removed + modified
+    }
+    lines.append("file_size: \(fileSize) bytes (\(elementCount) elements)")
+    lines.append("hint: grep -n 'AXButton' \(filepath)  # search by role or text")
+    if let screenshotPath = screenshotPath {
+        lines.append("screenshot: \(screenshotPath)")
+    }
 
     // Errors if any
     if let err = toolResponse.primaryActionError {
@@ -499,10 +705,19 @@ func buildCompactSummary(toolName: String, params: CallTool.Parameters, toolResp
         }
 
     case "macos-use_click_and_traverse":
-        let x = params.arguments?["x"]?.doubleValue ?? params.arguments?["x"]?.intValue.map(Double.init) ?? 0
-        let y = params.arguments?["y"]?.doubleValue ?? params.arguments?["y"]?.intValue.map(Double.init) ?? 0
+        let isDoubleClick = params.arguments?["doubleClick"]?.boolValue ?? false
+        let isRightClick = params.arguments?["rightClick"]?.boolValue ?? false
+        let clickType = isDoubleClick ? "Double-clicked" : isRightClick ? "Right-clicked" : "Clicked"
         let diffSummary = buildDiffSummary(toolResponse.diff)
-        summaryLine = "Clicked at (\(Int(x)),\(Int(y))). \(diffSummary)"
+        if let elemSearch = params.arguments?["element"]?.stringValue {
+            let roleFilter = params.arguments?["role"]?.stringValue
+            let roleDesc = roleFilter != nil ? " [\(roleFilter!)]" : ""
+            summaryLine = "\(clickType) element '\(elemSearch)'\(roleDesc). \(diffSummary)"
+        } else {
+            let x = params.arguments?["x"]?.doubleValue ?? params.arguments?["x"]?.intValue.map(Double.init) ?? 0
+            let y = params.arguments?["y"]?.doubleValue ?? params.arguments?["y"]?.intValue.map(Double.init) ?? 0
+            summaryLine = "\(clickType) at (\(Int(x)),\(Int(y))). \(diffSummary)"
+        }
 
     case "macos-use_type_and_traverse":
         let text = params.arguments?["text"]?.stringValue ?? ""
@@ -562,6 +777,30 @@ func buildCompactSummary(toolName: String, params: CallTool.Parameters, toolResp
         }
     }
 
+    // Inline visible interactive elements
+    if let traversal = toolResponse.traversal {
+        // Full traversal (open/refresh): show all visible interactive elements
+        let visLines = buildVisibleElementsSection(elements: traversal.elements, label: "visible_elements")
+        lines.append(contentsOf: visLines)
+    } else if let diff = toolResponse.diff, !diff.added.isEmpty {
+        // Diff (click/type/press/scroll): show newly added visible elements
+        let visLines = buildVisibleElementsSection(elements: diff.added, label: "visible_elements", interactiveCap: 20, textCap: 10)
+        lines.append(contentsOf: visLines)
+    }
+
+    // Cross-app handoff: a different app became frontmost after the action
+    if let switchPid = toolResponse.appSwitchPid {
+        let switchName = toolResponse.appSwitchAppName ?? "Unknown"
+        lines.append("app_switch: \(switchName) (PID: \(switchPid)) is now frontmost")
+        if let switchTraversal = toolResponse.appSwitchTraversal {
+            let total = switchTraversal.elements.count
+            let visible = switchTraversal.elements.filter { $0.in_viewport == true }.count
+            lines.append("app_switch_elements: \(total) total, \(visible) visible")
+            let visLines = buildVisibleElementsSection(elements: switchTraversal.elements, label: "app_switch_visible_elements")
+            lines.append(contentsOf: visLines)
+        }
+    }
+
     return lines.joined(separator: "\n")
 }
 
@@ -580,6 +819,154 @@ func truncate(_ s: String, maxLen: Int) -> String {
     s.count > maxLen ? String(s.prefix(maxLen)) + "..." : s
 }
 
+/// Protocol for element types that can be displayed in visible elements section
+protocol VisibleElement {
+    var role: String { get }
+    var text: String? { get }
+    var in_viewport: Bool? { get }
+    var x: Double? { get }
+    var y: Double? { get }
+    var width: Double? { get }
+    var height: Double? { get }
+}
+extension EnrichedElementData: VisibleElement {}
+extension DiffElementData: VisibleElement {}
+
+/// Interactive role prefixes worth showing inline in the compact summary
+private let interactiveRolePrefixes: [String] = [
+    "AXButton", "AXLink", "AXTextField", "AXTextArea", "AXCheckBox",
+    "AXRadioButton", "AXPopUpButton", "AXComboBox", "AXSlider",
+    "AXMenuItem", "AXMenuButton", "AXTab"
+]
+
+/// Check if a role string matches any interactive prefix
+private func isInteractiveRole(_ role: String) -> Bool {
+    interactiveRolePrefixes.contains { role.hasPrefix($0) }
+}
+
+/// Check if a role string is static text
+private func isStaticTextRole(_ role: String) -> Bool {
+    role.hasPrefix("AXStaticText")
+}
+
+/// Build a visible_elements section from a list of elements
+func buildVisibleElementsSection<T: VisibleElement>(elements: [T], label: String, interactiveCap: Int = 30, textCap: Int = 10) -> [String] {
+    var interactive: [String] = []
+    var staticText: [String] = []
+
+    for el in elements {
+        guard el.in_viewport == true else { continue }
+        guard let text = el.text, !text.isEmpty else { continue }
+
+        let truncatedText = truncate(text, maxLen: 50)
+        let pos: String
+        if let x = el.x, let y = el.y, let w = el.width, let h = el.height {
+            pos = " (\(Int(x)),\(Int(y)) \(Int(w))×\(Int(h)))"
+        } else {
+            pos = ""
+        }
+        let line = "  [\(el.role)] \"\(truncatedText)\"\(pos)"
+
+        if isInteractiveRole(el.role) {
+            if interactive.count < interactiveCap {
+                interactive.append(line)
+            }
+        } else if isStaticTextRole(el.role) {
+            if staticText.count < textCap {
+                staticText.append(line)
+            }
+        }
+    }
+
+    if interactive.isEmpty && staticText.isEmpty { return [] }
+
+    var result = ["\(label):"]
+    result.append(contentsOf: interactive)
+    result.append(contentsOf: staticText)
+    return result
+}
+
+// --- Flat Text Response File Builder ---
+
+/// Format a single element as a grep-friendly text line
+func formatElementLine(_ el: VisibleElement, prefix: String = " ") -> String {
+    var parts: [String] = []
+    parts.append("[\(el.role)]")
+    if let text = el.text, !text.isEmpty {
+        let truncated = text.count > 80 ? String(text.prefix(80)) + "..." : text
+        parts.append("\"\(truncated)\"")
+    }
+    if let x = el.x, let y = el.y {
+        parts.append("x:\(Int(x)) y:\(Int(y))")
+    }
+    if let w = el.width, let h = el.height {
+        parts.append("w:\(Int(w)) h:\(Int(h))")
+    }
+    if el.in_viewport == true {
+        parts.append("visible")
+    }
+    return "\(prefix)\(parts.joined(separator: " "))"
+}
+
+/// Build a flat text representation of a ToolResponse for writing to .txt files
+func buildFlatTextResponse(_ toolResponse: ToolResponse) -> String {
+    var lines: [String] = []
+
+    if let traversal = toolResponse.traversal {
+        // Full traversal
+        lines.append("# \(traversal.app_name) — \(traversal.elements.count) elements (\(traversal.processing_time_seconds)s)")
+        if toolResponse.sheetDetected == true {
+            lines.append("# dialog: AXSheet detected")
+        }
+        lines.append("")
+        for el in traversal.elements {
+            lines.append(formatElementLine(el))
+        }
+    }
+
+    if let diff = toolResponse.diff {
+        lines.append("# diff: +\(diff.added.count) added, -\(diff.removed.count) removed, ~\(diff.modified.count) modified")
+        if toolResponse.sheetDetected == true {
+            lines.append("# dialog: AXSheet detected")
+        }
+        lines.append("")
+        for el in diff.added {
+            lines.append(formatElementLine(el, prefix: "+ "))
+        }
+        for el in diff.removed {
+            lines.append(formatElementLine(el, prefix: "- "))
+        }
+        for mod in diff.modified {
+            var changeParts: [String] = []
+            for change in mod.changes {
+                let old = change.oldValue ?? change.removedText ?? ""
+                let new = change.newValue ?? change.addedText ?? ""
+                changeParts.append("\(change.attributeName): '\(old)' -> '\(new)'")
+            }
+            lines.append("~ [\(mod.after.role)] \"\(mod.after.text ?? "")\" | \(changeParts.joined(separator: ", "))")
+        }
+    }
+
+    // Cross-app handoff
+    if let switchTraversal = toolResponse.appSwitchTraversal {
+        lines.append("")
+        lines.append("# app_switch: \(toolResponse.appSwitchAppName ?? "Unknown") (PID: \(toolResponse.appSwitchPid ?? 0))")
+        for el in switchTraversal.elements {
+            lines.append(formatElementLine(el))
+        }
+    }
+
+    // Errors
+    if let err = toolResponse.primaryActionError {
+        lines.append("# error: \(err)")
+    }
+    if let err = toolResponse.traversalError {
+        lines.append("# traversal_error: \(err)")
+    }
+
+    return lines.joined(separator: "\n")
+}
+
 // --- Direct AX Element Interaction ---
 
 // --- Auto-Scroll via Scroll Wheel Events ---
@@ -691,6 +1078,7 @@ func findElementByText(root: AXUIElement, text: String, viewport: CGRect, maxDep
 /// scroll toward the target and keep probing until an element with text appears.
 func scrollIntoViewIfNeeded(pid: pid_t, point: CGPoint) async -> CGPoint {
     let appElement = AXUIElementCreateApplication(pid)
+    AXUIElementSetMessagingTimeout(appElement, 5.0)
 
     guard let (windowElement, windowBounds) = getWindowContainingPoint(appElement: appElement, point: point) else {
         fputs("log: scrollIntoViewIfNeeded: could not get window bounds, using original point\n", stderr)
@@ -839,12 +1227,16 @@ func setupAndStartServer() async throws -> Server {
         "type": .string("object"),
         "properties": .object([
             "pid": .object(["type": .string("number"), "description": .string("REQUIRED. PID of the target application window.")]),
-            "x": .object(["type": .string("number"), "description": .string("REQUIRED. X coordinate for the click (top-left of element).")]),
-            "y": .object(["type": .string("number"), "description": .string("REQUIRED. Y coordinate for the click (top-left of element).")]),
+            "x": .object(["type": .string("number"), "description": .string("X coordinate for the click (top-left of element). Required unless 'element' is provided.")]),
+            "y": .object(["type": .string("number"), "description": .string("Y coordinate for the click (top-left of element). Required unless 'element' is provided.")]),
             "width": .object(["type": .string("number"), "description": .string("Optional. Element width from traversal. When provided with height, click lands at center (x+width/2, y+height/2).")]),
-            "height": .object(["type": .string("number"), "description": .string("Optional. Element height from traversal. When provided with width, click lands at center (x+width/2, y+height/2).")])
+            "height": .object(["type": .string("number"), "description": .string("Optional. Element height from traversal. When provided with width, click lands at center (x+width/2, y+height/2).")]),
+            "element": .object(["type": .string("string"), "description": .string("Optional. Case-insensitive partial text match to find and click an element (e.g. \"Open\", \"Submit\"). Searches visible elements in the accessibility tree. First match is clicked. Alternative to x/y coordinates.")]),
+            "role": .object(["type": .string("string"), "description": .string("Optional. Filter element search by accessibility role. Common roles: AXButton, AXLink, AXTextField, AXTextArea, AXCheckBox, AXRadioButton, AXPopUpButton, AXComboBox, AXSlider, AXMenuItem, AXMenuButton, AXTab, AXStaticText, AXImage, AXGroup, AXCell, AXRow.")]),
+            "doubleClick": .object(["type": .string("boolean"), "description": .string("Optional. If true, performs a double-click instead of a single click.")]),
+            "rightClick": .object(["type": .string("boolean"), "description": .string("Optional. If true, performs a right-click (context menu) instead of a left click.")])
         ]),
-        "required": .array([.string("pid"), .string("x"), .string("y")])
+        "required": .array([.string("pid")])
     ])
     let clickTool = Tool(
         name: "macos-use_click_and_traverse",
@@ -926,7 +1318,7 @@ func setupAndStartServer() async throws -> Server {
 
     let server = Server(
         name: "SwiftMacOSServerDirect", // Renamed slightly
-        version: "1.3.0", // Incremented version for major change
+        version: "1.6.0", // Screenshot capture, AX timeout protection
         capabilities: .init(
             tools: .init(listChanged: true)
         )
@@ -974,6 +1366,7 @@ func setupAndStartServer() async throws -> Server {
         do {
             // --- Determine Action and Options from MCP Params ---
             let primaryAction: PrimaryAction
+            var lastClickPoint: CGPoint? = nil // Track click location for screenshot annotation
             var options = ActionOptions(traverseAfter: true, showAnimation: false) // MCP tools should return the tree by default, no visual highlighting
 
             // PID is required for click, type, press, refresh
@@ -1016,18 +1409,55 @@ func setupAndStartServer() async throws -> Server {
 
             case clickTool.name:
                 guard let reqPid = pidForOptions else { throw MCPError.invalidParams("Missing required 'pid' for click tool") }
-                let x = try getRequiredDouble(from: params.arguments, key: "x")
-                let y = try getRequiredDouble(from: params.arguments, key: "y")
-                let w = try getOptionalDouble(from: params.arguments, key: "width")
-                let h = try getOptionalDouble(from: params.arguments, key: "height")
-                // If width+height provided, compute exact center; otherwise use raw point (AX will center via lookup)
+
+                let elementSearch = params.arguments?["element"]?.stringValue
+                let roleFilter = params.arguments?["role"]?.stringValue
+
                 let rawPoint: CGPoint
-                if let w = w, let h = h {
-                    rawPoint = CGPoint(x: x + w / 2, y: y + h / 2)
-                    fputs("log: click_and_traverse: centering (\(x),\(y)) + size(\(w)×\(h)) → \(rawPoint)\n", stderr)
+                if let elementSearch = elementSearch {
+                    // --- Search mode: find element by text match ---
+                    fputs("log: click_and_traverse: searching for element '\(elementSearch)' (role: \(roleFilter ?? "any"))\n", stderr)
+                    let traversal: ResponseData = try await Task { @MainActor in
+                        return try traverseAccessibilityTree(pid: reqPid)
+                    }.value
+                    let windowBounds = getWindowBoundsFromTraversal(traversal) ?? getWindowBoundsFromAPI(pid: reqPid)
+                    let enriched = enrichResponseData(traversal, windowBounds: windowBounds)
+
+                    let searchLower = elementSearch.lowercased()
+                    let matches = enriched.elements.filter { elem in
+                        guard let text = elem.text, !text.isEmpty,
+                              let _ = elem.x, let _ = elem.y,
+                              let w = elem.width, let h = elem.height,
+                              w > 0, h > 0 else { return false }
+                        let textMatch = text.lowercased().contains(searchLower)
+                        let roleMatch = roleFilter == nil || elem.role.lowercased().hasPrefix(roleFilter!.lowercased())
+                        return textMatch && roleMatch
+                    }
+
+                    guard let match = matches.first else {
+                        let roleHint = roleFilter != nil ? " with role '\(roleFilter!)'" : ""
+                        throw MCPError.invalidParams("No visible element matching '\(elementSearch)'\(roleHint) found in app (PID \(reqPid)). Use the traversal file to find available elements.")
+                    }
+                    let matchX = match.x!, matchY = match.y!, matchW = match.width!, matchH = match.height!
+                    rawPoint = CGPoint(x: matchX + matchW / 2, y: matchY + matchH / 2)
+                    let matchCount = matches.count
+                    fputs("log: click_and_traverse: found \(matchCount) match(es) for '\(elementSearch)'. Clicking '\(match.text ?? "")' [\(match.role)] at center (\(rawPoint.x),\(rawPoint.y))\n", stderr)
                 } else {
-                    rawPoint = CGPoint(x: x, y: y)
+                    // --- Coordinate mode ---
+                    guard let x = try getOptionalDouble(from: params.arguments, key: "x"),
+                          let y = try getOptionalDouble(from: params.arguments, key: "y") else {
+                        throw MCPError.invalidParams("Either 'element' or both 'x' and 'y' must be provided for click tool")
+                    }
+                    let w = try getOptionalDouble(from: params.arguments, key: "width")
+                    let h = try getOptionalDouble(from: params.arguments, key: "height")
+                    if let w = w, let h = h {
+                        rawPoint = CGPoint(x: x + w / 2, y: y + h / 2)
+                        fputs("log: click_and_traverse: centering (\(x),\(y)) + size(\(w)×\(h)) → \(rawPoint)\n", stderr)
+                    } else {
+                        rawPoint = CGPoint(x: x, y: y)
+                    }
                 }
+
                 // Activate the target app before clicking so the click registers correctly
                 if let runningApp = NSRunningApplication(processIdentifier: reqPid) {
                     runningApp.activate(options: [])
@@ -1036,7 +1466,16 @@ func setupAndStartServer() async throws -> Server {
                 }
                 // Auto-scroll element into view if it's outside the visible window area
                 let adjustedPoint = await scrollIntoViewIfNeeded(pid: reqPid, point: rawPoint)
-                primaryAction = .input(action: .click(point: adjustedPoint))
+                lastClickPoint = adjustedPoint
+                let isDoubleClick = params.arguments?["doubleClick"]?.boolValue ?? false
+                let isRightClick = params.arguments?["rightClick"]?.boolValue ?? false
+                if isDoubleClick {
+                    primaryAction = .input(action: .doubleClick(point: adjustedPoint))
+                } else if isRightClick {
+                    primaryAction = .input(action: .rightClick(point: adjustedPoint))
+                } else {
+                    primaryAction = .input(action: .click(point: adjustedPoint))
+                }
                 options.pidForTraversal = reqPid
                 options.showDiff = true // enables traverseBefore automatically
                 hasDiff = true
@@ -1116,11 +1555,33 @@ func setupAndStartServer() async throws -> Server {
             }
 
             // --- Build simplified response and serialize to JSON ---
-            let toolResponse = buildToolResponse(actionResult, hasDiff: hasDiff)
-            guard let resultJsonString = serializeToJsonString(toolResponse) else {
-                fputs("error: handler(CallTool): failed to serialize ToolResponse to JSON for tool \(params.name).\n", stderr)
-                throw MCPError.internalError("failed to serialize ToolResponse to JSON")
+            var toolResponse = buildToolResponse(actionResult, hasDiff: hasDiff)
+
+            // --- Detect cross-app handoff ---
+            // After diff-based actions, check if a different app became frontmost
+            if hasDiff, let originalPid = options.pidForTraversal {
+                let frontmostPid = NSWorkspace.shared.frontmostApplication?.processIdentifier
+                if let newPid = frontmostPid, newPid != originalPid {
+                    let frontmostName = NSWorkspace.shared.frontmostApplication?.localizedName ?? "Unknown"
+                    fputs("log: handler(CallTool): app switch detected! Original PID \(originalPid) -> new frontmost PID \(newPid) (\(frontmostName))\n", stderr)
+                    toolResponse.appSwitchPid = newPid
+                    toolResponse.appSwitchAppName = frontmostName
+
+                    // Traverse the new frontmost app
+                    do {
+                        let newTraversal: ResponseData = try await Task { @MainActor in
+                            return try traverseAccessibilityTree(pid: newPid)
+                        }.value
+                        let newWindowBounds = getWindowBoundsFromTraversal(newTraversal)
+                            ?? getWindowBoundsFromAPI(pid: newPid)
+                        toolResponse.appSwitchTraversal = enrichResponseData(newTraversal, windowBounds: newWindowBounds)
+                        fputs("log: handler(CallTool): traversed new frontmost app \(frontmostName) (PID \(newPid)): \(newTraversal.elements.count) elements\n", stderr)
+                    } catch {
+                        fputs("warning: handler(CallTool): failed to traverse new frontmost app \(frontmostName) (PID \(newPid)): \(error)\n", stderr)
+                    }
+                }
             }
+            let resultTextString = buildFlatTextResponse(toolResponse)
 
             // --- Determine if it was an error overall ---
             let isError = actionResult.primaryActionError != nil ||
@@ -1131,18 +1592,28 @@ func setupAndStartServer() async throws -> Server {
                  fputs("warning: handler(CallTool): Action resulted in an error state (primary: \(actionResult.primaryActionError ?? "nil"), before: \(actionResult.traversalBeforeError ?? "nil"), after: \(actionResult.traversalAfterError ?? "nil")).\n", stderr)
             }
 
-            // --- Write full JSON to file, return compact summary ---
+            // --- Write flat text to file, return compact summary ---
             let outputDir = "/tmp/macos-use"
             try? FileManager.default.createDirectory(atPath: outputDir, withIntermediateDirectories: true)
 
             let timestamp = Int(Date().timeIntervalSince1970 * 1000) // ms precision to avoid collisions
             let safeName = params.name.replacingOccurrences(of: "macos-use_", with: "")
-            let filename = "\(timestamp)_\(safeName).json"
+            let filename = "\(timestamp)_\(safeName).txt"
             let filepath = "\(outputDir)/\(filename)"
-            try? resultJsonString.write(toFile: filepath, atomically: true, encoding: .utf8)
-            fputs("log: handler(CallTool): wrote full response to \(filepath) (\(resultJsonString.count) bytes)\n", stderr)
+            try? resultTextString.write(toFile: filepath, atomically: true, encoding: .utf8)
+            fputs("log: handler(CallTool): wrote full response to \(filepath) (\(resultTextString.count) bytes)\n", stderr)
+
+            // --- Capture window screenshot ---
+            var screenshotPath: String? = nil
+            let screenshotFilename = "\(timestamp)_\(safeName).png"
+            let screenshotFilepath = "\(outputDir)/\(screenshotFilename)"
+            // Use the effective PID (could be app-switched)
+            let screenshotPid = toolResponse.appSwitchPid ?? toolResponse.traversalPid ?? options.pidForTraversal
+            if let pid = screenshotPid {
+                screenshotPath = captureWindowScreenshot(pid: pid, outputPath: screenshotFilepath, clickPoint: lastClickPoint)
+            }
 
-            let summary = buildCompactSummary(toolName: params.name, params: params, toolResponse: toolResponse, filepath: filepath)
+            let summary = buildCompactSummary(toolName: params.name, params: params, toolResponse: toolResponse, filepath: filepath, fileSize: resultTextString.count, screenshotPath: screenshotPath)
             fputs("log: handler(CallTool): returning compact summary (\(summary.count) chars)\n", stderr)
 
             return .init(content: [.text(summary)], isError: isError)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-server-macos-use",
-  "version": "0.1.7",
+  "version": "0.1.10",
   "description": "MCP server that lets AI agents control any macOS application through accessibility APIs",
   "bin": {
     "mcp-server-macos-use": "bin/mcp-server-macos-use"