@mseep/clawdcursor 1.5.5

package/scripts/linux/ocr-recognize.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+"""
+Linux OCR via Tesseract (pytesseract) or tesseract CLI.
+Takes an image path, outputs JSON result to stdout.
+Matches the same JSON format as ocr-recognize.ps1 (Windows).
+
+Usage: python3 ocr-recognize.py /path/to/image.png
+
+Requires: tesseract-ocr package
+  Ubuntu/Debian: sudo apt install tesseract-ocr
+  Fedora:        sudo dnf install tesseract
+  Arch:          sudo pacman -S tesseract
+
+Optional: pip install pytesseract (for bounding boxes)
+"""
+
+import json
+import subprocess
+import sys
+import os
+import shutil
+
+def ocr_with_tesseract_cli(image_path):
+    """Use tesseract CLI with TSV output for bounding boxes."""
+    try:
+        result = subprocess.run(
+            ['tesseract', image_path, '-', 'tsv'],
+            capture_output=True, text=True, timeout=30
+        )
+        if result.returncode != 0:
+            return {"error": f"tesseract failed: {result.stderr.strip()}"}
+
+        elements = []
+        lines_text = []
+        current_line = -1
+
+        for line in result.stdout.strip().split('\n')[1:]:  # skip header
+            parts = line.split('\t')
+            if len(parts) < 12:
+                continue
+
+            level, page, block, par, line_num, word_num = parts[:6]
+            left, top, width, height = parts[6:10]
+            conf = parts[10]
+            text = parts[11].strip() if len(parts) > 11 else ''
+
+            if not text or conf == '-1':
+                continue
+
+            line_idx = int(line_num)
+            if line_idx != current_line:
+                current_line = line_idx
+                if text:
+                    lines_text.append(text)
+                else:
+                    lines_text.append('')
+            else:
+                if lines_text:
+                    lines_text[-1] += ' ' + text
+
+            elements.append({
+                "text": text,
+                "x": int(left),
+                "y": int(top),
+                "width": int(width),
+                "height": int(height),
+                "confidence": round(max(0, int(conf)) / 100, 2),
+                "line": line_idx
+            })
+
+        return {
+            "elements": elements,
+            "fullText": '\n'.join(lines_text)
+        }
+    except FileNotFoundError:
+        return {"error": "tesseract not found. Install: sudo apt install tesseract-ocr"}
+    except subprocess.TimeoutExpired:
+        return {"error": "tesseract timed out after 30s"}
+    except Exception as e:
+        return {"error": f"tesseract error: {str(e)}"}
+
+
+def ocr_with_pytesseract(image_path):
+    """Use pytesseract for bounding boxes (if installed)."""
+    try:
+        import pytesseract
+        from PIL import Image
+
+        img = Image.open(image_path)
+        data = pytesseract.image_to_data(img, output_type=pytesseract.Output.DICT)
+
+        elements = []
+        lines_text = []
+        current_line = -1
+
+        for i in range(len(data['text'])):
+            text = data['text'][i].strip()
+            conf = int(data['conf'][i])
+
+            if not text or conf < 0:
+                continue
+
+            line_idx = data['line_num'][i]
+            if line_idx != current_line:
+                current_line = line_idx
+                lines_text.append(text)
+            else:
+                if lines_text:
+                    lines_text[-1] += ' ' + text
+
+            elements.append({
+                "text": text,
+                "x": data['left'][i],
+                "y": data['top'][i],
+                "width": data['width'][i],
+                "height": data['height'][i],
+                "confidence": round(conf / 100, 2),
+                "line": line_idx
+            })
+
+        return {
+            "elements": elements,
+            "fullText": '\n'.join(lines_text)
+        }
+    except ImportError:
+        return None  # Fall back to CLI
+    except Exception as e:
+        return {"error": f"pytesseract error: {str(e)}"}
+
+
+def main():
+    if len(sys.argv) < 2:
+        print(json.dumps({"error": "Usage: ocr-recognize.py <image-path>"}))
+        return
+
+    image_path = sys.argv[1]
+    if not os.path.isfile(image_path):
+        print(json.dumps({"error": f"Image not found: {image_path}"}))
+        return
+
+    # Try pytesseract first (better bounding boxes), fall back to CLI
+    result = ocr_with_pytesseract(image_path)
+    if result is None:
+        # pytesseract not installed, use CLI
+        if shutil.which('tesseract'):
+            result = ocr_with_tesseract_cli(image_path)
+        else:
+            result = {"error": "No OCR available. Install: sudo apt install tesseract-ocr"}
+
+    print(json.dumps(result))
+
+
+if __name__ == '__main__':
+    main()

package/scripts/mac/_window-picker.jxa

@@ -0,0 +1,163 @@
+#!/usr/bin/env osascript -l JavaScript
+
+/**
+ * _window-picker.jxa
+ *
+ * Shared reference implementation for "which window of a process is the
+ * user's real interaction window?". osascript cannot `require()` other
+ * scripts, so the helpers below are also INLINED into the consumer scripts.
+ * Treat this file as the source of truth — if you change scoring here, mirror
+ * the change in:
+ *   - scripts/mac/focus-window.jxa
+ *   - scripts/mac/get-foreground-window.jxa
+ *
+ * Intended consumers:
+ *   - focus-window.jxa            (uses findTitleMatchWindow + pickInteractionWindow fallback)
+ *   - get-foreground-window.jxa   (uses pickInteractionWindow)
+ *
+ * Heuristics, in order of weight:
+ *   1. Visible area (largest wins; rejects 0x0 outright)
+ *   2. Subrole penalty: AXFloatingWindow / AXDialog / AXSystemDialog are
+ *      almost always popups; AXStandardWindow is preferred.
+ *   3. Title penalty: titles like "Downloads" / "Notifications" / "Updates"
+ *      are commonly system-tray-ish popovers in apps like Xcode, Safari.
+ *   4. Z-order tiebreak: lower index ~= newer / more recently activated,
+ *      so older-but-larger windows still beat tiny new popups via (1)+(2)+(3),
+ *      but among equally-scored windows the older one wins (it's the window
+ *      the user was already working in before the popup appeared).
+ *
+ * Bugs this addresses (clawdcursor #101):
+ *   #2 focus_window ignored title= on multi-window apps and grabbed windows[0]
+ *   #4 getActiveWindow grabbed windows[0] of the frontmost process, which on
+ *      Xcode would surface the "Downloads" popup instead of the project window
+ */
+
+function safeGet(obj, property, defaultValue) {
+    try {
+        var val = obj[property]();
+        return (val !== undefined && val !== null) ? val : defaultValue;
+    } catch (e) {
+        return defaultValue;
+    }
+}
+
+// Reads {x, y, width, height} for a window; returns zeros if AXPosition/AXSize
+// are not accessible (some background processes return EACCESS here).
+function getWinBounds(win) {
+    var bounds = { x: 0, y: 0, width: 0, height: 0 };
+    try {
+        var p = win.position();
+        var s = win.size();
+        if (p && s) {
+            bounds.x = p[0] || 0;
+            bounds.y = p[1] || 0;
+            bounds.width = s[0] || 0;
+            bounds.height = s[1] || 0;
+        }
+    } catch (e) {}
+    return bounds;
+}
+
+// Pulls the AXSubrole accessibility attribute, e.g. "AXStandardWindow",
+// "AXFloatingWindow", "AXDialog", "AXSystemDialog". Returns '' if unknown.
+function getWinSubrole(win) {
+    try {
+        return win.subrole() || '';
+    } catch (e) {}
+    try {
+        return win.attributes.byName('AXSubrole').value() || '';
+    } catch (e) {}
+    return '';
+}
+
+function isLikelyTrayTitle(title) {
+    if (!title) return false;
+    var t = String(title).trim().toLowerCase();
+    return t === 'downloads' || t === 'notifications' || t === 'updates' ||
+           t === 'inspector' || t === 'activity';
+}
+
+// Scores a window for "is this the user's real interaction window?".
+// Higher score wins. Returns -Infinity for 0x0 / invisible windows.
+function scoreWindow(win, index) {
+    var bounds = getWinBounds(win);
+    var area = bounds.width * bounds.height;
+    if (area <= 0) return -Infinity;
+
+    var score = area;
+
+    var subrole = getWinSubrole(win);
+    if (subrole === 'AXStandardWindow') {
+        score += 2000000;            // strongly prefer standard windows
+    } else if (subrole === 'AXFloatingWindow') {
+        score -= 5000000;            // popups, palettes
+    } else if (subrole === 'AXDialog' || subrole === 'AXSystemDialog') {
+        score -= 3000000;            // modal dialogs
+    }
+
+    var title = '';
+    try { title = win.name() || ''; } catch (e) {}
+    if (isLikelyTrayTitle(title)) score -= 1500000;
+
+    // Tiebreak: older windows (higher index) get a tiny nudge so popups
+    // (which arrive at index 0) lose ties to the underlying work window.
+    score += index * 10;
+
+    return score;
+}
+
+// Picks the most likely user-interaction window from a window list.
+// Returns { window, index } or null if no window has visible area.
+function pickInteractionWindow(windows) {
+    var best = null;
+    var bestScore = -Infinity;
+    var bestIndex = -1;
+    for (var i = 0; i < windows.length; i++) {
+        try {
+            var w = windows[i];
+            var s = scoreWindow(w, i);
+            if (s > bestScore) {
+                bestScore = s;
+                best = w;
+                bestIndex = i;
+            }
+        } catch (e) {}
+    }
+    if (!best) return null;
+    return { window: best, index: bestIndex };
+}
+
+// Finds first window whose name contains `needle` (case-insensitive)
+// AND has non-zero visible area. Used by focus-window.jxa.
+function findTitleMatchWindow(windows, needle) {
+    if (!needle) return null;
+    var n = String(needle).toLowerCase();
+    for (var i = 0; i < windows.length; i++) {
+        try {
+            var w = windows[i];
+            var title = '';
+            try { title = w.name() || ''; } catch (e) {}
+            if (!title) continue;
+            if (title.toLowerCase().indexOf(n) === -1) continue;
+            var b = getWinBounds(w);
+            if (b.width * b.height <= 0) continue;
+            return { window: w, index: i, title: title };
+        } catch (e) {}
+    }
+    return null;
+}
+
+// This file is reference-only when invoked directly — print the helpers
+// list so a curious developer running `osascript _window-picker.jxa` gets
+// something useful back instead of nothing.
+function run(argv) {
+    return JSON.stringify({
+        kind: 'reference',
+        helpers: [
+            'safeGet', 'getWinBounds', 'getWinSubrole',
+            'isLikelyTrayTitle', 'scoreWindow',
+            'pickInteractionWindow', 'findTitleMatchWindow'
+        ],
+        note: 'osascript cannot require() — these helpers are inlined in focus-window.jxa and get-foreground-window.jxa. Keep all three in sync.'
+    });
+}

package/scripts/mac/find-element.sh

@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+# find-element.sh — Find UI elements on macOS using JXA + System Events
+#
+# Wrapper around find-element.jxa for UIDriver cross-platform support.
+# Outputs JSON matching the Windows find-element.ps1 format:
+#   [{ name, automationId, controlType, className, processId, isEnabled, bounds }]
+#
+# Parameters:
+#   -Name <string>           Match elements by name (contains, case-insensitive)
+#   -AutomationId <string>   Match by description/identifier (mapped to name search)
+#   -ControlType <string>    Match by role (e.g. Button, Edit, MenuItem)
+#   -ProcessId <number>      Limit to a specific process
+#   -MaxResults <number>     Max results (default 20)
+#
+# Usage:
+#   chmod +x scripts/mac/find-element.sh
+#   ./scripts/mac/find-element.sh -Name "OK" -ControlType "Button" -ProcessId 1234
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Parse arguments
+NAME=""
+AUTOMATION_ID=""
+CONTROL_TYPE=""
+PROCESS_ID=""
+MAX_RESULTS="20"
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -Name)         NAME="$2"; shift 2 ;;
+    -AutomationId) AUTOMATION_ID="$2"; shift 2 ;;
+    -ControlType)  CONTROL_TYPE="$2"; shift 2 ;;
+    -ProcessId)    PROCESS_ID="$2"; shift 2 ;;
+    -MaxResults)   MAX_RESULTS="$2"; shift 2 ;;
+    *)             shift ;;
+  esac
+done
+
+# Map Windows ControlType names to macOS accessibility roles
+map_control_type() {
+  local ct="$1"
+  case "$ct" in
+    Button)       echo "button" ;;
+    Edit)         echo "text field" ;;
+    Text)         echo "static text" ;;
+    CheckBox)     echo "checkbox" ;;
+    RadioButton)  echo "radio button" ;;
+    ComboBox)     echo "pop up button" ;;
+    MenuItem)     echo "menu item" ;;
+    Menu)         echo "menu" ;;
+    MenuBar)      echo "menu bar" ;;
+    List)         echo "list" ;;
+    ListItem)     echo "row" ;;
+    Tree)         echo "outline" ;;
+    TreeItem)     echo "row" ;;
+    Tab)          echo "tab group" ;;
+    TabItem)      echo "radio button" ;;
+    Slider)       echo "slider" ;;
+    ProgressBar)  echo "progress indicator" ;;
+    ToolBar)      echo "toolbar" ;;
+    Window)       echo "window" ;;
+    Group)        echo "group" ;;
+    Image)        echo "image" ;;
+    Hyperlink)    echo "link" ;;
+    ScrollBar)    echo "scroll bar" ;;
+    Table)        echo "table" ;;
+    Document)     echo "text area" ;;
+    *)            echo "$ct" ;;
+  esac
+}
+
+# Build JXA arguments
+JXA_ARGS=()
+
+# Use Name or AutomationId as the search name
+SEARCH_NAME="${NAME:-$AUTOMATION_ID}"
+if [[ -n "$SEARCH_NAME" ]]; then
+  JXA_ARGS+=("-name" "$SEARCH_NAME")
+fi
+
+# Map control type to macOS role
+if [[ -n "$CONTROL_TYPE" ]]; then
+  MAPPED_ROLE=$(map_control_type "$CONTROL_TYPE")
+  JXA_ARGS+=("-role" "$MAPPED_ROLE")
+fi
+
+if [[ -n "$PROCESS_ID" && "$PROCESS_ID" != "0" ]]; then
+  JXA_ARGS+=("-processId" "$PROCESS_ID")
+fi
+
+# Run the JXA script
+RAW_OUTPUT=$(osascript -l JavaScript "$SCRIPT_DIR/find-element.jxa" "${JXA_ARGS[@]}" 2>/dev/null || echo "[]")
+
+# Transform the JXA output to match Windows format:
+# JXA returns: { name, role, value, bounds, processId }
+# Windows expects: { name, automationId, controlType, className, processId, isEnabled, bounds }
+echo "$RAW_OUTPUT" | /usr/bin/python3 -c "
+import sys, json
+
+try:
+    data = json.load(sys.stdin)
+except:
+    print('[]')
+    sys.exit(0)
+
+if isinstance(data, dict) and 'error' in data:
+    print(json.dumps(data))
+    sys.exit(0)
+
+if not isinstance(data, list):
+    data = [data]
+
+# Map macOS roles back to Windows ControlType names
+ROLE_MAP = {
+    'AXButton': 'Button', 'button': 'Button',
+    'AXTextField': 'Edit', 'text field': 'Edit',
+    'AXStaticText': 'Text', 'static text': 'Text',
+    'AXCheckBox': 'CheckBox', 'checkbox': 'CheckBox',
+    'AXRadioButton': 'RadioButton', 'radio button': 'RadioButton',
+    'AXPopUpButton': 'ComboBox', 'pop up button': 'ComboBox',
+    'AXMenuItem': 'MenuItem', 'menu item': 'MenuItem',
+    'AXMenu': 'Menu', 'menu': 'Menu',
+    'AXMenuBar': 'MenuBar', 'menu bar': 'MenuBar',
+    'AXList': 'List', 'list': 'List',
+    'AXRow': 'ListItem', 'row': 'ListItem',
+    'AXOutline': 'Tree', 'outline': 'Tree',
+    'AXTabGroup': 'Tab', 'tab group': 'Tab',
+    'AXSlider': 'Slider', 'slider': 'Slider',
+    'AXProgressIndicator': 'ProgressBar', 'progress indicator': 'ProgressBar',
+    'AXToolbar': 'ToolBar', 'toolbar': 'ToolBar',
+    'AXWindow': 'Window', 'window': 'Window',
+    'AXGroup': 'Group', 'group': 'Group',
+    'AXImage': 'Image', 'image': 'Image',
+    'AXLink': 'Hyperlink', 'link': 'Hyperlink',
+    'AXScrollBar': 'ScrollBar', 'scroll bar': 'ScrollBar',
+    'AXTable': 'Table', 'table': 'Table',
+    'AXTextArea': 'Document', 'text area': 'Document',
+    'AXScrollArea': 'Pane', 'scroll area': 'Pane',
+}
+
+max_results = int('${MAX_RESULTS}')
+results = []
+
+for item in data[:max_results]:
+    role = item.get('role', '')
+    control_type = ROLE_MAP.get(role, role)
+    
+    results.append({
+        'name': item.get('name', ''),
+        'automationId': '',  # macOS doesn't have AutomationId; use identifier if available
+        'controlType': control_type,
+        'className': role,  # Store original macOS role as className
+        'processId': item.get('processId', 0),
+        'isEnabled': True,  # macOS doesn't easily expose enabled state in bulk
+        'bounds': item.get('bounds', {'x': 0, 'y': 0, 'width': 0, 'height': 0})
+    })
+
+print(json.dumps(results, separators=(',', ':')))
+"