@cloudcli-ai/cloudcli 1.33.3 → 1.35.0

package/server/modules/computer-use/semantics/helpers/macos/CloudCLISemantics.swift

@@ -0,0 +1,467 @@
+import AppKit
+import ApplicationServices
+import Foundation
+
+typealias JSON = [String: Any]
+
+struct ElementRecord {
+    let index: String
+    let role: String
+    let title: String?
+    let value: String?
+    let bounds: [String: Double]?
+    let actions: [String]
+}
+
+var stateElements: [String: [ElementRecord]] = [:]
+var stateAxElements: [String: [String: AXUIElement]] = [:]
+var stateOrder: [String] = []
+let maxStoredStates = 100
+
+func jsonLine(_ object: Any) {
+    guard JSONSerialization.isValidJSONObject(object),
+          let data = try? JSONSerialization.data(withJSONObject: object),
+          let text = String(data: data, encoding: .utf8)
+    else {
+        print("{\"error\":\"Failed to encode JSON\"}")
+        fflush(stdout)
+        return
+    }
+    print(text)
+    fflush(stdout)
+}
+
+func respond(id: Any?, result: Any) {
+    jsonLine(["id": id ?? NSNull(), "result": result])
+}
+
+func respondError(id: Any?, _ message: String) {
+    jsonLine(["id": id ?? NSNull(), "error": message])
+}
+
+func stringAttr(_ element: AXUIElement, _ attr: CFString) -> String? {
+    var value: CFTypeRef?
+    guard AXUIElementCopyAttributeValue(element, attr, &value) == .success else { return nil }
+    return value as? String
+}
+
+func boolAttr(_ element: AXUIElement, _ attr: CFString) -> Bool? {
+    var value: CFTypeRef?
+    guard AXUIElementCopyAttributeValue(element, attr, &value) == .success else { return nil }
+    return value as? Bool
+}
+
+func arrayAttr(_ element: AXUIElement, _ attr: CFString) -> [AXUIElement] {
+    var value: CFTypeRef?
+    guard AXUIElementCopyAttributeValue(element, attr, &value) == .success else { return [] }
+    return value as? [AXUIElement] ?? []
+}
+
+func actions(_ element: AXUIElement) -> [String] {
+    var names: CFArray?
+    guard AXUIElementCopyActionNames(element, &names) == .success else { return [] }
+    return names as? [String] ?? []
+}
+
+func bounds(_ element: AXUIElement) -> [String: Double]? {
+    var positionRef: CFTypeRef?
+    var sizeRef: CFTypeRef?
+    guard AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionRef) == .success,
+          AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeRef) == .success,
+          let positionValue = positionRef,
+          let sizeValue = sizeRef
+    else { return nil }
+
+    var point = CGPoint.zero
+    var size = CGSize.zero
+    guard CFGetTypeID(positionValue) == AXValueGetTypeID(),
+          CFGetTypeID(sizeValue) == AXValueGetTypeID()
+    else { return nil }
+
+    let positionAxValue = positionValue as! AXValue
+    let sizeAxValue = sizeValue as! AXValue
+    guard AXValueGetValue(positionAxValue, .cgPoint, &point),
+          AXValueGetValue(sizeAxValue, .cgSize, &size)
+    else { return nil }
+
+    return [
+        "x": Double(point.x),
+        "y": Double(point.y),
+        "width": Double(size.width),
+        "height": Double(size.height),
+    ]
+}
+
+func record(_ element: AXUIElement, index: String) -> ElementRecord {
+    ElementRecord(
+        index: index,
+        role: stringAttr(element, kAXRoleAttribute as CFString) ?? "AXUnknown",
+        title: stringAttr(element, kAXTitleAttribute as CFString) ?? stringAttr(element, kAXDescriptionAttribute as CFString),
+        value: stringAttr(element, kAXValueAttribute as CFString),
+        bounds: bounds(element),
+        actions: actions(element)
+    )
+}
+
+func cachedElement(_ params: JSON) -> AXUIElement? {
+    guard let stateId = params["stateId"] as? String,
+          let elementIndex = params["element_index"] as? String
+    else {
+        return nil
+    }
+    return stateAxElements[stateId]?[elementIndex]
+}
+
+func dictionary(_ record: ElementRecord) -> JSON {
+    var output: JSON = [
+        "index": record.index,
+        "role": record.role,
+        "actions": record.actions,
+    ]
+    if let title = record.title { output["title"] = title }
+    if let value = record.value { output["value"] = value }
+    if let bounds = record.bounds { output["bounds"] = bounds }
+    return output
+}
+
+func pruneStoredStates() {
+    while stateOrder.count > maxStoredStates {
+        let evicted = stateOrder.removeFirst()
+        stateElements.removeValue(forKey: evicted)
+        stateAxElements.removeValue(forKey: evicted)
+    }
+}
+
+func resolveApp(_ query: String) throws -> NSRunningApplication {
+    let normalized = query.lowercased()
+    let apps = NSWorkspace.shared.runningApplications.filter { app in
+        app.activationPolicy == .regular
+    }
+    if let app = apps.first(where: { $0.bundleIdentifier?.lowercased() == normalized }) {
+        return app
+    }
+    if let app = apps.first(where: { ($0.localizedName ?? "").lowercased() == normalized }) {
+        return app
+    }
+    if let app = apps.first(where: { ($0.localizedName ?? "").lowercased().contains(normalized) }) {
+        return app
+    }
+    throw NSError(domain: "CloudCLISemantics", code: 404, userInfo: [NSLocalizedDescriptionKey: "App is not running: \(query)"])
+}
+
+func listApps() -> [[String: Any]] {
+    NSWorkspace.shared.runningApplications
+        .filter { $0.activationPolicy == .regular }
+        .map { app in
+            [
+                "id": app.bundleIdentifier ?? app.localizedName ?? "\(app.processIdentifier)",
+                "name": app.localizedName ?? app.bundleIdentifier ?? "Unknown",
+                "bundleIdentifier": app.bundleIdentifier ?? "",
+                "pid": Int(app.processIdentifier),
+                "running": true,
+            ]
+        }
+}
+
+func walk(_ element: AXUIElement, depth: Int, maxDepth: Int, records: inout [ElementRecord], axRecords: inout [String: AXUIElement], limit: Int) {
+    if depth > maxDepth || records.count >= limit { return }
+    let index = "\(records.count + 1)"
+    records.append(record(element, index: index))
+    axRecords[index] = element
+    for child in arrayAttr(element, kAXChildrenAttribute as CFString) {
+        walk(child, depth: depth + 1, maxDepth: maxDepth, records: &records, axRecords: &axRecords, limit: limit)
+        if records.count >= limit { return }
+    }
+}
+
+func pngDataUrlForMainDisplay() -> String? {
+    let fileURL = URL(fileURLWithPath: NSTemporaryDirectory()).appendingPathComponent("cloudcli-semantics-\(UUID().uuidString).png")
+    let process = Process()
+    process.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture")
+    process.arguments = ["-x", "-t", "png", fileURL.path]
+
+    do {
+        try process.run()
+        process.waitUntilExit()
+        guard process.terminationStatus == 0 else { return nil }
+        let png = try Data(contentsOf: fileURL)
+        try? FileManager.default.removeItem(at: fileURL)
+        return png.isEmpty ? nil : "data:image/png;base64,\(png.base64EncodedString())"
+    } catch {
+        try? FileManager.default.removeItem(at: fileURL)
+        return nil
+    }
+}
+
+func getAppState(_ params: JSON) throws -> JSON {
+    let appName = params["app"] as? String ?? ""
+    let app = try resolveApp(appName)
+    let axApp = AXUIElementCreateApplication(app.processIdentifier)
+    let windows = arrayAttr(axApp, kAXWindowsAttribute as CFString)
+    let root = windows.first ?? axApp
+    var records: [ElementRecord] = []
+    var axRecords: [String: AXUIElement] = [:]
+    walk(root, depth: 0, maxDepth: 5, records: &records, axRecords: &axRecords, limit: 300)
+    let stateId = "state_\(UUID().uuidString)"
+    stateElements[stateId] = records
+    stateAxElements[stateId] = axRecords
+    stateOrder.append(stateId)
+    pruneStoredStates()
+
+    let elements = records.map(dictionary)
+    return [
+        "stateId": stateId,
+        "app": app.localizedName ?? app.bundleIdentifier ?? appName,
+        "platform": "darwin",
+        "screenshotDataUrl": pngDataUrlForMainDisplay() ?? NSNull(),
+        "displaySize": [
+            "width": Int(CGDisplayPixelsWide(CGMainDisplayID())),
+            "height": Int(CGDisplayPixelsHigh(CGMainDisplayID())),
+        ],
+        "elements": elements,
+        "accessibilityTree": elements,
+        "treeText": elements.map { "\($0["index"] ?? "") \($0["role"] ?? "") \($0["title"] ?? "")" }.joined(separator: "\n"),
+    ]
+}
+
+func cgMouseButton(_ value: Any?) -> CGMouseButton {
+    guard let button = value as? String else { return .left }
+    switch button {
+    case "right": return .right
+    case "middle": return .center
+    default: return .left
+    }
+}
+
+func mouseEventTypes(_ button: CGMouseButton) -> (CGEventType, CGEventType) {
+    switch button {
+    case .right: return (.rightMouseDown, .rightMouseUp)
+    case .center: return (.otherMouseDown, .otherMouseUp)
+    default: return (.leftMouseDown, .leftMouseUp)
+    }
+}
+
+func postMouseClick(point: CGPoint, button: CGMouseButton, clickCount: Int = 1) throws {
+    guard let source = CGEventSource(stateID: .combinedSessionState) else {
+        throw NSError(domain: "CloudCLISemantics", code: 500, userInfo: [NSLocalizedDescriptionKey: "Failed to create CGEventSource"])
+    }
+    let eventTypes = mouseEventTypes(button)
+    for _ in 0..<max(1, clickCount) {
+        let down = CGEvent(mouseEventSource: source, mouseType: eventTypes.0, mouseCursorPosition: point, mouseButton: button)
+        let up = CGEvent(mouseEventSource: source, mouseType: eventTypes.1, mouseCursorPosition: point, mouseButton: button)
+        down?.post(tap: .cghidEventTap)
+        up?.post(tap: .cghidEventTap)
+        usleep(80_000)
+    }
+}
+
+func postDrag(from: CGPoint, to: CGPoint, button: CGMouseButton) throws {
+    guard let source = CGEventSource(stateID: .combinedSessionState) else {
+        throw NSError(domain: "CloudCLISemantics", code: 500, userInfo: [NSLocalizedDescriptionKey: "Failed to create CGEventSource"])
+    }
+    let eventTypes = mouseEventTypes(button)
+    CGEvent(mouseEventSource: source, mouseType: eventTypes.0, mouseCursorPosition: from, mouseButton: button)?.post(tap: .cghidEventTap)
+    usleep(80_000)
+    CGEvent(mouseEventSource: source, mouseType: .leftMouseDragged, mouseCursorPosition: to, mouseButton: button)?.post(tap: .cghidEventTap)
+    usleep(80_000)
+    CGEvent(mouseEventSource: source, mouseType: eventTypes.1, mouseCursorPosition: to, mouseButton: button)?.post(tap: .cghidEventTap)
+}
+
+func runAppleScript(_ script: String) throws {
+    let process = Process()
+    process.executableURL = URL(fileURLWithPath: "/usr/bin/osascript")
+    process.arguments = ["-e", script]
+    process.standardOutput = Pipe()
+    let stderr = Pipe()
+    process.standardError = stderr
+    try process.run()
+    process.waitUntilExit()
+    if process.terminationStatus != 0 {
+        let data = stderr.fileHandleForReading.readDataToEndOfFile()
+        let message = String(data: data, encoding: .utf8) ?? "AppleScript failed."
+        throw NSError(domain: "CloudCLISemantics", code: Int(process.terminationStatus), userInfo: [NSLocalizedDescriptionKey: message])
+    }
+}
+
+func escapedAppleScriptString(_ value: String) -> String {
+    value.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "\"", with: "\\\"")
+}
+
+func pointForElement(_ params: JSON) -> CGPoint? {
+    if let x = params["x"] as? Double, let y = params["y"] as? Double {
+        return CGPoint(x: x, y: y)
+    }
+    guard let stateId = params["stateId"] as? String,
+          let elementIndex = params["element_index"] as? String,
+          let element = stateElements[stateId]?.first(where: { $0.index == elementIndex }),
+          let b = element.bounds,
+          let x = b["x"], let y = b["y"], let width = b["width"], let height = b["height"]
+    else {
+        return nil
+    }
+    return CGPoint(x: x + width / 2, y: y + height / 2)
+}
+
+func click(_ params: JSON) throws -> JSON {
+    if let element = cachedElement(params),
+       cgMouseButton(params["mouse_button"]) == .left,
+       (params["click_count"] as? Int ?? 1) == 1,
+       actions(element).contains(kAXPressAction as String),
+       AXUIElementPerformAction(element, kAXPressAction as CFString) == .success {
+        return try getAppState(params)
+    }
+
+    guard let point = pointForElement(params) else {
+        throw NSError(domain: "CloudCLISemantics", code: 400, userInfo: [NSLocalizedDescriptionKey: "click_element requires x/y or stateId + element_index"])
+    }
+    let clickCount = params["click_count"] as? Int ?? 1
+    try postMouseClick(point: point, button: cgMouseButton(params["mouse_button"]), clickCount: clickCount)
+    return try getAppState(params)
+}
+
+func performSecondaryAction(_ params: JSON) throws -> JSON {
+    if let element = cachedElement(params),
+       actions(element).contains(kAXShowMenuAction as String),
+       AXUIElementPerformAction(element, kAXShowMenuAction as CFString) == .success {
+        return try getAppState(params)
+    }
+    guard let point = pointForElement(params) else {
+        throw NSError(domain: "CloudCLISemantics", code: 400, userInfo: [NSLocalizedDescriptionKey: "perform_secondary_action requires x/y or stateId + element_index"])
+    }
+    try postMouseClick(point: point, button: .right)
+    return try getAppState(params)
+}
+
+func setValue(_ params: JSON) throws -> JSON {
+    guard let value = params["value"] as? String else {
+        throw NSError(domain: "CloudCLISemantics", code: 400, userInfo: [NSLocalizedDescriptionKey: "set_value requires value"])
+    }
+    if let element = cachedElement(params),
+       AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFTypeRef) == .success {
+        return try getAppState(params)
+    }
+    guard let point = pointForElement(params) else {
+        throw NSError(domain: "CloudCLISemantics", code: 400, userInfo: [NSLocalizedDescriptionKey: "set_value requires x/y or stateId + element_index"])
+    }
+    try postMouseClick(point: point, button: .left)
+    try runAppleScript("tell application \"System Events\" to keystroke \"a\" using command down")
+    try runAppleScript("tell application \"System Events\" to keystroke \"\(escapedAppleScriptString(value))\"")
+    return try getAppState(params)
+}
+
+func typeText(_ params: JSON) throws -> JSON {
+    let text = params["text"] as? String ?? ""
+    try runAppleScript("tell application \"System Events\" to keystroke \"\(escapedAppleScriptString(text))\"")
+    return try getAppState(params)
+}
+
+func appleScriptModifiers(_ parts: [String]) -> String {
+    let modifiers = parts.compactMap { part -> String? in
+        switch part.lowercased() {
+        case "cmd", "command", "meta": return "command down"
+        case "ctrl", "control": return "control down"
+        case "alt", "option": return "option down"
+        case "shift": return "shift down"
+        default: return nil
+        }
+    }
+    return modifiers.isEmpty ? "" : " using {\(modifiers.joined(separator: ", "))}"
+}
+
+func appleScriptKeyCode(_ key: String) -> Int? {
+    switch key.lowercased() {
+    case "return", "enter": return 36
+    case "tab": return 48
+    case "space": return 49
+    case "delete", "backspace": return 51
+    case "escape", "esc": return 53
+    case "left": return 123
+    case "right": return 124
+    case "down": return 125
+    case "up": return 126
+    default: return nil
+    }
+}
+
+func pressKey(_ params: JSON) throws -> JSON {
+    let raw = params["key"] as? String ?? ""
+    let parts = raw.split(separator: "+").map { String($0).trimmingCharacters(in: .whitespacesAndNewlines) }.filter { !$0.isEmpty }
+    let key = parts.last ?? raw
+    let modifiers = appleScriptModifiers(Array(parts.dropLast()))
+    if let keyCode = appleScriptKeyCode(key) {
+        try runAppleScript("tell application \"System Events\" to key code \(keyCode)\(modifiers)")
+    } else {
+        try runAppleScript("tell application \"System Events\" to keystroke \"\(escapedAppleScriptString(key))\"\(modifiers)")
+    }
+    return try getAppState(params)
+}
+
+func scrollElement(_ params: JSON) throws -> JSON {
+    guard let point = pointForElement(params) else {
+        throw NSError(domain: "CloudCLISemantics", code: 400, userInfo: [NSLocalizedDescriptionKey: "scroll_element requires x/y or stateId + element_index"])
+    }
+    CGWarpMouseCursorPosition(point)
+    let direction = params["direction"] as? String ?? "down"
+    let pages = params["pages"] as? Double ?? 1.0
+    let amount = Int32(max(1.0, abs(pages) * 8.0))
+    let vertical = direction == "up" ? amount : direction == "down" ? -amount : 0
+    let horizontal = direction == "left" ? amount : direction == "right" ? -amount : 0
+    CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 2, wheel1: vertical, wheel2: horizontal, wheel3: 0)?.post(tap: .cghidEventTap)
+    return try getAppState(params)
+}
+
+func drag(_ params: JSON) throws -> JSON {
+    guard let fromX = params["from_x"] as? Double,
+          let fromY = params["from_y"] as? Double,
+          let toX = params["to_x"] as? Double,
+          let toY = params["to_y"] as? Double
+    else {
+        throw NSError(domain: "CloudCLISemantics", code: 400, userInfo: [NSLocalizedDescriptionKey: "drag requires from_x/from_y/to_x/to_y"])
+    }
+    try postDrag(from: CGPoint(x: fromX, y: fromY), to: CGPoint(x: toX, y: toY), button: cgMouseButton(params["mouse_button"]))
+    return try getAppState(params)
+}
+
+func handle(_ request: JSON) {
+    let id = request["id"]
+    let method = request["method"] as? String ?? ""
+    let params = request["params"] as? JSON ?? [:]
+
+    do {
+        switch method {
+        case "list_apps":
+            respond(id: id, result: listApps())
+        case "get_app_state":
+            respond(id: id, result: try getAppState(params))
+        case "click_element":
+            respond(id: id, result: try click(params))
+        case "perform_secondary_action":
+            respond(id: id, result: try performSecondaryAction(params))
+        case "set_value":
+            respond(id: id, result: try setValue(params))
+        case "type_text":
+            respond(id: id, result: try typeText(params))
+        case "press_key":
+            respond(id: id, result: try pressKey(params))
+        case "scroll_element":
+            respond(id: id, result: try scrollElement(params))
+        case "drag":
+            respond(id: id, result: try drag(params))
+        default:
+            respondError(id: id, "Method is not implemented yet: \(method)")
+        }
+    } catch {
+        respondError(id: id, error.localizedDescription)
+    }
+}
+
+while let line = readLine() {
+    guard let data = line.data(using: .utf8),
+          let object = try? JSONSerialization.jsonObject(with: data),
+          let request = object as? JSON
+    else {
+        respondError(id: nil, "Invalid JSON request")
+        continue
+    }
+    handle(request)
+}

package/server/modules/computer-use/semantics/helpers/semantic-helper-process.ts

@@ -0,0 +1,124 @@
+import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process';
+import readline from 'node:readline';
+
+type JsonRecord = Record<string, unknown>;
+
+type PendingRequest = {
+  resolve: (value: unknown) => void;
+  reject: (error: Error) => void;
+  timer: ReturnType<typeof setTimeout>;
+};
+
+const DEFAULT_TIMEOUT_MS = Number.parseInt(process.env.CLOUDCLI_SEMANTICS_HELPER_TIMEOUT_MS || '60000', 10);
+
+function timeoutMs(): number {
+  return Number.isFinite(DEFAULT_TIMEOUT_MS) && DEFAULT_TIMEOUT_MS > 0 ? DEFAULT_TIMEOUT_MS : 60000;
+}
+
+function errorMessage(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}
+
+export class SemanticHelperProcess {
+  private child: ChildProcessWithoutNullStreams | null = null;
+  private reader: readline.Interface | null = null;
+  private nextId = 1;
+  private pending = new Map<number, PendingRequest>();
+
+  constructor(private readonly executablePath: string) {}
+
+  async request(method: string, params: JsonRecord): Promise<unknown> {
+    this.ensureStarted();
+    const child = this.child;
+    if (!child?.stdin.writable) {
+      throw new Error('Semantic helper process is not running.');
+    }
+
+    const id = this.nextId++;
+    return new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.pending.delete(id);
+        reject(new Error(`Semantic helper request timed out: ${method}`));
+      }, timeoutMs());
+      this.pending.set(id, { resolve, reject, timer });
+      child.stdin.write(`${JSON.stringify({ id, method, params })}\n`);
+    });
+  }
+
+  stop(): void {
+    const child = this.child;
+    this.child = null;
+    this.reader?.close();
+    this.reader = null;
+    this.rejectAll('Semantic helper stopped.');
+    if (child) {
+      try { child.kill('SIGTERM'); } catch { /* noop */ }
+    }
+  }
+
+  private ensureStarted(): void {
+    if (this.child) {
+      return;
+    }
+
+    this.child = spawn(this.executablePath, [], {
+      stdio: ['pipe', 'pipe', 'pipe'],
+      windowsHide: true,
+    });
+
+    this.reader = readline.createInterface({ input: this.child.stdout });
+    this.reader.on('line', (line) => this.handleLine(line));
+
+    this.child.stderr.on('data', (chunk) => {
+      const text = String(chunk).trim();
+      if (text) {
+        console.error('[SemanticHelper]', text);
+      }
+    });
+
+    this.child.once('error', (error) => {
+      this.child = null;
+      this.rejectAll(`Failed to start semantic helper: ${error.message}`);
+    });
+
+    this.child.once('exit', (code) => {
+      this.child = null;
+      this.rejectAll(`Semantic helper exited with code ${code ?? 'null'}.`);
+    });
+  }
+
+  private handleLine(line: string): void {
+    let message: JsonRecord;
+    try {
+      message = JSON.parse(line) as JsonRecord;
+    } catch (error) {
+      console.error('[SemanticHelper] Invalid JSON response:', errorMessage(error));
+      return;
+    }
+
+    const id = typeof message.id === 'number' ? message.id : null;
+    if (id === null) {
+      return;
+    }
+    const pending = this.pending.get(id);
+    if (!pending) {
+      return;
+    }
+    clearTimeout(pending.timer);
+    this.pending.delete(id);
+
+    if (message.error) {
+      pending.reject(new Error(typeof message.error === 'string' ? message.error : 'Semantic helper request failed.'));
+      return;
+    }
+    pending.resolve(message.result);
+  }
+
+  private rejectAll(reason: string): void {
+    for (const [id, request] of this.pending.entries()) {
+      clearTimeout(request.timer);
+      request.reject(new Error(reason));
+      this.pending.delete(id);
+    }
+  }
+}

package/server/modules/computer-use/semantics/helpers/semantic-helper-resolver.ts

@@ -0,0 +1,97 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+export type SemanticHelperPlatform = 'darwin' | 'win32';
+
+export type SemanticHelperResolution = {
+  available: boolean;
+  path: string | null;
+  source: 'bundled' | 'dev' | 'missing';
+  platform: NodeJS.Platform;
+  arch: NodeJS.Architecture;
+  reason?: string;
+};
+
+function helperExecutableName(platform: NodeJS.Platform): string | null {
+  if (platform === 'darwin') {
+    return 'CloudCLISemantics';
+  }
+  if (platform === 'win32') {
+    return 'CloudCLISemantics.exe';
+  }
+  return null;
+}
+
+function pathExists(filePath: string): boolean {
+  try {
+    fs.accessSync(filePath, fs.constants.X_OK);
+    return true;
+  } catch {
+    try {
+      fs.accessSync(filePath, fs.constants.F_OK);
+      return true;
+    } catch {
+      return false;
+    }
+  }
+}
+
+function candidatePaths(platform: NodeJS.Platform, arch: NodeJS.Architecture): Array<{ source: 'bundled' | 'dev'; path: string }> {
+  const executable = helperExecutableName(platform);
+  if (!executable) {
+    return [];
+  }
+
+  const platformArch = `${platform}-${arch}`;
+  return [
+    {
+      source: 'bundled',
+      path: path.resolve(__dirname, '..', 'bin', platformArch, executable),
+    },
+    {
+      source: 'dev',
+      path: path.resolve(process.cwd(), 'server', 'modules', 'computer-use', 'semantics', 'bin', platformArch, executable),
+    },
+  ];
+}
+
+export function resolveSemanticHelper(
+  platform: NodeJS.Platform = process.platform,
+  arch: NodeJS.Architecture = process.arch,
+): SemanticHelperResolution {
+  const executable = helperExecutableName(platform);
+  if (!executable) {
+    return {
+      available: false,
+      path: null,
+      source: 'missing',
+      platform,
+      arch,
+      reason: `Semantic Computer Use helper is not supported on ${platform}.`,
+    };
+  }
+
+  for (const candidate of candidatePaths(platform, arch)) {
+    if (pathExists(candidate.path)) {
+      return {
+        available: true,
+        path: candidate.path,
+        source: candidate.source,
+        platform,
+        arch,
+      };
+    }
+  }
+
+  return {
+    available: false,
+    path: null,
+    source: 'missing',
+    platform,
+    arch,
+    reason: `Bundled semantic helper was not found for ${platform}-${arch} (${executable}).`,
+  };
+}

package/server/modules/computer-use/semantics/helpers/windows/CloudCLISemantics.csproj

@@ -0,0 +1,11 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net8.0-windows</TargetFramework>
+    <Nullable>enable</Nullable>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <UseWindowsForms>true</UseWindowsForms>
+    <UseWPF>true</UseWPF>
+    <AssemblyName>CloudCLISemantics</AssemblyName>
+  </PropertyGroup>
+</Project>