@cloudcli-ai/cloudcli 1.35.0 → 1.36.0

package/server/modules/computer-use/semantics/helpers/windows/Program.cs

@@ -1,534 +0,0 @@
-using System.Diagnostics;
-using System.Drawing;
-using System.Drawing.Imaging;
-using System.IO;
-using System.Runtime.InteropServices;
-using System.Text.Json;
-using System.Windows.Automation;
-
-static class Program
-{
-    private const int MaxStoredStates = 100;
-    private static readonly Dictionary<string, List<ElementRecord>> StateElements = new();
-    private static readonly Dictionary<string, Dictionary<string, AutomationElement>> StateAutomationElements = new();
-    private static readonly Queue<string> StateOrder = new();
-
-    public static void Main()
-    {
-        string? line;
-        while ((line = Console.ReadLine()) != null)
-        {
-            try
-            {
-                using var doc = JsonDocument.Parse(line);
-                var root = doc.RootElement;
-                var id = root.TryGetProperty("id", out var idValue) ? idValue.Clone() : default;
-                var method = root.TryGetProperty("method", out var methodValue) ? methodValue.GetString() ?? "" : "";
-                var parameters = root.TryGetProperty("params", out var paramsValue) && paramsValue.ValueKind == JsonValueKind.Object
-                    ? paramsValue.Clone()
-                    : JsonDocument.Parse("{}").RootElement.Clone();
-
-                try
-                {
-                    object result = method switch
-                    {
-                        "list_apps" => ListApps(),
-                        "get_app_state" => GetAppState(parameters),
-                        "click_element" => ClickElement(parameters),
-                        "perform_secondary_action" => PerformSecondaryAction(parameters),
-                        "set_value" => SetValue(parameters),
-                        "type_text" => TypeText(parameters),
-                        "press_key" => PressKey(parameters),
-                        "scroll_element" => ScrollElement(parameters),
-                        "drag" => Drag(parameters),
-                        _ => throw new InvalidOperationException($"Method is not implemented yet: {method}")
-                    };
-                    Write(new Dictionary<string, object?> { ["id"] = JsonValue(id), ["result"] = result });
-                }
-                catch (Exception ex)
-                {
-                    Write(new Dictionary<string, object?> { ["id"] = JsonValue(id), ["error"] = ex.Message });
-                }
-            }
-            catch (Exception ex)
-            {
-                Write(new Dictionary<string, object?> { ["id"] = null, ["error"] = $"Invalid JSON request: {ex.Message}" });
-            }
-        }
-    }
-
-    private static object? JsonValue(JsonElement element)
-    {
-        return element.ValueKind switch
-        {
-            JsonValueKind.String => element.GetString(),
-            JsonValueKind.Number => element.TryGetInt64(out var number) ? number : element.GetDouble(),
-            JsonValueKind.True => true,
-            JsonValueKind.False => false,
-            _ => null
-        };
-    }
-
-    private static void Write(object value)
-    {
-        Console.WriteLine(JsonSerializer.Serialize(value));
-        Console.Out.Flush();
-    }
-
-    private static List<Dictionary<string, object?>> ListApps()
-    {
-        return Process.GetProcesses()
-            .Where(process => process.MainWindowHandle != IntPtr.Zero)
-            .OrderBy(process => process.ProcessName)
-            .Select(process => new Dictionary<string, object?>
-            {
-                ["id"] = process.Id.ToString(),
-                ["name"] = process.ProcessName,
-                ["processName"] = process.ProcessName,
-                ["pid"] = process.Id,
-                ["running"] = true,
-                ["windowTitle"] = process.MainWindowTitle
-            })
-            .ToList();
-    }
-
-    private static Process ResolveProcess(string query)
-    {
-        var normalized = query.Trim();
-        if (string.IsNullOrWhiteSpace(normalized))
-        {
-            throw new InvalidOperationException("app is required.");
-        }
-
-        var processes = Process.GetProcesses()
-            .Where(process => process.MainWindowHandle != IntPtr.Zero)
-            .ToList();
-
-        return processes.FirstOrDefault(process => process.ProcessName.Equals(normalized, StringComparison.OrdinalIgnoreCase))
-            ?? processes.FirstOrDefault(process => process.MainWindowTitle.Equals(normalized, StringComparison.OrdinalIgnoreCase))
-            ?? processes.FirstOrDefault(process => process.MainWindowTitle.Contains(normalized, StringComparison.OrdinalIgnoreCase))
-            ?? throw new InvalidOperationException($"App is not running: {query}");
-    }
-
-    private static Dictionary<string, object?> GetAppState(JsonElement parameters)
-    {
-        var appQuery = ReadString(parameters, "app");
-        var process = ResolveProcess(appQuery);
-        var root = AutomationElement.FromHandle(process.MainWindowHandle)
-            ?? throw new InvalidOperationException("No UI Automation root window is available.");
-
-        var records = new List<ElementRecord>();
-        var automationElements = new Dictionary<string, AutomationElement>();
-        Walk(root, records, automationElements, 0, 5, 300);
-        var stateId = $"state_{Guid.NewGuid()}";
-        StateElements[stateId] = records;
-        StateAutomationElements[stateId] = automationElements;
-        StateOrder.Enqueue(stateId);
-        PruneStoredStates();
-
-        var elements = records.Select(record => record.ToDictionary()).ToList();
-        var bounds = root.Current.BoundingRectangle;
-        return new Dictionary<string, object?>
-        {
-            ["stateId"] = stateId,
-            ["app"] = process.ProcessName,
-            ["platform"] = "win32",
-            ["screenshotDataUrl"] = CaptureScreen(),
-            ["displaySize"] = new Dictionary<string, object?>
-            {
-                ["width"] = (int)System.Windows.Forms.Screen.PrimaryScreen!.Bounds.Width,
-                ["height"] = (int)System.Windows.Forms.Screen.PrimaryScreen!.Bounds.Height
-            },
-            ["window"] = new Dictionary<string, object?>
-            {
-                ["title"] = process.MainWindowTitle,
-                ["bounds"] = BoundsDictionary(bounds)
-            },
-            ["elements"] = elements,
-            ["accessibilityTree"] = elements,
-            ["treeText"] = string.Join("\n", elements.Select(element => $"{element["index"]} {element["role"]} {element.GetValueOrDefault("title")}"))
-        };
-    }
-
-    private static Dictionary<string, object?> ClickElement(JsonElement parameters)
-    {
-        var mouseButton = ReadString(parameters, "mouse_button");
-        if ((mouseButton == "" || mouseButton == "left") && ReadInt(parameters, "click_count", 1) == 1)
-        {
-            var element = AutomationElementFor(parameters);
-            if (element != null && TryInvoke(element))
-            {
-                return GetAppState(parameters);
-            }
-        }
-
-        var point = PointFor(parameters);
-        if (point == null)
-        {
-            throw new InvalidOperationException("click_element requires x/y or stateId + element_index.");
-        }
-
-        SendMouseClick(point.Value.X, point.Value.Y, ReadString(parameters, "mouse_button"), ReadInt(parameters, "click_count", 1));
-        return GetAppState(parameters);
-    }
-
-    private static Dictionary<string, object?> PerformSecondaryAction(JsonElement parameters)
-    {
-        var point = PointFor(parameters);
-        if (point == null)
-        {
-            throw new InvalidOperationException("perform_secondary_action requires x/y or stateId + element_index.");
-        }
-
-        SendMouseClick(point.Value.X, point.Value.Y, "right", 1);
-        return GetAppState(parameters);
-    }
-
-    private static Dictionary<string, object?> SetValue(JsonElement parameters)
-    {
-        var value = ReadString(parameters, "value");
-        var element = AutomationElementFor(parameters);
-        var focused = false;
-        if (element != null)
-        {
-            if (element.TryGetCurrentPattern(ValuePattern.Pattern, out var valuePattern))
-            {
-                ((ValuePattern)valuePattern).SetValue(value);
-                return GetAppState(parameters);
-            }
-
-            try
-            {
-                element.SetFocus();
-                focused = true;
-            }
-            catch
-            {
-                // Fall through to coordinate focus below.
-            }
-        }
-
-        var point = PointFor(parameters);
-        if (point != null)
-        {
-            SendMouseClick(point.Value.X, point.Value.Y, "left", 1);
-            focused = true;
-        }
-        else if (!focused && element == null)
-        {
-            throw new InvalidOperationException("set_value requires x/y or stateId + element_index.");
-        }
-        else if (!focused)
-        {
-            throw new InvalidOperationException("set_value could not focus the requested element.");
-        }
-        System.Windows.Forms.SendKeys.SendWait("^a");
-        System.Windows.Forms.SendKeys.SendWait(EscapeSendKeys(value));
-        return GetAppState(parameters);
-    }
-
-    private static Dictionary<string, object?> TypeText(JsonElement parameters)
-    {
-        var text = ReadString(parameters, "text");
-        System.Windows.Forms.SendKeys.SendWait(EscapeSendKeys(text));
-        return GetAppState(parameters);
-    }
-
-    private static Dictionary<string, object?> PressKey(JsonElement parameters)
-    {
-        var key = ReadString(parameters, "key");
-        System.Windows.Forms.SendKeys.SendWait(ToSendKeysChord(key));
-        return GetAppState(parameters);
-    }
-
-    private static Dictionary<string, object?> ScrollElement(JsonElement parameters)
-    {
-        var element = AutomationElementFor(parameters);
-        var direction = ReadString(parameters, "direction");
-        var pages = ReadDouble(parameters, "pages", 1);
-        if (element != null && element.TryGetCurrentPattern(ScrollPattern.Pattern, out var scrollPatternValue))
-        {
-            var scrollPattern = (ScrollPattern)scrollPatternValue;
-            var vertical = direction == "up" ? ScrollAmount.LargeDecrement : direction == "down" ? ScrollAmount.LargeIncrement : ScrollAmount.NoAmount;
-            var horizontal = direction == "left" ? ScrollAmount.LargeDecrement : direction == "right" ? ScrollAmount.LargeIncrement : ScrollAmount.NoAmount;
-            scrollPattern.Scroll(horizontal, vertical);
-            return GetAppState(parameters);
-        }
-
-        var point = PointFor(parameters);
-        if (point == null)
-        {
-            throw new InvalidOperationException("scroll_element requires x/y or stateId + element_index.");
-        }
-        SetCursorPos(point.Value.X, point.Value.Y);
-        var wheel = (int)Math.Round(Math.Max(1, pages) * 120);
-        if (direction == "down") wheel = -wheel;
-        mouse_event(0x0800, 0, 0, unchecked((uint)wheel), UIntPtr.Zero);
-        return GetAppState(parameters);
-    }
-
-    private static void PruneStoredStates()
-    {
-        while (StateOrder.Count > MaxStoredStates)
-        {
-            var evicted = StateOrder.Dequeue();
-            StateElements.Remove(evicted);
-            StateAutomationElements.Remove(evicted);
-        }
-    }
-
-    private static Dictionary<string, object?> Drag(JsonElement parameters)
-    {
-        var fromX = ReadDouble(parameters, "from_x", double.NaN);
-        var fromY = ReadDouble(parameters, "from_y", double.NaN);
-        var toX = ReadDouble(parameters, "to_x", double.NaN);
-        var toY = ReadDouble(parameters, "to_y", double.NaN);
-        if (double.IsNaN(fromX) || double.IsNaN(fromY) || double.IsNaN(toX) || double.IsNaN(toY))
-        {
-            throw new InvalidOperationException("drag requires from_x/from_y/to_x/to_y.");
-        }
-
-        SetCursorPos((int)Math.Round(fromX), (int)Math.Round(fromY));
-        mouse_event(0x0002, 0, 0, 0, UIntPtr.Zero);
-        Thread.Sleep(80);
-        SetCursorPos((int)Math.Round(toX), (int)Math.Round(toY));
-        Thread.Sleep(80);
-        mouse_event(0x0004, 0, 0, 0, UIntPtr.Zero);
-        return GetAppState(parameters);
-    }
-
-    private static void Walk(AutomationElement element, List<ElementRecord> records, Dictionary<string, AutomationElement> automationElements, int depth, int maxDepth, int limit)
-    {
-        if (depth > maxDepth || records.Count >= limit) return;
-        var index = (records.Count + 1).ToString();
-        records.Add(ElementRecord.From(element, index));
-        automationElements[index] = element;
-        var children = element.FindAll(TreeScope.Children, Condition.TrueCondition);
-        foreach (AutomationElement child in children)
-        {
-            Walk(child, records, automationElements, depth + 1, maxDepth, limit);
-            if (records.Count >= limit) return;
-        }
-    }
-
-    private static string ReadString(JsonElement element, string property)
-    {
-        return element.TryGetProperty(property, out var value) && value.ValueKind == JsonValueKind.String
-            ? value.GetString() ?? ""
-            : "";
-    }
-
-    private static int ReadInt(JsonElement element, string property, int defaultValue)
-    {
-        return element.TryGetProperty(property, out var value) && value.TryGetInt32(out var number)
-            ? number
-            : defaultValue;
-    }
-
-    private static double ReadDouble(JsonElement element, string property, double defaultValue)
-    {
-        return element.TryGetProperty(property, out var value) && value.TryGetDouble(out var number)
-            ? number
-            : defaultValue;
-    }
-
-    private static AutomationElement? AutomationElementFor(JsonElement parameters)
-    {
-        var stateId = ReadString(parameters, "stateId");
-        var elementIndex = ReadString(parameters, "element_index");
-        return !string.IsNullOrWhiteSpace(stateId)
-            && !string.IsNullOrWhiteSpace(elementIndex)
-            && StateAutomationElements.TryGetValue(stateId, out var elements)
-            && elements.TryGetValue(elementIndex, out var element)
-            ? element
-            : null;
-    }
-
-    private static System.Drawing.Point? PointFor(JsonElement parameters)
-    {
-        if (parameters.TryGetProperty("x", out var xValue) && parameters.TryGetProperty("y", out var yValue)
-            && xValue.TryGetDouble(out var x) && yValue.TryGetDouble(out var y))
-        {
-            return new System.Drawing.Point((int)Math.Round(x), (int)Math.Round(y));
-        }
-
-        var stateId = ReadString(parameters, "stateId");
-        var elementIndex = ReadString(parameters, "element_index");
-        if (string.IsNullOrWhiteSpace(stateId) || string.IsNullOrWhiteSpace(elementIndex)) return null;
-        if (!StateElements.TryGetValue(stateId, out var elements)) return null;
-        var element = elements.FirstOrDefault(item => item.Index == elementIndex);
-        if (element?.Bounds == null) return null;
-        return new System.Drawing.Point(
-            (int)Math.Round(element.Bounds.Value.Left + element.Bounds.Value.Width / 2),
-            (int)Math.Round(element.Bounds.Value.Top + element.Bounds.Value.Height / 2)
-        );
-    }
-
-    private static string CaptureScreen()
-    {
-        var bounds = System.Windows.Forms.Screen.PrimaryScreen!.Bounds;
-        using var bitmap = new Bitmap(bounds.Width, bounds.Height);
-        using var graphics = Graphics.FromImage(bitmap);
-        graphics.CopyFromScreen(bounds.Left, bounds.Top, 0, 0, bounds.Size);
-        using var stream = new MemoryStream();
-        bitmap.Save(stream, ImageFormat.Png);
-        return $"data:image/png;base64,{Convert.ToBase64String(stream.ToArray())}";
-    }
-
-    private static Dictionary<string, object?> BoundsDictionary(System.Windows.Rect rect)
-    {
-        return new Dictionary<string, object?>
-        {
-            ["x"] = rect.X,
-            ["y"] = rect.Y,
-            ["width"] = rect.Width,
-            ["height"] = rect.Height
-        };
-    }
-
-    [DllImport("user32.dll")]
-    private static extern bool SetCursorPos(int x, int y);
-
-    [DllImport("user32.dll")]
-    private static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, UIntPtr dwExtraInfo);
-
-    private static void SendMouseClick(int x, int y, string button, int clickCount)
-    {
-        var (down, up) = button switch
-        {
-            "right" => (0x0008u, 0x0010u),
-            "middle" => (0x0020u, 0x0040u),
-            _ => (0x0002u, 0x0004u)
-        };
-        SetCursorPos(x, y);
-        for (var i = 0; i < Math.Max(1, clickCount); i++)
-        {
-            mouse_event(down, 0, 0, 0, UIntPtr.Zero);
-            mouse_event(up, 0, 0, 0, UIntPtr.Zero);
-            Thread.Sleep(80);
-        }
-    }
-
-    private static bool TryInvoke(AutomationElement element)
-    {
-        try
-        {
-            if (!element.TryGetCurrentPattern(InvokePattern.Pattern, out var pattern)) return false;
-            ((InvokePattern)pattern).Invoke();
-            return true;
-        }
-        catch
-        {
-            return false;
-        }
-    }
-
-    private static string EscapeSendKeys(string value)
-    {
-        return value
-            .Replace("{", "{{}")
-            .Replace("}", "{}}")
-            .Replace("+", "{+}")
-            .Replace("^", "{^}")
-            .Replace("%", "{%}")
-            .Replace("~", "{~}")
-            .Replace("(", "{(}")
-            .Replace(")", "{)}")
-            .Replace("[", "{[}")
-            .Replace("]", "{]}");
-    }
-
-    private static string ToSendKeysChord(string key)
-    {
-        var normalized = key.Trim();
-        if (normalized.Contains('+'))
-        {
-            var parts = normalized.Split('+', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
-            var modifiers = "";
-            var last = parts.LastOrDefault() ?? "";
-            foreach (var part in parts.Take(parts.Length - 1))
-            {
-                modifiers += part.ToLowerInvariant() switch
-                {
-                    "ctrl" or "control" => "^",
-                    "alt" => "%",
-                    "shift" => "+",
-                    "cmd" or "win" or "windows" => "^",
-                    _ => ""
-                };
-            }
-            return modifiers + SendKeyName(last);
-        }
-        return SendKeyName(normalized);
-    }
-
-    private static string SendKeyName(string key)
-    {
-        return key.ToLowerInvariant() switch
-        {
-            "return" or "enter" => "{ENTER}",
-            "escape" or "esc" => "{ESC}",
-            "tab" => "{TAB}",
-            "backspace" => "{BACKSPACE}",
-            "delete" or "del" => "{DELETE}",
-            "left" => "{LEFT}",
-            "right" => "{RIGHT}",
-            "up" => "{UP}",
-            "down" => "{DOWN}",
-            "space" => " ",
-            _ => key.Length == 1 ? EscapeSendKeys(key) : $"{{{key.ToUpperInvariant()}}}"
-        };
-    }
-
-    private sealed record ElementRecord(
-        string Index,
-        string Role,
-        string? Title,
-        string? Value,
-        System.Windows.Rect? Bounds,
-        List<string> Actions)
-    {
-        public static ElementRecord From(AutomationElement element, string index)
-        {
-            var patterns = element.GetSupportedPatterns().Select(pattern => pattern.ProgrammaticName).ToList();
-            return new ElementRecord(
-                index,
-                element.Current.ControlType.ProgrammaticName.Replace("ControlType.", ""),
-                element.Current.Name,
-                TryValue(element),
-                element.Current.BoundingRectangle,
-                patterns
-            );
-        }
-
-        public Dictionary<string, object?> ToDictionary()
-        {
-            var output = new Dictionary<string, object?>
-            {
-                ["index"] = Index,
-                ["role"] = Role,
-                ["actions"] = Actions
-            };
-            if (!string.IsNullOrEmpty(Title)) output["title"] = Title;
-            if (!string.IsNullOrEmpty(Value)) output["value"] = Value;
-            if (Bounds != null) output["bounds"] = BoundsDictionary(Bounds.Value);
-            return output;
-        }
-
-        private static string? TryValue(AutomationElement element)
-        {
-            try
-            {
-                if (element.TryGetCurrentPattern(ValuePattern.Pattern, out var pattern))
-                {
-                    return ((ValuePattern)pattern).Current.Value;
-                }
-            }
-            catch
-            {
-                return null;
-            }
-            return null;
-        }
-    }
-}

package/server/modules/computer-use/semantics/semantic-session-store.ts

@@ -1,87 +0,0 @@
-import { randomUUID } from 'node:crypto';
-
-import type { SemanticAppState, SemanticElement } from '@/modules/computer-use/semantics/semantic-types.js';
-
-const DEFAULT_STATE_TTL_MS = Number.parseInt(process.env.CLOUDCLI_COMPUTER_SEMANTIC_STATE_TTL_MS || String(10 * 60 * 1000), 10);
-
-type StoredState = {
-  sessionId: string;
-  appKey: string;
-  state: SemanticAppState;
-  updatedAt: number;
-};
-
-function normalizeAppKey(app: string): string {
-  return app.trim().toLowerCase();
-}
-
-export class SemanticSessionStore {
-  private states = new Map<string, StoredState>();
-  private latestBySessionApp = new Map<string, string>();
-
-  createStateId(): string {
-    return `state_${randomUUID()}`;
-  }
-
-  save(sessionId: string, state: SemanticAppState): SemanticAppState {
-    const appKey = normalizeAppKey(state.app);
-    const nextState = {
-      ...state,
-      stateId: state.stateId || this.createStateId(),
-    };
-    this.states.set(nextState.stateId, {
-      sessionId,
-      appKey,
-      state: nextState,
-      updatedAt: Date.now(),
-    });
-    this.latestBySessionApp.set(this.latestKey(sessionId, appKey), nextState.stateId);
-    return nextState;
-  }
-
-  getState(sessionId: string, app: string, stateId?: string): SemanticAppState | null {
-    this.expire();
-    if (stateId) {
-      const entry = this.states.get(stateId);
-      const appKey = normalizeAppKey(app);
-      return entry && entry.sessionId === sessionId && entry.appKey === appKey ? entry.state : null;
-    }
-    const latestStateId = this.latestBySessionApp.get(this.latestKey(sessionId, normalizeAppKey(app)));
-    return latestStateId ? this.states.get(latestStateId)?.state || null : null;
-  }
-
-  getElement(sessionId: string, app: string, elementIndex: string, stateId?: string): SemanticElement | null {
-    const state = this.getState(sessionId, app, stateId);
-    return state?.elements.find((element) => element.index === elementIndex) || null;
-  }
-
-  clearSession(sessionId: string): void {
-    for (const [stateId, entry] of this.states.entries()) {
-      if (entry.sessionId === sessionId) {
-        this.states.delete(stateId);
-        this.latestBySessionApp.delete(this.latestKey(entry.sessionId, entry.appKey));
-      }
-    }
-  }
-
-  expire(now = Date.now()): void {
-    const ttl = Number.isFinite(DEFAULT_STATE_TTL_MS) && DEFAULT_STATE_TTL_MS > 0
-      ? DEFAULT_STATE_TTL_MS
-      : 10 * 60 * 1000;
-    for (const [stateId, entry] of this.states.entries()) {
-      if (now - entry.updatedAt > ttl) {
-        this.states.delete(stateId);
-        const key = this.latestKey(entry.sessionId, entry.appKey);
-        if (this.latestBySessionApp.get(key) === stateId) {
-          this.latestBySessionApp.delete(key);
-        }
-      }
-    }
-  }
-
-  private latestKey(sessionId: string, appKey: string): string {
-    return `${sessionId}:${appKey}`;
-  }
-}
-
-export const semanticSessionStore = new SemanticSessionStore();

package/server/modules/computer-use/semantics/semantic-tool-dispatcher.ts

@@ -1,17 +0,0 @@
-export const semanticMcpToolMap: Record<string, string> = {
-  computer_app_drag: 'drag',
-  computer_click_element: 'click_element',
-  computer_get_app_state: 'get_app_state',
-  computer_list_apps: 'list_apps',
-  computer_perform_secondary_action: 'perform_secondary_action',
-  computer_press_key: 'press_key',
-  computer_scroll_element: 'scroll_element',
-  computer_set_value: 'set_value',
-  computer_type_text: 'type_text',
-};
-
-export const semanticOperationNames = new Set(Object.values(semanticMcpToolMap));
-
-export function semanticOperationForMcpTool(toolName: string): string | null {
-  return semanticMcpToolMap[toolName] || null;
-}

package/server/modules/computer-use/semantics/semantic-types.ts

@@ -1,58 +0,0 @@
-import type { DisplaySize, Point } from '@/modules/computer-use/computer-executor.js';
-
-export type SemanticBounds = {
-  x: number;
-  y: number;
-  width: number;
-  height: number;
-};
-
-export type SemanticApp = {
-  id?: string;
-  name: string;
-  bundleIdentifier?: string;
-  processName?: string;
-  pid?: number;
-  running: boolean;
-  windowTitle?: string;
-};
-
-export type SemanticElement = {
-  index: string;
-  role: string;
-  title?: string;
-  value?: string;
-  description?: string;
-  enabled?: boolean;
-  focused?: boolean;
-  selected?: boolean;
-  bounds?: SemanticBounds;
-  actions?: string[];
-  settableValue?: boolean;
-};
-
-export type SemanticAppState = {
-  stateId: string;
-  app: string;
-  platform: NodeJS.Platform;
-  screenshotDataUrl: string | null;
-  displaySize: DisplaySize | null;
-  elements: SemanticElement[];
-  accessibilityTree: SemanticElement[];
-  treeText?: string;
-  message?: string;
-};
-
-export type SemanticToolInput = Record<string, unknown> & {
-  sessionId?: string;
-  app?: string;
-  stateId?: string;
-  element_index?: string;
-};
-
-export type SemanticToolResult = SemanticAppState | {
-  apps: SemanticApp[];
-  platform: NodeJS.Platform;
-};
-
-export type SemanticActionPoint = Point;