@cloudcli-ai/cloudcli 1.35.0 → 1.36.0

package/server/modules/computer-use/computer-executor.ts

@@ -1,242 +0,0 @@
-import { createRequire } from 'node:module';
-
-const require = createRequire(import.meta.url);
-
-export type Point = { x: number; y: number };
-export type ClickButton = 'left' | 'right' | 'middle';
-export type ScrollDirection = 'up' | 'down' | 'left' | 'right';
-export type DisplaySize = { width: number; height: number };
-
-export type RuntimeReadiness = {
-  nut: any | null;
-  screenshot: any | null;
-  nutInstalled: boolean;
-  screenshotInstalled: boolean;
-};
-
-/**
- * Coordinate space the executor reports/accepts. The screenshot pixel space is
- * the canonical space agents and users address; it is mapped to the nut-js
- * logical mouse space before any action runs.
- */
-export type ExecutorTarget = {
-  displaySize: DisplaySize | null;
-};
-
-export function getNut(): any | null {
-  try {
-    return require('@nut-tree-fork/nut-js');
-  } catch {
-    return null;
-  }
-}
-
-export function getScreenshot(): any | null {
-  try {
-    const mod = require('screenshot-desktop');
-    return mod?.default || mod;
-  } catch {
-    return null;
-  }
-}
-
-export function getRuntimeReadiness(): RuntimeReadiness {
-  const nut = getNut();
-  const screenshot = getScreenshot();
-  return {
-    nut,
-    screenshot,
-    nutInstalled: Boolean(nut),
-    screenshotInstalled: typeof screenshot === 'function',
-  };
-}
-
-/** Reads the pixel dimensions from a PNG/JPEG buffer header without decoding it. */
-export function readImageSize(buffer: Buffer): DisplaySize | null {
-  // PNG: 8-byte signature, then IHDR chunk with width/height as big-endian uint32.
-  if (buffer.length >= 24 && buffer[0] === 0x89 && buffer[1] === 0x50) {
-    return { width: buffer.readUInt32BE(16), height: buffer.readUInt32BE(20) };
-  }
-  // JPEG: scan for a Start-Of-Frame marker (0xFFC0..0xFFCF, excluding C4/C8/CC).
-  if (buffer.length >= 4 && buffer[0] === 0xff && buffer[1] === 0xd8) {
-    let offset = 2;
-    while (offset + 9 < buffer.length) {
-      if (buffer[offset] !== 0xff) {
-        offset += 1;
-        continue;
-      }
-      const marker = buffer[offset + 1];
-      if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
-        return { height: buffer.readUInt16BE(offset + 5), width: buffer.readUInt16BE(offset + 7) };
-      }
-      offset += 2 + buffer.readUInt16BE(offset + 2);
-    }
-  }
-  return null;
-}
-
-export async function captureScreenshot(): Promise<{ dataUrl: string; size: DisplaySize | null }> {
-  const screenshot = getScreenshot();
-  if (typeof screenshot !== 'function') {
-    throw new Error('Computer Use runtime is not available.');
-  }
-  const buffer: Buffer = await screenshot({ format: 'png' });
-  return {
-    dataUrl: `data:image/png;base64,${buffer.toString('base64')}`,
-    size: readImageSize(buffer),
-  };
-}
-
-/** Returns the mouse coordinate space size (logical screen pixels). */
-export async function getMouseSpaceSize(): Promise<DisplaySize> {
-  const nut = getNut();
-  if (!nut) {
-    throw new Error('Computer Use runtime is not available.');
-  }
-  const width = await nut.screen.width();
-  const height = await nut.screen.height();
-  return { width, height };
-}
-
-/** Maps a point from screenshot/image space to the mouse coordinate space. */
-export async function toMouseSpace(target: ExecutorTarget, point: Point): Promise<Point> {
-  const mouseSize = await getMouseSpaceSize();
-  const image = target.displaySize || mouseSize;
-  const scaleX = image.width ? mouseSize.width / image.width : 1;
-  const scaleY = image.height ? mouseSize.height / image.height : 1;
-  return {
-    x: Math.round(point.x * scaleX),
-    y: Math.round(point.y * scaleY),
-  };
-}
-
-/** Maps a point from the mouse coordinate space back to screenshot/image space. */
-export function toImageSpace(target: ExecutorTarget, point: Point, mouseSize: DisplaySize): Point {
-  const image = target.displaySize || mouseSize;
-  const scaleX = mouseSize.width ? image.width / mouseSize.width : 1;
-  const scaleY = mouseSize.height ? image.height / mouseSize.height : 1;
-  return {
-    x: Math.round(point.x * scaleX),
-    y: Math.round(point.y * scaleY),
-  };
-}
-
-function nutButton(nut: any, button: ClickButton) {
-  if (button === 'right') return nut.Button.RIGHT;
-  if (button === 'middle') return nut.Button.MIDDLE;
-  return nut.Button.LEFT;
-}
-
-/** Maps a key name (xdotool-style, as Anthropic's computer tool emits) to a nut-js Key. */
-function nutKey(nut: any, token: string): any {
-  const map: Record<string, string> = {
-    return: 'Enter', enter: 'Enter', esc: 'Escape', escape: 'Escape', tab: 'Tab',
-    space: 'Space', backspace: 'Backspace', delete: 'Delete', del: 'Delete', insert: 'Insert',
-    up: 'Up', down: 'Down', left: 'Left', right: 'Right',
-    home: 'Home', end: 'End', pageup: 'PageUp', page_up: 'PageUp', pagedown: 'PageDown', page_down: 'PageDown',
-    ctrl: 'LeftControl', control: 'LeftControl', alt: 'LeftAlt', shift: 'LeftShift',
-    meta: 'LeftSuper', super: 'LeftSuper', cmd: 'LeftSuper', win: 'LeftSuper',
-    capslock: 'CapsLock',
-  };
-  const lower = token.toLowerCase();
-  if (map[lower]) {
-    return nut.Key[map[lower]];
-  }
-  if (/^f([1-9]|1[0-9]|2[0-4])$/.test(lower)) {
-    return nut.Key[`F${lower.slice(1)}`];
-  }
-  if (token.length === 1) {
-    const upper = token.toUpperCase();
-    if (nut.Key[upper] !== undefined) {
-      return nut.Key[upper];
-    }
-    if (nut.Key[`Num${token}`] !== undefined && /[0-9]/.test(token)) {
-      return nut.Key[`Num${token}`];
-    }
-  }
-  throw new Error(`Unsupported key: ${token}`);
-}
-
-/**
- * The cross-platform OS executor. It is intentionally free of any server,
- * database, or session dependencies so it can run both inside the local server
- * process (OSS mode) and inside the standalone desktop agent (cloud relay).
- */
-export const executor = {
-  async configure() {
-    const nut = getNut();
-    if (nut) {
-      // Make actions responsive; the agent loop already paces itself with screenshots.
-      nut.mouse.config.autoDelayMs = 2;
-      nut.keyboard.config.autoDelayMs = 2;
-    }
-    return nut;
-  },
-
-  async cursorPosition(target: ExecutorTarget): Promise<Point> {
-    const nut = await this.configure();
-    const mouseSize = await getMouseSpaceSize();
-    const pos = await nut.mouse.getPosition();
-    return toImageSpace(target, { x: pos.x, y: pos.y }, mouseSize);
-  },
-
-  async moveTo(target: ExecutorTarget, point: Point): Promise<void> {
-    const nut = await this.configure();
-    const dest = await toMouseSpace(target, point);
-    await nut.mouse.setPosition(new nut.Point(dest.x, dest.y));
-  },
-
-  async click(target: ExecutorTarget, button: ClickButton, point?: Point, doubleClick = false): Promise<void> {
-    const nut = await this.configure();
-    if (point) {
-      await this.moveTo(target, point);
-    }
-    if (doubleClick) {
-      await nut.mouse.doubleClick(nutButton(nut, button));
-    } else {
-      await nut.mouse.click(nutButton(nut, button));
-    }
-  },
-
-  async drag(target: ExecutorTarget, from: Point, to: Point, button: ClickButton = 'left'): Promise<void> {
-    const nut = await this.configure();
-    const start = await toMouseSpace(target, from);
-    const end = await toMouseSpace(target, to);
-    await nut.mouse.setPosition(new nut.Point(start.x, start.y));
-    await nut.mouse.pressButton(nutButton(nut, button));
-    await nut.mouse.setPosition(new nut.Point(end.x, end.y));
-    await nut.mouse.releaseButton(nutButton(nut, button));
-  },
-
-  async type(text: string): Promise<void> {
-    const nut = await this.configure();
-    await nut.keyboard.type(text);
-  },
-
-  async pressChord(chord: string): Promise<void> {
-    const nut = await this.configure();
-    const tokens = chord.split('+').map((token) => token.trim()).filter(Boolean);
-    if (tokens.length === 0) {
-      return;
-    }
-    const keys = tokens.map((token) => nutKey(nut, token));
-    for (const key of keys) {
-      await nut.keyboard.pressKey(key);
-    }
-    for (const key of [...keys].reverse()) {
-      await nut.keyboard.releaseKey(key);
-    }
-  },
-
-  async scroll(target: ExecutorTarget, direction: ScrollDirection, amount: number, point?: Point): Promise<void> {
-    const nut = await this.configure();
-    if (point) {
-      await this.moveTo(target, point);
-    }
-    const steps = Math.max(1, Math.round(amount));
-    if (direction === 'up') await nut.mouse.scrollUp(steps);
-    else if (direction === 'down') await nut.mouse.scrollDown(steps);
-    else if (direction === 'left') await nut.mouse.scrollLeft(steps);
-    else await nut.mouse.scrollRight(steps);
-  },
-};

package/server/modules/computer-use/computer-semantics.service.ts

@@ -1,460 +0,0 @@
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
-
-import {
-  captureScreenshot,
-  executor,
-  type ClickButton,
-  type ExecutorTarget,
-  type Point,
-  type ScrollDirection,
-} from '@/modules/computer-use/computer-executor.js';
-import type { SemanticAdapter } from '@/modules/computer-use/semantics/adapters/semantic-adapter.js';
-import { createMacOsSemanticAdapter } from '@/modules/computer-use/semantics/adapters/macos/macos-semantic-adapter.js';
-import { createWindowsSemanticAdapter } from '@/modules/computer-use/semantics/adapters/windows/windows-semantic-adapter.js';
-import { resolveSemanticHelper } from '@/modules/computer-use/semantics/helpers/semantic-helper-resolver.js';
-import { semanticSessionStore } from '@/modules/computer-use/semantics/semantic-session-store.js';
-import type { SemanticAppState, SemanticElement } from '@/modules/computer-use/semantics/semantic-types.js';
-
-const execFileAsync = promisify(execFile);
-const MAX_APP_STATE_ELEMENTS = 250;
-let helperAdapter: SemanticAdapter | null | undefined;
-
-function readString(value: unknown): string {
-  return typeof value === 'string' ? value.trim() : '';
-}
-
-function requireApp(input: Record<string, unknown>): string {
-  const app = readString(input.app);
-  if (!app) {
-    throw new Error('app is required.');
-  }
-  return app;
-}
-
-function readNumber(value: unknown): number | undefined {
-  return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
-}
-
-function readButton(value: unknown): ClickButton {
-  return value === 'right' || value === 'middle' ? value : 'left';
-}
-
-function readClickCount(value: unknown): number {
-  const count = readNumber(value);
-  if (count === undefined) {
-    return 1;
-  }
-  return Math.max(1, Math.min(5, Math.trunc(count)));
-}
-
-function readDirection(value: unknown): ScrollDirection {
-  return value === 'up' || value === 'left' || value === 'right' ? value : 'down';
-}
-
-function readSessionId(input: Record<string, unknown>): string {
-  return readString(input.sessionId) || 'default';
-}
-
-function centerOf(element: SemanticElement): Point | null {
-  const bounds = element.bounds;
-  if (!bounds) {
-    return null;
-  }
-  return {
-    x: Math.round(bounds.x + bounds.width / 2),
-    y: Math.round(bounds.y + bounds.height / 2),
-  };
-}
-
-function getCachedElement(sessionId: string, app: string, index: string, stateId?: string): SemanticElement | null {
-  return semanticSessionStore.getElement(sessionId, app, index, stateId);
-}
-
-function getPoint(input: Record<string, unknown>, sessionId: string, app: string): Point | undefined {
-  const x = readNumber(input.x);
-  const y = readNumber(input.y);
-  if (x !== undefined && y !== undefined) {
-    return { x, y };
-  }
-
-  const elementIndex = readString(input.element_index);
-  if (!elementIndex) {
-    return undefined;
-  }
-  const element = getCachedElement(sessionId, app, elementIndex, readString(input.stateId) || undefined);
-  return element ? centerOf(element) || undefined : undefined;
-}
-
-function getHelperAdapter(): SemanticAdapter | null {
-  if (helperAdapter !== undefined) {
-    return helperAdapter;
-  }
-
-  if (process.platform !== 'darwin' && process.platform !== 'win32') {
-    helperAdapter = null;
-    return helperAdapter;
-  }
-
-  const resolution = resolveSemanticHelper();
-  if (!resolution.available) {
-    helperAdapter = null;
-    return helperAdapter;
-  }
-
-  helperAdapter = process.platform === 'darwin'
-    ? createMacOsSemanticAdapter()
-    : createWindowsSemanticAdapter();
-  return helperAdapter;
-}
-
-function shouldFallbackFromHelper(error: unknown): boolean {
-  const message = error instanceof Error ? error.message : String(error);
-  return /not implemented|unavailable|not found|does not exist|timed out|not running|exited with code|failed to start/i.test(message);
-}
-
-async function withHelperState(
-  sessionId: string,
-  operation: (adapter: SemanticAdapter) => Promise<SemanticAppState>,
-): Promise<SemanticAppState | null> {
-  const adapter = getHelperAdapter();
-  if (!adapter) {
-    return null;
-  }
-  try {
-    return semanticSessionStore.save(sessionId, await operation(adapter));
-  } catch (error) {
-    if (shouldFallbackFromHelper(error)) {
-      console.warn('[ComputerSemantics] Falling back from helper:', error instanceof Error ? error.message : String(error));
-      return null;
-    }
-    throw error;
-  }
-}
-
-async function run(command: string, args: string[], timeout = 5000): Promise<string> {
-  const { stdout } = await execFileAsync(command, args, {
-    timeout,
-    windowsHide: true,
-    maxBuffer: 1024 * 1024 * 4,
-  });
-  return stdout;
-}
-
-async function listMacApps(): Promise<Array<Record<string, unknown>>> {
-  const script = [
-    'tell application "System Events"',
-    'set appRows to {}',
-    'repeat with p in (application processes whose background only is false)',
-    'set end of appRows to (name of p as text)',
-    'end repeat',
-    'return appRows',
-    'end tell',
-  ].join('\n');
-  const output = await run('osascript', ['-e', script]);
-  return output.split(', ')
-    .map((name) => name.trim())
-    .filter(Boolean)
-    .map((name) => ({ name, running: true }));
-}
-
-async function listWindowsApps(): Promise<Array<Record<string, unknown>>> {
-  const script = [
-    'Get-Process | Where-Object { $_.MainWindowTitle } |',
-    'Select-Object ProcessName, Id, MainWindowTitle | ConvertTo-Json -Depth 3',
-  ].join(' ');
-  const output = await run('powershell.exe', ['-NoProfile', '-Command', script]);
-  const parsed = JSON.parse(output || '[]');
-  const rows = Array.isArray(parsed) ? parsed : [parsed];
-  return rows.map((row) => ({
-    name: row.ProcessName,
-    pid: row.Id,
-    windowTitle: row.MainWindowTitle,
-    running: true,
-  }));
-}
-
-async function listLinuxApps(): Promise<Array<Record<string, unknown>>> {
-  try {
-    const output = await run('wmctrl', ['-lx']);
-    return output.split(/\r?\n/)
-      .map((line) => line.trim())
-      .filter(Boolean)
-      .map((line) => {
-        const parts = line.split(/\s+/);
-        return {
-          windowId: parts[0],
-          desktop: parts[1],
-          host: parts[2],
-          className: parts[3],
-          windowTitle: parts.slice(4).join(' '),
-          running: true,
-        };
-      });
-  } catch {
-    const output = await run('ps', ['-eo', 'comm=']);
-    return [...new Set(output.split(/\r?\n/).map((name) => name.trim()).filter(Boolean))]
-      .slice(0, 200)
-      .map((name) => ({ name, running: true }));
-  }
-}
-
-async function listApps(): Promise<Array<Record<string, unknown>>> {
-  if (process.platform === 'darwin') {
-    return listMacApps();
-  }
-  if (process.platform === 'win32') {
-    return listWindowsApps();
-  }
-  return listLinuxApps();
-}
-
-async function macAccessibilityTree(app: string): Promise<SemanticElement[]> {
-  const escapedApp = app.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
-  const script = `
-on safeText(v)
-  try
-    return v as text
-  on error
-    return ""
-  end try
-end safeText
-
-on emitElement(e, depth, maxDepth, counter)
-  if depth > maxDepth then return {}
-  set rows to {}
-  try
-    set roleText to my safeText(role of e)
-  on error
-    set roleText to "element"
-  end try
-  try
-    set titleText to my safeText(title of e)
-  on error
-    set titleText to ""
-  end try
-  try
-    set valueText to my safeText(value of e)
-  on error
-    set valueText to ""
-  end try
-  try
-    set posValue to position of e
-    set sizeValue to size of e
-    set boundsText to ((item 1 of posValue) as text) & "," & ((item 2 of posValue) as text) & "," & ((item 1 of sizeValue) as text) & "," & ((item 2 of sizeValue) as text)
-  on error
-    set boundsText to ""
-  end try
-  set end of rows to ((counter as text) & tab & roleText & tab & titleText & tab & valueText & tab & boundsText)
-  if counter > ${MAX_APP_STATE_ELEMENTS} then return rows
-  try
-    repeat with childElement in UI elements of e
-      set childRows to my emitElement(childElement, depth + 1, maxDepth, counter + (count of rows))
-      set rows to rows & childRows
-      if (count of rows) > ${MAX_APP_STATE_ELEMENTS} then return rows
-    end repeat
-  end try
-  return rows
-end emitElement
-
-tell application "System Events"
-  if not (exists process "${escapedApp}") then error "App is not running: ${escapedApp}"
-  tell process "${escapedApp}"
-    set rows to {}
-    repeat with w in windows
-      set rows to rows & my emitElement(w, 0, 4, (count of rows) + 1)
-      if (count of rows) > ${MAX_APP_STATE_ELEMENTS} then exit repeat
-    end repeat
-    return rows
-  end tell
-end tell
-`;
-  const output = await run('osascript', ['-e', script], 10000);
-  return output.split(/\r?\n|, /)
-    .map((line) => line.trim())
-    .filter(Boolean)
-    .map((line, index) => {
-      const [rawIndex, role, title, value, boundsText] = line.split('\t');
-      const boundsParts = (boundsText || '').split(',').map((part) => Number.parseFloat(part));
-      const hasBounds = boundsParts.length === 4 && boundsParts.every(Number.isFinite);
-      return {
-        index: rawIndex || String(index + 1),
-        role: role || 'element',
-        title: title || undefined,
-        value: value || undefined,
-        bounds: hasBounds
-          ? { x: boundsParts[0], y: boundsParts[1], width: boundsParts[2], height: boundsParts[3] }
-          : undefined,
-      };
-    });
-}
-
-async function getAccessibilityTree(app: string): Promise<{ elements: SemanticElement[]; message?: string }> {
-  if (process.platform === 'darwin') {
-    try {
-      return { elements: await macAccessibilityTree(app) };
-    } catch (error) {
-      return { elements: [], message: error instanceof Error ? error.message : String(error) };
-    }
-  }
-
-  return {
-    elements: [],
-    message: 'Native accessibility tree capture is not implemented for this platform yet.',
-  };
-}
-
-async function getAppState(sessionId: string, app: string): Promise<SemanticAppState> {
-  if (!app) {
-    throw new Error('app is required.');
-  }
-  const helperState = await withHelperState(sessionId, (adapter) => adapter.getAppState({ sessionId, app }));
-  if (helperState) {
-    return helperState;
-  }
-
-  const screenshot = await captureScreenshot();
-  const tree = await getAccessibilityTree(app);
-  const state: SemanticAppState = {
-    stateId: semanticSessionStore.createStateId(),
-    app,
-    platform: process.platform,
-    screenshotDataUrl: screenshot.dataUrl,
-    displaySize: screenshot.size,
-    elements: tree.elements,
-    accessibilityTree: tree.elements,
-    message: tree.message,
-  };
-  return semanticSessionStore.save(sessionId, state);
-}
-
-async function targetFor(sessionId: string, app: string, stateId?: string): Promise<ExecutorTarget> {
-  const cached = semanticSessionStore.getState(sessionId, app, stateId);
-  return { displaySize: cached?.displaySize || (await captureScreenshot()).size };
-}
-
-export const computerSemanticsService = {
-  async callTool(name: string, input: Record<string, unknown>): Promise<unknown> {
-    const sessionId = readSessionId(input);
-    switch (name) {
-      case 'list_apps': {
-        const adapter = getHelperAdapter();
-        if (adapter) {
-          try {
-            return { apps: await adapter.listApps(), platform: process.platform };
-          } catch (error) {
-            if (!shouldFallbackFromHelper(error)) {
-              throw error;
-            }
-            console.warn('[ComputerSemantics] Falling back from helper:', error instanceof Error ? error.message : String(error));
-          }
-        }
-        return { apps: await listApps(), platform: process.platform };
-      }
-      case 'get_app_state':
-        return getAppState(sessionId, readString(input.app));
-      case 'click':
-      case 'click_element': {
-        const app = requireApp(input);
-        const helperState = await withHelperState(sessionId, (adapter) => adapter.clickElement({ ...input, sessionId, app }));
-        if (helperState) {
-          return helperState;
-        }
-        const stateId = readString(input.stateId) || undefined;
-        const point = getPoint(input, sessionId, app);
-        if (!point) {
-          throw new Error('click requires x/y or an element_index from computer_get_app_state.');
-        }
-        const target = await targetFor(sessionId, app, stateId);
-        const button = readButton(input.mouse_button ?? input.mouseButton);
-        const clickCount = readClickCount(input.click_count ?? input.clickCount);
-        for (let index = 0; index < clickCount; index += 1) {
-          await executor.click(target, button, point, false);
-        }
-        return getAppState(sessionId, app);
-      }
-      case 'drag': {
-        const app = requireApp(input);
-        const helperState = await withHelperState(sessionId, (adapter) => adapter.drag({ ...input, sessionId, app }));
-        if (helperState) {
-          return helperState;
-        }
-        const stateId = readString(input.stateId) || undefined;
-        const fromX = readNumber(input.from_x);
-        const fromY = readNumber(input.from_y);
-        const toX = readNumber(input.to_x);
-        const toY = readNumber(input.to_y);
-        if (fromX === undefined || fromY === undefined || toX === undefined || toY === undefined) {
-          throw new Error('drag requires from_x/from_y/to_x/to_y.');
-        }
-        await executor.drag(await targetFor(sessionId, app, stateId), { x: fromX, y: fromY }, { x: toX, y: toY }, readButton(input.mouse_button ?? input.mouseButton));
-        return getAppState(sessionId, app);
-      }
-      case 'scroll':
-      case 'scroll_element': {
-        const app = requireApp(input);
-        const helperState = await withHelperState(sessionId, (adapter) => adapter.scrollElement({ ...input, sessionId, app }));
-        if (helperState) {
-          return helperState;
-        }
-        const stateId = readString(input.stateId) || undefined;
-        const point = getPoint(input, sessionId, app);
-        if (!point) {
-          throw new Error('scroll requires x/y or an element_index from computer_get_app_state.');
-        }
-        await executor.scroll(await targetFor(sessionId, app, stateId), readDirection(input.direction), readNumber(input.pages) ?? 1, point);
-        return getAppState(sessionId, app);
-      }
-      case 'type_text': {
-        const app = requireApp(input);
-        const helperState = await withHelperState(sessionId, (adapter) => adapter.typeText({ ...input, sessionId, app }));
-        if (helperState) {
-          return helperState;
-        }
-        await executor.type(readString(input.text));
-        return getAppState(sessionId, app);
-      }
-      case 'press_key': {
-        const app = requireApp(input);
-        const helperState = await withHelperState(sessionId, (adapter) => adapter.pressKey({ ...input, sessionId, app }));
-        if (helperState) {
-          return helperState;
-        }
-        await executor.pressChord(readString(input.key));
-        return getAppState(sessionId, app);
-      }
-      case 'set_value': {
-        const app = requireApp(input);
-        const helperState = await withHelperState(sessionId, (adapter) => adapter.setValue({ ...input, sessionId, app }));
-        if (helperState) {
-          return helperState;
-        }
-        const stateId = readString(input.stateId) || undefined;
-        const point = getPoint(input, sessionId, app);
-        if (!point) {
-          throw new Error('set_value requires x/y or an element_index from computer_get_app_state.');
-        }
-        await executor.click(await targetFor(sessionId, app, stateId), 'left', point, false);
-        await executor.pressChord(process.platform === 'darwin' ? 'cmd+a' : 'ctrl+a');
-        await executor.type(readString(input.value));
-        return getAppState(sessionId, app);
-      }
-      case 'perform_secondary_action': {
-        const app = requireApp(input);
-        const helperState = await withHelperState(sessionId, (adapter) => adapter.performSecondaryAction({ ...input, sessionId, app }));
-        if (helperState) {
-          return helperState;
-        }
-        const stateId = readString(input.stateId) || undefined;
-        const point = getPoint(input, sessionId, app);
-        if (!point) {
-          throw new Error('perform_secondary_action requires x/y or an element_index from computer_get_app_state.');
-        }
-        await executor.click(await targetFor(sessionId, app, stateId), 'right', point, false);
-        return getAppState(sessionId, app);
-      }
-      default:
-        throw new Error(`Unknown semantic Computer Use tool: ${name}`);
-    }
-  },
-};