@cloudcli-ai/cloudcli 1.35.0 → 1.36.0

package/server/modules/computer-use/desktop-agent-relay.service.ts

@@ -1,158 +0,0 @@
-import { randomUUID } from 'node:crypto';
-
-import type { WebSocket } from 'ws';
-
-const RELAY_TIMEOUT_MS = Number.parseInt(process.env.CLOUDCLI_COMPUTER_USE_RELAY_TIMEOUT_MS || '60000', 10);
-const WS_OPEN = 1;
-
-type PendingRelay = {
-  resolve: (value: unknown) => void;
-  reject: (reason: Error) => void;
-  timer: ReturnType<typeof setTimeout>;
-};
-
-type ConnectedAgent = {
-  ws: WebSocket;
-  label: string;
-  registeredAt: string;
-};
-
-type RelayLifecycleHooks = {
-  canAcceptConnection?: () => boolean;
-  onFirstConnect?: () => void | Promise<void>;
-  onLastDisconnect?: () => void | Promise<void>;
-};
-
-const agents = new Map<WebSocket, ConnectedAgent>();
-const pending = new Map<string, PendingRelay>();
-let hooks: RelayLifecycleHooks = {};
-
-function rejectAllPending(reason: string): void {
-  for (const [callId, call] of pending.entries()) {
-    clearTimeout(call.timer);
-    call.reject(new Error(reason));
-    pending.delete(callId);
-  }
-}
-
-function pickAgent(): ConnectedAgent | undefined {
-  for (const agent of agents.values()) {
-    if (agent.ws.readyState === WS_OPEN) {
-      return agent;
-    }
-  }
-  return undefined;
-}
-
-/**
- * Cloud-side registry of linked desktop agents and the request/response relay
- * used to drive the user's real desktop. The hosted server never touches the OS
- * itself — it only forwards `computer_*` actions to a connected desktop agent
- * and awaits the screenshot it returns.
- */
-export const desktopAgentRelay = {
-  setHooks(next: RelayLifecycleHooks): void {
-    hooks = next;
-  },
-
-  register(ws: WebSocket, label = 'desktop-agent'): boolean {
-    if (hooks.canAcceptConnection && !hooks.canAcceptConnection()) {
-      console.log(`[DesktopAgent] Rejected (${label}); Computer Use is disabled.`);
-      try {
-        ws.close(1008, 'Computer Use is disabled in this environment.');
-      } catch {
-        // ignore close failures
-      }
-      return false;
-    }
-
-    const wasEmpty = pickAgent() === undefined;
-    agents.set(ws, { ws, label, registeredAt: new Date().toISOString() });
-    console.log(`[DesktopAgent] Registered (${label}); ${agents.size} connected.`);
-
-    ws.on('close', () => {
-      const wasRegistered = agents.delete(ws);
-      console.log(`[DesktopAgent] Disconnected (${label}); ${agents.size} remain.`);
-      if (wasRegistered && pickAgent() === undefined) {
-        rejectAllPending('Desktop agent disconnected.');
-        void hooks.onLastDisconnect?.();
-      }
-    });
-
-    if (wasEmpty) {
-      void hooks.onFirstConnect?.();
-    }
-    return true;
-  },
-
-  disconnectAll(reason = 'Desktop agent disconnected.'): void {
-    const hadAgent = pickAgent() !== undefined;
-    const sockets = [...agents.keys()];
-    agents.clear();
-    for (const ws of sockets) {
-      try {
-        ws.close(1008, reason);
-      } catch {
-        // ignore close failures
-      }
-    }
-    rejectAllPending(reason);
-    if (hadAgent) {
-      void hooks.onLastDisconnect?.();
-    }
-  },
-
-  /** Resolves a pending relay call with the desktop agent's reply. */
-  handleResult(id: string, result: unknown, error?: string): void {
-    const call = pending.get(id);
-    if (!call) {
-      return;
-    }
-    clearTimeout(call.timer);
-    pending.delete(id);
-    if (error) {
-      call.reject(new Error(error));
-    } else {
-      call.resolve(result);
-    }
-  },
-
-  isConnected(): boolean {
-    return pickAgent() !== undefined;
-  },
-
-  connectedCount(): number {
-    let count = 0;
-    for (const agent of agents.values()) {
-      if (agent.ws.readyState === WS_OPEN) {
-        count++;
-      }
-    }
-    return count;
-  },
-
-  async relay(type: string, params: Record<string, unknown>): Promise<unknown> {
-    const agent = pickAgent();
-    if (!agent) {
-      throw new Error(
-        'No desktop is linked. Open CloudCLI Desktop on this computer, connect the same account, and enable Computer Use.'
-      );
-    }
-
-    const id = randomUUID();
-    return new Promise<unknown>((resolve, reject) => {
-      const timer = setTimeout(() => {
-        pending.delete(id);
-        reject(new Error('Desktop agent did not respond in time.'));
-      }, RELAY_TIMEOUT_MS);
-      pending.set(id, { resolve, reject, timer });
-      try {
-        agent.ws.send(JSON.stringify({ kind: 'computer_relay', id, type, params }));
-      } catch (error) {
-        clearTimeout(timer);
-        pending.delete(id);
-        reject(error instanceof Error ? error : new Error('Failed to send to desktop agent.'));
-      }
-    });
-  },
-};

package/server/modules/computer-use/index.ts

@@ -1,2 +0,0 @@
-export { computerUseService } from '@/modules/computer-use/computer-use.service.js';
-export { desktopAgentRelay } from '@/modules/computer-use/desktop-agent-relay.service.js';

package/server/modules/computer-use/semantics/adapters/helper-semantic-adapter.ts

@@ -1,82 +0,0 @@
-import { SemanticHelperProcess } from '@/modules/computer-use/semantics/helpers/semantic-helper-process.js';
-import { resolveSemanticHelper } from '@/modules/computer-use/semantics/helpers/semantic-helper-resolver.js';
-import type { SemanticAdapter, SemanticAdapterCapabilities } from '@/modules/computer-use/semantics/adapters/semantic-adapter.js';
-import type { SemanticApp, SemanticAppState, SemanticToolInput } from '@/modules/computer-use/semantics/semantic-types.js';
-
-type HelperMethod =
-  | 'list_apps'
-  | 'get_app_state'
-  | 'click_element'
-  | 'perform_secondary_action'
-  | 'set_value'
-  | 'type_text'
-  | 'press_key'
-  | 'scroll_element'
-  | 'drag';
-
-export class HelperSemanticAdapter implements SemanticAdapter {
-  private helper: SemanticHelperProcess | null = null;
-
-  constructor(
-    private readonly platform: NodeJS.Platform,
-    private readonly arch: NodeJS.Architecture = process.arch,
-  ) {}
-
-  capabilities(): SemanticAdapterCapabilities {
-    return {
-      platform: this.platform,
-      appDiscovery: true,
-      accessibilityTree: true,
-      nativeElementActions: true,
-      nativeValueSetting: true,
-      targetedInput: true,
-    };
-  }
-
-  async listApps(): Promise<SemanticApp[]> {
-    return await this.request('list_apps', {}) as SemanticApp[];
-  }
-
-  async getAppState(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('get_app_state', input) as SemanticAppState;
-  }
-
-  async clickElement(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('click_element', input) as SemanticAppState;
-  }
-
-  async performSecondaryAction(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('perform_secondary_action', input) as SemanticAppState;
-  }
-
-  async setValue(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('set_value', input) as SemanticAppState;
-  }
-
-  async typeText(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('type_text', input) as SemanticAppState;
-  }
-
-  async pressKey(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('press_key', input) as SemanticAppState;
-  }
-
-  async scrollElement(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('scroll_element', input) as SemanticAppState;
-  }
-
-  async drag(input: SemanticToolInput): Promise<SemanticAppState> {
-    return await this.request('drag', input) as SemanticAppState;
-  }
-
-  private async request(method: HelperMethod, params: Record<string, unknown>): Promise<unknown> {
-    if (!this.helper) {
-      const resolution = resolveSemanticHelper(this.platform, this.arch);
-      if (!resolution.available || !resolution.path) {
-        throw new Error(resolution.reason || `Semantic helper is unavailable for ${this.platform}-${this.arch}.`);
-      }
-      this.helper = new SemanticHelperProcess(resolution.path);
-    }
-    return this.helper.request(method, params);
-  }
-}

package/server/modules/computer-use/semantics/adapters/macos/macos-semantic-adapter.ts

@@ -1,5 +0,0 @@
-import { HelperSemanticAdapter } from '@/modules/computer-use/semantics/adapters/helper-semantic-adapter.js';
-
-export function createMacOsSemanticAdapter(): HelperSemanticAdapter {
-  return new HelperSemanticAdapter('darwin');
-}

package/server/modules/computer-use/semantics/adapters/semantic-adapter.ts

@@ -1,23 +0,0 @@
-import type { SemanticApp, SemanticAppState, SemanticToolInput } from '@/modules/computer-use/semantics/semantic-types.js';
-
-export type SemanticAdapterCapabilities = {
-  platform: NodeJS.Platform;
-  appDiscovery: boolean;
-  accessibilityTree: boolean;
-  nativeElementActions: boolean;
-  nativeValueSetting: boolean;
-  targetedInput: boolean;
-};
-
-export type SemanticAdapter = {
-  capabilities(): SemanticAdapterCapabilities;
-  listApps(): Promise<SemanticApp[]>;
-  getAppState(input: SemanticToolInput): Promise<SemanticAppState>;
-  clickElement(input: SemanticToolInput): Promise<SemanticAppState>;
-  performSecondaryAction(input: SemanticToolInput): Promise<SemanticAppState>;
-  setValue(input: SemanticToolInput): Promise<SemanticAppState>;
-  typeText(input: SemanticToolInput): Promise<SemanticAppState>;
-  pressKey(input: SemanticToolInput): Promise<SemanticAppState>;
-  scrollElement(input: SemanticToolInput): Promise<SemanticAppState>;
-  drag(input: SemanticToolInput): Promise<SemanticAppState>;
-};

package/server/modules/computer-use/semantics/adapters/windows/windows-semantic-adapter.ts

@@ -1,5 +0,0 @@
-import { HelperSemanticAdapter } from '@/modules/computer-use/semantics/adapters/helper-semantic-adapter.js';
-
-export function createWindowsSemanticAdapter(): HelperSemanticAdapter {
-  return new HelperSemanticAdapter('win32');
-}