@kortix/agent-tunnel 0.1.0

package/src/agent/capabilities/desktop/types.ts

@@ -0,0 +1,201 @@
+export interface ScreenshotOptions {
+  region?: { x: number; y: number; width: number; height: number };
+  windowId?: number;
+}
+
+export interface ScreenshotResult {
+  image: string;
+  width: number;
+  height: number;
+  format: 'png' | 'jpeg';
+}
+
+export interface MouseClickOptions {
+  x: number;
+  y: number;
+  button?: 'left' | 'right' | 'middle';
+  clicks?: number;
+  modifiers?: string[];
+}
+
+export interface MouseMoveOptions {
+  x: number;
+  y: number;
+}
+
+export interface MouseDragOptions {
+  fromX: number;
+  fromY: number;
+  toX: number;
+  toY: number;
+  button?: 'left' | 'right';
+}
+
+export interface MouseScrollOptions {
+  x: number;
+  y: number;
+  deltaX?: number;
+  deltaY?: number;
+}
+
+export interface MousePosition {
+  x: number;
+  y: number;
+}
+
+export interface KeyboardTypeOptions {
+  text: string;
+  delay?: number;
+}
+
+export interface KeyboardKeyOptions {
+  keys: string[];
+}
+
+export interface WindowInfo {
+  id: number;
+  app: string;
+  title: string;
+  bounds: WindowBounds;
+  minimized: boolean;
+}
+
+export interface WindowBounds {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export interface AppInfo {
+  name: string;
+  pid: number;
+  bundleId?: string;
+}
+
+export interface ScreenInfo {
+  width: number;
+  height: number;
+  scaleFactor: number;
+}
+
+export interface AXElement {
+  id: string;
+  role: string;
+  subrole?: string;
+  title: string;
+  value: string;
+  description: string;
+  label?: string;
+  placeholder?: string;
+  identifier?: string;
+  bounds: { x: number; y: number; width: number; height: number };
+  children: AXElement[];
+  actions: string[];
+  enabled: boolean;
+  focused: boolean;
+}
+
+export interface AXTreeOptions {
+  pid?: number;
+  maxDepth?: number;
+  roles?: string[];
+}
+
+export interface AXTreeResult {
+  root: AXElement;
+  elementCount: number;
+}
+
+export interface AXActionOptions {
+  elementId: string;
+  action: string;
+  pid?: number;
+}
+
+export interface AXActionResult {
+  ok: boolean;
+  action: string;
+  elementId: string;
+  before: { focused: boolean; value: string };
+  after: { focused: boolean; value: string };
+  stateChanged: boolean;
+  role: string;
+  title: string;
+}
+
+export interface AXSetValueOptions {
+  elementId: string;
+  value: string;
+  pid?: number;
+}
+
+export interface AXSetValueResult {
+  ok: boolean;
+  elementId: string;
+  requestedValue: string;
+  actualValue: string;
+  error?: string;
+}
+
+export interface AXFocusOptions {
+  elementId: string;
+  pid?: number;
+}
+
+export interface AXFocusResult {
+  ok: boolean;
+  elementId: string;
+  role: string;
+  title: string;
+  before: { focused: boolean };
+  after: { focused: boolean };
+  error?: string;
+}
+
+export interface AXSearchOptions {
+  query: string;
+  role?: string;
+  pid?: number;
+  maxResults?: number;
+}
+
+export interface AXSearchResult {
+  elements: AXElement[];
+}
+
+export interface DesktopDriver {
+  screenshot(options: ScreenshotOptions): Promise<ScreenshotResult>;
+
+  mouseClick(options: MouseClickOptions): Promise<void>;
+  mouseMove(options: MouseMoveOptions): Promise<void>;
+  mouseDrag(options: MouseDragOptions): Promise<void>;
+  mouseScroll(options: MouseScrollOptions): Promise<void>;
+  mousePosition(): Promise<MousePosition>;
+
+  keyboardType(options: KeyboardTypeOptions): Promise<void>;
+  keyboardKey(options: KeyboardKeyOptions): Promise<void>;
+
+  windowList(): Promise<WindowInfo[]>;
+  windowFocus(windowId: number): Promise<void>;
+  windowResize(windowId: number, bounds: Partial<WindowBounds>): Promise<void>;
+  windowClose(windowId: number): Promise<void>;
+  windowMinimize(windowId: number): Promise<void>;
+
+  appLaunch(name: string): Promise<void>;
+  appQuit(name: string): Promise<void>;
+  appList(): Promise<AppInfo[]>;
+
+  clipboardRead(): Promise<string>;
+  clipboardWrite(text: string): Promise<void>;
+
+  screenInfo(): Promise<ScreenInfo>;
+
+  cursorImage(radius?: number): Promise<ScreenshotResult>;
+
+  axTree(options: AXTreeOptions): Promise<AXTreeResult>;
+  axAction(options: AXActionOptions): Promise<AXActionResult>;
+  axSetValue(options: AXSetValueOptions): Promise<AXSetValueResult>;
+  axFocus(options: AXFocusOptions): Promise<AXFocusResult>;
+  axSearch(options: AXSearchOptions): Promise<AXSearchResult>;
+}

package/src/agent/capabilities/desktop/windows-driver.ts

@@ -0,0 +1,220 @@
+import { execHelper } from './csharp-helper';
+import type {
+  DesktopDriver,
+  ScreenshotOptions,
+  ScreenshotResult,
+  MouseClickOptions,
+  MouseMoveOptions,
+  MouseDragOptions,
+  MouseScrollOptions,
+  MousePosition,
+  KeyboardTypeOptions,
+  KeyboardKeyOptions,
+  WindowInfo,
+  WindowBounds,
+  AppInfo,
+  ScreenInfo,
+  AXTreeOptions,
+  AXTreeResult,
+  AXActionOptions,
+  AXActionResult,
+  AXSetValueOptions,
+  AXSetValueResult,
+  AXFocusOptions,
+  AXFocusResult,
+  AXSearchOptions,
+  AXSearchResult,
+} from './types';
+
+export class WindowsDriver implements DesktopDriver {
+  async screenshot(_options: ScreenshotOptions): Promise<ScreenshotResult> {
+    const res = await execHelper({ action: 'screenshot' });
+    return {
+      image: res.image!,
+      width: res.width!,
+      height: res.height!,
+      format: (res.format as 'png' | 'jpeg') || 'jpeg',
+    };
+  }
+
+  async mouseClick(options: MouseClickOptions): Promise<void> {
+    await execHelper({
+      action: 'click',
+      x: options.x,
+      y: options.y,
+      button: options.button || 'left',
+      clicks: options.clicks || 1,
+    });
+  }
+
+  async mouseMove(options: MouseMoveOptions): Promise<void> {
+    await execHelper({ action: 'move', x: options.x, y: options.y });
+  }
+
+  async mouseDrag(options: MouseDragOptions): Promise<void> {
+    await execHelper({
+      action: 'drag',
+      x: options.fromX,
+      y: options.fromY,
+      toX: options.toX,
+      toY: options.toY,
+      button: options.button || 'left',
+    });
+  }
+
+  async mouseScroll(options: MouseScrollOptions): Promise<void> {
+    await execHelper({
+      action: 'scroll',
+      x: options.x,
+      y: options.y,
+      deltaX: options.deltaX || 0,
+      deltaY: options.deltaY || 0,
+    });
+  }
+
+  async mousePosition(): Promise<MousePosition> {
+    const res = await execHelper({ action: 'position' });
+    return { x: res.x!, y: res.y! };
+  }
+
+  async keyboardType(options: KeyboardTypeOptions): Promise<void> {
+    await execHelper({ action: 'type', text: options.text });
+    if (options.delay) {
+      await new Promise(r => setTimeout(r, options.delay));
+    }
+  }
+
+  async keyboardKey(options: KeyboardKeyOptions): Promise<void> {
+    await execHelper({ action: 'key', keys: options.keys });
+  }
+
+  async windowList(): Promise<WindowInfo[]> {
+    const res = await execHelper({ action: 'window_list' });
+    return (res.windows || []) as WindowInfo[];
+  }
+
+  async windowFocus(windowId: number): Promise<void> {
+    await execHelper({ action: 'window_focus', windowId });
+  }
+
+  async windowResize(windowId: number, bounds: Partial<WindowBounds>): Promise<void> {
+    const windows = await this.windowList();
+    const win = windows.find(w => w.id === windowId);
+    if (!win) throw new Error(`Window ${windowId} not found`);
+
+    await execHelper({
+      action: 'window_resize',
+      windowId,
+      x: bounds.x ?? win.bounds.x,
+      y: bounds.y ?? win.bounds.y,
+      width: bounds.width ?? win.bounds.width,
+      height: bounds.height ?? win.bounds.height,
+    });
+  }
+
+  async windowClose(windowId: number): Promise<void> {
+    await execHelper({ action: 'window_close', windowId });
+  }
+
+  async windowMinimize(windowId: number): Promise<void> {
+    await execHelper({ action: 'window_minimize', windowId });
+  }
+
+  async appLaunch(name: string): Promise<void> {
+    await execHelper({ action: 'app_launch', name });
+  }
+
+  async appQuit(name: string): Promise<void> {
+    await execHelper({ action: 'app_quit', name });
+  }
+
+  async appList(): Promise<AppInfo[]> {
+    const res = await execHelper({ action: 'app_list' });
+    return (res.apps || []) as AppInfo[];
+  }
+
+  async clipboardRead(): Promise<string> {
+    const res = await execHelper({ action: 'clipboard_read' });
+    return res.text || '';
+  }
+
+  async clipboardWrite(text: string): Promise<void> {
+    await execHelper({ action: 'clipboard_write', text });
+  }
+
+  async screenInfo(): Promise<ScreenInfo> {
+    const res = await execHelper({ action: 'screen_info' });
+    return {
+      width: res.width!,
+      height: res.height!,
+      scaleFactor: res.scaleFactor!,
+    };
+  }
+
+  async cursorImage(radius: number = 50): Promise<ScreenshotResult> {
+    // Windows: take full screenshot and crop around cursor
+    const pos = await this.mousePosition();
+    // For now, return full screenshot — region capture can be added later
+    return this.screenshot({
+      region: {
+        x: Math.max(0, Math.round(pos.x - radius)),
+        y: Math.max(0, Math.round(pos.y - radius)),
+        width: radius * 2,
+        height: radius * 2,
+      },
+    });
+  }
+
+  async axTree(options: AXTreeOptions): Promise<AXTreeResult> {
+    const res = await execHelper({
+      action: 'ax_tree',
+      pid: options.pid,
+      maxDepth: options.maxDepth ?? 8,
+      roles: options.roles,
+    });
+    return {
+      root: res.root,
+      elementCount: res.elementCount!,
+    };
+  }
+
+  async axAction(options: AXActionOptions): Promise<AXActionResult> {
+    const res = await execHelper({
+      action: 'ax_action',
+      elementId: options.elementId,
+      action_name: options.action,
+      pid: options.pid,
+    });
+    return res as unknown as AXActionResult;
+  }
+
+  async axSetValue(options: AXSetValueOptions): Promise<AXSetValueResult> {
+    const res = await execHelper({
+      action: 'ax_set_value',
+      elementId: options.elementId,
+      value: options.value,
+      pid: options.pid,
+    });
+    return res as unknown as AXSetValueResult;
+  }
+
+  async axFocus(options: AXFocusOptions): Promise<AXFocusResult> {
+    const res = await execHelper({
+      action: 'ax_focus',
+      elementId: options.elementId,
+      pid: options.pid,
+    });
+    return res as unknown as AXFocusResult;
+  }
+
+  async axSearch(options: AXSearchOptions): Promise<AXSearchResult> {
+    const res = await execHelper({
+      action: 'ax_search',
+      query: options.query,
+      role: options.role,
+      pid: options.pid,
+      maxResults: options.maxResults ?? 20,
+    });
+    return { elements: res.elements || [] };
+  }
+}

package/src/agent/capabilities/desktop.ts

@@ -0,0 +1,196 @@
+import { platform } from 'os';
+import type { Capability, RpcHandler } from './index';
+import type { DesktopDriver } from './desktop/types';
+
+function createDriver(): DesktopDriver {
+  switch (platform()) {
+    case 'darwin': {
+      const { MacOSDriver } = require('./desktop/macos-driver');
+      return new MacOSDriver();
+    }
+    case 'win32': {
+      const { WindowsDriver } = require('./desktop/windows-driver');
+      return new WindowsDriver();
+    }
+    default: {
+      const { LinuxDriver } = require('./desktop/linux-driver');
+      return new LinuxDriver();
+    }
+  }
+}
+
+export function createDesktopCapability(): Capability {
+  const driver = createDriver();
+  const methods = new Map<string, RpcHandler>();
+
+  methods.set('desktop.screenshot', async (params) => {
+    return driver.screenshot({
+      region: params.region as any,
+      windowId: params.windowId as number | undefined,
+    });
+  });
+
+  methods.set('desktop.mouse.click', async (params) => {
+    await driver.mouseClick({
+      x: params.x as number,
+      y: params.y as number,
+      button: params.button as any,
+      clicks: params.clicks as number | undefined,
+      modifiers: params.modifiers as string[] | undefined,
+    });
+    return { ok: true };
+  });
+
+  methods.set('desktop.mouse.move', async (params) => {
+    await driver.mouseMove({
+      x: params.x as number,
+      y: params.y as number,
+    });
+    return { ok: true };
+  });
+
+  methods.set('desktop.mouse.drag', async (params) => {
+    await driver.mouseDrag({
+      fromX: params.fromX as number,
+      fromY: params.fromY as number,
+      toX: params.toX as number,
+      toY: params.toY as number,
+      button: params.button as any,
+    });
+    return { ok: true };
+  });
+
+  methods.set('desktop.mouse.scroll', async (params) => {
+    await driver.mouseScroll({
+      x: params.x as number,
+      y: params.y as number,
+      deltaX: params.deltaX as number | undefined,
+      deltaY: params.deltaY as number | undefined,
+    });
+    return { ok: true };
+  });
+
+  methods.set('desktop.mouse.position', async () => {
+    return driver.mousePosition();
+  });
+
+  methods.set('desktop.keyboard.type', async (params) => {
+    await driver.keyboardType({
+      text: params.text as string,
+      delay: params.delay as number | undefined,
+    });
+    return { ok: true };
+  });
+
+  methods.set('desktop.keyboard.key', async (params) => {
+    await driver.keyboardKey({
+      keys: params.keys as string[],
+    });
+    return { ok: true };
+  });
+
+  methods.set('desktop.window.list', async () => {
+    return { windows: await driver.windowList() };
+  });
+
+  methods.set('desktop.window.focus', async (params) => {
+    await driver.windowFocus(params.windowId as number);
+    return { ok: true };
+  });
+
+  methods.set('desktop.window.resize', async (params) => {
+    await driver.windowResize(params.windowId as number, {
+      x: params.x as number | undefined,
+      y: params.y as number | undefined,
+      width: params.width as number | undefined,
+      height: params.height as number | undefined,
+    });
+    return { ok: true };
+  });
+
+  methods.set('desktop.window.close', async (params) => {
+    await driver.windowClose(params.windowId as number);
+    return { ok: true };
+  });
+
+  methods.set('desktop.window.minimize', async (params) => {
+    await driver.windowMinimize(params.windowId as number);
+    return { ok: true };
+  });
+
+  methods.set('desktop.app.launch', async (params) => {
+    await driver.appLaunch(params.app as string);
+    return { ok: true };
+  });
+
+  methods.set('desktop.app.quit', async (params) => {
+    await driver.appQuit(params.app as string);
+    return { ok: true };
+  });
+
+  methods.set('desktop.app.list', async () => {
+    return { apps: await driver.appList() };
+  });
+
+  methods.set('desktop.clipboard.read', async () => {
+    return { text: await driver.clipboardRead() };
+  });
+
+  methods.set('desktop.clipboard.write', async (params) => {
+    await driver.clipboardWrite(params.text as string);
+    return { ok: true };
+  });
+
+  methods.set('desktop.screen.info', async () => {
+    return driver.screenInfo();
+  });
+
+  methods.set('desktop.cursor.image', async (params) => {
+    return driver.cursorImage(params.radius as number | undefined);
+  });
+
+  methods.set('desktop.ax.tree', async (params) => {
+    return driver.axTree({
+      pid: params.pid as number | undefined,
+      maxDepth: params.maxDepth as number | undefined,
+      roles: params.roles as string[] | undefined,
+    });
+  });
+
+  methods.set('desktop.ax.action', async (params) => {
+    return driver.axAction({
+      elementId: params.elementId as string,
+      action: params.action as string,
+      pid: params.pid as number | undefined,
+    });
+  });
+
+  methods.set('desktop.ax.set_value', async (params) => {
+    return driver.axSetValue({
+      elementId: params.elementId as string,
+      value: params.value as string,
+      pid: params.pid as number | undefined,
+    });
+  });
+
+  methods.set('desktop.ax.focus', async (params) => {
+    return driver.axFocus({
+      elementId: params.elementId as string,
+      pid: params.pid as number | undefined,
+    });
+  });
+
+  methods.set('desktop.ax.search', async (params) => {
+    return driver.axSearch({
+      query: params.query as string,
+      role: params.role as string | undefined,
+      pid: params.pid as number | undefined,
+      maxResults: params.maxResults as number | undefined,
+    });
+  });
+
+  return {
+    name: 'desktop',
+    methods,
+  };
+}