@xagent-ai/cli 1.0.1

package/src/gui-subagent/agent/index.ts

@@ -0,0 +1,5 @@
+/**
+ * Agent Index for gui-subagent
+ */
+
+export { GUIAgent, type GUIAgentConfig, type GUIAgentData, type Conversation } from './gui-agent.js';

package/src/gui-subagent/index.ts

@@ -0,0 +1,139 @@
+/**
+ * GUI Subagent - Computer Automation for xagent
+ *
+ * A powerful automation subagent that can:
+ * - Control the entire desktop via computer actions (ComputerOperator)
+ * - Perform clicks, typing, scrolling, drag operations
+ * - Execute complex automation workflows
+ *
+ * Based on UI-TARS architecture
+ */
+
+export * from './types/index.js';
+export * from './operator/index.js';
+export * from './agent/index.js';
+
+import { ComputerOperator, type ComputerOperatorOptions } from './operator/computer-operator.js';
+import { GUIAgent, type GUIAgentConfig, type GUIAgentData, type Conversation, GUIAgentStatus } from './agent/gui-agent.js';
+import type { Operator } from './operator/base-operator.js';
+import { getCancellationManager } from '../cancellation.js';
+
+/**
+ * GUI Subagent configuration
+ */
+export interface GUISubAgentConfig {
+  model?: string;
+  modelBaseUrl?: string;
+  modelApiKey?: string;
+  headless?: boolean;
+  loopIntervalInMs?: number;
+  maxLoopCount?: number;
+  showAIDebugInfo?: boolean;
+}
+
+/**
+ * Default configuration values (aligned with UI-TARS)
+ */
+export const DEFAULT_GUI_CONFIG: Required<GUISubAgentConfig> = {
+  model: 'gpt-4o',
+  modelBaseUrl: '',
+  modelApiKey: '',
+  headless: false,
+  loopIntervalInMs: 0,
+  maxLoopCount: 100,
+  showAIDebugInfo: false,
+};
+
+/**
+ * Create a GUI subagent with the specified configuration
+ */
+export async function createGUISubAgent<T extends Operator>(
+  config: GUISubAgentConfig = {},
+  operator?: T
+): Promise<GUIAgent<T>> {
+  const mergedConfig = { ...DEFAULT_GUI_CONFIG, ...config };
+
+  // Default to ComputerOperator for initial screenshot
+  // The actual operator type will be determined by LLM in run()
+  const agentOperator = operator ?? new ComputerOperator({
+    config: {
+      headless: mergedConfig.headless,
+    },
+  }) as unknown as T;
+
+  // Create AbortController for cancellation support
+  const abortController = new AbortController();
+
+  // Listen to cancellationManager for ESC key
+  const cancellationManager = getCancellationManager();
+  const cancelHandler = () => {
+    abortController.abort();
+  };
+  cancellationManager.on('cancelled', cancelHandler);
+
+  const agentConfig: GUIAgentConfig<T> = {
+    operator: agentOperator,
+    model: mergedConfig.model,
+    modelBaseUrl: mergedConfig.modelBaseUrl,
+    modelApiKey: mergedConfig.modelApiKey,
+    loopIntervalInMs: mergedConfig.loopIntervalInMs,
+    maxLoopCount: mergedConfig.maxLoopCount,
+    showAIDebugInfo: mergedConfig.showAIDebugInfo,
+    signal: abortController.signal,
+  };
+
+  const agent = new GUIAgent<T>(agentConfig);
+
+  // Store cancel handler for cleanup
+  (agent as any)._cancelHandler = cancelHandler;
+  (agent as any)._cancellationManager = cancellationManager;
+
+  await agent.initialize();
+
+  return agent;
+}
+
+/**
+ * Create a GUI Agent with a specific operator
+ */
+export async function createGUIAgent<T extends Operator>(
+  operator: T,
+  config?: Partial<GUIAgentConfig<T>>
+): Promise<GUIAgent<T>> {
+  // Create AbortController for cancellation support if not provided
+  const abortController = config?.signal ? undefined : new AbortController();
+
+  // Listen to cancellationManager for ESC key if no signal provided
+  let cancelHandler: (() => void) | undefined;
+  if (!config?.signal) {
+    const cancellationManager = getCancellationManager();
+    cancelHandler = () => {
+      abortController?.abort();
+    };
+    cancellationManager.on('cancelled', cancelHandler);
+  }
+
+  const agent = new GUIAgent<T>({
+    operator,
+    ...config,
+    signal: config?.signal ?? abortController?.signal,
+  });
+
+  // Store cancel handler for cleanup
+  if (cancelHandler) {
+    const cancellationManager = getCancellationManager();
+    (agent as any)._cancelHandler = cancelHandler;
+    (agent as any)._cancellationManager = cancellationManager;
+  }
+
+  await agent.initialize();
+  return agent;
+}
+
+export { ComputerOperator, GUIAgent, GUIAgentStatus };
+export type {
+  ComputerOperatorOptions,
+  GUIAgentConfig,
+  GUIAgentData,
+  Conversation,
+};

package/src/gui-subagent/operator/base-operator.ts

@@ -0,0 +1,246 @@
+/**
+ * Base Operator for gui-subagent
+ * Abstract base class for browser/desktop automation operators
+ * Based on UI-TARS architecture (@ui-tars/sdk/core)
+ *
+ * This implementation is aligned with @ui-tars/sdk/core.ts Operator class
+ */
+
+import type {
+  ScreenContext,
+  ScreenshotOutput,
+  ExecuteParams,
+  ExecuteOutput,
+} from '../types/operator.js';
+
+/**
+ * Operator manual configuration for AI agents
+ * Aligned with @ui-tars/sdk/core.ts Operator.MANUAL
+ */
+export interface OperatorManual {
+  ACTION_SPACES: string[];
+  KEY_SPACE?: {
+    [key: string]: string;
+  };
+  KEY_DESCRIPTION?: {
+    [key: string]: string;
+  };
+}
+
+/**
+ * Abstract base Operator class
+ * Provides common functionality for all operators (browser, computer, etc.)
+ */
+export abstract class Operator {
+  protected _initialized = false;
+  protected _initializing = false;
+  protected _initPromise: Promise<void> | null = null;
+
+  constructor() {}
+
+  /**
+   * Static manual configuration for AI agents
+   * Override this in subclasses to provide custom action spaces
+   */
+  static get MANUAL(): OperatorManual {
+    return {
+      ACTION_SPACES: [
+        `click(start_box='[x1, y1, x2, y2]')`,
+        `left_double(start_box='[x1, y1, x2, y2]')`,
+        `right_single(start_box='[x1, y1, x2, y2]')`,
+        `drag(start_box='[x1, y1, x2, y2]', end_box='[x3, y3, x4, y4]')`,
+        `hotkey(key='')`,
+        `type(content='')`,
+        `scroll(start_box='[x1, y1, x2, y2]', direction='down or up or right or left')`,
+        `wait()`,
+        `finished()`,
+        `call_user()`,
+      ],
+    };
+  }
+
+  /**
+   * Get manual configuration from instance
+   */
+  protected get manual(): OperatorManual {
+    return (this.constructor as typeof Operator).MANUAL;
+  }
+
+  /**
+   * Initialize the operator
+   * Must be called before any other operations
+   */
+  async doInitialize(): Promise<void> {
+    if (this._initialized) {
+      return;
+    }
+
+    if (this._initializing && this._initPromise) {
+      return this._initPromise;
+    }
+
+    this._initializing = true;
+    const initPromise = (async () => {
+      try {
+        await this.initialize();
+        this._initialized = true;
+      } finally {
+        this._initializing = false;
+      }
+    })();
+
+    this._initPromise = initPromise;
+    return initPromise;
+  }
+
+  /**
+   * Internal initialize method - implement in subclass
+   */
+  protected abstract initialize(): Promise<void>;
+
+  /**
+   * Ensure operator is initialized
+   */
+  protected async ensureInitialized(): Promise<void> {
+    if (!this._initialized && !this._initializing) {
+      await this.doInitialize();
+    } else if (this._initializing && this._initPromise) {
+      await this._initPromise;
+    }
+  }
+
+  /**
+   * Get supported action types
+   */
+  abstract getSupportedActions(): string[];
+
+  /**
+   * Get screen context (width, height, scaleFactor)
+   */
+  async getScreenContext(): Promise<ScreenContext> {
+    await this.ensureInitialized();
+    return this.screenContext();
+  }
+
+  /**
+   * Internal screen context - implement in subclass
+   */
+  protected abstract screenContext(): ScreenContext;
+
+  /**
+   * Take a screenshot
+   * Returns base64 encoded image
+   */
+  async doScreenshot(): Promise<ScreenshotOutput> {
+    try {
+      await this.ensureInitialized();
+      return await this.screenshot();
+    } catch (error) {
+      return {
+        status: 'failed',
+        errorMessage: error instanceof Error ? error.message : 'Unknown error',
+      };
+    }
+  }
+
+  /**
+   * Internal screenshot method - implement in subclass
+   */
+  protected abstract screenshot(): Promise<ScreenshotOutput>;
+
+  /**
+   * Execute an action based on parsed prediction
+   * Aligned with @ui-tars/sdk/core.ts Operator.execute()
+   */
+  async doExecute(params: ExecuteParams): Promise<ExecuteOutput> {
+    try {
+      await this.ensureInitialized();
+      return await this.execute(params);
+    } catch (error) {
+      return {
+        status: 'failed',
+        errorMessage: error instanceof Error ? error.message : 'Unknown error',
+      };
+    }
+  }
+
+  /**
+   * Internal execute method - implement in subclass
+   */
+  protected abstract execute(params: ExecuteParams): Promise<ExecuteOutput>;
+
+  /**
+   * Cleanup resources
+   */
+  abstract cleanup(): Promise<void>;
+
+  /**
+   * Destroy operator instance
+   */
+  abstract destroyInstance(): Promise<void>;
+
+  /**
+   * Check if operator is initialized
+   */
+  isInitialized(): boolean {
+    return this._initialized;
+  }
+}
+
+/**
+ * Parse box string to screen coordinates
+ * Aligned with @ui-tars/sdk/core.ts parseBoxToScreenCoords()
+ */
+export function parseBoxToScreenCoords(params: {
+  boxStr: string;
+  screenWidth: number;
+  screenHeight: number;
+}): { x: number; y: number } {
+  const { boxStr, screenWidth, screenHeight } = params;
+
+  if (!boxStr) {
+    return { x: 0, y: 0 };
+  }
+
+  // Support multiple formats:
+  // [x1, y1, x2, y2], (x1,y1,x2,y2), "[x1, y1, x2, y2]", etc.
+  const numbers = boxStr
+    .replace(/[()[\]]/g, '')
+    .split(',')
+    .filter((n) => n.trim())
+    .map((n) => parseFloat(n.trim()));
+
+  if (numbers.length < 2 || numbers.some((n) => Number.isNaN(n))) {
+    return { x: 0, y: 0 };
+  }
+
+  // Calculate center of the box
+  const x1 = numbers[0];
+  const y1 = numbers[1];
+  const x2 = numbers.length > 2 ? numbers[2] : x1;
+  const y2 = numbers.length > 3 ? numbers[3] : y1;
+
+  // Normalize to screen coordinates
+  let normalizedX: number;
+  let normalizedY: number;
+
+  // Check if coordinates are normalized (0-1 range) or absolute pixels (> 1)
+  if (x1 >= 0 && x1 <= 1 && y1 >= 0 && y1 <= 1) {
+    // Already normalized coordinates (0-1 range)
+    normalizedX = (x1 + x2) / 2;
+    normalizedY = (y1 + y2) / 2;
+  } else if (x1 > 1 || y1 > 1) {
+    // Absolute pixel coordinates
+    normalizedX = x1 / screenWidth;
+    normalizedY = y1 / screenHeight;
+  } else {
+    // Fallback: treat as normalized
+    normalizedX = x1;
+    normalizedY = y1;
+  }
+
+  return {
+    x: Math.round(normalizedX * screenWidth),
+    y: Math.round(normalizedY * screenHeight),
+  };
+}