@ui-tars-test/shared 0.3.2

package/src/types/actions.ts

@@ -0,0 +1,382 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+export type Factors = [number, number];
+
+/**
+ * Coordinate data structure
+ * - Supports pixel coordinates (raw)
+ * - Supports normalized coordinates (normalized)
+ */
+export interface Coordinates {
+  raw?: { x: number; y: number }; // Raw pixels
+  normalized?: { x: number; y: number }; // Normalized coordinates (0–1)
+  referenceBox?: { x1: number; y1: number; x2: number; y2: number };
+  referenceSystem?: 'screen' | 'window' | 'browserPage' | string; // Coordinate reference system
+}
+
+/**
+ * Standard structure for GUI Actions
+ */
+export interface BaseAction<
+  T extends string = string,
+  I extends Record<string, any> = Record<string, any>,
+> {
+  type: T; // Action type (e.g., "click", "key", "swipe")
+  inputs: I; // Parameters required for the action
+  meta?: {
+    toolHint?: string; // Suggested execution tool (xdotool / adb / pyautogui etc.)
+    comment?: string; // Notes / Debug information
+  };
+}
+
+// ---------- ScreenShot Action ----------
+
+/**
+ * ScreenShot action
+ */
+export type ScreenShotAction = BaseAction<
+  'screenshot',
+  {
+    start?: Coordinates;
+    end?: Coordinates;
+  }
+>;
+
+// ---------- Mouse Actions ----------
+
+/**
+ * Click action with coordinates
+ */
+export type ClickAction = BaseAction<
+  'click',
+  {
+    point: Coordinates;
+  }
+>;
+
+/**
+ * Right click action with coordinates
+ */
+export type RightClickAction = BaseAction<
+  'right_click',
+  {
+    point: Coordinates;
+  }
+>;
+
+/**
+ * Double click action with coordinates
+ */
+export type DoubleClickAction = BaseAction<
+  'double_click',
+  {
+    point: Coordinates;
+  }
+>;
+
+/**
+ * Middle click action with coordinates
+ */
+export type MiddleClickAction = BaseAction<
+  'middle_click',
+  {
+    point: Coordinates;
+  }
+>;
+
+/**
+ * Mouse down action
+ */
+export type MouseDownAction = BaseAction<
+  'mouse_down',
+  {
+    point?: Coordinates; // Mouse down position. If not specified, default to execute on the current mouse position.
+    button?: 'left' | 'right'; // Down button. Default to left.
+  }
+>;
+
+/**
+ * Mouse up action
+ */
+export type MouseUpAction = BaseAction<
+  'mouse_up',
+  {
+    point?: Coordinates; // Mouse up position. If not specified, default to execute on the current mouse position.
+    button?: 'left' | 'right'; // Up button. Default to left.
+  }
+>;
+
+/**
+ * Mouse move action
+ */
+export type MouseMoveAction = BaseAction<
+  'mouse_move', // 'move' | 'move_to' | 'hover',
+  {
+    point: Coordinates; // Target coordinates
+  }
+>;
+
+/**
+ * Drag action with start and end coordinates
+ */
+export type DragAction = BaseAction<
+  'drag', // 'left_click_drag' | 'select',
+  {
+    start: Coordinates;
+    end: Coordinates;
+    direction?: 'up' | 'down' | 'left' | 'right';
+  }
+>;
+
+/**
+ * Scroll action with coordinates and direction
+ */
+export type ScrollAction = BaseAction<
+  'scroll',
+  {
+    point?: Coordinates;
+    direction: 'up' | 'down' | 'left' | 'right';
+  }
+>;
+
+// ---------- Keyboard Actions ----------
+
+/**
+ * Type action with text content
+ */
+export type TypeAction = BaseAction<
+  'type',
+  {
+    content: string;
+  }
+>;
+
+/**
+ * Hotkey action with key combination
+ */
+export type HotkeyAction = BaseAction<
+  'hotkey',
+  {
+    key: string;
+  }
+>;
+
+/**
+ * Press key action
+ */
+export type PressAction = BaseAction<
+  'press',
+  {
+    key: string; // Key you want to press. Only one key can be pressed at one time.
+  }
+>;
+
+/**
+ * Release key action
+ */
+export type ReleaseAction = BaseAction<
+  'release',
+  {
+    key: string; // Key you want to release. Only one key can be released at one time.
+  }
+>;
+
+// ---------- Browser Actions ----------
+
+/**
+ * Navigate action with URL
+ */
+export type NavigateAction = BaseAction<
+  'navigate',
+  {
+    url: string;
+  }
+>;
+
+/**
+ * Navigate back action
+ */
+export type NavigateBackAction = BaseAction<'navigate_back', Record<string, never>>;
+
+// ---------- App Actions ----------
+
+/**
+ * Long press action with coordinates
+ */
+export type LongPressAction = BaseAction<
+  'long_press',
+  {
+    point: Coordinates;
+  }
+>;
+
+export type SwipeAction = BaseAction<
+  'swipe', // 'drag',
+  {
+    start: Coordinates;
+    end: Coordinates;
+    direction: 'up' | 'down' | 'left' | 'right';
+  }
+>;
+
+/**
+ * Home action
+ */
+export type HomeAction = BaseAction<'home' | 'press_home', Record<string, never>>;
+
+/**
+ * Back action
+ */
+export type BackAction = BaseAction<'back' | 'press_back', Record<string, never>>;
+
+/**
+ * Open app action
+ */
+export type OpenAppAction = BaseAction<
+  'open_app',
+  {
+    name: string;
+  }
+>;
+
+// ---------- Wait Actions ----------
+
+/**
+ * Wait action with no inputs
+ */
+export type WaitAction = BaseAction<
+  'wait',
+  {
+    time?: number; // in seconds (optional)
+  }
+>;
+
+/**
+ * Finished - Complete the current operation.
+ */
+export type FinishAction = BaseAction<
+  'finished',
+  {
+    content?: string;
+  }
+>;
+
+/**
+ * CallUser - Request user interaction.
+ */
+export type CallUserAction = BaseAction<
+  'call_user',
+  {
+    content?: string;
+  }
+>;
+
+/**
+ * Operational action types (excluding screenshot which has special handling)
+ */
+export type OperationalGUIAction =
+  | ClickAction
+  | DoubleClickAction
+  | RightClickAction
+  | MiddleClickAction
+  | MouseDownAction
+  | MouseUpAction
+  | MouseMoveAction
+  | DragAction
+  | ScrollAction
+  | TypeAction
+  | HotkeyAction
+  | PressAction
+  | ReleaseAction
+  | NavigateAction
+  | NavigateBackAction
+  | LongPressAction
+  | SwipeAction
+  | HomeAction
+  | BackAction
+  | OpenAppAction
+  | WaitAction
+  | FinishAction
+  | CallUserAction;
+
+/**
+ * Complete GUI action types including screenshot
+ */
+export type GUIAction = ScreenShotAction | OperationalGUIAction;
+
+/**
+ * Extract action type from action interface
+ */
+export type ExtractActionType<T> = T extends BaseAction<infer U, any> ? U : never;
+
+/**
+ * Supported operational action types (excluding screenshot)
+ */
+export type SupportedActionType = ExtractActionType<OperationalGUIAction>;
+
+/**
+ * All action types including screenshot
+ */
+export type AllActionType = ExtractActionType<GUIAction>;
+
+/**
+ * Action metadata for documentation and serialization
+ */
+export interface ActionMetadata {
+  description: string;
+  category: 'mouse' | 'keyboard' | 'navigation' | 'mobile' | 'system' | 'wait';
+}
+
+/**
+ * Comprehensive action metadata registry
+ */
+export const ACTION_METADATA: Record<SupportedActionType, ActionMetadata> = {
+  click: { category: 'mouse', description: 'Click on an element' },
+  right_click: { category: 'mouse', description: 'Right click on an element' },
+  double_click: { category: 'mouse', description: 'Double click on an element' },
+  middle_click: { category: 'mouse', description: 'Middle click on an element' },
+  mouse_down: { category: 'mouse', description: 'Press mouse button down' },
+  mouse_up: { category: 'mouse', description: 'Release mouse button' },
+  mouse_move: { category: 'mouse', description: 'Move mouse to position' },
+  drag: { category: 'mouse', description: 'Drag from one position to another' },
+  scroll: { category: 'mouse', description: 'Scroll in a direction' },
+  type: { category: 'keyboard', description: 'Type text' },
+  hotkey: { category: 'keyboard', description: 'Press hotkey combination' },
+  press: { category: 'keyboard', description: 'Press a key' },
+  release: { category: 'keyboard', description: 'Release a key' },
+  navigate: { category: 'navigation', description: 'Navigate to URL' },
+  navigate_back: { category: 'navigation', description: 'Navigate back' },
+  long_press: { category: 'mobile', description: 'Long press on element' },
+  swipe: { category: 'mobile', description: 'Swipe gesture' },
+  home: { category: 'mobile', description: 'Go to home' },
+  press_home: { category: 'mobile', description: 'Press home button' },
+  back: { category: 'mobile', description: 'Go back' },
+  press_back: { category: 'mobile', description: 'Press back button' },
+  open_app: { category: 'mobile', description: 'Open application' },
+  wait: { category: 'wait', description: 'Wait for specified time' },
+  finished: { category: 'system', description: 'Mark task as finished' },
+  call_user: { category: 'system', description: 'Request user interaction' },
+} as const;
+
+/**
+ * Type guard function to check if a string is a valid operational action type
+ * @param type - The string to check
+ * @returns Whether the string is a valid SupportedActionType
+ */
+export function isSupportedActionType(type: string): type is SupportedActionType {
+  return type in ACTION_METADATA;
+}
+
+/**
+ * Type guard function to check if a string is any valid action type (including screenshot)
+ * @param type - The string to check
+ * @returns Whether the string is a valid AllActionType
+ */
+export function isValidActionType(type: string): type is AllActionType {
+  return type === 'screenshot' || isSupportedActionType(type);
+}

package/src/types/agents.ts

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { AgentOptions } from '@tarko/agent-interface';
+import { Factors, BaseAction, Coordinates, SupportedActionType, ActionMetadata } from './actions';
+
+/**
+ * Type definition for parsed GUI response structure
+ * Represents the components extracted from a model's output string
+ * Aligned with tarko's web UI design
+ */
+export interface ParsedGUIResponse {
+  /** raw prediction string */
+  rawContent: string;
+  /** parsed from Thought: `<thought>` */
+  reasoningContent?: string;
+  /** parsed from Action: action(params=`action`) */
+  rawActionStrings?: string[];
+  /** parsed from Action: action(params=`action`) */
+  actions: BaseAction[];
+  /** error message to feedback to LLM */
+  errorMessage?: string;
+}
+
+/**
+ * Type definition for function to normalize raw coordinates
+ * Converts raw pixel coordinates to normalized coordinates (0-1)
+ * @param rawX - Raw X coordinate in pixels
+ * @param rawY - Raw Y coordinate in pixels
+ * @returns Normalized coordinates {x, y} with values between 0 and 1
+ */
+export type NormalizeCoordinates = (
+  rawCoords: Coordinates,
+  factors?: Factors,
+) => { normalized: Coordinates };
+
+/**
+ * Type definition for handler function to parse model output into ParsedGUIResponse object
+ * @param prediction - The raw output from the model to be parsed
+ * @returns ParsedGUIResponse object if parsing is successful, null otherwise
+ */
+export type CustomActionParser = (prediction: string) => ParsedGUIResponse | null;
+
+/**
+ * Function type for serializing supported actions to string format
+ * @param actions - Array of supported action types
+ * @returns String representation of the actions for agent processing
+ */
+export type SerializeSupportedActions = (actions: Array<SupportedActionType>) => string;
+
+export type ExecuteParams = {
+  /** Required actions to execute */
+  actions: BaseAction[];
+} & Partial<Omit<ParsedGUIResponse, 'actions'>> &
+  Record<string, any>;
+
+export type ExecuteOutput = {
+  status: 'success' | 'failed';
+  errorMessage?: string;
+  url?: string; // url of the page
+} & Record<string, any>;
+
+/**
+ * Function type for calculating detail level based on image dimensions
+ */
+export type ImageDetailCalculator = (width: number, height: number) => 'low' | 'high' | 'auto';
+
+export interface ScreenshotOutput extends ExecuteOutput {
+  /** screenshot base64, `keep screenshot size as physical pixels` */
+  base64: string;
+}
+
+/**
+ * Reserved placeholder name for action space in system prompt template
+ */
+export const ACTION_SPACE_PLACEHOLDER = 'action_space';
+
+/**
+ * Interface for system prompt template configuration
+ */
+export interface SystemPromptTemplate {
+  /**
+   * Template string with placeholders. Must include an action space placeholder
+   * `{{${ACTION_SPACE_PLACEHOLDER}}}` that will be replaced with the string representation of available actions
+   */
+  template: string;
+
+  /**
+   * Function to convert BaseAction array to string representation for the action space
+   * This will be used to fill the action space placeholder in the template
+   */
+  actionsToString?: SerializeSupportedActions;
+
+  /**
+   * Optional map of additional placeholder values to be replaced in the template
+   * Keys are placeholder names, values are the replacement strings
+   * Note: '${ACTION_SPACE_PLACEHOLDER}' is a reserved placeholder and should NOT be included here
+   * as it will be automatically filled using the actionsToString function
+   */
+  // placeholders?: Omit<Record<string, string>, typeof ACTION_SPACE_PLACEHOLDER>;
+  placeholders?: Record<string, string>;
+}
+
+export interface GUIAgentConfig<TOperator> extends AgentOptions {
+  operator: TOperator;
+  // ===== Optional =====
+  /**
+   * System prompt configuration. Can be either:
+   * - A simple string (legacy mode)
+   * - An array of strings or chat messages (will be concatenated)
+   * - A SystemPromptTemplate object with template and actionsToString function
+   */
+  systemPrompt?: string | Array<string | { role: string; content: string }> | SystemPromptTemplate;
+  /** The handler function to parse model output into PredictionParsed object */
+  customeActionParser?: CustomActionParser;
+  /** The function to normalize raw coordinates */
+  normalizeCoordinates?: NormalizeCoordinates;
+  /** The function to calculate detail level based on image dimensions */
+  detailCalculator?: ImageDetailCalculator;
+  /** Maximum number of turns for Agent to execute, @default 1000 */
+  maxLoopCount?: number;
+  /** Time interval between two loop iterations (in milliseconds), @default 0 */
+  loopIntervalInMs?: number;
+}

package/src/types/archived.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export enum ErrorStatusEnum {
+  /** 100000 */
+  SCREENSHOT_RETRY_ERROR = -100000,
+  /** 100001 */
+  INVOKE_RETRY_ERROR = -100001,
+  /** 100002 */
+  EXECUTE_RETRY_ERROR = -100002,
+  /** 100003 */
+  MODEL_SERVICE_ERROR = -100003,
+  /** 100004 */
+  REACH_MAXLOOP_ERROR = -100004,
+  /** 100005 */
+  ENVIRONMENT_ERROR = -100005,
+  /** 100099 */
+  UNKNOWN_ERROR = -100099,
+}
+
+export class GUIAgentError extends Error {
+  status: ErrorStatusEnum;
+  message: string;
+  stack?: string;
+
+  constructor(status: ErrorStatusEnum, message: string, stack?: string) {
+    super(message);
+    this.status = status;
+    this.message = message;
+    this.stack = stack;
+  }
+}
+
+export type Status = `${StatusEnum}`;
+export enum StatusEnum {
+  INIT = 'init',
+  RUNNING = 'running',
+  PAUSE = 'pause',
+  END = 'end',
+  CALL_USER = 'call_user',
+  /**
+   * @deprecated kept for backward compatibility
+   */
+  MAX_LOOP = 'max_loop',
+  USER_STOPPED = 'user_stopped',
+  ERROR = 'error',
+}
+export interface VlmResponse {
+  generate_resp: {
+    input: string;
+    prediction: string;
+    uid: string;
+  }[];
+}

package/src/types/index.ts

@@ -0,0 +1,8 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+export * from './archived';
+export * from './actions';
+export * from './agents';