@ui-tars-test/shared 0.3.2

package/dist/base/parser.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"base/parser.mjs","sources":["webpack://@ui-tars-test/shared/./src/base/parser.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\n\nimport { ParsedGUIResponse } from '../types';\n\nexport abstract class BaseActionParser {\n  /**\n   * Parse model output\n   * @param input Model output string\n   * @returns Parsed ParsedGUIResponse object, returns null if parsing fails\n   * There is no need to throw error, the error message is returned in ParsedGUIResponse\n   */\n  abstract parsePrediction(input: string): ParsedGUIResponse | null;\n}\n"],"names":["BaseActionParser"],"mappings":";;;;AAOO,MAAeA;AAQtB"}

package/dist/types/actions.d.ts

@@ -0,0 +1,224 @@
+export type Factors = [number, number];
+/**
+ * Coordinate data structure
+ * - Supports pixel coordinates (raw)
+ * - Supports normalized coordinates (normalized)
+ */
+export interface Coordinates {
+    raw?: {
+        x: number;
+        y: number;
+    };
+    normalized?: {
+        x: number;
+        y: number;
+    };
+    referenceBox?: {
+        x1: number;
+        y1: number;
+        x2: number;
+        y2: number;
+    };
+    referenceSystem?: 'screen' | 'window' | 'browserPage' | string;
+}
+/**
+ * Standard structure for GUI Actions
+ */
+export interface BaseAction<T extends string = string, I extends Record<string, any> = Record<string, any>> {
+    type: T;
+    inputs: I;
+    meta?: {
+        toolHint?: string;
+        comment?: string;
+    };
+}
+/**
+ * ScreenShot action
+ */
+export type ScreenShotAction = BaseAction<'screenshot', {
+    start?: Coordinates;
+    end?: Coordinates;
+}>;
+/**
+ * Click action with coordinates
+ */
+export type ClickAction = BaseAction<'click', {
+    point: Coordinates;
+}>;
+/**
+ * Right click action with coordinates
+ */
+export type RightClickAction = BaseAction<'right_click', {
+    point: Coordinates;
+}>;
+/**
+ * Double click action with coordinates
+ */
+export type DoubleClickAction = BaseAction<'double_click', {
+    point: Coordinates;
+}>;
+/**
+ * Middle click action with coordinates
+ */
+export type MiddleClickAction = BaseAction<'middle_click', {
+    point: Coordinates;
+}>;
+/**
+ * Mouse down action
+ */
+export type MouseDownAction = BaseAction<'mouse_down', {
+    point?: Coordinates;
+    button?: 'left' | 'right';
+}>;
+/**
+ * Mouse up action
+ */
+export type MouseUpAction = BaseAction<'mouse_up', {
+    point?: Coordinates;
+    button?: 'left' | 'right';
+}>;
+/**
+ * Mouse move action
+ */
+export type MouseMoveAction = BaseAction<'mouse_move', // 'move' | 'move_to' | 'hover',
+{
+    point: Coordinates;
+}>;
+/**
+ * Drag action with start and end coordinates
+ */
+export type DragAction = BaseAction<'drag', // 'left_click_drag' | 'select',
+{
+    start: Coordinates;
+    end: Coordinates;
+    direction?: 'up' | 'down' | 'left' | 'right';
+}>;
+/**
+ * Scroll action with coordinates and direction
+ */
+export type ScrollAction = BaseAction<'scroll', {
+    point?: Coordinates;
+    direction: 'up' | 'down' | 'left' | 'right';
+}>;
+/**
+ * Type action with text content
+ */
+export type TypeAction = BaseAction<'type', {
+    content: string;
+}>;
+/**
+ * Hotkey action with key combination
+ */
+export type HotkeyAction = BaseAction<'hotkey', {
+    key: string;
+}>;
+/**
+ * Press key action
+ */
+export type PressAction = BaseAction<'press', {
+    key: string;
+}>;
+/**
+ * Release key action
+ */
+export type ReleaseAction = BaseAction<'release', {
+    key: string;
+}>;
+/**
+ * Navigate action with URL
+ */
+export type NavigateAction = BaseAction<'navigate', {
+    url: string;
+}>;
+/**
+ * Navigate back action
+ */
+export type NavigateBackAction = BaseAction<'navigate_back', Record<string, never>>;
+/**
+ * Long press action with coordinates
+ */
+export type LongPressAction = BaseAction<'long_press', {
+    point: Coordinates;
+}>;
+export type SwipeAction = BaseAction<'swipe', // 'drag',
+{
+    start: Coordinates;
+    end: Coordinates;
+    direction: 'up' | 'down' | 'left' | 'right';
+}>;
+/**
+ * Home action
+ */
+export type HomeAction = BaseAction<'home' | 'press_home', Record<string, never>>;
+/**
+ * Back action
+ */
+export type BackAction = BaseAction<'back' | 'press_back', Record<string, never>>;
+/**
+ * Open app action
+ */
+export type OpenAppAction = BaseAction<'open_app', {
+    name: string;
+}>;
+/**
+ * Wait action with no inputs
+ */
+export type WaitAction = BaseAction<'wait', {
+    time?: number;
+}>;
+/**
+ * Finished - Complete the current operation.
+ */
+export type FinishAction = BaseAction<'finished', {
+    content?: string;
+}>;
+/**
+ * CallUser - Request user interaction.
+ */
+export type CallUserAction = BaseAction<'call_user', {
+    content?: string;
+}>;
+/**
+ * Operational action types (excluding screenshot which has special handling)
+ */
+export type OperationalGUIAction = ClickAction | DoubleClickAction | RightClickAction | MiddleClickAction | MouseDownAction | MouseUpAction | MouseMoveAction | DragAction | ScrollAction | TypeAction | HotkeyAction | PressAction | ReleaseAction | NavigateAction | NavigateBackAction | LongPressAction | SwipeAction | HomeAction | BackAction | OpenAppAction | WaitAction | FinishAction | CallUserAction;
+/**
+ * Complete GUI action types including screenshot
+ */
+export type GUIAction = ScreenShotAction | OperationalGUIAction;
+/**
+ * Extract action type from action interface
+ */
+export type ExtractActionType<T> = T extends BaseAction<infer U, any> ? U : never;
+/**
+ * Supported operational action types (excluding screenshot)
+ */
+export type SupportedActionType = ExtractActionType<OperationalGUIAction>;
+/**
+ * All action types including screenshot
+ */
+export type AllActionType = ExtractActionType<GUIAction>;
+/**
+ * Action metadata for documentation and serialization
+ */
+export interface ActionMetadata {
+    description: string;
+    category: 'mouse' | 'keyboard' | 'navigation' | 'mobile' | 'system' | 'wait';
+}
+/**
+ * Comprehensive action metadata registry
+ */
+export declare const ACTION_METADATA: Record<SupportedActionType, ActionMetadata>;
+/**
+ * Type guard function to check if a string is a valid operational action type
+ * @param type - The string to check
+ * @returns Whether the string is a valid SupportedActionType
+ */
+export declare function isSupportedActionType(type: string): type is SupportedActionType;
+/**
+ * Type guard function to check if a string is any valid action type (including screenshot)
+ * @param type - The string to check
+ * @returns Whether the string is a valid AllActionType
+ */
+export declare function isValidActionType(type: string): type is AllActionType;
+//# sourceMappingURL=actions.d.ts.map

package/dist/types/actions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"actions.d.ts","sourceRoot":"","sources":["../../src/types/actions.ts"],"names":[],"mappings":"AAOA,MAAM,MAAM,OAAO,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEvC;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,GAAG,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC/B,UAAU,CAAC,EAAE;QAAE,CAAC,EAAE,MAAM,CAAC;QAAC,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IACtC,YAAY,CAAC,EAAE;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,MAAM,CAAA;KAAE,CAAC;IAClE,eAAe,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,aAAa,GAAG,MAAM,CAAC;CAChE;AAED;;GAEG;AACH,MAAM,WAAW,UAAU,CACzB,CAAC,SAAS,MAAM,GAAG,MAAM,EACzB,CAAC,SAAS,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC;IAEnD,IAAI,EAAE,CAAC,CAAC;IACR,MAAM,EAAE,CAAC,CAAC;IACV,IAAI,CAAC,EAAE;QACL,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH;AAID;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CACvC,YAAY,EACZ;IACE,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,GAAG,CAAC,EAAE,WAAW,CAAC;CACnB,CACF,CAAC;AAIF;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,UAAU,CAClC,OAAO,EACP;IACE,KAAK,EAAE,WAAW,CAAC;CACpB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,UAAU,CACvC,aAAa,EACb;IACE,KAAK,EAAE,WAAW,CAAC;CACpB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,UAAU,CACxC,cAAc,EACd;IACE,KAAK,EAAE,WAAW,CAAC;CACpB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,UAAU,CACxC,cAAc,EACd;IACE,KAAK,EAAE,WAAW,CAAC;CACpB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,UAAU,CACtC,YAAY,EACZ;IACE,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,CACpC,UAAU,EACV;IACE,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,MAAM,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;CAC3B,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,UAAU,CACtC,YAAY,EAAE,gCAAgC;AAC9C;IACE,KAAK,EAAE,WAAW,CAAC;CACpB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,CACjC,MAAM,EAAE,gCAAgC;AACxC;IACE,KAAK,EAAE,WAAW,CAAC;IACnB,GAAG,EAAE,WAAW,CAAC;IACjB,SAAS,CAAC,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;CAC9C,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,UAAU,CACnC,QAAQ,EACR;IACE,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;CAC7C,CACF,CAAC;AAIF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,CACjC,MAAM,EACN;IACE,OAAO,EAAE,MAAM,CAAC;CACjB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,UAAU,CACnC,QAAQ,EACR;IACE,GAAG,EAAE,MAAM,CAAC;CACb,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,UAAU,CAClC,OAAO,EACP;IACE,GAAG,EAAE,MAAM,CAAC;CACb,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,CACpC,SAAS,EACT;IACE,GAAG,EAAE,MAAM,CAAC;CACb,CACF,CAAC;AAIF;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,UAAU,CACrC,UAAU,EACV;IACE,GAAG,EAAE,MAAM,CAAC;CACb,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG,UAAU,CAAC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AAIpF;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,UAAU,CACtC,YAAY,EACZ;IACE,KAAK,EAAE,WAAW,CAAC;CACpB,CACF,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG,UAAU,CAClC,OAAO,EAAE,UAAU;AACnB;IACE,KAAK,EAAE,WAAW,CAAC;IACnB,GAAG,EAAE,WAAW,CAAC;IACjB,SAAS,EAAE,IAAI,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;CAC7C,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AAElF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;AAElF;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,UAAU,CACpC,UAAU,EACV;IACE,IAAI,EAAE,MAAM,CAAC;CACd,CACF,CAAC;AAIF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,UAAU,CACjC,MAAM,EACN;IACE,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,UAAU,CACnC,UAAU,EACV;IACE,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,UAAU,CACrC,WAAW,EACX;IACE,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB,CACF,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAC5B,WAAW,GACX,iBAAiB,GACjB,gBAAgB,GAChB,iBAAiB,GACjB,eAAe,GACf,aAAa,GACb,eAAe,GACf,UAAU,GACV,YAAY,GACZ,UAAU,GACV,YAAY,GACZ,WAAW,GACX,aAAa,GACb,cAAc,GACd,kBAAkB,GAClB,eAAe,GACf,WAAW,GACX,UAAU,GACV,UAAU,GACV,aAAa,GACb,UAAU,GACV,YAAY,GACZ,cAAc,CAAC;AAEnB;;GAEG;AACH,MAAM,MAAM,SAAS,GAAG,gBAAgB,GAAG,oBAAoB,CAAC;AAEhE;;GAEG;AACH,MAAM,MAAM,iBAAiB,CAAC,CAAC,IAAI,CAAC,SAAS,UAAU,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AAElF;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,iBAAiB,CAAC,oBAAoB,CAAC,CAAC;AAE1E;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,iBAAiB,CAAC,SAAS,CAAC,CAAC;AAEzD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,GAAG,UAAU,GAAG,YAAY,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;CAC9E;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,cAAc,CA0B9D,CAAC;AAEX;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,IAAI,mBAAmB,CAE/E;AAED;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,IAAI,aAAa,CAErE"}

package/dist/types/actions.js

@@ -0,0 +1,155 @@
+/**
+* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+* SPDX-License-Identifier: Apache-2.0
+*/
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    ACTION_METADATA: ()=>ACTION_METADATA,
+    isSupportedActionType: ()=>isSupportedActionType,
+    isValidActionType: ()=>isValidActionType
+});
+const ACTION_METADATA = {
+    click: {
+        category: 'mouse',
+        description: 'Click on an element'
+    },
+    right_click: {
+        category: 'mouse',
+        description: 'Right click on an element'
+    },
+    double_click: {
+        category: 'mouse',
+        description: 'Double click on an element'
+    },
+    middle_click: {
+        category: 'mouse',
+        description: 'Middle click on an element'
+    },
+    mouse_down: {
+        category: 'mouse',
+        description: 'Press mouse button down'
+    },
+    mouse_up: {
+        category: 'mouse',
+        description: 'Release mouse button'
+    },
+    mouse_move: {
+        category: 'mouse',
+        description: 'Move mouse to position'
+    },
+    drag: {
+        category: 'mouse',
+        description: 'Drag from one position to another'
+    },
+    scroll: {
+        category: 'mouse',
+        description: 'Scroll in a direction'
+    },
+    type: {
+        category: 'keyboard',
+        description: 'Type text'
+    },
+    hotkey: {
+        category: 'keyboard',
+        description: 'Press hotkey combination'
+    },
+    press: {
+        category: 'keyboard',
+        description: 'Press a key'
+    },
+    release: {
+        category: 'keyboard',
+        description: 'Release a key'
+    },
+    navigate: {
+        category: 'navigation',
+        description: 'Navigate to URL'
+    },
+    navigate_back: {
+        category: 'navigation',
+        description: 'Navigate back'
+    },
+    long_press: {
+        category: 'mobile',
+        description: 'Long press on element'
+    },
+    swipe: {
+        category: 'mobile',
+        description: 'Swipe gesture'
+    },
+    home: {
+        category: 'mobile',
+        description: 'Go to home'
+    },
+    press_home: {
+        category: 'mobile',
+        description: 'Press home button'
+    },
+    back: {
+        category: 'mobile',
+        description: 'Go back'
+    },
+    press_back: {
+        category: 'mobile',
+        description: 'Press back button'
+    },
+    open_app: {
+        category: 'mobile',
+        description: 'Open application'
+    },
+    wait: {
+        category: 'wait',
+        description: 'Wait for specified time'
+    },
+    finished: {
+        category: 'system',
+        description: 'Mark task as finished'
+    },
+    call_user: {
+        category: 'system',
+        description: 'Request user interaction'
+    }
+};
+function isSupportedActionType(type) {
+    return type in ACTION_METADATA;
+}
+function isValidActionType(type) {
+    return 'screenshot' === type || isSupportedActionType(type);
+}
+exports.ACTION_METADATA = __webpack_exports__.ACTION_METADATA;
+exports.isSupportedActionType = __webpack_exports__.isSupportedActionType;
+exports.isValidActionType = __webpack_exports__.isValidActionType;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "ACTION_METADATA",
+    "isSupportedActionType",
+    "isValidActionType"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=actions.js.map

package/dist/types/actions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types/actions.js","sources":["webpack://@ui-tars-test/shared/webpack/runtime/define_property_getters","webpack://@ui-tars-test/shared/webpack/runtime/has_own_property","webpack://@ui-tars-test/shared/webpack/runtime/make_namespace_object","webpack://@ui-tars-test/shared/./src/types/actions.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\n\n/* eslint-disable @typescript-eslint/no-explicit-any */\n\nexport type Factors = [number, number];\n\n/**\n * Coordinate data structure\n * - Supports pixel coordinates (raw)\n * - Supports normalized coordinates (normalized)\n */\nexport interface Coordinates {\n  raw?: { x: number; y: number }; // Raw pixels\n  normalized?: { x: number; y: number }; // Normalized coordinates (0–1)\n  referenceBox?: { x1: number; y1: number; x2: number; y2: number };\n  referenceSystem?: 'screen' | 'window' | 'browserPage' | string; // Coordinate reference system\n}\n\n/**\n * Standard structure for GUI Actions\n */\nexport interface BaseAction<\n  T extends string = string,\n  I extends Record<string, any> = Record<string, any>,\n> {\n  type: T; // Action type (e.g., \"click\", \"key\", \"swipe\")\n  inputs: I; // Parameters required for the action\n  meta?: {\n    toolHint?: string; // Suggested execution tool (xdotool / adb / pyautogui etc.)\n    comment?: string; // Notes / Debug information\n  };\n}\n\n// ---------- ScreenShot Action ----------\n\n/**\n * ScreenShot action\n */\nexport type ScreenShotAction = BaseAction<\n  'screenshot',\n  {\n    start?: Coordinates;\n    end?: Coordinates;\n  }\n>;\n\n// ---------- Mouse Actions ----------\n\n/**\n * Click action with coordinates\n */\nexport type ClickAction = BaseAction<\n  'click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Right click action with coordinates\n */\nexport type RightClickAction = BaseAction<\n  'right_click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Double click action with coordinates\n */\nexport type DoubleClickAction = BaseAction<\n  'double_click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Middle click action with coordinates\n */\nexport type MiddleClickAction = BaseAction<\n  'middle_click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Mouse down action\n */\nexport type MouseDownAction = BaseAction<\n  'mouse_down',\n  {\n    point?: Coordinates; // Mouse down position. If not specified, default to execute on the current mouse position.\n    button?: 'left' | 'right'; // Down button. Default to left.\n  }\n>;\n\n/**\n * Mouse up action\n */\nexport type MouseUpAction = BaseAction<\n  'mouse_up',\n  {\n    point?: Coordinates; // Mouse up position. If not specified, default to execute on the current mouse position.\n    button?: 'left' | 'right'; // Up button. Default to left.\n  }\n>;\n\n/**\n * Mouse move action\n */\nexport type MouseMoveAction = BaseAction<\n  'mouse_move', // 'move' | 'move_to' | 'hover',\n  {\n    point: Coordinates; // Target coordinates\n  }\n>;\n\n/**\n * Drag action with start and end coordinates\n */\nexport type DragAction = BaseAction<\n  'drag', // 'left_click_drag' | 'select',\n  {\n    start: Coordinates;\n    end: Coordinates;\n    direction?: 'up' | 'down' | 'left' | 'right';\n  }\n>;\n\n/**\n * Scroll action with coordinates and direction\n */\nexport type ScrollAction = BaseAction<\n  'scroll',\n  {\n    point?: Coordinates;\n    direction: 'up' | 'down' | 'left' | 'right';\n  }\n>;\n\n// ---------- Keyboard Actions ----------\n\n/**\n * Type action with text content\n */\nexport type TypeAction = BaseAction<\n  'type',\n  {\n    content: string;\n  }\n>;\n\n/**\n * Hotkey action with key combination\n */\nexport type HotkeyAction = BaseAction<\n  'hotkey',\n  {\n    key: string;\n  }\n>;\n\n/**\n * Press key action\n */\nexport type PressAction = BaseAction<\n  'press',\n  {\n    key: string; // Key you want to press. Only one key can be pressed at one time.\n  }\n>;\n\n/**\n * Release key action\n */\nexport type ReleaseAction = BaseAction<\n  'release',\n  {\n    key: string; // Key you want to release. Only one key can be released at one time.\n  }\n>;\n\n// ---------- Browser Actions ----------\n\n/**\n * Navigate action with URL\n */\nexport type NavigateAction = BaseAction<\n  'navigate',\n  {\n    url: string;\n  }\n>;\n\n/**\n * Navigate back action\n */\nexport type NavigateBackAction = BaseAction<'navigate_back', Record<string, never>>;\n\n// ---------- App Actions ----------\n\n/**\n * Long press action with coordinates\n */\nexport type LongPressAction = BaseAction<\n  'long_press',\n  {\n    point: Coordinates;\n  }\n>;\n\nexport type SwipeAction = BaseAction<\n  'swipe', // 'drag',\n  {\n    start: Coordinates;\n    end: Coordinates;\n    direction: 'up' | 'down' | 'left' | 'right';\n  }\n>;\n\n/**\n * Home action\n */\nexport type HomeAction = BaseAction<'home' | 'press_home', Record<string, never>>;\n\n/**\n * Back action\n */\nexport type BackAction = BaseAction<'back' | 'press_back', Record<string, never>>;\n\n/**\n * Open app action\n */\nexport type OpenAppAction = BaseAction<\n  'open_app',\n  {\n    name: string;\n  }\n>;\n\n// ---------- Wait Actions ----------\n\n/**\n * Wait action with no inputs\n */\nexport type WaitAction = BaseAction<\n  'wait',\n  {\n    time?: number; // in seconds (optional)\n  }\n>;\n\n/**\n * Finished - Complete the current operation.\n */\nexport type FinishAction = BaseAction<\n  'finished',\n  {\n    content?: string;\n  }\n>;\n\n/**\n * CallUser - Request user interaction.\n */\nexport type CallUserAction = BaseAction<\n  'call_user',\n  {\n    content?: string;\n  }\n>;\n\n/**\n * Operational action types (excluding screenshot which has special handling)\n */\nexport type OperationalGUIAction =\n  | ClickAction\n  | DoubleClickAction\n  | RightClickAction\n  | MiddleClickAction\n  | MouseDownAction\n  | MouseUpAction\n  | MouseMoveAction\n  | DragAction\n  | ScrollAction\n  | TypeAction\n  | HotkeyAction\n  | PressAction\n  | ReleaseAction\n  | NavigateAction\n  | NavigateBackAction\n  | LongPressAction\n  | SwipeAction\n  | HomeAction\n  | BackAction\n  | OpenAppAction\n  | WaitAction\n  | FinishAction\n  | CallUserAction;\n\n/**\n * Complete GUI action types including screenshot\n */\nexport type GUIAction = ScreenShotAction | OperationalGUIAction;\n\n/**\n * Extract action type from action interface\n */\nexport type ExtractActionType<T> = T extends BaseAction<infer U, any> ? U : never;\n\n/**\n * Supported operational action types (excluding screenshot)\n */\nexport type SupportedActionType = ExtractActionType<OperationalGUIAction>;\n\n/**\n * All action types including screenshot\n */\nexport type AllActionType = ExtractActionType<GUIAction>;\n\n/**\n * Action metadata for documentation and serialization\n */\nexport interface ActionMetadata {\n  description: string;\n  category: 'mouse' | 'keyboard' | 'navigation' | 'mobile' | 'system' | 'wait';\n}\n\n/**\n * Comprehensive action metadata registry\n */\nexport const ACTION_METADATA: Record<SupportedActionType, ActionMetadata> = {\n  click: { category: 'mouse', description: 'Click on an element' },\n  right_click: { category: 'mouse', description: 'Right click on an element' },\n  double_click: { category: 'mouse', description: 'Double click on an element' },\n  middle_click: { category: 'mouse', description: 'Middle click on an element' },\n  mouse_down: { category: 'mouse', description: 'Press mouse button down' },\n  mouse_up: { category: 'mouse', description: 'Release mouse button' },\n  mouse_move: { category: 'mouse', description: 'Move mouse to position' },\n  drag: { category: 'mouse', description: 'Drag from one position to another' },\n  scroll: { category: 'mouse', description: 'Scroll in a direction' },\n  type: { category: 'keyboard', description: 'Type text' },\n  hotkey: { category: 'keyboard', description: 'Press hotkey combination' },\n  press: { category: 'keyboard', description: 'Press a key' },\n  release: { category: 'keyboard', description: 'Release a key' },\n  navigate: { category: 'navigation', description: 'Navigate to URL' },\n  navigate_back: { category: 'navigation', description: 'Navigate back' },\n  long_press: { category: 'mobile', description: 'Long press on element' },\n  swipe: { category: 'mobile', description: 'Swipe gesture' },\n  home: { category: 'mobile', description: 'Go to home' },\n  press_home: { category: 'mobile', description: 'Press home button' },\n  back: { category: 'mobile', description: 'Go back' },\n  press_back: { category: 'mobile', description: 'Press back button' },\n  open_app: { category: 'mobile', description: 'Open application' },\n  wait: { category: 'wait', description: 'Wait for specified time' },\n  finished: { category: 'system', description: 'Mark task as finished' },\n  call_user: { category: 'system', description: 'Request user interaction' },\n} as const;\n\n/**\n * Type guard function to check if a string is a valid operational action type\n * @param type - The string to check\n * @returns Whether the string is a valid SupportedActionType\n */\nexport function isSupportedActionType(type: string): type is SupportedActionType {\n  return type in ACTION_METADATA;\n}\n\n/**\n * Type guard function to check if a string is any valid action type (including screenshot)\n * @param type - The string to check\n * @returns Whether the string is a valid AllActionType\n */\nexport function isValidActionType(type: string): type is AllActionType {\n  return type === 'screenshot' || isSupportedActionType(type);\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ACTION_METADATA","isSupportedActionType","type","isValidActionType"],"mappings":";;;;;;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;AC2UO,MAAMI,kBAA+D;IAC1E,OAAO;QAAE,UAAU;QAAS,aAAa;IAAsB;IAC/D,aAAa;QAAE,UAAU;QAAS,aAAa;IAA4B;IAC3E,cAAc;QAAE,UAAU;QAAS,aAAa;IAA6B;IAC7E,cAAc;QAAE,UAAU;QAAS,aAAa;IAA6B;IAC7E,YAAY;QAAE,UAAU;QAAS,aAAa;IAA0B;IACxE,UAAU;QAAE,UAAU;QAAS,aAAa;IAAuB;IACnE,YAAY;QAAE,UAAU;QAAS,aAAa;IAAyB;IACvE,MAAM;QAAE,UAAU;QAAS,aAAa;IAAoC;IAC5E,QAAQ;QAAE,UAAU;QAAS,aAAa;IAAwB;IAClE,MAAM;QAAE,UAAU;QAAY,aAAa;IAAY;IACvD,QAAQ;QAAE,UAAU;QAAY,aAAa;IAA2B;IACxE,OAAO;QAAE,UAAU;QAAY,aAAa;IAAc;IAC1D,SAAS;QAAE,UAAU;QAAY,aAAa;IAAgB;IAC9D,UAAU;QAAE,UAAU;QAAc,aAAa;IAAkB;IACnE,eAAe;QAAE,UAAU;QAAc,aAAa;IAAgB;IACtE,YAAY;QAAE,UAAU;QAAU,aAAa;IAAwB;IACvE,OAAO;QAAE,UAAU;QAAU,aAAa;IAAgB;IAC1D,MAAM;QAAE,UAAU;QAAU,aAAa;IAAa;IACtD,YAAY;QAAE,UAAU;QAAU,aAAa;IAAoB;IACnE,MAAM;QAAE,UAAU;QAAU,aAAa;IAAU;IACnD,YAAY;QAAE,UAAU;QAAU,aAAa;IAAoB;IACnE,UAAU;QAAE,UAAU;QAAU,aAAa;IAAmB;IAChE,MAAM;QAAE,UAAU;QAAQ,aAAa;IAA0B;IACjE,UAAU;QAAE,UAAU;QAAU,aAAa;IAAwB;IACrE,WAAW;QAAE,UAAU;QAAU,aAAa;IAA2B;AAC3E;AAOO,SAASC,sBAAsBC,IAAY;IAChD,OAAOA,QAAQF;AACjB;AAOO,SAASG,kBAAkBD,IAAY;IAC5C,OAAOA,AAAS,iBAATA,QAAyBD,sBAAsBC;AACxD"}

package/dist/types/actions.mjs

@@ -0,0 +1,115 @@
+/**
+* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+* SPDX-License-Identifier: Apache-2.0
+*/
+const ACTION_METADATA = {
+    click: {
+        category: 'mouse',
+        description: 'Click on an element'
+    },
+    right_click: {
+        category: 'mouse',
+        description: 'Right click on an element'
+    },
+    double_click: {
+        category: 'mouse',
+        description: 'Double click on an element'
+    },
+    middle_click: {
+        category: 'mouse',
+        description: 'Middle click on an element'
+    },
+    mouse_down: {
+        category: 'mouse',
+        description: 'Press mouse button down'
+    },
+    mouse_up: {
+        category: 'mouse',
+        description: 'Release mouse button'
+    },
+    mouse_move: {
+        category: 'mouse',
+        description: 'Move mouse to position'
+    },
+    drag: {
+        category: 'mouse',
+        description: 'Drag from one position to another'
+    },
+    scroll: {
+        category: 'mouse',
+        description: 'Scroll in a direction'
+    },
+    type: {
+        category: 'keyboard',
+        description: 'Type text'
+    },
+    hotkey: {
+        category: 'keyboard',
+        description: 'Press hotkey combination'
+    },
+    press: {
+        category: 'keyboard',
+        description: 'Press a key'
+    },
+    release: {
+        category: 'keyboard',
+        description: 'Release a key'
+    },
+    navigate: {
+        category: 'navigation',
+        description: 'Navigate to URL'
+    },
+    navigate_back: {
+        category: 'navigation',
+        description: 'Navigate back'
+    },
+    long_press: {
+        category: 'mobile',
+        description: 'Long press on element'
+    },
+    swipe: {
+        category: 'mobile',
+        description: 'Swipe gesture'
+    },
+    home: {
+        category: 'mobile',
+        description: 'Go to home'
+    },
+    press_home: {
+        category: 'mobile',
+        description: 'Press home button'
+    },
+    back: {
+        category: 'mobile',
+        description: 'Go back'
+    },
+    press_back: {
+        category: 'mobile',
+        description: 'Press back button'
+    },
+    open_app: {
+        category: 'mobile',
+        description: 'Open application'
+    },
+    wait: {
+        category: 'wait',
+        description: 'Wait for specified time'
+    },
+    finished: {
+        category: 'system',
+        description: 'Mark task as finished'
+    },
+    call_user: {
+        category: 'system',
+        description: 'Request user interaction'
+    }
+};
+function isSupportedActionType(type) {
+    return type in ACTION_METADATA;
+}
+function isValidActionType(type) {
+    return 'screenshot' === type || isSupportedActionType(type);
+}
+export { ACTION_METADATA, isSupportedActionType, isValidActionType };
+
+//# sourceMappingURL=actions.mjs.map

package/dist/types/actions.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"types/actions.mjs","sources":["webpack://@ui-tars-test/shared/./src/types/actions.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\n\n/* eslint-disable @typescript-eslint/no-explicit-any */\n\nexport type Factors = [number, number];\n\n/**\n * Coordinate data structure\n * - Supports pixel coordinates (raw)\n * - Supports normalized coordinates (normalized)\n */\nexport interface Coordinates {\n  raw?: { x: number; y: number }; // Raw pixels\n  normalized?: { x: number; y: number }; // Normalized coordinates (0–1)\n  referenceBox?: { x1: number; y1: number; x2: number; y2: number };\n  referenceSystem?: 'screen' | 'window' | 'browserPage' | string; // Coordinate reference system\n}\n\n/**\n * Standard structure for GUI Actions\n */\nexport interface BaseAction<\n  T extends string = string,\n  I extends Record<string, any> = Record<string, any>,\n> {\n  type: T; // Action type (e.g., \"click\", \"key\", \"swipe\")\n  inputs: I; // Parameters required for the action\n  meta?: {\n    toolHint?: string; // Suggested execution tool (xdotool / adb / pyautogui etc.)\n    comment?: string; // Notes / Debug information\n  };\n}\n\n// ---------- ScreenShot Action ----------\n\n/**\n * ScreenShot action\n */\nexport type ScreenShotAction = BaseAction<\n  'screenshot',\n  {\n    start?: Coordinates;\n    end?: Coordinates;\n  }\n>;\n\n// ---------- Mouse Actions ----------\n\n/**\n * Click action with coordinates\n */\nexport type ClickAction = BaseAction<\n  'click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Right click action with coordinates\n */\nexport type RightClickAction = BaseAction<\n  'right_click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Double click action with coordinates\n */\nexport type DoubleClickAction = BaseAction<\n  'double_click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Middle click action with coordinates\n */\nexport type MiddleClickAction = BaseAction<\n  'middle_click',\n  {\n    point: Coordinates;\n  }\n>;\n\n/**\n * Mouse down action\n */\nexport type MouseDownAction = BaseAction<\n  'mouse_down',\n  {\n    point?: Coordinates; // Mouse down position. If not specified, default to execute on the current mouse position.\n    button?: 'left' | 'right'; // Down button. Default to left.\n  }\n>;\n\n/**\n * Mouse up action\n */\nexport type MouseUpAction = BaseAction<\n  'mouse_up',\n  {\n    point?: Coordinates; // Mouse up position. If not specified, default to execute on the current mouse position.\n    button?: 'left' | 'right'; // Up button. Default to left.\n  }\n>;\n\n/**\n * Mouse move action\n */\nexport type MouseMoveAction = BaseAction<\n  'mouse_move', // 'move' | 'move_to' | 'hover',\n  {\n    point: Coordinates; // Target coordinates\n  }\n>;\n\n/**\n * Drag action with start and end coordinates\n */\nexport type DragAction = BaseAction<\n  'drag', // 'left_click_drag' | 'select',\n  {\n    start: Coordinates;\n    end: Coordinates;\n    direction?: 'up' | 'down' | 'left' | 'right';\n  }\n>;\n\n/**\n * Scroll action with coordinates and direction\n */\nexport type ScrollAction = BaseAction<\n  'scroll',\n  {\n    point?: Coordinates;\n    direction: 'up' | 'down' | 'left' | 'right';\n  }\n>;\n\n// ---------- Keyboard Actions ----------\n\n/**\n * Type action with text content\n */\nexport type TypeAction = BaseAction<\n  'type',\n  {\n    content: string;\n  }\n>;\n\n/**\n * Hotkey action with key combination\n */\nexport type HotkeyAction = BaseAction<\n  'hotkey',\n  {\n    key: string;\n  }\n>;\n\n/**\n * Press key action\n */\nexport type PressAction = BaseAction<\n  'press',\n  {\n    key: string; // Key you want to press. Only one key can be pressed at one time.\n  }\n>;\n\n/**\n * Release key action\n */\nexport type ReleaseAction = BaseAction<\n  'release',\n  {\n    key: string; // Key you want to release. Only one key can be released at one time.\n  }\n>;\n\n// ---------- Browser Actions ----------\n\n/**\n * Navigate action with URL\n */\nexport type NavigateAction = BaseAction<\n  'navigate',\n  {\n    url: string;\n  }\n>;\n\n/**\n * Navigate back action\n */\nexport type NavigateBackAction = BaseAction<'navigate_back', Record<string, never>>;\n\n// ---------- App Actions ----------\n\n/**\n * Long press action with coordinates\n */\nexport type LongPressAction = BaseAction<\n  'long_press',\n  {\n    point: Coordinates;\n  }\n>;\n\nexport type SwipeAction = BaseAction<\n  'swipe', // 'drag',\n  {\n    start: Coordinates;\n    end: Coordinates;\n    direction: 'up' | 'down' | 'left' | 'right';\n  }\n>;\n\n/**\n * Home action\n */\nexport type HomeAction = BaseAction<'home' | 'press_home', Record<string, never>>;\n\n/**\n * Back action\n */\nexport type BackAction = BaseAction<'back' | 'press_back', Record<string, never>>;\n\n/**\n * Open app action\n */\nexport type OpenAppAction = BaseAction<\n  'open_app',\n  {\n    name: string;\n  }\n>;\n\n// ---------- Wait Actions ----------\n\n/**\n * Wait action with no inputs\n */\nexport type WaitAction = BaseAction<\n  'wait',\n  {\n    time?: number; // in seconds (optional)\n  }\n>;\n\n/**\n * Finished - Complete the current operation.\n */\nexport type FinishAction = BaseAction<\n  'finished',\n  {\n    content?: string;\n  }\n>;\n\n/**\n * CallUser - Request user interaction.\n */\nexport type CallUserAction = BaseAction<\n  'call_user',\n  {\n    content?: string;\n  }\n>;\n\n/**\n * Operational action types (excluding screenshot which has special handling)\n */\nexport type OperationalGUIAction =\n  | ClickAction\n  | DoubleClickAction\n  | RightClickAction\n  | MiddleClickAction\n  | MouseDownAction\n  | MouseUpAction\n  | MouseMoveAction\n  | DragAction\n  | ScrollAction\n  | TypeAction\n  | HotkeyAction\n  | PressAction\n  | ReleaseAction\n  | NavigateAction\n  | NavigateBackAction\n  | LongPressAction\n  | SwipeAction\n  | HomeAction\n  | BackAction\n  | OpenAppAction\n  | WaitAction\n  | FinishAction\n  | CallUserAction;\n\n/**\n * Complete GUI action types including screenshot\n */\nexport type GUIAction = ScreenShotAction | OperationalGUIAction;\n\n/**\n * Extract action type from action interface\n */\nexport type ExtractActionType<T> = T extends BaseAction<infer U, any> ? U : never;\n\n/**\n * Supported operational action types (excluding screenshot)\n */\nexport type SupportedActionType = ExtractActionType<OperationalGUIAction>;\n\n/**\n * All action types including screenshot\n */\nexport type AllActionType = ExtractActionType<GUIAction>;\n\n/**\n * Action metadata for documentation and serialization\n */\nexport interface ActionMetadata {\n  description: string;\n  category: 'mouse' | 'keyboard' | 'navigation' | 'mobile' | 'system' | 'wait';\n}\n\n/**\n * Comprehensive action metadata registry\n */\nexport const ACTION_METADATA: Record<SupportedActionType, ActionMetadata> = {\n  click: { category: 'mouse', description: 'Click on an element' },\n  right_click: { category: 'mouse', description: 'Right click on an element' },\n  double_click: { category: 'mouse', description: 'Double click on an element' },\n  middle_click: { category: 'mouse', description: 'Middle click on an element' },\n  mouse_down: { category: 'mouse', description: 'Press mouse button down' },\n  mouse_up: { category: 'mouse', description: 'Release mouse button' },\n  mouse_move: { category: 'mouse', description: 'Move mouse to position' },\n  drag: { category: 'mouse', description: 'Drag from one position to another' },\n  scroll: { category: 'mouse', description: 'Scroll in a direction' },\n  type: { category: 'keyboard', description: 'Type text' },\n  hotkey: { category: 'keyboard', description: 'Press hotkey combination' },\n  press: { category: 'keyboard', description: 'Press a key' },\n  release: { category: 'keyboard', description: 'Release a key' },\n  navigate: { category: 'navigation', description: 'Navigate to URL' },\n  navigate_back: { category: 'navigation', description: 'Navigate back' },\n  long_press: { category: 'mobile', description: 'Long press on element' },\n  swipe: { category: 'mobile', description: 'Swipe gesture' },\n  home: { category: 'mobile', description: 'Go to home' },\n  press_home: { category: 'mobile', description: 'Press home button' },\n  back: { category: 'mobile', description: 'Go back' },\n  press_back: { category: 'mobile', description: 'Press back button' },\n  open_app: { category: 'mobile', description: 'Open application' },\n  wait: { category: 'wait', description: 'Wait for specified time' },\n  finished: { category: 'system', description: 'Mark task as finished' },\n  call_user: { category: 'system', description: 'Request user interaction' },\n} as const;\n\n/**\n * Type guard function to check if a string is a valid operational action type\n * @param type - The string to check\n * @returns Whether the string is a valid SupportedActionType\n */\nexport function isSupportedActionType(type: string): type is SupportedActionType {\n  return type in ACTION_METADATA;\n}\n\n/**\n * Type guard function to check if a string is any valid action type (including screenshot)\n * @param type - The string to check\n * @returns Whether the string is a valid AllActionType\n */\nexport function isValidActionType(type: string): type is AllActionType {\n  return type === 'screenshot' || isSupportedActionType(type);\n}\n"],"names":["ACTION_METADATA","isSupportedActionType","type","isValidActionType"],"mappings":";;;;AAiVO,MAAMA,kBAA+D;IAC1E,OAAO;QAAE,UAAU;QAAS,aAAa;IAAsB;IAC/D,aAAa;QAAE,UAAU;QAAS,aAAa;IAA4B;IAC3E,cAAc;QAAE,UAAU;QAAS,aAAa;IAA6B;IAC7E,cAAc;QAAE,UAAU;QAAS,aAAa;IAA6B;IAC7E,YAAY;QAAE,UAAU;QAAS,aAAa;IAA0B;IACxE,UAAU;QAAE,UAAU;QAAS,aAAa;IAAuB;IACnE,YAAY;QAAE,UAAU;QAAS,aAAa;IAAyB;IACvE,MAAM;QAAE,UAAU;QAAS,aAAa;IAAoC;IAC5E,QAAQ;QAAE,UAAU;QAAS,aAAa;IAAwB;IAClE,MAAM;QAAE,UAAU;QAAY,aAAa;IAAY;IACvD,QAAQ;QAAE,UAAU;QAAY,aAAa;IAA2B;IACxE,OAAO;QAAE,UAAU;QAAY,aAAa;IAAc;IAC1D,SAAS;QAAE,UAAU;QAAY,aAAa;IAAgB;IAC9D,UAAU;QAAE,UAAU;QAAc,aAAa;IAAkB;IACnE,eAAe;QAAE,UAAU;QAAc,aAAa;IAAgB;IACtE,YAAY;QAAE,UAAU;QAAU,aAAa;IAAwB;IACvE,OAAO;QAAE,UAAU;QAAU,aAAa;IAAgB;IAC1D,MAAM;QAAE,UAAU;QAAU,aAAa;IAAa;IACtD,YAAY;QAAE,UAAU;QAAU,aAAa;IAAoB;IACnE,MAAM;QAAE,UAAU;QAAU,aAAa;IAAU;IACnD,YAAY;QAAE,UAAU;QAAU,aAAa;IAAoB;IACnE,UAAU;QAAE,UAAU;QAAU,aAAa;IAAmB;IAChE,MAAM;QAAE,UAAU;QAAQ,aAAa;IAA0B;IACjE,UAAU;QAAE,UAAU;QAAU,aAAa;IAAwB;IACrE,WAAW;QAAE,UAAU;QAAU,aAAa;IAA2B;AAC3E;AAOO,SAASC,sBAAsBC,IAAY;IAChD,OAAOA,QAAQF;AACjB;AAOO,SAASG,kBAAkBD,IAAY;IAC5C,OAAOA,AAAS,iBAATA,QAAyBD,sBAAsBC;AACxD"}

package/dist/types/agents.d.ts

@@ -0,0 +1,108 @@
+import { AgentOptions } from '@tarko/agent-interface';
+import { Factors, BaseAction, Coordinates, SupportedActionType } from './actions';
+/**
+ * Type definition for parsed GUI response structure
+ * Represents the components extracted from a model's output string
+ * Aligned with tarko's web UI design
+ */
+export interface ParsedGUIResponse {
+    /** raw prediction string */
+    rawContent: string;
+    /** parsed from Thought: `<thought>` */
+    reasoningContent?: string;
+    /** parsed from Action: action(params=`action`) */
+    rawActionStrings?: string[];
+    /** parsed from Action: action(params=`action`) */
+    actions: BaseAction[];
+    /** error message to feedback to LLM */
+    errorMessage?: string;
+}
+/**
+ * Type definition for function to normalize raw coordinates
+ * Converts raw pixel coordinates to normalized coordinates (0-1)
+ * @param rawX - Raw X coordinate in pixels
+ * @param rawY - Raw Y coordinate in pixels
+ * @returns Normalized coordinates {x, y} with values between 0 and 1
+ */
+export type NormalizeCoordinates = (rawCoords: Coordinates, factors?: Factors) => {
+    normalized: Coordinates;
+};
+/**
+ * Type definition for handler function to parse model output into ParsedGUIResponse object
+ * @param prediction - The raw output from the model to be parsed
+ * @returns ParsedGUIResponse object if parsing is successful, null otherwise
+ */
+export type CustomActionParser = (prediction: string) => ParsedGUIResponse | null;
+/**
+ * Function type for serializing supported actions to string format
+ * @param actions - Array of supported action types
+ * @returns String representation of the actions for agent processing
+ */
+export type SerializeSupportedActions = (actions: Array<SupportedActionType>) => string;
+export type ExecuteParams = {
+    /** Required actions to execute */
+    actions: BaseAction[];
+} & Partial<Omit<ParsedGUIResponse, 'actions'>> & Record<string, any>;
+export type ExecuteOutput = {
+    status: 'success' | 'failed';
+    errorMessage?: string;
+    url?: string;
+} & Record<string, any>;
+/**
+ * Function type for calculating detail level based on image dimensions
+ */
+export type ImageDetailCalculator = (width: number, height: number) => 'low' | 'high' | 'auto';
+export interface ScreenshotOutput extends ExecuteOutput {
+    /** screenshot base64, `keep screenshot size as physical pixels` */
+    base64: string;
+}
+/**
+ * Reserved placeholder name for action space in system prompt template
+ */
+export declare const ACTION_SPACE_PLACEHOLDER = "action_space";
+/**
+ * Interface for system prompt template configuration
+ */
+export interface SystemPromptTemplate {
+    /**
+     * Template string with placeholders. Must include an action space placeholder
+     * `{{${ACTION_SPACE_PLACEHOLDER}}}` that will be replaced with the string representation of available actions
+     */
+    template: string;
+    /**
+     * Function to convert BaseAction array to string representation for the action space
+     * This will be used to fill the action space placeholder in the template
+     */
+    actionsToString?: SerializeSupportedActions;
+    /**
+     * Optional map of additional placeholder values to be replaced in the template
+     * Keys are placeholder names, values are the replacement strings
+     * Note: '${ACTION_SPACE_PLACEHOLDER}' is a reserved placeholder and should NOT be included here
+     * as it will be automatically filled using the actionsToString function
+     */
+    placeholders?: Record<string, string>;
+}
+export interface GUIAgentConfig<TOperator> extends AgentOptions {
+    operator: TOperator;
+    /**
+     * System prompt configuration. Can be either:
+     * - A simple string (legacy mode)
+     * - An array of strings or chat messages (will be concatenated)
+     * - A SystemPromptTemplate object with template and actionsToString function
+     */
+    systemPrompt?: string | Array<string | {
+        role: string;
+        content: string;
+    }> | SystemPromptTemplate;
+    /** The handler function to parse model output into PredictionParsed object */
+    customeActionParser?: CustomActionParser;
+    /** The function to normalize raw coordinates */
+    normalizeCoordinates?: NormalizeCoordinates;
+    /** The function to calculate detail level based on image dimensions */
+    detailCalculator?: ImageDetailCalculator;
+    /** Maximum number of turns for Agent to execute, @default 1000 */
+    maxLoopCount?: number;
+    /** Time interval between two loop iterations (in milliseconds), @default 0 */
+    loopIntervalInMs?: number;
+}
+//# sourceMappingURL=agents.d.ts.map