@ui-tars-test/shared 0.3.2

package/dist/utils/systemPromptProcessor.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils/systemPromptProcessor.mjs","sources":["webpack://@ui-tars-test/shared/./src/utils/systemPromptProcessor.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\n\nimport { SystemPromptTemplate, ACTION_SPACE_PLACEHOLDER } from '../types/agents';\nimport { SupportedActionType } from '../types/actions';\n\n/**\n * Assemble system prompt template by replacing placeholders with actual values\n * @param template - The system prompt template configuration\n * @param supportedActions - Array of supported action types\n * @returns Assembled system prompt string with placeholders replaced\n */\nexport function assembleSystemPrompt(\n  template: SystemPromptTemplate,\n  supportedActions: SupportedActionType[],\n): string {\n  let assembledPrompt = template.template;\n\n  // Replace action space placeholder if actionsToString function is provided\n  if (template.actionsToString) {\n    const actionSpacePlaceholder = `{{${ACTION_SPACE_PLACEHOLDER}}}`;\n    const realActionSpaces = template.actionsToString(supportedActions);\n    assembledPrompt = assembledPrompt.replace(actionSpacePlaceholder, realActionSpaces || '');\n  }\n\n  // Handle other custom placeholders replacement\n  if (template.placeholders) {\n    Object.entries(template.placeholders).forEach(([key, value]) => {\n      const placeholder = `{{${key}}}`;\n      assembledPrompt = assembledPrompt.replace(new RegExp(placeholder, 'g'), String(value));\n    });\n  }\n\n  return assembledPrompt;\n}\n\n/**\n * Type guard to check if a system prompt is a template object\n * @param systemPrompt - The system prompt to check\n * @returns Whether the system prompt is a SystemPromptTemplate\n */\nexport function isSystemPromptTemplate(\n  systemPrompt: string | SystemPromptTemplate,\n): systemPrompt is SystemPromptTemplate {\n  return typeof systemPrompt === 'object' && 'template' in systemPrompt;\n}\n"],"names":["assembleSystemPrompt","template","supportedActions","assembledPrompt","actionSpacePlaceholder","ACTION_SPACE_PLACEHOLDER","realActionSpaces","Object","key","value","placeholder","RegExp","String","isSystemPromptTemplate","systemPrompt"],"mappings":";;;;;AAcO,SAASA,qBACdC,QAA8B,EAC9BC,gBAAuC;IAEvC,IAAIC,kBAAkBF,SAAS,QAAQ;IAGvC,IAAIA,SAAS,eAAe,EAAE;QAC5B,MAAMG,yBAAyB,CAAC,EAAE,EAAEC,yBAAyB,EAAE,CAAC;QAChE,MAAMC,mBAAmBL,SAAS,eAAe,CAACC;QAClDC,kBAAkBA,gBAAgB,OAAO,CAACC,wBAAwBE,oBAAoB;IACxF;IAGA,IAAIL,SAAS,YAAY,EACvBM,OAAO,OAAO,CAACN,SAAS,YAAY,EAAE,OAAO,CAAC,CAAC,CAACO,KAAKC,MAAM;QACzD,MAAMC,cAAc,CAAC,EAAE,EAAEF,IAAI,EAAE,CAAC;QAChCL,kBAAkBA,gBAAgB,OAAO,CAAC,IAAIQ,OAAOD,aAAa,MAAME,OAAOH;IACjF;IAGF,OAAON;AACT;AAOO,SAASU,uBACdC,YAA2C;IAE3C,OAAO,AAAwB,YAAxB,OAAOA,gBAA6B,cAAcA;AAC3D"}

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "@ui-tars-test/shared",
+  "version": "0.3.2",
+  "description": "Shared types for GUI-Agent",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/bytedance/UI-TARS-desktop"
+  },
+  "bugs": {
+    "url": "https://github.com/bytedance/UI-TARS-desktop/issues"
+  },
+  "exports": {
+    "./types": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/types/index.mjs",
+      "require": "./dist/types/index.js"
+    },
+    "./base": {
+      "types": "./dist/base/index.d.ts",
+      "import": "./dist/base/index.mjs",
+      "require": "./dist/base/index.js"
+    },
+    "./utils": {
+      "types": "./dist/utils/index.d.ts",
+      "import": "./dist/utils/index.mjs",
+      "require": "./dist/utils/index.js"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      "*": [
+        "./src/*/index.ts"
+      ]
+    }
+  },
+  "scripts": {
+    "dev": "rslib build --watch",
+    "build": "rslib build",
+    "build:watch": "rslib build --watch",
+    "test": "vitest"
+  },
+  "keywords": [
+    "UI-TARS",
+    "GUI Agent",
+    "SDK"
+  ],
+  "license": "Apache-2.0",
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@agent-infra/logger": "0.0.2-beta.2",
+    "@tarko/agent": "^0.3.0",
+    "@tarko/agent-interface": "^0.3.0"
+  },
+  "devDependencies": {
+    "@rslib/core": "0.10.0",
+    "typescript": "^5.7.2",
+    "vitest": "^3.0.2"
+  }
+}

package/src/base/agent.ts

@@ -0,0 +1,13 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import Agent from '@tarko/agent';
+
+/**
+ * @abstract
+ * @class BaseGUIAgent
+ * @classdesc Abstract base class for GUI Agents.
+ */
+export abstract class BaseGUIAgent extends Agent {}

package/src/base/index.ts

@@ -0,0 +1,7 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+export * from './agent';
+export * from './parser';
+export * from './operator';

package/src/base/operator.ts

@@ -0,0 +1,221 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ExecuteParams, ScreenshotOutput, ExecuteOutput, SupportedActionType } from '../types';
+
+export interface ScreenContext {
+  screenWidth: number;
+  screenHeight: number;
+  scaleX: number;
+  scaleY: number;
+}
+
+/**
+ * @abstract
+ * @class BaseOperator
+ * @classdesc Abstract base class for Operators.
+ */
+export abstract class BaseOperator {
+  abstract doScreenshot(params?: unknown): Promise<unknown>;
+  abstract doExecute(params: unknown): Promise<unknown>;
+}
+
+/**
+ * @abstract
+ * @class Operator
+ * @classdesc Abstract base class for Operators.
+ *
+ * @example
+ * // Example of defining ACTION_SPACES for a custom Operator
+ * import type { GUIAction, ClickAction, DoubleClickAction, TypeAction, ScreenShotAction, SupportedActionType } from '../types/actions';
+ *
+ * class MyDesktopOperator extends Operator {
+ *
+ *   // Implement the required abstract methods
+ *   protected async initialize(): Promise<void> {
+ *     // Implementation for initializing the operator
+ *     // e.g., validate connections, setup resources
+ *     // ...
+ *   }
+ *
+ *   supportedActions(): Array<SupportedActionType> {
+ *     return [
+ *       'click',
+ *       'double_click',
+ *       'right_click',
+ *       'type',
+ *       'hotkey',
+ *       'scroll',
+ *       'drag',
+ *       'screenshot'
+ *     ] as SupportedActionType[];
+ *   }
+ *
+ *   screenContext(): ScreenContext {
+ *     // Implementation for getting screen context
+ *     // ...
+ *     return {
+ *       screenWidth: 1920,
+ *       screenHeight: 1080,
+ *       scaleX: 1,
+ *       scaleY: 1
+ *     };
+ *   }
+ *
+ *   async screenshot(): Promise<ScreenshotOutput> {
+ *     // Implementation for taking screenshots
+ *     // ...
+ *     return { }; // screenshot output
+ *   }
+ *
+ *   async execute(params: ExecuteParams): Promise<ExecuteOutput> {
+ *     // Implementation for executing actions
+ *     // ...
+ *     return { }; // execution output
+ *   }
+ * }
+ */
+export abstract class Operator extends BaseOperator {
+  // Track initialization state
+  private _initialized = false;
+  private _initializing = false;
+  private _initPromise: Promise<void> | null = null;
+
+  constructor() {
+    super();
+    // this.ensureInitialized();
+  }
+
+  /**
+   * Initializes the operator
+   * @description Performs initialization operations for the operator, such as validating connections,
+   * setting up resources, and preparing the operation environment.
+   * @returns Promise that resolves when initialization is complete
+   * @throws Error if initialization fails
+   */
+  async doInitialize(): Promise<void> {
+    // If already initialized, return immediately
+    if (this._initialized) {
+      return;
+    }
+
+    // If initialization is in progress, wait for it to complete
+    if (this._initializing && this._initPromise) {
+      return this._initPromise;
+    }
+
+    // Start initialization
+    this._initializing = true;
+    this._initPromise = (async () => {
+      try {
+        await this.initialize();
+        this._initialized = true;
+      } finally {
+        this._initializing = false;
+      }
+    })();
+
+    return this._initPromise;
+  }
+
+  /**
+   * Implementation of initialization logic
+   * @description Subclasses should implement this method to perform their specific initialization
+   * @returns Promise that resolves when initialization is complete
+   * @throws Error if initialization fails
+   */
+  protected abstract initialize(): Promise<void>;
+
+  /**
+   * Ensures the operator is initialized before performing operations
+   * @private
+   */
+  private async ensureInitialized(): Promise<void> {
+    if (!this._initialized && !this._initializing) {
+      await this.doInitialize();
+    } else if (this._initializing && this._initPromise) {
+      await this._initPromise;
+    }
+  }
+
+  /**
+   * Safely returns an array of supported action types with initialization guarantee
+   * @returns Array of action types supported by this operator
+   */
+  getSupportedActions(): Array<SupportedActionType> {
+    // await this.ensureInitialized();
+    return this.supportedActions();
+  }
+
+  /**
+   * Returns an array of supported action types
+   * @returns Array of action types supported by this operator
+   */
+  protected abstract supportedActions(): Array<SupportedActionType>;
+
+  /**
+   * Safely returns the screen context with initialization guarantee
+   * @returns The screen context
+   */
+  async getScreenContext(): Promise<ScreenContext> {
+    await this.ensureInitialized();
+    return this.screenContext();
+  }
+
+  /**
+   * Returns the screen context
+   * @returns The screen context
+   */
+  protected abstract screenContext(): ScreenContext;
+
+  /**
+   * Safely takes a screenshot with initialization guarantee
+   * @returns Promise that resolves to the screenshot output
+   */
+  async doScreenshot(): Promise<ScreenshotOutput> {
+    try {
+      await this.ensureInitialized();
+      return await this.screenshot();
+    } catch (error) {
+      console.error('Error in doScreenshot:', error);
+      return {
+        base64: '',
+        status: 'failed',
+        errorMessage: (error as Error).message,
+      };
+    }
+  }
+
+  /**
+   * Takes a screenshot
+   * @returns Promise that resolves to the screenshot output
+   */
+  protected abstract screenshot(): Promise<ScreenshotOutput>;
+
+  /**
+   * Safely executes actions with initialization guarantee
+   * @param params - The parameters for the actions
+   * @returns Promise that resolves to the execution output
+   */
+  async doExecute(params: ExecuteParams): Promise<ExecuteOutput> {
+    try {
+      await this.ensureInitialized();
+      return await this.execute(params);
+    } catch (error) {
+      console.error('Error in doExecute:', error);
+      return {
+        status: 'failed',
+        errorMessage: (error as Error).message,
+      };
+    }
+  }
+
+  /**
+   * Executes actions
+   * @param params - The parameters for the actions
+   * @returns Promise that resolves to the execution output
+   */
+  protected abstract execute(params: ExecuteParams): Promise<ExecuteOutput>;
+}

package/src/base/parser.ts

@@ -0,0 +1,16 @@
+/*
+ * Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import { ParsedGUIResponse } from '../types';
+
+export abstract class BaseActionParser {
+  /**
+   * Parse model output
+   * @param input Model output string
+   * @returns Parsed ParsedGUIResponse object, returns null if parsing fails
+   * There is no need to throw error, the error message is returned in ParsedGUIResponse
+   */
+  abstract parsePrediction(input: string): ParsedGUIResponse | null;
+}