@ui-tars-test/agent-sdk 0.3.0

package/README.md

@@ -0,0 +1,13 @@
+# @multimodal/seed-gui-agent
+
+Seed GUI Agent.
+
+## Installation
+
+```bash
+npm install @multimodal/seed-gui-agent
+```
+
+## Usage
+
+// TODO

package/dist/GUIAgent.d.ts

@@ -0,0 +1,24 @@
+import { LLMRequestHookPayload } from '@tarko/agent';
+import { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';
+import { GUIAgentConfig } from '@ui-tars-test/shared/types';
+export declare class GUIAgent<T extends Operator> extends BaseGUIAgent {
+    static label: string;
+    private operator;
+    private normalizeCoordinates;
+    private detailCalculator;
+    private loopIntervalInMs;
+    constructor(config: GUIAgentConfig<T>);
+    initialize(): Promise<void>;
+    onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void>;
+    onEachAgentLoopStart(sessionId: string): Promise<void>;
+    onAgentLoopEnd(id: string): Promise<void>;
+    onBeforeToolCall(id: string, toolCall: {
+        toolCallId: string;
+        name: string;
+    }, args: unknown): Promise<unknown>;
+    onAfterToolCall(id: string, toolCall: {
+        toolCallId: string;
+        name: string;
+    }, result: unknown): Promise<unknown>;
+}
+//# sourceMappingURL=GUIAgent.d.ts.map

package/dist/GUIAgent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GUIAgent.d.ts","sourceRoot":"","sources":["../src/GUIAgent.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,qBAAqB,EAKtB,MAAM,cAAc,CAAC;AAItB,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EACL,cAAc,EAGf,MAAM,4BAA4B,CAAC;AAapC,qBAAa,QAAQ,CAAC,CAAC,SAAS,QAAQ,CAAE,SAAQ,YAAY;IAC5D,MAAM,CAAC,KAAK,SAAe;IAE3B,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,gBAAgB,CAAS;gBAErB,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IA+C/B,UAAU;IA0CV,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvE,oBAAoB,CAAC,SAAS,EAAE,MAAM;IAItC,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,gBAAgB,CACpB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAC9C,IAAI,EAAE,OAAO;IAKT,eAAe,CACnB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAC9C,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,OAAO,CAAC;CA4DpB"}

package/dist/GUIAgent.js

@@ -0,0 +1,207 @@
+/**
+* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+* SPDX-License-Identifier: Apache-2.0
+*/
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    GUIAgent: ()=>GUIAgent
+});
+const agent_namespaceObject = require("@tarko/agent");
+const external_ToolCallEngine_js_namespaceObject = require("./ToolCallEngine.js");
+const external_prompts_js_namespaceObject = require("./prompts.js");
+const media_utils_namespaceObject = require("@agent-infra/media-utils");
+const base_namespaceObject = require("@ui-tars-test/shared/base");
+const utils_namespaceObject = require("@ui-tars-test/shared/utils");
+const external_constants_js_namespaceObject = require("./constants.js");
+const external_utils_js_namespaceObject = require("./utils.js");
+const external_defaultImpls_js_namespaceObject = require("./defaultImpls.js");
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const defaultLogger = new agent_namespaceObject.ConsoleLogger('[GUIAgent]', agent_namespaceObject.LogLevel.DEBUG);
+class GUIAgent extends base_namespaceObject.BaseGUIAgent {
+    async initialize() {
+        this.registerTool(new agent_namespaceObject.Tool({
+            id: external_constants_js_namespaceObject.GUI_ADAPTED_TOOL_NAME,
+            description: 'operator tool',
+            parameters: {},
+            function: async (input)=>{
+                this.logger.log(`${external_constants_js_namespaceObject.GUI_ADAPTED_TOOL_NAME} input:`, input);
+                if (!this.operator) return (0, external_utils_js_namespaceObject.createGUIErrorResponse)(input.action, 'Operator not initialized');
+                if (input.errorMessage) return (0, external_utils_js_namespaceObject.createGUIErrorResponse)(input.action, input.errorMessage);
+                if (input.operator_action) input.operator_action = (0, utils_namespaceObject.normalizeActionCoords)(input.operator_action, this.normalizeCoordinates);
+                this.logger.info('action to execute:', JSON.stringify(input.operator_action));
+                const result = await this.operator.doExecute({
+                    actions: [
+                        input.operator_action
+                    ]
+                });
+                if (result.errorMessage) return (0, external_utils_js_namespaceObject.createGUIErrorResponse)(input.action, result.errorMessage);
+                return {
+                    success: true,
+                    action: input.action,
+                    normalizedAction: (0, external_utils_js_namespaceObject.convertToAgentUIAction)(input.operator_action),
+                    observation: void 0
+                };
+            }
+        }));
+        super.initialize();
+    }
+    async onLLMRequest(id, payload) {
+        try {
+            const safeStringify = (obj, max = 800)=>{
+                try {
+                    const s = JSON.stringify(obj);
+                    return s.length > max ? s.slice(0, max) + "\u2026(truncated)" : s;
+                } catch  {
+                    return '[unserializable]';
+                }
+            };
+            const req = null == payload ? void 0 : payload.request;
+            const messages = req && Array.isArray(req.messages) ? req.messages : [];
+            const model = req && 'string' == typeof req.model ? req.model : 'unknown';
+            const hasImages = (()=>{
+                try {
+                    let cnt = 0;
+                    for (const m of messages){
+                        const content = null == m ? void 0 : m.content;
+                        if (Array.isArray(content)) {
+                            for (const part of content)if ((null == part ? void 0 : part.type) === 'image_url') cnt++;
+                        }
+                    }
+                    return cnt;
+                } catch  {
+                    return;
+                }
+            })();
+            const summary = {
+                id,
+                model,
+                messagesCount: messages.length,
+                hasImages
+            };
+            this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));
+            const firstMsg = messages[0];
+            if (firstMsg) this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));
+        } catch (e) {
+            this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);
+        }
+    }
+    async onEachAgentLoopStart(sessionId) {
+        this.logger.info('onEachAgentLoopStart', sessionId);
+    }
+    async onAgentLoopEnd(id) {}
+    async onBeforeToolCall(id, toolCall, args) {
+        return args;
+    }
+    async onAfterToolCall(id, toolCall, result) {
+        this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));
+        if (toolCall.name !== external_constants_js_namespaceObject.GUI_ADAPTED_TOOL_NAME) return void this.logger.info('onAfterToolCall: skipping screenshot');
+        await (0, utils_namespaceObject.sleep)(this.loopIntervalInMs);
+        const output = await this.operator.doScreenshot();
+        if (!output) return void this.logger.error('Failed to get screenshot');
+        const base64Tool = new media_utils_namespaceObject.Base64ImageParser(output.base64);
+        const base64Uri = base64Tool.getDataUri();
+        if (!base64Uri) return void this.logger.error('Failed to get base64 image uri');
+        const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() || {
+            width: -1,
+            height: -1
+        };
+        const content = [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: base64Uri,
+                    detail: this.detailCalculator(imageWidth, imageHeight)
+                }
+            }
+        ];
+        if (null == output ? void 0 : output.url) content.push({
+            type: 'text',
+            text: `The current page's url: ${output.url}`
+        });
+        const eventStream = this.getEventStream();
+        const events = eventStream.getEvents();
+        this.logger.info('onAfterToolCall events length:', events.length);
+        const event = eventStream.createEvent('environment_input', {
+            description: 'Browser Screenshot',
+            content,
+            metadata: {
+                type: 'screenshot',
+                url: null == output ? void 0 : output.url
+            }
+        });
+        eventStream.sendEvent(event);
+        return result;
+    }
+    constructor(config){
+        const { operator, model, systemPrompt, customeActionParser, normalizeCoordinates, detailCalculator, maxLoopCount, loopIntervalInMs = 500 } = config;
+        let finalSystemPrompt = external_prompts_js_namespaceObject.SYSTEM_PROMPT;
+        if ('string' == typeof systemPrompt) finalSystemPrompt = systemPrompt;
+        else if (Array.isArray(systemPrompt)) finalSystemPrompt = systemPrompt.map((p)=>'string' == typeof p ? p : p.content).join('\n\n');
+        else if (systemPrompt && (0, utils_namespaceObject.isSystemPromptTemplate)(systemPrompt)) finalSystemPrompt = (0, utils_namespaceObject.assembleSystemPrompt)(systemPrompt, operator.getSupportedActions());
+        defaultLogger.debug('final instructions for sp:', finalSystemPrompt);
+        const AdaptedToolCallEngine = class extends external_ToolCallEngine_js_namespaceObject.GUIAgentToolCallEngine {
+            constructor(){
+                super(customeActionParser);
+            }
+        };
+        super({
+            name: GUIAgent.label,
+            instructions: finalSystemPrompt,
+            tools: [],
+            toolCallEngine: AdaptedToolCallEngine,
+            model: model,
+            ...maxLoopCount && {
+                maxIterations: maxLoopCount
+            },
+            logLevel: agent_namespaceObject.LogLevel.DEBUG
+        }), _define_property(this, "operator", void 0), _define_property(this, "normalizeCoordinates", void 0), _define_property(this, "detailCalculator", void 0), _define_property(this, "loopIntervalInMs", void 0);
+        this.operator = operator;
+        this.normalizeCoordinates = normalizeCoordinates ?? external_defaultImpls_js_namespaceObject.defaultNormalizeCoords;
+        this.detailCalculator = detailCalculator ?? external_defaultImpls_js_namespaceObject.defaultDetailCalculator;
+        this.loopIntervalInMs = loopIntervalInMs;
+        this.logger = this.logger.spawn('[GUIAgent]');
+    }
+}
+_define_property(GUIAgent, "label", 'GUI Agent');
+exports.GUIAgent = __webpack_exports__.GUIAgent;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "GUIAgent"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=GUIAgent.js.map

package/dist/GUIAgent.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GUIAgent.js","sources":["webpack://@ui-tars-test/agent-sdk/webpack/runtime/define_property_getters","webpack://@ui-tars-test/agent-sdk/webpack/runtime/has_own_property","webpack://@ui-tars-test/agent-sdk/webpack/runtime/make_namespace_object","webpack://@ui-tars-test/agent-sdk/./src/GUIAgent.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n  LLMRequestHookPayload,\n  ChatCompletionContentPart,\n  LogLevel,\n  Tool,\n  ConsoleLogger,\n} from '@tarko/agent';\nimport { GUIAgentToolCallEngine } from './ToolCallEngine';\nimport { SYSTEM_PROMPT } from './prompts';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';\nimport {\n  GUIAgentConfig,\n  NormalizeCoordinates,\n  ImageDetailCalculator,\n} from '@ui-tars-test/shared/types';\nimport {\n  assembleSystemPrompt,\n  isSystemPromptTemplate,\n  normalizeActionCoords,\n  sleep,\n} from '@ui-tars-test/shared/utils';\nimport { GUI_ADAPTED_TOOL_NAME } from './constants';\nimport { convertToAgentUIAction, createGUIErrorResponse } from './utils';\nimport { defaultNormalizeCoords, defaultDetailCalculator } from './defaultImpls';\n\nconst defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);\n\nexport class GUIAgent<T extends Operator> extends BaseGUIAgent {\n  static label = 'GUI Agent';\n\n  private operator: Operator | undefined;\n  private normalizeCoordinates: NormalizeCoordinates;\n  private detailCalculator: ImageDetailCalculator;\n  private loopIntervalInMs: number;\n\n  constructor(config: GUIAgentConfig<T>) {\n    const {\n      operator,\n      model,\n      systemPrompt,\n      customeActionParser,\n      normalizeCoordinates,\n      detailCalculator,\n      maxLoopCount,\n      loopIntervalInMs = 500,\n    } = config;\n    let finalSystemPrompt = SYSTEM_PROMPT;\n    if (typeof systemPrompt === 'string') {\n      finalSystemPrompt = systemPrompt;\n    } else if (Array.isArray(systemPrompt)) {\n      finalSystemPrompt = systemPrompt\n        .map((p) => (typeof p === 'string' ? p : p.content))\n        .join('\\n\\n');\n    } else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) {\n      finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());\n    }\n    defaultLogger.debug('final instructions for sp:', finalSystemPrompt);\n\n    // Create a adapted ToolCallEngine constructor that captures the customeActionParser\n    const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {\n      constructor() {\n        super(customeActionParser);\n      }\n    };\n\n    super({\n      name: GUIAgent.label,\n      instructions: finalSystemPrompt,\n      tools: [],\n      toolCallEngine: AdaptedToolCallEngine,\n      model: model,\n      ...(maxLoopCount && { maxIterations: maxLoopCount }),\n      logLevel: LogLevel.DEBUG,\n    });\n    this.operator = operator;\n    this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;\n    // Default detail calculator implementation\n    this.detailCalculator = detailCalculator ?? defaultDetailCalculator;\n    this.loopIntervalInMs = loopIntervalInMs;\n    this.logger = this.logger.spawn('[GUIAgent]');\n  }\n\n  async initialize() {\n    // Register the GUI tool\n    this.registerTool(\n      new Tool({\n        id: GUI_ADAPTED_TOOL_NAME,\n        description: 'operator tool',\n        parameters: {}, // no need to pass parameters\n        function: async (input) => {\n          this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);\n          if (!this.operator) {\n            return createGUIErrorResponse(input.action, 'Operator not initialized');\n          }\n          if (input.errorMessage) {\n            return createGUIErrorResponse(input.action, input.errorMessage);\n          }\n          // normalize coordinates\n          if (input.operator_action) {\n            input.operator_action = normalizeActionCoords(\n              input.operator_action,\n              this.normalizeCoordinates,\n            );\n          }\n          this.logger.info('action to execute:', JSON.stringify(input.operator_action));\n          const result = await this.operator!.doExecute({\n            actions: [input.operator_action],\n          });\n          if (result.errorMessage) {\n            return createGUIErrorResponse(input.action, result.errorMessage);\n          }\n          // return { action: input.action, status: 'success', result };\n          return {\n            success: true,\n            action: input.action,\n            normalizedAction: convertToAgentUIAction(input.operator_action),\n            observation: undefined, // Reserved for future implementation\n          };\n        },\n      }),\n    );\n    super.initialize();\n  }\n\n  async onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void> {\n    try {\n      const safeStringify = (obj: unknown, max = 800): string => {\n        try {\n          const s = JSON.stringify(obj);\n          return s.length > max ? s.slice(0, max) + '…(truncated)' : s;\n        } catch {\n          return '[unserializable]';\n        }\n      };\n\n      const req = payload?.request;\n      const messages = req && Array.isArray(req.messages) ? req.messages : [];\n      const model = req && typeof req.model === 'string' ? req.model : 'unknown';\n\n      const hasImages = (() => {\n        try {\n          let cnt = 0;\n          for (const m of messages as Array<{ content?: unknown }>) {\n            const content = m?.content;\n            if (Array.isArray(content)) {\n              for (const part of content as Array<{ type?: string }>) {\n                if (part?.type === 'image_url') cnt++;\n              }\n            }\n          }\n          return cnt;\n        } catch {\n          return undefined as unknown as number | undefined;\n        }\n      })();\n\n      const summary = {\n        id,\n        model,\n        messagesCount: messages.length,\n        hasImages,\n      };\n\n      this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));\n\n      const firstMsg = messages[0];\n      if (firstMsg) {\n        this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));\n      }\n    } catch (e) {\n      this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);\n    }\n  }\n\n  async onEachAgentLoopStart(sessionId: string) {\n    this.logger.info('onEachAgentLoopStart', sessionId);\n  }\n\n  async onAgentLoopEnd(id: string): Promise<void> {\n    // await this.browserOperator.cleanup();\n  }\n\n  async onBeforeToolCall(\n    id: string,\n    toolCall: { toolCallId: string; name: string },\n    args: unknown,\n  ) {\n    return args;\n  }\n\n  async onAfterToolCall(\n    id: string,\n    toolCall: { toolCallId: string; name: string },\n    result: unknown,\n  ): Promise<unknown> {\n    this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));\n\n    if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) {\n      this.logger.info('onAfterToolCall: skipping screenshot');\n      return;\n    }\n\n    await sleep(this.loopIntervalInMs);\n\n    const output = await this.operator!.doScreenshot();\n    if (!output) {\n      this.logger.error('Failed to get screenshot');\n      return;\n    }\n\n    const base64Tool = new Base64ImageParser(output.base64);\n    const base64Uri = base64Tool.getDataUri();\n    if (!base64Uri) {\n      this.logger.error('Failed to get base64 image uri');\n      return;\n    }\n\n    const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() || {\n      width: -1,\n      height: -1,\n    };\n\n    const content: ChatCompletionContentPart[] = [\n      {\n        type: 'image_url',\n        image_url: {\n          url: base64Uri,\n          detail: this.detailCalculator(imageWidth, imageHeight),\n        },\n      },\n    ];\n\n    if (output?.url) {\n      content.push({\n        type: 'text',\n        text: `The current page's url: ${output.url}`,\n      });\n    }\n\n    const eventStream = this.getEventStream();\n    const events = eventStream.getEvents();\n    this.logger.info('onAfterToolCall events length:', events.length);\n\n    const event = eventStream.createEvent('environment_input', {\n      description: 'Browser Screenshot',\n      content,\n      metadata: {\n        type: 'screenshot',\n        url: output?.url,\n      },\n    });\n    eventStream.sendEvent(event);\n    return result;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultLogger","ConsoleLogger","LogLevel","GUIAgent","BaseGUIAgent","Tool","GUI_ADAPTED_TOOL_NAME","input","createGUIErrorResponse","normalizeActionCoords","JSON","result","convertToAgentUIAction","undefined","id","payload","safeStringify","max","s","req","messages","Array","model","hasImages","cnt","m","content","part","summary","firstMsg","e","sessionId","toolCall","args","sleep","output","base64Tool","Base64ImageParser","base64Uri","imageWidth","imageHeight","eventStream","events","event","config","operator","systemPrompt","customeActionParser","normalizeCoordinates","detailCalculator","maxLoopCount","loopIntervalInMs","finalSystemPrompt","SYSTEM_PROMPT","p","isSystemPromptTemplate","assembleSystemPrompt","AdaptedToolCallEngine","GUIAgentToolCallEngine","defaultNormalizeCoords","defaultDetailCalculator"],"mappings":";;;;;;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;ACHC;;;;;;;;;;AA2BD,MAAMI,gBAAgB,IAAIC,sBAAAA,aAAaA,CAAC,cAAcC,sBAAAA,QAAAA,CAAAA,KAAc;AAE7D,MAAMC,iBAAqCC,qBAAAA,YAAYA;IAuD5D,MAAM,aAAa;QAEjB,IAAI,CAAC,YAAY,CACf,IAAIC,sBAAAA,IAAIA,CAAC;YACP,IAAIC,sCAAAA,qBAAqBA;YACzB,aAAa;YACb,YAAY,CAAC;YACb,UAAU,OAAOC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAGD,sCAAAA,qBAAqBA,CAAC,OAAO,CAAC,EAAEC;gBACnD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAChB,OAAOC,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAE;gBAE9C,IAAIA,MAAM,YAAY,EACpB,OAAOC,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAEA,MAAM,YAAY;gBAGhE,IAAIA,MAAM,eAAe,EACvBA,MAAM,eAAe,GAAGE,AAAAA,IAAAA,sBAAAA,qBAAAA,AAAAA,EACtBF,MAAM,eAAe,EACrB,IAAI,CAAC,oBAAoB;gBAG7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsBG,KAAK,SAAS,CAACH,MAAM,eAAe;gBAC3E,MAAMI,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,SAAS,CAAC;oBAC5C,SAAS;wBAACJ,MAAM,eAAe;qBAAC;gBAClC;gBACA,IAAII,OAAO,YAAY,EACrB,OAAOH,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAEI,OAAO,YAAY;gBAGjE,OAAO;oBACL,SAAS;oBACT,QAAQJ,MAAM,MAAM;oBACpB,kBAAkBK,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBL,MAAM,eAAe;oBAC9D,aAAaM;gBACf;YACF;QACF;QAEF,KAAK,CAAC;IACR;IAEA,MAAM,aAAaC,EAAU,EAAEC,OAA8B,EAAiB;QAC5E,IAAI;YACF,MAAMC,gBAAgB,CAACnB,KAAcoB,MAAM,GAAG;gBAC5C,IAAI;oBACF,MAAMC,IAAIR,KAAK,SAAS,CAACb;oBACzB,OAAOqB,EAAE,MAAM,GAAGD,MAAMC,EAAE,KAAK,CAAC,GAAGD,OAAO,sBAAiBC;gBAC7D,EAAE,OAAM;oBACN,OAAO;gBACT;YACF;YAEA,MAAMC,MAAMJ,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO;YAC5B,MAAMK,WAAWD,OAAOE,MAAM,OAAO,CAACF,IAAI,QAAQ,IAAIA,IAAI,QAAQ,GAAG,EAAE;YACvE,MAAMG,QAAQH,OAAO,AAAqB,YAArB,OAAOA,IAAI,KAAK,GAAgBA,IAAI,KAAK,GAAG;YAEjE,MAAMI,YAAa,AAAC;gBAClB,IAAI;oBACF,IAAIC,MAAM;oBACV,KAAK,MAAMC,KAAKL,SAA0C;wBACxD,MAAMM,UAAUD,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO;wBAC1B,IAAIJ,MAAM,OAAO,CAACK,UAChB;4BAAA,KAAK,MAAMC,QAAQD,QACjB,IAAIC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,AAAD,MAAM,aAAaH;wBAClC;oBAEJ;oBACA,OAAOA;gBACT,EAAE,OAAM;oBACN;gBACF;YACF;YAEA,MAAMI,UAAU;gBACdd;gBACAQ;gBACA,eAAeF,SAAS,MAAM;gBAC9BG;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoCP,cAAcY;YAEnE,MAAMC,WAAWT,QAAQ,CAAC,EAAE;YAC5B,IAAIS,UACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0Cb,cAAca,UAAU;QAExF,EAAE,OAAOC,GAAG;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2CA;QAC/D;IACF;IAEA,MAAM,qBAAqBC,SAAiB,EAAE;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwBA;IAC3C;IAEA,MAAM,eAAejB,EAAU,EAAiB,CAEhD;IAEA,MAAM,iBACJA,EAAU,EACVkB,QAA8C,EAC9CC,IAAa,EACb;QACA,OAAOA;IACT;IAEA,MAAM,gBACJnB,EAAU,EACVkB,QAA8C,EAC9CrB,MAAe,EACG;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4BD,KAAK,SAAS,CAACsB;QAE5D,IAAIA,SAAS,IAAI,KAAK1B,sCAAAA,qBAAqBA,EAAE,YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAInB,MAAM4B,AAAAA,IAAAA,sBAAAA,KAAAA,AAAAA,EAAM,IAAI,CAAC,gBAAgB;QAEjC,MAAMC,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,YAAY;QAChD,IAAI,CAACA,QAAQ,YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAMC,aAAa,IAAIC,4BAAAA,iBAAiBA,CAACF,OAAO,MAAM;QACtD,MAAMG,YAAYF,WAAW,UAAU;QACvC,IAAI,CAACE,WAAW,YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAM,EAAE,OAAOC,UAAU,EAAE,QAAQC,WAAW,EAAE,GAAGJ,WAAW,aAAa,MAAM;YAC/E,OAAO;YACP,QAAQ;QACV;QAEA,MAAMV,UAAuC;YAC3C;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKY;oBACL,QAAQ,IAAI,CAAC,gBAAgB,CAACC,YAAYC;gBAC5C;YACF;SACD;QAED,IAAIL,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG,EACbT,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,wBAAwB,EAAES,OAAO,GAAG,EAAE;QAC/C;QAGF,MAAMM,cAAc,IAAI,CAAC,cAAc;QACvC,MAAMC,SAASD,YAAY,SAAS;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkCC,OAAO,MAAM;QAEhE,MAAMC,QAAQF,YAAY,WAAW,CAAC,qBAAqB;YACzD,aAAa;YACbf;YACA,UAAU;gBACR,MAAM;gBACN,KAAKS,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG;YAClB;QACF;QACAM,YAAY,SAAS,CAACE;QACtB,OAAOhC;IACT;IA1NA,YAAYiC,MAAyB,CAAE;QACrC,MAAM,EACJC,QAAQ,EACRvB,KAAK,EACLwB,YAAY,EACZC,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,YAAY,EACZC,mBAAmB,GAAG,EACvB,GAAGP;QACJ,IAAIQ,oBAAoBC,oCAAAA,aAAaA;QACrC,IAAI,AAAwB,YAAxB,OAAOP,cACTM,oBAAoBN;aACf,IAAIzB,MAAM,OAAO,CAACyB,eACvBM,oBAAoBN,aACjB,GAAG,CAAC,CAACQ,IAAO,AAAa,YAAb,OAAOA,IAAiBA,IAAIA,EAAE,OAAO,EACjD,IAAI,CAAC;aACH,IAAIR,gBAAgBS,AAAAA,IAAAA,sBAAAA,sBAAAA,AAAAA,EAAuBT,eAChDM,oBAAoBI,AAAAA,IAAAA,sBAAAA,oBAAAA,AAAAA,EAAqBV,cAAcD,SAAS,mBAAmB;QAErF7C,cAAc,KAAK,CAAC,8BAA8BoD;QAGlD,MAAMK,wBAAwB,cAAcC,2CAAAA,sBAAsBA;YAChE,aAAc;gBACZ,KAAK,CAACX;YACR;QACF;QAEA,KAAK,CAAC;YACJ,MAAM5C,SAAS,KAAK;YACpB,cAAciD;YACd,OAAO,EAAE;YACT,gBAAgBK;YAChB,OAAOnC;YACP,GAAI4B,gBAAgB;gBAAE,eAAeA;YAAa,CAAC;YACnD,UAAUhD,sBAAAA,QAAAA,CAAAA,KAAc;QAC1B,IA3CF,uBAAQ,YAAR,SACA,uBAAQ,wBAAR,SACA,uBAAQ,oBAAR,SACA,uBAAQ,oBAAR;QAyCE,IAAI,CAAC,QAAQ,GAAG2C;QAChB,IAAI,CAAC,oBAAoB,GAAGG,wBAAwBW,yCAAAA,sBAAsBA;QAE1E,IAAI,CAAC,gBAAgB,GAAGV,oBAAoBW,yCAAAA,uBAAuBA;QACnE,IAAI,CAAC,gBAAgB,GAAGT;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC;AA8KF;AAlOE,iBADWhD,UACJ,SAAQ"}

package/dist/GUIAgent.mjs

@@ -0,0 +1,173 @@
+/**
+* Copyright (c) 2025 Bytedance, Inc. and its affiliates.
+* SPDX-License-Identifier: Apache-2.0
+*/
+import { ConsoleLogger, LogLevel, Tool } from "@tarko/agent";
+import { GUIAgentToolCallEngine } from "./ToolCallEngine.mjs";
+import { SYSTEM_PROMPT } from "./prompts.mjs";
+import { Base64ImageParser } from "@agent-infra/media-utils";
+import { BaseGUIAgent } from "@ui-tars-test/shared/base";
+import { assembleSystemPrompt, isSystemPromptTemplate, normalizeActionCoords, sleep } from "@ui-tars-test/shared/utils";
+import { GUI_ADAPTED_TOOL_NAME } from "./constants.mjs";
+import { convertToAgentUIAction, createGUIErrorResponse } from "./utils.mjs";
+import { defaultDetailCalculator, defaultNormalizeCoords } from "./defaultImpls.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);
+class GUIAgent extends BaseGUIAgent {
+    async initialize() {
+        this.registerTool(new Tool({
+            id: GUI_ADAPTED_TOOL_NAME,
+            description: 'operator tool',
+            parameters: {},
+            function: async (input)=>{
+                this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);
+                if (!this.operator) return createGUIErrorResponse(input.action, 'Operator not initialized');
+                if (input.errorMessage) return createGUIErrorResponse(input.action, input.errorMessage);
+                if (input.operator_action) input.operator_action = normalizeActionCoords(input.operator_action, this.normalizeCoordinates);
+                this.logger.info('action to execute:', JSON.stringify(input.operator_action));
+                const result = await this.operator.doExecute({
+                    actions: [
+                        input.operator_action
+                    ]
+                });
+                if (result.errorMessage) return createGUIErrorResponse(input.action, result.errorMessage);
+                return {
+                    success: true,
+                    action: input.action,
+                    normalizedAction: convertToAgentUIAction(input.operator_action),
+                    observation: void 0
+                };
+            }
+        }));
+        super.initialize();
+    }
+    async onLLMRequest(id, payload) {
+        try {
+            const safeStringify = (obj, max = 800)=>{
+                try {
+                    const s = JSON.stringify(obj);
+                    return s.length > max ? s.slice(0, max) + "\u2026(truncated)" : s;
+                } catch  {
+                    return '[unserializable]';
+                }
+            };
+            const req = null == payload ? void 0 : payload.request;
+            const messages = req && Array.isArray(req.messages) ? req.messages : [];
+            const model = req && 'string' == typeof req.model ? req.model : 'unknown';
+            const hasImages = (()=>{
+                try {
+                    let cnt = 0;
+                    for (const m of messages){
+                        const content = null == m ? void 0 : m.content;
+                        if (Array.isArray(content)) {
+                            for (const part of content)if ((null == part ? void 0 : part.type) === 'image_url') cnt++;
+                        }
+                    }
+                    return cnt;
+                } catch  {
+                    return;
+                }
+            })();
+            const summary = {
+                id,
+                model,
+                messagesCount: messages.length,
+                hasImages
+            };
+            this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));
+            const firstMsg = messages[0];
+            if (firstMsg) this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));
+        } catch (e) {
+            this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);
+        }
+    }
+    async onEachAgentLoopStart(sessionId) {
+        this.logger.info('onEachAgentLoopStart', sessionId);
+    }
+    async onAgentLoopEnd(id) {}
+    async onBeforeToolCall(id, toolCall, args) {
+        return args;
+    }
+    async onAfterToolCall(id, toolCall, result) {
+        this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));
+        if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) return void this.logger.info('onAfterToolCall: skipping screenshot');
+        await sleep(this.loopIntervalInMs);
+        const output = await this.operator.doScreenshot();
+        if (!output) return void this.logger.error('Failed to get screenshot');
+        const base64Tool = new Base64ImageParser(output.base64);
+        const base64Uri = base64Tool.getDataUri();
+        if (!base64Uri) return void this.logger.error('Failed to get base64 image uri');
+        const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() || {
+            width: -1,
+            height: -1
+        };
+        const content = [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: base64Uri,
+                    detail: this.detailCalculator(imageWidth, imageHeight)
+                }
+            }
+        ];
+        if (null == output ? void 0 : output.url) content.push({
+            type: 'text',
+            text: `The current page's url: ${output.url}`
+        });
+        const eventStream = this.getEventStream();
+        const events = eventStream.getEvents();
+        this.logger.info('onAfterToolCall events length:', events.length);
+        const event = eventStream.createEvent('environment_input', {
+            description: 'Browser Screenshot',
+            content,
+            metadata: {
+                type: 'screenshot',
+                url: null == output ? void 0 : output.url
+            }
+        });
+        eventStream.sendEvent(event);
+        return result;
+    }
+    constructor(config){
+        const { operator, model, systemPrompt, customeActionParser, normalizeCoordinates, detailCalculator, maxLoopCount, loopIntervalInMs = 500 } = config;
+        let finalSystemPrompt = SYSTEM_PROMPT;
+        if ('string' == typeof systemPrompt) finalSystemPrompt = systemPrompt;
+        else if (Array.isArray(systemPrompt)) finalSystemPrompt = systemPrompt.map((p)=>'string' == typeof p ? p : p.content).join('\n\n');
+        else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());
+        defaultLogger.debug('final instructions for sp:', finalSystemPrompt);
+        const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {
+            constructor(){
+                super(customeActionParser);
+            }
+        };
+        super({
+            name: GUIAgent.label,
+            instructions: finalSystemPrompt,
+            tools: [],
+            toolCallEngine: AdaptedToolCallEngine,
+            model: model,
+            ...maxLoopCount && {
+                maxIterations: maxLoopCount
+            },
+            logLevel: LogLevel.DEBUG
+        }), _define_property(this, "operator", void 0), _define_property(this, "normalizeCoordinates", void 0), _define_property(this, "detailCalculator", void 0), _define_property(this, "loopIntervalInMs", void 0);
+        this.operator = operator;
+        this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;
+        this.detailCalculator = detailCalculator ?? defaultDetailCalculator;
+        this.loopIntervalInMs = loopIntervalInMs;
+        this.logger = this.logger.spawn('[GUIAgent]');
+    }
+}
+_define_property(GUIAgent, "label", 'GUI Agent');
+export { GUIAgent };
+
+//# sourceMappingURL=GUIAgent.mjs.map

package/dist/GUIAgent.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"GUIAgent.mjs","sources":["webpack://@ui-tars-test/agent-sdk/./src/GUIAgent.ts"],"sourcesContent":["/*\n * Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n  LLMRequestHookPayload,\n  ChatCompletionContentPart,\n  LogLevel,\n  Tool,\n  ConsoleLogger,\n} from '@tarko/agent';\nimport { GUIAgentToolCallEngine } from './ToolCallEngine';\nimport { SYSTEM_PROMPT } from './prompts';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';\nimport {\n  GUIAgentConfig,\n  NormalizeCoordinates,\n  ImageDetailCalculator,\n} from '@ui-tars-test/shared/types';\nimport {\n  assembleSystemPrompt,\n  isSystemPromptTemplate,\n  normalizeActionCoords,\n  sleep,\n} from '@ui-tars-test/shared/utils';\nimport { GUI_ADAPTED_TOOL_NAME } from './constants';\nimport { convertToAgentUIAction, createGUIErrorResponse } from './utils';\nimport { defaultNormalizeCoords, defaultDetailCalculator } from './defaultImpls';\n\nconst defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);\n\nexport class GUIAgent<T extends Operator> extends BaseGUIAgent {\n  static label = 'GUI Agent';\n\n  private operator: Operator | undefined;\n  private normalizeCoordinates: NormalizeCoordinates;\n  private detailCalculator: ImageDetailCalculator;\n  private loopIntervalInMs: number;\n\n  constructor(config: GUIAgentConfig<T>) {\n    const {\n      operator,\n      model,\n      systemPrompt,\n      customeActionParser,\n      normalizeCoordinates,\n      detailCalculator,\n      maxLoopCount,\n      loopIntervalInMs = 500,\n    } = config;\n    let finalSystemPrompt = SYSTEM_PROMPT;\n    if (typeof systemPrompt === 'string') {\n      finalSystemPrompt = systemPrompt;\n    } else if (Array.isArray(systemPrompt)) {\n      finalSystemPrompt = systemPrompt\n        .map((p) => (typeof p === 'string' ? p : p.content))\n        .join('\\n\\n');\n    } else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) {\n      finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());\n    }\n    defaultLogger.debug('final instructions for sp:', finalSystemPrompt);\n\n    // Create a adapted ToolCallEngine constructor that captures the customeActionParser\n    const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {\n      constructor() {\n        super(customeActionParser);\n      }\n    };\n\n    super({\n      name: GUIAgent.label,\n      instructions: finalSystemPrompt,\n      tools: [],\n      toolCallEngine: AdaptedToolCallEngine,\n      model: model,\n      ...(maxLoopCount && { maxIterations: maxLoopCount }),\n      logLevel: LogLevel.DEBUG,\n    });\n    this.operator = operator;\n    this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;\n    // Default detail calculator implementation\n    this.detailCalculator = detailCalculator ?? defaultDetailCalculator;\n    this.loopIntervalInMs = loopIntervalInMs;\n    this.logger = this.logger.spawn('[GUIAgent]');\n  }\n\n  async initialize() {\n    // Register the GUI tool\n    this.registerTool(\n      new Tool({\n        id: GUI_ADAPTED_TOOL_NAME,\n        description: 'operator tool',\n        parameters: {}, // no need to pass parameters\n        function: async (input) => {\n          this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);\n          if (!this.operator) {\n            return createGUIErrorResponse(input.action, 'Operator not initialized');\n          }\n          if (input.errorMessage) {\n            return createGUIErrorResponse(input.action, input.errorMessage);\n          }\n          // normalize coordinates\n          if (input.operator_action) {\n            input.operator_action = normalizeActionCoords(\n              input.operator_action,\n              this.normalizeCoordinates,\n            );\n          }\n          this.logger.info('action to execute:', JSON.stringify(input.operator_action));\n          const result = await this.operator!.doExecute({\n            actions: [input.operator_action],\n          });\n          if (result.errorMessage) {\n            return createGUIErrorResponse(input.action, result.errorMessage);\n          }\n          // return { action: input.action, status: 'success', result };\n          return {\n            success: true,\n            action: input.action,\n            normalizedAction: convertToAgentUIAction(input.operator_action),\n            observation: undefined, // Reserved for future implementation\n          };\n        },\n      }),\n    );\n    super.initialize();\n  }\n\n  async onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void> {\n    try {\n      const safeStringify = (obj: unknown, max = 800): string => {\n        try {\n          const s = JSON.stringify(obj);\n          return s.length > max ? s.slice(0, max) + '…(truncated)' : s;\n        } catch {\n          return '[unserializable]';\n        }\n      };\n\n      const req = payload?.request;\n      const messages = req && Array.isArray(req.messages) ? req.messages : [];\n      const model = req && typeof req.model === 'string' ? req.model : 'unknown';\n\n      const hasImages = (() => {\n        try {\n          let cnt = 0;\n          for (const m of messages as Array<{ content?: unknown }>) {\n            const content = m?.content;\n            if (Array.isArray(content)) {\n              for (const part of content as Array<{ type?: string }>) {\n                if (part?.type === 'image_url') cnt++;\n              }\n            }\n          }\n          return cnt;\n        } catch {\n          return undefined as unknown as number | undefined;\n        }\n      })();\n\n      const summary = {\n        id,\n        model,\n        messagesCount: messages.length,\n        hasImages,\n      };\n\n      this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));\n\n      const firstMsg = messages[0];\n      if (firstMsg) {\n        this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));\n      }\n    } catch (e) {\n      this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);\n    }\n  }\n\n  async onEachAgentLoopStart(sessionId: string) {\n    this.logger.info('onEachAgentLoopStart', sessionId);\n  }\n\n  async onAgentLoopEnd(id: string): Promise<void> {\n    // await this.browserOperator.cleanup();\n  }\n\n  async onBeforeToolCall(\n    id: string,\n    toolCall: { toolCallId: string; name: string },\n    args: unknown,\n  ) {\n    return args;\n  }\n\n  async onAfterToolCall(\n    id: string,\n    toolCall: { toolCallId: string; name: string },\n    result: unknown,\n  ): Promise<unknown> {\n    this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));\n\n    if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) {\n      this.logger.info('onAfterToolCall: skipping screenshot');\n      return;\n    }\n\n    await sleep(this.loopIntervalInMs);\n\n    const output = await this.operator!.doScreenshot();\n    if (!output) {\n      this.logger.error('Failed to get screenshot');\n      return;\n    }\n\n    const base64Tool = new Base64ImageParser(output.base64);\n    const base64Uri = base64Tool.getDataUri();\n    if (!base64Uri) {\n      this.logger.error('Failed to get base64 image uri');\n      return;\n    }\n\n    const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() || {\n      width: -1,\n      height: -1,\n    };\n\n    const content: ChatCompletionContentPart[] = [\n      {\n        type: 'image_url',\n        image_url: {\n          url: base64Uri,\n          detail: this.detailCalculator(imageWidth, imageHeight),\n        },\n      },\n    ];\n\n    if (output?.url) {\n      content.push({\n        type: 'text',\n        text: `The current page's url: ${output.url}`,\n      });\n    }\n\n    const eventStream = this.getEventStream();\n    const events = eventStream.getEvents();\n    this.logger.info('onAfterToolCall events length:', events.length);\n\n    const event = eventStream.createEvent('environment_input', {\n      description: 'Browser Screenshot',\n      content,\n      metadata: {\n        type: 'screenshot',\n        url: output?.url,\n      },\n    });\n    eventStream.sendEvent(event);\n    return result;\n  }\n}\n"],"names":["defaultLogger","ConsoleLogger","LogLevel","GUIAgent","BaseGUIAgent","Tool","GUI_ADAPTED_TOOL_NAME","input","createGUIErrorResponse","normalizeActionCoords","JSON","result","convertToAgentUIAction","undefined","id","payload","safeStringify","obj","max","s","req","messages","Array","model","hasImages","cnt","m","content","part","summary","firstMsg","e","sessionId","toolCall","args","sleep","output","base64Tool","Base64ImageParser","base64Uri","imageWidth","imageHeight","eventStream","events","event","config","operator","systemPrompt","customeActionParser","normalizeCoordinates","detailCalculator","maxLoopCount","loopIntervalInMs","finalSystemPrompt","SYSTEM_PROMPT","p","isSystemPromptTemplate","assembleSystemPrompt","AdaptedToolCallEngine","GUIAgentToolCallEngine","defaultNormalizeCoords","defaultDetailCalculator"],"mappings":";;;;;;;;;;;;;AAGC;;;;;;;;;;AA2BD,MAAMA,gBAAgB,IAAIC,cAAc,cAAcC,SAAS,KAAK;AAE7D,MAAMC,iBAAqCC;IAuDhD,MAAM,aAAa;QAEjB,IAAI,CAAC,YAAY,CACf,IAAIC,KAAK;YACP,IAAIC;YACJ,aAAa;YACb,YAAY,CAAC;YACb,UAAU,OAAOC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAGD,sBAAsB,OAAO,CAAC,EAAEC;gBACnD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAChB,OAAOC,uBAAuBD,MAAM,MAAM,EAAE;gBAE9C,IAAIA,MAAM,YAAY,EACpB,OAAOC,uBAAuBD,MAAM,MAAM,EAAEA,MAAM,YAAY;gBAGhE,IAAIA,MAAM,eAAe,EACvBA,MAAM,eAAe,GAAGE,sBACtBF,MAAM,eAAe,EACrB,IAAI,CAAC,oBAAoB;gBAG7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsBG,KAAK,SAAS,CAACH,MAAM,eAAe;gBAC3E,MAAMI,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,SAAS,CAAC;oBAC5C,SAAS;wBAACJ,MAAM,eAAe;qBAAC;gBAClC;gBACA,IAAII,OAAO,YAAY,EACrB,OAAOH,uBAAuBD,MAAM,MAAM,EAAEI,OAAO,YAAY;gBAGjE,OAAO;oBACL,SAAS;oBACT,QAAQJ,MAAM,MAAM;oBACpB,kBAAkBK,uBAAuBL,MAAM,eAAe;oBAC9D,aAAaM;gBACf;YACF;QACF;QAEF,KAAK,CAAC;IACR;IAEA,MAAM,aAAaC,EAAU,EAAEC,OAA8B,EAAiB;QAC5E,IAAI;YACF,MAAMC,gBAAgB,CAACC,KAAcC,MAAM,GAAG;gBAC5C,IAAI;oBACF,MAAMC,IAAIT,KAAK,SAAS,CAACO;oBACzB,OAAOE,EAAE,MAAM,GAAGD,MAAMC,EAAE,KAAK,CAAC,GAAGD,OAAO,sBAAiBC;gBAC7D,EAAE,OAAM;oBACN,OAAO;gBACT;YACF;YAEA,MAAMC,MAAML,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO;YAC5B,MAAMM,WAAWD,OAAOE,MAAM,OAAO,CAACF,IAAI,QAAQ,IAAIA,IAAI,QAAQ,GAAG,EAAE;YACvE,MAAMG,QAAQH,OAAO,AAAqB,YAArB,OAAOA,IAAI,KAAK,GAAgBA,IAAI,KAAK,GAAG;YAEjE,MAAMI,YAAa,AAAC;gBAClB,IAAI;oBACF,IAAIC,MAAM;oBACV,KAAK,MAAMC,KAAKL,SAA0C;wBACxD,MAAMM,UAAUD,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO;wBAC1B,IAAIJ,MAAM,OAAO,CAACK,UAChB;4BAAA,KAAK,MAAMC,QAAQD,QACjB,IAAIC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,AAAD,MAAM,aAAaH;wBAClC;oBAEJ;oBACA,OAAOA;gBACT,EAAE,OAAM;oBACN;gBACF;YACF;YAEA,MAAMI,UAAU;gBACdf;gBACAS;gBACA,eAAeF,SAAS,MAAM;gBAC9BG;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoCR,cAAca;YAEnE,MAAMC,WAAWT,QAAQ,CAAC,EAAE;YAC5B,IAAIS,UACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0Cd,cAAcc,UAAU;QAExF,EAAE,OAAOC,GAAG;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2CA;QAC/D;IACF;IAEA,MAAM,qBAAqBC,SAAiB,EAAE;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwBA;IAC3C;IAEA,MAAM,eAAelB,EAAU,EAAiB,CAEhD;IAEA,MAAM,iBACJA,EAAU,EACVmB,QAA8C,EAC9CC,IAAa,EACb;QACA,OAAOA;IACT;IAEA,MAAM,gBACJpB,EAAU,EACVmB,QAA8C,EAC9CtB,MAAe,EACG;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4BD,KAAK,SAAS,CAACuB;QAE5D,IAAIA,SAAS,IAAI,KAAK3B,uBAAuB,YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAInB,MAAM6B,MAAM,IAAI,CAAC,gBAAgB;QAEjC,MAAMC,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,YAAY;QAChD,IAAI,CAACA,QAAQ,YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAMC,aAAa,IAAIC,kBAAkBF,OAAO,MAAM;QACtD,MAAMG,YAAYF,WAAW,UAAU;QACvC,IAAI,CAACE,WAAW,YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAM,EAAE,OAAOC,UAAU,EAAE,QAAQC,WAAW,EAAE,GAAGJ,WAAW,aAAa,MAAM;YAC/E,OAAO;YACP,QAAQ;QACV;QAEA,MAAMV,UAAuC;YAC3C;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKY;oBACL,QAAQ,IAAI,CAAC,gBAAgB,CAACC,YAAYC;gBAC5C;YACF;SACD;QAED,IAAIL,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG,EACbT,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,wBAAwB,EAAES,OAAO,GAAG,EAAE;QAC/C;QAGF,MAAMM,cAAc,IAAI,CAAC,cAAc;QACvC,MAAMC,SAASD,YAAY,SAAS;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkCC,OAAO,MAAM;QAEhE,MAAMC,QAAQF,YAAY,WAAW,CAAC,qBAAqB;YACzD,aAAa;YACbf;YACA,UAAU;gBACR,MAAM;gBACN,KAAKS,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG;YAClB;QACF;QACAM,YAAY,SAAS,CAACE;QACtB,OAAOjC;IACT;IA1NA,YAAYkC,MAAyB,CAAE;QACrC,MAAM,EACJC,QAAQ,EACRvB,KAAK,EACLwB,YAAY,EACZC,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,YAAY,EACZC,mBAAmB,GAAG,EACvB,GAAGP;QACJ,IAAIQ,oBAAoBC;QACxB,IAAI,AAAwB,YAAxB,OAAOP,cACTM,oBAAoBN;aACf,IAAIzB,MAAM,OAAO,CAACyB,eACvBM,oBAAoBN,aACjB,GAAG,CAAC,CAACQ,IAAO,AAAa,YAAb,OAAOA,IAAiBA,IAAIA,EAAE,OAAO,EACjD,IAAI,CAAC;aACH,IAAIR,gBAAgBS,uBAAuBT,eAChDM,oBAAoBI,qBAAqBV,cAAcD,SAAS,mBAAmB;QAErF9C,cAAc,KAAK,CAAC,8BAA8BqD;QAGlD,MAAMK,wBAAwB,cAAcC;YAC1C,aAAc;gBACZ,KAAK,CAACX;YACR;QACF;QAEA,KAAK,CAAC;YACJ,MAAM7C,SAAS,KAAK;YACpB,cAAckD;YACd,OAAO,EAAE;YACT,gBAAgBK;YAChB,OAAOnC;YACP,GAAI4B,gBAAgB;gBAAE,eAAeA;YAAa,CAAC;YACnD,UAAUjD,SAAS,KAAK;QAC1B,IA3CF,uBAAQ,YAAR,SACA,uBAAQ,wBAAR,SACA,uBAAQ,oBAAR,SACA,uBAAQ,oBAAR;QAyCE,IAAI,CAAC,QAAQ,GAAG4C;QAChB,IAAI,CAAC,oBAAoB,GAAGG,wBAAwBW;QAEpD,IAAI,CAAC,gBAAgB,GAAGV,oBAAoBW;QAC5C,IAAI,CAAC,gBAAgB,GAAGT;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC;AA8KF;AAlOE,iBADWjD,UACJ,SAAQ"}

package/dist/ToolCallEngine.d.ts

@@ -0,0 +1,61 @@
+import { ToolCallEngine, Tool, ToolCallEnginePrepareRequestContext, ChatCompletionCreateParams, ChatCompletionAssistantMessageParam, ChatCompletionChunk, MultimodalToolCallResult, AgentEventStream, ChatCompletionMessageParam, ParsedModelResponse, StreamProcessingState, StreamChunkResult } from '@tarko/agent-interface';
+import { CustomActionParser } from '@ui-tars-test/shared/types';
+/**
+ * GUIAgentToolCallEngine - Minimal prompt engineering tool call engine
+ *
+ * This is the simplest possible implementation of a tool call engine that:
+ * 1. Uses prompt engineering to instruct the LLM to output tool calls in a specific format
+ * 2. Parses tool calls from LLM response text using simple regex matching
+ * 3. Does not support streaming (focuses on core functionality only)
+ *
+ * Format used: <tool_call>{"name": "tool_name", "arguments": {...}}</tool_call>
+ */
+export declare class GUIAgentToolCallEngine extends ToolCallEngine {
+    private customActionParser?;
+    constructor(customActionParser?: CustomActionParser);
+    /**
+     * Prepare system prompt with tool information and instructions
+     */
+    preparePrompt(instructions: string, tools: Tool[]): string;
+    /**
+     * Prepare request parameters for the LLM
+     *
+     * FIXME: move to base tool call engine.
+     */
+    prepareRequest(context: ToolCallEnginePrepareRequestContext): ChatCompletionCreateParams;
+    /**
+     * Initialize processing state (minimal implementation)
+     *
+     * FIXME: move to base tool call engine.
+     */
+    initStreamProcessingState(): StreamProcessingState;
+    /**
+     * Process streaming chunks - simply accumulate content
+     *
+     * FIXME: make it optional
+     */
+    processStreamingChunk(chunk: ChatCompletionChunk, state: StreamProcessingState): StreamChunkResult;
+    /**
+     * Generate a tool call ID
+     */
+    private generateToolCallId;
+    /**
+     * Extract tool calls from complete response text
+     */
+    finalizeStreamProcessing(state: StreamProcessingState): ParsedModelResponse;
+    /**
+     * Build assistant message for conversation history
+     * For PE engines, we preserve the raw content including tool call markup
+     *
+     * FIXME: move to base tool call engine.
+     */
+    buildHistoricalAssistantMessage(currentLoopAssistantEvent: AgentEventStream.AssistantMessageEvent): ChatCompletionAssistantMessageParam;
+    /**
+     * Build tool result messages as user messages
+     * PE engines format tool results as user input for next iteration
+     *
+     * FIXME: move to base tool call engine.
+     */
+    buildHistoricalToolCallResultMessages(toolCallResults: MultimodalToolCallResult[]): ChatCompletionMessageParam[];
+}
+//# sourceMappingURL=ToolCallEngine.d.ts.map

package/dist/ToolCallEngine.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ToolCallEngine.d.ts","sourceRoot":"","sources":["../src/ToolCallEngine.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,cAAc,EACd,IAAI,EACJ,mCAAmC,EACnC,0BAA0B,EAC1B,mCAAmC,EACnC,mBAAmB,EACnB,wBAAwB,EACxB,gBAAgB,EAChB,0BAA0B,EAE1B,mBAAmB,EACnB,qBAAqB,EACrB,iBAAiB,EAClB,MAAM,wBAAwB,CAAC;AAKhC,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAC;AAKhE;;;;;;;;;GASG;AACH,qBAAa,sBAAuB,SAAQ,cAAc;IACxD,OAAO,CAAC,kBAAkB,CAAC,CAAqB;gBAEpC,kBAAkB,CAAC,EAAE,kBAAkB;IAKnD;;OAEG;IACH,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,MAAM;IAI1D;;;;OAIG;IACH,cAAc,CAAC,OAAO,EAAE,mCAAmC,GAAG,0BAA0B;IAaxF;;;;OAIG;IACH,yBAAyB,IAAI,qBAAqB;IASlD;;;;OAIG;IACH,qBAAqB,CACnB,KAAK,EAAE,mBAAmB,EAC1B,KAAK,EAAE,qBAAqB,GAC3B,iBAAiB;IAsBpB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAI1B;;OAEG;IACH,wBAAwB,CAAC,KAAK,EAAE,qBAAqB,GAAG,mBAAmB;IAiG3E;;;;;OAKG;IACH,+BAA+B,CAC7B,yBAAyB,EAAE,gBAAgB,CAAC,qBAAqB,GAChE,mCAAmC;IAOtC;;;;;OAKG;IACH,qCAAqC,CACnC,eAAe,EAAE,wBAAwB,EAAE,GAC1C,0BAA0B,EAAE;CAchC"}