npm - @ui-tars-test/agent-sdk - Versions diffs - 0.3.15 → 0.3.17 - Mend

@ui-tars-test/agent-sdk 0.3.15 → 0.3.17

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (6) hide show

package/dist/GUIAgent.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"GUIAgent.d.ts","sourceRoot":"","sources":["../src/GUIAgent.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,qBAAqB,EAKtB,MAAM,2BAA2B,CAAC;AAInC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EACL,cAAc,EAGf,MAAM,4BAA4B,CAAC;AAapC,qBAAa,QAAQ,CAAC,CAAC,SAAS,QAAQ,CAAE,SAAQ,YAAY;IAC5D,MAAM,CAAC,KAAK,SAAe;IAE3B,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,gBAAgB,CAAS;gBAErB,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IA+C/B,UAAU;IA0CV,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvE,oBAAoB,CAAC,SAAS,EAAE,MAAM;IAItC,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,gBAAgB,CACpB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAC9C,IAAI,EAAE,OAAO;IAKT,eAAe,CACnB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAC9C,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,OAAO,CAAC;~~CA4DpB~~"}
1	+ {"version":3,"file":"GUIAgent.d.ts","sourceRoot":"","sources":["../src/GUIAgent.ts"],"names":[],"mappings":"AAIA,OAAO,EACL,qBAAqB,EAKtB,MAAM,2BAA2B,CAAC;AAInC,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACnE,OAAO,EACL,cAAc,EAGf,MAAM,4BAA4B,CAAC;AAapC,qBAAa,QAAQ,CAAC,CAAC,SAAS,QAAQ,CAAE,SAAQ,YAAY;IAC5D,MAAM,CAAC,KAAK,SAAe;IAE3B,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,oBAAoB,CAAuB;IACnD,OAAO,CAAC,gBAAgB,CAAwB;IAChD,OAAO,CAAC,gBAAgB,CAAS;gBAErB,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IA+C/B,UAAU;IA0CV,YAAY,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvE,oBAAoB,CAAC,SAAS,EAAE,MAAM;IAItC,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIzC,gBAAgB,CACpB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAC9C,IAAI,EAAE,OAAO;IAKT,eAAe,CACnB,EAAE,EAAE,MAAM,EACV,QAAQ,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAC9C,MAAM,EAAE,OAAO,GACd,OAAO,CAAC,OAAO,CAAC;CAgEpB"}

package/dist/GUIAgent.js CHANGED Viewed

@@ -145,6 +145,10 @@ class GUIAgent extends base_namespaceObject.BaseGUIAgent {
                     url: base64Uri,
                     detail: this.detailCalculator(imageWidth, imageHeight)
                 }
+            },
+            {
+                type: 'text',
+                text: `Screen resolution: ${imageWidth}x${imageHeight}. System is running on macOS. Please return actions with absolute coordinates within this resolution.`
             }
         ];
         if (null == output ? void 0 : output.url) content.push({

package/dist/GUIAgent.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"GUIAgent.js","sources":["webpack://@ui-tars-test/agent-sdk/webpack/runtime/define_property_getters","webpack://@ui-tars-test/agent-sdk/webpack/runtime/has_own_property","webpack://@ui-tars-test/agent-sdk/webpack/runtime/make_namespace_object","webpack://@ui-tars-test/agent-sdk/./src/GUIAgent.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/\n Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n LLMRequestHookPayload,\n ChatCompletionContentPart,\n LogLevel,\n Tool,\n ConsoleLogger,\n} from '@ui-tars-test/tarko-agent';\nimport { GUIAgentToolCallEngine } from './ToolCallEngine';\nimport { SYSTEM_PROMPT } from './prompts';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';\nimport {\n GUIAgentConfig,\n NormalizeCoordinates,\n ImageDetailCalculator,\n} from '@ui-tars-test/shared/types';\nimport {\n assembleSystemPrompt,\n isSystemPromptTemplate,\n normalizeActionCoords,\n sleep,\n} from '@ui-tars-test/shared/utils';\nimport { GUI_ADAPTED_TOOL_NAME } from './constants';\nimport { convertToAgentUIAction, createGUIErrorResponse } from './utils';\nimport { defaultNormalizeCoords, defaultDetailCalculator } from './defaultImpls';\n\nconst defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);\n\nexport class GUIAgent<T extends Operator> extends BaseGUIAgent {\n static label = 'GUI Agent';\n\n private operator: Operator \| undefined;\n private normalizeCoordinates: NormalizeCoordinates;\n private detailCalculator: ImageDetailCalculator;\n private loopIntervalInMs: number;\n\n constructor(config: GUIAgentConfig<T>) {\n const {\n operator,\n model,\n systemPrompt,\n customeActionParser,\n normalizeCoordinates,\n detailCalculator,\n maxLoopCount,\n loopIntervalInMs = 500,\n } = config;\n let finalSystemPrompt = SYSTEM_PROMPT;\n if (typeof systemPrompt === 'string') {\n finalSystemPrompt = systemPrompt;\n } else if (Array.isArray(systemPrompt)) {\n finalSystemPrompt = systemPrompt\n .map((p) => (typeof p === 'string' ? p : p.content))\n .join('\\n\\n');\n } else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) {\n finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());\n }\n defaultLogger.debug('final instructions for sp:', finalSystemPrompt);\n\n // Create a adapted ToolCallEngine constructor that captures the customeActionParser\n const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {\n constructor() {\n super(customeActionParser);\n }\n };\n\n super({\n name: GUIAgent.label,\n instructions: finalSystemPrompt,\n tools: [],\n toolCallEngine: AdaptedToolCallEngine,\n model: model,\n ...(maxLoopCount && { maxIterations: maxLoopCount }),\n logLevel: LogLevel.DEBUG,\n });\n this.operator = operator;\n this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;\n // Default detail calculator implementation\n this.detailCalculator = detailCalculator ?? defaultDetailCalculator;\n this.loopIntervalInMs = loopIntervalInMs;\n this.logger = this.logger.spawn('[GUIAgent]');\n }\n\n async initialize() {\n // Register the GUI tool\n this.registerTool(\n new Tool({\n id: GUI_ADAPTED_TOOL_NAME,\n description: 'operator tool',\n parameters: {}, // no need to pass parameters\n function: async (input) => {\n this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);\n if (!this.operator) {\n return createGUIErrorResponse(input.action, 'Operator not initialized');\n }\n if (input.errorMessage) {\n return createGUIErrorResponse(input.action, input.errorMessage);\n }\n // normalize coordinates\n if (input.operator_action) {\n input.operator_action = normalizeActionCoords(\n input.operator_action,\n this.normalizeCoordinates,\n );\n }\n this.logger.info('action to execute:', JSON.stringify(input.operator_action));\n const result = await this.operator!.doExecute({\n actions: [input.operator_action],\n });\n if (result.errorMessage) {\n return createGUIErrorResponse(input.action, result.errorMessage);\n }\n // return { action: input.action, status: 'success', result };\n return {\n success: true,\n action: input.action,\n normalizedAction: convertToAgentUIAction(input.operator_action),\n observation: undefined, // Reserved for future implementation\n };\n },\n }),\n );\n super.initialize();\n }\n\n async onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void> {\n try {\n const safeStringify = (obj: unknown, max = 800): string => {\n try {\n const s = JSON.stringify(obj);\n return s.length > max ? s.slice(0, max) + '…(truncated)' : s;\n } catch {\n return '[unserializable]';\n }\n };\n\n const req = payload?.request;\n const messages = req && Array.isArray(req.messages) ? req.messages : [];\n const model = req && typeof req.model === 'string' ? req.model : 'unknown';\n\n const hasImages = (() => {\n try {\n let cnt = 0;\n for (const m of messages as Array<{ content?: unknown }>) {\n const content = m?.content;\n if (Array.isArray(content)) {\n for (const part of content as Array<{ type?: string }>) {\n if (part?.type === 'image_url') cnt++;\n }\n }\n }\n return cnt;\n } catch {\n return undefined as unknown as number \| undefined;\n }\n })();\n\n const summary = {\n id,\n model,\n messagesCount: messages.length,\n hasImages,\n };\n\n this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));\n\n const firstMsg = messages[0];\n if (firstMsg) {\n this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));\n }\n } catch (e) {\n this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);\n }\n }\n\n async onEachAgentLoopStart(sessionId: string) {\n this.logger.info('onEachAgentLoopStart', sessionId);\n }\n\n async onAgentLoopEnd(id: string): Promise<void> {\n // await this.browserOperator.cleanup();\n }\n\n async onBeforeToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n args: unknown,\n ) {\n return args;\n }\n\n async onAfterToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n result: unknown,\n ): Promise<unknown> {\n this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));\n\n if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) {\n this.logger.info('onAfterToolCall: skipping screenshot');\n return;\n }\n\n await sleep(this.loopIntervalInMs);\n\n const output = await this.operator!.doScreenshot();\n if (!output) {\n this.logger.error('Failed to get screenshot');\n return;\n }\n\n const base64Tool = new Base64ImageParser(output.base64);\n const base64Uri = base64Tool.getDataUri();\n if (!base64Uri) {\n this.logger.error('Failed to get base64 image uri');\n return;\n }\n\n const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() \|\| {\n width: -1,\n height: -1,\n };\n\n const content: ChatCompletionContentPart[] = [\n {\n type: 'image_url',\n image_url: {\n url: base64Uri,\n detail: this.detailCalculator(imageWidth, imageHeight),\n },\n },\n ];\n\n if (output?.url) {\n content.push({\n type: 'text',\n text: `The current page's url: ${output.url}`,\n });\n }\n\n const eventStream = this.getEventStream();\n const events = eventStream.getEvents();\n this.logger.info('onAfterToolCall events length:', events.length);\n\n const event = eventStream.createEvent('environment_input', {\n description: 'Browser Screenshot',\n content,\n metadata: {\n type: 'screenshot',\n url: output?.url,\n },\n });\n eventStream.sendEvent(event);\n return result;\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultLogger","ConsoleLogger","LogLevel","GUIAgent","BaseGUIAgent","Tool","GUI_ADAPTED_TOOL_NAME","input","createGUIErrorResponse","normalizeActionCoords","JSON","result","convertToAgentUIAction","undefined","id","payload","safeStringify","max","s","req","messages","Array","model","hasImages","cnt","m","content","part","summary","firstMsg","e","sessionId","toolCall","args","sleep","output","base64Tool","Base64ImageParser","base64Uri","imageWidth","imageHeight","eventStream","events","event","config","operator","systemPrompt","customeActionParser","normalizeCoordinates","detailCalculator","maxLoopCount","loopIntervalInMs","finalSystemPrompt","SYSTEM_PROMPT","p","isSystemPromptTemplate","assembleSystemPrompt","AdaptedToolCallEngine","GUIAgentToolCallEngine","defaultNormalizeCoords","defaultDetailCalculator"],"mappings":";;;;;;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;ACHC;;;;;;;;;;AA2BD,MAAMI,gBAAgB,IAAIC,4BAAAA,aAAaA,CAAC,cAAcC,4BAAAA,QAAAA,CAAAA,KAAc;AAE7D,MAAMC,iBAAqCC,qBAAAA,YAAYA;IAuD5D,MAAM,aAAa;QAEjB,IAAI,CAAC,YAAY,CACf,IAAIC,4BAAAA,IAAIA,CAAC;YACP,IAAIC,sCAAAA,qBAAqBA;YACzB,aAAa;YACb,YAAY,CAAC;YACb,UAAU,OAAOC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAGD,sCAAAA,qBAAqBA,CAAC,OAAO,CAAC,EAAEC;gBACnD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAChB,OAAOC,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAE;gBAE9C,IAAIA,MAAM,YAAY,EACpB,OAAOC,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAEA,MAAM,YAAY;gBAGhE,IAAIA,MAAM,eAAe,EACvBA,MAAM,eAAe,GAAGE,AAAAA,IAAAA,sBAAAA,qBAAAA,AAAAA,EACtBF,MAAM,eAAe,EACrB,IAAI,CAAC,oBAAoB;gBAG7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsBG,KAAK,SAAS,CAACH,MAAM,eAAe;gBAC3E,MAAMI,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,SAAS,CAAC;oBAC5C,SAAS;wBAACJ,MAAM,eAAe;qBAAC;gBAClC;gBACA,IAAII,OAAO,YAAY,EACrB,OAAOH,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAEI,OAAO,YAAY;gBAGjE,OAAO;oBACL,SAAS;oBACT,QAAQJ,MAAM,MAAM;oBACpB,kBAAkBK,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBL,MAAM,eAAe;oBAC9D,aAAaM;gBACf;YACF;QACF;QAEF,KAAK,CAAC;IACR;IAEA,MAAM,aAAaC,EAAU,EAAEC,OAA8B,EAAiB;QAC5E,IAAI;YACF,MAAMC,gBAAgB,CAACnB,KAAcoB,MAAM,GAAG;gBAC5C,IAAI;oBACF,MAAMC,IAAIR,KAAK,SAAS,CAACb;oBACzB,OAAOqB,EAAE,MAAM,GAAGD,MAAMC,EAAE,KAAK,CAAC,GAAGD,OAAO,sBAAiBC;gBAC7D,EAAE,OAAM;oBACN,OAAO;gBACT;YACF;YAEA,MAAMC,MAAMJ,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO;YAC5B,MAAMK,WAAWD,OAAOE,MAAM,OAAO,CAACF,IAAI,QAAQ,IAAIA,IAAI,QAAQ,GAAG,EAAE;YACvE,MAAMG,QAAQH,OAAO,AAAqB,YAArB,OAAOA,IAAI,KAAK,GAAgBA,IAAI,KAAK,GAAG;YAEjE,MAAMI,YAAa,AAAC;gBAClB,IAAI;oBACF,IAAIC,MAAM;oBACV,KAAK,MAAMC,KAAKL,SAA0C;wBACxD,MAAMM,UAAUD,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO;wBAC1B,IAAIJ,MAAM,OAAO,CAACK,UAChB;4BAAA,KAAK,MAAMC,QAAQD,QACjB,IAAIC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,AAAD,MAAM,aAAaH;wBAClC;oBAEJ;oBACA,OAAOA;gBACT,EAAE,OAAM;oBACN;gBACF;YACF;YAEA,MAAMI,UAAU;gBACdd;gBACAQ;gBACA,eAAeF,SAAS,MAAM;gBAC9BG;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoCP,cAAcY;YAEnE,MAAMC,WAAWT,QAAQ,CAAC,EAAE;YAC5B,IAAIS,UACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0Cb,cAAca,UAAU;QAExF,EAAE,OAAOC,GAAG;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2CA;QAC/D;IACF;IAEA,MAAM,qBAAqBC,SAAiB,EAAE;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwBA;IAC3C;IAEA,MAAM,eAAejB,EAAU,EAAiB,CAEhD;IAEA,MAAM,iBACJA,EAAU,EACVkB,QAA8C,EAC9CC,IAAa,EACb;QACA,OAAOA;IACT;IAEA,MAAM,gBACJnB,EAAU,EACVkB,QAA8C,EAC9CrB,MAAe,EACG;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4BD,KAAK,SAAS,CAACsB;QAE5D,IAAIA,SAAS,IAAI,KAAK1B,sCAAAA,qBAAqBA,EAAE,YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAInB,MAAM4B,AAAAA,IAAAA,sBAAAA,KAAAA,AAAAA,EAAM,IAAI,CAAC,gBAAgB;QAEjC,MAAMC,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,YAAY;QAChD,IAAI,CAACA,QAAQ,YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAMC,aAAa,IAAIC,4BAAAA,iBAAiBA,CAACF,OAAO,MAAM;QACtD,MAAMG,YAAYF,WAAW,UAAU;QACvC,IAAI,CAACE,WAAW,YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAM,EAAE,OAAOC,UAAU,EAAE,QAAQC,WAAW,EAAE,GAAGJ,WAAW,aAAa,MAAM;YAC/E,OAAO;YACP,QAAQ;QACV;QAEA,MAAMV,UAAuC;YAC3C;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKY;oBACL,QAAQ,IAAI,CAAC,gBAAgB,CAACC,YAAYC;gBAC5C;YACF;SACD;QAED,IAAIL,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG,EACbT,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,wBAAwB,EAAES,OAAO,GAAG,EAAE;QAC/C;QAGF,MAAMM,cAAc,IAAI,CAAC,cAAc;QACvC,MAAMC,SAASD,YAAY,SAAS;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkCC,OAAO,MAAM;QAEhE,MAAMC,QAAQF,YAAY,WAAW,CAAC,qBAAqB;YACzD,aAAa;YACbf;YACA,UAAU;gBACR,MAAM;gBACN,KAAKS,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG;YAClB;QACF;QACAM,YAAY,SAAS,CAACE;QACtB,OAAOhC;IACT;IA1NA,YAAYiC,MAAyB,CAAE;QACrC,MAAM,EACJC,QAAQ,EACRvB,KAAK,EACLwB,YAAY,EACZC,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,YAAY,EACZC,mBAAmB,GAAG,EACvB,GAAGP;QACJ,IAAIQ,oBAAoBC,oCAAAA,aAAaA;QACrC,IAAI,AAAwB,YAAxB,OAAOP,cACTM,oBAAoBN;aACf,IAAIzB,MAAM,OAAO,CAACyB,eACvBM,oBAAoBN,aACjB,GAAG,CAAC,CAACQ,IAAO,AAAa,YAAb,OAAOA,IAAiBA,IAAIA,EAAE,OAAO,EACjD,IAAI,CAAC;aACH,IAAIR,gBAAgBS,AAAAA,IAAAA,sBAAAA,sBAAAA,AAAAA,EAAuBT,eAChDM,oBAAoBI,AAAAA,IAAAA,sBAAAA,oBAAAA,AAAAA,EAAqBV,cAAcD,SAAS,mBAAmB;QAErF7C,cAAc,KAAK,CAAC,8BAA8BoD;QAGlD,MAAMK,wBAAwB,cAAcC,2CAAAA,sBAAsBA;YAChE,aAAc;gBACZ,KAAK,CAACX;YACR;QACF;QAEA,KAAK,CAAC;YACJ,MAAM5C,SAAS,KAAK;YACpB,cAAciD;YACd,OAAO,EAAE;YACT,gBAAgBK;YAChB,OAAOnC;YACP,GAAI4B,gBAAgB;gBAAE,eAAeA;YAAa,CAAC;YACnD,UAAUhD,4BAAAA,QAAAA,CAAAA,KAAc;QAC1B,IA3CF,uBAAQ,YAAR,SACA,uBAAQ,wBAAR,SACA,uBAAQ,oBAAR,SACA,uBAAQ,oBAAR;QAyCE,IAAI,CAAC,QAAQ,GAAG2C;QAChB,IAAI,CAAC,oBAAoB,GAAGG,wBAAwBW,yCAAAA,sBAAsBA;QAE1E,IAAI,CAAC,gBAAgB,GAAGV,oBAAoBW,yCAAAA,uBAAuBA;QACnE,IAAI,CAAC,gBAAgB,GAAGT;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC;AA8KF;AAlOE,iBADWhD,UACJ,SAAQ"}
1	+ {"version":3,"file":"GUIAgent.js","sources":["webpack://@ui-tars-test/agent-sdk/webpack/runtime/define_property_getters","webpack://@ui-tars-test/agent-sdk/webpack/runtime/has_own_property","webpack://@ui-tars-test/agent-sdk/webpack/runtime/make_namespace_object","webpack://@ui-tars-test/agent-sdk/./src/GUIAgent.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n }\n }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/\n Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n LLMRequestHookPayload,\n ChatCompletionContentPart,\n LogLevel,\n Tool,\n ConsoleLogger,\n} from '@ui-tars-test/tarko-agent';\nimport { GUIAgentToolCallEngine } from './ToolCallEngine';\nimport { SYSTEM_PROMPT } from './prompts';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';\nimport {\n GUIAgentConfig,\n NormalizeCoordinates,\n ImageDetailCalculator,\n} from '@ui-tars-test/shared/types';\nimport {\n assembleSystemPrompt,\n isSystemPromptTemplate,\n normalizeActionCoords,\n sleep,\n} from '@ui-tars-test/shared/utils';\nimport { GUI_ADAPTED_TOOL_NAME } from './constants';\nimport { convertToAgentUIAction, createGUIErrorResponse } from './utils';\nimport { defaultNormalizeCoords, defaultDetailCalculator } from './defaultImpls';\n\nconst defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);\n\nexport class GUIAgent<T extends Operator> extends BaseGUIAgent {\n static label = 'GUI Agent';\n\n private operator: Operator \| undefined;\n private normalizeCoordinates: NormalizeCoordinates;\n private detailCalculator: ImageDetailCalculator;\n private loopIntervalInMs: number;\n\n constructor(config: GUIAgentConfig<T>) {\n const {\n operator,\n model,\n systemPrompt,\n customeActionParser,\n normalizeCoordinates,\n detailCalculator,\n maxLoopCount,\n loopIntervalInMs = 500,\n } = config;\n let finalSystemPrompt = SYSTEM_PROMPT;\n if (typeof systemPrompt === 'string') {\n finalSystemPrompt = systemPrompt;\n } else if (Array.isArray(systemPrompt)) {\n finalSystemPrompt = systemPrompt\n .map((p) => (typeof p === 'string' ? p : p.content))\n .join('\\n\\n');\n } else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) {\n finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());\n }\n defaultLogger.debug('final instructions for sp:', finalSystemPrompt);\n\n // Create a adapted ToolCallEngine constructor that captures the customeActionParser\n const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {\n constructor() {\n super(customeActionParser);\n }\n };\n\n super({\n name: GUIAgent.label,\n instructions: finalSystemPrompt,\n tools: [],\n toolCallEngine: AdaptedToolCallEngine,\n model: model,\n ...(maxLoopCount && { maxIterations: maxLoopCount }),\n logLevel: LogLevel.DEBUG,\n });\n this.operator = operator;\n this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;\n // Default detail calculator implementation\n this.detailCalculator = detailCalculator ?? defaultDetailCalculator;\n this.loopIntervalInMs = loopIntervalInMs;\n this.logger = this.logger.spawn('[GUIAgent]');\n }\n\n async initialize() {\n // Register the GUI tool\n this.registerTool(\n new Tool({\n id: GUI_ADAPTED_TOOL_NAME,\n description: 'operator tool',\n parameters: {}, // no need to pass parameters\n function: async (input) => {\n this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);\n if (!this.operator) {\n return createGUIErrorResponse(input.action, 'Operator not initialized');\n }\n if (input.errorMessage) {\n return createGUIErrorResponse(input.action, input.errorMessage);\n }\n // normalize coordinates\n if (input.operator_action) {\n input.operator_action = normalizeActionCoords(\n input.operator_action,\n this.normalizeCoordinates,\n );\n }\n this.logger.info('action to execute:', JSON.stringify(input.operator_action));\n const result = await this.operator!.doExecute({\n actions: [input.operator_action],\n });\n if (result.errorMessage) {\n return createGUIErrorResponse(input.action, result.errorMessage);\n }\n // return { action: input.action, status: 'success', result };\n return {\n success: true,\n action: input.action,\n normalizedAction: convertToAgentUIAction(input.operator_action),\n observation: undefined, // Reserved for future implementation\n };\n },\n }),\n );\n super.initialize();\n }\n\n async onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void> {\n try {\n const safeStringify = (obj: unknown, max = 800): string => {\n try {\n const s = JSON.stringify(obj);\n return s.length > max ? s.slice(0, max) + '…(truncated)' : s;\n } catch {\n return '[unserializable]';\n }\n };\n\n const req = payload?.request;\n const messages = req && Array.isArray(req.messages) ? req.messages : [];\n const model = req && typeof req.model === 'string' ? req.model : 'unknown';\n\n const hasImages = (() => {\n try {\n let cnt = 0;\n for (const m of messages as Array<{ content?: unknown }>) {\n const content = m?.content;\n if (Array.isArray(content)) {\n for (const part of content as Array<{ type?: string }>) {\n if (part?.type === 'image_url') cnt++;\n }\n }\n }\n return cnt;\n } catch {\n return undefined as unknown as number \| undefined;\n }\n })();\n\n const summary = {\n id,\n model,\n messagesCount: messages.length,\n hasImages,\n };\n\n this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));\n\n const firstMsg = messages[0];\n if (firstMsg) {\n this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));\n }\n } catch (e) {\n this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);\n }\n }\n\n async onEachAgentLoopStart(sessionId: string) {\n this.logger.info('onEachAgentLoopStart', sessionId);\n }\n\n async onAgentLoopEnd(id: string): Promise<void> {\n // await this.browserOperator.cleanup();\n }\n\n async onBeforeToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n args: unknown,\n ) {\n return args;\n }\n\n async onAfterToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n result: unknown,\n ): Promise<unknown> {\n this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));\n\n if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) {\n this.logger.info('onAfterToolCall: skipping screenshot');\n return;\n }\n\n await sleep(this.loopIntervalInMs);\n\n const output = await this.operator!.doScreenshot();\n if (!output) {\n this.logger.error('Failed to get screenshot');\n return;\n }\n\n const base64Tool = new Base64ImageParser(output.base64);\n const base64Uri = base64Tool.getDataUri();\n if (!base64Uri) {\n this.logger.error('Failed to get base64 image uri');\n return;\n }\n\n const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() \|\| {\n width: -1,\n height: -1,\n };\n\n const content: ChatCompletionContentPart[] = [\n {\n type: 'image_url',\n image_url: {\n url: base64Uri,\n detail: this.detailCalculator(imageWidth, imageHeight),\n },\n },\n {\n type: 'text',\n text: `Screen resolution: ${imageWidth}x${imageHeight}. System is running on macOS. Please return actions with absolute coordinates within this resolution.`,\n },\n ];\n\n if (output?.url) {\n content.push({\n type: 'text',\n text: `The current page's url: ${output.url}`,\n });\n }\n\n const eventStream = this.getEventStream();\n const events = eventStream.getEvents();\n this.logger.info('onAfterToolCall events length:', events.length);\n\n const event = eventStream.createEvent('environment_input', {\n description: 'Browser Screenshot',\n content,\n metadata: {\n type: 'screenshot',\n url: output?.url,\n },\n });\n eventStream.sendEvent(event);\n return result;\n }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","defaultLogger","ConsoleLogger","LogLevel","GUIAgent","BaseGUIAgent","Tool","GUI_ADAPTED_TOOL_NAME","input","createGUIErrorResponse","normalizeActionCoords","JSON","result","convertToAgentUIAction","undefined","id","payload","safeStringify","max","s","req","messages","Array","model","hasImages","cnt","m","content","part","summary","firstMsg","e","sessionId","toolCall","args","sleep","output","base64Tool","Base64ImageParser","base64Uri","imageWidth","imageHeight","eventStream","events","event","config","operator","systemPrompt","customeActionParser","normalizeCoordinates","detailCalculator","maxLoopCount","loopIntervalInMs","finalSystemPrompt","SYSTEM_PROMPT","p","isSystemPromptTemplate","assembleSystemPrompt","AdaptedToolCallEngine","GUIAgentToolCallEngine","defaultNormalizeCoords","defaultDetailCalculator"],"mappings":";;;;;;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;ACHC;;;;;;;;;;AA2BD,MAAMI,gBAAgB,IAAIC,4BAAAA,aAAaA,CAAC,cAAcC,4BAAAA,QAAAA,CAAAA,KAAc;AAE7D,MAAMC,iBAAqCC,qBAAAA,YAAYA;IAuD5D,MAAM,aAAa;QAEjB,IAAI,CAAC,YAAY,CACf,IAAIC,4BAAAA,IAAIA,CAAC;YACP,IAAIC,sCAAAA,qBAAqBA;YACzB,aAAa;YACb,YAAY,CAAC;YACb,UAAU,OAAOC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAGD,sCAAAA,qBAAqBA,CAAC,OAAO,CAAC,EAAEC;gBACnD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAChB,OAAOC,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAE;gBAE9C,IAAIA,MAAM,YAAY,EACpB,OAAOC,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAEA,MAAM,YAAY;gBAGhE,IAAIA,MAAM,eAAe,EACvBA,MAAM,eAAe,GAAGE,AAAAA,IAAAA,sBAAAA,qBAAAA,AAAAA,EACtBF,MAAM,eAAe,EACrB,IAAI,CAAC,oBAAoB;gBAG7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsBG,KAAK,SAAS,CAACH,MAAM,eAAe;gBAC3E,MAAMI,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,SAAS,CAAC;oBAC5C,SAAS;wBAACJ,MAAM,eAAe;qBAAC;gBAClC;gBACA,IAAII,OAAO,YAAY,EACrB,OAAOH,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBD,MAAM,MAAM,EAAEI,OAAO,YAAY;gBAGjE,OAAO;oBACL,SAAS;oBACT,QAAQJ,MAAM,MAAM;oBACpB,kBAAkBK,AAAAA,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuBL,MAAM,eAAe;oBAC9D,aAAaM;gBACf;YACF;QACF;QAEF,KAAK,CAAC;IACR;IAEA,MAAM,aAAaC,EAAU,EAAEC,OAA8B,EAAiB;QAC5E,IAAI;YACF,MAAMC,gBAAgB,CAACnB,KAAcoB,MAAM,GAAG;gBAC5C,IAAI;oBACF,MAAMC,IAAIR,KAAK,SAAS,CAACb;oBACzB,OAAOqB,EAAE,MAAM,GAAGD,MAAMC,EAAE,KAAK,CAAC,GAAGD,OAAO,sBAAiBC;gBAC7D,EAAE,OAAM;oBACN,OAAO;gBACT;YACF;YAEA,MAAMC,MAAMJ,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO;YAC5B,MAAMK,WAAWD,OAAOE,MAAM,OAAO,CAACF,IAAI,QAAQ,IAAIA,IAAI,QAAQ,GAAG,EAAE;YACvE,MAAMG,QAAQH,OAAO,AAAqB,YAArB,OAAOA,IAAI,KAAK,GAAgBA,IAAI,KAAK,GAAG;YAEjE,MAAMI,YAAa,AAAC;gBAClB,IAAI;oBACF,IAAIC,MAAM;oBACV,KAAK,MAAMC,KAAKL,SAA0C;wBACxD,MAAMM,UAAUD,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO;wBAC1B,IAAIJ,MAAM,OAAO,CAACK,UAChB;4BAAA,KAAK,MAAMC,QAAQD,QACjB,IAAIC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,AAAD,MAAM,aAAaH;wBAClC;oBAEJ;oBACA,OAAOA;gBACT,EAAE,OAAM;oBACN;gBACF;YACF;YAEA,MAAMI,UAAU;gBACdd;gBACAQ;gBACA,eAAeF,SAAS,MAAM;gBAC9BG;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoCP,cAAcY;YAEnE,MAAMC,WAAWT,QAAQ,CAAC,EAAE;YAC5B,IAAIS,UACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0Cb,cAAca,UAAU;QAExF,EAAE,OAAOC,GAAG;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2CA;QAC/D;IACF;IAEA,MAAM,qBAAqBC,SAAiB,EAAE;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwBA;IAC3C;IAEA,MAAM,eAAejB,EAAU,EAAiB,CAEhD;IAEA,MAAM,iBACJA,EAAU,EACVkB,QAA8C,EAC9CC,IAAa,EACb;QACA,OAAOA;IACT;IAEA,MAAM,gBACJnB,EAAU,EACVkB,QAA8C,EAC9CrB,MAAe,EACG;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4BD,KAAK,SAAS,CAACsB;QAE5D,IAAIA,SAAS,IAAI,KAAK1B,sCAAAA,qBAAqBA,EAAE,YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAInB,MAAM4B,AAAAA,IAAAA,sBAAAA,KAAAA,AAAAA,EAAM,IAAI,CAAC,gBAAgB;QAEjC,MAAMC,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,YAAY;QAChD,IAAI,CAACA,QAAQ,YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAMC,aAAa,IAAIC,4BAAAA,iBAAiBA,CAACF,OAAO,MAAM;QACtD,MAAMG,YAAYF,WAAW,UAAU;QACvC,IAAI,CAACE,WAAW,YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAM,EAAE,OAAOC,UAAU,EAAE,QAAQC,WAAW,EAAE,GAAGJ,WAAW,aAAa,MAAM;YAC/E,OAAO;YACP,QAAQ;QACV;QAEA,MAAMV,UAAuC;YAC3C;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKY;oBACL,QAAQ,IAAI,CAAC,gBAAgB,CAACC,YAAYC;gBAC5C;YACF;YACA;gBACE,MAAM;gBACN,MAAM,CAAC,mBAAmB,EAAED,WAAW,CAAC,EAAEC,YAAY,qGAAqG,CAAC;YAC9J;SACD;QAED,IAAIL,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG,EACbT,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,wBAAwB,EAAES,OAAO,GAAG,EAAE;QAC/C;QAGF,MAAMM,cAAc,IAAI,CAAC,cAAc;QACvC,MAAMC,SAASD,YAAY,SAAS;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkCC,OAAO,MAAM;QAEhE,MAAMC,QAAQF,YAAY,WAAW,CAAC,qBAAqB;YACzD,aAAa;YACbf;YACA,UAAU;gBACR,MAAM;gBACN,KAAKS,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG;YAClB;QACF;QACAM,YAAY,SAAS,CAACE;QACtB,OAAOhC;IACT;IA9NA,YAAYiC,MAAyB,CAAE;QACrC,MAAM,EACJC,QAAQ,EACRvB,KAAK,EACLwB,YAAY,EACZC,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,YAAY,EACZC,mBAAmB,GAAG,EACvB,GAAGP;QACJ,IAAIQ,oBAAoBC,oCAAAA,aAAaA;QACrC,IAAI,AAAwB,YAAxB,OAAOP,cACTM,oBAAoBN;aACf,IAAIzB,MAAM,OAAO,CAACyB,eACvBM,oBAAoBN,aACjB,GAAG,CAAC,CAACQ,IAAO,AAAa,YAAb,OAAOA,IAAiBA,IAAIA,EAAE,OAAO,EACjD,IAAI,CAAC;aACH,IAAIR,gBAAgBS,AAAAA,IAAAA,sBAAAA,sBAAAA,AAAAA,EAAuBT,eAChDM,oBAAoBI,AAAAA,IAAAA,sBAAAA,oBAAAA,AAAAA,EAAqBV,cAAcD,SAAS,mBAAmB;QAErF7C,cAAc,KAAK,CAAC,8BAA8BoD;QAGlD,MAAMK,wBAAwB,cAAcC,2CAAAA,sBAAsBA;YAChE,aAAc;gBACZ,KAAK,CAACX;YACR;QACF;QAEA,KAAK,CAAC;YACJ,MAAM5C,SAAS,KAAK;YACpB,cAAciD;YACd,OAAO,EAAE;YACT,gBAAgBK;YAChB,OAAOnC;YACP,GAAI4B,gBAAgB;gBAAE,eAAeA;YAAa,CAAC;YACnD,UAAUhD,4BAAAA,QAAAA,CAAAA,KAAc;QAC1B,IA3CF,uBAAQ,YAAR,SACA,uBAAQ,wBAAR,SACA,uBAAQ,oBAAR,SACA,uBAAQ,oBAAR;QAyCE,IAAI,CAAC,QAAQ,GAAG2C;QAChB,IAAI,CAAC,oBAAoB,GAAGG,wBAAwBW,yCAAAA,sBAAsBA;QAE1E,IAAI,CAAC,gBAAgB,GAAGV,oBAAoBW,yCAAAA,uBAAuBA;QACnE,IAAI,CAAC,gBAAgB,GAAGT;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC;AAkLF;AAtOE,iBADWhD,UACJ,SAAQ"}

package/dist/GUIAgent.mjs CHANGED Viewed

@@ -117,6 +117,10 @@ class GUIAgent extends BaseGUIAgent {
                     url: base64Uri,
                     detail: this.detailCalculator(imageWidth, imageHeight)
                 }
+            },
+            {
+                type: 'text',
+                text: `Screen resolution: ${imageWidth}x${imageHeight}. System is running on macOS. Please return actions with absolute coordinates within this resolution.`
             }
         ];
         if (null == output ? void 0 : output.url) content.push({

package/dist/GUIAgent.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"GUIAgent.mjs","sources":["webpack://@ui-tars-test/agent-sdk/./src/GUIAgent.ts"],"sourcesContent":["/\n Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n LLMRequestHookPayload,\n ChatCompletionContentPart,\n LogLevel,\n Tool,\n ConsoleLogger,\n} from '@ui-tars-test/tarko-agent';\nimport { GUIAgentToolCallEngine } from './ToolCallEngine';\nimport { SYSTEM_PROMPT } from './prompts';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';\nimport {\n GUIAgentConfig,\n NormalizeCoordinates,\n ImageDetailCalculator,\n} from '@ui-tars-test/shared/types';\nimport {\n assembleSystemPrompt,\n isSystemPromptTemplate,\n normalizeActionCoords,\n sleep,\n} from '@ui-tars-test/shared/utils';\nimport { GUI_ADAPTED_TOOL_NAME } from './constants';\nimport { convertToAgentUIAction, createGUIErrorResponse } from './utils';\nimport { defaultNormalizeCoords, defaultDetailCalculator } from './defaultImpls';\n\nconst defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);\n\nexport class GUIAgent<T extends Operator> extends BaseGUIAgent {\n static label = 'GUI Agent';\n\n private operator: Operator \| undefined;\n private normalizeCoordinates: NormalizeCoordinates;\n private detailCalculator: ImageDetailCalculator;\n private loopIntervalInMs: number;\n\n constructor(config: GUIAgentConfig<T>) {\n const {\n operator,\n model,\n systemPrompt,\n customeActionParser,\n normalizeCoordinates,\n detailCalculator,\n maxLoopCount,\n loopIntervalInMs = 500,\n } = config;\n let finalSystemPrompt = SYSTEM_PROMPT;\n if (typeof systemPrompt === 'string') {\n finalSystemPrompt = systemPrompt;\n } else if (Array.isArray(systemPrompt)) {\n finalSystemPrompt = systemPrompt\n .map((p) => (typeof p === 'string' ? p : p.content))\n .join('\\n\\n');\n } else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) {\n finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());\n }\n defaultLogger.debug('final instructions for sp:', finalSystemPrompt);\n\n // Create a adapted ToolCallEngine constructor that captures the customeActionParser\n const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {\n constructor() {\n super(customeActionParser);\n }\n };\n\n super({\n name: GUIAgent.label,\n instructions: finalSystemPrompt,\n tools: [],\n toolCallEngine: AdaptedToolCallEngine,\n model: model,\n ...(maxLoopCount && { maxIterations: maxLoopCount }),\n logLevel: LogLevel.DEBUG,\n });\n this.operator = operator;\n this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;\n // Default detail calculator implementation\n this.detailCalculator = detailCalculator ?? defaultDetailCalculator;\n this.loopIntervalInMs = loopIntervalInMs;\n this.logger = this.logger.spawn('[GUIAgent]');\n }\n\n async initialize() {\n // Register the GUI tool\n this.registerTool(\n new Tool({\n id: GUI_ADAPTED_TOOL_NAME,\n description: 'operator tool',\n parameters: {}, // no need to pass parameters\n function: async (input) => {\n this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);\n if (!this.operator) {\n return createGUIErrorResponse(input.action, 'Operator not initialized');\n }\n if (input.errorMessage) {\n return createGUIErrorResponse(input.action, input.errorMessage);\n }\n // normalize coordinates\n if (input.operator_action) {\n input.operator_action = normalizeActionCoords(\n input.operator_action,\n this.normalizeCoordinates,\n );\n }\n this.logger.info('action to execute:', JSON.stringify(input.operator_action));\n const result = await this.operator!.doExecute({\n actions: [input.operator_action],\n });\n if (result.errorMessage) {\n return createGUIErrorResponse(input.action, result.errorMessage);\n }\n // return { action: input.action, status: 'success', result };\n return {\n success: true,\n action: input.action,\n normalizedAction: convertToAgentUIAction(input.operator_action),\n observation: undefined, // Reserved for future implementation\n };\n },\n }),\n );\n super.initialize();\n }\n\n async onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void> {\n try {\n const safeStringify = (obj: unknown, max = 800): string => {\n try {\n const s = JSON.stringify(obj);\n return s.length > max ? s.slice(0, max) + '…(truncated)' : s;\n } catch {\n return '[unserializable]';\n }\n };\n\n const req = payload?.request;\n const messages = req && Array.isArray(req.messages) ? req.messages : [];\n const model = req && typeof req.model === 'string' ? req.model : 'unknown';\n\n const hasImages = (() => {\n try {\n let cnt = 0;\n for (const m of messages as Array<{ content?: unknown }>) {\n const content = m?.content;\n if (Array.isArray(content)) {\n for (const part of content as Array<{ type?: string }>) {\n if (part?.type === 'image_url') cnt++;\n }\n }\n }\n return cnt;\n } catch {\n return undefined as unknown as number \| undefined;\n }\n })();\n\n const summary = {\n id,\n model,\n messagesCount: messages.length,\n hasImages,\n };\n\n this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));\n\n const firstMsg = messages[0];\n if (firstMsg) {\n this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));\n }\n } catch (e) {\n this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);\n }\n }\n\n async onEachAgentLoopStart(sessionId: string) {\n this.logger.info('onEachAgentLoopStart', sessionId);\n }\n\n async onAgentLoopEnd(id: string): Promise<void> {\n // await this.browserOperator.cleanup();\n }\n\n async onBeforeToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n args: unknown,\n ) {\n return args;\n }\n\n async onAfterToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n result: unknown,\n ): Promise<unknown> {\n this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));\n\n if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) {\n this.logger.info('onAfterToolCall: skipping screenshot');\n return;\n }\n\n await sleep(this.loopIntervalInMs);\n\n const output = await this.operator!.doScreenshot();\n if (!output) {\n this.logger.error('Failed to get screenshot');\n return;\n }\n\n const base64Tool = new Base64ImageParser(output.base64);\n const base64Uri = base64Tool.getDataUri();\n if (!base64Uri) {\n this.logger.error('Failed to get base64 image uri');\n return;\n }\n\n const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() \|\| {\n width: -1,\n height: -1,\n };\n\n const content: ChatCompletionContentPart[] = [\n {\n type: 'image_url',\n image_url: {\n url: base64Uri,\n detail: this.detailCalculator(imageWidth, imageHeight),\n },\n },\n ];\n\n if (output?.url) {\n content.push({\n type: 'text',\n text: `The current page's url: ${output.url}`,\n });\n }\n\n const eventStream = this.getEventStream();\n const events = eventStream.getEvents();\n this.logger.info('onAfterToolCall events length:', events.length);\n\n const event = eventStream.createEvent('environment_input', {\n description: 'Browser Screenshot',\n content,\n metadata: {\n type: 'screenshot',\n url: output?.url,\n },\n });\n eventStream.sendEvent(event);\n return result;\n }\n}\n"],"names":["defaultLogger","ConsoleLogger","LogLevel","GUIAgent","BaseGUIAgent","Tool","GUI_ADAPTED_TOOL_NAME","input","createGUIErrorResponse","normalizeActionCoords","JSON","result","convertToAgentUIAction","undefined","id","payload","safeStringify","obj","max","s","req","messages","Array","model","hasImages","cnt","m","content","part","summary","firstMsg","e","sessionId","toolCall","args","sleep","output","base64Tool","Base64ImageParser","base64Uri","imageWidth","imageHeight","eventStream","events","event","config","operator","systemPrompt","customeActionParser","normalizeCoordinates","detailCalculator","maxLoopCount","loopIntervalInMs","finalSystemPrompt","SYSTEM_PROMPT","p","isSystemPromptTemplate","assembleSystemPrompt","AdaptedToolCallEngine","GUIAgentToolCallEngine","defaultNormalizeCoords","defaultDetailCalculator"],"mappings":";;;;;;;;;;;;;AAGC;;;;;;;;;;AA2BD,MAAMA,gBAAgB,IAAIC,cAAc,cAAcC,SAAS,KAAK;AAE7D,MAAMC,iBAAqCC;IAuDhD,MAAM,aAAa;QAEjB,IAAI,CAAC,YAAY,CACf,IAAIC,KAAK;YACP,IAAIC;YACJ,aAAa;YACb,YAAY,CAAC;YACb,UAAU,OAAOC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAGD,sBAAsB,OAAO,CAAC,EAAEC;gBACnD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAChB,OAAOC,uBAAuBD,MAAM,MAAM,EAAE;gBAE9C,IAAIA,MAAM,YAAY,EACpB,OAAOC,uBAAuBD,MAAM,MAAM,EAAEA,MAAM,YAAY;gBAGhE,IAAIA,MAAM,eAAe,EACvBA,MAAM,eAAe,GAAGE,sBACtBF,MAAM,eAAe,EACrB,IAAI,CAAC,oBAAoB;gBAG7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsBG,KAAK,SAAS,CAACH,MAAM,eAAe;gBAC3E,MAAMI,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,SAAS,CAAC;oBAC5C,SAAS;wBAACJ,MAAM,eAAe;qBAAC;gBAClC;gBACA,IAAII,OAAO,YAAY,EACrB,OAAOH,uBAAuBD,MAAM,MAAM,EAAEI,OAAO,YAAY;gBAGjE,OAAO;oBACL,SAAS;oBACT,QAAQJ,MAAM,MAAM;oBACpB,kBAAkBK,uBAAuBL,MAAM,eAAe;oBAC9D,aAAaM;gBACf;YACF;QACF;QAEF,KAAK,CAAC;IACR;IAEA,MAAM,aAAaC,EAAU,EAAEC,OAA8B,EAAiB;QAC5E,IAAI;YACF,MAAMC,gBAAgB,CAACC,KAAcC,MAAM,GAAG;gBAC5C,IAAI;oBACF,MAAMC,IAAIT,KAAK,SAAS,CAACO;oBACzB,OAAOE,EAAE,MAAM,GAAGD,MAAMC,EAAE,KAAK,CAAC,GAAGD,OAAO,sBAAiBC;gBAC7D,EAAE,OAAM;oBACN,OAAO;gBACT;YACF;YAEA,MAAMC,MAAML,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO;YAC5B,MAAMM,WAAWD,OAAOE,MAAM,OAAO,CAACF,IAAI,QAAQ,IAAIA,IAAI,QAAQ,GAAG,EAAE;YACvE,MAAMG,QAAQH,OAAO,AAAqB,YAArB,OAAOA,IAAI,KAAK,GAAgBA,IAAI,KAAK,GAAG;YAEjE,MAAMI,YAAa,AAAC;gBAClB,IAAI;oBACF,IAAIC,MAAM;oBACV,KAAK,MAAMC,KAAKL,SAA0C;wBACxD,MAAMM,UAAUD,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO;wBAC1B,IAAIJ,MAAM,OAAO,CAACK,UAChB;4BAAA,KAAK,MAAMC,QAAQD,QACjB,IAAIC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,AAAD,MAAM,aAAaH;wBAClC;oBAEJ;oBACA,OAAOA;gBACT,EAAE,OAAM;oBACN;gBACF;YACF;YAEA,MAAMI,UAAU;gBACdf;gBACAS;gBACA,eAAeF,SAAS,MAAM;gBAC9BG;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoCR,cAAca;YAEnE,MAAMC,WAAWT,QAAQ,CAAC,EAAE;YAC5B,IAAIS,UACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0Cd,cAAcc,UAAU;QAExF,EAAE,OAAOC,GAAG;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2CA;QAC/D;IACF;IAEA,MAAM,qBAAqBC,SAAiB,EAAE;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwBA;IAC3C;IAEA,MAAM,eAAelB,EAAU,EAAiB,CAEhD;IAEA,MAAM,iBACJA,EAAU,EACVmB,QAA8C,EAC9CC,IAAa,EACb;QACA,OAAOA;IACT;IAEA,MAAM,gBACJpB,EAAU,EACVmB,QAA8C,EAC9CtB,MAAe,EACG;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4BD,KAAK,SAAS,CAACuB;QAE5D,IAAIA,SAAS,IAAI,KAAK3B,uBAAuB,YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAInB,MAAM6B,MAAM,IAAI,CAAC,gBAAgB;QAEjC,MAAMC,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,YAAY;QAChD,IAAI,CAACA,QAAQ,YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAMC,aAAa,IAAIC,kBAAkBF,OAAO,MAAM;QACtD,MAAMG,YAAYF,WAAW,UAAU;QACvC,IAAI,CAACE,WAAW,YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAM,EAAE,OAAOC,UAAU,EAAE,QAAQC,WAAW,EAAE,GAAGJ,WAAW,aAAa,MAAM;YAC/E,OAAO;YACP,QAAQ;QACV;QAEA,MAAMV,UAAuC;YAC3C;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKY;oBACL,QAAQ,IAAI,CAAC,gBAAgB,CAACC,YAAYC;gBAC5C;YACF;SACD;QAED,IAAIL,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG,EACbT,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,wBAAwB,EAAES,OAAO,GAAG,EAAE;QAC/C;QAGF,MAAMM,cAAc,IAAI,CAAC,cAAc;QACvC,MAAMC,SAASD,YAAY,SAAS;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkCC,OAAO,MAAM;QAEhE,MAAMC,QAAQF,YAAY,WAAW,CAAC,qBAAqB;YACzD,aAAa;YACbf;YACA,UAAU;gBACR,MAAM;gBACN,KAAKS,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG;YAClB;QACF;QACAM,YAAY,SAAS,CAACE;QACtB,OAAOjC;IACT;IA1NA,YAAYkC,MAAyB,CAAE;QACrC,MAAM,EACJC,QAAQ,EACRvB,KAAK,EACLwB,YAAY,EACZC,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,YAAY,EACZC,mBAAmB,GAAG,EACvB,GAAGP;QACJ,IAAIQ,oBAAoBC;QACxB,IAAI,AAAwB,YAAxB,OAAOP,cACTM,oBAAoBN;aACf,IAAIzB,MAAM,OAAO,CAACyB,eACvBM,oBAAoBN,aACjB,GAAG,CAAC,CAACQ,IAAO,AAAa,YAAb,OAAOA,IAAiBA,IAAIA,EAAE,OAAO,EACjD,IAAI,CAAC;aACH,IAAIR,gBAAgBS,uBAAuBT,eAChDM,oBAAoBI,qBAAqBV,cAAcD,SAAS,mBAAmB;QAErF9C,cAAc,KAAK,CAAC,8BAA8BqD;QAGlD,MAAMK,wBAAwB,cAAcC;YAC1C,aAAc;gBACZ,KAAK,CAACX;YACR;QACF;QAEA,KAAK,CAAC;YACJ,MAAM7C,SAAS,KAAK;YACpB,cAAckD;YACd,OAAO,EAAE;YACT,gBAAgBK;YAChB,OAAOnC;YACP,GAAI4B,gBAAgB;gBAAE,eAAeA;YAAa,CAAC;YACnD,UAAUjD,SAAS,KAAK;QAC1B,IA3CF,uBAAQ,YAAR,SACA,uBAAQ,wBAAR,SACA,uBAAQ,oBAAR,SACA,uBAAQ,oBAAR;QAyCE,IAAI,CAAC,QAAQ,GAAG4C;QAChB,IAAI,CAAC,oBAAoB,GAAGG,wBAAwBW;QAEpD,IAAI,CAAC,gBAAgB,GAAGV,oBAAoBW;QAC5C,IAAI,CAAC,gBAAgB,GAAGT;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC;AA8KF;AAlOE,iBADWjD,UACJ,SAAQ"}
1	+ {"version":3,"file":"GUIAgent.mjs","sources":["webpack://@ui-tars-test/agent-sdk/./src/GUIAgent.ts"],"sourcesContent":["/\n Copyright (c) 2025 Bytedance, Inc. and its affiliates.\n * SPDX-License-Identifier: Apache-2.0\n */\nimport {\n LLMRequestHookPayload,\n ChatCompletionContentPart,\n LogLevel,\n Tool,\n ConsoleLogger,\n} from '@ui-tars-test/tarko-agent';\nimport { GUIAgentToolCallEngine } from './ToolCallEngine';\nimport { SYSTEM_PROMPT } from './prompts';\nimport { Base64ImageParser } from '@agent-infra/media-utils';\nimport { Operator, BaseGUIAgent } from '@ui-tars-test/shared/base';\nimport {\n GUIAgentConfig,\n NormalizeCoordinates,\n ImageDetailCalculator,\n} from '@ui-tars-test/shared/types';\nimport {\n assembleSystemPrompt,\n isSystemPromptTemplate,\n normalizeActionCoords,\n sleep,\n} from '@ui-tars-test/shared/utils';\nimport { GUI_ADAPTED_TOOL_NAME } from './constants';\nimport { convertToAgentUIAction, createGUIErrorResponse } from './utils';\nimport { defaultNormalizeCoords, defaultDetailCalculator } from './defaultImpls';\n\nconst defaultLogger = new ConsoleLogger('[GUIAgent]', LogLevel.DEBUG);\n\nexport class GUIAgent<T extends Operator> extends BaseGUIAgent {\n static label = 'GUI Agent';\n\n private operator: Operator \| undefined;\n private normalizeCoordinates: NormalizeCoordinates;\n private detailCalculator: ImageDetailCalculator;\n private loopIntervalInMs: number;\n\n constructor(config: GUIAgentConfig<T>) {\n const {\n operator,\n model,\n systemPrompt,\n customeActionParser,\n normalizeCoordinates,\n detailCalculator,\n maxLoopCount,\n loopIntervalInMs = 500,\n } = config;\n let finalSystemPrompt = SYSTEM_PROMPT;\n if (typeof systemPrompt === 'string') {\n finalSystemPrompt = systemPrompt;\n } else if (Array.isArray(systemPrompt)) {\n finalSystemPrompt = systemPrompt\n .map((p) => (typeof p === 'string' ? p : p.content))\n .join('\\n\\n');\n } else if (systemPrompt && isSystemPromptTemplate(systemPrompt)) {\n finalSystemPrompt = assembleSystemPrompt(systemPrompt, operator.getSupportedActions());\n }\n defaultLogger.debug('final instructions for sp:', finalSystemPrompt);\n\n // Create a adapted ToolCallEngine constructor that captures the customeActionParser\n const AdaptedToolCallEngine = class extends GUIAgentToolCallEngine {\n constructor() {\n super(customeActionParser);\n }\n };\n\n super({\n name: GUIAgent.label,\n instructions: finalSystemPrompt,\n tools: [],\n toolCallEngine: AdaptedToolCallEngine,\n model: model,\n ...(maxLoopCount && { maxIterations: maxLoopCount }),\n logLevel: LogLevel.DEBUG,\n });\n this.operator = operator;\n this.normalizeCoordinates = normalizeCoordinates ?? defaultNormalizeCoords;\n // Default detail calculator implementation\n this.detailCalculator = detailCalculator ?? defaultDetailCalculator;\n this.loopIntervalInMs = loopIntervalInMs;\n this.logger = this.logger.spawn('[GUIAgent]');\n }\n\n async initialize() {\n // Register the GUI tool\n this.registerTool(\n new Tool({\n id: GUI_ADAPTED_TOOL_NAME,\n description: 'operator tool',\n parameters: {}, // no need to pass parameters\n function: async (input) => {\n this.logger.log(`${GUI_ADAPTED_TOOL_NAME} input:`, input);\n if (!this.operator) {\n return createGUIErrorResponse(input.action, 'Operator not initialized');\n }\n if (input.errorMessage) {\n return createGUIErrorResponse(input.action, input.errorMessage);\n }\n // normalize coordinates\n if (input.operator_action) {\n input.operator_action = normalizeActionCoords(\n input.operator_action,\n this.normalizeCoordinates,\n );\n }\n this.logger.info('action to execute:', JSON.stringify(input.operator_action));\n const result = await this.operator!.doExecute({\n actions: [input.operator_action],\n });\n if (result.errorMessage) {\n return createGUIErrorResponse(input.action, result.errorMessage);\n }\n // return { action: input.action, status: 'success', result };\n return {\n success: true,\n action: input.action,\n normalizedAction: convertToAgentUIAction(input.operator_action),\n observation: undefined, // Reserved for future implementation\n };\n },\n }),\n );\n super.initialize();\n }\n\n async onLLMRequest(id: string, payload: LLMRequestHookPayload): Promise<void> {\n try {\n const safeStringify = (obj: unknown, max = 800): string => {\n try {\n const s = JSON.stringify(obj);\n return s.length > max ? s.slice(0, max) + '…(truncated)' : s;\n } catch {\n return '[unserializable]';\n }\n };\n\n const req = payload?.request;\n const messages = req && Array.isArray(req.messages) ? req.messages : [];\n const model = req && typeof req.model === 'string' ? req.model : 'unknown';\n\n const hasImages = (() => {\n try {\n let cnt = 0;\n for (const m of messages as Array<{ content?: unknown }>) {\n const content = m?.content;\n if (Array.isArray(content)) {\n for (const part of content as Array<{ type?: string }>) {\n if (part?.type === 'image_url') cnt++;\n }\n }\n }\n return cnt;\n } catch {\n return undefined as unknown as number \| undefined;\n }\n })();\n\n const summary = {\n id,\n model,\n messagesCount: messages.length,\n hasImages,\n };\n\n this.logger.info('[GUIAgent] onLLMRequest summary:', safeStringify(summary));\n\n const firstMsg = messages[0];\n if (firstMsg) {\n this.logger.debug('[GUIAgent] onLLMRequest first message:', safeStringify(firstMsg, 1200));\n }\n } catch (e) {\n this.logger.error('[GUIAgent] onLLMRequest logging failed:', e);\n }\n }\n\n async onEachAgentLoopStart(sessionId: string) {\n this.logger.info('onEachAgentLoopStart', sessionId);\n }\n\n async onAgentLoopEnd(id: string): Promise<void> {\n // await this.browserOperator.cleanup();\n }\n\n async onBeforeToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n args: unknown,\n ) {\n return args;\n }\n\n async onAfterToolCall(\n id: string,\n toolCall: { toolCallId: string; name: string },\n result: unknown,\n ): Promise<unknown> {\n this.logger.info('onAfterToolCall toolCall', JSON.stringify(toolCall));\n\n if (toolCall.name !== GUI_ADAPTED_TOOL_NAME) {\n this.logger.info('onAfterToolCall: skipping screenshot');\n return;\n }\n\n await sleep(this.loopIntervalInMs);\n\n const output = await this.operator!.doScreenshot();\n if (!output) {\n this.logger.error('Failed to get screenshot');\n return;\n }\n\n const base64Tool = new Base64ImageParser(output.base64);\n const base64Uri = base64Tool.getDataUri();\n if (!base64Uri) {\n this.logger.error('Failed to get base64 image uri');\n return;\n }\n\n const { width: imageWidth, height: imageHeight } = base64Tool.getDimensions() \|\| {\n width: -1,\n height: -1,\n };\n\n const content: ChatCompletionContentPart[] = [\n {\n type: 'image_url',\n image_url: {\n url: base64Uri,\n detail: this.detailCalculator(imageWidth, imageHeight),\n },\n },\n {\n type: 'text',\n text: `Screen resolution: ${imageWidth}x${imageHeight}. System is running on macOS. Please return actions with absolute coordinates within this resolution.`,\n },\n ];\n\n if (output?.url) {\n content.push({\n type: 'text',\n text: `The current page's url: ${output.url}`,\n });\n }\n\n const eventStream = this.getEventStream();\n const events = eventStream.getEvents();\n this.logger.info('onAfterToolCall events length:', events.length);\n\n const event = eventStream.createEvent('environment_input', {\n description: 'Browser Screenshot',\n content,\n metadata: {\n type: 'screenshot',\n url: output?.url,\n },\n });\n eventStream.sendEvent(event);\n return result;\n }\n}\n"],"names":["defaultLogger","ConsoleLogger","LogLevel","GUIAgent","BaseGUIAgent","Tool","GUI_ADAPTED_TOOL_NAME","input","createGUIErrorResponse","normalizeActionCoords","JSON","result","convertToAgentUIAction","undefined","id","payload","safeStringify","obj","max","s","req","messages","Array","model","hasImages","cnt","m","content","part","summary","firstMsg","e","sessionId","toolCall","args","sleep","output","base64Tool","Base64ImageParser","base64Uri","imageWidth","imageHeight","eventStream","events","event","config","operator","systemPrompt","customeActionParser","normalizeCoordinates","detailCalculator","maxLoopCount","loopIntervalInMs","finalSystemPrompt","SYSTEM_PROMPT","p","isSystemPromptTemplate","assembleSystemPrompt","AdaptedToolCallEngine","GUIAgentToolCallEngine","defaultNormalizeCoords","defaultDetailCalculator"],"mappings":";;;;;;;;;;;;;AAGC;;;;;;;;;;AA2BD,MAAMA,gBAAgB,IAAIC,cAAc,cAAcC,SAAS,KAAK;AAE7D,MAAMC,iBAAqCC;IAuDhD,MAAM,aAAa;QAEjB,IAAI,CAAC,YAAY,CACf,IAAIC,KAAK;YACP,IAAIC;YACJ,aAAa;YACb,YAAY,CAAC;YACb,UAAU,OAAOC;gBACf,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAGD,sBAAsB,OAAO,CAAC,EAAEC;gBACnD,IAAI,CAAC,IAAI,CAAC,QAAQ,EAChB,OAAOC,uBAAuBD,MAAM,MAAM,EAAE;gBAE9C,IAAIA,MAAM,YAAY,EACpB,OAAOC,uBAAuBD,MAAM,MAAM,EAAEA,MAAM,YAAY;gBAGhE,IAAIA,MAAM,eAAe,EACvBA,MAAM,eAAe,GAAGE,sBACtBF,MAAM,eAAe,EACrB,IAAI,CAAC,oBAAoB;gBAG7B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsBG,KAAK,SAAS,CAACH,MAAM,eAAe;gBAC3E,MAAMI,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,SAAS,CAAC;oBAC5C,SAAS;wBAACJ,MAAM,eAAe;qBAAC;gBAClC;gBACA,IAAII,OAAO,YAAY,EACrB,OAAOH,uBAAuBD,MAAM,MAAM,EAAEI,OAAO,YAAY;gBAGjE,OAAO;oBACL,SAAS;oBACT,QAAQJ,MAAM,MAAM;oBACpB,kBAAkBK,uBAAuBL,MAAM,eAAe;oBAC9D,aAAaM;gBACf;YACF;QACF;QAEF,KAAK,CAAC;IACR;IAEA,MAAM,aAAaC,EAAU,EAAEC,OAA8B,EAAiB;QAC5E,IAAI;YACF,MAAMC,gBAAgB,CAACC,KAAcC,MAAM,GAAG;gBAC5C,IAAI;oBACF,MAAMC,IAAIT,KAAK,SAAS,CAACO;oBACzB,OAAOE,EAAE,MAAM,GAAGD,MAAMC,EAAE,KAAK,CAAC,GAAGD,OAAO,sBAAiBC;gBAC7D,EAAE,OAAM;oBACN,OAAO;gBACT;YACF;YAEA,MAAMC,MAAML,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO;YAC5B,MAAMM,WAAWD,OAAOE,MAAM,OAAO,CAACF,IAAI,QAAQ,IAAIA,IAAI,QAAQ,GAAG,EAAE;YACvE,MAAMG,QAAQH,OAAO,AAAqB,YAArB,OAAOA,IAAI,KAAK,GAAgBA,IAAI,KAAK,GAAG;YAEjE,MAAMI,YAAa,AAAC;gBAClB,IAAI;oBACF,IAAIC,MAAM;oBACV,KAAK,MAAMC,KAAKL,SAA0C;wBACxD,MAAMM,UAAUD,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO;wBAC1B,IAAIJ,MAAM,OAAO,CAACK,UAChB;4BAAA,KAAK,MAAMC,QAAQD,QACjB,IAAIC,AAAAA,CAAAA,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,IAAI,AAAD,MAAM,aAAaH;wBAClC;oBAEJ;oBACA,OAAOA;gBACT,EAAE,OAAM;oBACN;gBACF;YACF;YAEA,MAAMI,UAAU;gBACdf;gBACAS;gBACA,eAAeF,SAAS,MAAM;gBAC9BG;YACF;YAEA,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoCR,cAAca;YAEnE,MAAMC,WAAWT,QAAQ,CAAC,EAAE;YAC5B,IAAIS,UACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,0CAA0Cd,cAAcc,UAAU;QAExF,EAAE,OAAOC,GAAG;YACV,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,2CAA2CA;QAC/D;IACF;IAEA,MAAM,qBAAqBC,SAAiB,EAAE;QAC5C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwBA;IAC3C;IAEA,MAAM,eAAelB,EAAU,EAAiB,CAEhD;IAEA,MAAM,iBACJA,EAAU,EACVmB,QAA8C,EAC9CC,IAAa,EACb;QACA,OAAOA;IACT;IAEA,MAAM,gBACJpB,EAAU,EACVmB,QAA8C,EAC9CtB,MAAe,EACG;QAClB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,4BAA4BD,KAAK,SAAS,CAACuB;QAE5D,IAAIA,SAAS,IAAI,KAAK3B,uBAAuB,YAC3C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;QAInB,MAAM6B,MAAM,IAAI,CAAC,gBAAgB;QAEjC,MAAMC,SAAS,MAAM,IAAI,CAAC,QAAQ,CAAE,YAAY;QAChD,IAAI,CAACA,QAAQ,YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAMC,aAAa,IAAIC,kBAAkBF,OAAO,MAAM;QACtD,MAAMG,YAAYF,WAAW,UAAU;QACvC,IAAI,CAACE,WAAW,YACd,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAIpB,MAAM,EAAE,OAAOC,UAAU,EAAE,QAAQC,WAAW,EAAE,GAAGJ,WAAW,aAAa,MAAM;YAC/E,OAAO;YACP,QAAQ;QACV;QAEA,MAAMV,UAAuC;YAC3C;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKY;oBACL,QAAQ,IAAI,CAAC,gBAAgB,CAACC,YAAYC;gBAC5C;YACF;YACA;gBACE,MAAM;gBACN,MAAM,CAAC,mBAAmB,EAAED,WAAW,CAAC,EAAEC,YAAY,qGAAqG,CAAC;YAC9J;SACD;QAED,IAAIL,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG,EACbT,QAAQ,IAAI,CAAC;YACX,MAAM;YACN,MAAM,CAAC,wBAAwB,EAAES,OAAO,GAAG,EAAE;QAC/C;QAGF,MAAMM,cAAc,IAAI,CAAC,cAAc;QACvC,MAAMC,SAASD,YAAY,SAAS;QACpC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkCC,OAAO,MAAM;QAEhE,MAAMC,QAAQF,YAAY,WAAW,CAAC,qBAAqB;YACzD,aAAa;YACbf;YACA,UAAU;gBACR,MAAM;gBACN,KAAKS,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,GAAG;YAClB;QACF;QACAM,YAAY,SAAS,CAACE;QACtB,OAAOjC;IACT;IA9NA,YAAYkC,MAAyB,CAAE;QACrC,MAAM,EACJC,QAAQ,EACRvB,KAAK,EACLwB,YAAY,EACZC,mBAAmB,EACnBC,oBAAoB,EACpBC,gBAAgB,EAChBC,YAAY,EACZC,mBAAmB,GAAG,EACvB,GAAGP;QACJ,IAAIQ,oBAAoBC;QACxB,IAAI,AAAwB,YAAxB,OAAOP,cACTM,oBAAoBN;aACf,IAAIzB,MAAM,OAAO,CAACyB,eACvBM,oBAAoBN,aACjB,GAAG,CAAC,CAACQ,IAAO,AAAa,YAAb,OAAOA,IAAiBA,IAAIA,EAAE,OAAO,EACjD,IAAI,CAAC;aACH,IAAIR,gBAAgBS,uBAAuBT,eAChDM,oBAAoBI,qBAAqBV,cAAcD,SAAS,mBAAmB;QAErF9C,cAAc,KAAK,CAAC,8BAA8BqD;QAGlD,MAAMK,wBAAwB,cAAcC;YAC1C,aAAc;gBACZ,KAAK,CAACX;YACR;QACF;QAEA,KAAK,CAAC;YACJ,MAAM7C,SAAS,KAAK;YACpB,cAAckD;YACd,OAAO,EAAE;YACT,gBAAgBK;YAChB,OAAOnC;YACP,GAAI4B,gBAAgB;gBAAE,eAAeA;YAAa,CAAC;YACnD,UAAUjD,SAAS,KAAK;QAC1B,IA3CF,uBAAQ,YAAR,SACA,uBAAQ,wBAAR,SACA,uBAAQ,oBAAR,SACA,uBAAQ,oBAAR;QAyCE,IAAI,CAAC,QAAQ,GAAG4C;QAChB,IAAI,CAAC,oBAAoB,GAAGG,wBAAwBW;QAEpD,IAAI,CAAC,gBAAgB,GAAGV,oBAAoBW;QAC5C,IAAI,CAAC,gBAAgB,GAAGT;QACxB,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;IAClC;AAkLF;AAtOE,iBADWjD,UACJ,SAAQ"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@ui-tars-test/agent-sdk",
-  "version": "0.3.15",
+  "version": "0.3.17",
   "description": "GUI Agent",
   "main": "dist/index.js",
   "module": "dist/index.mjs",
@@ -38,7 +38,7 @@
     "@ui-tars-test/action-parser": "^0.3.0",
     "@ui-tars-test/operator-adb": "^0.3.0",
     "@ui-tars-test/operator-browser": "^0.3.0",
-    "@ui-tars-test/operator-nutjs": "^0.3.0",
+    "@ui-tars-test/operator-nutjs": "^0.3.5",
     "@ui-tars-test/shared": "^0.3.7",
     "@ui-tars-test/tarko-agent": "^0.3.1",
     "@ui-tars-test/tarko-agent-interface": "^0.3.1",