@aiscene/core 1.6.2

package/dist/es/agent/tasks.mjs

@@ -0,0 +1,424 @@
+import { AIResponseParseError, ConversationHistory, autoGLMPlanning, plan, uiTarsPlanning } from "../ai-model/index.mjs";
+import { isAutoGLM, isUITars } from "../ai-model/auto-glm/util.mjs";
+import { getReadableTimeString } from "../common.mjs";
+import { TaskExecutionError } from "../task-runner.mjs";
+import { ServiceError } from "../types.mjs";
+import { getCurrentTime } from "@midscene/shared/env";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { ExecutionSession } from "./execution-session.mjs";
+import { TaskBuilder, locatePlanForLocate } from "./task-builder.mjs";
+import { setTimingFieldOnce } from "../task-timing.mjs";
+import { descriptionOfTree } from "@midscene/shared/extractor";
+import { taskTitleStr } from "./ui-utils.mjs";
+import { parsePrompt } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('device-task-executor');
+const maxErrorCountAllowedInOnePlanningLoop = 5;
+class TaskExecutor {
+    get page() {
+        return this.interface;
+    }
+    createExecutionSession(title, options) {
+        return new ExecutionSession(title, ()=>Promise.resolve(this.service.contextRetrieverFn()), {
+            onTaskStart: this.onTaskStartCallback,
+            tasks: options?.tasks,
+            onTaskUpdate: this.hooks?.onTaskUpdate
+        });
+    }
+    getActionSpace() {
+        return this.providedActionSpace;
+    }
+    async getTimeString(format) {
+        const timestamp = await getCurrentTime(this.interface, this.useDeviceTimestamp);
+        return getReadableTimeString(format, timestamp);
+    }
+    async convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
+        return this.taskBuilder.build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options);
+    }
+    async loadYamlFlowAsPlanning(userInstruction, yamlString) {
+        const session = this.createExecutionSession(taskTitleStr('Act', userInstruction));
+        const task = {
+            type: 'Planning',
+            subType: 'LoadYaml',
+            param: {
+                userInstruction
+            },
+            executor: async (param, executorContext)=>{
+                const { uiContext } = executorContext;
+                assert(uiContext, 'uiContext is required for Planning task');
+                return {
+                    output: {
+                        actions: [],
+                        shouldContinuePlanning: false,
+                        log: '',
+                        yamlString
+                    },
+                    cache: {
+                        hit: true
+                    },
+                    hitBy: {
+                        from: 'Cache',
+                        context: {
+                            yamlString
+                        }
+                    }
+                };
+            }
+        };
+        const runner = session.getRunner();
+        await session.appendAndRun(task);
+        return {
+            runner
+        };
+    }
+    async runPlans(title, plans, modelConfigForPlanning, modelConfigForDefaultIntent) {
+        const session = this.createExecutionSession(title);
+        const { tasks } = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent);
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(tasks);
+        const { output } = result ?? {};
+        return {
+            output,
+            runner
+        };
+    }
+    async action(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal) {
+        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal));
+    }
+    async runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal) {
+        const conversationHistory = new ConversationHistory();
+        const session = this.createExecutionSession(taskTitleStr('Act', userPrompt));
+        const runner = session.getRunner();
+        let replanCount = 0;
+        const yamlFlow = [];
+        const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
+        assert(void 0 !== replanningCycleLimit, 'replanningCycleLimit is required for TaskExecutor.action');
+        let errorCountInOnePlanningLoop = 0;
+        let outputString;
+        while(true){
+            if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
+            const subGoalStatus = conversationHistory.subGoalsToText() || void 0;
+            const memoriesStatus = conversationHistory.memoriesToText() || void 0;
+            const result = await session.appendAndRun({
+                type: 'Planning',
+                subType: 'Plan',
+                param: {
+                    userInstruction: userPrompt,
+                    aiActContext,
+                    imagesIncludeCount,
+                    deepThink,
+                    ...subGoalStatus ? {
+                        subGoalStatus
+                    } : {},
+                    ...memoriesStatus ? {
+                        memoriesStatus
+                    } : {}
+                },
+                executor: async (param, executorContext)=>{
+                    const { uiContext } = executorContext;
+                    assert(uiContext, 'uiContext is required for Planning task');
+                    const { modelFamily } = modelConfigForPlanning;
+                    const timing = executorContext.task.timing;
+                    const actionSpace = this.getActionSpace();
+                    debug('actionSpace for this interface is:', actionSpace.map((action)=>action.name).join(', '));
+                    assert(Array.isArray(actionSpace), 'actionSpace must be an array');
+                    if (0 === actionSpace.length) console.warn(`ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`);
+                    const planImpl = isUITars(modelFamily) ? uiTarsPlanning : isAutoGLM(modelFamily) ? autoGLMPlanning : plan;
+                    let planResult;
+                    try {
+                        setTimingFieldOnce(timing, 'callAiStart');
+                        planResult = await planImpl(param.userInstruction, {
+                            context: uiContext,
+                            actionContext: param.aiActContext,
+                            interfaceType: this.interface.interfaceType,
+                            actionSpace,
+                            modelConfig: modelConfigForPlanning,
+                            conversationHistory,
+                            includeBbox: includeBboxInPlanning,
+                            imagesIncludeCount,
+                            deepThink,
+                            abortSignal
+                        });
+                    } catch (planError) {
+                        if (planError instanceof AIResponseParseError) {
+                            executorContext.task.usage = planError.usage;
+                            executorContext.task.log = {
+                                ...executorContext.task.log || {},
+                                rawResponse: planError.rawResponse
+                            };
+                        }
+                        throw planError;
+                    } finally{
+                        setTimingFieldOnce(timing, 'callAiEnd');
+                    }
+                    debug('planResult', JSON.stringify(planResult, null, 2));
+                    const { actions, thought, log, memory, error, usage, rawResponse, reasoning_content, finalizeSuccess, finalizeMessage, updateSubGoals, markFinishedIndexes } = planResult;
+                    outputString = finalizeMessage;
+                    executorContext.task.log = {
+                        ...executorContext.task.log || {},
+                        rawResponse
+                    };
+                    executorContext.task.usage = usage;
+                    executorContext.task.reasoning_content = reasoning_content;
+                    executorContext.task.output = {
+                        actions: actions || [],
+                        log,
+                        thought,
+                        memory,
+                        yamlFlow: planResult.yamlFlow,
+                        output: finalizeMessage,
+                        shouldContinuePlanning: planResult.shouldContinuePlanning,
+                        updateSubGoals,
+                        markFinishedIndexes
+                    };
+                    executorContext.uiContext = uiContext;
+                    assert(!error, `Failed to continue: ${error}\n${log || ''}`);
+                    if (false === finalizeSuccess) assert(false, `Task failed: ${finalizeMessage || 'No error message provided'}\n${log || ''}`);
+                    return {
+                        cache: {
+                            hit: false
+                        }
+                    };
+                }
+            }, {
+                allowWhenError: true
+            });
+            const planResult = result?.output;
+            const plans = planResult?.actions || [];
+            yamlFlow.push(...planResult?.yamlFlow || []);
+            let executables;
+            try {
+                executables = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, {
+                    cacheable,
+                    deepLocate,
+                    abortSignal
+                });
+            } catch (error) {
+                return session.appendErrorPlan(`Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`);
+            }
+            if (conversationHistory.pendingFeedbackMessage) console.warn('unconsumed pending feedback message detected, this may lead to unexpected planning result:', conversationHistory.pendingFeedbackMessage);
+            const initialTimeString = await this.getTimeString();
+            conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;
+            try {
+                await session.appendAndRun(executables.tasks);
+            } catch (error) {
+                errorCountInOnePlanningLoop++;
+                const timeString = await this.getTimeString();
+                conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;
+                debug('error when executing running tasks, but continue to run if it is not too many errors:', error instanceof Error ? error.message : String(error), 'current error count in one planning loop:', errorCountInOnePlanningLoop);
+            }
+            if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) return session.appendErrorPlan('Too many errors in one planning loop');
+            if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
+            if (!planResult?.shouldContinuePlanning) break;
+            ++replanCount;
+            if (replanCount > replanningCycleLimit) {
+                const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;
+                return session.appendErrorPlan(errorMsg);
+            }
+            if (!conversationHistory.pendingFeedbackMessage) {
+                const timeString = await this.getTimeString();
+                conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;
+            }
+        }
+        return {
+            output: {
+                yamlFlow,
+                output: outputString
+            },
+            runner
+        };
+    }
+    createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt) {
+        const queryTask = {
+            type: 'Insight',
+            subType: type,
+            param: {
+                dataDemand: multimodalPrompt ? {
+                    demand,
+                    multimodalPrompt
+                } : demand
+            },
+            executor: async (param, taskContext)=>{
+                const { task } = taskContext;
+                let queryDump;
+                const applyDump = (dump)=>{
+                    queryDump = dump;
+                    task.log = {
+                        dump,
+                        rawResponse: dump.taskInfo?.rawResponse
+                    };
+                    task.usage = dump.taskInfo?.usage;
+                    if (dump.taskInfo?.reasoning_content) task.reasoning_content = dump.taskInfo.reasoning_content;
+                };
+                const uiContext = taskContext.uiContext;
+                assert(uiContext, 'uiContext is required for Query task');
+                const ifTypeRestricted = 'Query' !== type;
+                let demandInput = demand;
+                let keyOfResult = 'result';
+                if (ifTypeRestricted && ('Assert' === type || 'WaitFor' === type)) {
+                    keyOfResult = 'StatementIsTruthy';
+                    const booleanPrompt = 'Assert' === type ? `Boolean, whether the following statement is true: ${demand}` : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;
+                    demandInput = {
+                        [keyOfResult]: booleanPrompt
+                    };
+                } else if (ifTypeRestricted) {
+                    keyOfResult = type;
+                    const enhancedPrompt = 'Boolean' === type ? `${type}, CRITICAL: Answer ONLY based on what is CLEARLY VISIBLE in the screenshot. If not 100% certain, return false. Query: ${demand}` : `${type}, ${demand}`;
+                    demandInput = {
+                        [keyOfResult]: enhancedPrompt
+                    };
+                }
+                let extractResult;
+                let extraPageDescription = '';
+                if (opt?.domIncluded && this.interface.getElementsNodeTree) {
+                    debug('appending tree info for page');
+                    const tree = await this.interface.getElementsNodeTree();
+                    extraPageDescription = await descriptionOfTree(tree, 200, false, opt?.domIncluded === 'visible-only');
+                }
+                try {
+                    extractResult = await this.service.extract(demandInput, modelConfig, opt, extraPageDescription, multimodalPrompt, uiContext);
+                } catch (error) {
+                    if (error instanceof ServiceError) applyDump(error.dump);
+                    throw error;
+                }
+                const { data, thought, dump } = extractResult;
+                applyDump(dump);
+                let outputResult = data;
+                if (ifTypeRestricted) if ('string' == typeof data) outputResult = data;
+                else if ('WaitFor' === type) outputResult = null == data ? false : data[keyOfResult];
+                else if (null == data) outputResult = null;
+                else {
+                    assert(data?.[keyOfResult] !== void 0, 'No result in query data');
+                    outputResult = data[keyOfResult];
+                }
+                if ('Assert' === type && !outputResult) {
+                    task.thought = thought;
+                    throw new Error(`Assertion failed: ${thought}`);
+                }
+                return {
+                    output: outputResult,
+                    log: queryDump,
+                    thought
+                };
+            }
+        };
+        return queryTask;
+    }
+    async createTypeQueryExecution(type, demand, modelConfig, opt, multimodalPrompt) {
+        const session = this.createExecutionSession(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)));
+        const queryTask = await this.createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt);
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(queryTask);
+        if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');
+        const { output, thought } = result;
+        return {
+            output,
+            thought,
+            runner
+        };
+    }
+    async waitFor(assertion, opt, modelConfig) {
+        const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
+        const description = `waitFor: ${textPrompt}`;
+        const session = this.createExecutionSession(taskTitleStr('WaitFor', description));
+        const runner = session.getRunner();
+        const { timeoutMs, checkIntervalMs, domIncluded, screenshotIncluded, ...restOpt } = opt;
+        const serviceExtractOpt = {
+            domIncluded,
+            screenshotIncluded,
+            ...restOpt
+        };
+        assert(assertion, 'No assertion for waitFor');
+        assert(timeoutMs, 'No timeoutMs for waitFor');
+        assert(checkIntervalMs, 'No checkIntervalMs for waitFor');
+        assert(checkIntervalMs <= timeoutMs, `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`);
+        const overallStartTime = Date.now();
+        let lastCheckStart = overallStartTime;
+        let errorThought = '';
+        while(lastCheckStart - overallStartTime <= timeoutMs){
+            const currentCheckStart = Date.now();
+            lastCheckStart = currentCheckStart;
+            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, modelConfig, serviceExtractOpt, multimodalPrompt);
+            const result = await session.appendAndRun(queryTask);
+            if (result?.output) return {
+                output: void 0,
+                runner
+            };
+            errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
+            const now = Date.now();
+            if (now - currentCheckStart < checkIntervalMs) {
+                const elapsed = now - currentCheckStart;
+                const timeRemaining = checkIntervalMs - elapsed;
+                const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
+                const { tasks: sleepTasks } = await this.convertPlanToExecutable([
+                    {
+                        type: 'Sleep',
+                        param: {
+                            timeMs: timeRemaining
+                        },
+                        thought
+                    }
+                ], modelConfig, modelConfig);
+                if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
+            }
+        }
+        return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
+    }
+    constructor(interfaceInstance, service, opts){
+        _define_property(this, "interface", void 0);
+        _define_property(this, "service", void 0);
+        _define_property(this, "taskCache", void 0);
+        _define_property(this, "providedActionSpace", void 0);
+        _define_property(this, "taskBuilder", void 0);
+        _define_property(this, "onTaskStartCallback", void 0);
+        _define_property(this, "hooks", void 0);
+        _define_property(this, "replanningCycleLimit", void 0);
+        _define_property(this, "waitAfterAction", void 0);
+        _define_property(this, "useDeviceTimestamp", void 0);
+        this.interface = interfaceInstance;
+        this.service = service;
+        this.taskCache = opts.taskCache;
+        this.onTaskStartCallback = opts?.onTaskStart;
+        this.replanningCycleLimit = opts.replanningCycleLimit;
+        this.waitAfterAction = opts.waitAfterAction;
+        this.useDeviceTimestamp = opts.useDeviceTimestamp;
+        this.hooks = opts.hooks;
+        this.providedActionSpace = opts.actionSpace;
+        this.taskBuilder = new TaskBuilder({
+            interfaceInstance,
+            service,
+            taskCache: opts.taskCache,
+            actionSpace: this.getActionSpace(),
+            waitAfterAction: opts.waitAfterAction
+        });
+    }
+}
+async function withFileChooser(interfaceInstance, fileChooserAccept, action) {
+    if (!fileChooserAccept?.length) return action();
+    if (!interfaceInstance.registerFileChooserListener) throw new Error(`File upload is not supported on ${interfaceInstance.interfaceType}`);
+    const handler = async (chooser)=>{
+        await chooser.accept(fileChooserAccept);
+    };
+    const { dispose, getError } = await interfaceInstance.registerFileChooserListener(handler);
+    try {
+        const result = await action();
+        const error = getError();
+        if (error) throw error;
+        return result;
+    } finally{
+        dispose();
+    }
+}
+export { TaskExecutionError, TaskExecutor, locatePlanForLocate, withFileChooser };
+
+//# sourceMappingURL=tasks.mjs.map

package/dist/es/agent/tasks.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import {\n  AIResponseParseError,\n  ConversationHistory,\n  autoGLMPlanning,\n  plan,\n  uiTarsPlanning,\n} from '@/ai-model';\nimport { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport {\n  type TMultimodalPrompt,\n  type TUserPrompt,\n  getReadableTimeString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n  DeepThinkOption,\n  DeviceAction,\n  ExecutionTaskApply,\n  ExecutionTaskInsightQueryApply,\n  ExecutionTaskPlanningApply,\n  ExecutionTaskProgressOptions,\n  InterfaceType,\n  MidsceneYamlFlowItem,\n  PlanningAIResponse,\n  PlanningAction,\n  PlanningActionParamWaitFor,\n  ServiceDump,\n  ServiceExtractOption,\n  ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { type IModelConfig, getCurrentTime } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n  output: OutputType;\n  thought?: string;\n  runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n  interface: AbstractInterface;\n\n  service: Service;\n\n  taskCache?: TaskCache;\n\n  private readonly providedActionSpace: DeviceAction[];\n\n  private readonly taskBuilder: TaskBuilder;\n\n  onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly hooks?: TaskExecutorHooks;\n\n  replanningCycleLimit?: number;\n\n  waitAfterAction?: number;\n\n  useDeviceTimestamp?: boolean;\n\n  // @deprecated use .interface instead\n  get page() {\n    return this.interface;\n  }\n\n  constructor(\n    interfaceInstance: AbstractInterface,\n    service: Service,\n    opts: {\n      taskCache?: TaskCache;\n      onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n      replanningCycleLimit?: number;\n      waitAfterAction?: number;\n      useDeviceTimestamp?: boolean;\n      hooks?: TaskExecutorHooks;\n      actionSpace: DeviceAction[];\n    },\n  ) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = opts.taskCache;\n    this.onTaskStartCallback = opts?.onTaskStart;\n    this.replanningCycleLimit = opts.replanningCycleLimit;\n    this.waitAfterAction = opts.waitAfterAction;\n    this.useDeviceTimestamp = opts.useDeviceTimestamp;\n    this.hooks = opts.hooks;\n    this.providedActionSpace = opts.actionSpace;\n    this.taskBuilder = new TaskBuilder({\n      interfaceInstance,\n      service,\n      taskCache: opts.taskCache,\n      actionSpace: this.getActionSpace(),\n      waitAfterAction: opts.waitAfterAction,\n    });\n  }\n\n  private createExecutionSession(\n    title: string,\n    options?: { tasks?: ExecutionTaskApply[] },\n  ) {\n    return new ExecutionSession(\n      title,\n      () => Promise.resolve(this.service.contextRetrieverFn()),\n      {\n        onTaskStart: this.onTaskStartCallback,\n        tasks: options?.tasks,\n        onTaskUpdate: this.hooks?.onTaskUpdate,\n      },\n    );\n  }\n\n  private getActionSpace(): DeviceAction[] {\n    return this.providedActionSpace;\n  }\n\n  /**\n   * Get a readable time string using device time when configured.\n   * This method respects the useDeviceTimestamp configuration.\n   * @param format - Optional format string\n   * @returns A formatted time string\n   */\n  private async getTimeString(format?: string): Promise<string> {\n    const timestamp = await getCurrentTime(\n      this.interface,\n      this.useDeviceTimestamp,\n    );\n    return getReadableTimeString(format, timestamp);\n  }\n\n  public async convertPlanToExecutable(\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    options?: {\n      cacheable?: boolean;\n      deepLocate?: boolean;\n      abortSignal?: AbortSignal;\n    },\n  ) {\n    return this.taskBuilder.build(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n      options,\n    );\n  }\n\n  async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userInstruction),\n    );\n\n    const task: ExecutionTaskPlanningApply = {\n      type: 'Planning',\n      subType: 'LoadYaml',\n      param: {\n        userInstruction,\n      },\n      executor: async (param, executorContext) => {\n        const { uiContext } = executorContext;\n        assert(uiContext, 'uiContext is required for Planning task');\n        return {\n          output: {\n            actions: [],\n            shouldContinuePlanning: false,\n            log: '',\n            yamlString,\n          },\n          cache: {\n            hit: true,\n          },\n          hitBy: {\n            from: 'Cache',\n            context: {\n              yamlString,\n            },\n          },\n        };\n      },\n    };\n    const runner = session.getRunner();\n    await session.appendAndRun(task);\n\n    return {\n      runner,\n    };\n  }\n\n  async runPlans(\n    title: string,\n    plans: PlanningAction[],\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n  ): Promise<ExecutionResult> {\n    const session = this.createExecutionSession(title);\n    const { tasks } = await this.convertPlanToExecutable(\n      plans,\n      modelConfigForPlanning,\n      modelConfigForDefaultIntent,\n    );\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(tasks);\n    const { output } = result ?? {};\n    return {\n      output,\n      runner,\n    };\n  }\n\n  async action(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n    fileChooserAccept?: string[],\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    return withFileChooser(this.interface, fileChooserAccept, async () => {\n      return this.runAction(\n        userPrompt,\n        modelConfigForPlanning,\n        modelConfigForDefaultIntent,\n        includeBboxInPlanning,\n        aiActContext,\n        cacheable,\n        replanningCycleLimitOverride,\n        imagesIncludeCount,\n        deepThink,\n        deepLocate,\n        abortSignal,\n      );\n    });\n  }\n\n  private async runAction(\n    userPrompt: string,\n    modelConfigForPlanning: IModelConfig,\n    modelConfigForDefaultIntent: IModelConfig,\n    includeBboxInPlanning: boolean,\n    aiActContext?: string,\n    cacheable?: boolean,\n    replanningCycleLimitOverride?: number,\n    imagesIncludeCount?: number,\n    deepThink?: DeepThinkOption,\n    deepLocate?: boolean,\n    abortSignal?: AbortSignal,\n  ): Promise<\n    ExecutionResult<\n      | {\n          yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n          output?: string;\n        }\n      | undefined\n    >\n  > {\n    const conversationHistory = new ConversationHistory();\n\n    const session = this.createExecutionSession(\n      taskTitleStr('Act', userPrompt),\n    );\n    const runner = session.getRunner();\n\n    let replanCount = 0;\n    const yamlFlow: MidsceneYamlFlowItem[] = [];\n    const replanningCycleLimit =\n      replanningCycleLimitOverride ?? this.replanningCycleLimit;\n    assert(\n      replanningCycleLimit !== undefined,\n      'replanningCycleLimit is required for TaskExecutor.action',\n    );\n\n    let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n    let outputString: string | undefined;\n\n    // Main planning loop - unified plan/replan logic\n    while (true) {\n      // Check abort signal before each planning cycle\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // Get sub-goal status text if available\n      const subGoalStatus = conversationHistory.subGoalsToText() || undefined;\n\n      // Get memories text if available\n      const memoriesStatus = conversationHistory.memoriesToText() || undefined;\n\n      const result = await session.appendAndRun(\n        {\n          type: 'Planning',\n          subType: 'Plan',\n          param: {\n            userInstruction: userPrompt,\n            aiActContext,\n            imagesIncludeCount,\n            deepThink,\n            ...(subGoalStatus ? { subGoalStatus } : {}),\n            ...(memoriesStatus ? { memoriesStatus } : {}),\n          },\n          executor: async (param, executorContext) => {\n            const { uiContext } = executorContext;\n            assert(uiContext, 'uiContext is required for Planning task');\n            const { modelFamily } = modelConfigForPlanning;\n            const timing = executorContext.task.timing;\n\n            const actionSpace = this.getActionSpace();\n            debug(\n              'actionSpace for this interface is:',\n              actionSpace.map((action) => action.name).join(', '),\n            );\n            assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n            if (actionSpace.length === 0) {\n              console.warn(\n                `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n              );\n            }\n\n            const planImpl = isUITars(modelFamily)\n              ? uiTarsPlanning\n              : isAutoGLM(modelFamily)\n                ? autoGLMPlanning\n                : plan;\n\n            let planResult: Awaited<ReturnType<typeof planImpl>>;\n            try {\n              setTimingFieldOnce(timing, 'callAiStart');\n              planResult = await planImpl(param.userInstruction, {\n                context: uiContext,\n                actionContext: param.aiActContext,\n                interfaceType: this.interface.interfaceType as InterfaceType,\n                actionSpace,\n                modelConfig: modelConfigForPlanning,\n                conversationHistory,\n                includeBbox: includeBboxInPlanning,\n                imagesIncludeCount,\n                deepThink,\n                abortSignal,\n              });\n            } catch (planError) {\n              if (planError instanceof AIResponseParseError) {\n                // Record usage and rawResponse even when parsing fails\n                executorContext.task.usage = planError.usage;\n                executorContext.task.log = {\n                  ...(executorContext.task.log || {}),\n                  rawResponse: planError.rawResponse,\n                };\n              }\n              throw planError;\n            } finally {\n              setTimingFieldOnce(timing, 'callAiEnd');\n            }\n            debug('planResult', JSON.stringify(planResult, null, 2));\n\n            const {\n              actions,\n              thought,\n              log,\n              memory,\n              error,\n              usage,\n              rawResponse,\n              reasoning_content,\n              finalizeSuccess,\n              finalizeMessage,\n              updateSubGoals,\n              markFinishedIndexes,\n            } = planResult;\n            outputString = finalizeMessage;\n\n            executorContext.task.log = {\n              ...(executorContext.task.log || {}),\n              rawResponse,\n            };\n            executorContext.task.usage = usage;\n            executorContext.task.reasoning_content = reasoning_content;\n            executorContext.task.output = {\n              actions: actions || [],\n              log,\n              thought,\n              memory,\n              yamlFlow: planResult.yamlFlow,\n              output: finalizeMessage,\n              shouldContinuePlanning: planResult.shouldContinuePlanning,\n              updateSubGoals,\n              markFinishedIndexes,\n            };\n            executorContext.uiContext = uiContext;\n\n            assert(!error, `Failed to continue: ${error}\\n${log || ''}`);\n\n            // Check if task was finalized with failure\n            if (finalizeSuccess === false) {\n              assert(\n                false,\n                `Task failed: ${finalizeMessage || 'No error message provided'}\\n${log || ''}`,\n              );\n            }\n\n            return {\n              cache: {\n                hit: false,\n              },\n            } as any;\n          },\n        },\n        {\n          allowWhenError: true,\n        },\n      );\n\n      const planResult = result?.output as PlanningAIResponse | undefined;\n\n      // Execute planned actions\n      const plans = planResult?.actions || [];\n      yamlFlow.push(...(planResult?.yamlFlow || []));\n\n      let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n      try {\n        executables = await this.convertPlanToExecutable(\n          plans,\n          modelConfigForPlanning,\n          modelConfigForDefaultIntent,\n          {\n            cacheable,\n            deepLocate,\n            abortSignal,\n          },\n        );\n      } catch (error) {\n        return session.appendErrorPlan(\n          `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n            plans,\n          )}`,\n        );\n      }\n      if (conversationHistory.pendingFeedbackMessage) {\n        console.warn(\n          'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n          conversationHistory.pendingFeedbackMessage,\n        );\n      }\n\n      // Set initial time context for the first planning call\n      const initialTimeString = await this.getTimeString();\n      conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;\n\n      try {\n        await session.appendAndRun(executables.tasks);\n      } catch (error: any) {\n        // errorFlag = true;\n        errorCountInOnePlanningLoop++;\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;\n        debug(\n          'error when executing running tasks, but continue to run if it is not too many errors:',\n          error instanceof Error ? error.message : String(error),\n          'current error count in one planning loop:',\n          errorCountInOnePlanningLoop,\n        );\n      }\n\n      if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n        return session.appendErrorPlan('Too many errors in one planning loop');\n      }\n\n      // Check abort signal after executing actions\n      if (abortSignal?.aborted) {\n        return session.appendErrorPlan(\n          `Task aborted: ${abortSignal.reason || 'abort signal received'}`,\n        );\n      }\n\n      // // Check if task is complete\n      if (!planResult?.shouldContinuePlanning) {\n        break;\n      }\n\n      // Increment replan count for next iteration\n      ++replanCount;\n\n      if (replanCount > replanningCycleLimit) {\n        const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n        return session.appendErrorPlan(errorMsg);\n      }\n\n      if (!conversationHistory.pendingFeedbackMessage) {\n        const timeString = await this.getTimeString();\n        conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n      }\n    }\n\n    return {\n      output: {\n        yamlFlow,\n        output: outputString,\n      },\n      runner,\n    };\n  }\n\n  private createTypeQueryTask(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert' | 'WaitFor',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ) {\n    const queryTask: ExecutionTaskInsightQueryApply = {\n      type: 'Insight',\n      subType: type,\n      param: {\n        dataDemand: multimodalPrompt\n          ? ({\n              demand,\n              multimodalPrompt,\n            } as never)\n          : demand, // for user param presentation in report right sidebar\n      },\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let queryDump: ServiceDump | undefined;\n        const applyDump = (dump: ServiceDump) => {\n          queryDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n          };\n          task.usage = dump.taskInfo?.usage;\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        // Get context for query operations\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Query task');\n\n        const ifTypeRestricted = type !== 'Query';\n        let demandInput = demand;\n        let keyOfResult = 'result';\n        if (ifTypeRestricted && (type === 'Assert' || type === 'WaitFor')) {\n          keyOfResult = 'StatementIsTruthy';\n          const booleanPrompt =\n            type === 'Assert'\n              ? `Boolean, whether the following statement is true: ${demand}`\n              : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;\n          demandInput = {\n            [keyOfResult]: booleanPrompt,\n          };\n        } else if (ifTypeRestricted) {\n          keyOfResult = type;\n          const enhancedPrompt = type === 'Boolean'\n            ? `${type}, CRITICAL: Answer ONLY based on what is CLEARLY VISIBLE in the screenshot. If not 100% certain, return false. Query: ${demand}`\n            : `${type}, ${demand}`;\n          demandInput = {\n            [keyOfResult]: enhancedPrompt,\n          };\n        }\n\n        let extractResult;\n\n        let extraPageDescription = '';\n        if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n          debug('appending tree info for page');\n          const tree = await this.interface.getElementsNodeTree();\n          extraPageDescription = await descriptionOfTree(\n            tree,\n            200,\n            false,\n            opt?.domIncluded === 'visible-only',\n          );\n        }\n\n        try {\n          extractResult = await this.service.extract<any>(\n            demandInput,\n            modelConfig,\n            opt,\n            extraPageDescription,\n            multimodalPrompt,\n            uiContext,\n          );\n        } catch (error) {\n          if (error instanceof ServiceError) {\n            applyDump(error.dump);\n          }\n          throw error;\n        }\n\n        const { data, thought, dump } = extractResult;\n        applyDump(dump);\n\n        let outputResult = data;\n        if (ifTypeRestricted) {\n          // If AI returned a plain string instead of structured format, use it directly\n          if (typeof data === 'string') {\n            outputResult = data;\n          } else if (type === 'WaitFor') {\n            if (data === null || data === undefined) {\n              outputResult = false;\n            } else {\n              outputResult = (data as any)[keyOfResult];\n            }\n          } else if (data === null || data === undefined) {\n            outputResult = null;\n          } else {\n            assert(\n              data?.[keyOfResult] !== undefined,\n              'No result in query data',\n            );\n            outputResult = (data as any)[keyOfResult];\n          }\n        }\n\n        if (type === 'Assert' && !outputResult) {\n          task.thought = thought;\n          throw new Error(`Assertion failed: ${thought}`);\n        }\n\n        return {\n          output: outputResult,\n          log: queryDump,\n          thought,\n        };\n      },\n    };\n\n    return queryTask;\n  }\n  async createTypeQueryExecution<T>(\n    type: 'Query' | 'Boolean' | 'Number' | 'String' | 'Assert',\n    demand: ServiceExtractParam,\n    modelConfig: IModelConfig,\n    opt?: ServiceExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<ExecutionResult<T>> {\n    const session = this.createExecutionSession(\n      taskTitleStr(\n        type,\n        typeof demand === 'string' ? demand : JSON.stringify(demand),\n      ),\n    );\n\n    const queryTask = await this.createTypeQueryTask(\n      type,\n      demand,\n      modelConfig,\n      opt,\n      multimodalPrompt,\n    );\n\n    const runner = session.getRunner();\n    const result = await session.appendAndRun(queryTask);\n\n    if (!result) {\n      throw new Error(\n        'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n      );\n    }\n\n    const { output, thought } = result;\n\n    return {\n      output,\n      thought,\n      runner,\n    };\n  }\n\n  async waitFor(\n    assertion: TUserPrompt,\n    opt: PlanningActionParamWaitFor,\n    modelConfig: IModelConfig,\n  ): Promise<ExecutionResult<void>> {\n    const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n    const description = `waitFor: ${textPrompt}`;\n    const session = this.createExecutionSession(\n      taskTitleStr('WaitFor', description),\n    );\n    const runner = session.getRunner();\n    const {\n      timeoutMs,\n      checkIntervalMs,\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt\n    } = opt;\n    const serviceExtractOpt: ServiceExtractOption = {\n      domIncluded,\n      screenshotIncluded,\n      ...restOpt,\n    };\n\n    assert(assertion, 'No assertion for waitFor');\n    assert(timeoutMs, 'No timeoutMs for waitFor');\n    assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n    assert(\n      checkIntervalMs <= timeoutMs,\n      `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n    );\n\n    const overallStartTime = Date.now();\n    let lastCheckStart = overallStartTime;\n    let errorThought = '';\n    // Continue checking as long as the previous iteration began within the timeout window.\n    while (lastCheckStart - overallStartTime <= timeoutMs) {\n      const currentCheckStart = Date.now();\n      lastCheckStart = currentCheckStart;\n      const queryTask = await this.createTypeQueryTask(\n        'WaitFor',\n        textPrompt,\n        modelConfig,\n        serviceExtractOpt,\n        multimodalPrompt,\n      );\n\n      const result = (await session.appendAndRun(queryTask)) as\n        | {\n            output: boolean;\n            thought?: string;\n          }\n        | undefined;\n\n      if (result?.output) {\n        return {\n          output: undefined,\n          runner,\n        };\n      }\n\n      errorThought =\n        result?.thought ||\n        (!result && `No result from assertion: ${textPrompt}`) ||\n        `unknown error when waiting for assertion: ${textPrompt}`;\n      const now = Date.now();\n      if (now - currentCheckStart < checkIntervalMs) {\n        const elapsed = now - currentCheckStart;\n        const timeRemaining = checkIntervalMs - elapsed;\n        const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n        const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n          [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n          modelConfig,\n          modelConfig,\n        );\n        if (sleepTasks[0]) {\n          await session.appendAndRun(sleepTasks[0]);\n        }\n      }\n    }\n\n    return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n  }\n}\n\nexport async function withFileChooser<T>(\n  interfaceInstance: AbstractInterface,\n  fileChooserAccept: string[] | undefined,\n  action: () => Promise<T>,\n): Promise<T> {\n  if (!fileChooserAccept?.length) {\n    return action();\n  }\n\n  if (!interfaceInstance.registerFileChooserListener) {\n    throw new Error(\n      `File upload is not supported on ${interfaceInstance.interfaceType}`,\n    );\n  }\n\n  const handler = async (chooser: FileChooserHandler) => {\n    await chooser.accept(fileChooserAccept);\n  };\n\n  const { dispose, getError } =\n    await interfaceInstance.registerFileChooserListener(handler);\n  try {\n    const result = await action();\n    // Check for errors that occurred during file chooser handling\n    const error = getError();\n    if (error) {\n      throw error;\n    }\n    return result;\n  } finally {\n    dispose();\n  }\n}\n"],"names":["debug","getDebug","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","format","timestamp","getCurrentTime","getReadableTimeString","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","conversationHistory","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","outputString","subGoalStatus","memoriesStatus","modelFamily","timing","actionSpace","action","Array","console","planImpl","isUITars","uiTarsPlanning","isAutoGLM","autoGLMPlanning","plan","planResult","setTimingFieldOnce","planError","AIResponseParseError","JSON","actions","thought","log","memory","error","usage","rawResponse","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","timeString","String","Error","errorMsg","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","booleanPrompt","enhancedPrompt","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AA2DA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,wCAAwC;AAIvC,MAAMC;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IAQA,MAAc,cAAcG,MAAe,EAAmB;QAC5D,MAAMC,YAAY,MAAMC,eACtB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,kBAAkB;QAEzB,OAAOC,sBAAsBH,QAAQC;IACvC;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCT,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BO,OACAC,wBACAC,6BACAT;IAEJ;IAEA,MAAM,uBAAuBU,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOH;QAGtB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJpB,KAAa,EACbQ,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACb;QAC5C,MAAM,EAAEqB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAC3BC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EASzB;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,wBACAC,6BACAe,uBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC;IAGN;IAEA,MAAc,UACZT,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAA2B,EAC3BE,UAAoB,EACpBC,WAAyB,EASzB;QACA,MAAME,sBAAsB,IAAIC;QAEhC,MAAMvB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOU;QAEtB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIwB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJX,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEoB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAClC,IAAIC;QAGJ,MAAO,KAAM;YAEX,IAAIT,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMU,gBAAgBR,oBAAoB,cAAc,MAAMK;YAG9D,MAAMI,iBAAiBT,oBAAoB,cAAc,MAAMK;YAE/D,MAAMlB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;oBACAC;oBACA,GAAIa,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO5B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM,EAAE2B,WAAW,EAAE,GAAGpC;oBACxB,MAAMqC,SAAS7B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAM8B,cAAc,IAAI,CAAC,cAAc;oBACvCnD,MACE,sCACAmD,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD7B,OAAO8B,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WAAWC,SAASP,eACtBQ,iBACAC,UAAUT,eACRU,kBACAC;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBZ,QAAQ;wBAC3BW,aAAa,MAAMN,SAASnC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;4BAC3C+B;4BACA,aAAatC;4BACb0B;4BACA,aAAaV;4BACbI;4BACAC;4BACAG;wBACF;oBACF,EAAE,OAAO0B,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7C3C,gBAAgB,IAAI,CAAC,KAAK,GAAG0C,UAAU,KAAK;4BAC5C1C,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAa0C,UAAU,WAAW;4BACpC;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBZ,QAAQ;oBAC7B;oBACAlD,MAAM,cAAciE,KAAK,SAAS,CAACJ,YAAY,MAAM;oBAErD,MAAM,EACJK,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACNC,KAAK,EACLC,KAAK,EACLC,WAAW,EACXC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGhB;oBACJf,eAAe6B;oBAEftD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCmD;oBACF;oBACAnD,gBAAgB,IAAI,CAAC,KAAK,GAAGkD;oBAC7BlD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGoD;oBACzCpD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS6C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUR,WAAW,QAAQ;wBAC7B,QAAQc;wBACR,wBAAwBd,WAAW,sBAAsB;wBACzDe;wBACAC;oBACF;oBACAxD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAAC+C,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAEF,OAAO,IAAI;oBAG3D,IAAIM,AAAoB,UAApBA,iBACFnD,OACE,OACA,CAAC,aAAa,EAAEoD,mBAAmB,4BAA4B,EAAE,EAAEP,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMP,aAAanC,QAAQ;YAG3B,MAAMd,QAAQiD,YAAY,WAAW,EAAE;YACvCnB,SAAS,IAAI,IAAKmB,YAAY,YAAY,EAAE;YAE5C,IAAIiB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9ClE,OACAC,wBACAC,6BACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAOiC,OAAO;gBACd,OAAOrD,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEqD,MAAM,SAAS,EAAEL,KAAK,SAAS,CAC5ErD,QACC;YAEP;YACA,IAAI2B,oBAAoB,sBAAsB,EAC5Ce,QAAQ,IAAI,CACV,8FACAf,oBAAoB,sBAAsB;YAK9C,MAAMwC,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAClDxC,oBAAoB,sBAAsB,IAAI,CAAC,cAAc,EAAEwC,mBAAmB;YAElF,IAAI;gBACF,MAAM9D,QAAQ,YAAY,CAAC6D,YAAY,KAAK;YAC9C,EAAE,OAAOR,OAAY;gBAEnBzB;gBACA,MAAMmC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,iCAAiC,EAAEV,OAAO,WAAWW,OAAOX,QAAQ;gBACrItE,MACE,yFACAsE,iBAAiBY,QAAQZ,MAAM,OAAO,GAAGW,OAAOX,QAChD,6CACAzB;YAEJ;YAEA,IAAIA,8BAA8B3C,uCAChC,OAAOe,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACwB,YAAY,wBACf;YAIF,EAAEpB;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMwC,WAAW,CAAC,UAAU,EAAExC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO1B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAC5C,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMyC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACNtC;gBACA,QAAQI;YACV;YACAtB;QACF;IACF;IAEQ,oBACN4D,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,YAAYI,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOjE,OAAOsE;gBACtB,MAAM,EAAEvE,IAAI,EAAE,GAAGuE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ1E,KAAK,GAAG,GAAG;wBACT0E;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1E,KAAK,KAAK,GAAG0E,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,mBACjB1E,KAAK,iBAAiB,GAAG0E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMvE,YAAYoE,YAAY,SAAS;gBACvCnE,OAAOD,WAAW;gBAElB,MAAMwE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACd,MAAMC,gBACJb,AAAS,aAATA,OACI,CAAC,kDAAkD,EAAEC,QAAQ,GAC7D,CAAC,+GAA+G,EAAEA,QAAQ;oBAChIU,cAAc;wBACZ,CAACC,YAAY,EAAEC;oBACjB;gBACF,OAAO,IAAIH,kBAAkB;oBAC3BE,cAAcZ;oBACd,MAAMc,iBAAiBd,AAAS,cAATA,OACnB,GAAGA,KAAK,sHAAsH,EAAEC,QAAQ,GACxI,GAAGD,KAAK,EAAE,EAAEC,QAAQ;oBACxBU,cAAc;wBACZ,CAACC,YAAY,EAAEE;oBACjB;gBACF;gBAEA,IAAIC;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIb,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DvF,MAAM;oBACN,MAAMqG,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAd,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFY,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCJ,aACAT,aACAC,KACAa,sBACAZ,kBACAlE;gBAEJ,EAAE,OAAOgD,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBX,UAAUtB,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAEkC,IAAI,EAAErC,OAAO,EAAE0B,IAAI,EAAE,GAAGM;gBAChCP,UAAUC;gBAEV,IAAIY,eAAeD;gBACnB,IAAIV,kBAEF,IAAI,AAAgB,YAAhB,OAAOU,MACTC,eAAeD;qBACV,IAAIpB,AAAS,cAATA,MAEPqB,eADED,QAAAA,OACa,QAECA,IAAY,CAACR,YAAY;qBAEtC,IAAIQ,QAAAA,MACTC,eAAe;qBACV;oBACLlF,OACEiF,MAAM,CAACR,YAAY,KAAKpD,QACxB;oBAEF6D,eAAgBD,IAAY,CAACR,YAAY;gBAC3C;gBAGF,IAAIZ,AAAS,aAATA,QAAqB,CAACqB,cAAc;oBACtCtF,KAAK,OAAO,GAAGgD;oBACf,MAAM,IAAIe,MAAM,CAAC,kBAAkB,EAAEf,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQsC;oBACR,KAAKd;oBACLxB;gBACF;YACF;QACF;QAEA,OAAOsB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASpB,KAAK,SAAS,CAACoB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAMhE,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEwC,OAAO,EAAE,GAAGzC;QAE5B,OAAO;YACLC;YACAwC;YACA3C;QACF;IACF;IAEA,MAAM,QACJkF,SAAsB,EACtBnB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEqB,UAAU,EAAEnB,gBAAgB,EAAE,GAAGoB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAM1F,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW2F;QAE1B,MAAMrF,SAASP,QAAQ,SAAS;QAChC,MAAM,EACJ6F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG3B;QACJ,MAAM4B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA3F,OAAOmF,WAAW;QAClBnF,OAAOuF,WAAW;QAClBvF,OAAOwF,iBAAiB;QAExBxF,OACEwF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM/B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAkB,YACArB,aACA6B,mBACA3B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQkB;gBACRpB;YACF;YAGF+F,eACE7F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEiF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMvD,UAAU,CAAC,kBAAkB,EAAE4C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGxD;oBAAQ;iBAAE,EAC9DmB,aACAA;gBAEF,IAAIsC,UAAU,CAAC,EAAE,EACf,MAAM3G,QAAQ,YAAY,CAAC2G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO3G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEsG,cAAc;IACnE;IAhsBA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,kBAAkB,GAAGA,KAAK,kBAAkB;QACjD,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAoqBF;AAEO,eAAezF,gBACpBuF,iBAAoC,EACpC1F,iBAAuC,EACvCiB,MAAwB;IAExB,IAAI,CAACjB,mBAAmB,QACtB,OAAOiB;IAGT,IAAI,CAACyE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI3C,MACR,CAAC,gCAAgC,EAAE2C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC/F;IACvB;IAEA,MAAM,EAAEgG,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMvG,SAAS,MAAM0B;QAErB,MAAMkB,QAAQ8D;QACd,IAAI9D,OACF,MAAMA;QAER,OAAO5C;IACT,SAAU;QACRyG;IACF;AACF"}

package/dist/es/agent/ui-utils.mjs

@@ -0,0 +1,91 @@
+function typeStr(task) {
+    return task.subType || task.type;
+}
+function locateParamStr(locate) {
+    if (!locate) return '';
+    if ('string' == typeof locate) return locate;
+    if ('object' == typeof locate) {
+        if ('object' == typeof locate.prompt && null !== locate.prompt && locate.prompt.prompt) {
+            const prompt = locate.prompt.prompt;
+            return prompt;
+        }
+        if ('string' == typeof locate.prompt) return locate.prompt;
+        if ('string' == typeof locate.description) return locate.description;
+    }
+    return '';
+}
+function scrollParamStr(scrollParam) {
+    if (!scrollParam) return '';
+    return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;
+}
+function pullParamStr(pullParam) {
+    if (!pullParam) return '';
+    const parts = [];
+    parts.push(`direction: ${pullParam.direction || 'down'}`);
+    if (pullParam.distance) parts.push(`distance: ${pullParam.distance}`);
+    if (pullParam.duration) parts.push(`duration: ${pullParam.duration}ms`);
+    return parts.join(', ');
+}
+function extractInsightParam(taskParam) {
+    if (!taskParam) return {
+        content: ''
+    };
+    const extractImages = (source)=>source?.multimodalPrompt?.images && Array.isArray(source.multimodalPrompt.images) ? source.multimodalPrompt.images : void 0;
+    const toContent = (value)=>'string' == typeof value ? value : JSON.stringify(value);
+    if (taskParam.demand) return {
+        content: toContent(taskParam.demand),
+        images: extractImages(taskParam)
+    };
+    if (taskParam.assertion) return {
+        content: toContent(taskParam.assertion),
+        images: extractImages(taskParam)
+    };
+    if (taskParam.dataDemand) {
+        const { dataDemand } = taskParam;
+        if ('string' == typeof dataDemand) return {
+            content: dataDemand
+        };
+        if ('object' == typeof dataDemand) return {
+            content: toContent(dataDemand.demand || dataDemand),
+            images: extractImages(dataDemand)
+        };
+    }
+    return {
+        content: ''
+    };
+}
+function taskTitleStr(type, prompt) {
+    if (prompt) return `${type} - ${prompt}`;
+    return type;
+}
+function paramStr(task) {
+    let value;
+    if ('Planning' === task.type) if ('Locate' === task.subType) value = locateParamStr(task?.param);
+    else {
+        const planTask = task;
+        value = planTask.output?.log || planTask.param?.userInstruction;
+    }
+    if ('Insight' === task.type) value = extractInsightParam(task?.param).content;
+    if ('Action Space' === task.type) {
+        const locate = task?.param?.locate;
+        const locateStr = locate ? locateParamStr(locate) : '';
+        value = task.thought || '';
+        if ('number' == typeof task?.param?.timeMs) value = `${task?.param?.timeMs}ms`;
+        else if ('string' == typeof task?.param?.scrollType) value = scrollParamStr(task?.param);
+        else if ('string' == typeof task?.param?.direction && task?.subType === 'PullGesture') value = pullParamStr(task?.param);
+        else if (void 0 !== task?.param?.value) value = task?.param?.value;
+        else if (task?.param && 'object' == typeof task?.param && Object.keys(task?.param || {}).length > 0) value = task?.param;
+        if (locateStr) value = value && 'object' != typeof value ? `${locateStr} - ${value}` : locateStr;
+    }
+    if (void 0 === value) return '';
+    if ('string' == typeof value) return value;
+    if ('object' == typeof value) {
+        const locateStr = locateParamStr(value);
+        if (locateStr) return locateStr;
+        return JSON.stringify(value, void 0, 2);
+    }
+    return String(value);
+}
+export { extractInsightParam, locateParamStr, paramStr, pullParamStr, scrollParamStr, taskTitleStr, typeStr };
+
+//# sourceMappingURL=ui-utils.mjs.map

package/dist/es/agent/ui-utils.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/ui-utils.mjs","sources":["../../../src/agent/ui-utils.ts"],"sourcesContent":["import type {\n  DetailedLocateParam,\n  ExecutionTask,\n  ExecutionTaskAction,\n  ExecutionTaskInsightAssertion,\n  ExecutionTaskInsightQuery,\n  ExecutionTaskPlanning,\n  ExecutionTaskPlanningLocate,\n  PullParam,\n  ScrollParam,\n} from '@/types';\n\nexport function typeStr(task: ExecutionTask) {\n  // // For Action tasks with subType, show \"Action Space / subType\"\n  // if (task.type === 'Action' && task.subType) {\n  //   return `Action Space / ${task.subType}`;\n  // }\n\n  // // For all other cases with subType, show \"type / subType\"\n  // if (task.subType) {\n  //   return `${task.type} / ${task.subType}`;\n  // }\n\n  // No subType, just show type\n  return task.subType || task.type;\n}\n\nexport function locateParamStr(locate?: DetailedLocateParam | string): string {\n  if (!locate) {\n    return '';\n  }\n\n  if (typeof locate === 'string') {\n    return locate;\n  }\n\n  if (typeof locate === 'object') {\n    // Check for nested prompt.prompt (Planning Locate tasks)\n    if (\n      typeof locate.prompt === 'object' &&\n      locate.prompt !== null &&\n      locate.prompt.prompt\n    ) {\n      const prompt = locate.prompt.prompt;\n      return prompt;\n    }\n\n    // Check for direct prompt string\n    if (typeof locate.prompt === 'string') {\n      return locate.prompt;\n    }\n\n    // Check for description field (Action Space tasks like Tap, Hover)\n    if (typeof (locate as any).description === 'string') {\n      return (locate as any).description;\n    }\n  }\n\n  return '';\n}\n\nexport function scrollParamStr(scrollParam?: ScrollParam) {\n  if (!scrollParam) {\n    return '';\n  }\n  return `${scrollParam.direction || 'down'}, ${scrollParam.scrollType || 'singleAction'}, ${scrollParam.distance || 'distance-not-set'}`;\n}\n\nexport function pullParamStr(pullParam?: PullParam) {\n  if (!pullParam) {\n    return '';\n  }\n  const parts: string[] = [];\n  parts.push(`direction: ${pullParam.direction || 'down'}`);\n  if (pullParam.distance) {\n    parts.push(`distance: ${pullParam.distance}`);\n  }\n  if (pullParam.duration) {\n    parts.push(`duration: ${pullParam.duration}ms`);\n  }\n  return parts.join(', ');\n}\n\nexport function extractInsightParam(taskParam: any): {\n  content: string;\n  images?: Array<{ name: string; url: string }>;\n} {\n  if (!taskParam) {\n    return { content: '' };\n  }\n\n  // Helper to extract images from multimodalPrompt\n  const extractImages = (source: any) => {\n    return source?.multimodalPrompt?.images &&\n      Array.isArray(source.multimodalPrompt.images)\n      ? source.multimodalPrompt.images\n      : undefined;\n  };\n\n  // Helper to stringify if needed\n  const toContent = (value: any) =>\n    typeof value === 'string' ? value : JSON.stringify(value);\n\n  // Extract from demand\n  if (taskParam.demand) {\n    return {\n      content: toContent(taskParam.demand),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from assertion\n  if (taskParam.assertion) {\n    return {\n      content: toContent(taskParam.assertion),\n      images: extractImages(taskParam),\n    };\n  }\n\n  // Extract from dataDemand\n  if (taskParam.dataDemand) {\n    const { dataDemand } = taskParam;\n\n    if (typeof dataDemand === 'string') {\n      return { content: dataDemand };\n    }\n\n    if (typeof dataDemand === 'object') {\n      return {\n        content: toContent(dataDemand.demand || dataDemand),\n        images: extractImages(dataDemand),\n      };\n    }\n  }\n\n  return { content: '' };\n}\n\nexport function taskTitleStr(\n  type:\n    | 'Tap'\n    | 'Hover'\n    | 'Input'\n    | 'RightClick'\n    | 'KeyboardPress'\n    | 'Scroll'\n    | 'Act'\n    | 'Query'\n    | 'Assert'\n    | 'WaitFor'\n    | 'Locate'\n    | 'Boolean'\n    | 'Number'\n    | 'String',\n  prompt: string,\n) {\n  if (prompt) {\n    return `${type} - ${prompt}`;\n  }\n  return type;\n}\n\nexport function paramStr(task: ExecutionTask) {\n  let value: string | undefined | object;\n  if (task.type === 'Planning') {\n    if (task.subType === 'Locate') {\n      value = locateParamStr((task as ExecutionTaskPlanningLocate)?.param);\n    } else {\n      // Prefer AI-generated output.log over user input\n      const planTask = task as ExecutionTaskPlanning;\n      value = planTask.output?.log || planTask.param?.userInstruction;\n    }\n  }\n\n  if (task.type === 'Insight') {\n    value = extractInsightParam((task as any)?.param).content;\n  }\n\n  if (task.type === 'Action Space') {\n    const locate = (task as ExecutionTaskAction)?.param?.locate;\n    const locateStr = locate ? locateParamStr(locate) : '';\n\n    value = task.thought || '';\n    if (typeof (task as ExecutionTaskAction)?.param?.timeMs === 'number') {\n      value = `${(task as ExecutionTaskAction)?.param?.timeMs}ms`;\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.scrollType === 'string'\n    ) {\n      value = scrollParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.direction === 'string' &&\n      (task as ExecutionTaskAction)?.subType === 'PullGesture'\n    ) {\n      value = pullParamStr((task as ExecutionTaskAction)?.param);\n    } else if (\n      typeof (task as ExecutionTaskAction)?.param?.value !== 'undefined'\n    ) {\n      value = (task as ExecutionTaskAction)?.param?.value;\n    } else if (\n      (task as ExecutionTaskAction)?.param &&\n      typeof (task as ExecutionTaskAction)?.param === 'object' &&\n      Object.keys((task as ExecutionTaskAction)?.param || {}).length > 0\n    ) {\n      // General parameter handling for actions with custom parameters\n      // (e.g., runWdaRequest, runAdbShell)\n      value = (task as ExecutionTaskAction)?.param;\n    }\n\n    if (locateStr) {\n      if (value && typeof value !== 'object') {\n        value = `${locateStr} - ${value}`;\n      } else {\n        value = locateStr;\n      }\n    }\n  }\n\n  if (typeof value === 'undefined') return '';\n\n  if (typeof value === 'string') return value;\n\n  if (typeof value === 'object') {\n    const locateStr = locateParamStr(value as any);\n    if (locateStr) {\n      return locateStr;\n    }\n    return JSON.stringify(value, undefined, 2);\n  }\n\n  return String(value);\n}\n"],"names":["typeStr","task","locateParamStr","locate","prompt","scrollParamStr","scrollParam","pullParamStr","pullParam","parts","extractInsightParam","taskParam","extractImages","source","Array","undefined","toContent","value","JSON","dataDemand","taskTitleStr","type","paramStr","planTask","locateStr","Object","String"],"mappings":"AAYO,SAASA,QAAQC,IAAmB;IAYzC,OAAOA,KAAK,OAAO,IAAIA,KAAK,IAAI;AAClC;AAEO,SAASC,eAAeC,MAAqC;IAClE,IAAI,CAACA,QACH,OAAO;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAGT,IAAI,AAAkB,YAAlB,OAAOA,QAAqB;QAE9B,IACE,AAAyB,YAAzB,OAAOA,OAAO,MAAM,IACpBA,AAAkB,SAAlBA,OAAO,MAAM,IACbA,OAAO,MAAM,CAAC,MAAM,EACpB;YACA,MAAMC,SAASD,OAAO,MAAM,CAAC,MAAM;YACnC,OAAOC;QACT;QAGA,IAAI,AAAyB,YAAzB,OAAOD,OAAO,MAAM,EACtB,OAAOA,OAAO,MAAM;QAItB,IAAI,AAAuC,YAAvC,OAAQA,OAAe,WAAW,EACpC,OAAQA,OAAe,WAAW;IAEtC;IAEA,OAAO;AACT;AAEO,SAASE,eAAeC,WAAyB;IACtD,IAAI,CAACA,aACH,OAAO;IAET,OAAO,GAAGA,YAAY,SAAS,IAAI,OAAO,EAAE,EAAEA,YAAY,UAAU,IAAI,eAAe,EAAE,EAAEA,YAAY,QAAQ,IAAI,oBAAoB;AACzI;AAEO,SAASC,aAAaC,SAAqB;IAChD,IAAI,CAACA,WACH,OAAO;IAET,MAAMC,QAAkB,EAAE;IAC1BA,MAAM,IAAI,CAAC,CAAC,WAAW,EAAED,UAAU,SAAS,IAAI,QAAQ;IACxD,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,EAAE;IAE9C,IAAIA,UAAU,QAAQ,EACpBC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAED,UAAU,QAAQ,CAAC,EAAE,CAAC;IAEhD,OAAOC,MAAM,IAAI,CAAC;AACpB;AAEO,SAASC,oBAAoBC,SAAc;IAIhD,IAAI,CAACA,WACH,OAAO;QAAE,SAAS;IAAG;IAIvB,MAAMC,gBAAgB,CAACC,SACdA,QAAQ,kBAAkB,UAC/BC,MAAM,OAAO,CAACD,OAAO,gBAAgB,CAAC,MAAM,IAC1CA,OAAO,gBAAgB,CAAC,MAAM,GAC9BE;IAIN,MAAMC,YAAY,CAACC,QACjB,AAAiB,YAAjB,OAAOA,QAAqBA,QAAQC,KAAK,SAAS,CAACD;IAGrD,IAAIN,UAAU,MAAM,EAClB,OAAO;QACL,SAASK,UAAUL,UAAU,MAAM;QACnC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,SAAS,EACrB,OAAO;QACL,SAASK,UAAUL,UAAU,SAAS;QACtC,QAAQC,cAAcD;IACxB;IAIF,IAAIA,UAAU,UAAU,EAAE;QACxB,MAAM,EAAEQ,UAAU,EAAE,GAAGR;QAEvB,IAAI,AAAsB,YAAtB,OAAOQ,YACT,OAAO;YAAE,SAASA;QAAW;QAG/B,IAAI,AAAsB,YAAtB,OAAOA,YACT,OAAO;YACL,SAASH,UAAUG,WAAW,MAAM,IAAIA;YACxC,QAAQP,cAAcO;QACxB;IAEJ;IAEA,OAAO;QAAE,SAAS;IAAG;AACvB;AAEO,SAASC,aACdC,IAcY,EACZjB,MAAc;IAEd,IAAIA,QACF,OAAO,GAAGiB,KAAK,GAAG,EAAEjB,QAAQ;IAE9B,OAAOiB;AACT;AAEO,SAASC,SAASrB,IAAmB;IAC1C,IAAIgB;IACJ,IAAIhB,AAAc,eAAdA,KAAK,IAAI,EACX,IAAIA,AAAiB,aAAjBA,KAAK,OAAO,EACdgB,QAAQf,eAAgBD,MAAsC;SACzD;QAEL,MAAMsB,WAAWtB;QACjBgB,QAAQM,SAAS,MAAM,EAAE,OAAOA,SAAS,KAAK,EAAE;IAClD;IAGF,IAAItB,AAAc,cAAdA,KAAK,IAAI,EACXgB,QAAQP,oBAAqBT,MAAc,OAAO,OAAO;IAG3D,IAAIA,AAAc,mBAAdA,KAAK,IAAI,EAAqB;QAChC,MAAME,SAAUF,MAA8B,OAAO;QACrD,MAAMuB,YAAYrB,SAASD,eAAeC,UAAU;QAEpDc,QAAQhB,KAAK,OAAO,IAAI;QACxB,IAAI,AAAwD,YAAxD,OAAQA,MAA8B,OAAO,QAC/CgB,QAAQ,GAAIhB,MAA8B,OAAO,OAAO,EAAE,CAAC;aACtD,IACL,AAA4D,YAA5D,OAAQA,MAA8B,OAAO,YAE7CgB,QAAQZ,eAAgBJ,MAA8B;aACjD,IACL,AAA2D,YAA3D,OAAQA,MAA8B,OAAO,aAC5CA,MAA8B,YAAY,eAE3CgB,QAAQV,aAAcN,MAA8B;aAC/C,IACL,AAAuD,WAA/CA,MAA8B,OAAO,OAE7CgB,QAAShB,MAA8B,OAAO;aACzC,IACJA,MAA8B,SAC/B,AAAgD,YAAhD,OAAQA,MAA8B,SACtCwB,OAAO,IAAI,CAAExB,MAA8B,SAAS,CAAC,GAAG,MAAM,GAAG,GAIjEgB,QAAShB,MAA8B;QAGzC,IAAIuB,WAEAP,QADEA,SAAS,AAAiB,YAAjB,OAAOA,QACV,GAAGO,UAAU,GAAG,EAAEP,OAAO,GAEzBO;IAGd;IAEA,IAAI,AAAiB,WAAVP,OAAuB,OAAO;IAEzC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB,OAAOA;IAEtC,IAAI,AAAiB,YAAjB,OAAOA,OAAoB;QAC7B,MAAMO,YAAYtB,eAAee;QACjC,IAAIO,WACF,OAAOA;QAET,OAAON,KAAK,SAAS,CAACD,OAAOF,QAAW;IAC1C;IAEA,OAAOW,OAAOT;AAChB"}