npm - @midscene/core - Versions diffs - 1.8.3-beta-20260518105651.0 → 1.8.3 - Mend

@midscene/core 1.8.3-beta-20260518105651.0 → 1.8.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (22) hide show

package/dist/es/agent/agent.mjs +5 -5
package/dist/es/agent/agent.mjs.map +1 -1
package/dist/es/agent/tasks.mjs +5 -5
package/dist/es/agent/tasks.mjs.map +1 -1
package/dist/es/agent/utils.mjs +1 -1
package/dist/es/ai-model/llm-planning.mjs +1 -1
package/dist/es/ai-model/llm-planning.mjs.map +1 -1
package/dist/es/types.mjs.map +1 -1
package/dist/es/utils.mjs +2 -2
package/dist/lib/agent/agent.js +5 -5
package/dist/lib/agent/agent.js.map +1 -1
package/dist/lib/agent/tasks.js +5 -5
package/dist/lib/agent/tasks.js.map +1 -1
package/dist/lib/agent/utils.js +1 -1
package/dist/lib/ai-model/llm-planning.js +1 -1
package/dist/lib/ai-model/llm-planning.js.map +1 -1
package/dist/lib/types.js.map +1 -1
package/dist/lib/utils.js +2 -2
package/dist/types/agent/tasks.d.ts +1 -1
package/dist/types/ai-model/llm-planning.d.ts +1 -1
package/dist/types/types.d.ts +7 -2
package/package.json +2 -2

package/dist/es/agent/tasks.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import {\n AIResponseParseError,\n ConversationHistory,\n autoGLMPlanning,\n plan,\n uiTarsPlanning,\n} from '@/ai-model';\nimport { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport {\n type TMultimodalPrompt,\n type TUserPrompt,\n getReadableTimeString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n DeviceAction,\n ExecutionTaskApply,\n ExecutionTaskInsightQueryApply,\n ExecutionTaskPlanningApply,\n ExecutionTaskProgressOptions,\n InterfaceType,\n MidsceneYamlFlowItem,\n PlanningAIResponse,\n PlanningAction,\n PlanningActionParamWaitFor,\n ServiceDump,\n ServiceExtractOption,\n ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n output: OutputType;\n thought?: string;\n runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n onTaskUpdate?: (\n runner: TaskRunner,\n error?: TaskExecutionError,\n ) => Promise<void> \| void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n interface: AbstractInterface;\n\n service: Service;\n\n taskCache?: TaskCache;\n\n private readonly providedActionSpace: DeviceAction[];\n\n private readonly taskBuilder: TaskBuilder;\n\n onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n private readonly hooks?: TaskExecutorHooks;\n\n replanningCycleLimit?: number;\n\n waitAfterAction?: number;\n\n useDeviceTime?: boolean;\n\n // @deprecated use .interface instead\n get page() {\n return this.interface;\n }\n\n constructor(\n interfaceInstance: AbstractInterface,\n service: Service,\n opts: {\n taskCache?: TaskCache;\n onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n replanningCycleLimit?: number;\n waitAfterAction?: number;\n useDeviceTime?: boolean;\n hooks?: TaskExecutorHooks;\n actionSpace: DeviceAction[];\n },\n ) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = opts.taskCache;\n this.onTaskStartCallback = opts?.onTaskStart;\n this.replanningCycleLimit = opts.replanningCycleLimit;\n this.waitAfterAction = opts.waitAfterAction;\n this.useDeviceTime = opts.useDeviceTime;\n this.hooks = opts.hooks;\n this.providedActionSpace = opts.actionSpace;\n this.taskBuilder = new TaskBuilder({\n interfaceInstance,\n service,\n taskCache: opts.taskCache,\n actionSpace: this.getActionSpace(),\n waitAfterAction: opts.waitAfterAction,\n });\n }\n\n private createExecutionSession(\n title: string,\n options?: { tasks?: ExecutionTaskApply[] },\n ) {\n return new ExecutionSession(\n title,\n () => Promise.resolve(this.service.contextRetrieverFn()),\n {\n onTaskStart: this.onTaskStartCallback,\n tasks: options?.tasks,\n onTaskUpdate: this.hooks?.onTaskUpdate,\n },\n );\n }\n\n private getActionSpace(): DeviceAction[] {\n return this.providedActionSpace;\n }\n\n /*\n Get a readable time string. When device time is enabled, use the\n * device-formatted wall-clock time directly so host timezone formatting does\n * not reinterpret a device timestamp.\n * @param format - Optional format string\n * @returns A formatted time string\n */\n private async getTimeString(format?: string): Promise<string> {\n if (this.useDeviceTime) {\n if (this.interface.getDeviceLocalTimeString) {\n try {\n return await this.interface.getDeviceLocalTimeString(format);\n } catch (error) {\n warnLog(\n `Failed to get device time string, falling back to runtime time: ${error}`,\n );\n }\n } else {\n warnLog(\n 'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n );\n }\n }\n\n return getReadableTimeString(format);\n }\n\n public async convertPlanToExecutable(\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n options?: {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n },\n ) {\n return this.taskBuilder.build(\n plans,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n options,\n );\n }\n\n async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n const session = this.createExecutionSession(\n taskTitleStr('Act', userInstruction),\n );\n\n const task: ExecutionTaskPlanningApply = {\n type: 'Planning',\n subType: 'LoadYaml',\n param: {\n userInstruction,\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n return {\n output: {\n actions: [],\n shouldContinuePlanning: false,\n log: '',\n yamlString,\n },\n cache: {\n hit: true,\n },\n hitBy: {\n from: 'Cache',\n context: {\n yamlString,\n },\n },\n };\n },\n };\n const runner = session.getRunner();\n await session.appendAndRun(task);\n\n return {\n runner,\n };\n }\n\n async runPlans(\n title: string,\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n ): Promise<ExecutionResult> {\n const session = this.createExecutionSession(title);\n const { tasks } = await this.convertPlanToExecutable(\n plans,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n );\n const runner = session.getRunner();\n const result = await session.appendAndRun(tasks);\n const { output } = result ?? {};\n return {\n output,\n runner,\n };\n }\n\n async action(\n userPrompt: string,\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n includeBboxInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n planningModeDeepThink?: boolean,\n fileChooserAccept?: string[],\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n return withFileChooser(this.interface, fileChooserAccept, async () => {\n return this.runAction(\n userPrompt,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n includeBboxInPlanning,\n aiActContext,\n cacheable,\n replanningCycleLimitOverride,\n imagesIncludeCount,\n planningModeDeepThink,\n deepLocate,\n abortSignal,\n );\n });\n }\n\n private async runAction(\n userPrompt: string,\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n includeBboxInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n planningModeDeepThink?: boolean,\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n const conversationHistory = new ConversationHistory();\n\n const session = this.createExecutionSession(\n taskTitleStr('Act', userPrompt),\n );\n const runner = session.getRunner();\n\n let replanCount = 0;\n const yamlFlow: MidsceneYamlFlowItem[] = [];\n const replanningCycleLimit =\n replanningCycleLimitOverride ?? this.replanningCycleLimit;\n assert(\n replanningCycleLimit !== undefined,\n 'replanningCycleLimit is required for TaskExecutor.action',\n );\n\n let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n let outputString: string \| undefined;\n\n // Main planning loop - unified plan/replan logic\n while (true) {\n // Check abort signal before each planning cycle\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // Get sub-goal status text if available\n const subGoalStatus = conversationHistory.subGoalsToText() \|\| undefined;\n\n // Get memories text if available\n const memoriesStatus = conversationHistory.memoriesToText() \|\| undefined;\n\n const result = await session.appendAndRun(\n {\n type: 'Planning',\n subType: 'Plan',\n param: {\n userInstruction: userPrompt,\n aiActContext,\n imagesIncludeCount,\n planningModeDeepThink,\n ...(subGoalStatus ? { subGoalStatus } : {}),\n ...(memoriesStatus ? { memoriesStatus } : {}),\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n const { modelFamily } = modelConfigForPlanning;\n const timing = executorContext.task.timing;\n\n const actionSpace = this.getActionSpace();\n debug(\n 'actionSpace for this interface is:',\n actionSpace.map((action) => action.name).join(', '),\n );\n assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n if (actionSpace.length === 0) {\n console.warn(\n `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n );\n }\n\n const planImpl = isUITars(modelFamily)\n ? uiTarsPlanning\n : isAutoGLM(modelFamily)\n ? autoGLMPlanning\n : plan;\n\n let planResult: Awaited<ReturnType<typeof planImpl>>;\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n planResult = await planImpl(param.userInstruction, {\n context: uiContext,\n actionContext: param.aiActContext,\n interfaceType: this.interface.interfaceType as InterfaceType,\n actionSpace,\n modelConfig: modelConfigForPlanning,\n conversationHistory,\n includeBbox: includeBboxInPlanning,\n imagesIncludeCount,\n planningModeDeepThink,\n abortSignal,\n });\n } catch (planError) {\n if (planError instanceof AIResponseParseError) {\n // Record usage and rawResponse even when parsing fails\n executorContext.task.usage = withUsageIntent(\n planError.usage,\n 'planning',\n );\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse: planError.rawResponse,\n };\n }\n throw planError;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n debug('planResult', JSON.stringify(planResult, null, 2));\n\n const {\n actions,\n thought,\n log,\n memory,\n error,\n usage,\n rawResponse,\n reasoning_content,\n finalizeSuccess,\n finalizeMessage,\n updateSubGoals,\n markFinishedIndexes,\n } = planResult;\n outputString = finalizeMessage;\n\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse,\n };\n executorContext.task.usage = withUsageIntent(usage, 'planning');\n executorContext.task.reasoning_content = reasoning_content;\n executorContext.task.output = {\n actions: actions \|\| [],\n log,\n thought,\n memory,\n yamlFlow: planResult.yamlFlow,\n output: finalizeMessage,\n shouldContinuePlanning: planResult.shouldContinuePlanning,\n updateSubGoals,\n markFinishedIndexes,\n };\n executorContext.uiContext = uiContext;\n\n assert(!error, `Failed to continue: ${error}\\n${log \|\| ''}`);\n\n // Check if task was finalized with failure\n if (finalizeSuccess === false) {\n assert(\n false,\n `Task failed: ${finalizeMessage \|\| 'No error message provided'}\\n${log \|\| ''}`,\n );\n }\n\n return {\n cache: {\n hit: false,\n },\n } as any;\n },\n },\n {\n allowWhenError: true,\n },\n );\n\n const planResult = result?.output as PlanningAIResponse \| undefined;\n\n // Execute planned actions\n const plans = planResult?.actions \|\| [];\n yamlFlow.push(...(planResult?.yamlFlow \|\| []));\n\n let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n try {\n executables = await this.convertPlanToExecutable(\n plans,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n {\n cacheable,\n deepLocate,\n abortSignal,\n },\n );\n } catch (error) {\n return session.appendErrorPlan(\n `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n plans,\n )}`,\n );\n }\n if (conversationHistory.pendingFeedbackMessage) {\n console.warn(\n 'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n conversationHistory.pendingFeedbackMessage,\n );\n }\n\n // Set initial time context for the first planning call\n const initialTimeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;\n\n try {\n await session.appendAndRun(executables.tasks);\n } catch (error: any) {\n // errorFlag = true;\n errorCountInOnePlanningLoop++;\n const timeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message \|\| String(error)}`;\n debug(\n 'error when executing running tasks, but continue to run if it is not too many errors:',\n error instanceof Error ? error.message : String(error),\n 'current error count in one planning loop:',\n errorCountInOnePlanningLoop,\n );\n }\n\n if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n return session.appendErrorPlan('Too many errors in one planning loop');\n }\n\n // Check abort signal after executing actions\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // // Check if task is complete\n if (!planResult?.shouldContinuePlanning) {\n break;\n }\n\n // Increment replan count for next iteration\n ++replanCount;\n\n if (replanCount > replanningCycleLimit) {\n const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n return session.appendErrorPlan(errorMsg);\n }\n\n if (!conversationHistory.pendingFeedbackMessage) {\n const timeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n }\n }\n\n return {\n output: {\n yamlFlow,\n output: outputString,\n },\n runner,\n };\n }\n\n private createTypeQueryTask(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert' \| 'WaitFor',\n demand: ServiceExtractParam,\n modelConfig: IModelConfig,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n ) {\n const queryTask: ExecutionTaskInsightQueryApply = {\n type: 'Insight',\n subType: type,\n param: {\n domIncluded: opt?.domIncluded,\n dataDemand: multimodalPrompt\n ? ({\n demand,\n multimodalPrompt,\n } as never)\n : demand, // for user param presentation in report right sidebar\n },\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let queryDump: ServiceDump \| undefined;\n const applyDump = (dump: ServiceDump) => {\n queryDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n // Get context for query operations\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Query task');\n\n const ifTypeRestricted = type !== 'Query';\n let demandInput = demand;\n let keyOfResult = 'result';\n if (ifTypeRestricted && (type === 'Assert' \|\| type === 'WaitFor')) {\n keyOfResult = 'StatementIsTruthy';\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n } else if (ifTypeRestricted) {\n keyOfResult = type;\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n }\n\n let extractResult;\n\n let extraPageDescription = '';\n if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n debug('appending tree info for page');\n const tree = await this.interface.getElementsNodeTree();\n extraPageDescription = await descriptionOfTree(\n tree,\n 200,\n false,\n opt?.domIncluded === 'visible-only',\n );\n }\n\n try {\n extractResult = await this.service.extract<any>(\n demandInput,\n modelConfig,\n opt,\n extraPageDescription,\n multimodalPrompt,\n uiContext,\n );\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n }\n\n const { data, thought, dump } = extractResult;\n applyDump(dump);\n\n let outputResult = data;\n if (ifTypeRestricted) {\n // If AI returned a plain string instead of structured format, use it directly\n if (typeof data === 'string') {\n outputResult = data;\n } else if (type === 'WaitFor') {\n if (data === null \|\| data === undefined) {\n outputResult = false;\n } else {\n outputResult = (data as any)[keyOfResult];\n }\n } else if (data === null \|\| data === undefined) {\n outputResult = null;\n } else {\n // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n if (data?.[keyOfResult] !== undefined) {\n outputResult = (data as any)[keyOfResult];\n } else if (data?.result !== undefined) {\n outputResult = (data as any).result;\n } else {\n assert(false, 'No result in query data');\n }\n }\n }\n\n if (type === 'Assert' && !outputResult) {\n task.thought = thought;\n throw new Error(`Assertion failed: ${thought}`);\n }\n\n return {\n output: outputResult,\n log: queryDump,\n thought,\n };\n },\n };\n\n return queryTask;\n }\n async createTypeQueryExecution<T>(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert',\n demand: ServiceExtractParam,\n modelConfig: IModelConfig,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n ): Promise<ExecutionResult<T>> {\n const session = this.createExecutionSession(\n taskTitleStr(\n type,\n typeof demand === 'string' ? demand : JSON.stringify(demand),\n ),\n );\n\n const queryTask = await this.createTypeQueryTask(\n type,\n demand,\n modelConfig,\n opt,\n multimodalPrompt,\n );\n\n const runner = session.getRunner();\n const result = await session.appendAndRun(queryTask);\n\n if (!result) {\n throw new Error(\n 'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n );\n }\n\n const { output, thought } = result;\n\n return {\n output,\n thought,\n runner,\n };\n }\n\n async waitFor(\n assertion: TUserPrompt,\n opt: PlanningActionParamWaitFor,\n modelConfig: IModelConfig,\n ): Promise<ExecutionResult<void>> {\n const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n const description = `waitFor: ${textPrompt}`;\n const session = this.createExecutionSession(\n taskTitleStr('WaitFor', description),\n );\n const runner = session.getRunner();\n const {\n timeoutMs,\n checkIntervalMs,\n domIncluded,\n screenshotIncluded,\n ...restOpt\n } = opt;\n const serviceExtractOpt: ServiceExtractOption = {\n domIncluded,\n screenshotIncluded,\n ...restOpt,\n };\n\n assert(assertion, 'No assertion for waitFor');\n assert(timeoutMs, 'No timeoutMs for waitFor');\n assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n assert(\n checkIntervalMs <= timeoutMs,\n `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n );\n\n const overallStartTime = Date.now();\n let lastCheckStart = overallStartTime;\n let errorThought = '';\n // Continue checking as long as the previous iteration began within the timeout window.\n while (lastCheckStart - overallStartTime <= timeoutMs) {\n const currentCheckStart = Date.now();\n lastCheckStart = currentCheckStart;\n const queryTask = await this.createTypeQueryTask(\n 'WaitFor',\n textPrompt,\n modelConfig,\n serviceExtractOpt,\n multimodalPrompt,\n );\n\n const result = (await session.appendAndRun(queryTask)) as\n \| {\n output: boolean;\n thought?: string;\n }\n \| undefined;\n\n if (result?.output) {\n return {\n output: undefined,\n runner,\n };\n }\n\n errorThought =\n result?.thought \|\|\n (!result && `No result from assertion: ${textPrompt}`) \|\|\n `unknown error when waiting for assertion: ${textPrompt}`;\n const now = Date.now();\n if (now - currentCheckStart < checkIntervalMs) {\n const elapsed = now - currentCheckStart;\n const timeRemaining = checkIntervalMs - elapsed;\n const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n modelConfig,\n modelConfig,\n );\n if (sleepTasks[0]) {\n await session.appendAndRun(sleepTasks[0]);\n }\n }\n }\n\n return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n }\n}\n\nexport async function withFileChooser<T>(\n interfaceInstance: AbstractInterface,\n fileChooserAccept: string[] \| undefined,\n action: () => Promise<T>,\n): Promise<T> {\n if (!fileChooserAccept?.length) {\n return action();\n }\n\n if (!interfaceInstance.registerFileChooserListener) {\n throw new Error(\n `File upload is not supported on ${interfaceInstance.interfaceType}`,\n );\n }\n\n const handler = async (chooser: FileChooserHandler) => {\n await chooser.accept(fileChooserAccept);\n };\n\n const { dispose, getError } =\n await interfaceInstance.registerFileChooserListener(handler);\n try {\n const result = await action();\n // Check for errors that occurred during file chooser handling\n const error = getError();\n if (error) {\n throw error;\n }\n return result;\n } finally {\n dispose();\n }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","format","error","getReadableTimeString","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","planningModeDeepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","conversationHistory","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","outputString","subGoalStatus","memoriesStatus","modelFamily","timing","actionSpace","action","Array","console","planImpl","isUITars","uiTarsPlanning","isAutoGLM","autoGLMPlanning","plan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","timeString","String","Error","errorMsg","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAIvC,MAAMC;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASA,MAAc,cAAcG,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdR,QACE,CAAC,gEAAgE,EAAEQ,OAAO;QAE9E;aAEAR,QACE;QAKN,OAAOS,sBAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCR,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BM,OACAC,wBACAC,6BACAR;IAEJ;IAEA,MAAM,uBAAuBS,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOH;QAGtB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJnB,KAAa,EACbO,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACZ;QAC5C,MAAM,EAAEoB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,qBAA+B,EAC/BC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EASzB;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,wBACAC,6BACAe,uBACAC,cACAC,WACAC,8BACAC,oBACAC,uBACAE,YACAC;IAGN;IAEA,MAAc,UACZT,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,qBAA+B,EAC/BE,UAAoB,EACpBC,WAAyB,EASzB;QACA,MAAME,sBAAsB,IAAIC;QAEhC,MAAMvB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOU;QAEtB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIwB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJX,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEoB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAClC,IAAIC;QAGJ,MAAO,KAAM;YAEX,IAAIT,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMU,gBAAgBR,oBAAoB,cAAc,MAAMK;YAG9D,MAAMI,iBAAiBT,oBAAoB,cAAc,MAAMK;YAE/D,MAAMlB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;oBACAC;oBACA,GAAIa,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO5B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM,EAAE2B,WAAW,EAAE,GAAGpC;oBACxB,MAAMqC,SAAS7B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAM8B,cAAc,IAAI,CAAC,cAAc;oBACvCnD,MACE,sCACAmD,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD7B,OAAO8B,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WAAWC,SAASP,eACtBQ,iBACAC,UAAUT,eACRU,kBACAC;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBZ,QAAQ;wBAC3BW,aAAa,MAAMN,SAASnC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;4BAC3C+B;4BACA,aAAatC;4BACb0B;4BACA,aAAaV;4BACbI;4BACAC;4BACAG;wBACF;oBACF,EAAE,OAAO0B,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7C3C,gBAAgB,IAAI,CAAC,KAAK,GAAG4C,gBAC3BF,UAAU,KAAK,EACf;4BAEF1C,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAa0C,UAAU,WAAW;4BACpC;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBZ,QAAQ;oBAC7B;oBACAlD,MAAM,cAAckE,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGhB;oBACJf,eAAe6B;oBAEftD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCmD;oBACF;oBACAnD,gBAAgB,IAAI,CAAC,KAAK,GAAG4C,gBAAgBM,OAAO;oBACpDlD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGoD;oBACzCpD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS8C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQc;wBACR,wBAAwBd,WAAW,sBAAsB;wBACzDe;wBACAC;oBACF;oBACAxD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAACb,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIK,AAAoB,UAApBA,iBACFnD,OACE,OACA,CAAC,aAAa,EAAEoD,mBAAmB,4BAA4B,EAAE,EAAEN,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAanC,QAAQ;YAG3B,MAAMd,QAAQiD,YAAY,WAAW,EAAE;YACvCnB,SAAS,IAAI,IAAKmB,YAAY,YAAY,EAAE;YAE5C,IAAIiB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9ClE,OACAC,wBACAC,6BACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO3B,OAAO;gBACd,OAAOO,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEP,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAI2B,oBAAoB,sBAAsB,EAC5Ce,QAAQ,IAAI,CACV,8FACAf,oBAAoB,sBAAsB;YAK9C,MAAMwC,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAClDxC,oBAAoB,sBAAsB,IAAI,CAAC,cAAc,EAAEwC,mBAAmB;YAElF,IAAI;gBACF,MAAM9D,QAAQ,YAAY,CAAC6D,YAAY,KAAK;YAC9C,EAAE,OAAOpE,OAAY;gBAEnBmC;gBACA,MAAMmC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,iCAAiC,EAAEtE,OAAO,WAAWuE,OAAOvE,QAAQ;gBACrIV,MACE,yFACAU,iBAAiBwE,QAAQxE,MAAM,OAAO,GAAGuE,OAAOvE,QAChD,6CACAmC;YAEJ;YAEA,IAAIA,8BAA8B1C,uCAChC,OAAOc,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACwB,YAAY,wBACf;YAIF,EAAEpB;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMwC,WAAW,CAAC,UAAU,EAAExC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO1B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAC5C,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMyC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACNtC;gBACA,QAAQI;YACV;YACAtB;QACF;IACF;IAEQ,oBACN4D,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOjE,OAAOsE;gBACtB,MAAM,EAAEvE,IAAI,EAAE,GAAGuE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ1E,KAAK,GAAG,GAAG;wBACT0E;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1E,KAAK,KAAK,GAAG8C,gBAAgB4B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1E,KAAK,iBAAiB,GAAG0E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMvE,YAAYoE,YAAY,SAAS;gBACvCnE,OAAOD,WAAW;gBAElB,MAAMwE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DvF,MAAM;oBACN,MAAMoG,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX,kBACAlE;gBAEJ,EAAE,OAAOZ,OAAO;oBACd,IAAIA,iBAAiB4F,cACnBV,UAAUlF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE6F,IAAI,EAAEnC,OAAO,EAAEyB,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKpD,QAC1B4D,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAW3D,QAC1B4D,eAAgBD,KAAa,MAAM;qBAEnChF,OAAO,OAAO;gBAKpB,IAAI6D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCrF,KAAK,OAAO,GAAGiD;oBACf,MAAM,IAAIc,MAAM,CAAC,kBAAkB,EAAEd,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQoC;oBACR,KAAKb;oBACLvB;gBACF;YACF;QACF;QAEA,OAAOqB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASnB,KAAK,SAAS,CAACmB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAMhE,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEyC,OAAO,EAAE,GAAG1C;QAE5B,OAAO;YACLC;YACAyC;YACA5C;QACF;IACF;IAEA,MAAM,QACJiF,SAAsB,EACtBlB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW0F;QAE1B,MAAMpF,SAASP,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA1F,OAAOkF,WAAW;QAClBlF,OAAOsF,WAAW;QAClBtF,OAAOuF,iBAAiB;QAExBvF,OACEuF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,aACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQkB;gBACRpB;YACF;YAGF8F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMrD,UAAU,CAAC,kBAAkB,EAAE0C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGtD;oBAAQ;iBAAE,EAC9DkB,aACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IA7sBA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAirBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCiB,MAAwB;IAExB,IAAI,CAACjB,mBAAmB,QACtB,OAAOiB;IAGT,IAAI,CAACwE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM0B;QAErB,MAAM1C,QAAQyH;QACd,IAAIzH,OACF,MAAMA;QAER,OAAOgB;IACT,SAAU;QACRwG;IACF;AACF"}
1	+ {"version":3,"file":"agent/tasks.mjs","sources":["../../../src/agent/tasks.ts"],"sourcesContent":["import {\n AIResponseParseError,\n ConversationHistory,\n autoGLMPlanning,\n plan,\n uiTarsPlanning,\n} from '@/ai-model';\nimport { isAutoGLM, isUITars } from '@/ai-model/auto-glm/util';\nimport { buildTypeQueryDemandValue } from '@/ai-model/prompt/extraction';\nimport {\n type TMultimodalPrompt,\n type TUserPrompt,\n getReadableTimeString,\n} from '@/common';\nimport type { AbstractInterface, FileChooserHandler } from '@/device';\nimport type Service from '@/service';\nimport type { TaskRunner } from '@/task-runner';\nimport { TaskExecutionError } from '@/task-runner';\nimport type {\n DeviceAction,\n ExecutionTaskApply,\n ExecutionTaskInsightQueryApply,\n ExecutionTaskPlanningApply,\n ExecutionTaskProgressOptions,\n InterfaceType,\n MidsceneYamlFlowItem,\n PlanningAIResponse,\n PlanningAction,\n PlanningActionParamWaitFor,\n ServiceDump,\n ServiceExtractOption,\n ServiceExtractParam,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ExecutionSession } from './execution-session';\nimport { TaskBuilder } from './task-builder';\nimport type { TaskCache } from './task-cache';\nexport { locatePlanForLocate } from './task-builder';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport { descriptionOfTree } from '@midscene/shared/extractor';\nimport { taskTitleStr } from './ui-utils';\nimport { withUsageIntent } from './usage-intent';\nimport { parsePrompt } from './utils';\n\ninterface ExecutionResult<OutputType = any> {\n output: OutputType;\n thought?: string;\n runner: TaskRunner;\n}\n\ninterface TaskExecutorHooks {\n onTaskUpdate?: (\n runner: TaskRunner,\n error?: TaskExecutionError,\n ) => Promise<void> \| void;\n}\n\nconst debug = getDebug('device-task-executor');\nconst warnLog = getDebug('device-task-executor', { console: true });\nconst maxErrorCountAllowedInOnePlanningLoop = 5;\n\nexport { TaskExecutionError };\n\nexport class TaskExecutor {\n interface: AbstractInterface;\n\n service: Service;\n\n taskCache?: TaskCache;\n\n private readonly providedActionSpace: DeviceAction[];\n\n private readonly taskBuilder: TaskBuilder;\n\n onTaskStartCallback?: ExecutionTaskProgressOptions['onTaskStart'];\n\n private readonly hooks?: TaskExecutorHooks;\n\n replanningCycleLimit?: number;\n\n waitAfterAction?: number;\n\n useDeviceTime?: boolean;\n\n // @deprecated use .interface instead\n get page() {\n return this.interface;\n }\n\n constructor(\n interfaceInstance: AbstractInterface,\n service: Service,\n opts: {\n taskCache?: TaskCache;\n onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n replanningCycleLimit?: number;\n waitAfterAction?: number;\n useDeviceTime?: boolean;\n hooks?: TaskExecutorHooks;\n actionSpace: DeviceAction[];\n },\n ) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = opts.taskCache;\n this.onTaskStartCallback = opts?.onTaskStart;\n this.replanningCycleLimit = opts.replanningCycleLimit;\n this.waitAfterAction = opts.waitAfterAction;\n this.useDeviceTime = opts.useDeviceTime;\n this.hooks = opts.hooks;\n this.providedActionSpace = opts.actionSpace;\n this.taskBuilder = new TaskBuilder({\n interfaceInstance,\n service,\n taskCache: opts.taskCache,\n actionSpace: this.getActionSpace(),\n waitAfterAction: opts.waitAfterAction,\n });\n }\n\n private createExecutionSession(\n title: string,\n options?: { tasks?: ExecutionTaskApply[] },\n ) {\n return new ExecutionSession(\n title,\n () => Promise.resolve(this.service.contextRetrieverFn()),\n {\n onTaskStart: this.onTaskStartCallback,\n tasks: options?.tasks,\n onTaskUpdate: this.hooks?.onTaskUpdate,\n },\n );\n }\n\n private getActionSpace(): DeviceAction[] {\n return this.providedActionSpace;\n }\n\n /*\n Get a readable time string. When device time is enabled, use the\n * device-formatted wall-clock time directly so host timezone formatting does\n * not reinterpret a device timestamp.\n * @param format - Optional format string\n * @returns A formatted time string\n */\n private async getTimeString(format?: string): Promise<string> {\n if (this.useDeviceTime) {\n if (this.interface.getDeviceLocalTimeString) {\n try {\n return await this.interface.getDeviceLocalTimeString(format);\n } catch (error) {\n warnLog(\n `Failed to get device time string, falling back to runtime time: ${error}`,\n );\n }\n } else {\n warnLog(\n 'useDeviceTime is enabled but getDeviceLocalTimeString is not implemented, falling back to runtime time.',\n );\n }\n }\n\n return getReadableTimeString(format);\n }\n\n public async convertPlanToExecutable(\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n options?: {\n cacheable?: boolean;\n deepLocate?: boolean;\n abortSignal?: AbortSignal;\n },\n ) {\n return this.taskBuilder.build(\n plans,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n options,\n );\n }\n\n async loadYamlFlowAsPlanning(userInstruction: string, yamlString: string) {\n const session = this.createExecutionSession(\n taskTitleStr('Act', userInstruction),\n );\n\n const task: ExecutionTaskPlanningApply = {\n type: 'Planning',\n subType: 'LoadYaml',\n param: {\n userInstruction,\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n return {\n output: {\n actions: [],\n shouldContinuePlanning: false,\n log: '',\n yamlString,\n },\n cache: {\n hit: true,\n },\n hitBy: {\n from: 'Cache',\n context: {\n yamlString,\n },\n },\n };\n },\n };\n const runner = session.getRunner();\n await session.appendAndRun(task);\n\n return {\n runner,\n };\n }\n\n async runPlans(\n title: string,\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n ): Promise<ExecutionResult> {\n const session = this.createExecutionSession(title);\n const { tasks } = await this.convertPlanToExecutable(\n plans,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n );\n const runner = session.getRunner();\n const result = await session.appendAndRun(tasks);\n const { output } = result ?? {};\n return {\n output,\n runner,\n };\n }\n\n async action(\n userPrompt: string,\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n includeBboxInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n deepThink?: boolean,\n fileChooserAccept?: string[],\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n return withFileChooser(this.interface, fileChooserAccept, async () => {\n return this.runAction(\n userPrompt,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n includeBboxInPlanning,\n aiActContext,\n cacheable,\n replanningCycleLimitOverride,\n imagesIncludeCount,\n deepThink,\n deepLocate,\n abortSignal,\n );\n });\n }\n\n private async runAction(\n userPrompt: string,\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n includeBboxInPlanning: boolean,\n aiActContext?: string,\n cacheable?: boolean,\n replanningCycleLimitOverride?: number,\n imagesIncludeCount?: number,\n deepThink?: boolean,\n deepLocate?: boolean,\n abortSignal?: AbortSignal,\n ): Promise<\n ExecutionResult<\n \| {\n yamlFlow?: MidsceneYamlFlowItem[]; // for cache use\n output?: string;\n }\n \| undefined\n >\n > {\n const conversationHistory = new ConversationHistory();\n\n const session = this.createExecutionSession(\n taskTitleStr('Act', userPrompt),\n );\n const runner = session.getRunner();\n\n let replanCount = 0;\n const yamlFlow: MidsceneYamlFlowItem[] = [];\n const replanningCycleLimit =\n replanningCycleLimitOverride ?? this.replanningCycleLimit;\n assert(\n replanningCycleLimit !== undefined,\n 'replanningCycleLimit is required for TaskExecutor.action',\n );\n\n let errorCountInOnePlanningLoop = 0; // count the number of errors in one planning loop\n let outputString: string \| undefined;\n\n // Main planning loop - unified plan/replan logic\n while (true) {\n // Check abort signal before each planning cycle\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // Get sub-goal status text if available\n const subGoalStatus = conversationHistory.subGoalsToText() \|\| undefined;\n\n // Get memories text if available\n const memoriesStatus = conversationHistory.memoriesToText() \|\| undefined;\n\n const result = await session.appendAndRun(\n {\n type: 'Planning',\n subType: 'Plan',\n param: {\n userInstruction: userPrompt,\n aiActContext,\n imagesIncludeCount,\n deepThink,\n ...(subGoalStatus ? { subGoalStatus } : {}),\n ...(memoriesStatus ? { memoriesStatus } : {}),\n },\n executor: async (param, executorContext) => {\n const { uiContext } = executorContext;\n assert(uiContext, 'uiContext is required for Planning task');\n const { modelFamily } = modelConfigForPlanning;\n const timing = executorContext.task.timing;\n\n const actionSpace = this.getActionSpace();\n debug(\n 'actionSpace for this interface is:',\n actionSpace.map((action) => action.name).join(', '),\n );\n assert(Array.isArray(actionSpace), 'actionSpace must be an array');\n if (actionSpace.length === 0) {\n console.warn(\n `ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`,\n );\n }\n\n const planImpl = isUITars(modelFamily)\n ? uiTarsPlanning\n : isAutoGLM(modelFamily)\n ? autoGLMPlanning\n : plan;\n\n let planResult: Awaited<ReturnType<typeof planImpl>>;\n try {\n setTimingFieldOnce(timing, 'callAiStart');\n planResult = await planImpl(param.userInstruction, {\n context: uiContext,\n actionContext: param.aiActContext,\n interfaceType: this.interface.interfaceType as InterfaceType,\n actionSpace,\n modelConfig: modelConfigForPlanning,\n conversationHistory,\n includeBbox: includeBboxInPlanning,\n imagesIncludeCount,\n deepThink,\n abortSignal,\n });\n } catch (planError) {\n if (planError instanceof AIResponseParseError) {\n // Record usage and rawResponse even when parsing fails\n executorContext.task.usage = withUsageIntent(\n planError.usage,\n 'planning',\n );\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse: planError.rawResponse,\n };\n }\n throw planError;\n } finally {\n setTimingFieldOnce(timing, 'callAiEnd');\n }\n debug('planResult', JSON.stringify(planResult, null, 2));\n\n const {\n actions,\n thought,\n log,\n memory,\n error,\n usage,\n rawResponse,\n reasoning_content,\n finalizeSuccess,\n finalizeMessage,\n updateSubGoals,\n markFinishedIndexes,\n } = planResult;\n outputString = finalizeMessage;\n\n executorContext.task.log = {\n ...(executorContext.task.log \|\| {}),\n rawResponse,\n };\n executorContext.task.usage = withUsageIntent(usage, 'planning');\n executorContext.task.reasoning_content = reasoning_content;\n executorContext.task.output = {\n actions: actions \|\| [],\n log,\n thought,\n memory,\n yamlFlow: planResult.yamlFlow,\n output: finalizeMessage,\n shouldContinuePlanning: planResult.shouldContinuePlanning,\n updateSubGoals,\n markFinishedIndexes,\n };\n executorContext.uiContext = uiContext;\n\n assert(!error, `Failed to continue: ${error}\\n${log \|\| ''}`);\n\n // Check if task was finalized with failure\n if (finalizeSuccess === false) {\n assert(\n false,\n `Task failed: ${finalizeMessage \|\| 'No error message provided'}\\n${log \|\| ''}`,\n );\n }\n\n return {\n cache: {\n hit: false,\n },\n } as any;\n },\n },\n {\n allowWhenError: true,\n },\n );\n\n const planResult = result?.output as PlanningAIResponse \| undefined;\n\n // Execute planned actions\n const plans = planResult?.actions \|\| [];\n yamlFlow.push(...(planResult?.yamlFlow \|\| []));\n\n let executables: Awaited<ReturnType<typeof this.convertPlanToExecutable>>;\n try {\n executables = await this.convertPlanToExecutable(\n plans,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n {\n cacheable,\n deepLocate,\n abortSignal,\n },\n );\n } catch (error) {\n return session.appendErrorPlan(\n `Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(\n plans,\n )}`,\n );\n }\n if (conversationHistory.pendingFeedbackMessage) {\n console.warn(\n 'unconsumed pending feedback message detected, this may lead to unexpected planning result:',\n conversationHistory.pendingFeedbackMessage,\n );\n }\n\n // Set initial time context for the first planning call\n const initialTimeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;\n\n try {\n await session.appendAndRun(executables.tasks);\n } catch (error: any) {\n // errorFlag = true;\n errorCountInOnePlanningLoop++;\n const timeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message \|\| String(error)}`;\n debug(\n 'error when executing running tasks, but continue to run if it is not too many errors:',\n error instanceof Error ? error.message : String(error),\n 'current error count in one planning loop:',\n errorCountInOnePlanningLoop,\n );\n }\n\n if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) {\n return session.appendErrorPlan('Too many errors in one planning loop');\n }\n\n // Check abort signal after executing actions\n if (abortSignal?.aborted) {\n return session.appendErrorPlan(\n `Task aborted: ${abortSignal.reason \|\| 'abort signal received'}`,\n );\n }\n\n // // Check if task is complete\n if (!planResult?.shouldContinuePlanning) {\n break;\n }\n\n // Increment replan count for next iteration\n ++replanCount;\n\n if (replanCount > replanningCycleLimit) {\n const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;\n return session.appendErrorPlan(errorMsg);\n }\n\n if (!conversationHistory.pendingFeedbackMessage) {\n const timeString = await this.getTimeString();\n conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;\n }\n }\n\n return {\n output: {\n yamlFlow,\n output: outputString,\n },\n runner,\n };\n }\n\n private createTypeQueryTask(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert' \| 'WaitFor',\n demand: ServiceExtractParam,\n modelConfig: IModelConfig,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n ) {\n const queryTask: ExecutionTaskInsightQueryApply = {\n type: 'Insight',\n subType: type,\n param: {\n domIncluded: opt?.domIncluded,\n dataDemand: multimodalPrompt\n ? ({\n demand,\n multimodalPrompt,\n } as never)\n : demand, // for user param presentation in report right sidebar\n },\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let queryDump: ServiceDump \| undefined;\n const applyDump = (dump: ServiceDump) => {\n queryDump = dump;\n task.log = {\n dump,\n rawResponse: dump.taskInfo?.rawResponse,\n };\n task.usage = withUsageIntent(dump.taskInfo?.usage, 'insight');\n if (dump.taskInfo?.reasoning_content) {\n task.reasoning_content = dump.taskInfo.reasoning_content;\n }\n };\n\n // Get context for query operations\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Query task');\n\n const ifTypeRestricted = type !== 'Query';\n let demandInput = demand;\n let keyOfResult = 'result';\n if (ifTypeRestricted && (type === 'Assert' \|\| type === 'WaitFor')) {\n keyOfResult = 'StatementIsTruthy';\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n } else if (ifTypeRestricted) {\n keyOfResult = type;\n demandInput = {\n [keyOfResult]: buildTypeQueryDemandValue(type, demand),\n };\n }\n\n let extractResult;\n\n let extraPageDescription = '';\n if (opt?.domIncluded && this.interface.getElementsNodeTree) {\n debug('appending tree info for page');\n const tree = await this.interface.getElementsNodeTree();\n extraPageDescription = await descriptionOfTree(\n tree,\n 200,\n false,\n opt?.domIncluded === 'visible-only',\n );\n }\n\n try {\n extractResult = await this.service.extract<any>(\n demandInput,\n modelConfig,\n opt,\n extraPageDescription,\n multimodalPrompt,\n uiContext,\n );\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n }\n\n const { data, thought, dump } = extractResult;\n applyDump(dump);\n\n let outputResult = data;\n if (ifTypeRestricted) {\n // If AI returned a plain string instead of structured format, use it directly\n if (typeof data === 'string') {\n outputResult = data;\n } else if (type === 'WaitFor') {\n if (data === null \|\| data === undefined) {\n outputResult = false;\n } else {\n outputResult = (data as any)[keyOfResult];\n }\n } else if (data === null \|\| data === undefined) {\n outputResult = null;\n } else {\n // AI model may return {result: ...} instead of {[keyOfResult]: ...}\n if (data?.[keyOfResult] !== undefined) {\n outputResult = (data as any)[keyOfResult];\n } else if (data?.result !== undefined) {\n outputResult = (data as any).result;\n } else {\n assert(false, 'No result in query data');\n }\n }\n }\n\n if (type === 'Assert' && !outputResult) {\n task.thought = thought;\n throw new Error(`Assertion failed: ${thought}`);\n }\n\n return {\n output: outputResult,\n log: queryDump,\n thought,\n };\n },\n };\n\n return queryTask;\n }\n async createTypeQueryExecution<T>(\n type: 'Query' \| 'Boolean' \| 'Number' \| 'String' \| 'Assert',\n demand: ServiceExtractParam,\n modelConfig: IModelConfig,\n opt?: ServiceExtractOption,\n multimodalPrompt?: TMultimodalPrompt,\n ): Promise<ExecutionResult<T>> {\n const session = this.createExecutionSession(\n taskTitleStr(\n type,\n typeof demand === 'string' ? demand : JSON.stringify(demand),\n ),\n );\n\n const queryTask = await this.createTypeQueryTask(\n type,\n demand,\n modelConfig,\n opt,\n multimodalPrompt,\n );\n\n const runner = session.getRunner();\n const result = await session.appendAndRun(queryTask);\n\n if (!result) {\n throw new Error(\n 'result of taskExecutor.flush() is undefined in function createTypeQueryTask',\n );\n }\n\n const { output, thought } = result;\n\n return {\n output,\n thought,\n runner,\n };\n }\n\n async waitFor(\n assertion: TUserPrompt,\n opt: PlanningActionParamWaitFor,\n modelConfig: IModelConfig,\n ): Promise<ExecutionResult<void>> {\n const { textPrompt, multimodalPrompt } = parsePrompt(assertion);\n\n const description = `waitFor: ${textPrompt}`;\n const session = this.createExecutionSession(\n taskTitleStr('WaitFor', description),\n );\n const runner = session.getRunner();\n const {\n timeoutMs,\n checkIntervalMs,\n domIncluded,\n screenshotIncluded,\n ...restOpt\n } = opt;\n const serviceExtractOpt: ServiceExtractOption = {\n domIncluded,\n screenshotIncluded,\n ...restOpt,\n };\n\n assert(assertion, 'No assertion for waitFor');\n assert(timeoutMs, 'No timeoutMs for waitFor');\n assert(checkIntervalMs, 'No checkIntervalMs for waitFor');\n\n assert(\n checkIntervalMs <= timeoutMs,\n `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`,\n );\n\n const overallStartTime = Date.now();\n let lastCheckStart = overallStartTime;\n let errorThought = '';\n // Continue checking as long as the previous iteration began within the timeout window.\n while (lastCheckStart - overallStartTime <= timeoutMs) {\n const currentCheckStart = Date.now();\n lastCheckStart = currentCheckStart;\n const queryTask = await this.createTypeQueryTask(\n 'WaitFor',\n textPrompt,\n modelConfig,\n serviceExtractOpt,\n multimodalPrompt,\n );\n\n const result = (await session.appendAndRun(queryTask)) as\n \| {\n output: boolean;\n thought?: string;\n }\n \| undefined;\n\n if (result?.output) {\n return {\n output: undefined,\n runner,\n };\n }\n\n errorThought =\n result?.thought \|\|\n (!result && `No result from assertion: ${textPrompt}`) \|\|\n `unknown error when waiting for assertion: ${textPrompt}`;\n const now = Date.now();\n if (now - currentCheckStart < checkIntervalMs) {\n const elapsed = now - currentCheckStart;\n const timeRemaining = checkIntervalMs - elapsed;\n const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;\n const { tasks: sleepTasks } = await this.convertPlanToExecutable(\n [{ type: 'Sleep', param: { timeMs: timeRemaining }, thought }],\n modelConfig,\n modelConfig,\n );\n if (sleepTasks[0]) {\n await session.appendAndRun(sleepTasks[0]);\n }\n }\n }\n\n return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);\n }\n}\n\nexport async function withFileChooser<T>(\n interfaceInstance: AbstractInterface,\n fileChooserAccept: string[] \| undefined,\n action: () => Promise<T>,\n): Promise<T> {\n if (!fileChooserAccept?.length) {\n return action();\n }\n\n if (!interfaceInstance.registerFileChooserListener) {\n throw new Error(\n `File upload is not supported on ${interfaceInstance.interfaceType}`,\n );\n }\n\n const handler = async (chooser: FileChooserHandler) => {\n await chooser.accept(fileChooserAccept);\n };\n\n const { dispose, getError } =\n await interfaceInstance.registerFileChooserListener(handler);\n try {\n const result = await action();\n // Check for errors that occurred during file chooser handling\n const error = getError();\n if (error) {\n throw error;\n }\n return result;\n } finally {\n dispose();\n }\n}\n"],"names":["debug","getDebug","warnLog","maxErrorCountAllowedInOnePlanningLoop","TaskExecutor","title","options","ExecutionSession","Promise","format","error","getReadableTimeString","plans","modelConfigForPlanning","modelConfigForDefaultIntent","userInstruction","yamlString","session","taskTitleStr","task","param","executorContext","uiContext","assert","runner","tasks","result","output","userPrompt","includeBboxInPlanning","aiActContext","cacheable","replanningCycleLimitOverride","imagesIncludeCount","deepThink","fileChooserAccept","deepLocate","abortSignal","withFileChooser","conversationHistory","ConversationHistory","replanCount","yamlFlow","replanningCycleLimit","undefined","errorCountInOnePlanningLoop","outputString","subGoalStatus","memoriesStatus","modelFamily","timing","actionSpace","action","Array","console","planImpl","isUITars","uiTarsPlanning","isAutoGLM","autoGLMPlanning","plan","planResult","setTimingFieldOnce","planError","AIResponseParseError","withUsageIntent","JSON","actions","thought","log","memory","usage","rawResponse","reasoning_content","finalizeSuccess","finalizeMessage","updateSubGoals","markFinishedIndexes","executables","initialTimeString","timeString","String","Error","errorMsg","type","demand","modelConfig","opt","multimodalPrompt","queryTask","taskContext","queryDump","applyDump","dump","ifTypeRestricted","demandInput","keyOfResult","buildTypeQueryDemandValue","extractResult","extraPageDescription","tree","descriptionOfTree","ServiceError","data","outputResult","assertion","textPrompt","parsePrompt","description","timeoutMs","checkIntervalMs","domIncluded","screenshotIncluded","restOpt","serviceExtractOpt","overallStartTime","Date","lastCheckStart","errorThought","currentCheckStart","now","elapsed","timeRemaining","sleepTasks","interfaceInstance","service","opts","TaskBuilder","handler","chooser","dispose","getError"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;AA4DA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,wBAAwB;IAAE,SAAS;AAAK;AACjE,MAAME,wCAAwC;AAIvC,MAAMC;IAsBX,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,SAAS;IACvB;IAiCQ,uBACNC,KAAa,EACbC,OAA0C,EAC1C;QACA,OAAO,IAAIC,iBACTF,OACA,IAAMG,QAAQ,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,kBAAkB,KACrD;YACE,aAAa,IAAI,CAAC,mBAAmB;YACrC,OAAOF,SAAS;YAChB,cAAc,IAAI,CAAC,KAAK,EAAE;QAC5B;IAEJ;IAEQ,iBAAiC;QACvC,OAAO,IAAI,CAAC,mBAAmB;IACjC;IASA,MAAc,cAAcG,MAAe,EAAmB;QAC5D,IAAI,IAAI,CAAC,aAAa,EACpB,IAAI,IAAI,CAAC,SAAS,CAAC,wBAAwB,EACzC,IAAI;YACF,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,wBAAwB,CAACA;QACvD,EAAE,OAAOC,OAAO;YACdR,QACE,CAAC,gEAAgE,EAAEQ,OAAO;QAE9E;aAEAR,QACE;QAKN,OAAOS,sBAAsBF;IAC/B;IAEA,MAAa,wBACXG,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCR,OAIC,EACD;QACA,OAAO,IAAI,CAAC,WAAW,CAAC,KAAK,CAC3BM,OACAC,wBACAC,6BACAR;IAEJ;IAEA,MAAM,uBAAuBS,eAAuB,EAAEC,UAAkB,EAAE;QACxE,MAAMC,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOH;QAGtB,MAAMI,OAAmC;YACvC,MAAM;YACN,SAAS;YACT,OAAO;gBACLJ;YACF;YACA,UAAU,OAAOK,OAAOC;gBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;gBACtBE,OAAOD,WAAW;gBAClB,OAAO;oBACL,QAAQ;wBACN,SAAS,EAAE;wBACX,wBAAwB;wBACxB,KAAK;wBACLN;oBACF;oBACA,OAAO;wBACL,KAAK;oBACP;oBACA,OAAO;wBACL,MAAM;wBACN,SAAS;4BACPA;wBACF;oBACF;gBACF;YACF;QACF;QACA,MAAMQ,SAASP,QAAQ,SAAS;QAChC,MAAMA,QAAQ,YAAY,CAACE;QAE3B,OAAO;YACLK;QACF;IACF;IAEA,MAAM,SACJnB,KAAa,EACbO,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACf;QAC1B,MAAMG,UAAU,IAAI,CAAC,sBAAsB,CAACZ;QAC5C,MAAM,EAAEoB,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAClDb,OACAC,wBACAC;QAEF,MAAMU,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACQ;QAC1C,MAAM,EAAEE,MAAM,EAAE,GAAGD,UAAU,CAAC;QAC9B,OAAO;YACLC;YACAH;QACF;IACF;IAEA,MAAM,OACJI,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBC,iBAA4B,EAC5BC,UAAoB,EACpBC,WAAyB,EASzB;QACA,OAAOC,gBAAgB,IAAI,CAAC,SAAS,EAAEH,mBAAmB,UACjD,IAAI,CAAC,SAAS,CACnBP,YACAf,wBACAC,6BACAe,uBACAC,cACAC,WACAC,8BACAC,oBACAC,WACAE,YACAC;IAGN;IAEA,MAAc,UACZT,UAAkB,EAClBf,sBAAoC,EACpCC,2BAAyC,EACzCe,qBAA8B,EAC9BC,YAAqB,EACrBC,SAAmB,EACnBC,4BAAqC,EACrCC,kBAA2B,EAC3BC,SAAmB,EACnBE,UAAoB,EACpBC,WAAyB,EASzB;QACA,MAAME,sBAAsB,IAAIC;QAEhC,MAAMvB,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,OAAOU;QAEtB,MAAMJ,SAASP,QAAQ,SAAS;QAEhC,IAAIwB,cAAc;QAClB,MAAMC,WAAmC,EAAE;QAC3C,MAAMC,uBACJX,gCAAgC,IAAI,CAAC,oBAAoB;QAC3DT,OACEoB,AAAyBC,WAAzBD,sBACA;QAGF,IAAIE,8BAA8B;QAClC,IAAIC;QAGJ,MAAO,KAAM;YAEX,IAAIT,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,MAAMU,gBAAgBR,oBAAoB,cAAc,MAAMK;YAG9D,MAAMI,iBAAiBT,oBAAoB,cAAc,MAAMK;YAE/D,MAAMlB,SAAS,MAAMT,QAAQ,YAAY,CACvC;gBACE,MAAM;gBACN,SAAS;gBACT,OAAO;oBACL,iBAAiBW;oBACjBE;oBACAG;oBACAC;oBACA,GAAIa,gBAAgB;wBAAEA;oBAAc,IAAI,CAAC,CAAC;oBAC1C,GAAIC,iBAAiB;wBAAEA;oBAAe,IAAI,CAAC,CAAC;gBAC9C;gBACA,UAAU,OAAO5B,OAAOC;oBACtB,MAAM,EAAEC,SAAS,EAAE,GAAGD;oBACtBE,OAAOD,WAAW;oBAClB,MAAM,EAAE2B,WAAW,EAAE,GAAGpC;oBACxB,MAAMqC,SAAS7B,gBAAgB,IAAI,CAAC,MAAM;oBAE1C,MAAM8B,cAAc,IAAI,CAAC,cAAc;oBACvCnD,MACE,sCACAmD,YAAY,GAAG,CAAC,CAACC,SAAWA,OAAO,IAAI,EAAE,IAAI,CAAC;oBAEhD7B,OAAO8B,MAAM,OAAO,CAACF,cAAc;oBACnC,IAAIA,AAAuB,MAAvBA,YAAY,MAAM,EACpBG,QAAQ,IAAI,CACV,CAAC,gBAAgB,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,gDAAgD,CAAC;oBAIrG,MAAMC,WAAWC,SAASP,eACtBQ,iBACAC,UAAUT,eACRU,kBACAC;oBAEN,IAAIC;oBACJ,IAAI;wBACFC,mBAAmBZ,QAAQ;wBAC3BW,aAAa,MAAMN,SAASnC,MAAM,eAAe,EAAE;4BACjD,SAASE;4BACT,eAAeF,MAAM,YAAY;4BACjC,eAAe,IAAI,CAAC,SAAS,CAAC,aAAa;4BAC3C+B;4BACA,aAAatC;4BACb0B;4BACA,aAAaV;4BACbI;4BACAC;4BACAG;wBACF;oBACF,EAAE,OAAO0B,WAAW;wBAClB,IAAIA,qBAAqBC,sBAAsB;4BAE7C3C,gBAAgB,IAAI,CAAC,KAAK,GAAG4C,gBAC3BF,UAAU,KAAK,EACf;4BAEF1C,gBAAgB,IAAI,CAAC,GAAG,GAAG;gCACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;gCAClC,aAAa0C,UAAU,WAAW;4BACpC;wBACF;wBACA,MAAMA;oBACR,SAAU;wBACRD,mBAAmBZ,QAAQ;oBAC7B;oBACAlD,MAAM,cAAckE,KAAK,SAAS,CAACL,YAAY,MAAM;oBAErD,MAAM,EACJM,OAAO,EACPC,OAAO,EACPC,GAAG,EACHC,MAAM,EACN5D,KAAK,EACL6D,KAAK,EACLC,WAAW,EACXC,iBAAiB,EACjBC,eAAe,EACfC,eAAe,EACfC,cAAc,EACdC,mBAAmB,EACpB,GAAGhB;oBACJf,eAAe6B;oBAEftD,gBAAgB,IAAI,CAAC,GAAG,GAAG;wBACzB,GAAIA,gBAAgB,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC;wBAClCmD;oBACF;oBACAnD,gBAAgB,IAAI,CAAC,KAAK,GAAG4C,gBAAgBM,OAAO;oBACpDlD,gBAAgB,IAAI,CAAC,iBAAiB,GAAGoD;oBACzCpD,gBAAgB,IAAI,CAAC,MAAM,GAAG;wBAC5B,SAAS8C,WAAW,EAAE;wBACtBE;wBACAD;wBACAE;wBACA,UAAUT,WAAW,QAAQ;wBAC7B,QAAQc;wBACR,wBAAwBd,WAAW,sBAAsB;wBACzDe;wBACAC;oBACF;oBACAxD,gBAAgB,SAAS,GAAGC;oBAE5BC,OAAO,CAACb,OAAO,CAAC,oBAAoB,EAAEA,MAAM,EAAE,EAAE2D,OAAO,IAAI;oBAG3D,IAAIK,AAAoB,UAApBA,iBACFnD,OACE,OACA,CAAC,aAAa,EAAEoD,mBAAmB,4BAA4B,EAAE,EAAEN,OAAO,IAAI;oBAIlF,OAAO;wBACL,OAAO;4BACL,KAAK;wBACP;oBACF;gBACF;YACF,GACA;gBACE,gBAAgB;YAClB;YAGF,MAAMR,aAAanC,QAAQ;YAG3B,MAAMd,QAAQiD,YAAY,WAAW,EAAE;YACvCnB,SAAS,IAAI,IAAKmB,YAAY,YAAY,EAAE;YAE5C,IAAIiB;YACJ,IAAI;gBACFA,cAAc,MAAM,IAAI,CAAC,uBAAuB,CAC9ClE,OACAC,wBACAC,6BACA;oBACEiB;oBACAK;oBACAC;gBACF;YAEJ,EAAE,OAAO3B,OAAO;gBACd,OAAOO,QAAQ,eAAe,CAC5B,CAAC,4CAA4C,EAAEP,MAAM,SAAS,EAAEwD,KAAK,SAAS,CAC5EtD,QACC;YAEP;YACA,IAAI2B,oBAAoB,sBAAsB,EAC5Ce,QAAQ,IAAI,CACV,8FACAf,oBAAoB,sBAAsB;YAK9C,MAAMwC,oBAAoB,MAAM,IAAI,CAAC,aAAa;YAClDxC,oBAAoB,sBAAsB,IAAI,CAAC,cAAc,EAAEwC,mBAAmB;YAElF,IAAI;gBACF,MAAM9D,QAAQ,YAAY,CAAC6D,YAAY,KAAK;YAC9C,EAAE,OAAOpE,OAAY;gBAEnBmC;gBACA,MAAMmC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,iCAAiC,EAAEtE,OAAO,WAAWuE,OAAOvE,QAAQ;gBACrIV,MACE,yFACAU,iBAAiBwE,QAAQxE,MAAM,OAAO,GAAGuE,OAAOvE,QAChD,6CACAmC;YAEJ;YAEA,IAAIA,8BAA8B1C,uCAChC,OAAOc,QAAQ,eAAe,CAAC;YAIjC,IAAIoB,aAAa,SACf,OAAOpB,QAAQ,eAAe,CAC5B,CAAC,cAAc,EAAEoB,YAAY,MAAM,IAAI,yBAAyB;YAKpE,IAAI,CAACwB,YAAY,wBACf;YAIF,EAAEpB;YAEF,IAAIA,cAAcE,sBAAsB;gBACtC,MAAMwC,WAAW,CAAC,UAAU,EAAExC,qBAAqB,4JAA4J,CAAC;gBAChN,OAAO1B,QAAQ,eAAe,CAACkE;YACjC;YAEA,IAAI,CAAC5C,oBAAoB,sBAAsB,EAAE;gBAC/C,MAAMyC,aAAa,MAAM,IAAI,CAAC,aAAa;gBAC3CzC,oBAAoB,sBAAsB,GAAG,CAAC,MAAM,EAAEyC,WAAW,gDAAgD,CAAC;YACpH;QACF;QAEA,OAAO;YACL,QAAQ;gBACNtC;gBACA,QAAQI;YACV;YACAtB;QACF;IACF;IAEQ,oBACN4D,IAAsE,EACtEC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACpC;QACA,MAAMC,YAA4C;YAChD,MAAM;YACN,SAASL;YACT,OAAO;gBACL,aAAaG,KAAK;gBAClB,YAAYC,mBACP;oBACCH;oBACAG;gBACF,IACAH;YACN;YACA,UAAU,OAAOjE,OAAOsE;gBACtB,MAAM,EAAEvE,IAAI,EAAE,GAAGuE;gBACjB,IAAIC;gBACJ,MAAMC,YAAY,CAACC;oBACjBF,YAAYE;oBACZ1E,KAAK,GAAG,GAAG;wBACT0E;wBACA,aAAaA,KAAK,QAAQ,EAAE;oBAC9B;oBACA1E,KAAK,KAAK,GAAG8C,gBAAgB4B,KAAK,QAAQ,EAAE,OAAO;oBACnD,IAAIA,KAAK,QAAQ,EAAE,mBACjB1E,KAAK,iBAAiB,GAAG0E,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAGA,MAAMvE,YAAYoE,YAAY,SAAS;gBACvCnE,OAAOD,WAAW;gBAElB,MAAMwE,mBAAmBV,AAAS,YAATA;gBACzB,IAAIW,cAAcV;gBAClB,IAAIW,cAAc;gBAClB,IAAIF,oBAAqBV,CAAAA,AAAS,aAATA,QAAqBA,AAAS,cAATA,IAAiB,GAAI;oBACjEY,cAAc;oBACdD,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF,OAAO,IAAIS,kBAAkB;oBAC3BE,cAAcZ;oBACdW,cAAc;wBACZ,CAACC,YAAY,EAAEC,0BAA0Bb,MAAMC;oBACjD;gBACF;gBAEA,IAAIa;gBAEJ,IAAIC,uBAAuB;gBAC3B,IAAIZ,KAAK,eAAe,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE;oBAC1DvF,MAAM;oBACN,MAAMoG,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB;oBACrDD,uBAAuB,MAAME,kBAC3BD,MACA,KACA,OACAb,KAAK,gBAAgB;gBAEzB;gBAEA,IAAI;oBACFW,gBAAgB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,CACxCH,aACAT,aACAC,KACAY,sBACAX,kBACAlE;gBAEJ,EAAE,OAAOZ,OAAO;oBACd,IAAIA,iBAAiB4F,cACnBV,UAAUlF,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAEA,MAAM,EAAE6F,IAAI,EAAEnC,OAAO,EAAEyB,IAAI,EAAE,GAAGK;gBAChCN,UAAUC;gBAEV,IAAIW,eAAeD;gBACnB,IAAIT,kBAEF,IAAI,AAAgB,YAAhB,OAAOS,MACTC,eAAeD;qBACV,IAAInB,AAAS,cAATA,MAEPoB,eADED,QAAAA,OACa,QAECA,IAAY,CAACP,YAAY;qBAEtC,IAAIO,QAAAA,MACTC,eAAe;qBAGf,IAAID,MAAM,CAACP,YAAY,KAAKpD,QAC1B4D,eAAgBD,IAAY,CAACP,YAAY;qBACpC,IAAIO,MAAM,WAAW3D,QAC1B4D,eAAgBD,KAAa,MAAM;qBAEnChF,OAAO,OAAO;gBAKpB,IAAI6D,AAAS,aAATA,QAAqB,CAACoB,cAAc;oBACtCrF,KAAK,OAAO,GAAGiD;oBACf,MAAM,IAAIc,MAAM,CAAC,kBAAkB,EAAEd,SAAS;gBAChD;gBAEA,OAAO;oBACL,QAAQoC;oBACR,KAAKb;oBACLvB;gBACF;YACF;QACF;QAEA,OAAOqB;IACT;IACA,MAAM,yBACJL,IAA0D,EAC1DC,MAA2B,EAC3BC,WAAyB,EACzBC,GAA0B,EAC1BC,gBAAoC,EACP;QAC7B,MAAMvE,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aACEkE,MACA,AAAkB,YAAlB,OAAOC,SAAsBA,SAASnB,KAAK,SAAS,CAACmB;QAIzD,MAAMI,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9CL,MACAC,QACAC,aACAC,KACAC;QAGF,MAAMhE,SAASP,QAAQ,SAAS;QAChC,MAAMS,SAAS,MAAMT,QAAQ,YAAY,CAACwE;QAE1C,IAAI,CAAC/D,QACH,MAAM,IAAIwD,MACR;QAIJ,MAAM,EAAEvD,MAAM,EAAEyC,OAAO,EAAE,GAAG1C;QAE5B,OAAO;YACLC;YACAyC;YACA5C;QACF;IACF;IAEA,MAAM,QACJiF,SAAsB,EACtBlB,GAA+B,EAC/BD,WAAyB,EACO;QAChC,MAAM,EAAEoB,UAAU,EAAElB,gBAAgB,EAAE,GAAGmB,YAAYF;QAErD,MAAMG,cAAc,CAAC,SAAS,EAAEF,YAAY;QAC5C,MAAMzF,UAAU,IAAI,CAAC,sBAAsB,CACzCC,aAAa,WAAW0F;QAE1B,MAAMpF,SAASP,QAAQ,SAAS;QAChC,MAAM,EACJ4F,SAAS,EACTC,eAAe,EACfC,WAAW,EACXC,kBAAkB,EAClB,GAAGC,SACJ,GAAG1B;QACJ,MAAM2B,oBAA0C;YAC9CH;YACAC;YACA,GAAGC,OAAO;QACZ;QAEA1F,OAAOkF,WAAW;QAClBlF,OAAOsF,WAAW;QAClBtF,OAAOuF,iBAAiB;QAExBvF,OACEuF,mBAAmBD,WACnB,CAAC,iGAAiG,EAAEC,gBAAgB,aAAa,EAAED,UAAU,CAAC,CAAC;QAGjJ,MAAMM,mBAAmBC,KAAK,GAAG;QACjC,IAAIC,iBAAiBF;QACrB,IAAIG,eAAe;QAEnB,MAAOD,iBAAiBF,oBAAoBN,UAAW;YACrD,MAAMU,oBAAoBH,KAAK,GAAG;YAClCC,iBAAiBE;YACjB,MAAM9B,YAAY,MAAM,IAAI,CAAC,mBAAmB,CAC9C,WACAiB,YACApB,aACA4B,mBACA1B;YAGF,MAAM9D,SAAU,MAAMT,QAAQ,YAAY,CAACwE;YAO3C,IAAI/D,QAAQ,QACV,OAAO;gBACL,QAAQkB;gBACRpB;YACF;YAGF8F,eACE5F,QAAQ,WACP,CAACA,UAAU,CAAC,0BAA0B,EAAEgF,YAAY,IACrD,CAAC,0CAA0C,EAAEA,YAAY;YAC3D,MAAMc,MAAMJ,KAAK,GAAG;YACpB,IAAII,MAAMD,oBAAoBT,iBAAiB;gBAC7C,MAAMW,UAAUD,MAAMD;gBACtB,MAAMG,gBAAgBZ,kBAAkBW;gBACxC,MAAMrD,UAAU,CAAC,kBAAkB,EAAE0C,gBAAgB,IAAI,EAAEW,QAAQ,0CAA0C,EAAEC,cAAc,EAAE,CAAC;gBAChI,MAAM,EAAE,OAAOC,UAAU,EAAE,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAC9D;oBAAC;wBAAE,MAAM;wBAAS,OAAO;4BAAE,QAAQD;wBAAc;wBAAGtD;oBAAQ;iBAAE,EAC9DkB,aACAA;gBAEF,IAAIqC,UAAU,CAAC,EAAE,EACf,MAAM1G,QAAQ,YAAY,CAAC0G,UAAU,CAAC,EAAE;YAE5C;QACF;QAEA,OAAO1G,QAAQ,eAAe,CAAC,CAAC,iBAAiB,EAAEqG,cAAc;IACnE;IA7sBA,YACEM,iBAAoC,EACpCC,OAAgB,EAChBC,IAQC,CACD;QArCF;QAEA;QAEA;QAEA,uBAAiB,uBAAjB;QAEA,uBAAiB,eAAjB;QAEA;QAEA,uBAAiB,SAAjB;QAEA;QAEA;QAEA;QAoBE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC,KAAK,SAAS;QAC/B,IAAI,CAAC,mBAAmB,GAAGA,MAAM;QACjC,IAAI,CAAC,oBAAoB,GAAGA,KAAK,oBAAoB;QACrD,IAAI,CAAC,eAAe,GAAGA,KAAK,eAAe;QAC3C,IAAI,CAAC,aAAa,GAAGA,KAAK,aAAa;QACvC,IAAI,CAAC,KAAK,GAAGA,KAAK,KAAK;QACvB,IAAI,CAAC,mBAAmB,GAAGA,KAAK,WAAW;QAC3C,IAAI,CAAC,WAAW,GAAG,IAAIC,YAAY;YACjCH;YACAC;YACA,WAAWC,KAAK,SAAS;YACzB,aAAa,IAAI,CAAC,cAAc;YAChC,iBAAiBA,KAAK,eAAe;QACvC;IACF;AAirBF;AAEO,eAAexF,gBACpBsF,iBAAoC,EACpCzF,iBAAuC,EACvCiB,MAAwB;IAExB,IAAI,CAACjB,mBAAmB,QACtB,OAAOiB;IAGT,IAAI,CAACwE,kBAAkB,2BAA2B,EAChD,MAAM,IAAI1C,MACR,CAAC,gCAAgC,EAAE0C,kBAAkB,aAAa,EAAE;IAIxE,MAAMI,UAAU,OAAOC;QACrB,MAAMA,QAAQ,MAAM,CAAC9F;IACvB;IAEA,MAAM,EAAE+F,OAAO,EAAEC,QAAQ,EAAE,GACzB,MAAMP,kBAAkB,2BAA2B,CAACI;IACtD,IAAI;QACF,MAAMtG,SAAS,MAAM0B;QAErB,MAAM1C,QAAQyH;QACd,IAAIzH,OACF,MAAMA;QAER,OAAOgB;IACT,SAAU;QACRwG;IACF;AACF"}

package/dist/es/agent/utils.mjs CHANGED Viewed

@@ -123,7 +123,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.8.3-beta-20260518105651.0";
+const getMidsceneVersion = ()=>"1.8.3";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/llm-planning.mjs CHANGED Viewed

@@ -75,7 +75,7 @@ async function plan(userInstruction, opts) {
     const { shotSize } = context;
     const screenshotBase64 = context.screenshot.base64;
     const { modelFamily } = modelConfig;
-    const includeSubGoals = true === opts.planningModeDeepThink;
+    const includeSubGoals = true === opts.deepThink;
     const systemPrompt = await systemPromptToTaskPlanning({
         actionSpace: opts.actionSpace,
         modelFamily,

package/dist/es/ai-model/llm-planning.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n DeviceAction,\n InterfaceType,\n PlanningAIResponse,\n RawResponsePlanningAIResponse,\n UIContext,\n} from '@/types';\nimport type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n buildYamlFlowFromPlans,\n fillBboxParam,\n findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n extractXMLTag,\n parseMarkFinishedIndexes,\n parseSubGoalsFromXML,\n} from './prompt/util';\nimport {\n AIResponseParseError,\n callAI,\n safeParseJson,\n} from './service-caller/index';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\n/*\n Parse XML response from LLM and convert to RawResponsePlanningAIResponse\n /\nexport function parseXMLPlanningResponse(\n xmlString: string,\n modelFamily: TModelFamily \| undefined,\n): RawResponsePlanningAIResponse {\n const thought = extractXMLTag(xmlString, 'thought');\n const memory = extractXMLTag(xmlString, 'memory');\n const log = extractXMLTag(xmlString, 'log') \|\| '';\n const error = extractXMLTag(xmlString, 'error');\n const actionType = extractXMLTag(xmlString, 'action-type');\n const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n // Parse <complete> tag with success attribute\n const completeGoalRegex =\n /<complete\\s+success=\"(true\|false)\">([\\s\\S]?)<\\/complete>/i;\n const completeGoalMatch = xmlString.match(completeGoalRegex);\n let finalizeMessage: string \| undefined;\n let finalizeSuccess: boolean \| undefined;\n\n if (completeGoalMatch) {\n finalizeSuccess = completeGoalMatch[1] === 'true';\n finalizeMessage = completeGoalMatch[2]?.trim() \|\| undefined;\n }\n\n // Parse sub-goal related tags\n const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n const updateSubGoals = updatePlanContent\n ? parseSubGoalsFromXML(updatePlanContent)\n : undefined;\n const markFinishedIndexes = markSubGoalDone\n ? parseMarkFinishedIndexes(markSubGoalDone)\n : undefined;\n\n // Parse action\n let action: any = null;\n if (actionType && actionType.toLowerCase() !== 'null') {\n // Strip any trailing XML tags that LLM might have leaked into the action type\n // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n const type = actionType.split('<')[0].trim();\n let param: any = undefined;\n\n if (actionParamStr) {\n try {\n // Parse the JSON string in action-param-json\n param = safeParseJson(actionParamStr, modelFamily);\n } catch (e) {\n throw new Error(`Failed to parse action-param-json: ${e}`);\n }\n }\n\n action = {\n type,\n ...(param !== undefined ? { param } : {}),\n };\n }\n\n return {\n ...(thought ? { thought } : {}),\n ...(memory ? { memory } : {}),\n log,\n ...(error ? { error } : {}),\n action,\n ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n };\n}\n\nexport async function plan(\n userInstruction: string,\n opts: {\n context: UIContext;\n interfaceType: InterfaceType;\n actionSpace: DeviceAction<any>[];\n actionContext?: string;\n modelConfig: IModelConfig;\n conversationHistory: ConversationHistory;\n includeBbox: boolean;\n imagesIncludeCount?: number;\n // Controls aiAct planning prompt shape and state updates, such as sub-goals.\n planningModeDeepThink?: boolean;\n abortSignal?: AbortSignal;\n },\n): Promise<PlanningAIResponse> {\n const { context, modelConfig, conversationHistory } = opts;\n const { shotSize } = context;\n const screenshotBase64 = context.screenshot.base64;\n\n const { modelFamily } = modelConfig;\n\n // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n const includeSubGoals = opts.planningModeDeepThink === true;\n\n const systemPrompt = await systemPromptToTaskPlanning({\n actionSpace: opts.actionSpace,\n modelFamily,\n includeBbox: opts.includeBbox,\n includeThought: true, // always include thought\n includeSubGoals,\n });\n\n let imagePayload = screenshotBase64;\n let imageWidth = shotSize.width;\n let imageHeight = shotSize.height;\n const rightLimit = imageWidth;\n const bottomLimit = imageHeight;\n\n // Process image based on VL mode requirements\n if (modelFamily === 'qwen2.5-vl') {\n const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n imageWidth = paddedResult.width;\n imageHeight = paddedResult.height;\n imagePayload = paddedResult.imageBase64;\n }\n\n const actionContext = opts.actionContext\n ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n : '';\n\n const instruction: ChatCompletionMessageParam[] = [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n },\n ],\n },\n ];\n\n let latestFeedbackMessage: ChatCompletionMessageParam;\n\n // Build sub-goal status text to include in the message\n // In planning deep-think mode: show full sub-goals with logs\n // Otherwise: show historical execution logs\n const subGoalsText = includeSubGoals\n ? conversationHistory.subGoalsToText()\n : conversationHistory.historicalLogsToText();\n const subGoalsSection = subGoalsText ? `\\n\\n${subGoalsText}` : '';\n\n // Build memories text to include in the message\n const memoriesText = conversationHistory.memoriesToText();\n const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n if (conversationHistory.pendingFeedbackMessage) {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${subGoalsSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n\n conversationHistory.resetPendingFeedbackMessageIfExists();\n } else {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `this is the latest screenshot${memoriesSection}${subGoalsSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n }\n conversationHistory.append(latestFeedbackMessage);\n\n // Compress history if it exceeds the threshold to avoid context overflow\n conversationHistory.compressHistory(50, 20);\n\n const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n const msgs: ChatCompletionMessageParam[] = [\n { role: 'system', content: systemPrompt },\n ...instruction,\n ...historyLog,\n ];\n\n let {\n content: rawResponse,\n usage,\n reasoning_content,\n } = await callAI(msgs, modelConfig, {\n abortSignal: opts.abortSignal,\n });\n\n // Parse XML response to JSON object, retry once on parse failure\n let planFromAI: RawResponsePlanningAIResponse;\n try {\n try {\n planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n } catch {\n const retry = await callAI(msgs, modelConfig, {\n abortSignal: opts.abortSignal,\n });\n rawResponse = retry.content;\n usage = retry.usage;\n reasoning_content = retry.reasoning_content;\n planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n }\n\n if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n warnLog(\n 'Planning response included both an action and <complete>; ignoring <complete> output.',\n );\n planFromAI.finalizeMessage = undefined;\n planFromAI.finalizeSuccess = undefined;\n }\n\n const actions = planFromAI.action ? [planFromAI.action] : [];\n let shouldContinuePlanning = true;\n\n // Check if task is completed via <complete> tag\n if (planFromAI.finalizeSuccess !== undefined) {\n debug('task completed via <complete> tag, stop planning');\n shouldContinuePlanning = false;\n // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n if (includeSubGoals) {\n conversationHistory.markAllSubGoalsFinished();\n }\n }\n\n const returnValue: PlanningAIResponse = {\n ...planFromAI,\n actions,\n rawResponse,\n usage,\n reasoning_content,\n yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n shouldContinuePlanning,\n };\n\n assert(planFromAI, \"can't get plans from AI\");\n\n actions.forEach((action) => {\n const type = action.type;\n const actionInActionSpace = opts.actionSpace.find(\n (action) => action.name === type,\n );\n\n debug('actionInActionSpace matched', actionInActionSpace);\n const locateFields = actionInActionSpace\n ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n : [];\n\n debug('locateFields', locateFields);\n\n locateFields.forEach((field) => {\n const locateResult = action.param[field];\n if (locateResult && modelFamily !== undefined) {\n // Always use model family to fill bbox parameters\n action.param[field] = fillBboxParam(\n locateResult,\n imageWidth,\n imageHeight,\n modelFamily,\n );\n }\n });\n });\n\n // Update sub-goals in conversation history only in planning deep-think mode.\n if (includeSubGoals) {\n if (planFromAI.updateSubGoals?.length) {\n conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n }\n if (planFromAI.markFinishedIndexes?.length) {\n for (const index of planFromAI.markFinishedIndexes) {\n conversationHistory.markSubGoalFinished(index);\n }\n }\n // Append the planning log to the currently running sub-goal\n if (planFromAI.log) {\n conversationHistory.appendSubGoalLog(planFromAI.log);\n }\n } else {\n // Without planning deep-think mode, accumulate logs as historical execution steps.\n if (planFromAI.log) {\n conversationHistory.appendHistoricalLog(planFromAI.log);\n }\n }\n\n // Append memory to conversation history if present\n if (planFromAI.memory) {\n conversationHistory.appendMemory(planFromAI.memory);\n }\n\n conversationHistory.append({\n role: 'assistant',\n content: [\n {\n type: 'text',\n text: rawResponse,\n },\n ],\n });\n\n return returnValue;\n } catch (parseError) {\n // Throw AIResponseParseError with usage and rawResponse preserved\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n throw new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n rawResponse,\n usage,\n );\n }\n}\n"],"names":["debug","getDebug","warnLog","parseXMLPlanningResponse","xmlString","modelFamily","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","safeParseJson","e","Error","plan","userInstruction","opts","context","modelConfig","conversationHistory","shotSize","screenshotBase64","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","subGoalsText","subGoalsSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","callAI","planFromAI","retry","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;AA8BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAK9C,SAASE,yBACdC,SAAiB,EACjBC,WAAqC;IAErC,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQC,cAAcf,gBAAgBP;QACxC,EAAE,OAAOuB,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFJ,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeQ,KACpBC,eAAuB,EACvBC,IAYC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM,EAAE5B,WAAW,EAAE,GAAG6B;IAGxB,MAAMI,kBAAkBN,AAA+B,SAA/BA,KAAK,qBAAqB;IAElD,MAAMO,eAAe,MAAMC,2BAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7B3B;QACA,aAAa2B,KAAK,WAAW;QAC7B,gBAAgB;QAChBM;IACF;IAEA,IAAIG,eAAeJ;IACnB,IAAIK,aAAaN,SAAS,KAAK;IAC/B,IAAIO,cAAcP,SAAS,MAAM;IAKjC,IAAI/B,AAAgB,iBAAhBA,aAA8B;QAChC,MAAMuC,eAAe,MAAMC,4BAA4BJ;QACvDC,aAAaE,aAAa,KAAK;QAC/BD,cAAcC,aAAa,MAAM;QACjCH,eAAeG,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBd,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMe,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEf,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAIiB;IAKJ,MAAMC,eAAeX,kBACjBH,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMe,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAG/D,MAAME,eAAehB,oBAAoB,cAAc;IACvD,MAAMiB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIhB,oBAAoB,sBAAsB,EAAE;QAC9Ca,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGb,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEiB,kBAAkBF,iBAAiB;gBAChN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKT;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEa,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,6BAA6B,EAAEI,kBAAkBF,iBAAiB;YAC3E;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACa;IAG3Bb,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMkB,aAAalB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMsB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,IAAI,EACF,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,OAAOJ,MAAMpB,aAAa;QAClC,aAAaF,KAAK,WAAW;IAC/B;IAGA,IAAI2B;IACJ,IAAI;QACF,IAAI;YACFA,aAAaxD,yBAAyBoD,aAAalD;QACrD,EAAE,OAAM;YACN,MAAMuD,QAAQ,MAAMF,OAAOJ,MAAMpB,aAAa;gBAC5C,aAAaF,KAAK,WAAW;YAC/B;YACAuB,cAAcK,MAAM,OAAO;YAC3BJ,QAAQI,MAAM,KAAK;YACnBH,oBAAoBG,MAAM,iBAAiB;YAC3CD,aAAaxD,yBAAyBoD,aAAalD;QACrD;QAEA,IAAIsD,WAAW,MAAM,IAAIA,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YACjEzD,QACE;YAEFyD,WAAW,eAAe,GAAG1C;YAC7B0C,WAAW,eAAe,GAAG1C;QAC/B;QAEA,MAAM4C,UAAUF,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIG,yBAAyB;QAG7B,IAAIH,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YAC5C3D,MAAM;YACN8D,yBAAyB;YAEzB,IAAIxB,iBACFH,oBAAoB,uBAAuB;QAE/C;QAEA,MAAM4B,cAAkC;YACtC,GAAGJ,UAAU;YACbE;YACAN;YACAC;YACAC;YACA,UAAUO,uBAAuBH,SAAS7B,KAAK,WAAW;YAC1D8B;QACF;QAEAG,OAAON,YAAY;QAEnBE,QAAQ,OAAO,CAAC,CAACrC;YACf,MAAMC,OAAOD,OAAO,IAAI;YACxB,MAAM0C,sBAAsBlC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACR,SAAWA,OAAO,IAAI,KAAKC;YAG9BzB,MAAM,+BAA+BkE;YACrC,MAAMC,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;YAENlE,MAAM,gBAAgBmE;YAEtBA,aAAa,OAAO,CAAC,CAACE;gBACpB,MAAMC,eAAe9C,OAAO,KAAK,CAAC6C,MAAM;gBACxC,IAAIC,gBAAgBjE,AAAgBY,WAAhBZ,aAElBmB,OAAO,KAAK,CAAC6C,MAAM,GAAGE,cACpBD,cACA5B,YACAC,aACAtC;YAGN;QACF;QAGA,IAAIiC,iBAAiB;YACnB,IAAIqB,WAAW,cAAc,EAAE,QAC7BxB,oBAAoB,aAAa,CAACwB,WAAW,cAAc;YAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMa,SAASb,WAAW,mBAAmB,CAChDxB,oBAAoB,mBAAmB,CAACqC;YAI5C,IAAIb,WAAW,GAAG,EAChBxB,oBAAoB,gBAAgB,CAACwB,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBxB,oBAAoB,mBAAmB,CAACwB,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnBxB,oBAAoB,YAAY,CAACwB,WAAW,MAAM;QAGpDxB,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMoB;gBACR;aACD;QACH;QAEA,OAAOQ;IACT,EAAE,OAAOU,YAAY;QAEnB,MAAMC,eACJD,sBAAsB5C,QAAQ4C,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClCnB,aACAC;IAEJ;AACF"}
1	+ {"version":3,"file":"ai-model/llm-planning.mjs","sources":["../../../src/ai-model/llm-planning.ts"],"sourcesContent":["import type {\n DeviceAction,\n InterfaceType,\n PlanningAIResponse,\n RawResponsePlanningAIResponse,\n UIContext,\n} from '@/types';\nimport type { IModelConfig, TModelFamily } from '@midscene/shared/env';\nimport { paddingToMatchBlockByBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport {\n buildYamlFlowFromPlans,\n fillBboxParam,\n findAllMidsceneLocatorField,\n} from '../common';\nimport type { ConversationHistory } from './conversation-history';\nimport { systemPromptToTaskPlanning } from './prompt/llm-planning';\nimport {\n extractXMLTag,\n parseMarkFinishedIndexes,\n parseSubGoalsFromXML,\n} from './prompt/util';\nimport {\n AIResponseParseError,\n callAI,\n safeParseJson,\n} from './service-caller/index';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\n/*\n Parse XML response from LLM and convert to RawResponsePlanningAIResponse\n /\nexport function parseXMLPlanningResponse(\n xmlString: string,\n modelFamily: TModelFamily \| undefined,\n): RawResponsePlanningAIResponse {\n const thought = extractXMLTag(xmlString, 'thought');\n const memory = extractXMLTag(xmlString, 'memory');\n const log = extractXMLTag(xmlString, 'log') \|\| '';\n const error = extractXMLTag(xmlString, 'error');\n const actionType = extractXMLTag(xmlString, 'action-type');\n const actionParamStr = extractXMLTag(xmlString, 'action-param-json');\n\n // Parse <complete> tag with success attribute\n const completeGoalRegex =\n /<complete\\s+success=\"(true\|false)\">([\\s\\S]?)<\\/complete>/i;\n const completeGoalMatch = xmlString.match(completeGoalRegex);\n let finalizeMessage: string \| undefined;\n let finalizeSuccess: boolean \| undefined;\n\n if (completeGoalMatch) {\n finalizeSuccess = completeGoalMatch[1] === 'true';\n finalizeMessage = completeGoalMatch[2]?.trim() \|\| undefined;\n }\n\n // Parse sub-goal related tags\n const updatePlanContent = extractXMLTag(xmlString, 'update-plan-content');\n const markSubGoalDone = extractXMLTag(xmlString, 'mark-sub-goal-done');\n\n const updateSubGoals = updatePlanContent\n ? parseSubGoalsFromXML(updatePlanContent)\n : undefined;\n const markFinishedIndexes = markSubGoalDone\n ? parseMarkFinishedIndexes(markSubGoalDone)\n : undefined;\n\n // Parse action\n let action: any = null;\n if (actionType && actionType.toLowerCase() !== 'null') {\n // Strip any trailing XML tags that LLM might have leaked into the action type\n // e.g. \"KeyboardPress</action-type>\\n<action-param-json>\" -> \"KeyboardPress\"\n const type = actionType.split('<')[0].trim();\n let param: any = undefined;\n\n if (actionParamStr) {\n try {\n // Parse the JSON string in action-param-json\n param = safeParseJson(actionParamStr, modelFamily);\n } catch (e) {\n throw new Error(`Failed to parse action-param-json: ${e}`);\n }\n }\n\n action = {\n type,\n ...(param !== undefined ? { param } : {}),\n };\n }\n\n return {\n ...(thought ? { thought } : {}),\n ...(memory ? { memory } : {}),\n log,\n ...(error ? { error } : {}),\n action,\n ...(finalizeMessage !== undefined ? { finalizeMessage } : {}),\n ...(finalizeSuccess !== undefined ? { finalizeSuccess } : {}),\n ...(updateSubGoals?.length ? { updateSubGoals } : {}),\n ...(markFinishedIndexes?.length ? { markFinishedIndexes } : {}),\n };\n}\n\nexport async function plan(\n userInstruction: string,\n opts: {\n context: UIContext;\n interfaceType: InterfaceType;\n actionSpace: DeviceAction<any>[];\n actionContext?: string;\n modelConfig: IModelConfig;\n conversationHistory: ConversationHistory;\n includeBbox: boolean;\n imagesIncludeCount?: number;\n // Controls aiAct planning prompt shape and state updates, such as sub-goals.\n deepThink?: boolean;\n abortSignal?: AbortSignal;\n },\n): Promise<PlanningAIResponse> {\n const { context, modelConfig, conversationHistory } = opts;\n const { shotSize } = context;\n const screenshotBase64 = context.screenshot.base64;\n\n const { modelFamily } = modelConfig;\n\n // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n const includeSubGoals = opts.deepThink === true;\n\n const systemPrompt = await systemPromptToTaskPlanning({\n actionSpace: opts.actionSpace,\n modelFamily,\n includeBbox: opts.includeBbox,\n includeThought: true, // always include thought\n includeSubGoals,\n });\n\n let imagePayload = screenshotBase64;\n let imageWidth = shotSize.width;\n let imageHeight = shotSize.height;\n const rightLimit = imageWidth;\n const bottomLimit = imageHeight;\n\n // Process image based on VL mode requirements\n if (modelFamily === 'qwen2.5-vl') {\n const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n imageWidth = paddedResult.width;\n imageHeight = paddedResult.height;\n imagePayload = paddedResult.imageBase64;\n }\n\n const actionContext = opts.actionContext\n ? `<high_priority_knowledge>${opts.actionContext}</high_priority_knowledge>\\n`\n : '';\n\n const instruction: ChatCompletionMessageParam[] = [\n {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${actionContext}<user_instruction>${userInstruction}</user_instruction>`,\n },\n ],\n },\n ];\n\n let latestFeedbackMessage: ChatCompletionMessageParam;\n\n // Build sub-goal status text to include in the message\n // In planning deep-think mode: show full sub-goals with logs\n // Otherwise: show historical execution logs\n const subGoalsText = includeSubGoals\n ? conversationHistory.subGoalsToText()\n : conversationHistory.historicalLogsToText();\n const subGoalsSection = subGoalsText ? `\\n\\n${subGoalsText}` : '';\n\n // Build memories text to include in the message\n const memoriesText = conversationHistory.memoriesToText();\n const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n if (conversationHistory.pendingFeedbackMessage) {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${subGoalsSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n\n conversationHistory.resetPendingFeedbackMessageIfExists();\n } else {\n latestFeedbackMessage = {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `this is the latest screenshot${memoriesSection}${subGoalsSection}`,\n },\n {\n type: 'image_url',\n image_url: {\n url: imagePayload,\n detail: 'high',\n },\n },\n ],\n };\n }\n conversationHistory.append(latestFeedbackMessage);\n\n // Compress history if it exceeds the threshold to avoid context overflow\n conversationHistory.compressHistory(50, 20);\n\n const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n const msgs: ChatCompletionMessageParam[] = [\n { role: 'system', content: systemPrompt },\n ...instruction,\n ...historyLog,\n ];\n\n let {\n content: rawResponse,\n usage,\n reasoning_content,\n } = await callAI(msgs, modelConfig, {\n abortSignal: opts.abortSignal,\n });\n\n // Parse XML response to JSON object, retry once on parse failure\n let planFromAI: RawResponsePlanningAIResponse;\n try {\n try {\n planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n } catch {\n const retry = await callAI(msgs, modelConfig, {\n abortSignal: opts.abortSignal,\n });\n rawResponse = retry.content;\n usage = retry.usage;\n reasoning_content = retry.reasoning_content;\n planFromAI = parseXMLPlanningResponse(rawResponse, modelFamily);\n }\n\n if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n warnLog(\n 'Planning response included both an action and <complete>; ignoring <complete> output.',\n );\n planFromAI.finalizeMessage = undefined;\n planFromAI.finalizeSuccess = undefined;\n }\n\n const actions = planFromAI.action ? [planFromAI.action] : [];\n let shouldContinuePlanning = true;\n\n // Check if task is completed via <complete> tag\n if (planFromAI.finalizeSuccess !== undefined) {\n debug('task completed via <complete> tag, stop planning');\n shouldContinuePlanning = false;\n // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n if (includeSubGoals) {\n conversationHistory.markAllSubGoalsFinished();\n }\n }\n\n const returnValue: PlanningAIResponse = {\n ...planFromAI,\n actions,\n rawResponse,\n usage,\n reasoning_content,\n yamlFlow: buildYamlFlowFromPlans(actions, opts.actionSpace),\n shouldContinuePlanning,\n };\n\n assert(planFromAI, \"can't get plans from AI\");\n\n actions.forEach((action) => {\n const type = action.type;\n const actionInActionSpace = opts.actionSpace.find(\n (action) => action.name === type,\n );\n\n debug('actionInActionSpace matched', actionInActionSpace);\n const locateFields = actionInActionSpace\n ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema)\n : [];\n\n debug('locateFields', locateFields);\n\n locateFields.forEach((field) => {\n const locateResult = action.param[field];\n if (locateResult && modelFamily !== undefined) {\n // Always use model family to fill bbox parameters\n action.param[field] = fillBboxParam(\n locateResult,\n imageWidth,\n imageHeight,\n modelFamily,\n );\n }\n });\n });\n\n // Update sub-goals in conversation history only in planning deep-think mode.\n if (includeSubGoals) {\n if (planFromAI.updateSubGoals?.length) {\n conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n }\n if (planFromAI.markFinishedIndexes?.length) {\n for (const index of planFromAI.markFinishedIndexes) {\n conversationHistory.markSubGoalFinished(index);\n }\n }\n // Append the planning log to the currently running sub-goal\n if (planFromAI.log) {\n conversationHistory.appendSubGoalLog(planFromAI.log);\n }\n } else {\n // Without planning deep-think mode, accumulate logs as historical execution steps.\n if (planFromAI.log) {\n conversationHistory.appendHistoricalLog(planFromAI.log);\n }\n }\n\n // Append memory to conversation history if present\n if (planFromAI.memory) {\n conversationHistory.appendMemory(planFromAI.memory);\n }\n\n conversationHistory.append({\n role: 'assistant',\n content: [\n {\n type: 'text',\n text: rawResponse,\n },\n ],\n });\n\n return returnValue;\n } catch (parseError) {\n // Throw AIResponseParseError with usage and rawResponse preserved\n const errorMessage =\n parseError instanceof Error ? parseError.message : String(parseError);\n throw new AIResponseParseError(\n `XML parse error: ${errorMessage}`,\n rawResponse,\n usage,\n );\n }\n}\n"],"names":["debug","getDebug","warnLog","parseXMLPlanningResponse","xmlString","modelFamily","thought","extractXMLTag","memory","log","error","actionType","actionParamStr","completeGoalRegex","completeGoalMatch","finalizeMessage","finalizeSuccess","undefined","updatePlanContent","markSubGoalDone","updateSubGoals","parseSubGoalsFromXML","markFinishedIndexes","parseMarkFinishedIndexes","action","type","param","safeParseJson","e","Error","plan","userInstruction","opts","context","modelConfig","conversationHistory","shotSize","screenshotBase64","includeSubGoals","systemPrompt","systemPromptToTaskPlanning","imagePayload","imageWidth","imageHeight","paddedResult","paddingToMatchBlockByBase64","actionContext","instruction","latestFeedbackMessage","subGoalsText","subGoalsSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","callAI","planFromAI","retry","actions","shouldContinuePlanning","returnValue","buildYamlFlowFromPlans","assert","actionInActionSpace","locateFields","findAllMidsceneLocatorField","field","locateResult","fillBboxParam","index","parseError","errorMessage","String","AIResponseParseError"],"mappings":";;;;;;;AA8BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAK9C,SAASE,yBACdC,SAAiB,EACjBC,WAAqC;IAErC,MAAMC,UAAUC,cAAcH,WAAW;IACzC,MAAMI,SAASD,cAAcH,WAAW;IACxC,MAAMK,MAAMF,cAAcH,WAAW,UAAU;IAC/C,MAAMM,QAAQH,cAAcH,WAAW;IACvC,MAAMO,aAAaJ,cAAcH,WAAW;IAC5C,MAAMQ,iBAAiBL,cAAcH,WAAW;IAGhD,MAAMS,oBACJ;IACF,MAAMC,oBAAoBV,UAAU,KAAK,CAACS;IAC1C,IAAIE;IACJ,IAAIC;IAEJ,IAAIF,mBAAmB;QACrBE,kBAAkBF,AAAyB,WAAzBA,iBAAiB,CAAC,EAAE;QACtCC,kBAAkBD,iBAAiB,CAAC,EAAE,EAAE,UAAUG;IACpD;IAGA,MAAMC,oBAAoBX,cAAcH,WAAW;IACnD,MAAMe,kBAAkBZ,cAAcH,WAAW;IAEjD,MAAMgB,iBAAiBF,oBACnBG,qBAAqBH,qBACrBD;IACJ,MAAMK,sBAAsBH,kBACxBI,yBAAyBJ,mBACzBF;IAGJ,IAAIO,SAAc;IAClB,IAAIb,cAAcA,AAA6B,WAA7BA,WAAW,WAAW,IAAe;QAGrD,MAAMc,OAAOd,WAAW,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI;QAC1C,IAAIe;QAEJ,IAAId,gBACF,IAAI;YAEFc,QAAQC,cAAcf,gBAAgBP;QACxC,EAAE,OAAOuB,GAAG;YACV,MAAM,IAAIC,MAAM,CAAC,mCAAmC,EAAED,GAAG;QAC3D;QAGFJ,SAAS;YACPC;YACA,GAAIC,AAAUT,WAAVS,QAAsB;gBAAEA;YAAM,IAAI,CAAC,CAAC;QAC1C;IACF;IAEA,OAAO;QACL,GAAIpB,UAAU;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAC9B,GAAIE,SAAS;YAAEA;QAAO,IAAI,CAAC,CAAC;QAC5BC;QACA,GAAIC,QAAQ;YAAEA;QAAM,IAAI,CAAC,CAAC;QAC1Bc;QACA,GAAIT,AAAoBE,WAApBF,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAIC,AAAoBC,WAApBD,kBAAgC;YAAEA;QAAgB,IAAI,CAAC,CAAC;QAC5D,GAAII,gBAAgB,SAAS;YAAEA;QAAe,IAAI,CAAC,CAAC;QACpD,GAAIE,qBAAqB,SAAS;YAAEA;QAAoB,IAAI,CAAC,CAAC;IAChE;AACF;AAEO,eAAeQ,KACpBC,eAAuB,EACvBC,IAYC;IAED,MAAM,EAAEC,OAAO,EAAEC,WAAW,EAAEC,mBAAmB,EAAE,GAAGH;IACtD,MAAM,EAAEI,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM,EAAE5B,WAAW,EAAE,GAAG6B;IAGxB,MAAMI,kBAAkBN,AAAmB,SAAnBA,KAAK,SAAS;IAEtC,MAAMO,eAAe,MAAMC,2BAA2B;QACpD,aAAaR,KAAK,WAAW;QAC7B3B;QACA,aAAa2B,KAAK,WAAW;QAC7B,gBAAgB;QAChBM;IACF;IAEA,IAAIG,eAAeJ;IACnB,IAAIK,aAAaN,SAAS,KAAK;IAC/B,IAAIO,cAAcP,SAAS,MAAM;IAKjC,IAAI/B,AAAgB,iBAAhBA,aAA8B;QAChC,MAAMuC,eAAe,MAAMC,4BAA4BJ;QACvDC,aAAaE,aAAa,KAAK;QAC/BD,cAAcC,aAAa,MAAM;QACjCH,eAAeG,aAAa,WAAW;IACzC;IAEA,MAAME,gBAAgBd,KAAK,aAAa,GACpC,CAAC,yBAAyB,EAAEA,KAAK,aAAa,CAAC,4BAA4B,CAAC,GAC5E;IAEJ,MAAMe,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGD,cAAc,kBAAkB,EAAEf,gBAAgB,mBAAmB,CAAC;gBACjF;aACD;QACH;KACD;IAED,IAAIiB;IAKJ,MAAMC,eAAeX,kBACjBH,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMe,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAG/D,MAAME,eAAehB,oBAAoB,cAAc;IACvD,MAAMiB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAIhB,oBAAoB,sBAAsB,EAAE;QAC9Ca,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGb,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEiB,kBAAkBF,iBAAiB;gBAChN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKT;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAN,oBAAoB,mCAAmC;IACzD,OACEa,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,6BAA6B,EAAEI,kBAAkBF,iBAAiB;YAC3E;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKT;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFN,oBAAoB,MAAM,CAACa;IAG3Bb,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMkB,aAAalB,oBAAoB,QAAQ,CAACH,KAAK,kBAAkB;IAEvE,MAAMsB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,IAAI,EACF,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EAClB,GAAG,MAAMC,OAAOJ,MAAMpB,aAAa;QAClC,aAAaF,KAAK,WAAW;IAC/B;IAGA,IAAI2B;IACJ,IAAI;QACF,IAAI;YACFA,aAAaxD,yBAAyBoD,aAAalD;QACrD,EAAE,OAAM;YACN,MAAMuD,QAAQ,MAAMF,OAAOJ,MAAMpB,aAAa;gBAC5C,aAAaF,KAAK,WAAW;YAC/B;YACAuB,cAAcK,MAAM,OAAO;YAC3BJ,QAAQI,MAAM,KAAK;YACnBH,oBAAoBG,MAAM,iBAAiB;YAC3CD,aAAaxD,yBAAyBoD,aAAalD;QACrD;QAEA,IAAIsD,WAAW,MAAM,IAAIA,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YACjEzD,QACE;YAEFyD,WAAW,eAAe,GAAG1C;YAC7B0C,WAAW,eAAe,GAAG1C;QAC/B;QAEA,MAAM4C,UAAUF,WAAW,MAAM,GAAG;YAACA,WAAW,MAAM;SAAC,GAAG,EAAE;QAC5D,IAAIG,yBAAyB;QAG7B,IAAIH,AAA+B1C,WAA/B0C,WAAW,eAAe,EAAgB;YAC5C3D,MAAM;YACN8D,yBAAyB;YAEzB,IAAIxB,iBACFH,oBAAoB,uBAAuB;QAE/C;QAEA,MAAM4B,cAAkC;YACtC,GAAGJ,UAAU;YACbE;YACAN;YACAC;YACAC;YACA,UAAUO,uBAAuBH,SAAS7B,KAAK,WAAW;YAC1D8B;QACF;QAEAG,OAAON,YAAY;QAEnBE,QAAQ,OAAO,CAAC,CAACrC;YACf,MAAMC,OAAOD,OAAO,IAAI;YACxB,MAAM0C,sBAAsBlC,KAAK,WAAW,CAAC,IAAI,CAC/C,CAACR,SAAWA,OAAO,IAAI,KAAKC;YAG9BzB,MAAM,+BAA+BkE;YACrC,MAAMC,eAAeD,sBACjBE,4BAA4BF,oBAAoB,WAAW,IAC3D,EAAE;YAENlE,MAAM,gBAAgBmE;YAEtBA,aAAa,OAAO,CAAC,CAACE;gBACpB,MAAMC,eAAe9C,OAAO,KAAK,CAAC6C,MAAM;gBACxC,IAAIC,gBAAgBjE,AAAgBY,WAAhBZ,aAElBmB,OAAO,KAAK,CAAC6C,MAAM,GAAGE,cACpBD,cACA5B,YACAC,aACAtC;YAGN;QACF;QAGA,IAAIiC,iBAAiB;YACnB,IAAIqB,WAAW,cAAc,EAAE,QAC7BxB,oBAAoB,aAAa,CAACwB,WAAW,cAAc;YAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAMa,SAASb,WAAW,mBAAmB,CAChDxB,oBAAoB,mBAAmB,CAACqC;YAI5C,IAAIb,WAAW,GAAG,EAChBxB,oBAAoB,gBAAgB,CAACwB,WAAW,GAAG;QAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBxB,oBAAoB,mBAAmB,CAACwB,WAAW,GAAG;QAK1D,IAAIA,WAAW,MAAM,EACnBxB,oBAAoB,YAAY,CAACwB,WAAW,MAAM;QAGpDxB,oBAAoB,MAAM,CAAC;YACzB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAMoB;gBACR;aACD;QACH;QAEA,OAAOQ;IACT,EAAE,OAAOU,YAAY;QAEnB,MAAMC,eACJD,sBAAsB5C,QAAQ4C,WAAW,OAAO,GAAGE,OAAOF;QAC5D,MAAM,IAAIG,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClCnB,aACAC;IAEJ;AACF"}

package/dist/es/types.mjs.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any /\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n BaseElement,\n LocateResultElement,\n Rect,\n Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n DetailedLocateParam,\n MidsceneYamlFlowItem,\n ServiceExtractOption,\n} from './yaml';\n\nexport type {\n ElementTreeNode,\n BaseElement,\n Rect,\n Size,\n Point,\n} from '@midscene/shared/types';\nexport from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n ExecutionDump,\n ReportActionDump,\n GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type AIUsageInfo = Record<string, any> & {\n prompt_tokens: number \| undefined;\n completion_tokens: number \| undefined;\n total_tokens: number \| undefined;\n cached_input: number \| undefined;\n time_cost: number \| undefined;\n model_name: string \| undefined;\n model_description: string \| undefined;\n /*\n Semantic intent of the model call, such as default, planning, or insight.\n /\n intent: string \| undefined;\n /\n Config slot where the model config was resolved from. For example, a\n * planning call may use the default slot when no planning model is configured.\n /\n slot: string \| undefined;\n request_id: string \| undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n position?: {\n x: number;\n y: number;\n };\n bbox?: [number, number, number, number];\n reason: string;\n text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n bbox: [number, number, number, number];\n errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n data: DataDemand;\n errors?: string[];\n thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n bbox: [number, number, number, number];\n references_bbox?: [number, number, number, number][];\n error?: string;\n}\n\nexport interface AIAssertionResponse {\n pass: boolean;\n thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n description: string;\n error?: string;\n}\n\nexport interface LocatorValidatorOption {\n centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n pass: boolean;\n rect: Rect;\n center: [number, number];\n centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n prompt: string;\n deepLocate: boolean;\n verifyResult?: LocateValidatorResult;\n}\n\n/\n context\n /\n\nexport abstract class UIContext {\n /\n screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n /\n abstract screenshot: ScreenshotItem;\n\n /\n screenshot size after shrinking\n /\n abstract shotSize: Size;\n\n /\n The ratio for converting shrunk screenshot coordinates to logical coordinates.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - User-defined screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n /\n abstract shrunkShotToLogicalRatio: number;\n\n abstract _isFrozen?: boolean;\n\n // @deprecated - backward compatibility for aiLocate\n abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' \| 'extract' \| 'assert' \| 'describe';\n\nexport type ServiceExtractParam = string \| Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n element: LocateResultElement \| null;\n rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' \| 'medium' \| 'high';\n\nexport type DeepThinkOption = 'unset' \| true \| false;\n\nexport interface ServiceTaskInfo {\n durationMs: number;\n formatResponse?: string;\n rawResponse?: string;\n usage?: AIUsageInfo;\n searchArea?: Rect;\n searchAreaRawResponse?: string;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n logTime: number;\n}\n\nexport type ReportAttributes = Record<\n string,\n string \| number \| boolean \| null \| undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n dumpString: string;\n attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n type: 'locate' \| 'extract' \| 'assert';\n logId: string;\n userQuery: {\n element?: TUserPrompt;\n dataDemand?: ServiceExtractParam;\n assertion?: TUserPrompt;\n };\n matchedElement: LocateResultElement[];\n matchedRect?: Rect;\n deepLocate?: boolean;\n data: any;\n assertionPass?: boolean;\n assertionThought?: string;\n taskInfo: ServiceTaskInfo;\n error?: string;\n output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n ServiceDump,\n 'logTime' \| 'logId' \| 'model_name'\n>;\n\nexport interface ServiceResultBase {\n dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n data: T;\n thought?: string;\n usage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n name: string;\n description: string;\n sectionCharacteristics: string;\n textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement \| null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n usage?: AIUsageInfo;\n};\n\n/\n agent\n /\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> \| void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n checkIntervalMs?: number;\n timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n keepRawResponse?: boolean;\n}\n\n/\n planning\n \n /\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n thought?: string;\n log?: string; // a brief preamble to the user explaining what you’re about to do\n type: string;\n param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' \| 'running' \| 'finished';\n\nexport interface SubGoal {\n index: number;\n status: SubGoalStatus;\n description: string;\n logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n action: PlanningAction;\n thought?: string;\n log: string;\n memory?: string;\n error?: string;\n finalizeMessage?: string;\n finalizeSuccess?: boolean;\n updateSubGoals?: SubGoal[];\n markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n extends Omit<RawResponsePlanningAIResponse, 'action'> {\n actions?: PlanningAction[];\n usage?: AIUsageInfo;\n rawResponse?: string;\n yamlFlow?: MidsceneYamlFlowItem[];\n yamlString?: string;\n error?: string;\n reasoning_content?: string;\n shouldContinuePlanning: boolean;\n output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n duration?: number;\n}\n\nexport interface PullParam {\n direction: 'up' \| 'down';\n distance?: number;\n duration?: number;\n}\n/*\n misc\n /\n\nexport interface Color {\n name: string;\n hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/\naction\n/\nexport interface ExecutionTaskProgressOptions {\n onTaskStart?: (task: ExecutionTask) => Promise<void> \| void;\n}\n\nexport interface ExecutionRecorderItem {\n type: 'screenshot';\n ts: number;\n screenshot?: ScreenshotItem;\n timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' \| 'Insight' \| 'Action Space' \| 'Log';\n\nexport interface ExecutorContext {\n task: ExecutionTask;\n element?: LocateResultElement \| null;\n uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n Type extends ExecutionTaskType = any,\n TaskParam = any,\n TaskOutput = any,\n TaskLog = any,\n> {\n type: Type;\n subType?: string;\n param?: TaskParam;\n thought?: string;\n uiContext?: UIContext;\n executor: (\n param: TaskParam,\n context: ExecutorContext,\n ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n \| Promise<ExecutionTaskReturn<TaskOutput, TaskLog> \| undefined \| void>\n \| undefined\n \| void;\n}\n\nexport interface ExecutionTaskHitBy {\n from: string;\n context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n output?: TaskOutput;\n log?: TaskLog;\n recorder?: ExecutionRecorderItem[];\n hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n any,\n any,\n any\n >,\n> = E &\n ExecutionTaskReturn<\n E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n ? TaskOutput\n : unknown,\n E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n ? TaskLog\n : unknown\n > & {\n taskId: string;\n status: 'pending' \| 'running' \| 'finished' \| 'failed' \| 'cancelled';\n error?: Error;\n errorMessage?: string;\n errorStack?: string;\n timing?: {\n start: number;\n getUiContextStart?: number;\n getUiContextEnd?: number;\n callAiStart?: number;\n callAiEnd?: number;\n beforeInvokeActionHookStart?: number;\n beforeInvokeActionHookEnd?: number;\n callActionStart?: number;\n callActionEnd?: number;\n afterInvokeActionHookStart?: number;\n afterInvokeActionHookEnd?: number;\n captureAfterCallingSnapshotStart?: number;\n captureAfterCallingSnapshotEnd?: number;\n end?: number;\n cost?: number;\n };\n usage?: AIUsageInfo;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n };\n\nexport interface IExecutionDump extends DumpMeta {\n /* Stable unique identifier for this execution run /\n id?: string;\n name: string;\n description?: string;\n tasks: ExecutionTask[];\n aiActContext?: string;\n}\n\n/\ntask - service-locate\n/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightLocateParam,\n ExecutionTaskInsightLocateOutput,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/\ntask - service-query\n/\nexport interface ExecutionTaskInsightQueryParam {\n dataDemand: ServiceExtractParam;\n domIncluded?: boolean \| 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightQueryParam,\n any,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/\ntask - assertion\n/\nexport interface ExecutionTaskInsightAssertionParam {\n assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightAssertionParam,\n ServiceAssertionResponse,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/\ntask - action (i.e. interact) \n/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n 'Action Space',\n ActionParam,\n void,\n void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/\ntask - Log\n/\n\nexport type ExecutionTaskLogApply<\n LogParam = {\n content: string;\n },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/\ntask - planning\n/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n 'Planning',\n {\n userInstruction: string;\n aiActContext?: string;\n },\n PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/\ntask - planning-locate\n/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n 'Planning',\n ExecutionTaskPlanningLocateParam,\n ExecutionTaskPlanningLocateOutput,\n ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/\nReport metadata - extracted from ReportActionDump for per-execution writes\n/\nexport interface ReportMeta {\n groupName: string;\n groupDescription?: string;\n sdkVersion: string;\n modelBriefs: ModelBrief[];\n deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/\nReport dump\n/\nexport interface IReportActionDump {\n sdkVersion: string;\n groupName: string;\n groupDescription?: string;\n modelBriefs: ModelBrief[];\n executions: IExecutionDump[];\n deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n /\n The intent/category of the model call, for example \"planning\" or \"insight\".\n /\n intent?: string;\n\n /\n The model name returned by usage metadata, for example \"gpt-4o\".\n /\n name?: string;\n\n /\n Optional human-readable model description, for example \"qwen2.5-vl mode\".\n /\n modelDescription?: string;\n}\n\nexport type InterfaceType =\n \| 'puppeteer'\n \| 'playwright'\n \| 'static'\n \| 'chrome-extension-proxy'\n \| 'android'\n \| string;\n\nexport interface StreamingCodeGenerationOptions {\n /* Whether to enable streaming output /\n stream?: boolean;\n /* Callback function to handle streaming chunks /\n onChunk?: StreamingCallback;\n /* Callback function to handle streaming completion /\n onComplete?: (finalCode: string) => void;\n /* Callback function to handle streaming errors /\n onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n /* The incremental content chunk /\n content: string;\n /* The reasoning content /\n reasoning_content: string;\n /* The accumulated content so far /\n accumulated: string;\n /* Whether this is the final chunk /\n isComplete: boolean;\n /* Token usage information if available /\n usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n /* The final accumulated content /\n content: string;\n /* Token usage information /\n usage?: AIUsageInfo;\n /* Whether the response was streamed /\n isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n name: string;\n description?: string;\n interfaceAlias?: string;\n paramSchema?: z.ZodType<TParam>;\n call: (param: TParam, context: ExecutorContext) => Promise<TReturn> \| TReturn;\n delayBeforeRunner?: number;\n delayAfterRunner?: number;\n /\n An example param object for this action.\n * Locate fields with { prompt } will automatically get bbox injected when needed.\n /\n sample?: { [K in keyof TParam]?: any };\n}\n\n/\n Type utilities for extracting types from DeviceAction definitions\n /\n\n/\n Extract parameter type from a DeviceAction\n /\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<infer P, any> ? P : never;\n\n/\n Extract return type from a DeviceAction\n /\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<any, infer R> ? R : never;\n\n/\n Web-specific types\n /\nexport interface WebElementInfo extends BaseElement {\n id: string;\n attributes: {\n nodeType: NodeType;\n [key: string]: string;\n };\n}\n\n/\n Agent\n /\n\nexport type CacheConfig = {\n strategy?: 'read-only' \| 'read-write' \| 'write-only';\n id: string;\n};\n\nexport type Cache =\n \| false // No read, no write\n \| true // Will throw error at runtime - deprecated\n \| CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n // @deprecated Use `reportFileName` and `cache.id` instead.\n testId?: string;\n // @deprecated\n cacheId?: string; // Keep backward compatibility, but marked as deprecated\n groupName?: string;\n groupDescription?: string;\n / if auto generate report, default true /\n generateReport?: boolean;\n / if persist per-execution dump files next to the report, default false /\n persistExecutionDump?: boolean;\n / if auto print report msg, default true /\n autoPrintReportMsg?: boolean;\n\n /\n Use directory-based report format with separate image files.\n \n When enabled:\n * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n * - Report is generated as `index.html` with relative image paths\n * - Reduces memory usage and report file size\n \n IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n * work due to browser CORS restrictions.\n \n @default 'single-html'\n /\n outputFormat?: 'single-html' \| 'html-and-external-assets';\n\n onTaskStartTip?: OnTaskStartTip;\n aiActContext?: string;\n aiActionContext?: string;\n / custom report file name /\n reportFileName?: string;\n reportAttributes?: ReportAttributes;\n modelConfig?: TModelConfig;\n cache?: Cache;\n /\n Maximum number of replanning cycles for aiAct.\n * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n /\n replanningCycleLimit?: number;\n\n /\n Wait time in milliseconds after each action execution.\n * This allows the UI to settle and stabilize before the next action.\n * Defaults to 300ms when not provided.\n /\n waitAfterAction?: number;\n\n /\n When set to true, Midscene will use the target device's formatted local\n * time instead of the runtime system time. The target interface must implement\n * getDeviceLocalTimeString to provide device-local wall-clock time.\n * Default: false\n /\n useDeviceTime?: boolean;\n\n /\n Custom screenshot shrink factor to reduce AI token usage.\n * When set, the screenshot will be scaled down by this factor from the physical resolution.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - AI analyzes the 1500px screenshot\n * - Coordinates are transformed back to logical (500px) before actions execute\n \n Benefits:\n * - Reduces token usage for high-resolution screenshots\n * - Maintains accuracy by scaling coordinates appropriately\n \n Must be >= 1 (shrinking only, enlarging is not supported).\n \n @default 1 (no shrinking, uses original physical screenshot)\n /\n screenshotShrinkFactor?: number;\n\n /\n Custom OpenAI client factory function\n \n If provided, this function will be called to create OpenAI client instances\n * for each AI call, allowing you to:\n * - Wrap clients with observability tools (langsmith, langfuse)\n * - Use custom OpenAI-compatible clients\n * - Apply different configurations based on intent\n \n @param config - Resolved model configuration\n * @returns OpenAI client instance (original or wrapped)\n \n @example\n * ```typescript\n * createOpenAIClient: async (openai, opts) => {\n * // Wrap with langsmith for planning tasks\n * if (opts.baseURL?.includes('planning')) {\n * return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n * }\n \n return openai;\n * }\n * ```\n */\n createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n \| 'passed'\n \| 'failed'\n \| 'timedOut'\n \| 'skipped'\n \| 'interrupted';\n\nexport interface ReportFileAttributes {\n testDuration: number;\n testStatus: TestStatus;\n testTitle: string;\n testId: string;\n testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n \| {\n reportFilePath: string;\n reportAttributes: ReportFileAttributes;\n }\n \| {\n reportFilePath?: string;\n reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n };\n"],"names":["UIContext"],"mappings":";;;AAqHO,MAAeA;AA4BtB"}
1	+ {"version":3,"file":"types.mjs","sources":["../../src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any /\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n BaseElement,\n LocateResultElement,\n Rect,\n Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { ScreenshotItem } from './screenshot-item';\nimport type {\n DetailedLocateParam,\n MidsceneYamlFlowItem,\n ServiceExtractOption,\n} from './yaml';\n\nexport type {\n ElementTreeNode,\n BaseElement,\n Rect,\n Size,\n Point,\n} from '@midscene/shared/types';\nexport from './yaml';\n\nexport { ServiceError } from './errors';\nexport {\n ExecutionDump,\n ReportActionDump,\n GroupedActionDump,\n} from './dump/report-action-dump';\n\nexport type AIUsageInfo = Record<string, any> & {\n prompt_tokens: number \| undefined;\n completion_tokens: number \| undefined;\n total_tokens: number \| undefined;\n cached_input: number \| undefined;\n time_cost: number \| undefined;\n model_name: string \| undefined;\n model_description: string \| undefined;\n /*\n Semantic intent of the model call, such as default, planning, or insight.\n /\n intent: string \| undefined;\n /\n Config slot where the model config was resolved from. For example, a\n * planning call may use the default slot when no planning model is configured.\n /\n slot: string \| undefined;\n request_id: string \| undefined;\n};\n\nexport type { LocateResultElement };\n\nexport type AISingleElementResponseByPosition = {\n position?: {\n x: number;\n y: number;\n };\n bbox?: [number, number, number, number];\n reason: string;\n text: string;\n};\n\nexport interface AIElementCoordinatesResponse {\n bbox: [number, number, number, number];\n errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n data: DataDemand;\n errors?: string[];\n thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n bbox: [number, number, number, number];\n references_bbox?: [number, number, number, number][];\n error?: string;\n}\n\nexport interface AIAssertionResponse {\n pass: boolean;\n thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n description: string;\n error?: string;\n}\n\nexport interface LocatorValidatorOption {\n centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n pass: boolean;\n rect: Rect;\n center: [number, number];\n centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n prompt: string;\n deepLocate: boolean;\n verifyResult?: LocateValidatorResult;\n}\n\n/\n context\n /\n\nexport abstract class UIContext {\n /\n screenshot of the current UI state. which size is shotSize(be shrunk by screenshotShrinkFactor),\n /\n abstract screenshot: ScreenshotItem;\n\n /\n screenshot size after shrinking\n /\n abstract shotSize: Size;\n\n /\n The ratio for converting shrunk screenshot coordinates to logical coordinates.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - User-defined screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - shrunkShotToLogicalRatio: dpr / screenshotShrinkFactor = 6 / 2 = 3\n * - To map back to logical coordinates: 1500 / shrunkShotToLogicalRatio = 500px\n /\n abstract shrunkShotToLogicalRatio: number;\n\n abstract _isFrozen?: boolean;\n\n // @deprecated - backward compatibility for aiLocate\n abstract deprecatedDpr?: number;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' \| 'extract' \| 'assert' \| 'describe';\n\nexport type ServiceExtractParam = string \| Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n element: LocateResultElement \| null;\n rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' \| 'medium' \| 'high';\n\nexport type DeepThinkOption = 'unset' \| true \| false;\n\nexport interface ServiceTaskInfo {\n durationMs: number;\n formatResponse?: string;\n rawResponse?: string;\n usage?: AIUsageInfo;\n searchArea?: Rect;\n searchAreaRawResponse?: string;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\nexport interface DumpMeta {\n logTime: number;\n}\n\nexport type ReportAttributes = Record<\n string,\n string \| number \| boolean \| null \| undefined\n>;\n\nexport interface ReportDumpWithAttributes {\n dumpString: string;\n attributes?: ReportAttributes;\n}\n\nexport interface ServiceDump extends DumpMeta {\n type: 'locate' \| 'extract' \| 'assert';\n logId: string;\n userQuery: {\n element?: TUserPrompt;\n dataDemand?: ServiceExtractParam;\n assertion?: TUserPrompt;\n };\n matchedElement: LocateResultElement[];\n matchedRect?: Rect;\n deepLocate?: boolean;\n data: any;\n assertionPass?: boolean;\n assertionThought?: string;\n taskInfo: ServiceTaskInfo;\n error?: string;\n output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n ServiceDump,\n 'logTime' \| 'logId' \| 'model_name'\n>;\n\nexport interface ServiceResultBase {\n dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n data: T;\n thought?: string;\n usage?: AIUsageInfo;\n reasoning_content?: string;\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n name: string;\n description: string;\n sectionCharacteristics: string;\n textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement \| null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n usage?: AIUsageInfo;\n};\n\n/\n agent\n /\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> \| void;\n\nexport interface AgentWaitForOpt extends ServiceExtractOption {\n checkIntervalMs?: number;\n timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n keepRawResponse?: boolean;\n}\n\n/\n planning\n \n /\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n thought?: string;\n log?: string; // a brief preamble to the user explaining what you’re about to do\n type: string;\n param: ParamType;\n}\n\nexport type SubGoalStatus = 'pending' \| 'running' \| 'finished';\n\nexport interface SubGoal {\n index: number;\n status: SubGoalStatus;\n description: string;\n logs?: string[];\n}\n\nexport interface RawResponsePlanningAIResponse {\n action: PlanningAction;\n thought?: string;\n log: string;\n memory?: string;\n error?: string;\n finalizeMessage?: string;\n finalizeSuccess?: boolean;\n updateSubGoals?: SubGoal[];\n markFinishedIndexes?: number[];\n}\n\nexport interface PlanningAIResponse\n extends Omit<RawResponsePlanningAIResponse, 'action'> {\n actions?: PlanningAction[];\n usage?: AIUsageInfo;\n rawResponse?: string;\n yamlFlow?: MidsceneYamlFlowItem[];\n yamlString?: string;\n error?: string;\n reasoning_content?: string;\n shouldContinuePlanning: boolean;\n output?: string; // Output message from <complete> tag (same as finalizeMessage)\n}\n\nexport interface PlanningActionParamSleep {\n timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n duration?: number;\n}\n\nexport interface PullParam {\n direction: 'up' \| 'down';\n distance?: number;\n duration?: number;\n}\n/*\n misc\n /\n\nexport interface Color {\n name: string;\n hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/\naction\n/\nexport interface ExecutionTaskProgressOptions {\n onTaskStart?: (task: ExecutionTask) => Promise<void> \| void;\n}\n\nexport interface ExecutionRecorderItem {\n type: 'screenshot';\n ts: number;\n screenshot?: ScreenshotItem;\n timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' \| 'Insight' \| 'Action Space' \| 'Log';\n\nexport interface ExecutorContext {\n task: ExecutionTask;\n element?: LocateResultElement \| null;\n uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n Type extends ExecutionTaskType = any,\n TaskParam = any,\n TaskOutput = any,\n TaskLog = any,\n> {\n type: Type;\n subType?: string;\n param?: TaskParam;\n thought?: string;\n uiContext?: UIContext;\n executor: (\n param: TaskParam,\n context: ExecutorContext,\n ) => // biome-ignore lint/suspicious/noConfusingVoidType: void is intentionally allowed as some executors may not return a value\n \| Promise<ExecutionTaskReturn<TaskOutput, TaskLog> \| undefined \| void>\n \| undefined\n \| void;\n}\n\nexport interface ExecutionTaskHitBy {\n from: string;\n context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n output?: TaskOutput;\n log?: TaskLog;\n recorder?: ExecutionRecorderItem[];\n hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n any,\n any,\n any\n >,\n> = E &\n ExecutionTaskReturn<\n E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n ? TaskOutput\n : unknown,\n E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n ? TaskLog\n : unknown\n > & {\n taskId: string;\n status: 'pending' \| 'running' \| 'finished' \| 'failed' \| 'cancelled';\n error?: Error;\n errorMessage?: string;\n errorStack?: string;\n timing?: {\n start: number;\n getUiContextStart?: number;\n getUiContextEnd?: number;\n callAiStart?: number;\n callAiEnd?: number;\n beforeInvokeActionHookStart?: number;\n beforeInvokeActionHookEnd?: number;\n callActionStart?: number;\n callActionEnd?: number;\n afterInvokeActionHookStart?: number;\n afterInvokeActionHookEnd?: number;\n captureAfterCallingSnapshotStart?: number;\n captureAfterCallingSnapshotEnd?: number;\n end?: number;\n cost?: number;\n };\n usage?: AIUsageInfo;\n searchAreaUsage?: AIUsageInfo;\n reasoning_content?: string;\n };\n\nexport interface IExecutionDump extends DumpMeta {\n /* Stable unique identifier for this execution run /\n id?: string;\n name: string;\n description?: string;\n tasks: ExecutionTask[];\n aiActContext?: string;\n}\n\n/\ntask - service-locate\n/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightLocateParam,\n ExecutionTaskInsightLocateOutput,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/\ntask - service-query\n/\nexport interface ExecutionTaskInsightQueryParam {\n dataDemand: ServiceExtractParam;\n domIncluded?: boolean \| 'visible-only';\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightQueryParam,\n any,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/\ntask - assertion\n/\nexport interface ExecutionTaskInsightAssertionParam {\n assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n 'Insight',\n ExecutionTaskInsightAssertionParam,\n ServiceAssertionResponse,\n ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/\ntask - action (i.e. interact) \n/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n 'Action Space',\n ActionParam,\n void,\n void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/\ntask - Log\n/\n\nexport type ExecutionTaskLogApply<\n LogParam = {\n content: string;\n },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/\ntask - planning\n/\n\nexport interface ExecutionTaskPlanningParam {\n userInstruction: string;\n aiActContext?: string;\n imagesIncludeCount?: number;\n deepThink?: DeepThinkOption;\n subGoalStatus?: string;\n memoriesStatus?: string;\n}\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n 'Planning',\n ExecutionTaskPlanningParam,\n PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/\ntask - planning-locate\n/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n element: LocateResultElement \| null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n 'Planning',\n ExecutionTaskPlanningLocateParam,\n ExecutionTaskPlanningLocateOutput,\n ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/\nReport metadata - extracted from ReportActionDump for per-execution writes\n/\nexport interface ReportMeta {\n groupName: string;\n groupDescription?: string;\n sdkVersion: string;\n modelBriefs: ModelBrief[];\n deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type GroupMeta = ReportMeta;\n\n/\nReport dump\n/\nexport interface IReportActionDump {\n sdkVersion: string;\n groupName: string;\n groupDescription?: string;\n modelBriefs: ModelBrief[];\n executions: IExecutionDump[];\n deviceType?: string;\n}\n\n// Backward-compatible aliases for existing external consumers.\nexport type IGroupedActionDump = IReportActionDump;\n\nexport interface ModelBrief {\n /\n The intent/category of the model call, for example \"planning\" or \"insight\".\n /\n intent?: string;\n\n /\n The model name returned by usage metadata, for example \"gpt-4o\".\n /\n name?: string;\n\n /\n Optional human-readable model description, for example \"qwen2.5-vl mode\".\n /\n modelDescription?: string;\n}\n\nexport type InterfaceType =\n \| 'puppeteer'\n \| 'playwright'\n \| 'static'\n \| 'chrome-extension-proxy'\n \| 'android'\n \| string;\n\nexport interface StreamingCodeGenerationOptions {\n /* Whether to enable streaming output /\n stream?: boolean;\n /* Callback function to handle streaming chunks /\n onChunk?: StreamingCallback;\n /* Callback function to handle streaming completion /\n onComplete?: (finalCode: string) => void;\n /* Callback function to handle streaming errors /\n onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n /* The incremental content chunk /\n content: string;\n /* The reasoning content /\n reasoning_content: string;\n /* The accumulated content so far /\n accumulated: string;\n /* Whether this is the final chunk /\n isComplete: boolean;\n /* Token usage information if available /\n usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n /* The final accumulated content /\n content: string;\n /* Token usage information /\n usage?: AIUsageInfo;\n /* Whether the response was streamed /\n isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n name: string;\n description?: string;\n interfaceAlias?: string;\n paramSchema?: z.ZodType<TParam>;\n call: (param: TParam, context: ExecutorContext) => Promise<TReturn> \| TReturn;\n delayBeforeRunner?: number;\n delayAfterRunner?: number;\n /\n An example param object for this action.\n * Locate fields with { prompt } will automatically get bbox injected when needed.\n /\n sample?: { [K in keyof TParam]?: any };\n}\n\n/\n Type utilities for extracting types from DeviceAction definitions\n /\n\n/\n Extract parameter type from a DeviceAction\n /\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<infer P, any> ? P : never;\n\n/\n Extract return type from a DeviceAction\n /\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n Action extends DeviceAction<any, infer R> ? R : never;\n\n/\n Web-specific types\n /\nexport interface WebElementInfo extends BaseElement {\n id: string;\n attributes: {\n nodeType: NodeType;\n [key: string]: string;\n };\n}\n\n/\n Agent\n /\n\nexport type CacheConfig = {\n strategy?: 'read-only' \| 'read-write' \| 'write-only';\n id: string;\n};\n\nexport type Cache =\n \| false // No read, no write\n \| true // Will throw error at runtime - deprecated\n \| CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n // @deprecated Use `reportFileName` and `cache.id` instead.\n testId?: string;\n // @deprecated\n cacheId?: string; // Keep backward compatibility, but marked as deprecated\n groupName?: string;\n groupDescription?: string;\n / if auto generate report, default true /\n generateReport?: boolean;\n / if persist per-execution dump files next to the report, default false /\n persistExecutionDump?: boolean;\n / if auto print report msg, default true /\n autoPrintReportMsg?: boolean;\n\n /\n Use directory-based report format with separate image files.\n \n When enabled:\n * - Screenshots are saved as PNG files in a `screenshots/` subdirectory\n * - Report is generated as `index.html` with relative image paths\n * - Reduces memory usage and report file size\n \n IMPORTANT: 'html-and-external-assets' reports must be served via HTTP server\n * (e.g., `npx serve ./report-dir`). The file:// protocol will not\n * work due to browser CORS restrictions.\n \n @default 'single-html'\n /\n outputFormat?: 'single-html' \| 'html-and-external-assets';\n\n onTaskStartTip?: OnTaskStartTip;\n aiActContext?: string;\n aiActionContext?: string;\n / custom report file name /\n reportFileName?: string;\n reportAttributes?: ReportAttributes;\n modelConfig?: TModelConfig;\n cache?: Cache;\n /\n Maximum number of replanning cycles for aiAct.\n * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n /\n replanningCycleLimit?: number;\n\n /\n Wait time in milliseconds after each action execution.\n * This allows the UI to settle and stabilize before the next action.\n * Defaults to 300ms when not provided.\n /\n waitAfterAction?: number;\n\n /\n When set to true, Midscene will use the target device's formatted local\n * time instead of the runtime system time. The target interface must implement\n * getDeviceLocalTimeString to provide device-local wall-clock time.\n * Default: false\n /\n useDeviceTime?: boolean;\n\n /\n Custom screenshot shrink factor to reduce AI token usage.\n * When set, the screenshot will be scaled down by this factor from the physical resolution.\n \n Example:\n * - Physical screen width: 3000px, dpr=6\n * - Logical width: 500px\n * - screenshotShrinkFactor: 2\n * - Actual shrunk screenshot width: 3000 / 2 = 1500px\n * - AI analyzes the 1500px screenshot\n * - Coordinates are transformed back to logical (500px) before actions execute\n \n Benefits:\n * - Reduces token usage for high-resolution screenshots\n * - Maintains accuracy by scaling coordinates appropriately\n \n Must be >= 1 (shrinking only, enlarging is not supported).\n \n @default 1 (no shrinking, uses original physical screenshot)\n /\n screenshotShrinkFactor?: number;\n\n /\n Custom OpenAI client factory function\n \n If provided, this function will be called to create OpenAI client instances\n * for each AI call, allowing you to:\n * - Wrap clients with observability tools (langsmith, langfuse)\n * - Use custom OpenAI-compatible clients\n * - Apply different configurations based on intent\n \n @param config - Resolved model configuration\n * @returns OpenAI client instance (original or wrapped)\n \n @example\n * ```typescript\n * createOpenAIClient: async (openai, opts) => {\n * // Wrap with langsmith for planning tasks\n * if (opts.baseURL?.includes('planning')) {\n * return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n * }\n \n return openai;\n * }\n * ```\n */\n createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n \| 'passed'\n \| 'failed'\n \| 'timedOut'\n \| 'skipped'\n \| 'interrupted';\n\nexport interface ReportFileAttributes {\n testDuration: number;\n testStatus: TestStatus;\n testTitle: string;\n testId: string;\n testDescription: string;\n}\n\nexport type ReportFileWithAttributes =\n \| {\n reportFilePath: string;\n reportAttributes: ReportFileAttributes;\n }\n \| {\n reportFilePath?: string;\n reportAttributes: ReportFileAttributes & { testStatus: 'skipped' };\n };\n"],"names":["UIContext"],"mappings":";;;AAqHO,MAAeA;AA4BtB"}