@midscene/core 1.0.1-beta-20251205094204.0 → 1.0.1-beta-20251208031823.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (9) hide show
  1. package/dist/es/agent/task-builder.mjs +26 -23
  2. package/dist/es/agent/task-builder.mjs.map +1 -1
  3. package/dist/es/agent/utils.mjs +1 -1
  4. package/dist/es/utils.mjs +2 -2
  5. package/dist/lib/agent/task-builder.js +26 -23
  6. package/dist/lib/agent/task-builder.js.map +1 -1
  7. package/dist/lib/agent/utils.js +1 -1
  8. package/dist/lib/utils.js +2 -2
  9. package/package.json +2 -2
package/dist/es/agent/task-builder.mjs CHANGED
@@ -16,6 +16,9 @@ function _define_property(obj, key, value) {
16
16
  return obj;
17
17
  }
18
18
  const debug = getDebug('agent:task-builder');
19
+ function hasNonEmptyCache(cache) {
20
+ return null != cache && 'object' == typeof cache && Object.keys(cache).length > 0;
21
+ }
19
22
  function locatePlanForLocate(param) {
20
23
  const locate = 'string' == typeof param ? {
21
24
  prompt: param
@@ -103,19 +106,14 @@ class TaskBuilder {
103
106
  const locateFields = action ? findAllMidsceneLocatorField(action.paramSchema) : [];
104
107
  const requiredLocateFields = action ? findAllMidsceneLocatorField(action.paramSchema, true) : [];
105
108
  locateFields.forEach((field)=>{
106
- if (param[field]) if (ifPlanLocateParamIsBbox(param[field])) {
107
- debug('plan locate param is bbox, will match element from plan', param[field]);
108
- const elementFromPlan = matchElementFromPlan(param[field]);
109
- if (elementFromPlan) param[field] = elementFromPlan;
110
- } else {
109
+ if (param[field]) {
111
110
  const locatePlan = locatePlanForLocate(param[field]);
112
- debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`);
111
+ debug('will prepend locate param for field', `action.type=${planType}`, `param=${JSON.stringify(param[field])}`, `locatePlan=${JSON.stringify(locatePlan)}`, `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`);
113
112
  const locateTask = this.createLocateTask(locatePlan, param[field], context, (result)=>{
114
113
  param[field] = result;
115
114
  });
116
115
  context.tasks.push(locateTask);
117
- }
118
- else {
116
+ } else {
119
117
  assert(!requiredLocateFields.includes(field), `Required locate field '${field}' is not provided for action ${planType}`);
120
118
  debug(`field '${field}' is not provided for action ${planType}`);
121
119
  }
@@ -185,7 +183,6 @@ class TaskBuilder {
185
183
  createLocateTask(plan, detailedLocateParam, context, onResult) {
186
184
  const { cacheable, modelConfigForDefaultIntent } = context;
187
185
  let locateParam = detailedLocateParam;
188
- console.log('modelConfigForDefaultIntent-locateParam', modelConfigForDefaultIntent, locateParam);
189
186
  if ('string' == typeof locateParam) locateParam = {
190
187
  prompt: locateParam
191
188
  };
@@ -216,32 +213,31 @@ class TaskBuilder {
216
213
  task.usage = dump.taskInfo?.usage;
217
214
  if (dump.taskInfo?.searchAreaUsage) task.searchAreaUsage = dump.taskInfo.searchAreaUsage;
218
215
  };
216
+ const elementFromBbox = ifPlanLocateParamIsBbox(param) ? matchElementFromPlan(param) : void 0;
217
+ const isPlanHit = !!elementFromBbox;
219
218
  let rectFromXpath;
220
- if (param.xpath && this.interface.rectMatchesCacheFeature) try {
219
+ if (!isPlanHit && param.xpath && this.interface.rectMatchesCacheFeature) try {
221
220
  rectFromXpath = await this.interface.rectMatchesCacheFeature({
222
221
  xpaths: [
223
222
  param.xpath
224
223
  ]
225
224
  });
226
- } catch (error) {
227
- rectFromXpath = void 0;
228
- }
225
+ } catch {}
229
226
  const elementFromXpath = rectFromXpath ? generateElementByPosition({
230
227
  x: rectFromXpath.left + rectFromXpath.width / 2,
231
228
  y: rectFromXpath.top + rectFromXpath.height / 2
232
229
  }, 'string' == typeof param.prompt ? param.prompt : param.prompt?.prompt || '') : void 0;
233
- const userExpectedPathHitFlag = !!elementFromXpath;
230
+ const isXpathHit = !!elementFromXpath;
234
231
  const cachePrompt = param.prompt;
235
232
  const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);
236
233
  const cacheEntry = locateCacheRecord?.cacheContent?.cache;
237
- const elementFromCache = userExpectedPathHitFlag ? null : await matchElementFromCache({
234
+ const elementFromCache = isPlanHit || isXpathHit ? null : await matchElementFromCache({
238
235
  taskCache: this.taskCache,
239
236
  interfaceInstance: this.interface
240
237
  }, cacheEntry, cachePrompt, param.cacheable);
241
- const cacheHitFlag = !!elementFromCache;
242
- const planHitFlag = false;
238
+ const isCacheHit = !!elementFromCache;
243
239
  let elementFromAiLocate;
244
- if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) try {
240
+ if (!isXpathHit && !isCacheHit && !isPlanHit) try {
245
241
  locateResult = await this.service.locate(param, {
246
242
  context: uiContext
247
243
  }, modelConfigForDefaultIntent);
@@ -251,14 +247,15 @@ class TaskBuilder {
251
247
  if (error instanceof ServiceError) applyDump(error.dump);
252
248
  throw error;
253
249
  }
254
- const element = elementFromXpath || elementFromCache || elementFromAiLocate;
250
+ const element = elementFromBbox || elementFromXpath || elementFromCache || elementFromAiLocate;
251
+ const locateCacheAlreadyExists = hasNonEmptyCache(locateCacheRecord?.cacheContent?.cache);
255
252
  let currentCacheEntry;
256
- if (element && this.taskCache && !cacheHitFlag && param?.cacheable !== false) if (this.interface.cacheFeatureForRect) try {
253
+ if (element && this.taskCache && !isCacheHit && !locateCacheAlreadyExists && param?.cacheable !== false) if (this.interface.cacheFeatureForRect) try {
257
254
  const feature = await this.interface.cacheFeatureForRect(element.rect, {
258
255
  targetDescription: 'string' == typeof param.prompt ? param.prompt : param.prompt?.prompt,
259
256
  modelConfig: modelConfigForDefaultIntent
260
257
  });
261
- if (feature && Object.keys(feature).length > 0) {
258
+ if (hasNonEmptyCache(feature)) {
262
259
  debug('update cache, prompt: %s, cache: %o', cachePrompt, feature);
263
260
  currentCacheEntry = feature;
264
261
  this.taskCache.updateOrAppendCacheRecord({
@@ -276,13 +273,19 @@ class TaskBuilder {
276
273
  throw new Error(`Element not found: ${param.prompt}`);
277
274
  }
278
275
  let hitBy;
279
- if (userExpectedPathHitFlag) hitBy = {
276
+ if (isPlanHit) hitBy = {
277
+ from: 'Plan',
278
+ context: {
279
+ bbox: param.bbox
280
+ }
281
+ };
282
+ else if (isXpathHit) hitBy = {
280
283
  from: 'User expected path',
281
284
  context: {
282
285
  xpath: param.xpath
283
286
  }
284
287
  };
285
- else if (cacheHitFlag) hitBy = {
288
+ else if (isCacheHit) hitBy = {
286
289
  from: 'Cache',
287
290
  context: {
288
291
  cacheEntry,
package/dist/es/agent/task-builder.mjs.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type {\n DetailedLocateParam,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningActionParamSleep,\n PlanningLocateParam,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n ifPlanLocateParamIsBbox,\n matchElementFromCache,\n matchElementFromPlan,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n const locate = typeof param === 'string' ? { prompt: param } : param;\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n subTask?: boolean;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n modelConfigForPlanning: IModelConfig;\n modelConfigForDefaultIntent: IModelConfig;\n cacheable?: boolean;\n subTask: boolean;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n constructor({ interfaceInstance, service, taskCache }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n }\n\n public async build(\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n cacheable,\n subTask: !!options?.subTask,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n [\n 'Sleep',\n (plan) =>\n this.handleSleepPlan(\n plan as PlanningAction<PlanningActionParamSleep>,\n context,\n ),\n ],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n subTask: context.subTask || undefined,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private handleSleepPlan(\n plan: PlanningAction<PlanningActionParamSleep>,\n context: PlanBuildContext,\n ): void {\n const sleepTask = this.createSleepTask(plan.param, {\n thought: plan.thought,\n });\n if (context.subTask) {\n sleepTask.subTask = true;\n }\n context.tasks.push(sleepTask);\n }\n\n public createSleepTask(\n param: PlanningActionParamSleep,\n meta?: { thought?: string },\n ): ExecutionTaskActionApply<PlanningActionParamSleep> {\n return {\n type: 'Action Space',\n subType: 'Sleep',\n param,\n thought: meta?.thought,\n executor: async (taskParam) => {\n await sleep(taskParam?.timeMs || 3000);\n },\n };\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = await this.interface.actionSpace();\n const action = actionSpace.find((item) => item.name === planType);\n const param = plan.param;\n\n if (!action) {\n throw new Error(`Action type '${planType}' not found`);\n }\n\n const locateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema)\n : [];\n\n const requiredLocateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema, true)\n : [];\n\n locateFields.forEach((field) => {\n if (param[field]) {\n if (ifPlanLocateParamIsBbox(param[field])) {\n debug(\n 'plan locate param is bbox, will match element from plan',\n param[field],\n );\n const elementFromPlan = matchElementFromPlan(param[field]);\n if (elementFromPlan) {\n param[field] = elementFromPlan;\n }\n } else {\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n }\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n subTask: context.subTask || undefined,\n executor: async (param, taskContext) => {\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug('will call \"beforeInvokeAction\" for interface');\n await this.interface.beforeInvokeAction(action.name, param);\n debug('called \"beforeInvokeAction\" for interface');\n }\n })(),\n sleep(200),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message || String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema);\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n debug('called action', action.name, 'result:', actionResult);\n\n const delayAfterRunner = action.delayAfterRunner ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug('will call \"afterInvokeAction\" for interface');\n await this.interface.afterInvokeAction(action.name, param);\n debug('called \"afterInvokeAction\" for interface');\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message || String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam | string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, modelConfigForDefaultIntent } = context;\n\n let locateParam = detailedLocateParam;\n console.log(\n 'modelConfigForDefaultIntent-locateParam',\n modelConfigForDefaultIntent,\n locateParam,\n );\n\n if (typeof locateParam === 'string') {\n locateParam = {\n prompt: locateParam,\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n subTask: context.subTask || undefined,\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n\n assert(\n param?.prompt || param?.bbox,\n `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n let locateDump: ServiceDump | undefined;\n let locateResult: LocateResultWithDump | undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n };\n task.usage = dump.taskInfo?.usage;\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n };\n\n // from xpath\n let rectFromXpath: Rect | undefined;\n if (param.xpath && this.interface.rectMatchesCacheFeature) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch (error) {\n // xpath locate failed, allow fallback to cache or AI locate\n rectFromXpath = undefined;\n }\n }\n const elementFromXpath = rectFromXpath\n ? generateElementByPosition(\n {\n x: rectFromXpath.left + rectFromXpath.width / 2,\n y: rectFromXpath.top + rectFromXpath.height / 2,\n },\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt || '',\n )\n : undefined;\n const userExpectedPathHitFlag = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCache = userExpectedPathHitFlag\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n const cacheHitFlag = !!elementFromCache;\n\n const planHitFlag = false;\n\n let elementFromAiLocate: LocateResultElement | null | undefined;\n if (!userExpectedPathHitFlag && !cacheHitFlag && !planHitFlag) {\n try {\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n },\n modelConfigForDefaultIntent,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n }\n }\n\n const element =\n elementFromXpath || elementFromCache || elementFromAiLocate;\n\n let currentCacheEntry: ElementCacheFeature | undefined;\n if (\n element &&\n this.taskCache &&\n !cacheHitFlag &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForRect) {\n try {\n const feature = await this.interface.cacheFeatureForRect(\n element.rect,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelConfig: modelConfigForDefaultIntent,\n },\n );\n if (feature && Object.keys(feature).length > 0) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForRect failed: %s', error);\n }\n } else {\n debug('cacheFeatureForRect is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy | undefined;\n\n if (userExpectedPathHitFlag) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (cacheHitFlag) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element,\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","undefined","sleepTask","meta","taskParam","sleep","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamIsBbox","elementFromPlan","matchElementFromPlan","JSON","locateTask","result","assert","task","taskContext","uiContext","Promise","originalError","originalMessage","String","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","console","taskLocator","locateDump","locateResult","applyDump","dump","rectFromXpath","elementFromXpath","generateElementByPosition","userExpectedPathHitFlag","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","cacheHitFlag","planHitFlag","elementFromAiLocate","ServiceError","element","currentCacheEntry","feature","Object","hitBy","interfaceInstance","service","taskCache"],"mappings":";;;;;;;;;;;;;;;;;AA+BA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAqBO,MAAMC;IAaX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,SAAS,CAAC,CAACF,SAAS;QACtB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;YAC9D;gBACE;gBACA,CAACG,OACC,IAAI,CAAC,eAAe,CAClBA,MACAH;aAEL;SACF;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,WAAa;QACzB;QACAP,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEQ,gBACNH,IAA8C,EAC9CH,OAAyB,EACnB;QACN,MAAMQ,YAAY,IAAI,CAAC,eAAe,CAACL,KAAK,KAAK,EAAE;YACjD,SAASA,KAAK,OAAO;QACvB;QACA,IAAIH,QAAQ,OAAO,EACjBQ,UAAU,OAAO,GAAG;QAEtBR,QAAQ,KAAK,CAAC,IAAI,CAACQ;IACrB;IAEO,gBACLlB,KAA+B,EAC/BmB,IAA2B,EACyB;QACpD,OAAO;YACL,MAAM;YACN,SAAS;YACTnB;YACA,SAASmB,MAAM;YACf,UAAU,OAAOC;gBACf,MAAMC,MAAMD,WAAW,UAAU;YACnC;QACF;IACF;IAEA,MAAc,iBACZP,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMY,aAAa,IAAI,CAAC,gBAAgB,CAACT,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACY;IACrB;IAEA,MAAc,iBACZT,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMa,WAAWV,KAAK,IAAI;QAC1B,MAAMW,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;QACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMvB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACY,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI/B,KAAK,CAAC+B,MAAM,EACd,IAAIC,wBAAwBhC,KAAK,CAAC+B,MAAM,GAAG;gBACzClC,MACE,2DACAG,KAAK,CAAC+B,MAAM;gBAEd,MAAME,kBAAkBC,qBAAqBlC,KAAK,CAAC+B,MAAM;gBACzD,IAAIE,iBACFjC,KAAK,CAAC+B,MAAM,GAAGE;YAEnB,OAAO;gBACL,MAAM/B,aAAaH,oBAAoBC,KAAK,CAAC+B,MAAM;gBACnDlC,MACE,uCACA,CAAC,YAAY,EAAE0B,UAAU,EACzB,CAAC,MAAM,EAAEY,KAAK,SAAS,CAACnC,KAAK,CAAC+B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEI,KAAK,SAAS,CAACjC,aAAa;gBAE5C,MAAMkC,aAAa,IAAI,CAAC,gBAAgB,CACtClC,YACAF,KAAK,CAAC+B,MAAM,EACZrB,SACA,CAAC2B;oBACCrC,KAAK,CAAC+B,MAAM,GAAGM;gBACjB;gBAEF3B,QAAQ,KAAK,CAAC,IAAI,CAAC0B;YACrB;iBACK;gBACLE,OACE,CAACR,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3E1B,MAAM,CAAC,OAAO,EAAEkC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMgB,OAKF;YACF,MAAM;YACN,SAAShB;YACT,SAASV,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,OAAOjB,OAAOwC;gBACtB3C,MACE,oBACA0B,UACAvB,OACA,CAAC,4BAA4B,EAAEwC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMC,YAAYD,YAAY,SAAS;gBACvCF,OAAOG,WAAW;gBAElBX,qBAAqB,OAAO,CAAC,CAACC;oBAC5BO,OACEtC,KAAK,CAAC+B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEA,IAAI;oBACF,MAAMmB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC7C,MAAM;gCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC4B,OAAO,IAAI,EAAEzB;gCACrDH,MAAM;4BACR;wBACF;wBACAwB,MAAM;qBACP;gBACH,EAAE,OAAOsB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIhB,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,IAAIlB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQ8C,iBAAiB9C,OAAOyB,OAAO,WAAW;gBACpD,EAAE,OAAOsB,OAAY;oBACnB,MAAM,IAAIpB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEsB,MAAM,OAAO,CAAC,cAAc,EAAEZ,KAAK,SAAS,CAACnC,QAAQ,EACtG;wBAAE,OAAO+C;oBAAM;gBAEnB;gBAGFlD,MAAM,kBAAkB4B,OAAO,IAAI;gBACnC,MAAMuB,WAAWvB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAMwB,eAAe,MAAMD,SAAShD,OAAOwC;gBAC3C3C,MAAM,iBAAiB4B,OAAO,IAAI,EAAE,WAAWwB;gBAE/C,MAAMC,mBAAmBzB,OAAO,gBAAgB,IAAI;gBACpD,IAAIyB,mBAAmB,GACrB,MAAM7B,MAAM6B;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCrD,MAAM;wBACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC4B,OAAO,IAAI,EAAEzB;wBACpDH,MAAM;oBACR;gBACF,EAAE,OAAO8C,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIhB,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,OAAO;oBACL,QAAQM;gBACV;YACF;QACF;QAEAvC,QAAQ,KAAK,CAAC,IAAI,CAAC6B;IACrB;IAEQ,iBACN1B,IAAyC,EACzCsC,mBAAiD,EACjDzC,OAAyB,EACzB0C,QAAgD,EACd;QAClC,MAAM,EAAE3C,SAAS,EAAEH,2BAA2B,EAAE,GAAGI;QAEnD,IAAI2C,cAAcF;QAClBG,QAAQ,GAAG,CACT,2CACAhD,6BACA+C;QAGF,IAAI,AAAuB,YAAvB,OAAOA,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI5C,AAAcQ,WAAdR,WACF4C,cAAc;YACZ,GAAGA,WAAW;YACd5C;QACF;QAGF,MAAM8C,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,SAAS7C,QAAQ,OAAO,IAAIO;YAC5B,OAAOoC;YACP,SAASxC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOwC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEC,SAAS,EAAE,GAAGD;gBAEpBF,OACEtC,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAEmC,KAAK,SAAS,CACpEnC,QACC;gBAGL,IAAI,CAACyC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDH,OAAOG,WAAW;gBAElB,IAAIe;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbpB,KAAK,GAAG,GAAG;wBACToB;oBACF;oBACApB,KAAK,KAAK,GAAGoB,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjBpB,KAAK,eAAe,GAAGoB,KAAK,QAAQ,CAAC,eAAe;gBAExD;gBAGA,IAAIC;gBACJ,IAAI5D,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,uBAAuB,EACvD,IAAI;oBACF4D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC5D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAO+C,OAAO;oBAEda,gBAAgB3C;gBAClB;gBAEF,MAAM4C,mBAAmBD,gBACrBE,0BACE;oBACE,GAAGF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;oBAC9C,GAAGA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;gBAChD,GACA,AAAwB,YAAxB,OAAO5D,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BiB;gBACJ,MAAM8C,0BAA0B,CAAC,CAACF;gBAElC,MAAMG,cAAchE,MAAM,MAAM;gBAChC,MAAMiE,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,mBAAmBJ,0BACrB,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhE,MAAM,SAAS;gBAErB,MAAMqE,eAAe,CAAC,CAACF;gBAEvB,MAAMG,cAAc;gBAEpB,IAAIC;gBACJ,IAAI,CAACR,2BAA2B,CAACM,gBAAgB,CAACC,aAChD,IAAI;oBACFb,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCzD,OACA;wBACE,SAASyC;oBACX,GACAnC;oBAEFoD,UAAUD,aAAa,IAAI;oBAC3Bc,sBAAsBd,aAAa,OAAO;gBAC5C,EAAE,OAAOV,OAAO;oBACd,IAAIA,iBAAiByB,cACnBd,UAAUX,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAGF,MAAM0B,UACJZ,oBAAoBM,oBAAoBI;gBAE1C,IAAIG;gBACJ,IACED,WACA,IAAI,CAAC,SAAS,IACd,CAACJ,gBACDrE,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;oBACF,MAAM2E,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDF,QAAQ,IAAI,EACZ;wBACE,mBACE,AAAwB,YAAxB,OAAOzE,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIqE,WAAWC,OAAO,IAAI,CAACD,SAAS,MAAM,GAAG,GAAG;wBAC9C9E,MACE,uCACAmE,aACAW;wBAEFD,oBAAoBC;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQX;4BACR,OAAOW;wBACT,GACAV;oBAEJ,OACEpE,MACE,yDACAmE;gBAGN,EAAE,OAAOjB,OAAO;oBACdlD,MAAM,kCAAkCkD;gBAC1C;qBAEAlD,MAAM;gBAIV,IAAI,CAAC4E,SAAS;oBACZ,IAAIjB,YACF,MAAM,IAAIgB,aACR,CAAC,oBAAoB,EAAExE,MAAM,MAAM,EAAE,EACrCwD;oBAGJ,MAAM,IAAI7B,MAAM,CAAC,mBAAmB,EAAE3B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAI6E;gBAEJ,IAAId,yBACFc,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO7E,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIqE,cACTQ,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPX;wBACA,aAAaQ;oBACf;gBACF;gBAGFtB,WAAWqB;gBAEX,OAAO;oBACL,QAAQ;wBACNA;oBACF;oBACAI;gBACF;YACF;QACF;QAEA,OAAOtB;IACT;IAteA,YAAY,EAAEuB,iBAAiB,EAAEC,OAAO,EAAEC,SAAS,EAAmB,CAAE;QANxE,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAGE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;IACnB;AAmeF"}
1
+ {"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport type {\n DetailedLocateParam,\n ElementCacheFeature,\n ExecutionTaskActionApply,\n ExecutionTaskApply,\n ExecutionTaskHitBy,\n ExecutionTaskPlanningLocateApply,\n LocateResultElement,\n LocateResultWithDump,\n PlanningAction,\n PlanningActionParamSleep,\n PlanningLocateParam,\n Rect,\n ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport {\n ifPlanLocateParamIsBbox,\n matchElementFromCache,\n matchElementFromPlan,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n return (\n cache !== null &&\n cache !== undefined &&\n typeof cache === 'object' &&\n Object.keys(cache).length > 0\n );\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n const locate = typeof param === 'string' ? { prompt: param } : param;\n const locatePlan: PlanningAction<PlanningLocateParam> = {\n type: 'Locate',\n param: locate,\n thought: '',\n };\n return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n interfaceInstance: AbstractInterface;\n service: Service;\n taskCache?: TaskCache;\n}\n\ninterface BuildOptions {\n cacheable?: boolean;\n subTask?: boolean;\n}\n\ninterface PlanBuildContext {\n tasks: ExecutionTaskApply[];\n modelConfigForPlanning: IModelConfig;\n modelConfigForDefaultIntent: IModelConfig;\n cacheable?: boolean;\n subTask: boolean;\n}\n\nexport class TaskBuilder {\n private readonly interface: AbstractInterface;\n\n private readonly service: Service;\n\n private readonly taskCache?: TaskCache;\n\n constructor({ interfaceInstance, service, taskCache }: TaskBuilderDeps) {\n this.interface = interfaceInstance;\n this.service = service;\n this.taskCache = taskCache;\n }\n\n public async build(\n plans: PlanningAction[],\n modelConfigForPlanning: IModelConfig,\n modelConfigForDefaultIntent: IModelConfig,\n options?: BuildOptions,\n ): Promise<{ tasks: ExecutionTaskApply[] }> {\n const tasks: ExecutionTaskApply[] = [];\n const cacheable = options?.cacheable;\n\n const context: PlanBuildContext = {\n tasks,\n modelConfigForPlanning,\n modelConfigForDefaultIntent,\n cacheable,\n subTask: !!options?.subTask,\n };\n\n type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n const planHandlers = new Map<string, PlanHandler>([\n [\n 'Locate',\n (plan) =>\n this.handleLocatePlan(\n plan as PlanningAction<PlanningLocateParam>,\n context,\n ),\n ],\n ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n [\n 'Sleep',\n (plan) =>\n this.handleSleepPlan(\n plan as PlanningAction<PlanningActionParamSleep>,\n context,\n ),\n ],\n ]);\n\n const defaultHandler: PlanHandler = (plan) =>\n this.handleActionPlan(plan, context);\n\n for (const plan of plans) {\n const handler = planHandlers.get(plan.type) ?? defaultHandler;\n await handler(plan);\n }\n\n return {\n tasks,\n };\n }\n\n private handleFinishedPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): void {\n const taskActionFinished: ExecutionTaskActionApply<null> = {\n type: 'Action Space',\n subType: 'Finished',\n param: null,\n thought: plan.thought,\n subTask: context.subTask || undefined,\n executor: async () => {},\n };\n context.tasks.push(taskActionFinished);\n }\n\n private handleSleepPlan(\n plan: PlanningAction<PlanningActionParamSleep>,\n context: PlanBuildContext,\n ): void {\n const sleepTask = this.createSleepTask(plan.param, {\n thought: plan.thought,\n });\n if (context.subTask) {\n sleepTask.subTask = true;\n }\n context.tasks.push(sleepTask);\n }\n\n public createSleepTask(\n param: PlanningActionParamSleep,\n meta?: { thought?: string },\n ): ExecutionTaskActionApply<PlanningActionParamSleep> {\n return {\n type: 'Action Space',\n subType: 'Sleep',\n param,\n thought: meta?.thought,\n executor: async (taskParam) => {\n await sleep(taskParam?.timeMs || 3000);\n },\n };\n }\n\n private async handleLocatePlan(\n plan: PlanningAction<PlanningLocateParam>,\n context: PlanBuildContext,\n ): Promise<void> {\n const taskLocate = this.createLocateTask(plan, plan.param, context);\n context.tasks.push(taskLocate);\n }\n\n private async handleActionPlan(\n plan: PlanningAction,\n context: PlanBuildContext,\n ): Promise<void> {\n const planType = plan.type;\n const actionSpace = await this.interface.actionSpace();\n const action = actionSpace.find((item) => item.name === planType);\n const param = plan.param;\n\n if (!action) {\n throw new Error(`Action type '${planType}' not found`);\n }\n\n const locateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema)\n : [];\n\n const requiredLocateFields = action\n ? findAllMidsceneLocatorField(action.paramSchema, true)\n : [];\n\n locateFields.forEach((field) => {\n if (param[field]) {\n // Always use createLocateTask for all locate params (including bbox)\n // This ensures cache writing happens even when bbox is available\n const locatePlan = locatePlanForLocate(param[field]);\n debug(\n 'will prepend locate param for field',\n `action.type=${planType}`,\n `param=${JSON.stringify(param[field])}`,\n `locatePlan=${JSON.stringify(locatePlan)}`,\n `hasBbox=${ifPlanLocateParamIsBbox(param[field])}`,\n );\n const locateTask = this.createLocateTask(\n locatePlan,\n param[field],\n context,\n (result) => {\n param[field] = result;\n },\n );\n context.tasks.push(locateTask);\n } else {\n assert(\n !requiredLocateFields.includes(field),\n `Required locate field '${field}' is not provided for action ${planType}`,\n );\n debug(`field '${field}' is not provided for action ${planType}`);\n }\n });\n\n const task: ExecutionTaskApply<\n 'Action Space',\n any,\n { success: boolean; action: string; param: any },\n void\n > = {\n type: 'Action Space',\n subType: planType,\n thought: plan.thought,\n param: plan.param,\n subTask: context.subTask || undefined,\n executor: async (param, taskContext) => {\n debug(\n 'executing action',\n planType,\n param,\n `taskContext.element.center: ${taskContext.element?.center}`,\n );\n\n const uiContext = taskContext.uiContext;\n assert(uiContext, 'uiContext is required for Action task');\n\n requiredLocateFields.forEach((field) => {\n assert(\n param[field],\n `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n );\n });\n\n try {\n await Promise.all([\n (async () => {\n if (this.interface.beforeInvokeAction) {\n debug('will call \"beforeInvokeAction\" for interface');\n await this.interface.beforeInvokeAction(action.name, param);\n debug('called \"beforeInvokeAction\" for interface');\n }\n })(),\n sleep(200),\n ]);\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message || String(originalError);\n throw new Error(\n `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n if (action.paramSchema) {\n try {\n param = parseActionParam(param, action.paramSchema);\n } catch (error: any) {\n throw new Error(\n `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n { cause: error },\n );\n }\n }\n\n debug('calling action', action.name);\n const actionFn = action.call.bind(this.interface);\n const actionResult = await actionFn(param, taskContext);\n debug('called action', action.name, 'result:', actionResult);\n\n const delayAfterRunner = action.delayAfterRunner ?? 300;\n if (delayAfterRunner > 0) {\n await sleep(delayAfterRunner);\n }\n\n try {\n if (this.interface.afterInvokeAction) {\n debug('will call \"afterInvokeAction\" for interface');\n await this.interface.afterInvokeAction(action.name, param);\n debug('called \"afterInvokeAction\" for interface');\n }\n } catch (originalError: any) {\n const originalMessage =\n originalError?.message || String(originalError);\n throw new Error(\n `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n { cause: originalError },\n );\n }\n\n return {\n output: actionResult,\n };\n },\n };\n\n context.tasks.push(task);\n }\n\n private createLocateTask(\n plan: PlanningAction<PlanningLocateParam>,\n detailedLocateParam: DetailedLocateParam | string,\n context: PlanBuildContext,\n onResult?: (result: LocateResultElement) => void,\n ): ExecutionTaskPlanningLocateApply {\n const { cacheable, modelConfigForDefaultIntent } = context;\n\n let locateParam = detailedLocateParam;\n\n if (typeof locateParam === 'string') {\n locateParam = {\n prompt: locateParam,\n };\n }\n\n if (cacheable !== undefined) {\n locateParam = {\n ...locateParam,\n cacheable,\n };\n }\n\n const taskLocator: ExecutionTaskPlanningLocateApply = {\n type: 'Planning',\n subType: 'Locate',\n subTask: context.subTask || undefined,\n param: locateParam,\n thought: plan.thought,\n executor: async (param, taskContext) => {\n const { task } = taskContext;\n let { uiContext } = taskContext;\n\n assert(\n param?.prompt || param?.bbox,\n `No prompt or id or position or bbox to locate, param=${JSON.stringify(\n param,\n )}`,\n );\n\n if (!uiContext) {\n uiContext = await this.service.contextRetrieverFn();\n }\n\n assert(uiContext, 'uiContext is required for Service task');\n\n let locateDump: ServiceDump | undefined;\n let locateResult: LocateResultWithDump | undefined;\n\n const applyDump = (dump?: ServiceDump) => {\n if (!dump) {\n return;\n }\n locateDump = dump;\n task.log = {\n dump,\n };\n task.usage = dump.taskInfo?.usage;\n if (dump.taskInfo?.searchAreaUsage) {\n task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n }\n };\n\n // from bbox (plan hit)\n const elementFromBbox = ifPlanLocateParamIsBbox(param)\n ? matchElementFromPlan(param)\n : undefined;\n const isPlanHit = !!elementFromBbox;\n\n // from xpath\n let rectFromXpath: Rect | undefined;\n if (\n !isPlanHit &&\n param.xpath &&\n this.interface.rectMatchesCacheFeature\n ) {\n try {\n rectFromXpath = await this.interface.rectMatchesCacheFeature({\n xpaths: [param.xpath],\n });\n } catch {\n // xpath locate failed, allow fallback to cache or AI locate\n }\n }\n const elementFromXpath = rectFromXpath\n ? generateElementByPosition(\n {\n x: rectFromXpath.left + rectFromXpath.width / 2,\n y: rectFromXpath.top + rectFromXpath.height / 2,\n },\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt || '',\n )\n : undefined;\n const isXpathHit = !!elementFromXpath;\n\n const cachePrompt = param.prompt;\n const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n const elementFromCache =\n isPlanHit || isXpathHit\n ? null\n : await matchElementFromCache(\n {\n taskCache: this.taskCache,\n interfaceInstance: this.interface,\n },\n cacheEntry,\n cachePrompt,\n param.cacheable,\n );\n const isCacheHit = !!elementFromCache;\n\n let elementFromAiLocate: LocateResultElement | null | undefined;\n if (!isXpathHit && !isCacheHit && !isPlanHit) {\n try {\n locateResult = await this.service.locate(\n param,\n {\n context: uiContext,\n },\n modelConfigForDefaultIntent,\n );\n applyDump(locateResult.dump);\n elementFromAiLocate = locateResult.element;\n } catch (error) {\n if (error instanceof ServiceError) {\n applyDump(error.dump);\n }\n throw error;\n }\n }\n\n const element =\n elementFromBbox ||\n elementFromXpath ||\n elementFromCache ||\n elementFromAiLocate;\n\n // Check if locate cache already exists (for planHitFlag case)\n const locateCacheAlreadyExists = hasNonEmptyCache(\n locateCacheRecord?.cacheContent?.cache,\n );\n\n let currentCacheEntry: ElementCacheFeature | undefined;\n // Write cache if:\n // 1. element found\n // 2. taskCache enabled\n // 3. not a cache hit (otherwise we'd be writing what we just read)\n // 4. not already cached (for bbox/plan hit case, avoid redundant writes)\n // 5. cacheable is not explicitly false\n if (\n element &&\n this.taskCache &&\n !isCacheHit &&\n !locateCacheAlreadyExists &&\n param?.cacheable !== false\n ) {\n if (this.interface.cacheFeatureForRect) {\n try {\n const feature = await this.interface.cacheFeatureForRect(\n element.rect,\n {\n targetDescription:\n typeof param.prompt === 'string'\n ? param.prompt\n : param.prompt?.prompt,\n modelConfig: modelConfigForDefaultIntent,\n },\n );\n if (hasNonEmptyCache(feature)) {\n debug(\n 'update cache, prompt: %s, cache: %o',\n cachePrompt,\n feature,\n );\n currentCacheEntry = feature;\n this.taskCache.updateOrAppendCacheRecord(\n {\n type: 'locate',\n prompt: cachePrompt,\n cache: feature,\n },\n locateCacheRecord,\n );\n } else {\n debug(\n 'no cache data returned, skip cache update, prompt: %s',\n cachePrompt,\n );\n }\n } catch (error) {\n debug('cacheFeatureForRect failed: %s', error);\n }\n } else {\n debug('cacheFeatureForRect is not supported, skip cache update');\n }\n }\n\n if (!element) {\n if (locateDump) {\n throw new ServiceError(\n `Element not found : ${param.prompt}`,\n locateDump,\n );\n }\n throw new Error(`Element not found: ${param.prompt}`);\n }\n\n let hitBy: ExecutionTaskHitBy | undefined;\n\n if (isPlanHit) {\n hitBy = {\n from: 'Plan',\n context: {\n bbox: param.bbox,\n },\n };\n } else if (isXpathHit) {\n hitBy = {\n from: 'User expected path',\n context: {\n xpath: param.xpath,\n },\n };\n } else if (isCacheHit) {\n hitBy = {\n from: 'Cache',\n context: {\n cacheEntry,\n cacheToSave: currentCacheEntry,\n },\n };\n }\n\n onResult?.(element);\n\n return {\n output: {\n element,\n },\n hitBy,\n };\n },\n };\n\n return taskLocator;\n }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","locatePlanForLocate","param","locate","locatePlan","TaskBuilder","plans","modelConfigForPlanning","modelConfigForDefaultIntent","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","undefined","sleepTask","meta","taskParam","sleep","taskLocate","planType","actionSpace","action","item","Error","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","JSON","ifPlanLocateParamIsBbox","locateTask","result","assert","task","taskContext","uiContext","Promise","originalError","originalMessage","String","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","locateParam","taskLocator","locateDump","locateResult","applyDump","dump","elementFromBbox","matchElementFromPlan","isPlanHit","rectFromXpath","elementFromXpath","generateElementByPosition","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCache","matchElementFromCache","isCacheHit","elementFromAiLocate","ServiceError","element","locateCacheAlreadyExists","currentCacheEntry","feature","hitBy","interfaceInstance","service","taskCache"],"mappings":";;;;;;;;;;;;;;;;;AA+BA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEO,SAASE,oBAAoBC,KAAmC;IACrE,MAAMC,SAAS,AAAiB,YAAjB,OAAOD,QAAqB;QAAE,QAAQA;IAAM,IAAIA;IAC/D,MAAME,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAqBO,MAAMC;IAaX,MAAa,MACXC,KAAuB,EACvBC,sBAAoC,EACpCC,2BAAyC,EACzCC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,SAAS,CAAC,CAACF,SAAS;QACtB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;YAC9D;gBACE;gBACA,CAACG,OACC,IAAI,CAAC,eAAe,CAClBA,MACAH;aAEL;SACF;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,WAAa;QACzB;QACAP,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEQ,gBACNH,IAA8C,EAC9CH,OAAyB,EACnB;QACN,MAAMQ,YAAY,IAAI,CAAC,eAAe,CAACL,KAAK,KAAK,EAAE;YACjD,SAASA,KAAK,OAAO;QACvB;QACA,IAAIH,QAAQ,OAAO,EACjBQ,UAAU,OAAO,GAAG;QAEtBR,QAAQ,KAAK,CAAC,IAAI,CAACQ;IACrB;IAEO,gBACLlB,KAA+B,EAC/BmB,IAA2B,EACyB;QACpD,OAAO;YACL,MAAM;YACN,SAAS;YACTnB;YACA,SAASmB,MAAM;YACf,UAAU,OAAOC;gBACf,MAAMC,MAAMD,WAAW,UAAU;YACnC;QACF;IACF;IAEA,MAAc,iBACZP,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMY,aAAa,IAAI,CAAC,gBAAgB,CAACT,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACY;IACrB;IAEA,MAAc,iBACZT,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMa,WAAWV,KAAK,IAAI;QAC1B,MAAMW,cAAc,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW;QACpD,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;QACxD,MAAMvB,QAAQa,KAAK,KAAK;QAExB,IAAI,CAACY,QACH,MAAM,IAAIE,MAAM,CAAC,aAAa,EAAEJ,SAAS,WAAW,CAAC;QAGvD,MAAMK,eAAeH,SACjBI,4BAA4BJ,OAAO,WAAW,IAC9C,EAAE;QAEN,MAAMK,uBAAuBL,SACzBI,4BAA4BJ,OAAO,WAAW,EAAE,QAChD,EAAE;QAENG,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI/B,KAAK,CAAC+B,MAAM,EAAE;gBAGhB,MAAM7B,aAAaH,oBAAoBC,KAAK,CAAC+B,MAAM;gBACnDrC,MACE,uCACA,CAAC,YAAY,EAAE6B,UAAU,EACzB,CAAC,MAAM,EAAES,KAAK,SAAS,CAAChC,KAAK,CAAC+B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEC,KAAK,SAAS,CAAC9B,aAAa,EAC1C,CAAC,QAAQ,EAAE+B,wBAAwBjC,KAAK,CAAC+B,MAAM,GAAG;gBAEpD,MAAMG,aAAa,IAAI,CAAC,gBAAgB,CACtChC,YACAF,KAAK,CAAC+B,MAAM,EACZrB,SACA,CAACyB;oBACCnC,KAAK,CAAC+B,MAAM,GAAGI;gBACjB;gBAEFzB,QAAQ,KAAK,CAAC,IAAI,CAACwB;YACrB,OAAO;gBACLE,OACE,CAACN,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAER,UAAU;gBAE3E7B,MAAM,CAAC,OAAO,EAAEqC,MAAM,6BAA6B,EAAER,UAAU;YACjE;QACF;QAEA,MAAMc,OAKF;YACF,MAAM;YACN,SAASd;YACT,SAASV,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,SAASH,QAAQ,OAAO,IAAIO;YAC5B,UAAU,OAAOjB,OAAOsC;gBACtB5C,MACE,oBACA6B,UACAvB,OACA,CAAC,4BAA4B,EAAEsC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAMC,YAAYD,YAAY,SAAS;gBACvCF,OAAOG,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BK,OACEpC,KAAK,CAAC+B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAER,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEA,IAAI;oBACF,MAAMiB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrC9C,MAAM;gCACN,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC+B,OAAO,IAAI,EAAEzB;gCACrDN,MAAM;4BACR;wBACF;wBACA2B,MAAM;qBACP;gBACH,EAAE,OAAOoB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,wCAAwC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,IAAIhB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQ4C,iBAAiB5C,OAAOyB,OAAO,WAAW;gBACpD,EAAE,OAAOoB,OAAY;oBACnB,MAAM,IAAIlB,MACR,CAAC,8BAA8B,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEoB,MAAM,OAAO,CAAC,cAAc,EAAEb,KAAK,SAAS,CAAChC,QAAQ,EACtG;wBAAE,OAAO6C;oBAAM;gBAEnB;gBAGFnD,MAAM,kBAAkB+B,OAAO,IAAI;gBACnC,MAAMqB,WAAWrB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAMsB,eAAe,MAAMD,SAAS9C,OAAOsC;gBAC3C5C,MAAM,iBAAiB+B,OAAO,IAAI,EAAE,WAAWsB;gBAE/C,MAAMC,mBAAmBvB,OAAO,gBAAgB,IAAI;gBACpD,IAAIuB,mBAAmB,GACrB,MAAM3B,MAAM2B;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpCtD,MAAM;wBACN,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC+B,OAAO,IAAI,EAAEzB;wBACpDN,MAAM;oBACR;gBACF,EAAE,OAAO+C,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAId,MACR,CAAC,uCAAuC,EAAEF,OAAO,IAAI,CAAC,EAAE,EAAEiB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEA,OAAO;oBACL,QAAQM;gBACV;YACF;QACF;QAEArC,QAAQ,KAAK,CAAC,IAAI,CAAC2B;IACrB;IAEQ,iBACNxB,IAAyC,EACzCoC,mBAAiD,EACjDvC,OAAyB,EACzBwC,QAAgD,EACd;QAClC,MAAM,EAAEzC,SAAS,EAAEH,2BAA2B,EAAE,GAAGI;QAEnD,IAAIyC,cAAcF;QAElB,IAAI,AAAuB,YAAvB,OAAOE,aACTA,cAAc;YACZ,QAAQA;QACV;QAGF,IAAI1C,AAAcQ,WAAdR,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,MAAM2C,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,SAAS1C,QAAQ,OAAO,IAAIO;YAC5B,OAAOkC;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOb,OAAOsC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEC,SAAS,EAAE,GAAGD;gBAEpBF,OACEpC,OAAO,UAAUA,OAAO,MACxB,CAAC,qDAAqD,EAAEgC,KAAK,SAAS,CACpEhC,QACC;gBAGL,IAAI,CAACuC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDH,OAAOG,WAAW;gBAElB,IAAIc;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACbnB,KAAK,GAAG,GAAG;wBACTmB;oBACF;oBACAnB,KAAK,KAAK,GAAGmB,KAAK,QAAQ,EAAE;oBAC5B,IAAIA,KAAK,QAAQ,EAAE,iBACjBnB,KAAK,eAAe,GAAGmB,KAAK,QAAQ,CAAC,eAAe;gBAExD;gBAGA,MAAMC,kBAAkBxB,wBAAwBjC,SAC5C0D,qBAAqB1D,SACrBiB;gBACJ,MAAM0C,YAAY,CAAC,CAACF;gBAGpB,IAAIG;gBACJ,IACE,CAACD,aACD3D,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACF4D,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAAC5D,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAEF,MAAM6D,mBAAmBD,gBACrBE,0BACE;oBACE,GAAGF,cAAc,IAAI,GAAGA,cAAc,KAAK,GAAG;oBAC9C,GAAGA,cAAc,GAAG,GAAGA,cAAc,MAAM,GAAG;gBAChD,GACA,AAAwB,YAAxB,OAAO5D,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BiB;gBACJ,MAAM8C,aAAa,CAAC,CAACF;gBAErB,MAAMG,cAAchE,MAAM,MAAM;gBAChC,MAAMiE,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,mBACJR,aAAaI,aACT,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAhE,MAAM,SAAS;gBAEvB,MAAMqE,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,IAAI,CAACP,cAAc,CAACM,cAAc,CAACV,WACjC,IAAI;oBACFL,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtCtD,OACA;wBACE,SAASuC;oBACX,GACAjC;oBAEFiD,UAAUD,aAAa,IAAI;oBAC3BgB,sBAAsBhB,aAAa,OAAO;gBAC5C,EAAE,OAAOT,OAAO;oBACd,IAAIA,iBAAiB0B,cACnBhB,UAAUV,MAAM,IAAI;oBAEtB,MAAMA;gBACR;gBAGF,MAAM2B,UACJf,mBACAI,oBACAM,oBACAG;gBAGF,MAAMG,2BAA2B7E,iBAC/BqE,mBAAmB,cAAc;gBAGnC,IAAIS;gBAOJ,IACEF,WACA,IAAI,CAAC,SAAS,IACd,CAACH,cACD,CAACI,4BACDzE,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,mBAAmB,EACpC,IAAI;oBACF,MAAM2E,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,mBAAmB,CACtDH,QAAQ,IAAI,EACZ;wBACE,mBACE,AAAwB,YAAxB,OAAOxE,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,aAAaM;oBACf;oBAEF,IAAIV,iBAAiB+E,UAAU;wBAC7BjF,MACE,uCACAsE,aACAW;wBAEFD,oBAAoBC;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQX;4BACR,OAAOW;wBACT,GACAV;oBAEJ,OACEvE,MACE,yDACAsE;gBAGN,EAAE,OAAOnB,OAAO;oBACdnD,MAAM,kCAAkCmD;gBAC1C;qBAEAnD,MAAM;gBAIV,IAAI,CAAC8E,SAAS;oBACZ,IAAInB,YACF,MAAM,IAAIkB,aACR,CAAC,oBAAoB,EAAEvE,MAAM,MAAM,EAAE,EACrCqD;oBAGJ,MAAM,IAAI1B,MAAM,CAAC,mBAAmB,EAAE3B,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAI4E;gBAEJ,IAAIjB,WACFiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,MAAM5E,MAAM,IAAI;oBAClB;gBACF;qBACK,IAAI+D,YACTa,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAO5E,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIqE,YACTO,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPV;wBACA,aAAaQ;oBACf;gBACF;gBAGFxB,WAAWsB;gBAEX,OAAO;oBACL,QAAQ;wBACNA;oBACF;oBACAI;gBACF;YACF;QACF;QAEA,OAAOxB;IACT;IAvfA,YAAY,EAAEyB,iBAAiB,EAAEC,OAAO,EAAEC,SAAS,EAAmB,CAAE;QANxE,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAGE,IAAI,CAAC,SAAS,GAAGF;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;IACnB;AAofF"}
package/dist/es/agent/utils.mjs CHANGED
@@ -100,7 +100,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
100
100
  return;
101
101
  }
102
102
  }
103
- const getMidsceneVersion = ()=>"1.0.1-beta-20251205094204.0";
103
+ const getMidsceneVersion = ()=>"1.0.1-beta-20251208031823.0";
104
104
  const parsePrompt = (prompt)=>{
105
105
  if ('string' == typeof prompt) return {
106
106
  textPrompt: prompt,