@midscene/core 1.10.2 → 1.10.3

package/dist/es/agent/task-builder.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifPlanLocateParamHasLocatedPixelBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [\n    element.center?.[0],\n    element.center?.[1],\n    element.rect?.left,\n    element.rect?.top,\n    element.rect?.width,\n    element.rect?.height,\n  ];\n  if (\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  if (element.rect.width <= 0 || element.rect.height <= 0) {\n    return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    const requiredLocateFields = findAllMidsceneLocatorField(\n      action.paramSchema,\n      true,\n    );\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelBbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n          param,\n        )\n          ? param\n          : undefined;\n\n        assert(\n          param?.prompt || paramWithLocatedPixelBbox,\n          `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            task.searchAreaUsage = dump.taskInfo.searchAreaUsage;\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelBbox\n          ? matchElementFromPlan(paramWithLocatedPixelBbox)\n          : undefined;\n\n        // from locatedPixelBbox (direct plan hit)\n        // when deepLocate is enabled, locatedPixelBbox should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,4BAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEwB,qCAAqC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIrB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,iBAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOoC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpChE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDZ,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMsB,4BAA4B3B,qCAChC/B,SAEEA,QACAI;gBAEJ8B,OACElC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACsC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIa;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3B,KAAK,KAAK,GAAG4B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD3B,KAAK,UAAU,GAAG2B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBACjB3B,KAAK,eAAe,GAAG2B,KAAK,QAAQ,CAAC,eAAe;oBAEtD,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,oCACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF5B,mBAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASsC;wBACT0B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAIb,MAAMqC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B9F,iBAC/BoF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD3D,MACE,8DACAM,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIrB,iBAAiBkG,UAAU;wBAC7BpG,MACE,uCACAqF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEtF,MACE,yDACAqF;gBAGN,EAAE,OAAOxB,OAAO;oBACd7D,MAAM,mCAAmC6D;gBAC3C;qBAEA7D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIb,MAAM,CAAC,mBAAmB,EAAE9C,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW5D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK4C,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IA/jBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTpE,WAAW,EACXqE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGpE;QACnB,IAAI,CAAC,eAAe,GAAGqE;IACzB;AAojBF"}
+{"version":3,"file":"agent/task-builder.mjs","sources":["../../../src/agent/task-builder.ts"],"sourcesContent":["import { findAllMidsceneLocatorField, parseActionParam } from '@/ai-model';\nimport type { ModelRuntime } from '@/ai-model/models';\nimport { findActionInActionSpaceOrThrow } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport type Service from '@/service';\nimport { setTimingFieldOnce } from '@/task-timing';\nimport type {\n  AIUsageInfo,\n  DetailedLocateParam,\n  DeviceAction,\n  ElementCacheFeature,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskHitBy,\n  ExecutionTaskPlanningLocateApply,\n  LocateResultElement,\n  LocateResultWithDump,\n  PlanningAction,\n  PlanningLocateParam,\n  Rect,\n  ServiceDump,\n} from '@/types';\nimport { ServiceError } from '@/types';\nimport { sleep } from '@/utils';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TaskCache } from './task-cache';\nimport { withUsageIntent } from './usage-intent';\nimport {\n  ifPlanLocateParamHasLocatedPixelBbox,\n  matchElementFromCache,\n  matchElementFromPlan,\n  transformLogicalElementToScreenshot,\n  transformLogicalRectToScreenshotRect,\n} from './utils';\n\nconst debug = getDebug('agent:task-builder');\n\n/**\n * Check if a cache object is non-empty\n */\nfunction hasNonEmptyCache(cache: unknown): boolean {\n  return (\n    cache !== null &&\n    cache !== undefined &&\n    typeof cache === 'object' &&\n    Object.keys(cache).length > 0\n  );\n}\n\nfunction invalidLocateElementReason(\n  element: LocateResultElement,\n): string | undefined {\n  const values = [\n    element.center?.[0],\n    element.center?.[1],\n    element.rect?.left,\n    element.rect?.top,\n    element.rect?.width,\n    element.rect?.height,\n  ];\n  if (\n    values.some((value) => typeof value !== 'number' || !Number.isFinite(value))\n  ) {\n    return `Invalid locate result coordinates: ${JSON.stringify(element)}`;\n  }\n  if (element.rect.width <= 0 || element.rect.height <= 0) {\n    return `Invalid locate result rect size: ${JSON.stringify(element)}`;\n  }\n  return undefined;\n}\n\ntype LocateParamWithDeprecatedAlias = DetailedLocateParam & {\n  deepThink?: boolean;\n};\n\nfunction normalizeLocateParam(\n  param: string | DetailedLocateParam,\n): DetailedLocateParam {\n  if (typeof param === 'string') {\n    return { prompt: param };\n  }\n\n  const { deepThink, ...rest } = param as LocateParamWithDeprecatedAlias;\n  const deepLocate = rest.deepLocate ?? deepThink;\n\n  return deepLocate === undefined ? rest : { ...rest, deepLocate };\n}\n\nexport function locatePlanForLocate(param: string | DetailedLocateParam) {\n  const locate = normalizeLocateParam(param);\n  const locatePlan: PlanningAction<PlanningLocateParam> = {\n    type: 'Locate',\n    param: locate,\n    thought: '',\n  };\n  return locatePlan;\n}\n\ninterface TaskBuilderDeps {\n  interfaceInstance: AbstractInterface;\n  service: Service;\n  taskCache?: TaskCache;\n  actionSpace: DeviceAction[];\n  waitAfterAction?: number;\n}\n\ninterface BuildOptions {\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\ninterface PlanBuildContext {\n  tasks: ExecutionTaskApply[];\n  planningModel: ModelRuntime;\n  defaultModel: ModelRuntime;\n  cacheable?: boolean;\n  deepLocate?: boolean;\n  abortSignal?: AbortSignal;\n}\n\nexport class TaskBuilder {\n  private readonly interface: AbstractInterface;\n\n  private readonly service: Service;\n\n  private readonly taskCache?: TaskCache;\n\n  private readonly actionSpace: DeviceAction[];\n\n  private readonly waitAfterAction?: number;\n\n  constructor({\n    interfaceInstance,\n    service,\n    taskCache,\n    actionSpace,\n    waitAfterAction,\n  }: TaskBuilderDeps) {\n    this.interface = interfaceInstance;\n    this.service = service;\n    this.taskCache = taskCache;\n    this.actionSpace = actionSpace;\n    this.waitAfterAction = waitAfterAction;\n  }\n\n  public async build(\n    plans: PlanningAction[],\n    planningModel: ModelRuntime,\n    defaultModel: ModelRuntime,\n    options?: BuildOptions,\n  ): Promise<{ tasks: ExecutionTaskApply[] }> {\n    const tasks: ExecutionTaskApply[] = [];\n    const cacheable = options?.cacheable;\n\n    const context: PlanBuildContext = {\n      tasks,\n      planningModel,\n      defaultModel,\n      cacheable,\n      deepLocate: options?.deepLocate,\n      abortSignal: options?.abortSignal,\n    };\n\n    type PlanHandler = (plan: PlanningAction) => Promise<void> | void;\n\n    const planHandlers = new Map<string, PlanHandler>([\n      [\n        'Locate',\n        (plan) =>\n          this.handleLocatePlan(\n            plan as PlanningAction<PlanningLocateParam>,\n            context,\n          ),\n      ],\n      ['Finished', (plan) => this.handleFinishedPlan(plan, context)],\n    ]);\n\n    const defaultHandler: PlanHandler = (plan) =>\n      this.handleActionPlan(plan, context);\n\n    for (const plan of plans) {\n      const handler = planHandlers.get(plan.type) ?? defaultHandler;\n      await handler(plan);\n    }\n\n    return {\n      tasks,\n    };\n  }\n\n  private handleFinishedPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): void {\n    const taskActionFinished: ExecutionTaskActionApply<null> = {\n      type: 'Action Space',\n      subType: 'Finished',\n      param: null,\n      thought: plan.thought,\n      executor: async () => {},\n    };\n    context.tasks.push(taskActionFinished);\n  }\n\n  private async handleLocatePlan(\n    plan: PlanningAction<PlanningLocateParam>,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const taskLocate = this.createLocateTask(plan, plan.param, context);\n    context.tasks.push(taskLocate);\n  }\n\n  private async handleActionPlan(\n    plan: PlanningAction,\n    context: PlanBuildContext,\n  ): Promise<void> {\n    const planType = plan.type;\n    const actionSpace = this.actionSpace;\n    const action = findActionInActionSpaceOrThrow(planType, actionSpace);\n    const param = plan.param;\n\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n\n    const requiredLocateFields = findAllMidsceneLocatorField(\n      action.paramSchema,\n      true,\n    );\n\n    locateFields.forEach((field) => {\n      if (param[field]) {\n        // Always use createLocateTask for all locate params.\n        // This ensures cache writing happens even when locatedPixelBbox is available\n        const locatePlan = locatePlanForLocate(param[field]);\n        debug(\n          'will prepend locate param for field',\n          `action.type=${planType}`,\n          `param=${JSON.stringify(param[field])}`,\n          `locatePlan=${JSON.stringify(locatePlan)}`,\n          `hasLocatedPixelBbox=${ifPlanLocateParamHasLocatedPixelBbox(param[field])}`,\n        );\n        const locateTask = this.createLocateTask(\n          locatePlan,\n          param[field],\n          context,\n          (result) => {\n            param[field] = result;\n          },\n        );\n        context.tasks.push(locateTask);\n      } else {\n        assert(\n          !requiredLocateFields.includes(field),\n          `Required locate field '${field}' is not provided for action ${planType}`,\n        );\n        debug(`field '${field}' is not provided for action ${planType}`);\n      }\n    });\n\n    const task: ExecutionTaskApply<\n      'Action Space',\n      any,\n      { success: boolean; action: string; param: any },\n      void\n    > = {\n      type: 'Action Space',\n      subType: planType,\n      thought: plan.thought,\n      param: plan.param,\n      executor: async (param, taskContext) => {\n        const timing = taskContext.task.timing;\n\n        debug(\n          'executing action',\n          planType,\n          param,\n          `taskContext.element.center: ${taskContext.element?.center}`,\n        );\n\n        const uiContext = taskContext.uiContext;\n        assert(uiContext, 'uiContext is required for Action task');\n\n        requiredLocateFields.forEach((field) => {\n          assert(\n            param[field],\n            `field '${field}' is required for action ${planType} but not provided. Cannot execute action ${planType}.`,\n          );\n        });\n\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookStart');\n        const delayBeforeRunner = action.delayBeforeRunner ?? 200;\n        try {\n          await Promise.all([\n            (async () => {\n              if (this.interface.beforeInvokeAction) {\n                debug(\n                  `will call \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n                await this.interface.beforeInvokeAction(action.name, param);\n                debug(\n                  `called \"beforeInvokeAction\" for interface with action name ${action.name}`,\n                );\n              }\n            })(),\n            delayBeforeRunner > 0\n              ? sleep(delayBeforeRunner)\n              : Promise.resolve(),\n          ]);\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running beforeInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n        setTimingFieldOnce(timing, 'beforeInvokeActionHookEnd');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in Action task',\n          );\n        }\n\n        if (action.paramSchema) {\n          try {\n            param = parseActionParam(param, action.paramSchema, {\n              shrunkShotToLogicalRatio,\n            });\n          } catch (error: any) {\n            throw new Error(\n              `Invalid parameters for action ${action.name}: ${error.message}\\nParameters: ${JSON.stringify(param)}`,\n              { cause: error },\n            );\n          }\n        }\n\n        setTimingFieldOnce(timing, 'callActionStart');\n\n        debug('calling action', action.name);\n        const actionFn = action.call.bind(this.interface);\n        const actionResult = await actionFn(param, taskContext);\n        setTimingFieldOnce(timing, 'callActionEnd');\n        debug('called action', action.name, 'result:', actionResult);\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookStart');\n\n        const delayAfterRunner =\n          action.delayAfterRunner ?? this.waitAfterAction ?? 300;\n        if (delayAfterRunner > 0) {\n          await sleep(delayAfterRunner);\n        }\n\n        try {\n          if (this.interface.afterInvokeAction) {\n            debug(\n              `will call \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n            await this.interface.afterInvokeAction(action.name, param);\n            debug(\n              `called \"afterInvokeAction\" for interface with action name ${action.name}`,\n            );\n          }\n        } catch (originalError: any) {\n          const originalMessage =\n            originalError?.message || String(originalError);\n          throw new Error(\n            `error in running afterInvokeAction for ${action.name}: ${originalMessage}`,\n            { cause: originalError },\n          );\n        }\n\n        setTimingFieldOnce(timing, 'afterInvokeActionHookEnd');\n\n        return {\n          output: actionResult,\n        };\n      },\n    };\n\n    context.tasks.push(task);\n  }\n\n  private createLocateTask(\n    plan: PlanningAction<PlanningLocateParam>,\n    detailedLocateParam: DetailedLocateParam | string,\n    context: PlanBuildContext,\n    onResult?: (result: LocateResultElement) => void,\n  ): ExecutionTaskPlanningLocateApply {\n    const { cacheable, defaultModel, deepLocate, abortSignal } = context;\n\n    let locateParam = normalizeLocateParam(detailedLocateParam);\n\n    if (cacheable !== undefined) {\n      locateParam = {\n        ...locateParam,\n        cacheable,\n      };\n    }\n\n    if (deepLocate && !locateParam.deepLocate) {\n      locateParam = {\n        ...locateParam,\n        deepLocate: true,\n      };\n    }\n\n    const taskLocator: ExecutionTaskPlanningLocateApply = {\n      type: 'Planning',\n      subType: 'Locate',\n      param: locateParam,\n      thought: plan.thought,\n      executor: async (param, taskContext) => {\n        const { task } = taskContext;\n        let { uiContext } = taskContext;\n        const paramWithLocatedPixelBbox = ifPlanLocateParamHasLocatedPixelBbox(\n          param,\n        )\n          ? param\n          : undefined;\n\n        assert(\n          param?.prompt || paramWithLocatedPixelBbox,\n          `No prompt or id or position or locatedPixelBbox to locate, param=${JSON.stringify(\n            param,\n          )}`,\n        );\n\n        if (!uiContext) {\n          uiContext = await this.service.contextRetrieverFn();\n        }\n\n        assert(uiContext, 'uiContext is required for Service task');\n\n        const { shrunkShotToLogicalRatio } = uiContext;\n\n        if (shrunkShotToLogicalRatio === undefined) {\n          throw new Error(\n            'shrunkShotToLogicalRatio is not defined in locate task',\n          );\n        }\n\n        let locateDump: ServiceDump | undefined;\n        let locateResult: LocateResultWithDump | undefined;\n\n        const applyDump = (dump?: ServiceDump) => {\n          if (!dump) {\n            return;\n          }\n          locateDump = dump;\n          task.log = {\n            dump,\n            rawResponse: dump.taskInfo?.rawResponse,\n            rawChoiceMessage: dump.taskInfo?.rawChoiceMessage,\n            searchAreaRawChoiceMessage:\n              dump.taskInfo?.searchAreaRawChoiceMessage,\n          };\n          task.usage = withUsageIntent(dump.taskInfo?.usage, 'default');\n          task.searchArea = dump.taskInfo?.searchArea;\n          if (dump.taskInfo?.searchAreaUsage) {\n            // The deepLocate search-area call belongs to the same locate task,\n            // so it shares the companion usage intent.\n            task.searchAreaUsage = withUsageIntent(\n              dump.taskInfo.searchAreaUsage,\n              'default',\n            );\n          }\n          if (dump.taskInfo?.reasoning_content) {\n            task.reasoning_content = dump.taskInfo.reasoning_content;\n          }\n        };\n\n        const planLocatedElement = paramWithLocatedPixelBbox\n          ? matchElementFromPlan(paramWithLocatedPixelBbox)\n          : undefined;\n\n        // from locatedPixelBbox (direct plan hit)\n        // when deepLocate is enabled, locatedPixelBbox should be used as search\n        // area hint, not as a final direct hit\n        const elementFromPlan = param.deepLocate\n          ? undefined\n          : planLocatedElement;\n        const isPlanDirectHit = !!elementFromPlan;\n\n        // from xpath\n        let rectFromXpath: Rect | undefined;\n        if (\n          !isPlanDirectHit &&\n          param.xpath &&\n          this.interface.rectMatchesCacheFeature\n        ) {\n          try {\n            rectFromXpath = await this.interface.rectMatchesCacheFeature({\n              xpaths: [param.xpath],\n            });\n          } catch {\n            // xpath locate failed, allow fallback to cache or AI locate\n          }\n        }\n\n        const elementFromXpath = rectFromXpath\n          ? generateElementByRect(\n              // rectFromXpath is in logical coordinates, which should be transformed to screenshot coordinates;\n              transformLogicalRectToScreenshotRect(\n                rectFromXpath,\n                shrunkShotToLogicalRatio,\n              ),\n              typeof param.prompt === 'string'\n                ? param.prompt\n                : param.prompt?.prompt || '',\n            )\n          : undefined;\n\n        const isXpathHit = !!elementFromXpath;\n\n        const cachePrompt = param.prompt;\n        const locateCacheRecord = this.taskCache?.matchLocateCache(cachePrompt);\n        const cacheEntry = locateCacheRecord?.cacheContent?.cache;\n\n        const elementFromCacheResult =\n          isPlanDirectHit || isXpathHit\n            ? null\n            : await matchElementFromCache(\n                {\n                  taskCache: this.taskCache,\n                  interfaceInstance: this.interface,\n                },\n                cacheEntry,\n                cachePrompt,\n                param.cacheable,\n              );\n\n        // elementFromCacheResult is in logical coordinates, which should be transformed to screenshot coordinates;\n        const elementFromCache = elementFromCacheResult\n          ? transformLogicalElementToScreenshot(\n              elementFromCacheResult,\n              shrunkShotToLogicalRatio,\n            )\n          : undefined;\n\n        const isCacheHit = !!elementFromCache;\n\n        let elementFromAiLocate: LocateResultElement | null | undefined;\n        const timing = taskContext.task.timing;\n        if (!isXpathHit && !isCacheHit && !isPlanDirectHit) {\n          try {\n            setTimingFieldOnce(timing, 'callAiStart');\n            locateResult = await this.service.locate(\n              param,\n              {\n                context: uiContext,\n                planLocatedElement,\n              },\n              defaultModel,\n              abortSignal,\n            );\n            applyDump(locateResult.dump);\n            elementFromAiLocate = locateResult.element;\n          } catch (error) {\n            if (error instanceof ServiceError) {\n              applyDump(error.dump);\n            }\n            throw error;\n          } finally {\n            setTimingFieldOnce(timing, 'callAiEnd');\n          }\n        }\n\n        const element =\n          elementFromPlan ||\n          elementFromXpath ||\n          elementFromCache ||\n          elementFromAiLocate;\n\n        if (element) {\n          const invalidElementReason = invalidLocateElementReason(element);\n          if (invalidElementReason) {\n            if (locateDump) {\n              throw new ServiceError(invalidElementReason, locateDump);\n            }\n            throw new Error(invalidElementReason);\n          }\n        }\n\n        // Check if locate cache already exists (for planHitFlag case)\n        const locateCacheAlreadyExists = hasNonEmptyCache(\n          locateCacheRecord?.cacheContent?.cache,\n        );\n\n        let currentCacheEntry: ElementCacheFeature | undefined;\n        // Write cache if:\n        // 1. element found\n        // 2. taskCache enabled\n        // 3. not a cache hit (otherwise we'd be writing what we just read)\n        // 4. not already cached for plan hit case (avoid redundant writes), OR allow update if cache validation failed\n        // 5. cacheable is not explicitly false\n        if (\n          element &&\n          this.taskCache &&\n          !isCacheHit &&\n          (!isPlanDirectHit || !locateCacheAlreadyExists) &&\n          param?.cacheable !== false\n        ) {\n          if (this.interface.cacheFeatureForPoint) {\n            try {\n              // Transform coordinates to logical space for cacheFeatureForPoint\n              // cacheFeatureForPoint needs logical coordinates to locate elements in DOM\n              let pointForCache: [number, number] = element.center;\n              if (shrunkShotToLogicalRatio !== 1) {\n                pointForCache = [\n                  Math.round(element.center[0] / shrunkShotToLogicalRatio),\n                  Math.round(element.center[1] / shrunkShotToLogicalRatio),\n                ];\n                debug(\n                  'Transformed coordinates for cacheFeatureForPoint: %o -> %o',\n                  element.center,\n                  pointForCache,\n                );\n              }\n\n              const feature = await this.interface.cacheFeatureForPoint(\n                pointForCache,\n                {\n                  targetDescription:\n                    typeof param.prompt === 'string'\n                      ? param.prompt\n                      : param.prompt?.prompt,\n                  modelRuntime: defaultModel,\n                },\n              );\n              if (hasNonEmptyCache(feature)) {\n                debug(\n                  'update cache, prompt: %s, cache: %o',\n                  cachePrompt,\n                  feature,\n                );\n                currentCacheEntry = feature;\n                this.taskCache.updateOrAppendCacheRecord(\n                  {\n                    type: 'locate',\n                    prompt: cachePrompt,\n                    cache: feature,\n                  },\n                  locateCacheRecord,\n                );\n              } else {\n                debug(\n                  'no cache data returned, skip cache update, prompt: %s',\n                  cachePrompt,\n                );\n              }\n            } catch (error) {\n              debug('cacheFeatureForPoint failed: %s', error);\n            }\n          } else {\n            debug('cacheFeatureForPoint is not supported, skip cache update');\n          }\n        }\n\n        if (!element) {\n          if (locateDump) {\n            throw new ServiceError(\n              `Element not found : ${param.prompt}`,\n              locateDump,\n            );\n          }\n          throw new Error(`Element not found: ${param.prompt}`);\n        }\n\n        let hitBy: ExecutionTaskHitBy | undefined;\n\n        if (isPlanDirectHit && paramWithLocatedPixelBbox) {\n          hitBy = {\n            from: 'Plan',\n            context: {\n              locatedPixelBbox: paramWithLocatedPixelBbox.locatedPixelBbox,\n            },\n          };\n        } else if (isXpathHit) {\n          hitBy = {\n            from: 'User expected path',\n            context: {\n              xpath: param.xpath,\n            },\n          };\n        } else if (isCacheHit) {\n          hitBy = {\n            from: 'Cache',\n            context: {\n              cacheEntry,\n              cacheToSave: currentCacheEntry,\n            },\n          };\n        }\n\n        onResult?.(element);\n\n        return {\n          output: {\n            element: {\n              ...element,\n              // backward compatibility for aiLocate, which return value needs a dpr field\n              dpr: uiContext.deprecatedDpr,\n            },\n          },\n          hitBy,\n        };\n      },\n    };\n\n    return taskLocator;\n  }\n}\n"],"names":["debug","getDebug","hasNonEmptyCache","cache","Object","invalidLocateElementReason","element","values","value","Number","JSON","normalizeLocateParam","param","deepThink","rest","deepLocate","undefined","locatePlanForLocate","locate","locatePlan","TaskBuilder","plans","planningModel","defaultModel","options","tasks","cacheable","context","planHandlers","Map","plan","defaultHandler","handler","taskActionFinished","taskLocate","planType","actionSpace","action","findActionInActionSpaceOrThrow","locateFields","findAllMidsceneLocatorField","requiredLocateFields","field","ifPlanLocateParamHasLocatedPixelBbox","locateTask","result","assert","task","taskContext","timing","uiContext","setTimingFieldOnce","delayBeforeRunner","Promise","sleep","originalError","originalMessage","String","Error","shrunkShotToLogicalRatio","parseActionParam","error","actionFn","actionResult","delayAfterRunner","detailedLocateParam","onResult","abortSignal","locateParam","taskLocator","paramWithLocatedPixelBbox","locateDump","locateResult","applyDump","dump","withUsageIntent","planLocatedElement","matchElementFromPlan","elementFromPlan","isPlanDirectHit","rectFromXpath","elementFromXpath","generateElementByRect","transformLogicalRectToScreenshotRect","isXpathHit","cachePrompt","locateCacheRecord","cacheEntry","elementFromCacheResult","matchElementFromCache","elementFromCache","transformLogicalElementToScreenshot","isCacheHit","elementFromAiLocate","ServiceError","invalidElementReason","locateCacheAlreadyExists","currentCacheEntry","pointForCache","Math","feature","hitBy","interfaceInstance","service","taskCache","waitAfterAction"],"mappings":";;;;;;;;;;;;;;;;;;;;AAqCA,MAAMA,QAAQC,SAAS;AAKvB,SAASC,iBAAiBC,KAAc;IACtC,OACEA,QAAAA,SAEA,AAAiB,YAAjB,OAAOA,SACPC,OAAO,IAAI,CAACD,OAAO,MAAM,GAAG;AAEhC;AAEA,SAASE,2BACPC,OAA4B;IAE5B,MAAMC,SAAS;QACbD,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,MAAM,EAAE,CAAC,EAAE;QACnBA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;QACdA,QAAQ,IAAI,EAAE;KACf;IACD,IACEC,OAAO,IAAI,CAAC,CAACC,QAAU,AAAiB,YAAjB,OAAOA,SAAsB,CAACC,OAAO,QAAQ,CAACD,SAErE,OAAO,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACJ,UAAU;IAExE,IAAIA,QAAQ,IAAI,CAAC,KAAK,IAAI,KAAKA,QAAQ,IAAI,CAAC,MAAM,IAAI,GACpD,OAAO,CAAC,iCAAiC,EAAEI,KAAK,SAAS,CAACJ,UAAU;AAGxE;AAMA,SAASK,qBACPC,KAAmC;IAEnC,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAO;QAAE,QAAQA;IAAM;IAGzB,MAAM,EAAEC,SAAS,EAAE,GAAGC,MAAM,GAAGF;IAC/B,MAAMG,aAAaD,KAAK,UAAU,IAAID;IAEtC,OAAOE,AAAeC,WAAfD,aAA2BD,OAAO;QAAE,GAAGA,IAAI;QAAEC;IAAW;AACjE;AAEO,SAASE,oBAAoBL,KAAmC;IACrE,MAAMM,SAASP,qBAAqBC;IACpC,MAAMO,aAAkD;QACtD,MAAM;QACN,OAAOD;QACP,SAAS;IACX;IACA,OAAOC;AACT;AAyBO,MAAMC;IAyBX,MAAa,MACXC,KAAuB,EACvBC,aAA2B,EAC3BC,YAA0B,EAC1BC,OAAsB,EACoB;QAC1C,MAAMC,QAA8B,EAAE;QACtC,MAAMC,YAAYF,SAAS;QAE3B,MAAMG,UAA4B;YAChCF;YACAH;YACAC;YACAG;YACA,YAAYF,SAAS;YACrB,aAAaA,SAAS;QACxB;QAIA,MAAMI,eAAe,IAAIC,IAAyB;YAChD;gBACE;gBACA,CAACC,OACC,IAAI,CAAC,gBAAgB,CACnBA,MACAH;aAEL;YACD;gBAAC;gBAAY,CAACG,OAAS,IAAI,CAAC,kBAAkB,CAACA,MAAMH;aAAS;SAC/D;QAED,MAAMI,iBAA8B,CAACD,OACnC,IAAI,CAAC,gBAAgB,CAACA,MAAMH;QAE9B,KAAK,MAAMG,QAAQT,MAAO;YACxB,MAAMW,UAAUJ,aAAa,GAAG,CAACE,KAAK,IAAI,KAAKC;YAC/C,MAAMC,QAAQF;QAChB;QAEA,OAAO;YACLL;QACF;IACF;IAEQ,mBACNK,IAAoB,EACpBH,OAAyB,EACnB;QACN,MAAMM,qBAAqD;YACzD,MAAM;YACN,SAAS;YACT,OAAO;YACP,SAASH,KAAK,OAAO;YACrB,UAAU,WAAa;QACzB;QACAH,QAAQ,KAAK,CAAC,IAAI,CAACM;IACrB;IAEA,MAAc,iBACZH,IAAyC,EACzCH,OAAyB,EACV;QACf,MAAMO,aAAa,IAAI,CAAC,gBAAgB,CAACJ,MAAMA,KAAK,KAAK,EAAEH;QAC3DA,QAAQ,KAAK,CAAC,IAAI,CAACO;IACrB;IAEA,MAAc,iBACZJ,IAAoB,EACpBH,OAAyB,EACV;QACf,MAAMQ,WAAWL,KAAK,IAAI;QAC1B,MAAMM,cAAc,IAAI,CAAC,WAAW;QACpC,MAAMC,SAASC,+BAA+BH,UAAUC;QACxD,MAAMxB,QAAQkB,KAAK,KAAK;QAExB,MAAMS,eAAeC,4BAA4BH,OAAO,WAAW;QAEnE,MAAMI,uBAAuBD,4BAC3BH,OAAO,WAAW,EAClB;QAGFE,aAAa,OAAO,CAAC,CAACG;YACpB,IAAI9B,KAAK,CAAC8B,MAAM,EAAE;gBAGhB,MAAMvB,aAAaF,oBAAoBL,KAAK,CAAC8B,MAAM;gBACnD1C,MACE,uCACA,CAAC,YAAY,EAAEmC,UAAU,EACzB,CAAC,MAAM,EAAEzB,KAAK,SAAS,CAACE,KAAK,CAAC8B,MAAM,GAAG,EACvC,CAAC,WAAW,EAAEhC,KAAK,SAAS,CAACS,aAAa,EAC1C,CAAC,oBAAoB,EAAEwB,qCAAqC/B,KAAK,CAAC8B,MAAM,GAAG;gBAE7E,MAAME,aAAa,IAAI,CAAC,gBAAgB,CACtCzB,YACAP,KAAK,CAAC8B,MAAM,EACZf,SACA,CAACkB;oBACCjC,KAAK,CAAC8B,MAAM,GAAGG;gBACjB;gBAEFlB,QAAQ,KAAK,CAAC,IAAI,CAACiB;YACrB,OAAO;gBACLE,OACE,CAACL,qBAAqB,QAAQ,CAACC,QAC/B,CAAC,uBAAuB,EAAEA,MAAM,6BAA6B,EAAEP,UAAU;gBAE3EnC,MAAM,CAAC,OAAO,EAAE0C,MAAM,6BAA6B,EAAEP,UAAU;YACjE;QACF;QAEA,MAAMY,OAKF;YACF,MAAM;YACN,SAASZ;YACT,SAASL,KAAK,OAAO;YACrB,OAAOA,KAAK,KAAK;YACjB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAMC,SAASD,YAAY,IAAI,CAAC,MAAM;gBAEtChD,MACE,oBACAmC,UACAvB,OACA,CAAC,4BAA4B,EAAEoC,YAAY,OAAO,EAAE,QAAQ;gBAG9D,MAAME,YAAYF,YAAY,SAAS;gBACvCF,OAAOI,WAAW;gBAElBT,qBAAqB,OAAO,CAAC,CAACC;oBAC5BI,OACElC,KAAK,CAAC8B,MAAM,EACZ,CAAC,OAAO,EAAEA,MAAM,yBAAyB,EAAEP,SAAS,yCAAyC,EAAEA,SAAS,CAAC,CAAC;gBAE9G;gBAEAgB,mBAAmBF,QAAQ;gBAC3B,MAAMG,oBAAoBf,OAAO,iBAAiB,IAAI;gBACtD,IAAI;oBACF,MAAMgB,QAAQ,GAAG,CAAC;wBACf;4BACC,IAAI,IAAI,CAAC,SAAS,CAAC,kBAAkB,EAAE;gCACrCrD,MACE,CAAC,8DAA8D,EAAEqC,OAAO,IAAI,EAAE;gCAEhF,MAAM,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAACA,OAAO,IAAI,EAAEzB;gCACrDZ,MACE,CAAC,2DAA2D,EAAEqC,OAAO,IAAI,EAAE;4BAE/E;wBACF;wBACAe,oBAAoB,IAChBE,MAAMF,qBACNC,QAAQ,OAAO;qBACpB;gBACH,EAAE,OAAOE,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,wCAAwC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC5E;wBAAE,OAAOD;oBAAc;gBAE3B;gBACAJ,mBAAmBF,QAAQ;gBAE3B,MAAM,EAAEU,wBAAwB,EAAE,GAAGT;gBACrC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIrB,OAAO,WAAW,EACpB,IAAI;oBACFzB,QAAQgD,iBAAiBhD,OAAOyB,OAAO,WAAW,EAAE;wBAClDsB;oBACF;gBACF,EAAE,OAAOE,OAAY;oBACnB,MAAM,IAAIH,MACR,CAAC,8BAA8B,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEwB,MAAM,OAAO,CAAC,cAAc,EAAEnD,KAAK,SAAS,CAACE,QAAQ,EACtG;wBAAE,OAAOiD;oBAAM;gBAEnB;gBAGFV,mBAAmBF,QAAQ;gBAE3BjD,MAAM,kBAAkBqC,OAAO,IAAI;gBACnC,MAAMyB,WAAWzB,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS;gBAChD,MAAM0B,eAAe,MAAMD,SAASlD,OAAOoC;gBAC3CG,mBAAmBF,QAAQ;gBAC3BjD,MAAM,iBAAiBqC,OAAO,IAAI,EAAE,WAAW0B;gBAE/CZ,mBAAmBF,QAAQ;gBAE3B,MAAMe,mBACJ3B,OAAO,gBAAgB,IAAI,IAAI,CAAC,eAAe,IAAI;gBACrD,IAAI2B,mBAAmB,GACrB,MAAMV,MAAMU;gBAGd,IAAI;oBACF,IAAI,IAAI,CAAC,SAAS,CAAC,iBAAiB,EAAE;wBACpChE,MACE,CAAC,6DAA6D,EAAEqC,OAAO,IAAI,EAAE;wBAE/E,MAAM,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAACA,OAAO,IAAI,EAAEzB;wBACpDZ,MACE,CAAC,0DAA0D,EAAEqC,OAAO,IAAI,EAAE;oBAE9E;gBACF,EAAE,OAAOkB,eAAoB;oBAC3B,MAAMC,kBACJD,eAAe,WAAWE,OAAOF;oBACnC,MAAM,IAAIG,MACR,CAAC,uCAAuC,EAAErB,OAAO,IAAI,CAAC,EAAE,EAAEmB,iBAAiB,EAC3E;wBAAE,OAAOD;oBAAc;gBAE3B;gBAEAJ,mBAAmBF,QAAQ;gBAE3B,OAAO;oBACL,QAAQc;gBACV;YACF;QACF;QAEApC,QAAQ,KAAK,CAAC,IAAI,CAACoB;IACrB;IAEQ,iBACNjB,IAAyC,EACzCmC,mBAAiD,EACjDtC,OAAyB,EACzBuC,QAAgD,EACd;QAClC,MAAM,EAAExC,SAAS,EAAEH,YAAY,EAAER,UAAU,EAAEoD,WAAW,EAAE,GAAGxC;QAE7D,IAAIyC,cAAczD,qBAAqBsD;QAEvC,IAAIvC,AAAcV,WAAdU,WACF0C,cAAc;YACZ,GAAGA,WAAW;YACd1C;QACF;QAGF,IAAIX,cAAc,CAACqD,YAAY,UAAU,EACvCA,cAAc;YACZ,GAAGA,WAAW;YACd,YAAY;QACd;QAGF,MAAMC,cAAgD;YACpD,MAAM;YACN,SAAS;YACT,OAAOD;YACP,SAAStC,KAAK,OAAO;YACrB,UAAU,OAAOlB,OAAOoC;gBACtB,MAAM,EAAED,IAAI,EAAE,GAAGC;gBACjB,IAAI,EAAEE,SAAS,EAAE,GAAGF;gBACpB,MAAMsB,4BAA4B3B,qCAChC/B,SAEEA,QACAI;gBAEJ8B,OACElC,OAAO,UAAU0D,2BACjB,CAAC,iEAAiE,EAAE5D,KAAK,SAAS,CAChFE,QACC;gBAGL,IAAI,CAACsC,WACHA,YAAY,MAAM,IAAI,CAAC,OAAO,CAAC,kBAAkB;gBAGnDJ,OAAOI,WAAW;gBAElB,MAAM,EAAES,wBAAwB,EAAE,GAAGT;gBAErC,IAAIS,AAA6B3C,WAA7B2C,0BACF,MAAM,IAAID,MACR;gBAIJ,IAAIa;gBACJ,IAAIC;gBAEJ,MAAMC,YAAY,CAACC;oBACjB,IAAI,CAACA,MACH;oBAEFH,aAAaG;oBACb3B,KAAK,GAAG,GAAG;wBACT2B;wBACA,aAAaA,KAAK,QAAQ,EAAE;wBAC5B,kBAAkBA,KAAK,QAAQ,EAAE;wBACjC,4BACEA,KAAK,QAAQ,EAAE;oBACnB;oBACA3B,KAAK,KAAK,GAAG4B,gBAAgBD,KAAK,QAAQ,EAAE,OAAO;oBACnD3B,KAAK,UAAU,GAAG2B,KAAK,QAAQ,EAAE;oBACjC,IAAIA,KAAK,QAAQ,EAAE,iBAGjB3B,KAAK,eAAe,GAAG4B,gBACrBD,KAAK,QAAQ,CAAC,eAAe,EAC7B;oBAGJ,IAAIA,KAAK,QAAQ,EAAE,mBACjB3B,KAAK,iBAAiB,GAAG2B,KAAK,QAAQ,CAAC,iBAAiB;gBAE5D;gBAEA,MAAME,qBAAqBN,4BACvBO,qBAAqBP,6BACrBtD;gBAKJ,MAAM8D,kBAAkBlE,MAAM,UAAU,GACpCI,SACA4D;gBACJ,MAAMG,kBAAkB,CAAC,CAACD;gBAG1B,IAAIE;gBACJ,IACE,CAACD,mBACDnE,MAAM,KAAK,IACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,EAEtC,IAAI;oBACFoE,gBAAgB,MAAM,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC;wBAC3D,QAAQ;4BAACpE,MAAM,KAAK;yBAAC;oBACvB;gBACF,EAAE,OAAM,CAER;gBAGF,MAAMqE,mBAAmBD,gBACrBE,sBAEEC,qCACEH,eACArB,2BAEF,AAAwB,YAAxB,OAAO/C,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE,UAAU,MAE9BI;gBAEJ,MAAMoE,aAAa,CAAC,CAACH;gBAErB,MAAMI,cAAczE,MAAM,MAAM;gBAChC,MAAM0E,oBAAoB,IAAI,CAAC,SAAS,EAAE,iBAAiBD;gBAC3D,MAAME,aAAaD,mBAAmB,cAAc;gBAEpD,MAAME,yBACJT,mBAAmBK,aACf,OACA,MAAMK,sBACJ;oBACE,WAAW,IAAI,CAAC,SAAS;oBACzB,mBAAmB,IAAI,CAAC,SAAS;gBACnC,GACAF,YACAF,aACAzE,MAAM,SAAS;gBAIvB,MAAM8E,mBAAmBF,yBACrBG,oCACEH,wBACA7B,4BAEF3C;gBAEJ,MAAM4E,aAAa,CAAC,CAACF;gBAErB,IAAIG;gBACJ,MAAM5C,SAASD,YAAY,IAAI,CAAC,MAAM;gBACtC,IAAI,CAACoC,cAAc,CAACQ,cAAc,CAACb,iBACjC,IAAI;oBACF5B,mBAAmBF,QAAQ;oBAC3BuB,eAAe,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CACtC5D,OACA;wBACE,SAASsC;wBACT0B;oBACF,GACArD,cACA4C;oBAEFM,UAAUD,aAAa,IAAI;oBAC3BqB,sBAAsBrB,aAAa,OAAO;gBAC5C,EAAE,OAAOX,OAAO;oBACd,IAAIA,iBAAiBiC,cACnBrB,UAAUZ,MAAM,IAAI;oBAEtB,MAAMA;gBACR,SAAU;oBACRV,mBAAmBF,QAAQ;gBAC7B;gBAGF,MAAM3C,UACJwE,mBACAG,oBACAS,oBACAG;gBAEF,IAAIvF,SAAS;oBACX,MAAMyF,uBAAuB1F,2BAA2BC;oBACxD,IAAIyF,sBAAsB;wBACxB,IAAIxB,YACF,MAAM,IAAIuB,aAAaC,sBAAsBxB;wBAE/C,MAAM,IAAIb,MAAMqC;oBAClB;gBACF;gBAGA,MAAMC,2BAA2B9F,iBAC/BoF,mBAAmB,cAAc;gBAGnC,IAAIW;gBAOJ,IACE3F,WACA,IAAI,CAAC,SAAS,IACd,CAACsF,cACA,EAACb,mBAAmB,CAACiB,wBAAuB,KAC7CpF,OAAO,cAAc,OAErB,IAAI,IAAI,CAAC,SAAS,CAAC,oBAAoB,EACrC,IAAI;oBAGF,IAAIsF,gBAAkC5F,QAAQ,MAAM;oBACpD,IAAIqD,AAA6B,MAA7BA,0BAAgC;wBAClCuC,gBAAgB;4BACdC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;4BAC/BwC,KAAK,KAAK,CAAC7F,QAAQ,MAAM,CAAC,EAAE,GAAGqD;yBAChC;wBACD3D,MACE,8DACAM,QAAQ,MAAM,EACd4F;oBAEJ;oBAEA,MAAME,UAAU,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CACvDF,eACA;wBACE,mBACE,AAAwB,YAAxB,OAAOtF,MAAM,MAAM,GACfA,MAAM,MAAM,GACZA,MAAM,MAAM,EAAE;wBACpB,cAAcW;oBAChB;oBAEF,IAAIrB,iBAAiBkG,UAAU;wBAC7BpG,MACE,uCACAqF,aACAe;wBAEFH,oBAAoBG;wBACpB,IAAI,CAAC,SAAS,CAAC,yBAAyB,CACtC;4BACE,MAAM;4BACN,QAAQf;4BACR,OAAOe;wBACT,GACAd;oBAEJ,OACEtF,MACE,yDACAqF;gBAGN,EAAE,OAAOxB,OAAO;oBACd7D,MAAM,mCAAmC6D;gBAC3C;qBAEA7D,MAAM;gBAIV,IAAI,CAACM,SAAS;oBACZ,IAAIiE,YACF,MAAM,IAAIuB,aACR,CAAC,oBAAoB,EAAElF,MAAM,MAAM,EAAE,EACrC2D;oBAGJ,MAAM,IAAIb,MAAM,CAAC,mBAAmB,EAAE9C,MAAM,MAAM,EAAE;gBACtD;gBAEA,IAAIyF;gBAEJ,IAAItB,mBAAmBT,2BACrB+B,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,kBAAkB/B,0BAA0B,gBAAgB;oBAC9D;gBACF;qBACK,IAAIc,YACTiB,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACP,OAAOzF,MAAM,KAAK;oBACpB;gBACF;qBACK,IAAIgF,YACTS,QAAQ;oBACN,MAAM;oBACN,SAAS;wBACPd;wBACA,aAAaU;oBACf;gBACF;gBAGF/B,WAAW5D;gBAEX,OAAO;oBACL,QAAQ;wBACN,SAAS;4BACP,GAAGA,OAAO;4BAEV,KAAK4C,UAAU,aAAa;wBAC9B;oBACF;oBACAmD;gBACF;YACF;QACF;QAEA,OAAOhC;IACT;IApkBA,YAAY,EACViC,iBAAiB,EACjBC,OAAO,EACPC,SAAS,EACTpE,WAAW,EACXqE,eAAe,EACC,CAAE;QAhBpB,uBAAiB,aAAjB;QAEA,uBAAiB,WAAjB;QAEA,uBAAiB,aAAjB;QAEA,uBAAiB,eAAjB;QAEA,uBAAiB,mBAAjB;QASE,IAAI,CAAC,SAAS,GAAGH;QACjB,IAAI,CAAC,OAAO,GAAGC;QACf,IAAI,CAAC,SAAS,GAAGC;QACjB,IAAI,CAAC,WAAW,GAAGpE;QACnB,IAAI,CAAC,eAAe,GAAGqE;IACzB;AAyjBF"}

package/dist/es/agent/utils.mjs

@@ -173,7 +173,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.10.2";
+const getMidsceneVersion = ()=>"1.10.3";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/connectivity/run-connectivity-test.mjs

@@ -32,6 +32,12 @@ function buildCheckResult(name, modelRuntime, result) {
         ...result
     };
 }
+function buildConnectivityModelRuntime(modelConfig) {
+    return getModelRuntime({
+        ...modelConfig,
+        retryCount: 0
+    });
+}
 function formatConnectivityCheckName(check) {
     const checkName = 'aiLocate' === check.name ? 'AI locate check' : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;
     const modelLabel = check.modelName || check.intent;
@@ -135,9 +141,9 @@ async function runAiLocateConnectivityCheck(modelRuntime) {
     }
 }
 async function runConnectivityTest(config) {
-    const planningModelRuntime = getModelRuntime(config.planningModelConfig);
-    const insightModelRuntime = getModelRuntime(config.insightModelConfig);
-    const defaultModelRuntime = getModelRuntime(config.defaultModelConfig);
+    const planningModelRuntime = buildConnectivityModelRuntime(config.planningModelConfig);
+    const insightModelRuntime = buildConnectivityModelRuntime(config.insightModelConfig);
+    const defaultModelRuntime = buildConnectivityModelRuntime(config.defaultModelConfig);
     const checks = await Promise.all([
         runTextConnectivityCheck(planningModelRuntime),
         runVisionConnectivityCheck(insightModelRuntime),

package/dist/es/ai-model/connectivity/run-connectivity-test.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/connectivity/run-connectivity-test.mjs","sources":["../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callAI } from '../service-caller';\nimport {\n  CONNECTIVITY_FIXTURE_IMAGE,\n  CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\ninterface ConnectivityCheckResultItem {\n  name: 'text' | 'vision' | 'aiLocate';\n  intent: TIntent;\n  modelName: string;\n  modelFamily?: string;\n  passed: boolean;\n  durationMs: number;\n  message: string;\n}\n\nexport interface ConnectivityTestResult {\n  passed: boolean;\n  message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n  defaultModelConfig: IModelConfig;\n  planningModelConfig: IModelConfig;\n  insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidRect(value: unknown): boolean {\n  if (!value || typeof value !== 'object') {\n    return false;\n  }\n\n  const rect = value as {\n    left?: unknown;\n    top?: unknown;\n    width?: unknown;\n    height?: unknown;\n  };\n\n  return (\n    isFiniteNumber(rect.left) &&\n    isFiniteNumber(rect.top) &&\n    isFiniteNumber(rect.width) &&\n    isFiniteNumber(rect.height)\n  );\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n  return (\n    Array.isArray(value) &&\n    value.length === 2 &&\n    isFiniteNumber(value[0]) &&\n    isFiniteNumber(value[1])\n  );\n}\n\nfunction buildFixtureContext(): UIContext {\n  return {\n    screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n    shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n    shrunkShotToLogicalRatio: 1,\n  };\n}\n\nfunction buildCheckResult(\n  name: ConnectivityCheckResultItem['name'],\n  modelRuntime: ModelRuntime,\n  result: Omit<\n    ConnectivityCheckResultItem,\n    'name' | 'intent' | 'modelName' | 'modelFamily'\n  >,\n): ConnectivityCheckResultItem {\n  const { config } = modelRuntime;\n  return {\n    name,\n    intent: config.intent,\n    modelName: config.modelName,\n    modelFamily: config.modelFamily,\n    ...result,\n  };\n}\n\nfunction formatConnectivityCheckName(\n  check: ConnectivityCheckResultItem,\n): string {\n  const checkName =\n    check.name === 'aiLocate'\n      ? 'AI locate check'\n      : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n  const modelLabel = check.modelName || check.intent;\n  return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n  checks: ConnectivityCheckResultItem[],\n): string {\n  const failedChecks = checks.filter((item) => !item.passed);\n  if (failedChecks.length === 0) {\n    return 'Connectivity test failed, but no failed check details were generated.';\n  }\n\n  return failedChecks\n    .map((item) => {\n      const detail = item.message || 'Failed without details.';\n      return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n    })\n    .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const result = await callAI(\n      [\n        {\n          role: 'system',\n          content: 'Reply with the exact token the user asks for.',\n        },\n        {\n          role: 'user',\n          content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n        },\n      ],\n      modelRuntime,\n    );\n    const content = result.content.trim();\n    const passed = content.includes(TEXT_EXPECTED_TOKEN);\n    return buildCheckResult('text', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed ? '' : `Unexpected response: ${content}`,\n    });\n  } catch (error) {\n    return buildCheckResult('text', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runVisionConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    await callAI(\n      [\n        {\n          role: 'user',\n          content: [\n            {\n              type: 'text',\n              text: 'What is the main content of this image ? It is a photo or a form ?',\n            },\n            {\n              type: 'image_url',\n              image_url: {\n                url: CONNECTIVITY_FIXTURE_IMAGE,\n                detail: 'high',\n              },\n            },\n          ],\n        },\n      ],\n      modelRuntime,\n    );\n    return buildCheckResult('vision', modelRuntime, {\n      passed: true,\n      durationMs: Date.now() - startTime,\n      message: '',\n    });\n  } catch (error) {\n    return buildCheckResult('vision', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runAiLocateConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const context = buildFixtureContext();\n    const service = new Service(context);\n    const locateResult = await service.locate(\n      { prompt: 'the main todo input box' },\n      {},\n      modelRuntime,\n    );\n    const targetRect = locateResult.rect || locateResult.element?.rect;\n    const center = locateResult.element?.center;\n    const passed = hasValidRect(targetRect) && hasValidCenter(center);\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed\n        ? ''\n        : `Invalid locate result: ${JSON.stringify({\n            rect: targetRect,\n            center,\n          })}`,\n    });\n  } catch (error) {\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nexport async function runConnectivityTest(\n  config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n  const planningModelRuntime = getModelRuntime(config.planningModelConfig);\n  const insightModelRuntime = getModelRuntime(config.insightModelConfig);\n  const defaultModelRuntime = getModelRuntime(config.defaultModelConfig);\n  const checks = await Promise.all([\n    runTextConnectivityCheck(planningModelRuntime),\n    runVisionConnectivityCheck(insightModelRuntime),\n    runAiLocateConnectivityCheck(defaultModelRuntime),\n  ]);\n\n  const passed = checks.every((item) => item.passed);\n  return {\n    passed,\n    message: passed ? undefined : buildConnectivityFailureMessage(checks),\n  };\n}\n"],"names":["TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidRect","rect","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","callAI","content","passed","error","Error","String","runVisionConnectivityCheck","runAiLocateConnectivityCheck","context","service","Service","locateResult","targetRect","center","JSON","runConnectivityTest","planningModelRuntime","getModelRuntime","insightModelRuntime","defaultModelRuntime","Promise","undefined"],"mappings":";;;;;AAYA,MAAMA,sBAAsB;AAuB5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,aAAaF,KAAc;IAClC,IAAI,CAACA,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAMG,OAAOH;IAOb,OACED,eAAeI,KAAK,IAAI,KACxBJ,eAAeI,KAAK,GAAG,KACvBJ,eAAeI,KAAK,KAAK,KACzBJ,eAAeI,KAAK,MAAM;AAE9B;AAEA,SAASC,eAAeJ,KAAc;IACpC,OACEK,MAAM,OAAO,CAACL,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASM;IACP,OAAO;QACL,YAAYC,eAAe,MAAM,CAACC,4BAA4BC,KAAK,GAAG;QACtE,UAAUC;QACV,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAGC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbZ,YAA0B;IAE1B,MAAMa,YAAYjB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMK,SAAS,MAAMa,OACnB;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAE7B,qBAAqB;YAClD;SACD,EACDe;QAEF,MAAMe,UAAUd,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMe,SAASD,QAAQ,QAAQ,CAAC9B;QAChC,OAAOa,iBAAiB,QAAQE,cAAc;YAC5CgB;YACA,YAAYpB,KAAK,GAAG,KAAKiB;YACzB,SAASG,SAAS,KAAK,CAAC,qBAAqB,EAAED,SAAS;QAC1D;IACF,EAAE,OAAOE,OAAO;QACd,OAAOnB,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKiB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACbpB,YAA0B;IAE1B,MAAMa,YAAYjB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMkB,OACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKnB;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD,EACDK;QAEF,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKiB;YACzB,SAAS;QACX;IACF,EAAE,OAAOI,OAAO;QACd,OAAOnB,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKiB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeI,6BACbrB,YAA0B;IAE1B,MAAMa,YAAYjB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAM0B,UAAU7B;QAChB,MAAM8B,UAAU,IAAIC,UAAQF;QAC5B,MAAMG,eAAe,MAAMF,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACDvB;QAEF,MAAM0B,aAAaD,aAAa,IAAI,IAAIA,aAAa,OAAO,EAAE;QAC9D,MAAME,SAASF,aAAa,OAAO,EAAE;QACrC,MAAMT,SAAS3B,aAAaqC,eAAenC,eAAeoC;QAC1D,OAAO7B,iBAAiB,YAAYE,cAAc;YAChDgB;YACA,YAAYpB,KAAK,GAAG,KAAKiB;YACzB,SAASG,SACL,KACA,CAAC,uBAAuB,EAAEY,KAAK,SAAS,CAAC;gBACvC,MAAMF;gBACNC;YACF,IAAI;QACV;IACF,EAAE,OAAOV,OAAO;QACd,OAAOnB,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKiB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeY,oBACpB3B,MAA8B;IAE9B,MAAM4B,uBAAuBC,gBAAgB7B,OAAO,mBAAmB;IACvE,MAAM8B,sBAAsBD,gBAAgB7B,OAAO,kBAAkB;IACrE,MAAM+B,sBAAsBF,gBAAgB7B,OAAO,kBAAkB;IACrE,MAAMM,SAAS,MAAM0B,QAAQ,GAAG,CAAC;QAC/BtB,yBAAyBkB;QACzBV,2BAA2BY;QAC3BX,6BAA6BY;KAC9B;IAED,MAAMjB,SAASR,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLM;QACA,SAASA,SAASmB,SAAY5B,gCAAgCC;IAChE;AACF"}
+{"version":3,"file":"ai-model/connectivity/run-connectivity-test.mjs","sources":["../../../../src/ai-model/connectivity/run-connectivity-test.ts"],"sourcesContent":["import { ScreenshotItem } from '@/screenshot-item';\nimport Service from '@/service';\nimport type { UIContext } from '@/types';\nimport type { IModelConfig, TIntent } from '@midscene/shared/env';\nimport { getModelRuntime } from '../models';\nimport type { ModelRuntime } from '../models';\nimport { callAI } from '../service-caller';\nimport {\n  CONNECTIVITY_FIXTURE_IMAGE,\n  CONNECTIVITY_FIXTURE_SHOT_SIZE,\n} from './fixture';\n\nconst TEXT_EXPECTED_TOKEN = 'CONNECTIVITY_OK';\n\ninterface ConnectivityCheckResultItem {\n  name: 'text' | 'vision' | 'aiLocate';\n  intent: TIntent;\n  modelName: string;\n  modelFamily?: string;\n  passed: boolean;\n  durationMs: number;\n  message: string;\n}\n\nexport interface ConnectivityTestResult {\n  passed: boolean;\n  message?: string;\n}\n\nexport interface ConnectivityTestConfig {\n  defaultModelConfig: IModelConfig;\n  planningModelConfig: IModelConfig;\n  insightModelConfig: IModelConfig;\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value);\n}\n\nfunction hasValidRect(value: unknown): boolean {\n  if (!value || typeof value !== 'object') {\n    return false;\n  }\n\n  const rect = value as {\n    left?: unknown;\n    top?: unknown;\n    width?: unknown;\n    height?: unknown;\n  };\n\n  return (\n    isFiniteNumber(rect.left) &&\n    isFiniteNumber(rect.top) &&\n    isFiniteNumber(rect.width) &&\n    isFiniteNumber(rect.height)\n  );\n}\n\nfunction hasValidCenter(value: unknown): boolean {\n  return (\n    Array.isArray(value) &&\n    value.length === 2 &&\n    isFiniteNumber(value[0]) &&\n    isFiniteNumber(value[1])\n  );\n}\n\nfunction buildFixtureContext(): UIContext {\n  return {\n    screenshot: ScreenshotItem.create(CONNECTIVITY_FIXTURE_IMAGE, Date.now()),\n    shotSize: CONNECTIVITY_FIXTURE_SHOT_SIZE,\n    shrunkShotToLogicalRatio: 1,\n  };\n}\n\nfunction buildCheckResult(\n  name: ConnectivityCheckResultItem['name'],\n  modelRuntime: ModelRuntime,\n  result: Omit<\n    ConnectivityCheckResultItem,\n    'name' | 'intent' | 'modelName' | 'modelFamily'\n  >,\n): ConnectivityCheckResultItem {\n  const { config } = modelRuntime;\n  return {\n    name,\n    intent: config.intent,\n    modelName: config.modelName,\n    modelFamily: config.modelFamily,\n    ...result,\n  };\n}\n\nfunction buildConnectivityModelRuntime(\n  modelConfig: IModelConfig,\n): ModelRuntime {\n  return getModelRuntime({\n    ...modelConfig,\n    retryCount: 0,\n  });\n}\n\nfunction formatConnectivityCheckName(\n  check: ConnectivityCheckResultItem,\n): string {\n  const checkName =\n    check.name === 'aiLocate'\n      ? 'AI locate check'\n      : `${check.name[0]?.toUpperCase()}${check.name.slice(1)} check`;\n  const modelLabel = check.modelName || check.intent;\n  return `${checkName} - ${modelLabel} (${check.intent})`;\n}\n\nfunction buildConnectivityFailureMessage(\n  checks: ConnectivityCheckResultItem[],\n): string {\n  const failedChecks = checks.filter((item) => !item.passed);\n  if (failedChecks.length === 0) {\n    return 'Connectivity test failed, but no failed check details were generated.';\n  }\n\n  return failedChecks\n    .map((item) => {\n      const detail = item.message || 'Failed without details.';\n      return `[${formatConnectivityCheckName(item)}]: ${detail}`;\n    })\n    .join('\\n');\n}\n\nasync function runTextConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const result = await callAI(\n      [\n        {\n          role: 'system',\n          content: 'Reply with the exact token the user asks for.',\n        },\n        {\n          role: 'user',\n          content: `Return exactly ${TEXT_EXPECTED_TOKEN}`,\n        },\n      ],\n      modelRuntime,\n    );\n    const content = result.content.trim();\n    const passed = content.includes(TEXT_EXPECTED_TOKEN);\n    return buildCheckResult('text', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed ? '' : `Unexpected response: ${content}`,\n    });\n  } catch (error) {\n    return buildCheckResult('text', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runVisionConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    await callAI(\n      [\n        {\n          role: 'user',\n          content: [\n            {\n              type: 'text',\n              text: 'What is the main content of this image ? It is a photo or a form ?',\n            },\n            {\n              type: 'image_url',\n              image_url: {\n                url: CONNECTIVITY_FIXTURE_IMAGE,\n                detail: 'high',\n              },\n            },\n          ],\n        },\n      ],\n      modelRuntime,\n    );\n    return buildCheckResult('vision', modelRuntime, {\n      passed: true,\n      durationMs: Date.now() - startTime,\n      message: '',\n    });\n  } catch (error) {\n    return buildCheckResult('vision', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nasync function runAiLocateConnectivityCheck(\n  modelRuntime: ModelRuntime,\n): Promise<ConnectivityCheckResultItem> {\n  const startTime = Date.now();\n  try {\n    const context = buildFixtureContext();\n    const service = new Service(context);\n    const locateResult = await service.locate(\n      { prompt: 'the main todo input box' },\n      {},\n      modelRuntime,\n    );\n    const targetRect = locateResult.rect || locateResult.element?.rect;\n    const center = locateResult.element?.center;\n    const passed = hasValidRect(targetRect) && hasValidCenter(center);\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed,\n      durationMs: Date.now() - startTime,\n      message: passed\n        ? ''\n        : `Invalid locate result: ${JSON.stringify({\n            rect: targetRect,\n            center,\n          })}`,\n    });\n  } catch (error) {\n    return buildCheckResult('aiLocate', modelRuntime, {\n      passed: false,\n      durationMs: Date.now() - startTime,\n      message: error instanceof Error ? error.message : String(error),\n    });\n  }\n}\n\nexport async function runConnectivityTest(\n  config: ConnectivityTestConfig,\n): Promise<ConnectivityTestResult> {\n  const planningModelRuntime = buildConnectivityModelRuntime(\n    config.planningModelConfig,\n  );\n  const insightModelRuntime = buildConnectivityModelRuntime(\n    config.insightModelConfig,\n  );\n  const defaultModelRuntime = buildConnectivityModelRuntime(\n    config.defaultModelConfig,\n  );\n  const checks = await Promise.all([\n    runTextConnectivityCheck(planningModelRuntime),\n    runVisionConnectivityCheck(insightModelRuntime),\n    runAiLocateConnectivityCheck(defaultModelRuntime),\n  ]);\n\n  const passed = checks.every((item) => item.passed);\n  return {\n    passed,\n    message: passed ? undefined : buildConnectivityFailureMessage(checks),\n  };\n}\n"],"names":["TEXT_EXPECTED_TOKEN","isFiniteNumber","value","Number","hasValidRect","rect","hasValidCenter","Array","buildFixtureContext","ScreenshotItem","CONNECTIVITY_FIXTURE_IMAGE","Date","CONNECTIVITY_FIXTURE_SHOT_SIZE","buildCheckResult","name","modelRuntime","result","config","buildConnectivityModelRuntime","modelConfig","getModelRuntime","formatConnectivityCheckName","check","checkName","modelLabel","buildConnectivityFailureMessage","checks","failedChecks","item","detail","runTextConnectivityCheck","startTime","callAI","content","passed","error","Error","String","runVisionConnectivityCheck","runAiLocateConnectivityCheck","context","service","Service","locateResult","targetRect","center","JSON","runConnectivityTest","planningModelRuntime","insightModelRuntime","defaultModelRuntime","Promise","undefined"],"mappings":";;;;;AAYA,MAAMA,sBAAsB;AAuB5B,SAASC,eAAeC,KAAc;IACpC,OAAO,AAAiB,YAAjB,OAAOA,SAAsBC,OAAO,QAAQ,CAACD;AACtD;AAEA,SAASE,aAAaF,KAAc;IAClC,IAAI,CAACA,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAMG,OAAOH;IAOb,OACED,eAAeI,KAAK,IAAI,KACxBJ,eAAeI,KAAK,GAAG,KACvBJ,eAAeI,KAAK,KAAK,KACzBJ,eAAeI,KAAK,MAAM;AAE9B;AAEA,SAASC,eAAeJ,KAAc;IACpC,OACEK,MAAM,OAAO,CAACL,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZD,eAAeC,KAAK,CAAC,EAAE,KACvBD,eAAeC,KAAK,CAAC,EAAE;AAE3B;AAEA,SAASM;IACP,OAAO;QACL,YAAYC,eAAe,MAAM,CAACC,4BAA4BC,KAAK,GAAG;QACtE,UAAUC;QACV,0BAA0B;IAC5B;AACF;AAEA,SAASC,iBACPC,IAAyC,EACzCC,YAA0B,EAC1BC,MAGC;IAED,MAAM,EAAEC,MAAM,EAAE,GAAGF;IACnB,OAAO;QACLD;QACA,QAAQG,OAAO,MAAM;QACrB,WAAWA,OAAO,SAAS;QAC3B,aAAaA,OAAO,WAAW;QAC/B,GAAGD,MAAM;IACX;AACF;AAEA,SAASE,8BACPC,WAAyB;IAEzB,OAAOC,gBAAgB;QACrB,GAAGD,WAAW;QACd,YAAY;IACd;AACF;AAEA,SAASE,4BACPC,KAAkC;IAElC,MAAMC,YACJD,AAAe,eAAfA,MAAM,IAAI,GACN,oBACA,GAAGA,MAAM,IAAI,CAAC,EAAE,EAAE,gBAAgBA,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;IACnE,MAAME,aAAaF,MAAM,SAAS,IAAIA,MAAM,MAAM;IAClD,OAAO,GAAGC,UAAU,GAAG,EAAEC,WAAW,EAAE,EAAEF,MAAM,MAAM,CAAC,CAAC,CAAC;AACzD;AAEA,SAASG,gCACPC,MAAqC;IAErC,MAAMC,eAAeD,OAAO,MAAM,CAAC,CAACE,OAAS,CAACA,KAAK,MAAM;IACzD,IAAID,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAO;IAGT,OAAOA,aACJ,GAAG,CAAC,CAACC;QACJ,MAAMC,SAASD,KAAK,OAAO,IAAI;QAC/B,OAAO,CAAC,CAAC,EAAEP,4BAA4BO,MAAM,GAAG,EAAEC,QAAQ;IAC5D,GACC,IAAI,CAAC;AACV;AAEA,eAAeC,yBACbf,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMK,SAAS,MAAMgB,OACnB;YACE;gBACE,MAAM;gBACN,SAAS;YACX;YACA;gBACE,MAAM;gBACN,SAAS,CAAC,eAAe,EAAEhC,qBAAqB;YAClD;SACD,EACDe;QAEF,MAAMkB,UAAUjB,OAAO,OAAO,CAAC,IAAI;QACnC,MAAMkB,SAASD,QAAQ,QAAQ,CAACjC;QAChC,OAAOa,iBAAiB,QAAQE,cAAc;YAC5CmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SAAS,KAAK,CAAC,qBAAqB,EAAED,SAAS;QAC1D;IACF,EAAE,OAAOE,OAAO;QACd,OAAOtB,iBAAiB,QAAQE,cAAc;YAC5C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeG,2BACbvB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAMqB,OACJ;YACE;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM;oBACR;oBACA;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKtB;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD,EACDK;QAEF,OAAOF,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAAS;QACX;IACF,EAAE,OAAOI,OAAO;QACd,OAAOtB,iBAAiB,UAAUE,cAAc;YAC9C,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEA,eAAeI,6BACbxB,YAA0B;IAE1B,MAAMgB,YAAYpB,KAAK,GAAG;IAC1B,IAAI;QACF,MAAM6B,UAAUhC;QAChB,MAAMiC,UAAU,IAAIC,UAAQF;QAC5B,MAAMG,eAAe,MAAMF,QAAQ,MAAM,CACvC;YAAE,QAAQ;QAA0B,GACpC,CAAC,GACD1B;QAEF,MAAM6B,aAAaD,aAAa,IAAI,IAAIA,aAAa,OAAO,EAAE;QAC9D,MAAME,SAASF,aAAa,OAAO,EAAE;QACrC,MAAMT,SAAS9B,aAAawC,eAAetC,eAAeuC;QAC1D,OAAOhC,iBAAiB,YAAYE,cAAc;YAChDmB;YACA,YAAYvB,KAAK,GAAG,KAAKoB;YACzB,SAASG,SACL,KACA,CAAC,uBAAuB,EAAEY,KAAK,SAAS,CAAC;gBACvC,MAAMF;gBACNC;YACF,IAAI;QACV;IACF,EAAE,OAAOV,OAAO;QACd,OAAOtB,iBAAiB,YAAYE,cAAc;YAChD,QAAQ;YACR,YAAYJ,KAAK,GAAG,KAAKoB;YACzB,SAASI,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;QAC3D;IACF;AACF;AAEO,eAAeY,oBACpB9B,MAA8B;IAE9B,MAAM+B,uBAAuB9B,8BAC3BD,OAAO,mBAAmB;IAE5B,MAAMgC,sBAAsB/B,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMiC,sBAAsBhC,8BAC1BD,OAAO,kBAAkB;IAE3B,MAAMS,SAAS,MAAMyB,QAAQ,GAAG,CAAC;QAC/BrB,yBAAyBkB;QACzBV,2BAA2BW;QAC3BV,6BAA6BW;KAC9B;IAED,MAAMhB,SAASR,OAAO,KAAK,CAAC,CAACE,OAASA,KAAK,MAAM;IACjD,OAAO;QACLM;QACA,SAASA,SAASkB,SAAY3B,gCAAgCC;IAChE;AACF"}

package/dist/es/ai-model/inspect.mjs

@@ -159,7 +159,7 @@ async function genericLocate(_elementDescription, options, locateRequest) {
             ]
         };
     }
-    const rawResponse = JSON.stringify(res.content);
+    const rawResponse = res.contentString;
     let errors = 'errors' in res.content ? res.content.errors : [];
     const resultAdapter = adapter.locate.resultAdapter;
     if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) return {
@@ -256,7 +256,7 @@ async function AiLocateSection(options) {
     if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) return {
         searchAreaConfig: void 0,
         error: sectionError,
-        rawResponse: JSON.stringify(result.content),
+        rawResponse: result.contentString,
         rawChoiceMessage: result.rawChoiceMessage,
         usage: result.usage
     };
@@ -286,7 +286,7 @@ async function AiLocateSection(options) {
     return {
         searchAreaConfig,
         error: sectionError,
-        rawResponse: JSON.stringify(result.content),
+        rawResponse: result.contentString,
         rawChoiceMessage: result.rawChoiceMessage,
         usage: result.usage
     };

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  let res: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AIElementLocateResponse>>\n  >;\n  try {\n    res = await callAIWithObjectResponse<AIElementLocateResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'locate',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      rawResponse,\n      rawChoiceMessage: res.rawChoiceMessage,\n      usage: res.usage,\n      reasoningContent: res.reasoning_content,\n      errors: errors as string[],\n    };\n  }\n\n  let targetPixelBbox;\n  try {\n    targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(\n      res.content,\n      {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    );\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse locate result: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    locatedPixelBbox: targetPixelBbox,\n    rawResponse,\n    rawChoiceMessage: res.rawChoiceMessage,\n    usage: res.usage,\n    reasoningContent: res.reasoning_content,\n    errors: errors as string[],\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let result: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n  >;\n  try {\n    result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'section-locator',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n    };\n  }\n\n  let searchAreaConfig:\n    | Awaited<ReturnType<typeof buildSearchAreaConfig>>\n    | undefined;\n  let sectionError = result.content.error;\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: JSON.stringify(result.content),\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const adaptedResult =\n      resultAdapter.adaptSectionLocateResultToPixelBboxGroup(result.content, {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      });\n    const mergedRect = mergePixelBboxesToRect([\n      adaptedResult.target,\n      ...(adaptedResult.references ?? []),\n    ]);\n    debugSection('mergedRect %j', mergedRect);\n\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    sectionError = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n  }\n\n  return {\n    searchAreaConfig,\n    error: sectionError,\n    rawResponse: JSON.stringify(result.content),\n    rawChoiceMessage: result.rawChoiceMessage,\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'text',\n      text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n    });\n\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime, {\n    abortSignal: options.abortSignal,\n  });\n\n  let parseResult: AIDataExtractionResponse<T>;\n  try {\n    parseResult = parseXMLExtractionResponse<T>(rawResponse);\n  } catch (parseError) {\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  return {\n    parseResult,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","res","callAIWithObjectResponse","callError","errorMessage","String","AIResponseParseError","JSON","resultAdapter","targetPixelBbox","e","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","result","searchAreaConfig","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","parseErrorMessage","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AAkEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAEO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,OACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,+BACJC,6BAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,mBAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,gBACXC,0CACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,wBAAwBC,kBAC5BxB,cAAc,sBAAsB;IAEtC,MAAMyB,eAAeC,4BACnBJ,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMK,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAa5B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMO,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIvB,cAAc,sBAAsB,EACtC8B,KAAK,IAAI,IAAI9B,cAAc,sBAAsB;IAGnD,IAAI+B;IAGJ,IAAI;QACFA,MAAM,MAAMC,yBACVF,MACAT,cACA;YACE,aAAavC,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOmD,WAAW;QAClB,MAAMC,eACJD,qBAAqBd,QAAQc,UAAU,OAAO,GAAGE,OAAOF;QAC1D,MAAM1B,cACJ0B,qBAAqBG,uBACjBH,UAAU,WAAW,GACrBC;QACN,MAAMzB,QACJwB,qBAAqBG,uBAAuBH,UAAU,KAAK,GAAGrD;QAChE,MAAM4B,mBACJyB,qBAAqBG,uBACjBH,UAAU,gBAAgB,GAC1BrD;QACN,OAAO;YACL2B;YACAC;YACAC;YACA,QAAQ;gBAAC,CAAC,eAAe,EAAEyB,cAAc;aAAC;QAC5C;IACF;IAEA,MAAM3B,cAAc8B,KAAK,SAAS,CAACN,IAAI,OAAO;IAE9C,IAAIpB,SACF,YAAYoB,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,MAAMO,gBAAgBhB,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAAChD,gBAAgByD,IAAI,OAAO,EAAEO,cAAc,UAAU,CAAC,SAAS,GAClE,OAAO;QACL/B;QACA,kBAAkBwB,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,kBAAkBA,IAAI,iBAAiB;QACvC,QAAQpB;IACV;IAGF,IAAI4B;IACJ,IAAI;QACFA,kBAAkBD,cAAc,mCAAmC,CACjEP,IAAI,OAAO,EACX;YACE,cAAcJ,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IAEJ,EAAE,OAAOa,GAAG;QACV,MAAMtB,MACJsB,aAAarB,QACT,CAAC,+BAA+B,EAAEqB,EAAE,OAAO,EAAE,GAC7C;QACN,IAAI,AAAC7B,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;aAFtBP,SAAS;YAACO;SAAI;IAIlB;IAEA,OAAO;QACL,kBAAkBqB;QAClBhC;QACA,kBAAkBwB,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,kBAAkBA,IAAI,iBAAiB;QACvC,QAAQpB;IACV;AACF;AAEO,eAAe8B,gBAAgB3D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE2D,kBAAkB,EAAE,GAAG5D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMqB,mBAAmB5D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM4C,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAae;QACb,OAAO5D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMG,eAAemB,4BACnBtB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMuB,gCAAgCC,0BACpC7C,mBAAmByC;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMjD,+BACnBC,6BAA6B2C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAGJ,IAAI;QACFA,SAAS,MAAMhB,yBACbF,MACAT,cACA;YACE,aAAavC,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOmD,WAAW;QAClB,MAAMC,eACJD,qBAAqBd,QAAQc,UAAU,OAAO,GAAGE,OAAOF;QAC1D,MAAM1B,cACJ0B,qBAAqBG,uBACjBH,UAAU,WAAW,GACrBC;QACN,MAAMzB,QACJwB,qBAAqBG,uBAAuBH,UAAU,KAAK,GAAGrD;QAChE,MAAM4B,mBACJyB,qBAAqBG,uBACjBH,UAAU,gBAAgB,GAC1BrD;QACN,OAAO;YACL,kBAAkBA;YAClB,OAAO,CAAC,eAAe,EAAEsD,cAAc;YACvC3B;YACAC;YACAC;QACF;IACF;IAEA,IAAIwC;IAGJ,IAAIC,eAAeF,OAAO,OAAO,CAAC,KAAK;IACvC,MAAMV,gBAAgBhB,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAAChD,gBAAgB0E,OAAO,OAAO,EAAEV,cAAc,UAAU,CAAC,SAAS,GACrE,OAAO;QACL,kBAAkB1D;QAClB,OAAOsE;QACP,aAAab,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMG,gBACJb,cAAc,wCAAwC,CAACU,OAAO,OAAO,EAAE;YACrE,cAAcrB,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACF,MAAMyB,aAAaC,uBAAuB;YACxCF,cAAc,MAAM;eAChBA,cAAc,UAAU,IAAI,EAAE;SACnC;QACD9E,aAAa,iBAAiB+E;QAE9B,MAAME,eAAenE,iBAAiBiE,YAAYrE,QAAQ,QAAQ;QAClE,MAAMwE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1CjF,aAAa,2BAA2BiF;QAExCL,mBAAmB,MAAMpE,sBAAsB;YAC7CE;YACA,UAAUqE;QACZ;QAEA/E,aACE,uDACAkF,eACAC,gBACAP,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;IAElC,EAAE,OAAOhC,OAAO;QACd,MAAMwC,oBACJxC,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACNiC,eAAeA,eACX,GAAGA,aAAa,EAAE,EAAEO,kBAAkB,CAAC,CAAC,GACxCA;IACN;IAEA,OAAO;QACLR;QACA,OAAOC;QACP,aAAab,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeU,qBAAwB5E,OAQ7C;IACC,MAAM,EAAE6E,SAAS,EAAE5E,OAAO,EAAE6E,aAAa,EAAEC,gBAAgB,EAAExC,YAAY,EAAE,GACzEvC;IACF,MAAM2C,eAAeqC,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB5D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMgF,wBAAwBC,uBAC5BlF,QAAQ,eAAe,IAAI,IAC3B6E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/CK,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QAEAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBACT,KAAKtB;gBACL,QAAQ;YACV;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMjD,+BAA+B+D;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,MAAM,EACJ,SAASxC,WAAW,EACpBE,KAAK,EACLyD,iBAAiB,EACjB1D,gBAAgB,EACjB,GAAG,MAAM2D,OAAOrC,MAAMT,cAAc;QACnC,aAAavC,QAAQ,WAAW;IAClC;IAEA,IAAIsF;IACJ,IAAI;QACFA,cAAcC,2BAA8B9D;IAC9C,EAAE,OAAO+D,YAAY;QACnB,MAAMpC,eACJoC,sBAAsBnD,QAAQmD,WAAW,OAAO,GAAGnC,OAAOmC;QAC5D,MAAM,IAAIlC,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClC3B,aACAE,OACAD;IAEJ;IAEA,OAAO;QACL4D;QACA7D;QACAC;QACAC;QACAyD;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBnD,YAA0B;IAK1B,MAAMI,eAAegD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAM1C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASiD;QACX;KACD;IAEDvG,aAAa,yCAAyCqG;IAEtD,MAAMxB,SAAS,MAAMhB,yBACnBF,MACAT,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB2B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  let res: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AIElementLocateResponse>>\n  >;\n  try {\n    res = await callAIWithObjectResponse<AIElementLocateResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'locate',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n\n  const rawResponse = res.contentString;\n\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      rawResponse,\n      rawChoiceMessage: res.rawChoiceMessage,\n      usage: res.usage,\n      reasoningContent: res.reasoning_content,\n      errors: errors as string[],\n    };\n  }\n\n  let targetPixelBbox;\n  try {\n    targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(\n      res.content,\n      {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    );\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse locate result: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    locatedPixelBbox: targetPixelBbox,\n    rawResponse,\n    rawChoiceMessage: res.rawChoiceMessage,\n    usage: res.usage,\n    reasoningContent: res.reasoning_content,\n    errors: errors as string[],\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let result: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n  >;\n  try {\n    result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'section-locator',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n    };\n  }\n\n  let searchAreaConfig:\n    | Awaited<ReturnType<typeof buildSearchAreaConfig>>\n    | undefined;\n  let sectionError = result.content.error;\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.contentString,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const adaptedResult =\n      resultAdapter.adaptSectionLocateResultToPixelBboxGroup(result.content, {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      });\n    const mergedRect = mergePixelBboxesToRect([\n      adaptedResult.target,\n      ...(adaptedResult.references ?? []),\n    ]);\n    debugSection('mergedRect %j', mergedRect);\n\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    sectionError = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n  }\n\n  return {\n    searchAreaConfig,\n    error: sectionError,\n    rawResponse: result.contentString,\n    rawChoiceMessage: result.rawChoiceMessage,\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'text',\n      text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n    });\n\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime, {\n    abortSignal: options.abortSignal,\n  });\n\n  let parseResult: AIDataExtractionResponse<T>;\n  try {\n    parseResult = parseXMLExtractionResponse<T>(rawResponse);\n  } catch (parseError) {\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  return {\n    parseResult,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","res","callAIWithObjectResponse","callError","errorMessage","String","AIResponseParseError","resultAdapter","targetPixelBbox","e","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","result","searchAreaConfig","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","parseErrorMessage","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AAkEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAEO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,OACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,+BACJC,6BAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,mBAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,gBACXC,0CACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,wBAAwBC,kBAC5BxB,cAAc,sBAAsB;IAEtC,MAAMyB,eAAeC,4BACnBJ,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMK,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAa5B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMO,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIvB,cAAc,sBAAsB,EACtC8B,KAAK,IAAI,IAAI9B,cAAc,sBAAsB;IAGnD,IAAI+B;IAGJ,IAAI;QACFA,MAAM,MAAMC,yBACVF,MACAT,cACA;YACE,aAAavC,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOmD,WAAW;QAClB,MAAMC,eACJD,qBAAqBd,QAAQc,UAAU,OAAO,GAAGE,OAAOF;QAC1D,MAAM1B,cACJ0B,qBAAqBG,uBACjBH,UAAU,WAAW,GACrBC;QACN,MAAMzB,QACJwB,qBAAqBG,uBAAuBH,UAAU,KAAK,GAAGrD;QAChE,MAAM4B,mBACJyB,qBAAqBG,uBACjBH,UAAU,gBAAgB,GAC1BrD;QACN,OAAO;YACL2B;YACAC;YACAC;YACA,QAAQ;gBAAC,CAAC,eAAe,EAAEyB,cAAc;aAAC;QAC5C;IACF;IAEA,MAAM3B,cAAcwB,IAAI,aAAa;IAErC,IAAIpB,SACF,YAAYoB,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,MAAMM,gBAAgBf,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAAChD,gBAAgByD,IAAI,OAAO,EAAEM,cAAc,UAAU,CAAC,SAAS,GAClE,OAAO;QACL9B;QACA,kBAAkBwB,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,kBAAkBA,IAAI,iBAAiB;QACvC,QAAQpB;IACV;IAGF,IAAI2B;IACJ,IAAI;QACFA,kBAAkBD,cAAc,mCAAmC,CACjEN,IAAI,OAAO,EACX;YACE,cAAcJ,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IAEJ,EAAE,OAAOY,GAAG;QACV,MAAMrB,MACJqB,aAAapB,QACT,CAAC,+BAA+B,EAAEoB,EAAE,OAAO,EAAE,GAC7C;QACN,IAAI,AAAC5B,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;aAFtBP,SAAS;YAACO;SAAI;IAIlB;IAEA,OAAO;QACL,kBAAkBoB;QAClB/B;QACA,kBAAkBwB,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,kBAAkBA,IAAI,iBAAiB;QACvC,QAAQpB;IACV;AACF;AAEO,eAAe6B,gBAAgB1D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE0D,kBAAkB,EAAE,GAAG3D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMoB,mBAAmB3D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM4C,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAac;QACb,OAAO3D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMG,eAAekB,4BACnBrB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMsB,gCAAgCC,0BACpC5C,mBAAmBwC;IAErB,MAAMX,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMiB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMhD,+BACnBC,6BAA6B0C;QAE/BX,KAAK,IAAI,IAAIgB;IACf;IAEA,IAAIC;IAGJ,IAAI;QACFA,SAAS,MAAMf,yBACbF,MACAT,cACA;YACE,aAAavC,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOmD,WAAW;QAClB,MAAMC,eACJD,qBAAqBd,QAAQc,UAAU,OAAO,GAAGE,OAAOF;QAC1D,MAAM1B,cACJ0B,qBAAqBG,uBACjBH,UAAU,WAAW,GACrBC;QACN,MAAMzB,QACJwB,qBAAqBG,uBAAuBH,UAAU,KAAK,GAAGrD;QAChE,MAAM4B,mBACJyB,qBAAqBG,uBACjBH,UAAU,gBAAgB,GAC1BrD;QACN,OAAO;YACL,kBAAkBA;YAClB,OAAO,CAAC,eAAe,EAAEsD,cAAc;YACvC3B;YACAC;YACAC;QACF;IACF;IAEA,IAAIuC;IAGJ,IAAIC,eAAeF,OAAO,OAAO,CAAC,KAAK;IACvC,MAAMV,gBAAgBf,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAAChD,gBAAgByE,OAAO,OAAO,EAAEV,cAAc,UAAU,CAAC,SAAS,GACrE,OAAO;QACL,kBAAkBzD;QAClB,OAAOqE;QACP,aAAaF,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMG,gBACJb,cAAc,wCAAwC,CAACU,OAAO,OAAO,EAAE;YACrE,cAAcpB,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACF,MAAMwB,aAAaC,uBAAuB;YACxCF,cAAc,MAAM;eAChBA,cAAc,UAAU,IAAI,EAAE;SACnC;QACD7E,aAAa,iBAAiB8E;QAE9B,MAAME,eAAelE,iBAAiBgE,YAAYpE,QAAQ,QAAQ;QAClE,MAAMuE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1ChF,aAAa,2BAA2BgF;QAExCL,mBAAmB,MAAMnE,sBAAsB;YAC7CE;YACA,UAAUoE;QACZ;QAEA9E,aACE,uDACAiF,eACAC,gBACAP,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;IAElC,EAAE,OAAO/B,OAAO;QACd,MAAMuC,oBACJvC,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACNgC,eAAeA,eACX,GAAGA,aAAa,EAAE,EAAEO,kBAAkB,CAAC,CAAC,GACxCA;IACN;IAEA,OAAO;QACLR;QACA,OAAOC;QACP,aAAaF,OAAO,aAAa;QACjC,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeU,qBAAwB3E,OAQ7C;IACC,MAAM,EAAE4E,SAAS,EAAE3E,OAAO,EAAE4E,aAAa,EAAEC,gBAAgB,EAAEvC,YAAY,EAAE,GACzEvC;IACF,MAAM2C,eAAeoC,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB3D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM+E,wBAAwBC,uBAC5BjF,QAAQ,eAAe,IAAI,IAC3B4E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/CK,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QAEAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBACT,KAAKtB;gBACL,QAAQ;YACV;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMhC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASuC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMhD,+BAA+B8D;QACpD9B,KAAK,IAAI,IAAIgB;IACf;IAEA,MAAM,EACJ,SAASvC,WAAW,EACpBE,KAAK,EACLwD,iBAAiB,EACjBzD,gBAAgB,EACjB,GAAG,MAAM0D,OAAOpC,MAAMT,cAAc;QACnC,aAAavC,QAAQ,WAAW;IAClC;IAEA,IAAIqF;IACJ,IAAI;QACFA,cAAcC,2BAA8B7D;IAC9C,EAAE,OAAO8D,YAAY;QACnB,MAAMnC,eACJmC,sBAAsBlD,QAAQkD,WAAW,OAAO,GAAGlC,OAAOkC;QAC5D,MAAM,IAAIjC,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClC3B,aACAE,OACAD;IAEJ;IAEA,OAAO;QACL2D;QACA5D;QACAC;QACAC;QACAwD;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBlD,YAA0B;IAK1B,MAAMI,eAAe+C;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMzC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASgD;QACX;KACD;IAEDtG,aAAa,yCAAyCoG;IAEtD,MAAMxB,SAAS,MAAMf,yBACnBF,MACAT,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB0B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/model-adapter/resolve.mjs

@@ -1,4 +1,4 @@
-import { normalJsonParser } from "../service-caller/json.mjs";
+import { parseModelResponseJson } from "../service-caller/json.mjs";
 import { resolveChatCompletion } from "./chat-completion.mjs";
 import { resolveLocate } from "./locate.mjs";
 import { resolveCustomPlanningDefinition, resolvePlanning } from "./planning.mjs";
@@ -13,7 +13,7 @@ function _define_property(obj, key, value) {
     return obj;
 }
 function resolveJsonParser(jsonParser) {
-    if (!jsonParser || 'lenient-json' === jsonParser) return normalJsonParser;
+    if (!jsonParser || 'lenient-json' === jsonParser) return parseModelResponseJson;
     if ('function' == typeof jsonParser) return jsonParser;
     throw new Error(`Unknown json parser preset: ${jsonParser}`);
 }

package/dist/es/ai-model/model-adapter/resolve.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { normalJsonParser } from '../service-caller/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport type {\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return normalJsonParser;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.planning = resolvePlanning(config.planning, resolvedCustomPlanner);\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner);\n  }\n}\n"],"names":["resolveJsonParser","jsonParser","normalJsonParser","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolvePlanning","resolveLocate"],"mappings":";;;;;;;;;;;;;;AAcA,SAASA,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAOX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QANjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGR,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,eAAe,GAAGH,uBAAuBG,OAAO,eAAe;QACpE,MAAMG,gBACJH,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGI;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,QAAQ,GAAGG,gBAAgBP,OAAO,QAAQ,EAAEK;QACjD,IAAI,CAAC,MAAM,GAAGG,cAAcR,OAAO,MAAM,EAAEK;IAC7C;AACF"}
+{"version":3,"file":"ai-model/model-adapter/resolve.mjs","sources":["../../../../src/ai-model/model-adapter/resolve.ts"],"sourcesContent":["import { parseModelResponseJson } from '../service-caller/json';\nimport { resolveChatCompletion } from './chat-completion';\nimport { resolveLocate } from './locate';\nimport { resolveCustomPlanningDefinition, resolvePlanning } from './planning';\nimport type {\n  ChatCompletionAdapter,\n  ImagePreprocessPolicy,\n  JsonParser,\n  LocateAdapter,\n  ModelAdapter,\n  ModelAdapterDefinition,\n  PlanningAdapter,\n} from './types';\n\nfunction resolveJsonParser(\n  jsonParser: ModelAdapterDefinition['jsonParser'],\n): JsonParser {\n  if (!jsonParser || jsonParser === 'lenient-json') {\n    return parseModelResponseJson;\n  }\n\n  if (typeof jsonParser === 'function') {\n    return jsonParser;\n  }\n\n  throw new Error(`Unknown json parser preset: ${jsonParser}`);\n}\n\nfunction resolveImagePreprocess(\n  imagePreprocess: ModelAdapterDefinition['imagePreprocess'],\n): ImagePreprocessPolicy {\n  return {\n    padBlockSize: imagePreprocess?.padBlockSize,\n  };\n}\n\nexport class ResolvedModelAdapter implements ModelAdapter {\n  readonly jsonParser: JsonParser;\n  readonly chatCompletion: ChatCompletionAdapter;\n  readonly imagePreprocess: ImagePreprocessPolicy;\n  readonly planning: PlanningAdapter;\n  readonly locate: LocateAdapter;\n\n  constructor(config: ModelAdapterDefinition, modelFamily: string) {\n    this.jsonParser = resolveJsonParser(config.jsonParser);\n    this.chatCompletion = resolveChatCompletion(config.chatCompletion);\n    this.imagePreprocess = resolveImagePreprocess(config.imagePreprocess);\n    const customPlanner =\n      config.planning?.kind === 'custom' ? config.planning.planner : undefined;\n    const resolvedCustomPlanner = customPlanner\n      ? resolveCustomPlanningDefinition(customPlanner)\n      : undefined;\n    this.planning = resolvePlanning(config.planning, resolvedCustomPlanner);\n    this.locate = resolveLocate(config.locate, resolvedCustomPlanner);\n  }\n}\n"],"names":["resolveJsonParser","jsonParser","parseModelResponseJson","Error","resolveImagePreprocess","imagePreprocess","ResolvedModelAdapter","config","modelFamily","resolveChatCompletion","customPlanner","undefined","resolvedCustomPlanner","resolveCustomPlanningDefinition","resolvePlanning","resolveLocate"],"mappings":";;;;;;;;;;;;;;AAcA,SAASA,kBACPC,UAAgD;IAEhD,IAAI,CAACA,cAAcA,AAAe,mBAAfA,YACjB,OAAOC;IAGT,IAAI,AAAsB,cAAtB,OAAOD,YACT,OAAOA;IAGT,MAAM,IAAIE,MAAM,CAAC,4BAA4B,EAAEF,YAAY;AAC7D;AAEA,SAASG,uBACPC,eAA0D;IAE1D,OAAO;QACL,cAAcA,iBAAiB;IACjC;AACF;AAEO,MAAMC;IAOX,YAAYC,MAA8B,EAAEC,WAAmB,CAAE;QANjE,uBAAS,cAAT;QACA,uBAAS,kBAAT;QACA,uBAAS,mBAAT;QACA,uBAAS,YAAT;QACA,uBAAS,UAAT;QAGE,IAAI,CAAC,UAAU,GAAGR,kBAAkBO,OAAO,UAAU;QACrD,IAAI,CAAC,cAAc,GAAGE,sBAAsBF,OAAO,cAAc;QACjE,IAAI,CAAC,eAAe,GAAGH,uBAAuBG,OAAO,eAAe;QACpE,MAAMG,gBACJH,OAAO,QAAQ,EAAE,SAAS,WAAWA,OAAO,QAAQ,CAAC,OAAO,GAAGI;QACjE,MAAMC,wBAAwBF,gBAC1BG,gCAAgCH,iBAChCC;QACJ,IAAI,CAAC,QAAQ,GAAGG,gBAAgBP,OAAO,QAAQ,EAAEK;QACjD,IAAI,CAAC,MAAM,GAAGG,cAAcR,OAAO,MAAM,EAAEK;IAC7C;AACF"}

package/dist/es/ai-model/models/doubao.mjs

@@ -1,6 +1,4 @@
-import { assert } from "@midscene/shared/utils";
-import { jsonrepair } from "jsonrepair";
-import { extractJSONFromCodeBlock, safeParseJson } from "../service-caller/json.mjs";
+import { parseModelResponseJson } from "../service-caller/json.mjs";
 import { createLocateResultValue, unwrapCoordinateListLikeInput } from "../shared/model-locate-result/index.mjs";
 const doubaoBboxCoordinatesMeta = {
     shape: 'bbox',
@@ -12,68 +10,48 @@ const doubaoPointCoordinatesMeta = {
     order: 'xy',
     normalizedBy: 1000
 };
-function normalizeDoubaoJsonObject(obj, context = {}) {
-    if (null == obj) return obj;
-    if (Array.isArray(obj)) return obj.map((item)=>normalizeDoubaoJsonObject(item, context));
-    if ('object' == typeof obj) {
-        const normalized = {};
-        for (const [key, value] of Object.entries(obj)){
-            const trimmedKey = key.trim();
-            const preserveStringValue = context.preserveStringValueKeys?.includes(trimmedKey) ?? false;
-            const normalizedValue = 'string' == typeof value ? preserveStringValue ? value : value.trim() : normalizeDoubaoJsonObject(value, context);
-            normalized[trimmedKey] = normalizedValue;
-        }
-        return normalized;
-    }
-    return 'string' == typeof obj ? obj.trim() : obj;
+function parseNumbersFromBboxString(input) {
+    return (input.match(/\d+/g) ?? []).map(Number).filter(Number.isFinite);
 }
-function shouldRepairDoubaoLocateJson(source) {
-    return 'locate' === source || 'section-locator' === source || 'planning-action-param' === source;
+function parseLeadingNumberFromString(input) {
+    const match = input.match(/^\s*(\d+)/);
+    return match ? Number(match[1]) : void 0;
 }
-function preprocessDoubaoLocateJson(input) {
-    if (input.includes('bbox')) while(/\d+\s+\d+/.test(input))input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
-    return input;
+function isCleanCoordinateString(input) {
+    return /^\s*\d+(?:[\s,;]+\d+)*\s*[,;]?\s*$/.test(input);
 }
-const doubaoJsonParser = (raw, context = {
-    source: 'generic-object'
-})=>{
-    const { source } = context;
-    try {
-        return safeParseJson(raw, context);
-    } catch (firstError) {
-        if (!shouldRepairDoubaoLocateJson(source)) throw firstError;
-        const jsonString = preprocessDoubaoLocateJson(extractJSONFromCodeBlock(raw));
-        try {
-            return normalizeDoubaoJsonObject(JSON.parse(jsonrepair(jsonString)), context);
-        } catch (error) {
-            throw Error(`failed to parse LLM response into JSON. Error - ${String(error ?? firstError ?? 'unknown error')}. Response - \n ${raw}`);
+function parseNumbersFromBboxArray(input) {
+    const numbers = [];
+    for (const item of input){
+        if ('number' == typeof item) {
+            numbers.push(item);
+            continue;
+        }
+        if ('string' == typeof item) if (isCleanCoordinateString(item)) {
+            numbers.push(...parseNumbersFromBboxString(item));
+            continue;
+        } else {
+            const leadingNumber = parseLeadingNumberFromString(item);
+            if (void 0 !== leadingNumber) numbers.push(leadingNumber);
         }
+        break;
     }
-};
+    return numbers.filter(Number.isFinite);
+}
+function parseNumbersFromTaggedBboxArray(input) {
+    if (input.length < 2) return null;
+    const taggedBbox = input[1];
+    if ('string' == typeof taggedBbox) return parseNumbersFromBboxString(taggedBbox);
+    if (Array.isArray(taggedBbox)) return parseNumbersFromBboxArray(taggedBbox);
+    return null;
+}
 function parseDoubaoRawLocateValue(input) {
     const bbox = unwrapCoordinateListLikeInput(input);
-    if ('string' == typeof bbox) {
-        assert(/^(\d+)\s(\d+)\s(\d+)\s(\d+)$/.test(bbox.trim()), `invalid bbox data string for doubao-vision mode: ${bbox}`);
-        const splitted = bbox.split(' ');
-        if (4 === splitted.length) return createLocateResultValue(doubaoBboxCoordinatesMeta, [
-            Number(splitted[0]),
-            Number(splitted[1]),
-            Number(splitted[2]),
-            Number(splitted[3])
-        ]);
-        throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);
-    }
     let bboxList = [];
-    if (Array.isArray(bbox) && 'string' == typeof bbox[0]) bbox.forEach((item)=>{
-        if ('string' == typeof item && item.includes(',')) {
-            const [x, y] = item.split(',');
-            bboxList.push(Number(x.trim()), Number(y.trim()));
-        } else if ('string' == typeof item && item.includes(' ')) {
-            const [x, y] = item.split(' ');
-            bboxList.push(Number(x.trim()), Number(y.trim()));
-        } else bboxList.push(Number(item));
-    });
-    else bboxList = bbox;
+    if ('string' == typeof bbox) {
+        bboxList = parseNumbersFromBboxString(bbox);
+        if (4 !== bboxList.length) throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);
+    } else bboxList = Array.isArray(bbox) ? 'string' == typeof bbox[0] && 'bbox' === bbox[0].trim() ? parseNumbersFromTaggedBboxArray(bbox) ?? [] : parseNumbersFromBboxArray(bbox) : bbox;
     if (4 === bboxList.length || 5 === bboxList.length) return createLocateResultValue(doubaoBboxCoordinatesMeta, [
         bboxList[0],
         bboxList[1],
@@ -84,7 +62,7 @@ function parseDoubaoRawLocateValue(input) {
         bboxList[0],
         bboxList[1]
     ]);
-    if (8 === bbox.length) return createLocateResultValue(doubaoBboxCoordinatesMeta, [
+    if (8 === bboxList.length) return createLocateResultValue(doubaoBboxCoordinatesMeta, [
         bboxList[0],
         bboxList[1],
         bboxList[4],
@@ -114,7 +92,7 @@ const buildDoubaoChatCompletionParams = (input)=>{
     };
 };
 const doubaoVisionAdapter = {
-    jsonParser: doubaoJsonParser,
+    jsonParser: parseModelResponseJson,
     chatCompletion: {
         unsupportedUserConfig: [
             'reasoningBudget'
@@ -133,6 +111,6 @@ const doubaoAdapters = {
     'doubao-vision': doubaoVisionAdapter,
     'doubao-seed': doubaoVisionAdapter
 };
-export { doubaoAdapters, normalizeDoubaoJsonObject, parseDoubaoRawLocateValue, preprocessDoubaoLocateJson, shouldRepairDoubaoLocateJson };
+export { doubaoAdapters, parseDoubaoRawLocateValue };
 
 //# sourceMappingURL=doubao.mjs.map