@donggui/core 1.5.12 → 1.5.14

package/dist/es/agent/task-cache.mjs

@@ -220,7 +220,9 @@ class TaskCache {
         else this.cacheAdapter = cacheAdapterOrPath;
         if (!this.cacheAdapter) this.cacheFilePath = ifInBrowser || ifInWorker ? void 0 : join(getMidsceneRunSubDir('cache'), `${this.cacheId}${cacheFileExt}`);
         let cacheContent;
-        if (!this.writeOnlyMode) cacheContent = this.loadCacheFromFile();
+        if (!this.writeOnlyMode) {
+            if (!this.cacheAdapter) cacheContent = this.loadCacheFromFile();
+        }
         if (!cacheContent) cacheContent = {
             midsceneVersion: getMidsceneVersion(),
             cacheId: this.cacheId,

package/dist/es/agent/task-cache.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-cache.mjs","sources":["../../../src/agent/task-cache.ts"],"sourcesContent":["import assert from 'node:assert';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { isDeepStrictEqual } from 'node:util';\nimport type { TUserPrompt } from '@/ai-model';\nimport type { ElementCacheFeature } from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport {\n  MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { generateHashId } from '@midscene/shared/utils';\nimport { replaceIllegalPathCharsAndSpace } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\nimport semver from 'semver';\nimport type { CacheAdapter } from './cache-adapter';\nimport { getMidsceneVersion } from './utils';\n\nconst DEFAULT_CACHE_MAX_FILENAME_LENGTH = 200;\n\nexport const debug = getDebug('cache');\n\nexport interface PlanningCache {\n  type: 'plan';\n  prompt: string;\n  yamlWorkflow: string;\n}\n\nexport interface LocateCache {\n  type: 'locate';\n  prompt: TUserPrompt;\n  cache?: ElementCacheFeature;\n  /** @deprecated kept for backward compatibility */\n  xpaths?: string[];\n}\n\nexport interface MatchCacheResult<T extends PlanningCache | LocateCache> {\n  cacheContent: T;\n  updateFn: (\n    cb: (cache: T) => void,\n    options?: { ttl?: number },\n  ) => Promise<void>;\n}\n\nexport type CacheFileContent = {\n  midsceneVersion: string;\n  cacheId: string;\n  caches: Array<PlanningCache | LocateCache>;\n};\n\nconst lowestSupportedMidsceneVersion = '0.16.10';\nexport const cacheFileExt = '.cache.yaml';\n\nexport class TaskCache {\n  cacheId: string;\n\n  cacheFilePath?: string;\n\n  cache: CacheFileContent;\n\n  isCacheResultUsed: boolean;\n\n  cacheOriginalLength: number;\n\n  readOnlyMode: boolean;\n\n  writeOnlyMode: boolean;\n\n  private matchedCacheIndices: Set<string> = new Set();\n\n  private cacheAdapter?: CacheAdapter;\n\n  constructor(\n    cacheId: string,\n    isCacheResultUsed: boolean,\n    cacheAdapterOrPath?: CacheAdapter | string,\n    options: { readOnly?: boolean; writeOnly?: boolean } = {},\n  ) {\n    assert(cacheId, 'cacheId is required');\n    let safeCacheId = replaceIllegalPathCharsAndSpace(cacheId);\n    const cacheMaxFilenameLength =\n      globalConfigManager.getEnvConfigValueAsNumber(\n        MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n      ) ?? DEFAULT_CACHE_MAX_FILENAME_LENGTH;\n    if (Buffer.byteLength(safeCacheId, 'utf8') > cacheMaxFilenameLength) {\n      const prefix = safeCacheId.slice(0, 32);\n      const hash = generateHashId(undefined, safeCacheId);\n      safeCacheId = `${prefix}-${hash}`;\n    }\n    this.cacheId = safeCacheId;\n\n    const readOnlyMode = Boolean(options?.readOnly);\n    const writeOnlyMode = Boolean(options?.writeOnly);\n\n    if (readOnlyMode && writeOnlyMode) {\n      throw new Error('TaskCache cannot be both read-only and write-only');\n    }\n\n    this.isCacheResultUsed = writeOnlyMode ? false : isCacheResultUsed;\n    this.readOnlyMode = readOnlyMode;\n    this.writeOnlyMode = writeOnlyMode;\n\n    if (cacheAdapterOrPath) {\n      if (typeof cacheAdapterOrPath === 'string') {\n        this.cacheFilePath =\n          ifInBrowser || ifInWorker ? undefined : cacheAdapterOrPath;\n      } else {\n        this.cacheAdapter = cacheAdapterOrPath;\n      }\n    }\n\n    if (!this.cacheAdapter) {\n      this.cacheFilePath =\n        ifInBrowser || ifInWorker\n          ? undefined\n          : join(\n              getMidsceneRunSubDir('cache'),\n              `${this.cacheId}${cacheFileExt}`,\n            );\n    }\n\n    let cacheContent;\n    if (!this.writeOnlyMode) {\n      // 同步加载缓存，对于文件系统缓存是同步的，对于适配器缓存会返回undefined\n      // 实际的异步加载会在需要时进行\n      cacheContent = this.loadCacheFromFile();\n    }\n    if (!cacheContent) {\n      cacheContent = {\n        midsceneVersion: getMidsceneVersion(),\n        cacheId: this.cacheId,\n        caches: [],\n      };\n    }\n    this.cache = cacheContent;\n    this.cacheOriginalLength = this.isCacheResultUsed\n      ? this.cache.caches.length\n      : 0;\n  }\n\n  async matchCache(\n    prompt: TUserPrompt,\n    type: 'plan' | 'locate',\n  ): Promise<MatchCacheResult<PlanningCache | LocateCache> | undefined> {\n    if (!this.isCacheResultUsed) {\n      return undefined;\n    }\n\n    // 如果有缓存适配器且缓存未加载，尝试异步加载\n    if (this.cacheAdapter && this.cache.caches.length === 0) {\n      const loadedCache = await this.loadCacheFromAdapter();\n      if (loadedCache) {\n        this.cache = loadedCache;\n        this.cacheOriginalLength = loadedCache.caches.length;\n      }\n    }\n\n    const promptStr =\n      typeof prompt === 'string' ? prompt : JSON.stringify(prompt);\n    for (let i = 0; i < this.cacheOriginalLength; i++) {\n      const item = this.cache.caches[i];\n      const key = `${type}:${promptStr}:${i}`;\n      if (\n        item.type === type &&\n        isDeepStrictEqual(item.prompt, prompt) &&\n        !this.matchedCacheIndices.has(key)\n      ) {\n        if (item.type === 'locate') {\n          const locateItem = item as LocateCache;\n          if (!locateItem.cache && Array.isArray(locateItem.xpaths)) {\n            locateItem.cache = { xpaths: locateItem.xpaths };\n          }\n          if ('xpaths' in locateItem) {\n            locateItem.xpaths = undefined;\n          }\n        }\n        this.matchedCacheIndices.add(key);\n        debug(\n          'cache found and marked as used, type: %s, prompt: %s, index: %d',\n          type,\n          prompt,\n          i,\n        );\n        return {\n          cacheContent: item,\n          updateFn: async (\n            cb: (cache: PlanningCache | LocateCache) => void,\n            options?: { ttl?: number },\n          ) => {\n            debug(\n              'will call updateFn to update cache, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            cb(item);\n\n            if (this.readOnlyMode) {\n              debug('read-only mode, cache updated in memory but not flushed');\n              return;\n            }\n\n            debug(\n              'cache updated, will flush, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            await this.flushCache(options);\n          },\n        };\n      }\n    }\n    debug('no unused cache found, type: %s, prompt: %s', type, prompt);\n    return undefined;\n  }\n\n  async matchPlanCache(\n    prompt: string,\n  ): Promise<MatchCacheResult<PlanningCache> | undefined> {\n    return this.matchCache(prompt, 'plan') as Promise<\n      MatchCacheResult<PlanningCache> | undefined\n    >;\n  }\n\n  async matchLocateCache(\n    prompt: TUserPrompt,\n  ): Promise<MatchCacheResult<LocateCache> | undefined> {\n    return this.matchCache(prompt, 'locate') as Promise<\n      MatchCacheResult<LocateCache> | undefined\n    >;\n  }\n\n  async appendCache(cache: PlanningCache | LocateCache) {\n    debug('will append cache', cache);\n    this.cache.caches.push(cache);\n\n    if (this.readOnlyMode) {\n      debug('read-only mode, cache appended to memory but not flushed');\n      return;\n    }\n\n    await this.flushCache();\n  }\n\n  private async loadCache(): Promise<CacheFileContent | undefined> {\n    if (this.cacheAdapter) {\n      return this.loadCacheFromAdapter();\n    }\n    return this.loadCacheFromFile();\n  }\n\n  private async loadCacheFromAdapter(): Promise<CacheFileContent | undefined> {\n    try {\n      const content = await this.cacheAdapter!.get(this.cacheId);\n      if (!content) {\n        debug('no cache found from adapter, cacheId: %s', this.cacheId);\n        return undefined;\n      }\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from adapter');\n        return undefined;\n      }\n\n      if (\n        semver.lt(content.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !content.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache, and we cannot match any info from it.\\ncacheId: ${this.cacheId}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from adapter, cacheId: %s, cache version: %s, record length: %s',\n        this.cacheId,\n        content.midsceneVersion,\n        content.caches.length,\n      );\n      content.midsceneVersion = getMidsceneVersion();\n      return content;\n    } catch (err) {\n      debug(\n        'load cache from adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  loadCacheFromFile(): CacheFileContent | undefined {\n    const cacheFile = this.cacheFilePath;\n    assert(cacheFile, 'cache file path is required');\n\n    if (!existsSync(cacheFile)) {\n      debug('no cache file found, path: %s', cacheFile);\n      return undefined;\n    }\n\n    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, '.json');\n    if (existsSync(jsonTypeCacheFile) && this.isCacheResultUsed) {\n      console.warn(\n        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`,\n      );\n      return undefined;\n    }\n\n    try {\n      const data = readFileSync(cacheFile, 'utf8');\n      const jsonData = yaml.load(data) as CacheFileContent;\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from file');\n        return undefined;\n      }\n\n      if (\n        semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !jsonData.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.\\nPlease delete the existing cache and rebuild it. Sorry for the inconvenience.\\ncache file: ${cacheFile}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from file, path: %s, cache version: %s, record length: %s',\n        cacheFile,\n        jsonData.midsceneVersion,\n        jsonData.caches.length,\n      );\n      jsonData.midsceneVersion = getMidsceneVersion();\n      return jsonData;\n    } catch (err) {\n      debug(\n        'cache file exists but load failed, path: %s, error: %s',\n        cacheFile,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  private async flushCache(options?: { cleanUnused?: boolean; ttl?: number }) {\n    if (this.cacheAdapter) {\n      return this.flushCacheToAdapter(options);\n    }\n    return this.flushCacheToFile(options);\n  }\n\n  private async flushCacheToAdapter(options?: {\n    cleanUnused?: boolean;\n    ttl?: number;\n  }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to adapter');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      await this.cacheAdapter!.set(\n        this.cacheId,\n        cacheToWrite,\n        undefined,\n        options?.ttl,\n      );\n      debug('cache flushed to adapter, cacheId: %s', this.cacheId);\n    } catch (err) {\n      debug(\n        'write cache to adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n    }\n  }\n\n  flushCacheToFile(options?: { cleanUnused?: boolean }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to file');\n      return;\n    }\n\n    if (!this.cacheFilePath) {\n      debug('no cache file path, will not write cache to file');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      const dir = dirname(this.cacheFilePath);\n      if (!existsSync(dir)) {\n        mkdirSync(dir, { recursive: true });\n        debug('created cache directory: %s', dir);\n      }\n\n      const yamlData = yaml.dump(cacheToWrite, { lineWidth: -1 });\n      writeFileSync(this.cacheFilePath, yamlData);\n      debug('cache flushed to file: %s', this.cacheFilePath);\n    } catch (err) {\n      debug(\n        'write cache to file failed, path: %s, error: %s',\n        this.cacheFilePath,\n        err,\n      );\n    }\n  }\n\n  private prepareCacheToWrite(options?: {\n    cleanUnused?: boolean;\n  }): CacheFileContent {\n    let caches = [...this.cache.caches];\n\n    if (options?.cleanUnused) {\n      if (this.isCacheResultUsed) {\n        const usedIndices = new Set<number>();\n        for (const key of this.matchedCacheIndices) {\n          const parts = key.split(':');\n          const index = Number.parseInt(parts[parts.length - 1], 10);\n          if (!Number.isNaN(index)) {\n            usedIndices.add(index);\n          }\n        }\n\n        caches = caches.filter((_, index) => {\n          const isUsed = usedIndices.has(index);\n          const isNew = index >= this.cacheOriginalLength;\n          return isUsed || isNew;\n        });\n\n        const removedCount = this.cache.caches.length - caches.length;\n        if (removedCount > 0) {\n          debug('cleaned %d unused cache record(s)', removedCount);\n        } else {\n          debug('no unused cache to clean');\n        }\n      } else {\n        debug('skip cleaning: cache is not used for reading');\n      }\n    }\n\n    const sortedCaches = [...caches].sort((a, b) => {\n      if (a.type === 'plan' && b.type === 'locate') return -1;\n      if (a.type === 'locate' && b.type === 'plan') return 1;\n      return 0;\n    });\n\n    return {\n      ...this.cache,\n      caches: sortedCaches,\n    };\n  }\n\n  async updateOrAppendCacheRecord(\n    newRecord: PlanningCache | LocateCache,\n    cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>,\n  ) {\n    if (cachedRecord) {\n      if (newRecord.type === 'plan') {\n        await cachedRecord.updateFn((cache) => {\n          (cache as PlanningCache).yamlWorkflow = newRecord.yamlWorkflow;\n        });\n      } else {\n        await cachedRecord.updateFn((cache) => {\n          const locateCache = cache as LocateCache;\n          locateCache.cache = newRecord.cache;\n          if ('xpaths' in locateCache) {\n            locateCache.xpaths = undefined;\n          }\n        });\n      }\n    } else {\n      await this.appendCache(newRecord);\n    }\n  }\n}\n"],"names":["DEFAULT_CACHE_MAX_FILENAME_LENGTH","debug","getDebug","lowestSupportedMidsceneVersion","cacheFileExt","TaskCache","prompt","type","loadedCache","promptStr","JSON","i","item","key","isDeepStrictEqual","locateItem","Array","undefined","cb","options","cache","content","version","getMidsceneVersion","semver","console","err","cacheFile","assert","existsSync","jsonTypeCacheFile","data","readFileSync","jsonData","yaml","cacheToWrite","dir","dirname","mkdirSync","yamlData","writeFileSync","caches","usedIndices","Set","parts","index","Number","_","isUsed","isNew","removedCount","sortedCaches","a","b","newRecord","cachedRecord","locateCache","cacheId","isCacheResultUsed","cacheAdapterOrPath","safeCacheId","replaceIllegalPathCharsAndSpace","cacheMaxFilenameLength","globalConfigManager","MIDSCENE_CACHE_MAX_FILENAME_LENGTH","Buffer","prefix","hash","generateHashId","readOnlyMode","Boolean","writeOnlyMode","Error","ifInBrowser","ifInWorker","join","getMidsceneRunSubDir","cacheContent"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoBA,MAAMA,oCAAoC;AAEnC,MAAMC,QAAQC,SAAS;AA8B9B,MAAMC,iCAAiC;AAChC,MAAMC,eAAe;AAErB,MAAMC;IAuFX,MAAM,WACJC,MAAmB,EACnBC,IAAuB,EAC6C;QACpE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EACzB;QAIF,IAAI,IAAI,CAAC,YAAY,IAAI,AAA6B,MAA7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAQ;YACvD,MAAMC,cAAc,MAAM,IAAI,CAAC,oBAAoB;YACnD,IAAIA,aAAa;gBACf,IAAI,CAAC,KAAK,GAAGA;gBACb,IAAI,CAAC,mBAAmB,GAAGA,YAAY,MAAM,CAAC,MAAM;YACtD;QACF;QAEA,MAAMC,YACJ,AAAkB,YAAlB,OAAOH,SAAsBA,SAASI,KAAK,SAAS,CAACJ;QACvD,IAAK,IAAIK,IAAI,GAAGA,IAAI,IAAI,CAAC,mBAAmB,EAAEA,IAAK;YACjD,MAAMC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAACD,EAAE;YACjC,MAAME,MAAM,GAAGN,KAAK,CAAC,EAAEE,UAAU,CAAC,EAAEE,GAAG;YACvC,IACEC,KAAK,IAAI,KAAKL,QACdO,kBAAkBF,KAAK,MAAM,EAAEN,WAC/B,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACO,MAC9B;gBACA,IAAID,AAAc,aAAdA,KAAK,IAAI,EAAe;oBAC1B,MAAMG,aAAaH;oBACnB,IAAI,CAACG,WAAW,KAAK,IAAIC,MAAM,OAAO,CAACD,WAAW,MAAM,GACtDA,WAAW,KAAK,GAAG;wBAAE,QAAQA,WAAW,MAAM;oBAAC;oBAEjD,IAAI,YAAYA,YACdA,WAAW,MAAM,GAAGE;gBAExB;gBACA,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACJ;gBAC7BZ,MACE,mEACAM,MACAD,QACAK;gBAEF,OAAO;oBACL,cAAcC;oBACd,UAAU,OACRM,IACAC;wBAEAlB,MACE,uEACAM,MACAD,QACAK;wBAEFO,GAAGN;wBAEH,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBX,MAAM;wBAIRA,MACE,8DACAM,MACAD,QACAK;wBAEF,MAAM,IAAI,CAAC,UAAU,CAACQ;oBACxB;gBACF;YACF;QACF;QACAlB,MAAM,+CAA+CM,MAAMD;IAE7D;IAEA,MAAM,eACJA,MAAc,EACwC;QACtD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,iBACJA,MAAmB,EACiC;QACpD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,YAAYc,KAAkC,EAAE;QACpDnB,MAAM,qBAAqBmB;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAACA;QAEvB,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBnB,MAAM;QAIR,MAAM,IAAI,CAAC,UAAU;IACvB;IAEA,MAAc,YAAmD;QAC/D,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,oBAAoB;QAElC,OAAO,IAAI,CAAC,iBAAiB;IAC/B;IAEA,MAAc,uBAA8D;QAC1E,IAAI;YACF,MAAMoB,UAAU,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAAC,IAAI,CAAC,OAAO;YACzD,IAAI,CAACA,SAAS,YACZpB,MAAM,4CAA4C,IAAI,CAAC,OAAO;YAIhE,MAAMqB,UAAUC;YAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;YAIR,IACEuB,OAAO,EAAE,CAACH,QAAQ,eAAe,EAAElB,mCACnC,CAACkB,QAAQ,eAAe,CAAC,QAAQ,CAAC,SAClC,YACAI,QAAQ,IAAI,CACV,CAAC,gGAAgG,EAAE,IAAI,CAAC,OAAO,EAAE;YAKrHxB,MACE,gFACA,IAAI,CAAC,OAAO,EACZoB,QAAQ,eAAe,EACvBA,QAAQ,MAAM,CAAC,MAAM;YAEvBA,QAAQ,eAAe,GAAGE;YAC1B,OAAOF;QACT,EAAE,OAAOK,KAAK;YACZzB,MACE,0DACA,IAAI,CAAC,OAAO,EACZyB;YAEF;QACF;IACF;IAEA,oBAAkD;QAChD,MAAMC,YAAY,IAAI,CAAC,aAAa;QACpCC,YAAOD,WAAW;QAElB,IAAI,CAACE,WAAWF,YAAY,YAC1B1B,MAAM,iCAAiC0B;QAIzC,MAAMG,oBAAoBH,UAAU,OAAO,CAACvB,cAAc;QAC1D,IAAIyB,WAAWC,sBAAsB,IAAI,CAAC,iBAAiB,EAAE,YAC3DL,QAAQ,IAAI,CACV,CAAC,2LAA2L,EAAEK,kBAAkB,CAAC,CAAC;QAKtN,IAAI;YACF,MAAMC,OAAOC,aAAaL,WAAW;YACrC,MAAMM,WAAWC,QAAAA,IAAS,CAACH;YAE3B,MAAMT,UAAUC;YAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;YAIR,IACEuB,OAAO,EAAE,CAACS,SAAS,eAAe,EAAE9B,mCACpC,CAAC8B,SAAS,eAAe,CAAC,QAAQ,CAAC,SACnC,YACAR,QAAQ,IAAI,CACV,CAAC,wSAAwS,EAAEE,WAAW;YAK1T1B,MACE,0EACA0B,WACAM,SAAS,eAAe,EACxBA,SAAS,MAAM,CAAC,MAAM;YAExBA,SAAS,eAAe,GAAGV;YAC3B,OAAOU;QACT,EAAE,OAAOP,KAAK;YACZzB,MACE,0DACA0B,WACAD;YAEF;QACF;IACF;IAEA,MAAc,WAAWP,OAAiD,EAAE;QAC1E,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,mBAAmB,CAACA;QAElC,OAAO,IAAI,CAAC,gBAAgB,CAACA;IAC/B;IAEA,MAAc,oBAAoBA,OAGjC,EAAE;QACD,MAAMG,UAAUC;QAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;QAIR,MAAMkC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAC1B,IAAI,CAAC,OAAO,EACZgB,cACAlB,QACAE,SAAS;YAEXlB,MAAM,yCAAyC,IAAI,CAAC,OAAO;QAC7D,EAAE,OAAOyB,KAAK;YACZzB,MACE,yDACA,IAAI,CAAC,OAAO,EACZyB;QAEJ;IACF;IAEA,iBAAiBP,OAAmC,EAAE;QACpD,MAAMG,UAAUC;QAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;QAIR,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,YACvBA,MAAM;QAIR,MAAMkC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAMiB,MAAMC,QAAQ,IAAI,CAAC,aAAa;YACtC,IAAI,CAACR,WAAWO,MAAM;gBACpBE,UAAUF,KAAK;oBAAE,WAAW;gBAAK;gBACjCnC,MAAM,+BAA+BmC;YACvC;YAEA,MAAMG,WAAWL,QAAAA,IAAS,CAACC,cAAc;gBAAE,WAAW;YAAG;YACzDK,cAAc,IAAI,CAAC,aAAa,EAAED;YAClCtC,MAAM,6BAA6B,IAAI,CAAC,aAAa;QACvD,EAAE,OAAOyB,KAAK;YACZzB,MACE,mDACA,IAAI,CAAC,aAAa,EAClByB;QAEJ;IACF;IAEQ,oBAAoBP,OAE3B,EAAoB;QACnB,IAAIsB,SAAS;eAAI,IAAI,CAAC,KAAK,CAAC,MAAM;SAAC;QAEnC,IAAItB,SAAS,aACX,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,MAAMuB,cAAc,IAAIC;YACxB,KAAK,MAAM9B,OAAO,IAAI,CAAC,mBAAmB,CAAE;gBAC1C,MAAM+B,QAAQ/B,IAAI,KAAK,CAAC;gBACxB,MAAMgC,QAAQC,OAAO,QAAQ,CAACF,KAAK,CAACA,MAAM,MAAM,GAAG,EAAE,EAAE;gBACvD,IAAI,CAACE,OAAO,KAAK,CAACD,QAChBH,YAAY,GAAG,CAACG;YAEpB;YAEAJ,SAASA,OAAO,MAAM,CAAC,CAACM,GAAGF;gBACzB,MAAMG,SAASN,YAAY,GAAG,CAACG;gBAC/B,MAAMI,QAAQJ,SAAS,IAAI,CAAC,mBAAmB;gBAC/C,OAAOG,UAAUC;YACnB;YAEA,MAAMC,eAAe,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAGT,OAAO,MAAM;YACzDS,eAAe,IACjBjD,MAAM,qCAAqCiD,gBAE3CjD,MAAM;QAEV,OACEA,MAAM;QAIV,MAAMkD,eAAe;eAAIV;SAAO,CAAC,IAAI,CAAC,CAACW,GAAGC;YACxC,IAAID,AAAW,WAAXA,EAAE,IAAI,IAAeC,AAAW,aAAXA,EAAE,IAAI,EAAe,OAAO;YACrD,IAAID,AAAW,aAAXA,EAAE,IAAI,IAAiBC,AAAW,WAAXA,EAAE,IAAI,EAAa,OAAO;YACrD,OAAO;QACT;QAEA,OAAO;YACL,GAAG,IAAI,CAAC,KAAK;YACb,QAAQF;QACV;IACF;IAEA,MAAM,0BACJG,SAAsC,EACtCC,YAA4D,EAC5D;QACA,IAAIA,cACF,IAAID,AAAmB,WAAnBA,UAAU,IAAI,EAChB,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC1BA,MAAwB,YAAY,GAAGkC,UAAU,YAAY;QAChE;aAEA,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC3B,MAAMoC,cAAcpC;YACpBoC,YAAY,KAAK,GAAGF,UAAU,KAAK;YACnC,IAAI,YAAYE,aACdA,YAAY,MAAM,GAAGvC;QAEzB;aAGF,MAAM,IAAI,CAAC,WAAW,CAACqC;IAE3B;IA5ZA,YACEG,OAAe,EACfC,iBAA0B,EAC1BC,kBAA0C,EAC1CxC,UAAuD,CAAC,CAAC,CACzD;QAvBF;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA,uBAAQ,uBAAmC,IAAIwB;QAE/C,uBAAQ,gBAAR;QAQEf,YAAO6B,SAAS;QAChB,IAAIG,cAAcC,gCAAgCJ;QAClD,MAAMK,yBACJC,oBAAoB,yBAAyB,CAC3CC,uCACGhE;QACP,IAAIiE,OAAO,UAAU,CAACL,aAAa,UAAUE,wBAAwB;YACnE,MAAMI,SAASN,YAAY,KAAK,CAAC,GAAG;YACpC,MAAMO,OAAOC,eAAenD,QAAW2C;YACvCA,cAAc,GAAGM,OAAO,CAAC,EAAEC,MAAM;QACnC;QACA,IAAI,CAAC,OAAO,GAAGP;QAEf,MAAMS,eAAeC,QAAQnD,SAAS;QACtC,MAAMoD,gBAAgBD,QAAQnD,SAAS;QAEvC,IAAIkD,gBAAgBE,eAClB,MAAM,IAAIC,MAAM;QAGlB,IAAI,CAAC,iBAAiB,GAAGD,gBAAgB,QAAQb;QACjD,IAAI,CAAC,YAAY,GAAGW;QACpB,IAAI,CAAC,aAAa,GAAGE;QAErB,IAAIZ,oBACF,IAAI,AAA8B,YAA9B,OAAOA,oBACT,IAAI,CAAC,aAAa,GAChBc,eAAeC,aAAazD,SAAY0C;aAE1C,IAAI,CAAC,YAAY,GAAGA;QAIxB,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB,IAAI,CAAC,aAAa,GAChBc,eAAeC,aACXzD,SACA0D,KACEC,qBAAqB,UACrB,GAAG,IAAI,CAAC,OAAO,GAAGxE,cAAc;QAI1C,IAAIyE;QACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAGrBA,eAAe,IAAI,CAAC,iBAAiB;QAEvC,IAAI,CAACA,cACHA,eAAe;YACb,iBAAiBtD;YACjB,SAAS,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE;QACZ;QAEF,IAAI,CAAC,KAAK,GAAGsD;QACb,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,GAC7C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GACxB;IACN;AA2VF"}
+{"version":3,"file":"agent/task-cache.mjs","sources":["../../../src/agent/task-cache.ts"],"sourcesContent":["import assert from 'node:assert';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { isDeepStrictEqual } from 'node:util';\nimport type { TUserPrompt } from '@/ai-model';\nimport type { ElementCacheFeature } from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport {\n  MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { generateHashId } from '@midscene/shared/utils';\nimport { replaceIllegalPathCharsAndSpace } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\nimport semver from 'semver';\nimport type { CacheAdapter } from './cache-adapter';\nimport { getMidsceneVersion } from './utils';\n\nconst DEFAULT_CACHE_MAX_FILENAME_LENGTH = 200;\n\nexport const debug = getDebug('cache');\n\nexport interface PlanningCache {\n  type: 'plan';\n  prompt: string;\n  yamlWorkflow: string;\n}\n\nexport interface LocateCache {\n  type: 'locate';\n  prompt: TUserPrompt;\n  cache?: ElementCacheFeature;\n  /** @deprecated kept for backward compatibility */\n  xpaths?: string[];\n}\n\nexport interface MatchCacheResult<T extends PlanningCache | LocateCache> {\n  cacheContent: T;\n  updateFn: (\n    cb: (cache: T) => void,\n    options?: { ttl?: number },\n  ) => Promise<void>;\n}\n\nexport type CacheFileContent = {\n  midsceneVersion: string;\n  cacheId: string;\n  caches: Array<PlanningCache | LocateCache>;\n};\n\nconst lowestSupportedMidsceneVersion = '0.16.10';\nexport const cacheFileExt = '.cache.yaml';\n\nexport class TaskCache {\n  cacheId: string;\n\n  cacheFilePath?: string;\n\n  cache: CacheFileContent;\n\n  isCacheResultUsed: boolean;\n\n  cacheOriginalLength: number;\n\n  readOnlyMode: boolean;\n\n  writeOnlyMode: boolean;\n\n  private matchedCacheIndices: Set<string> = new Set();\n\n  private cacheAdapter?: CacheAdapter;\n\n  constructor(\n    cacheId: string,\n    isCacheResultUsed: boolean,\n    cacheAdapterOrPath?: CacheAdapter | string,\n    options: { readOnly?: boolean; writeOnly?: boolean } = {},\n  ) {\n    assert(cacheId, 'cacheId is required');\n    let safeCacheId = replaceIllegalPathCharsAndSpace(cacheId);\n    const cacheMaxFilenameLength =\n      globalConfigManager.getEnvConfigValueAsNumber(\n        MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n      ) ?? DEFAULT_CACHE_MAX_FILENAME_LENGTH;\n    if (Buffer.byteLength(safeCacheId, 'utf8') > cacheMaxFilenameLength) {\n      const prefix = safeCacheId.slice(0, 32);\n      const hash = generateHashId(undefined, safeCacheId);\n      safeCacheId = `${prefix}-${hash}`;\n    }\n    this.cacheId = safeCacheId;\n\n    const readOnlyMode = Boolean(options?.readOnly);\n    const writeOnlyMode = Boolean(options?.writeOnly);\n\n    if (readOnlyMode && writeOnlyMode) {\n      throw new Error('TaskCache cannot be both read-only and write-only');\n    }\n\n    this.isCacheResultUsed = writeOnlyMode ? false : isCacheResultUsed;\n    this.readOnlyMode = readOnlyMode;\n    this.writeOnlyMode = writeOnlyMode;\n\n    if (cacheAdapterOrPath) {\n      if (typeof cacheAdapterOrPath === 'string') {\n        this.cacheFilePath =\n          ifInBrowser || ifInWorker ? undefined : cacheAdapterOrPath;\n      } else {\n        this.cacheAdapter = cacheAdapterOrPath;\n      }\n    }\n\n    if (!this.cacheAdapter) {\n      this.cacheFilePath =\n        ifInBrowser || ifInWorker\n          ? undefined\n          : join(\n              getMidsceneRunSubDir('cache'),\n              `${this.cacheId}${cacheFileExt}`,\n            );\n    }\n\n    let cacheContent;\n    if (!this.writeOnlyMode) {\n      // 同步加载缓存，对于文件系统缓存是同步的，对于适配器缓存会返回undefined\n      // 实际的异步加载会在需要时进行\n      // 如果有 cacheAdapter，就不调用 loadCacheFromFile()，因为适配器缓存是异步加载的\n      if (!this.cacheAdapter) {\n        cacheContent = this.loadCacheFromFile();\n      }\n    }\n    if (!cacheContent) {\n      cacheContent = {\n        midsceneVersion: getMidsceneVersion(),\n        cacheId: this.cacheId,\n        caches: [],\n      };\n    }\n    this.cache = cacheContent;\n    this.cacheOriginalLength = this.isCacheResultUsed\n      ? this.cache.caches.length\n      : 0;\n  }\n\n  async matchCache(\n    prompt: TUserPrompt,\n    type: 'plan' | 'locate',\n  ): Promise<MatchCacheResult<PlanningCache | LocateCache> | undefined> {\n    if (!this.isCacheResultUsed) {\n      return undefined;\n    }\n\n    // 如果有缓存适配器且缓存未加载，尝试异步加载\n    if (this.cacheAdapter && this.cache.caches.length === 0) {\n      const loadedCache = await this.loadCacheFromAdapter();\n      if (loadedCache) {\n        this.cache = loadedCache;\n        this.cacheOriginalLength = loadedCache.caches.length;\n      }\n    }\n\n    const promptStr =\n      typeof prompt === 'string' ? prompt : JSON.stringify(prompt);\n    for (let i = 0; i < this.cacheOriginalLength; i++) {\n      const item = this.cache.caches[i];\n      const key = `${type}:${promptStr}:${i}`;\n      if (\n        item.type === type &&\n        isDeepStrictEqual(item.prompt, prompt) &&\n        !this.matchedCacheIndices.has(key)\n      ) {\n        if (item.type === 'locate') {\n          const locateItem = item as LocateCache;\n          if (!locateItem.cache && Array.isArray(locateItem.xpaths)) {\n            locateItem.cache = { xpaths: locateItem.xpaths };\n          }\n          if ('xpaths' in locateItem) {\n            locateItem.xpaths = undefined;\n          }\n        }\n        this.matchedCacheIndices.add(key);\n        debug(\n          'cache found and marked as used, type: %s, prompt: %s, index: %d',\n          type,\n          prompt,\n          i,\n        );\n        return {\n          cacheContent: item,\n          updateFn: async (\n            cb: (cache: PlanningCache | LocateCache) => void,\n            options?: { ttl?: number },\n          ) => {\n            debug(\n              'will call updateFn to update cache, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            cb(item);\n\n            if (this.readOnlyMode) {\n              debug('read-only mode, cache updated in memory but not flushed');\n              return;\n            }\n\n            debug(\n              'cache updated, will flush, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            await this.flushCache(options);\n          },\n        };\n      }\n    }\n    debug('no unused cache found, type: %s, prompt: %s', type, prompt);\n    return undefined;\n  }\n\n  async matchPlanCache(\n    prompt: string,\n  ): Promise<MatchCacheResult<PlanningCache> | undefined> {\n    return this.matchCache(prompt, 'plan') as Promise<\n      MatchCacheResult<PlanningCache> | undefined\n    >;\n  }\n\n  async matchLocateCache(\n    prompt: TUserPrompt,\n  ): Promise<MatchCacheResult<LocateCache> | undefined> {\n    return this.matchCache(prompt, 'locate') as Promise<\n      MatchCacheResult<LocateCache> | undefined\n    >;\n  }\n\n  async appendCache(cache: PlanningCache | LocateCache) {\n    debug('will append cache', cache);\n    this.cache.caches.push(cache);\n\n    if (this.readOnlyMode) {\n      debug('read-only mode, cache appended to memory but not flushed');\n      return;\n    }\n\n    await this.flushCache();\n  }\n\n  private async loadCache(): Promise<CacheFileContent | undefined> {\n    if (this.cacheAdapter) {\n      return this.loadCacheFromAdapter();\n    }\n    return this.loadCacheFromFile();\n  }\n\n  private async loadCacheFromAdapter(): Promise<CacheFileContent | undefined> {\n    try {\n      const content = await this.cacheAdapter!.get(this.cacheId);\n      if (!content) {\n        debug('no cache found from adapter, cacheId: %s', this.cacheId);\n        return undefined;\n      }\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from adapter');\n        return undefined;\n      }\n\n      if (\n        semver.lt(content.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !content.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache, and we cannot match any info from it.\\ncacheId: ${this.cacheId}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from adapter, cacheId: %s, cache version: %s, record length: %s',\n        this.cacheId,\n        content.midsceneVersion,\n        content.caches.length,\n      );\n      content.midsceneVersion = getMidsceneVersion();\n      return content;\n    } catch (err) {\n      debug(\n        'load cache from adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  loadCacheFromFile(): CacheFileContent | undefined {\n    const cacheFile = this.cacheFilePath;\n    assert(cacheFile, 'cache file path is required');\n\n    if (!existsSync(cacheFile)) {\n      debug('no cache file found, path: %s', cacheFile);\n      return undefined;\n    }\n\n    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, '.json');\n    if (existsSync(jsonTypeCacheFile) && this.isCacheResultUsed) {\n      console.warn(\n        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`,\n      );\n      return undefined;\n    }\n\n    try {\n      const data = readFileSync(cacheFile, 'utf8');\n      const jsonData = yaml.load(data) as CacheFileContent;\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from file');\n        return undefined;\n      }\n\n      if (\n        semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !jsonData.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.\\nPlease delete the existing cache and rebuild it. Sorry for the inconvenience.\\ncache file: ${cacheFile}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from file, path: %s, cache version: %s, record length: %s',\n        cacheFile,\n        jsonData.midsceneVersion,\n        jsonData.caches.length,\n      );\n      jsonData.midsceneVersion = getMidsceneVersion();\n      return jsonData;\n    } catch (err) {\n      debug(\n        'cache file exists but load failed, path: %s, error: %s',\n        cacheFile,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  private async flushCache(options?: { cleanUnused?: boolean; ttl?: number }) {\n    if (this.cacheAdapter) {\n      return this.flushCacheToAdapter(options);\n    }\n    return this.flushCacheToFile(options);\n  }\n\n  private async flushCacheToAdapter(options?: {\n    cleanUnused?: boolean;\n    ttl?: number;\n  }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to adapter');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      await this.cacheAdapter!.set(\n        this.cacheId,\n        cacheToWrite,\n        undefined,\n        options?.ttl,\n      );\n      debug('cache flushed to adapter, cacheId: %s', this.cacheId);\n    } catch (err) {\n      debug(\n        'write cache to adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n    }\n  }\n\n  flushCacheToFile(options?: { cleanUnused?: boolean }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to file');\n      return;\n    }\n\n    if (!this.cacheFilePath) {\n      debug('no cache file path, will not write cache to file');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      const dir = dirname(this.cacheFilePath);\n      if (!existsSync(dir)) {\n        mkdirSync(dir, { recursive: true });\n        debug('created cache directory: %s', dir);\n      }\n\n      const yamlData = yaml.dump(cacheToWrite, { lineWidth: -1 });\n      writeFileSync(this.cacheFilePath, yamlData);\n      debug('cache flushed to file: %s', this.cacheFilePath);\n    } catch (err) {\n      debug(\n        'write cache to file failed, path: %s, error: %s',\n        this.cacheFilePath,\n        err,\n      );\n    }\n  }\n\n  private prepareCacheToWrite(options?: {\n    cleanUnused?: boolean;\n  }): CacheFileContent {\n    let caches = [...this.cache.caches];\n\n    if (options?.cleanUnused) {\n      if (this.isCacheResultUsed) {\n        const usedIndices = new Set<number>();\n        for (const key of this.matchedCacheIndices) {\n          const parts = key.split(':');\n          const index = Number.parseInt(parts[parts.length - 1], 10);\n          if (!Number.isNaN(index)) {\n            usedIndices.add(index);\n          }\n        }\n\n        caches = caches.filter((_, index) => {\n          const isUsed = usedIndices.has(index);\n          const isNew = index >= this.cacheOriginalLength;\n          return isUsed || isNew;\n        });\n\n        const removedCount = this.cache.caches.length - caches.length;\n        if (removedCount > 0) {\n          debug('cleaned %d unused cache record(s)', removedCount);\n        } else {\n          debug('no unused cache to clean');\n        }\n      } else {\n        debug('skip cleaning: cache is not used for reading');\n      }\n    }\n\n    const sortedCaches = [...caches].sort((a, b) => {\n      if (a.type === 'plan' && b.type === 'locate') return -1;\n      if (a.type === 'locate' && b.type === 'plan') return 1;\n      return 0;\n    });\n\n    return {\n      ...this.cache,\n      caches: sortedCaches,\n    };\n  }\n\n  async updateOrAppendCacheRecord(\n    newRecord: PlanningCache | LocateCache,\n    cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>,\n  ) {\n    if (cachedRecord) {\n      if (newRecord.type === 'plan') {\n        await cachedRecord.updateFn((cache) => {\n          (cache as PlanningCache).yamlWorkflow = newRecord.yamlWorkflow;\n        });\n      } else {\n        await cachedRecord.updateFn((cache) => {\n          const locateCache = cache as LocateCache;\n          locateCache.cache = newRecord.cache;\n          if ('xpaths' in locateCache) {\n            locateCache.xpaths = undefined;\n          }\n        });\n      }\n    } else {\n      await this.appendCache(newRecord);\n    }\n  }\n}\n"],"names":["DEFAULT_CACHE_MAX_FILENAME_LENGTH","debug","getDebug","lowestSupportedMidsceneVersion","cacheFileExt","TaskCache","prompt","type","loadedCache","promptStr","JSON","i","item","key","isDeepStrictEqual","locateItem","Array","undefined","cb","options","cache","content","version","getMidsceneVersion","semver","console","err","cacheFile","assert","existsSync","jsonTypeCacheFile","data","readFileSync","jsonData","yaml","cacheToWrite","dir","dirname","mkdirSync","yamlData","writeFileSync","caches","usedIndices","Set","parts","index","Number","_","isUsed","isNew","removedCount","sortedCaches","a","b","newRecord","cachedRecord","locateCache","cacheId","isCacheResultUsed","cacheAdapterOrPath","safeCacheId","replaceIllegalPathCharsAndSpace","cacheMaxFilenameLength","globalConfigManager","MIDSCENE_CACHE_MAX_FILENAME_LENGTH","Buffer","prefix","hash","generateHashId","readOnlyMode","Boolean","writeOnlyMode","Error","ifInBrowser","ifInWorker","join","getMidsceneRunSubDir","cacheContent"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAoBA,MAAMA,oCAAoC;AAEnC,MAAMC,QAAQC,SAAS;AA8B9B,MAAMC,iCAAiC;AAChC,MAAMC,eAAe;AAErB,MAAMC;IA0FX,MAAM,WACJC,MAAmB,EACnBC,IAAuB,EAC6C;QACpE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EACzB;QAIF,IAAI,IAAI,CAAC,YAAY,IAAI,AAA6B,MAA7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAQ;YACvD,MAAMC,cAAc,MAAM,IAAI,CAAC,oBAAoB;YACnD,IAAIA,aAAa;gBACf,IAAI,CAAC,KAAK,GAAGA;gBACb,IAAI,CAAC,mBAAmB,GAAGA,YAAY,MAAM,CAAC,MAAM;YACtD;QACF;QAEA,MAAMC,YACJ,AAAkB,YAAlB,OAAOH,SAAsBA,SAASI,KAAK,SAAS,CAACJ;QACvD,IAAK,IAAIK,IAAI,GAAGA,IAAI,IAAI,CAAC,mBAAmB,EAAEA,IAAK;YACjD,MAAMC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAACD,EAAE;YACjC,MAAME,MAAM,GAAGN,KAAK,CAAC,EAAEE,UAAU,CAAC,EAAEE,GAAG;YACvC,IACEC,KAAK,IAAI,KAAKL,QACdO,kBAAkBF,KAAK,MAAM,EAAEN,WAC/B,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACO,MAC9B;gBACA,IAAID,AAAc,aAAdA,KAAK,IAAI,EAAe;oBAC1B,MAAMG,aAAaH;oBACnB,IAAI,CAACG,WAAW,KAAK,IAAIC,MAAM,OAAO,CAACD,WAAW,MAAM,GACtDA,WAAW,KAAK,GAAG;wBAAE,QAAQA,WAAW,MAAM;oBAAC;oBAEjD,IAAI,YAAYA,YACdA,WAAW,MAAM,GAAGE;gBAExB;gBACA,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACJ;gBAC7BZ,MACE,mEACAM,MACAD,QACAK;gBAEF,OAAO;oBACL,cAAcC;oBACd,UAAU,OACRM,IACAC;wBAEAlB,MACE,uEACAM,MACAD,QACAK;wBAEFO,GAAGN;wBAEH,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBX,MAAM;wBAIRA,MACE,8DACAM,MACAD,QACAK;wBAEF,MAAM,IAAI,CAAC,UAAU,CAACQ;oBACxB;gBACF;YACF;QACF;QACAlB,MAAM,+CAA+CM,MAAMD;IAE7D;IAEA,MAAM,eACJA,MAAc,EACwC;QACtD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,iBACJA,MAAmB,EACiC;QACpD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,YAAYc,KAAkC,EAAE;QACpDnB,MAAM,qBAAqBmB;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAACA;QAEvB,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBnB,MAAM;QAIR,MAAM,IAAI,CAAC,UAAU;IACvB;IAEA,MAAc,YAAmD;QAC/D,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,oBAAoB;QAElC,OAAO,IAAI,CAAC,iBAAiB;IAC/B;IAEA,MAAc,uBAA8D;QAC1E,IAAI;YACF,MAAMoB,UAAU,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAAC,IAAI,CAAC,OAAO;YACzD,IAAI,CAACA,SAAS,YACZpB,MAAM,4CAA4C,IAAI,CAAC,OAAO;YAIhE,MAAMqB,UAAUC;YAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;YAIR,IACEuB,OAAO,EAAE,CAACH,QAAQ,eAAe,EAAElB,mCACnC,CAACkB,QAAQ,eAAe,CAAC,QAAQ,CAAC,SAClC,YACAI,QAAQ,IAAI,CACV,CAAC,gGAAgG,EAAE,IAAI,CAAC,OAAO,EAAE;YAKrHxB,MACE,gFACA,IAAI,CAAC,OAAO,EACZoB,QAAQ,eAAe,EACvBA,QAAQ,MAAM,CAAC,MAAM;YAEvBA,QAAQ,eAAe,GAAGE;YAC1B,OAAOF;QACT,EAAE,OAAOK,KAAK;YACZzB,MACE,0DACA,IAAI,CAAC,OAAO,EACZyB;YAEF;QACF;IACF;IAEA,oBAAkD;QAChD,MAAMC,YAAY,IAAI,CAAC,aAAa;QACpCC,YAAOD,WAAW;QAElB,IAAI,CAACE,WAAWF,YAAY,YAC1B1B,MAAM,iCAAiC0B;QAIzC,MAAMG,oBAAoBH,UAAU,OAAO,CAACvB,cAAc;QAC1D,IAAIyB,WAAWC,sBAAsB,IAAI,CAAC,iBAAiB,EAAE,YAC3DL,QAAQ,IAAI,CACV,CAAC,2LAA2L,EAAEK,kBAAkB,CAAC,CAAC;QAKtN,IAAI;YACF,MAAMC,OAAOC,aAAaL,WAAW;YACrC,MAAMM,WAAWC,QAAAA,IAAS,CAACH;YAE3B,MAAMT,UAAUC;YAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;YAIR,IACEuB,OAAO,EAAE,CAACS,SAAS,eAAe,EAAE9B,mCACpC,CAAC8B,SAAS,eAAe,CAAC,QAAQ,CAAC,SACnC,YACAR,QAAQ,IAAI,CACV,CAAC,wSAAwS,EAAEE,WAAW;YAK1T1B,MACE,0EACA0B,WACAM,SAAS,eAAe,EACxBA,SAAS,MAAM,CAAC,MAAM;YAExBA,SAAS,eAAe,GAAGV;YAC3B,OAAOU;QACT,EAAE,OAAOP,KAAK;YACZzB,MACE,0DACA0B,WACAD;YAEF;QACF;IACF;IAEA,MAAc,WAAWP,OAAiD,EAAE;QAC1E,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,mBAAmB,CAACA;QAElC,OAAO,IAAI,CAAC,gBAAgB,CAACA;IAC/B;IAEA,MAAc,oBAAoBA,OAGjC,EAAE;QACD,MAAMG,UAAUC;QAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;QAIR,MAAMkC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAC1B,IAAI,CAAC,OAAO,EACZgB,cACAlB,QACAE,SAAS;YAEXlB,MAAM,yCAAyC,IAAI,CAAC,OAAO;QAC7D,EAAE,OAAOyB,KAAK;YACZzB,MACE,yDACA,IAAI,CAAC,OAAO,EACZyB;QAEJ;IACF;IAEA,iBAAiBP,OAAmC,EAAE;QACpD,MAAMG,UAAUC;QAChB,IAAI,CAACD,SAAS,YACZrB,MAAM;QAIR,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,YACvBA,MAAM;QAIR,MAAMkC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAMiB,MAAMC,QAAQ,IAAI,CAAC,aAAa;YACtC,IAAI,CAACR,WAAWO,MAAM;gBACpBE,UAAUF,KAAK;oBAAE,WAAW;gBAAK;gBACjCnC,MAAM,+BAA+BmC;YACvC;YAEA,MAAMG,WAAWL,QAAAA,IAAS,CAACC,cAAc;gBAAE,WAAW;YAAG;YACzDK,cAAc,IAAI,CAAC,aAAa,EAAED;YAClCtC,MAAM,6BAA6B,IAAI,CAAC,aAAa;QACvD,EAAE,OAAOyB,KAAK;YACZzB,MACE,mDACA,IAAI,CAAC,aAAa,EAClByB;QAEJ;IACF;IAEQ,oBAAoBP,OAE3B,EAAoB;QACnB,IAAIsB,SAAS;eAAI,IAAI,CAAC,KAAK,CAAC,MAAM;SAAC;QAEnC,IAAItB,SAAS,aACX,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,MAAMuB,cAAc,IAAIC;YACxB,KAAK,MAAM9B,OAAO,IAAI,CAAC,mBAAmB,CAAE;gBAC1C,MAAM+B,QAAQ/B,IAAI,KAAK,CAAC;gBACxB,MAAMgC,QAAQC,OAAO,QAAQ,CAACF,KAAK,CAACA,MAAM,MAAM,GAAG,EAAE,EAAE;gBACvD,IAAI,CAACE,OAAO,KAAK,CAACD,QAChBH,YAAY,GAAG,CAACG;YAEpB;YAEAJ,SAASA,OAAO,MAAM,CAAC,CAACM,GAAGF;gBACzB,MAAMG,SAASN,YAAY,GAAG,CAACG;gBAC/B,MAAMI,QAAQJ,SAAS,IAAI,CAAC,mBAAmB;gBAC/C,OAAOG,UAAUC;YACnB;YAEA,MAAMC,eAAe,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAGT,OAAO,MAAM;YACzDS,eAAe,IACjBjD,MAAM,qCAAqCiD,gBAE3CjD,MAAM;QAEV,OACEA,MAAM;QAIV,MAAMkD,eAAe;eAAIV;SAAO,CAAC,IAAI,CAAC,CAACW,GAAGC;YACxC,IAAID,AAAW,WAAXA,EAAE,IAAI,IAAeC,AAAW,aAAXA,EAAE,IAAI,EAAe,OAAO;YACrD,IAAID,AAAW,aAAXA,EAAE,IAAI,IAAiBC,AAAW,WAAXA,EAAE,IAAI,EAAa,OAAO;YACrD,OAAO;QACT;QAEA,OAAO;YACL,GAAG,IAAI,CAAC,KAAK;YACb,QAAQF;QACV;IACF;IAEA,MAAM,0BACJG,SAAsC,EACtCC,YAA4D,EAC5D;QACA,IAAIA,cACF,IAAID,AAAmB,WAAnBA,UAAU,IAAI,EAChB,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC1BA,MAAwB,YAAY,GAAGkC,UAAU,YAAY;QAChE;aAEA,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC3B,MAAMoC,cAAcpC;YACpBoC,YAAY,KAAK,GAAGF,UAAU,KAAK;YACnC,IAAI,YAAYE,aACdA,YAAY,MAAM,GAAGvC;QAEzB;aAGF,MAAM,IAAI,CAAC,WAAW,CAACqC;IAE3B;IA/ZA,YACEG,OAAe,EACfC,iBAA0B,EAC1BC,kBAA0C,EAC1CxC,UAAuD,CAAC,CAAC,CACzD;QAvBF;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA,uBAAQ,uBAAmC,IAAIwB;QAE/C,uBAAQ,gBAAR;QAQEf,YAAO6B,SAAS;QAChB,IAAIG,cAAcC,gCAAgCJ;QAClD,MAAMK,yBACJC,oBAAoB,yBAAyB,CAC3CC,uCACGhE;QACP,IAAIiE,OAAO,UAAU,CAACL,aAAa,UAAUE,wBAAwB;YACnE,MAAMI,SAASN,YAAY,KAAK,CAAC,GAAG;YACpC,MAAMO,OAAOC,eAAenD,QAAW2C;YACvCA,cAAc,GAAGM,OAAO,CAAC,EAAEC,MAAM;QACnC;QACA,IAAI,CAAC,OAAO,GAAGP;QAEf,MAAMS,eAAeC,QAAQnD,SAAS;QACtC,MAAMoD,gBAAgBD,QAAQnD,SAAS;QAEvC,IAAIkD,gBAAgBE,eAClB,MAAM,IAAIC,MAAM;QAGlB,IAAI,CAAC,iBAAiB,GAAGD,gBAAgB,QAAQb;QACjD,IAAI,CAAC,YAAY,GAAGW;QACpB,IAAI,CAAC,aAAa,GAAGE;QAErB,IAAIZ,oBACF,IAAI,AAA8B,YAA9B,OAAOA,oBACT,IAAI,CAAC,aAAa,GAChBc,eAAeC,aAAazD,SAAY0C;aAE1C,IAAI,CAAC,YAAY,GAAGA;QAIxB,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB,IAAI,CAAC,aAAa,GAChBc,eAAeC,aACXzD,SACA0D,KACEC,qBAAqB,UACrB,GAAG,IAAI,CAAC,OAAO,GAAGxE,cAAc;QAI1C,IAAIyE;QACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAIrB;YAAA,IAAI,CAAC,IAAI,CAAC,YAAY,EACpBA,eAAe,IAAI,CAAC,iBAAiB;QACvC;QAEF,IAAI,CAACA,cACHA,eAAe;YACb,iBAAiBtD;YACjB,SAAS,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE;QACZ;QAEF,IAAI,CAAC,KAAK,GAAGsD;QACb,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,GAC7C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GACxB;IACN;AA2VF"}

package/dist/es/agent/utils.mjs

@@ -152,7 +152,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.5.11";
+const getMidsceneVersion = ()=>"1.5.14";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/prompt/llm-planning.mjs

@@ -351,7 +351,7 @@ ${shouldIncludeSubGoals ? `
 ${shouldIncludeSubGoals ? `
 ## Example
 
-Instruction: "fill the form with name 'John' and email 'test@midscene.ai', return the email"
+Instruction: "fill the form with name 'John' and email 'testmidscene.ai', return the email"
 
 **Turn 1:** Empty form → <thought>Plan: fill Name, Email, return email</thought><update-plan-content><sub-goal index="1" status="pending">Fill Name</sub-goal><sub-goal index="2" status="pending">Fill Email</sub-goal></update-plan-content><action-type>Tap</action-type><action-param-json>{"locate": ${locateNameField}}</action-param-json>
 
@@ -359,13 +359,13 @@ Instruction: "fill the form with name 'John' and email 'test@midscene.ai', retur
 
 **Turn 3:** Name='John' → <mark-sub-goal-done><sub-goal index="1" status="finished" /></mark-sub-goal-done><action-type>Tap</action-type><action-param-json>{"locate": ${locateEmailField}}</action-param-json>
 
-**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "test@midscene.ai"}</action-param-json>
+**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "testmidscene.ai"}</action-param-json>
 
-**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index="2" status="finished" /></mark-sub-goal-done><complete success="true">test@midscene.ai</complete>
+**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index="2" status="finished" /></mark-sub-goal-done><complete success="true">testmidscene.ai</complete>
 ` : `
 ## Example
 
-Instruction: "fill the form with name 'John' and email 'test@midscene.ai', return the email"
+Instruction: "fill the form with name 'John' and email 'testmidscene.ai', return the email"
 
 **Turn 1:** Empty form → <thought>Fill Name then Email, return email</thought><action-type>Tap</action-type><action-param-json>{"locate": ${locateNameField}}</action-param-json>
 
@@ -373,9 +373,9 @@ Instruction: "fill the form with name 'John' and email 'test@midscene.ai', retur
 
 **Turn 3:** Name='John' → <action-type>Tap</action-type><action-param-json>{"locate": ${locateEmailField}}</action-param-json>
 
-**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "test@midscene.ai"}</action-param-json>
+**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "testmidscene.ai"}</action-param-json>
 
-**Turn 5:** Form complete → <complete success="true">test@midscene.ai</complete>
+**Turn 5:** Form complete → <complete success="true">testmidscene.ai</complete>
 `}`;
 }
 export { descriptionForAction, systemPromptToTaskPlanning };

package/dist/es/ai-model/prompt/llm-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-planning.mjs","sources":["../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\n/**\n * Find ZodDefault in the wrapper chain and return its default value\n */\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    // Continue unwrapping if it's a wrapper type\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\n/**\n * Inject bbox into locate fields of a sample object.\n * Walks the sample and for any locate field (identified by paramSchema),\n * adds a fake bbox array when includeBbox is true.\n */\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name using extracted helper\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          // Get description using extracted helper\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          // Check if field has a default value by searching the wrapper chain\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  // Render sample if provided, using the same XML tag format as the real output\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  // Validate parameters: if includeBbox is true, modelFamily must be defined\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  // Generate locate object examples based on includeBbox\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  // Locate examples for multi-turn conversation\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const thoughtTag = (content: string) =>\n    shouldIncludeThought ? `<thought>${content}</thought>\\n` : '';\n\n  // Sub-goals related content - only included when shouldIncludeSubGoals is true\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)'\n    : '## Step 1: Observe (related tags: <thought>)';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\"\n    : 'First, observe the current screenshot and previous logs to understand the current state.';\n\n  const explicitInstructionRule = `CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`;\n\n  const thoughtTagDescription = shouldIncludeSubGoals\n    ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`\n    : `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`;\n\n  const subGoalTags = shouldIncludeSubGoals\n    ? `\n\n* <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n* Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<thought>...</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`\n    : '';\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const memoryStepNumber = 2; // Only used when shouldIncludeSubGoals is true\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalTags}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)\n\nWhile observing the current screenshot, if you notice any information that might be needed in follow-up actions, record it here. The current screenshot will NOT be available in subsequent steps, so this memory is your only way to preserve essential information. Examples: extracted data, element states, content that needs to be referenced.\n\nDon't use this tag if no information needs to be preserved.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: ${shouldIncludeSubGoals ? 'Check if Goal is Accomplished' : 'Check if the Instruction is Fulfilled'} (related tags: <complete>)\n\n${shouldIncludeSubGoals ? 'Based on the current screenshot and the status of all sub-goals, determine' : 'Determine'} if the entire task is completed.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${shouldIncludeSubGoals ? 'goal accomplished' : 'instruction fulfilled'}\" means:**\n- The ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when selected. Do NOT proceed to checkout.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${shouldIncludeSubGoals ? 'the goal' : 'it'} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n${\n  !shouldIncludeSubGoals\n    ? `\n**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.\n`\n    : ''\n}\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${shouldIncludeSubGoals ? 'It means whether the expected goal is accomplished based on what you observe in the current screenshot. ' : ''}No matter what actions were executed or what errors occurred during execution, if the ${shouldIncludeSubGoals ? 'expected goal is accomplished' : 'instruction is fulfilled'}, set success=\"true\". If the ${shouldIncludeSubGoals ? 'expected goal is not accomplished and cannot be accomplished' : 'instruction is not fulfilled and cannot be fulfilled'}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output <action-type> or <action-param-json>. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${shouldIncludeSubGoals ? ' and the plan' : ''}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Supporting actions list\n\n${actionList}\n\n### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>\n\n### If there is some action to do ...\n\n- Use the <action-type> and <action-param-json> tags to output the action to be executed.\n- The <action-type> MUST be one of the supporting actions. 'complete' is NOT a valid action-type.\n- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If an action has a \"sample\" in its description, follow that structure.\nFor example:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output <action-type> or <action-param-json> if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n**Always include (REQUIRED):**\n<!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<!-- Step ${memoryStepNumber}: Memory data from current screenshot if needed -->\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} or failed (Step ${checkGoalStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${shouldIncludeSubGoals ? 'goal is NOT complete' : 'instruction is NOT fulfilled'} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n${\n  shouldIncludeSubGoals\n    ? `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'test@midscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Plan: fill Name, Email, return email</thought><update-plan-content><sub-goal index=\"1\" status=\"pending\">Fill Name</sub-goal><sub-goal index=\"2\" status=\"pending\">Fill Email</sub-goal></update-plan-content><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <mark-sub-goal-done><sub-goal index=\"1\" status=\"finished\" /></mark-sub-goal-done><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"test@midscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index=\"2\" status=\"finished\" /></mark-sub-goal-done><complete success=\"true\">test@midscene.ai</complete>\n`\n    : `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'test@midscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Fill Name then Email, return email</thought><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"test@midscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <complete success=\"true\">test@midscene.ai</complete>\n`\n}`;\n}\n"],"names":["vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","key","Object","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","systemPromptToTaskPlanning","actionSpace","includeThought","includeSubGoals","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","locateNameField","locateEmailField","step1Title","step1Description","explicitInstructionRule","thoughtTagDescription","subGoalTags","memoryStepNumber","checkGoalStepNumber","actionStepNumber"],"mappings":";;;;AAWA,MAAMA,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,gBAAgBD,cAAc;IAEvG,OAAO;AACT;AAKA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAIzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAOA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAACG,KAAKrB,MAAM,IAAIsB,OAAO,OAAO,CAACF,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMuB,aACJ,AACE,cADF,OAAQvB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMwB,kBAAkBD,aAAa,GAAGF,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMI,WAAWC,eAAe1B,OAAOc;gBAGvC,MAAMa,cAAcC,kBAAkB5B;gBAGtC,MAAM6B,eAAe9B,iBAAiBC;gBACtC,MAAM8B,aAAaD,AAAiBE,WAAjBF;gBAGnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3ChB,WAAW,IAAI,CAACe;YAClB;YAIF,IAAIf,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACmB;oBAClBpB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEoB,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAMX,WAAWC,eAAeR;YAChC,MAAMS,cAAcC,kBAAkBV;YAGtC,IAAImB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBrB,OAAO,IAAI,CAACqB;QACd;IACF;IAGA,IAAIxB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe8B,4BAA4BzB,OAAO,WAAW;QACnE,MAAM0B,iBAAiBjC,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM+B,YAAY,CAAC,WAAW,EAAEzB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMoB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAExB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACwB;IACd;IAEA,OAAO,CAAC,EAAE,EAAE3B,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAe0B,2BAA2B,EAC/CC,WAAW,EACX7C,WAAW,EACXY,WAAW,EACXkC,cAAc,EACdC,eAAe,EAOhB;IACC,MAAMC,oBAAoBC;IAG1B,IAAIrC,eAAe,CAACZ,aAClB,MAAM,IAAIkD,MACR;IAIJ,MAAMC,wBAAwBN,YAAY,GAAG,CAAC,CAAC7B,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAckC,SAC1CtB;IAGJ,MAAMwC,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBN,mBAAmB;IAGjD,MAAMO,iBAAiB1C,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAGF,MAAM2C,kBAAkB3C,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAM4C,mBAAmB5C,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAMF,MAAM6C,aAAaJ,wBACf,uGACA;IAEJ,MAAMK,mBAAmBL,wBACrB,wNACA;IAEJ,MAAMM,0BAA0B;IAEhC,MAAMC,wBAAwBP,wBAC1B,CAAC;;;;AAIP,EAAEM,yBAAyB,GACrB,CAAC;;;;AAIP,EAAEA,yBAAyB;IAEzB,MAAME,cAAcR,wBAChB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAuDc,CAAC,GAChB;IAIJ,MAAMS,mBAAmB;IACzB,MAAMC,sBAAsBV,wBAAwB,IAAI;IACxD,MAAMW,mBAAmBX,wBAAwB,IAAI;IAErD,OAAO,CAAC;;;AAGV,EAAEI,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEE,sBAAsB;AACxB,EAAEC,YAAY;AACd,EACER,wBACI,CAAC;QACC,EAAES,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,EAAE,EAAEV,wBAAwB,kCAAkC,wCAAwC;;AAEpI,EAAEA,wBAAwB,+EAA+E,YAAY;;;;;;;;;;QAU7G,EAAEA,wBAAwB,sBAAsB,wBAAwB;MAC1E,EAAEA,wBAAwB,yBAAyB,2BAA2B;;;;wBAI5D,EAAEA,wBAAwB,sBAAsB,wBAAwB;6BACnE,EAAEA,wBAAwB,sBAAsB,wBAAwB;qCAChE,EAAEA,wBAAwB,sBAAsB,wBAAwB;4BACjF,EAAEA,wBAAwB,sBAAsB,wBAAwB;;;0NAGsH,EAAEA,wBAAwB,aAAa,KAAK;;AAEtQ,EACE,CAACA,wBACG,CAAC;;;;;AAKP,CAAC,GACK,GACL;;;sEAGqE,EAAEW,iBAAiB;;yCAEhD,EAAEX,wBAAwB,6GAA6G,GAAG,sFAAsF,EAAEA,wBAAwB,kCAAkC,2BAA2B,6BAA6B,EAAEA,wBAAwB,iEAAiE,uDAAuD;;;;QAIvgB,EAAEW,iBAAiB;;mGAEwE,EAAEX,wBAAwB,kBAAkB,GAAG;;;;;;;;;;AAUlJ,EAAED,WAAW;;;;;;QAML,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;;;;;;;;oBAoBT,EAAED,wBAAwB,cAAc,GAAG;;AAE/D,EACEA,wBACI,CAAC;;;;;;;;AAQP,CAAC,GACK,KAEJA,wBACI,CAAC;UACG,EAAES,iBAAiB;;AAE7B,CAAC,GACK,GACL;;;iBAGgB,EAAET,wBAAwB,yBAAyB,2BAA2B,iBAAiB,EAAEU,oBAAoB;;;iBAGrH,EAAEV,wBAAwB,yBAAyB,+BAA+B,WAAW,EAAEW,iBAAiB;;;;;;;;AAQjI,EACEX,wBACI,CAAC;;;;;0SAKmS,EAAEE,gBAAgB;;;;uKAIrJ,EAAEC,iBAAiB;;;;;AAK1L,CAAC,GACK,CAAC;;;;;0IAKmI,EAAED,gBAAgB;;;;sFAItE,EAAEC,iBAAiB;;;;;AAKzG,CAAC,EACC;AACF"}
+{"version":3,"file":"ai-model/prompt/llm-planning.mjs","sources":["../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\n/**\n * Find ZodDefault in the wrapper chain and return its default value\n */\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    // Continue unwrapping if it's a wrapper type\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\n/**\n * Inject bbox into locate fields of a sample object.\n * Walks the sample and for any locate field (identified by paramSchema),\n * adds a fake bbox array when includeBbox is true.\n */\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name using extracted helper\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          // Get description using extracted helper\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          // Check if field has a default value by searching the wrapper chain\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  // Render sample if provided, using the same XML tag format as the real output\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  // Validate parameters: if includeBbox is true, modelFamily must be defined\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  // Generate locate object examples based on includeBbox\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  // Locate examples for multi-turn conversation\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const thoughtTag = (content: string) =>\n    shouldIncludeThought ? `<thought>${content}</thought>\\n` : '';\n\n  // Sub-goals related content - only included when shouldIncludeSubGoals is true\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)'\n    : '## Step 1: Observe (related tags: <thought>)';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\"\n    : 'First, observe the current screenshot and previous logs to understand the current state.';\n\n  const explicitInstructionRule = `CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`;\n\n  const thoughtTagDescription = shouldIncludeSubGoals\n    ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`\n    : `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`;\n\n  const subGoalTags = shouldIncludeSubGoals\n    ? `\n\n* <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n* Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<thought>...</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`\n    : '';\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const memoryStepNumber = 2; // Only used when shouldIncludeSubGoals is true\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalTags}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)\n\nWhile observing the current screenshot, if you notice any information that might be needed in follow-up actions, record it here. The current screenshot will NOT be available in subsequent steps, so this memory is your only way to preserve essential information. Examples: extracted data, element states, content that needs to be referenced.\n\nDon't use this tag if no information needs to be preserved.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: ${shouldIncludeSubGoals ? 'Check if Goal is Accomplished' : 'Check if the Instruction is Fulfilled'} (related tags: <complete>)\n\n${shouldIncludeSubGoals ? 'Based on the current screenshot and the status of all sub-goals, determine' : 'Determine'} if the entire task is completed.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${shouldIncludeSubGoals ? 'goal accomplished' : 'instruction fulfilled'}\" means:**\n- The ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when selected. Do NOT proceed to checkout.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${shouldIncludeSubGoals ? 'the goal' : 'it'} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n${\n  !shouldIncludeSubGoals\n    ? `\n**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.\n`\n    : ''\n}\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${shouldIncludeSubGoals ? 'It means whether the expected goal is accomplished based on what you observe in the current screenshot. ' : ''}No matter what actions were executed or what errors occurred during execution, if the ${shouldIncludeSubGoals ? 'expected goal is accomplished' : 'instruction is fulfilled'}, set success=\"true\". If the ${shouldIncludeSubGoals ? 'expected goal is not accomplished and cannot be accomplished' : 'instruction is not fulfilled and cannot be fulfilled'}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output <action-type> or <action-param-json>. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${shouldIncludeSubGoals ? ' and the plan' : ''}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Supporting actions list\n\n${actionList}\n\n### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>\n\n### If there is some action to do ...\n\n- Use the <action-type> and <action-param-json> tags to output the action to be executed.\n- The <action-type> MUST be one of the supporting actions. 'complete' is NOT a valid action-type.\n- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If an action has a \"sample\" in its description, follow that structure.\nFor example:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output <action-type> or <action-param-json> if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n**Always include (REQUIRED):**\n<!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<!-- Step ${memoryStepNumber}: Memory data from current screenshot if needed -->\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} or failed (Step ${checkGoalStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${shouldIncludeSubGoals ? 'goal is NOT complete' : 'instruction is NOT fulfilled'} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n${\n  shouldIncludeSubGoals\n    ? `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'testmidscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Plan: fill Name, Email, return email</thought><update-plan-content><sub-goal index=\"1\" status=\"pending\">Fill Name</sub-goal><sub-goal index=\"2\" status=\"pending\">Fill Email</sub-goal></update-plan-content><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <mark-sub-goal-done><sub-goal index=\"1\" status=\"finished\" /></mark-sub-goal-done><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"testmidscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index=\"2\" status=\"finished\" /></mark-sub-goal-done><complete success=\"true\">testmidscene.ai</complete>\n`\n    : `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'testmidscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Fill Name then Email, return email</thought><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"testmidscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <complete success=\"true\">testmidscene.ai</complete>\n`\n}`;\n}\n"],"names":["vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","key","Object","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","systemPromptToTaskPlanning","actionSpace","includeThought","includeSubGoals","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","locateNameField","locateEmailField","step1Title","step1Description","explicitInstructionRule","thoughtTagDescription","subGoalTags","memoryStepNumber","checkGoalStepNumber","actionStepNumber"],"mappings":";;;;AAWA,MAAMA,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,gBAAgBD,cAAc;IAEvG,OAAO;AACT;AAKA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAIzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAOA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAACG,KAAKrB,MAAM,IAAIsB,OAAO,OAAO,CAACF,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMuB,aACJ,AACE,cADF,OAAQvB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMwB,kBAAkBD,aAAa,GAAGF,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMI,WAAWC,eAAe1B,OAAOc;gBAGvC,MAAMa,cAAcC,kBAAkB5B;gBAGtC,MAAM6B,eAAe9B,iBAAiBC;gBACtC,MAAM8B,aAAaD,AAAiBE,WAAjBF;gBAGnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3ChB,WAAW,IAAI,CAACe;YAClB;YAIF,IAAIf,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACmB;oBAClBpB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEoB,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAMX,WAAWC,eAAeR;YAChC,MAAMS,cAAcC,kBAAkBV;YAGtC,IAAImB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBrB,OAAO,IAAI,CAACqB;QACd;IACF;IAGA,IAAIxB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe8B,4BAA4BzB,OAAO,WAAW;QACnE,MAAM0B,iBAAiBjC,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM+B,YAAY,CAAC,WAAW,EAAEzB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMoB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAExB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACwB;IACd;IAEA,OAAO,CAAC,EAAE,EAAE3B,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAe0B,2BAA2B,EAC/CC,WAAW,EACX7C,WAAW,EACXY,WAAW,EACXkC,cAAc,EACdC,eAAe,EAOhB;IACC,MAAMC,oBAAoBC;IAG1B,IAAIrC,eAAe,CAACZ,aAClB,MAAM,IAAIkD,MACR;IAIJ,MAAMC,wBAAwBN,YAAY,GAAG,CAAC,CAAC7B,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAckC,SAC1CtB;IAGJ,MAAMwC,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBN,mBAAmB;IAGjD,MAAMO,iBAAiB1C,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAGF,MAAM2C,kBAAkB3C,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAM4C,mBAAmB5C,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAMF,MAAM6C,aAAaJ,wBACf,uGACA;IAEJ,MAAMK,mBAAmBL,wBACrB,wNACA;IAEJ,MAAMM,0BAA0B;IAEhC,MAAMC,wBAAwBP,wBAC1B,CAAC;;;;AAIP,EAAEM,yBAAyB,GACrB,CAAC;;;;AAIP,EAAEA,yBAAyB;IAEzB,MAAME,cAAcR,wBAChB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAuDc,CAAC,GAChB;IAIJ,MAAMS,mBAAmB;IACzB,MAAMC,sBAAsBV,wBAAwB,IAAI;IACxD,MAAMW,mBAAmBX,wBAAwB,IAAI;IAErD,OAAO,CAAC;;;AAGV,EAAEI,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEE,sBAAsB;AACxB,EAAEC,YAAY;AACd,EACER,wBACI,CAAC;QACC,EAAES,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,EAAE,EAAEV,wBAAwB,kCAAkC,wCAAwC;;AAEpI,EAAEA,wBAAwB,+EAA+E,YAAY;;;;;;;;;;QAU7G,EAAEA,wBAAwB,sBAAsB,wBAAwB;MAC1E,EAAEA,wBAAwB,yBAAyB,2BAA2B;;;;wBAI5D,EAAEA,wBAAwB,sBAAsB,wBAAwB;6BACnE,EAAEA,wBAAwB,sBAAsB,wBAAwB;qCAChE,EAAEA,wBAAwB,sBAAsB,wBAAwB;4BACjF,EAAEA,wBAAwB,sBAAsB,wBAAwB;;;0NAGsH,EAAEA,wBAAwB,aAAa,KAAK;;AAEtQ,EACE,CAACA,wBACG,CAAC;;;;;AAKP,CAAC,GACK,GACL;;;sEAGqE,EAAEW,iBAAiB;;yCAEhD,EAAEX,wBAAwB,6GAA6G,GAAG,sFAAsF,EAAEA,wBAAwB,kCAAkC,2BAA2B,6BAA6B,EAAEA,wBAAwB,iEAAiE,uDAAuD;;;;QAIvgB,EAAEW,iBAAiB;;mGAEwE,EAAEX,wBAAwB,kBAAkB,GAAG;;;;;;;;;;AAUlJ,EAAED,WAAW;;;;;;QAML,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;;;;;;;;oBAoBT,EAAED,wBAAwB,cAAc,GAAG;;AAE/D,EACEA,wBACI,CAAC;;;;;;;;AAQP,CAAC,GACK,KAEJA,wBACI,CAAC;UACG,EAAES,iBAAiB;;AAE7B,CAAC,GACK,GACL;;;iBAGgB,EAAET,wBAAwB,yBAAyB,2BAA2B,iBAAiB,EAAEU,oBAAoB;;;iBAGrH,EAAEV,wBAAwB,yBAAyB,+BAA+B,WAAW,EAAEW,iBAAiB;;;;;;;;AAQjI,EACEX,wBACI,CAAC;;;;;0SAKmS,EAAEE,gBAAgB;;;;uKAIrJ,EAAEC,iBAAiB;;;;;AAK1L,CAAC,GACK,CAAC;;;;;0IAKmI,EAAED,gBAAgB;;;;sFAItE,EAAEC,iBAAiB;;;;;AAKzG,CAAC,EACC;AACF"}

package/dist/es/device/index.mjs

@@ -96,7 +96,7 @@ const defineActionInput = (call)=>defineAction({
         interfaceAlias: 'aiInput',
         paramSchema: actionInputParamSchema,
         sample: {
-            value: 'test@example.com',
+            value: 'testexample.com',
             locate: {
                 prompt: 'the email input field'
             }

package/dist/es/device/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"device/index.mjs","sources":["../../../src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/common';\nimport type {\n  ActionScrollParam,\n  DeviceAction,\n  LocateResultElement,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n  accept(files: string[]): Promise<void>;\n}\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[];\n\n  abstract cacheFeatureForPoint?(\n    center: [number, number],\n    options?: {\n      targetDescription?: string;\n      modelConfig?: IModelConfig;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // for web only\n  registerFileChooserListener?(\n    handler: (chooser: FileChooserHandler) => Promise<void>,\n  ): Promise<{ dispose: () => void; getError: () => Error | undefined }>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  /**\n   * Get the current time from the device.\n   * Returns the device's current timestamp in milliseconds.\n   * This is useful when the system time and device time are not synchronized.\n   */\n  getTimestamp?(): Promise<number>;\n\n  /** URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) */\n  mjpegStreamUrl?: string;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n  TSchema extends z.ZodType | undefined = undefined,\n  TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n  TReturn = any,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema?: TSchema;\n    call: (param: TRuntime) => Promise<TReturn> | TReturn;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime, TReturn>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime, TReturn> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    sample: {\n      locate: { prompt: 'the \"Submit\" button' },\n    },\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    sample: {\n      locate: { prompt: 'the file icon on the desktop' },\n    },\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    sample: {\n      locate: { prompt: 'the folder icon' },\n    },\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    sample: {\n      locate: { prompt: 'the navigation menu item \"Products\"' },\n    },\n    call,\n  });\n};\n\n// Input\nconst inputLocateDescription =\n  'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe(inputLocateDescription)\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'typeOnly'])\n    .default('replace')\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'typeOnly' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    sample: {\n      value: 'test@example.com',\n      locate: { prompt: 'the email input field' },\n    },\n    call: (param) => {\n      // backward compat: convert deprecated 'append' to 'typeOnly'\n      if ((param.mode as string) === 'append') {\n        param.mode = 'typeOnly';\n      }\n      return call(param);\n    },\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    sample: {\n      keyName: 'Enter',\n    },\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  scrollType: z\n    .enum([\n      'singleAction',\n      'scrollToBottom',\n      'scrollToTop',\n      'scrollToRight',\n      'scrollToLeft',\n    ])\n    .default('singleAction')\n    .describe(\n      'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n    ),\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe(\n      'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n    ),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n    ),\n});\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    sample: {\n      direction: 'down',\n      scrollType: 'singleAction',\n      locate: { prompt: 'the center of the product list area' },\n    },\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description:\n      'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    sample: {\n      from: { prompt: 'the \"report.pdf\" file icon' },\n      to: { prompt: 'the upload drop zone' },\n    },\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    sample: {\n      locate: { prompt: 'the message bubble' },\n    },\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n  param: ActionSwipeParam,\n  screenSize: { width: number; height: number },\n): {\n  startPoint: { x: number; y: number };\n  endPoint: { x: number; y: number };\n  duration: number;\n  repeatCount: number;\n} {\n  const { width, height } = screenSize;\n  const { start, end } = param;\n\n  const startPoint = start\n    ? { x: start.center[0], y: start.center[1] }\n    : { x: width / 2, y: height / 2 };\n\n  let endPoint: { x: number; y: number };\n\n  if (end) {\n    endPoint = { x: end.center[0], y: end.center[1] };\n  } else if (param.distance) {\n    const direction = param.direction;\n    if (!direction) {\n      throw new Error('direction is required for swipe gesture');\n    }\n    endPoint = {\n      x:\n        startPoint.x +\n        (direction === 'right'\n          ? param.distance\n          : direction === 'left'\n            ? -param.distance\n            : 0),\n      y:\n        startPoint.y +\n        (direction === 'down'\n          ? param.distance\n          : direction === 'up'\n            ? -param.distance\n            : 0),\n    };\n  } else {\n    throw new Error(\n      'Either end or distance must be specified for swipe gesture',\n    );\n  }\n\n  endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n  endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n  const duration = param.duration ?? 300;\n\n  let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n  if (repeatCount === 0) {\n    repeatCount = 10;\n  }\n\n  return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n    paramSchema: ActionSwipeParamSchema,\n    sample: {\n      start: { prompt: 'center of the notification' },\n      end: { prompt: 'upper edge of the screen' },\n    },\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The input field to be cleared')\n    .optional(),\n});\nexport type ActionClearInputParam = {\n  locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: inputLocateDescription,\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    sample: {\n      locate: { prompt: 'the search input field' },\n    },\n    call,\n  });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n  direction: z\n    .enum(['left', 'right'])\n    .describe('The direction to move the cursor'),\n  times: z\n    .number()\n    .int()\n    .min(1)\n    .default(1)\n    .describe(\n      'The number of times to move the cursor in the specified direction',\n    ),\n});\nexport type ActionCursorMoveParam = {\n  direction: 'left' | 'right';\n  times?: number;\n};\n\nexport const defineActionCursorMove = (\n  call: (param: ActionCursorMoveParam) => Promise<void>,\n): DeviceAction<ActionCursorMoveParam> => {\n  return defineAction<\n    typeof actionCursorMoveParamSchema,\n    ActionCursorMoveParam\n  >({\n    name: 'CursorMove',\n    description:\n      'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n    paramSchema: actionCursorMoveParamSchema,\n    sample: {\n      direction: 'left',\n      times: 3,\n    },\n    call,\n  });\n};\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n  timeMs: z\n    .number()\n    .default(1000)\n    .optional()\n    .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n  timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n  return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n    name: 'Sleep',\n    description:\n      'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n    paramSchema: ActionSleepParamSchema,\n    sample: {\n      timeMs: 2000,\n    },\n    call: async (param) => {\n      const duration = param?.timeMs ?? 1000;\n      getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n      await new Promise((resolve) => setTimeout(resolve, duration));\n    },\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n  HarmonyDeviceOpt,\n  HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","param","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Error","Math","duration","repeatCount","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput","actionCursorMoveParamSchema","defineActionCursorMove","ActionSleepParamSchema","defineActionSleep","getDebug","Promise","resolve","setTimeout"],"mappings":";;;;;;;;;;;;;AAiBO,MAAeA;;QA8CpB;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACAI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACAD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACAH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAL;IACF;AAIF,MAAMO,yBACJ;AACK,MAAMC,yBAAyBX,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACY,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQX,4BACL,QAAQ,CAACS,wBACT,QAAQ;IACX,MAAMV,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;AAEN;AAOO,MAAMc,oBAAoB,CAC/BX,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,CAACI;YAEL,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAEf,OAAOZ,KAAKY;QACd;IACF;AAIK,MAAMC,iCAAiChB,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMiB,4BAA4B,CACvCd,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACb,QAAQ;YACN,SAAS;QACX;QACAb;IACF;AAIK,MAAMe,0BAA0BlB,EAAE,MAAM,CAAC;IAC9C,YAAYA,CAAC,CAADA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMkB,qBAAqB,CAChChB,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaqB;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAf;IACF;AAIK,MAAMiB,+BAA+BpB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMoB,0BAA0B,CACrClB,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAauB;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACAjB;IACF;AAGK,MAAMmB,6BAA6BtB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMuB,wBAAwB,CACnCpB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAayB;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACAnB;IACF;AAGK,MAAMqB,yBAAyBxB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASyB,0BACdV,KAAuB,EACvBW,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAGf;IAEvB,MAAMgB,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAIf,MAAM,QAAQ,EAAE;QACzB,MAAMkB,YAAYlB,MAAM,SAAS;QACjC,IAAI,CAACkB,WACH,MAAM,IAAIC,MAAM;QAElBF,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,WAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;YACR,GACEgB,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,SAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAImB,MACR;IAIJF,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEJ;IAE9C,MAAMQ,WAAWrB,MAAM,QAAQ,IAAI;IAEnC,IAAIsB,cAAc,AAAwB,YAAxB,OAAOtB,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIsB,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEN;QAAYC;QAAUI;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAC/BnC,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2B;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACArB;IACF;AAIK,MAAMoC,8BAA8BvC,EAAE,MAAM,CAAC;IAClD,QAAQC,4BACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMuC,yBAAyB,CACpCrC,OAEON,aAGL;QACA,MAAM;QACN,aAAaa;QACb,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACApC;IACF;AAIK,MAAMsC,8BAA8BzC,EAAE,MAAM,CAAC;IAClD,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,EAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAM0C,yBAAyB,CACpCvC,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa4C;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACAtC;IACF;AAGK,MAAMwC,yBAAyB3C,EAAE,MAAM,CAAC;IAC7C,QAAQA,EAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAM4C,oBAAoB,IACxB/C,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8C;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAO5B;YACX,MAAMqB,WAAWrB,OAAO,UAAU;YAClC8B,SAAS,wBAAwB,CAAC,aAAa,EAAET,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIU,QAAQ,CAACC,UAAYC,WAAWD,SAASX;QACrD;IACF"}
+{"version":3,"file":"device/index.mjs","sources":["../../../src/device/index.ts"],"sourcesContent":["import { getMidsceneLocationSchema } from '@/common';\nimport type {\n  ActionScrollParam,\n  DeviceAction,\n  LocateResultElement,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n  accept(files: string[]): Promise<void>;\n}\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[];\n\n  abstract cacheFeatureForPoint?(\n    center: [number, number],\n    options?: {\n      targetDescription?: string;\n      modelConfig?: IModelConfig;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // for web only\n  registerFileChooserListener?(\n    handler: (chooser: FileChooserHandler) => Promise<void>,\n  ): Promise<{ dispose: () => void; getError: () => Error | undefined }>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  /**\n   * Get the current time from the device.\n   * Returns the device's current timestamp in milliseconds.\n   * This is useful when the system time and device time are not synchronized.\n   */\n  getTimestamp?(): Promise<number>;\n\n  /** URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) */\n  mjpegStreamUrl?: string;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n  TSchema extends z.ZodType | undefined = undefined,\n  TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n  TReturn = any,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema?: TSchema;\n    call: (param: TRuntime) => Promise<TReturn> | TReturn;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime, TReturn>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime, TReturn> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    sample: {\n      locate: { prompt: 'the \"Submit\" button' },\n    },\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    sample: {\n      locate: { prompt: 'the file icon on the desktop' },\n    },\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    sample: {\n      locate: { prompt: 'the folder icon' },\n    },\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    sample: {\n      locate: { prompt: 'the navigation menu item \"Products\"' },\n    },\n    call,\n  });\n};\n\n// Input\nconst inputLocateDescription =\n  'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe(inputLocateDescription)\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'typeOnly'])\n    .default('replace')\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'typeOnly' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    sample: {\n      value: 'testexample.com',\n      locate: { prompt: 'the email input field' },\n    },\n    call: (param) => {\n      // backward compat: convert deprecated 'append' to 'typeOnly'\n      if ((param.mode as string) === 'append') {\n        param.mode = 'typeOnly';\n      }\n      return call(param);\n    },\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    sample: {\n      keyName: 'Enter',\n    },\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  scrollType: z\n    .enum([\n      'singleAction',\n      'scrollToBottom',\n      'scrollToTop',\n      'scrollToRight',\n      'scrollToLeft',\n    ])\n    .default('singleAction')\n    .describe(\n      'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n    ),\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe(\n      'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n    ),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n    ),\n});\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    sample: {\n      direction: 'down',\n      scrollType: 'singleAction',\n      locate: { prompt: 'the center of the product list area' },\n    },\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description:\n      'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    sample: {\n      from: { prompt: 'the \"report.pdf\" file icon' },\n      to: { prompt: 'the upload drop zone' },\n    },\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    sample: {\n      locate: { prompt: 'the message bubble' },\n    },\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n  param: ActionSwipeParam,\n  screenSize: { width: number; height: number },\n): {\n  startPoint: { x: number; y: number };\n  endPoint: { x: number; y: number };\n  duration: number;\n  repeatCount: number;\n} {\n  const { width, height } = screenSize;\n  const { start, end } = param;\n\n  const startPoint = start\n    ? { x: start.center[0], y: start.center[1] }\n    : { x: width / 2, y: height / 2 };\n\n  let endPoint: { x: number; y: number };\n\n  if (end) {\n    endPoint = { x: end.center[0], y: end.center[1] };\n  } else if (param.distance) {\n    const direction = param.direction;\n    if (!direction) {\n      throw new Error('direction is required for swipe gesture');\n    }\n    endPoint = {\n      x:\n        startPoint.x +\n        (direction === 'right'\n          ? param.distance\n          : direction === 'left'\n            ? -param.distance\n            : 0),\n      y:\n        startPoint.y +\n        (direction === 'down'\n          ? param.distance\n          : direction === 'up'\n            ? -param.distance\n            : 0),\n    };\n  } else {\n    throw new Error(\n      'Either end or distance must be specified for swipe gesture',\n    );\n  }\n\n  endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n  endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n  const duration = param.duration ?? 300;\n\n  let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n  if (repeatCount === 0) {\n    repeatCount = 10;\n  }\n\n  return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n    paramSchema: ActionSwipeParamSchema,\n    sample: {\n      start: { prompt: 'center of the notification' },\n      end: { prompt: 'upper edge of the screen' },\n    },\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The input field to be cleared')\n    .optional(),\n});\nexport type ActionClearInputParam = {\n  locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: inputLocateDescription,\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    sample: {\n      locate: { prompt: 'the search input field' },\n    },\n    call,\n  });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n  direction: z\n    .enum(['left', 'right'])\n    .describe('The direction to move the cursor'),\n  times: z\n    .number()\n    .int()\n    .min(1)\n    .default(1)\n    .describe(\n      'The number of times to move the cursor in the specified direction',\n    ),\n});\nexport type ActionCursorMoveParam = {\n  direction: 'left' | 'right';\n  times?: number;\n};\n\nexport const defineActionCursorMove = (\n  call: (param: ActionCursorMoveParam) => Promise<void>,\n): DeviceAction<ActionCursorMoveParam> => {\n  return defineAction<\n    typeof actionCursorMoveParamSchema,\n    ActionCursorMoveParam\n  >({\n    name: 'CursorMove',\n    description:\n      'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n    paramSchema: actionCursorMoveParamSchema,\n    sample: {\n      direction: 'left',\n      times: 3,\n    },\n    call,\n  });\n};\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n  timeMs: z\n    .number()\n    .default(1000)\n    .optional()\n    .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n  timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n  return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n    name: 'Sleep',\n    description:\n      'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n    paramSchema: ActionSleepParamSchema,\n    sample: {\n      timeMs: 2000,\n    },\n    call: async (param) => {\n      const duration = param?.timeMs ?? 1000;\n      getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n      await new Promise((resolve) => setTimeout(resolve, duration));\n    },\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n  HarmonyDeviceOpt,\n  HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","param","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Error","Math","duration","repeatCount","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput","actionCursorMoveParamSchema","defineActionCursorMove","ActionSleepParamSchema","defineActionSleep","getDebug","Promise","resolve","setTimeout"],"mappings":";;;;;;;;;;;;;AAiBO,MAAeA;;QA8CpB;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAIF,MAAMC,uBAAuBC,EAAE,MAAM,CAAC;IAC3C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACAI;IACF;AAIK,MAAMC,8BAA8BJ,EAAE,MAAM,CAAC;IAClD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACAD;IACF;AAIK,MAAMG,+BAA+BN,EAAE,MAAM,CAAC;IACnD,QAAQC,4BAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACAH;IACF;AAIK,MAAMK,yBAAyBR,EAAE,MAAM,CAAC;IAC7C,QAAQC,4BAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAL;IACF;AAIF,MAAMO,yBACJ;AACK,MAAMC,yBAAyBX,EAAE,MAAM,CAAC;IAC7C,OAAOA,EAAAA,KACC,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG,EAC9B,SAAS,CAAC,CAACY,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQX,4BACL,QAAQ,CAACS,wBACT,QAAQ;IACX,MAAMV,CAAC,CAADA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;AAEN;AAOO,MAAMc,oBAAoB,CAC/BX,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,CAACI;YAEL,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAEf,OAAOZ,KAAKY;QACd;IACF;AAIK,MAAMC,iCAAiChB,EAAE,MAAM,CAAC;IACrD,QAAQC,4BACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,EAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMiB,4BAA4B,CACvCd,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACb,QAAQ;YACN,SAAS;QACX;QACAb;IACF;AAIK,MAAMe,0BAA0BlB,EAAE,MAAM,CAAC;IAC9C,YAAYA,CAAC,CAADA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,4BACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMkB,qBAAqB,CAChChB,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaqB;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAf;IACF;AAIK,MAAMiB,+BAA+BpB,EAAE,MAAM,CAAC;IACnD,MAAMC,4BAA4B,QAAQ,CAAC;IAC3C,IAAIA,4BAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMoB,0BAA0B,CACrClB,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAauB;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACAjB;IACF;AAGK,MAAMmB,6BAA6BtB,EAAE,MAAM,CAAC;IACjD,QAAQC,4BAA4B,QAAQ,CAC1C;IAEF,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMuB,wBAAwB,CACnCpB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAayB;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACAnB;IACF;AAGK,MAAMqB,yBAAyBxB,EAAE,MAAM,CAAC;IAC7C,OAAOC,4BACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,CAAC,CAADA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,EAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,4BACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,EAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,EAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASyB,0BACdV,KAAuB,EACvBW,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAGf;IAEvB,MAAMgB,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAIf,MAAM,QAAQ,EAAE;QACzB,MAAMkB,YAAYlB,MAAM,SAAS;QACjC,IAAI,CAACkB,WACH,MAAM,IAAIC,MAAM;QAElBF,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,WAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;YACR,GACEgB,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,SAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAImB,MACR;IAIJF,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEJ;IAE9C,MAAMQ,WAAWrB,MAAM,QAAQ,IAAI;IAEnC,IAAIsB,cAAc,AAAwB,YAAxB,OAAOtB,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIsB,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEN;QAAYC;QAAUI;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAC/BnC,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2B;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACArB;IACF;AAIK,MAAMoC,8BAA8BvC,EAAE,MAAM,CAAC;IAClD,QAAQC,4BACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMuC,yBAAyB,CACpCrC,OAEON,aAGL;QACA,MAAM;QACN,aAAaa;QACb,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACApC;IACF;AAIK,MAAMsC,8BAA8BzC,EAAE,MAAM,CAAC;IAClD,WAAWA,CAAC,CAADA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,EAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAM0C,yBAAyB,CACpCvC,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa4C;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACAtC;IACF;AAGK,MAAMwC,yBAAyB3C,EAAE,MAAM,CAAC;IAC7C,QAAQA,EAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAM4C,oBAAoB,IACxB/C,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8C;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAO5B;YACX,MAAMqB,WAAWrB,OAAO,UAAU;YAClC8B,SAAS,wBAAwB,CAAC,aAAa,EAAET,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIU,QAAQ,CAACC,UAAYC,WAAWD,SAASX;QACrD;IACF"}

package/dist/es/utils.mjs

@@ -159,7 +159,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.5.11";
+    return "1.5.14";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];

package/dist/lib/agent/task-cache.js

@@ -262,7 +262,9 @@ class TaskCache {
         else this.cacheAdapter = cacheAdapterOrPath;
         if (!this.cacheAdapter) this.cacheFilePath = utils_namespaceObject.ifInBrowser || utils_namespaceObject.ifInWorker ? void 0 : (0, external_node_path_namespaceObject.join)((0, common_namespaceObject.getMidsceneRunSubDir)('cache'), `${this.cacheId}${cacheFileExt}`);
         let cacheContent;
-        if (!this.writeOnlyMode) cacheContent = this.loadCacheFromFile();
+        if (!this.writeOnlyMode) {
+            if (!this.cacheAdapter) cacheContent = this.loadCacheFromFile();
+        }
         if (!cacheContent) cacheContent = {
             midsceneVersion: (0, external_utils_js_namespaceObject.getMidsceneVersion)(),
             cacheId: this.cacheId,

package/dist/lib/agent/task-cache.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/task-cache.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-cache.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import assert from 'node:assert';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { isDeepStrictEqual } from 'node:util';\nimport type { TUserPrompt } from '@/ai-model';\nimport type { ElementCacheFeature } from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport {\n  MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { generateHashId } from '@midscene/shared/utils';\nimport { replaceIllegalPathCharsAndSpace } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\nimport semver from 'semver';\nimport type { CacheAdapter } from './cache-adapter';\nimport { getMidsceneVersion } from './utils';\n\nconst DEFAULT_CACHE_MAX_FILENAME_LENGTH = 200;\n\nexport const debug = getDebug('cache');\n\nexport interface PlanningCache {\n  type: 'plan';\n  prompt: string;\n  yamlWorkflow: string;\n}\n\nexport interface LocateCache {\n  type: 'locate';\n  prompt: TUserPrompt;\n  cache?: ElementCacheFeature;\n  /** @deprecated kept for backward compatibility */\n  xpaths?: string[];\n}\n\nexport interface MatchCacheResult<T extends PlanningCache | LocateCache> {\n  cacheContent: T;\n  updateFn: (\n    cb: (cache: T) => void,\n    options?: { ttl?: number },\n  ) => Promise<void>;\n}\n\nexport type CacheFileContent = {\n  midsceneVersion: string;\n  cacheId: string;\n  caches: Array<PlanningCache | LocateCache>;\n};\n\nconst lowestSupportedMidsceneVersion = '0.16.10';\nexport const cacheFileExt = '.cache.yaml';\n\nexport class TaskCache {\n  cacheId: string;\n\n  cacheFilePath?: string;\n\n  cache: CacheFileContent;\n\n  isCacheResultUsed: boolean;\n\n  cacheOriginalLength: number;\n\n  readOnlyMode: boolean;\n\n  writeOnlyMode: boolean;\n\n  private matchedCacheIndices: Set<string> = new Set();\n\n  private cacheAdapter?: CacheAdapter;\n\n  constructor(\n    cacheId: string,\n    isCacheResultUsed: boolean,\n    cacheAdapterOrPath?: CacheAdapter | string,\n    options: { readOnly?: boolean; writeOnly?: boolean } = {},\n  ) {\n    assert(cacheId, 'cacheId is required');\n    let safeCacheId = replaceIllegalPathCharsAndSpace(cacheId);\n    const cacheMaxFilenameLength =\n      globalConfigManager.getEnvConfigValueAsNumber(\n        MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n      ) ?? DEFAULT_CACHE_MAX_FILENAME_LENGTH;\n    if (Buffer.byteLength(safeCacheId, 'utf8') > cacheMaxFilenameLength) {\n      const prefix = safeCacheId.slice(0, 32);\n      const hash = generateHashId(undefined, safeCacheId);\n      safeCacheId = `${prefix}-${hash}`;\n    }\n    this.cacheId = safeCacheId;\n\n    const readOnlyMode = Boolean(options?.readOnly);\n    const writeOnlyMode = Boolean(options?.writeOnly);\n\n    if (readOnlyMode && writeOnlyMode) {\n      throw new Error('TaskCache cannot be both read-only and write-only');\n    }\n\n    this.isCacheResultUsed = writeOnlyMode ? false : isCacheResultUsed;\n    this.readOnlyMode = readOnlyMode;\n    this.writeOnlyMode = writeOnlyMode;\n\n    if (cacheAdapterOrPath) {\n      if (typeof cacheAdapterOrPath === 'string') {\n        this.cacheFilePath =\n          ifInBrowser || ifInWorker ? undefined : cacheAdapterOrPath;\n      } else {\n        this.cacheAdapter = cacheAdapterOrPath;\n      }\n    }\n\n    if (!this.cacheAdapter) {\n      this.cacheFilePath =\n        ifInBrowser || ifInWorker\n          ? undefined\n          : join(\n              getMidsceneRunSubDir('cache'),\n              `${this.cacheId}${cacheFileExt}`,\n            );\n    }\n\n    let cacheContent;\n    if (!this.writeOnlyMode) {\n      // 同步加载缓存，对于文件系统缓存是同步的，对于适配器缓存会返回undefined\n      // 实际的异步加载会在需要时进行\n      cacheContent = this.loadCacheFromFile();\n    }\n    if (!cacheContent) {\n      cacheContent = {\n        midsceneVersion: getMidsceneVersion(),\n        cacheId: this.cacheId,\n        caches: [],\n      };\n    }\n    this.cache = cacheContent;\n    this.cacheOriginalLength = this.isCacheResultUsed\n      ? this.cache.caches.length\n      : 0;\n  }\n\n  async matchCache(\n    prompt: TUserPrompt,\n    type: 'plan' | 'locate',\n  ): Promise<MatchCacheResult<PlanningCache | LocateCache> | undefined> {\n    if (!this.isCacheResultUsed) {\n      return undefined;\n    }\n\n    // 如果有缓存适配器且缓存未加载，尝试异步加载\n    if (this.cacheAdapter && this.cache.caches.length === 0) {\n      const loadedCache = await this.loadCacheFromAdapter();\n      if (loadedCache) {\n        this.cache = loadedCache;\n        this.cacheOriginalLength = loadedCache.caches.length;\n      }\n    }\n\n    const promptStr =\n      typeof prompt === 'string' ? prompt : JSON.stringify(prompt);\n    for (let i = 0; i < this.cacheOriginalLength; i++) {\n      const item = this.cache.caches[i];\n      const key = `${type}:${promptStr}:${i}`;\n      if (\n        item.type === type &&\n        isDeepStrictEqual(item.prompt, prompt) &&\n        !this.matchedCacheIndices.has(key)\n      ) {\n        if (item.type === 'locate') {\n          const locateItem = item as LocateCache;\n          if (!locateItem.cache && Array.isArray(locateItem.xpaths)) {\n            locateItem.cache = { xpaths: locateItem.xpaths };\n          }\n          if ('xpaths' in locateItem) {\n            locateItem.xpaths = undefined;\n          }\n        }\n        this.matchedCacheIndices.add(key);\n        debug(\n          'cache found and marked as used, type: %s, prompt: %s, index: %d',\n          type,\n          prompt,\n          i,\n        );\n        return {\n          cacheContent: item,\n          updateFn: async (\n            cb: (cache: PlanningCache | LocateCache) => void,\n            options?: { ttl?: number },\n          ) => {\n            debug(\n              'will call updateFn to update cache, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            cb(item);\n\n            if (this.readOnlyMode) {\n              debug('read-only mode, cache updated in memory but not flushed');\n              return;\n            }\n\n            debug(\n              'cache updated, will flush, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            await this.flushCache(options);\n          },\n        };\n      }\n    }\n    debug('no unused cache found, type: %s, prompt: %s', type, prompt);\n    return undefined;\n  }\n\n  async matchPlanCache(\n    prompt: string,\n  ): Promise<MatchCacheResult<PlanningCache> | undefined> {\n    return this.matchCache(prompt, 'plan') as Promise<\n      MatchCacheResult<PlanningCache> | undefined\n    >;\n  }\n\n  async matchLocateCache(\n    prompt: TUserPrompt,\n  ): Promise<MatchCacheResult<LocateCache> | undefined> {\n    return this.matchCache(prompt, 'locate') as Promise<\n      MatchCacheResult<LocateCache> | undefined\n    >;\n  }\n\n  async appendCache(cache: PlanningCache | LocateCache) {\n    debug('will append cache', cache);\n    this.cache.caches.push(cache);\n\n    if (this.readOnlyMode) {\n      debug('read-only mode, cache appended to memory but not flushed');\n      return;\n    }\n\n    await this.flushCache();\n  }\n\n  private async loadCache(): Promise<CacheFileContent | undefined> {\n    if (this.cacheAdapter) {\n      return this.loadCacheFromAdapter();\n    }\n    return this.loadCacheFromFile();\n  }\n\n  private async loadCacheFromAdapter(): Promise<CacheFileContent | undefined> {\n    try {\n      const content = await this.cacheAdapter!.get(this.cacheId);\n      if (!content) {\n        debug('no cache found from adapter, cacheId: %s', this.cacheId);\n        return undefined;\n      }\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from adapter');\n        return undefined;\n      }\n\n      if (\n        semver.lt(content.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !content.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache, and we cannot match any info from it.\\ncacheId: ${this.cacheId}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from adapter, cacheId: %s, cache version: %s, record length: %s',\n        this.cacheId,\n        content.midsceneVersion,\n        content.caches.length,\n      );\n      content.midsceneVersion = getMidsceneVersion();\n      return content;\n    } catch (err) {\n      debug(\n        'load cache from adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  loadCacheFromFile(): CacheFileContent | undefined {\n    const cacheFile = this.cacheFilePath;\n    assert(cacheFile, 'cache file path is required');\n\n    if (!existsSync(cacheFile)) {\n      debug('no cache file found, path: %s', cacheFile);\n      return undefined;\n    }\n\n    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, '.json');\n    if (existsSync(jsonTypeCacheFile) && this.isCacheResultUsed) {\n      console.warn(\n        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`,\n      );\n      return undefined;\n    }\n\n    try {\n      const data = readFileSync(cacheFile, 'utf8');\n      const jsonData = yaml.load(data) as CacheFileContent;\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from file');\n        return undefined;\n      }\n\n      if (\n        semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !jsonData.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.\\nPlease delete the existing cache and rebuild it. Sorry for the inconvenience.\\ncache file: ${cacheFile}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from file, path: %s, cache version: %s, record length: %s',\n        cacheFile,\n        jsonData.midsceneVersion,\n        jsonData.caches.length,\n      );\n      jsonData.midsceneVersion = getMidsceneVersion();\n      return jsonData;\n    } catch (err) {\n      debug(\n        'cache file exists but load failed, path: %s, error: %s',\n        cacheFile,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  private async flushCache(options?: { cleanUnused?: boolean; ttl?: number }) {\n    if (this.cacheAdapter) {\n      return this.flushCacheToAdapter(options);\n    }\n    return this.flushCacheToFile(options);\n  }\n\n  private async flushCacheToAdapter(options?: {\n    cleanUnused?: boolean;\n    ttl?: number;\n  }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to adapter');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      await this.cacheAdapter!.set(\n        this.cacheId,\n        cacheToWrite,\n        undefined,\n        options?.ttl,\n      );\n      debug('cache flushed to adapter, cacheId: %s', this.cacheId);\n    } catch (err) {\n      debug(\n        'write cache to adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n    }\n  }\n\n  flushCacheToFile(options?: { cleanUnused?: boolean }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to file');\n      return;\n    }\n\n    if (!this.cacheFilePath) {\n      debug('no cache file path, will not write cache to file');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      const dir = dirname(this.cacheFilePath);\n      if (!existsSync(dir)) {\n        mkdirSync(dir, { recursive: true });\n        debug('created cache directory: %s', dir);\n      }\n\n      const yamlData = yaml.dump(cacheToWrite, { lineWidth: -1 });\n      writeFileSync(this.cacheFilePath, yamlData);\n      debug('cache flushed to file: %s', this.cacheFilePath);\n    } catch (err) {\n      debug(\n        'write cache to file failed, path: %s, error: %s',\n        this.cacheFilePath,\n        err,\n      );\n    }\n  }\n\n  private prepareCacheToWrite(options?: {\n    cleanUnused?: boolean;\n  }): CacheFileContent {\n    let caches = [...this.cache.caches];\n\n    if (options?.cleanUnused) {\n      if (this.isCacheResultUsed) {\n        const usedIndices = new Set<number>();\n        for (const key of this.matchedCacheIndices) {\n          const parts = key.split(':');\n          const index = Number.parseInt(parts[parts.length - 1], 10);\n          if (!Number.isNaN(index)) {\n            usedIndices.add(index);\n          }\n        }\n\n        caches = caches.filter((_, index) => {\n          const isUsed = usedIndices.has(index);\n          const isNew = index >= this.cacheOriginalLength;\n          return isUsed || isNew;\n        });\n\n        const removedCount = this.cache.caches.length - caches.length;\n        if (removedCount > 0) {\n          debug('cleaned %d unused cache record(s)', removedCount);\n        } else {\n          debug('no unused cache to clean');\n        }\n      } else {\n        debug('skip cleaning: cache is not used for reading');\n      }\n    }\n\n    const sortedCaches = [...caches].sort((a, b) => {\n      if (a.type === 'plan' && b.type === 'locate') return -1;\n      if (a.type === 'locate' && b.type === 'plan') return 1;\n      return 0;\n    });\n\n    return {\n      ...this.cache,\n      caches: sortedCaches,\n    };\n  }\n\n  async updateOrAppendCacheRecord(\n    newRecord: PlanningCache | LocateCache,\n    cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>,\n  ) {\n    if (cachedRecord) {\n      if (newRecord.type === 'plan') {\n        await cachedRecord.updateFn((cache) => {\n          (cache as PlanningCache).yamlWorkflow = newRecord.yamlWorkflow;\n        });\n      } else {\n        await cachedRecord.updateFn((cache) => {\n          const locateCache = cache as LocateCache;\n          locateCache.cache = newRecord.cache;\n          if ('xpaths' in locateCache) {\n            locateCache.xpaths = undefined;\n          }\n        });\n      }\n    } else {\n      await this.appendCache(newRecord);\n    }\n  }\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","DEFAULT_CACHE_MAX_FILENAME_LENGTH","debug","getDebug","lowestSupportedMidsceneVersion","cacheFileExt","TaskCache","prompt","type","loadedCache","promptStr","JSON","i","item","isDeepStrictEqual","locateItem","Array","undefined","cb","options","cache","content","version","getMidsceneVersion","semver","console","err","cacheFile","assert","existsSync","jsonTypeCacheFile","data","readFileSync","jsonData","yaml","cacheToWrite","dir","dirname","mkdirSync","yamlData","writeFileSync","caches","usedIndices","Set","parts","index","Number","_","isUsed","isNew","removedCount","sortedCaches","a","b","newRecord","cachedRecord","locateCache","cacheId","isCacheResultUsed","cacheAdapterOrPath","safeCacheId","replaceIllegalPathCharsAndSpace","cacheMaxFilenameLength","globalConfigManager","MIDSCENE_CACHE_MAX_FILENAME_LENGTH","Buffer","prefix","hash","generateHashId","readOnlyMode","Boolean","writeOnlyMode","Error","ifInBrowser","ifInWorker","join","getMidsceneRunSubDir","cacheContent"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACcA,MAAMI,oCAAoC;AAEnC,MAAMC,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AA8B9B,MAAMC,iCAAiC;AAChC,MAAMC,eAAe;AAErB,MAAMC;IAuFX,MAAM,WACJC,MAAmB,EACnBC,IAAuB,EAC6C;QACpE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EACzB;QAIF,IAAI,IAAI,CAAC,YAAY,IAAI,AAA6B,MAA7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAQ;YACvD,MAAMC,cAAc,MAAM,IAAI,CAAC,oBAAoB;YACnD,IAAIA,aAAa;gBACf,IAAI,CAAC,KAAK,GAAGA;gBACb,IAAI,CAAC,mBAAmB,GAAGA,YAAY,MAAM,CAAC,MAAM;YACtD;QACF;QAEA,MAAMC,YACJ,AAAkB,YAAlB,OAAOH,SAAsBA,SAASI,KAAK,SAAS,CAACJ;QACvD,IAAK,IAAIK,IAAI,GAAGA,IAAI,IAAI,CAAC,mBAAmB,EAAEA,IAAK;YACjD,MAAMC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAACD,EAAE;YACjC,MAAMhB,MAAM,GAAGY,KAAK,CAAC,EAAEE,UAAU,CAAC,EAAEE,GAAG;YACvC,IACEC,KAAK,IAAI,KAAKL,QACdM,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EAAkBD,KAAK,MAAM,EAAEN,WAC/B,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACX,MAC9B;gBACA,IAAIiB,AAAc,aAAdA,KAAK,IAAI,EAAe;oBAC1B,MAAME,aAAaF;oBACnB,IAAI,CAACE,WAAW,KAAK,IAAIC,MAAM,OAAO,CAACD,WAAW,MAAM,GACtDA,WAAW,KAAK,GAAG;wBAAE,QAAQA,WAAW,MAAM;oBAAC;oBAEjD,IAAI,YAAYA,YACdA,WAAW,MAAM,GAAGE;gBAExB;gBACA,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACrB;gBAC7BM,MACE,mEACAM,MACAD,QACAK;gBAEF,OAAO;oBACL,cAAcC;oBACd,UAAU,OACRK,IACAC;wBAEAjB,MACE,uEACAM,MACAD,QACAK;wBAEFM,GAAGL;wBAEH,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBX,MAAM;wBAIRA,MACE,8DACAM,MACAD,QACAK;wBAEF,MAAM,IAAI,CAAC,UAAU,CAACO;oBACxB;gBACF;YACF;QACF;QACAjB,MAAM,+CAA+CM,MAAMD;IAE7D;IAEA,MAAM,eACJA,MAAc,EACwC;QACtD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,iBACJA,MAAmB,EACiC;QACpD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,YAAYa,KAAkC,EAAE;QACpDlB,MAAM,qBAAqBkB;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAACA;QAEvB,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBlB,MAAM;QAIR,MAAM,IAAI,CAAC,UAAU;IACvB;IAEA,MAAc,YAAmD;QAC/D,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,oBAAoB;QAElC,OAAO,IAAI,CAAC,iBAAiB;IAC/B;IAEA,MAAc,uBAA8D;QAC1E,IAAI;YACF,MAAMmB,UAAU,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAAC,IAAI,CAAC,OAAO;YACzD,IAAI,CAACA,SAAS,YACZnB,MAAM,4CAA4C,IAAI,CAAC,OAAO;YAIhE,MAAMoB,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;YAIR,IACEsB,0BAAAA,EAAS,CAACH,QAAQ,eAAe,EAAEjB,mCACnC,CAACiB,QAAQ,eAAe,CAAC,QAAQ,CAAC,SAClC,YACAI,QAAQ,IAAI,CACV,CAAC,gGAAgG,EAAE,IAAI,CAAC,OAAO,EAAE;YAKrHvB,MACE,gFACA,IAAI,CAAC,OAAO,EACZmB,QAAQ,eAAe,EACvBA,QAAQ,MAAM,CAAC,MAAM;YAEvBA,QAAQ,eAAe,GAAGE,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAC1B,OAAOF;QACT,EAAE,OAAOK,KAAK;YACZxB,MACE,0DACA,IAAI,CAAC,OAAO,EACZwB;YAEF;QACF;IACF;IAEA,oBAAkD;QAChD,MAAMC,YAAY,IAAI,CAAC,aAAa;QACpCC,+BAAOD,WAAW;QAElB,IAAI,CAACE,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YAAY,YAC1BzB,MAAM,iCAAiCyB;QAIzC,MAAMG,oBAAoBH,UAAU,OAAO,CAACtB,cAAc;QAC1D,IAAIwB,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWC,sBAAsB,IAAI,CAAC,iBAAiB,EAAE,YAC3DL,QAAQ,IAAI,CACV,CAAC,2LAA2L,EAAEK,kBAAkB,CAAC,CAAC;QAKtN,IAAI;YACF,MAAMC,OAAOC,AAAAA,IAAAA,iCAAAA,YAAAA,AAAAA,EAAaL,WAAW;YACrC,MAAMM,WAAWC,2BAAAA,IAAS,CAACH;YAE3B,MAAMT,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;YAIR,IACEsB,0BAAAA,EAAS,CAACS,SAAS,eAAe,EAAE7B,mCACpC,CAAC6B,SAAS,eAAe,CAAC,QAAQ,CAAC,SACnC,YACAR,QAAQ,IAAI,CACV,CAAC,wSAAwS,EAAEE,WAAW;YAK1TzB,MACE,0EACAyB,WACAM,SAAS,eAAe,EACxBA,SAAS,MAAM,CAAC,MAAM;YAExBA,SAAS,eAAe,GAAGV,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAC3B,OAAOU;QACT,EAAE,OAAOP,KAAK;YACZxB,MACE,0DACAyB,WACAD;YAEF;QACF;IACF;IAEA,MAAc,WAAWP,OAAiD,EAAE;QAC1E,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,mBAAmB,CAACA;QAElC,OAAO,IAAI,CAAC,gBAAgB,CAACA;IAC/B;IAEA,MAAc,oBAAoBA,OAGjC,EAAE;QACD,MAAMG,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;QAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;QAIR,MAAMiC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAC1B,IAAI,CAAC,OAAO,EACZgB,cACAlB,QACAE,SAAS;YAEXjB,MAAM,yCAAyC,IAAI,CAAC,OAAO;QAC7D,EAAE,OAAOwB,KAAK;YACZxB,MACE,yDACA,IAAI,CAAC,OAAO,EACZwB;QAEJ;IACF;IAEA,iBAAiBP,OAAmC,EAAE;QACpD,MAAMG,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;QAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;QAIR,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,YACvBA,MAAM;QAIR,MAAMiC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAMiB,MAAMC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQ,IAAI,CAAC,aAAa;YACtC,IAAI,CAACR,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWO,MAAM;gBACpBE,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUF,KAAK;oBAAE,WAAW;gBAAK;gBACjClC,MAAM,+BAA+BkC;YACvC;YAEA,MAAMG,WAAWL,2BAAAA,IAAS,CAACC,cAAc;gBAAE,WAAW;YAAG;YACzDK,IAAAA,iCAAAA,aAAAA,AAAAA,EAAc,IAAI,CAAC,aAAa,EAAED;YAClCrC,MAAM,6BAA6B,IAAI,CAAC,aAAa;QACvD,EAAE,OAAOwB,KAAK;YACZxB,MACE,mDACA,IAAI,CAAC,aAAa,EAClBwB;QAEJ;IACF;IAEQ,oBAAoBP,OAE3B,EAAoB;QACnB,IAAIsB,SAAS;eAAI,IAAI,CAAC,KAAK,CAAC,MAAM;SAAC;QAEnC,IAAItB,SAAS,aACX,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,MAAMuB,cAAc,IAAIC;YACxB,KAAK,MAAM/C,OAAO,IAAI,CAAC,mBAAmB,CAAE;gBAC1C,MAAMgD,QAAQhD,IAAI,KAAK,CAAC;gBACxB,MAAMiD,QAAQC,OAAO,QAAQ,CAACF,KAAK,CAACA,MAAM,MAAM,GAAG,EAAE,EAAE;gBACvD,IAAI,CAACE,OAAO,KAAK,CAACD,QAChBH,YAAY,GAAG,CAACG;YAEpB;YAEAJ,SAASA,OAAO,MAAM,CAAC,CAACM,GAAGF;gBACzB,MAAMG,SAASN,YAAY,GAAG,CAACG;gBAC/B,MAAMI,QAAQJ,SAAS,IAAI,CAAC,mBAAmB;gBAC/C,OAAOG,UAAUC;YACnB;YAEA,MAAMC,eAAe,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAGT,OAAO,MAAM;YACzDS,eAAe,IACjBhD,MAAM,qCAAqCgD,gBAE3ChD,MAAM;QAEV,OACEA,MAAM;QAIV,MAAMiD,eAAe;eAAIV;SAAO,CAAC,IAAI,CAAC,CAACW,GAAGC;YACxC,IAAID,AAAW,WAAXA,EAAE,IAAI,IAAeC,AAAW,aAAXA,EAAE,IAAI,EAAe,OAAO;YACrD,IAAID,AAAW,aAAXA,EAAE,IAAI,IAAiBC,AAAW,WAAXA,EAAE,IAAI,EAAa,OAAO;YACrD,OAAO;QACT;QAEA,OAAO;YACL,GAAG,IAAI,CAAC,KAAK;YACb,QAAQF;QACV;IACF;IAEA,MAAM,0BACJG,SAAsC,EACtCC,YAA4D,EAC5D;QACA,IAAIA,cACF,IAAID,AAAmB,WAAnBA,UAAU,IAAI,EAChB,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC1BA,MAAwB,YAAY,GAAGkC,UAAU,YAAY;QAChE;aAEA,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC3B,MAAMoC,cAAcpC;YACpBoC,YAAY,KAAK,GAAGF,UAAU,KAAK;YACnC,IAAI,YAAYE,aACdA,YAAY,MAAM,GAAGvC;QAEzB;aAGF,MAAM,IAAI,CAAC,WAAW,CAACqC;IAE3B;IA5ZA,YACEG,OAAe,EACfC,iBAA0B,EAC1BC,kBAA0C,EAC1CxC,UAAuD,CAAC,CAAC,CACzD;QAvBF;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA,uBAAQ,uBAAmC,IAAIwB;QAE/C,uBAAQ,gBAAR;QAQEf,+BAAO6B,SAAS;QAChB,IAAIG,cAAcC,AAAAA,IAAAA,sBAAAA,+BAAAA,AAAAA,EAAgCJ;QAClD,MAAMK,yBACJC,oBAAAA,mBAAAA,CAAAA,yBAA6C,CAC3CC,oBAAAA,kCAAkCA,KAC/B/D;QACP,IAAIgE,OAAO,UAAU,CAACL,aAAa,UAAUE,wBAAwB;YACnE,MAAMI,SAASN,YAAY,KAAK,CAAC,GAAG;YACpC,MAAMO,OAAOC,AAAAA,IAAAA,sBAAAA,cAAAA,AAAAA,EAAenD,QAAW2C;YACvCA,cAAc,GAAGM,OAAO,CAAC,EAAEC,MAAM;QACnC;QACA,IAAI,CAAC,OAAO,GAAGP;QAEf,MAAMS,eAAeC,QAAQnD,SAAS;QACtC,MAAMoD,gBAAgBD,QAAQnD,SAAS;QAEvC,IAAIkD,gBAAgBE,eAClB,MAAM,IAAIC,MAAM;QAGlB,IAAI,CAAC,iBAAiB,GAAGD,gBAAgB,QAAQb;QACjD,IAAI,CAAC,YAAY,GAAGW;QACpB,IAAI,CAAC,aAAa,GAAGE;QAErB,IAAIZ,oBACF,IAAI,AAA8B,YAA9B,OAAOA,oBACT,IAAI,CAAC,aAAa,GAChBc,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,GAAGzD,SAAY0C;aAE1C,IAAI,CAAC,YAAY,GAAGA;QAIxB,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB,IAAI,CAAC,aAAa,GAChBc,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,GACrBzD,SACA0D,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACEC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,UACrB,GAAG,IAAI,CAAC,OAAO,GAAGvE,cAAc;QAI1C,IAAIwE;QACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAGrBA,eAAe,IAAI,CAAC,iBAAiB;QAEvC,IAAI,CAACA,cACHA,eAAe;YACb,iBAAiBtD,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YACjB,SAAS,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE;QACZ;QAEF,IAAI,CAAC,KAAK,GAAGsD;QACb,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,GAC7C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GACxB;IACN;AA2VF"}
+{"version":3,"file":"agent/task-cache.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/task-cache.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import assert from 'node:assert';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { isDeepStrictEqual } from 'node:util';\nimport type { TUserPrompt } from '@/ai-model';\nimport type { ElementCacheFeature } from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport {\n  MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { generateHashId } from '@midscene/shared/utils';\nimport { replaceIllegalPathCharsAndSpace } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\nimport semver from 'semver';\nimport type { CacheAdapter } from './cache-adapter';\nimport { getMidsceneVersion } from './utils';\n\nconst DEFAULT_CACHE_MAX_FILENAME_LENGTH = 200;\n\nexport const debug = getDebug('cache');\n\nexport interface PlanningCache {\n  type: 'plan';\n  prompt: string;\n  yamlWorkflow: string;\n}\n\nexport interface LocateCache {\n  type: 'locate';\n  prompt: TUserPrompt;\n  cache?: ElementCacheFeature;\n  /** @deprecated kept for backward compatibility */\n  xpaths?: string[];\n}\n\nexport interface MatchCacheResult<T extends PlanningCache | LocateCache> {\n  cacheContent: T;\n  updateFn: (\n    cb: (cache: T) => void,\n    options?: { ttl?: number },\n  ) => Promise<void>;\n}\n\nexport type CacheFileContent = {\n  midsceneVersion: string;\n  cacheId: string;\n  caches: Array<PlanningCache | LocateCache>;\n};\n\nconst lowestSupportedMidsceneVersion = '0.16.10';\nexport const cacheFileExt = '.cache.yaml';\n\nexport class TaskCache {\n  cacheId: string;\n\n  cacheFilePath?: string;\n\n  cache: CacheFileContent;\n\n  isCacheResultUsed: boolean;\n\n  cacheOriginalLength: number;\n\n  readOnlyMode: boolean;\n\n  writeOnlyMode: boolean;\n\n  private matchedCacheIndices: Set<string> = new Set();\n\n  private cacheAdapter?: CacheAdapter;\n\n  constructor(\n    cacheId: string,\n    isCacheResultUsed: boolean,\n    cacheAdapterOrPath?: CacheAdapter | string,\n    options: { readOnly?: boolean; writeOnly?: boolean } = {},\n  ) {\n    assert(cacheId, 'cacheId is required');\n    let safeCacheId = replaceIllegalPathCharsAndSpace(cacheId);\n    const cacheMaxFilenameLength =\n      globalConfigManager.getEnvConfigValueAsNumber(\n        MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n      ) ?? DEFAULT_CACHE_MAX_FILENAME_LENGTH;\n    if (Buffer.byteLength(safeCacheId, 'utf8') > cacheMaxFilenameLength) {\n      const prefix = safeCacheId.slice(0, 32);\n      const hash = generateHashId(undefined, safeCacheId);\n      safeCacheId = `${prefix}-${hash}`;\n    }\n    this.cacheId = safeCacheId;\n\n    const readOnlyMode = Boolean(options?.readOnly);\n    const writeOnlyMode = Boolean(options?.writeOnly);\n\n    if (readOnlyMode && writeOnlyMode) {\n      throw new Error('TaskCache cannot be both read-only and write-only');\n    }\n\n    this.isCacheResultUsed = writeOnlyMode ? false : isCacheResultUsed;\n    this.readOnlyMode = readOnlyMode;\n    this.writeOnlyMode = writeOnlyMode;\n\n    if (cacheAdapterOrPath) {\n      if (typeof cacheAdapterOrPath === 'string') {\n        this.cacheFilePath =\n          ifInBrowser || ifInWorker ? undefined : cacheAdapterOrPath;\n      } else {\n        this.cacheAdapter = cacheAdapterOrPath;\n      }\n    }\n\n    if (!this.cacheAdapter) {\n      this.cacheFilePath =\n        ifInBrowser || ifInWorker\n          ? undefined\n          : join(\n              getMidsceneRunSubDir('cache'),\n              `${this.cacheId}${cacheFileExt}`,\n            );\n    }\n\n    let cacheContent;\n    if (!this.writeOnlyMode) {\n      // 同步加载缓存，对于文件系统缓存是同步的，对于适配器缓存会返回undefined\n      // 实际的异步加载会在需要时进行\n      // 如果有 cacheAdapter，就不调用 loadCacheFromFile()，因为适配器缓存是异步加载的\n      if (!this.cacheAdapter) {\n        cacheContent = this.loadCacheFromFile();\n      }\n    }\n    if (!cacheContent) {\n      cacheContent = {\n        midsceneVersion: getMidsceneVersion(),\n        cacheId: this.cacheId,\n        caches: [],\n      };\n    }\n    this.cache = cacheContent;\n    this.cacheOriginalLength = this.isCacheResultUsed\n      ? this.cache.caches.length\n      : 0;\n  }\n\n  async matchCache(\n    prompt: TUserPrompt,\n    type: 'plan' | 'locate',\n  ): Promise<MatchCacheResult<PlanningCache | LocateCache> | undefined> {\n    if (!this.isCacheResultUsed) {\n      return undefined;\n    }\n\n    // 如果有缓存适配器且缓存未加载，尝试异步加载\n    if (this.cacheAdapter && this.cache.caches.length === 0) {\n      const loadedCache = await this.loadCacheFromAdapter();\n      if (loadedCache) {\n        this.cache = loadedCache;\n        this.cacheOriginalLength = loadedCache.caches.length;\n      }\n    }\n\n    const promptStr =\n      typeof prompt === 'string' ? prompt : JSON.stringify(prompt);\n    for (let i = 0; i < this.cacheOriginalLength; i++) {\n      const item = this.cache.caches[i];\n      const key = `${type}:${promptStr}:${i}`;\n      if (\n        item.type === type &&\n        isDeepStrictEqual(item.prompt, prompt) &&\n        !this.matchedCacheIndices.has(key)\n      ) {\n        if (item.type === 'locate') {\n          const locateItem = item as LocateCache;\n          if (!locateItem.cache && Array.isArray(locateItem.xpaths)) {\n            locateItem.cache = { xpaths: locateItem.xpaths };\n          }\n          if ('xpaths' in locateItem) {\n            locateItem.xpaths = undefined;\n          }\n        }\n        this.matchedCacheIndices.add(key);\n        debug(\n          'cache found and marked as used, type: %s, prompt: %s, index: %d',\n          type,\n          prompt,\n          i,\n        );\n        return {\n          cacheContent: item,\n          updateFn: async (\n            cb: (cache: PlanningCache | LocateCache) => void,\n            options?: { ttl?: number },\n          ) => {\n            debug(\n              'will call updateFn to update cache, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            cb(item);\n\n            if (this.readOnlyMode) {\n              debug('read-only mode, cache updated in memory but not flushed');\n              return;\n            }\n\n            debug(\n              'cache updated, will flush, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            await this.flushCache(options);\n          },\n        };\n      }\n    }\n    debug('no unused cache found, type: %s, prompt: %s', type, prompt);\n    return undefined;\n  }\n\n  async matchPlanCache(\n    prompt: string,\n  ): Promise<MatchCacheResult<PlanningCache> | undefined> {\n    return this.matchCache(prompt, 'plan') as Promise<\n      MatchCacheResult<PlanningCache> | undefined\n    >;\n  }\n\n  async matchLocateCache(\n    prompt: TUserPrompt,\n  ): Promise<MatchCacheResult<LocateCache> | undefined> {\n    return this.matchCache(prompt, 'locate') as Promise<\n      MatchCacheResult<LocateCache> | undefined\n    >;\n  }\n\n  async appendCache(cache: PlanningCache | LocateCache) {\n    debug('will append cache', cache);\n    this.cache.caches.push(cache);\n\n    if (this.readOnlyMode) {\n      debug('read-only mode, cache appended to memory but not flushed');\n      return;\n    }\n\n    await this.flushCache();\n  }\n\n  private async loadCache(): Promise<CacheFileContent | undefined> {\n    if (this.cacheAdapter) {\n      return this.loadCacheFromAdapter();\n    }\n    return this.loadCacheFromFile();\n  }\n\n  private async loadCacheFromAdapter(): Promise<CacheFileContent | undefined> {\n    try {\n      const content = await this.cacheAdapter!.get(this.cacheId);\n      if (!content) {\n        debug('no cache found from adapter, cacheId: %s', this.cacheId);\n        return undefined;\n      }\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from adapter');\n        return undefined;\n      }\n\n      if (\n        semver.lt(content.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !content.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache, and we cannot match any info from it.\\ncacheId: ${this.cacheId}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from adapter, cacheId: %s, cache version: %s, record length: %s',\n        this.cacheId,\n        content.midsceneVersion,\n        content.caches.length,\n      );\n      content.midsceneVersion = getMidsceneVersion();\n      return content;\n    } catch (err) {\n      debug(\n        'load cache from adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  loadCacheFromFile(): CacheFileContent | undefined {\n    const cacheFile = this.cacheFilePath;\n    assert(cacheFile, 'cache file path is required');\n\n    if (!existsSync(cacheFile)) {\n      debug('no cache file found, path: %s', cacheFile);\n      return undefined;\n    }\n\n    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, '.json');\n    if (existsSync(jsonTypeCacheFile) && this.isCacheResultUsed) {\n      console.warn(\n        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`,\n      );\n      return undefined;\n    }\n\n    try {\n      const data = readFileSync(cacheFile, 'utf8');\n      const jsonData = yaml.load(data) as CacheFileContent;\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from file');\n        return undefined;\n      }\n\n      if (\n        semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !jsonData.midsceneVersion.includes('beta')\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.\\nPlease delete the existing cache and rebuild it. Sorry for the inconvenience.\\ncache file: ${cacheFile}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from file, path: %s, cache version: %s, record length: %s',\n        cacheFile,\n        jsonData.midsceneVersion,\n        jsonData.caches.length,\n      );\n      jsonData.midsceneVersion = getMidsceneVersion();\n      return jsonData;\n    } catch (err) {\n      debug(\n        'cache file exists but load failed, path: %s, error: %s',\n        cacheFile,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  private async flushCache(options?: { cleanUnused?: boolean; ttl?: number }) {\n    if (this.cacheAdapter) {\n      return this.flushCacheToAdapter(options);\n    }\n    return this.flushCacheToFile(options);\n  }\n\n  private async flushCacheToAdapter(options?: {\n    cleanUnused?: boolean;\n    ttl?: number;\n  }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to adapter');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      await this.cacheAdapter!.set(\n        this.cacheId,\n        cacheToWrite,\n        undefined,\n        options?.ttl,\n      );\n      debug('cache flushed to adapter, cacheId: %s', this.cacheId);\n    } catch (err) {\n      debug(\n        'write cache to adapter failed, cacheId: %s, error: %s',\n        this.cacheId,\n        err,\n      );\n    }\n  }\n\n  flushCacheToFile(options?: { cleanUnused?: boolean }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to file');\n      return;\n    }\n\n    if (!this.cacheFilePath) {\n      debug('no cache file path, will not write cache to file');\n      return;\n    }\n\n    const cacheToWrite = this.prepareCacheToWrite(options);\n\n    try {\n      const dir = dirname(this.cacheFilePath);\n      if (!existsSync(dir)) {\n        mkdirSync(dir, { recursive: true });\n        debug('created cache directory: %s', dir);\n      }\n\n      const yamlData = yaml.dump(cacheToWrite, { lineWidth: -1 });\n      writeFileSync(this.cacheFilePath, yamlData);\n      debug('cache flushed to file: %s', this.cacheFilePath);\n    } catch (err) {\n      debug(\n        'write cache to file failed, path: %s, error: %s',\n        this.cacheFilePath,\n        err,\n      );\n    }\n  }\n\n  private prepareCacheToWrite(options?: {\n    cleanUnused?: boolean;\n  }): CacheFileContent {\n    let caches = [...this.cache.caches];\n\n    if (options?.cleanUnused) {\n      if (this.isCacheResultUsed) {\n        const usedIndices = new Set<number>();\n        for (const key of this.matchedCacheIndices) {\n          const parts = key.split(':');\n          const index = Number.parseInt(parts[parts.length - 1], 10);\n          if (!Number.isNaN(index)) {\n            usedIndices.add(index);\n          }\n        }\n\n        caches = caches.filter((_, index) => {\n          const isUsed = usedIndices.has(index);\n          const isNew = index >= this.cacheOriginalLength;\n          return isUsed || isNew;\n        });\n\n        const removedCount = this.cache.caches.length - caches.length;\n        if (removedCount > 0) {\n          debug('cleaned %d unused cache record(s)', removedCount);\n        } else {\n          debug('no unused cache to clean');\n        }\n      } else {\n        debug('skip cleaning: cache is not used for reading');\n      }\n    }\n\n    const sortedCaches = [...caches].sort((a, b) => {\n      if (a.type === 'plan' && b.type === 'locate') return -1;\n      if (a.type === 'locate' && b.type === 'plan') return 1;\n      return 0;\n    });\n\n    return {\n      ...this.cache,\n      caches: sortedCaches,\n    };\n  }\n\n  async updateOrAppendCacheRecord(\n    newRecord: PlanningCache | LocateCache,\n    cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>,\n  ) {\n    if (cachedRecord) {\n      if (newRecord.type === 'plan') {\n        await cachedRecord.updateFn((cache) => {\n          (cache as PlanningCache).yamlWorkflow = newRecord.yamlWorkflow;\n        });\n      } else {\n        await cachedRecord.updateFn((cache) => {\n          const locateCache = cache as LocateCache;\n          locateCache.cache = newRecord.cache;\n          if ('xpaths' in locateCache) {\n            locateCache.xpaths = undefined;\n          }\n        });\n      }\n    } else {\n      await this.appendCache(newRecord);\n    }\n  }\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","DEFAULT_CACHE_MAX_FILENAME_LENGTH","debug","getDebug","lowestSupportedMidsceneVersion","cacheFileExt","TaskCache","prompt","type","loadedCache","promptStr","JSON","i","item","isDeepStrictEqual","locateItem","Array","undefined","cb","options","cache","content","version","getMidsceneVersion","semver","console","err","cacheFile","assert","existsSync","jsonTypeCacheFile","data","readFileSync","jsonData","yaml","cacheToWrite","dir","dirname","mkdirSync","yamlData","writeFileSync","caches","usedIndices","Set","parts","index","Number","_","isUsed","isNew","removedCount","sortedCaches","a","b","newRecord","cachedRecord","locateCache","cacheId","isCacheResultUsed","cacheAdapterOrPath","safeCacheId","replaceIllegalPathCharsAndSpace","cacheMaxFilenameLength","globalConfigManager","MIDSCENE_CACHE_MAX_FILENAME_LENGTH","Buffer","prefix","hash","generateHashId","readOnlyMode","Boolean","writeOnlyMode","Error","ifInBrowser","ifInWorker","join","getMidsceneRunSubDir","cacheContent"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACcA,MAAMI,oCAAoC;AAEnC,MAAMC,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AA8B9B,MAAMC,iCAAiC;AAChC,MAAMC,eAAe;AAErB,MAAMC;IA0FX,MAAM,WACJC,MAAmB,EACnBC,IAAuB,EAC6C;QACpE,IAAI,CAAC,IAAI,CAAC,iBAAiB,EACzB;QAIF,IAAI,IAAI,CAAC,YAAY,IAAI,AAA6B,MAA7B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,EAAQ;YACvD,MAAMC,cAAc,MAAM,IAAI,CAAC,oBAAoB;YACnD,IAAIA,aAAa;gBACf,IAAI,CAAC,KAAK,GAAGA;gBACb,IAAI,CAAC,mBAAmB,GAAGA,YAAY,MAAM,CAAC,MAAM;YACtD;QACF;QAEA,MAAMC,YACJ,AAAkB,YAAlB,OAAOH,SAAsBA,SAASI,KAAK,SAAS,CAACJ;QACvD,IAAK,IAAIK,IAAI,GAAGA,IAAI,IAAI,CAAC,mBAAmB,EAAEA,IAAK;YACjD,MAAMC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAACD,EAAE;YACjC,MAAMhB,MAAM,GAAGY,KAAK,CAAC,EAAEE,UAAU,CAAC,EAAEE,GAAG;YACvC,IACEC,KAAK,IAAI,KAAKL,QACdM,AAAAA,IAAAA,mCAAAA,iBAAAA,AAAAA,EAAkBD,KAAK,MAAM,EAAEN,WAC/B,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACX,MAC9B;gBACA,IAAIiB,AAAc,aAAdA,KAAK,IAAI,EAAe;oBAC1B,MAAME,aAAaF;oBACnB,IAAI,CAACE,WAAW,KAAK,IAAIC,MAAM,OAAO,CAACD,WAAW,MAAM,GACtDA,WAAW,KAAK,GAAG;wBAAE,QAAQA,WAAW,MAAM;oBAAC;oBAEjD,IAAI,YAAYA,YACdA,WAAW,MAAM,GAAGE;gBAExB;gBACA,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACrB;gBAC7BM,MACE,mEACAM,MACAD,QACAK;gBAEF,OAAO;oBACL,cAAcC;oBACd,UAAU,OACRK,IACAC;wBAEAjB,MACE,uEACAM,MACAD,QACAK;wBAEFM,GAAGL;wBAEH,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBX,MAAM;wBAIRA,MACE,8DACAM,MACAD,QACAK;wBAEF,MAAM,IAAI,CAAC,UAAU,CAACO;oBACxB;gBACF;YACF;QACF;QACAjB,MAAM,+CAA+CM,MAAMD;IAE7D;IAEA,MAAM,eACJA,MAAc,EACwC;QACtD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,iBACJA,MAAmB,EACiC;QACpD,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,MAAM,YAAYa,KAAkC,EAAE;QACpDlB,MAAM,qBAAqBkB;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAACA;QAEvB,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBlB,MAAM;QAIR,MAAM,IAAI,CAAC,UAAU;IACvB;IAEA,MAAc,YAAmD;QAC/D,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,oBAAoB;QAElC,OAAO,IAAI,CAAC,iBAAiB;IAC/B;IAEA,MAAc,uBAA8D;QAC1E,IAAI;YACF,MAAMmB,UAAU,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAAC,IAAI,CAAC,OAAO;YACzD,IAAI,CAACA,SAAS,YACZnB,MAAM,4CAA4C,IAAI,CAAC,OAAO;YAIhE,MAAMoB,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;YAIR,IACEsB,0BAAAA,EAAS,CAACH,QAAQ,eAAe,EAAEjB,mCACnC,CAACiB,QAAQ,eAAe,CAAC,QAAQ,CAAC,SAClC,YACAI,QAAQ,IAAI,CACV,CAAC,gGAAgG,EAAE,IAAI,CAAC,OAAO,EAAE;YAKrHvB,MACE,gFACA,IAAI,CAAC,OAAO,EACZmB,QAAQ,eAAe,EACvBA,QAAQ,MAAM,CAAC,MAAM;YAEvBA,QAAQ,eAAe,GAAGE,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAC1B,OAAOF;QACT,EAAE,OAAOK,KAAK;YACZxB,MACE,0DACA,IAAI,CAAC,OAAO,EACZwB;YAEF;QACF;IACF;IAEA,oBAAkD;QAChD,MAAMC,YAAY,IAAI,CAAC,aAAa;QACpCC,+BAAOD,WAAW;QAElB,IAAI,CAACE,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,YAAY,YAC1BzB,MAAM,iCAAiCyB;QAIzC,MAAMG,oBAAoBH,UAAU,OAAO,CAACtB,cAAc;QAC1D,IAAIwB,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWC,sBAAsB,IAAI,CAAC,iBAAiB,EAAE,YAC3DL,QAAQ,IAAI,CACV,CAAC,2LAA2L,EAAEK,kBAAkB,CAAC,CAAC;QAKtN,IAAI;YACF,MAAMC,OAAOC,AAAAA,IAAAA,iCAAAA,YAAAA,AAAAA,EAAaL,WAAW;YACrC,MAAMM,WAAWC,2BAAAA,IAAS,CAACH;YAE3B,MAAMT,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;YAIR,IACEsB,0BAAAA,EAAS,CAACS,SAAS,eAAe,EAAE7B,mCACpC,CAAC6B,SAAS,eAAe,CAAC,QAAQ,CAAC,SACnC,YACAR,QAAQ,IAAI,CACV,CAAC,wSAAwS,EAAEE,WAAW;YAK1TzB,MACE,0EACAyB,WACAM,SAAS,eAAe,EACxBA,SAAS,MAAM,CAAC,MAAM;YAExBA,SAAS,eAAe,GAAGV,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YAC3B,OAAOU;QACT,EAAE,OAAOP,KAAK;YACZxB,MACE,0DACAyB,WACAD;YAEF;QACF;IACF;IAEA,MAAc,WAAWP,OAAiD,EAAE;QAC1E,IAAI,IAAI,CAAC,YAAY,EACnB,OAAO,IAAI,CAAC,mBAAmB,CAACA;QAElC,OAAO,IAAI,CAAC,gBAAgB,CAACA;IAC/B;IAEA,MAAc,oBAAoBA,OAGjC,EAAE;QACD,MAAMG,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;QAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;QAIR,MAAMiC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAM,IAAI,CAAC,YAAY,CAAE,GAAG,CAC1B,IAAI,CAAC,OAAO,EACZgB,cACAlB,QACAE,SAAS;YAEXjB,MAAM,yCAAyC,IAAI,CAAC,OAAO;QAC7D,EAAE,OAAOwB,KAAK;YACZxB,MACE,yDACA,IAAI,CAAC,OAAO,EACZwB;QAEJ;IACF;IAEA,iBAAiBP,OAAmC,EAAE;QACpD,MAAMG,UAAUC,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;QAChB,IAAI,CAACD,SAAS,YACZpB,MAAM;QAIR,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,YACvBA,MAAM;QAIR,MAAMiC,eAAe,IAAI,CAAC,mBAAmB,CAAChB;QAE9C,IAAI;YACF,MAAMiB,MAAMC,AAAAA,IAAAA,mCAAAA,OAAAA,AAAAA,EAAQ,IAAI,CAAC,aAAa;YACtC,IAAI,CAACR,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWO,MAAM;gBACpBE,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUF,KAAK;oBAAE,WAAW;gBAAK;gBACjClC,MAAM,+BAA+BkC;YACvC;YAEA,MAAMG,WAAWL,2BAAAA,IAAS,CAACC,cAAc;gBAAE,WAAW;YAAG;YACzDK,IAAAA,iCAAAA,aAAAA,AAAAA,EAAc,IAAI,CAAC,aAAa,EAAED;YAClCrC,MAAM,6BAA6B,IAAI,CAAC,aAAa;QACvD,EAAE,OAAOwB,KAAK;YACZxB,MACE,mDACA,IAAI,CAAC,aAAa,EAClBwB;QAEJ;IACF;IAEQ,oBAAoBP,OAE3B,EAAoB;QACnB,IAAIsB,SAAS;eAAI,IAAI,CAAC,KAAK,CAAC,MAAM;SAAC;QAEnC,IAAItB,SAAS,aACX,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,MAAMuB,cAAc,IAAIC;YACxB,KAAK,MAAM/C,OAAO,IAAI,CAAC,mBAAmB,CAAE;gBAC1C,MAAMgD,QAAQhD,IAAI,KAAK,CAAC;gBACxB,MAAMiD,QAAQC,OAAO,QAAQ,CAACF,KAAK,CAACA,MAAM,MAAM,GAAG,EAAE,EAAE;gBACvD,IAAI,CAACE,OAAO,KAAK,CAACD,QAChBH,YAAY,GAAG,CAACG;YAEpB;YAEAJ,SAASA,OAAO,MAAM,CAAC,CAACM,GAAGF;gBACzB,MAAMG,SAASN,YAAY,GAAG,CAACG;gBAC/B,MAAMI,QAAQJ,SAAS,IAAI,CAAC,mBAAmB;gBAC/C,OAAOG,UAAUC;YACnB;YAEA,MAAMC,eAAe,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAGT,OAAO,MAAM;YACzDS,eAAe,IACjBhD,MAAM,qCAAqCgD,gBAE3ChD,MAAM;QAEV,OACEA,MAAM;QAIV,MAAMiD,eAAe;eAAIV;SAAO,CAAC,IAAI,CAAC,CAACW,GAAGC;YACxC,IAAID,AAAW,WAAXA,EAAE,IAAI,IAAeC,AAAW,aAAXA,EAAE,IAAI,EAAe,OAAO;YACrD,IAAID,AAAW,aAAXA,EAAE,IAAI,IAAiBC,AAAW,WAAXA,EAAE,IAAI,EAAa,OAAO;YACrD,OAAO;QACT;QAEA,OAAO;YACL,GAAG,IAAI,CAAC,KAAK;YACb,QAAQF;QACV;IACF;IAEA,MAAM,0BACJG,SAAsC,EACtCC,YAA4D,EAC5D;QACA,IAAIA,cACF,IAAID,AAAmB,WAAnBA,UAAU,IAAI,EAChB,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC1BA,MAAwB,YAAY,GAAGkC,UAAU,YAAY;QAChE;aAEA,MAAMC,aAAa,QAAQ,CAAC,CAACnC;YAC3B,MAAMoC,cAAcpC;YACpBoC,YAAY,KAAK,GAAGF,UAAU,KAAK;YACnC,IAAI,YAAYE,aACdA,YAAY,MAAM,GAAGvC;QAEzB;aAGF,MAAM,IAAI,CAAC,WAAW,CAACqC;IAE3B;IA/ZA,YACEG,OAAe,EACfC,iBAA0B,EAC1BC,kBAA0C,EAC1CxC,UAAuD,CAAC,CAAC,CACzD;QAvBF;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA;QAEA,uBAAQ,uBAAmC,IAAIwB;QAE/C,uBAAQ,gBAAR;QAQEf,+BAAO6B,SAAS;QAChB,IAAIG,cAAcC,AAAAA,IAAAA,sBAAAA,+BAAAA,AAAAA,EAAgCJ;QAClD,MAAMK,yBACJC,oBAAAA,mBAAAA,CAAAA,yBAA6C,CAC3CC,oBAAAA,kCAAkCA,KAC/B/D;QACP,IAAIgE,OAAO,UAAU,CAACL,aAAa,UAAUE,wBAAwB;YACnE,MAAMI,SAASN,YAAY,KAAK,CAAC,GAAG;YACpC,MAAMO,OAAOC,AAAAA,IAAAA,sBAAAA,cAAAA,AAAAA,EAAenD,QAAW2C;YACvCA,cAAc,GAAGM,OAAO,CAAC,EAAEC,MAAM;QACnC;QACA,IAAI,CAAC,OAAO,GAAGP;QAEf,MAAMS,eAAeC,QAAQnD,SAAS;QACtC,MAAMoD,gBAAgBD,QAAQnD,SAAS;QAEvC,IAAIkD,gBAAgBE,eAClB,MAAM,IAAIC,MAAM;QAGlB,IAAI,CAAC,iBAAiB,GAAGD,gBAAgB,QAAQb;QACjD,IAAI,CAAC,YAAY,GAAGW;QACpB,IAAI,CAAC,aAAa,GAAGE;QAErB,IAAIZ,oBACF,IAAI,AAA8B,YAA9B,OAAOA,oBACT,IAAI,CAAC,aAAa,GAChBc,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,GAAGzD,SAAY0C;aAE1C,IAAI,CAAC,YAAY,GAAGA;QAIxB,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB,IAAI,CAAC,aAAa,GAChBc,sBAAAA,WAAWA,IAAIC,sBAAAA,UAAUA,GACrBzD,SACA0D,AAAAA,IAAAA,mCAAAA,IAAAA,AAAAA,EACEC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,UACrB,GAAG,IAAI,CAAC,OAAO,GAAGvE,cAAc;QAI1C,IAAIwE;QACJ,IAAI,CAAC,IAAI,CAAC,aAAa,EAIrB;YAAA,IAAI,CAAC,IAAI,CAAC,YAAY,EACpBA,eAAe,IAAI,CAAC,iBAAiB;QACvC;QAEF,IAAI,CAACA,cACHA,eAAe;YACb,iBAAiBtD,AAAAA,IAAAA,kCAAAA,kBAAAA,AAAAA;YACjB,SAAS,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE;QACZ;QAEF,IAAI,CAAC,KAAK,GAAGsD;QACb,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,GAC7C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GACxB;IACN;AA2VF"}

package/dist/lib/agent/utils.js

@@ -201,7 +201,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.5.11";
+const getMidsceneVersion = ()=>"1.5.14";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/ai-model/prompt/llm-planning.js

@@ -380,7 +380,7 @@ ${shouldIncludeSubGoals ? `
 ${shouldIncludeSubGoals ? `
 ## Example
 
-Instruction: "fill the form with name 'John' and email 'test@midscene.ai', return the email"
+Instruction: "fill the form with name 'John' and email 'testmidscene.ai', return the email"
 
 **Turn 1:** Empty form → <thought>Plan: fill Name, Email, return email</thought><update-plan-content><sub-goal index="1" status="pending">Fill Name</sub-goal><sub-goal index="2" status="pending">Fill Email</sub-goal></update-plan-content><action-type>Tap</action-type><action-param-json>{"locate": ${locateNameField}}</action-param-json>
 
@@ -388,13 +388,13 @@ Instruction: "fill the form with name 'John' and email 'test@midscene.ai', retur
 
 **Turn 3:** Name='John' → <mark-sub-goal-done><sub-goal index="1" status="finished" /></mark-sub-goal-done><action-type>Tap</action-type><action-param-json>{"locate": ${locateEmailField}}</action-param-json>
 
-**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "test@midscene.ai"}</action-param-json>
+**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "testmidscene.ai"}</action-param-json>
 
-**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index="2" status="finished" /></mark-sub-goal-done><complete success="true">test@midscene.ai</complete>
+**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index="2" status="finished" /></mark-sub-goal-done><complete success="true">testmidscene.ai</complete>
 ` : `
 ## Example
 
-Instruction: "fill the form with name 'John' and email 'test@midscene.ai', return the email"
+Instruction: "fill the form with name 'John' and email 'testmidscene.ai', return the email"
 
 **Turn 1:** Empty form → <thought>Fill Name then Email, return email</thought><action-type>Tap</action-type><action-param-json>{"locate": ${locateNameField}}</action-param-json>
 
@@ -402,9 +402,9 @@ Instruction: "fill the form with name 'John' and email 'test@midscene.ai', retur
 
 **Turn 3:** Name='John' → <action-type>Tap</action-type><action-param-json>{"locate": ${locateEmailField}}</action-param-json>
 
-**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "test@midscene.ai"}</action-param-json>
+**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{"value": "testmidscene.ai"}</action-param-json>
 
-**Turn 5:** Form complete → <complete success="true">test@midscene.ai</complete>
+**Turn 5:** Form complete → <complete success="true">testmidscene.ai</complete>
 `}`;
 }
 exports.descriptionForAction = __webpack_exports__.descriptionForAction;

package/dist/lib/ai-model/prompt/llm-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\n/**\n * Find ZodDefault in the wrapper chain and return its default value\n */\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    // Continue unwrapping if it's a wrapper type\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\n/**\n * Inject bbox into locate fields of a sample object.\n * Walks the sample and for any locate field (identified by paramSchema),\n * adds a fake bbox array when includeBbox is true.\n */\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name using extracted helper\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          // Get description using extracted helper\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          // Check if field has a default value by searching the wrapper chain\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  // Render sample if provided, using the same XML tag format as the real output\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  // Validate parameters: if includeBbox is true, modelFamily must be defined\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  // Generate locate object examples based on includeBbox\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  // Locate examples for multi-turn conversation\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const thoughtTag = (content: string) =>\n    shouldIncludeThought ? `<thought>${content}</thought>\\n` : '';\n\n  // Sub-goals related content - only included when shouldIncludeSubGoals is true\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)'\n    : '## Step 1: Observe (related tags: <thought>)';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\"\n    : 'First, observe the current screenshot and previous logs to understand the current state.';\n\n  const explicitInstructionRule = `CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`;\n\n  const thoughtTagDescription = shouldIncludeSubGoals\n    ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`\n    : `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`;\n\n  const subGoalTags = shouldIncludeSubGoals\n    ? `\n\n* <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n* Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<thought>...</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`\n    : '';\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const memoryStepNumber = 2; // Only used when shouldIncludeSubGoals is true\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalTags}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)\n\nWhile observing the current screenshot, if you notice any information that might be needed in follow-up actions, record it here. The current screenshot will NOT be available in subsequent steps, so this memory is your only way to preserve essential information. Examples: extracted data, element states, content that needs to be referenced.\n\nDon't use this tag if no information needs to be preserved.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: ${shouldIncludeSubGoals ? 'Check if Goal is Accomplished' : 'Check if the Instruction is Fulfilled'} (related tags: <complete>)\n\n${shouldIncludeSubGoals ? 'Based on the current screenshot and the status of all sub-goals, determine' : 'Determine'} if the entire task is completed.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${shouldIncludeSubGoals ? 'goal accomplished' : 'instruction fulfilled'}\" means:**\n- The ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when selected. Do NOT proceed to checkout.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${shouldIncludeSubGoals ? 'the goal' : 'it'} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n${\n  !shouldIncludeSubGoals\n    ? `\n**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.\n`\n    : ''\n}\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${shouldIncludeSubGoals ? 'It means whether the expected goal is accomplished based on what you observe in the current screenshot. ' : ''}No matter what actions were executed or what errors occurred during execution, if the ${shouldIncludeSubGoals ? 'expected goal is accomplished' : 'instruction is fulfilled'}, set success=\"true\". If the ${shouldIncludeSubGoals ? 'expected goal is not accomplished and cannot be accomplished' : 'instruction is not fulfilled and cannot be fulfilled'}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output <action-type> or <action-param-json>. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${shouldIncludeSubGoals ? ' and the plan' : ''}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Supporting actions list\n\n${actionList}\n\n### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>\n\n### If there is some action to do ...\n\n- Use the <action-type> and <action-param-json> tags to output the action to be executed.\n- The <action-type> MUST be one of the supporting actions. 'complete' is NOT a valid action-type.\n- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If an action has a \"sample\" in its description, follow that structure.\nFor example:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output <action-type> or <action-param-json> if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n**Always include (REQUIRED):**\n<!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<!-- Step ${memoryStepNumber}: Memory data from current screenshot if needed -->\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} or failed (Step ${checkGoalStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${shouldIncludeSubGoals ? 'goal is NOT complete' : 'instruction is NOT fulfilled'} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n${\n  shouldIncludeSubGoals\n    ? `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'test@midscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Plan: fill Name, Email, return email</thought><update-plan-content><sub-goal index=\"1\" status=\"pending\">Fill Name</sub-goal><sub-goal index=\"2\" status=\"pending\">Fill Email</sub-goal></update-plan-content><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <mark-sub-goal-done><sub-goal index=\"1\" status=\"finished\" /></mark-sub-goal-done><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"test@midscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index=\"2\" status=\"finished\" /></mark-sub-goal-done><complete success=\"true\">test@midscene.ai</complete>\n`\n    : `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'test@midscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Fill Name then Email, return email</thought><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"test@midscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <complete success=\"true\">test@midscene.ai</complete>\n`\n}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","systemPromptToTaskPlanning","actionSpace","includeThought","includeSubGoals","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","locateNameField","locateEmailField","step1Title","step1Description","explicitInstructionRule","thoughtTagDescription","subGoalTags","memoryStepNumber","checkGoalStepNumber","actionStepNumber"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACKA,MAAMI,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,cAAc;IAEvG,OAAO;AACT;AAKA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAIzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAOA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAAC3B,KAAKS,MAAM,IAAIR,OAAO,OAAO,CAAC4B,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMqB,aACJ,AACE,cADF,OAAQrB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMsB,kBAAkBD,aAAa,GAAG9B,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMgC,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAexB,OAAOc;gBAGvC,MAAMW,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkB1B;gBAGtC,MAAM2B,eAAe5B,iBAAiBC;gBACtC,MAAM4B,aAAaD,AAAiBE,WAAjBF;gBAGnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3Cd,WAAW,IAAI,CAACa;YAClB;YAIF,IAAIb,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACiB;oBAClBlB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEkB,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAMX,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeN;YAChC,MAAMO,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBR;YAGtC,IAAIiB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBnB,OAAO,IAAI,CAACmB;QACd;IACF;IAGA,IAAItB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe4B,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BvB,OAAO,WAAW;QACnE,MAAMwB,iBAAiB/B,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM6B,YAAY,CAAC,WAAW,EAAEvB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMkB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAEtB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACsB;IACd;IAEA,OAAO,CAAC,EAAE,EAAEzB,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAewB,2BAA2B,EAC/CC,WAAW,EACX3C,WAAW,EACXY,WAAW,EACXgC,cAAc,EACdC,eAAe,EAOhB;IACC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAG1B,IAAInC,eAAe,CAACZ,aAClB,MAAM,IAAIgD,MACR;IAIJ,MAAMC,wBAAwBN,YAAY,GAAG,CAAC,CAAC3B,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAcgC,SAC1CpB;IAGJ,MAAMsC,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBN,mBAAmB;IAGjD,MAAMO,iBAAiBxC,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAGF,MAAMyC,kBAAkBzC,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAM0C,mBAAmB1C,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAMF,MAAM2C,aAAaJ,wBACf,uGACA;IAEJ,MAAMK,mBAAmBL,wBACrB,wNACA;IAEJ,MAAMM,0BAA0B;IAEhC,MAAMC,wBAAwBP,wBAC1B,CAAC;;;;AAIP,EAAEM,yBAAyB,GACrB,CAAC;;;;AAIP,EAAEA,yBAAyB;IAEzB,MAAME,cAAcR,wBAChB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAuDc,CAAC,GAChB;IAIJ,MAAMS,mBAAmB;IACzB,MAAMC,sBAAsBV,wBAAwB,IAAI;IACxD,MAAMW,mBAAmBX,wBAAwB,IAAI;IAErD,OAAO,CAAC;;;AAGV,EAAEI,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEE,sBAAsB;AACxB,EAAEC,YAAY;AACd,EACER,wBACI,CAAC;QACC,EAAES,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,EAAE,EAAEV,wBAAwB,kCAAkC,wCAAwC;;AAEpI,EAAEA,wBAAwB,+EAA+E,YAAY;;;;;;;;;;QAU7G,EAAEA,wBAAwB,sBAAsB,wBAAwB;MAC1E,EAAEA,wBAAwB,yBAAyB,2BAA2B;;;;wBAI5D,EAAEA,wBAAwB,sBAAsB,wBAAwB;6BACnE,EAAEA,wBAAwB,sBAAsB,wBAAwB;qCAChE,EAAEA,wBAAwB,sBAAsB,wBAAwB;4BACjF,EAAEA,wBAAwB,sBAAsB,wBAAwB;;;0NAGsH,EAAEA,wBAAwB,aAAa,KAAK;;AAEtQ,EACE,CAACA,wBACG,CAAC;;;;;AAKP,CAAC,GACK,GACL;;;sEAGqE,EAAEW,iBAAiB;;yCAEhD,EAAEX,wBAAwB,6GAA6G,GAAG,sFAAsF,EAAEA,wBAAwB,kCAAkC,2BAA2B,6BAA6B,EAAEA,wBAAwB,iEAAiE,uDAAuD;;;;QAIvgB,EAAEW,iBAAiB;;mGAEwE,EAAEX,wBAAwB,kBAAkB,GAAG;;;;;;;;;;AAUlJ,EAAED,WAAW;;;;;;QAML,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;;;;;;;;oBAoBT,EAAED,wBAAwB,cAAc,GAAG;;AAE/D,EACEA,wBACI,CAAC;;;;;;;;AAQP,CAAC,GACK,KAEJA,wBACI,CAAC;UACG,EAAES,iBAAiB;;AAE7B,CAAC,GACK,GACL;;;iBAGgB,EAAET,wBAAwB,yBAAyB,2BAA2B,iBAAiB,EAAEU,oBAAoB;;;iBAGrH,EAAEV,wBAAwB,yBAAyB,+BAA+B,WAAW,EAAEW,iBAAiB;;;;;;;;AAQjI,EACEX,wBACI,CAAC;;;;;0SAKmS,EAAEE,gBAAgB;;;;uKAIrJ,EAAEC,iBAAiB;;;;;AAK1L,CAAC,GACK,CAAC;;;;;0IAKmI,EAAED,gBAAgB;;;;sFAItE,EAAEC,iBAAiB;;;;;AAKzG,CAAC,EACC;AACF"}
+{"version":3,"file":"ai-model/prompt/llm-planning.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../../src/ai-model/prompt/llm-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/types';\nimport type { TModelFamily } from '@midscene/shared/env';\nimport { getPreferredLanguage } from '@midscene/shared/env';\nimport {\n  getZodDescription,\n  getZodTypeName,\n} from '@midscene/shared/zod-schema-utils';\nimport type { z } from 'zod';\nimport { bboxDescription } from './common';\n\nconst vlLocateParam = (modelFamily: TModelFamily | undefined) => {\n  if (modelFamily) {\n    return `{bbox: [number, number, number, number], prompt: string } // ${bboxDescription(modelFamily)}`;\n  }\n  return '{ prompt: string /* description of the target element */ }';\n};\n\n/**\n * Find ZodDefault in the wrapper chain and return its default value\n */\nconst findDefaultValue = (field: unknown): any | undefined => {\n  let current = field;\n  const visited = new Set<unknown>();\n\n  while (current && !visited.has(current)) {\n    visited.add(current);\n    const currentWithDef = current as {\n      _def?: {\n        typeName?: string;\n        defaultValue?: () => any;\n        innerType?: unknown;\n      };\n    };\n\n    if (!currentWithDef._def?.typeName) break;\n\n    if (currentWithDef._def.typeName === 'ZodDefault') {\n      return currentWithDef._def.defaultValue?.();\n    }\n\n    // Continue unwrapping if it's a wrapper type\n    if (\n      currentWithDef._def.typeName === 'ZodOptional' ||\n      currentWithDef._def.typeName === 'ZodNullable'\n    ) {\n      current = currentWithDef._def.innerType;\n    } else {\n      break;\n    }\n  }\n\n  return undefined;\n};\n\n/**\n * Inject bbox into locate fields of a sample object.\n * Walks the sample and for any locate field (identified by paramSchema),\n * adds a fake bbox array when includeBbox is true.\n */\nconst SAMPLE_BBOXES: [number, number, number, number][] = [\n  [50, 100, 200, 200],\n  [300, 400, 500, 500],\n  [600, 100, 800, 250],\n  [50, 600, 250, 750],\n];\n\nconst injectBboxIntoSample = (\n  sample: Record<string, any>,\n  locateFields: string[],\n  includeBbox: boolean,\n): Record<string, any> => {\n  if (!includeBbox) return sample;\n  const result = { ...sample };\n  let bboxIndex = 0;\n  for (const field of locateFields) {\n    if (\n      result[field] &&\n      typeof result[field] === 'object' &&\n      result[field].prompt\n    ) {\n      result[field] = {\n        ...result[field],\n        bbox: SAMPLE_BBOXES[bboxIndex % SAMPLE_BBOXES.length],\n      };\n      bboxIndex++;\n    }\n  }\n  return result;\n};\n\nexport const descriptionForAction = (\n  action: DeviceAction<any>,\n  locatorSchemaTypeDescription: string,\n  includeBbox = false,\n) => {\n  const tab = '  ';\n  const fields: string[] = [];\n\n  // Add the action type field\n  fields.push(`- type: \"${action.name}\"`);\n\n  // Handle paramSchema if it exists\n  if (action.paramSchema) {\n    const paramLines: string[] = [];\n\n    // Check if paramSchema is a ZodObject with shape\n    const schema = action.paramSchema as {\n      _def?: { typeName?: string };\n      shape?: Record<string, unknown>;\n    };\n    const isZodObject = schema._def?.typeName === 'ZodObject';\n\n    if (isZodObject && schema.shape) {\n      // Original logic for ZodObject schemas\n      const shape = schema.shape;\n\n      for (const [key, field] of Object.entries(shape)) {\n        if (field && typeof field === 'object') {\n          // Check if field is optional\n          const isOptional =\n            typeof (field as { isOptional?: () => boolean }).isOptional ===\n              'function' &&\n            (field as { isOptional: () => boolean }).isOptional();\n          const keyWithOptional = isOptional ? `${key}?` : key;\n\n          // Get the type name using extracted helper\n          const typeName = getZodTypeName(field, locatorSchemaTypeDescription);\n\n          // Get description using extracted helper\n          const description = getZodDescription(field as z.ZodTypeAny);\n\n          // Check if field has a default value by searching the wrapper chain\n          const defaultValue = findDefaultValue(field);\n          const hasDefault = defaultValue !== undefined;\n\n          // Build param line for this field\n          let paramLine = `${keyWithOptional}: ${typeName}`;\n          const comments: string[] = [];\n          if (description) {\n            comments.push(description);\n          }\n          if (hasDefault) {\n            const defaultStr =\n              typeof defaultValue === 'string'\n                ? `\"${defaultValue}\"`\n                : JSON.stringify(defaultValue);\n            comments.push(`default: ${defaultStr}`);\n          }\n          if (comments.length > 0) {\n            paramLine += ` // ${comments.join(', ')}`;\n          }\n\n          paramLines.push(paramLine);\n        }\n      }\n\n      // Add the param section to fields if there are paramLines\n      if (paramLines.length > 0) {\n        fields.push('- param:');\n        paramLines.forEach((line) => {\n          fields.push(`  - ${line}`);\n        });\n      }\n    } else {\n      // Handle non-object schemas (string, number, etc.)\n      const typeName = getZodTypeName(schema);\n      const description = getZodDescription(schema as z.ZodTypeAny);\n\n      // For simple types, indicate that param should be the direct value, not an object\n      let paramDescription = `- param: ${typeName}`;\n      if (description) {\n        paramDescription += ` // ${description}`;\n      }\n      paramDescription += ' (pass the value directly, not as an object)';\n\n      fields.push(paramDescription);\n    }\n  }\n\n  // Render sample if provided, using the same XML tag format as the real output\n  if (action.sample && typeof action.sample === 'object') {\n    const locateFields = findAllMidsceneLocatorField(action.paramSchema);\n    const sampleWithBbox = injectBboxIntoSample(\n      action.sample,\n      locateFields,\n      includeBbox,\n    );\n    const sampleStr = `- sample:\\n${tab}${tab}<action-type>${action.name}</action-type>\\n${tab}${tab}<action-param-json>\\n${tab}${tab}${JSON.stringify(sampleWithBbox, null, 2).replace(/\\n/g, `\\n${tab}${tab}`)}\\n${tab}${tab}</action-param-json>`;\n    fields.push(sampleStr);\n  }\n\n  return `- ${action.name}, ${action.description || 'No description provided'}\n${tab}${fields.join(`\\n${tab}`)}\n`.trim();\n};\n\nexport async function systemPromptToTaskPlanning({\n  actionSpace,\n  modelFamily,\n  includeBbox,\n  includeThought,\n  includeSubGoals,\n}: {\n  actionSpace: DeviceAction<any>[];\n  modelFamily: TModelFamily | undefined;\n  includeBbox: boolean;\n  includeThought?: boolean;\n  includeSubGoals?: boolean;\n}) {\n  const preferredLanguage = getPreferredLanguage();\n\n  // Validate parameters: if includeBbox is true, modelFamily must be defined\n  if (includeBbox && !modelFamily) {\n    throw new Error(\n      'modelFamily cannot be undefined when includeBbox is true. A valid modelFamily is required for bbox-based location.',\n    );\n  }\n\n  const actionDescriptionList = actionSpace.map((action) => {\n    return descriptionForAction(\n      action,\n      vlLocateParam(includeBbox ? modelFamily : undefined),\n      includeBbox,\n    );\n  });\n  const actionList = actionDescriptionList.join('\\n');\n\n  const shouldIncludeThought = includeThought ?? true;\n  const shouldIncludeSubGoals = includeSubGoals ?? false;\n\n  // Generate locate object examples based on includeBbox\n  const locateExample1 = includeBbox\n    ? `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\",\n    \"bbox\": [345, 442, 458, 483]\n  }`\n    : `{\n    \"prompt\": \"Add to cart button for Sauce Labs Backpack\"\n  }`;\n\n  // Locate examples for multi-turn conversation\n  const locateNameField = includeBbox\n    ? `{\n    \"prompt\": \"Name input field in the registration form\",\n    \"bbox\": [120, 180, 380, 210]\n  }`\n    : `{\n    \"prompt\": \"Name input field in the registration form\"\n  }`;\n\n  const locateEmailField = includeBbox\n    ? `{\n    \"prompt\": \"Email input field in the registration form\",\n    \"bbox\": [120, 240, 380, 270]\n  }`\n    : `{\n    \"prompt\": \"Email input field in the registration form\"\n  }`;\n\n  const thoughtTag = (content: string) =>\n    shouldIncludeThought ? `<thought>${content}</thought>\\n` : '';\n\n  // Sub-goals related content - only included when shouldIncludeSubGoals is true\n  const step1Title = shouldIncludeSubGoals\n    ? '## Step 1: Observe and Plan (related tags: <thought>, <update-plan-content>, <mark-sub-goal-done>)'\n    : '## Step 1: Observe (related tags: <thought>)';\n\n  const step1Description = shouldIncludeSubGoals\n    ? \"First, observe the current screenshot and previous logs, then break down the user's instruction into multiple high-level sub-goals. Update the status of sub-goals based on what you see in the current screenshot.\"\n    : 'First, observe the current screenshot and previous logs to understand the current state.';\n\n  const explicitInstructionRule = `CRITICAL - Following Explicit Instructions: When the user gives you specific operation steps (not high-level goals), you MUST execute ONLY those exact steps - nothing more, nothing less. Do NOT add extra actions even if they seem logical. For example: \"fill out the form\" means only fill fields, do NOT submit; \"click the button\" means only click, do NOT wait for page load or verify results; \"type 'hello'\" means only type, do NOT press Enter.`;\n\n  const thoughtTagDescription = shouldIncludeSubGoals\n    ? `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the user's requirement? What is the current state based on the screenshot? Are all sub-goals completed? If not, what should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`\n    : `REQUIRED: You MUST always output the <thought> tag. Never skip it.\n\nInclude your thought process in the <thought> tag. It should answer: What is the current state based on the screenshot? What should be the next action? Write your thoughts naturally without numbering or section headers.\n\n${explicitInstructionRule}`;\n\n  const subGoalTags = shouldIncludeSubGoals\n    ? `\n\n* <update-plan-content> tag\n\nUse this structure to give or update your plan:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished|pending\">sub goal description</sub-goal>\n  <sub-goal index=\"2\" status=\"finished|pending\">sub goal description</sub-goal>\n  ...\n</update-plan-content>\n\n* <mark-sub-goal-done> tag\n\nUse this structure to mark a sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nIMPORTANT: You MUST only mark a sub-goal as \"finished\" AFTER you have confirmed the task is actually completed by observing the result in the screenshot. Do NOT mark a sub-goal as done just because you expect the next action will complete it. Wait until you see visual confirmation in the screenshot that the sub-goal has been achieved.\n\n* Note\n\nDuring execution, you can call <update-plan-content> at any time to update the plan based on the latest screenshot and completed sub-goals.\n\n### Example\n\nIf the user wants to \"log in to a system using username and password, complete all to-do items, and submit a registration form\", you can break it down into the following sub-goals:\n\n<thought>...</thought>\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"pending\">Log in to the system</sub-goal>\n  <sub-goal index=\"2\" status=\"pending\">Complete all to-do items</sub-goal>\n  <sub-goal index=\"3\" status=\"pending\">Submit the registration form</sub-goal>\n</update-plan-content>\n\nAfter logging in and seeing the to-do items, you can mark the sub-goal as done:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n\nAt this point, the status of all sub-goals is:\n\n<update-plan-content>\n  <sub-goal index=\"1\" status=\"finished\" />\n  <sub-goal index=\"2\" status=\"pending\" />\n  <sub-goal index=\"3\" status=\"pending\" />\n</update-plan-content>\n\nAfter some time, when the last sub-goal is also completed, you can mark it as done as well:\n\n<mark-sub-goal-done>\n  <sub-goal index=\"3\" status=\"finished\" />\n</mark-sub-goal-done>`\n    : '';\n\n  // Step numbering adjusts based on whether sub-goals are included\n  // When includeSubGoals=false, memory step is skipped\n  const memoryStepNumber = 2; // Only used when shouldIncludeSubGoals is true\n  const checkGoalStepNumber = shouldIncludeSubGoals ? 3 : 2;\n  const actionStepNumber = shouldIncludeSubGoals ? 4 : 3;\n\n  return `\nTarget: You are an expert to manipulate the UI to accomplish the user's instruction. User will give you an instruction, some screenshots, background knowledge and previous logs indicating what have been done. Your task is to accomplish the instruction by thinking through the path to complete the task and give the next action to execute.\n\n${step1Title}\n\n${step1Description}\n\n* <thought> tag (REQUIRED)\n\n${thoughtTagDescription}\n${subGoalTags}\n${\n  shouldIncludeSubGoals\n    ? `\n## Step ${memoryStepNumber}: Memory Data from Current Screenshot (related tags: <memory>)\n\nWhile observing the current screenshot, if you notice any information that might be needed in follow-up actions, record it here. The current screenshot will NOT be available in subsequent steps, so this memory is your only way to preserve essential information. Examples: extracted data, element states, content that needs to be referenced.\n\nDon't use this tag if no information needs to be preserved.\n`\n    : ''\n}\n## Step ${checkGoalStepNumber}: ${shouldIncludeSubGoals ? 'Check if Goal is Accomplished' : 'Check if the Instruction is Fulfilled'} (related tags: <complete>)\n\n${shouldIncludeSubGoals ? 'Based on the current screenshot and the status of all sub-goals, determine' : 'Determine'} if the entire task is completed.\n\n### CRITICAL: The User's Instruction is the Supreme Authority\n\nThe user's instruction defines the EXACT scope of what you must accomplish. You MUST follow it precisely - nothing more, nothing less. Violating this rule may cause severe consequences such as data loss, unintended operations, or system failures.\n\n**Explicit instructions vs. High-level goals:**\n- If the user gives you **explicit operation steps** (e.g., \"click X\", \"type Y\", \"fill out the form\"), treat them as exact commands. Execute ONLY those steps, nothing more.\n- If the user gives you a **high-level goal** (e.g., \"log in to the system\", \"complete the purchase\"), you may determine the necessary steps to achieve it.\n\n**What \"${shouldIncludeSubGoals ? 'goal accomplished' : 'instruction fulfilled'}\" means:**\n- The ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} when you have done EXACTLY what the user asked - no extra steps, no assumptions.\n- Do NOT perform any action beyond the explicit instruction, even if it seems logical or helpful.\n\n**Examples - Explicit instructions (execute exactly, no extra steps):**\n- \"fill out the form\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when all fields are filled. Do NOT submit the form.\n- \"click the login button\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} once clicked. Do NOT wait for page load or verify login success.\n- \"type 'hello' in the search box\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when 'hello' is typed. Do NOT press Enter or trigger search.\n- \"select the first item\" → ${shouldIncludeSubGoals ? 'Goal accomplished' : 'Instruction fulfilled'} when selected. Do NOT proceed to checkout.\n\n**Special case - Assertion instructions:**\n- If the user's instruction includes an assertion (e.g., \"verify that...\", \"check that...\", \"assert...\"), and you observe from the screenshot that the assertion condition is NOT satisfied and cannot be satisfied, mark ${shouldIncludeSubGoals ? 'the goal' : 'it'} as failed (success=\"false\").\n- If the page is still loading (e.g., you see a loading spinner, skeleton screen, or progress bar), do NOT assert yet. Wait for the page to finish loading before evaluating the assertion.\n${\n  !shouldIncludeSubGoals\n    ? `\n**Page navigation restriction:**\n- Unless the user's instruction explicitly asks you to click a link, jump to another page, or navigate to a URL, you MUST complete the task on the current page only.\n- Do NOT navigate away from the current page on your own initiative (e.g., do not click links that lead to other pages, do not use browser back/forward, do not open new URLs).\n- If the task cannot be accomplished on the current page and the user has not instructed you to navigate, report it as a failure (success=\"false\") instead of attempting to navigate to other pages.\n`\n    : ''\n}\n### Output Rules\n\n- If the task is NOT complete, skip this section and continue to Step ${actionStepNumber}.\n- Use the <complete success=\"true|false\">message</complete> tag to output the result if the goal is accomplished or failed.\n  - the 'success' attribute is required. ${shouldIncludeSubGoals ? 'It means whether the expected goal is accomplished based on what you observe in the current screenshot. ' : ''}No matter what actions were executed or what errors occurred during execution, if the ${shouldIncludeSubGoals ? 'expected goal is accomplished' : 'instruction is fulfilled'}, set success=\"true\". If the ${shouldIncludeSubGoals ? 'expected goal is not accomplished and cannot be accomplished' : 'instruction is not fulfilled and cannot be fulfilled'}, set success=\"false\".\n  - the 'message' is the information that will be provided to the user. If the user asks for a specific format, strictly follow that.\n- If you output <complete>, do NOT output <action-type> or <action-param-json>. The task ends here.\n\n## Step ${actionStepNumber}: Determine Next Action (related tags: <log>, <action-type>, <action-param-json>, <error>)\n\nONLY if the task is not complete: Think what the next action is according to the current screenshot${shouldIncludeSubGoals ? ' and the plan' : ''}.\n\n- Don't give extra actions or plans beyond the instruction or the plan. For example, don't try to submit the form if the instruction is only to fill something.\n- Consider the current screenshot and give the action that is most likely to accomplish the instruction. For example, if the next step is to click a button but it's not visible in the screenshot, you should try to find it first instead of give a click action.\n- Make sure the previous actions are completed successfully. Otherwise, retry or do something else to recover.\n- Give just the next ONE action you should do (if any)\n- If there are some error messages reported by the previous actions, don't give up, try parse a new action to recover. If the error persists for more than 3 times, you should think this is an error and set the \"error\" field to the error message.\n\n### Supporting actions list\n\n${actionList}\n\n### Log to give user feedback (preamble message)\n\nThe <log> tag is a brief preamble message to the user explaining what you're about to do. It should follow these principles and examples:\n\n- **Use ${preferredLanguage}**\n- **Keep it concise**: be no more than 1-2 sentences, focused on immediate, tangible next steps. (8–12 words or Chinese characters for quick updates).\n- **Build on prior context**: if this is not the first action to be done, use the preamble message to connect the dots with what's been done so far and create a sense of momentum and clarity for the user to understand your next actions.\n- **Keep your tone light, friendly and curious**: add small touches of personality in preambles feel collaborative and engaging.\n\n**Examples:**\n- <log>Click the login button</log>\n- <log>Scroll to find the 'Yes' button in popup</log>\n- <log>Previous actions failed to find the 'Yes' button, i will try again</log>\n- <log>Go back to find the login button</log>\n\n### If there is some action to do ...\n\n- Use the <action-type> and <action-param-json> tags to output the action to be executed.\n- The <action-type> MUST be one of the supporting actions. 'complete' is NOT a valid action-type.\n- Parameter names are strict. Use EXACTLY the field names listed for the selected action. Do NOT invent alias fields. If an action has a \"sample\" in its description, follow that structure.\nFor example:\n<action-type>Tap</action-type>\n<action-param-json>\n{\n  \"locate\": ${locateExample1}\n}\n</action-param-json>\n\n### If you think there is an error ...\n\n- Use the <error> tag to output the error message.\n\nFor example:\n<error>Unable to find the required element on the page</error>\n\n### If there is no action to do ...\n\n- Don't output <action-type> or <action-param-json> if there is no action to do.\n\n## Return Format\n\nReturn in XML format following this decision flow:\n\n**Always include (REQUIRED):**\n<!-- Step 1: Observe${shouldIncludeSubGoals ? ' and Plan' : ''} -->\n<thought>Your thought process here. NEVER skip this tag.</thought>\n${\n  shouldIncludeSubGoals\n    ? `\n<!-- required when no update-plan-content is provided in the previous response -->\n<update-plan-content>...</update-plan-content>\n\n<!-- required when any sub-goal is completed -->\n<mark-sub-goal-done>\n  <sub-goal index=\"1\" status=\"finished\" />\n</mark-sub-goal-done>\n`\n    : ''\n}${\n  shouldIncludeSubGoals\n    ? `\n<!-- Step ${memoryStepNumber}: Memory data from current screenshot if needed -->\n<memory>...</memory>\n`\n    : ''\n}\n**Then choose ONE of the following paths:**\n\n**Path A: If the ${shouldIncludeSubGoals ? 'goal is accomplished' : 'instruction is fulfilled'} or failed (Step ${checkGoalStepNumber})**\n<complete success=\"true|false\">...</complete>\n\n**Path B: If the ${shouldIncludeSubGoals ? 'goal is NOT complete' : 'instruction is NOT fulfilled'} yet (Step ${actionStepNumber})**\n<!-- Determine next action -->\n<log>...</log>\n<action-type>...</action-type>\n<action-param-json>...</action-param-json>\n\n<!-- OR if there's an error -->\n<error>...</error>\n${\n  shouldIncludeSubGoals\n    ? `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'testmidscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Plan: fill Name, Email, return email</thought><update-plan-content><sub-goal index=\"1\" status=\"pending\">Fill Name</sub-goal><sub-goal index=\"2\" status=\"pending\">Fill Email</sub-goal></update-plan-content><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <mark-sub-goal-done><sub-goal index=\"1\" status=\"finished\" /></mark-sub-goal-done><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"testmidscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <mark-sub-goal-done><sub-goal index=\"2\" status=\"finished\" /></mark-sub-goal-done><complete success=\"true\">testmidscene.ai</complete>\n`\n    : `\n## Example\n\nInstruction: \"fill the form with name 'John' and email 'testmidscene.ai', return the email\"\n\n**Turn 1:** Empty form → <thought>Fill Name then Email, return email</thought><action-type>Tap</action-type><action-param-json>{\"locate\": ${locateNameField}}</action-param-json>\n\n**Turn 2:** Name focused → <action-type>Input</action-type><action-param-json>{\"value\": \"John\"}</action-param-json>\n\n**Turn 3:** Name='John' → <action-type>Tap</action-type><action-param-json>{\"locate\": ${locateEmailField}}</action-param-json>\n\n**Turn 4:** Email focused → <action-type>Input</action-type><action-param-json>{\"value\": \"testmidscene.ai\"}</action-param-json>\n\n**Turn 5:** Form complete → <complete success=\"true\">testmidscene.ai</complete>\n`\n}`;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","vlLocateParam","modelFamily","bboxDescription","findDefaultValue","field","current","visited","Set","currentWithDef","SAMPLE_BBOXES","injectBboxIntoSample","sample","locateFields","includeBbox","result","bboxIndex","descriptionForAction","action","locatorSchemaTypeDescription","tab","fields","paramLines","schema","isZodObject","shape","isOptional","keyWithOptional","typeName","getZodTypeName","description","getZodDescription","defaultValue","hasDefault","undefined","paramLine","comments","defaultStr","JSON","line","paramDescription","findAllMidsceneLocatorField","sampleWithBbox","sampleStr","systemPromptToTaskPlanning","actionSpace","includeThought","includeSubGoals","preferredLanguage","getPreferredLanguage","Error","actionDescriptionList","actionList","shouldIncludeSubGoals","locateExample1","locateNameField","locateEmailField","step1Title","step1Description","explicitInstructionRule","thoughtTagDescription","subGoalTags","memoryStepNumber","checkGoalStepNumber","actionStepNumber"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;ACKA,MAAMI,gBAAgB,CAACC;IACrB,IAAIA,aACF,OAAO,CAAC,6DAA6D,EAAEC,AAAAA,IAAAA,qCAAAA,eAAAA,AAAAA,EAAgBD,cAAc;IAEvG,OAAO;AACT;AAKA,MAAME,mBAAmB,CAACC;IACxB,IAAIC,UAAUD;IACd,MAAME,UAAU,IAAIC;IAEpB,MAAOF,WAAW,CAACC,QAAQ,GAAG,CAACD,SAAU;QACvCC,QAAQ,GAAG,CAACD;QACZ,MAAMG,iBAAiBH;QAQvB,IAAI,CAACG,eAAe,IAAI,EAAE,UAAU;QAEpC,IAAIA,AAAiC,iBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAC9B,OAAOA,eAAe,IAAI,CAAC,YAAY;QAIzC,IACEA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,IAC5BA,AAAiC,kBAAjCA,eAAe,IAAI,CAAC,QAAQ,EAE5BH,UAAUG,eAAe,IAAI,CAAC,SAAS;aAEvC;IAEJ;AAGF;AAOA,MAAMC,gBAAoD;IACxD;QAAC;QAAI;QAAK;QAAK;KAAI;IACnB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAK;QAAK;QAAK;KAAI;IACpB;QAAC;QAAI;QAAK;QAAK;KAAI;CACpB;AAED,MAAMC,uBAAuB,CAC3BC,QACAC,cACAC;IAEA,IAAI,CAACA,aAAa,OAAOF;IACzB,MAAMG,SAAS;QAAE,GAAGH,MAAM;IAAC;IAC3B,IAAII,YAAY;IAChB,KAAK,MAAMX,SAASQ,aAClB,IACEE,MAAM,CAACV,MAAM,IACb,AAAyB,YAAzB,OAAOU,MAAM,CAACV,MAAM,IACpBU,MAAM,CAACV,MAAM,CAAC,MAAM,EACpB;QACAU,MAAM,CAACV,MAAM,GAAG;YACd,GAAGU,MAAM,CAACV,MAAM;YAChB,MAAMK,aAAa,CAACM,YAAYN,cAAc,MAAM,CAAC;QACvD;QACAM;IACF;IAEF,OAAOD;AACT;AAEO,MAAME,uBAAuB,CAClCC,QACAC,8BACAL,cAAc,KAAK;IAEnB,MAAMM,MAAM;IACZ,MAAMC,SAAmB,EAAE;IAG3BA,OAAO,IAAI,CAAC,CAAC,SAAS,EAAEH,OAAO,IAAI,CAAC,CAAC,CAAC;IAGtC,IAAIA,OAAO,WAAW,EAAE;QACtB,MAAMI,aAAuB,EAAE;QAG/B,MAAMC,SAASL,OAAO,WAAW;QAIjC,MAAMM,cAAcD,OAAO,IAAI,EAAE,aAAa;QAE9C,IAAIC,eAAeD,OAAO,KAAK,EAAE;YAE/B,MAAME,QAAQF,OAAO,KAAK;YAE1B,KAAK,MAAM,CAAC3B,KAAKS,MAAM,IAAIR,OAAO,OAAO,CAAC4B,OACxC,IAAIpB,SAAS,AAAiB,YAAjB,OAAOA,OAAoB;gBAEtC,MAAMqB,aACJ,AACE,cADF,OAAQrB,MAAyC,UAAU,IAE1DA,MAAwC,UAAU;gBACrD,MAAMsB,kBAAkBD,aAAa,GAAG9B,IAAI,CAAC,CAAC,GAAGA;gBAGjD,MAAMgC,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAexB,OAAOc;gBAGvC,MAAMW,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkB1B;gBAGtC,MAAM2B,eAAe5B,iBAAiBC;gBACtC,MAAM4B,aAAaD,AAAiBE,WAAjBF;gBAGnB,IAAIG,YAAY,GAAGR,gBAAgB,EAAE,EAAEC,UAAU;gBACjD,MAAMQ,WAAqB,EAAE;gBAC7B,IAAIN,aACFM,SAAS,IAAI,CAACN;gBAEhB,IAAIG,YAAY;oBACd,MAAMI,aACJ,AAAwB,YAAxB,OAAOL,eACH,CAAC,CAAC,EAAEA,aAAa,CAAC,CAAC,GACnBM,KAAK,SAAS,CAACN;oBACrBI,SAAS,IAAI,CAAC,CAAC,SAAS,EAAEC,YAAY;gBACxC;gBACA,IAAID,SAAS,MAAM,GAAG,GACpBD,aAAa,CAAC,IAAI,EAAEC,SAAS,IAAI,CAAC,OAAO;gBAG3Cd,WAAW,IAAI,CAACa;YAClB;YAIF,IAAIb,WAAW,MAAM,GAAG,GAAG;gBACzBD,OAAO,IAAI,CAAC;gBACZC,WAAW,OAAO,CAAC,CAACiB;oBAClBlB,OAAO,IAAI,CAAC,CAAC,IAAI,EAAEkB,MAAM;gBAC3B;YACF;QACF,OAAO;YAEL,MAAMX,WAAWC,AAAAA,IAAAA,iCAAAA,cAAAA,AAAAA,EAAeN;YAChC,MAAMO,cAAcC,AAAAA,IAAAA,iCAAAA,iBAAAA,AAAAA,EAAkBR;YAGtC,IAAIiB,mBAAmB,CAAC,SAAS,EAAEZ,UAAU;YAC7C,IAAIE,aACFU,oBAAoB,CAAC,IAAI,EAAEV,aAAa;YAE1CU,oBAAoB;YAEpBnB,OAAO,IAAI,CAACmB;QACd;IACF;IAGA,IAAItB,OAAO,MAAM,IAAI,AAAyB,YAAzB,OAAOA,OAAO,MAAM,EAAe;QACtD,MAAML,eAAe4B,AAAAA,IAAAA,mCAAAA,2BAAAA,AAAAA,EAA4BvB,OAAO,WAAW;QACnE,MAAMwB,iBAAiB/B,qBACrBO,OAAO,MAAM,EACbL,cACAC;QAEF,MAAM6B,YAAY,CAAC,WAAW,EAAEvB,MAAMA,IAAI,aAAa,EAAEF,OAAO,IAAI,CAAC,gBAAgB,EAAEE,MAAMA,IAAI,qBAAqB,EAAEA,MAAMA,MAAMkB,KAAK,SAAS,CAACI,gBAAgB,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,EAAE,EAAEtB,MAAMA,KAAK,EAAE,EAAE,EAAEA,MAAMA,IAAI,oBAAoB,CAAC;QAChPC,OAAO,IAAI,CAACsB;IACd;IAEA,OAAO,CAAC,EAAE,EAAEzB,OAAO,IAAI,CAAC,EAAE,EAAEA,OAAO,WAAW,IAAI,0BAA0B;AAC9E,EAAEE,MAAMC,OAAO,IAAI,CAAC,CAAC,EAAE,EAAED,KAAK,EAAE;AAChC,CAAC,CAAC,IAAI;AACN;AAEO,eAAewB,2BAA2B,EAC/CC,WAAW,EACX3C,WAAW,EACXY,WAAW,EACXgC,cAAc,EACdC,eAAe,EAOhB;IACC,MAAMC,oBAAoBC,AAAAA,IAAAA,oBAAAA,oBAAAA,AAAAA;IAG1B,IAAInC,eAAe,CAACZ,aAClB,MAAM,IAAIgD,MACR;IAIJ,MAAMC,wBAAwBN,YAAY,GAAG,CAAC,CAAC3B,SACtCD,qBACLC,QACAjB,cAAca,cAAcZ,cAAcgC,SAC1CpB;IAGJ,MAAMsC,aAAaD,sBAAsB,IAAI,CAAC;IAG9C,MAAME,wBAAwBN,mBAAmB;IAGjD,MAAMO,iBAAiBxC,cACnB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAGF,MAAMyC,kBAAkBzC,cACpB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAEF,MAAM0C,mBAAmB1C,cACrB,CAAC;;;GAGJ,CAAC,GACE,CAAC;;GAEJ,CAAC;IAMF,MAAM2C,aAAaJ,wBACf,uGACA;IAEJ,MAAMK,mBAAmBL,wBACrB,wNACA;IAEJ,MAAMM,0BAA0B;IAEhC,MAAMC,wBAAwBP,wBAC1B,CAAC;;;;AAIP,EAAEM,yBAAyB,GACrB,CAAC;;;;AAIP,EAAEA,yBAAyB;IAEzB,MAAME,cAAcR,wBAChB,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;qBAuDc,CAAC,GAChB;IAIJ,MAAMS,mBAAmB;IACzB,MAAMC,sBAAsBV,wBAAwB,IAAI;IACxD,MAAMW,mBAAmBX,wBAAwB,IAAI;IAErD,OAAO,CAAC;;;AAGV,EAAEI,WAAW;;AAEb,EAAEC,iBAAiB;;;;AAInB,EAAEE,sBAAsB;AACxB,EAAEC,YAAY;AACd,EACER,wBACI,CAAC;QACC,EAAES,iBAAiB;;;;;AAK3B,CAAC,GACK,GACL;QACO,EAAEC,oBAAoB,EAAE,EAAEV,wBAAwB,kCAAkC,wCAAwC;;AAEpI,EAAEA,wBAAwB,+EAA+E,YAAY;;;;;;;;;;QAU7G,EAAEA,wBAAwB,sBAAsB,wBAAwB;MAC1E,EAAEA,wBAAwB,yBAAyB,2BAA2B;;;;wBAI5D,EAAEA,wBAAwB,sBAAsB,wBAAwB;6BACnE,EAAEA,wBAAwB,sBAAsB,wBAAwB;qCAChE,EAAEA,wBAAwB,sBAAsB,wBAAwB;4BACjF,EAAEA,wBAAwB,sBAAsB,wBAAwB;;;0NAGsH,EAAEA,wBAAwB,aAAa,KAAK;;AAEtQ,EACE,CAACA,wBACG,CAAC;;;;;AAKP,CAAC,GACK,GACL;;;sEAGqE,EAAEW,iBAAiB;;yCAEhD,EAAEX,wBAAwB,6GAA6G,GAAG,sFAAsF,EAAEA,wBAAwB,kCAAkC,2BAA2B,6BAA6B,EAAEA,wBAAwB,iEAAiE,uDAAuD;;;;QAIvgB,EAAEW,iBAAiB;;mGAEwE,EAAEX,wBAAwB,kBAAkB,GAAG;;;;;;;;;;AAUlJ,EAAED,WAAW;;;;;;QAML,EAAEJ,kBAAkB;;;;;;;;;;;;;;;;;;;;YAoBhB,EAAEM,eAAe;;;;;;;;;;;;;;;;;;;;oBAoBT,EAAED,wBAAwB,cAAc,GAAG;;AAE/D,EACEA,wBACI,CAAC;;;;;;;;AAQP,CAAC,GACK,KAEJA,wBACI,CAAC;UACG,EAAES,iBAAiB;;AAE7B,CAAC,GACK,GACL;;;iBAGgB,EAAET,wBAAwB,yBAAyB,2BAA2B,iBAAiB,EAAEU,oBAAoB;;;iBAGrH,EAAEV,wBAAwB,yBAAyB,+BAA+B,WAAW,EAAEW,iBAAiB;;;;;;;;AAQjI,EACEX,wBACI,CAAC;;;;;0SAKmS,EAAEE,gBAAgB;;;;uKAIrJ,EAAEC,iBAAiB;;;;;AAK1L,CAAC,GACK,CAAC;;;;;0IAKmI,EAAED,gBAAgB;;;;sFAItE,EAAEC,iBAAiB;;;;;AAKzG,CAAC,EACC;AACF"}

package/dist/lib/device/index.js

@@ -152,7 +152,7 @@ const defineActionInput = (call)=>defineAction({
         interfaceAlias: 'aiInput',
         paramSchema: actionInputParamSchema,
         sample: {
-            value: 'test@example.com',
+            value: 'testexample.com',
             locate: {
                 prompt: 'the email input field'
             }

package/dist/lib/device/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"device/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/device/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getMidsceneLocationSchema } from '@/common';\nimport type {\n  ActionScrollParam,\n  DeviceAction,\n  LocateResultElement,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n  accept(files: string[]): Promise<void>;\n}\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[];\n\n  abstract cacheFeatureForPoint?(\n    center: [number, number],\n    options?: {\n      targetDescription?: string;\n      modelConfig?: IModelConfig;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // for web only\n  registerFileChooserListener?(\n    handler: (chooser: FileChooserHandler) => Promise<void>,\n  ): Promise<{ dispose: () => void; getError: () => Error | undefined }>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  /**\n   * Get the current time from the device.\n   * Returns the device's current timestamp in milliseconds.\n   * This is useful when the system time and device time are not synchronized.\n   */\n  getTimestamp?(): Promise<number>;\n\n  /** URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) */\n  mjpegStreamUrl?: string;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n  TSchema extends z.ZodType | undefined = undefined,\n  TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n  TReturn = any,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema?: TSchema;\n    call: (param: TRuntime) => Promise<TReturn> | TReturn;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime, TReturn>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime, TReturn> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    sample: {\n      locate: { prompt: 'the \"Submit\" button' },\n    },\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    sample: {\n      locate: { prompt: 'the file icon on the desktop' },\n    },\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    sample: {\n      locate: { prompt: 'the folder icon' },\n    },\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    sample: {\n      locate: { prompt: 'the navigation menu item \"Products\"' },\n    },\n    call,\n  });\n};\n\n// Input\nconst inputLocateDescription =\n  'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe(inputLocateDescription)\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'typeOnly'])\n    .default('replace')\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'typeOnly' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    sample: {\n      value: 'test@example.com',\n      locate: { prompt: 'the email input field' },\n    },\n    call: (param) => {\n      // backward compat: convert deprecated 'append' to 'typeOnly'\n      if ((param.mode as string) === 'append') {\n        param.mode = 'typeOnly';\n      }\n      return call(param);\n    },\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    sample: {\n      keyName: 'Enter',\n    },\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  scrollType: z\n    .enum([\n      'singleAction',\n      'scrollToBottom',\n      'scrollToTop',\n      'scrollToRight',\n      'scrollToLeft',\n    ])\n    .default('singleAction')\n    .describe(\n      'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n    ),\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe(\n      'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n    ),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n    ),\n});\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    sample: {\n      direction: 'down',\n      scrollType: 'singleAction',\n      locate: { prompt: 'the center of the product list area' },\n    },\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description:\n      'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    sample: {\n      from: { prompt: 'the \"report.pdf\" file icon' },\n      to: { prompt: 'the upload drop zone' },\n    },\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    sample: {\n      locate: { prompt: 'the message bubble' },\n    },\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n  param: ActionSwipeParam,\n  screenSize: { width: number; height: number },\n): {\n  startPoint: { x: number; y: number };\n  endPoint: { x: number; y: number };\n  duration: number;\n  repeatCount: number;\n} {\n  const { width, height } = screenSize;\n  const { start, end } = param;\n\n  const startPoint = start\n    ? { x: start.center[0], y: start.center[1] }\n    : { x: width / 2, y: height / 2 };\n\n  let endPoint: { x: number; y: number };\n\n  if (end) {\n    endPoint = { x: end.center[0], y: end.center[1] };\n  } else if (param.distance) {\n    const direction = param.direction;\n    if (!direction) {\n      throw new Error('direction is required for swipe gesture');\n    }\n    endPoint = {\n      x:\n        startPoint.x +\n        (direction === 'right'\n          ? param.distance\n          : direction === 'left'\n            ? -param.distance\n            : 0),\n      y:\n        startPoint.y +\n        (direction === 'down'\n          ? param.distance\n          : direction === 'up'\n            ? -param.distance\n            : 0),\n    };\n  } else {\n    throw new Error(\n      'Either end or distance must be specified for swipe gesture',\n    );\n  }\n\n  endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n  endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n  const duration = param.duration ?? 300;\n\n  let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n  if (repeatCount === 0) {\n    repeatCount = 10;\n  }\n\n  return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n    paramSchema: ActionSwipeParamSchema,\n    sample: {\n      start: { prompt: 'center of the notification' },\n      end: { prompt: 'upper edge of the screen' },\n    },\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The input field to be cleared')\n    .optional(),\n});\nexport type ActionClearInputParam = {\n  locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: inputLocateDescription,\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    sample: {\n      locate: { prompt: 'the search input field' },\n    },\n    call,\n  });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n  direction: z\n    .enum(['left', 'right'])\n    .describe('The direction to move the cursor'),\n  times: z\n    .number()\n    .int()\n    .min(1)\n    .default(1)\n    .describe(\n      'The number of times to move the cursor in the specified direction',\n    ),\n});\nexport type ActionCursorMoveParam = {\n  direction: 'left' | 'right';\n  times?: number;\n};\n\nexport const defineActionCursorMove = (\n  call: (param: ActionCursorMoveParam) => Promise<void>,\n): DeviceAction<ActionCursorMoveParam> => {\n  return defineAction<\n    typeof actionCursorMoveParamSchema,\n    ActionCursorMoveParam\n  >({\n    name: 'CursorMove',\n    description:\n      'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n    paramSchema: actionCursorMoveParamSchema,\n    sample: {\n      direction: 'left',\n      times: 3,\n    },\n    call,\n  });\n};\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n  timeMs: z\n    .number()\n    .default(1000)\n    .optional()\n    .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n  timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n  return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n    name: 'Sleep',\n    description:\n      'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n    paramSchema: ActionSleepParamSchema,\n    sample: {\n      timeMs: 2000,\n    },\n    call: async (param) => {\n      const duration = param?.timeMs ?? 1000;\n      getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n      await new Promise((resolve) => setTimeout(resolve, duration));\n    },\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n  HarmonyDeviceOpt,\n  HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","param","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Error","Math","duration","repeatCount","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput","actionCursorMoveParamSchema","defineActionCursorMove","ActionSleepParamSchema","defineActionSleep","getDebug","Promise","resolve","setTimeout"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACWO,MAAeI;;QA8CpB;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAIF,MAAMC,uBAAuBC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC3C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACAI;IACF;AAIK,MAAMC,8BAA8BJ,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACAD;IACF;AAIK,MAAMG,+BAA+BN,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACAH;IACF;AAIK,MAAMK,yBAAyBR,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAL;IACF;AAIF,MAAMO,yBACJ;AACK,MAAMC,yBAAyBX,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOA,6BAAAA,CAAAA,CAAAA,KACC,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG,EAC9B,SAAS,CAAC,CAACY,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQX,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAACS,wBACT,QAAQ;IACX,MAAMV,6BAAAA,CAAAA,CAAAA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;AAEN;AAOO,MAAMc,oBAAoB,CAC/BX,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,CAACI;YAEL,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAEf,OAAOZ,KAAKY;QACd;IACF;AAIK,MAAMC,iCAAiChB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACrD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,6BAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMiB,4BAA4B,CACvCd,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACb,QAAQ;YACN,SAAS;QACX;QACAb;IACF;AAIK,MAAMe,0BAA0BlB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,YAAYA,6BAAAA,CAAAA,CAAAA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMkB,qBAAqB,CAChChB,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaqB;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAf;IACF;AAIK,MAAMiB,+BAA+BpB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,MAAMC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;IAC3C,IAAIA,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMoB,0BAA0B,CACrClB,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAauB;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACAjB;IACF;AAGK,MAAMmB,6BAA6BtB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;IAEF,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMuB,wBAAwB,CACnCpB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAayB;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACAnB;IACF;AAGK,MAAMqB,yBAAyBxB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASyB,0BACdV,KAAuB,EACvBW,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAGf;IAEvB,MAAMgB,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAIf,MAAM,QAAQ,EAAE;QACzB,MAAMkB,YAAYlB,MAAM,SAAS;QACjC,IAAI,CAACkB,WACH,MAAM,IAAIC,MAAM;QAElBF,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,WAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;YACR,GACEgB,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,SAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAImB,MACR;IAIJF,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEJ;IAE9C,MAAMQ,WAAWrB,MAAM,QAAQ,IAAI;IAEnC,IAAIsB,cAAc,AAAwB,YAAxB,OAAOtB,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIsB,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEN;QAAYC;QAAUI;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAC/BnC,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2B;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACArB;IACF;AAIK,MAAMoC,8BAA8BvC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMuC,yBAAyB,CACpCrC,OAEON,aAGL;QACA,MAAM;QACN,aAAaa;QACb,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACApC;IACF;AAIK,MAAMsC,8BAA8BzC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,6BAAAA,CAAAA,CAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAM0C,yBAAyB,CACpCvC,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa4C;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACAtC;IACF;AAGK,MAAMwC,yBAAyB3C,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAM4C,oBAAoB,IACxB/C,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8C;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAO5B;YACX,MAAMqB,WAAWrB,OAAO,UAAU;YAClC8B,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB,CAAC,aAAa,EAAET,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIU,QAAQ,CAACC,UAAYC,WAAWD,SAASX;QACrD;IACF"}
+{"version":3,"file":"device/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/device/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getMidsceneLocationSchema } from '@/common';\nimport type {\n  ActionScrollParam,\n  DeviceAction,\n  LocateResultElement,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport interface FileChooserHandler {\n  accept(files: string[]): Promise<void>;\n}\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[];\n\n  abstract cacheFeatureForPoint?(\n    center: [number, number],\n    options?: {\n      targetDescription?: string;\n      modelConfig?: IModelConfig;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // for web only\n  registerFileChooserListener?(\n    handler: (chooser: FileChooserHandler) => Promise<void>,\n  ): Promise<{ dispose: () => void; getError: () => Error | undefined }>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  /**\n   * Get the current time from the device.\n   * Returns the device's current timestamp in milliseconds.\n   * This is useful when the system time and device time are not synchronized.\n   */\n  getTimestamp?(): Promise<number>;\n\n  /** URL of native MJPEG stream for real-time screen preview (e.g. WDA MJPEG server) */\n  mjpegStreamUrl?: string;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n  TSchema extends z.ZodType | undefined = undefined,\n  TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n  TReturn = any,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema?: TSchema;\n    call: (param: TRuntime) => Promise<TReturn> | TReturn;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime, TReturn>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime, TReturn> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    sample: {\n      locate: { prompt: 'the \"Submit\" button' },\n    },\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    sample: {\n      locate: { prompt: 'the file icon on the desktop' },\n    },\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    sample: {\n      locate: { prompt: 'the folder icon' },\n    },\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    sample: {\n      locate: { prompt: 'the navigation menu item \"Products\"' },\n    },\n    call,\n  });\n};\n\n// Input\nconst inputLocateDescription =\n  'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe(inputLocateDescription)\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'typeOnly'])\n    .default('replace')\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"typeOnly\" - type the value directly without clearing the field first; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'typeOnly' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    sample: {\n      value: 'testexample.com',\n      locate: { prompt: 'the email input field' },\n    },\n    call: (param) => {\n      // backward compat: convert deprecated 'append' to 'typeOnly'\n      if ((param.mode as string) === 'append') {\n        param.mode = 'typeOnly';\n      }\n      return call(param);\n    },\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    sample: {\n      keyName: 'Enter',\n    },\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  scrollType: z\n    .enum([\n      'singleAction',\n      'scrollToBottom',\n      'scrollToTop',\n      'scrollToRight',\n      'scrollToLeft',\n    ])\n    .default('singleAction')\n    .describe(\n      'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling all the way to the bottom by rapidly scrolling 5-10 times (skipping intermediate content until reaching the bottom), \"scrollToTop\" for scrolling all the way to the top by rapidly scrolling 5-10 times (skipping intermediate content until reaching the top), \"scrollToRight\" for scrolling all the way to the right by rapidly scrolling multiple times, \"scrollToLeft\" for scrolling all the way to the left by rapidly scrolling multiple times',\n    ),\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe(\n      'The direction to scroll. Only effective when scrollType is \"singleAction\".',\n    ),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Describe the target element to be scrolled on, like \"the table\" or \"the list\" or \"the content area\" or \"the scrollable area\". Do NOT provide a general intent like \"scroll to find some element\"',\n    ),\n});\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or a scrollable element to browse content. This is the preferred way to scroll on all platforms, including mobile. Supports scrollToBottom/scrollToTop for boundary navigation. Default: direction `down`, scrollType `singleAction`, distance `null`.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    sample: {\n      direction: 'down',\n      scrollType: 'singleAction',\n      locate: { prompt: 'the center of the product list area' },\n    },\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description:\n      'Pick up a specific UI element and move it to a new position (e.g., reorder a card, move a file into a folder, sort list items). The element itself moves with your finger/mouse.',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    sample: {\n      from: { prompt: 'the \"report.pdf\" file icon' },\n      to: { prompt: 'the upload drop zone' },\n    },\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    sample: {\n      locate: { prompt: 'the message bubble' },\n    },\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport function normalizeMobileSwipeParam(\n  param: ActionSwipeParam,\n  screenSize: { width: number; height: number },\n): {\n  startPoint: { x: number; y: number };\n  endPoint: { x: number; y: number };\n  duration: number;\n  repeatCount: number;\n} {\n  const { width, height } = screenSize;\n  const { start, end } = param;\n\n  const startPoint = start\n    ? { x: start.center[0], y: start.center[1] }\n    : { x: width / 2, y: height / 2 };\n\n  let endPoint: { x: number; y: number };\n\n  if (end) {\n    endPoint = { x: end.center[0], y: end.center[1] };\n  } else if (param.distance) {\n    const direction = param.direction;\n    if (!direction) {\n      throw new Error('direction is required for swipe gesture');\n    }\n    endPoint = {\n      x:\n        startPoint.x +\n        (direction === 'right'\n          ? param.distance\n          : direction === 'left'\n            ? -param.distance\n            : 0),\n      y:\n        startPoint.y +\n        (direction === 'down'\n          ? param.distance\n          : direction === 'up'\n            ? -param.distance\n            : 0),\n    };\n  } else {\n    throw new Error(\n      'Either end or distance must be specified for swipe gesture',\n    );\n  }\n\n  endPoint.x = Math.max(0, Math.min(endPoint.x, width));\n  endPoint.y = Math.max(0, Math.min(endPoint.y, height));\n\n  const duration = param.duration ?? 300;\n\n  let repeatCount = typeof param.repeat === 'number' ? param.repeat : 1;\n  if (repeatCount === 0) {\n    repeatCount = 10;\n  }\n\n  return { startPoint, endPoint, duration, repeatCount };\n}\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a touch gesture for interactions beyond regular scrolling (e.g., flip pages in a carousel, dismiss a notification, swipe-to-delete a list item). For regular content scrolling, use Scroll instead. Use \"distance\" + \"direction\" for relative movement, or \"end\" for precise endpoint.',\n    paramSchema: ActionSwipeParamSchema,\n    sample: {\n      start: { prompt: 'center of the notification' },\n      end: { prompt: 'upper edge of the screen' },\n    },\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The input field to be cleared')\n    .optional(),\n});\nexport type ActionClearInputParam = {\n  locate?: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: inputLocateDescription,\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    sample: {\n      locate: { prompt: 'the search input field' },\n    },\n    call,\n  });\n};\n\n// CursorMove\nexport const actionCursorMoveParamSchema = z.object({\n  direction: z\n    .enum(['left', 'right'])\n    .describe('The direction to move the cursor'),\n  times: z\n    .number()\n    .int()\n    .min(1)\n    .default(1)\n    .describe(\n      'The number of times to move the cursor in the specified direction',\n    ),\n});\nexport type ActionCursorMoveParam = {\n  direction: 'left' | 'right';\n  times?: number;\n};\n\nexport const defineActionCursorMove = (\n  call: (param: ActionCursorMoveParam) => Promise<void>,\n): DeviceAction<ActionCursorMoveParam> => {\n  return defineAction<\n    typeof actionCursorMoveParamSchema,\n    ActionCursorMoveParam\n  >({\n    name: 'CursorMove',\n    description:\n      'Move the text cursor (caret) left or right within an input field or text area. Use this to reposition the cursor without selecting text.',\n    paramSchema: actionCursorMoveParamSchema,\n    sample: {\n      direction: 'left',\n      times: 3,\n    },\n    call,\n  });\n};\n// Sleep\nexport const ActionSleepParamSchema = z.object({\n  timeMs: z\n    .number()\n    .default(1000)\n    .optional()\n    .describe('Sleep duration in milliseconds, defaults to 1000ms (1 second)'),\n});\n\nexport type ActionSleepParam = {\n  timeMs?: number;\n};\n\nexport const defineActionSleep = (): DeviceAction<ActionSleepParam> => {\n  return defineAction<typeof ActionSleepParamSchema, ActionSleepParam>({\n    name: 'Sleep',\n    description:\n      'Wait for a specified duration before continuing. Defaults to 1 second (1000ms) if not specified.',\n    paramSchema: ActionSleepParamSchema,\n    sample: {\n      timeMs: 2000,\n    },\n    call: async (param) => {\n      const duration = param?.timeMs ?? 1000;\n      getDebug('device:common-action')(`Sleeping for ${duration}ms`);\n      await new Promise((resolve) => setTimeout(resolve, duration));\n    },\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n  HarmonyDeviceOpt,\n  HarmonyDeviceInputOpt,\n} from './device-options';\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","param","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","normalizeMobileSwipeParam","screenSize","width","height","start","end","startPoint","endPoint","direction","Error","Math","duration","repeatCount","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput","actionCursorMoveParamSchema","defineActionCursorMove","ActionSleepParamSchema","defineActionSleep","getDebug","Promise","resolve","setTimeout"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACWO,MAAeI;;QA8CpB;;AACF;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAIF,MAAMC,uBAAuBC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC3C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsB;QAC1C;QACAI;IACF;AAIK,MAAMC,8BAA8BJ,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAA+B;QACnD;QACAD;IACF;AAIK,MAAMG,+BAA+BN,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAkB;QACtC;QACAH;IACF;AAIK,MAAMK,yBAAyBR,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAL;IACF;AAIF,MAAMO,yBACJ;AACK,MAAMC,yBAAyBX,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOA,6BAAAA,CAAAA,CAAAA,KACC,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG,EAC9B,SAAS,CAAC,CAACY,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQX,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAACS,wBACT,QAAQ;IACX,MAAMV,6BAAAA,CAAAA,CAAAA,OACC,CAAC;QAAC;QAAW;QAAS;KAAW,EACrC,OAAO,CAAC,WACR,QAAQ,CACP;AAEN;AAOO,MAAMc,oBAAoB,CAC/BX,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACb,QAAQ;YACN,OAAO;YACP,QAAQ;gBAAE,QAAQ;YAAwB;QAC5C;QACA,MAAM,CAACI;YAEL,IAAKA,AAA0B,aAA1BA,MAAM,IAAI,EACbA,MAAM,IAAI,GAAG;YAEf,OAAOZ,KAAKY;QACd;IACF;AAIK,MAAMC,iCAAiChB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACrD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,6BAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMiB,4BAA4B,CACvCd,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACb,QAAQ;YACN,SAAS;QACX;QACAb;IACF;AAIK,MAAMe,0BAA0BlB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,YAAYA,6BAAAA,CAAAA,CAAAA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CACP;AAEN;AAEO,MAAMkB,qBAAqB,CAChChB,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaqB;QACb,QAAQ;YACN,WAAW;YACX,YAAY;YACZ,QAAQ;gBAAE,QAAQ;YAAsC;QAC1D;QACAf;IACF;AAIK,MAAMiB,+BAA+BpB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,MAAMC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;IAC3C,IAAIA,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMoB,0BAA0B,CACrClB,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAauB;QACb,QAAQ;YACN,MAAM;gBAAE,QAAQ;YAA6B;YAC7C,IAAI;gBAAE,QAAQ;YAAuB;QACvC;QACAjB;IACF;AAGK,MAAMmB,6BAA6BtB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;IAEF,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMuB,wBAAwB,CACnCpB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAayB;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAqB;QACzC;QACAnB;IACF;AAGK,MAAMqB,yBAAyBxB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,SAASyB,0BACdV,KAAuB,EACvBW,UAA6C;IAO7C,MAAM,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGF;IAC1B,MAAM,EAAEG,KAAK,EAAEC,GAAG,EAAE,GAAGf;IAEvB,MAAMgB,aAAaF,QACf;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;QAAE,GAAGA,MAAM,MAAM,CAAC,EAAE;IAAC,IACzC;QAAE,GAAGF,QAAQ;QAAG,GAAGC,SAAS;IAAE;IAElC,IAAII;IAEJ,IAAIF,KACFE,WAAW;QAAE,GAAGF,IAAI,MAAM,CAAC,EAAE;QAAE,GAAGA,IAAI,MAAM,CAAC,EAAE;IAAC;SAC3C,IAAIf,MAAM,QAAQ,EAAE;QACzB,MAAMkB,YAAYlB,MAAM,SAAS;QACjC,IAAI,CAACkB,WACH,MAAM,IAAIC,MAAM;QAElBF,WAAW;YACT,GACED,WAAW,CAAC,GACXE,CAAAA,AAAc,YAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,WAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;YACR,GACEgB,WAAW,CAAC,GACXE,CAAAA,AAAc,WAAdA,YACGlB,MAAM,QAAQ,GACdkB,AAAc,SAAdA,YACE,CAAClB,MAAM,QAAQ,GACf;QACV;IACF,OACE,MAAM,IAAImB,MACR;IAIJF,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEL;IAC9CK,SAAS,CAAC,GAAGG,KAAK,GAAG,CAAC,GAAGA,KAAK,GAAG,CAACH,SAAS,CAAC,EAAEJ;IAE9C,MAAMQ,WAAWrB,MAAM,QAAQ,IAAI;IAEnC,IAAIsB,cAAc,AAAwB,YAAxB,OAAOtB,MAAM,MAAM,GAAgBA,MAAM,MAAM,GAAG;IACpE,IAAIsB,AAAgB,MAAhBA,aACFA,cAAc;IAGhB,OAAO;QAAEN;QAAYC;QAAUI;QAAUC;IAAY;AACvD;AAEO,MAAMC,oBAAoB,CAC/BnC,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa2B;QACb,QAAQ;YACN,OAAO;gBAAE,QAAQ;YAA6B;YAC9C,KAAK;gBAAE,QAAQ;YAA2B;QAC5C;QACArB;IACF;AAIK,MAAMoC,8BAA8BvC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,iCACT,QAAQ;AACb;AAKO,MAAMuC,yBAAyB,CACpCrC,OAEON,aAGL;QACA,MAAM;QACN,aAAaa;QACb,gBAAgB;QAChB,aAAa6B;QACb,QAAQ;YACN,QAAQ;gBAAE,QAAQ;YAAyB;QAC7C;QACApC;IACF;AAIK,MAAMsC,8BAA8BzC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;KAAQ,EACtB,QAAQ,CAAC;IACZ,OAAOA,6BAAAA,CAAAA,CAAAA,MACE,GACN,GAAG,GACH,GAAG,CAAC,GACJ,OAAO,CAAC,GACR,QAAQ,CACP;AAEN;AAMO,MAAM0C,yBAAyB,CACpCvC,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,aAAa4C;QACb,QAAQ;YACN,WAAW;YACX,OAAO;QACT;QACAtC;IACF;AAGK,MAAMwC,yBAAyB3C,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,OAAO,CAAC,MACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAM4C,oBAAoB,IACxB/C,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa8C;QACb,QAAQ;YACN,QAAQ;QACV;QACA,MAAM,OAAO5B;YACX,MAAMqB,WAAWrB,OAAO,UAAU;YAClC8B,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBAAwB,CAAC,aAAa,EAAET,SAAS,EAAE,CAAC;YAC7D,MAAM,IAAIU,QAAQ,CAACC,UAAYC,WAAWD,SAASX;QACrD;IACF"}

package/dist/lib/utils.js

@@ -202,7 +202,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "1.5.11";
+    return "1.5.14";
 }
 function debugLog(...message) {
     const debugMode = process.env[env_namespaceObject.MIDSCENE_DEBUG_MODE];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@donggui/core",
   "description": "DongGUI Core - Automate browser actions, extract data, and perform assertions using AI.",
-  "version": "1.5.12",
+  "version": "1.5.14",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",