@donggui/core 1.5.4-donggui.3

package/dist/es/agent/task-cache.mjs

@@ -0,0 +1,186 @@
+import node_assert from "node:assert";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { isDeepStrictEqual } from "node:util";
+import { getMidsceneRunSubDir } from "@midscene/shared/common";
+import { MIDSCENE_CACHE_MAX_FILENAME_LENGTH, globalConfigManager } from "@midscene/shared/env";
+import { getDebug } from "@midscene/shared/logger";
+import { generateHashId, ifInBrowser, ifInWorker, replaceIllegalPathCharsAndSpace } from "@midscene/shared/utils";
+import js_yaml from "js-yaml";
+import semver from "semver";
+import { getMidsceneVersion } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const DEFAULT_CACHE_MAX_FILENAME_LENGTH = 200;
+const debug = getDebug('cache');
+const lowestSupportedMidsceneVersion = '0.16.10';
+const cacheFileExt = '.cache.yaml';
+class TaskCache {
+    matchCache(prompt, type) {
+        if (!this.isCacheResultUsed) return;
+        const promptStr = 'string' == typeof prompt ? prompt : JSON.stringify(prompt);
+        for(let i = 0; i < this.cacheOriginalLength; i++){
+            const item = this.cache.caches[i];
+            const key = `${type}:${promptStr}:${i}`;
+            if (item.type === type && isDeepStrictEqual(item.prompt, prompt) && !this.matchedCacheIndices.has(key)) {
+                if ('locate' === item.type) {
+                    const locateItem = item;
+                    if (!locateItem.cache && Array.isArray(locateItem.xpaths)) locateItem.cache = {
+                        xpaths: locateItem.xpaths
+                    };
+                    if ('xpaths' in locateItem) locateItem.xpaths = void 0;
+                }
+                this.matchedCacheIndices.add(key);
+                debug('cache found and marked as used, type: %s, prompt: %s, index: %d', type, prompt, i);
+                return {
+                    cacheContent: item,
+                    updateFn: (cb)=>{
+                        debug('will call updateFn to update cache, type: %s, prompt: %s, index: %d', type, prompt, i);
+                        cb(item);
+                        if (this.readOnlyMode) return void debug('read-only mode, cache updated in memory but not flushed to file');
+                        debug('cache updated, will flush to file, type: %s, prompt: %s, index: %d', type, prompt, i);
+                        this.flushCacheToFile();
+                    }
+                };
+            }
+        }
+        debug('no unused cache found, type: %s, prompt: %s', type, prompt);
+    }
+    matchPlanCache(prompt) {
+        return this.matchCache(prompt, 'plan');
+    }
+    matchLocateCache(prompt) {
+        return this.matchCache(prompt, 'locate');
+    }
+    appendCache(cache) {
+        debug('will append cache', cache);
+        this.cache.caches.push(cache);
+        if (this.readOnlyMode) return void debug('read-only mode, cache appended to memory but not flushed to file');
+        this.flushCacheToFile();
+    }
+    loadCacheFromFile() {
+        const cacheFile = this.cacheFilePath;
+        node_assert(cacheFile, 'cache file path is required');
+        if (!existsSync(cacheFile)) return void debug('no cache file found, path: %s', cacheFile);
+        const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, '.json');
+        if (existsSync(jsonTypeCacheFile) && this.isCacheResultUsed) return void console.warn(`An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`);
+        try {
+            const data = readFileSync(cacheFile, 'utf8');
+            const jsonData = js_yaml.load(data);
+            const version = getMidsceneVersion();
+            if (!version) return void debug('no midscene version info, will not read cache from file');
+            if (semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) && !jsonData.midsceneVersion.includes('beta')) return void console.warn(`You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.\nPlease delete the existing cache and rebuild it. Sorry for the inconvenience.\ncache file: ${cacheFile}`);
+            debug('cache loaded from file, path: %s, cache version: %s, record length: %s', cacheFile, jsonData.midsceneVersion, jsonData.caches.length);
+            jsonData.midsceneVersion = getMidsceneVersion();
+            return jsonData;
+        } catch (err) {
+            debug('cache file exists but load failed, path: %s, error: %s', cacheFile, err);
+            return;
+        }
+    }
+    flushCacheToFile(options) {
+        const version = getMidsceneVersion();
+        if (!version) return void debug('no midscene version info, will not write cache to file');
+        if (!this.cacheFilePath) return void debug('no cache file path, will not write cache to file');
+        if (options?.cleanUnused) if (this.isCacheResultUsed) {
+            const originalLength = this.cache.caches.length;
+            const usedIndices = new Set();
+            for (const key of this.matchedCacheIndices){
+                const parts = key.split(':');
+                const index = Number.parseInt(parts[parts.length - 1], 10);
+                if (!Number.isNaN(index)) usedIndices.add(index);
+            }
+            this.cache.caches = this.cache.caches.filter((_, index)=>{
+                const isUsed = usedIndices.has(index);
+                const isNew = index >= this.cacheOriginalLength;
+                return isUsed || isNew;
+            });
+            const removedCount = originalLength - this.cache.caches.length;
+            removedCount > 0 ? debug('cleaned %d unused cache record(s)', removedCount) : debug('no unused cache to clean');
+        } else debug('skip cleaning: cache is not used for reading');
+        try {
+            const dir = dirname(this.cacheFilePath);
+            if (!existsSync(dir)) {
+                mkdirSync(dir, {
+                    recursive: true
+                });
+                debug('created cache directory: %s', dir);
+            }
+            const sortedCaches = [
+                ...this.cache.caches
+            ].sort((a, b)=>{
+                if ('plan' === a.type && 'locate' === b.type) return -1;
+                if ('locate' === a.type && 'plan' === b.type) return 1;
+                return 0;
+            });
+            const cacheToWrite = {
+                ...this.cache,
+                caches: sortedCaches
+            };
+            const yamlData = js_yaml.dump(cacheToWrite, {
+                lineWidth: -1
+            });
+            writeFileSync(this.cacheFilePath, yamlData);
+            debug('cache flushed to file: %s', this.cacheFilePath);
+        } catch (err) {
+            debug('write cache to file failed, path: %s, error: %s', this.cacheFilePath, err);
+        }
+    }
+    updateOrAppendCacheRecord(newRecord, cachedRecord) {
+        if (cachedRecord) if ('plan' === newRecord.type) cachedRecord.updateFn((cache)=>{
+            cache.yamlWorkflow = newRecord.yamlWorkflow;
+        });
+        else cachedRecord.updateFn((cache)=>{
+            const locateCache = cache;
+            locateCache.cache = newRecord.cache;
+            if ('xpaths' in locateCache) locateCache.xpaths = void 0;
+        });
+        else this.appendCache(newRecord);
+    }
+    constructor(cacheId, isCacheResultUsed, cacheFilePath, options = {}){
+        _define_property(this, "cacheId", void 0);
+        _define_property(this, "cacheFilePath", void 0);
+        _define_property(this, "cache", void 0);
+        _define_property(this, "isCacheResultUsed", void 0);
+        _define_property(this, "cacheOriginalLength", void 0);
+        _define_property(this, "readOnlyMode", void 0);
+        _define_property(this, "writeOnlyMode", void 0);
+        _define_property(this, "matchedCacheIndices", new Set());
+        node_assert(cacheId, 'cacheId is required');
+        let safeCacheId = replaceIllegalPathCharsAndSpace(cacheId);
+        const cacheMaxFilenameLength = globalConfigManager.getEnvConfigValueAsNumber(MIDSCENE_CACHE_MAX_FILENAME_LENGTH) ?? DEFAULT_CACHE_MAX_FILENAME_LENGTH;
+        if (Buffer.byteLength(safeCacheId, 'utf8') > cacheMaxFilenameLength) {
+            const prefix = safeCacheId.slice(0, 32);
+            const hash = generateHashId(void 0, safeCacheId);
+            safeCacheId = `${prefix}-${hash}`;
+        }
+        this.cacheId = safeCacheId;
+        this.cacheFilePath = ifInBrowser || ifInWorker ? void 0 : cacheFilePath || join(getMidsceneRunSubDir('cache'), `${this.cacheId}${cacheFileExt}`);
+        const readOnlyMode = Boolean(options?.readOnly);
+        const writeOnlyMode = Boolean(options?.writeOnly);
+        if (readOnlyMode && writeOnlyMode) throw new Error('TaskCache cannot be both read-only and write-only');
+        this.isCacheResultUsed = writeOnlyMode ? false : isCacheResultUsed;
+        this.readOnlyMode = readOnlyMode;
+        this.writeOnlyMode = writeOnlyMode;
+        let cacheContent;
+        if (this.cacheFilePath && !this.writeOnlyMode) cacheContent = this.loadCacheFromFile();
+        if (!cacheContent) cacheContent = {
+            midsceneVersion: getMidsceneVersion(),
+            cacheId: this.cacheId,
+            caches: []
+        };
+        this.cache = cacheContent;
+        this.cacheOriginalLength = this.isCacheResultUsed ? this.cache.caches.length : 0;
+    }
+}
+export { TaskCache, cacheFileExt, debug };
+
+//# sourceMappingURL=task-cache.mjs.map

package/dist/es/agent/task-cache.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"agent/task-cache.mjs","sources":["../../../src/agent/task-cache.ts"],"sourcesContent":["import assert from 'node:assert';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { dirname, join } from 'node:path';\nimport { isDeepStrictEqual } from 'node:util';\nimport type { TUserPrompt } from '@/ai-model';\nimport type { ElementCacheFeature } from '@/types';\nimport { getMidsceneRunSubDir } from '@midscene/shared/common';\nimport {\n  MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { getDebug } from '@midscene/shared/logger';\nimport { ifInBrowser, ifInWorker } from '@midscene/shared/utils';\nimport { generateHashId } from '@midscene/shared/utils';\nimport { replaceIllegalPathCharsAndSpace } from '@midscene/shared/utils';\nimport yaml from 'js-yaml';\nimport semver from 'semver';\nimport { getMidsceneVersion } from './utils';\n\nconst DEFAULT_CACHE_MAX_FILENAME_LENGTH = 200;\n\nexport const debug = getDebug('cache');\n\nexport interface PlanningCache {\n  type: 'plan';\n  prompt: string;\n  yamlWorkflow: string;\n}\n\nexport interface LocateCache {\n  type: 'locate';\n  prompt: TUserPrompt;\n  cache?: ElementCacheFeature;\n  /** @deprecated kept for backward compatibility */\n  xpaths?: string[];\n}\n\nexport interface MatchCacheResult<T extends PlanningCache | LocateCache> {\n  cacheContent: T;\n  updateFn: (cb: (cache: T) => void) => void;\n}\n\nexport type CacheFileContent = {\n  midsceneVersion: string;\n  cacheId: string;\n  caches: Array<PlanningCache | LocateCache>;\n};\n\nconst lowestSupportedMidsceneVersion = '0.16.10';\nexport const cacheFileExt = '.cache.yaml';\n\nexport class TaskCache {\n  cacheId: string;\n\n  cacheFilePath?: string;\n\n  cache: CacheFileContent;\n\n  isCacheResultUsed: boolean; // a flag to indicate if the cache result should be used\n  cacheOriginalLength: number;\n\n  readOnlyMode: boolean; // a flag to indicate if the cache is in read-only mode\n\n  writeOnlyMode: boolean; // a flag to indicate if the cache is in write-only mode\n\n  private matchedCacheIndices: Set<string> = new Set(); // Track matched records\n\n  constructor(\n    cacheId: string,\n    isCacheResultUsed: boolean,\n    cacheFilePath?: string,\n    options: { readOnly?: boolean; writeOnly?: boolean } = {},\n  ) {\n    assert(cacheId, 'cacheId is required');\n    let safeCacheId = replaceIllegalPathCharsAndSpace(cacheId);\n    const cacheMaxFilenameLength =\n      globalConfigManager.getEnvConfigValueAsNumber(\n        MIDSCENE_CACHE_MAX_FILENAME_LENGTH,\n      ) ?? DEFAULT_CACHE_MAX_FILENAME_LENGTH;\n    if (Buffer.byteLength(safeCacheId, 'utf8') > cacheMaxFilenameLength) {\n      const prefix = safeCacheId.slice(0, 32);\n      const hash = generateHashId(undefined, safeCacheId);\n      safeCacheId = `${prefix}-${hash}`;\n    }\n    this.cacheId = safeCacheId;\n\n    this.cacheFilePath =\n      ifInBrowser || ifInWorker\n        ? undefined\n        : cacheFilePath ||\n          join(getMidsceneRunSubDir('cache'), `${this.cacheId}${cacheFileExt}`);\n    const readOnlyMode = Boolean(options?.readOnly);\n    const writeOnlyMode = Boolean(options?.writeOnly);\n\n    if (readOnlyMode && writeOnlyMode) {\n      throw new Error('TaskCache cannot be both read-only and write-only');\n    }\n\n    this.isCacheResultUsed = writeOnlyMode ? false : isCacheResultUsed;\n    this.readOnlyMode = readOnlyMode;\n    this.writeOnlyMode = writeOnlyMode;\n\n    let cacheContent;\n    if (this.cacheFilePath && !this.writeOnlyMode) {\n      cacheContent = this.loadCacheFromFile();\n    }\n    if (!cacheContent) {\n      cacheContent = {\n        midsceneVersion: getMidsceneVersion(),\n        cacheId: this.cacheId,\n        caches: [],\n      };\n    }\n    this.cache = cacheContent;\n    this.cacheOriginalLength = this.isCacheResultUsed\n      ? this.cache.caches.length\n      : 0;\n  }\n\n  matchCache(\n    prompt: TUserPrompt,\n    type: 'plan' | 'locate',\n  ): MatchCacheResult<PlanningCache | LocateCache> | undefined {\n    if (!this.isCacheResultUsed) {\n      return undefined;\n    }\n    // Find the first unused matching cache\n    const promptStr =\n      typeof prompt === 'string' ? prompt : JSON.stringify(prompt);\n    for (let i = 0; i < this.cacheOriginalLength; i++) {\n      const item = this.cache.caches[i];\n      const key = `${type}:${promptStr}:${i}`;\n      if (\n        item.type === type &&\n        isDeepStrictEqual(item.prompt, prompt) &&\n        !this.matchedCacheIndices.has(key)\n      ) {\n        if (item.type === 'locate') {\n          const locateItem = item as LocateCache;\n          if (!locateItem.cache && Array.isArray(locateItem.xpaths)) {\n            locateItem.cache = { xpaths: locateItem.xpaths };\n          }\n          if ('xpaths' in locateItem) {\n            locateItem.xpaths = undefined;\n          }\n        }\n        this.matchedCacheIndices.add(key);\n        debug(\n          'cache found and marked as used, type: %s, prompt: %s, index: %d',\n          type,\n          prompt,\n          i,\n        );\n        return {\n          cacheContent: item,\n          updateFn: (cb: (cache: PlanningCache | LocateCache) => void) => {\n            debug(\n              'will call updateFn to update cache, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            cb(item);\n\n            if (this.readOnlyMode) {\n              debug(\n                'read-only mode, cache updated in memory but not flushed to file',\n              );\n              return;\n            }\n\n            debug(\n              'cache updated, will flush to file, type: %s, prompt: %s, index: %d',\n              type,\n              prompt,\n              i,\n            );\n            this.flushCacheToFile();\n          },\n        };\n      }\n    }\n    debug('no unused cache found, type: %s, prompt: %s', type, prompt);\n    return undefined;\n  }\n\n  matchPlanCache(prompt: string): MatchCacheResult<PlanningCache> | undefined {\n    return this.matchCache(prompt, 'plan') as\n      | MatchCacheResult<PlanningCache>\n      | undefined;\n  }\n\n  matchLocateCache(\n    prompt: TUserPrompt,\n  ): MatchCacheResult<LocateCache> | undefined {\n    return this.matchCache(prompt, 'locate') as\n      | MatchCacheResult<LocateCache>\n      | undefined;\n  }\n\n  appendCache(cache: PlanningCache | LocateCache) {\n    debug('will append cache', cache);\n    this.cache.caches.push(cache);\n\n    if (this.readOnlyMode) {\n      debug('read-only mode, cache appended to memory but not flushed to file');\n      return;\n    }\n\n    this.flushCacheToFile();\n  }\n\n  loadCacheFromFile() {\n    const cacheFile = this.cacheFilePath;\n    assert(cacheFile, 'cache file path is required');\n\n    if (!existsSync(cacheFile)) {\n      debug('no cache file found, path: %s', cacheFile);\n      return undefined;\n    }\n\n    // detect old cache file\n    const jsonTypeCacheFile = cacheFile.replace(cacheFileExt, '.json');\n    if (existsSync(jsonTypeCacheFile) && this.isCacheResultUsed) {\n      console.warn(\n        `An outdated cache file from an earlier version of Midscene has been detected. Since version 0.17, we have implemented an improved caching strategy. Please delete the old file located at: ${jsonTypeCacheFile}.`,\n      );\n      return undefined;\n    }\n\n    try {\n      const data = readFileSync(cacheFile, 'utf8');\n      const jsonData = yaml.load(data) as CacheFileContent;\n\n      const version = getMidsceneVersion();\n      if (!version) {\n        debug('no midscene version info, will not read cache from file');\n        return undefined;\n      }\n\n      if (\n        semver.lt(jsonData.midsceneVersion, lowestSupportedMidsceneVersion) &&\n        !jsonData.midsceneVersion.includes('beta') // for internal test\n      ) {\n        console.warn(\n          `You are using an old version of Midscene cache file, and we cannot match any info from it. Starting from Midscene v0.17, we changed our strategy to use xpath for cache info, providing better performance.\\nPlease delete the existing cache and rebuild it. Sorry for the inconvenience.\\ncache file: ${cacheFile}`,\n        );\n        return undefined;\n      }\n\n      debug(\n        'cache loaded from file, path: %s, cache version: %s, record length: %s',\n        cacheFile,\n        jsonData.midsceneVersion,\n        jsonData.caches.length,\n      );\n      jsonData.midsceneVersion = getMidsceneVersion(); // update the version\n      return jsonData;\n    } catch (err) {\n      debug(\n        'cache file exists but load failed, path: %s, error: %s',\n        cacheFile,\n        err,\n      );\n      return undefined;\n    }\n  }\n\n  flushCacheToFile(options?: { cleanUnused?: boolean }) {\n    const version = getMidsceneVersion();\n    if (!version) {\n      debug('no midscene version info, will not write cache to file');\n      return;\n    }\n\n    if (!this.cacheFilePath) {\n      debug('no cache file path, will not write cache to file');\n      return;\n    }\n\n    // Clean unused caches if requested\n    if (options?.cleanUnused) {\n      // Skip cleaning in write-only mode or when cache is not used\n      if (this.isCacheResultUsed) {\n        const originalLength = this.cache.caches.length;\n\n        // Collect indices of used caches\n        const usedIndices = new Set<number>();\n        for (const key of this.matchedCacheIndices) {\n          // key format: \"type:prompt:index\"\n          const parts = key.split(':');\n          const index = Number.parseInt(parts[parts.length - 1], 10);\n          if (!Number.isNaN(index)) {\n            usedIndices.add(index);\n          }\n        }\n\n        // Filter: keep used caches and newly added caches\n        this.cache.caches = this.cache.caches.filter((_, index) => {\n          const isUsed = usedIndices.has(index);\n          const isNew = index >= this.cacheOriginalLength;\n          return isUsed || isNew;\n        });\n\n        const removedCount = originalLength - this.cache.caches.length;\n        if (removedCount > 0) {\n          debug('cleaned %d unused cache record(s)', removedCount);\n        } else {\n          debug('no unused cache to clean');\n        }\n      } else {\n        debug('skip cleaning: cache is not used for reading');\n      }\n    }\n\n    try {\n      const dir = dirname(this.cacheFilePath);\n      if (!existsSync(dir)) {\n        mkdirSync(dir, { recursive: true });\n        debug('created cache directory: %s', dir);\n      }\n\n      // Sort caches to ensure plan entries come before locate entries for better readability\n      // Create a sorted copy for writing to disk while keeping in-memory order unchanged\n      const sortedCaches = [...this.cache.caches].sort((a, b) => {\n        if (a.type === 'plan' && b.type === 'locate') return -1;\n        if (a.type === 'locate' && b.type === 'plan') return 1;\n        return 0;\n      });\n\n      const cacheToWrite = {\n        ...this.cache,\n        caches: sortedCaches,\n      };\n\n      const yamlData = yaml.dump(cacheToWrite, { lineWidth: -1 });\n      writeFileSync(this.cacheFilePath, yamlData);\n      debug('cache flushed to file: %s', this.cacheFilePath);\n    } catch (err) {\n      debug(\n        'write cache to file failed, path: %s, error: %s',\n        this.cacheFilePath,\n        err,\n      );\n    }\n  }\n\n  updateOrAppendCacheRecord(\n    newRecord: PlanningCache | LocateCache,\n    cachedRecord?: MatchCacheResult<PlanningCache | LocateCache>,\n  ) {\n    if (cachedRecord) {\n      // update existing record\n      if (newRecord.type === 'plan') {\n        cachedRecord.updateFn((cache) => {\n          (cache as PlanningCache).yamlWorkflow = newRecord.yamlWorkflow;\n        });\n      } else {\n        cachedRecord.updateFn((cache) => {\n          const locateCache = cache as LocateCache;\n          locateCache.cache = newRecord.cache;\n          if ('xpaths' in locateCache) {\n            locateCache.xpaths = undefined;\n          }\n        });\n      }\n    } else {\n      this.appendCache(newRecord);\n    }\n  }\n}\n"],"names":["DEFAULT_CACHE_MAX_FILENAME_LENGTH","debug","getDebug","lowestSupportedMidsceneVersion","cacheFileExt","TaskCache","prompt","type","promptStr","JSON","i","item","key","isDeepStrictEqual","locateItem","Array","undefined","cb","cache","cacheFile","assert","existsSync","jsonTypeCacheFile","console","data","readFileSync","jsonData","yaml","version","getMidsceneVersion","semver","err","options","originalLength","usedIndices","Set","parts","index","Number","_","isUsed","isNew","removedCount","dir","dirname","mkdirSync","sortedCaches","a","b","cacheToWrite","yamlData","writeFileSync","newRecord","cachedRecord","locateCache","cacheId","isCacheResultUsed","cacheFilePath","safeCacheId","replaceIllegalPathCharsAndSpace","cacheMaxFilenameLength","globalConfigManager","MIDSCENE_CACHE_MAX_FILENAME_LENGTH","Buffer","prefix","hash","generateHashId","ifInBrowser","ifInWorker","join","getMidsceneRunSubDir","readOnlyMode","Boolean","writeOnlyMode","Error","cacheContent"],"mappings":";;;;;;;;;;;;;;;;;;;;;AAmBA,MAAMA,oCAAoC;AAEnC,MAAMC,QAAQC,SAAS;AA2B9B,MAAMC,iCAAiC;AAChC,MAAMC,eAAe;AAErB,MAAMC;IAoEX,WACEC,MAAmB,EACnBC,IAAuB,EACoC;QAC3D,IAAI,CAAC,IAAI,CAAC,iBAAiB,EACzB;QAGF,MAAMC,YACJ,AAAkB,YAAlB,OAAOF,SAAsBA,SAASG,KAAK,SAAS,CAACH;QACvD,IAAK,IAAII,IAAI,GAAGA,IAAI,IAAI,CAAC,mBAAmB,EAAEA,IAAK;YACjD,MAAMC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAACD,EAAE;YACjC,MAAME,MAAM,GAAGL,KAAK,CAAC,EAAEC,UAAU,CAAC,EAAEE,GAAG;YACvC,IACEC,KAAK,IAAI,KAAKJ,QACdM,kBAAkBF,KAAK,MAAM,EAAEL,WAC/B,CAAC,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACM,MAC9B;gBACA,IAAID,AAAc,aAAdA,KAAK,IAAI,EAAe;oBAC1B,MAAMG,aAAaH;oBACnB,IAAI,CAACG,WAAW,KAAK,IAAIC,MAAM,OAAO,CAACD,WAAW,MAAM,GACtDA,WAAW,KAAK,GAAG;wBAAE,QAAQA,WAAW,MAAM;oBAAC;oBAEjD,IAAI,YAAYA,YACdA,WAAW,MAAM,GAAGE;gBAExB;gBACA,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAACJ;gBAC7BX,MACE,mEACAM,MACAD,QACAI;gBAEF,OAAO;oBACL,cAAcC;oBACd,UAAU,CAACM;wBACThB,MACE,uEACAM,MACAD,QACAI;wBAEFO,GAAGN;wBAEH,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBV,MACE;wBAKJA,MACE,sEACAM,MACAD,QACAI;wBAEF,IAAI,CAAC,gBAAgB;oBACvB;gBACF;YACF;QACF;QACAT,MAAM,+CAA+CM,MAAMD;IAE7D;IAEA,eAAeA,MAAc,EAA+C;QAC1E,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,iBACEA,MAAmB,EACwB;QAC3C,OAAO,IAAI,CAAC,UAAU,CAACA,QAAQ;IAGjC;IAEA,YAAYY,KAAkC,EAAE;QAC9CjB,MAAM,qBAAqBiB;QAC3B,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAACA;QAEvB,IAAI,IAAI,CAAC,YAAY,EAAE,YACrBjB,MAAM;QAIR,IAAI,CAAC,gBAAgB;IACvB;IAEA,oBAAoB;QAClB,MAAMkB,YAAY,IAAI,CAAC,aAAa;QACpCC,YAAOD,WAAW;QAElB,IAAI,CAACE,WAAWF,YAAY,YAC1BlB,MAAM,iCAAiCkB;QAKzC,MAAMG,oBAAoBH,UAAU,OAAO,CAACf,cAAc;QAC1D,IAAIiB,WAAWC,sBAAsB,IAAI,CAAC,iBAAiB,EAAE,YAC3DC,QAAQ,IAAI,CACV,CAAC,2LAA2L,EAAED,kBAAkB,CAAC,CAAC;QAKtN,IAAI;YACF,MAAME,OAAOC,aAAaN,WAAW;YACrC,MAAMO,WAAWC,QAAAA,IAAS,CAACH;YAE3B,MAAMI,UAAUC;YAChB,IAAI,CAACD,SAAS,YACZ3B,MAAM;YAIR,IACE6B,OAAO,EAAE,CAACJ,SAAS,eAAe,EAAEvB,mCACpC,CAACuB,SAAS,eAAe,CAAC,QAAQ,CAAC,SACnC,YACAH,QAAQ,IAAI,CACV,CAAC,wSAAwS,EAAEJ,WAAW;YAK1TlB,MACE,0EACAkB,WACAO,SAAS,eAAe,EACxBA,SAAS,MAAM,CAAC,MAAM;YAExBA,SAAS,eAAe,GAAGG;YAC3B,OAAOH;QACT,EAAE,OAAOK,KAAK;YACZ9B,MACE,0DACAkB,WACAY;YAEF;QACF;IACF;IAEA,iBAAiBC,OAAmC,EAAE;QACpD,MAAMJ,UAAUC;QAChB,IAAI,CAACD,SAAS,YACZ3B,MAAM;QAIR,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,YACvBA,MAAM;QAKR,IAAI+B,SAAS,aAEX,IAAI,IAAI,CAAC,iBAAiB,EAAE;YAC1B,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;YAG/C,MAAMC,cAAc,IAAIC;YACxB,KAAK,MAAMvB,OAAO,IAAI,CAAC,mBAAmB,CAAE;gBAE1C,MAAMwB,QAAQxB,IAAI,KAAK,CAAC;gBACxB,MAAMyB,QAAQC,OAAO,QAAQ,CAACF,KAAK,CAACA,MAAM,MAAM,GAAG,EAAE,EAAE;gBACvD,IAAI,CAACE,OAAO,KAAK,CAACD,QAChBH,YAAY,GAAG,CAACG;YAEpB;YAGA,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAACE,GAAGF;gBAC/C,MAAMG,SAASN,YAAY,GAAG,CAACG;gBAC/B,MAAMI,QAAQJ,SAAS,IAAI,CAAC,mBAAmB;gBAC/C,OAAOG,UAAUC;YACnB;YAEA,MAAMC,eAAeT,iBAAiB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM;YAC1DS,eAAe,IACjBzC,MAAM,qCAAqCyC,gBAE3CzC,MAAM;QAEV,OACEA,MAAM;QAIV,IAAI;YACF,MAAM0C,MAAMC,QAAQ,IAAI,CAAC,aAAa;YACtC,IAAI,CAACvB,WAAWsB,MAAM;gBACpBE,UAAUF,KAAK;oBAAE,WAAW;gBAAK;gBACjC1C,MAAM,+BAA+B0C;YACvC;YAIA,MAAMG,eAAe;mBAAI,IAAI,CAAC,KAAK,CAAC,MAAM;aAAC,CAAC,IAAI,CAAC,CAACC,GAAGC;gBACnD,IAAID,AAAW,WAAXA,EAAE,IAAI,IAAeC,AAAW,aAAXA,EAAE,IAAI,EAAe,OAAO;gBACrD,IAAID,AAAW,aAAXA,EAAE,IAAI,IAAiBC,AAAW,WAAXA,EAAE,IAAI,EAAa,OAAO;gBACrD,OAAO;YACT;YAEA,MAAMC,eAAe;gBACnB,GAAG,IAAI,CAAC,KAAK;gBACb,QAAQH;YACV;YAEA,MAAMI,WAAWvB,QAAAA,IAAS,CAACsB,cAAc;gBAAE,WAAW;YAAG;YACzDE,cAAc,IAAI,CAAC,aAAa,EAAED;YAClCjD,MAAM,6BAA6B,IAAI,CAAC,aAAa;QACvD,EAAE,OAAO8B,KAAK;YACZ9B,MACE,mDACA,IAAI,CAAC,aAAa,EAClB8B;QAEJ;IACF;IAEA,0BACEqB,SAAsC,EACtCC,YAA4D,EAC5D;QACA,IAAIA,cAEF,IAAID,AAAmB,WAAnBA,UAAU,IAAI,EAChBC,aAAa,QAAQ,CAAC,CAACnC;YACpBA,MAAwB,YAAY,GAAGkC,UAAU,YAAY;QAChE;aAEAC,aAAa,QAAQ,CAAC,CAACnC;YACrB,MAAMoC,cAAcpC;YACpBoC,YAAY,KAAK,GAAGF,UAAU,KAAK;YACnC,IAAI,YAAYE,aACdA,YAAY,MAAM,GAAGtC;QAEzB;aAGF,IAAI,CAAC,WAAW,CAACoC;IAErB;IA9SA,YACEG,OAAe,EACfC,iBAA0B,EAC1BC,aAAsB,EACtBzB,UAAuD,CAAC,CAAC,CACzD;QApBF;QAEA;QAEA;QAEA;QACA;QAEA;QAEA;QAEA,uBAAQ,uBAAmC,IAAIG;QAQ7Cf,YAAOmC,SAAS;QAChB,IAAIG,cAAcC,gCAAgCJ;QAClD,MAAMK,yBACJC,oBAAoB,yBAAyB,CAC3CC,uCACG9D;QACP,IAAI+D,OAAO,UAAU,CAACL,aAAa,UAAUE,wBAAwB;YACnE,MAAMI,SAASN,YAAY,KAAK,CAAC,GAAG;YACpC,MAAMO,OAAOC,eAAelD,QAAW0C;YACvCA,cAAc,GAAGM,OAAO,CAAC,EAAEC,MAAM;QACnC;QACA,IAAI,CAAC,OAAO,GAAGP;QAEf,IAAI,CAAC,aAAa,GAChBS,eAAeC,aACXpD,SACAyC,iBACAY,KAAKC,qBAAqB,UAAU,GAAG,IAAI,CAAC,OAAO,GAAGlE,cAAc;QAC1E,MAAMmE,eAAeC,QAAQxC,SAAS;QACtC,MAAMyC,gBAAgBD,QAAQxC,SAAS;QAEvC,IAAIuC,gBAAgBE,eAClB,MAAM,IAAIC,MAAM;QAGlB,IAAI,CAAC,iBAAiB,GAAGD,gBAAgB,QAAQjB;QACjD,IAAI,CAAC,YAAY,GAAGe;QACpB,IAAI,CAAC,aAAa,GAAGE;QAErB,IAAIE;QACJ,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,aAAa,EAC3CA,eAAe,IAAI,CAAC,iBAAiB;QAEvC,IAAI,CAACA,cACHA,eAAe;YACb,iBAAiB9C;YACjB,SAAS,IAAI,CAAC,OAAO;YACrB,QAAQ,EAAE;QACZ;QAEF,IAAI,CAAC,KAAK,GAAG8C;QACb,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,iBAAiB,GAC7C,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GACxB;IACN;AA6PF"}

package/dist/es/agent/tasks.mjs

@@ -0,0 +1,422 @@
+import { AIResponseParseError, ConversationHistory, autoGLMPlanning, plan, uiTarsPlanning } from "../ai-model/index.mjs";
+import { isAutoGLM, isUITars } from "../ai-model/auto-glm/util.mjs";
+import { getReadableTimeString } from "../common.mjs";
+import { TaskExecutionError } from "../task-runner.mjs";
+import { ServiceError } from "../types.mjs";
+import { getCurrentTime } from "@midscene/shared/env";
+import { getDebug } from "@midscene/shared/logger";
+import { assert } from "@midscene/shared/utils";
+import { ExecutionSession } from "./execution-session.mjs";
+import { TaskBuilder, locatePlanForLocate } from "./task-builder.mjs";
+import { setTimingFieldOnce } from "../task-timing.mjs";
+import { descriptionOfTree } from "@midscene/shared/extractor";
+import { taskTitleStr } from "./ui-utils.mjs";
+import { parsePrompt } from "./utils.mjs";
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+const debug = getDebug('device-task-executor');
+const maxErrorCountAllowedInOnePlanningLoop = 5;
+class TaskExecutor {
+    get page() {
+        return this.interface;
+    }
+    createExecutionSession(title, options) {
+        return new ExecutionSession(title, ()=>Promise.resolve(this.service.contextRetrieverFn()), {
+            onTaskStart: this.onTaskStartCallback,
+            tasks: options?.tasks,
+            onTaskUpdate: this.hooks?.onTaskUpdate
+        });
+    }
+    getActionSpace() {
+        return this.providedActionSpace;
+    }
+    async getTimeString(format) {
+        const timestamp = await getCurrentTime(this.interface, this.useDeviceTimestamp);
+        return getReadableTimeString(format, timestamp);
+    }
+    async convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options) {
+        return this.taskBuilder.build(plans, modelConfigForPlanning, modelConfigForDefaultIntent, options);
+    }
+    async loadYamlFlowAsPlanning(userInstruction, yamlString) {
+        const session = this.createExecutionSession(taskTitleStr('Act', userInstruction));
+        const task = {
+            type: 'Planning',
+            subType: 'LoadYaml',
+            param: {
+                userInstruction
+            },
+            executor: async (param, executorContext)=>{
+                const { uiContext } = executorContext;
+                assert(uiContext, 'uiContext is required for Planning task');
+                return {
+                    output: {
+                        actions: [],
+                        shouldContinuePlanning: false,
+                        log: '',
+                        yamlString
+                    },
+                    cache: {
+                        hit: true
+                    },
+                    hitBy: {
+                        from: 'Cache',
+                        context: {
+                            yamlString
+                        }
+                    }
+                };
+            }
+        };
+        const runner = session.getRunner();
+        await session.appendAndRun(task);
+        return {
+            runner
+        };
+    }
+    async runPlans(title, plans, modelConfigForPlanning, modelConfigForDefaultIntent) {
+        const session = this.createExecutionSession(title);
+        const { tasks } = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent);
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(tasks);
+        const { output } = result ?? {};
+        return {
+            output,
+            runner
+        };
+    }
+    async action(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, fileChooserAccept, deepLocate, abortSignal) {
+        return withFileChooser(this.interface, fileChooserAccept, async ()=>this.runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal));
+    }
+    async runAction(userPrompt, modelConfigForPlanning, modelConfigForDefaultIntent, includeBboxInPlanning, aiActContext, cacheable, replanningCycleLimitOverride, imagesIncludeCount, deepThink, deepLocate, abortSignal) {
+        this.conversationHistory.reset();
+        const session = this.createExecutionSession(taskTitleStr('Act', userPrompt));
+        const runner = session.getRunner();
+        let replanCount = 0;
+        const yamlFlow = [];
+        const replanningCycleLimit = replanningCycleLimitOverride ?? this.replanningCycleLimit;
+        assert(void 0 !== replanningCycleLimit, 'replanningCycleLimit is required for TaskExecutor.action');
+        let errorCountInOnePlanningLoop = 0;
+        let outputString;
+        while(true){
+            if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
+            const subGoalStatus = this.conversationHistory.subGoalsToText() || void 0;
+            const memoriesStatus = this.conversationHistory.memoriesToText() || void 0;
+            const result = await session.appendAndRun({
+                type: 'Planning',
+                subType: 'Plan',
+                param: {
+                    userInstruction: userPrompt,
+                    aiActContext,
+                    imagesIncludeCount,
+                    deepThink,
+                    ...subGoalStatus ? {
+                        subGoalStatus
+                    } : {},
+                    ...memoriesStatus ? {
+                        memoriesStatus
+                    } : {}
+                },
+                executor: async (param, executorContext)=>{
+                    const { uiContext } = executorContext;
+                    assert(uiContext, 'uiContext is required for Planning task');
+                    const { modelFamily } = modelConfigForPlanning;
+                    const timing = executorContext.task.timing;
+                    const actionSpace = this.getActionSpace();
+                    debug('actionSpace for this interface is:', actionSpace.map((action)=>action.name).join(', '));
+                    assert(Array.isArray(actionSpace), 'actionSpace must be an array');
+                    if (0 === actionSpace.length) console.warn(`ActionSpace for ${this.interface.interfaceType} is empty. This may lead to unexpected behavior.`);
+                    const planImpl = isUITars(modelFamily) ? uiTarsPlanning : isAutoGLM(modelFamily) ? autoGLMPlanning : plan;
+                    let planResult;
+                    try {
+                        setTimingFieldOnce(timing, 'callAiStart');
+                        planResult = await planImpl(param.userInstruction, {
+                            context: uiContext,
+                            actionContext: param.aiActContext,
+                            interfaceType: this.interface.interfaceType,
+                            actionSpace,
+                            modelConfig: modelConfigForPlanning,
+                            conversationHistory: this.conversationHistory,
+                            includeBbox: includeBboxInPlanning,
+                            imagesIncludeCount,
+                            deepThink,
+                            abortSignal
+                        });
+                    } catch (planError) {
+                        if (planError instanceof AIResponseParseError) {
+                            executorContext.task.usage = planError.usage;
+                            executorContext.task.log = {
+                                ...executorContext.task.log || {},
+                                rawResponse: planError.rawResponse
+                            };
+                        }
+                        throw planError;
+                    } finally{
+                        setTimingFieldOnce(timing, 'callAiEnd');
+                    }
+                    debug('planResult', JSON.stringify(planResult, null, 2));
+                    const { actions, thought, log, memory, error, usage, rawResponse, reasoning_content, finalizeSuccess, finalizeMessage, updateSubGoals, markFinishedIndexes } = planResult;
+                    outputString = finalizeMessage;
+                    executorContext.task.log = {
+                        ...executorContext.task.log || {},
+                        rawResponse
+                    };
+                    executorContext.task.usage = usage;
+                    executorContext.task.reasoning_content = reasoning_content;
+                    executorContext.task.output = {
+                        actions: actions || [],
+                        log,
+                        thought,
+                        memory,
+                        yamlFlow: planResult.yamlFlow,
+                        output: finalizeMessage,
+                        shouldContinuePlanning: planResult.shouldContinuePlanning,
+                        updateSubGoals,
+                        markFinishedIndexes
+                    };
+                    executorContext.uiContext = uiContext;
+                    assert(!error, `Failed to continue: ${error}\n${log || ''}`);
+                    if (false === finalizeSuccess) assert(false, `Task failed: ${finalizeMessage || 'No error message provided'}\n${log || ''}`);
+                    return {
+                        cache: {
+                            hit: false
+                        }
+                    };
+                }
+            }, {
+                allowWhenError: true
+            });
+            const planResult = result?.output;
+            const plans = planResult?.actions || [];
+            yamlFlow.push(...planResult?.yamlFlow || []);
+            let executables;
+            try {
+                executables = await this.convertPlanToExecutable(plans, modelConfigForPlanning, modelConfigForDefaultIntent, {
+                    cacheable,
+                    deepLocate,
+                    abortSignal
+                });
+            } catch (error) {
+                return session.appendErrorPlan(`Error converting plans to executable tasks: ${error}, plans: ${JSON.stringify(plans)}`);
+            }
+            if (this.conversationHistory.pendingFeedbackMessage) console.warn('unconsumed pending feedback message detected, this may lead to unexpected planning result:', this.conversationHistory.pendingFeedbackMessage);
+            const initialTimeString = await this.getTimeString();
+            this.conversationHistory.pendingFeedbackMessage += `Current time: ${initialTimeString}`;
+            try {
+                await session.appendAndRun(executables.tasks);
+            } catch (error) {
+                errorCountInOnePlanningLoop++;
+                const timeString = await this.getTimeString();
+                this.conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, Error executing running tasks: ${error?.message || String(error)}`;
+                debug('error when executing running tasks, but continue to run if it is not too many errors:', error instanceof Error ? error.message : String(error), 'current error count in one planning loop:', errorCountInOnePlanningLoop);
+            }
+            if (errorCountInOnePlanningLoop > maxErrorCountAllowedInOnePlanningLoop) return session.appendErrorPlan('Too many errors in one planning loop');
+            if (abortSignal?.aborted) return session.appendErrorPlan(`Task aborted: ${abortSignal.reason || 'abort signal received'}`);
+            if (!planResult?.shouldContinuePlanning) break;
+            ++replanCount;
+            if (replanCount > replanningCycleLimit) {
+                const errorMsg = `Replanned ${replanningCycleLimit} times, exceeding the limit. Please configure a larger value for replanningCycleLimit (or use MIDSCENE_REPLANNING_CYCLE_LIMIT) to handle more complex tasks.`;
+                return session.appendErrorPlan(errorMsg);
+            }
+            if (!this.conversationHistory.pendingFeedbackMessage) {
+                const timeString = await this.getTimeString();
+                this.conversationHistory.pendingFeedbackMessage = `Time: ${timeString}, I have finished the action previously planned.`;
+            }
+        }
+        return {
+            output: {
+                yamlFlow,
+                output: outputString
+            },
+            runner
+        };
+    }
+    createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt) {
+        const queryTask = {
+            type: 'Insight',
+            subType: type,
+            param: {
+                dataDemand: multimodalPrompt ? {
+                    demand,
+                    multimodalPrompt
+                } : demand
+            },
+            executor: async (param, taskContext)=>{
+                const { task } = taskContext;
+                let queryDump;
+                const applyDump = (dump)=>{
+                    queryDump = dump;
+                    task.log = {
+                        dump,
+                        rawResponse: dump.taskInfo?.rawResponse
+                    };
+                    task.usage = dump.taskInfo?.usage;
+                    if (dump.taskInfo?.reasoning_content) task.reasoning_content = dump.taskInfo.reasoning_content;
+                };
+                const uiContext = taskContext.uiContext;
+                assert(uiContext, 'uiContext is required for Query task');
+                const ifTypeRestricted = 'Query' !== type;
+                let demandInput = demand;
+                let keyOfResult = 'result';
+                if (ifTypeRestricted && ('Assert' === type || 'WaitFor' === type)) {
+                    keyOfResult = 'StatementIsTruthy';
+                    const booleanPrompt = 'Assert' === type ? `Boolean, whether the following statement is true: ${demand}` : `Boolean, the user wants to do some 'wait for' operation, please check whether the following statement is true: ${demand}`;
+                    demandInput = {
+                        [keyOfResult]: booleanPrompt
+                    };
+                } else if (ifTypeRestricted) demandInput = {
+                    [keyOfResult]: `${type}, ${demand}`
+                };
+                let extractResult;
+                let extraPageDescription = '';
+                if (opt?.domIncluded && this.interface.getElementsNodeTree) {
+                    debug('appending tree info for page');
+                    const tree = await this.interface.getElementsNodeTree();
+                    extraPageDescription = await descriptionOfTree(tree, 200, false, opt?.domIncluded === 'visible-only');
+                }
+                try {
+                    extractResult = await this.service.extract(demandInput, modelConfig, opt, extraPageDescription, multimodalPrompt, uiContext);
+                } catch (error) {
+                    if (error instanceof ServiceError) applyDump(error.dump);
+                    throw error;
+                }
+                const { data, thought, dump } = extractResult;
+                applyDump(dump);
+                let outputResult = data;
+                if (ifTypeRestricted) if ('string' == typeof data) outputResult = data;
+                else if ('WaitFor' === type) outputResult = null == data ? false : data[keyOfResult];
+                else if (null == data) outputResult = null;
+                else {
+                    assert(data?.[keyOfResult] !== void 0, 'No result in query data');
+                    outputResult = data[keyOfResult];
+                }
+                if ('Assert' === type && !outputResult) {
+                    task.thought = thought;
+                    throw new Error(`Assertion failed: ${thought}`);
+                }
+                return {
+                    output: outputResult,
+                    log: queryDump,
+                    thought
+                };
+            }
+        };
+        return queryTask;
+    }
+    async createTypeQueryExecution(type, demand, modelConfig, opt, multimodalPrompt) {
+        const session = this.createExecutionSession(taskTitleStr(type, 'string' == typeof demand ? demand : JSON.stringify(demand)));
+        const queryTask = await this.createTypeQueryTask(type, demand, modelConfig, opt, multimodalPrompt);
+        const runner = session.getRunner();
+        const result = await session.appendAndRun(queryTask);
+        if (!result) throw new Error('result of taskExecutor.flush() is undefined in function createTypeQueryTask');
+        const { output, thought } = result;
+        return {
+            output,
+            thought,
+            runner
+        };
+    }
+    async waitFor(assertion, opt, modelConfig) {
+        const { textPrompt, multimodalPrompt } = parsePrompt(assertion);
+        const description = `waitFor: ${textPrompt}`;
+        const session = this.createExecutionSession(taskTitleStr('WaitFor', description));
+        const runner = session.getRunner();
+        const { timeoutMs, checkIntervalMs, domIncluded, screenshotIncluded, ...restOpt } = opt;
+        const serviceExtractOpt = {
+            domIncluded,
+            screenshotIncluded,
+            ...restOpt
+        };
+        assert(assertion, 'No assertion for waitFor');
+        assert(timeoutMs, 'No timeoutMs for waitFor');
+        assert(checkIntervalMs, 'No checkIntervalMs for waitFor');
+        assert(checkIntervalMs <= timeoutMs, `wrong config for waitFor: checkIntervalMs must be less than timeoutMs, config: {checkIntervalMs: ${checkIntervalMs}, timeoutMs: ${timeoutMs}}`);
+        const overallStartTime = Date.now();
+        let lastCheckStart = overallStartTime;
+        let errorThought = '';
+        while(lastCheckStart - overallStartTime <= timeoutMs){
+            const currentCheckStart = Date.now();
+            lastCheckStart = currentCheckStart;
+            const queryTask = await this.createTypeQueryTask('WaitFor', textPrompt, modelConfig, serviceExtractOpt, multimodalPrompt);
+            const result = await session.appendAndRun(queryTask);
+            if (result?.output) return {
+                output: void 0,
+                runner
+            };
+            errorThought = result?.thought || !result && `No result from assertion: ${textPrompt}` || `unknown error when waiting for assertion: ${textPrompt}`;
+            const now = Date.now();
+            if (now - currentCheckStart < checkIntervalMs) {
+                const elapsed = now - currentCheckStart;
+                const timeRemaining = checkIntervalMs - elapsed;
+                const thought = `Check interval is ${checkIntervalMs}ms, ${elapsed}ms elapsed since last check, sleeping for ${timeRemaining}ms`;
+                const { tasks: sleepTasks } = await this.convertPlanToExecutable([
+                    {
+                        type: 'Sleep',
+                        param: {
+                            timeMs: timeRemaining
+                        },
+                        thought
+                    }
+                ], modelConfig, modelConfig);
+                if (sleepTasks[0]) await session.appendAndRun(sleepTasks[0]);
+            }
+        }
+        return session.appendErrorPlan(`waitFor timeout: ${errorThought}`);
+    }
+    constructor(interfaceInstance, service, opts){
+        _define_property(this, "interface", void 0);
+        _define_property(this, "service", void 0);
+        _define_property(this, "taskCache", void 0);
+        _define_property(this, "providedActionSpace", void 0);
+        _define_property(this, "taskBuilder", void 0);
+        _define_property(this, "conversationHistory", void 0);
+        _define_property(this, "onTaskStartCallback", void 0);
+        _define_property(this, "hooks", void 0);
+        _define_property(this, "replanningCycleLimit", void 0);
+        _define_property(this, "waitAfterAction", void 0);
+        _define_property(this, "useDeviceTimestamp", void 0);
+        this.interface = interfaceInstance;
+        this.service = service;
+        this.taskCache = opts.taskCache;
+        this.onTaskStartCallback = opts?.onTaskStart;
+        this.replanningCycleLimit = opts.replanningCycleLimit;
+        this.waitAfterAction = opts.waitAfterAction;
+        this.useDeviceTimestamp = opts.useDeviceTimestamp;
+        this.hooks = opts.hooks;
+        this.conversationHistory = new ConversationHistory();
+        this.providedActionSpace = opts.actionSpace;
+        this.taskBuilder = new TaskBuilder({
+            interfaceInstance,
+            service,
+            taskCache: opts.taskCache,
+            actionSpace: this.getActionSpace(),
+            waitAfterAction: opts.waitAfterAction
+        });
+    }
+}
+async function withFileChooser(interfaceInstance, fileChooserAccept, action) {
+    if (!fileChooserAccept?.length) return action();
+    if (!interfaceInstance.registerFileChooserListener) throw new Error(`File upload is not supported on ${interfaceInstance.interfaceType}`);
+    const handler = async (chooser)=>{
+        await chooser.accept(fileChooserAccept);
+    };
+    const { dispose, getError } = await interfaceInstance.registerFileChooserListener(handler);
+    try {
+        const result = await action();
+        const error = getError();
+        if (error) throw error;
+        return result;
+    } finally{
+        dispose();
+    }
+}
+export { TaskExecutionError, TaskExecutor, locatePlanForLocate, withFileChooser };
+
+//# sourceMappingURL=tasks.mjs.map