@midscene/core 0.26.7-beta-20250818035341.0 → 0.26.7-beta-20250820105545.0

package/dist/lib/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"utils.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { execSync } from 'node:child_process';\nimport * as fs from 'node:fs';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { tmpdir } from 'node:os';\nimport * as path from 'node:path';\nimport {\n  defaultRunDirName,\n  getMidsceneRunSubDir,\n} from '@midscene/shared/common';\nimport {\n  MIDSCENE_DEBUG_MODE,\n  MIDSCENE_OPENAI_INIT_CONFIG_JSON,\n  getAIConfig,\n  getAIConfigInJson,\n} from '@midscene/shared/env';\nimport { getRunningPkgInfo } from '@midscene/shared/node';\nimport { assert, logMsg } from '@midscene/shared/utils';\nimport {\n  escapeScriptTag,\n  ifInBrowser,\n  ifInWorker,\n  uuid,\n} from '@midscene/shared/utils';\nimport type { Rect, ReportDumpWithAttributes } from './types';\n\nlet logEnvReady = false;\n\nexport const groupedActionDumpFileExt = 'web-dump.json';\n\nconst reportInitializedMap = new Map<string, boolean>();\n\ndeclare const __DEV_REPORT_PATH__: string;\n\nfunction getReportTpl() {\n  if (__DEV_REPORT_PATH__) {\n    return fs.readFileSync(__DEV_REPORT_PATH__, 'utf-8');\n  }\n  const reportTpl = 'REPLACE_ME_WITH_REPORT_HTML';\n\n  return reportTpl;\n}\n\n/**\n * high performance, insert script before </html> in HTML file\n * only truncate and append, no temporary file\n */\nexport function insertScriptBeforeClosingHtml(\n  filePath: string,\n  scriptContent: string,\n): void {\n  const htmlEndTag = '</html>';\n  const stat = fs.statSync(filePath);\n\n  const readSize = Math.min(stat.size, 4096);\n  const start = Math.max(0, stat.size - readSize);\n  const buffer = Buffer.alloc(stat.size - start);\n  const fd = fs.openSync(filePath, 'r');\n  fs.readSync(fd, buffer, 0, buffer.length, start);\n  fs.closeSync(fd);\n\n  const tailStr = buffer.toString('utf8');\n  const htmlEndIdx = tailStr.lastIndexOf(htmlEndTag);\n  if (htmlEndIdx === -1) {\n    throw new Error(`No </html> found in file：${filePath}`);\n  }\n\n  // calculate the correct byte position: char position to byte position\n  const beforeHtmlInTail = tailStr.slice(0, htmlEndIdx);\n  const htmlEndPos = start + Buffer.byteLength(beforeHtmlInTail, 'utf8');\n\n  // truncate to </html> before\n  fs.truncateSync(filePath, htmlEndPos);\n  // append script and </html>\n  fs.appendFileSync(filePath, `${scriptContent}\\n${htmlEndTag}\\n`);\n}\n\nexport function reportHTMLContent(\n  dumpData: string | ReportDumpWithAttributes,\n  reportPath?: string,\n  appendReport?: boolean,\n): string {\n  const tpl = getReportTpl();\n\n  if (!tpl) {\n    console.warn('reportTpl is not set, will not write report');\n    return '';\n  }\n\n  // if reportPath is set, it means we are in write to file mode\n  const writeToFile = reportPath && !ifInBrowser;\n  let dumpContent = '';\n\n  if (typeof dumpData === 'string') {\n    // do not use template string here, will cause bundle error\n    dumpContent =\n      // biome-ignore lint/style/useTemplate: <explanation>\n      '<script type=\"midscene_web_dump\" type=\"application/json\">\\n' +\n      escapeScriptTag(dumpData) +\n      '\\n</script>';\n  } else {\n    const { dumpString, attributes } = dumpData;\n    const attributesArr = Object.keys(attributes || {}).map((key) => {\n      return `${key}=\"${encodeURIComponent(attributes![key])}\"`;\n    });\n\n    dumpContent =\n      // do not use template string here, will cause bundle error\n      // biome-ignore lint/style/useTemplate: <explanation>\n      '<script type=\"midscene_web_dump\" type=\"application/json\" ' +\n      attributesArr.join(' ') +\n      '>\\n' +\n      escapeScriptTag(dumpString) +\n      '\\n</script>';\n  }\n\n  if (writeToFile) {\n    if (!appendReport) {\n      writeFileSync(reportPath!, tpl + dumpContent, { flag: 'w' });\n      return reportPath!;\n    }\n\n    if (!reportInitializedMap.get(reportPath!)) {\n      writeFileSync(reportPath!, tpl, { flag: 'w' });\n      reportInitializedMap.set(reportPath!, true);\n    }\n\n    insertScriptBeforeClosingHtml(reportPath!, dumpContent);\n    return reportPath!;\n  }\n\n  return tpl + dumpContent;\n}\n\nexport function writeDumpReport(\n  fileName: string,\n  dumpData: string | ReportDumpWithAttributes,\n  appendReport?: boolean,\n): string | null {\n  if (ifInBrowser || ifInWorker) {\n    console.log('will not write report in browser');\n    return null;\n  }\n\n  const reportPath = path.join(\n    getMidsceneRunSubDir('report'),\n    `${fileName}.html`,\n  );\n\n  reportHTMLContent(dumpData, reportPath, appendReport);\n\n  if (process.env.MIDSCENE_DEBUG_LOG_JSON) {\n    const jsonPath = `${reportPath}.json`;\n    let data;\n\n    if (typeof dumpData === 'string') {\n      data = JSON.parse(dumpData) as ReportDumpWithAttributes;\n    } else {\n      data = dumpData;\n    }\n\n    writeFileSync(jsonPath, JSON.stringify(data, null, 2), {\n      flag: appendReport ? 'a' : 'w',\n    });\n\n    logMsg(`Midscene - dump file written: ${jsonPath}`);\n  }\n\n  return reportPath;\n}\n\nexport function writeLogFile(opts: {\n  fileName: string;\n  fileExt: string;\n  fileContent: string;\n  type: 'dump' | 'cache' | 'report' | 'tmp';\n  generateReport?: boolean;\n  appendReport?: boolean;\n}) {\n  if (ifInBrowser || ifInWorker) {\n    return '/mock/report.html';\n  }\n  const { fileName, fileExt, fileContent, type = 'dump' } = opts;\n  const targetDir = getMidsceneRunSubDir(type);\n  // Ensure directory exists\n  if (!logEnvReady) {\n    assert(targetDir, 'logDir should be set before writing dump file');\n\n    // gitIgnore in the parent directory\n    const gitIgnorePath = path.join(targetDir, '../../.gitignore');\n    const gitPath = path.join(targetDir, '../../.git');\n    let gitIgnoreContent = '';\n\n    if (existsSync(gitPath)) {\n      // if the git path exists, we need to add the log folder to the git ignore file\n      if (existsSync(gitIgnorePath)) {\n        gitIgnoreContent = readFileSync(gitIgnorePath, 'utf-8');\n      }\n\n      // ignore the log folder\n      if (!gitIgnoreContent.includes(`${defaultRunDirName}/`)) {\n        writeFileSync(\n          gitIgnorePath,\n          `${gitIgnoreContent}\\n# Midscene.js dump files\\n${defaultRunDirName}/dump\\n${defaultRunDirName}/report\\n${defaultRunDirName}/tmp\\n${defaultRunDirName}/log\\n`,\n          'utf-8',\n        );\n      }\n    }\n\n    logEnvReady = true;\n  }\n\n  const filePath = path.join(targetDir, `${fileName}.${fileExt}`);\n\n  if (type !== 'dump') {\n    // do not write dump file any more\n    writeFileSync(filePath, fileContent);\n  }\n\n  if (opts?.generateReport) {\n    return writeDumpReport(fileName, fileContent, opts.appendReport);\n  }\n\n  return filePath;\n}\n\nexport function getTmpDir(): string | null {\n  try {\n    const runningPkgInfo = getRunningPkgInfo();\n    if (!runningPkgInfo) {\n      return null;\n    }\n    const { name } = runningPkgInfo;\n    const tmpPath = path.join(tmpdir(), name);\n    mkdirSync(tmpPath, { recursive: true });\n    return tmpPath;\n  } catch (e) {\n    return null;\n  }\n}\n\nexport function getTmpFile(fileExtWithoutDot: string): string | null {\n  if (ifInBrowser || ifInWorker) {\n    return null;\n  }\n  const tmpDir = getTmpDir();\n  const filename = `${uuid()}.${fileExtWithoutDot}`;\n  return path.join(tmpDir!, filename);\n}\n\nexport function overlapped(container: Rect, target: Rect) {\n  // container and the target have some part overlapped\n  return (\n    container.left < target.left + target.width &&\n    container.left + container.width > target.left &&\n    container.top < target.top + target.height &&\n    container.top + container.height > target.top\n  );\n}\n\nexport async function sleep(ms: number) {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport function replacerForPageObject(key: string, value: any) {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  return value;\n}\n\nexport function stringifyDumpData(data: any, indents?: number) {\n  return JSON.stringify(data, replacerForPageObject, indents);\n}\n\ndeclare const __VERSION__: string;\n\nexport function getVersion() {\n  return __VERSION__;\n}\n\nfunction debugLog(...message: any[]) {\n  const debugMode = getAIConfig(MIDSCENE_DEBUG_MODE);\n  if (debugMode) {\n    console.log('[Midscene]', ...message);\n  }\n}\n\nlet lastReportedRepoUrl = '';\nexport function uploadTestInfoToServer({ testUrl }: { testUrl: string }) {\n  let repoUrl = '';\n  let userEmail = '';\n\n  const extraConfig = getAIConfigInJson(MIDSCENE_OPENAI_INIT_CONFIG_JSON);\n  const serverUrl = extraConfig?.REPORT_SERVER_URL;\n\n  try {\n    repoUrl = execSync('git config --get remote.origin.url').toString().trim();\n    userEmail = execSync('git config --get user.email').toString().trim();\n  } catch (error) {\n    debugLog('Failed to get git info:', error);\n  }\n\n  // Only upload test info if:\n  // 1. Server URL is configured AND\n  // 2. Either:\n  //    - We have a repo URL that's different from last reported one (to avoid duplicate reports)\n  //    - OR we don't have a repo URL but have a test URL (for non-git environments)\n  if (\n    serverUrl &&\n    ((repoUrl && repoUrl !== lastReportedRepoUrl) || (!repoUrl && testUrl))\n  ) {\n    debugLog('Uploading test info to server', {\n      serverUrl,\n      repoUrl,\n      testUrl,\n      userEmail,\n    });\n\n    fetch(serverUrl, {\n      method: 'POST',\n      headers: {\n        'Content-Type': 'application/json',\n      },\n      body: JSON.stringify({\n        repo_url: repoUrl,\n        test_url: testUrl,\n        user_email: userEmail,\n      }),\n    })\n      .then((response) => response.json())\n      .then((data) => {\n        debugLog('Successfully uploaded test info to server:', data);\n      })\n      .catch((error) =>\n        debugLog('Failed to upload test info to server:', error),\n      );\n    lastReportedRepoUrl = repoUrl;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","logEnvReady","groupedActionDumpFileExt","reportInitializedMap","Map","getReportTpl","reportTpl","insertScriptBeforeClosingHtml","filePath","scriptContent","htmlEndTag","stat","fs","readSize","Math","start","buffer","Buffer","fd","tailStr","htmlEndIdx","Error","beforeHtmlInTail","htmlEndPos","reportHTMLContent","dumpData","reportPath","appendReport","tpl","console","writeToFile","ifInBrowser","dumpContent","escapeScriptTag","dumpString","attributes","attributesArr","encodeURIComponent","writeFileSync","writeDumpReport","fileName","ifInWorker","path","getMidsceneRunSubDir","process","jsonPath","data","JSON","logMsg","writeLogFile","opts","fileExt","fileContent","type","targetDir","assert","gitIgnorePath","gitPath","gitIgnoreContent","existsSync","readFileSync","defaultRunDirName","getTmpDir","runningPkgInfo","getRunningPkgInfo","name","tmpPath","tmpdir","mkdirSync","e","getTmpFile","fileExtWithoutDot","tmpDir","filename","uuid","overlapped","container","target","sleep","ms","Promise","resolve","setTimeout","replacerForPageObject","value","_value_constructor","_value_constructor1","stringifyDumpData","indents","getVersion","__VERSION__","debugLog","message","debugMode","getAIConfig","MIDSCENE_DEBUG_MODE","lastReportedRepoUrl","uploadTestInfoToServer","testUrl","repoUrl","userEmail","extraConfig","getAIConfigInJson","MIDSCENE_OPENAI_INIT_CONFIG_JSON","serverUrl","execSync","error","fetch","response"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;ACmBA,IAAII,cAAc;AAEX,MAAMC,2BAA2B;AAExC,MAAMC,uBAAuB,IAAIC;AAIjC,SAASC;IAIP,MAAMC,YAAY;IAElB,OAAOA;AACT;AAMO,SAASC,8BACdC,QAAgB,EAChBC,aAAqB;IAErB,MAAMC,aAAa;IACnB,MAAMC,OAAOC,iCAAAA,QAAW,CAACJ;IAEzB,MAAMK,WAAWC,KAAK,GAAG,CAACH,KAAK,IAAI,EAAE;IACrC,MAAMI,QAAQD,KAAK,GAAG,CAAC,GAAGH,KAAK,IAAI,GAAGE;IACtC,MAAMG,SAASC,OAAO,KAAK,CAACN,KAAK,IAAI,GAAGI;IACxC,MAAMG,KAAKN,iCAAAA,QAAW,CAACJ,UAAU;IACjCI,iCAAAA,QAAW,CAACM,IAAIF,QAAQ,GAAGA,OAAO,MAAM,EAAED;IAC1CH,iCAAAA,SAAY,CAACM;IAEb,MAAMC,UAAUH,OAAO,QAAQ,CAAC;IAChC,MAAMI,aAAaD,QAAQ,WAAW,CAACT;IACvC,IAAIU,AAAe,OAAfA,YACF,MAAM,IAAIC,MAAM,CAAC,gCAAyB,EAAEb,UAAU;IAIxD,MAAMc,mBAAmBH,QAAQ,KAAK,CAAC,GAAGC;IAC1C,MAAMG,aAAaR,QAAQE,OAAO,UAAU,CAACK,kBAAkB;IAG/DV,iCAAAA,YAAe,CAACJ,UAAUe;IAE1BX,iCAAAA,cAAiB,CAACJ,UAAU,GAAGC,cAAc,EAAE,EAAEC,WAAW,EAAE,CAAC;AACjE;AAEO,SAASc,kBACdC,QAA2C,EAC3CC,UAAmB,EACnBC,YAAsB;IAEtB,MAAMC,MAAMvB;IAEZ,IAAI,CAACuB,KAAK;QACRC,QAAQ,IAAI,CAAC;QACb,OAAO;IACT;IAGA,MAAMC,cAAcJ,cAAc,CAACK,sBAAAA,WAAWA;IAC9C,IAAIC,cAAc;IAElB,IAAI,AAAoB,YAApB,OAAOP,UAETO,cAEE,gEACAC,AAAAA,IAAAA,sBAAAA,eAAAA,AAAAA,EAAgBR,YAChB;SACG;QACL,MAAM,EAAES,UAAU,EAAEC,UAAU,EAAE,GAAGV;QACnC,MAAMW,gBAAgBvC,OAAO,IAAI,CAACsC,cAAc,CAAC,GAAG,GAAG,CAAC,CAACvC,MAChD,GAAGA,IAAI,EAAE,EAAEyC,mBAAmBF,UAAW,CAACvC,IAAI,EAAE,CAAC,CAAC;QAG3DoC,cAGE,8DACAI,cAAc,IAAI,CAAC,OACnB,QACAH,AAAAA,IAAAA,sBAAAA,eAAAA,AAAAA,EAAgBC,cAChB;IACJ;IAEA,IAAIJ,aAAa;QACf,IAAI,CAACH,cAAc;YACjBW,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcZ,YAAaE,MAAMI,aAAa;gBAAE,MAAM;YAAI;YAC1D,OAAON;QACT;QAEA,IAAI,CAACvB,qBAAqB,GAAG,CAACuB,aAAc;YAC1CY,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcZ,YAAaE,KAAK;gBAAE,MAAM;YAAI;YAC5CzB,qBAAqB,GAAG,CAACuB,YAAa;QACxC;QAEAnB,8BAA8BmB,YAAaM;QAC3C,OAAON;IACT;IAEA,OAAOE,MAAMI;AACf;AAEO,SAASO,gBACdC,QAAgB,EAChBf,QAA2C,EAC3CE,YAAsB;IAEtB,IAAII,sBAAAA,WAAWA,IAAIU,sBAAAA,UAAUA,EAAE;QAC7BZ,QAAQ,GAAG,CAAC;QACZ,OAAO;IACT;IAEA,MAAMH,aAAagB,mCAAAA,IAAS,CAC1BC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,SAAS,KAAK,CAAC;IAGpBhB,kBAAkBC,UAAUC,YAAYC;IAExC,IAAIiB,QAAQ,GAAG,CAAC,uBAAuB,EAAE;QACvC,MAAMC,WAAW,GAAGnB,WAAW,KAAK,CAAC;QACrC,IAAIoB;QAGFA,OADE,AAAoB,YAApB,OAAOrB,WACFsB,KAAK,KAAK,CAACtB,YAEXA;QAGTa,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcO,UAAUE,KAAK,SAAS,CAACD,MAAM,MAAM,IAAI;YACrD,MAAMnB,eAAe,MAAM;QAC7B;QAEAqB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,8BAA8B,EAAEH,UAAU;IACpD;IAEA,OAAOnB;AACT;AAEO,SAASuB,aAAaC,IAO5B;IACC,IAAInB,sBAAAA,WAAWA,IAAIU,sBAAAA,UAAUA,EAC3B,OAAO;IAET,MAAM,EAAED,QAAQ,EAAEW,OAAO,EAAEC,WAAW,EAAEC,OAAO,MAAM,EAAE,GAAGH;IAC1D,MAAMI,YAAYX,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqBU;IAEvC,IAAI,CAACpD,aAAa;QAChBsD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;QAGlB,MAAME,gBAAgBd,mCAAAA,IAAS,CAACY,WAAW;QAC3C,MAAMG,UAAUf,mCAAAA,IAAS,CAACY,WAAW;QACrC,IAAII,mBAAmB;QAEvB,IAAIC,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,UAAU;YAEvB,IAAIE,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWH,gBACbE,mBAAmBE,AAAAA,IAAAA,iCAAAA,YAAAA,AAAAA,EAAaJ,eAAe;YAIjD,IAAI,CAACE,iBAAiB,QAAQ,CAAC,GAAGG,uBAAAA,iBAAiBA,CAAC,CAAC,CAAC,GACpDvB,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EACEkB,eACA,GAAGE,iBAAiB,4BAA4B,EAAEG,uBAAAA,iBAAiBA,CAAC,OAAO,EAAEA,uBAAAA,iBAAiBA,CAAC,SAAS,EAAEA,uBAAAA,iBAAiBA,CAAC,MAAM,EAAEA,uBAAAA,iBAAiBA,CAAC,MAAM,CAAC,EAC7J;QAGN;QAEA5D,cAAc;IAChB;IAEA,MAAMO,WAAWkC,mCAAAA,IAAS,CAACY,WAAW,GAAGd,SAAS,CAAC,EAAEW,SAAS;IAE9D,IAAIE,AAAS,WAATA,MAEFf,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAc9B,UAAU4C;IAG1B,IAAIF,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,cAAc,EACtB,OAAOX,gBAAgBC,UAAUY,aAAaF,KAAK,YAAY;IAGjE,OAAO1C;AACT;AAEO,SAASsD;IACd,IAAI;QACF,MAAMC,iBAAiBC,AAAAA,IAAAA,qBAAAA,iBAAAA,AAAAA;QACvB,IAAI,CAACD,gBACH,OAAO;QAET,MAAM,EAAEE,IAAI,EAAE,GAAGF;QACjB,MAAMG,UAAUxB,mCAAAA,IAAS,CAACyB,AAAAA,IAAAA,iCAAAA,MAAAA,AAAAA,KAAUF;QACpCG,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUF,SAAS;YAAE,WAAW;QAAK;QACrC,OAAOA;IACT,EAAE,OAAOG,GAAG;QACV,OAAO;IACT;AACF;AAEO,SAASC,WAAWC,iBAAyB;IAClD,IAAIxC,sBAAAA,WAAWA,IAAIU,sBAAAA,UAAUA,EAC3B,OAAO;IAET,MAAM+B,SAASV;IACf,MAAMW,WAAW,GAAGC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,CAAC,EAAEH,mBAAmB;IACjD,OAAO7B,mCAAAA,IAAS,CAAC8B,QAASC;AAC5B;AAEO,SAASE,WAAWC,SAAe,EAAEC,MAAY;IAEtD,OACED,UAAU,IAAI,GAAGC,OAAO,IAAI,GAAGA,OAAO,KAAK,IAC3CD,UAAU,IAAI,GAAGA,UAAU,KAAK,GAAGC,OAAO,IAAI,IAC9CD,UAAU,GAAG,GAAGC,OAAO,GAAG,GAAGA,OAAO,MAAM,IAC1CD,UAAU,GAAG,GAAGA,UAAU,MAAM,GAAGC,OAAO,GAAG;AAEjD;AAEO,eAAeC,MAAMC,EAAU;IACpC,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEO,SAASI,sBAAsBvF,GAAW,EAAEwF,KAAU;QAC9CC,oBAGAC;IAHb,IAAIF,SAASC,AAAAA,SAAAA,CAAAA,qBAAAA,MAAM,WAAW,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,MAAM,QACvC,OAAO;IAET,IAAID,SAASE,AAAAA,SAAAA,CAAAA,sBAAAA,MAAM,WAAW,AAAD,IAAhBA,KAAAA,IAAAA,oBAAmB,IAAI,AAAD,MAAM,WACvC,OAAO;IAET,OAAOF;AACT;AAEO,SAASG,kBAAkBzC,IAAS,EAAE0C,OAAgB;IAC3D,OAAOzC,KAAK,SAAS,CAACD,MAAMqC,uBAAuBK;AACrD;AAIO,SAASC;IACd,OAAOC;AACT;AAEA,SAASC,SAAS,GAAGC,OAAc;IACjC,MAAMC,YAAYC,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAYC,oBAAAA,mBAAmBA;IACjD,IAAIF,WACFhE,QAAQ,GAAG,CAAC,iBAAiB+D;AAEjC;AAEA,IAAII,sBAAsB;AACnB,SAASC,uBAAuB,EAAEC,OAAO,EAAuB;IACrE,IAAIC,UAAU;IACd,IAAIC,YAAY;IAEhB,MAAMC,cAAcC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBC,oBAAAA,gCAAgCA;IACtE,MAAMC,YAAYH,QAAAA,cAAAA,KAAAA,IAAAA,YAAa,iBAAiB;IAEhD,IAAI;QACFF,UAAUM,AAAAA,IAAAA,4CAAAA,QAAAA,AAAAA,EAAS,sCAAsC,QAAQ,GAAG,IAAI;QACxEL,YAAYK,AAAAA,IAAAA,4CAAAA,QAAAA,AAAAA,EAAS,+BAA+B,QAAQ,GAAG,IAAI;IACrE,EAAE,OAAOC,OAAO;QACdf,SAAS,2BAA2Be;IACtC;IAOA,IACEF,aACEL,CAAAA,WAAWA,YAAYH,uBAAyB,CAACG,WAAWD,OAAM,GACpE;QACAP,SAAS,iCAAiC;YACxCa;YACAL;YACAD;YACAE;QACF;QAEAO,MAAMH,WAAW;YACf,QAAQ;YACR,SAAS;gBACP,gBAAgB;YAClB;YACA,MAAMzD,KAAK,SAAS,CAAC;gBACnB,UAAUoD;gBACV,UAAUD;gBACV,YAAYE;YACd;QACF,GACG,IAAI,CAAC,CAACQ,WAAaA,SAAS,IAAI,IAChC,IAAI,CAAC,CAAC9D;YACL6C,SAAS,8CAA8C7C;QACzD,GACC,KAAK,CAAC,CAAC4D,QACNf,SAAS,yCAAyCe;QAEtDV,sBAAsBG;IACxB;AACF"}
+{"version":3,"file":"utils.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/utils.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { execSync } from 'node:child_process';\nimport * as fs from 'node:fs';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { tmpdir } from 'node:os';\nimport * as path from 'node:path';\nimport {\n  defaultRunDirName,\n  getMidsceneRunSubDir,\n} from '@midscene/shared/common';\nimport {\n  MIDSCENE_DEBUG_MODE,\n  getUploadTestServerUrl,\n} from '@midscene/shared/env';\nimport { getRunningPkgInfo } from '@midscene/shared/node';\nimport { assert, logMsg } from '@midscene/shared/utils';\nimport {\n  escapeScriptTag,\n  ifInBrowser,\n  ifInWorker,\n  uuid,\n} from '@midscene/shared/utils';\nimport type { Rect, ReportDumpWithAttributes } from './types';\n\nlet logEnvReady = false;\n\nexport const groupedActionDumpFileExt = 'web-dump.json';\n\nconst reportInitializedMap = new Map<string, boolean>();\n\ndeclare const __DEV_REPORT_PATH__: string;\n\nfunction getReportTpl() {\n  if (__DEV_REPORT_PATH__) {\n    return fs.readFileSync(__DEV_REPORT_PATH__, 'utf-8');\n  }\n  const reportTpl = 'REPLACE_ME_WITH_REPORT_HTML';\n\n  return reportTpl;\n}\n\n/**\n * high performance, insert script before </html> in HTML file\n * only truncate and append, no temporary file\n */\nexport function insertScriptBeforeClosingHtml(\n  filePath: string,\n  scriptContent: string,\n): void {\n  const htmlEndTag = '</html>';\n  const stat = fs.statSync(filePath);\n\n  const readSize = Math.min(stat.size, 4096);\n  const start = Math.max(0, stat.size - readSize);\n  const buffer = Buffer.alloc(stat.size - start);\n  const fd = fs.openSync(filePath, 'r');\n  fs.readSync(fd, buffer, 0, buffer.length, start);\n  fs.closeSync(fd);\n\n  const tailStr = buffer.toString('utf8');\n  const htmlEndIdx = tailStr.lastIndexOf(htmlEndTag);\n  if (htmlEndIdx === -1) {\n    throw new Error(`No </html> found in file：${filePath}`);\n  }\n\n  // calculate the correct byte position: char position to byte position\n  const beforeHtmlInTail = tailStr.slice(0, htmlEndIdx);\n  const htmlEndPos = start + Buffer.byteLength(beforeHtmlInTail, 'utf8');\n\n  // truncate to </html> before\n  fs.truncateSync(filePath, htmlEndPos);\n  // append script and </html>\n  fs.appendFileSync(filePath, `${scriptContent}\\n${htmlEndTag}\\n`);\n}\n\nexport function reportHTMLContent(\n  dumpData: string | ReportDumpWithAttributes,\n  reportPath?: string,\n  appendReport?: boolean,\n): string {\n  const tpl = getReportTpl();\n\n  if (!tpl) {\n    console.warn('reportTpl is not set, will not write report');\n    return '';\n  }\n\n  // if reportPath is set, it means we are in write to file mode\n  const writeToFile = reportPath && !ifInBrowser;\n  let dumpContent = '';\n\n  if (typeof dumpData === 'string') {\n    // do not use template string here, will cause bundle error\n    dumpContent =\n      // biome-ignore lint/style/useTemplate: <explanation>\n      '<script type=\"midscene_web_dump\" type=\"application/json\">\\n' +\n      escapeScriptTag(dumpData) +\n      '\\n</script>';\n  } else {\n    const { dumpString, attributes } = dumpData;\n    const attributesArr = Object.keys(attributes || {}).map((key) => {\n      return `${key}=\"${encodeURIComponent(attributes![key])}\"`;\n    });\n\n    dumpContent =\n      // do not use template string here, will cause bundle error\n      // biome-ignore lint/style/useTemplate: <explanation>\n      '<script type=\"midscene_web_dump\" type=\"application/json\" ' +\n      attributesArr.join(' ') +\n      '>\\n' +\n      escapeScriptTag(dumpString) +\n      '\\n</script>';\n  }\n\n  if (writeToFile) {\n    if (!appendReport) {\n      writeFileSync(reportPath!, tpl + dumpContent, { flag: 'w' });\n      return reportPath!;\n    }\n\n    if (!reportInitializedMap.get(reportPath!)) {\n      writeFileSync(reportPath!, tpl, { flag: 'w' });\n      reportInitializedMap.set(reportPath!, true);\n    }\n\n    insertScriptBeforeClosingHtml(reportPath!, dumpContent);\n    return reportPath!;\n  }\n\n  return tpl + dumpContent;\n}\n\nexport function writeDumpReport(\n  fileName: string,\n  dumpData: string | ReportDumpWithAttributes,\n  appendReport?: boolean,\n): string | null {\n  if (ifInBrowser || ifInWorker) {\n    console.log('will not write report in browser');\n    return null;\n  }\n\n  const reportPath = path.join(\n    getMidsceneRunSubDir('report'),\n    `${fileName}.html`,\n  );\n\n  reportHTMLContent(dumpData, reportPath, appendReport);\n\n  if (process.env.MIDSCENE_DEBUG_LOG_JSON) {\n    const jsonPath = `${reportPath}.json`;\n    let data;\n\n    if (typeof dumpData === 'string') {\n      data = JSON.parse(dumpData) as ReportDumpWithAttributes;\n    } else {\n      data = dumpData;\n    }\n\n    writeFileSync(jsonPath, JSON.stringify(data, null, 2), {\n      flag: appendReport ? 'a' : 'w',\n    });\n\n    logMsg(`Midscene - dump file written: ${jsonPath}`);\n  }\n\n  return reportPath;\n}\n\nexport function writeLogFile(opts: {\n  fileName: string;\n  fileExt: string;\n  fileContent: string;\n  type: 'dump' | 'cache' | 'report' | 'tmp';\n  generateReport?: boolean;\n  appendReport?: boolean;\n}) {\n  if (ifInBrowser || ifInWorker) {\n    return '/mock/report.html';\n  }\n  const { fileName, fileExt, fileContent, type = 'dump' } = opts;\n  const targetDir = getMidsceneRunSubDir(type);\n  // Ensure directory exists\n  if (!logEnvReady) {\n    assert(targetDir, 'logDir should be set before writing dump file');\n\n    // gitIgnore in the parent directory\n    const gitIgnorePath = path.join(targetDir, '../../.gitignore');\n    const gitPath = path.join(targetDir, '../../.git');\n    let gitIgnoreContent = '';\n\n    if (existsSync(gitPath)) {\n      // if the git path exists, we need to add the log folder to the git ignore file\n      if (existsSync(gitIgnorePath)) {\n        gitIgnoreContent = readFileSync(gitIgnorePath, 'utf-8');\n      }\n\n      // ignore the log folder\n      if (!gitIgnoreContent.includes(`${defaultRunDirName}/`)) {\n        writeFileSync(\n          gitIgnorePath,\n          `${gitIgnoreContent}\\n# Midscene.js dump files\\n${defaultRunDirName}/dump\\n${defaultRunDirName}/report\\n${defaultRunDirName}/tmp\\n${defaultRunDirName}/log\\n`,\n          'utf-8',\n        );\n      }\n    }\n\n    logEnvReady = true;\n  }\n\n  const filePath = path.join(targetDir, `${fileName}.${fileExt}`);\n\n  if (type !== 'dump') {\n    // do not write dump file any more\n    writeFileSync(filePath, fileContent);\n  }\n\n  if (opts?.generateReport) {\n    return writeDumpReport(fileName, fileContent, opts.appendReport);\n  }\n\n  return filePath;\n}\n\nexport function getTmpDir(): string | null {\n  try {\n    const runningPkgInfo = getRunningPkgInfo();\n    if (!runningPkgInfo) {\n      return null;\n    }\n    const { name } = runningPkgInfo;\n    const tmpPath = path.join(tmpdir(), name);\n    mkdirSync(tmpPath, { recursive: true });\n    return tmpPath;\n  } catch (e) {\n    return null;\n  }\n}\n\nexport function getTmpFile(fileExtWithoutDot: string): string | null {\n  if (ifInBrowser || ifInWorker) {\n    return null;\n  }\n  const tmpDir = getTmpDir();\n  const filename = `${uuid()}.${fileExtWithoutDot}`;\n  return path.join(tmpDir!, filename);\n}\n\nexport function overlapped(container: Rect, target: Rect) {\n  // container and the target have some part overlapped\n  return (\n    container.left < target.left + target.width &&\n    container.left + container.width > target.left &&\n    container.top < target.top + target.height &&\n    container.top + container.height > target.top\n  );\n}\n\nexport async function sleep(ms: number) {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport function replacerForPageObject(key: string, value: any) {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  return value;\n}\n\nexport function stringifyDumpData(data: any, indents?: number) {\n  return JSON.stringify(data, replacerForPageObject, indents);\n}\n\ndeclare const __VERSION__: string;\n\nexport function getVersion() {\n  return __VERSION__;\n}\n\nfunction debugLog(...message: any[]) {\n  // always read from process.env, and cannot be override by modelConfig, overrideAIConfig, etc.\n  const debugMode = process.env[MIDSCENE_DEBUG_MODE];\n  if (debugMode) {\n    console.log('[Midscene]', ...message);\n  }\n}\n\nlet lastReportedRepoUrl = '';\nexport function uploadTestInfoToServer({ testUrl }: { testUrl: string }) {\n  let repoUrl = '';\n  let userEmail = '';\n\n  const serverUrl = getUploadTestServerUrl();\n\n  try {\n    repoUrl = execSync('git config --get remote.origin.url').toString().trim();\n    userEmail = execSync('git config --get user.email').toString().trim();\n  } catch (error) {\n    debugLog('Failed to get git info:', error);\n  }\n\n  // Only upload test info if:\n  // 1. Server URL is configured AND\n  // 2. Either:\n  //    - We have a repo URL that's different from last reported one (to avoid duplicate reports)\n  //    - OR we don't have a repo URL but have a test URL (for non-git environments)\n  if (\n    serverUrl &&\n    ((repoUrl && repoUrl !== lastReportedRepoUrl) || (!repoUrl && testUrl))\n  ) {\n    debugLog('Uploading test info to server', {\n      serverUrl,\n      repoUrl,\n      testUrl,\n      userEmail,\n    });\n\n    fetch(serverUrl, {\n      method: 'POST',\n      headers: {\n        'Content-Type': 'application/json',\n      },\n      body: JSON.stringify({\n        repo_url: repoUrl,\n        test_url: testUrl,\n        user_email: userEmail,\n      }),\n    })\n      .then((response) => response.json())\n      .then((data) => {\n        debugLog('Successfully uploaded test info to server:', data);\n      })\n      .catch((error) =>\n        debugLog('Failed to upload test info to server:', error),\n      );\n    lastReportedRepoUrl = repoUrl;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","logEnvReady","groupedActionDumpFileExt","reportInitializedMap","Map","getReportTpl","reportTpl","insertScriptBeforeClosingHtml","filePath","scriptContent","htmlEndTag","stat","fs","readSize","Math","start","buffer","Buffer","fd","tailStr","htmlEndIdx","Error","beforeHtmlInTail","htmlEndPos","reportHTMLContent","dumpData","reportPath","appendReport","tpl","console","writeToFile","ifInBrowser","dumpContent","escapeScriptTag","dumpString","attributes","attributesArr","encodeURIComponent","writeFileSync","writeDumpReport","fileName","ifInWorker","path","getMidsceneRunSubDir","process","jsonPath","data","JSON","logMsg","writeLogFile","opts","fileExt","fileContent","type","targetDir","assert","gitIgnorePath","gitPath","gitIgnoreContent","existsSync","readFileSync","defaultRunDirName","getTmpDir","runningPkgInfo","getRunningPkgInfo","name","tmpPath","tmpdir","mkdirSync","e","getTmpFile","fileExtWithoutDot","tmpDir","filename","uuid","overlapped","container","target","sleep","ms","Promise","resolve","setTimeout","replacerForPageObject","value","_value_constructor","_value_constructor1","stringifyDumpData","indents","getVersion","__VERSION__","debugLog","message","debugMode","MIDSCENE_DEBUG_MODE","lastReportedRepoUrl","uploadTestInfoToServer","testUrl","repoUrl","userEmail","serverUrl","getUploadTestServerUrl","execSync","error","fetch","response"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;ACiBA,IAAII,cAAc;AAEX,MAAMC,2BAA2B;AAExC,MAAMC,uBAAuB,IAAIC;AAIjC,SAASC;IAIP,MAAMC,YAAY;IAElB,OAAOA;AACT;AAMO,SAASC,8BACdC,QAAgB,EAChBC,aAAqB;IAErB,MAAMC,aAAa;IACnB,MAAMC,OAAOC,iCAAAA,QAAW,CAACJ;IAEzB,MAAMK,WAAWC,KAAK,GAAG,CAACH,KAAK,IAAI,EAAE;IACrC,MAAMI,QAAQD,KAAK,GAAG,CAAC,GAAGH,KAAK,IAAI,GAAGE;IACtC,MAAMG,SAASC,OAAO,KAAK,CAACN,KAAK,IAAI,GAAGI;IACxC,MAAMG,KAAKN,iCAAAA,QAAW,CAACJ,UAAU;IACjCI,iCAAAA,QAAW,CAACM,IAAIF,QAAQ,GAAGA,OAAO,MAAM,EAAED;IAC1CH,iCAAAA,SAAY,CAACM;IAEb,MAAMC,UAAUH,OAAO,QAAQ,CAAC;IAChC,MAAMI,aAAaD,QAAQ,WAAW,CAACT;IACvC,IAAIU,AAAe,OAAfA,YACF,MAAM,IAAIC,MAAM,CAAC,gCAAyB,EAAEb,UAAU;IAIxD,MAAMc,mBAAmBH,QAAQ,KAAK,CAAC,GAAGC;IAC1C,MAAMG,aAAaR,QAAQE,OAAO,UAAU,CAACK,kBAAkB;IAG/DV,iCAAAA,YAAe,CAACJ,UAAUe;IAE1BX,iCAAAA,cAAiB,CAACJ,UAAU,GAAGC,cAAc,EAAE,EAAEC,WAAW,EAAE,CAAC;AACjE;AAEO,SAASc,kBACdC,QAA2C,EAC3CC,UAAmB,EACnBC,YAAsB;IAEtB,MAAMC,MAAMvB;IAEZ,IAAI,CAACuB,KAAK;QACRC,QAAQ,IAAI,CAAC;QACb,OAAO;IACT;IAGA,MAAMC,cAAcJ,cAAc,CAACK,sBAAAA,WAAWA;IAC9C,IAAIC,cAAc;IAElB,IAAI,AAAoB,YAApB,OAAOP,UAETO,cAEE,gEACAC,AAAAA,IAAAA,sBAAAA,eAAAA,AAAAA,EAAgBR,YAChB;SACG;QACL,MAAM,EAAES,UAAU,EAAEC,UAAU,EAAE,GAAGV;QACnC,MAAMW,gBAAgBvC,OAAO,IAAI,CAACsC,cAAc,CAAC,GAAG,GAAG,CAAC,CAACvC,MAChD,GAAGA,IAAI,EAAE,EAAEyC,mBAAmBF,UAAW,CAACvC,IAAI,EAAE,CAAC,CAAC;QAG3DoC,cAGE,8DACAI,cAAc,IAAI,CAAC,OACnB,QACAH,AAAAA,IAAAA,sBAAAA,eAAAA,AAAAA,EAAgBC,cAChB;IACJ;IAEA,IAAIJ,aAAa;QACf,IAAI,CAACH,cAAc;YACjBW,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcZ,YAAaE,MAAMI,aAAa;gBAAE,MAAM;YAAI;YAC1D,OAAON;QACT;QAEA,IAAI,CAACvB,qBAAqB,GAAG,CAACuB,aAAc;YAC1CY,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcZ,YAAaE,KAAK;gBAAE,MAAM;YAAI;YAC5CzB,qBAAqB,GAAG,CAACuB,YAAa;QACxC;QAEAnB,8BAA8BmB,YAAaM;QAC3C,OAAON;IACT;IAEA,OAAOE,MAAMI;AACf;AAEO,SAASO,gBACdC,QAAgB,EAChBf,QAA2C,EAC3CE,YAAsB;IAEtB,IAAII,sBAAAA,WAAWA,IAAIU,sBAAAA,UAAUA,EAAE;QAC7BZ,QAAQ,GAAG,CAAC;QACZ,OAAO;IACT;IAEA,MAAMH,aAAagB,mCAAAA,IAAS,CAC1BC,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqB,WACrB,GAAGH,SAAS,KAAK,CAAC;IAGpBhB,kBAAkBC,UAAUC,YAAYC;IAExC,IAAIiB,QAAQ,GAAG,CAAC,uBAAuB,EAAE;QACvC,MAAMC,WAAW,GAAGnB,WAAW,KAAK,CAAC;QACrC,IAAIoB;QAGFA,OADE,AAAoB,YAApB,OAAOrB,WACFsB,KAAK,KAAK,CAACtB,YAEXA;QAGTa,IAAAA,iCAAAA,aAAAA,AAAAA,EAAcO,UAAUE,KAAK,SAAS,CAACD,MAAM,MAAM,IAAI;YACrD,MAAMnB,eAAe,MAAM;QAC7B;QAEAqB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,8BAA8B,EAAEH,UAAU;IACpD;IAEA,OAAOnB;AACT;AAEO,SAASuB,aAAaC,IAO5B;IACC,IAAInB,sBAAAA,WAAWA,IAAIU,sBAAAA,UAAUA,EAC3B,OAAO;IAET,MAAM,EAAED,QAAQ,EAAEW,OAAO,EAAEC,WAAW,EAAEC,OAAO,MAAM,EAAE,GAAGH;IAC1D,MAAMI,YAAYX,AAAAA,IAAAA,uBAAAA,oBAAAA,AAAAA,EAAqBU;IAEvC,IAAI,CAACpD,aAAa;QAChBsD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,WAAW;QAGlB,MAAME,gBAAgBd,mCAAAA,IAAS,CAACY,WAAW;QAC3C,MAAMG,UAAUf,mCAAAA,IAAS,CAACY,WAAW;QACrC,IAAII,mBAAmB;QAEvB,IAAIC,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWF,UAAU;YAEvB,IAAIE,AAAAA,IAAAA,iCAAAA,UAAAA,AAAAA,EAAWH,gBACbE,mBAAmBE,AAAAA,IAAAA,iCAAAA,YAAAA,AAAAA,EAAaJ,eAAe;YAIjD,IAAI,CAACE,iBAAiB,QAAQ,CAAC,GAAGG,uBAAAA,iBAAiBA,CAAC,CAAC,CAAC,GACpDvB,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EACEkB,eACA,GAAGE,iBAAiB,4BAA4B,EAAEG,uBAAAA,iBAAiBA,CAAC,OAAO,EAAEA,uBAAAA,iBAAiBA,CAAC,SAAS,EAAEA,uBAAAA,iBAAiBA,CAAC,MAAM,EAAEA,uBAAAA,iBAAiBA,CAAC,MAAM,CAAC,EAC7J;QAGN;QAEA5D,cAAc;IAChB;IAEA,MAAMO,WAAWkC,mCAAAA,IAAS,CAACY,WAAW,GAAGd,SAAS,CAAC,EAAEW,SAAS;IAE9D,IAAIE,AAAS,WAATA,MAEFf,AAAAA,IAAAA,iCAAAA,aAAAA,AAAAA,EAAc9B,UAAU4C;IAG1B,IAAIF,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,cAAc,EACtB,OAAOX,gBAAgBC,UAAUY,aAAaF,KAAK,YAAY;IAGjE,OAAO1C;AACT;AAEO,SAASsD;IACd,IAAI;QACF,MAAMC,iBAAiBC,AAAAA,IAAAA,qBAAAA,iBAAAA,AAAAA;QACvB,IAAI,CAACD,gBACH,OAAO;QAET,MAAM,EAAEE,IAAI,EAAE,GAAGF;QACjB,MAAMG,UAAUxB,mCAAAA,IAAS,CAACyB,AAAAA,IAAAA,iCAAAA,MAAAA,AAAAA,KAAUF;QACpCG,IAAAA,iCAAAA,SAAAA,AAAAA,EAAUF,SAAS;YAAE,WAAW;QAAK;QACrC,OAAOA;IACT,EAAE,OAAOG,GAAG;QACV,OAAO;IACT;AACF;AAEO,SAASC,WAAWC,iBAAyB;IAClD,IAAIxC,sBAAAA,WAAWA,IAAIU,sBAAAA,UAAUA,EAC3B,OAAO;IAET,MAAM+B,SAASV;IACf,MAAMW,WAAW,GAAGC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,CAAC,EAAEH,mBAAmB;IACjD,OAAO7B,mCAAAA,IAAS,CAAC8B,QAASC;AAC5B;AAEO,SAASE,WAAWC,SAAe,EAAEC,MAAY;IAEtD,OACED,UAAU,IAAI,GAAGC,OAAO,IAAI,GAAGA,OAAO,KAAK,IAC3CD,UAAU,IAAI,GAAGA,UAAU,KAAK,GAAGC,OAAO,IAAI,IAC9CD,UAAU,GAAG,GAAGC,OAAO,GAAG,GAAGA,OAAO,MAAM,IAC1CD,UAAU,GAAG,GAAGA,UAAU,MAAM,GAAGC,OAAO,GAAG;AAEjD;AAEO,eAAeC,MAAMC,EAAU;IACpC,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEO,SAASI,sBAAsBvF,GAAW,EAAEwF,KAAU;QAC9CC,oBAGAC;IAHb,IAAIF,SAASC,AAAAA,SAAAA,CAAAA,qBAAAA,MAAM,WAAW,AAAD,IAAhBA,KAAAA,IAAAA,mBAAmB,IAAI,AAAD,MAAM,QACvC,OAAO;IAET,IAAID,SAASE,AAAAA,SAAAA,CAAAA,sBAAAA,MAAM,WAAW,AAAD,IAAhBA,KAAAA,IAAAA,oBAAmB,IAAI,AAAD,MAAM,WACvC,OAAO;IAET,OAAOF;AACT;AAEO,SAASG,kBAAkBzC,IAAS,EAAE0C,OAAgB;IAC3D,OAAOzC,KAAK,SAAS,CAACD,MAAMqC,uBAAuBK;AACrD;AAIO,SAASC;IACd,OAAOC;AACT;AAEA,SAASC,SAAS,GAAGC,OAAc;IAEjC,MAAMC,YAAYjD,QAAQ,GAAG,CAACkD,oBAAAA,mBAAmBA,CAAC;IAClD,IAAID,WACFhE,QAAQ,GAAG,CAAC,iBAAiB+D;AAEjC;AAEA,IAAIG,sBAAsB;AACnB,SAASC,uBAAuB,EAAEC,OAAO,EAAuB;IACrE,IAAIC,UAAU;IACd,IAAIC,YAAY;IAEhB,MAAMC,YAAYC,AAAAA,IAAAA,oBAAAA,sBAAAA,AAAAA;IAElB,IAAI;QACFH,UAAUI,AAAAA,IAAAA,4CAAAA,QAAAA,AAAAA,EAAS,sCAAsC,QAAQ,GAAG,IAAI;QACxEH,YAAYG,AAAAA,IAAAA,4CAAAA,QAAAA,AAAAA,EAAS,+BAA+B,QAAQ,GAAG,IAAI;IACrE,EAAE,OAAOC,OAAO;QACdZ,SAAS,2BAA2BY;IACtC;IAOA,IACEH,aACEF,CAAAA,WAAWA,YAAYH,uBAAyB,CAACG,WAAWD,OAAM,GACpE;QACAN,SAAS,iCAAiC;YACxCS;YACAF;YACAD;YACAE;QACF;QAEAK,MAAMJ,WAAW;YACf,QAAQ;YACR,SAAS;gBACP,gBAAgB;YAClB;YACA,MAAMrD,KAAK,SAAS,CAAC;gBACnB,UAAUmD;gBACV,UAAUD;gBACV,YAAYE;YACd;QACF,GACG,IAAI,CAAC,CAACM,WAAaA,SAAS,IAAI,IAChC,IAAI,CAAC,CAAC3D;YACL6C,SAAS,8CAA8C7C;QACzD,GACC,KAAK,CAAC,CAACyD,QACNZ,SAAS,yCAAyCY;QAEtDR,sBAAsBG;IACxB;AACF"}

package/dist/types/ai-model/common.d.ts

@@ -1,6 +1,8 @@
-import type { AIUsageInfo, BaseElement, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
+import type { AIUsageInfo, BaseElement, DeviceAction, ElementTreeNode, MidsceneYamlFlowItem, PlanningAction, Rect, Size } from '../types';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import type { PlanningLocateParam } from '../types';
+import { type IModelPreferences } from '@midscene/shared/env';
+import { z } from 'zod';
 export type AIArgs = [
     ChatCompletionSystemMessageParam,
     ...ChatCompletionUserMessageParam[]
@@ -12,23 +14,175 @@ export declare enum AIActionType {
     PLAN = 3,
     DESCRIBE_ELEMENT = 4
 }
-export declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
+export declare const actionSpaceTypePrefix = "action_space_";
+export declare function callAiFn<T>(msgs: AIArgs, AIActionTypeValue: AIActionType, modelPreferences: IModelPreferences): Promise<{
     content: T;
     usage?: AIUsageInfo;
 }>;
-export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number): PlanningLocateParam;
+export declare function fillBboxParam(locate: PlanningLocateParam, width: number, height: number, modelPreferences: IModelPreferences): PlanningLocateParam;
 export declare function adaptQwenBbox(bbox: number[]): [number, number, number, number];
 export declare function adaptDoubaoBbox(bbox: string[] | number[] | string, width: number, height: number): [number, number, number, number];
-export declare function adaptBbox(bbox: number[], width: number, height: number): [number, number, number, number];
+export declare function adaptBbox(bbox: number[], width: number, height: number, modelPreferences: IModelPreferences): [number, number, number, number];
 export declare function adaptGeminiBbox(bbox: number[], width: number, height: number): [number, number, number, number];
-export declare function adaptBboxToRect(bbox: number[], width: number, height: number, offsetX?: number, offsetY?: number): Rect;
-export declare function warnGPT4oSizeLimit(size: Size): void;
+export declare function adaptBboxToRect(bbox: number[], width: number, height: number, modelPreferences: IModelPreferences, offsetX?: number, offsetY?: number): Rect;
+export declare function warnGPT4oSizeLimit(size: Size, modelPreferences: IModelPreferences): void;
 export declare function mergeRects(rects: Rect[]): {
     left: number;
     top: number;
     width: number;
     height: number;
 };
-export declare function expandSearchArea(rect: Rect, screenSize: Size): Rect;
+export declare function expandSearchArea(rect: Rect, screenSize: Size, modelPreferences: IModelPreferences): Rect;
 export declare function markupImageForLLM(screenshotBase64: string, tree: ElementTreeNode<BaseElement>, size: Size): Promise<string>;
-export declare function buildYamlFlowFromPlans(plans: PlanningAction[], sleep?: number): MidsceneYamlFlowItem[];
+export declare function buildYamlFlowFromPlans(plans: PlanningAction[], actionSpace: DeviceAction<any>[], sleep?: number): MidsceneYamlFlowItem[];
+export declare const PointSchema: z.ZodObject<{
+    left: z.ZodNumber;
+    top: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    left: number;
+    top: number;
+}, {
+    left: number;
+    top: number;
+}>;
+export declare const SizeSchema: z.ZodObject<{
+    width: z.ZodNumber;
+    height: z.ZodNumber;
+    dpr: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}>;
+export declare const RectSchema: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+    left: z.ZodNumber;
+    top: z.ZodNumber;
+}, "strip", z.ZodTypeAny, {
+    left: number;
+    top: number;
+}, {
+    left: number;
+    top: number;
+}>, z.ZodObject<{
+    width: z.ZodNumber;
+    height: z.ZodNumber;
+    dpr: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}, {
+    width: number;
+    height: number;
+    dpr?: number | undefined;
+}>>, z.ZodObject<{
+    zoom: z.ZodOptional<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    zoom?: number | undefined;
+}, {
+    zoom?: number | undefined;
+}>>;
+export declare const MidsceneLocation: z.ZodObject<{
+    midscene_location_field_flag: z.ZodLiteral<true>;
+    prompt: z.ZodString;
+    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
+    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+        left: z.ZodNumber;
+        top: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        left: number;
+        top: number;
+    }, {
+        left: number;
+        top: number;
+    }>, z.ZodObject<{
+        width: z.ZodNumber;
+        height: z.ZodNumber;
+        dpr: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }>>, z.ZodObject<{
+        zoom: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        zoom?: number | undefined;
+    }, {
+        zoom?: number | undefined;
+    }>>;
+}, "passthrough", z.ZodTypeAny, z.objectOutputType<{
+    midscene_location_field_flag: z.ZodLiteral<true>;
+    prompt: z.ZodString;
+    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
+    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+        left: z.ZodNumber;
+        top: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        left: number;
+        top: number;
+    }, {
+        left: number;
+        top: number;
+    }>, z.ZodObject<{
+        width: z.ZodNumber;
+        height: z.ZodNumber;
+        dpr: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }>>, z.ZodObject<{
+        zoom: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        zoom?: number | undefined;
+    }, {
+        zoom?: number | undefined;
+    }>>;
+}, z.ZodTypeAny, "passthrough">, z.objectInputType<{
+    midscene_location_field_flag: z.ZodLiteral<true>;
+    prompt: z.ZodString;
+    center: z.ZodTuple<[z.ZodNumber, z.ZodNumber], null>;
+    rect: z.ZodIntersection<z.ZodIntersection<z.ZodObject<{
+        left: z.ZodNumber;
+        top: z.ZodNumber;
+    }, "strip", z.ZodTypeAny, {
+        left: number;
+        top: number;
+    }, {
+        left: number;
+        top: number;
+    }>, z.ZodObject<{
+        width: z.ZodNumber;
+        height: z.ZodNumber;
+        dpr: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }, {
+        width: number;
+        height: number;
+        dpr?: number | undefined;
+    }>>, z.ZodObject<{
+        zoom: z.ZodOptional<z.ZodNumber>;
+    }, "strip", z.ZodTypeAny, {
+        zoom?: number | undefined;
+    }, {
+        zoom?: number | undefined;
+    }>>;
+}, z.ZodTypeAny, "passthrough">>;
+export type MidsceneLocationType = z.infer<typeof MidsceneLocation>;
+export declare const ifMidsceneLocatorField: (field: any) => boolean;
+export declare const findAllMidsceneLocatorField: (zodType?: z.ZodType<any>) => string[];

package/dist/types/ai-model/index.d.ts

@@ -4,8 +4,9 @@ export { describeUserPage, elementByPositionWithElementInfo, } from './prompt/ut
 export { generatePlaywrightTest, generatePlaywrightTestStream, } from './prompt/playwright-generator';
 export { generateYamlTest, generateYamlTestStream, } from './prompt/yaml-generator';
 export type { ChatCompletionMessageParam } from 'openai/resources/index';
-export { AiLocateElement, AiExtractElementInfo, AiAssert, AiLocateSection, } from './inspect';
+export { AiLocateElement, AiExtractElementInfo, AiLocateSection, } from './inspect';
 export { plan } from './llm-planning';
 export { callAiFn, adaptBboxToRect, } from './common';
 export { vlmPlanning, resizeImageForUiTars } from './ui-tars-planning';
 export { AIActionType, type AIArgs } from './common';
+export { actionSpaceTypePrefix, MidsceneLocation, type MidsceneLocationType, PointSchema, SizeSchema, RectSchema, } from './common';

package/dist/types/ai-model/inspect.d.ts

@@ -1,4 +1,5 @@
-import type { AIAssertionResponse, AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AISectionLocatorResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import type { AIDataExtractionResponse, AIElementLocatorResponse, AIElementResponse, AISectionLocatorResponse, AIUsageInfo, BaseElement, ElementById, InsightExtractOption, Rect, ReferenceImage, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import { type IModelPreferences } from '@midscene/shared/env';
 import type { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources/index';
 import { callAiFn } from './common';
 export type AIArgs = [
@@ -35,15 +36,9 @@ export declare function AiExtractElementInfo<T, ElementType extends BaseElement
     multimodalPrompt?: TMultimodalPrompt;
     context: UIContext<ElementType>;
     extractOption?: InsightExtractOption;
+    modelPreferences: IModelPreferences;
 }): Promise<{
     parseResult: AIDataExtractionResponse<T>;
     elementById: (idOrIndexId: string) => ElementType;
     usage: AIUsageInfo | undefined;
 }>;
-export declare function AiAssert<ElementType extends BaseElement = BaseElement>(options: {
-    assertion: TUserPrompt;
-    context: UIContext<ElementType>;
-}): Promise<{
-    content: AIAssertionResponse;
-    usage: AIUsageInfo | undefined;
-}>;

package/dist/types/ai-model/llm-planning.d.ts

@@ -3,7 +3,7 @@ import { callAiFn } from './common';
 export declare function plan(userInstruction: string, opts: {
     context: UIContext;
     pageType: PageType;
-    actionSpace: DeviceAction[];
+    actionSpace: DeviceAction<any>[];
     callAI?: typeof callAiFn<PlanningAIResponse>;
     log?: string;
     actionContext?: string;

package/dist/types/ai-model/prompt/assertion.d.ts

@@ -1,5 +1,2 @@
 import type { ResponseFormatJSONSchema } from 'openai/resources/index';
-export declare function systemPromptToAssert(model: {
-    isUITars: boolean;
-}): string;
 export declare const assertSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/llm-planning.d.ts

@@ -2,9 +2,9 @@ import type { DeviceAction } from '../../types';
 import { PromptTemplate } from '@langchain/core/prompts';
 import type { vlLocateMode } from '@midscene/shared/env';
 import type { ResponseFormatJSONSchema } from 'openai/resources/index';
-export declare const descriptionForAction: (action: DeviceAction, locatorScheme: string) => string;
+export declare const descriptionForAction: (action: DeviceAction<any>, locatorSchemaTypeDescription: string) => string;
 export declare function systemPromptToTaskPlanning({ actionSpace, vlMode, }: {
-    actionSpace: DeviceAction[];
+    actionSpace: DeviceAction<any>[];
     vlMode: ReturnType<typeof vlLocateMode>;
 }): Promise<string>;
 export declare const planSchema: ResponseFormatJSONSchema;

package/dist/types/ai-model/prompt/util.d.ts

@@ -1,4 +1,5 @@
 import type { BaseElement, ElementTreeNode, Size, UIContext } from '../../types';
+import { type IModelPreferences } from '@midscene/shared/env';
 export declare function describeSize(size: Size): string;
 export declare function describeElement(elements: (Pick<BaseElement, 'rect' | 'content'> & {
     id: string;
@@ -19,7 +20,7 @@ export declare function distance(point1: {
     y: number;
 }): number;
 export declare const samplePageDescription = "\nAnd the page is described as follows:\n====================\nThe size of the page: 1280 x 720\nSome of the elements are marked with a rectangle in the screenshot corresponding to the markerId, some are not.\n\nDescription of all the elements in screenshot:\n<div id=\"969f1637\" markerId=\"1\" left=\"100\" top=\"100\" width=\"100\" height=\"100\"> // The markerId indicated by the rectangle label in the screenshot\n  <h4 id=\"b211ecb2\" markerId=\"5\" left=\"150\" top=\"150\" width=\"90\" height=\"60\">\n    The username is accepted\n  </h4>\n  ...many more\n</div>\n====================\n";
-export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, opt?: {
+export declare function describeUserPage<ElementType extends BaseElement = BaseElement>(context: Omit<UIContext<ElementType>, 'describer'>, modelPreferences: IModelPreferences, opt?: {
     truncateTextLength?: number;
     filterNonTextContent?: boolean;
     domIncluded?: boolean | 'visible-only';

package/dist/types/ai-model/service-caller/index.d.ts

@@ -1,11 +1,10 @@
 import { type AIUsageInfo } from '../../types';
 import type { StreamingCallback } from '../../types';
+import { type IModelPreferences } from '@midscene/shared/env';
 import OpenAI from 'openai';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
 import { AIActionType, type AIArgs } from '../common';
-export declare function checkAIConfig(): boolean;
-export declare function getModelName(): string;
-export declare function call(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, responseFormat?: OpenAI.ChatCompletionCreateParams['response_format'] | OpenAI.ResponseFormatJSONObject, options?: {
+export declare function call(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelPreferences: IModelPreferences, options?: {
     stream?: boolean;
     onChunk?: StreamingCallback;
 }): Promise<{
@@ -13,14 +12,15 @@ export declare function call(messages: ChatCompletionMessageParam[], AIActionTyp
     usage?: AIUsageInfo;
     isStreamed: boolean;
 }>;
-export declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType): Promise<{
+export declare const getResponseFormat: (modelName: string, AIActionTypeValue: AIActionType) => OpenAI.ChatCompletionCreateParams["response_format"] | OpenAI.ResponseFormatJSONObject;
+export declare function callToGetJSONObject<T>(messages: ChatCompletionMessageParam[], AIActionTypeValue: AIActionType, modelPreferences: IModelPreferences): Promise<{
     content: T;
     usage?: AIUsageInfo;
 }>;
-export declare function callAiFnWithStringResponse<T>(msgs: AIArgs, AIActionTypeValue: AIActionType): Promise<{
+export declare function callAiFnWithStringResponse<T>(msgs: AIArgs, AIActionTypeValue: AIActionType, modelPreferences: IModelPreferences): Promise<{
     content: string;
     usage?: AIUsageInfo;
 }>;
 export declare function extractJSONFromCodeBlock(response: string): string;
 export declare function preprocessDoubaoBboxJson(input: string): string;
-export declare function safeParseJson(input: string): any;
+export declare function safeParseJson(input: string, modelPreferences: IModelPreferences): any;

package/dist/types/ai-model/ui-tars-planning.d.ts

@@ -1,4 +1,5 @@
 import type { AIUsageInfo, MidsceneYamlFlowItem, PlanningAction, Size } from '../types';
+import { type IModelPreferences } from '@midscene/shared/env';
 import { actionParser } from '@ui-tars/action-parser';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
 type ActionType = 'click' | 'drag' | 'type' | 'hotkey' | 'finished' | 'scroll' | 'wait' | 'androidBackButton' | 'androidHomeButton' | 'androidRecentAppsButton' | 'androidLongPress' | 'androidPull';
@@ -9,6 +10,7 @@ export declare function vlmPlanning(options: {
         width: number;
         height: number;
     };
+    modelPreferences: IModelPreferences;
 }): Promise<{
     actions: PlanningAction<any>[];
     actionsFromModel: ReturnType<typeof actionParser>['parsed'];
@@ -72,5 +74,5 @@ interface AndroidLongPressAction extends BaseAction {
     };
 }
 export type Action = ClickAction | DragAction | TypeAction | HotkeyAction | ScrollAction | FinishedAction | WaitAction | AndroidLongPressAction;
-export declare function resizeImageForUiTars(imageBase64: string, size: Size): Promise<string>;
+export declare function resizeImageForUiTars(imageBase64: string, size: Size, modelPreferences: IModelPreferences): Promise<string>;
 export {};

package/dist/types/index.d.ts

@@ -1,9 +1,11 @@
+import { z } from 'zod';
 import { Executor } from './ai-model/action-executor';
 import Insight from './insight/index';
 import { getVersion } from './utils';
-export { plan, describeUserPage, AiLocateElement, AiAssert, } from './ai-model/index';
+export { plan, describeUserPage, AiLocateElement, MidsceneLocation, type MidsceneLocationType, PointSchema, SizeSchema, RectSchema, } from './ai-model/index';
 export { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';
 export type * from './types';
+export { z };
 export default Insight;
 export { Executor, Insight, getVersion };
 export type { MidsceneYamlScript, MidsceneYamlTask, MidsceneYamlFlowItem, MidsceneYamlFlowItemAIRightClick, MidsceneYamlConfigResult, LocateOption, DetailedLocateParam, } from './yaml';

package/dist/types/insight/index.d.ts

@@ -1,5 +1,5 @@
 import { callAiFn } from '../ai-model/common';
-import type { AIDescribeElementResponse, AIElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightAssertionResponse, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, Rect, TMultimodalPrompt, TUserPrompt, UIContext } from '../types';
+import type { AIElementResponse, AIUsageInfo, BaseElement, DetailedLocateParam, DumpSubscriber, InsightAction, InsightExtractOption, InsightExtractParam, InsightOptions, InsightTaskInfo, LocateResult, TMultimodalPrompt, UIContext } from '../types';
 export interface LocateOpts {
     context?: UIContext<BaseElement>;
     callAI?: typeof callAiFn<AIElementResponse>;
@@ -19,8 +19,4 @@ export default class Insight<ElementType extends BaseElement = BaseElement, Cont
         thought?: string;
         usage?: AIUsageInfo;
     }>;
-    assert(assertion: TUserPrompt): Promise<InsightAssertionResponse>;
-    describe(target: Rect | [number, number], opt?: {
-        deepThink?: boolean;
-    }): Promise<Pick<AIDescribeElementResponse, 'description'>>;
 }

package/dist/types/types.d.ts

@@ -1,6 +1,7 @@
 import type { NodeType } from '@midscene/shared/constants';
 import type { BaseElement, ElementTreeNode, Rect, Size } from '@midscene/shared/types';
 import type { ChatCompletionMessageParam } from 'openai/resources/index';
+import type { z } from 'zod';
 import type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';
 export type { ElementTreeNode, BaseElement, Rect, Size, Point, } from '@midscene/shared/types';
 export * from './yaml';
@@ -9,6 +10,7 @@ export type AIUsageInfo = Record<string, any> & {
     completion_tokens: number | undefined;
     total_tokens: number | undefined;
     time_cost: number | undefined;
+    model_name: string | undefined;
 };
 /**
  * openai
@@ -105,10 +107,10 @@ export type EnsureObject<T> = {
 export type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';
 export type InsightExtractParam = string | Record<string, string>;
 export type LocateResultElement = {
-    id: string;
-    indexId?: number;
     center: [number, number];
     rect: Rect;
+    id: string;
+    indexId?: number;
     xpaths: string[];
     attributes: {
         nodeType: NodeType;
@@ -132,8 +134,6 @@ export interface InsightTaskInfo {
 export interface DumpMeta {
     sdkVersion: string;
     logTime: number;
-    model_name: string;
-    model_description?: string;
 }
 export interface ReportDumpWithAttributes {
     dumpString: string;
@@ -190,7 +190,7 @@ export interface PlanningLocateParam extends DetailedLocateParam {
 }
 export interface PlanningAction<ParamType = any> {
     thought?: string;
-    type: 'Locate' | 'Tap' | 'RightClick' | 'Hover' | 'Drag' | 'Input' | 'KeyboardPress' | 'Scroll' | 'Error' | 'Assert' | 'AssertWithoutThrow' | 'Sleep' | 'Finished' | 'AndroidBackButton' | 'AndroidHomeButton' | 'AndroidRecentAppsButton' | 'AndroidLongPress' | 'AndroidPull';
+    type: string;
     param: ParamType;
     locate?: PlanningLocateParam | null;
 }
@@ -242,7 +242,6 @@ export interface Color {
 }
 export interface BaseAgentParserOpt {
     selector?: string;
-    ignoreMarker?: boolean;
 }
 export interface PuppeteerParserOpt extends BaseAgentParserOpt {
 }
@@ -334,6 +333,8 @@ export type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;
 export interface GroupedActionDump {
     groupName: string;
     groupDescription?: string;
+    modelName: string;
+    modelDescription: string;
     executions: ExecutionDump[];
 }
 export type PageType = 'puppeteer' | 'playwright' | 'static' | 'chrome-extension-proxy' | 'android';
@@ -388,12 +389,10 @@ export type TMultimodalPrompt = {
 export type TUserPrompt = string | ({
     prompt: string;
 } & Partial<TMultimodalPrompt>);
-export interface DeviceAction<ParamType = any> {
+export interface DeviceAction<T = {}> {
     name: string;
     description?: string;
-    paramSchema?: string;
-    paramDescription?: string;
-    location?: 'required' | 'optional' | false;
-    whatToLocate?: string;
-    call: (context: ExecutorContext, param: ParamType) => Promise<void> | void;
+    interfaceAlias?: string;
+    paramSchema?: z.ZodType<T>;
+    call: (param: T, context: ExecutorContext) => Promise<void> | void;
 }

package/dist/types/yaml.d.ts

@@ -10,6 +10,8 @@ export interface InsightExtractOption {
     domIncluded?: boolean | 'visible-only';
     screenshotIncluded?: boolean;
     returnThought?: boolean;
+    isWaitForAssert?: boolean;
+    doNotThrowError?: boolean;
 }
 export interface ReferenceImage {
     base64: string;
@@ -110,16 +112,15 @@ export interface MidsceneYamlFlowItemAIHover extends LocateOption {
     aiHover: TUserPrompt;
 }
 export interface MidsceneYamlFlowItemAIInput extends LocateOption {
-    aiInput: string;
-    locate: TUserPrompt;
+    aiInput: TUserPrompt | undefined;
+    value: string;
 }
 export interface MidsceneYamlFlowItemAIKeyboardPress extends LocateOption {
-    aiKeyboardPress: string;
-    locate?: TUserPrompt;
+    aiKeyboardPress: TUserPrompt | undefined;
+    key: string;
 }
 export interface MidsceneYamlFlowItemAIScroll extends LocateOption, ScrollParam {
-    aiScroll: null;
-    locate?: TUserPrompt;
+    aiScroll: TUserPrompt | undefined;
 }
 export interface MidsceneYamlFlowItemEvaluateJavaScript {
     javascript: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.26.7-beta-20250818035341.0",
+  "version": "0.26.7-beta-20250820105545.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -60,8 +60,9 @@
     "langsmith": "0.3.7",
     "openai": "4.81.0",
     "socks-proxy-agent": "8.0.4",
-    "@midscene/shared": "0.26.7-beta-20250818035341.0",
-    "@midscene/recorder": "0.26.7-beta-20250818035341.0"
+    "zod": "3.24.3",
+    "@midscene/recorder": "0.26.7-beta-20250820105545.0",
+    "@midscene/shared": "0.26.7-beta-20250820105545.0"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.52.10",