@midscene/core 0.17.6-beta-20250607054355.0 → 0.18.1-beta-20250611081529.0

package/dist/lib/chunk-TO33YH6H.js.map

@@ -0,0 +1 @@
+{"version":3,"mappings":";AAAA,SAAS,gBAAgB;AACzB,SAAS,YAAY,WAAW,cAAc,qBAAqB;AACnE,SAAS,cAAc;AACvB,YAAY,UAAU;AACtB,SAAS,eAAe;AACxB;AAAA,EACE;AAAA,EACA;AAAA,OACK;AACP;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,yBAAyB;AAClC,SAAS,QAAQ,cAAc;AAC/B,SAAS,YAAY,aAAa,YAAY;AAC9C,OAAO,SAAS;AAGhB,IAAI,cAAc;AAElB,IAAM,aAAa;AAAA,EACjB;AACF;AACO,IAAM,2BAA2B;AAExC,IAAM,YAAY;AAElB,SAAS,eAAe;AACtB,SAAO;AACT;AAEO,SAAS,iCACd,KACA,QACA,aACA;AACA,QAAM,QAAQ,IAAI,QAAQ,MAAM;AAChC,SAAO,IAAI,MAAM,GAAG,KAAK,IAAI,cAAc,IAAI,MAAM,QAAQ,OAAO,MAAM;AAC5E;AAEO,SAAS,kBACd,UACA,YACQ;AACR,QAAM,MAAM,aAAa;AACzB,MAAI,CAAC,KAAK;AACR,YAAQ,KAAK,6CAA6C;AAC1D,WAAO;AAAA,EACT;AAEA,QAAM,kBAAkB;AAGxB,MAAI,CAAC,IAAI,SAAS,eAAe,GAAG;AAClC,YAAQ;AAAA,MACN;AAAA,IACF;AACA,WAAO;AAAA,EACT;AAGA,QAAM,mBAAmB,IAAI,QAAQ,eAAe;AAGpD,QAAM,YAAY,IAAI,UAAU,GAAG,gBAAgB;AACnD,QAAM,aAAa,IAAI,UAAU,mBAAmB,gBAAgB,MAAM;AAG1E,QAAM,cAAc,cAAc,CAAC;AACnC,MAAI,gBAAgB;AAGpB,QAAM,gBAAgB,CAAC,YAA0B;AAC/C,QAAI,aAAa;AACf,oBAAc,YAAa,GAAG,OAAO;AAAA,GAAM;AAAA,QACzC,MAAM;AAAA,MACR,CAAC;AAAA,IACH,OAAO;AACL,uBAAiB,GAAG,OAAO;AAAA;AAAA,IAC7B;AAAA,EACF;AAGA,MAAI,aAAa;AACf,kBAAc,YAAa,WAAW,EAAE,MAAM,IAAI,CAAC;AAAA,EACrD,OAAO;AACL,oBAAgB;AAAA,EAClB;AAIA,MACG,MAAM,QAAQ,QAAQ,KAAK,SAAS,WAAW,KAChD,OAAO,aAAa,aACpB;AACA,UAAM,cACJ;AACF,kBAAc,WAAW;AAAA,EAC3B,WAES,OAAO,aAAa,UAAU;AACrC,UAAM;AAAA;AAAA,MAEJ,gEACA,IAAI,UAAU,UAAU,IACxB;AAAA;AACF,kBAAc,WAAW;AAAA,EAC3B,OAEK;AAEH,aAAS,IAAI,GAAG,IAAI,SAAS,QAAQ,KAAK;AACxC,YAAM,EAAE,YAAY,WAAW,IAAI,SAAS,CAAC;AAC7C,YAAM,gBAAgB,OAAO,KAAK,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,QAAQ;AAC/D,eAAO,GAAG,GAAG,KAAK,mBAAmB,WAAY,GAAG,CAAC,CAAC;AAAA,MACxD,CAAC;AAED,YAAM;AAAA;AAAA,QAEJ,8DACA,cAAc,KAAK,GAAG,IACtB,QACA,IAAI,YAAY,UAAU,IAC1B;AAAA;AACF,oBAAc,WAAW;AAAA,IAC3B;AAAA,EACF;AAGA,MAAI,aAAa;AACf,kBAAc,YAAa,YAAY,EAAE,MAAM,IAAI,CAAC;AACpD,WAAO;AAAA,EACT;AAEA,mBAAiB;AACjB,SAAO;AACT;AAEO,SAAS,gBACd,UACA,UACe;AACf,MAAI,aAAa;AACf,YAAQ,IAAI,kCAAkC;AAC9C,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,QAAQ,UAAU;AACpC,QAAM,kBAAkB,kBAAkB,SAAS;AACnD,MAAI,CAAC,iBAAiB;AACpB,YAAQ,KAAK,kDAAkD;AAC/D,WAAO;AAAA,EACT;AAEA,QAAM,aAAkB;AAAA,IACtB,qBAAqB,QAAQ;AAAA,IAC7B,GAAG,QAAQ;AAAA,EACb;AAEA,oBAAkB,UAAU,UAAU;AAEtC,MAAI,QAAQ,IAAI,yBAAyB;AACvC,UAAM,WAAW,GAAG,UAAU;AAC9B,QAAI,OAAO;AAEX,QAAI,OAAO,aAAa,UAAU;AAChC,aAAO,KAAK,MAAM,QAAQ;AAAA,IAC5B;AAEA;AAAA,MACE;AAAA,MACA,KAAK;AAAA,QACH,KAAK,IAAI,CAAC,UAAU;AAAA,UAClB,YAAY,KAAK,MAAM,KAAK,UAAU;AAAA,UACtC,YAAY,KAAK;AAAA,QACnB,EAAE;AAAA,QACF;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAEA,WAAO,iCAAiC,QAAQ,EAAE;AAAA,EACpD;AAEA,SAAO;AACT;AAEO,SAAS,aAAa,MAM1B;AACD,MAAI,aAAa;AACf,WAAO;AAAA,EACT;AACA,QAAM,EAAE,UAAU,SAAS,aAAa,OAAO,OAAO,IAAI;AAC1D,QAAM,YAAY,qBAAqB,IAAI;AAE3C,MAAI,CAAC,aAAa;AAChB,WAAO,WAAW,+CAA+C;AAGjE,UAAM,gBAAqB,UAAK,WAAW,kBAAkB;AAC7D,UAAM,UAAe,UAAK,WAAW,YAAY;AACjD,QAAI,mBAAmB;AAEvB,QAAI,WAAW,OAAO,GAAG;AAEvB,UAAI,WAAW,aAAa,GAAG;AAC7B,2BAAmB,aAAa,eAAe,OAAO;AAAA,MACxD;AAGA,UAAI,CAAC,iBAAiB,SAAS,GAAG,iBAAiB,GAAG,GAAG;AACvD;AAAA,UACE;AAAA,UACA,GAAG,gBAAgB;AAAA;AAAA,EAA+B,iBAAiB;AAAA,EAAU,iBAAiB;AAAA,EAAY,iBAAiB;AAAA,EAAS,iBAAiB;AAAA;AAAA,UACrJ;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,kBAAc;AAAA,EAChB;AAEA,QAAM,WAAgB,UAAK,WAAW,GAAG,QAAQ,IAAI,OAAO,EAAE;AAE9D,MAAI,SAAS,QAAQ;AAEnB,kBAAc,UAAU,WAAW;AAAA,EACrC;AAEA,MAAI,MAAM,gBAAgB;AACxB,WAAO,gBAAgB,UAAU,WAAW;AAAA,EAC9C;AAEA,SAAO;AACT;AAEO,SAAS,YAA2B;AACzC,MAAI;AACF,UAAM,iBAAiB,kBAAkB;AACzC,QAAI,CAAC,gBAAgB;AACnB,aAAO;AAAA,IACT;AACA,UAAM,EAAE,KAAK,IAAI;AACjB,UAAM,UAAe,UAAK,OAAO,GAAG,IAAI;AACxC,cAAU,SAAS,EAAE,WAAW,KAAK,CAAC;AACtC,WAAO;AAAA,EACT,SAAS,GAAG;AACV,WAAO;AAAA,EACT;AACF;AAEO,SAAS,WAAW,mBAA0C;AACnE,MAAI,aAAa;AACf,WAAO;AAAA,EACT;AACA,QAAM,SAAS,UAAU;AACzB,QAAM,WAAW,GAAG,KAAK,CAAC,IAAI,iBAAiB;AAC/C,SAAY,UAAK,QAAS,QAAQ;AACpC;AAEO,SAAS,WAAW,WAAiB,QAAc;AAExD,SACE,UAAU,OAAO,OAAO,OAAO,OAAO,SACtC,UAAU,OAAO,UAAU,QAAQ,OAAO,QAC1C,UAAU,MAAM,OAAO,MAAM,OAAO,UACpC,UAAU,MAAM,UAAU,SAAS,OAAO;AAE9C;AAEA,eAAsB,MAAM,IAAY;AACtC,SAAO,IAAI,QAAQ,CAAC,YAAY,WAAW,SAAS,EAAE,CAAC;AACzD;AAEO,SAAS,sBAAsB,KAAa,OAAY;AAC7D,MAAI,SAAS,MAAM,aAAa,SAAS,QAAQ;AAC/C,WAAO;AAAA,EACT;AACA,MAAI,SAAS,MAAM,aAAa,SAAS,WAAW;AAClD,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEO,SAAS,kBAAkB,MAAW,SAAkB;AAC7D,SAAO,KAAK,UAAU,MAAM,uBAAuB,OAAO;AAC5D;AAIO,SAAS,aAAa;AAC3B,SAAO;AACT;AAEA,SAAS,YAAY,SAAgB;AACnC,QAAM,YAAY,YAAY,mBAAmB;AACjD,MAAI,WAAW;AACb,YAAQ,IAAI,cAAc,GAAG,OAAO;AAAA,EACtC;AACF;AAEA,IAAI,sBAAsB;AACnB,SAAS,uBAAuB,EAAE,QAAQ,GAAwB;AACvE,MAAI,UAAU;AACd,MAAI,YAAY;AAEhB,QAAM,cAAc,kBAAkB,gCAAgC;AACtE,QAAM,YAAY,aAAa;AAE/B,MAAI;AACF,cAAU,SAAS,oCAAoC,EAAE,SAAS,EAAE,KAAK;AACzE,gBAAY,SAAS,6BAA6B,EAAE,SAAS,EAAE,KAAK;AAAA,EACtE,SAAS,OAAO;AACd,aAAS,2BAA2B,KAAK;AAAA,EAC3C;AAOA,MACE,cACE,WAAW,YAAY,uBAAyB,CAAC,WAAW,UAC9D;AACA,aAAS,iCAAiC;AAAA,MACxC;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW;AAAA,MACf,QAAQ;AAAA,MACR,SAAS;AAAA,QACP,gBAAgB;AAAA,MAClB;AAAA,MACA,MAAM,KAAK,UAAU;AAAA,QACnB,UAAU;AAAA,QACV,UAAU;AAAA,QACV,YAAY;AAAA,MACd,CAAC;AAAA,IACH,CAAC,EACE,KAAK,CAAC,aAAa,SAAS,KAAK,CAAC,EAClC,KAAK,CAAC,SAAS;AACd,eAAS,8CAA8C,IAAI;AAAA,IAC7D,CAAC,EACA;AAAA,MAAM,CAAC,UACN,SAAS,yCAAyC,KAAK;AAAA,IACzD;AACF,0BAAsB;AAAA,EACxB;AACF","names":[],"ignoreList":[],"sources":["../../src/utils.ts"],"sourcesContent":["import { execSync } from 'node:child_process';\nimport { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';\nimport { tmpdir } from 'node:os';\nimport * as path from 'node:path';\nimport { dirname } from 'node:path';\nimport {\n  defaultRunDirName,\n  getMidsceneRunSubDir,\n} from '@midscene/shared/common';\nimport {\n  MIDSCENE_DEBUG_MODE,\n  MIDSCENE_OPENAI_INIT_CONFIG_JSON,\n  getAIConfig,\n  getAIConfigInJson,\n} from '@midscene/shared/env';\nimport { getRunningPkgInfo } from '@midscene/shared/fs';\nimport { assert, logMsg } from '@midscene/shared/utils';\nimport { escapeHtml, ifInBrowser, uuid } from '@midscene/shared/utils';\nimport xss from 'xss';\nimport type { Rect, ReportDumpWithAttributes } from './types';\n\nlet logEnvReady = false;\n\nconst xssOptions = {\n  escapeHtml,\n};\nexport const groupedActionDumpFileExt = 'web-dump.json';\n\nconst reportTpl = 'REPLACE_ME_WITH_REPORT_HTML';\n\nfunction getReportTpl() {\n  return reportTpl;\n}\n\nexport function replaceStringWithFirstAppearance(\n  str: string,\n  target: string,\n  replacement: string,\n) {\n  const index = str.indexOf(target);\n  return str.slice(0, index) + replacement + str.slice(index + target.length);\n}\n\nexport function reportHTMLContent(\n  dumpData: string | ReportDumpWithAttributes[],\n  reportPath?: string,\n): string {\n  const tpl = getReportTpl();\n  if (!tpl) {\n    console.warn('reportTpl is not set, will not write report');\n    return '';\n  }\n\n  const dumpPlaceholder = '{{dump}}';\n\n  // verify the template contains the placeholder\n  if (!tpl.includes(dumpPlaceholder)) {\n    console.warn(\n      'Failed to get the Midscene report template due to the lack of the {{dump}} placeholder. If you are building Midscene.js by yourself, please refer to the contribution guide for more information: https://github.com/web-infra-dev/midscene/blob/main/CONTRIBUTING.md#FAQ',\n    );\n    return '';\n  }\n\n  // find the first placeholder position\n  const placeholderIndex = tpl.indexOf(dumpPlaceholder);\n\n  // split the template into two parts before and after the placeholder\n  const firstPart = tpl.substring(0, placeholderIndex);\n  const secondPart = tpl.substring(placeholderIndex + dumpPlaceholder.length);\n\n  // if reportPath is set, it means we are in write to file mode\n  const writeToFile = reportPath && !ifInBrowser;\n  let resultContent = '';\n\n  // helper function: decide to write to file or append to resultContent\n  const appendOrWrite = (content: string): void => {\n    if (writeToFile) {\n      writeFileSync(reportPath!, `${content}\\n`, {\n        flag: 'a',\n      });\n    } else {\n      resultContent += `${content}\\n`;\n    }\n  };\n\n  // if writeToFile is true, write the first part to file, otherwise set the first part to the initial value of resultContent\n  if (writeToFile) {\n    writeFileSync(reportPath!, firstPart, { flag: 'w' }); // use 'w' flag to overwrite the existing file\n  } else {\n    resultContent = firstPart;\n  }\n\n  // generate dump content\n  // handle empty data or undefined\n  if (\n    (Array.isArray(dumpData) && dumpData.length === 0) ||\n    typeof dumpData === 'undefined'\n  ) {\n    const dumpContent =\n      '<script type=\"midscene_web_dump\" type=\"application/json\"></script>';\n    appendOrWrite(dumpContent);\n  }\n  // handle string type dumpData\n  else if (typeof dumpData === 'string') {\n    const dumpContent =\n      // biome-ignore lint/style/useTemplate: <explanation> do not use template string here, will cause bundle error\n      '<script type=\"midscene_web_dump\" type=\"application/json\">\\n' +\n      xss(dumpData, xssOptions) +\n      '\\n</script>';\n    appendOrWrite(dumpContent);\n  }\n  // handle array type dumpData\n  else {\n    // for array, handle each item\n    for (let i = 0; i < dumpData.length; i++) {\n      const { dumpString, attributes } = dumpData[i];\n      const attributesArr = Object.keys(attributes || {}).map((key) => {\n        return `${key}=\"${encodeURIComponent(attributes![key])}\"`;\n      });\n\n      const dumpContent =\n        // biome-ignore lint/style/useTemplate: <explanation> do not use template string here, will cause bundle error\n        '<script type=\"midscene_web_dump\" type=\"application/json\" ' +\n        attributesArr.join(' ') +\n        '>\\n' +\n        xss(dumpString, xssOptions) +\n        '\\n</script>';\n      appendOrWrite(dumpContent);\n    }\n  }\n\n  // add the second part\n  if (writeToFile) {\n    writeFileSync(reportPath!, secondPart, { flag: 'a' });\n    return reportPath!;\n  }\n\n  resultContent += secondPart;\n  return resultContent;\n}\n\nexport function writeDumpReport(\n  fileName: string,\n  dumpData: string | ReportDumpWithAttributes[],\n): string | null {\n  if (ifInBrowser) {\n    console.log('will not write report in browser');\n    return null;\n  }\n\n  const __dirname = dirname(__filename);\n  const midscenePkgInfo = getRunningPkgInfo(__dirname);\n  if (!midscenePkgInfo) {\n    console.warn('midscenePkgInfo not found, will not write report');\n    return null;\n  }\n\n  const reportPath = path.join(\n    getMidsceneRunSubDir('report'),\n    `${fileName}.html`,\n  );\n\n  reportHTMLContent(dumpData, reportPath);\n\n  if (process.env.MIDSCENE_DEBUG_LOG_JSON) {\n    const jsonPath = `${reportPath}.json`;\n    let data = dumpData as ReportDumpWithAttributes[];\n\n    if (typeof dumpData === 'string') {\n      data = JSON.parse(dumpData) as ReportDumpWithAttributes[];\n    }\n\n    writeFileSync(\n      jsonPath,\n      JSON.stringify(\n        data.map((item) => ({\n          dumpString: JSON.parse(item.dumpString),\n          attributes: item.attributes,\n        })),\n        null,\n        2,\n      ),\n    );\n\n    logMsg(`Midscene - dump file written: ${jsonPath}`);\n  }\n\n  return reportPath;\n}\n\nexport function writeLogFile(opts: {\n  fileName: string;\n  fileExt: string;\n  fileContent: string;\n  type: 'dump' | 'cache' | 'report' | 'tmp';\n  generateReport?: boolean;\n}) {\n  if (ifInBrowser) {\n    return '/mock/report.html';\n  }\n  const { fileName, fileExt, fileContent, type = 'dump' } = opts;\n  const targetDir = getMidsceneRunSubDir(type);\n  // Ensure directory exists\n  if (!logEnvReady) {\n    assert(targetDir, 'logDir should be set before writing dump file');\n\n    // gitIgnore in the parent directory\n    const gitIgnorePath = path.join(targetDir, '../../.gitignore');\n    const gitPath = path.join(targetDir, '../../.git');\n    let gitIgnoreContent = '';\n\n    if (existsSync(gitPath)) {\n      // if the git path exists, we need to add the log folder to the git ignore file\n      if (existsSync(gitIgnorePath)) {\n        gitIgnoreContent = readFileSync(gitIgnorePath, 'utf-8');\n      }\n\n      // ignore the log folder\n      if (!gitIgnoreContent.includes(`${defaultRunDirName}/`)) {\n        writeFileSync(\n          gitIgnorePath,\n          `${gitIgnoreContent}\\n# Midscene.js dump files\\n${defaultRunDirName}/dump\\n${defaultRunDirName}/report\\n${defaultRunDirName}/tmp\\n${defaultRunDirName}/log\\n`,\n          'utf-8',\n        );\n      }\n    }\n\n    logEnvReady = true;\n  }\n\n  const filePath = path.join(targetDir, `${fileName}.${fileExt}`);\n\n  if (type !== 'dump') {\n    // do not write dump file any more\n    writeFileSync(filePath, fileContent);\n  }\n\n  if (opts?.generateReport) {\n    return writeDumpReport(fileName, fileContent);\n  }\n\n  return filePath;\n}\n\nexport function getTmpDir(): string | null {\n  try {\n    const runningPkgInfo = getRunningPkgInfo();\n    if (!runningPkgInfo) {\n      return null;\n    }\n    const { name } = runningPkgInfo;\n    const tmpPath = path.join(tmpdir(), name);\n    mkdirSync(tmpPath, { recursive: true });\n    return tmpPath;\n  } catch (e) {\n    return null;\n  }\n}\n\nexport function getTmpFile(fileExtWithoutDot: string): string | null {\n  if (ifInBrowser) {\n    return null;\n  }\n  const tmpDir = getTmpDir();\n  const filename = `${uuid()}.${fileExtWithoutDot}`;\n  return path.join(tmpDir!, filename);\n}\n\nexport function overlapped(container: Rect, target: Rect) {\n  // container and the target have some part overlapped\n  return (\n    container.left < target.left + target.width &&\n    container.left + container.width > target.left &&\n    container.top < target.top + target.height &&\n    container.top + container.height > target.top\n  );\n}\n\nexport async function sleep(ms: number) {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\nexport function replacerForPageObject(key: string, value: any) {\n  if (value && value.constructor?.name === 'Page') {\n    return '[Page object]';\n  }\n  if (value && value.constructor?.name === 'Browser') {\n    return '[Browser object]';\n  }\n  return value;\n}\n\nexport function stringifyDumpData(data: any, indents?: number) {\n  return JSON.stringify(data, replacerForPageObject, indents);\n}\n\ndeclare const __VERSION__: string;\n\nexport function getVersion() {\n  return __VERSION__;\n}\n\nfunction debugLog(...message: any[]) {\n  const debugMode = getAIConfig(MIDSCENE_DEBUG_MODE);\n  if (debugMode) {\n    console.log('[Midscene]', ...message);\n  }\n}\n\nlet lastReportedRepoUrl = '';\nexport function uploadTestInfoToServer({ testUrl }: { testUrl: string }) {\n  let repoUrl = '';\n  let userEmail = '';\n\n  const extraConfig = getAIConfigInJson(MIDSCENE_OPENAI_INIT_CONFIG_JSON);\n  const serverUrl = extraConfig?.REPORT_SERVER_URL;\n\n  try {\n    repoUrl = execSync('git config --get remote.origin.url').toString().trim();\n    userEmail = execSync('git config --get user.email').toString().trim();\n  } catch (error) {\n    debugLog('Failed to get git info:', error);\n  }\n\n  // Only upload test info if:\n  // 1. Server URL is configured AND\n  // 2. Either:\n  //    - We have a repo URL that's different from last reported one (to avoid duplicate reports)\n  //    - OR we don't have a repo URL but have a test URL (for non-git environments)\n  if (\n    serverUrl &&\n    ((repoUrl && repoUrl !== lastReportedRepoUrl) || (!repoUrl && testUrl))\n  ) {\n    debugLog('Uploading test info to server', {\n      serverUrl,\n      repoUrl,\n      testUrl,\n      userEmail,\n    });\n\n    fetch(serverUrl, {\n      method: 'POST',\n      headers: {\n        'Content-Type': 'application/json',\n      },\n      body: JSON.stringify({\n        repo_url: repoUrl,\n        test_url: testUrl,\n        user_email: userEmail,\n      }),\n    })\n      .then((response) => response.json())\n      .then((data) => {\n        debugLog('Successfully uploaded test info to server:', data);\n      })\n      .catch((error) =>\n        debugLog('Failed to upload test info to server:', error),\n      );\n    lastReportedRepoUrl = repoUrl;\n  }\n}\n"]}

package/dist/lib/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractOption, h as InsightAssertionResponse, A as AIDescribeElementResponse } from './types-00c9086f.js';
-export { v as AIAssertionResponse, t as AIDataExtractionResponse, r as AIElementCoordinatesResponse, q as AIElementLocatorResponse, s as AIElementResponse, m as AIResponseFormat, u as AISectionLocatorResponse, p as AISingleElementResponse, n as AISingleElementResponseById, o as AISingleElementResponseByPosition, l as AIUsageInfo, Q as AgentAssertOpt, y as AgentDescribeElementAtPointResult, N as AgentWaitForOpt, a4 as BaseAgentParserOpt, C as CallAIFn, a3 as Color, G as DumpMeta, K as ElementById, z as EnsureObject, a7 as ExecutionRecorderItem, ap as ExecutionTaskAction, ao as ExecutionTaskActionApply, an as ExecutionTaskInsightAssertion, am as ExecutionTaskInsightAssertionApply, al as ExecutionTaskInsightAssertionParam, ae as ExecutionTaskInsightDumpLog, ag as ExecutionTaskInsightLocate, af as ExecutionTaskInsightLocateApply, ad as ExecutionTaskInsightLocateOutput, ac as ExecutionTaskInsightLocateParam, ak as ExecutionTaskInsightQuery, aj as ExecutionTaskInsightQueryApply, ai as ExecutionTaskInsightQueryOutput, ah as ExecutionTaskInsightQueryParam, ar as ExecutionTaskLog, aq as ExecutionTaskLogApply, at as ExecutionTaskPlanning, as as ExecutionTaskPlanningApply, ab as ExecutionTaskReturn, a8 as ExecutionTaskType, a9 as ExecutorContext, aT as FreeFn, au as GroupedActionDump, H as InsightDump, B as InsightExtractParam, J as LiteUISection, aw as LocateOption, F as LocateResultElement, x as LocateValidatorResult, w as LocatorValidatorOption, j as MidsceneYamlFlowItem, aD as MidsceneYamlFlowItemAIAction, aE as MidsceneYamlFlowItemAIAssert, aI as MidsceneYamlFlowItemAIBoolean, aM as MidsceneYamlFlowItemAIHover, aN as MidsceneYamlFlowItemAIInput, aO as MidsceneYamlFlowItemAIKeyboardPress, aJ as MidsceneYamlFlowItemAILocate, aH as MidsceneYamlFlowItemAINString, aG as MidsceneYamlFlowItemAINumber, aF as MidsceneYamlFlowItemAIQuery, k as MidsceneYamlFlowItemAIRightClick, aP as MidsceneYamlFlowItemAIScroll, aL as MidsceneYamlFlowItemAITap, aK as MidsceneYamlFlowItemAIWaitFor, aQ as MidsceneYamlFlowItemEvaluateJavaScript, aS as MidsceneYamlFlowItemLogScreenshot, aR as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aB as MidsceneYamlScriptAndroidEnv, aC as MidsceneYamlScriptEnv, az as MidsceneYamlScriptEnvBase, aA as MidsceneYamlScriptWebEnv, i as MidsceneYamlTask, O as OnTaskStartTip, av as PageType, P as PartialInsightDumpFromSDK, V as PlanningAIResponse, T as PlanningAction, $ as PlanningActionParamAssert, a1 as PlanningActionParamError, X as PlanningActionParamHover, Z as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamRightClick, _ as PlanningActionParamScroll, a0 as PlanningActionParamSleep, W as PlanningActionParamTap, a2 as PlanningActionParamWaitFor, S as PlanningLocateParam, a6 as PlaywrightParserOpt, a5 as PuppeteerParserOpt, ax as ReferenceImage, R as ReportDumpWithAttributes, aV as ScriptPlayerStatusValue, aU as ScriptPlayerTaskStatus, aa as TaskCacheInfo, ay as scrollParam } from './types-00c9086f.js';
-import { c as callAiFn } from './llm-planning-453ffce5.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-453ffce5.js';
+import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractOption, h as InsightAssertionResponse, A as AIDescribeElementResponse } from './types-e7be1eb0.js';
+export { v as AIAssertionResponse, t as AIDataExtractionResponse, r as AIElementCoordinatesResponse, q as AIElementLocatorResponse, s as AIElementResponse, m as AIResponseFormat, u as AISectionLocatorResponse, p as AISingleElementResponse, n as AISingleElementResponseById, o as AISingleElementResponseByPosition, l as AIUsageInfo, Q as AgentAssertOpt, y as AgentDescribeElementAtPointResult, N as AgentWaitForOpt, a4 as BaseAgentParserOpt, C as CallAIFn, a3 as Color, G as DumpMeta, K as ElementById, z as EnsureObject, a7 as ExecutionRecorderItem, ap as ExecutionTaskAction, ao as ExecutionTaskActionApply, an as ExecutionTaskInsightAssertion, am as ExecutionTaskInsightAssertionApply, al as ExecutionTaskInsightAssertionParam, ae as ExecutionTaskInsightDumpLog, ag as ExecutionTaskInsightLocate, af as ExecutionTaskInsightLocateApply, ad as ExecutionTaskInsightLocateOutput, ac as ExecutionTaskInsightLocateParam, ak as ExecutionTaskInsightQuery, aj as ExecutionTaskInsightQueryApply, ai as ExecutionTaskInsightQueryOutput, ah as ExecutionTaskInsightQueryParam, ar as ExecutionTaskLog, aq as ExecutionTaskLogApply, at as ExecutionTaskPlanning, as as ExecutionTaskPlanningApply, ab as ExecutionTaskReturn, a8 as ExecutionTaskType, a9 as ExecutorContext, aT as FreeFn, au as GroupedActionDump, H as InsightDump, B as InsightExtractParam, J as LiteUISection, aw as LocateOption, F as LocateResultElement, x as LocateValidatorResult, w as LocatorValidatorOption, j as MidsceneYamlFlowItem, aD as MidsceneYamlFlowItemAIAction, aE as MidsceneYamlFlowItemAIAssert, aI as MidsceneYamlFlowItemAIBoolean, aM as MidsceneYamlFlowItemAIHover, aN as MidsceneYamlFlowItemAIInput, aO as MidsceneYamlFlowItemAIKeyboardPress, aJ as MidsceneYamlFlowItemAILocate, aH as MidsceneYamlFlowItemAINString, aG as MidsceneYamlFlowItemAINumber, aF as MidsceneYamlFlowItemAIQuery, k as MidsceneYamlFlowItemAIRightClick, aP as MidsceneYamlFlowItemAIScroll, aL as MidsceneYamlFlowItemAITap, aK as MidsceneYamlFlowItemAIWaitFor, aQ as MidsceneYamlFlowItemEvaluateJavaScript, aS as MidsceneYamlFlowItemLogScreenshot, aR as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aB as MidsceneYamlScriptAndroidEnv, aC as MidsceneYamlScriptEnv, az as MidsceneYamlScriptEnvBase, aA as MidsceneYamlScriptWebEnv, i as MidsceneYamlTask, O as OnTaskStartTip, av as PageType, P as PartialInsightDumpFromSDK, V as PlanningAIResponse, T as PlanningAction, $ as PlanningActionParamAssert, a1 as PlanningActionParamError, X as PlanningActionParamHover, Z as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamRightClick, _ as PlanningActionParamScroll, a0 as PlanningActionParamSleep, W as PlanningActionParamTap, a2 as PlanningActionParamWaitFor, S as PlanningLocateParam, a6 as PlaywrightParserOpt, a5 as PuppeteerParserOpt, ax as ReferenceImage, R as ReportDumpWithAttributes, aV as ScriptPlayerStatusValue, aU as ScriptPlayerTaskStatus, aa as TaskCacheInfo, ay as scrollParam } from './types-e7be1eb0.js';
+import { c as callAiFn } from './llm-planning-45954424.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-45954424.js';
 import { BaseElement, Rect } from '@midscene/shared/types';
 export { BaseElement, ElementTreeNode, Point, Rect, Size } from '@midscene/shared/types';
 export { getVersion } from './utils.js';

package/dist/lib/index.js

@@ -1,6 +1,6 @@
 "use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
 
-var _chunkXIJPQF2Gjs = require('./chunk-XIJPQF2G.js');
+var _chunkTO33YH6Hjs = require('./chunk-TO33YH6H.js');
 
 
 
@@ -11,7 +11,7 @@ var _chunkXIJPQF2Gjs = require('./chunk-XIJPQF2G.js');
 
 
 
-var _chunk7MNTWX2Ajs = require('./chunk-7MNTWX2A.js');
+var _chunkCLWM3F4Jjs = require('./chunk-CLWM3F4J.js');
 
 // src/ai-model/action-executor.ts
 
@@ -121,11 +121,10 @@ ${_optionalChain([this, 'access', _7 => _7.latestErrorTask, 'call', _8 => _8(),
         task.status = "finished";
         task.timing.end = Date.now();
         task.timing.cost = task.timing.end - task.timing.start;
-        task.timing.aiCost = _optionalChain([returnValue, 'optionalAccess', _12 => _12.aiCost]) || 0;
         taskIndex++;
       } catch (e) {
         successfullyCompleted = false;
-        task.error = _optionalChain([e, 'optionalAccess', _13 => _13.message]) || (typeof e === "string" ? e : "error-without-message");
+        task.error = _optionalChain([e, 'optionalAccess', _12 => _12.message]) || (typeof e === "string" ? e : "error-without-message");
         task.errorStack = e.stack;
         task.status = "failed";
         task.timing.end = Date.now();
@@ -172,7 +171,7 @@ ${_optionalChain([this, 'access', _7 => _7.latestErrorTask, 'call', _8 => _8(),
       }
     }
     const dumpData = {
-      sdkVersion: _chunkXIJPQF2Gjs.getVersion.call(void 0, ),
+      sdkVersion: _chunkTO33YH6Hjs.getVersion.call(void 0, ),
       model_name: _env.getAIConfig.call(void 0, _env.MIDSCENE_MODEL_NAME) || "",
       model_description: modelDescription,
       logTime: Date.now(),
@@ -222,12 +221,9 @@ var _logger = require('@midscene/shared/logger');
 // src/insight/utils.ts
 
 
-
-
-
 function emitInsightDump(data, dumpSubscriber) {
   const baseData = {
-    sdkVersion: _chunkXIJPQF2Gjs.getVersion.call(void 0, ),
+    sdkVersion: _chunkTO33YH6Hjs.getVersion.call(void 0, ),
     logTime: Date.now(),
     model_name: _env.getAIConfig.call(void 0, _env.MIDSCENE_MODEL_NAME) || ""
   };
@@ -236,24 +232,24 @@ function emitInsightDump(data, dumpSubscriber) {
     ...baseData,
     ...data
   };
-  _optionalChain([dumpSubscriber, 'optionalCall', _14 => _14(finalData)]);
+  _optionalChain([dumpSubscriber, 'optionalCall', _13 => _13(finalData)]);
 }
 
 // src/insight/index.ts
 var debug = _logger.getDebug.call(void 0, "ai:insight");
 var Insight = class {
   constructor(context, opt) {
-    this.aiVendorFn = _chunk7MNTWX2Ajs.callAiFn;
+    this.aiVendorFn = _chunkCLWM3F4Jjs.callAiFn;
     _utils.assert.call(void 0, context, "context is required for Insight");
     if (typeof context === "function") {
       this.contextRetrieverFn = context;
     } else {
       this.contextRetrieverFn = () => Promise.resolve(context);
     }
-    if (typeof _optionalChain([opt, 'optionalAccess', _15 => _15.aiVendorFn]) !== "undefined") {
+    if (typeof _optionalChain([opt, 'optionalAccess', _14 => _14.aiVendorFn]) !== "undefined") {
       this.aiVendorFn = opt.aiVendorFn;
     }
-    if (typeof _optionalChain([opt, 'optionalAccess', _16 => _16.taskInfo]) !== "undefined") {
+    if (typeof _optionalChain([opt, 'optionalAccess', _15 => _15.taskInfo]) !== "undefined") {
       this.taskInfo = opt.taskInfo;
     }
   }
@@ -280,13 +276,13 @@ var Insight = class {
       );
       searchAreaPrompt = void 0;
     }
-    const context = _optionalChain([opt, 'optionalAccess', _17 => _17.context]) || await this.contextRetrieverFn("locate");
+    const context = _optionalChain([opt, 'optionalAccess', _16 => _16.context]) || await this.contextRetrieverFn("locate");
     let searchArea = void 0;
     let searchAreaRawResponse = void 0;
     let searchAreaUsage = void 0;
     let searchAreaResponse = void 0;
     if (searchAreaPrompt) {
-      searchAreaResponse = await _chunk7MNTWX2Ajs.AiLocateSection.call(void 0, {
+      searchAreaResponse = await _chunkCLWM3F4Jjs.AiLocateSection.call(void 0, {
         context,
         sectionDescription: searchAreaPrompt
       });
@@ -299,7 +295,7 @@ var Insight = class {
       searchArea = searchAreaResponse.rect;
     }
     const startTime = Date.now();
-    const { parseResult, rect, elementById, rawResponse, usage } = await _chunk7MNTWX2Ajs.AiLocateElement.call(void 0, {
+    const { parseResult, rect, elementById, rawResponse, usage } = await _chunkCLWM3F4Jjs.AiLocateElement.call(void 0, {
       callAI: callAI || this.aiVendorFn,
       context,
       targetElementDescription: queryPrompt,
@@ -317,7 +313,7 @@ var Insight = class {
       searchAreaUsage
     };
     let errorLog;
-    if (_optionalChain([parseResult, 'access', _18 => _18.errors, 'optionalAccess', _19 => _19.length])) {
+    if (_optionalChain([parseResult, 'access', _17 => _17.errors, 'optionalAccess', _18 => _18.length])) {
       errorLog = `AI model failed to locate: 
 ${parseResult.errors.join("\n")}`;
     }
@@ -336,7 +332,7 @@ ${parseResult.errors.join("\n")}`;
     const elements = [];
     (parseResult.elements || []).forEach((item) => {
       if ("id" in item) {
-        const element = elementById(_optionalChain([item, 'optionalAccess', _20 => _20.id]));
+        const element = elementById(_optionalChain([item, 'optionalAccess', _19 => _19.id]));
         if (!element) {
           console.warn(
             `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`
@@ -387,7 +383,7 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("extract");
     const startTime = Date.now();
-    const { parseResult, usage } = await _chunk7MNTWX2Ajs.AiExtractElementInfo.call(void 0, {
+    const { parseResult, usage } = await _chunkCLWM3F4Jjs.AiExtractElementInfo.call(void 0, {
       context,
       dataQuery: dataDemand,
       extractOption: opt
@@ -399,7 +395,7 @@ ${parseResult.errors.join("\n")}`;
       rawResponse: JSON.stringify(parseResult)
     };
     let errorLog;
-    if (_optionalChain([parseResult, 'access', _21 => _21.errors, 'optionalAccess', _22 => _22.length])) {
+    if (_optionalChain([parseResult, 'access', _20 => _20.errors, 'optionalAccess', _21 => _21.length])) {
       errorLog = `AI response error: 
 ${parseResult.errors.join("\n")}`;
     }
@@ -439,7 +435,7 @@ ${parseResult.errors.join("\n")}`;
     this.onceDumpUpdatedFn = void 0;
     const context = await this.contextRetrieverFn("assert");
     const startTime = Date.now();
-    const assertResult = await _chunk7MNTWX2Ajs.AiAssert.call(void 0, {
+    const assertResult = await _chunkCLWM3F4Jjs.AiAssert.call(void 0, {
       assertion,
       context
     });
@@ -491,8 +487,8 @@ ${parseResult.errors.join("\n")}`;
       ],
       borderThickness: 3
     });
-    if (_optionalChain([opt, 'optionalAccess', _23 => _23.deepThink])) {
-      const searchArea = _chunk7MNTWX2Ajs.expandSearchArea.call(void 0, targetRect, context.size);
+    if (_optionalChain([opt, 'optionalAccess', _22 => _22.deepThink])) {
+      const searchArea = _chunkCLWM3F4Jjs.expandSearchArea.call(void 0, targetRect, context.size);
       debug("describe: set searchArea", searchArea);
       imagePayload = await _img.cropByRect.call(void 0, 
         imagePayload,
@@ -515,7 +511,7 @@ ${parseResult.errors.join("\n")}`;
         ]
       }
     ];
-    const callAIFn = this.aiVendorFn || _chunk7MNTWX2Ajs.callToGetJSONObject;
+    const callAIFn = this.aiVendorFn || _chunkCLWM3F4Jjs.callToGetJSONObject;
     const res = await callAIFn(msgs, 4 /* DESCRIBE_ELEMENT */);
     const { content } = res;
     _utils.assert.call(void 0, !content.error, `describe failed: ${content.error}`);
@@ -538,6 +534,6 @@ var src_default = Insight;
 
 
 
-exports.AiAssert = _chunk7MNTWX2Ajs.AiAssert; exports.AiLocateElement = _chunk7MNTWX2Ajs.AiLocateElement; exports.Executor = Executor; exports.Insight = Insight; exports.MIDSCENE_MODEL_NAME = _env.MIDSCENE_MODEL_NAME; exports.default = src_default; exports.describeUserPage = _chunk7MNTWX2Ajs.describeUserPage; exports.getAIConfig = _env.getAIConfig; exports.getVersion = _chunkXIJPQF2Gjs.getVersion; exports.plan = _chunk7MNTWX2Ajs.plan;
+exports.AiAssert = _chunkCLWM3F4Jjs.AiAssert; exports.AiLocateElement = _chunkCLWM3F4Jjs.AiLocateElement; exports.Executor = Executor; exports.Insight = Insight; exports.MIDSCENE_MODEL_NAME = _env.MIDSCENE_MODEL_NAME; exports.default = src_default; exports.describeUserPage = _chunkCLWM3F4Jjs.describeUserPage; exports.getAIConfig = _env.getAIConfig; exports.getVersion = _chunkTO33YH6Hjs.getVersion; exports.plan = _chunkCLWM3F4Jjs.plan;
 
 //# sourceMappingURL=index.js.map

package/dist/lib/index.js.map

@@ -1 +1 @@
-{"version":3,"mappings":";;;;;;;;;;;;;;;;AAUA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY,YACjB,KAAK,YAAY,aACjB,KAAK,YAAY,YACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD,aAAK,OAAO,SAAU,aAAqB,UAAU;AACrD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,QAAI,mBAAmB;AAEvB,QAAI,aAAa,GAAG;AAClB,YAAM,iBAAiB,mBAAmB;AAC1C,UAAI,gBAAgB;AAClB,2BAAmB,WAAW,cAAc;AAAA,MAC9C,OAAO;AACL,2BAAmB,GAAG,aAAa,CAAC;AAAA,MACtC;AAAA,IACF;AACA,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,mBAAmB;AAAA,MACnB,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AC9NA,SAAS,4BAA4B;AAE9B,IAAM,8BAA8B,MAAM;AAC/C,SAAO,mNAAmN,qBAAqB,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAoBlP;;;ACQA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA,gBAAAA;AAAA,OACK;AACP,SAAS,yBAAyB,kBAAkB;AACpD,SAAS,gBAAgB;AACzB,SAAS,UAAAC,eAAc;;;AChCvB;AAAA,EACE,uBAAAC;AAAA,EACA,eAAAC;AAAA,OAGK;AACP,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAYA,aAAYD,oBAAmB,KAAK;AAAA,EAClD;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;ADoBA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAD,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,KACuB;AACvB,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA,QAAO,aAAa,8BAA8B;AAClD,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,IAAAA,QAAO,OAAO,UAAU,UAAU,sCAAsC;AAExE,UAAM,wBAAwB;AAAA,MAC5B;AAAA,IACF;AACA,QAAI,uBAAuB;AACzB,YAAM,yBAAyB,qBAAqB;AAAA,IACtD;AACA,QAAI;AACJ,QAAI,MAAM,aAAa,uBAAuB;AAC5C,yBAAmB,MAAM;AAAA,IAC3B;AAEA,QAAI,oBAAoB,CAACD,cAAa,GAAG;AACvC,cAAQ;AAAA,QACN;AAAA,MACF;AACA,yBAAmB;AAAA,IACrB;AAEA,UAAM,UAAU,KAAK,WAAY,MAAM,KAAK,mBAAmB,QAAQ;AAEvE,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAC;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,aAAa,aAAa,MAAM,IACzD,MAAM,gBAAgB;AAAA,MACpB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,cAAc;AAAA,IAChB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAgC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC1E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,CAAC,CAAC;AAAA,MACb,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,MAAM,EAAE;AAEpC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,UACnB,QAAQ,SAAS,CAAC,EAAG,UAAU,CAAC;AAAA,UAChC,YAAY,SAAS,CAAC,EAAG;AAAA,QAC3B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAYA,MAAM,QACJ,YACA,KACc;AACd,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,MACX,eAAe;AAAA,IACjB,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,KAAK,IAAI,eAAe,CAAC;AAGjC;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AAAA,EACA,MAAM,SACJ,QACA,KAGyD;AACzD,IAAAA,QAAO,QAAQ,yCAAyC;AACxD,UAAM,UAAU,MAAM,KAAK,mBAAmB,UAAU;AACxD,UAAM,EAAE,iBAAiB,IAAI;AAC7B,IAAAA,QAAO,kBAAkB,6CAA6C;AAEtE,UAAM,eAAe,4BAA4B;AAGjD,UAAM,kBAAkB;AACxB,UAAM,aAAmB,MAAM,QAAQ,MAAM,IACzC;AAAA,MACE,MAAM,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAChD,KAAK,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAC/C,OAAO;AAAA,MACP,QAAQ;AAAA,IACV,IACA;AAEJ,QAAI,eAAe,MAAM,wBAAwB;AAAA,MAC/C,gBAAgB;AAAA,MAChB,sBAAsB;AAAA,QACpB;AAAA,UACE,MAAM;AAAA,QACR;AAAA,MACF;AAAA,MACA,iBAAiB;AAAA,IACnB,CAAC;AAED,QAAI,KAAK,WAAW;AAClB,YAAM,aAAa,iBAAiB,YAAY,QAAQ,IAAI;AAC5D,YAAM,4BAA4B,UAAU;AAC5C,qBAAe,MAAM;AAAA,QACnB;AAAA,QACA;AAAA,QACA,qBAAqB,oBAAoB;AAAA,MAC3C;AAAA,IACF;AAEA,UAAM,OAAe;AAAA,MACnB,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,MACxC;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP;AAAA,YACE,MAAM;AAAA,YACN,WAAW;AAAA,cACT,KAAK;AAAA,cACL,QAAQ;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,WACJ,KAAK,cAAc;AAErB,UAAM,MAAM,MAAM,SAAS,8BAAmC;AAE9D,UAAM,EAAE,QAAQ,IAAI;AACpB,IAAAA,QAAO,CAAC,QAAQ,OAAO,oBAAoB,QAAQ,KAAK,EAAE;AAC1D,IAAAA,QAAO,QAAQ,aAAa,gCAAgC;AAC5D,WAAO;AAAA,EACT;AACF;;;AEzZA,SAAS,eAAAE,cAAa,uBAAAD,4BAA2B;AAGjD,IAAO,cAAQ","names":["vlLocateMode","assert","MIDSCENE_MODEL_NAME","getAIConfig"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/ai-model/prompt/describe.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        task.timing.aiCost = (returnValue as any)?.aiCost || 0;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    let modelDescription = '';\n\n    if (vlLocateMode()) {\n      const uiTarsModelVer = uiTarsModelVersion();\n      if (uiTarsModelVer) {\n        modelDescription = `UI-TARS=${uiTarsModelVer}`;\n      } else {\n        modelDescription = `${vlLocateMode()} mode`;\n      }\n    }\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      model_description: modelDescription,\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const elementDescriberInstruction = () => {\n  return `Tell what is the content of the element wrapped by the read rectangle in the screenshot. Your description is expected to be used to precisely locate the element from other similar elements on screenshot. Use ${getPreferredLanguage()} in the description.\n\nPlease follow the following rules:\n1. The description should be start with a brief description, like \"a button for confirming the action\".\n\n2. Include these information in the description to distinguish the element from its siblings and other similar elements, as much as possible:\n- The text of the element, like \"with text 'Confirm'\"\n- What the element looks like if it's an image, like \"with image '...'\"\n- The relative position of the element, like \"on the left of ..., around ...\"\n- How to distinguish the element from its siblings elements, like \"it is the icon instead of the text\"\n\n3. Do NOT mention the red rectangle in the description.\n\n4. Use the error field to describe the unexpected situations, if any. If not, put null.\n\nReturn in JSON:\n{\n  \"description\": \"[{brief description}]: {text of the element} {image of the element} {relative position of the element} ... \",\n  \"error\"?: \"...\"\n}`;\n};\n","import {\n  AIActionType,\n  type AIArgs,\n  callAiFn,\n  expandSearchArea,\n} from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callToGetJSONObject,\n} from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  MIDSCENE_USE_QWEN_VL,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n  callAI?: typeof callAiFn<AIElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, elementById, rawResponse, usage } =\n      await AiLocateElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T = any>(input: string, opt?: InsightExtractOption): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n    opt?: InsightExtractOption,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(\n    input: Record<keyof T, string>,\n    opt?: InsightExtractOption,\n  ): Promise<T>;\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    opt?: InsightExtractOption,\n  ): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      extractOption: opt,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n  async describe(\n    target: Rect | [number, number],\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64 } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size);\n      debug('describe: set searchArea', searchArea);\n      imagePayload = await cropByRect(\n        imagePayload,\n        searchArea,\n        getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n      );\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn =\n      this.aiVendorFn || callToGetJSONObject<AIDescribeElementResponse>;\n\n    const res = await callAIFn(msgs, AIActionType.DESCRIBE_ELEMENT);\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n","import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIRightClick,\n} from './yaml';\n"]}
+{"version":3,"mappings":";;;;;;;;;;;;;;;;AAUA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,cAAc;AAEhB,IAAM,WAAN,MAAe;AAAA,EAUpB,YACE,MACA,SAGA;AACA,SAAK,SACH,SAAS,SAAS,QAAQ,MAAM,SAAS,IAAI,YAAY;AAC3D,SAAK,OAAO;AACZ,SAAK,SAAS,SAAS,SAAS,CAAC,GAAG;AAAA,MAAI,CAAC,SACvC,KAAK,kBAAkB,IAAI;AAAA,IAC7B;AACA,SAAK,cAAc,SAAS;AAAA,EAC9B;AAAA,EAEQ,kBAAkB,MAAyC;AACjE,WAAO;AAAA,MACL,QAAQ;AAAA,MACR,GAAG;AAAA,IACL;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,MAAgE;AAC3E;AAAA,MACE,KAAK,WAAW;AAAA,MAChB;AAAA,QAAyD,KAAK,gBAAgB,GAAG,KAAK;AAAA,EAAK,KAAK,gBAAgB,GAAG,UAAU;AAAA,IAC/H;AACA,QAAI,MAAM,QAAQ,IAAI,GAAG;AACvB,WAAK,MAAM,KAAK,GAAG,KAAK,IAAI,CAAC,SAAS,KAAK,kBAAkB,IAAI,CAAC,CAAC;AAAA,IACrE,OAAO;AACL,WAAK,MAAM,KAAK,KAAK,kBAAkB,IAAI,CAAC;AAAA,IAC9C;AACA,QAAI,KAAK,WAAW,WAAW;AAC7B,WAAK,SAAS;AAAA,IAChB;AAAA,EACF;AAAA,EAEA,MAAM,QAAsB;AAC1B,QAAI,KAAK,WAAW,UAAU,KAAK,MAAM,SAAS,GAAG;AACnD,cAAQ;AAAA,QACN;AAAA,MACF;AAAA,IACF;AAEA,WAAO,KAAK,WAAW,WAAW,6BAA6B;AAC/D,WAAO,KAAK,WAAW,aAAa,+BAA+B;AACnE,WAAO,KAAK,WAAW,SAAS,4BAA4B;AAE5D,UAAM,mBAAmB,KAAK,MAAM;AAAA,MAClC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,mBAAmB,GAAG;AAExB;AAAA,IACF;AAEA,SAAK,SAAS;AACd,QAAI,YAAY;AAChB,QAAI,wBAAwB;AAE5B,QAAI;AAEJ,WAAO,YAAY,KAAK,MAAM,QAAQ;AACpC,YAAM,OAAO,KAAK,MAAM,SAAS;AACjC;AAAA,QACE,KAAK,WAAW;AAAA,QAChB,2CAA2C,KAAK,MAAM;AAAA,MACxD;AACA,WAAK,SAAS;AAAA,QACZ,OAAO,KAAK,IAAI;AAAA,MAClB;AACA,UAAI;AACF,aAAK,SAAS;AACd,YAAI;AACF,cAAI,KAAK,aAAa;AACpB,kBAAM,KAAK,YAAY,IAAI;AAAA,UAC7B;AAAA,QACF,SAAS,GAAG;AACV,kBAAQ,MAAM,wBAAwB,CAAC;AAAA,QACzC;AACA;AAAA,UACE,CAAC,WAAW,UAAU,UAAU,EAAE,QAAQ,KAAK,IAAI,KAAK;AAAA,UACxD,0BAA0B,KAAK,IAAI;AAAA,QACrC;AAEA,cAAM,EAAE,UAAU,MAAM,IAAI;AAC5B,eAAO,UAAU,uCAAuC,KAAK,IAAI,EAAE;AAEnE,YAAI;AACJ,cAAM,kBAAmC;AAAA,UACvC;AAAA,UACA,SAAS,oBAAoB;AAAA,QAC/B;AAEA,YAAI,KAAK,SAAS,WAAW;AAC3B;AAAA,YACE,KAAK,YAAY,YACf,KAAK,YAAY,WACjB,KAAK,YAAY,YACjB,KAAK,YAAY,aACjB,KAAK,YAAY,YACjB,KAAK,YAAY;AAAA,YACnB,gCAAgC,KAAK,OAAO;AAAA,UAC9C;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AACxD,cAAI,KAAK,YAAY,UAAU;AAC7B,iCACE,aACC;AAAA,UACL;AAAA,QACF,WAAW,KAAK,SAAS,YAAY,KAAK,SAAS,YAAY;AAC7D,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D,OAAO;AACL,kBAAQ;AAAA,YACN,0BAA0B,KAAK,IAAI;AAAA,UACrC;AACA,wBAAc,MAAM,KAAK,SAAS,OAAO,eAAe;AAAA,QAC1D;AAEA,eAAO,OAAO,MAAM,WAAW;AAC/B,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF,SAAS,GAAQ;AACf,gCAAwB;AACxB,aAAK,QACH,GAAG,YAAY,OAAO,MAAM,WAAW,IAAI;AAC7C,aAAK,aAAa,EAAE;AAEpB,aAAK,SAAS;AACd,aAAK,OAAO,MAAM,KAAK,IAAI;AAC3B,aAAK,OAAO,OAAO,KAAK,OAAO,MAAM,KAAK,OAAO;AACjD;AAAA,MACF;AAAA,IACF;AAGA,aAAS,IAAI,YAAY,GAAG,IAAI,KAAK,MAAM,QAAQ,KAAK;AACtD,WAAK,MAAM,CAAC,EAAE,SAAS;AAAA,IACzB;AAEA,QAAI,uBAAuB;AACzB,WAAK,SAAS;AAAA,IAChB,OAAO;AACL,WAAK,SAAS;AAAA,IAChB;AAEA,QAAI,KAAK,MAAM,QAAQ;AAErB,YAAM,cAAc,KAAK,IAAI,WAAW,KAAK,MAAM,SAAS,CAAC;AAC7D,aAAO,KAAK,MAAM,WAAW,EAAE;AAAA,IACjC;AAAA,EACF;AAAA,EAEA,iBAA0B;AACxB,WAAO,KAAK,WAAW;AAAA,EACzB;AAAA,EAEA,kBAAwC;AACtC,QAAI,KAAK,WAAW,SAAS;AAC3B,aAAO;AAAA,IACT;AACA,UAAM,iBAAiB,KAAK,MAAM;AAAA,MAChC,CAAC,SAAS,KAAK,WAAW;AAAA,IAC5B;AACA,QAAI,kBAAkB,GAAG;AACvB,aAAO,KAAK,MAAM,cAAc;AAAA,IAClC;AACA,WAAO;AAAA,EACT;AAAA,EAEA,OAAsB;AACpB,QAAI,mBAAmB;AAEvB,QAAI,aAAa,GAAG;AAClB,YAAM,iBAAiB,mBAAmB;AAC1C,UAAI,gBAAgB;AAClB,2BAAmB,WAAW,cAAc;AAAA,MAC9C,OAAO;AACL,2BAAmB,GAAG,aAAa,CAAC;AAAA,MACtC;AAAA,IACF;AACA,UAAM,WAA0B;AAAA,MAC9B,YAAY,WAAW;AAAA,MACvB,YAAY,YAAY,mBAAmB,KAAK;AAAA,MAChD,mBAAmB;AAAA,MACnB,SAAS,KAAK,IAAI;AAAA,MAClB,MAAM,KAAK;AAAA,MACX,OAAO,KAAK;AAAA,IACd;AACA,WAAO;AAAA,EACT;AACF;;;AC7NA,SAAS,4BAA4B;AAE9B,IAAM,8BAA8B,MAAM;AAC/C,SAAO,mNAAmN,qBAAqB,CAAC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAoBlP;;;ACQA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA,gBAAAA;AAAA,OACK;AACP,SAAS,yBAAyB,kBAAkB;AACpD,SAAS,gBAAgB;AACzB,SAAS,UAAAC,eAAc;;;AChCvB,SAAS,uBAAAC,sBAAqB,eAAAC,oBAAmB;AACjD,SAAS,YAAY;AAEd,SAAS,gBACd,MACA,gBACA;AACA,QAAM,WAAqB;AAAA,IACzB,YAAY,WAAW;AAAA,IACvB,SAAS,KAAK,IAAI;AAAA,IAClB,YAAYA,aAAYD,oBAAmB,KAAK;AAAA,EAClD;AACA,QAAM,YAAyB;AAAA,IAC7B,OAAO,KAAK;AAAA,IACZ,GAAG;AAAA,IACH,GAAG;AAAA,EACL;AAEA,mBAAiB,SAAS;AAC5B;;;ADyBA,IAAM,QAAQ,SAAS,YAAY;AACnC,IAAqB,UAArB,MAGE;AAAA,EAWA,YACE,SAGA,KACA;AAXF,sBAAoD;AAYlD,IAAAD,QAAO,SAAS,iCAAiC;AACjD,QAAI,OAAO,YAAY,YAAY;AACjC,WAAK,qBAAqB;AAAA,IAC5B,OAAO;AACL,WAAK,qBAAqB,MAAM,QAAQ,QAAQ,OAAO;AAAA,IACzD;AAEA,QAAI,OAAO,KAAK,eAAe,aAAa;AAC1C,WAAK,aAAa,IAAI;AAAA,IACxB;AACA,QAAI,OAAO,KAAK,aAAa,aAAa;AACxC,WAAK,WAAW,IAAI;AAAA,IACtB;AAAA,EACF;AAAA,EAEA,MAAM,OACJ,OACA,KACuB;AACvB,UAAM,EAAE,OAAO,IAAI,OAAO,CAAC;AAC3B,UAAM,cAAc,OAAO,UAAU,WAAW,QAAQ,MAAM;AAC9D,IAAAA,QAAO,aAAa,8BAA8B;AAClD,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,IAAAA,QAAO,OAAO,UAAU,UAAU,sCAAsC;AAExE,UAAM,wBAAwB;AAAA,MAC5B;AAAA,IACF;AACA,QAAI,uBAAuB;AACzB,YAAM,yBAAyB,qBAAqB;AAAA,IACtD;AACA,QAAI;AACJ,QAAI,MAAM,aAAa,uBAAuB;AAC5C,yBAAmB,MAAM;AAAA,IAC3B;AAEA,QAAI,oBAAoB,CAACD,cAAa,GAAG;AACvC,cAAQ;AAAA,QACN;AAAA,MACF;AACA,yBAAmB;AAAA,IACrB;AAEA,UAAM,UAAU,KAAK,WAAY,MAAM,KAAK,mBAAmB,QAAQ;AAEvE,QAAI,aAA+B;AACnC,QAAI,wBAA4C;AAChD,QAAI,kBAA2C;AAC/C,QAAI,qBAEY;AAChB,QAAI,kBAAkB;AACpB,2BAAqB,MAAM,gBAAgB;AAAA,QACzC;AAAA,QACA,oBAAoB;AAAA,MACtB,CAAC;AACD,MAAAC;AAAA,QACE,mBAAmB;AAAA,QACnB,gCAAgC,gBAAgB,IAC9C,mBAAmB,QAAQ,KAAK,mBAAmB,KAAK,KAAK,EAC/D;AAAA,MACF;AACA,8BAAwB,mBAAmB;AAC3C,wBAAkB,mBAAmB;AACrC,mBAAa,mBAAmB;AAAA,IAClC;AAEA,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,aAAa,aAAa,MAAM,IACzD,MAAM,gBAAgB;AAAA,MACpB,QAAQ,UAAU,KAAK;AAAA,MACvB;AAAA,MACA,0BAA0B;AAAA,MAC1B,cAAc;AAAA,IAChB,CAAC;AAEH,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,MACvC,gBAAgB,KAAK,UAAU,WAAW;AAAA,MAC1C;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAgC,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAC1E;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT,SAAS;AAAA,MACX;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,aAAa;AAAA,MACb,MAAM;AAAA,MACN;AAAA,MACA,WAAW,CAAC,CAAC;AAAA,MACb,OAAO;AAAA,IACT;AAEA,UAAM,WAA0B,CAAC;AACjC,KAAC,YAAY,YAAY,CAAC,GAAG,QAAQ,CAAC,SAAS;AAC7C,UAAI,QAAQ,MAAM;AAChB,cAAM,UAAU,YAAY,MAAM,EAAE;AAEpC,YAAI,CAAC,SAAS;AACZ,kBAAQ;AAAA,YACN,kCAAkC,KAAK,EAAE;AAAA,UAC3C;AACA;AAAA,QACF;AACA,iBAAS,KAAK,OAAO;AAAA,MACvB;AAAA,IACF,CAAC;AAED;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH,gBAAgB;AAAA,MAClB;AAAA,MACA;AAAA,IACF;AAEA,QAAI,UAAU;AACZ,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,IAAAA;AAAA,MACE,SAAS,UAAU;AAAA,MACnB,6CAA6C,SAAS,MAAM;AAAA,IAC9D;AAEA,QAAI,SAAS,WAAW,GAAG;AACzB,aAAO;AAAA,QACL,SAAS;AAAA,UACP,IAAI,SAAS,CAAC,EAAG;AAAA,UACjB,SAAS,SAAS,CAAC,EAAG;AAAA,UACtB,QAAQ,SAAS,CAAC,EAAG;AAAA,UACrB,MAAM,SAAS,CAAC,EAAG;AAAA,UACnB,QAAQ,SAAS,CAAC,EAAG,UAAU,CAAC;AAAA,UAChC,YAAY,SAAS,CAAC,EAAG;AAAA,QAC3B;AAAA,QACA;AAAA,MACF;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS;AAAA,MACT;AAAA,IACF;AAAA,EACF;AAAA,EAYA,MAAM,QACJ,YACA,KACc;AACd,IAAAA;AAAA,MACE,OAAO,eAAe,YAAY,OAAO,eAAe;AAAA,MACxD,kDAAkD,OAAO,UAAU;AAAA,IACrE;AACA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,SAAS;AAEvD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,EAAE,aAAa,MAAM,IAAI,MAAM,qBAAwB;AAAA,MAC3D;AAAA,MACA,WAAW;AAAA,MACX,eAAe;AAAA,IACjB,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,WAAW;AAAA,IACzC;AAEA,QAAI;AACJ,QAAI,YAAY,QAAQ,QAAQ;AAC9B,iBAAW;AAAA,EAAwB,YAAY,OAAO,KAAK,IAAI,CAAC;AAAA,IAClE;AAEA,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,OAAO;AAAA,IACT;AAEA,UAAM,EAAE,KAAK,IAAI,eAAe,CAAC;AAGjC;AAAA,MACE;AAAA,QACE,GAAG;AAAA,QACH;AAAA,MACF;AAAA,MACA;AAAA,IACF;AAEA,QAAI,YAAY,CAAC,MAAM;AACrB,YAAM,IAAI,MAAM,QAAQ;AAAA,IAC1B;AAEA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA,EAEA,MAAM,OAAO,WAAsD;AACjE,QAAI,OAAO,cAAc,UAAU;AACjC,YAAM,IAAI;AAAA,QACR;AAAA,MACF;AAAA,IACF;AAEA,UAAM,iBAAiB,KAAK;AAC5B,SAAK,oBAAoB;AAEzB,UAAM,UAAU,MAAM,KAAK,mBAAmB,QAAQ;AACtD,UAAM,YAAY,KAAK,IAAI;AAC3B,UAAM,eAAe,MAAM,SAAS;AAAA,MAClC;AAAA,MACA;AAAA,IACF,CAAC;AAED,UAAM,WAAW,KAAK,IAAI,IAAI;AAC9B,UAAM,WAA4B;AAAA,MAChC,GAAI,KAAK,WAAW,KAAK,WAAW,CAAC;AAAA,MACrC,YAAY;AAAA,MACZ,aAAa,KAAK,UAAU,aAAa,OAAO;AAAA,IAClD;AAEA,UAAM,EAAE,SAAS,KAAK,IAAI,aAAa;AACvC,UAAM,WAAsC;AAAA,MAC1C,MAAM;AAAA,MACN,WAAW;AAAA,QACT;AAAA,MACF;AAAA,MACA,gBAAgB,CAAC;AAAA,MACjB,MAAM;AAAA,MACN;AAAA,MACA,eAAe;AAAA,MACf,kBAAkB;AAAA,MAClB,OAAO,OAAO,SAAY;AAAA,IAC5B;AACA,oBAAgB,UAAU,cAAc;AAExC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,OAAO,aAAa;AAAA,IACtB;AAAA,EACF;AAAA,EACA,MAAM,SACJ,QACA,KAGyD;AACzD,IAAAA,QAAO,QAAQ,yCAAyC;AACxD,UAAM,UAAU,MAAM,KAAK,mBAAmB,UAAU;AACxD,UAAM,EAAE,iBAAiB,IAAI;AAC7B,IAAAA,QAAO,kBAAkB,6CAA6C;AAEtE,UAAM,eAAe,4BAA4B;AAGjD,UAAM,kBAAkB;AACxB,UAAM,aAAmB,MAAM,QAAQ,MAAM,IACzC;AAAA,MACE,MAAM,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAChD,KAAK,KAAK,MAAM,OAAO,CAAC,IAAI,kBAAkB,CAAC;AAAA,MAC/C,OAAO;AAAA,MACP,QAAQ;AAAA,IACV,IACA;AAEJ,QAAI,eAAe,MAAM,wBAAwB;AAAA,MAC/C,gBAAgB;AAAA,MAChB,sBAAsB;AAAA,QACpB;AAAA,UACE,MAAM;AAAA,QACR;AAAA,MACF;AAAA,MACA,iBAAiB;AAAA,IACnB,CAAC;AAED,QAAI,KAAK,WAAW;AAClB,YAAM,aAAa,iBAAiB,YAAY,QAAQ,IAAI;AAC5D,YAAM,4BAA4B,UAAU;AAC5C,qBAAe,MAAM;AAAA,QACnB;AAAA,QACA;AAAA,QACA,qBAAqB,oBAAoB;AAAA,MAC3C;AAAA,IACF;AAEA,UAAM,OAAe;AAAA,MACnB,EAAE,MAAM,UAAU,SAAS,aAAa;AAAA,MACxC;AAAA,QACE,MAAM;AAAA,QACN,SAAS;AAAA,UACP;AAAA,YACE,MAAM;AAAA,YACN,WAAW;AAAA,cACT,KAAK;AAAA,cACL,QAAQ;AAAA,YACV;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,WACJ,KAAK,cAAc;AAErB,UAAM,MAAM,MAAM,SAAS,8BAAmC;AAE9D,UAAM,EAAE,QAAQ,IAAI;AACpB,IAAAA,QAAO,CAAC,QAAQ,OAAO,oBAAoB,QAAQ,KAAK,EAAE;AAC1D,IAAAA,QAAO,QAAQ,aAAa,gCAAgC;AAC5D,WAAO;AAAA,EACT;AACF;;;AEzZA,SAAS,eAAAE,cAAa,uBAAAD,4BAA2B;AAGjD,IAAO,cAAQ","names":["vlLocateMode","assert","MIDSCENE_MODEL_NAME","getAIConfig"],"ignoreList":[],"sources":["../../src/ai-model/action-executor.ts","../../src/ai-model/prompt/describe.ts","../../src/insight/index.ts","../../src/insight/utils.ts","../../src/index.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionTask,\n  ExecutionTaskApply,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport {\n  MIDSCENE_MODEL_NAME,\n  getAIConfig,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { assert } from '@midscene/shared/utils';\n\nexport class Executor {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of executor\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  constructor(\n    name: string,\n    options?: ExecutionTaskProgressOptions & {\n      tasks?: ExecutionTaskApply[];\n    },\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  async append(task: ExecutionTaskApply[] | ExecutionTaskApply): Promise<void> {\n    assert(\n      this.status !== 'error',\n      `executor is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n  }\n\n  async flush(): Promise<any> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for executor, status is init but tasks are not empty',\n      );\n    }\n\n    assert(this.status !== 'running', 'executor is already running');\n    assert(this.status !== 'completed', 'executor is already completed');\n    assert(this.status !== 'error', 'executor is in error state');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskInsightLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Locate' ||\n              task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported insight subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskInsightLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action' || task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n\n    if (successfullyCompleted) {\n      this.status = 'completed';\n    } else {\n      this.status = 'error';\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      return this.tasks[outputIndex].output;\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    let modelDescription = '';\n\n    if (vlLocateMode()) {\n      const uiTarsModelVer = uiTarsModelVersion();\n      if (uiTarsModelVer) {\n        modelDescription = `UI-TARS=${uiTarsModelVer}`;\n      } else {\n        modelDescription = `${vlLocateMode()} mode`;\n      }\n    }\n    const dumpData: ExecutionDump = {\n      sdkVersion: getVersion(),\n      model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n      model_description: modelDescription,\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n}\n","import { getPreferredLanguage } from '@midscene/shared/env';\n\nexport const elementDescriberInstruction = () => {\n  return `Tell what is the content of the element wrapped by the read rectangle in the screenshot. Your description is expected to be used to precisely locate the element from other similar elements on screenshot. Use ${getPreferredLanguage()} in the description.\n\nPlease follow the following rules:\n1. The description should be start with a brief description, like \"a button for confirming the action\".\n\n2. Include these information in the description to distinguish the element from its siblings and other similar elements, as much as possible:\n- The text of the element, like \"with text 'Confirm'\"\n- What the element looks like if it's an image, like \"with image '...'\"\n- The relative position of the element, like \"on the left of ..., around ...\"\n- How to distinguish the element from its siblings elements, like \"it is the icon instead of the text\"\n\n3. Do NOT mention the red rectangle in the description.\n\n4. Use the error field to describe the unexpected situations, if any. If not, put null.\n\nReturn in JSON:\n{\n  \"description\": \"[{brief description}]: {text of the element} {image of the element} {relative position of the element} ... \",\n  \"error\"?: \"...\"\n}`;\n};\n","import {\n  AIActionType,\n  type AIArgs,\n  callAiFn,\n  expandSearchArea,\n} from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callToGetJSONObject,\n} from '@/ai-model/index';\nimport { AiAssert, AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightAssertionResponse,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport {\n  MIDSCENE_FORCE_DEEP_THINK,\n  MIDSCENE_USE_QWEN_VL,\n  getAIConfigInBoolean,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n  callAI?: typeof callAiFn<AIElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = getAIConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    if (searchAreaPrompt && !vlLocateMode()) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const { parseResult, rect, elementById, rawResponse, usage } =\n      await AiLocateElement({\n        callAI: callAI || this.aiVendorFn,\n        context,\n        targetElementDescription: queryPrompt,\n        searchConfig: searchAreaResponse,\n      });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T = any>(input: string, opt?: InsightExtractOption): Promise<T>;\n  async extract<T extends Record<string, string>>(\n    input: T,\n    opt?: InsightExtractOption,\n  ): Promise<Record<keyof T, any>>;\n  async extract<T extends object>(\n    input: Record<keyof T, string>,\n    opt?: InsightExtractOption,\n  ): Promise<T>;\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    opt?: InsightExtractOption,\n  ): Promise<any> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      extractOption: opt,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      usage,\n    };\n  }\n\n  async assert(assertion: string): Promise<InsightAssertionResponse> {\n    if (typeof assertion !== 'string') {\n      throw new Error(\n        'This is the assert method for Midscene, the first argument should be a string. If you want to use the assert method from Node.js, please import it from the Node.js assert module.',\n      );\n    }\n\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('assert');\n    const startTime = Date.now();\n    const assertResult = await AiAssert({\n      assertion,\n      context,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(assertResult.content),\n    };\n\n    const { thought, pass } = assertResult.content;\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'assert',\n      userQuery: {\n        assertion,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      assertionPass: pass,\n      assertionThought: thought,\n      error: pass ? undefined : thought,\n    };\n    emitInsightDump(dumpData, dumpSubscriber);\n\n    return {\n      pass,\n      thought,\n      usage: assertResult.usage,\n    };\n  }\n  async describe(\n    target: Rect | [number, number],\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64 } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size);\n      debug('describe: set searchArea', searchArea);\n      imagePayload = await cropByRect(\n        imagePayload,\n        searchArea,\n        getAIConfigInBoolean(MIDSCENE_USE_QWEN_VL),\n      );\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn =\n      this.aiVendorFn || callToGetJSONObject<AIDescribeElementResponse>;\n\n    const res = await callAIFn(msgs, AIActionType.DESCRIBE_ELEMENT);\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n","import type {\n  DumpMeta,\n  DumpSubscriber,\n  InsightDump,\n  PartialInsightDumpFromSDK,\n} from '@/types';\nimport { getVersion } from '@/utils';\nimport { MIDSCENE_MODEL_NAME, getAIConfig } from '@midscene/shared/env';\nimport { uuid } from '@midscene/shared/utils';\n\nexport function emitInsightDump(\n  data: PartialInsightDumpFromSDK,\n  dumpSubscriber?: DumpSubscriber,\n) {\n  const baseData: DumpMeta = {\n    sdkVersion: getVersion(),\n    logTime: Date.now(),\n    model_name: getAIConfig(MIDSCENE_MODEL_NAME) || '',\n  };\n  const finalData: InsightDump = {\n    logId: uuid(),\n    ...baseData,\n    ...data,\n  };\n\n  dumpSubscriber?.(finalData);\n}\n","import { Executor } from './ai-model/action-executor';\nimport Insight from './insight/index';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  AiAssert,\n} from './ai-model/index';\n\nexport { getAIConfig, MIDSCENE_MODEL_NAME } from '@midscene/shared/env';\n\nexport type * from './types';\nexport default Insight;\nexport { Executor, Insight, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlFlowItemAIRightClick,\n} from './yaml';\n"]}

package/dist/lib/{llm-planning-453ffce5.d.ts → llm-planning-45954424.d.ts}

@@ -1,4 +1,4 @@
-import { l as AIUsageInfo, U as UIContext, ax as ReferenceImage, q as AIElementLocatorResponse, K as ElementById, g as InsightExtractOption, t as AIDataExtractionResponse, v as AIAssertionResponse, av as PageType, V as PlanningAIResponse } from './types-00c9086f.js';
+import { l as AIUsageInfo, U as UIContext, ax as ReferenceImage, q as AIElementLocatorResponse, K as ElementById, g as InsightExtractOption, t as AIDataExtractionResponse, v as AIAssertionResponse, av as PageType, V as PlanningAIResponse } from './types-e7be1eb0.js';
 import { Rect, ElementTreeNode, BaseElement } from '@midscene/shared/types';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 

package/dist/{types/types-00c9086f.d.ts → lib/types-e7be1eb0.d.ts}

@@ -37,6 +37,7 @@ interface MidsceneYamlTask {
 }
 interface MidsceneYamlScriptEnvBase {
     output?: string;
+    unstableLogContent?: boolean | string;
     aiActionContext?: string;
 }
 interface MidsceneYamlScriptWebEnv extends MidsceneYamlScriptEnvBase {
@@ -141,9 +142,10 @@ interface ScriptPlayerTaskStatus extends MidsceneYamlTask {
 type ScriptPlayerStatusValue = 'init' | 'running' | 'done' | 'error';
 
 type AIUsageInfo = Record<string, any> & {
-    prompt_tokens: number;
-    completion_tokens: number;
-    total_tokens: number;
+    prompt_tokens: number | undefined;
+    completion_tokens: number | undefined;
+    total_tokens: number | undefined;
+    time_cost: number | undefined;
 };
 /**
  * openai
@@ -413,7 +415,6 @@ type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = ExecutionTaskAp
         start: number;
         end?: number;
         cost?: number;
-        aiCost?: number;
     };
     usage?: AIUsageInfo;
 };

package/dist/lib/utils.d.ts

@@ -1,4 +1,4 @@
-import { R as ReportDumpWithAttributes } from './types-00c9086f.js';
+import { R as ReportDumpWithAttributes } from './types-e7be1eb0.js';
 import { Rect } from '@midscene/shared/types';
 import '@midscene/shared/constants';
 import 'openai/resources';

package/dist/lib/utils.js

@@ -12,7 +12,7 @@
 
 
 
-var _chunkXIJPQF2Gjs = require('./chunk-XIJPQF2G.js');
+var _chunkTO33YH6Hjs = require('./chunk-TO33YH6H.js');
 
 
 
@@ -27,4 +27,4 @@ var _chunkXIJPQF2Gjs = require('./chunk-XIJPQF2G.js');
 
 
 
-exports.getTmpDir = _chunkXIJPQF2Gjs.getTmpDir; exports.getTmpFile = _chunkXIJPQF2Gjs.getTmpFile; exports.getVersion = _chunkXIJPQF2Gjs.getVersion; exports.groupedActionDumpFileExt = _chunkXIJPQF2Gjs.groupedActionDumpFileExt; exports.overlapped = _chunkXIJPQF2Gjs.overlapped; exports.replaceStringWithFirstAppearance = _chunkXIJPQF2Gjs.replaceStringWithFirstAppearance; exports.replacerForPageObject = _chunkXIJPQF2Gjs.replacerForPageObject; exports.reportHTMLContent = _chunkXIJPQF2Gjs.reportHTMLContent; exports.sleep = _chunkXIJPQF2Gjs.sleep; exports.stringifyDumpData = _chunkXIJPQF2Gjs.stringifyDumpData; exports.uploadTestInfoToServer = _chunkXIJPQF2Gjs.uploadTestInfoToServer; exports.writeDumpReport = _chunkXIJPQF2Gjs.writeDumpReport; exports.writeLogFile = _chunkXIJPQF2Gjs.writeLogFile;
+exports.getTmpDir = _chunkTO33YH6Hjs.getTmpDir; exports.getTmpFile = _chunkTO33YH6Hjs.getTmpFile; exports.getVersion = _chunkTO33YH6Hjs.getVersion; exports.groupedActionDumpFileExt = _chunkTO33YH6Hjs.groupedActionDumpFileExt; exports.overlapped = _chunkTO33YH6Hjs.overlapped; exports.replaceStringWithFirstAppearance = _chunkTO33YH6Hjs.replaceStringWithFirstAppearance; exports.replacerForPageObject = _chunkTO33YH6Hjs.replacerForPageObject; exports.reportHTMLContent = _chunkTO33YH6Hjs.reportHTMLContent; exports.sleep = _chunkTO33YH6Hjs.sleep; exports.stringifyDumpData = _chunkTO33YH6Hjs.stringifyDumpData; exports.uploadTestInfoToServer = _chunkTO33YH6Hjs.uploadTestInfoToServer; exports.writeDumpReport = _chunkTO33YH6Hjs.writeDumpReport; exports.writeLogFile = _chunkTO33YH6Hjs.writeLogFile;

package/dist/types/ai-model.d.ts

@@ -1,8 +1,8 @@
-import { l as AIUsageInfo, T as PlanningAction, j as MidsceneYamlFlowItem } from './types-00c9086f.js';
+import { l as AIUsageInfo, T as PlanningAction, j as MidsceneYamlFlowItem } from './types-e7be1eb0.js';
 import { ChatCompletionMessageParam } from 'openai/resources';
 export { ChatCompletionMessageParam } from 'openai/resources';
-import { b as AIActionType } from './llm-planning-453ffce5.js';
-export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-453ffce5.js';
+import { b as AIActionType } from './llm-planning-45954424.js';
+export { a as AiAssert, f as AiExtractElementInfo, A as AiLocateElement, g as AiLocateSection, h as adaptBboxToRect, c as callAiFn, d as describeUserPage, e as elementByPositionWithElementInfo, p as plan } from './llm-planning-45954424.js';
 import { vlLocateMode } from '@midscene/shared/env';
 import { actionParser } from '@ui-tars/action-parser';
 import { Size } from '@midscene/shared/types';

package/dist/types/index.d.ts

@@ -1,7 +1,7 @@
-import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractOption, h as InsightAssertionResponse, A as AIDescribeElementResponse } from './types-00c9086f.js';
-export { v as AIAssertionResponse, t as AIDataExtractionResponse, r as AIElementCoordinatesResponse, q as AIElementLocatorResponse, s as AIElementResponse, m as AIResponseFormat, u as AISectionLocatorResponse, p as AISingleElementResponse, n as AISingleElementResponseById, o as AISingleElementResponseByPosition, l as AIUsageInfo, Q as AgentAssertOpt, y as AgentDescribeElementAtPointResult, N as AgentWaitForOpt, a4 as BaseAgentParserOpt, C as CallAIFn, a3 as Color, G as DumpMeta, K as ElementById, z as EnsureObject, a7 as ExecutionRecorderItem, ap as ExecutionTaskAction, ao as ExecutionTaskActionApply, an as ExecutionTaskInsightAssertion, am as ExecutionTaskInsightAssertionApply, al as ExecutionTaskInsightAssertionParam, ae as ExecutionTaskInsightDumpLog, ag as ExecutionTaskInsightLocate, af as ExecutionTaskInsightLocateApply, ad as ExecutionTaskInsightLocateOutput, ac as ExecutionTaskInsightLocateParam, ak as ExecutionTaskInsightQuery, aj as ExecutionTaskInsightQueryApply, ai as ExecutionTaskInsightQueryOutput, ah as ExecutionTaskInsightQueryParam, ar as ExecutionTaskLog, aq as ExecutionTaskLogApply, at as ExecutionTaskPlanning, as as ExecutionTaskPlanningApply, ab as ExecutionTaskReturn, a8 as ExecutionTaskType, a9 as ExecutorContext, aT as FreeFn, au as GroupedActionDump, H as InsightDump, B as InsightExtractParam, J as LiteUISection, aw as LocateOption, F as LocateResultElement, x as LocateValidatorResult, w as LocatorValidatorOption, j as MidsceneYamlFlowItem, aD as MidsceneYamlFlowItemAIAction, aE as MidsceneYamlFlowItemAIAssert, aI as MidsceneYamlFlowItemAIBoolean, aM as MidsceneYamlFlowItemAIHover, aN as MidsceneYamlFlowItemAIInput, aO as MidsceneYamlFlowItemAIKeyboardPress, aJ as MidsceneYamlFlowItemAILocate, aH as MidsceneYamlFlowItemAINString, aG as MidsceneYamlFlowItemAINumber, aF as MidsceneYamlFlowItemAIQuery, k as MidsceneYamlFlowItemAIRightClick, aP as MidsceneYamlFlowItemAIScroll, aL as MidsceneYamlFlowItemAITap, aK as MidsceneYamlFlowItemAIWaitFor, aQ as MidsceneYamlFlowItemEvaluateJavaScript, aS as MidsceneYamlFlowItemLogScreenshot, aR as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aB as MidsceneYamlScriptAndroidEnv, aC as MidsceneYamlScriptEnv, az as MidsceneYamlScriptEnvBase, aA as MidsceneYamlScriptWebEnv, i as MidsceneYamlTask, O as OnTaskStartTip, av as PageType, P as PartialInsightDumpFromSDK, V as PlanningAIResponse, T as PlanningAction, $ as PlanningActionParamAssert, a1 as PlanningActionParamError, X as PlanningActionParamHover, Z as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamRightClick, _ as PlanningActionParamScroll, a0 as PlanningActionParamSleep, W as PlanningActionParamTap, a2 as PlanningActionParamWaitFor, S as PlanningLocateParam, a6 as PlaywrightParserOpt, a5 as PuppeteerParserOpt, ax as ReferenceImage, R as ReportDumpWithAttributes, aV as ScriptPlayerStatusValue, aU as ScriptPlayerTaskStatus, aa as TaskCacheInfo, ay as scrollParam } from './types-00c9086f.js';
-import { c as callAiFn } from './llm-planning-453ffce5.js';
-export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-453ffce5.js';
+import { E as ExecutionTask, a as ExecutionTaskProgressOptions, b as ExecutionTaskApply, c as ExecutionDump, U as UIContext, I as InsightAction, D as DumpSubscriber, d as InsightTaskInfo, e as InsightOptions, f as DetailedLocateParam, L as LocateResult, g as InsightExtractOption, h as InsightAssertionResponse, A as AIDescribeElementResponse } from './types-e7be1eb0.js';
+export { v as AIAssertionResponse, t as AIDataExtractionResponse, r as AIElementCoordinatesResponse, q as AIElementLocatorResponse, s as AIElementResponse, m as AIResponseFormat, u as AISectionLocatorResponse, p as AISingleElementResponse, n as AISingleElementResponseById, o as AISingleElementResponseByPosition, l as AIUsageInfo, Q as AgentAssertOpt, y as AgentDescribeElementAtPointResult, N as AgentWaitForOpt, a4 as BaseAgentParserOpt, C as CallAIFn, a3 as Color, G as DumpMeta, K as ElementById, z as EnsureObject, a7 as ExecutionRecorderItem, ap as ExecutionTaskAction, ao as ExecutionTaskActionApply, an as ExecutionTaskInsightAssertion, am as ExecutionTaskInsightAssertionApply, al as ExecutionTaskInsightAssertionParam, ae as ExecutionTaskInsightDumpLog, ag as ExecutionTaskInsightLocate, af as ExecutionTaskInsightLocateApply, ad as ExecutionTaskInsightLocateOutput, ac as ExecutionTaskInsightLocateParam, ak as ExecutionTaskInsightQuery, aj as ExecutionTaskInsightQueryApply, ai as ExecutionTaskInsightQueryOutput, ah as ExecutionTaskInsightQueryParam, ar as ExecutionTaskLog, aq as ExecutionTaskLogApply, at as ExecutionTaskPlanning, as as ExecutionTaskPlanningApply, ab as ExecutionTaskReturn, a8 as ExecutionTaskType, a9 as ExecutorContext, aT as FreeFn, au as GroupedActionDump, H as InsightDump, B as InsightExtractParam, J as LiteUISection, aw as LocateOption, F as LocateResultElement, x as LocateValidatorResult, w as LocatorValidatorOption, j as MidsceneYamlFlowItem, aD as MidsceneYamlFlowItemAIAction, aE as MidsceneYamlFlowItemAIAssert, aI as MidsceneYamlFlowItemAIBoolean, aM as MidsceneYamlFlowItemAIHover, aN as MidsceneYamlFlowItemAIInput, aO as MidsceneYamlFlowItemAIKeyboardPress, aJ as MidsceneYamlFlowItemAILocate, aH as MidsceneYamlFlowItemAINString, aG as MidsceneYamlFlowItemAINumber, aF as MidsceneYamlFlowItemAIQuery, k as MidsceneYamlFlowItemAIRightClick, aP as MidsceneYamlFlowItemAIScroll, aL as MidsceneYamlFlowItemAITap, aK as MidsceneYamlFlowItemAIWaitFor, aQ as MidsceneYamlFlowItemEvaluateJavaScript, aS as MidsceneYamlFlowItemLogScreenshot, aR as MidsceneYamlFlowItemSleep, M as MidsceneYamlScript, aB as MidsceneYamlScriptAndroidEnv, aC as MidsceneYamlScriptEnv, az as MidsceneYamlScriptEnvBase, aA as MidsceneYamlScriptWebEnv, i as MidsceneYamlTask, O as OnTaskStartTip, av as PageType, P as PartialInsightDumpFromSDK, V as PlanningAIResponse, T as PlanningAction, $ as PlanningActionParamAssert, a1 as PlanningActionParamError, X as PlanningActionParamHover, Z as PlanningActionParamInputOrKeyPress, Y as PlanningActionParamRightClick, _ as PlanningActionParamScroll, a0 as PlanningActionParamSleep, W as PlanningActionParamTap, a2 as PlanningActionParamWaitFor, S as PlanningLocateParam, a6 as PlaywrightParserOpt, a5 as PuppeteerParserOpt, ax as ReferenceImage, R as ReportDumpWithAttributes, aV as ScriptPlayerStatusValue, aU as ScriptPlayerTaskStatus, aa as TaskCacheInfo, ay as scrollParam } from './types-e7be1eb0.js';
+import { c as callAiFn } from './llm-planning-45954424.js';
+export { a as AiAssert, A as AiLocateElement, d as describeUserPage, p as plan } from './llm-planning-45954424.js';
 import { BaseElement, Rect } from '@midscene/shared/types';
 export { BaseElement, ElementTreeNode, Point, Rect, Size } from '@midscene/shared/types';
 export { getVersion } from './utils.js';

package/dist/types/{llm-planning-453ffce5.d.ts → llm-planning-45954424.d.ts}

@@ -1,4 +1,4 @@
-import { l as AIUsageInfo, U as UIContext, ax as ReferenceImage, q as AIElementLocatorResponse, K as ElementById, g as InsightExtractOption, t as AIDataExtractionResponse, v as AIAssertionResponse, av as PageType, V as PlanningAIResponse } from './types-00c9086f.js';
+import { l as AIUsageInfo, U as UIContext, ax as ReferenceImage, q as AIElementLocatorResponse, K as ElementById, g as InsightExtractOption, t as AIDataExtractionResponse, v as AIAssertionResponse, av as PageType, V as PlanningAIResponse } from './types-e7be1eb0.js';
 import { Rect, ElementTreeNode, BaseElement } from '@midscene/shared/types';
 import { ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam } from 'openai/resources';
 

package/dist/{lib/types-00c9086f.d.ts → types/types-e7be1eb0.d.ts}

@@ -37,6 +37,7 @@ interface MidsceneYamlTask {
 }
 interface MidsceneYamlScriptEnvBase {
     output?: string;
+    unstableLogContent?: boolean | string;
     aiActionContext?: string;
 }
 interface MidsceneYamlScriptWebEnv extends MidsceneYamlScriptEnvBase {
@@ -141,9 +142,10 @@ interface ScriptPlayerTaskStatus extends MidsceneYamlTask {
 type ScriptPlayerStatusValue = 'init' | 'running' | 'done' | 'error';
 
 type AIUsageInfo = Record<string, any> & {
-    prompt_tokens: number;
-    completion_tokens: number;
-    total_tokens: number;
+    prompt_tokens: number | undefined;
+    completion_tokens: number | undefined;
+    total_tokens: number | undefined;
+    time_cost: number | undefined;
 };
 /**
  * openai
@@ -413,7 +415,6 @@ type ExecutionTask<E extends ExecutionTaskApply<any, any, any> = ExecutionTaskAp
         start: number;
         end?: number;
         cost?: number;
-        aiCost?: number;
     };
     usage?: AIUsageInfo;
 };

package/dist/types/utils.d.ts

@@ -1,4 +1,4 @@
-import { R as ReportDumpWithAttributes } from './types-00c9086f.js';
+import { R as ReportDumpWithAttributes } from './types-e7be1eb0.js';
 import { Rect } from '@midscene/shared/types';
 import '@midscene/shared/constants';
 import 'openai/resources';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "0.17.6-beta-20250607054355.0",
+  "version": "0.18.1-beta-20250611081529.0",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "jsnext:source": "./src/index.ts",
@@ -45,7 +45,7 @@
     "openai": "4.81.0",
     "socks-proxy-agent": "8.0.4",
     "xss": "1.0.15",
-    "@midscene/shared": "0.17.6-beta-20250607054355.0"
+    "@midscene/shared": "0.18.1-beta-20250611081529.0"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.60.6",