@aiscene/core 8.0.2 → 8.0.3

package/dist/es/agent/utils.mjs

@@ -26,14 +26,31 @@ async function commonContextParser(interfaceInstance, _opt) {
     if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) throw new Error(`Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
     if (logicalWidth <= 0 || logicalHeight <= 0) throw new Error(`Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
     debug(`size: ${logicalWidth}x${logicalHeight}`);
-    const screenshotBase64 = await interfaceInstance.screenshotBase64();
-    const screenshotCapturedAt = Date.now();
-    assert(screenshotBase64, 'screenshotBase64 is required');
-    debug('will get screenshot dimensions');
-    const { width: imgWidth, height: imgHeight } = await imageInfoOfBase64(screenshotBase64);
-    if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) throw new Error(`Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
-    if (imgWidth <= 0 || imgHeight <= 0) throw new Error(`Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
-    debug('screenshot dimensions', imgWidth, 'x', imgHeight);
+    const maxRetries = 3;
+    let lastError = null;
+    let screenshotBase64 = null;
+    let screenshotCapturedAt = 0;
+    let imgWidth = 0;
+    let imgHeight = 0;
+    for(let attempt = 1; attempt <= maxRetries; attempt++)try {
+        screenshotBase64 = await interfaceInstance.screenshotBase64();
+        screenshotCapturedAt = Date.now();
+        assert(screenshotBase64, 'screenshotBase64 is required');
+        debug('will get screenshot dimensions');
+        const dimensions = await imageInfoOfBase64(screenshotBase64);
+        imgWidth = dimensions.width;
+        imgHeight = dimensions.height;
+        if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) throw new Error(`Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
+        if (imgWidth <= 0 || imgHeight <= 0) throw new Error(`Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
+        debug('screenshot dimensions', imgWidth, 'x', imgHeight);
+        break;
+    } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+        debug(`Screenshot capture/decode failed (attempt ${attempt}/${maxRetries}): ${lastError.message}`);
+        if (attempt === maxRetries) throw new Error(`Failed to capture valid screenshot after ${maxRetries} attempts: ${lastError.message}`);
+        await new Promise((resolve)=>setTimeout(resolve, 500));
+    }
+    if (!screenshotBase64) throw new Error('Screenshot data is null after retry loop');
     const logicalIsPortrait = logicalWidth < logicalHeight;
     const screenshotIsPortrait = imgWidth < imgHeight;
     let finalLogicalWidth = logicalWidth;
@@ -123,7 +140,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"8.0.1";
+const getMidsceneVersion = ()=>"8.0.3";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport type { TModelFamily } from '@aiscene/shared/env';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@aiscene/shared/env';\nimport { generateElementByRect } from '@aiscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { _keyDefinitions } from '@aiscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@aiscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n    modelFamily?: TModelFamily;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    // Convert bbox [x1, y1, x2, y2] to rect {left, top, width, height}\n    const rect = {\n      left: planLocateParam.bbox[0],\n      top: planLocateParam.bbox[1],\n      width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,\n      height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,\n    };\n\n    const element = generateElementByRect(\n      rect,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","rect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;AA0BO,eAAeA,oBACpBC,iBAAoC,EACpCC,IAIC;IAED,MAAMC,QAAQC,SAAS;IAEvBC,OAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,uBAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMb,kBAAkB,gBAAgB;IACjE,MAAMc,uBAAuBC,KAAK,GAAG;IACrCX,OAAOS,kBAAmB;IAG1BX,MAAM;IACN,MAAM,EAAE,OAAOc,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjIf,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9ClB,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBrB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBnB,MAAM,mBAAmBqB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCpB,MAAM,4BAA4BsB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CpB,MACE,CAAC,mCAAmC,EAAEoB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,eAAe,MAAM,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QAExB,MAAMG,OAAO;YACX,MAAMH,gBAAgB,IAAI,CAAC,EAAE;YAC7B,KAAKA,gBAAgB,IAAI,CAAC,EAAE;YAC5B,OAAOA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;YAC3D,QAAQA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;QAC9D;QAEA,MAAMI,UAAUC,sBACdF,MACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOI;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMR,OACJ,MAAMI,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNvB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCtB,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOM,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,8BAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAzB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOyB;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNvB,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;YAC/BE,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;SAChC;QACD,MAAM;YACJ,GAAGyB,QAAQ,IAAI;YACf,MAAMvB,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,IAAI,GAAGzB;YACrC,KAAKE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,GAAG,GAAGzB;YACnC,OAAOE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,KAAK,GAAGzB;YACvC,QAAQE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,MAAM,GAAGzB;QAC3C;IACF;AACF;AAEO,MAAMwC,uCAAuC,CAClDhB,MACAxB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOwB;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMtB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGxB;QAC7B,KAAKE,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGxB;QAC3B,OAAOE,KAAK,KAAK,CAACsB,KAAK,KAAK,GAAGxB;QAC/B,QAAQE,KAAK,KAAK,CAACsB,KAAK,MAAM,GAAGxB;IACnC;AACF"}
+{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport type { TModelFamily } from '@aiscene/shared/env';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@aiscene/shared/env';\nimport { generateElementByRect } from '@aiscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { _keyDefinitions } from '@aiscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@aiscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n    modelFamily?: TModelFamily;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  // Retry logic for screenshot capture/decoding\n  const maxRetries = 3;\n  let lastError: Error | null = null;\n  let screenshotBase64: string | null = null;\n  let screenshotCapturedAt = 0;\n  let imgWidth: number = 0;\n  let imgHeight: number = 0;\n\n  for (let attempt = 1; attempt <= maxRetries; attempt++) {\n    try {\n      screenshotBase64 = await interfaceInstance.screenshotBase64();\n      screenshotCapturedAt = Date.now();\n      assert(screenshotBase64!, 'screenshotBase64 is required');\n\n      // Get physical screenshot dimensions\n      debug('will get screenshot dimensions');\n      const dimensions = await imageInfoOfBase64(screenshotBase64);\n      imgWidth = dimensions.width;\n      imgHeight = dimensions.height;\n\n      // Validate dimensions\n      if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n        throw new Error(\n          `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n        );\n      }\n      if (imgWidth <= 0 || imgHeight <= 0) {\n        throw new Error(\n          `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n        );\n      }\n\n      // If we get here, screenshot is valid\n      debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n      break;\n    } catch (error) {\n      lastError =\n        error instanceof Error ? error : new Error(String(error));\n      debug(\n        `Screenshot capture/decode failed (attempt ${attempt}/${maxRetries}): ${lastError.message}`,\n      );\n\n      if (attempt === maxRetries) {\n        throw new Error(\n          `Failed to capture valid screenshot after ${maxRetries} attempts: ${lastError.message}`,\n        );\n      }\n\n      // Wait before retrying\n      await new Promise((resolve) => setTimeout(resolve, 500));\n    }\n  }\n\n  // This should never be null if we got past the retry loop\n  if (!screenshotBase64) {\n    throw new Error('Screenshot data is null after retry loop');\n  }\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    // Convert bbox [x1, y1, x2, y2] to rect {left, top, width, height}\n    const rect = {\n      left: planLocateParam.bbox[0],\n      top: planLocateParam.bbox[1],\n      width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,\n      height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,\n    };\n\n    const element = generateElementByRect(\n      rect,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","maxRetries","lastError","screenshotBase64","screenshotCapturedAt","imgWidth","imgHeight","attempt","Date","dimensions","imageInfoOfBase64","error","String","Promise","resolve","setTimeout","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","rect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;AA0BO,eAAeA,oBACpBC,iBAAoC,EACpCC,IAIC;IAED,MAAMC,QAAQC,SAAS;IAEvBC,OAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,uBAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAG9C,MAAMI,aAAa;IACnB,IAAIC,YAA0B;IAC9B,IAAIC,mBAAkC;IACtC,IAAIC,uBAAuB;IAC3B,IAAIC,WAAmB;IACvB,IAAIC,YAAoB;IAExB,IAAK,IAAIC,UAAU,GAAGA,WAAWN,YAAYM,UAC3C,IAAI;QACFJ,mBAAmB,MAAMf,kBAAkB,gBAAgB;QAC3DgB,uBAAuBI,KAAK,GAAG;QAC/BhB,OAAOW,kBAAmB;QAG1Bb,MAAM;QACN,MAAMmB,aAAa,MAAMC,kBAAkBP;QAC3CE,WAAWI,WAAW,KAAK;QAC3BH,YAAYG,WAAW,MAAM;QAG7B,IAAI,CAACV,OAAO,QAAQ,CAACM,aAAa,CAACN,OAAO,QAAQ,CAACO,YACjD,MAAM,IAAIN,MACR,CAAC,wFAAwF,EAAEK,SAAS,UAAU,EAAEC,WAAW;QAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIN,MACR,CAAC,0FAA0F,EAAEK,SAAS,UAAU,EAAEC,WAAW;QAKjIhB,MAAM,yBAAyBe,UAAU,KAAKC;QAC9C;IACF,EAAE,OAAOK,OAAO;QACdT,YACES,iBAAiBX,QAAQW,QAAQ,IAAIX,MAAMY,OAAOD;QACpDrB,MACE,CAAC,0CAA0C,EAAEiB,QAAQ,CAAC,EAAEN,WAAW,GAAG,EAAEC,UAAU,OAAO,EAAE;QAG7F,IAAIK,YAAYN,YACd,MAAM,IAAID,MACR,CAAC,yCAAyC,EAAEC,WAAW,WAAW,EAAEC,UAAU,OAAO,EAAE;QAK3F,MAAM,IAAIW,QAAQ,CAACC,UAAYC,WAAWD,SAAS;IACrD;IAIF,IAAI,CAACX,kBACH,MAAM,IAAIH,MAAM;IAOlB,MAAMgB,oBAAoBpB,eAAeC;IACzC,MAAMoB,uBAAuBZ,WAAWC;IACxC,IAAIY,oBAAoBtB;IAExB,IAAIoB,sBAAsBC,sBAAsB;QAC9C3B,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEmB,oBAAoB,aAAa,YAAY,gBAAgB,EAAEX,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEW,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBrB;IAEtB;IAEA,MAAMsB,mBAAmB9B,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACoB,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAInB,MACR,CAAC,wEAAwE,EAAEmB,kBAAkB;IAIjG,MAAMC,MAAMf,WAAWa;IAEvB5B,MAAM,mBAAmB8B;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvC7B,MAAM,4BAA4B+B;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAAClB,WAAWc;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACjB,YAAYa;QAE5C7B,MACE,CAAC,mCAAmC,EAAE6B,iBAAiB,YAAY,EAAEd,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEgB,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBvB,kBAAkB;YAC5D,OAAOmB;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAerB;YACjDiB;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOhB;YACP,QAAQC;QACV;QACA,eAAec;QACf,YAAYO,eAAe,MAAM,CAACxB,kBAAkBC;QACpDiB;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QAExB,MAAMG,OAAO;YACX,MAAMH,gBAAgB,IAAI,CAAC,EAAE;YAC7B,KAAKA,gBAAgB,IAAI,CAAC,EAAE;YAC5B,OAAOA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;YAC3D,QAAQA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;QAC9D;QAEA,MAAMI,UAAUC,sBACdF,MACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOI;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMR,OACJ,MAAMI,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNvB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCtB,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOM,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOnC,OAAO;QACd0C,8BAAW,qCAAqC1C;QAChD;IACF;AACF;AAIO,MAAM2C,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDb,SACAzB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOyB;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNvB,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;YAC/BE,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;SAChC;QACD,MAAM;YACJ,GAAGyB,QAAQ,IAAI;YACf,MAAMvB,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,IAAI,GAAGzB;YACrC,KAAKE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,GAAG,GAAGzB;YACnC,OAAOE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,KAAK,GAAGzB;YACvC,QAAQE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,MAAM,GAAGzB;QAC3C;IACF;AACF;AAEO,MAAMuC,uCAAuC,CAClDf,MACAxB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOwB;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMtB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGxB;QAC7B,KAAKE,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGxB;QAC3B,OAAOE,KAAK,KAAK,CAACsB,KAAK,KAAK,GAAGxB;QAC/B,QAAQE,KAAK,KAAK,CAACsB,KAAK,MAAM,GAAGxB;IACnC;AACF"}

package/dist/es/utils.mjs

@@ -175,7 +175,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "8.0.1";
+    return "8.0.3";
 }
 function debugLog(...message) {
     const debugMode = process.env[MIDSCENE_DEBUG_MODE];

package/dist/lib/agent/utils.js

@@ -73,14 +73,31 @@ async function commonContextParser(interfaceInstance, _opt) {
     if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) throw new Error(`Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
     if (logicalWidth <= 0 || logicalHeight <= 0) throw new Error(`Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`);
     debug(`size: ${logicalWidth}x${logicalHeight}`);
-    const screenshotBase64 = await interfaceInstance.screenshotBase64();
-    const screenshotCapturedAt = Date.now();
-    (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshotBase64 is required');
-    debug('will get screenshot dimensions');
-    const { width: imgWidth, height: imgHeight } = await (0, img_namespaceObject.imageInfoOfBase64)(screenshotBase64);
-    if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) throw new Error(`Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
-    if (imgWidth <= 0 || imgHeight <= 0) throw new Error(`Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
-    debug('screenshot dimensions', imgWidth, 'x', imgHeight);
+    const maxRetries = 3;
+    let lastError = null;
+    let screenshotBase64 = null;
+    let screenshotCapturedAt = 0;
+    let imgWidth = 0;
+    let imgHeight = 0;
+    for(let attempt = 1; attempt <= maxRetries; attempt++)try {
+        screenshotBase64 = await interfaceInstance.screenshotBase64();
+        screenshotCapturedAt = Date.now();
+        (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshotBase64 is required');
+        debug('will get screenshot dimensions');
+        const dimensions = await (0, img_namespaceObject.imageInfoOfBase64)(screenshotBase64);
+        imgWidth = dimensions.width;
+        imgHeight = dimensions.height;
+        if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) throw new Error(`Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
+        if (imgWidth <= 0 || imgHeight <= 0) throw new Error(`Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`);
+        debug('screenshot dimensions', imgWidth, 'x', imgHeight);
+        break;
+    } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+        debug(`Screenshot capture/decode failed (attempt ${attempt}/${maxRetries}): ${lastError.message}`);
+        if (attempt === maxRetries) throw new Error(`Failed to capture valid screenshot after ${maxRetries} attempts: ${lastError.message}`);
+        await new Promise((resolve)=>setTimeout(resolve, 500));
+    }
+    if (!screenshotBase64) throw new Error('Screenshot data is null after retry loop');
     const logicalIsPortrait = logicalWidth < logicalHeight;
     const screenshotIsPortrait = imgWidth < imgHeight;
     let finalLogicalWidth = logicalWidth;
@@ -170,7 +187,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"8.0.1";
+const getMidsceneVersion = ()=>"8.0.3";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/lib/agent/utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport type { TModelFamily } from '@aiscene/shared/env';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@aiscene/shared/env';\nimport { generateElementByRect } from '@aiscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { _keyDefinitions } from '@aiscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@aiscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n    modelFamily?: TModelFamily;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    // Convert bbox [x1, y1, x2, y2] to rect {left, top, width, height}\n    const rect = {\n      left: planLocateParam.bbox[0],\n      top: planLocateParam.bbox[1],\n      width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,\n      height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,\n    };\n\n    const element = generateElementByRect(\n      rect,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","rect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;ACoBO,eAAeI,oBACpBC,iBAAoC,EACpCC,IAIC;IAED,MAAMC,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;IAEvBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMb,kBAAkB,gBAAgB;IACjE,MAAMc,uBAAuBC,KAAK,GAAG;IACrCX,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,kBAAmB;IAG1BX,MAAM;IACN,MAAM,EAAE,OAAOc,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjIf,MAAM,yBAAyBc,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9ClB,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBrB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBnB,MAAM,mBAAmBqB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCpB,MAAM,4BAA4BsB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CpB,MACE,CAAC,mCAAmC,EAAEoB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACQ,oBAAAA,qBAAqBA,GACjE;IAEFC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QAExB,MAAMG,OAAO;YACX,MAAMH,gBAAgB,IAAI,CAAC,EAAE;YAC7B,KAAKA,gBAAgB,IAAI,CAAC,EAAE;YAC5B,OAAOA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;YAC3D,QAAQA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;QAC9D;QAEA,MAAMI,UAAUC,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACdF,MACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOI;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMR,OACJ,MAAMI,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNvB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCtB,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOM,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAzB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOyB;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNvB,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;YAC/BE,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;SAChC;QACD,MAAM;YACJ,GAAGyB,QAAQ,IAAI;YACf,MAAMvB,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,IAAI,GAAGzB;YACrC,KAAKE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,GAAG,GAAGzB;YACnC,OAAOE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,KAAK,GAAGzB;YACvC,QAAQE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,MAAM,GAAGzB;QAC3C;IACF;AACF;AAEO,MAAMwC,uCAAuC,CAClDhB,MACAxB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOwB;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMtB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGxB;QAC7B,KAAKE,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGxB;QAC3B,OAAOE,KAAK,KAAK,CAACsB,KAAK,KAAK,GAAGxB;QAC/B,QAAQE,KAAK,KAAK,CAACsB,KAAK,MAAM,GAAGxB;IACnC;AACF"}
+{"version":3,"file":"agent/utils.js","sources":["webpack/runtime/compat_get_default_export","webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/agent/utils.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PlanningLocateParam,\n  Rect,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport type { TModelFamily } from '@aiscene/shared/env';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@aiscene/shared/env';\nimport { generateElementByRect } from '@aiscene/shared/extractor';\nimport { imageInfoOfBase64, resizeImgBase64 } from '@aiscene/shared/img';\nimport { getDebug } from '@aiscene/shared/logger';\nimport { _keyDefinitions } from '@aiscene/shared/us-keyboard-layout';\nimport { assert, logMsg, uuid } from '@aiscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n    modelFamily?: TModelFamily;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  // Retry logic for screenshot capture/decoding\n  const maxRetries = 3;\n  let lastError: Error | null = null;\n  let screenshotBase64: string | null = null;\n  let screenshotCapturedAt = 0;\n  let imgWidth: number = 0;\n  let imgHeight: number = 0;\n\n  for (let attempt = 1; attempt <= maxRetries; attempt++) {\n    try {\n      screenshotBase64 = await interfaceInstance.screenshotBase64();\n      screenshotCapturedAt = Date.now();\n      assert(screenshotBase64!, 'screenshotBase64 is required');\n\n      // Get physical screenshot dimensions\n      debug('will get screenshot dimensions');\n      const dimensions = await imageInfoOfBase64(screenshotBase64);\n      imgWidth = dimensions.width;\n      imgHeight = dimensions.height;\n\n      // Validate dimensions\n      if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n        throw new Error(\n          `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n        );\n      }\n      if (imgWidth <= 0 || imgHeight <= 0) {\n        throw new Error(\n          `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n        );\n      }\n\n      // If we get here, screenshot is valid\n      debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n      break;\n    } catch (error) {\n      lastError =\n        error instanceof Error ? error : new Error(String(error));\n      debug(\n        `Screenshot capture/decode failed (attempt ${attempt}/${maxRetries}): ${lastError.message}`,\n      );\n\n      if (attempt === maxRetries) {\n        throw new Error(\n          `Failed to capture valid screenshot after ${maxRetries} attempts: ${lastError.message}`,\n        );\n      }\n\n      // Wait before retrying\n      await new Promise((resolve) => setTimeout(resolve, 500));\n    }\n  }\n\n  // This should never be null if we got past the retry loop\n  if (!screenshotBase64) {\n    throw new Error('Screenshot data is null after retry loop');\n  }\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\nexport function ifPlanLocateParamIsBbox(\n  planLocateParam: PlanningLocateParam,\n): boolean {\n  return !!(\n    planLocateParam.bbox &&\n    Array.isArray(planLocateParam.bbox) &&\n    planLocateParam.bbox.length === 4\n  );\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParam,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  if (planLocateParam.bbox) {\n    // Convert bbox [x1, y1, x2, y2] to rect {left, top, width, height}\n    const rect = {\n      left: planLocateParam.bbox[0],\n      top: planLocateParam.bbox[1],\n      width: planLocateParam.bbox[2] - planLocateParam.bbox[0] + 1,\n      height: planLocateParam.bbox[3] - planLocateParam.bbox[1] + 1,\n    };\n\n    const element = generateElementByRect(\n      rect,\n      typeof planLocateParam.prompt === 'string'\n        ? planLocateParam.prompt\n        : planLocateParam.prompt?.prompt || '',\n    );\n    return element;\n  }\n\n  return undefined;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","commonContextParser","interfaceInstance","_opt","debug","getDebug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","maxRetries","lastError","screenshotBase64","screenshotCapturedAt","imgWidth","imgHeight","attempt","Date","dimensions","imageInfoOfBase64","error","String","Promise","resolve","setTimeout","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","ifPlanLocateParamIsBbox","planLocateParam","Array","matchElementFromPlan","rect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;ACoBO,eAAeI,oBACpBC,iBAAoC,EACpCC,IAIC;IAED,MAAMC,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;IAEvBC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOJ,mBAAmB;IAE1BE,MAAM;IACN,MAAMG,cAAcL,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNI,IAAAA,kCAAAA,sBAAAA,AAAAA,EAAuB;QACrB,SAASD;QACT,WAAWJ,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMK,gBAAgB,MAAMP,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOQ,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIP,MAAM,CAAC,MAAM,EAAEM,aAAa,CAAC,EAAEC,eAAe;IAG9C,MAAMI,aAAa;IACnB,IAAIC,YAA0B;IAC9B,IAAIC,mBAAkC;IACtC,IAAIC,uBAAuB;IAC3B,IAAIC,WAAmB;IACvB,IAAIC,YAAoB;IAExB,IAAK,IAAIC,UAAU,GAAGA,WAAWN,YAAYM,UAC3C,IAAI;QACFJ,mBAAmB,MAAMf,kBAAkB,gBAAgB;QAC3DgB,uBAAuBI,KAAK,GAAG;QAC/BhB,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOW,kBAAmB;QAG1Bb,MAAM;QACN,MAAMmB,aAAa,MAAMC,AAAAA,IAAAA,oBAAAA,iBAAAA,AAAAA,EAAkBP;QAC3CE,WAAWI,WAAW,KAAK;QAC3BH,YAAYG,WAAW,MAAM;QAG7B,IAAI,CAACV,OAAO,QAAQ,CAACM,aAAa,CAACN,OAAO,QAAQ,CAACO,YACjD,MAAM,IAAIN,MACR,CAAC,wFAAwF,EAAEK,SAAS,UAAU,EAAEC,WAAW;QAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIN,MACR,CAAC,0FAA0F,EAAEK,SAAS,UAAU,EAAEC,WAAW;QAKjIhB,MAAM,yBAAyBe,UAAU,KAAKC;QAC9C;IACF,EAAE,OAAOK,OAAO;QACdT,YACES,iBAAiBX,QAAQW,QAAQ,IAAIX,MAAMY,OAAOD;QACpDrB,MACE,CAAC,0CAA0C,EAAEiB,QAAQ,CAAC,EAAEN,WAAW,GAAG,EAAEC,UAAU,OAAO,EAAE;QAG7F,IAAIK,YAAYN,YACd,MAAM,IAAID,MACR,CAAC,yCAAyC,EAAEC,WAAW,WAAW,EAAEC,UAAU,OAAO,EAAE;QAK3F,MAAM,IAAIW,QAAQ,CAACC,UAAYC,WAAWD,SAAS;IACrD;IAIF,IAAI,CAACX,kBACH,MAAM,IAAIH,MAAM;IAOlB,MAAMgB,oBAAoBpB,eAAeC;IACzC,MAAMoB,uBAAuBZ,WAAWC;IACxC,IAAIY,oBAAoBtB;IAExB,IAAIoB,sBAAsBC,sBAAsB;QAC9C3B,MACE,CAAC,4CAA4C,EAAEM,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEmB,oBAAoB,aAAa,YAAY,gBAAgB,EAAEX,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEW,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBrB;IAEtB;IAEA,MAAMsB,mBAAmB9B,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACU,OAAO,QAAQ,CAACoB,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAInB,MACR,CAAC,wEAAwE,EAAEmB,kBAAkB;IAIjG,MAAMC,MAAMf,WAAWa;IAEvB5B,MAAM,mBAAmB8B;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvC7B,MAAM,4BAA4B+B;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAAClB,WAAWc;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACjB,YAAYa;QAE5C7B,MACE,CAAC,mCAAmC,EAAE6B,iBAAiB,YAAY,EAAEd,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEgB,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBvB,kBAAkB;YAC5D,OAAOmB;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACF,eAAerB;YACjDiB;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOhB;YACP,QAAQC;QACV;QACA,eAAec;QACf,YAAYO,4CAAAA,cAAAA,CAAAA,MAAqB,CAACxB,kBAAkBC;QACpDiB;IACF;AACF;AAEO,SAASO,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAAA,mBAAAA,CAAAA,iBAAqC,CACzDC,oBAAAA,wBAAwBA;IAE1B,MAAMC,qBAAqBC,2BAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,AAAAA,IAAAA,sBAAAA,IAAAA,AAAAA,IAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACQ,oBAAAA,qBAAqBA,GACjE;IAEFC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAEO,SAASG,wBACdC,eAAoC;IAEpC,OAAO,CAAC,CACNA,CAAAA,gBAAgB,IAAI,IACpBC,MAAM,OAAO,CAACD,gBAAgB,IAAI,KAClCA,AAAgC,MAAhCA,gBAAgB,IAAI,CAAC,MAAM,AAAK;AAEpC;AAEO,SAASE,qBACdF,eAAoC;IAEpC,IAAI,CAACA,iBACH;IAGF,IAAIA,gBAAgB,IAAI,EAAE;QAExB,MAAMG,OAAO;YACX,MAAMH,gBAAgB,IAAI,CAAC,EAAE;YAC7B,KAAKA,gBAAgB,IAAI,CAAC,EAAE;YAC5B,OAAOA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;YAC3D,QAAQA,gBAAgB,IAAI,CAAC,EAAE,GAAGA,gBAAgB,IAAI,CAAC,EAAE,GAAG;QAC9D;QAEA,MAAMI,UAAUC,AAAAA,IAAAA,0BAAAA,qBAAAA,AAAAA,EACdF,MACA,AAAkC,YAAlC,OAAOH,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;QAExC,OAAOI;IACT;AAGF;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,AAAAA,IAAAA,uCAAAA,KAAAA,AAAAA,EACE;IAKJ,IAAI;QACF,MAAMR,OACJ,MAAMI,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACNvB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpCtB,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOM,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOnC,OAAO;QACd0C,IAAAA,uCAAAA,KAAAA,AAAAA,EAAW,qCAAqC1C;QAChD;IACF;AACF;AAIO,MAAM2C,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDb,SACAzB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOyB;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACNvB,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;YAC/BE,KAAK,KAAK,CAACuB,QAAQ,MAAM,CAAC,EAAE,GAAGzB;SAChC;QACD,MAAM;YACJ,GAAGyB,QAAQ,IAAI;YACf,MAAMvB,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,IAAI,GAAGzB;YACrC,KAAKE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,GAAG,GAAGzB;YACnC,OAAOE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,KAAK,GAAGzB;YACvC,QAAQE,KAAK,KAAK,CAACuB,QAAQ,IAAI,CAAC,MAAM,GAAGzB;QAC3C;IACF;AACF;AAEO,MAAMuC,uCAAuC,CAClDf,MACAxB;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOwB;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAMtB,KAAK,KAAK,CAACsB,KAAK,IAAI,GAAGxB;QAC7B,KAAKE,KAAK,KAAK,CAACsB,KAAK,GAAG,GAAGxB;QAC3B,OAAOE,KAAK,KAAK,CAACsB,KAAK,KAAK,GAAGxB;QAC/B,QAAQE,KAAK,KAAK,CAACsB,KAAK,MAAM,GAAGxB;IACnC;AACF"}

package/dist/lib/types.js

@@ -6,7 +6,7 @@ var __webpack_modules__ = {
     "./dump/screenshot-store" (module) {
         module.exports = require("./dump/screenshot-store.js");
     },
-    "./screenshot-item" (module) {
+    "@/screenshot-item" (module) {
         module.exports = require("./screenshot-item.js");
     },
     "./yaml" (module) {
@@ -73,16 +73,16 @@ var __webpack_exports__ = {};
     var node_path__rspack_import_1 = __webpack_require__("node:path");
     var _dump_screenshot_restoration__rspack_import_2 = __webpack_require__("./dump/screenshot-restoration");
     var _dump_screenshot_store__rspack_import_3 = __webpack_require__("./dump/screenshot-store");
-    var _screenshot_item__rspack_import_4 = __webpack_require__("./screenshot-item");
+    var _screenshot_item__rspack_import_4 = __webpack_require__("@/screenshot-item");
     var _yaml__rspack_import_5 = __webpack_require__("./yaml");
     var __rspack_reexport = {};
     for(const __rspack_import_key in _yaml__rspack_import_5)if ([
         "UIContext",
-        "GroupedActionDump",
+        "ServiceError",
         "default",
         "ReportActionDump",
-        "ExecutionDump",
-        "ServiceError"
+        "GroupedActionDump",
+        "ExecutionDump"
     ].indexOf(__rspack_import_key) < 0) __rspack_reexport[__rspack_import_key] = ()=>_yaml__rspack_import_5[__rspack_import_key];
     __webpack_require__.d(__webpack_exports__, __rspack_reexport);
     function _define_property(obj, key, value) {

package/dist/lib/utils.js

@@ -219,7 +219,7 @@ function stringifyDumpData(data, indents) {
     return JSON.stringify(data, replacerForPageObject, indents);
 }
 function getVersion() {
-    return "8.0.1";
+    return "8.0.3";
 }
 function debugLog(...message) {
     const debugMode = process.env[env_namespaceObject.MIDSCENE_DEBUG_MODE];

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@aiscene/core",
   "description": "Automate browser actions, extract data, and perform assertions using AI. It offers JavaScript SDK, Chrome extension, and support for scripting in YAML. See https://midscenejs.com/ for details.",
-  "version": "8.0.2",
+  "version": "8.0.3",
   "repository": "https://github.com/web-infra-dev/midscene",
   "homepage": "https://midscenejs.com/",
   "main": "./dist/lib/index.js",
@@ -78,7 +78,7 @@
     "test:parse-action": "npm run test:ai -- tests/ai/parse-action.test.ts"
   },
   "dependencies": {
-    "@aiscene/shared": "^8.0.1",
+    "@aiscene/shared": "^8.0.2",
     "@ui-tars/action-parser": "1.2.3",
     "dayjs": "^1.11.11",
     "dotenv": "^16.4.5",

package/dist/types/agent/agent.d.ts

@@ -1,216 +0,0 @@
-import type { TUserPrompt } from '../ai-model/index';
-import Service from '../service/index';
-import { type ActionParam, type ActionReturn, type AgentAssertOpt, type AgentDescribeElementAtPointResult, type AgentOpt, type AgentWaitForOpt, type DeepThinkOption, type DeviceAction, ExecutionDump, type LocateOption, type LocateValidatorResult, type LocatorValidatorOption, type OnTaskStartTip, ReportActionDump, type ScrollParam, type ServiceAction, type ServiceExtractOption, type ServiceExtractParam, type UIContext } from '../types';
-import type { AbstractInterface } from '../device';
-import type { TaskRunner } from '../task-runner';
-import { ModelConfigManager } from '@aiscene/shared/env';
-import { TaskCache } from './task-cache';
-import { TaskExecutor } from './tasks';
-export type AiActOptions = {
-    cacheable?: boolean;
-    fileChooserAccept?: string | string[];
-    deepThink?: DeepThinkOption;
-    deepLocate?: boolean;
-    abortSignal?: AbortSignal;
-};
-export declare class Agent<InterfaceType extends AbstractInterface = AbstractInterface> {
-    interface: InterfaceType;
-    service: Service;
-    dump: ReportActionDump;
-    reportFile?: string | null;
-    reportFileName?: string;
-    taskExecutor: TaskExecutor;
-    opts: AgentOpt;
-    /**
-     * If true, the agent will not perform any actions
-     */
-    dryMode: boolean;
-    onTaskStartTip?: OnTaskStartTip;
-    taskCache?: TaskCache;
-    private dumpUpdateListeners;
-    get onDumpUpdate(): ((dump: string, executionDump?: ExecutionDump) => void) | undefined;
-    set onDumpUpdate(callback: ((dump: string, executionDump?: ExecutionDump) => void) | undefined);
-    destroyed: boolean;
-    modelConfigManager: ModelConfigManager;
-    /**
-     * Frozen page context for consistent AI operations
-     */
-    private frozenUIContext?;
-    private get aiActContext();
-    /**
-     * Flag to track if VL model warning has been shown
-     */
-    private hasWarnedNonVLModel;
-    private executionDumpIndexByRunner;
-    private fullActionSpace;
-    private reportGenerator;
-    get page(): InterfaceType;
-    /**
-     * Ensures VL model warning is shown once when needed
-     */
-    private ensureVLModelWarning;
-    private resolveReplanningCycleLimit;
-    constructor(interfaceInstance: InterfaceType, opts?: AgentOpt);
-    getActionSpace(): Promise<DeviceAction[]>;
-    private static readonly CONTEXT_RETRY_MAX;
-    private static readonly CONTEXT_RETRY_DELAY_MS;
-    /**
-     * Override in subclasses to indicate which errors are transient and should
-     * trigger an automatic retry when building the UI context.
-     * Returns `false` by default (no retry).
-     */
-    protected isRetryableContextError(_error: unknown): boolean;
-    getUIContext(action?: ServiceAction): Promise<UIContext>;
-    _snapshotContext(): Promise<UIContext>;
-    /**
-     * @deprecated Use {@link setAIActContext} instead.
-     */
-    setAIActionContext(prompt: string): Promise<void>;
-    setAIActContext(prompt: string): Promise<void>;
-    resetDump(): ReportActionDump;
-    appendExecutionDump(execution: ExecutionDump, runner?: TaskRunner): void;
-    dumpDataString(opt?: {
-        inlineScreenshots?: boolean;
-    }): string;
-    reportHTMLString(opt?: {
-        inlineScreenshots?: boolean;
-    }): string;
-    private lastExecutionDump?;
-    writeOutActionDumps(executionDump?: ExecutionDump): void;
-    private getReportMeta;
-    private callbackOnTaskStartTip;
-    wrapActionInActionSpace<T extends DeviceAction>(name: string): (param: ActionParam<T>) => Promise<ActionReturn<T>>;
-    callActionInActionSpace<T = any>(type: string, opt?: T): Promise<any>;
-    aiTap(locatePrompt: TUserPrompt, opt?: LocateOption & {
-        fileChooserAccept?: string | string[];
-    }): Promise<any>;
-    aiRightClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
-    aiDoubleClick(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
-    aiHover(locatePrompt: TUserPrompt, opt?: LocateOption): Promise<any>;
-    aiInput(locatePrompt: TUserPrompt, opt: LocateOption & {
-        value: string | number;
-    } & {
-        autoDismissKeyboard?: boolean;
-    } & {
-        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
-    }): Promise<any>;
-    /**
-     * @deprecated Use aiInput(locatePrompt, opt) instead where opt contains the value
-     */
-    aiInput(value: string | number, locatePrompt: TUserPrompt, opt?: LocateOption & {
-        autoDismissKeyboard?: boolean;
-    } & {
-        mode?: 'replace' | 'clear' | 'typeOnly' | 'append';
-    }): Promise<any>;
-    aiKeyboardPress(locatePrompt: TUserPrompt, opt: LocateOption & {
-        keyName: string;
-    }): Promise<any>;
-    /**
-     * @deprecated Use aiKeyboardPress(locatePrompt, opt) instead where opt contains the keyName
-     */
-    aiKeyboardPress(keyName: string, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
-    aiScroll(locatePrompt: TUserPrompt | undefined, opt: LocateOption & ScrollParam): Promise<any>;
-    /**
-     * @deprecated Use aiScroll(locatePrompt, opt) instead where opt contains the scroll parameters
-     */
-    aiScroll(scrollParam: ScrollParam, locatePrompt?: TUserPrompt, opt?: LocateOption): Promise<any>;
-    aiPinch(locatePrompt: TUserPrompt | undefined, opt: LocateOption & {
-        direction: 'in' | 'out';
-        distance?: number;
-        duration?: number;
-    }): Promise<any>;
-    aiAct(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
-    /**
-     * @deprecated Use {@link Agent.aiAct} instead.
-     */
-    aiAction(taskPrompt: string, opt?: AiActOptions): Promise<string | undefined>;
-    aiQuery<ReturnType = any>(demand: ServiceExtractParam, opt?: ServiceExtractOption): Promise<ReturnType>;
-    aiBoolean(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<boolean>;
-    aiNumber(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<number>;
-    aiString(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
-    aiAsk(prompt: TUserPrompt, opt?: ServiceExtractOption): Promise<string>;
-    describeElementAtPoint(center: [number, number], opt?: {
-        verifyPrompt?: boolean;
-        retryLimit?: number;
-        deepLocate?: boolean;
-    } & LocatorValidatorOption): Promise<AgentDescribeElementAtPointResult>;
-    verifyLocator(prompt: string, locateOpt: LocateOption | undefined, expectCenter: [number, number], verifyLocateOption?: LocatorValidatorOption): Promise<LocateValidatorResult>;
-    /**
-     * Locate an element and return both its center point and an approximate rect.
-     *
-     * - In most locate flows, `rect` represents the matched element boundary.
-     * - Some models only support point grounding instead of boundary grounding.
-     *   In those cases (for example, AutoGLM), `rect` falls back to a small 8x8
-     *   box centered on the located point.
-     *
-     * Because `rect` may vary with the underlying model capability, avoid relying
-     * on it too heavily for strict boundary semantics. If you need a stable click
-     * target, prefer `center`.
-     */
-    aiLocate(prompt: TUserPrompt, opt?: LocateOption): Promise<Pick<LocateResultElement, "rect" | "center">>;
-    aiAssert(assertion: TUserPrompt, msg?: string, opt?: AgentAssertOpt & ServiceExtractOption): Promise<{
-        pass: boolean;
-        thought: string | undefined;
-        message: string | undefined;
-    } | undefined>;
-    aiWaitFor(assertion: TUserPrompt, opt?: AgentWaitForOpt): Promise<void>;
-    ai(...args: Parameters<typeof this.aiAct>): Promise<string | undefined>;
-    runYaml(yamlScriptContent: string): Promise<{
-        result: Record<string, any>;
-    }>;
-    evaluateJavaScript(script: string): Promise<any>;
-    /**
-     * Add a dump update listener
-     * @param listener Listener function
-     * @returns A remove function that can be called to remove this listener
-     */
-    addDumpUpdateListener(listener: (dump: string, executionDump?: ExecutionDump) => void): () => void;
-    /**
-     * Remove a dump update listener
-     * @param listener The listener function to remove
-     */
-    removeDumpUpdateListener(listener: (dump: string, executionDump?: ExecutionDump) => void): void;
-    /**
-     * Clear all dump update listeners
-     */
-    clearDumpUpdateListeners(): void;
-    destroy(): Promise<void>;
-    recordToReport(title?: string, opt?: {
-        content: string;
-    }): Promise<void>;
-    /**
-     * @deprecated Use {@link Agent.recordToReport} instead.
-     */
-    logScreenshot(title?: string, opt?: {
-        content: string;
-    }): Promise<void>;
-    _unstableLogContent(): {
-        groupName: string;
-        groupDescription: string | undefined;
-        executions: ExecutionDump[];
-    };
-    /**
-     * Freezes the current page context to be reused in subsequent AI operations
-     * This avoids recalculating page context for each operation
-     */
-    freezePageContext(): Promise<void>;
-    /**
-     * Unfreezes the page context, allowing AI operations to calculate context dynamically
-     */
-    unfreezePageContext(): Promise<void>;
-    /**
-     * Process cache configuration and return normalized cache settings
-     */
-    private processCacheConfig;
-    private normalizeFilePaths;
-    private normalizeFileInput;
-    /**
-     * Manually flush cache to file
-     * @param options - Optional configuration
-     * @param options.cleanUnused - If true, removes unused cache records before flushing
-     */
-    flushCache(options?: {
-        cleanUnused?: boolean;
-    }): Promise<void>;
-}
-export declare const createAgent: (interfaceInstance: AbstractInterface, opts?: AgentOpt) => Agent<AbstractInterface>;

package/dist/types/agent/execution-session.d.ts

@@ -1,36 +0,0 @@
-import { type TaskExecutionError, TaskRunner } from '../task-runner';
-import type { ExecutionTaskApply, ExecutionTaskProgressOptions, UIContext } from '../types';
-type ExecutionSessionOptions = ExecutionTaskProgressOptions & {
-    tasks?: ExecutionTaskApply[];
-    onTaskUpdate?: (runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void;
-};
-/**
- * Thin wrapper around {@link TaskRunner} that represents a single linear execution run.
- */
-export declare class ExecutionSession {
-    private readonly runner;
-    constructor(name: string, contextProvider: () => Promise<UIContext>, options?: ExecutionSessionOptions);
-    append(tasks: ExecutionTaskApply[] | ExecutionTaskApply, options?: {
-        allowWhenError?: boolean;
-    }): Promise<void>;
-    appendAndRun(tasks: ExecutionTaskApply[] | ExecutionTaskApply, options?: {
-        allowWhenError?: boolean;
-    }): Promise<{
-        output: any;
-        thought?: string;
-    } | undefined>;
-    run(options?: {
-        allowWhenError?: boolean;
-    }): Promise<{
-        output: any;
-        thought?: string;
-    } | undefined>;
-    isInErrorState(): boolean;
-    latestErrorTask(): import("../types").ExecutionTask | null;
-    appendErrorPlan(errorMsg: string): Promise<{
-        output: undefined;
-        runner: TaskRunner;
-    }>;
-    getRunner(): TaskRunner;
-}
-export {};

package/dist/types/agent/index.d.ts

@@ -1,9 +0,0 @@
-export { Agent, createAgent } from './agent';
-export { commonContextParser } from './utils';
-export { getReportFileName, printReportMsg, } from './utils';
-export { extractInsightParam, locateParamStr, paramStr, taskTitleStr, typeStr, } from './ui-utils';
-export { type LocateCache, type PlanningCache, TaskCache } from './task-cache';
-export { cacheFileExt } from './task-cache';
-export { TaskExecutor } from './tasks';
-export type { AgentOpt } from '../types';
-export type { AiActOptions } from './agent';