@midscene/core 1.9.8 → 1.10.0

package/dist/es/agent/utils.mjs

@@ -5,12 +5,24 @@ import { ScreenshotItem } from "../screenshot-item.mjs";
 import { uploadTestInfoToServer } from "../utils.mjs";
 import { MIDSCENE_REPORT_QUIET, MIDSCENE_REPORT_TAG_NAME, globalConfigManager } from "@midscene/shared/env";
 import { generateElementByRect } from "@midscene/shared/extractor";
-import { imageInfoOfBase64, normalizeBase64Image, resizeImgBase64 } from "@midscene/shared/img";
+import { createImgBase64ByFormat, imageInfoOfBase64, resizeImgBase64 } from "@midscene/shared/img";
 import { getDebug } from "@midscene/shared/logger";
 import { assert, ifInBrowser, logMsg, uuid } from "@midscene/shared/utils";
 import dayjs from "dayjs";
 import { debug as external_task_cache_mjs_debug } from "./task-cache.mjs";
 const agentDebug = getDebug('agent');
+const screenshotDataUrlPattern = /^data:image\/[a-zA-Z0-9.+-]+;base64,/i;
+const inferBase64ImageFormat = (base64Body)=>{
+    if (base64Body.startsWith('iVBORw0KGgo')) return 'png';
+    return 'jpeg';
+};
+const normalizeScreenshotBase64 = (screenshotBase64)=>{
+    const trimmedBase64 = screenshotBase64.trim();
+    if (screenshotDataUrlPattern.test(trimmedBase64)) return trimmedBase64;
+    const base64Body = trimmedBase64.replace(/\s/g, '');
+    assert(base64Body, 'screenshotBase64 must include image data');
+    return createImgBase64ByFormat(inferBase64ImageFormat(base64Body), base64Body);
+};
 const legacyScrollTypeMap = {
     once: 'singleAction',
     untilBottom: 'scrollToBottom',
@@ -92,7 +104,7 @@ async function commonContextParser(interfaceInstance, _opt) {
     };
 }
 async function createScreenshotBoundUIContext(screenshotBase64, opt) {
-    const normalizedScreenshotBase64 = normalizeBase64Image(screenshotBase64);
+    const normalizedScreenshotBase64 = normalizeScreenshotBase64(screenshotBase64);
     const actualScreenshotSize = await imageInfoOfBase64(normalizedScreenshotBase64);
     if (opt.screenshotSize && (opt.screenshotSize.width !== actualScreenshotSize.width || opt.screenshotSize.height !== actualScreenshotSize.height)) agentDebug('describeElementAtPoint screenshotSize mismatch, use actual size', {
         provided: opt.screenshotSize,
@@ -161,7 +173,7 @@ async function matchElementFromCache(context, cacheEntry, cachePrompt, cacheable
         return;
     }
 }
-const getMidsceneVersion = ()=>"1.9.8";
+const getMidsceneVersion = ()=>"1.10.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/agent/utils.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport { pixelBboxToRect } from '@/ai-model/workflows/inspect/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  ScrollParam,\n  Size,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  imageInfoOfBase64,\n  normalizeBase64Image,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\n\nconst legacyScrollTypeMap = {\n  once: 'singleAction',\n  untilBottom: 'scrollToBottom',\n  untilTop: 'scrollToTop',\n  untilRight: 'scrollToRight',\n  untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n  scrollType: string | undefined,\n): ScrollParam['scrollType'] | undefined => {\n  if (!scrollType) {\n    return undefined;\n  }\n\n  if (scrollType in legacyScrollTypeMap) {\n    return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n  }\n\n  return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport async function createScreenshotBoundUIContext(\n  screenshotBase64: string,\n  opt: {\n    screenshotSize?: Size;\n  },\n): Promise<UIContext> {\n  const normalizedScreenshotBase64 = normalizeBase64Image(screenshotBase64);\n  const actualScreenshotSize = await imageInfoOfBase64(\n    normalizedScreenshotBase64,\n  );\n  if (\n    opt.screenshotSize &&\n    (opt.screenshotSize.width !== actualScreenshotSize.width ||\n      opt.screenshotSize.height !== actualScreenshotSize.height)\n  ) {\n    agentDebug(\n      'describeElementAtPoint screenshotSize mismatch, use actual size',\n      {\n        provided: opt.screenshotSize,\n        actual: actualScreenshotSize,\n      },\n    );\n  }\n\n  return {\n    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n    shotSize: actualScreenshotSize,\n    shrunkShotToLogicalRatio: 1,\n    _isFrozen: true,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n  fileExists?: (path: string) => boolean;\n  isInBrowser?: boolean;\n  resolvePath?: (path: string) => string;\n  wslDistroName?: string;\n  cwd?: string;\n};\n\nexport function normalizeFilePaths(\n  files: string[],\n  options: NormalizeFilePathsOptions = {},\n): string[] {\n  const {\n    fileExists = existsSync,\n    isInBrowser = ifInBrowser,\n    resolvePath = resolve,\n    wslDistroName = process.env.WSL_DISTRO_NAME,\n    cwd = process.cwd(),\n  } = options;\n\n  if (isInBrowser) {\n    throw new Error('File chooser is not supported in browser environment');\n  }\n\n  return files.map((file) => {\n    const absolutePath = resolvePath(file);\n    if (!fileExists(absolutePath)) {\n      throw new Error(\n        `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n      );\n    }\n\n    if (!wslDistroName) {\n      return absolutePath;\n    }\n\n    const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n    if (wslMount) {\n      return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n    }\n\n    return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n  });\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["agentDebug","getDebug","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","assert","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotBase64","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","normalizeBase64Image","actualScreenshotSize","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;;;;AAoCA,MAAMA,aAAaC,SAAS;AAE5B,MAAMC,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQP,SAAS;IAEvBQ,OAAOH,mBAAmB;IAE1BE,MAAM;IACN,MAAME,cAAcJ,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNG,uBAAuB;QACrB,SAASD;QACT,WAAWH,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMI,gBAAgB,MAAMN,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOO,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIN,MAAM,CAAC,MAAM,EAAEK,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMI,mBAAmB,MAAMZ,kBAAkB,gBAAgB;IACjE,MAAMa,uBAAuBC,KAAK,GAAG;IACrCX,OAAOS,kBAAmB;IAG1BV,MAAM;IACN,MAAM,EAAE,OAAOa,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBL;IAE1B,IAAI,CAACF,OAAO,QAAQ,CAACK,aAAa,CAACL,OAAO,QAAQ,CAACM,YACjD,MAAM,IAAIL,MACR,CAAC,wFAAwF,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIL,MACR,CAAC,0FAA0F,EAAEI,SAAS,UAAU,EAAEC,WAAW;IAGjId,MAAM,yBAAyBa,UAAU,KAAKC;IAM9C,MAAME,oBAAoBX,eAAeC;IACzC,MAAMW,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBb;IAExB,IAAIW,sBAAsBC,sBAAsB;QAC9CjB,MACE,CAAC,4CAA4C,EAAEK,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAEU,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBZ;IAEtB;IAEA,MAAMa,mBAAmBpB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACS,OAAO,QAAQ,CAACW,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIV,MACR,CAAC,wEAAwE,EAAEU,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBlB,MAAM,mBAAmBoB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCnB,MAAM,4BAA4BqB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CnB,MACE,CAAC,mCAAmC,EAAEmB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBhB,kBAAkB;YAC5D,OAAOY;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,eAAe,MAAM,CAACjB,kBAAkBC;QACpDU;IACF;AACF;AAEO,eAAeO,+BACpBlB,gBAAwB,EACxBmB,GAEC;IAED,MAAMC,6BAA6BC,qBAAqBrB;IACxD,MAAMsB,uBAAuB,MAAMjB,kBACjCe;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKG,qBAAqB,KAAK,IACtDH,IAAI,cAAc,CAAC,MAAM,KAAKG,qBAAqB,MAAK,GAE1DxC,WACE,mEACA;QACE,UAAUqC,IAAI,cAAc;QAC5B,QAAQG;IACV;IAIJ,OAAO;QACL,YAAYL,eAAe,MAAM,CAACG,4BAA4BlB,KAAK,GAAG;QACtE,UAAUoB;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASC,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAI1C,MAAM;IAGlB,OAAOsC,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIlD,MACR,CAAC,gBAAgB,EAAEiD,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAEO,SAASE,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqBxD,OAAO,QAAQ,CAACwD;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,gBAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,sBACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACN/C,KAAK,KAAK,CAAC6C,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpC7C,KAAK,KAAK,CAAC6C,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,8BAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAjD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOiD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACN/C,KAAK,KAAK,CAAC+C,QAAQ,MAAM,CAAC,EAAE,GAAGjD;YAC/BE,KAAK,KAAK,CAAC+C,QAAQ,MAAM,CAAC,EAAE,GAAGjD;SAChC;QACD,MAAM;YACJ,GAAGiD,QAAQ,IAAI;YACf,MAAM/C,KAAK,KAAK,CAAC+C,QAAQ,IAAI,CAAC,IAAI,GAAGjD;YACrC,KAAKE,KAAK,KAAK,CAAC+C,QAAQ,IAAI,CAAC,GAAG,GAAGjD;YACnC,OAAOE,KAAK,KAAK,CAAC+C,QAAQ,IAAI,CAAC,KAAK,GAAGjD;YACvC,QAAQE,KAAK,KAAK,CAAC+C,QAAQ,IAAI,CAAC,MAAM,GAAGjD;QAC3C;IACF;AACF;AAEO,MAAMgE,uCAAuC,CAClDjB,MACA/C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO+C;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAM7C,KAAK,KAAK,CAAC6C,KAAK,IAAI,GAAG/C;QAC7B,KAAKE,KAAK,KAAK,CAAC6C,KAAK,GAAG,GAAG/C;QAC3B,OAAOE,KAAK,KAAK,CAAC6C,KAAK,KAAK,GAAG/C;QAC/B,QAAQE,KAAK,KAAK,CAAC6C,KAAK,MAAM,GAAG/C;IACnC;AACF"}
+{"version":3,"file":"agent/utils.mjs","sources":["../../../src/agent/utils.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { resolve } from 'node:path';\nimport { pixelBboxToRect } from '@/ai-model/workflows/inspect/locate-result-rect';\nimport type { TMultimodalPrompt, TUserPrompt } from '@/common';\nimport type { AbstractInterface } from '@/device';\nimport { ScreenshotItem } from '@/screenshot-item';\nimport type {\n  ElementCacheFeature,\n  LocateResultElement,\n  PixelBbox,\n  PlanningLocateParam,\n  PlanningLocateParamWithLocatedPixelBbox,\n  Rect,\n  ScrollParam,\n  Size,\n  UIContext,\n} from '@/types';\nimport { uploadTestInfoToServer } from '@/utils';\nimport {\n  MIDSCENE_REPORT_QUIET,\n  MIDSCENE_REPORT_TAG_NAME,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport {\n  createImgBase64ByFormat,\n  imageInfoOfBase64,\n  resizeImgBase64,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { assert, ifInBrowser, logMsg, uuid } from '@midscene/shared/utils';\nimport dayjs from 'dayjs';\nimport type { TaskCache } from './task-cache';\nimport { debug as cacheDebug } from './task-cache';\n\nconst agentDebug = getDebug('agent');\nconst screenshotDataUrlPattern = /^data:image\\/[a-zA-Z0-9.+-]+;base64,/i;\n\nconst inferBase64ImageFormat = (base64Body: string) => {\n  if (base64Body.startsWith('iVBORw0KGgo')) {\n    return 'png';\n  }\n  return 'jpeg';\n};\n\nconst normalizeScreenshotBase64 = (screenshotBase64: string) => {\n  const trimmedBase64 = screenshotBase64.trim();\n  if (screenshotDataUrlPattern.test(trimmedBase64)) {\n    return trimmedBase64;\n  }\n\n  const base64Body = trimmedBase64.replace(/\\s/g, '');\n  assert(base64Body, 'screenshotBase64 must include image data');\n  return createImgBase64ByFormat(\n    inferBase64ImageFormat(base64Body),\n    base64Body,\n  );\n};\n\nconst legacyScrollTypeMap = {\n  once: 'singleAction',\n  untilBottom: 'scrollToBottom',\n  untilTop: 'scrollToTop',\n  untilRight: 'scrollToRight',\n  untilLeft: 'scrollToLeft',\n} as const;\n\nexport const normalizeScrollType = (\n  scrollType: string | undefined,\n): ScrollParam['scrollType'] | undefined => {\n  if (!scrollType) {\n    return undefined;\n  }\n\n  if (scrollType in legacyScrollTypeMap) {\n    return legacyScrollTypeMap[scrollType as keyof typeof legacyScrollTypeMap];\n  }\n\n  return scrollType as ScrollParam['scrollType'];\n};\n\nexport async function commonContextParser(\n  interfaceInstance: AbstractInterface,\n  _opt: {\n    uploadServerUrl?: string;\n    screenshotShrinkFactor?: number;\n  },\n): Promise<UIContext> {\n  const debug = getDebug('commonContextParser');\n\n  assert(interfaceInstance, 'interfaceInstance is required');\n\n  debug('Getting interface description');\n  const description = interfaceInstance.describe?.() || '';\n  debug('Interface description end');\n\n  debug('Uploading test info to server');\n  uploadTestInfoToServer({\n    testUrl: description,\n    serverUrl: _opt.uploadServerUrl,\n  });\n  debug('UploadTestInfoToServer end');\n\n  debug('will get size');\n  const interfaceSize = await interfaceInstance.size();\n  const { width: logicalWidth, height: logicalHeight } = interfaceSize;\n\n  if ((interfaceSize as unknown as { dpr: number }).dpr) {\n    console.warn(\n      'Warning: return value of interface.size() include a dpr property, which is not expected and ignored. ',\n    );\n  }\n\n  if (!Number.isFinite(logicalWidth) || !Number.isFinite(logicalHeight)) {\n    throw new Error(\n      `Invalid interface size: width and height must be finite numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  if (logicalWidth <= 0 || logicalHeight <= 0) {\n    throw new Error(\n      `Invalid interface size: width and height must be positive numbers. Received width: ${logicalWidth}, height: ${logicalHeight}`,\n    );\n  }\n\n  debug(`size: ${logicalWidth}x${logicalHeight}`);\n\n  const screenshotBase64 = await interfaceInstance.screenshotBase64();\n  const screenshotCapturedAt = Date.now();\n  assert(screenshotBase64!, 'screenshotBase64 is required');\n\n  // Get physical screenshot dimensions\n  debug('will get screenshot dimensions');\n  const { width: imgWidth, height: imgHeight } =\n    await imageInfoOfBase64(screenshotBase64);\n\n  if (!Number.isFinite(imgWidth) || !Number.isFinite(imgHeight)) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be finite numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  if (imgWidth <= 0 || imgHeight <= 0) {\n    throw new Error(\n      `Invalid screenshot dimensions: width and height must be positive numbers. Received width: ${imgWidth}, height: ${imgHeight}`,\n    );\n  }\n  debug('screenshot dimensions', imgWidth, 'x', imgHeight);\n\n  // Detect orientation mismatch between logical size and screenshot.\n  // Some devices (e.g. OPPO) report wrong orientation via ADB, causing\n  // size() to return portrait dimensions even when the device is landscape.\n  // We detect this by comparing aspect ratios and swap if they disagree.\n  const logicalIsPortrait = logicalWidth < logicalHeight;\n  const screenshotIsPortrait = imgWidth < imgHeight;\n  let finalLogicalWidth = logicalWidth;\n  let finalLogicalHeight = logicalHeight;\n  if (logicalIsPortrait !== screenshotIsPortrait) {\n    debug(\n      `Orientation mismatch detected: logical size ${logicalWidth}x${logicalHeight} (${logicalIsPortrait ? 'portrait' : 'landscape'}) vs screenshot ${imgWidth}x${imgHeight} (${screenshotIsPortrait ? 'portrait' : 'landscape'}). Swapping logical dimensions.`,\n    );\n    finalLogicalWidth = logicalHeight;\n    finalLogicalHeight = logicalWidth;\n  }\n\n  const userShrinkFactor = _opt.screenshotShrinkFactor ?? 1;\n\n  if (!Number.isFinite(userShrinkFactor) || userShrinkFactor < 1) {\n    throw new Error(\n      `Invalid screenshotShrinkFactor: must be a finite number >= 1. Received: ${userShrinkFactor}`,\n    );\n  }\n\n  const dpr = imgWidth / finalLogicalWidth;\n\n  debug('calculated dpr:', dpr);\n\n  const shrunkShotToLogicalRatio = dpr / userShrinkFactor;\n\n  debug('shrunkShotToLogicalRatio', shrunkShotToLogicalRatio);\n\n  if (userShrinkFactor !== 1) {\n    const targetWidth = Math.round(imgWidth / userShrinkFactor);\n    const targetHeight = Math.round(imgHeight / userShrinkFactor);\n\n    debug(\n      `Applying screenshot shrink factor: ${userShrinkFactor} (physical: ${imgWidth}x${imgHeight} -> target: ${targetWidth}x${targetHeight})`,\n    );\n\n    const resizedBase64 = await resizeImgBase64(screenshotBase64, {\n      width: targetWidth,\n      height: targetHeight,\n    });\n    return {\n      shotSize: {\n        width: targetWidth,\n        height: targetHeight,\n      },\n      deprecatedDpr: dpr,\n      screenshot: ScreenshotItem.create(resizedBase64, screenshotCapturedAt),\n      shrunkShotToLogicalRatio,\n    };\n  }\n\n  return {\n    shotSize: {\n      width: imgWidth,\n      height: imgHeight,\n    },\n    deprecatedDpr: dpr,\n    screenshot: ScreenshotItem.create(screenshotBase64, screenshotCapturedAt),\n    shrunkShotToLogicalRatio,\n  };\n}\n\nexport async function createScreenshotBoundUIContext(\n  screenshotBase64: string,\n  opt: {\n    screenshotSize?: Size;\n  },\n): Promise<UIContext> {\n  const normalizedScreenshotBase64 =\n    normalizeScreenshotBase64(screenshotBase64);\n  const actualScreenshotSize = await imageInfoOfBase64(\n    normalizedScreenshotBase64,\n  );\n  if (\n    opt.screenshotSize &&\n    (opt.screenshotSize.width !== actualScreenshotSize.width ||\n      opt.screenshotSize.height !== actualScreenshotSize.height)\n  ) {\n    agentDebug(\n      'describeElementAtPoint screenshotSize mismatch, use actual size',\n      {\n        provided: opt.screenshotSize,\n        actual: actualScreenshotSize,\n      },\n    );\n  }\n\n  return {\n    screenshot: ScreenshotItem.create(normalizedScreenshotBase64, Date.now()),\n    shotSize: actualScreenshotSize,\n    shrunkShotToLogicalRatio: 1,\n    _isFrozen: true,\n  };\n}\n\nexport function getReportFileName(tag = 'web') {\n  const reportTagName = globalConfigManager.getEnvConfigValue(\n    MIDSCENE_REPORT_TAG_NAME,\n  );\n  const dateTimeInFileName = dayjs().format('YYYY-MM-DD_HH-mm-ss');\n  // ensure uniqueness at the same time\n  const uniqueId = uuid().substring(0, 8);\n  return `${reportTagName || tag}-${dateTimeInFileName}-${uniqueId}`;\n}\n\nexport function printReportMsg(filepath: string) {\n  if (globalConfigManager.getEnvConfigInBoolean(MIDSCENE_REPORT_QUIET)) {\n    return;\n  }\n  logMsg(`Midscene - report file updated: ${filepath}`);\n}\n\ntype NormalizeFilePathsOptions = {\n  fileExists?: (path: string) => boolean;\n  isInBrowser?: boolean;\n  resolvePath?: (path: string) => string;\n  wslDistroName?: string;\n  cwd?: string;\n};\n\nexport function normalizeFilePaths(\n  files: string[],\n  options: NormalizeFilePathsOptions = {},\n): string[] {\n  const {\n    fileExists = existsSync,\n    isInBrowser = ifInBrowser,\n    resolvePath = resolve,\n    wslDistroName = process.env.WSL_DISTRO_NAME,\n    cwd = process.cwd(),\n  } = options;\n\n  if (isInBrowser) {\n    throw new Error('File chooser is not supported in browser environment');\n  }\n\n  return files.map((file) => {\n    const absolutePath = resolvePath(file);\n    if (!fileExists(absolutePath)) {\n      throw new Error(\n        `File not found: ${file}. Resolved to: ${absolutePath}. Current working directory: ${cwd}`,\n      );\n    }\n\n    if (!wslDistroName) {\n      return absolutePath;\n    }\n\n    const wslMount = absolutePath.match(/^\\/mnt\\/([a-z])\\//);\n    if (wslMount) {\n      return `${wslMount[1].toUpperCase()}:\\\\${absolutePath.slice(7).replace(/\\//g, '\\\\')}`;\n    }\n\n    return `\\\\\\\\wsl$\\\\${wslDistroName}${absolutePath.replace(/\\//g, '\\\\')}`;\n  });\n}\n\nexport function isPixelBbox(value: unknown): value is PixelBbox {\n  return (\n    Array.isArray(value) &&\n    value.length === 4 &&\n    value.every((item) => typeof item === 'number' && Number.isFinite(item))\n  );\n}\n\ntype PlanningLocateParamWithMaybeLocatedPixelBbox = PlanningLocateParam & {\n  locatedPixelBbox?: unknown;\n};\n\nexport function ifPlanLocateParamHasLocatedPixelBbox(\n  planLocateParam: PlanningLocateParamWithMaybeLocatedPixelBbox,\n): planLocateParam is PlanningLocateParamWithLocatedPixelBbox {\n  return isPixelBbox(planLocateParam.locatedPixelBbox);\n}\n\nexport function matchElementFromPlan(\n  planLocateParam: PlanningLocateParamWithLocatedPixelBbox,\n): LocateResultElement | undefined {\n  if (!planLocateParam) {\n    return undefined;\n  }\n\n  const rect = pixelBboxToRect(planLocateParam.locatedPixelBbox);\n\n  const element = generateElementByRect(\n    rect,\n    typeof planLocateParam.prompt === 'string'\n      ? planLocateParam.prompt\n      : planLocateParam.prompt?.prompt || '',\n  );\n  return element;\n}\n\nexport async function matchElementFromCache(\n  context: {\n    taskCache?: TaskCache;\n    interfaceInstance: AbstractInterface;\n  },\n  cacheEntry: ElementCacheFeature | undefined,\n  cachePrompt: TUserPrompt,\n  cacheable: boolean | undefined,\n): Promise<LocateResultElement | undefined> {\n  if (!cacheEntry) {\n    return undefined;\n  }\n\n  if (cacheable === false) {\n    cacheDebug('cache disabled for prompt: %s', cachePrompt);\n    return undefined;\n  }\n\n  if (!context.taskCache?.isCacheResultUsed) {\n    return undefined;\n  }\n\n  if (!context.interfaceInstance.rectMatchesCacheFeature) {\n    cacheDebug(\n      'interface does not implement rectMatchesCacheFeature, skip cache',\n    );\n    return undefined;\n  }\n\n  try {\n    const rect =\n      await context.interfaceInstance.rectMatchesCacheFeature(cacheEntry);\n    const element: LocateResultElement = {\n      center: [\n        Math.round(rect.left + rect.width / 2),\n        Math.round(rect.top + rect.height / 2),\n      ],\n      rect,\n      description:\n        typeof cachePrompt === 'string'\n          ? cachePrompt\n          : cachePrompt.prompt || '',\n    };\n\n    cacheDebug('cache hit, prompt: %s', cachePrompt);\n    return element;\n  } catch (error) {\n    cacheDebug('rectMatchesCacheFeature error: %s', error);\n    return undefined;\n  }\n}\n\ndeclare const __VERSION__: string | undefined;\n\nexport const getMidsceneVersion = (): string => {\n  if (typeof __VERSION__ !== 'undefined') {\n    return __VERSION__;\n  } else if (\n    process.env.__VERSION__ &&\n    process.env.__VERSION__ !== 'undefined'\n  ) {\n    return process.env.__VERSION__;\n  }\n  throw new Error('__VERSION__ inject failed during build');\n};\n\nexport const parsePrompt = (\n  prompt: TUserPrompt,\n): {\n  textPrompt: string;\n  multimodalPrompt?: TMultimodalPrompt;\n} => {\n  if (typeof prompt === 'string') {\n    return {\n      textPrompt: prompt,\n      multimodalPrompt: undefined,\n    };\n  }\n  return {\n    textPrompt: prompt.prompt,\n    multimodalPrompt: prompt.images\n      ? {\n          images: prompt.images,\n          convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n        }\n      : undefined,\n  };\n};\n\nexport const transformLogicalElementToScreenshot = (\n  element: LocateResultElement,\n  shrunkShotToLogicalRatio: number,\n): LocateResultElement => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return element;\n  }\n\n  return {\n    ...element,\n    center: [\n      Math.round(element.center[0] * shrunkShotToLogicalRatio),\n      Math.round(element.center[1] * shrunkShotToLogicalRatio),\n    ],\n    rect: {\n      ...element.rect,\n      left: Math.round(element.rect.left * shrunkShotToLogicalRatio),\n      top: Math.round(element.rect.top * shrunkShotToLogicalRatio),\n      width: Math.round(element.rect.width * shrunkShotToLogicalRatio),\n      height: Math.round(element.rect.height * shrunkShotToLogicalRatio),\n    },\n  };\n};\n\nexport const transformLogicalRectToScreenshotRect = (\n  rect: Rect,\n  shrunkShotToLogicalRatio: number,\n): Rect => {\n  if (shrunkShotToLogicalRatio === 1) {\n    return rect;\n  }\n\n  return {\n    ...rect,\n    left: Math.round(rect.left * shrunkShotToLogicalRatio),\n    top: Math.round(rect.top * shrunkShotToLogicalRatio),\n    width: Math.round(rect.width * shrunkShotToLogicalRatio),\n    height: Math.round(rect.height * shrunkShotToLogicalRatio),\n  };\n};\n"],"names":["agentDebug","getDebug","screenshotDataUrlPattern","inferBase64ImageFormat","base64Body","normalizeScreenshotBase64","screenshotBase64","trimmedBase64","assert","createImgBase64ByFormat","legacyScrollTypeMap","normalizeScrollType","scrollType","commonContextParser","interfaceInstance","_opt","debug","description","uploadTestInfoToServer","interfaceSize","logicalWidth","logicalHeight","console","Number","Error","screenshotCapturedAt","Date","imgWidth","imgHeight","imageInfoOfBase64","logicalIsPortrait","screenshotIsPortrait","finalLogicalWidth","userShrinkFactor","dpr","shrunkShotToLogicalRatio","targetWidth","Math","targetHeight","resizedBase64","resizeImgBase64","ScreenshotItem","createScreenshotBoundUIContext","opt","normalizedScreenshotBase64","actualScreenshotSize","getReportFileName","tag","reportTagName","globalConfigManager","MIDSCENE_REPORT_TAG_NAME","dateTimeInFileName","dayjs","uniqueId","uuid","printReportMsg","filepath","MIDSCENE_REPORT_QUIET","logMsg","normalizeFilePaths","files","options","fileExists","existsSync","isInBrowser","ifInBrowser","resolvePath","resolve","wslDistroName","process","cwd","file","absolutePath","wslMount","isPixelBbox","value","Array","item","ifPlanLocateParamHasLocatedPixelBbox","planLocateParam","matchElementFromPlan","rect","pixelBboxToRect","element","generateElementByRect","matchElementFromCache","context","cacheEntry","cachePrompt","cacheable","cacheDebug","error","getMidsceneVersion","__VERSION__","parsePrompt","prompt","undefined","transformLogicalElementToScreenshot","transformLogicalRectToScreenshotRect"],"mappings":";;;;;;;;;;;;AAoCA,MAAMA,aAAaC,SAAS;AAC5B,MAAMC,2BAA2B;AAEjC,MAAMC,yBAAyB,CAACC;IAC9B,IAAIA,WAAW,UAAU,CAAC,gBACxB,OAAO;IAET,OAAO;AACT;AAEA,MAAMC,4BAA4B,CAACC;IACjC,MAAMC,gBAAgBD,iBAAiB,IAAI;IAC3C,IAAIJ,yBAAyB,IAAI,CAACK,gBAChC,OAAOA;IAGT,MAAMH,aAAaG,cAAc,OAAO,CAAC,OAAO;IAChDC,OAAOJ,YAAY;IACnB,OAAOK,wBACLN,uBAAuBC,aACvBA;AAEJ;AAEA,MAAMM,sBAAsB;IAC1B,MAAM;IACN,aAAa;IACb,UAAU;IACV,YAAY;IACZ,WAAW;AACb;AAEO,MAAMC,sBAAsB,CACjCC;IAEA,IAAI,CAACA,YACH;IAGF,IAAIA,cAAcF,qBAChB,OAAOA,mBAAmB,CAACE,WAA+C;IAG5E,OAAOA;AACT;AAEO,eAAeC,oBACpBC,iBAAoC,EACpCC,IAGC;IAED,MAAMC,QAAQf,SAAS;IAEvBO,OAAOM,mBAAmB;IAE1BE,MAAM;IACN,MAAMC,cAAcH,kBAAkB,QAAQ,QAAQ;IACtDE,MAAM;IAENA,MAAM;IACNE,uBAAuB;QACrB,SAASD;QACT,WAAWF,KAAK,eAAe;IACjC;IACAC,MAAM;IAENA,MAAM;IACN,MAAMG,gBAAgB,MAAML,kBAAkB,IAAI;IAClD,MAAM,EAAE,OAAOM,YAAY,EAAE,QAAQC,aAAa,EAAE,GAAGF;IAEvD,IAAKA,cAA6C,GAAG,EACnDG,QAAQ,IAAI,CACV;IAIJ,IAAI,CAACC,OAAO,QAAQ,CAACH,iBAAiB,CAACG,OAAO,QAAQ,CAACF,gBACrD,MAAM,IAAIG,MACR,CAAC,iFAAiF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIhI,IAAID,gBAAgB,KAAKC,iBAAiB,GACxC,MAAM,IAAIG,MACR,CAAC,mFAAmF,EAAEJ,aAAa,UAAU,EAAEC,eAAe;IAIlIL,MAAM,CAAC,MAAM,EAAEI,aAAa,CAAC,EAAEC,eAAe;IAE9C,MAAMf,mBAAmB,MAAMQ,kBAAkB,gBAAgB;IACjE,MAAMW,uBAAuBC,KAAK,GAAG;IACrClB,OAAOF,kBAAmB;IAG1BU,MAAM;IACN,MAAM,EAAE,OAAOW,QAAQ,EAAE,QAAQC,SAAS,EAAE,GAC1C,MAAMC,kBAAkBvB;IAE1B,IAAI,CAACiB,OAAO,QAAQ,CAACI,aAAa,CAACJ,OAAO,QAAQ,CAACK,YACjD,MAAM,IAAIJ,MACR,CAAC,wFAAwF,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAG/H,IAAID,YAAY,KAAKC,aAAa,GAChC,MAAM,IAAIJ,MACR,CAAC,0FAA0F,EAAEG,SAAS,UAAU,EAAEC,WAAW;IAGjIZ,MAAM,yBAAyBW,UAAU,KAAKC;IAM9C,MAAME,oBAAoBV,eAAeC;IACzC,MAAMU,uBAAuBJ,WAAWC;IACxC,IAAII,oBAAoBZ;IAExB,IAAIU,sBAAsBC,sBAAsB;QAC9Cf,MACE,CAAC,4CAA4C,EAAEI,aAAa,CAAC,EAAEC,cAAc,EAAE,EAAES,oBAAoB,aAAa,YAAY,gBAAgB,EAAEH,SAAS,CAAC,EAAEC,UAAU,EAAE,EAAEG,uBAAuB,aAAa,YAAY,+BAA+B,CAAC;QAE5PC,oBAAoBX;IAEtB;IAEA,MAAMY,mBAAmBlB,KAAK,sBAAsB,IAAI;IAExD,IAAI,CAACQ,OAAO,QAAQ,CAACU,qBAAqBA,mBAAmB,GAC3D,MAAM,IAAIT,MACR,CAAC,wEAAwE,EAAES,kBAAkB;IAIjG,MAAMC,MAAMP,WAAWK;IAEvBhB,MAAM,mBAAmBkB;IAEzB,MAAMC,2BAA2BD,MAAMD;IAEvCjB,MAAM,4BAA4BmB;IAElC,IAAIF,AAAqB,MAArBA,kBAAwB;QAC1B,MAAMG,cAAcC,KAAK,KAAK,CAACV,WAAWM;QAC1C,MAAMK,eAAeD,KAAK,KAAK,CAACT,YAAYK;QAE5CjB,MACE,CAAC,mCAAmC,EAAEiB,iBAAiB,YAAY,EAAEN,SAAS,CAAC,EAAEC,UAAU,YAAY,EAAEQ,YAAY,CAAC,EAAEE,aAAa,CAAC,CAAC;QAGzI,MAAMC,gBAAgB,MAAMC,gBAAgBlC,kBAAkB;YAC5D,OAAO8B;YACP,QAAQE;QACV;QACA,OAAO;YACL,UAAU;gBACR,OAAOF;gBACP,QAAQE;YACV;YACA,eAAeJ;YACf,YAAYO,eAAe,MAAM,CAACF,eAAed;YACjDU;QACF;IACF;IAEA,OAAO;QACL,UAAU;YACR,OAAOR;YACP,QAAQC;QACV;QACA,eAAeM;QACf,YAAYO,eAAe,MAAM,CAACnC,kBAAkBmB;QACpDU;IACF;AACF;AAEO,eAAeO,+BACpBpC,gBAAwB,EACxBqC,GAEC;IAED,MAAMC,6BACJvC,0BAA0BC;IAC5B,MAAMuC,uBAAuB,MAAMhB,kBACjCe;IAEF,IACED,IAAI,cAAc,IACjBA,CAAAA,IAAI,cAAc,CAAC,KAAK,KAAKE,qBAAqB,KAAK,IACtDF,IAAI,cAAc,CAAC,MAAM,KAAKE,qBAAqB,MAAK,GAE1D7C,WACE,mEACA;QACE,UAAU2C,IAAI,cAAc;QAC5B,QAAQE;IACV;IAIJ,OAAO;QACL,YAAYJ,eAAe,MAAM,CAACG,4BAA4BlB,KAAK,GAAG;QACtE,UAAUmB;QACV,0BAA0B;QAC1B,WAAW;IACb;AACF;AAEO,SAASC,kBAAkBC,MAAM,KAAK;IAC3C,MAAMC,gBAAgBC,oBAAoB,iBAAiB,CACzDC;IAEF,MAAMC,qBAAqBC,QAAQ,MAAM,CAAC;IAE1C,MAAMC,WAAWC,OAAO,SAAS,CAAC,GAAG;IACrC,OAAO,GAAGN,iBAAiBD,IAAI,CAAC,EAAEI,mBAAmB,CAAC,EAAEE,UAAU;AACpE;AAEO,SAASE,eAAeC,QAAgB;IAC7C,IAAIP,oBAAoB,qBAAqB,CAACQ,wBAC5C;IAEFC,OAAO,CAAC,gCAAgC,EAAEF,UAAU;AACtD;AAUO,SAASG,mBACdC,KAAe,EACfC,UAAqC,CAAC,CAAC;IAEvC,MAAM,EACJC,aAAaC,UAAU,EACvBC,cAAcC,WAAW,EACzBC,cAAcC,OAAO,EACrBC,gBAAgBC,QAAQ,GAAG,CAAC,eAAe,EAC3CC,MAAMD,QAAQ,GAAG,EAAE,EACpB,GAAGR;IAEJ,IAAIG,aACF,MAAM,IAAIxC,MAAM;IAGlB,OAAOoC,MAAM,GAAG,CAAC,CAACW;QAChB,MAAMC,eAAeN,YAAYK;QACjC,IAAI,CAACT,WAAWU,eACd,MAAM,IAAIhD,MACR,CAAC,gBAAgB,EAAE+C,KAAK,eAAe,EAAEC,aAAa,6BAA6B,EAAEF,KAAK;QAI9F,IAAI,CAACF,eACH,OAAOI;QAGT,MAAMC,WAAWD,aAAa,KAAK,CAAC;QACpC,IAAIC,UACF,OAAO,GAAGA,QAAQ,CAAC,EAAE,CAAC,WAAW,GAAG,GAAG,EAAED,aAAa,KAAK,CAAC,GAAG,OAAO,CAAC,OAAO,OAAO;QAGvF,OAAO,CAAC,UAAU,EAAEJ,gBAAgBI,aAAa,OAAO,CAAC,OAAO,OAAO;IACzE;AACF;AAEO,SAASE,YAAYC,KAAc;IACxC,OACEC,MAAM,OAAO,CAACD,UACdA,AAAiB,MAAjBA,MAAM,MAAM,IACZA,MAAM,KAAK,CAAC,CAACE,OAAS,AAAgB,YAAhB,OAAOA,QAAqBtD,OAAO,QAAQ,CAACsD;AAEtE;AAMO,SAASC,qCACdC,eAA6D;IAE7D,OAAOL,YAAYK,gBAAgB,gBAAgB;AACrD;AAEO,SAASC,qBACdD,eAAwD;IAExD,IAAI,CAACA,iBACH;IAGF,MAAME,OAAOC,gBAAgBH,gBAAgB,gBAAgB;IAE7D,MAAMI,UAAUC,sBACdH,MACA,AAAkC,YAAlC,OAAOF,gBAAgB,MAAM,GACzBA,gBAAgB,MAAM,GACtBA,gBAAgB,MAAM,EAAE,UAAU;IAExC,OAAOI;AACT;AAEO,eAAeE,sBACpBC,OAGC,EACDC,UAA2C,EAC3CC,WAAwB,EACxBC,SAA8B;IAE9B,IAAI,CAACF,YACH;IAGF,IAAIE,AAAc,UAAdA,WAAqB,YACvBC,8BAAW,iCAAiCF;IAI9C,IAAI,CAACF,QAAQ,SAAS,EAAE,mBACtB;IAGF,IAAI,CAACA,QAAQ,iBAAiB,CAAC,uBAAuB,EAAE,YACtDI,8BACE;IAKJ,IAAI;QACF,MAAMT,OACJ,MAAMK,QAAQ,iBAAiB,CAAC,uBAAuB,CAACC;QAC1D,MAAMJ,UAA+B;YACnC,QAAQ;gBACN9C,KAAK,KAAK,CAAC4C,KAAK,IAAI,GAAGA,KAAK,KAAK,GAAG;gBACpC5C,KAAK,KAAK,CAAC4C,KAAK,GAAG,GAAGA,KAAK,MAAM,GAAG;aACrC;YACDA;YACA,aACE,AAAuB,YAAvB,OAAOO,cACHA,cACAA,YAAY,MAAM,IAAI;QAC9B;QAEAE,8BAAW,yBAAyBF;QACpC,OAAOL;IACT,EAAE,OAAOQ,OAAO;QACdD,8BAAW,qCAAqCC;QAChD;IACF;AACF;AAIO,MAAMC,qBAAqB,IAEvBC;AAUJ,MAAMC,cAAc,CACzBC;IAKA,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAO;QACL,YAAYA;QACZ,kBAAkBC;IACpB;IAEF,OAAO;QACL,YAAYD,OAAO,MAAM;QACzB,kBAAkBA,OAAO,MAAM,GAC3B;YACE,QAAQA,OAAO,MAAM;YACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;QAC3D,IACAC;IACN;AACF;AAEO,MAAMC,sCAAsC,CACjDd,SACAhD;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAOgD;IAGT,OAAO;QACL,GAAGA,OAAO;QACV,QAAQ;YACN9C,KAAK,KAAK,CAAC8C,QAAQ,MAAM,CAAC,EAAE,GAAGhD;YAC/BE,KAAK,KAAK,CAAC8C,QAAQ,MAAM,CAAC,EAAE,GAAGhD;SAChC;QACD,MAAM;YACJ,GAAGgD,QAAQ,IAAI;YACf,MAAM9C,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,IAAI,GAAGhD;YACrC,KAAKE,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,GAAG,GAAGhD;YACnC,OAAOE,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,KAAK,GAAGhD;YACvC,QAAQE,KAAK,KAAK,CAAC8C,QAAQ,IAAI,CAAC,MAAM,GAAGhD;QAC3C;IACF;AACF;AAEO,MAAM+D,uCAAuC,CAClDjB,MACA9C;IAEA,IAAIA,AAA6B,MAA7BA,0BACF,OAAO8C;IAGT,OAAO;QACL,GAAGA,IAAI;QACP,MAAM5C,KAAK,KAAK,CAAC4C,KAAK,IAAI,GAAG9C;QAC7B,KAAKE,KAAK,KAAK,CAAC4C,KAAK,GAAG,GAAG9C;QAC3B,OAAOE,KAAK,KAAK,CAAC4C,KAAK,KAAK,GAAG9C;QAC/B,QAAQE,KAAK,KAAK,CAAC4C,KAAK,MAAM,GAAG9C;IACnC;AACF"}

package/dist/es/ai-model/index.mjs

@@ -6,9 +6,8 @@ import { generatePlaywrightTest, generatePlaywrightTestStream } from "./prompt/p
 import { convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, generateRecorderMarkdownReplay } from "./prompt/markdown-generator.mjs";
 import { generateRecorderSessionMetadata } from "./prompt/recorder-metadata-generator.mjs";
 import { generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream } from "./prompt/yaml-generator.mjs";
-import { describeRecorderUIEvent, describeRecorderUIEvents, getRecorderUIEventTargetRect } from "../recorder-ui-describer.mjs";
 import { AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection } from "./inspect.mjs";
 import { plan } from "./llm-planning.mjs";
 import { ConversationHistory } from "./conversation-history.mjs";
 import { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, dumpActionParam, findAllMidsceneLocatorField, getMidsceneLocationSchema, parseActionParam } from "../common.mjs";
-export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, callAI, callAIWithObjectResponse, callAIWithStringResponse, convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, describeRecorderUIEvent, describeRecorderUIEvents, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, getModelRuntime, getRecorderUIEventTargetRect, parseActionParam, plan, runConnectivityTest, systemPromptToLocateElement };
+export { AIResponseParseError, AiExtractElementInfo, AiJudgeOrderSensitive, AiLocateElement, AiLocateSection, ConversationHistory, PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, callAI, callAIWithObjectResponse, callAIWithStringResponse, convertRecordLogIntoMarkdown, createRecorderMarkdownReplayPrompt, dumpActionParam, findAllMidsceneLocatorField, generatePlaywrightTest, generatePlaywrightTestStream, generateRecorderMarkdownReplay, generateRecorderSessionMetadata, generateRecorderYamlTest, generateRecorderYamlTestStream, generateYamlTest, generateYamlTestStream, getMidsceneLocationSchema, getModelRuntime, parseActionParam, plan, runConnectivityTest, systemPromptToLocateElement };

package/dist/es/ai-model/inspect.mjs

@@ -43,29 +43,76 @@ async function buildSearchAreaConfig(options) {
 }
 async function AiLocateElement(options) {
     const { targetElementDescription, ...locateOptions } = options;
+    assert(targetElementDescription, "cannot find the target element description");
+    const { context } = locateOptions;
+    const locateImage = locateOptions.searchConfig?.image ?? {
+        imageBase64: context.screenshot.base64,
+        width: context.shotSize.width,
+        height: context.shotSize.height
+    };
+    const referenceImageMessages = 'string' == typeof targetElementDescription ? void 0 : await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(targetElementDescription));
+    const locateRequest = {
+        elementDescriptionText: userPromptToString(targetElementDescription),
+        locateImage,
+        referenceImageMessages,
+        options: locateOptions
+    };
     const locateAdapter = options.modelRuntime.adapter.locate;
-    if ('custom' === locateAdapter.kind) return locateAdapter.locateFn(targetElementDescription, locateOptions);
-    return genericLocate(targetElementDescription, locateOptions);
+    const locateFn = 'custom' === locateAdapter.kind ? locateAdapter.locateFn : genericLocate;
+    const locateResponse = await locateFn(targetElementDescription, locateOptions, locateRequest);
+    const { locatedPixelBbox, rawResponse, rawChoiceMessage, usage, reasoningContent, errors = [] } = locateResponse;
+    const baseLocateResult = {
+        rawResponse,
+        rawChoiceMessage,
+        usage,
+        reasoning_content: reasoningContent
+    };
+    if (!locatedPixelBbox) return {
+        rect: void 0,
+        parseResult: {
+            element: void 0,
+            errors
+        },
+        ...baseLocateResult
+    };
+    try {
+        const rect = pixelBboxToRect(mapSearchAreaPixelBboxToOriginalPixelBbox(locatedPixelBbox, locateOptions.searchConfig?.mapping));
+        debugInspect('resRect', rect);
+        return {
+            rect,
+            parseResult: {
+                element: generateElementByRect(rect, locateRequest.elementDescriptionText),
+                errors: []
+            },
+            ...baseLocateResult
+        };
+    } catch (error) {
+        const msg = error instanceof Error ? `Failed to parse locate result: ${error.message}` : 'unknown error in locate';
+        return {
+            rect: void 0,
+            parseResult: {
+                element: void 0,
+                errors: errors.length > 0 ? [
+                    ...errors,
+                    `(${msg})`
+                ] : [
+                    msg
+                ]
+            },
+            ...baseLocateResult
+        };
+    }
 }
-async function genericLocate(elementDescription, options) {
-    const { context } = options;
+async function genericLocate(_elementDescription, options, locateRequest) {
     const modelRuntime = options.modelRuntime;
     const { adapter } = modelRuntime;
     assert('standard' === adapter.locate.kind, 'generic locate requires a standard locate adapter');
-    const screenshotBase64 = context.screenshot.base64;
-    assert(elementDescription, "cannot find the target element description");
-    const elementDescriptionText = userPromptToString(elementDescription);
-    const userInstructionPrompt = findElementPrompt(elementDescriptionText);
+    const userInstructionPrompt = findElementPrompt(locateRequest.elementDescriptionText);
     const systemPrompt = systemPromptToLocateElement(adapter.locate.resultAdapter.promptSpec);
-    const modelImage = options.searchConfig?.image ?? {
-        imageBase64: screenshotBase64,
-        width: context.shotSize.width,
-        height: context.shotSize.height
-    };
     const preparedImage = await prepareModelImage({
-        imageBase64: modelImage.imageBase64,
-        width: modelImage.width,
-        height: modelImage.height,
+        imageBase64: locateRequest.locateImage.imageBase64,
+        width: locateRequest.locateImage.width,
+        height: locateRequest.locateImage.height,
         policy: adapter.imagePreprocess
     });
     const imagePayload = preparedImage.imageBase64;
@@ -91,10 +138,7 @@ async function genericLocate(elementDescription, options) {
             ]
         }
     ];
-    if ('string' != typeof elementDescription) {
-        const addOns = await multimodalPromptToChatMessages(userPromptToMultimodalPrompt(elementDescription));
-        msgs.push(...addOns);
-    }
+    if (locateRequest.referenceImageMessages) msgs.push(...locateRequest.referenceImageMessages);
     let res;
     try {
         res = await callAIWithObjectResponse(msgs, modelRuntime, {
@@ -107,46 +151,30 @@ async function genericLocate(elementDescription, options) {
         const usage = callError instanceof AIResponseParseError ? callError.usage : void 0;
         const rawChoiceMessage = callError instanceof AIResponseParseError ? callError.rawChoiceMessage : void 0;
         return {
-            rect: void 0,
-            parseResult: {
-                element: void 0,
-                errors: [
-                    `AI call error: ${errorMessage}`
-                ]
-            },
             rawResponse,
             rawChoiceMessage,
             usage,
-            reasoning_content: void 0
+            errors: [
+                `AI call error: ${errorMessage}`
+            ]
         };
     }
     const rawResponse = JSON.stringify(res.content);
-    let resRect;
-    let matchedElement;
     let errors = 'errors' in res.content ? res.content.errors : [];
     const resultAdapter = adapter.locate.resultAdapter;
     if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) return {
-        rect: void 0,
-        parseResult: {
-            element: void 0,
-            errors: errors
-        },
         rawResponse,
         rawChoiceMessage: res.rawChoiceMessage,
         usage: res.usage,
-        reasoning_content: res.reasoning_content
+        reasoningContent: res.reasoning_content,
+        errors: errors
     };
+    let targetPixelBbox;
     try {
-        const mapping = options.searchConfig?.mapping;
-        const targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(res.content, {
+        targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(res.content, {
             preparedSize: preparedImage.preparedSize,
             contentSize: preparedImage.contentSize
         });
-        resRect = pixelBboxToRect(mapSearchAreaPixelBboxToOriginalPixelBbox(targetPixelBbox, mapping));
-        debugInspect('resRect', resRect);
-        const element = generateElementByRect(resRect, elementDescriptionText);
-        errors = [];
-        if (element) matchedElement = element;
     } catch (e) {
         const msg = e instanceof Error ? `Failed to parse locate result: ${e.message}` : 'unknown error in locate';
         if (errors && errors?.length !== 0) errors.push(`(${msg})`);
@@ -155,15 +183,12 @@ async function genericLocate(elementDescription, options) {
         ];
     }
     return {
-        rect: resRect,
-        parseResult: {
-            element: matchedElement,
-            errors: errors
-        },
+        locatedPixelBbox: targetPixelBbox,
         rawResponse,
         rawChoiceMessage: res.rawChoiceMessage,
         usage: res.usage,
-        reasoning_content: res.reasoning_content
+        reasoningContent: res.reasoning_content,
+        errors: errors
     };
 }
 async function AiLocateSection(options) {
@@ -306,7 +331,9 @@ async function AiExtractElementInfo(options) {
         const addOns = await multimodalPromptToChatMessages(multimodalPrompt);
         msgs.push(...addOns);
     }
-    const { content: rawResponse, usage, reasoning_content, rawChoiceMessage } = await callAI(msgs, modelRuntime);
+    const { content: rawResponse, usage, reasoning_content, rawChoiceMessage } = await callAI(msgs, modelRuntime, {
+        abortSignal: options.abortSignal
+    });
     let parseResult;
     try {
         parseResult = parseXMLExtractionResponse(rawResponse);

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateOptions,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  if (locateAdapter.kind === 'custom') {\n    return locateAdapter.locateFn(targetElementDescription, locateOptions);\n  }\n  return genericLocate(targetElementDescription, locateOptions);\n}\n\nexport async function genericLocate(\n  elementDescription: TUserPrompt,\n  options: LocateOptions,\n): Promise<LocateResult> {\n  const { context } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n\n  assert(elementDescription, 'cannot find the target element description');\n  const elementDescriptionText = userPromptToString(elementDescription);\n  const userInstructionPrompt = findElementPrompt(elementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const modelImage = options.searchConfig?.image ?? {\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const preparedImage = await prepareModelImage({\n    imageBase64: modelImage.imageBase64,\n    width: modelImage.width,\n    height: modelImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof elementDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(elementDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let res: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AIElementLocateResponse>>\n  >;\n  try {\n    res = await callAIWithObjectResponse<AIElementLocateResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'locate',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: [`AI call error: ${errorMessage}`],\n      },\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      reasoning_content: undefined,\n    };\n  }\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElement: LocateResultElement | undefined;\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors as string[],\n      },\n      rawResponse,\n      rawChoiceMessage: res.rawChoiceMessage,\n      usage: res.usage,\n      reasoning_content: res.reasoning_content,\n    };\n  }\n\n  try {\n    const mapping = options.searchConfig?.mapping;\n    const targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(\n      res.content,\n      {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    );\n    resRect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(targetPixelBbox, mapping),\n    );\n\n    debugInspect('resRect', resRect);\n\n    const element: LocateResultElement = generateElementByRect(\n      resRect,\n      elementDescriptionText as string,\n    );\n    errors = [];\n\n    if (element) {\n      matchedElement = element;\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse locate result: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      element: matchedElement,\n      errors: errors as string[],\n    },\n    rawResponse,\n    rawChoiceMessage: res.rawChoiceMessage,\n    usage: res.usage,\n    reasoning_content: res.reasoning_content,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let result: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n  >;\n  try {\n    result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'section-locator',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n    };\n  }\n\n  let searchAreaConfig:\n    | Awaited<ReturnType<typeof buildSearchAreaConfig>>\n    | undefined;\n  let sectionError = result.content.error;\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: JSON.stringify(result.content),\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const adaptedResult =\n      resultAdapter.adaptSectionLocateResultToPixelBboxGroup(result.content, {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      });\n    const mergedRect = mergePixelBboxesToRect([\n      adaptedResult.target,\n      ...(adaptedResult.references ?? []),\n    ]);\n    debugSection('mergedRect %j', mergedRect);\n\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    sectionError = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n  }\n\n  return {\n    searchAreaConfig,\n    error: sectionError,\n    rawResponse: JSON.stringify(result.content),\n    rawChoiceMessage: result.rawChoiceMessage,\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'text',\n      text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n    });\n\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime);\n\n  let parseResult: AIDataExtractionResponse<T>;\n  try {\n    parseResult = parseXMLExtractionResponse<T>(rawResponse);\n  } catch (parseError) {\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  return {\n    parseResult,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","locateAdapter","genericLocate","elementDescription","modelRuntime","adapter","assert","screenshotBase64","elementDescriptionText","userPromptToString","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","modelImage","preparedImage","prepareModelImage","imagePayload","msgs","addOns","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","res","callAIWithObjectResponse","callError","errorMessage","Error","String","rawResponse","AIResponseParseError","usage","rawChoiceMessage","JSON","resRect","matchedElement","errors","resultAdapter","mapping","targetPixelBbox","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","element","generateElementByRect","e","msg","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","searchAreaConfig","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","error","parseErrorMessage","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AAiEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAEO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvD,MAAMa,gBAAgBb,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,IAAIa,AAAuB,aAAvBA,cAAc,IAAI,EACpB,OAAOA,cAAc,QAAQ,CAACF,0BAA0BC;IAE1D,OAAOE,cAAcH,0BAA0BC;AACjD;AAEO,eAAeE,cACpBC,kBAA+B,EAC/Bf,OAAsB;IAEtB,MAAM,EAAEC,OAAO,EAAE,GAAGD;IACpB,MAAMgB,eAAehB,QAAQ,YAAY;IACzC,MAAM,EAAEiB,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,mBAAmBlB,QAAQ,UAAU,CAAC,MAAM;IAElDiB,OAAOH,oBAAoB;IAC3B,MAAMK,yBAAyBC,mBAAmBN;IAClD,MAAMO,wBAAwBC,kBAAkBH;IAChD,MAAMI,eAAeC,4BACnBR,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMS,aAAa1B,QAAQ,YAAY,EAAE,SAAS;QAChD,aAAamB;QACb,OAAOlB,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAM0B,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaF,WAAW,WAAW;QACnC,OAAOA,WAAW,KAAK;QACvB,QAAQA,WAAW,MAAM;QACzB,QAAQT,QAAQ,eAAe;IACjC;IAEA,MAAMY,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKK;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMP;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOP,oBAAiC;QAC1C,MAAMgB,SAAS,MAAMC,+BACnBC,6BAA6BlB;QAE/Be,KAAK,IAAI,IAAIC;IACf;IAEA,IAAIG;IAGJ,IAAI;QACFA,MAAM,MAAMC,yBACVL,MACAd,cACA;YACE,aAAahB,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOoC,WAAW;QAClB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMK,QACJN,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGtC;QAChE,MAAM6C,mBACJP,qBAAqBK,uBACjBL,UAAU,gBAAgB,GAC1BtC;QACN,OAAO;YACL,MAAMA;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ;oBAAC,CAAC,eAAe,EAAEuC,cAAc;iBAAC;YAC5C;YACAG;YACAG;YACAD;YACA,mBAAmB5C;QACrB;IACF;IAEA,MAAM0C,cAAcI,KAAK,SAAS,CAACV,IAAI,OAAO;IAE9C,IAAIW;IACJ,IAAIC;IACJ,IAAIC,SACF,YAAYb,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,MAAMc,gBAAgB/B,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAACzB,gBAAgB0C,IAAI,OAAO,EAAEc,cAAc,UAAU,CAAC,SAAS,GAClE,OAAO;QACL,MAAMlD;QACN,aAAa;YACX,SAASA;YACT,QAAQiD;QACV;QACAP;QACA,kBAAkBN,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;IAGF,IAAI;QACF,MAAMe,UAAUjD,QAAQ,YAAY,EAAE;QACtC,MAAMkD,kBAAkBF,cAAc,mCAAmC,CACvEd,IAAI,OAAO,EACX;YACE,cAAcP,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QAEFkB,UAAUM,gBACRC,0CAA0CF,iBAAiBD;QAG7D5D,aAAa,WAAWwD;QAExB,MAAMQ,UAA+BC,sBACnCT,SACAzB;QAEF2B,SAAS,EAAE;QAEX,IAAIM,SACFP,iBAAiBO;IAErB,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAajB,QACT,CAAC,+BAA+B,EAAEiB,EAAE,OAAO,EAAE,GAC7C;QACN,IAAI,AAACR,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;aAFtBT,SAAS;YAACS;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMX;QACN,aAAa;YACX,SAASC;YACT,QAAQC;QACV;QACAP;QACA,kBAAkBN,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,mBAAmBA,IAAI,iBAAiB;IAC1C;AACF;AAEO,eAAeuB,gBAAgBzD,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEyD,kBAAkB,EAAE,GAAG1D;IACxC,MAAMgB,eAAehB,QAAQ,YAAY;IACzC,MAAM,EAAEiB,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,mBAAmBlB,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM0B,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaT;QACb,OAAOlB,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQgB,QAAQ,eAAe;IACjC;IAEA,MAAMO,eAAemC,4BACnB1C,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAM2C,gCAAgCC,0BACpCxC,mBAAmBqC;IAErB,MAAM5B,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKG,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMiC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAM3B,SAAS,MAAMC,+BACnBC,6BAA6ByB;QAE/B5B,KAAK,IAAI,IAAIC;IACf;IAEA,IAAI+B;IAGJ,IAAI;QACFA,SAAS,MAAM3B,yBACbL,MACAd,cACA;YACE,aAAahB,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOoC,WAAW;QAClB,MAAMC,eACJD,qBAAqBE,QAAQF,UAAU,OAAO,GAAGG,OAAOH;QAC1D,MAAMI,cACJJ,qBAAqBK,uBACjBL,UAAU,WAAW,GACrBC;QACN,MAAMK,QACJN,qBAAqBK,uBAAuBL,UAAU,KAAK,GAAGtC;QAChE,MAAM6C,mBACJP,qBAAqBK,uBACjBL,UAAU,gBAAgB,GAC1BtC;QACN,OAAO;YACL,kBAAkBA;YAClB,OAAO,CAAC,eAAe,EAAEuC,cAAc;YACvCG;YACAG;YACAD;QACF;IACF;IAEA,IAAIqB;IAGJ,IAAIC,eAAeF,OAAO,OAAO,CAAC,KAAK;IACvC,MAAMd,gBAAgB/B,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAACzB,gBAAgBsE,OAAO,OAAO,EAAEd,cAAc,UAAU,CAAC,SAAS,GACrE,OAAO;QACL,kBAAkBlD;QAClB,OAAOkE;QACP,aAAapB,KAAK,SAAS,CAACkB,OAAO,OAAO;QAC1C,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMG,gBACJjB,cAAc,wCAAwC,CAACc,OAAO,OAAO,EAAE;YACrE,cAAcnC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACF,MAAMuC,aAAaC,uBAAuB;YACxCF,cAAc,MAAM;eAChBA,cAAc,UAAU,IAAI,EAAE;SACnC;QACD1E,aAAa,iBAAiB2E;QAE9B,MAAME,eAAe/D,iBAAiB6D,YAAYjE,QAAQ,QAAQ;QAClE,MAAMoE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1C7E,aAAa,2BAA2B6E;QAExCL,mBAAmB,MAAMhE,sBAAsB;YAC7CE;YACA,UAAUiE;QACZ;QAEA3E,aACE,uDACA8E,eACAC,gBACAP,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;IAElC,EAAE,OAAOQ,OAAO;QACd,MAAMC,oBACJD,iBAAiBjC,QACb,CAAC,uCAAuC,EAAEiC,MAAM,OAAO,EAAE,GACzD;QACNP,eAAeA,eACX,GAAGA,aAAa,EAAE,EAAEQ,kBAAkB,CAAC,CAAC,GACxCA;IACN;IAEA,OAAO;QACLT;QACA,OAAOC;QACP,aAAapB,KAAK,SAAS,CAACkB,OAAO,OAAO;QAC1C,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeW,qBAAwBzE,OAO7C;IACC,MAAM,EAAE0E,SAAS,EAAEzE,OAAO,EAAE0E,aAAa,EAAEC,gBAAgB,EAAE5D,YAAY,EAAE,GACzEhB;IACF,MAAMwB,eAAeqD,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMzD,mBAAmBlB,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAM6E,wBAAwBC,uBAC5B/E,QAAQ,eAAe,IAAI,IAC3B0E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/CK,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QAEAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBACT,KAAK7D;gBACL,QAAQ;YACV;QACF;IACF;IAEA6D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMhD,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAASwD;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAM7C,SAAS,MAAMC,+BAA+B4C;QACpD9C,KAAK,IAAI,IAAIC;IACf;IAEA,MAAM,EACJ,SAASS,WAAW,EACpBE,KAAK,EACLuC,iBAAiB,EACjBtC,gBAAgB,EACjB,GAAG,MAAMuC,OAAOpD,MAAMd;IAEvB,IAAImE;IACJ,IAAI;QACFA,cAAcC,2BAA8B5C;IAC9C,EAAE,OAAO6C,YAAY;QACnB,MAAMhD,eACJgD,sBAAsB/C,QAAQ+C,WAAW,OAAO,GAAG9C,OAAO8C;QAC5D,MAAM,IAAI5C,qBACR,CAAC,iBAAiB,EAAEJ,cAAc,EAClCG,aACAE,OACAC;IAEJ;IAEA,OAAO;QACLwC;QACA3C;QACAG;QACAD;QACAuC;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBvE,YAA0B;IAK1B,MAAMQ,eAAegE;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMzD,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASN;QAAa;QACxC;YACE,MAAM;YACN,SAASiE;QACX;KACD;IAEDpG,aAAa,yCAAyCkG;IAEtD,MAAMzB,SAAS,MAAM3B,yBACnBL,MACAd,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB8C,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["../../../src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementLocateResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport { generateElementByRect } from '@midscene/shared/extractor';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  expandSearchArea,\n  multimodalPromptToChatMessages,\n  userPromptToMultimodalPrompt,\n  userPromptToString,\n} from '../common';\nimport type { ModelRuntime } from './models';\nimport {\n  extractDataQueryPrompt,\n  parseXMLExtractionResponse,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport {\n  AIResponseParseError,\n  callAI,\n  callAIWithObjectResponse,\n} from './service-caller/index';\nimport { prepareModelImage } from './workflows/image-preprocess';\nimport {\n  mergePixelBboxesToRect,\n  pixelBboxToRect,\n} from './workflows/inspect/locate-result-rect';\nimport { mapSearchAreaPixelBboxToOriginalPixelBbox } from './workflows/inspect/search-area-mapping';\nimport type {\n  LocateModelResponse,\n  LocateOptions,\n  LocateRequestContext,\n  LocateResult,\n  SearchAreaConfig,\n} from './workflows/inspect/types';\n\nexport type InspectAIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nexport {\n  userPromptToString as extraTextFromUserPrompt,\n  multimodalPromptToChatMessages as promptsToChatParam,\n} from '../common';\n\nfunction hasLocateResult(input: unknown, resultKey: string) {\n  if (!input || typeof input !== 'object') {\n    return false;\n  }\n\n  const record = input as Record<string, unknown>;\n  const locateResult = record[resultKey];\n  return Array.isArray(locateResult)\n    ? locateResult.length > 0\n    : locateResult !== undefined;\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n\nexport async function AiLocateElement(\n  options: LocateOptions & { targetElementDescription: TUserPrompt },\n): Promise<LocateResult> {\n  const { targetElementDescription, ...locateOptions } = options;\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n\n  const { context } = locateOptions;\n  const locateImage = locateOptions.searchConfig?.image ?? {\n    imageBase64: context.screenshot.base64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n  };\n  const referenceImageMessages =\n    typeof targetElementDescription === 'string'\n      ? undefined\n      : await multimodalPromptToChatMessages(\n          userPromptToMultimodalPrompt(targetElementDescription),\n        );\n  const locateRequest: LocateRequestContext = {\n    elementDescriptionText: userPromptToString(targetElementDescription),\n    locateImage,\n    referenceImageMessages,\n    options: locateOptions,\n  };\n\n  const locateAdapter = options.modelRuntime.adapter.locate;\n  const locateFn =\n    locateAdapter.kind === 'custom' ? locateAdapter.locateFn : genericLocate;\n  const locateResponse = await locateFn(\n    targetElementDescription,\n    locateOptions,\n    locateRequest,\n  );\n  const {\n    locatedPixelBbox,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoningContent,\n    errors = [],\n  } = locateResponse;\n  const baseLocateResult = {\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content: reasoningContent,\n  };\n\n  if (!locatedPixelBbox) {\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors,\n      },\n      ...baseLocateResult,\n    };\n  }\n\n  try {\n    const rect = pixelBboxToRect(\n      mapSearchAreaPixelBboxToOriginalPixelBbox(\n        locatedPixelBbox,\n        locateOptions.searchConfig?.mapping,\n      ),\n    );\n    debugInspect('resRect', rect);\n\n    return {\n      rect,\n      parseResult: {\n        element: generateElementByRect(\n          rect,\n          locateRequest.elementDescriptionText,\n        ),\n        errors: [],\n      },\n      ...baseLocateResult,\n    };\n  } catch (error) {\n    const msg =\n      error instanceof Error\n        ? `Failed to parse locate result: ${error.message}`\n        : 'unknown error in locate';\n    return {\n      rect: undefined,\n      parseResult: {\n        element: undefined,\n        errors: errors.length > 0 ? [...errors, `(${msg})`] : [msg],\n      },\n      ...baseLocateResult,\n    };\n  }\n}\n\nexport async function genericLocate(\n  _elementDescription: TUserPrompt,\n  options: LocateOptions,\n  locateRequest: LocateRequestContext,\n): Promise<LocateModelResponse> {\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'generic locate requires a standard locate adapter',\n  );\n  const userInstructionPrompt = findElementPrompt(\n    locateRequest.elementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: locateRequest.locateImage.imageBase64,\n    width: locateRequest.locateImage.width,\n    height: locateRequest.locateImage.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const imagePayload = preparedImage.imageBase64;\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (locateRequest.referenceImageMessages) {\n    msgs.push(...locateRequest.referenceImageMessages);\n  }\n\n  let res: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AIElementLocateResponse>>\n  >;\n  try {\n    res = await callAIWithObjectResponse<AIElementLocateResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'locate',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n      errors: [`AI call error: ${errorMessage}`],\n    };\n  }\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(res.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      rawResponse,\n      rawChoiceMessage: res.rawChoiceMessage,\n      usage: res.usage,\n      reasoningContent: res.reasoning_content,\n      errors: errors as string[],\n    };\n  }\n\n  let targetPixelBbox;\n  try {\n    targetPixelBbox = resultAdapter.adaptElementLocateResultToPixelBbox(\n      res.content,\n      {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      },\n    );\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse locate result: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    locatedPixelBbox: targetPixelBbox,\n    rawResponse,\n    rawChoiceMessage: res.rawChoiceMessage,\n    usage: res.usage,\n    reasoningContent: res.reasoning_content,\n    errors: errors as string[],\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const screenshotBase64 = context.screenshot.base64;\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: context.shotSize.width,\n    height: context.shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n\n  const systemPrompt = systemPromptToLocateSection(\n    adapter.locate.resultAdapter.promptSpec,\n  );\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    userPromptToString(sectionDescription),\n  );\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: preparedImage.imageBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await multimodalPromptToChatMessages(\n      userPromptToMultimodalPrompt(sectionDescription),\n    );\n    msgs.push(...addOns);\n  }\n\n  let result: Awaited<\n    ReturnType<typeof callAIWithObjectResponse<AISectionLocatorResponse>>\n  >;\n  try {\n    result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n      msgs,\n      modelRuntime,\n      {\n        abortSignal: options.abortSignal,\n        jsonParserSource: 'section-locator',\n      },\n    );\n  } catch (callError) {\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    const rawResponse =\n      callError instanceof AIResponseParseError\n        ? callError.rawResponse\n        : errorMessage;\n    const usage =\n      callError instanceof AIResponseParseError ? callError.usage : undefined;\n    const rawChoiceMessage =\n      callError instanceof AIResponseParseError\n        ? callError.rawChoiceMessage\n        : undefined;\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse,\n      rawChoiceMessage,\n      usage,\n    };\n  }\n\n  let searchAreaConfig:\n    | Awaited<ReturnType<typeof buildSearchAreaConfig>>\n    | undefined;\n  let sectionError = result.content.error;\n  const resultAdapter = adapter.locate.resultAdapter;\n  if (!hasLocateResult(result.content, resultAdapter.promptSpec.resultKey)) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: JSON.stringify(result.content),\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const adaptedResult =\n      resultAdapter.adaptSectionLocateResultToPixelBboxGroup(result.content, {\n        preparedSize: preparedImage.preparedSize,\n        contentSize: preparedImage.contentSize,\n      });\n    const mergedRect = mergePixelBboxesToRect([\n      adaptedResult.target,\n      ...(adaptedResult.references ?? []),\n    ]);\n    debugSection('mergedRect %j', mergedRect);\n\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    sectionError = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n  }\n\n  return {\n    searchAreaConfig,\n    error: sectionError,\n    rawResponse: JSON.stringify(result.content),\n    rawChoiceMessage: result.rawChoiceMessage,\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelRuntime } =\n    options;\n  const systemPrompt = systemPromptToExtract({\n    screenshotIncluded: extractOption?.screenshotIncluded !== false,\n    referenceImagesIncluded: !!multimodalPrompt?.images?.length,\n  });\n  const screenshotBase64 = context.screenshot.base64;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'text',\n      text: 'This is the current screenshot to evaluate. Unless <DATA_DEMAND> explicitly asks for comparison or matching against reference images, base your answer on this screenshot and its contents when provided.',\n    });\n\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await multimodalPromptToChatMessages(multimodalPrompt);\n    msgs.push(...addOns);\n  }\n\n  const {\n    content: rawResponse,\n    usage,\n    reasoning_content,\n    rawChoiceMessage,\n  } = await callAI(msgs, modelRuntime, {\n    abortSignal: options.abortSignal,\n  });\n\n  let parseResult: AIDataExtractionResponse<T>;\n  try {\n    parseResult = parseXMLExtractionResponse<T>(rawResponse);\n  } catch (parseError) {\n    const errorMessage =\n      parseError instanceof Error ? parseError.message : String(parseError);\n    throw new AIResponseParseError(\n      `XML parse error: ${errorMessage}`,\n      rawResponse,\n      usage,\n      rawChoiceMessage,\n    );\n  }\n\n  return {\n    parseResult,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  modelRuntime: ModelRuntime,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: InspectAIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  debugInspect('AiJudgeOrderSensitive: description=%s', description);\n\n  const result = await callAIWithObjectResponse<{ isOrderSensitive: boolean }>(\n    msgs,\n    modelRuntime,\n    {\n      jsonParserSource: 'generic-object',\n    },\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","hasLocateResult","input","resultKey","record","locateResult","Array","undefined","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","expandSearchArea","croppedResult","cropByRect","scaledResult","scaleImage","AiLocateElement","targetElementDescription","locateOptions","assert","locateImage","referenceImageMessages","multimodalPromptToChatMessages","userPromptToMultimodalPrompt","locateRequest","userPromptToString","locateAdapter","locateFn","genericLocate","locateResponse","locatedPixelBbox","rawResponse","rawChoiceMessage","usage","reasoningContent","errors","baseLocateResult","rect","pixelBboxToRect","mapSearchAreaPixelBboxToOriginalPixelBbox","generateElementByRect","error","msg","Error","_elementDescription","modelRuntime","adapter","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","preparedImage","prepareModelImage","imagePayload","msgs","res","callAIWithObjectResponse","callError","errorMessage","String","AIResponseParseError","JSON","resultAdapter","targetPixelBbox","e","AiLocateSection","sectionDescription","screenshotBase64","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","addOns","result","searchAreaConfig","sectionError","adaptedResult","mergedRect","mergePixelBboxesToRect","expandedRect","originalWidth","originalHeight","parseErrorMessage","AiExtractElementInfo","dataQuery","extractOption","multimodalPrompt","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","reasoning_content","callAI","parseResult","parseXMLExtractionResponse","parseError","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;;;;AAkEA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAO9B,SAASE,gBAAgBC,KAAc,EAAEC,SAAiB;IACxD,IAAI,CAACD,SAAS,AAAiB,YAAjB,OAAOA,OACnB,OAAO;IAGT,MAAME,SAASF;IACf,MAAMG,eAAeD,MAAM,CAACD,UAAU;IACtC,OAAOG,MAAM,OAAO,CAACD,gBACjBA,aAAa,MAAM,GAAG,IACtBA,AAAiBE,WAAjBF;AACN;AAEO,eAAeG,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcC,iBAAiBH,UAAUD,QAAQ,QAAQ;IAE/D,MAAMK,gBAAgB,MAAMC,WAC1BN,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMI,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEH;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaI,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGJ,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF;AAEO,eAAeO,gBACpBV,OAAkE;IAElE,MAAM,EAAEW,wBAAwB,EAAE,GAAGC,eAAe,GAAGZ;IACvDa,OACEF,0BACA;IAGF,MAAM,EAAEV,OAAO,EAAE,GAAGW;IACpB,MAAME,cAAcF,cAAc,YAAY,EAAE,SAAS;QACvD,aAAaX,QAAQ,UAAU,CAAC,MAAM;QACtC,OAAOA,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;IACjC;IACA,MAAMc,yBACJ,AAAoC,YAApC,OAAOJ,2BACHb,SACA,MAAMkB,+BACJC,6BAA6BN;IAErC,MAAMO,gBAAsC;QAC1C,wBAAwBC,mBAAmBR;QAC3CG;QACAC;QACA,SAASH;IACX;IAEA,MAAMQ,gBAAgBpB,QAAQ,YAAY,CAAC,OAAO,CAAC,MAAM;IACzD,MAAMqB,WACJD,AAAuB,aAAvBA,cAAc,IAAI,GAAgBA,cAAc,QAAQ,GAAGE;IAC7D,MAAMC,iBAAiB,MAAMF,SAC3BV,0BACAC,eACAM;IAEF,MAAM,EACJM,gBAAgB,EAChBC,WAAW,EACXC,gBAAgB,EAChBC,KAAK,EACLC,gBAAgB,EAChBC,SAAS,EAAE,EACZ,GAAGN;IACJ,MAAMO,mBAAmB;QACvBL;QACAC;QACAC;QACA,mBAAmBC;IACrB;IAEA,IAAI,CAACJ,kBACH,OAAO;QACL,MAAM1B;QACN,aAAa;YACX,SAASA;YACT+B;QACF;QACA,GAAGC,gBAAgB;IACrB;IAGF,IAAI;QACF,MAAMC,OAAOC,gBACXC,0CACET,kBACAZ,cAAc,YAAY,EAAE;QAGhCvB,aAAa,WAAW0C;QAExB,OAAO;YACLA;YACA,aAAa;gBACX,SAASG,sBACPH,MACAb,cAAc,sBAAsB;gBAEtC,QAAQ,EAAE;YACZ;YACA,GAAGY,gBAAgB;QACrB;IACF,EAAE,OAAOK,OAAO;QACd,MAAMC,MACJD,iBAAiBE,QACb,CAAC,+BAA+B,EAAEF,MAAM,OAAO,EAAE,GACjD;QACN,OAAO;YACL,MAAMrC;YACN,aAAa;gBACX,SAASA;gBACT,QAAQ+B,OAAO,MAAM,GAAG,IAAI;uBAAIA;oBAAQ,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;iBAAC,GAAG;oBAACA;iBAAI;YAC7D;YACA,GAAGN,gBAAgB;QACrB;IACF;AACF;AAEO,eAAeR,cACpBgB,mBAAgC,EAChCtC,OAAsB,EACtBkB,aAAmC;IAEnC,MAAMqB,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMC,wBAAwBC,kBAC5BxB,cAAc,sBAAsB;IAEtC,MAAMyB,eAAeC,4BACnBJ,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAGzC,MAAMK,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAa5B,cAAc,WAAW,CAAC,WAAW;QAClD,OAAOA,cAAc,WAAW,CAAC,KAAK;QACtC,QAAQA,cAAc,WAAW,CAAC,MAAM;QACxC,QAAQsB,QAAQ,eAAe;IACjC;IAEA,MAAMO,eAAeF,cAAc,WAAW;IAE9C,MAAMG,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKI;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMN;gBACR;aACD;QACH;KACD;IAED,IAAIvB,cAAc,sBAAsB,EACtC8B,KAAK,IAAI,IAAI9B,cAAc,sBAAsB;IAGnD,IAAI+B;IAGJ,IAAI;QACFA,MAAM,MAAMC,yBACVF,MACAT,cACA;YACE,aAAavC,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOmD,WAAW;QAClB,MAAMC,eACJD,qBAAqBd,QAAQc,UAAU,OAAO,GAAGE,OAAOF;QAC1D,MAAM1B,cACJ0B,qBAAqBG,uBACjBH,UAAU,WAAW,GACrBC;QACN,MAAMzB,QACJwB,qBAAqBG,uBAAuBH,UAAU,KAAK,GAAGrD;QAChE,MAAM4B,mBACJyB,qBAAqBG,uBACjBH,UAAU,gBAAgB,GAC1BrD;QACN,OAAO;YACL2B;YACAC;YACAC;YACA,QAAQ;gBAAC,CAAC,eAAe,EAAEyB,cAAc;aAAC;QAC5C;IACF;IAEA,MAAM3B,cAAc8B,KAAK,SAAS,CAACN,IAAI,OAAO;IAE9C,IAAIpB,SACF,YAAYoB,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,MAAMO,gBAAgBhB,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAAChD,gBAAgByD,IAAI,OAAO,EAAEO,cAAc,UAAU,CAAC,SAAS,GAClE,OAAO;QACL/B;QACA,kBAAkBwB,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,kBAAkBA,IAAI,iBAAiB;QACvC,QAAQpB;IACV;IAGF,IAAI4B;IACJ,IAAI;QACFA,kBAAkBD,cAAc,mCAAmC,CACjEP,IAAI,OAAO,EACX;YACE,cAAcJ,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;IAEJ,EAAE,OAAOa,GAAG;QACV,MAAMtB,MACJsB,aAAarB,QACT,CAAC,+BAA+B,EAAEqB,EAAE,OAAO,EAAE,GAC7C;QACN,IAAI,AAAC7B,UAAUA,QAAQ,WAAW,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAEO,IAAI,CAAC,CAAC;aAFtBP,SAAS;YAACO;SAAI;IAIlB;IAEA,OAAO;QACL,kBAAkBqB;QAClBhC;QACA,kBAAkBwB,IAAI,gBAAgB;QACtC,OAAOA,IAAI,KAAK;QAChB,kBAAkBA,IAAI,iBAAiB;QACvC,QAAQpB;IACV;AACF;AAEO,eAAe8B,gBAAgB3D,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAE2D,kBAAkB,EAAE,GAAG5D;IACxC,MAAMuC,eAAevC,QAAQ,YAAY;IACzC,MAAM,EAAEwC,OAAO,EAAE,GAAGD;IACpB1B,OACE2B,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAMqB,mBAAmB5D,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAM4C,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAae;QACb,OAAO5D,QAAQ,QAAQ,CAAC,KAAK;QAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QAC/B,QAAQuC,QAAQ,eAAe;IACjC;IAEA,MAAMG,eAAemB,4BACnBtB,QAAQ,MAAM,CAAC,aAAa,CAAC,UAAU;IAEzC,MAAMuB,gCAAgCC,0BACpC7C,mBAAmByC;IAErB,MAAMZ,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE,cAAc,WAAW;wBAC9B,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMkB;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOH,oBAAiC;QAC1C,MAAMK,SAAS,MAAMjD,+BACnBC,6BAA6B2C;QAE/BZ,KAAK,IAAI,IAAIiB;IACf;IAEA,IAAIC;IAGJ,IAAI;QACFA,SAAS,MAAMhB,yBACbF,MACAT,cACA;YACE,aAAavC,QAAQ,WAAW;YAChC,kBAAkB;QACpB;IAEJ,EAAE,OAAOmD,WAAW;QAClB,MAAMC,eACJD,qBAAqBd,QAAQc,UAAU,OAAO,GAAGE,OAAOF;QAC1D,MAAM1B,cACJ0B,qBAAqBG,uBACjBH,UAAU,WAAW,GACrBC;QACN,MAAMzB,QACJwB,qBAAqBG,uBAAuBH,UAAU,KAAK,GAAGrD;QAChE,MAAM4B,mBACJyB,qBAAqBG,uBACjBH,UAAU,gBAAgB,GAC1BrD;QACN,OAAO;YACL,kBAAkBA;YAClB,OAAO,CAAC,eAAe,EAAEsD,cAAc;YACvC3B;YACAC;YACAC;QACF;IACF;IAEA,IAAIwC;IAGJ,IAAIC,eAAeF,OAAO,OAAO,CAAC,KAAK;IACvC,MAAMV,gBAAgBhB,QAAQ,MAAM,CAAC,aAAa;IAClD,IAAI,CAAChD,gBAAgB0E,OAAO,OAAO,EAAEV,cAAc,UAAU,CAAC,SAAS,GACrE,OAAO;QACL,kBAAkB1D;QAClB,OAAOsE;QACP,aAAab,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMG,gBACJb,cAAc,wCAAwC,CAACU,OAAO,OAAO,EAAE;YACrE,cAAcrB,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACF,MAAMyB,aAAaC,uBAAuB;YACxCF,cAAc,MAAM;eAChBA,cAAc,UAAU,IAAI,EAAE;SACnC;QACD9E,aAAa,iBAAiB+E;QAE9B,MAAME,eAAenE,iBAAiBiE,YAAYrE,QAAQ,QAAQ;QAClE,MAAMwE,gBAAgBD,aAAa,KAAK;QACxC,MAAME,iBAAiBF,aAAa,MAAM;QAC1CjF,aAAa,2BAA2BiF;QAExCL,mBAAmB,MAAMpE,sBAAsB;YAC7CE;YACA,UAAUqE;QACZ;QAEA/E,aACE,uDACAkF,eACAC,gBACAP,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;IAElC,EAAE,OAAOhC,OAAO;QACd,MAAMwC,oBACJxC,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACNiC,eAAeA,eACX,GAAGA,aAAa,EAAE,EAAEO,kBAAkB,CAAC,CAAC,GACxCA;IACN;IAEA,OAAO;QACLR;QACA,OAAOC;QACP,aAAab,KAAK,SAAS,CAACW,OAAO,OAAO;QAC1C,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeU,qBAAwB5E,OAQ7C;IACC,MAAM,EAAE6E,SAAS,EAAE5E,OAAO,EAAE6E,aAAa,EAAEC,gBAAgB,EAAExC,YAAY,EAAE,GACzEvC;IACF,MAAM2C,eAAeqC,sBAAsB;QACzC,oBAAoBF,eAAe,uBAAuB;QAC1D,yBAAyB,CAAC,CAACC,kBAAkB,QAAQ;IACvD;IACA,MAAMlB,mBAAmB5D,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMgF,wBAAwBC,uBAC5BlF,QAAQ,eAAe,IAAI,IAC3B6E;IAGF,MAAMM,cAAyD,EAAE;IAEjE,IAAIL,eAAe,uBAAuB,OAAO;QAC/CK,YAAY,IAAI,CAAC;YACf,MAAM;YACN,MAAM;QACR;QAEAA,YAAY,IAAI,CAAC;YACf,MAAM;YACN,WAAW;gBACT,KAAKtB;gBACL,QAAQ;YACV;QACF;IACF;IAEAsB,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMjC,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASwC;QACX;KACD;IAED,IAAIJ,kBAAkB;QACpB,MAAMd,SAAS,MAAMjD,+BAA+B+D;QACpD/B,KAAK,IAAI,IAAIiB;IACf;IAEA,MAAM,EACJ,SAASxC,WAAW,EACpBE,KAAK,EACLyD,iBAAiB,EACjB1D,gBAAgB,EACjB,GAAG,MAAM2D,OAAOrC,MAAMT,cAAc;QACnC,aAAavC,QAAQ,WAAW;IAClC;IAEA,IAAIsF;IACJ,IAAI;QACFA,cAAcC,2BAA8B9D;IAC9C,EAAE,OAAO+D,YAAY;QACnB,MAAMpC,eACJoC,sBAAsBnD,QAAQmD,WAAW,OAAO,GAAGnC,OAAOmC;QAC5D,MAAM,IAAIlC,qBACR,CAAC,iBAAiB,EAAEF,cAAc,EAClC3B,aACAE,OACAD;IAEJ;IAEA,OAAO;QACL4D;QACA7D;QACAC;QACAC;QACAyD;IACF;AACF;AAEO,eAAeK,sBACpBC,WAAmB,EACnBnD,YAA0B;IAK1B,MAAMI,eAAegD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAM1C,OAAsB;QAC1B;YAAE,MAAM;YAAU,SAASL;QAAa;QACxC;YACE,MAAM;YACN,SAASiD;QACX;KACD;IAEDvG,aAAa,yCAAyCqG;IAEtD,MAAMxB,SAAS,MAAMhB,yBACnBF,MACAT,cACA;QACE,kBAAkB;IACpB;IAGF,OAAO;QACL,kBAAkB2B,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/llm-planning.mjs

@@ -1,4 +1,4 @@
-import { buildYamlFlowFromPlans, findAllMidsceneLocatorField, userPromptToString } from "../common.mjs";
+import { buildYamlFlowFromPlans, userPromptToString } from "../common.mjs";
 import { getDebug } from "@midscene/shared/logger";
 import { assert } from "@midscene/shared/utils";
 import { planningModelFamilyRequiredForLocateMessage } from "./errors.mjs";
@@ -6,6 +6,7 @@ import { systemPromptToTaskPlanning } from "./prompt/llm-planning.mjs";
 import { extractXMLTag, parseMarkFinishedIndexes, parseSubGoalsFromXML } from "./prompt/util.mjs";
 import { AIResponseParseError, callAI } from "./service-caller/index.mjs";
 import { prepareModelImage } from "./workflows/image-preprocess.mjs";
+import { normalizePlanningActionLocateFields } from "./workflows/planning/locate-normalization.mjs";
 const debug = getDebug('planning');
 const warnLog = getDebug('planning', {
     console: true
@@ -209,31 +210,14 @@ async function plan(userInstruction, opts) {
             shouldContinuePlanning
         };
         assert(planFromAI, "can't get plans from AI");
-        actions.forEach((action)=>{
-            const type = action.type;
-            const actionInActionSpace = opts.actionSpace.find((action)=>action.name === type);
-            debug('actionInActionSpace matched', actionInActionSpace);
-            const locateFields = actionInActionSpace ? findAllMidsceneLocatorField(actionInActionSpace.paramSchema) : [];
-            debug('locateFields', locateFields);
-            locateFields.forEach((field)=>{
-                const locateResult = action.param[field];
-                if (locateResult) {
-                    if (!opts.includeLocateInPlanning) {
-                        if ('object' == typeof locateResult) action.param[field] = {
-                            prompt: locateResult.prompt
-                        };
-                        return;
-                    }
-                    assert(locateResultAdapter, 'generic planning locate normalization requires a standard locate adapter');
-                    action.param[field] = {
-                        ...locateResult,
-                        locatedPixelBbox: locateResultAdapter.adaptPlanningParamToPixelBbox(locateResult, {
-                            preparedSize: preparedImage.preparedSize,
-                            contentSize: preparedImage.contentSize
-                        })
-                    };
-                }
-            });
+        normalizePlanningActionLocateFields(actions, {
+            actionSpace: opts.actionSpace,
+            includeLocateInPlanning: opts.includeLocateInPlanning,
+            locateResultAdapter,
+            locateResultContext: {
+                preparedSize: preparedImage.preparedSize,
+                contentSize: preparedImage.contentSize
+            }
         });
         if (includeSubGoals) {
             if (planFromAI.updateSubGoals?.length) conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);