@midscene/core 1.12.5-beta-20260908034641.0 → 1.12.5

package/dist/es/ai-model/workflows/grounding/search-area.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/search-area.mjs","sources":["../../../../../src/ai-model/workflows/grounding/search-area.ts"],"sourcesContent":["import type { Rect, Size, UIContext } from '@/types';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport type { SearchAreaConfig } from './types';\n\n/**\n * Expand the search area to at least 400 x 400 pixels\n *\n * Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n *\n * Step 2: Ensure the area is at least 400x400 pixels\n * - Scale up proportionally from the center if needed\n * - Final result is clamped to screen boundaries\n */\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n  const minArea = 400 * 400;\n  const expandSize = 100;\n\n  // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n  // Note: If element is near boundary, actual expansion may be less than 100px on that side\n  const expandedLeft = Math.max(rect.left - expandSize, 0);\n  const expandedTop = Math.max(rect.top - expandSize, 0);\n\n  const expandRect = {\n    left: expandedLeft,\n    top: expandedTop,\n    width: Math.min(\n      rect.left - expandedLeft + rect.width + expandSize,\n      screenSize.width - expandedLeft,\n    ),\n    height: Math.min(\n      rect.top - expandedTop + rect.height + expandSize,\n      screenSize.height - expandedTop,\n    ),\n  };\n\n  // Step 2: Check if area is already >= 400x400\n  const currentArea = expandRect.width * expandRect.height;\n\n  if (currentArea >= minArea) {\n    return expandRect;\n  }\n\n  // Step 2: Scale up from center to reach minimum 400x400 area\n  const centerX = expandRect.left + expandRect.width / 2;\n  const centerY = expandRect.top + expandRect.height / 2;\n\n  // Calculate scale factor needed to reach minimum area\n  const scaleFactor = Math.sqrt(minArea / currentArea);\n  const newWidth = Math.round(expandRect.width * scaleFactor);\n  const newHeight = Math.round(expandRect.height * scaleFactor);\n\n  // Calculate new position based on center point\n  const newLeft = Math.round(centerX - newWidth / 2);\n  const newTop = Math.round(centerY - newHeight / 2);\n\n  // Clamp to screen boundaries\n  const left = Math.max(newLeft, 0);\n  const top = Math.max(newTop, 0);\n\n  return {\n    left,\n    top,\n    width: Math.min(newWidth, screenSize.width - left),\n    height: Math.min(newHeight, screenSize.height - top),\n  };\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n"],"names":["expandSearchArea","rect","screenSize","minArea","expandSize","expandedLeft","Math","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","left","top","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","croppedResult","cropByRect","scaledResult","scaleImage"],"mappings":";AAeO,SAASA,iBAAiBC,IAAU,EAAEC,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAeC,KAAK,GAAG,CAACL,KAAK,IAAI,GAAGG,YAAY;IACtD,MAAMG,cAAcD,KAAK,GAAG,CAACL,KAAK,GAAG,GAAGG,YAAY;IAEpD,MAAMI,aAAa;QACjB,MAAMH;QACN,KAAKE;QACL,OAAOD,KAAK,GAAG,CACbL,KAAK,IAAI,GAAGI,eAAeJ,KAAK,KAAK,GAAGG,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQC,KAAK,GAAG,CACdL,KAAK,GAAG,GAAGM,cAAcN,KAAK,MAAM,GAAGG,YACvCF,WAAW,MAAM,GAAGK;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeN,SACjB,OAAOK;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAAcN,KAAK,IAAI,CAACH,UAAUM;IACxC,MAAMI,WAAWP,KAAK,KAAK,CAACE,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYR,KAAK,KAAK,CAACE,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUT,KAAK,KAAK,CAACI,UAAUG,WAAW;IAChD,MAAMG,SAASV,KAAK,KAAK,CAACK,UAAUG,YAAY;IAGhD,MAAMG,OAAOX,KAAK,GAAG,CAACS,SAAS;IAC/B,MAAMG,MAAMZ,KAAK,GAAG,CAACU,QAAQ;IAE7B,OAAO;QACLC;QACAC;QACA,OAAOZ,KAAK,GAAG,CAACO,UAAUX,WAAW,KAAK,GAAGe;QAC7C,QAAQX,KAAK,GAAG,CAACQ,WAAWZ,WAAW,MAAM,GAAGgB;IAClD;AACF;AAEO,eAAeC,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcxB,iBAAiBsB,UAAUD,QAAQ,QAAQ;IAE/D,MAAMI,gBAAgB,MAAMC,WAC1BL,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMG,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEF;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaG,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGH,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/search-area.mjs","sources":["../../../../../src/ai-model/workflows/grounding/search-area.ts"],"sourcesContent":["import type { Rect, Size, UIContext } from '@/types';\nimport { cropByRect, scaleImage } from '@midscene/shared/img';\nimport type { PixelLocateResult } from '../../shared/model-locate-result';\nimport type { SearchAreaConfig } from './types';\n\n/** Merge target and reference regions for DeepLocate; point-only results cover one pixel. */\nexport function mergeSearchAreaResults(\n  target: PixelLocateResult,\n  references: PixelLocateResult[] = [],\n): Rect {\n  const bounds = [target, ...references].map(({ rect, center }) => {\n    const left = rect?.left ?? Math.round(center[0]);\n    const top = rect?.top ?? Math.round(center[1]);\n    return {\n      left,\n      top,\n      right: left + (rect?.width ?? 1),\n      bottom: top + (rect?.height ?? 1),\n    };\n  });\n  const left = Math.min(...bounds.map((bound) => bound.left));\n  const top = Math.min(...bounds.map((bound) => bound.top));\n  const right = Math.max(...bounds.map((bound) => bound.right));\n  const bottom = Math.max(...bounds.map((bound) => bound.bottom));\n  return { left, top, width: right - left, height: bottom - top };\n}\n\n/**\n * Expand the search region toward a minimum area of 160,000 square pixels.\n *\n * Step 1: Extend 100px on each side (top, right, bottom, left)\n * - If the element is near a boundary, expansion on that side will be limited\n * - No compensation is made for boundary limitations (this is intentional)\n *\n * Step 2: Scale proportionally toward the minimum area.\n * - Scale up proportionally from the center if needed\n * - Clamping to screen boundaries may leave the final area below this target.\n */\nexport function expandSearchArea(rect: Rect, screenSize: Size): Rect {\n  const minArea = 400 * 400;\n  const expandSize = 100;\n\n  // Step 1: Extend each side by expandSize (100px), clamped to screen boundaries\n  // Note: If element is near boundary, actual expansion may be less than 100px on that side\n  const expandedLeft = Math.max(rect.left - expandSize, 0);\n  const expandedTop = Math.max(rect.top - expandSize, 0);\n\n  const expandRect = {\n    left: expandedLeft,\n    top: expandedTop,\n    width: Math.min(\n      rect.left - expandedLeft + rect.width + expandSize,\n      screenSize.width - expandedLeft,\n    ),\n    height: Math.min(\n      rect.top - expandedTop + rect.height + expandSize,\n      screenSize.height - expandedTop,\n    ),\n  };\n\n  // Step 2: Check if area is already >= 400x400\n  const currentArea = expandRect.width * expandRect.height;\n\n  if (currentArea >= minArea) {\n    return expandRect;\n  }\n\n  // Step 2: Scale up from center to reach minimum 400x400 area\n  const centerX = expandRect.left + expandRect.width / 2;\n  const centerY = expandRect.top + expandRect.height / 2;\n\n  // Calculate scale factor needed to reach minimum area\n  const scaleFactor = Math.sqrt(minArea / currentArea);\n  const newWidth = Math.round(expandRect.width * scaleFactor);\n  const newHeight = Math.round(expandRect.height * scaleFactor);\n\n  // Calculate new position based on center point\n  const newLeft = Math.round(centerX - newWidth / 2);\n  const newTop = Math.round(centerY - newHeight / 2);\n\n  // Clamp to screen boundaries\n  const left = Math.max(newLeft, 0);\n  const top = Math.max(newTop, 0);\n\n  return {\n    left,\n    top,\n    width: Math.min(newWidth, screenSize.width - left),\n    height: Math.min(newHeight, screenSize.height - top),\n  };\n}\n\nexport async function buildSearchAreaConfig(options: {\n  context: UIContext;\n  baseRect: Rect;\n}): Promise<SearchAreaConfig> {\n  const { context, baseRect } = options;\n  const scaleRatio = 2;\n  const sectionRect = expandSearchArea(baseRect, context.shotSize);\n\n  const croppedResult = await cropByRect(\n    context.screenshot.base64,\n    sectionRect,\n  );\n\n  const scaledResult = await scaleImage(croppedResult.imageBase64, scaleRatio);\n  return {\n    sourceRect: sectionRect,\n    image: {\n      imageBase64: scaledResult.imageBase64,\n      width: scaledResult.width,\n      height: scaledResult.height,\n    },\n    mapping: {\n      offset: {\n        x: sectionRect.left,\n        y: sectionRect.top,\n      },\n      scale: scaleRatio,\n    },\n  };\n}\n"],"names":["mergeSearchAreaResults","target","references","bounds","rect","center","left","Math","top","bound","right","bottom","expandSearchArea","screenSize","minArea","expandSize","expandedLeft","expandedTop","expandRect","currentArea","centerX","centerY","scaleFactor","newWidth","newHeight","newLeft","newTop","buildSearchAreaConfig","options","context","baseRect","scaleRatio","sectionRect","croppedResult","cropByRect","scaledResult","scaleImage"],"mappings":";AAMO,SAASA,uBACdC,MAAyB,EACzBC,aAAkC,EAAE;IAEpC,MAAMC,SAAS;QAACF;WAAWC;KAAW,CAAC,GAAG,CAAC,CAAC,EAAEE,IAAI,EAAEC,MAAM,EAAE;QAC1D,MAAMC,OAAOF,MAAM,QAAQG,KAAK,KAAK,CAACF,MAAM,CAAC,EAAE;QAC/C,MAAMG,MAAMJ,MAAM,OAAOG,KAAK,KAAK,CAACF,MAAM,CAAC,EAAE;QAC7C,OAAO;YACLC;YACAE;YACA,OAAOF,OAAQF,CAAAA,MAAM,SAAS;YAC9B,QAAQI,MAAOJ,CAAAA,MAAM,UAAU;QACjC;IACF;IACA,MAAME,OAAOC,KAAK,GAAG,IAAIJ,OAAO,GAAG,CAAC,CAACM,QAAUA,MAAM,IAAI;IACzD,MAAMD,MAAMD,KAAK,GAAG,IAAIJ,OAAO,GAAG,CAAC,CAACM,QAAUA,MAAM,GAAG;IACvD,MAAMC,QAAQH,KAAK,GAAG,IAAIJ,OAAO,GAAG,CAAC,CAACM,QAAUA,MAAM,KAAK;IAC3D,MAAME,SAASJ,KAAK,GAAG,IAAIJ,OAAO,GAAG,CAAC,CAACM,QAAUA,MAAM,MAAM;IAC7D,OAAO;QAAEH;QAAME;QAAK,OAAOE,QAAQJ;QAAM,QAAQK,SAASH;IAAI;AAChE;AAaO,SAASI,iBAAiBR,IAAU,EAAES,UAAgB;IAC3D,MAAMC,UAAU;IAChB,MAAMC,aAAa;IAInB,MAAMC,eAAeT,KAAK,GAAG,CAACH,KAAK,IAAI,GAAGW,YAAY;IACtD,MAAME,cAAcV,KAAK,GAAG,CAACH,KAAK,GAAG,GAAGW,YAAY;IAEpD,MAAMG,aAAa;QACjB,MAAMF;QACN,KAAKC;QACL,OAAOV,KAAK,GAAG,CACbH,KAAK,IAAI,GAAGY,eAAeZ,KAAK,KAAK,GAAGW,YACxCF,WAAW,KAAK,GAAGG;QAErB,QAAQT,KAAK,GAAG,CACdH,KAAK,GAAG,GAAGa,cAAcb,KAAK,MAAM,GAAGW,YACvCF,WAAW,MAAM,GAAGI;IAExB;IAGA,MAAME,cAAcD,WAAW,KAAK,GAAGA,WAAW,MAAM;IAExD,IAAIC,eAAeL,SACjB,OAAOI;IAIT,MAAME,UAAUF,WAAW,IAAI,GAAGA,WAAW,KAAK,GAAG;IACrD,MAAMG,UAAUH,WAAW,GAAG,GAAGA,WAAW,MAAM,GAAG;IAGrD,MAAMI,cAAcf,KAAK,IAAI,CAACO,UAAUK;IACxC,MAAMI,WAAWhB,KAAK,KAAK,CAACW,WAAW,KAAK,GAAGI;IAC/C,MAAME,YAAYjB,KAAK,KAAK,CAACW,WAAW,MAAM,GAAGI;IAGjD,MAAMG,UAAUlB,KAAK,KAAK,CAACa,UAAUG,WAAW;IAChD,MAAMG,SAASnB,KAAK,KAAK,CAACc,UAAUG,YAAY;IAGhD,MAAMlB,OAAOC,KAAK,GAAG,CAACkB,SAAS;IAC/B,MAAMjB,MAAMD,KAAK,GAAG,CAACmB,QAAQ;IAE7B,OAAO;QACLpB;QACAE;QACA,OAAOD,KAAK,GAAG,CAACgB,UAAUV,WAAW,KAAK,GAAGP;QAC7C,QAAQC,KAAK,GAAG,CAACiB,WAAWX,WAAW,MAAM,GAAGL;IAClD;AACF;AAEO,eAAemB,sBAAsBC,OAG3C;IACC,MAAM,EAAEC,OAAO,EAAEC,QAAQ,EAAE,GAAGF;IAC9B,MAAMG,aAAa;IACnB,MAAMC,cAAcpB,iBAAiBkB,UAAUD,QAAQ,QAAQ;IAE/D,MAAMI,gBAAgB,MAAMC,WAC1BL,QAAQ,UAAU,CAAC,MAAM,EACzBG;IAGF,MAAMG,eAAe,MAAMC,WAAWH,cAAc,WAAW,EAAEF;IACjE,OAAO;QACL,YAAYC;QACZ,OAAO;YACL,aAAaG,aAAa,WAAW;YACrC,OAAOA,aAAa,KAAK;YACzB,QAAQA,aAAa,MAAM;QAC7B;QACA,SAAS;YACP,QAAQ;gBACN,GAAGH,YAAY,IAAI;gBACnB,GAAGA,YAAY,GAAG;YACpB;YACA,OAAOD;QACT;IACF;AACF"}

package/dist/es/ai-model/workflows/grounding/section-locate.mjs

@@ -4,8 +4,7 @@ import { userPromptToString } from "../../../common.mjs";
 import { buildSearchAreaLocateSystemPrompt } from "../../prompt/locate/index.mjs";
 import { AIResponseParseError, callAI } from "../../service-caller/index.mjs";
 import { callAiAndParseWithRetry, withSemanticRetryFeedback } from "../../service-caller/semantic-retry.mjs";
-import { mergePixelBboxesToRect } from "./locate-result-rect.mjs";
-import { buildSearchAreaConfig, expandSearchArea } from "./search-area.mjs";
+import { buildSearchAreaConfig, expandSearchArea, mergeSearchAreaResults } from "./search-area.mjs";
 import { formatLocateModelContext, prepareLocateModelInput } from "./utils.mjs";
 const debugSection = getDebug('ai:grounding:section');
 async function AiLocateSection(options) {
@@ -54,12 +53,9 @@ async function AiLocateSection(options) {
                         preparedSize: preparedImage.preparedSize,
                         contentSize: preparedImage.contentSize
                     };
-                    const target = resultCodec.toPixelBbox(parsedLocateResult.target, locateResultContext);
-                    const references = parsedLocateResult.references?.map((reference)=>resultCodec.toPixelBbox(reference, locateResultContext));
-                    const mergedRect = mergePixelBboxesToRect([
-                        target,
-                        ...references ?? []
-                    ]);
+                    const target = resultCodec.toPixelResult(parsedLocateResult.target, locateResultContext);
+                    const references = parsedLocateResult.references?.map((reference)=>resultCodec.toPixelResult(reference, locateResultContext));
+                    const mergedRect = mergeSearchAreaResults(target, references);
                     debugSection('mergedRect %j', mergedRect);
                     return {
                         result: response,

package/dist/es/ai-model/workflows/grounding/section-locate.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport { mergePixelBboxesToRect } from './locate-result-rect';\nimport { buildSearchAreaConfig, expandSearchArea } from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelBbox(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelBbox(reference, locateResultContext),\n          );\n          const mergedRect = mergePixelBboxesToRect([\n            target,\n            ...(references ?? []),\n          ]);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergePixelBboxesToRect","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;;AAiBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,WAAW,CACpCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,WAAW,CAACqB,WAAWH;oBAErC,MAAMI,aAAaC,uBAAuB;wBACxCJ;2BACIC,cAAc,EAAE;qBACrB;oBACD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/grounding/section-locate.mjs","sources":["../../../../../src/ai-model/workflows/grounding/section-locate.ts"],"sourcesContent":["import type { AIUsageInfo, Rect, UIContext } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TUserPrompt } from '../../../common';\nimport { userPromptToString } from '../../../common';\nimport type { ModelRuntime } from '../../models';\nimport { buildSearchAreaLocateSystemPrompt } from '../../prompt/locate';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport {\n  buildSearchAreaConfig,\n  expandSearchArea,\n  mergeSearchAreaResults,\n} from './search-area';\nimport type { SearchAreaConfig } from './types';\nimport { formatLocateModelContext, prepareLocateModelInput } from './utils';\n\nconst debugSection = getDebug('ai:grounding:section');\n\ntype SectionLocateRawResponse = Awaited<ReturnType<typeof callAI>>;\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelRuntime: ModelRuntime;\n  abortSignal?: AbortSignal;\n}): Promise<{\n  searchAreaConfig?: SearchAreaConfig;\n  error?: string;\n  rawResponse: string;\n  rawChoiceMessage?: unknown;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription } = options;\n  const modelRuntime = options.modelRuntime;\n  const { adapter } = modelRuntime;\n  assert(\n    adapter.locate.kind === 'standard',\n    'section locate requires a standard locate adapter',\n  );\n  const searchArea = adapter.locate.searchArea;\n  assert(searchArea, 'section locate requires a search area operation');\n  const { protocol: searchAreaProtocol, resultCodec } = searchArea;\n  const screenshotBase64 = context.screenshot.base64;\n\n  const systemPrompt = buildSearchAreaLocateSystemPrompt({\n    systemPromptIntroduction: searchAreaProtocol.systemPromptIntroduction,\n    responseInstructions: searchAreaProtocol.buildResponseInstructions(\n      resultCodec.promptSpec,\n    ),\n  });\n\n  const userInstructionPrompt = searchAreaProtocol.buildUserPrompt(\n    userPromptToString(sectionDescription),\n  );\n\n  const { messages, preparedImage } = await prepareLocateModelInput({\n    systemPrompt,\n    userPrompt: userInstructionPrompt,\n    locateImage: {\n      imageBase64: screenshotBase64,\n      width: context.shotSize.width,\n      height: context.shotSize.height,\n    },\n    imagePreprocess: adapter.imagePreprocess,\n    targetDescription: sectionDescription,\n    userMessageContentOrder: adapter.locate.userMessageContentOrder,\n  });\n\n  let parsedResult:\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect?: undefined;\n      }\n    | {\n        result: SectionLocateRawResponse;\n        sectionError?: string;\n        mergedRect: Rect;\n      };\n\n  try {\n    parsedResult = await callAiAndParseWithRetry({\n      callAi: (retryAttempt, previousParseError) =>\n        callAI(\n          withSemanticRetryFeedback(messages, previousParseError),\n          modelRuntime,\n          {\n            abortSignal: options.abortSignal,\n            expectedJsonObjectResponse:\n              searchAreaProtocol.expectedJsonObjectResponse,\n            semanticRetryAttempt: retryAttempt,\n          },\n        ),\n      parseResponse: (response) => {\n        const parsedLocateResult = searchAreaProtocol.parseRawResponse(\n          response.content,\n          resultCodec.promptSpec,\n        );\n        const sectionError = parsedLocateResult.error;\n        if (parsedLocateResult.kind === 'not-found') {\n          return { result: response, sectionError };\n        }\n\n        try {\n          const locateResultContext = {\n            preparedSize: preparedImage.preparedSize,\n            contentSize: preparedImage.contentSize,\n          };\n          const target = resultCodec.toPixelResult(\n            parsedLocateResult.target,\n            locateResultContext,\n          );\n          const references = parsedLocateResult.references?.map((reference) =>\n            resultCodec.toPixelResult(reference, locateResultContext),\n          );\n          const mergedRect = mergeSearchAreaResults(target, references);\n          debugSection('mergedRect %j', mergedRect);\n          return {\n            result: response,\n            sectionError,\n            mergedRect,\n          };\n        } catch (error) {\n          const errorMessage =\n            error instanceof Error ? error.message : String(error);\n          const message = [\n            sectionError && `error in section locate result: ${sectionError}`,\n            `coordinate parsing error: ${errorMessage}`,\n            formatLocateModelContext(modelRuntime),\n          ]\n            .filter(Boolean)\n            .join('\\n');\n          throw new Error(message, { cause: error });\n        }\n      },\n      toParseError: (error, response) => {\n        const parseErrorMessage =\n          error instanceof Error\n            ? `Failed to parse section locate result: ${error.message}`\n            : 'unknown error in section locate';\n        return new AIResponseParseError(\n          parseErrorMessage,\n          response.content,\n          response.usage,\n          response.rawChoiceMessage,\n          response.reasoning_content,\n        );\n      },\n      parseRetryTimes: modelRuntime.config.retryCount,\n      parseRetryInterval: modelRuntime.config.retryInterval,\n      abortSignal: options.abortSignal,\n      onParseRetry: (error) => {\n        debugSection(\n          'retrying section locate after coordinate parsing failed: %s',\n          error instanceof Error ? error.message : String(error),\n        );\n      },\n    });\n  } catch (callError) {\n    if (callError instanceof AIResponseParseError) {\n      return {\n        searchAreaConfig: undefined,\n        error: callError.message,\n        rawResponse: callError.rawResponse,\n        rawChoiceMessage: callError.rawChoiceMessage,\n        usage: callError.usage,\n      };\n    }\n\n    const errorMessage =\n      callError instanceof Error ? callError.message : String(callError);\n    return {\n      searchAreaConfig: undefined,\n      error: `AI call error: ${errorMessage}`,\n      rawResponse: errorMessage,\n    };\n  }\n\n  const { result, sectionError, mergedRect } = parsedResult;\n  if (!mergedRect) {\n    return {\n      searchAreaConfig: undefined,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n\n  try {\n    const expandedRect = expandSearchArea(mergedRect, context.shotSize);\n    const originalWidth = expandedRect.width;\n    const originalHeight = expandedRect.height;\n    debugSection('expanded sectionRect %j', expandedRect);\n\n    const searchAreaConfig = await buildSearchAreaConfig({\n      context,\n      baseRect: mergedRect,\n    });\n\n    debugSection(\n      'scaled section image from %dx%d to %dx%d (scale=%d)',\n      originalWidth,\n      originalHeight,\n      searchAreaConfig.image.width,\n      searchAreaConfig.image.height,\n      searchAreaConfig.mapping.scale,\n    );\n    return {\n      searchAreaConfig,\n      error: sectionError,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  } catch (error) {\n    const parseErrorMessage =\n      error instanceof Error\n        ? `Failed to parse section locate result: ${error.message}`\n        : 'unknown error in section locate';\n    const errorMessage = sectionError\n      ? `${sectionError} (${parseErrorMessage})`\n      : parseErrorMessage;\n    return {\n      searchAreaConfig: undefined,\n      error: errorMessage,\n      rawResponse: result.content,\n      rawChoiceMessage: result.rawChoiceMessage,\n      usage: result.usage,\n    };\n  }\n}\n"],"names":["debugSection","getDebug","AiLocateSection","options","context","sectionDescription","modelRuntime","adapter","assert","searchArea","searchAreaProtocol","resultCodec","screenshotBase64","systemPrompt","buildSearchAreaLocateSystemPrompt","userInstructionPrompt","userPromptToString","messages","preparedImage","prepareLocateModelInput","parsedResult","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","parsedLocateResult","sectionError","locateResultContext","target","references","reference","mergedRect","mergeSearchAreaResults","error","errorMessage","Error","String","message","formatLocateModelContext","Boolean","parseErrorMessage","AIResponseParseError","callError","undefined","result","expandedRect","expandSearchArea","originalWidth","originalHeight","searchAreaConfig","buildSearchAreaConfig"],"mappings":";;;;;;;;AAoBA,MAAMA,eAAeC,SAAS;AAIvB,eAAeC,gBAAgBC,OAKrC;IAOC,MAAM,EAAEC,OAAO,EAAEC,kBAAkB,EAAE,GAAGF;IACxC,MAAMG,eAAeH,QAAQ,YAAY;IACzC,MAAM,EAAEI,OAAO,EAAE,GAAGD;IACpBE,OACED,AAAwB,eAAxBA,QAAQ,MAAM,CAAC,IAAI,EACnB;IAEF,MAAME,aAAaF,QAAQ,MAAM,CAAC,UAAU;IAC5CC,OAAOC,YAAY;IACnB,MAAM,EAAE,UAAUC,kBAAkB,EAAEC,WAAW,EAAE,GAAGF;IACtD,MAAMG,mBAAmBR,QAAQ,UAAU,CAAC,MAAM;IAElD,MAAMS,eAAeC,kCAAkC;QACrD,0BAA0BJ,mBAAmB,wBAAwB;QACrE,sBAAsBA,mBAAmB,yBAAyB,CAChEC,YAAY,UAAU;IAE1B;IAEA,MAAMI,wBAAwBL,mBAAmB,eAAe,CAC9DM,mBAAmBX;IAGrB,MAAM,EAAEY,QAAQ,EAAEC,aAAa,EAAE,GAAG,MAAMC,wBAAwB;QAChEN;QACA,YAAYE;QACZ,aAAa;YACX,aAAaH;YACb,OAAOR,QAAQ,QAAQ,CAAC,KAAK;YAC7B,QAAQA,QAAQ,QAAQ,CAAC,MAAM;QACjC;QACA,iBAAiBG,QAAQ,eAAe;QACxC,mBAAmBF;QACnB,yBAAyBE,QAAQ,MAAM,CAAC,uBAAuB;IACjE;IAEA,IAAIa;IAYJ,IAAI;QACFA,eAAe,MAAMC,wBAAwB;YAC3C,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0BR,UAAUM,qBACpCjB,cACA;oBACE,aAAaH,QAAQ,WAAW;oBAChC,4BACEO,mBAAmB,0BAA0B;oBAC/C,sBAAsBY;gBACxB;YAEJ,eAAe,CAACI;gBACd,MAAMC,qBAAqBjB,mBAAmB,gBAAgB,CAC5DgB,SAAS,OAAO,EAChBf,YAAY,UAAU;gBAExB,MAAMiB,eAAeD,mBAAmB,KAAK;gBAC7C,IAAIA,AAA4B,gBAA5BA,mBAAmB,IAAI,EACzB,OAAO;oBAAE,QAAQD;oBAAUE;gBAAa;gBAG1C,IAAI;oBACF,MAAMC,sBAAsB;wBAC1B,cAAcX,cAAc,YAAY;wBACxC,aAAaA,cAAc,WAAW;oBACxC;oBACA,MAAMY,SAASnB,YAAY,aAAa,CACtCgB,mBAAmB,MAAM,EACzBE;oBAEF,MAAME,aAAaJ,mBAAmB,UAAU,EAAE,IAAI,CAACK,YACrDrB,YAAY,aAAa,CAACqB,WAAWH;oBAEvC,MAAMI,aAAaC,uBAAuBJ,QAAQC;oBAClD/B,aAAa,iBAAiBiC;oBAC9B,OAAO;wBACL,QAAQP;wBACRE;wBACAK;oBACF;gBACF,EAAE,OAAOE,OAAO;oBACd,MAAMC,eACJD,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;oBAClD,MAAMI,UAAU;wBACdX,gBAAgB,CAAC,gCAAgC,EAAEA,cAAc;wBACjE,CAAC,0BAA0B,EAAEQ,cAAc;wBAC3CI,yBAAyBlC;qBAC1B,CACE,MAAM,CAACmC,SACP,IAAI,CAAC;oBACR,MAAM,IAAIJ,MAAME,SAAS;wBAAE,OAAOJ;oBAAM;gBAC1C;YACF;YACA,cAAc,CAACA,OAAOT;gBACpB,MAAMgB,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;gBACN,OAAO,IAAIQ,qBACTD,mBACAhB,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;YAE9B;YACA,iBAAiBpB,aAAa,MAAM,CAAC,UAAU;YAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;YACrD,aAAaH,QAAQ,WAAW;YAChC,cAAc,CAACgC;gBACbnC,aACE,+DACAmC,iBAAiBE,QAAQF,MAAM,OAAO,GAAGG,OAAOH;YAEpD;QACF;IACF,EAAE,OAAOS,WAAW;QAClB,IAAIA,qBAAqBD,sBACvB,OAAO;YACL,kBAAkBE;YAClB,OAAOD,UAAU,OAAO;YACxB,aAAaA,UAAU,WAAW;YAClC,kBAAkBA,UAAU,gBAAgB;YAC5C,OAAOA,UAAU,KAAK;QACxB;QAGF,MAAMR,eACJQ,qBAAqBP,QAAQO,UAAU,OAAO,GAAGN,OAAOM;QAC1D,OAAO;YACL,kBAAkBC;YAClB,OAAO,CAAC,eAAe,EAAET,cAAc;YACvC,aAAaA;QACf;IACF;IAEA,MAAM,EAAEU,MAAM,EAAElB,YAAY,EAAEK,UAAU,EAAE,GAAGb;IAC7C,IAAI,CAACa,YACH,OAAO;QACL,kBAAkBY;QAClB,OAAOjB;QACP,aAAakB,OAAO,OAAO;QAC3B,kBAAkBA,OAAO,gBAAgB;QACzC,OAAOA,OAAO,KAAK;IACrB;IAGF,IAAI;QACF,MAAMC,eAAeC,iBAAiBf,YAAY7B,QAAQ,QAAQ;QAClE,MAAM6C,gBAAgBF,aAAa,KAAK;QACxC,MAAMG,iBAAiBH,aAAa,MAAM;QAC1C/C,aAAa,2BAA2B+C;QAExC,MAAMI,mBAAmB,MAAMC,sBAAsB;YACnDhD;YACA,UAAU6B;QACZ;QAEAjC,aACE,uDACAiD,eACAC,gBACAC,iBAAiB,KAAK,CAAC,KAAK,EAC5BA,iBAAiB,KAAK,CAAC,MAAM,EAC7BA,iBAAiB,OAAO,CAAC,KAAK;QAEhC,OAAO;YACLA;YACA,OAAOvB;YACP,aAAakB,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF,EAAE,OAAOX,OAAO;QACd,MAAMO,oBACJP,iBAAiBE,QACb,CAAC,uCAAuC,EAAEF,MAAM,OAAO,EAAE,GACzD;QACN,MAAMC,eAAeR,eACjB,GAAGA,aAAa,EAAE,EAAEc,kBAAkB,CAAC,CAAC,GACxCA;QACJ,OAAO;YACL,kBAAkBG;YAClB,OAAOT;YACP,aAAaU,OAAO,OAAO;YAC3B,kBAAkBA,OAAO,gBAAgB;YACzC,OAAOA,OAAO,KAAK;QACrB;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/locate-normalization.mjs

@@ -29,9 +29,10 @@ function normalizePlanningActionLocateFields(actions, { actionSpace, includeLoca
                 ] : []
             ]);
             const locateParamWithoutRawCoordinates = Object.fromEntries(Object.entries(locateParameter).filter(([key])=>!rawCoordinateKeys.has(key)));
+            const result = locateResultCodec.toPixelResult(rawLocateValue, locateResultContext);
             action.param[field] = {
                 ...locateParamWithoutRawCoordinates,
-                locatedPixelBbox: locateResultCodec.toPixelBbox(rawLocateValue, locateResultContext)
+                locatedPixelResult: result
             };
         });
     });

package/dist/es/ai-model/workflows/planning/locate-normalization.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/locate-normalization.mjs","sources":["../../../../../src/ai-model/workflows/planning/locate-normalization.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result/types';\n\nconst debug = getDebug('planning');\n\nexport function normalizePlanningActionLocateFields(\n  actions: PlanningAction[],\n  {\n    actionSpace,\n    includeLocateInPlanning,\n    locateResultCodec,\n    locateResultContext,\n    acceptBbox2dAlias = false,\n    parseRawLocateParameter,\n  }: {\n    actionSpace: DeviceAction[];\n    includeLocateInPlanning: boolean;\n    locateResultCodec?: LocateResultCodec;\n    locateResultContext: LocateResultContext;\n    acceptBbox2dAlias?: boolean;\n    parseRawLocateParameter: (value: unknown) => ParsedPlanningLocateParameter;\n  },\n): void {\n  actions.forEach((action) => {\n    const actionInActionSpace = actionSpace.find(\n      (actionInSpace) => actionInSpace.name === action.type,\n    );\n    if (!actionInActionSpace) {\n      debug('skip locate normalization for action outside actionSpace', action);\n      return;\n    }\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = findAllMidsceneLocatorField(\n      actionInActionSpace.paramSchema,\n    );\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const rawLocateParameter = action.param?.[field];\n      if (!rawLocateParameter) {\n        return;\n      }\n\n      const locateParameter = parseRawLocateParameter(rawLocateParameter);\n\n      if (!includeLocateInPlanning) {\n        // In prompt-only planning mode, ignore any accidental coordinates from the model.\n        action.param[field] = { prompt: locateParameter.prompt };\n        return;\n      }\n\n      assert(\n        locateResultCodec,\n        'planning locate normalization requires a locate result codec',\n      );\n\n      const resultKey = locateResultCodec.promptSpec.resultKey;\n      const rawLocateValue =\n        locateParameter[resultKey] !== undefined\n          ? locateParameter[resultKey]\n          : acceptBbox2dAlias && resultKey === 'bbox'\n            ? locateParameter.bbox_2d\n            : undefined;\n\n      // The raw result field is replaced by locatedPixelBbox, so it should not\n      // remain in the normalized locate parameter.\n      const rawCoordinateKeys = new Set([\n        resultKey,\n        ...(acceptBbox2dAlias && resultKey === 'bbox' ? ['bbox_2d'] : []),\n      ]);\n\n      const locateParamWithoutRawCoordinates = Object.fromEntries(\n        Object.entries(locateParameter).filter(\n          ([key]) => !rawCoordinateKeys.has(key),\n        ),\n      );\n\n      action.param[field] = {\n        ...locateParamWithoutRawCoordinates,\n        locatedPixelBbox: locateResultCodec.toPixelBbox(\n          rawLocateValue,\n          locateResultContext,\n        ),\n      };\n    });\n  });\n}\n"],"names":["debug","getDebug","normalizePlanningActionLocateFields","actions","actionSpace","includeLocateInPlanning","locateResultCodec","locateResultContext","acceptBbox2dAlias","parseRawLocateParameter","action","actionInActionSpace","actionInSpace","locateFields","findAllMidsceneLocatorField","field","rawLocateParameter","locateParameter","assert","resultKey","rawLocateValue","undefined","rawCoordinateKeys","Set","locateParamWithoutRawCoordinates","Object","key"],"mappings":";;;AAWA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oCACdC,OAAyB,EACzB,EACEC,WAAW,EACXC,uBAAuB,EACvBC,iBAAiB,EACjBC,mBAAmB,EACnBC,oBAAoB,KAAK,EACzBC,uBAAuB,EAQxB;IAEDN,QAAQ,OAAO,CAAC,CAACO;QACf,MAAMC,sBAAsBP,YAAY,IAAI,CAC1C,CAACQ,gBAAkBA,cAAc,IAAI,KAAKF,OAAO,IAAI;QAEvD,IAAI,CAACC,qBAAqB,YACxBX,MAAM,4DAA4DU;QAIpEV,MAAM,+BAA+BW;QACrC,MAAME,eAAeC,4BACnBH,oBAAoB,WAAW;QAGjCX,MAAM,gBAAgBa;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,qBAAqBN,OAAO,KAAK,EAAE,CAACK,MAAM;YAChD,IAAI,CAACC,oBACH;YAGF,MAAMC,kBAAkBR,wBAAwBO;YAEhD,IAAI,CAACX,yBAAyB;gBAE5BK,OAAO,KAAK,CAACK,MAAM,GAAG;oBAAE,QAAQE,gBAAgB,MAAM;gBAAC;gBACvD;YACF;YAEAC,OACEZ,mBACA;YAGF,MAAMa,YAAYb,kBAAkB,UAAU,CAAC,SAAS;YACxD,MAAMc,iBACJH,AAA+BI,WAA/BJ,eAAe,CAACE,UAAU,GACtBF,eAAe,CAACE,UAAU,GAC1BX,qBAAqBW,AAAc,WAAdA,YACnBF,gBAAgB,OAAO,GACvBI;YAIR,MAAMC,oBAAoB,IAAIC,IAAI;gBAChCJ;mBACIX,qBAAqBW,AAAc,WAAdA,YAAuB;oBAAC;iBAAU,GAAG,EAAE;aACjE;YAED,MAAMK,mCAAmCC,OAAO,WAAW,CACzDA,OAAO,OAAO,CAACR,iBAAiB,MAAM,CACpC,CAAC,CAACS,IAAI,GAAK,CAACJ,kBAAkB,GAAG,CAACI;YAItChB,OAAO,KAAK,CAACK,MAAM,GAAG;gBACpB,GAAGS,gCAAgC;gBACnC,kBAAkBlB,kBAAkB,WAAW,CAC7Cc,gBACAb;YAEJ;QACF;IACF;AACF"}
+{"version":3,"file":"ai-model/workflows/planning/locate-normalization.mjs","sources":["../../../../../src/ai-model/workflows/planning/locate-normalization.ts"],"sourcesContent":["import { findAllMidsceneLocatorField } from '@/common';\nimport type { DeviceAction } from '@/device';\nimport type { PlanningAction } from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ParsedPlanningLocateParameter } from '../../model-adapter/planning-protocol';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result/types';\n\nconst debug = getDebug('planning');\n\nexport function normalizePlanningActionLocateFields(\n  actions: PlanningAction[],\n  {\n    actionSpace,\n    includeLocateInPlanning,\n    locateResultCodec,\n    locateResultContext,\n    acceptBbox2dAlias = false,\n    parseRawLocateParameter,\n  }: {\n    actionSpace: DeviceAction[];\n    includeLocateInPlanning: boolean;\n    locateResultCodec?: LocateResultCodec;\n    locateResultContext: LocateResultContext;\n    acceptBbox2dAlias?: boolean;\n    parseRawLocateParameter: (value: unknown) => ParsedPlanningLocateParameter;\n  },\n): void {\n  actions.forEach((action) => {\n    const actionInActionSpace = actionSpace.find(\n      (actionInSpace) => actionInSpace.name === action.type,\n    );\n    if (!actionInActionSpace) {\n      debug('skip locate normalization for action outside actionSpace', action);\n      return;\n    }\n\n    debug('actionInActionSpace matched', actionInActionSpace);\n    const locateFields = findAllMidsceneLocatorField(\n      actionInActionSpace.paramSchema,\n    );\n\n    debug('locateFields', locateFields);\n\n    locateFields.forEach((field) => {\n      const rawLocateParameter = action.param?.[field];\n      if (!rawLocateParameter) {\n        return;\n      }\n\n      const locateParameter = parseRawLocateParameter(rawLocateParameter);\n\n      if (!includeLocateInPlanning) {\n        // In prompt-only planning mode, ignore any accidental coordinates from the model.\n        action.param[field] = { prompt: locateParameter.prompt };\n        return;\n      }\n\n      assert(\n        locateResultCodec,\n        'planning locate normalization requires a locate result codec',\n      );\n\n      const resultKey = locateResultCodec.promptSpec.resultKey;\n      const rawLocateValue =\n        locateParameter[resultKey] !== undefined\n          ? locateParameter[resultKey]\n          : acceptBbox2dAlias && resultKey === 'bbox'\n            ? locateParameter.bbox_2d\n            : undefined;\n\n      // The raw result field is replaced by locatedPixelResult, so it should not\n      // remain in the normalized locate parameter.\n      const rawCoordinateKeys = new Set([\n        resultKey,\n        ...(acceptBbox2dAlias && resultKey === 'bbox' ? ['bbox_2d'] : []),\n      ]);\n\n      const locateParamWithoutRawCoordinates = Object.fromEntries(\n        Object.entries(locateParameter).filter(\n          ([key]) => !rawCoordinateKeys.has(key),\n        ),\n      );\n\n      const result = locateResultCodec.toPixelResult(\n        rawLocateValue,\n        locateResultContext,\n      );\n      action.param[field] = {\n        ...locateParamWithoutRawCoordinates,\n        locatedPixelResult: result,\n      };\n    });\n  });\n}\n"],"names":["debug","getDebug","normalizePlanningActionLocateFields","actions","actionSpace","includeLocateInPlanning","locateResultCodec","locateResultContext","acceptBbox2dAlias","parseRawLocateParameter","action","actionInActionSpace","actionInSpace","locateFields","findAllMidsceneLocatorField","field","rawLocateParameter","locateParameter","assert","resultKey","rawLocateValue","undefined","rawCoordinateKeys","Set","locateParamWithoutRawCoordinates","Object","key","result"],"mappings":";;;AAWA,MAAMA,QAAQC,SAAS;AAEhB,SAASC,oCACdC,OAAyB,EACzB,EACEC,WAAW,EACXC,uBAAuB,EACvBC,iBAAiB,EACjBC,mBAAmB,EACnBC,oBAAoB,KAAK,EACzBC,uBAAuB,EAQxB;IAEDN,QAAQ,OAAO,CAAC,CAACO;QACf,MAAMC,sBAAsBP,YAAY,IAAI,CAC1C,CAACQ,gBAAkBA,cAAc,IAAI,KAAKF,OAAO,IAAI;QAEvD,IAAI,CAACC,qBAAqB,YACxBX,MAAM,4DAA4DU;QAIpEV,MAAM,+BAA+BW;QACrC,MAAME,eAAeC,4BACnBH,oBAAoB,WAAW;QAGjCX,MAAM,gBAAgBa;QAEtBA,aAAa,OAAO,CAAC,CAACE;YACpB,MAAMC,qBAAqBN,OAAO,KAAK,EAAE,CAACK,MAAM;YAChD,IAAI,CAACC,oBACH;YAGF,MAAMC,kBAAkBR,wBAAwBO;YAEhD,IAAI,CAACX,yBAAyB;gBAE5BK,OAAO,KAAK,CAACK,MAAM,GAAG;oBAAE,QAAQE,gBAAgB,MAAM;gBAAC;gBACvD;YACF;YAEAC,OACEZ,mBACA;YAGF,MAAMa,YAAYb,kBAAkB,UAAU,CAAC,SAAS;YACxD,MAAMc,iBACJH,AAA+BI,WAA/BJ,eAAe,CAACE,UAAU,GACtBF,eAAe,CAACE,UAAU,GAC1BX,qBAAqBW,AAAc,WAAdA,YACnBF,gBAAgB,OAAO,GACvBI;YAIR,MAAMC,oBAAoB,IAAIC,IAAI;gBAChCJ;mBACIX,qBAAqBW,AAAc,WAAdA,YAAuB;oBAAC;iBAAU,GAAG,EAAE;aACjE;YAED,MAAMK,mCAAmCC,OAAO,WAAW,CACzDA,OAAO,OAAO,CAACR,iBAAiB,MAAM,CACpC,CAAC,CAACS,IAAI,GAAK,CAACJ,kBAAkB,GAAG,CAACI;YAItC,MAAMC,SAASrB,kBAAkB,aAAa,CAC5Cc,gBACAb;YAEFG,OAAO,KAAK,CAACK,MAAM,GAAG;gBACpB,GAAGS,gCAAgC;gBACnC,oBAAoBG;YACtB;QACF;IACF;AACF"}

package/dist/es/ai-model/workflows/planning/standard-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n      });\n      // dumpActionParam keeps only the locator prompt, so runtime-only\n      // locatedPixelBbox fields added during normalization are not serialized.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","actions","normalizePlanningActionLocateFields","yamlFlow","buildYamlFlowFromPlans","parseError","errorMessage","Error","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IAAIS,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAMC,UAAUH,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DI,oCAAoCD,SAAS;gBAC3ChB;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;gBACzD,yBAAyBS,qBAAqB,uBAAuB;YACvE;YAGA,MAAMY,WAAWC,uBAAuBH,SAAShB;YACjD,OAAO;gBAAEY;gBAAUC;gBAAYG;gBAASE;YAAS;QACnD;QACA,cAAc,CAACE,YAAYR;YACzB,MAAMS,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACsB;YACb9B,MACE,mDACA8B,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAE9D;IACF;AACF;AAEO,eAAeK,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM9B,eAAe8B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGjC;IACpB,MAAM,EAAEkC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDvB,OACEyB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC9B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIyB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrDiC,QAAQ,QAAQ,CAAC,iBAAiB,GAClCf;IAGJ,MAAMoB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMxB,iBAAiBwB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMvB,aAAauB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAAC1B,mBACnC,MAAM,IAAIqB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMuC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BxB;QACAC;QACA+B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAI1B,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMqC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpC,uBAAuB;IAGpC,MAAMuD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACD1C,UAAU,EACVG,OAAO,EACPE,QAAQ,EACT,GAAG,MAAMxB,6BAA6B;QACrC,UAAUyD;QACVtD;QACA,aAAa8B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7B1B;QACA,qBAAqB;YACnB,cAAcqC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAnC;QACAC;IACF;IAEA,IAAIoD,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIrB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAG5C,UAAU;QACbG;QACAoC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIsB,iBAAiB;QACnB,IAAItB,WAAW,cAAc,EAAE,QAC7BgB,oBAAoB,aAAa,CAAChB,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDgB,oBAAoB,mBAAmB,CAAC6B;QAI5C,IAAI7C,WAAW,GAAG,EAChBgB,oBAAoB,gBAAgB,CAAChB,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBgB,oBAAoB,mBAAmB,CAAChB,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBgB,oBAAoB,YAAY,CAAChB,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7D0D,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}
+{"version":3,"file":"ai-model/workflows/planning/standard-planning.mjs","sources":["../../../../../src/ai-model/workflows/planning/standard-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  RawResponsePlanningAIResponse,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { buildYamlFlowFromPlans } from '../../../common';\nimport { prepareModelImage } from '../../model-adapter/image-preprocess';\nimport { buildStandardPlanningSystemPrompt } from '../../prompt/planning';\nimport { AIResponseParseError, callAI } from '../../service-caller/index';\nimport {\n  callAiAndParseWithRetry,\n  withSemanticRetryFeedback,\n} from '../../service-caller/semantic-retry';\nimport type {\n  LocateResultCodec,\n  LocateResultContext,\n} from '../../shared/model-locate-result';\nimport { planningModelFamilyRequiredForLocateMessage } from '../../shared/model-locate-result/errors';\nimport {\n  type PreparedUserPrompt,\n  preparedReferenceImagesToChatMessages,\n} from '../../shared/multimodal-prompt';\nimport { normalizePlanningActionLocateFields } from './locate-normalization';\nimport { parseStandardPlanningResponse } from './standard-planning-parser';\nimport type { PlanOptions } from './types';\n\nconst debug = getDebug('planning');\nconst warnLog = getDebug('planning', { console: true });\n\nconst noPreviousActionsText =\n  'No previous actions have been executed in this aiAct execution yet. If the instruction asks for actions, choose the first action to execute.';\ntype PlanningCallResponse = Awaited<ReturnType<typeof callAI>>;\n\ntype CallAndParsePlanningResponseOptions = {\n  messages: ChatCompletionMessageParam[];\n  modelRuntime: PlanOptions['modelRuntime'];\n  abortSignal?: AbortSignal;\n  includeLocateInPlanning: boolean;\n  actionSpace: PlanOptions['actionSpace'];\n  locateResultCodec?: LocateResultCodec;\n  locateResultContext: LocateResultContext;\n  includeThought: boolean;\n  includeLog: boolean;\n};\n\nasync function callAndParsePlanningResponse(\n  options: CallAndParsePlanningResponseOptions,\n): Promise<{\n  response: PlanningCallResponse;\n  planFromAI: RawResponsePlanningAIResponse;\n  actions: PlanningAction[];\n  yamlFlow: ReturnType<typeof buildYamlFlowFromPlans>;\n}> {\n  const {\n    messages,\n    modelRuntime,\n    abortSignal,\n    includeLocateInPlanning,\n    actionSpace,\n    locateResultCodec,\n    locateResultContext,\n    includeThought,\n    includeLog,\n  } = options;\n  assert(\n    modelRuntime.adapter.planning.kind === 'standard',\n    'callAndParsePlanningResponse requires a standard planning adapter',\n  );\n  const actionOutputProtocol =\n    modelRuntime.adapter.planning.protocol.actionOutputProtocol;\n\n  return callAiAndParseWithRetry({\n    callAi: (retryAttempt, previousParseError) =>\n      callAI(\n        withSemanticRetryFeedback(messages, previousParseError),\n        modelRuntime,\n        {\n          abortSignal,\n          requiresOriginalImageDetail: includeLocateInPlanning,\n          semanticRetryAttempt: retryAttempt,\n        },\n      ),\n    parseResponse: (response) => {\n      const planFromAI = parseStandardPlanningResponse(response.content, {\n        includeThought,\n        actionOutputProtocol,\n        actionSpace,\n        logSource: includeLog ? 'model' : 'action',\n      });\n      if (planFromAI.action && planFromAI.finalizeSuccess !== undefined) {\n        warnLog(\n          'Planning response included both an action and <complete>; ignoring <complete> output.',\n        );\n        planFromAI.finalizeMessage = undefined;\n        planFromAI.finalizeSuccess = undefined;\n      }\n\n      const actions = planFromAI.action ? [planFromAI.action] : [];\n      normalizePlanningActionLocateFields(actions, {\n        actionSpace,\n        includeLocateInPlanning,\n        locateResultCodec,\n        locateResultContext,\n        acceptBbox2dAlias: modelRuntime.adapter.acceptBbox2dAlias,\n        parseRawLocateParameter: actionOutputProtocol.parseRawLocateParameter,\n      });\n      // dumpActionParam keeps only the locator prompt, so runtime-only\n      // locatedPixelResult fields added during normalization are not serialized.\n      const yamlFlow = buildYamlFlowFromPlans(actions, actionSpace);\n      return { response, planFromAI, actions, yamlFlow };\n    },\n    toParseError: (parseError, response) => {\n      const errorMessage =\n        parseError instanceof Error ? parseError.message : String(parseError);\n      return new AIResponseParseError(\n        `XML parse error: ${errorMessage}`,\n        response.content,\n        response.usage,\n        response.rawChoiceMessage,\n        response.reasoning_content,\n      );\n    },\n    parseRetryTimes: modelRuntime.config.retryCount,\n    parseRetryInterval: modelRuntime.config.retryInterval,\n    abortSignal,\n    onParseRetry: (parseError) => {\n      debug(\n        'retrying plan after response parsing failed: %s',\n        parseError instanceof Error ? parseError.message : String(parseError),\n      );\n    },\n  });\n}\n\nexport async function standardPlan(\n  userInstruction: PreparedUserPrompt,\n  opts: PlanOptions,\n): Promise<PlanningAIResponse> {\n  const { context, conversationHistory } = opts;\n  const modelRuntime = opts.modelRuntime;\n  const { adapter } = modelRuntime;\n  const { shotSize } = context;\n  const screenshotBase64 = context.screenshot.base64;\n  assert(\n    adapter.planning.kind === 'standard',\n    'standardPlan requires a standard planning adapter',\n  );\n  const planningProtocol = adapter.planning.protocol;\n\n  if (opts.includeLocateInPlanning && !modelRuntime.config.modelFamily) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const locateResultCodec = modelRuntime.config.modelFamily\n    ? adapter.planning.locateResultCodec\n    : undefined;\n\n  // Only enable sub-goals when aiAct is in deep-thinking planning mode.\n  const includeSubGoals = opts.effort === 'deepThink';\n  const includeThought = opts.effort !== 'fast';\n  const includeLog = opts.effort !== 'fast';\n\n  if (opts.includeLocateInPlanning && !locateResultCodec) {\n    throw new Error(\n      planningModelFamilyRequiredForLocateMessage(modelRuntime.config.slot),\n    );\n  }\n\n  const systemPrompt = await buildStandardPlanningSystemPrompt({\n    actionSpace: opts.actionSpace,\n    includeThought,\n    includeLog,\n    includeSubGoals,\n    planningProtocol,\n    ...(opts.includeLocateInPlanning && locateResultCodec\n      ? {\n          includeLocateInPlanning: true,\n          locatePromptSpec: locateResultCodec.promptSpec,\n        }\n      : { includeLocateInPlanning: false }),\n  });\n\n  const preparedImage = await prepareModelImage({\n    imageBase64: screenshotBase64,\n    width: shotSize.width,\n    height: shotSize.height,\n    policy: adapter.imagePreprocess,\n  });\n  const imagePayload = preparedImage.imageBase64;\n\n  const actionContext = opts.actionContext ? `${opts.actionContext}\\n` : '';\n\n  const referenceImageMessages = preparedReferenceImagesToChatMessages(\n    userInstruction.referenceImages,\n  );\n  const instruction: ChatCompletionMessageParam[] = [\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${actionContext}<user_instruction>${userInstruction.text}</user_instruction>`,\n        },\n      ],\n    },\n    ...referenceImageMessages,\n  ];\n\n  let latestFeedbackMessage: ChatCompletionMessageParam;\n\n  // Build sub-goal status text to include in the message\n  // In planning deep-think mode: show full sub-goals with logs\n  // Otherwise: show historical execution logs\n  const executionProgressText = includeSubGoals\n    ? conversationHistory.subGoalsToText()\n    : conversationHistory.historicalLogsToText();\n  const executionProgressSection = executionProgressText\n    ? `\\n\\n${executionProgressText}`\n    : conversationHistory.pendingFeedbackMessage\n      ? ''\n      : `\\n\\n${noPreviousActionsText}`;\n\n  // Build memories text to include in the message\n  const memoriesText = conversationHistory.memoriesToText();\n  const memoriesSection = memoriesText ? `\\n\\n${memoriesText}` : '';\n\n  if (conversationHistory.pendingFeedbackMessage) {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `${conversationHistory.pendingFeedbackMessage}. The previous action has been executed, here is the latest screenshot. Please continue according to the instruction.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n\n    conversationHistory.resetPendingFeedbackMessageIfExists();\n  } else {\n    latestFeedbackMessage = {\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: `This is the current screenshot.${memoriesSection}${executionProgressSection}`,\n        },\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n      ],\n    };\n  }\n  conversationHistory.append(latestFeedbackMessage);\n\n  // Compress history if it exceeds the threshold to avoid context overflow\n  conversationHistory.compressHistory(50, 20);\n\n  const historyLog = conversationHistory.snapshot(opts.imagesIncludeCount);\n\n  const msgs: ChatCompletionMessageParam[] = [\n    { role: 'system', content: systemPrompt },\n    ...instruction,\n    ...historyLog,\n  ];\n\n  const {\n    response: {\n      content: rawResponse,\n      usage,\n      reasoning_content,\n      rawChoiceMessage,\n    },\n    planFromAI,\n    actions,\n    yamlFlow,\n  } = await callAndParsePlanningResponse({\n    messages: msgs,\n    modelRuntime,\n    abortSignal: opts.abortSignal,\n    includeLocateInPlanning: opts.includeLocateInPlanning,\n    actionSpace: opts.actionSpace,\n    locateResultCodec,\n    locateResultContext: {\n      preparedSize: preparedImage.preparedSize,\n      contentSize: preparedImage.contentSize,\n    },\n    includeThought,\n    includeLog,\n  });\n\n  let shouldContinuePlanning = true;\n\n  // Check if task is completed via <complete> tag\n  if (planFromAI.finalizeSuccess !== undefined) {\n    debug('task completed via <complete> tag, stop planning');\n    shouldContinuePlanning = false;\n    // Mark all sub-goals as finished when goal is completed in planning deep-think mode.\n    if (includeSubGoals) {\n      conversationHistory.markAllSubGoalsFinished();\n    }\n  }\n\n  const returnValue: PlanningAIResponse = {\n    ...planFromAI,\n    actions,\n    rawResponse,\n    rawChoiceMessage,\n    usage,\n    reasoning_content,\n    yamlFlow,\n    shouldContinuePlanning,\n  };\n\n  assert(planFromAI, \"can't get plans from AI\");\n\n  // TODO: The plan log is recorded before its action has executed, so a failed\n  // action may still appear in the next round as an action already performed.\n  // Move this write to the successful action execution path in TaskExecutor.action.\n  // Update sub-goals in conversation history only in planning deep-think mode.\n  if (includeSubGoals) {\n    if (planFromAI.updateSubGoals?.length) {\n      conversationHistory.mergeSubGoals(planFromAI.updateSubGoals);\n    }\n    if (planFromAI.markFinishedIndexes?.length) {\n      for (const index of planFromAI.markFinishedIndexes) {\n        conversationHistory.markSubGoalFinished(index);\n      }\n    }\n    // Append the planning log to the currently running sub-goal\n    if (planFromAI.log) {\n      conversationHistory.appendSubGoalLog(planFromAI.log);\n    }\n  } else {\n    // Without planning deep-think mode, accumulate logs as historical execution steps.\n    if (planFromAI.log) {\n      conversationHistory.appendHistoricalLog(planFromAI.log);\n    }\n  }\n\n  // Append memory to conversation history if present\n  if (planFromAI.memory) {\n    conversationHistory.appendMemory(planFromAI.memory);\n  }\n\n  // Some model providers require opaque assistant fields to be replayed\n  // verbatim in later turns. Keep this opt-in per model adapter so that an\n  // unverified provider does not receive non-standard response fields.\n  if (\n    modelRuntime.adapter.chatCompletion.replayRawAssistantMessage &&\n    rawChoiceMessage\n  ) {\n    conversationHistory.append(rawChoiceMessage as ChatCompletionMessageParam);\n  } else {\n    conversationHistory.append({\n      role: 'assistant',\n      content: [\n        {\n          type: 'text',\n          text: rawResponse,\n        },\n      ],\n    });\n  }\n\n  return returnValue;\n}\n"],"names":["debug","getDebug","warnLog","noPreviousActionsText","callAndParsePlanningResponse","options","messages","modelRuntime","abortSignal","includeLocateInPlanning","actionSpace","locateResultCodec","locateResultContext","includeThought","includeLog","assert","actionOutputProtocol","callAiAndParseWithRetry","retryAttempt","previousParseError","callAI","withSemanticRetryFeedback","response","planFromAI","parseStandardPlanningResponse","undefined","actions","normalizePlanningActionLocateFields","yamlFlow","buildYamlFlowFromPlans","parseError","errorMessage","Error","String","AIResponseParseError","standardPlan","userInstruction","opts","context","conversationHistory","adapter","shotSize","screenshotBase64","planningProtocol","planningModelFamilyRequiredForLocateMessage","includeSubGoals","systemPrompt","buildStandardPlanningSystemPrompt","preparedImage","prepareModelImage","imagePayload","actionContext","referenceImageMessages","preparedReferenceImagesToChatMessages","instruction","latestFeedbackMessage","executionProgressText","executionProgressSection","memoriesText","memoriesSection","historyLog","msgs","rawResponse","usage","reasoning_content","rawChoiceMessage","shouldContinuePlanning","returnValue","index"],"mappings":";;;;;;;;;;;AA6BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,UAAUD,SAAS,YAAY;IAAE,SAAS;AAAK;AAErD,MAAME,wBACJ;AAeF,eAAeC,6BACbC,OAA4C;IAO5C,MAAM,EACJC,QAAQ,EACRC,YAAY,EACZC,WAAW,EACXC,uBAAuB,EACvBC,WAAW,EACXC,iBAAiB,EACjBC,mBAAmB,EACnBC,cAAc,EACdC,UAAU,EACX,GAAGT;IACJU,OACER,AAAuC,eAAvCA,aAAa,OAAO,CAAC,QAAQ,CAAC,IAAI,EAClC;IAEF,MAAMS,uBACJT,aAAa,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB;IAE7D,OAAOU,wBAAwB;QAC7B,QAAQ,CAACC,cAAcC,qBACrBC,OACEC,0BAA0Bf,UAAUa,qBACpCZ,cACA;gBACEC;gBACA,6BAA6BC;gBAC7B,sBAAsBS;YACxB;QAEJ,eAAe,CAACI;YACd,MAAMC,aAAaC,8BAA8BF,SAAS,OAAO,EAAE;gBACjET;gBACAG;gBACAN;gBACA,WAAWI,aAAa,UAAU;YACpC;YACA,IAAIS,WAAW,MAAM,IAAIA,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;gBACjErB,QACE;gBAEFqB,WAAW,eAAe,GAAGE;gBAC7BF,WAAW,eAAe,GAAGE;YAC/B;YAEA,MAAMC,UAAUH,WAAW,MAAM,GAAG;gBAACA,WAAW,MAAM;aAAC,GAAG,EAAE;YAC5DI,oCAAoCD,SAAS;gBAC3ChB;gBACAD;gBACAE;gBACAC;gBACA,mBAAmBL,aAAa,OAAO,CAAC,iBAAiB;gBACzD,yBAAyBS,qBAAqB,uBAAuB;YACvE;YAGA,MAAMY,WAAWC,uBAAuBH,SAAShB;YACjD,OAAO;gBAAEY;gBAAUC;gBAAYG;gBAASE;YAAS;QACnD;QACA,cAAc,CAACE,YAAYR;YACzB,MAAMS,eACJD,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;YAC5D,OAAO,IAAII,qBACT,CAAC,iBAAiB,EAAEH,cAAc,EAClCT,SAAS,OAAO,EAChBA,SAAS,KAAK,EACdA,SAAS,gBAAgB,EACzBA,SAAS,iBAAiB;QAE9B;QACA,iBAAiBf,aAAa,MAAM,CAAC,UAAU;QAC/C,oBAAoBA,aAAa,MAAM,CAAC,aAAa;QACrDC;QACA,cAAc,CAACsB;YACb9B,MACE,mDACA8B,sBAAsBE,QAAQF,WAAW,OAAO,GAAGG,OAAOH;QAE9D;IACF;AACF;AAEO,eAAeK,aACpBC,eAAmC,EACnCC,IAAiB;IAEjB,MAAM,EAAEC,OAAO,EAAEC,mBAAmB,EAAE,GAAGF;IACzC,MAAM9B,eAAe8B,KAAK,YAAY;IACtC,MAAM,EAAEG,OAAO,EAAE,GAAGjC;IACpB,MAAM,EAAEkC,QAAQ,EAAE,GAAGH;IACrB,MAAMI,mBAAmBJ,QAAQ,UAAU,CAAC,MAAM;IAClDvB,OACEyB,AAA0B,eAA1BA,QAAQ,QAAQ,CAAC,IAAI,EACrB;IAEF,MAAMG,mBAAmBH,QAAQ,QAAQ,CAAC,QAAQ;IAElD,IAAIH,KAAK,uBAAuB,IAAI,CAAC9B,aAAa,MAAM,CAAC,WAAW,EAClE,MAAM,IAAIyB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMI,oBAAoBJ,aAAa,MAAM,CAAC,WAAW,GACrDiC,QAAQ,QAAQ,CAAC,iBAAiB,GAClCf;IAGJ,MAAMoB,kBAAkBR,AAAgB,gBAAhBA,KAAK,MAAM;IACnC,MAAMxB,iBAAiBwB,AAAgB,WAAhBA,KAAK,MAAM;IAClC,MAAMvB,aAAauB,AAAgB,WAAhBA,KAAK,MAAM;IAE9B,IAAIA,KAAK,uBAAuB,IAAI,CAAC1B,mBACnC,MAAM,IAAIqB,MACRY,4CAA4CrC,aAAa,MAAM,CAAC,IAAI;IAIxE,MAAMuC,eAAe,MAAMC,kCAAkC;QAC3D,aAAaV,KAAK,WAAW;QAC7BxB;QACAC;QACA+B;QACAF;QACA,GAAIN,KAAK,uBAAuB,IAAI1B,oBAChC;YACE,yBAAyB;YACzB,kBAAkBA,kBAAkB,UAAU;QAChD,IACA;YAAE,yBAAyB;QAAM,CAAC;IACxC;IAEA,MAAMqC,gBAAgB,MAAMC,kBAAkB;QAC5C,aAAaP;QACb,OAAOD,SAAS,KAAK;QACrB,QAAQA,SAAS,MAAM;QACvB,QAAQD,QAAQ,eAAe;IACjC;IACA,MAAMU,eAAeF,cAAc,WAAW;IAE9C,MAAMG,gBAAgBd,KAAK,aAAa,GAAG,GAAGA,KAAK,aAAa,CAAC,EAAE,CAAC,GAAG;IAEvE,MAAMe,yBAAyBC,sCAC7BjB,gBAAgB,eAAe;IAEjC,MAAMkB,cAA4C;QAChD;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGH,cAAc,kBAAkB,EAAEf,gBAAgB,IAAI,CAAC,mBAAmB,CAAC;gBACtF;aACD;QACH;WACGgB;KACJ;IAED,IAAIG;IAKJ,MAAMC,wBAAwBX,kBAC1BN,oBAAoB,cAAc,KAClCA,oBAAoB,oBAAoB;IAC5C,MAAMkB,2BAA2BD,wBAC7B,CAAC,IAAI,EAAEA,uBAAuB,GAC9BjB,oBAAoB,sBAAsB,GACxC,KACA,CAAC,IAAI,EAAEpC,uBAAuB;IAGpC,MAAMuD,eAAenB,oBAAoB,cAAc;IACvD,MAAMoB,kBAAkBD,eAAe,CAAC,IAAI,EAAEA,cAAc,GAAG;IAE/D,IAAInB,oBAAoB,sBAAsB,EAAE;QAC9CgB,wBAAwB;YACtB,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM,GAAGhB,oBAAoB,sBAAsB,CAAC,qHAAqH,EAAEoB,kBAAkBF,0BAA0B;gBACzN;gBACA;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKP;wBACL,QAAQ;oBACV;gBACF;aACD;QACH;QAEAX,oBAAoB,mCAAmC;IACzD,OACEgB,wBAAwB;QACtB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAM,CAAC,+BAA+B,EAAEI,kBAAkBF,0BAA0B;YACtF;YACA;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKP;oBACL,QAAQ;gBACV;YACF;SACD;IACH;IAEFX,oBAAoB,MAAM,CAACgB;IAG3BhB,oBAAoB,eAAe,CAAC,IAAI;IAExC,MAAMqB,aAAarB,oBAAoB,QAAQ,CAACF,KAAK,kBAAkB;IAEvE,MAAMwB,OAAqC;QACzC;YAAE,MAAM;YAAU,SAASf;QAAa;WACrCQ;WACAM;KACJ;IAED,MAAM,EACJ,UAAU,EACR,SAASE,WAAW,EACpBC,KAAK,EACLC,iBAAiB,EACjBC,gBAAgB,EACjB,EACD1C,UAAU,EACVG,OAAO,EACPE,QAAQ,EACT,GAAG,MAAMxB,6BAA6B;QACrC,UAAUyD;QACVtD;QACA,aAAa8B,KAAK,WAAW;QAC7B,yBAAyBA,KAAK,uBAAuB;QACrD,aAAaA,KAAK,WAAW;QAC7B1B;QACA,qBAAqB;YACnB,cAAcqC,cAAc,YAAY;YACxC,aAAaA,cAAc,WAAW;QACxC;QACAnC;QACAC;IACF;IAEA,IAAIoD,yBAAyB;IAG7B,IAAI3C,AAA+BE,WAA/BF,WAAW,eAAe,EAAgB;QAC5CvB,MAAM;QACNkE,yBAAyB;QAEzB,IAAIrB,iBACFN,oBAAoB,uBAAuB;IAE/C;IAEA,MAAM4B,cAAkC;QACtC,GAAG5C,UAAU;QACbG;QACAoC;QACAG;QACAF;QACAC;QACApC;QACAsC;IACF;IAEAnD,OAAOQ,YAAY;IAMnB,IAAIsB,iBAAiB;QACnB,IAAItB,WAAW,cAAc,EAAE,QAC7BgB,oBAAoB,aAAa,CAAChB,WAAW,cAAc;QAE7D,IAAIA,WAAW,mBAAmB,EAAE,QAClC,KAAK,MAAM6C,SAAS7C,WAAW,mBAAmB,CAChDgB,oBAAoB,mBAAmB,CAAC6B;QAI5C,IAAI7C,WAAW,GAAG,EAChBgB,oBAAoB,gBAAgB,CAAChB,WAAW,GAAG;IAEvD,OAEE,IAAIA,WAAW,GAAG,EAChBgB,oBAAoB,mBAAmB,CAAChB,WAAW,GAAG;IAK1D,IAAIA,WAAW,MAAM,EACnBgB,oBAAoB,YAAY,CAAChB,WAAW,MAAM;IAMpD,IACEhB,aAAa,OAAO,CAAC,cAAc,CAAC,yBAAyB,IAC7D0D,kBAEA1B,oBAAoB,MAAM,CAAC0B;SAE3B1B,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,MAAMuB;YACR;SACD;IACH;IAGF,OAAOK;AACT"}

package/dist/es/common.mjs

@@ -138,19 +138,12 @@ const parseActionParam = (rawParam, zodSchema, options)=>{
     const ratio = options?.shrunkShotToLogicalRatio;
     for(const fieldName in locateFieldValues){
         let value = locateFieldValues[fieldName];
-        if (void 0 !== ratio && 1 !== ratio && value && 'object' == typeof value && value.center && value.rect) value = {
+        if (void 0 !== ratio && value && 'object' == typeof value && value.center) value = {
             ...value,
             center: [
                 Math.round(value.center[0] / ratio),
                 Math.round(value.center[1] / ratio)
-            ],
-            rect: {
-                ...value.rect,
-                left: Math.round(value.rect.left / ratio),
-                top: Math.round(value.rect.top / ratio),
-                width: Math.round(value.rect.width / ratio),
-                height: Math.round(value.rect.height / ratio)
-            }
+            ]
         };
         validated[fieldName] = value;
     }

package/dist/es/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n} from '@/types';\nimport { isPlainObject } from '@midscene/shared/utils';\nimport { z } from 'zod';\n\nexport function findActionInActionSpaceOrThrow(\n  planType: string,\n  actionSpace: DeviceAction<any>[],\n): DeviceAction<any> {\n  const action = actionSpace.find((item) => item.name === planType);\n  if (!action) {\n    const available = actionSpace.map((item) => item.name).join(', ');\n    throw new Error(\n      `Action type '${planType}' is not in the current action space. Available actions: ${available || '(none)'}`,\n    );\n  }\n  return action;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = findActionInActionSpaceOrThrow(verb, actionSpace);\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    // For actions whose param is a single string field (e.g. Launch/Terminate's\n    // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n    // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n    // string as the param and drop the sibling `uri`, so cache replay would\n    // call the action with an empty argument.\n    const shortcutField =\n      action.name === 'Launch' || action.interfaceAlias === 'launch'\n        ? 'uri'\n        : action.name === 'Terminate' || action.interfaceAlias === 'terminate'\n          ? 'uri'\n          : action.name === 'RunAdbShell' ||\n              action.interfaceAlias === 'runAdbShell' ||\n              action.name === 'RunHdcShell' ||\n              action.interfaceAlias === 'runHdcShell'\n            ? 'command'\n            : undefined;\n    const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n    const canInlineShortcut =\n      shortcutField &&\n      shortcutKeys.length === 1 &&\n      shortcutKeys[0] === shortcutField &&\n      typeof flowParam[shortcutField] === 'string';\n\n    const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n      ? { [flowKey]: flowParam[shortcutField as string] }\n      : { [flowKey]: '', ...flowParam };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nexport const userPromptToString = (prompt: TUserPrompt): string => {\n  return typeof prompt === 'string' ? prompt : prompt.prompt;\n};\n\nexport const userPromptToMultimodalPrompt = (\n  prompt: TUserPrompt,\n): TMultimodalPrompt | undefined => {\n  if (typeof prompt === 'string' || !prompt.images) {\n    return undefined;\n  }\n  return {\n    images: prompt.images,\n    convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n  };\n};\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepLocate: z.boolean().optional(),\n    deepThink: z\n      .boolean()\n      .optional()\n      .describe('@deprecated Use `deepLocate` instead.'),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nconst formatPromptWithImages = (\n  promptObj: Exclude<TUserPrompt, string>,\n): string => {\n  let promptString = promptObj.prompt;\n  if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n    const imageCount = promptObj.images.length;\n    promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n  }\n  return promptString;\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      // TODO: A similar locator-field check exists as isMidsceneLocatorField in\n      // @midscene/shared/zod-schema-utils.\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n *\n * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n  options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement),\n  // and transform coordinates from screenshot space to logical space if needed\n  const ratio = options?.shrunkShotToLogicalRatio;\n  for (const fieldName in locateFieldValues) {\n    let value = locateFieldValues[fieldName];\n    if (\n      ratio !== undefined &&\n      ratio !== 1 &&\n      value &&\n      typeof value === 'object' &&\n      value.center &&\n      value.rect\n    ) {\n      value = {\n        ...value,\n        center: [\n          Math.round(value.center[0] / ratio),\n          Math.round(value.center[1] / ratio),\n        ],\n        rect: {\n          ...value.rect,\n          left: Math.round(value.rect.left / ratio),\n          top: Math.round(value.rect.top / ratio),\n          width: Math.round(value.rect.width / ratio),\n          height: Math.round(value.rect.height / ratio),\n        },\n      };\n    }\n    validated[fieldName] = value;\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n  timestamp?: number,\n): string => {\n  const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["findActionInActionSpaceOrThrow","planType","actionSpace","action","item","available","Error","buildYamlFlowFromPlans","plans","flow","plan","verb","flowKey","flowParam","dumpActionParam","shortcutField","undefined","shortcutKeys","Object","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","userPromptToString","prompt","userPromptToMultimodalPrompt","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","Array","imageCount","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","key","jsonObject","zodSchema","isPlainObject","locatorFields","result","fieldName","fieldValue","parseActionParam","rawParam","options","param","locateFields","locateFieldValues","paramsForValidation","validated","ratio","value","Math","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","String","day","hours","minutes","seconds","timeString"],"mappings":";;AAQO,SAASA,+BACdC,QAAgB,EAChBC,WAAgC;IAEhC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;IACxD,IAAI,CAACE,QAAQ;QACX,MAAME,YAAYH,YAAY,GAAG,CAAC,CAACE,OAASA,KAAK,IAAI,EAAE,IAAI,CAAC;QAC5D,MAAM,IAAIE,MACR,CAAC,aAAa,EAAEL,SAAS,yDAAyD,EAAEI,aAAa,UAAU;IAE/G;IACA,OAAOF;AACT;AAEO,SAASI,uBACdC,KAAuB,EACvBN,WAAgC;IAEhC,MAAMO,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQF,MAAO;QACxB,MAAMG,OAAOD,KAAK,IAAI;QAEtB,MAAMP,SAASH,+BAA+BW,MAAMT;QAEpD,MAAMU,UAAUT,OAAO,cAAc,IAAIQ;QACzC,MAAME,YAAYV,OAAO,WAAW,GAChCW,gBAAgBJ,KAAK,KAAK,IAAI,CAAC,GAAGP,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMY,gBACJZ,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,IACrBA,AAAgB,kBAAhBA,OAAO,IAAI,IACXA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACAa;QACV,MAAMC,eAAeF,gBAAgBG,OAAO,IAAI,CAACL,aAAa,EAAE;QAChE,MAAMM,oBACJJ,iBACAE,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKF,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMK,WAAiCD,oBACnC;YAAE,CAACP,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCJ,KAAK,IAAI,CAACW;IACZ;IAEA,OAAOX;AACT;AAGO,MAAMY,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMM,MAAME,qBAAqB,CAACC,SAC1B,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAGrD,MAAMC,+BAA+B,CAC1CD;IAEA,IAAI,AAAkB,YAAlB,OAAOA,UAAuB,CAACA,OAAO,MAAM,EAC9C;IAEF,OAAO;QACL,QAAQA,OAAO,MAAM;QACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;IAC3D;AACF;AAGA,MAAME,wBAAwBR,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,EAAE,OAAO,GAAG,QAAQ;IAChC,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMS,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAI,YAAYC,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAIE,MAAM,OAAO,CAACF,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAMG,aAAaH,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEE,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOF;AACT;AAEO,MAAMG,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO3B,OAAO,IAAI,CAAC0B,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACC;YAClB,MAAMb,QAAQW,UAAU,KAAK,CAACE,IAAI;YAGlC,IAAI,CAACd,uBAAuBC,QAC1B,OAAO;YAIT,IAAIU,cACF,OAAOV,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMnB,kBAAkB,CAC7BiC,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBT,4BAA4BO;IAClD,MAAMG,SAAS;QAAE,GAAGJ,UAAU;IAAC;IAE/B,KAAK,MAAMK,aAAaF,cAAe;QACrC,MAAMG,aAAaF,MAAM,CAACC,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTF,MAAM,CAACC,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BF,MAAM,CAACC,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBF,MAAM,CAACC,UAAU,GAAGhB,uBAAuBiB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOF;AACT;AAaO,MAAMG,mBAAmB,CAC9BC,UACAP,WACAQ;IAGA,IAAI,CAACR,WACH;IAIF,MAAMS,QAAQF,YAAY,CAAC;IAG3B,MAAMG,eAAejB,4BAA4BO;IAGjD,IAAIU,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOV,UAAU,KAAK,CAACS;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMd,OAAOW,MAChB,IAAIC,aAAa,QAAQ,CAACZ,MAExBc,mBAAmB,CAACd,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Cc,mBAAmB,CAACd,IAAI,GAAGW,KAAK,CAACX,IAAI;IAKzC,MAAMe,YAAYb,UAAU,KAAK,CAACY;IAIlC,MAAME,QAAQN,SAAS;IACvB,IAAK,MAAMJ,aAAaO,kBAAmB;QACzC,IAAII,QAAQJ,iBAAiB,CAACP,UAAU;QACxC,IACEU,AAAU9C,WAAV8C,SACAA,AAAU,MAAVA,SACAC,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,IACZA,MAAM,IAAI,EAEVA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACNC,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;gBAC7BE,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;aAC9B;YACD,MAAM;gBACJ,GAAGC,MAAM,IAAI;gBACb,MAAMC,KAAK,KAAK,CAACD,MAAM,IAAI,CAAC,IAAI,GAAGD;gBACnC,KAAKE,KAAK,KAAK,CAACD,MAAM,IAAI,CAAC,GAAG,GAAGD;gBACjC,OAAOE,KAAK,KAAK,CAACD,MAAM,IAAI,CAAC,KAAK,GAAGD;gBACrC,QAAQE,KAAK,KAAK,CAACD,MAAM,IAAI,CAAC,MAAM,GAAGD;YACzC;QACF;QAEFD,SAAS,CAACT,UAAU,GAAGW;IACzB;IAEA,OAAOF;AACT;AAEO,MAAMI,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAAcpD,WAAdoD,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQC,OAAOJ,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMK,MAAMD,OAAOJ,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMM,QAAQF,OAAOJ,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMO,UAAUH,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMQ,UAAUJ,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMS,aAAaX,OAChB,OAAO,CAAC,QAAQM,OAAOF,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAME,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEX,OAAO,CAAC,CAAC;AACpC"}
+{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n  DeviceAction,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n} from '@/types';\nimport { isPlainObject } from '@midscene/shared/utils';\nimport { z } from 'zod';\n\nexport function findActionInActionSpaceOrThrow(\n  planType: string,\n  actionSpace: DeviceAction<any>[],\n): DeviceAction<any> {\n  const action = actionSpace.find((item) => item.name === planType);\n  if (!action) {\n    const available = actionSpace.map((item) => item.name).join(', ');\n    throw new Error(\n      `Action type '${planType}' is not in the current action space. Available actions: ${available || '(none)'}`,\n    );\n  }\n  return action;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = findActionInActionSpaceOrThrow(verb, actionSpace);\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    // For actions whose param is a single string field (e.g. Launch/Terminate's\n    // `uri`, RunAdbShell's `command`), inline the value on the flowKey. Writing\n    // `{ terminate: '', uri: '...' }` makes the YAML player treat the empty\n    // string as the param and drop the sibling `uri`, so cache replay would\n    // call the action with an empty argument.\n    const shortcutField =\n      action.name === 'Launch' || action.interfaceAlias === 'launch'\n        ? 'uri'\n        : action.name === 'Terminate' || action.interfaceAlias === 'terminate'\n          ? 'uri'\n          : action.name === 'RunAdbShell' ||\n              action.interfaceAlias === 'runAdbShell' ||\n              action.name === 'RunHdcShell' ||\n              action.interfaceAlias === 'runHdcShell'\n            ? 'command'\n            : undefined;\n    const shortcutKeys = shortcutField ? Object.keys(flowParam) : [];\n    const canInlineShortcut =\n      shortcutField &&\n      shortcutKeys.length === 1 &&\n      shortcutKeys[0] === shortcutField &&\n      typeof flowParam[shortcutField] === 'string';\n\n    const flowItem: MidsceneYamlFlowItem = canInlineShortcut\n      ? { [flowKey]: flowParam[shortcutField as string] }\n      : { [flowKey]: '', ...flowParam };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nexport const userPromptToString = (prompt: TUserPrompt): string => {\n  return typeof prompt === 'string' ? prompt : prompt.prompt;\n};\n\nexport const userPromptToMultimodalPrompt = (\n  prompt: TUserPrompt,\n): TMultimodalPrompt | undefined => {\n  if (typeof prompt === 'string' || !prompt.images) {\n    return undefined;\n  }\n  return {\n    images: prompt.images,\n    convertHttpImage2Base64: !!prompt.convertHttpImage2Base64,\n  };\n};\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepLocate: z.boolean().optional(),\n    deepThink: z\n      .boolean()\n      .optional()\n      .describe('@deprecated Use `deepLocate` instead.'),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nconst formatPromptWithImages = (\n  promptObj: Exclude<TUserPrompt, string>,\n): string => {\n  let promptString = promptObj.prompt;\n  if (Array.isArray(promptObj.images) && promptObj.images.length > 0) {\n    const imageCount = promptObj.images.length;\n    promptString += ` (with ${imageCount} image${imageCount > 1 ? 's' : ''})`;\n  }\n  return promptString;\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      // TODO: A similar locator-field check exists as isMidsceneLocatorField in\n      // @midscene/shared/zod-schema-utils.\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = formatPromptWithImages(fieldValue.prompt);\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n *\n * When shrunkShotToLogicalRatio is provided and !== 1, coordinates in locate fields\n * are transformed from screenshot space to logical space.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n  options?: { shrunkShotToLogicalRatio?: number },\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement),\n  // and transform coordinates from screenshot space to logical space if needed\n  const ratio = options?.shrunkShotToLogicalRatio;\n  for (const fieldName in locateFieldValues) {\n    let value = locateFieldValues[fieldName];\n    if (\n      ratio !== undefined &&\n      value &&\n      typeof value === 'object' &&\n      value.center\n    ) {\n      value = {\n        ...value,\n        center: [\n          Math.round(value.center[0] / ratio),\n          Math.round(value.center[1] / ratio),\n        ],\n      };\n    }\n    validated[fieldName] = value;\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for a given timestamp or the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @param timestamp - Optional timestamp in milliseconds. If not provided, uses current system time.\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n  timestamp?: number,\n): string => {\n  const now = timestamp !== undefined ? new Date(timestamp) : new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["findActionInActionSpaceOrThrow","planType","actionSpace","action","item","available","Error","buildYamlFlowFromPlans","plans","flow","plan","verb","flowKey","flowParam","dumpActionParam","shortcutField","undefined","shortcutKeys","Object","canInlineShortcut","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","userPromptToString","prompt","userPromptToMultimodalPrompt","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","formatPromptWithImages","promptObj","promptString","Array","imageCount","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","key","jsonObject","zodSchema","isPlainObject","locatorFields","result","fieldName","fieldValue","parseActionParam","rawParam","options","param","locateFields","locateFieldValues","paramsForValidation","validated","ratio","value","Math","finalizeActionName","getReadableTimeString","format","timestamp","now","Date","year","month","String","day","hours","minutes","seconds","timeString"],"mappings":";;AAQO,SAASA,+BACdC,QAAgB,EAChBC,WAAgC;IAEhC,MAAMC,SAASD,YAAY,IAAI,CAAC,CAACE,OAASA,KAAK,IAAI,KAAKH;IACxD,IAAI,CAACE,QAAQ;QACX,MAAME,YAAYH,YAAY,GAAG,CAAC,CAACE,OAASA,KAAK,IAAI,EAAE,IAAI,CAAC;QAC5D,MAAM,IAAIE,MACR,CAAC,aAAa,EAAEL,SAAS,yDAAyD,EAAEI,aAAa,UAAU;IAE/G;IACA,OAAOF;AACT;AAEO,SAASI,uBACdC,KAAuB,EACvBN,WAAgC;IAEhC,MAAMO,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQF,MAAO;QACxB,MAAMG,OAAOD,KAAK,IAAI;QAEtB,MAAMP,SAASH,+BAA+BW,MAAMT;QAEpD,MAAMU,UAAUT,OAAO,cAAc,IAAIQ;QACzC,MAAME,YAAYV,OAAO,WAAW,GAChCW,gBAAgBJ,KAAK,KAAK,IAAI,CAAC,GAAGP,OAAO,WAAW,IACpD,CAAC;QAOL,MAAMY,gBACJZ,AAAgB,aAAhBA,OAAO,IAAI,IAAiBA,AAA0B,aAA1BA,OAAO,cAAc,GAC7C,QACAA,AAAgB,gBAAhBA,OAAO,IAAI,IAAoBA,AAA0B,gBAA1BA,OAAO,cAAc,GAClD,QACAA,AAAgB,kBAAhBA,OAAO,IAAI,IACTA,AAA0B,kBAA1BA,OAAO,cAAc,IACrBA,AAAgB,kBAAhBA,OAAO,IAAI,IACXA,AAA0B,kBAA1BA,OAAO,cAAc,GACrB,YACAa;QACV,MAAMC,eAAeF,gBAAgBG,OAAO,IAAI,CAACL,aAAa,EAAE;QAChE,MAAMM,oBACJJ,iBACAE,AAAwB,MAAxBA,aAAa,MAAM,IACnBA,YAAY,CAAC,EAAE,KAAKF,iBACpB,AAAoC,YAApC,OAAOF,SAAS,CAACE,cAAc;QAEjC,MAAMK,WAAiCD,oBACnC;YAAE,CAACP,QAAQ,EAAEC,SAAS,CAACE,cAAwB;QAAC,IAChD;YAAE,CAACH,QAAQ,EAAE;YAAI,GAAGC,SAAS;QAAC;QAElCJ,KAAK,IAAI,CAACW;IACZ;IAEA,OAAOX;AACT;AAGO,MAAMY,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;AAClB;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMM,MAAME,qBAAqB,CAACC,SAC1B,AAAkB,YAAlB,OAAOA,SAAsBA,SAASA,OAAO,MAAM;AAGrD,MAAMC,+BAA+B,CAC1CD;IAEA,IAAI,AAAkB,YAAlB,OAAOA,UAAuB,CAACA,OAAO,MAAM,EAC9C;IAEF,OAAO;QACL,QAAQA,OAAO,MAAM;QACrB,yBAAyB,CAAC,CAACA,OAAO,uBAAuB;IAC3D;AACF;AAGA,MAAME,wBAAwBR,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,YAAYJ,EAAE,OAAO,GAAG,QAAQ;IAChC,WAAWA,EAAAA,OACD,GACP,QAAQ,GACR,QAAQ,CAAC;IACZ,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAMP,MAAMS,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAI,YAAYC,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEA,MAAMC,yBAAyB,CAC7BC;IAEA,IAAIC,eAAeD,UAAU,MAAM;IACnC,IAAIE,MAAM,OAAO,CAACF,UAAU,MAAM,KAAKA,UAAU,MAAM,CAAC,MAAM,GAAG,GAAG;QAClE,MAAMG,aAAaH,UAAU,MAAM,CAAC,MAAM;QAC1CC,gBAAgB,CAAC,OAAO,EAAEE,WAAW,MAAM,EAAEA,aAAa,IAAI,MAAM,GAAG,CAAC,CAAC;IAC3E;IACA,OAAOF;AACT;AAEO,MAAMG,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO3B,OAAO,IAAI,CAAC0B,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACC;YAClB,MAAMb,QAAQW,UAAU,KAAK,CAACE,IAAI;YAGlC,IAAI,CAACd,uBAAuBC,QAC1B,OAAO;YAIT,IAAIU,cACF,OAAOV,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMnB,kBAAkB,CAC7BiC,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBT,4BAA4BO;IAClD,MAAMG,SAAS;QAAE,GAAGJ,UAAU;IAAC;IAE/B,KAAK,MAAMK,aAAaF,cAAe;QACrC,MAAMG,aAAaF,MAAM,CAACC,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTF,MAAM,CAACC,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BF,MAAM,CAACC,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBF,MAAM,CAACC,UAAU,GAAGhB,uBAAuBiB,WAAW,MAAM;gBAC9D;YACF;QACF;IAEJ;IAEA,OAAOF;AACT;AAaO,MAAMG,mBAAmB,CAC9BC,UACAP,WACAQ;IAGA,IAAI,CAACR,WACH;IAIF,MAAMS,QAAQF,YAAY,CAAC;IAG3B,MAAMG,eAAejB,4BAA4BO;IAGjD,IAAIU,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOV,UAAU,KAAK,CAACS;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMd,OAAOW,MAChB,IAAIC,aAAa,QAAQ,CAACZ,MAExBc,mBAAmB,CAACd,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Cc,mBAAmB,CAACd,IAAI,GAAGW,KAAK,CAACX,IAAI;IAKzC,MAAMe,YAAYb,UAAU,KAAK,CAACY;IAIlC,MAAME,QAAQN,SAAS;IACvB,IAAK,MAAMJ,aAAaO,kBAAmB;QACzC,IAAII,QAAQJ,iBAAiB,CAACP,UAAU;QACxC,IACEU,AAAU9C,WAAV8C,SACAC,SACA,AAAiB,YAAjB,OAAOA,SACPA,MAAM,MAAM,EAEZA,QAAQ;YACN,GAAGA,KAAK;YACR,QAAQ;gBACNC,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;gBAC7BE,KAAK,KAAK,CAACD,MAAM,MAAM,CAAC,EAAE,GAAGD;aAC9B;QACH;QAEFD,SAAS,CAACT,UAAU,GAAGW;IACzB;IAEA,OAAOF;AACT;AAEO,MAAMI,qBAAqB;AAQ3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB,EAC9BC;IAEA,MAAMC,MAAMD,AAAcpD,WAAdoD,YAA0B,IAAIE,KAAKF,aAAa,IAAIE;IAChE,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQC,OAAOJ,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMK,MAAMD,OAAOJ,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMM,QAAQF,OAAOJ,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMO,UAAUH,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMQ,UAAUJ,OAAOJ,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMS,aAAaX,OAChB,OAAO,CAAC,QAAQM,OAAOF,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAME,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAEX,OAAO,CAAC,CAAC;AACpC"}

package/dist/es/element-describer.mjs

@@ -13,8 +13,8 @@ const includedInRect = (point, rect)=>{
 };
 const buildLocateValidatorResult = (expectCenter, located, verifyLocateOption)=>{
     const distance = distanceOfTwoPoints(expectCenter, located.center);
-    const included = includedInRect(expectCenter, located.rect);
-    const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;
+    const included = located.rect ? includedInRect(expectCenter, located.rect) : void 0;
+    const pass = distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || true === included;
     return {
         pass,
         rect: located.rect,

package/dist/es/element-describer.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"element-describer.mjs","sources":["../../src/element-describer.ts"],"sourcesContent":["import type { ModelRuntime } from '@/ai-model/models';\nimport { assert } from '@midscene/shared/utils';\nimport { createScreenshotBoundUIContext } from './agent/utils';\nimport type Service from './service';\nimport type {\n  AgentDescribeElementAtPointResult,\n  LocateOption,\n  LocateResultElement,\n  LocateValidatorResult,\n  LocatorValidatorOption,\n  Rect,\n  Size,\n  UIContext,\n} from './types';\nimport { buildDetailedLocateParam } from './yaml/utils';\n\nexport type DescribeElementCoordinateSpace = 'screenshot' | 'logical';\n\nexport type LocatorVerifyFn = (input: {\n  prompt: string;\n  expectCenter: [number, number];\n  retryCount: number;\n  verifyResult: LocateValidatorResult;\n}) => LocateValidatorResult | boolean;\n\nexport type DescribeElementAtPointOptions = {\n  verifyPrompt?: boolean;\n  retryLimit?: number;\n  deepDescribe?: boolean;\n  deepLocate?: boolean;\n  locatorVerifyFn?: LocatorVerifyFn;\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n  onProgress?: (progress: {\n    prompt?: string;\n    deepDescribe?: boolean;\n    deepLocate?: boolean;\n    verifyResult?: LocateValidatorResult;\n  }) => void;\n} & LocatorValidatorOption;\n\ntype ScreenshotBoundContextOptions = {\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n};\n\nexport type VerifyElementDescriptionAtPointOptions =\n  ScreenshotBoundContextOptions & LocatorValidatorOption;\n\ntype LocateAndVerifyOptions = VerifyElementDescriptionAtPointOptions &\n  Pick<LocateOption, 'cacheable' | 'deepLocate' | 'xpath'> & {\n    abortSignal?: AbortSignal;\n  };\n\nexport type ElementDescriberRuntime = {\n  service: Pick<Service, 'describe' | 'locate'>;\n  describeModelRuntime: ModelRuntime;\n  locateModelRuntime: ModelRuntime;\n};\n\ntype ServiceDescribeOptions = NonNullable<\n  Parameters<ElementDescriberRuntime['service']['describe']>[2]\n>;\n\nconst distanceOfTwoPoints = (p1: [number, number], p2: [number, number]) => {\n  const [x1, y1] = p1;\n  const [x2, y2] = p2;\n  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));\n};\n\nconst includedInRect = (point: [number, number], rect: Rect) => {\n  const [x, y] = point;\n  const { left, top, width, height } = rect;\n  return x >= left && x <= left + width && y >= top && y <= top + height;\n};\n\nconst buildLocateValidatorResult = (\n  expectCenter: [number, number],\n  located: Pick<LocateResultElement, 'center' | 'rect'>,\n  verifyLocateOption?: LocatorValidatorOption,\n): LocateValidatorResult => {\n  const distance = distanceOfTwoPoints(expectCenter, located.center);\n  const included = includedInRect(expectCenter, located.rect);\n  const pass =\n    distance <= (verifyLocateOption?.centerDistanceThreshold || 20) || included;\n  return {\n    pass,\n    rect: located.rect,\n    center: located.center,\n    centerDistance: distance,\n    includedInRect: included,\n  };\n};\n\nfunction assertPositiveSize(\n  size: Size | undefined,\n  label: string,\n): asserts size is Size {\n  assert(\n    size &&\n      Number.isFinite(size.width) &&\n      Number.isFinite(size.height) &&\n      size.width > 0 &&\n      size.height > 0,\n    `${label} must include positive width and height`,\n  );\n}\n\nconst mapPointToScreenshotSpace = (\n  center: [number, number],\n  screenshotSize: Size,\n  opt: ScreenshotBoundContextOptions,\n): [number, number] => {\n  const coordinateSpace = opt.coordinateSpace || 'screenshot';\n  if (coordinateSpace === 'screenshot') {\n    return center;\n  }\n\n  assertPositiveSize(\n    opt.logicalSize,\n    'logicalSize is required when coordinateSpace is logical',\n  );\n  return [\n    (center[0] * screenshotSize.width) / opt.logicalSize.width,\n    (center[1] * screenshotSize.height) / opt.logicalSize.height,\n  ];\n};\n\nconst createScreenshotBoundLocatorContext = async (\n  center: [number, number],\n  opt?: ScreenshotBoundContextOptions,\n): Promise<{\n  screenshotContext?: UIContext;\n  locateOpt?: LocateOption;\n  targetCenter: [number, number];\n}> => {\n  const screenshotContext = opt?.screenshotBase64\n    ? await createScreenshotBoundUIContext(opt.screenshotBase64, opt)\n    : undefined;\n  const targetCenter = screenshotContext\n    ? mapPointToScreenshotSpace(center, screenshotContext.shotSize, opt || {})\n    : center;\n  return {\n    screenshotContext,\n    locateOpt: screenshotContext ? { uiContext: screenshotContext } : undefined,\n    targetCenter,\n  };\n};\n\nexport async function verifyLocator(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  prompt: string,\n  locateOpt: LocateOption | undefined,\n  expectCenter: [number, number],\n  verifyLocateOption?: LocatorValidatorOption &\n    Pick<LocateOption, 'deepLocate'>,\n): Promise<LocateValidatorResult> {\n  return locateAndVerify(runtime, prompt, expectCenter, {\n    centerDistanceThreshold: verifyLocateOption?.centerDistanceThreshold,\n    deepLocate: verifyLocateOption?.deepLocate,\n    uiContext: locateOpt?.uiContext,\n  });\n}\n\nfunction applyLocatorVerifyFn(\n  locatorVerifyFn: LocatorVerifyFn | undefined,\n  input: Parameters<LocatorVerifyFn>[0],\n): LocateValidatorResult {\n  if (!locatorVerifyFn) {\n    return input.verifyResult;\n  }\n  const customResult = locatorVerifyFn(input);\n  if (typeof customResult === 'boolean') {\n    return {\n      ...input.verifyResult,\n      pass: customResult,\n    };\n  }\n  return customResult;\n}\n\nfunction errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport async function describeElementAtPoint(\n  runtime: ElementDescriberRuntime,\n  center: [number, number],\n  opt?: DescribeElementAtPointOptions,\n): Promise<AgentDescribeElementAtPointResult> {\n  const { verifyPrompt = true, retryLimit = 3 } = opt || {};\n  const { screenshotContext, locateOpt, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n\n  let success = false;\n  let retryCount = 0;\n  let resultPrompt = '';\n  const autoRetryDeepDescribe = opt?.deepDescribe === undefined;\n  const autoRetryDeepLocate = opt?.deepLocate === undefined;\n  let deepDescribe = opt?.deepDescribe || false;\n  let deepLocate = opt?.deepLocate || false;\n  let verifyResult: LocateValidatorResult | undefined;\n  let lastError: string | undefined;\n  let failureStage: AgentDescribeElementAtPointResult['failureStage'];\n\n  while (!success && retryCount < retryLimit) {\n    if (retryCount >= 1 && autoRetryDeepDescribe) {\n      deepDescribe = true;\n    }\n    if (retryCount >= 1 && autoRetryDeepLocate) {\n      deepLocate = true;\n    }\n    const describeModelRuntime = runtime.describeModelRuntime;\n    const locateModelRuntime = runtime.locateModelRuntime;\n    const retryRuntime: ElementDescriberRuntime = {\n      ...runtime,\n      describeModelRuntime,\n      locateModelRuntime,\n    };\n    const describeOpt: ServiceDescribeOptions = screenshotContext\n      ? {\n          deepDescribe,\n          context: screenshotContext,\n        }\n      : {\n          deepDescribe,\n        };\n    let text: Awaited<\n      ReturnType<ElementDescriberRuntime['service']['describe']>\n    >;\n    try {\n      text = await retryRuntime.service.describe(\n        targetCenter,\n        retryRuntime.describeModelRuntime,\n        describeOpt,\n      );\n    } catch (error) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        deepDescribe,\n        verifyResult,\n        success: false,\n        error: errorMessage(error),\n        failureStage: 'describe',\n      };\n    }\n    if (!text.description) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        deepDescribe,\n        verifyResult,\n        success: false,\n        error: `failed to describe element at [${targetCenter}]`,\n        failureStage: 'describe',\n      };\n    }\n    resultPrompt = text.description;\n    if (!verifyPrompt) {\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n      });\n      success = true;\n      break;\n    }\n\n    try {\n      const candidateVerifyResult = await verifyLocator(\n        retryRuntime,\n        resultPrompt,\n        locateOpt,\n        targetCenter,\n        {\n          ...opt,\n          deepLocate,\n        },\n      );\n      verifyResult = applyLocatorVerifyFn(opt?.locatorVerifyFn, {\n        prompt: resultPrompt,\n        expectCenter: targetCenter,\n        retryCount,\n        verifyResult: candidateVerifyResult,\n      });\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n        verifyResult,\n      });\n      if (verifyResult.pass) {\n        success = true;\n        break;\n      }\n      lastError = undefined;\n      failureStage = 'verify';\n    } catch (error) {\n      lastError = errorMessage(error);\n      failureStage = 'verify';\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n      });\n    }\n    retryCount++;\n  }\n\n  return {\n    prompt: resultPrompt,\n    deepLocate,\n    deepDescribe,\n    verifyResult,\n    success,\n    error:\n      success || !verifyPrompt\n        ? undefined\n        : lastError || 'describeElementAtPoint verify failed',\n    failureStage: success ? undefined : failureStage,\n  };\n}\n\nexport async function verifyElementDescriptionAtPoint(\n  runtime: ElementDescriberRuntime,\n  description: string,\n  center: [number, number],\n  opt?: VerifyElementDescriptionAtPointOptions,\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { locateOpt, targetCenter } = await createScreenshotBoundLocatorContext(\n    center,\n    opt,\n  );\n\n  return verifyLocator(runtime, description, locateOpt, targetCenter, opt);\n}\n\nasync function locateAndVerify(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  description: string,\n  center: [number, number],\n  opt?: LocateAndVerifyOptions & {\n    uiContext?: UIContext;\n  },\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { screenshotContext, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n  const context = opt?.uiContext || screenshotContext;\n  const locateParam = buildDetailedLocateParam(description, {\n    cacheable: opt?.cacheable,\n    deepLocate: opt?.deepLocate,\n    xpath: opt?.xpath,\n  });\n  assert(locateParam, 'cannot get locate param for service locate');\n\n  const locateResult = await runtime.service.locate(\n    locateParam,\n    context ? { context } : {},\n    runtime.locateModelRuntime,\n    opt?.abortSignal,\n  );\n  assert(locateResult.element, `Element not found: ${description}`);\n  const verifyResult = buildLocateValidatorResult(\n    targetCenter,\n    locateResult.element,\n    opt,\n  );\n  return verifyResult;\n}\n"],"names":["distanceOfTwoPoints","p1","p2","x1","y1","x2","y2","Math","includedInRect","point","rect","x","y","left","top","width","height","buildLocateValidatorResult","expectCenter","located","verifyLocateOption","distance","included","pass","assertPositiveSize","size","label","assert","Number","mapPointToScreenshotSpace","center","screenshotSize","opt","coordinateSpace","createScreenshotBoundLocatorContext","screenshotContext","createScreenshotBoundUIContext","undefined","targetCenter","verifyLocator","runtime","prompt","locateOpt","locateAndVerify","applyLocatorVerifyFn","locatorVerifyFn","input","customResult","errorMessage","error","Error","String","describeElementAtPoint","verifyPrompt","retryLimit","success","retryCount","resultPrompt","autoRetryDeepDescribe","autoRetryDeepLocate","deepDescribe","deepLocate","verifyResult","lastError","failureStage","describeModelRuntime","locateModelRuntime","retryRuntime","describeOpt","text","candidateVerifyResult","verifyElementDescriptionAtPoint","description","context","locateParam","buildDetailedLocateParam","locateResult"],"mappings":";;;AAoEA,MAAMA,sBAAsB,CAACC,IAAsBC;IACjD,MAAM,CAACC,IAAIC,GAAG,GAAGH;IACjB,MAAM,CAACI,IAAIC,GAAG,GAAGJ;IACjB,OAAOK,KAAK,KAAK,CAACA,KAAK,IAAI,CAAEJ,AAAAA,CAAAA,KAAKE,EAAC,KAAM,IAAKD,AAAAA,CAAAA,KAAKE,EAAC,KAAM;AAC5D;AAEA,MAAME,iBAAiB,CAACC,OAAyBC;IAC/C,MAAM,CAACC,GAAGC,EAAE,GAAGH;IACf,MAAM,EAAEI,IAAI,EAAEC,GAAG,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGN;IACrC,OAAOC,KAAKE,QAAQF,KAAKE,OAAOE,SAASH,KAAKE,OAAOF,KAAKE,MAAME;AAClE;AAEA,MAAMC,6BAA6B,CACjCC,cACAC,SACAC;IAEA,MAAMC,WAAWrB,oBAAoBkB,cAAcC,QAAQ,MAAM;IACjE,MAAMG,WAAWd,eAAeU,cAAcC,QAAQ,IAAI;IAC1D,MAAMI,OACJF,YAAaD,CAAAA,oBAAoB,2BAA2B,EAAC,KAAME;IACrE,OAAO;QACLC;QACA,MAAMJ,QAAQ,IAAI;QAClB,QAAQA,QAAQ,MAAM;QACtB,gBAAgBE;QAChB,gBAAgBC;IAClB;AACF;AAEA,SAASE,mBACPC,IAAsB,EACtBC,KAAa;IAEbC,OACEF,QACEG,OAAO,QAAQ,CAACH,KAAK,KAAK,KAC1BG,OAAO,QAAQ,CAACH,KAAK,MAAM,KAC3BA,KAAK,KAAK,GAAG,KACbA,KAAK,MAAM,GAAG,GAChB,GAAGC,MAAM,uCAAuC,CAAC;AAErD;AAEA,MAAMG,4BAA4B,CAChCC,QACAC,gBACAC;IAEA,MAAMC,kBAAkBD,IAAI,eAAe,IAAI;IAC/C,IAAIC,AAAoB,iBAApBA,iBACF,OAAOH;IAGTN,mBACEQ,IAAI,WAAW,EACf;IAEF,OAAO;QACJF,MAAM,CAAC,EAAE,GAAGC,eAAe,KAAK,GAAIC,IAAI,WAAW,CAAC,KAAK;QACzDF,MAAM,CAAC,EAAE,GAAGC,eAAe,MAAM,GAAIC,IAAI,WAAW,CAAC,MAAM;KAC7D;AACH;AAEA,MAAME,sCAAsC,OAC1CJ,QACAE;IAMA,MAAMG,oBAAoBH,KAAK,mBAC3B,MAAMI,+BAA+BJ,IAAI,gBAAgB,EAAEA,OAC3DK;IACJ,MAAMC,eAAeH,oBACjBN,0BAA0BC,QAAQK,kBAAkB,QAAQ,EAAEH,OAAO,CAAC,KACtEF;IACJ,OAAO;QACLK;QACA,WAAWA,oBAAoB;YAAE,WAAWA;QAAkB,IAAIE;QAClEC;IACF;AACF;AAEO,eAAeC,cACpBC,OAAwE,EACxEC,MAAc,EACdC,SAAmC,EACnCxB,YAA8B,EAC9BE,kBACkC;IAElC,OAAOuB,gBAAgBH,SAASC,QAAQvB,cAAc;QACpD,yBAAyBE,oBAAoB;QAC7C,YAAYA,oBAAoB;QAChC,WAAWsB,WAAW;IACxB;AACF;AAEA,SAASE,qBACPC,eAA4C,EAC5CC,KAAqC;IAErC,IAAI,CAACD,iBACH,OAAOC,MAAM,YAAY;IAE3B,MAAMC,eAAeF,gBAAgBC;IACrC,IAAI,AAAwB,aAAxB,OAAOC,cACT,OAAO;QACL,GAAGD,MAAM,YAAY;QACrB,MAAMC;IACR;IAEF,OAAOA;AACT;AAEA,SAASC,aAAaC,KAAc;IAClC,OAAOA,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;AACzD;AAEO,eAAeG,uBACpBZ,OAAgC,EAChCV,MAAwB,EACxBE,GAAmC;IAEnC,MAAM,EAAEqB,eAAe,IAAI,EAAEC,aAAa,CAAC,EAAE,GAAGtB,OAAO,CAAC;IACxD,MAAM,EAAEG,iBAAiB,EAAEO,SAAS,EAAEJ,YAAY,EAAE,GAClD,MAAMJ,oCAAoCJ,QAAQE;IAEpD,IAAIuB,UAAU;IACd,IAAIC,aAAa;IACjB,IAAIC,eAAe;IACnB,MAAMC,wBAAwB1B,KAAK,iBAAiBK;IACpD,MAAMsB,sBAAsB3B,KAAK,eAAeK;IAChD,IAAIuB,eAAe5B,KAAK,gBAAgB;IACxC,IAAI6B,aAAa7B,KAAK,cAAc;IACpC,IAAI8B;IACJ,IAAIC;IACJ,IAAIC;IAEJ,MAAO,CAACT,WAAWC,aAAaF,WAAY;QAC1C,IAAIE,cAAc,KAAKE,uBACrBE,eAAe;QAEjB,IAAIJ,cAAc,KAAKG,qBACrBE,aAAa;QAEf,MAAMI,uBAAuBzB,QAAQ,oBAAoB;QACzD,MAAM0B,qBAAqB1B,QAAQ,kBAAkB;QACrD,MAAM2B,eAAwC;YAC5C,GAAG3B,OAAO;YACVyB;YACAC;QACF;QACA,MAAME,cAAsCjC,oBACxC;YACEyB;YACA,SAASzB;QACX,IACA;YACEyB;QACF;QACJ,IAAIS;QAGJ,IAAI;YACFA,OAAO,MAAMF,aAAa,OAAO,CAAC,QAAQ,CACxC7B,cACA6B,aAAa,oBAAoB,EACjCC;QAEJ,EAAE,OAAOnB,OAAO;YACd,OAAO;gBACL,QAAQQ;gBACRI;gBACAD;gBACAE;gBACA,SAAS;gBACT,OAAOd,aAAaC;gBACpB,cAAc;YAChB;QACF;QACA,IAAI,CAACoB,KAAK,WAAW,EACnB,OAAO;YACL,QAAQZ;YACRI;YACAD;YACAE;YACA,SAAS;YACT,OAAO,CAAC,+BAA+B,EAAExB,aAAa,CAAC,CAAC;YACxD,cAAc;QAChB;QAEFmB,eAAeY,KAAK,WAAW;QAC/B,IAAI,CAAChB,cAAc;YACjBrB,KAAK,aAAa;gBAChB,QAAQyB;gBACRG;gBACAC;YACF;YACAN,UAAU;YACV;QACF;QAEA,IAAI;YACF,MAAMe,wBAAwB,MAAM/B,cAClC4B,cACAV,cACAf,WACAJ,cACA;gBACE,GAAGN,GAAG;gBACN6B;YACF;YAEFC,eAAelB,qBAAqBZ,KAAK,iBAAiB;gBACxD,QAAQyB;gBACR,cAAcnB;gBACdkB;gBACA,cAAcc;YAChB;YACAtC,KAAK,aAAa;gBAChB,QAAQyB;gBACRG;gBACAC;gBACAC;YACF;YACA,IAAIA,aAAa,IAAI,EAAE;gBACrBP,UAAU;gBACV;YACF;YACAQ,YAAY1B;YACZ2B,eAAe;QACjB,EAAE,OAAOf,OAAO;YACdc,YAAYf,aAAaC;YACzBe,eAAe;YACfhC,KAAK,aAAa;gBAChB,QAAQyB;gBACRG;gBACAC;YACF;QACF;QACAL;IACF;IAEA,OAAO;QACL,QAAQC;QACRI;QACAD;QACAE;QACAP;QACA,OACEA,WAAW,CAACF,eACRhB,SACA0B,aAAa;QACnB,cAAcR,UAAUlB,SAAY2B;IACtC;AACF;AAEO,eAAeO,gCACpB/B,OAAgC,EAChCgC,WAAmB,EACnB1C,MAAwB,EACxBE,GAA4C;IAE5CL,OAAO6C,aAAa,QAAQ;IAC5B,MAAM,EAAE9B,SAAS,EAAEJ,YAAY,EAAE,GAAG,MAAMJ,oCACxCJ,QACAE;IAGF,OAAOO,cAAcC,SAASgC,aAAa9B,WAAWJ,cAAcN;AACtE;AAEA,eAAeW,gBACbH,OAAwE,EACxEgC,WAAmB,EACnB1C,MAAwB,EACxBE,GAEC;IAEDL,OAAO6C,aAAa,QAAQ;IAC5B,MAAM,EAAErC,iBAAiB,EAAEG,YAAY,EAAE,GACvC,MAAMJ,oCAAoCJ,QAAQE;IACpD,MAAMyC,UAAUzC,KAAK,aAAaG;IAClC,MAAMuC,cAAcC,yBAAyBH,aAAa;QACxD,WAAWxC,KAAK;QAChB,YAAYA,KAAK;QACjB,OAAOA,KAAK;IACd;IACAL,OAAO+C,aAAa;IAEpB,MAAME,eAAe,MAAMpC,QAAQ,OAAO,CAAC,MAAM,CAC/CkC,aACAD,UAAU;QAAEA;IAAQ,IAAI,CAAC,GACzBjC,QAAQ,kBAAkB,EAC1BR,KAAK;IAEPL,OAAOiD,aAAa,OAAO,EAAE,CAAC,mBAAmB,EAAEJ,aAAa;IAChE,MAAMV,eAAe7C,2BACnBqB,cACAsC,aAAa,OAAO,EACpB5C;IAEF,OAAO8B;AACT"}
+{"version":3,"file":"element-describer.mjs","sources":["../../src/element-describer.ts"],"sourcesContent":["import type { ModelRuntime } from '@/ai-model/models';\nimport { assert } from '@midscene/shared/utils';\nimport { createScreenshotBoundUIContext } from './agent/utils';\nimport type Service from './service';\nimport type {\n  AgentDescribeElementAtPointResult,\n  LocateOption,\n  LocateResultElement,\n  LocateValidatorResult,\n  LocatorValidatorOption,\n  Rect,\n  Size,\n  UIContext,\n} from './types';\nimport { buildDetailedLocateParam } from './yaml/utils';\n\nexport type DescribeElementCoordinateSpace = 'screenshot' | 'logical';\n\nexport type LocatorVerifyFn = (input: {\n  prompt: string;\n  expectCenter: [number, number];\n  retryCount: number;\n  verifyResult: LocateValidatorResult;\n}) => LocateValidatorResult | boolean;\n\nexport type DescribeElementAtPointOptions = {\n  verifyPrompt?: boolean;\n  retryLimit?: number;\n  deepDescribe?: boolean;\n  deepLocate?: boolean;\n  locatorVerifyFn?: LocatorVerifyFn;\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n  onProgress?: (progress: {\n    prompt?: string;\n    deepDescribe?: boolean;\n    deepLocate?: boolean;\n    verifyResult?: LocateValidatorResult;\n  }) => void;\n} & LocatorValidatorOption;\n\ntype ScreenshotBoundContextOptions = {\n  screenshotBase64?: string;\n  screenshotSize?: Size;\n  coordinateSpace?: DescribeElementCoordinateSpace;\n  logicalSize?: Size;\n};\n\nexport type VerifyElementDescriptionAtPointOptions =\n  ScreenshotBoundContextOptions & LocatorValidatorOption;\n\ntype LocateAndVerifyOptions = VerifyElementDescriptionAtPointOptions &\n  Pick<LocateOption, 'cacheable' | 'deepLocate' | 'xpath'> & {\n    abortSignal?: AbortSignal;\n  };\n\nexport type ElementDescriberRuntime = {\n  service: Pick<Service, 'describe' | 'locate'>;\n  describeModelRuntime: ModelRuntime;\n  locateModelRuntime: ModelRuntime;\n};\n\ntype ServiceDescribeOptions = NonNullable<\n  Parameters<ElementDescriberRuntime['service']['describe']>[2]\n>;\n\nconst distanceOfTwoPoints = (p1: [number, number], p2: [number, number]) => {\n  const [x1, y1] = p1;\n  const [x2, y2] = p2;\n  return Math.round(Math.sqrt((x1 - x2) ** 2 + (y1 - y2) ** 2));\n};\n\nconst includedInRect = (point: [number, number], rect: Rect) => {\n  const [x, y] = point;\n  const { left, top, width, height } = rect;\n  return x >= left && x <= left + width && y >= top && y <= top + height;\n};\n\nconst buildLocateValidatorResult = (\n  expectCenter: [number, number],\n  located: Pick<LocateResultElement, 'center' | 'rect'>,\n  verifyLocateOption?: LocatorValidatorOption,\n): LocateValidatorResult => {\n  const distance = distanceOfTwoPoints(expectCenter, located.center);\n  const included = located.rect\n    ? includedInRect(expectCenter, located.rect)\n    : undefined;\n  const pass =\n    distance <= (verifyLocateOption?.centerDistanceThreshold || 20) ||\n    included === true;\n  return {\n    pass,\n    rect: located.rect,\n    center: located.center,\n    centerDistance: distance,\n    includedInRect: included,\n  };\n};\n\nfunction assertPositiveSize(\n  size: Size | undefined,\n  label: string,\n): asserts size is Size {\n  assert(\n    size &&\n      Number.isFinite(size.width) &&\n      Number.isFinite(size.height) &&\n      size.width > 0 &&\n      size.height > 0,\n    `${label} must include positive width and height`,\n  );\n}\n\nconst mapPointToScreenshotSpace = (\n  center: [number, number],\n  screenshotSize: Size,\n  opt: ScreenshotBoundContextOptions,\n): [number, number] => {\n  const coordinateSpace = opt.coordinateSpace || 'screenshot';\n  if (coordinateSpace === 'screenshot') {\n    return center;\n  }\n\n  assertPositiveSize(\n    opt.logicalSize,\n    'logicalSize is required when coordinateSpace is logical',\n  );\n  return [\n    (center[0] * screenshotSize.width) / opt.logicalSize.width,\n    (center[1] * screenshotSize.height) / opt.logicalSize.height,\n  ];\n};\n\nconst createScreenshotBoundLocatorContext = async (\n  center: [number, number],\n  opt?: ScreenshotBoundContextOptions,\n): Promise<{\n  screenshotContext?: UIContext;\n  locateOpt?: LocateOption;\n  targetCenter: [number, number];\n}> => {\n  const screenshotContext = opt?.screenshotBase64\n    ? await createScreenshotBoundUIContext(opt.screenshotBase64, opt)\n    : undefined;\n  const targetCenter = screenshotContext\n    ? mapPointToScreenshotSpace(center, screenshotContext.shotSize, opt || {})\n    : center;\n  return {\n    screenshotContext,\n    locateOpt: screenshotContext ? { uiContext: screenshotContext } : undefined,\n    targetCenter,\n  };\n};\n\nexport async function verifyLocator(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  prompt: string,\n  locateOpt: LocateOption | undefined,\n  expectCenter: [number, number],\n  verifyLocateOption?: LocatorValidatorOption &\n    Pick<LocateOption, 'deepLocate'>,\n): Promise<LocateValidatorResult> {\n  return locateAndVerify(runtime, prompt, expectCenter, {\n    centerDistanceThreshold: verifyLocateOption?.centerDistanceThreshold,\n    deepLocate: verifyLocateOption?.deepLocate,\n    uiContext: locateOpt?.uiContext,\n  });\n}\n\nfunction applyLocatorVerifyFn(\n  locatorVerifyFn: LocatorVerifyFn | undefined,\n  input: Parameters<LocatorVerifyFn>[0],\n): LocateValidatorResult {\n  if (!locatorVerifyFn) {\n    return input.verifyResult;\n  }\n  const customResult = locatorVerifyFn(input);\n  if (typeof customResult === 'boolean') {\n    return {\n      ...input.verifyResult,\n      pass: customResult,\n    };\n  }\n  return customResult;\n}\n\nfunction errorMessage(error: unknown): string {\n  return error instanceof Error ? error.message : String(error);\n}\n\nexport async function describeElementAtPoint(\n  runtime: ElementDescriberRuntime,\n  center: [number, number],\n  opt?: DescribeElementAtPointOptions,\n): Promise<AgentDescribeElementAtPointResult> {\n  const { verifyPrompt = true, retryLimit = 3 } = opt || {};\n  const { screenshotContext, locateOpt, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n\n  let success = false;\n  let retryCount = 0;\n  let resultPrompt = '';\n  const autoRetryDeepDescribe = opt?.deepDescribe === undefined;\n  const autoRetryDeepLocate = opt?.deepLocate === undefined;\n  let deepDescribe = opt?.deepDescribe || false;\n  let deepLocate = opt?.deepLocate || false;\n  let verifyResult: LocateValidatorResult | undefined;\n  let lastError: string | undefined;\n  let failureStage: AgentDescribeElementAtPointResult['failureStage'];\n\n  while (!success && retryCount < retryLimit) {\n    if (retryCount >= 1 && autoRetryDeepDescribe) {\n      deepDescribe = true;\n    }\n    if (retryCount >= 1 && autoRetryDeepLocate) {\n      deepLocate = true;\n    }\n    const describeModelRuntime = runtime.describeModelRuntime;\n    const locateModelRuntime = runtime.locateModelRuntime;\n    const retryRuntime: ElementDescriberRuntime = {\n      ...runtime,\n      describeModelRuntime,\n      locateModelRuntime,\n    };\n    const describeOpt: ServiceDescribeOptions = screenshotContext\n      ? {\n          deepDescribe,\n          context: screenshotContext,\n        }\n      : {\n          deepDescribe,\n        };\n    let text: Awaited<\n      ReturnType<ElementDescriberRuntime['service']['describe']>\n    >;\n    try {\n      text = await retryRuntime.service.describe(\n        targetCenter,\n        retryRuntime.describeModelRuntime,\n        describeOpt,\n      );\n    } catch (error) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        deepDescribe,\n        verifyResult,\n        success: false,\n        error: errorMessage(error),\n        failureStage: 'describe',\n      };\n    }\n    if (!text.description) {\n      return {\n        prompt: resultPrompt,\n        deepLocate,\n        deepDescribe,\n        verifyResult,\n        success: false,\n        error: `failed to describe element at [${targetCenter}]`,\n        failureStage: 'describe',\n      };\n    }\n    resultPrompt = text.description;\n    if (!verifyPrompt) {\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n      });\n      success = true;\n      break;\n    }\n\n    try {\n      const candidateVerifyResult = await verifyLocator(\n        retryRuntime,\n        resultPrompt,\n        locateOpt,\n        targetCenter,\n        {\n          ...opt,\n          deepLocate,\n        },\n      );\n      verifyResult = applyLocatorVerifyFn(opt?.locatorVerifyFn, {\n        prompt: resultPrompt,\n        expectCenter: targetCenter,\n        retryCount,\n        verifyResult: candidateVerifyResult,\n      });\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n        verifyResult,\n      });\n      if (verifyResult.pass) {\n        success = true;\n        break;\n      }\n      lastError = undefined;\n      failureStage = 'verify';\n    } catch (error) {\n      lastError = errorMessage(error);\n      failureStage = 'verify';\n      opt?.onProgress?.({\n        prompt: resultPrompt,\n        deepDescribe,\n        deepLocate,\n      });\n    }\n    retryCount++;\n  }\n\n  return {\n    prompt: resultPrompt,\n    deepLocate,\n    deepDescribe,\n    verifyResult,\n    success,\n    error:\n      success || !verifyPrompt\n        ? undefined\n        : lastError || 'describeElementAtPoint verify failed',\n    failureStage: success ? undefined : failureStage,\n  };\n}\n\nexport async function verifyElementDescriptionAtPoint(\n  runtime: ElementDescriberRuntime,\n  description: string,\n  center: [number, number],\n  opt?: VerifyElementDescriptionAtPointOptions,\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { locateOpt, targetCenter } = await createScreenshotBoundLocatorContext(\n    center,\n    opt,\n  );\n\n  return verifyLocator(runtime, description, locateOpt, targetCenter, opt);\n}\n\nasync function locateAndVerify(\n  runtime: Pick<ElementDescriberRuntime, 'service' | 'locateModelRuntime'>,\n  description: string,\n  center: [number, number],\n  opt?: LocateAndVerifyOptions & {\n    uiContext?: UIContext;\n  },\n): Promise<LocateValidatorResult> {\n  assert(description?.trim(), 'description must not be empty');\n  const { screenshotContext, targetCenter } =\n    await createScreenshotBoundLocatorContext(center, opt);\n  const context = opt?.uiContext || screenshotContext;\n  const locateParam = buildDetailedLocateParam(description, {\n    cacheable: opt?.cacheable,\n    deepLocate: opt?.deepLocate,\n    xpath: opt?.xpath,\n  });\n  assert(locateParam, 'cannot get locate param for service locate');\n\n  const locateResult = await runtime.service.locate(\n    locateParam,\n    context ? { context } : {},\n    runtime.locateModelRuntime,\n    opt?.abortSignal,\n  );\n  assert(locateResult.element, `Element not found: ${description}`);\n  const verifyResult = buildLocateValidatorResult(\n    targetCenter,\n    locateResult.element,\n    opt,\n  );\n  return verifyResult;\n}\n"],"names":["distanceOfTwoPoints","p1","p2","x1","y1","x2","y2","Math","includedInRect","point","rect","x","y","left","top","width","height","buildLocateValidatorResult","expectCenter","located","verifyLocateOption","distance","included","undefined","pass","assertPositiveSize","size","label","assert","Number","mapPointToScreenshotSpace","center","screenshotSize","opt","coordinateSpace","createScreenshotBoundLocatorContext","screenshotContext","createScreenshotBoundUIContext","targetCenter","verifyLocator","runtime","prompt","locateOpt","locateAndVerify","applyLocatorVerifyFn","locatorVerifyFn","input","customResult","errorMessage","error","Error","String","describeElementAtPoint","verifyPrompt","retryLimit","success","retryCount","resultPrompt","autoRetryDeepDescribe","autoRetryDeepLocate","deepDescribe","deepLocate","verifyResult","lastError","failureStage","describeModelRuntime","locateModelRuntime","retryRuntime","describeOpt","text","candidateVerifyResult","verifyElementDescriptionAtPoint","description","context","locateParam","buildDetailedLocateParam","locateResult"],"mappings":";;;AAoEA,MAAMA,sBAAsB,CAACC,IAAsBC;IACjD,MAAM,CAACC,IAAIC,GAAG,GAAGH;IACjB,MAAM,CAACI,IAAIC,GAAG,GAAGJ;IACjB,OAAOK,KAAK,KAAK,CAACA,KAAK,IAAI,CAAEJ,AAAAA,CAAAA,KAAKE,EAAC,KAAM,IAAKD,AAAAA,CAAAA,KAAKE,EAAC,KAAM;AAC5D;AAEA,MAAME,iBAAiB,CAACC,OAAyBC;IAC/C,MAAM,CAACC,GAAGC,EAAE,GAAGH;IACf,MAAM,EAAEI,IAAI,EAAEC,GAAG,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAGN;IACrC,OAAOC,KAAKE,QAAQF,KAAKE,OAAOE,SAASH,KAAKE,OAAOF,KAAKE,MAAME;AAClE;AAEA,MAAMC,6BAA6B,CACjCC,cACAC,SACAC;IAEA,MAAMC,WAAWrB,oBAAoBkB,cAAcC,QAAQ,MAAM;IACjE,MAAMG,WAAWH,QAAQ,IAAI,GACzBX,eAAeU,cAAcC,QAAQ,IAAI,IACzCI;IACJ,MAAMC,OACJH,YAAaD,CAAAA,oBAAoB,2BAA2B,EAAC,KAC7DE,AAAa,SAAbA;IACF,OAAO;QACLE;QACA,MAAML,QAAQ,IAAI;QAClB,QAAQA,QAAQ,MAAM;QACtB,gBAAgBE;QAChB,gBAAgBC;IAClB;AACF;AAEA,SAASG,mBACPC,IAAsB,EACtBC,KAAa;IAEbC,OACEF,QACEG,OAAO,QAAQ,CAACH,KAAK,KAAK,KAC1BG,OAAO,QAAQ,CAACH,KAAK,MAAM,KAC3BA,KAAK,KAAK,GAAG,KACbA,KAAK,MAAM,GAAG,GAChB,GAAGC,MAAM,uCAAuC,CAAC;AAErD;AAEA,MAAMG,4BAA4B,CAChCC,QACAC,gBACAC;IAEA,MAAMC,kBAAkBD,IAAI,eAAe,IAAI;IAC/C,IAAIC,AAAoB,iBAApBA,iBACF,OAAOH;IAGTN,mBACEQ,IAAI,WAAW,EACf;IAEF,OAAO;QACJF,MAAM,CAAC,EAAE,GAAGC,eAAe,KAAK,GAAIC,IAAI,WAAW,CAAC,KAAK;QACzDF,MAAM,CAAC,EAAE,GAAGC,eAAe,MAAM,GAAIC,IAAI,WAAW,CAAC,MAAM;KAC7D;AACH;AAEA,MAAME,sCAAsC,OAC1CJ,QACAE;IAMA,MAAMG,oBAAoBH,KAAK,mBAC3B,MAAMI,+BAA+BJ,IAAI,gBAAgB,EAAEA,OAC3DV;IACJ,MAAMe,eAAeF,oBACjBN,0BAA0BC,QAAQK,kBAAkB,QAAQ,EAAEH,OAAO,CAAC,KACtEF;IACJ,OAAO;QACLK;QACA,WAAWA,oBAAoB;YAAE,WAAWA;QAAkB,IAAIb;QAClEe;IACF;AACF;AAEO,eAAeC,cACpBC,OAAwE,EACxEC,MAAc,EACdC,SAAmC,EACnCxB,YAA8B,EAC9BE,kBACkC;IAElC,OAAOuB,gBAAgBH,SAASC,QAAQvB,cAAc;QACpD,yBAAyBE,oBAAoB;QAC7C,YAAYA,oBAAoB;QAChC,WAAWsB,WAAW;IACxB;AACF;AAEA,SAASE,qBACPC,eAA4C,EAC5CC,KAAqC;IAErC,IAAI,CAACD,iBACH,OAAOC,MAAM,YAAY;IAE3B,MAAMC,eAAeF,gBAAgBC;IACrC,IAAI,AAAwB,aAAxB,OAAOC,cACT,OAAO;QACL,GAAGD,MAAM,YAAY;QACrB,MAAMC;IACR;IAEF,OAAOA;AACT;AAEA,SAASC,aAAaC,KAAc;IAClC,OAAOA,iBAAiBC,QAAQD,MAAM,OAAO,GAAGE,OAAOF;AACzD;AAEO,eAAeG,uBACpBZ,OAAgC,EAChCT,MAAwB,EACxBE,GAAmC;IAEnC,MAAM,EAAEoB,eAAe,IAAI,EAAEC,aAAa,CAAC,EAAE,GAAGrB,OAAO,CAAC;IACxD,MAAM,EAAEG,iBAAiB,EAAEM,SAAS,EAAEJ,YAAY,EAAE,GAClD,MAAMH,oCAAoCJ,QAAQE;IAEpD,IAAIsB,UAAU;IACd,IAAIC,aAAa;IACjB,IAAIC,eAAe;IACnB,MAAMC,wBAAwBzB,KAAK,iBAAiBV;IACpD,MAAMoC,sBAAsB1B,KAAK,eAAeV;IAChD,IAAIqC,eAAe3B,KAAK,gBAAgB;IACxC,IAAI4B,aAAa5B,KAAK,cAAc;IACpC,IAAI6B;IACJ,IAAIC;IACJ,IAAIC;IAEJ,MAAO,CAACT,WAAWC,aAAaF,WAAY;QAC1C,IAAIE,cAAc,KAAKE,uBACrBE,eAAe;QAEjB,IAAIJ,cAAc,KAAKG,qBACrBE,aAAa;QAEf,MAAMI,uBAAuBzB,QAAQ,oBAAoB;QACzD,MAAM0B,qBAAqB1B,QAAQ,kBAAkB;QACrD,MAAM2B,eAAwC;YAC5C,GAAG3B,OAAO;YACVyB;YACAC;QACF;QACA,MAAME,cAAsChC,oBACxC;YACEwB;YACA,SAASxB;QACX,IACA;YACEwB;QACF;QACJ,IAAIS;QAGJ,IAAI;YACFA,OAAO,MAAMF,aAAa,OAAO,CAAC,QAAQ,CACxC7B,cACA6B,aAAa,oBAAoB,EACjCC;QAEJ,EAAE,OAAOnB,OAAO;YACd,OAAO;gBACL,QAAQQ;gBACRI;gBACAD;gBACAE;gBACA,SAAS;gBACT,OAAOd,aAAaC;gBACpB,cAAc;YAChB;QACF;QACA,IAAI,CAACoB,KAAK,WAAW,EACnB,OAAO;YACL,QAAQZ;YACRI;YACAD;YACAE;YACA,SAAS;YACT,OAAO,CAAC,+BAA+B,EAAExB,aAAa,CAAC,CAAC;YACxD,cAAc;QAChB;QAEFmB,eAAeY,KAAK,WAAW;QAC/B,IAAI,CAAChB,cAAc;YACjBpB,KAAK,aAAa;gBAChB,QAAQwB;gBACRG;gBACAC;YACF;YACAN,UAAU;YACV;QACF;QAEA,IAAI;YACF,MAAMe,wBAAwB,MAAM/B,cAClC4B,cACAV,cACAf,WACAJ,cACA;gBACE,GAAGL,GAAG;gBACN4B;YACF;YAEFC,eAAelB,qBAAqBX,KAAK,iBAAiB;gBACxD,QAAQwB;gBACR,cAAcnB;gBACdkB;gBACA,cAAcc;YAChB;YACArC,KAAK,aAAa;gBAChB,QAAQwB;gBACRG;gBACAC;gBACAC;YACF;YACA,IAAIA,aAAa,IAAI,EAAE;gBACrBP,UAAU;gBACV;YACF;YACAQ,YAAYxC;YACZyC,eAAe;QACjB,EAAE,OAAOf,OAAO;YACdc,YAAYf,aAAaC;YACzBe,eAAe;YACf/B,KAAK,aAAa;gBAChB,QAAQwB;gBACRG;gBACAC;YACF;QACF;QACAL;IACF;IAEA,OAAO;QACL,QAAQC;QACRI;QACAD;QACAE;QACAP;QACA,OACEA,WAAW,CAACF,eACR9B,SACAwC,aAAa;QACnB,cAAcR,UAAUhC,SAAYyC;IACtC;AACF;AAEO,eAAeO,gCACpB/B,OAAgC,EAChCgC,WAAmB,EACnBzC,MAAwB,EACxBE,GAA4C;IAE5CL,OAAO4C,aAAa,QAAQ;IAC5B,MAAM,EAAE9B,SAAS,EAAEJ,YAAY,EAAE,GAAG,MAAMH,oCACxCJ,QACAE;IAGF,OAAOM,cAAcC,SAASgC,aAAa9B,WAAWJ,cAAcL;AACtE;AAEA,eAAeU,gBACbH,OAAwE,EACxEgC,WAAmB,EACnBzC,MAAwB,EACxBE,GAEC;IAEDL,OAAO4C,aAAa,QAAQ;IAC5B,MAAM,EAAEpC,iBAAiB,EAAEE,YAAY,EAAE,GACvC,MAAMH,oCAAoCJ,QAAQE;IACpD,MAAMwC,UAAUxC,KAAK,aAAaG;IAClC,MAAMsC,cAAcC,yBAAyBH,aAAa;QACxD,WAAWvC,KAAK;QAChB,YAAYA,KAAK;QACjB,OAAOA,KAAK;IACd;IACAL,OAAO8C,aAAa;IAEpB,MAAME,eAAe,MAAMpC,QAAQ,OAAO,CAAC,MAAM,CAC/CkC,aACAD,UAAU;QAAEA;IAAQ,IAAI,CAAC,GACzBjC,QAAQ,kBAAkB,EAC1BP,KAAK;IAEPL,OAAOgD,aAAa,OAAO,EAAE,CAAC,mBAAmB,EAAEJ,aAAa;IAChE,MAAMV,eAAe7C,2BACnBqB,cACAsC,aAAa,OAAO,EACpB3C;IAEF,OAAO6B;AACT"}