@midscene/core 0.30.10 → 1.0.0

package/dist/lib/common.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"common.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../src/common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n  TEXT = 5,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\ntype AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(\n      locate.bbox,\n      width,\n      height,\n      rightLimit,\n      bottomLimit,\n      vlMode,\n    );\n  }\n\n  return locate;\n}\n\nexport function adaptQwen2_5Bbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction normalizeBboxInput(\n  bbox: AdaptBboxInput,\n): number[] | string[] | string {\n  if (Array.isArray(bbox)) {\n    if (Array.isArray(bbox[0])) {\n      return bbox[0] as number[] | string[];\n    }\n    return bbox as number[] | string[];\n  }\n  return bbox as string;\n}\n\nexport function adaptBbox(\n  bbox: AdaptBboxInput,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  const normalizedBbox = normalizeBboxInput(bbox);\n\n  let result: [number, number, number, number] = [0, 0, 0, 0];\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    result = adaptDoubaoBbox(normalizedBbox, width, height);\n  } else if (vlMode === 'gemini') {\n    result = adaptGeminiBbox(normalizedBbox as number[], width, height);\n  } else if (vlMode === 'qwen3-vl') {\n    result = normalized01000(normalizedBbox as number[], width, height);\n  } else {\n    result = adaptQwen2_5Bbox(normalizedBbox as number[]);\n  }\n\n  result[2] = Math.min(result[2], rightLimit);\n  result[3] = Math.min(result[3], bottomLimit);\n\n  return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  return [\n    Math.round((bbox[0] * width) / 1000),\n    Math.round((bbox[1] * height) / 1000),\n    Math.round((bbox[2] * width) / 1000),\n    Math.round((bbox[3] * height) / 1000),\n  ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  rightLimit = width,\n  bottomLimit = height,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils(\n    'adaptBboxToRect',\n    bbox,\n    width,\n    height,\n    'offset',\n    offsetX,\n    offsetY,\n    'limit',\n    rightLimit,\n    bottomLimit,\n    'vlMode',\n    vlMode,\n  );\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    rightLimit,\n    bottomLimit,\n    vlMode,\n  );\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n\n  return rect;\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  const minEdgeSize = 500; // vlMode === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepThink: z.boolean().optional(),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\n\n// Export the input type - this is the inferred type from getMidsceneLocationSchema()\nexport type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement)\n  for (const fieldName in locateFieldValues) {\n    validated[fieldName] = locateFieldValues[fieldName];\n  }\n\n  return validated;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwen2_5Bbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","bboxList","Array","item","x","y","normalizeBboxInput","normalizedBbox","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACiBO,IAAKI,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;;WAAZA;;AASZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAI5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,QAAQ,MAAM;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAQ,MACVA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,iBACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;QACN,YAAnB,OAAOY,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;KAC1B;IACD,OAAOgB;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEda,IAAAA,sBAAAA,MAAAA,AAAAA,EACEd,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BO,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;YAC5CW,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACX,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEJ,SAAS,IAAI,CAACD,OAAOG;IAEzB;SAEAF,WAAWV;IAGb,IAAIU,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLL,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAIH,IACEgB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLL,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;QAEhEiB,KAAK,GAAG,CACNZ,OACAY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACNX,QACAW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIY,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEA,SAASc,mBACPf,IAAoB;IAEpB,IAAIW,MAAM,OAAO,CAACX,OAChB;QAAA,IAAIW,MAAM,OAAO,CAACX,IAAI,CAAC,EAAE,GACvB,OAAOA,IAAI,CAAC,EAAE;IAChB;IAGF,OAAOA;AACT;AAEO,SAASF,UACdE,IAAoB,EACpBP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,MAAMmB,iBAAiBD,mBAAmBf;IAE1C,IAAII,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBS,gBAAgBU,gBAAgBvB,OAAOC,UACvCG,AAAW,aAAXA,SACAoB,gBAAgBD,gBAA4BvB,OAAOC,UACnDG,AAAW,eAAXA,SACAqB,gBAAgBF,gBAA4BvB,OAAOC,UAEnDK,iBAAiBiB;IAG5BZ,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASc,gBACdlB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASuB,gBACdjB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMyB,OAAOd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAM2B,MAAMf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAM2B,QAAQhB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM6B,SAASjB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACyB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdvB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd8B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX9B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjCR,kBACE,mBACAW,MACAP,OACAC,QACA,UACA8B,SACAC,SACA,SACA9B,YACAC,aACA,UACAC;IAEF,MAAM,CAACsB,MAAMC,KAAKC,OAAOC,OAAO,GAAGxB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM6B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYnC,OACzBmC,YAAYnC,QAAQiC;IAItB,IAAIC,UAAUE,aAAanC,QACzBmC,aAAanC,SAASiC;IAIxBC,YAAYvB,KAAK,GAAG,CAAC,GAAGuB;IACxBC,aAAaxB,KAAK,GAAG,CAAC,GAAGwB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAxC,kBAAkB,4BAA4ByC;IAE9C,OAAOA;AACT;AAEO,SAASC,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdR,IAAU,EACVS,UAAgB,EAChB1C,MAAgC;IAEhC,MAAM2C,cAAc;IACpB,MAAMC,iBAAiB;IAGvB,MAAMC,wBACJZ,KAAK,KAAK,GAAGU,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,KAAI,IAAK,KACvCW;IACN,MAAME,sBACJb,KAAK,MAAM,GAAGU,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,MAAK,IAAK,KACxCW;IAGN,IAAIG,WAAWvC,KAAK,GAAG,CAACmC,aAAaV,KAAK,KAAK,GAAGY,AAAwB,IAAxBA;IAClD,IAAIG,YAAYxC,KAAK,GAAG,CAACmC,aAAaV,KAAK,MAAM,GAAGa,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUhB,KAAK,IAAI,GAAGY;IAC1B,IAAIK,SAASjB,KAAK,GAAG,GAAGa;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAUzC,KAAK,GAAG,CAAC,GAAGyC;IACtBC,SAAS1C,KAAK,GAAG,CAAC,GAAG0C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCjB,KAAK,IAAI,GAAGgB;IACZhB,KAAK,GAAG,GAAGiB;IACXjB,KAAK,KAAK,GAAGc;IACbd,KAAK,MAAM,GAAGe;IAEd,OAAOf;AACT;AAEO,eAAekB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,0BAAAA,QAAAA,CAAAA,IAAa,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMK,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAN,KAAK,IAAI,CAACQ;IACZ;IAEA,IAAIT,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMS,cAAcC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClC,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;IACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;AACf;AAEO,MAAMC,aAAaD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjC,OAAOA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAChB,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,QAAQA,6BAAAA,CAAAA,CAAAA,KACA,CACJA,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;QACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;QACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,IAED,QAAQ;IACX,yBAAyBA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;IACvCA,6BAAAA,CAAAA,CAAAA,MAAQ;IACRA,6BAAAA,CAAAA,CAAAA,MACS,CAAC;QACN,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,6BAAAA,CAAAA,CAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,OAAS;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,6BAAAA,CAAAA,CAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,6BAAAA,CAAAA,CAAAA,OAAS,CAAC;IACjC,QAAQI;IAGR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAG3B,QAAQA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACH;IACvC3E,IAAAA,sBAAAA,MAAAA,AAAAA,EACE0E,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOI,OAAOJ;AAChB;AAEO,MAAMK,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAO5G,OAAO,IAAI,CAAC2G,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAAC7G;YAClB,MAAMoG,QAAQQ,UAAU,KAAK,CAAC5G,IAAI;YAClC,IAAI,CAACmG,uBAAuBC,QAC1B,OAAO;YAIT,IAAIO,cACF,OAAOP,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMZ,kBAAkB,CAC7BsB,YACAC;IAEA,MAAMC,gBAAgBP,4BAA4BM;IAClD,MAAMzF,SAAS;QAAE,GAAGwF,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAa5F,MAAM,CAAC2F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT5F,MAAM,CAAC2F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B5F,MAAM,CAAC2F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB5F,MAAM,CAAC2F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO5F;AACT;AAEO,MAAM6F,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBP,4BAA4BM;IAClD,MAAMzF,SAAS;QAAE,GAAGwF,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAa5F,MAAM,CAAC2F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB5F,MAAM,CAAC2F,UAAU,GAAG;YAClB,CAACjB,oBAAoB,EAAE;YACvB,QAAQkB;QACV;IAEJ;IAEA,OAAO5F;AACT;AAUO,MAAM8F,mBAAmB,CAC9BC,UACAN;IAGA,IAAI,CAACA,WACH;IAIF,MAAMO,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAed,4BAA4BM;IAGjD,IAAIQ,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOR,UAAU,KAAK,CAACO;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMzH,OAAOsH,MAChB,IAAIC,aAAa,QAAQ,CAACvH,MAExByH,mBAAmB,CAACzH,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/CyH,mBAAmB,CAACzH,IAAI,GAAGsH,KAAK,CAACtH,IAAI;IAKzC,MAAM0H,YAAYX,UAAU,KAAK,CAACU;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT"}

package/dist/lib/device/device-options.js

@@ -0,0 +1,20 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+for(var __rspack_i in __webpack_exports__)exports[__rspack_i] = __webpack_exports__[__rspack_i];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=device-options.js.map

package/dist/lib/device/device-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"device/device-options.js","sources":["webpack/runtime/make_namespace_object"],"sourcesContent":["// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","Symbol","Object"],"mappings":";;;IACAA,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOC,UAA0BA,OAAO,WAAW,EACrDC,OAAO,cAAc,CAAC,UAASD,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEC,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/device/index.js

@@ -24,38 +24,41 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    defineActionSwipe: ()=>defineActionSwipe,
-    actionRightClickParamSchema: ()=>actionRightClickParamSchema,
     ActionSwipeParamSchema: ()=>ActionSwipeParamSchema,
-    actionDoubleClickParamSchema: ()=>actionDoubleClickParamSchema,
-    defineAction: ()=>defineAction,
-    AbstractInterface: ()=>AbstractInterface,
-    defineActionLongPress: ()=>defineActionLongPress,
+    defineActionKeyboardPress: ()=>defineActionKeyboardPress,
+    defineActionRightClick: ()=>defineActionRightClick,
+    defineActionScroll: ()=>defineActionScroll,
+    ActionLongPressParamSchema: ()=>ActionLongPressParamSchema,
+    defineActionClearInput: ()=>defineActionClearInput,
+    actionScrollParamSchema: ()=>actionScrollParamSchema,
     defineActionTap: ()=>defineActionTap,
     defineActionDragAndDrop: ()=>defineActionDragAndDrop,
+    defineActionLongPress: ()=>defineActionLongPress,
+    actionRightClickParamSchema: ()=>actionRightClickParamSchema,
+    defineActionSwipe: ()=>defineActionSwipe,
+    actionDoubleClickParamSchema: ()=>actionDoubleClickParamSchema,
     actionTapParamSchema: ()=>actionTapParamSchema,
-    actionKeyboardPressParamSchema: ()=>actionKeyboardPressParamSchema,
+    defineActionDoubleClick: ()=>defineActionDoubleClick,
+    actionInputParamSchema: ()=>actionInputParamSchema,
     defineActionHover: ()=>defineActionHover,
-    actionScrollParamSchema: ()=>actionScrollParamSchema,
+    defineActionInput: ()=>defineActionInput,
+    defineAction: ()=>defineAction,
+    actionKeyboardPressParamSchema: ()=>actionKeyboardPressParamSchema,
+    actionAssertParamSchema: ()=>actionAssertParamSchema,
     actionDragAndDropParamSchema: ()=>actionDragAndDropParamSchema,
-    defineActionClearInput: ()=>defineActionClearInput,
-    defineActionRightClick: ()=>defineActionRightClick,
-    ActionLongPressParamSchema: ()=>ActionLongPressParamSchema,
+    AbstractInterface: ()=>AbstractInterface,
     actionClearInputParamSchema: ()=>actionClearInputParamSchema,
-    actionInputParamSchema: ()=>actionInputParamSchema,
-    defineActionDoubleClick: ()=>defineActionDoubleClick,
-    defineActionKeyboardPress: ()=>defineActionKeyboardPress,
-    defineActionScroll: ()=>defineActionScroll,
     actionHoverParamSchema: ()=>actionHoverParamSchema,
-    defineActionInput: ()=>defineActionInput
+    defineActionAssert: ()=>defineActionAssert
 });
-const index_js_namespaceObject = require("../ai-model/index.js");
+const external_common_js_namespaceObject = require("../common.js");
+const logger_namespaceObject = require("@midscene/shared/logger");
 const external_zod_namespaceObject = require("zod");
 class AbstractInterface {
 }
 const defineAction = (config)=>config;
 const actionTapParamSchema = external_zod_namespaceObject.z.object({
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be tapped')
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be tapped')
 });
 const defineActionTap = (call)=>defineAction({
         name: 'Tap',
@@ -65,7 +68,7 @@ const defineActionTap = (call)=>defineAction({
         call
     });
 const actionRightClickParamSchema = external_zod_namespaceObject.z.object({
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be right clicked')
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be right clicked')
 });
 const defineActionRightClick = (call)=>defineAction({
         name: 'RightClick',
@@ -75,7 +78,7 @@ const defineActionRightClick = (call)=>defineAction({
         call
     });
 const actionDoubleClickParamSchema = external_zod_namespaceObject.z.object({
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be double clicked')
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be double clicked')
 });
 const defineActionDoubleClick = (call)=>defineAction({
         name: 'DoubleClick',
@@ -85,7 +88,7 @@ const defineActionDoubleClick = (call)=>defineAction({
         call
     });
 const actionHoverParamSchema = external_zod_namespaceObject.z.object({
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be hovered')
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be hovered')
 });
 const defineActionHover = (call)=>defineAction({
         name: 'Hover',
@@ -94,12 +97,13 @@ const defineActionHover = (call)=>defineAction({
         paramSchema: actionHoverParamSchema,
         call
     });
+const inputLocateDescription = 'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';
 const actionInputParamSchema = external_zod_namespaceObject.z.object({
     value: external_zod_namespaceObject.z.union([
         external_zod_namespaceObject.z.string(),
         external_zod_namespaceObject.z.number()
     ]).transform((val)=>String(val)).describe('The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.'),
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be input').optional(),
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe(inputLocateDescription).optional(),
     mode: external_zod_namespaceObject.z["enum"]([
         'replace',
         'clear',
@@ -114,7 +118,7 @@ const defineActionInput = (call)=>defineAction({
         call
     });
 const actionKeyboardPressParamSchema = external_zod_namespaceObject.z.object({
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be clicked before pressing the key').optional(),
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be clicked before pressing the key').optional(),
     keyName: external_zod_namespaceObject.z.string().describe("The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'")
 });
 const defineActionKeyboardPress = (call)=>defineAction({
@@ -132,14 +136,14 @@ const actionScrollParamSchema = external_zod_namespaceObject.z.object({
         'left'
     ]).default('down').describe('The direction to scroll'),
     scrollType: external_zod_namespaceObject.z["enum"]([
-        'once',
-        'untilBottom',
-        'untilTop',
-        'untilRight',
-        'untilLeft'
-    ]).default('once').describe('The scroll type'),
+        'singleAction',
+        'scrollToBottom',
+        'scrollToTop',
+        'scrollToRight',
+        'scrollToLeft'
+    ]).default('singleAction').describe('The scroll behavior: "singleAction" for a single scroll action, "scrollToBottom" for scrolling to the bottom, "scrollToTop" for scrolling to the top, "scrollToRight" for scrolling to the right, "scrollToLeft" for scrolling to the left'),
     distance: external_zod_namespaceObject.z.number().nullable().optional().describe('The distance in pixels to scroll'),
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().optional().describe('The element to be scrolled')
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().optional().describe('The target element to be scrolled')
 });
 const defineActionScroll = (call)=>defineAction({
         name: 'Scroll',
@@ -149,18 +153,18 @@ const defineActionScroll = (call)=>defineAction({
         call
     });
 const actionDragAndDropParamSchema = external_zod_namespaceObject.z.object({
-    from: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The position to be dragged'),
-    to: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The position to be dropped')
+    from: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The position to be dragged'),
+    to: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The position to be dropped')
 });
 const defineActionDragAndDrop = (call)=>defineAction({
         name: 'DragAndDrop',
-        description: 'Drag and drop the element',
+        description: 'Drag and drop (hold the mouse or finger down and move the mouse) ',
         interfaceAlias: 'aiDragAndDrop',
         paramSchema: actionDragAndDropParamSchema,
         call
     });
 const ActionLongPressParamSchema = external_zod_namespaceObject.z.object({
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be long pressed'),
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The element to be long pressed'),
     duration: external_zod_namespaceObject.z.number().default(500).optional().describe('Long press duration in milliseconds')
 });
 const defineActionLongPress = (call)=>defineAction({
@@ -170,7 +174,7 @@ const defineActionLongPress = (call)=>defineAction({
         call
     });
 const ActionSwipeParamSchema = external_zod_namespaceObject.z.object({
-    start: (0, index_js_namespaceObject.getMidsceneLocationSchema)().optional().describe('Starting point of the swipe gesture, if not specified, the center of the page will be used'),
+    start: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().optional().describe('Starting point of the swipe gesture, if not specified, the center of the page will be used'),
     direction: external_zod_namespaceObject.z["enum"]([
         'up',
         'down',
@@ -178,7 +182,7 @@ const ActionSwipeParamSchema = external_zod_namespaceObject.z.object({
         'right'
     ]).optional().describe('The direction to swipe (required when using distance). The direction means the direction of the finger swipe.'),
     distance: external_zod_namespaceObject.z.number().optional().describe('The distance in pixels to swipe (mutually exclusive with end)'),
-    end: (0, index_js_namespaceObject.getMidsceneLocationSchema)().optional().describe('Ending point of the swipe gesture (mutually exclusive with distance)'),
+    end: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().optional().describe('Ending point of the swipe gesture (mutually exclusive with distance)'),
     duration: external_zod_namespaceObject.z.number().default(300).describe('Duration of the swipe gesture in milliseconds'),
     repeat: external_zod_namespaceObject.z.number().optional().describe('The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)')
 });
@@ -189,18 +193,34 @@ const defineActionSwipe = (call)=>defineAction({
         call
     });
 const actionClearInputParamSchema = external_zod_namespaceObject.z.object({
-    locate: (0, index_js_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be cleared')
+    locate: (0, external_common_js_namespaceObject.getMidsceneLocationSchema)().describe('The input field to be cleared')
 });
 const defineActionClearInput = (call)=>defineAction({
         name: 'ClearInput',
-        description: 'Clear the text content of an input field',
+        description: inputLocateDescription,
         interfaceAlias: 'aiClearInput',
         paramSchema: actionClearInputParamSchema,
         call
     });
+const actionAssertParamSchema = external_zod_namespaceObject.z.object({
+    condition: external_zod_namespaceObject.z.string().describe('The condition of the assertion'),
+    thought: external_zod_namespaceObject.z.string().describe('The thought of the assertion, like "I can see there are A, B, C elements on the page, which means ... , so the assertion is true"'),
+    result: external_zod_namespaceObject.z.boolean().describe('The result of the assertion, true or false')
+});
+const defineActionAssert = ()=>defineAction({
+        name: 'Print_Assert_Result',
+        description: 'Print the result of the assertion',
+        paramSchema: actionAssertParamSchema,
+        call: async (param)=>{
+            if ('boolean' != typeof param?.result) throw new Error(`The result of the assertion must be a boolean, but got: ${typeof param?.result}. ${param.thought || '(no thought)'}`);
+            (0, logger_namespaceObject.getDebug)('device:common-action')(`Assert: ${param.condition}, Thought: ${param.thought}, Result: ${param.result}`);
+            if (!param.result) throw new Error(`Assertion failed: ${param.thought || '(no thought)'} (Assertion = ${param.condition})`);
+        }
+    });
 exports.AbstractInterface = __webpack_exports__.AbstractInterface;
 exports.ActionLongPressParamSchema = __webpack_exports__.ActionLongPressParamSchema;
 exports.ActionSwipeParamSchema = __webpack_exports__.ActionSwipeParamSchema;
+exports.actionAssertParamSchema = __webpack_exports__.actionAssertParamSchema;
 exports.actionClearInputParamSchema = __webpack_exports__.actionClearInputParamSchema;
 exports.actionDoubleClickParamSchema = __webpack_exports__.actionDoubleClickParamSchema;
 exports.actionDragAndDropParamSchema = __webpack_exports__.actionDragAndDropParamSchema;
@@ -211,6 +231,7 @@ exports.actionRightClickParamSchema = __webpack_exports__.actionRightClickParamS
 exports.actionScrollParamSchema = __webpack_exports__.actionScrollParamSchema;
 exports.actionTapParamSchema = __webpack_exports__.actionTapParamSchema;
 exports.defineAction = __webpack_exports__.defineAction;
+exports.defineActionAssert = __webpack_exports__.defineActionAssert;
 exports.defineActionClearInput = __webpack_exports__.defineActionClearInput;
 exports.defineActionDoubleClick = __webpack_exports__.defineActionDoubleClick;
 exports.defineActionDragAndDrop = __webpack_exports__.defineActionDragAndDrop;
@@ -222,10 +243,11 @@ exports.defineActionRightClick = __webpack_exports__.defineActionRightClick;
 exports.defineActionScroll = __webpack_exports__.defineActionScroll;
 exports.defineActionSwipe = __webpack_exports__.defineActionSwipe;
 exports.defineActionTap = __webpack_exports__.defineActionTap;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "AbstractInterface",
     "ActionLongPressParamSchema",
     "ActionSwipeParamSchema",
+    "actionAssertParamSchema",
     "actionClearInputParamSchema",
     "actionDoubleClickParamSchema",
     "actionDragAndDropParamSchema",
@@ -236,6 +258,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "actionScrollParamSchema",
     "actionTapParamSchema",
     "defineAction",
+    "defineActionAssert",
     "defineActionClearInput",
     "defineActionDoubleClick",
     "defineActionDragAndDrop",
@@ -247,7 +270,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "defineActionScroll",
     "defineActionSwipe",
     "defineActionTap"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/device/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"device/index.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/device/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getMidsceneLocationSchema } from '@/ai-model';\nimport type { DeviceAction, LocateResultElement } from '@/types';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[] | Promise<DeviceAction[]>;\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    opt?: { _orderSensitive: boolean },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\nexport const defineAction = <\n  TSchema extends z.ZodType,\n  TRuntime = z.infer<TSchema>,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema: TSchema;\n    call: (param: TRuntime) => Promise<void>;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be input')\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum(['once', 'untilBottom', 'untilTop', 'untilRight', 'untilLeft'])\n    .default('once')\n    .describe('The scroll type'),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The element to be scrolled'),\n});\nexport type ActionScrollParam = {\n  direction?: 'down' | 'up' | 'right' | 'left';\n  scrollType?: 'once' | 'untilBottom' | 'untilTop' | 'untilRight' | 'untilLeft';\n  distance?: number | null;\n  locate?: LocateResultElement;\n};\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description: 'Drag and drop the element',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: 'Clear the text content of an input field',\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\nexport type { DeviceAction } from '../types';\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACCO,MAAeI;AAgCtB;AAIO,MAAMC,eAAe,CAI1BC,SAaOA;AAIF,MAAMC,uBAAuBC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC3C,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIK,MAAMO,yBAAyBV,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOA,6BAAAA,CAAAA,CAAAA,KACC,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG,EAC9B,SAAS,CAAC,CAACW,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQV,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,2BACT,QAAQ;IACX,MAAMD,6BAAAA,CAAAA,CAAAA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMa,oBAAoB,CAC/BV,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaa;QACbP;IACF;AAIK,MAAMW,iCAAiCd,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACrD,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,6BAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMe,4BAA4B,CACvCZ,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaiB;QACbX;IACF;AAIK,MAAMa,0BAA0BhB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,6BAAAA,CAAAA,CAAAA,OACL,CAAC;QAAC;QAAQ;QAAe;QAAY;QAAc;KAAY,EACnE,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAQO,MAAMgB,qBAAqB,CAChCd,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAamB;QACbb;IACF;AAIK,MAAMe,+BAA+BlB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,MAAMC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;IAC3C,IAAIA,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMkB,0BAA0B,CACrChB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaqB;QACbf;IACF;AAGK,MAAMiB,6BAA6BpB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjD,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;IAEF,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMqB,wBAAwB,CACnClB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAauB;QACbjB;IACF;AAGK,MAAMmB,yBAAyBtB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMuB,oBAAoB,CAC/BpB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAayB;QACbnB;IACF;AAIK,MAAMqB,8BAA8BxB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,yBAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMwB,yBAAyB,CACpCtB,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAa2B;QACbrB;IACF"}
+{"version":3,"file":"device/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object","../../../src/device/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { getMidsceneLocationSchema } from '@/common';\nimport type {\n  ActionScrollParam,\n  DeviceAction,\n  LocateResultElement,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ElementNode } from '@midscene/shared/extractor';\nimport { getDebug } from '@midscene/shared/logger';\nimport { _keyDefinitions } from '@midscene/shared/us-keyboard-layout';\nimport { z } from 'zod';\nimport type { ElementCacheFeature, Rect, Size, UIContext } from '../types';\n\nexport abstract class AbstractInterface {\n  abstract interfaceType: string;\n\n  abstract screenshotBase64(): Promise<string>;\n  abstract size(): Promise<Size>;\n  abstract actionSpace(): DeviceAction[];\n\n  abstract cacheFeatureForRect?(\n    rect: Rect,\n    options?: {\n      targetDescription?: string;\n      modelConfig?: IModelConfig;\n    },\n  ): Promise<ElementCacheFeature>;\n  abstract rectMatchesCacheFeature?(\n    feature: ElementCacheFeature,\n  ): Promise<Rect>;\n\n  abstract destroy?(): Promise<void>;\n\n  abstract describe?(): string;\n  abstract beforeInvokeAction?(actionName: string, param: any): Promise<void>;\n  abstract afterInvokeAction?(actionName: string, param: any): Promise<void>;\n\n  // @deprecated do NOT extend this method\n  abstract getElementsNodeTree?: () => Promise<ElementNode>;\n\n  // @deprecated do NOT extend this method\n  abstract url?: () => string | Promise<string>;\n\n  // @deprecated do NOT extend this method\n  abstract evaluateJavaScript?<T = any>(script: string): Promise<T>;\n\n  // @deprecated do NOT extend this method\n  abstract getContext?(): Promise<UIContext>;\n}\n\n// Generic function to define actions with proper type inference\n// TRuntime allows specifying a different type for the runtime parameter (after location resolution)\n// TReturn allows specifying the return type of the action\nexport const defineAction = <\n  TSchema extends z.ZodType | undefined = undefined,\n  TRuntime = TSchema extends z.ZodType ? z.infer<TSchema> : undefined,\n  TReturn = any,\n>(\n  config: {\n    name: string;\n    description: string;\n    interfaceAlias?: string;\n    paramSchema?: TSchema;\n    call: (param: TRuntime) => Promise<TReturn> | TReturn;\n  } & Partial<\n    Omit<\n      DeviceAction<TRuntime, TReturn>,\n      'name' | 'description' | 'interfaceAlias' | 'paramSchema' | 'call'\n    >\n  >,\n): DeviceAction<TRuntime, TReturn> => {\n  return config as any; // Type assertion needed because schema validation type differs from runtime type\n};\n\n// Tap\nexport const actionTapParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be tapped'),\n});\n// Override the inferred type to use LocateResultElement for the runtime locate field\nexport type ActionTapParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionTap = (\n  call: (param: ActionTapParam) => Promise<void>,\n): DeviceAction<ActionTapParam> => {\n  return defineAction<typeof actionTapParamSchema, ActionTapParam>({\n    name: 'Tap',\n    description: 'Tap the element',\n    interfaceAlias: 'aiTap',\n    paramSchema: actionTapParamSchema,\n    call,\n  });\n};\n\n// RightClick\nexport const actionRightClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be right clicked',\n  ),\n});\nexport type ActionRightClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionRightClick = (\n  call: (param: ActionRightClickParam) => Promise<void>,\n): DeviceAction<ActionRightClickParam> => {\n  return defineAction<\n    typeof actionRightClickParamSchema,\n    ActionRightClickParam\n  >({\n    name: 'RightClick',\n    description: 'Right click the element',\n    interfaceAlias: 'aiRightClick',\n    paramSchema: actionRightClickParamSchema,\n    call,\n  });\n};\n\n// DoubleClick\nexport const actionDoubleClickParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be double clicked',\n  ),\n});\nexport type ActionDoubleClickParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionDoubleClick = (\n  call: (param: ActionDoubleClickParam) => Promise<void>,\n): DeviceAction<ActionDoubleClickParam> => {\n  return defineAction<\n    typeof actionDoubleClickParamSchema,\n    ActionDoubleClickParam\n  >({\n    name: 'DoubleClick',\n    description: 'Double click the element',\n    interfaceAlias: 'aiDoubleClick',\n    paramSchema: actionDoubleClickParamSchema,\n    call,\n  });\n};\n\n// Hover\nexport const actionHoverParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The element to be hovered'),\n});\nexport type ActionHoverParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionHover = (\n  call: (param: ActionHoverParam) => Promise<void>,\n): DeviceAction<ActionHoverParam> => {\n  return defineAction<typeof actionHoverParamSchema, ActionHoverParam>({\n    name: 'Hover',\n    description: 'Move the mouse to the element',\n    interfaceAlias: 'aiHover',\n    paramSchema: actionHoverParamSchema,\n    call,\n  });\n};\n\n// Input\nconst inputLocateDescription =\n  'the position of the placeholder or text content in the target input field. If there is no content, locate the center of the input field.';\nexport const actionInputParamSchema = z.object({\n  value: z\n    .union([z.string(), z.number()])\n    .transform((val) => String(val))\n    .describe(\n      'The text to input. Provide the final content for replace/append modes, or an empty string when using clear mode to remove existing text.',\n    ),\n  locate: getMidsceneLocationSchema()\n    .describe(inputLocateDescription)\n    .optional(),\n  mode: z\n    .enum(['replace', 'clear', 'append'])\n    .default('replace')\n    .optional()\n    .describe(\n      'Input mode: \"replace\" (default) - clear the field and input the value; \"append\" - append the value to existing content; \"clear\" - clear the field without inputting new text.',\n    ),\n});\nexport type ActionInputParam = {\n  value: string;\n  locate?: LocateResultElement;\n  mode?: 'replace' | 'clear' | 'append';\n};\n\nexport const defineActionInput = (\n  call: (param: ActionInputParam) => Promise<void>,\n): DeviceAction<ActionInputParam> => {\n  return defineAction<typeof actionInputParamSchema, ActionInputParam>({\n    name: 'Input',\n    description: 'Input the value into the element',\n    interfaceAlias: 'aiInput',\n    paramSchema: actionInputParamSchema,\n    call,\n  });\n};\n\n// KeyboardPress\nexport const actionKeyboardPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema()\n    .describe('The element to be clicked before pressing the key')\n    .optional(),\n  keyName: z\n    .string()\n    .describe(\n      \"The key to be pressed. Use '+' for key combinations, e.g., 'Control+A', 'Shift+Enter'\",\n    ),\n});\nexport type ActionKeyboardPressParam = {\n  locate?: LocateResultElement;\n  keyName: string;\n};\n\nexport const defineActionKeyboardPress = (\n  call: (param: ActionKeyboardPressParam) => Promise<void>,\n): DeviceAction<ActionKeyboardPressParam> => {\n  return defineAction<\n    typeof actionKeyboardPressParamSchema,\n    ActionKeyboardPressParam\n  >({\n    name: 'KeyboardPress',\n    description:\n      'Press a key or key combination, like \"Enter\", \"Tab\", \"Escape\", or \"Control+A\", \"Shift+Enter\". Do not use this to type text.',\n    interfaceAlias: 'aiKeyboardPress',\n    paramSchema: actionKeyboardPressParamSchema,\n    call,\n  });\n};\n\n// Scroll\nexport const actionScrollParamSchema = z.object({\n  direction: z\n    .enum(['down', 'up', 'right', 'left'])\n    .default('down')\n    .describe('The direction to scroll'),\n  scrollType: z\n    .enum([\n      'singleAction',\n      'scrollToBottom',\n      'scrollToTop',\n      'scrollToRight',\n      'scrollToLeft',\n    ])\n    .default('singleAction')\n    .describe(\n      'The scroll behavior: \"singleAction\" for a single scroll action, \"scrollToBottom\" for scrolling to the bottom, \"scrollToTop\" for scrolling to the top, \"scrollToRight\" for scrolling to the right, \"scrollToLeft\" for scrolling to the left',\n    ),\n  distance: z\n    .number()\n    .nullable()\n    .optional()\n    .describe('The distance in pixels to scroll'),\n  locate: getMidsceneLocationSchema()\n    .optional()\n    .describe('The target element to be scrolled'),\n});\n\nexport const defineActionScroll = (\n  call: (param: ActionScrollParam) => Promise<void>,\n): DeviceAction<ActionScrollParam> => {\n  return defineAction<typeof actionScrollParamSchema, ActionScrollParam>({\n    name: 'Scroll',\n    description:\n      'Scroll the page or an element. The direction to scroll, the scroll type, and the distance to scroll. The distance is the number of pixels to scroll. If not specified, use `down` direction, `once` scroll type, and `null` distance.',\n    interfaceAlias: 'aiScroll',\n    paramSchema: actionScrollParamSchema,\n    call,\n  });\n};\n\n// DragAndDrop\nexport const actionDragAndDropParamSchema = z.object({\n  from: getMidsceneLocationSchema().describe('The position to be dragged'),\n  to: getMidsceneLocationSchema().describe('The position to be dropped'),\n});\nexport type ActionDragAndDropParam = {\n  from: LocateResultElement;\n  to: LocateResultElement;\n};\n\nexport const defineActionDragAndDrop = (\n  call: (param: ActionDragAndDropParam) => Promise<void>,\n): DeviceAction<ActionDragAndDropParam> => {\n  return defineAction<\n    typeof actionDragAndDropParamSchema,\n    ActionDragAndDropParam\n  >({\n    name: 'DragAndDrop',\n    description:\n      'Drag and drop (hold the mouse or finger down and move the mouse) ',\n    interfaceAlias: 'aiDragAndDrop',\n    paramSchema: actionDragAndDropParamSchema,\n    call,\n  });\n};\n\nexport const ActionLongPressParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe(\n    'The element to be long pressed',\n  ),\n  duration: z\n    .number()\n    .default(500)\n    .optional()\n    .describe('Long press duration in milliseconds'),\n});\n\nexport type ActionLongPressParam = {\n  locate: LocateResultElement;\n  duration?: number;\n};\nexport const defineActionLongPress = (\n  call: (param: ActionLongPressParam) => Promise<void>,\n): DeviceAction<ActionLongPressParam> => {\n  return defineAction<typeof ActionLongPressParamSchema, ActionLongPressParam>({\n    name: 'LongPress',\n    description: 'Long press the element',\n    paramSchema: ActionLongPressParamSchema,\n    call,\n  });\n};\n\nexport const ActionSwipeParamSchema = z.object({\n  start: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Starting point of the swipe gesture, if not specified, the center of the page will be used',\n    ),\n  direction: z\n    .enum(['up', 'down', 'left', 'right'])\n    .optional()\n    .describe(\n      'The direction to swipe (required when using distance). The direction means the direction of the finger swipe.',\n    ),\n  distance: z\n    .number()\n    .optional()\n    .describe('The distance in pixels to swipe (mutually exclusive with end)'),\n  end: getMidsceneLocationSchema()\n    .optional()\n    .describe(\n      'Ending point of the swipe gesture (mutually exclusive with distance)',\n    ),\n  duration: z\n    .number()\n    .default(300)\n    .describe('Duration of the swipe gesture in milliseconds'),\n  repeat: z\n    .number()\n    .optional()\n    .describe(\n      'The number of times to repeat the swipe gesture. 1 for default, 0 for infinite (e.g. endless swipe until the end of the page)',\n    ),\n});\n\nexport type ActionSwipeParam = {\n  start?: LocateResultElement;\n  direction?: 'up' | 'down' | 'left' | 'right';\n  distance?: number;\n  end?: LocateResultElement;\n  duration?: number;\n  repeat?: number;\n};\n\nexport const defineActionSwipe = (\n  call: (param: ActionSwipeParam) => Promise<void>,\n): DeviceAction<ActionSwipeParam> => {\n  return defineAction<typeof ActionSwipeParamSchema, ActionSwipeParam>({\n    name: 'Swipe',\n    description:\n      'Perform a swipe gesture. You must specify either \"end\" (target location) or \"distance\" + \"direction\" - they are mutually exclusive. Use \"end\" for precise location-based swipes, or \"distance\" + \"direction\" for relative movement.',\n    paramSchema: ActionSwipeParamSchema,\n    call,\n  });\n};\n\n// ClearInput\nexport const actionClearInputParamSchema = z.object({\n  locate: getMidsceneLocationSchema().describe('The input field to be cleared'),\n});\nexport type ActionClearInputParam = {\n  locate: LocateResultElement;\n};\n\nexport const defineActionClearInput = (\n  call: (param: ActionClearInputParam) => Promise<void>,\n): DeviceAction<ActionClearInputParam> => {\n  return defineAction<\n    typeof actionClearInputParamSchema,\n    ActionClearInputParam\n  >({\n    name: 'ClearInput',\n    description: inputLocateDescription,\n    interfaceAlias: 'aiClearInput',\n    paramSchema: actionClearInputParamSchema,\n    call,\n  });\n};\n\n// Assert\nexport const actionAssertParamSchema = z.object({\n  condition: z.string().describe('The condition of the assertion'),\n  thought: z\n    .string()\n    .describe(\n      'The thought of the assertion, like \"I can see there are A, B, C elements on the page, which means ... , so the assertion is true\"',\n    ),\n  result: z.boolean().describe('The result of the assertion, true or false'),\n});\nexport type ActionAssertParam = {\n  condition: string;\n  thought: string;\n  result: boolean;\n};\n\nexport const defineActionAssert = (): DeviceAction<ActionAssertParam> => {\n  return defineAction<typeof actionAssertParamSchema, ActionAssertParam>({\n    name: 'Print_Assert_Result',\n    description: 'Print the result of the assertion',\n    paramSchema: actionAssertParamSchema,\n    call: async (param) => {\n      if (typeof param?.result !== 'boolean') {\n        throw new Error(\n          `The result of the assertion must be a boolean, but got: ${typeof param?.result}. ${param.thought || '(no thought)'}`,\n        );\n      }\n\n      getDebug('device:common-action')(\n        `Assert: ${param.condition}, Thought: ${param.thought}, Result: ${param.result}`,\n      );\n\n      if (!param.result) {\n        throw new Error(\n          `Assertion failed: ${param.thought || '(no thought)'} (Assertion = ${param.condition})`,\n        );\n      }\n    },\n  });\n};\n\nexport type { DeviceAction } from '../types';\nexport type {\n  AndroidDeviceOpt,\n  AndroidDeviceInputOpt,\n  IOSDeviceOpt,\n  IOSDeviceInputOpt,\n} from './device-options';\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AbstractInterface","defineAction","config","actionTapParamSchema","z","getMidsceneLocationSchema","defineActionTap","call","actionRightClickParamSchema","defineActionRightClick","actionDoubleClickParamSchema","defineActionDoubleClick","actionHoverParamSchema","defineActionHover","inputLocateDescription","actionInputParamSchema","val","String","defineActionInput","actionKeyboardPressParamSchema","defineActionKeyboardPress","actionScrollParamSchema","defineActionScroll","actionDragAndDropParamSchema","defineActionDragAndDrop","ActionLongPressParamSchema","defineActionLongPress","ActionSwipeParamSchema","defineActionSwipe","actionClearInputParamSchema","defineActionClearInput","actionAssertParamSchema","defineActionAssert","param","Error","getDebug"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACOO,MAAeI;AAmCtB;AAKO,MAAMC,eAAe,CAK1BC,SAaOA;AAIF,MAAMC,uBAAuBC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC3C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAMO,MAAMC,kBAAkB,CAC7BC,OAEON,aAA0D;QAC/D,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaE;QACbI;IACF;AAIK,MAAMC,8BAA8BJ,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMI,yBAAyB,CACpCF,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaO;QACbD;IACF;AAIK,MAAMG,+BAA+BN,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;AAEJ;AAKO,MAAMM,0BAA0B,CACrCJ,OAEON,aAGL;QACA,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaS;QACbH;IACF;AAIK,MAAMK,yBAAyBR,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMQ,oBAAoB,CAC/BN,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAaW;QACbL;IACF;AAIF,MAAMO,yBACJ;AACK,MAAMC,yBAAyBX,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOA,6BAAAA,CAAAA,CAAAA,KACC,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG,EAC9B,SAAS,CAAC,CAACY,MAAQC,OAAOD,MAC1B,QAAQ,CACP;IAEJ,QAAQX,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAACS,wBACT,QAAQ;IACX,MAAMV,6BAAAA,CAAAA,CAAAA,OACC,CAAC;QAAC;QAAW;QAAS;KAAS,EACnC,OAAO,CAAC,WACR,QAAQ,GACR,QAAQ,CACP;AAEN;AAOO,MAAMc,oBAAoB,CAC/BX,OAEON,aAA8D;QACnE,MAAM;QACN,aAAa;QACb,gBAAgB;QAChB,aAAac;QACbR;IACF;AAIK,MAAMY,iCAAiCf,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACrD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,CAAC,qDACT,QAAQ;IACX,SAASD,6BAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,CACP;AAEN;AAMO,MAAMgB,4BAA4B,CACvCb,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAakB;QACbZ;IACF;AAIK,MAAMc,0BAA0BjB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,WAAWA,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAQ;QAAM;QAAS;KAAO,EACpC,OAAO,CAAC,QACR,QAAQ,CAAC;IACZ,YAAYA,6BAAAA,CAAAA,CAAAA,OACL,CAAC;QACJ;QACA;QACA;QACA;QACA;KACD,EACA,OAAO,CAAC,gBACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,CAAC;IACZ,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACL,QAAQ,GACR,QAAQ,CAAC;AACd;AAEO,MAAMiB,qBAAqB,CAChCf,OAEON,aAAgE;QACrE,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAaoB;QACbd;IACF;AAIK,MAAMgB,+BAA+BnB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACnD,MAAMC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;IAC3C,IAAIA,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC3C;AAMO,MAAMmB,0BAA0B,CACrCjB,OAEON,aAGL;QACA,MAAM;QACN,aACE;QACF,gBAAgB;QAChB,aAAasB;QACbhB;IACF;AAGK,MAAMkB,6BAA6BrB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAC1C;IAEF,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,GACR,QAAQ,CAAC;AACd;AAMO,MAAMsB,wBAAwB,CACnCnB,OAEON,aAAsE;QAC3E,MAAM;QACN,aAAa;QACb,aAAawB;QACblB;IACF;AAGK,MAAMoB,yBAAyBvB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC7C,OAAOC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACJ,QAAQ,GACR,QAAQ,CACP;IAEJ,WAAWD,6BAAAA,CAAAA,CAAAA,OACJ,CAAC;QAAC;QAAM;QAAQ;QAAQ;KAAQ,EACpC,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUA,6BAAAA,CAAAA,CAAAA,MACD,GACN,QAAQ,GACR,QAAQ,CAAC;IACZ,KAAKC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IACF,QAAQ,GACR,QAAQ,CACP;IAEJ,UAAUD,6BAAAA,CAAAA,CAAAA,MACD,GACN,OAAO,CAAC,KACR,QAAQ,CAAC;IACZ,QAAQA,6BAAAA,CAAAA,CAAAA,MACC,GACN,QAAQ,GACR,QAAQ,CACP;AAEN;AAWO,MAAMwB,oBAAoB,CAC/BrB,OAEON,aAA8D;QACnE,MAAM;QACN,aACE;QACF,aAAa0B;QACbpB;IACF;AAIK,MAAMsB,8BAA8BzB,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClD,QAAQC,AAAAA,IAAAA,mCAAAA,yBAAAA,AAAAA,IAA4B,QAAQ,CAAC;AAC/C;AAKO,MAAMyB,yBAAyB,CACpCvB,OAEON,aAGL;QACA,MAAM;QACN,aAAaa;QACb,gBAAgB;QAChB,aAAae;QACbtB;IACF;AAIK,MAAMwB,0BAA0B3B,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,WAAWA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ,CAAC;IAC/B,SAASA,6BAAAA,CAAAA,CAAAA,MACA,GACN,QAAQ,CACP;IAEJ,QAAQA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ,CAAC;AAC/B;AAOO,MAAM4B,qBAAqB,IACzB/B,aAAgE;QACrE,MAAM;QACN,aAAa;QACb,aAAa8B;QACb,MAAM,OAAOE;YACX,IAAI,AAAyB,aAAzB,OAAOA,OAAO,QAChB,MAAM,IAAIC,MACR,CAAC,wDAAwD,EAAE,OAAOD,OAAO,OAAO,EAAE,EAAEA,MAAM,OAAO,IAAI,gBAAgB;YAIzHE,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS,wBACP,CAAC,QAAQ,EAAEF,MAAM,SAAS,CAAC,WAAW,EAAEA,MAAM,OAAO,CAAC,UAAU,EAAEA,MAAM,MAAM,EAAE;YAGlF,IAAI,CAACA,MAAM,MAAM,EACf,MAAM,IAAIC,MACR,CAAC,kBAAkB,EAAED,MAAM,OAAO,IAAI,eAAe,cAAc,EAAEA,MAAM,SAAS,CAAC,CAAC,CAAC;QAG7F;IACF"}

package/dist/lib/image/index.js

@@ -24,13 +24,13 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    saveBase64Image: ()=>img_namespaceObject.saveBase64Image,
-    httpImg2Base64: ()=>img_namespaceObject.httpImg2Base64,
+    resizeAndConvertImgBuffer: ()=>img_namespaceObject.resizeAndConvertImgBuffer,
     localImg2Base64: ()=>img_namespaceObject.localImg2Base64,
+    saveBase64Image: ()=>img_namespaceObject.saveBase64Image,
     zoomForGPT4o: ()=>img_namespaceObject.zoomForGPT4o,
     imageInfoOfBase64: ()=>img_namespaceObject.imageInfoOfBase64,
     imageInfo: ()=>img_namespaceObject.imageInfo,
-    resizeAndConvertImgBuffer: ()=>img_namespaceObject.resizeAndConvertImgBuffer
+    httpImg2Base64: ()=>img_namespaceObject.httpImg2Base64
 });
 const img_namespaceObject = require("@midscene/shared/img");
 exports.httpImg2Base64 = __webpack_exports__.httpImg2Base64;
@@ -40,7 +40,7 @@ exports.localImg2Base64 = __webpack_exports__.localImg2Base64;
 exports.resizeAndConvertImgBuffer = __webpack_exports__.resizeAndConvertImgBuffer;
 exports.saveBase64Image = __webpack_exports__.saveBase64Image;
 exports.zoomForGPT4o = __webpack_exports__.zoomForGPT4o;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "httpImg2Base64",
     "imageInfo",
     "imageInfoOfBase64",
@@ -48,7 +48,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "resizeAndConvertImgBuffer",
     "saveBase64Image",
     "zoomForGPT4o"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });

package/dist/lib/image/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"image/index.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}
+{"version":3,"file":"image/index.js","sources":["webpack/runtime/define_property_getters","webpack/runtime/has_own_property","webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/index.js

@@ -33,43 +33,46 @@ var __webpack_require__ = {};
 var __webpack_exports__ = {};
 __webpack_require__.r(__webpack_exports__);
 __webpack_require__.d(__webpack_exports__, {
-    MIDSCENE_MODEL_NAME: ()=>env_namespaceObject.MIDSCENE_MODEL_NAME,
-    SizeSchema: ()=>external_ai_model_index_js_namespaceObject.SizeSchema,
-    getVersion: ()=>external_utils_js_namespaceObject.getVersion,
-    TMultimodalPromptSchema: ()=>external_ai_model_index_js_namespaceObject.TMultimodalPromptSchema,
     plan: ()=>external_ai_model_index_js_namespaceObject.plan,
+    Service: ()=>index_js_default(),
+    MIDSCENE_MODEL_NAME: ()=>env_namespaceObject.MIDSCENE_MODEL_NAME,
     TUserPromptSchema: ()=>external_ai_model_index_js_namespaceObject.TUserPromptSchema,
-    z: ()=>external_zod_namespaceObject.z,
+    describeUserPage: ()=>external_ai_model_index_js_namespaceObject.describeUserPage,
+    AiLocateElement: ()=>external_ai_model_index_js_namespaceObject.AiLocateElement,
     createAgent: ()=>external_agent_index_js_namespaceObject.createAgent,
-    RectSchema: ()=>external_ai_model_index_js_namespaceObject.RectSchema,
-    PointSchema: ()=>external_ai_model_index_js_namespaceObject.PointSchema,
+    z: ()=>external_zod_namespaceObject.z,
+    SizeSchema: ()=>external_ai_model_index_js_namespaceObject.SizeSchema,
+    TaskRunner: ()=>external_task_runner_js_namespaceObject.TaskRunner,
     default: ()=>src,
-    AiLocateElement: ()=>external_ai_model_index_js_namespaceObject.AiLocateElement,
-    Insight: ()=>index_js_default(),
-    describeUserPage: ()=>external_ai_model_index_js_namespaceObject.describeUserPage,
-    Executor: ()=>action_executor_js_namespaceObject.Executor,
+    getVersion: ()=>external_utils_js_namespaceObject.getVersion,
     getMidsceneLocationSchema: ()=>external_ai_model_index_js_namespaceObject.getMidsceneLocationSchema,
-    Agent: ()=>external_agent_index_js_namespaceObject.Agent
+    ServiceError: ()=>external_types_js_namespaceObject.ServiceError,
+    TMultimodalPromptSchema: ()=>external_ai_model_index_js_namespaceObject.TMultimodalPromptSchema,
+    PointSchema: ()=>external_ai_model_index_js_namespaceObject.PointSchema,
+    Agent: ()=>external_agent_index_js_namespaceObject.Agent,
+    RectSchema: ()=>external_ai_model_index_js_namespaceObject.RectSchema
 });
 const external_zod_namespaceObject = require("zod");
-const action_executor_js_namespaceObject = require("./ai-model/action-executor.js");
-const index_js_namespaceObject = require("./insight/index.js");
+const index_js_namespaceObject = require("./service/index.js");
 var index_js_default = /*#__PURE__*/ __webpack_require__.n(index_js_namespaceObject);
+const external_task_runner_js_namespaceObject = require("./task-runner.js");
 const external_utils_js_namespaceObject = require("./utils.js");
 const external_ai_model_index_js_namespaceObject = require("./ai-model/index.js");
 const env_namespaceObject = require("@midscene/shared/env");
+const external_types_js_namespaceObject = require("./types.js");
 const external_agent_index_js_namespaceObject = require("./agent/index.js");
 const src = index_js_default();
 exports.Agent = __webpack_exports__.Agent;
 exports.AiLocateElement = __webpack_exports__.AiLocateElement;
-exports.Executor = __webpack_exports__.Executor;
-exports.Insight = __webpack_exports__.Insight;
 exports.MIDSCENE_MODEL_NAME = __webpack_exports__.MIDSCENE_MODEL_NAME;
 exports.PointSchema = __webpack_exports__.PointSchema;
 exports.RectSchema = __webpack_exports__.RectSchema;
+exports.Service = __webpack_exports__.Service;
+exports.ServiceError = __webpack_exports__.ServiceError;
 exports.SizeSchema = __webpack_exports__.SizeSchema;
 exports.TMultimodalPromptSchema = __webpack_exports__.TMultimodalPromptSchema;
 exports.TUserPromptSchema = __webpack_exports__.TUserPromptSchema;
+exports.TaskRunner = __webpack_exports__.TaskRunner;
 exports.createAgent = __webpack_exports__.createAgent;
 exports["default"] = __webpack_exports__["default"];
 exports.describeUserPage = __webpack_exports__.describeUserPage;
@@ -77,17 +80,18 @@ exports.getMidsceneLocationSchema = __webpack_exports__.getMidsceneLocationSchem
 exports.getVersion = __webpack_exports__.getVersion;
 exports.plan = __webpack_exports__.plan;
 exports.z = __webpack_exports__.z;
-for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+for(var __rspack_i in __webpack_exports__)if (-1 === [
     "Agent",
     "AiLocateElement",
-    "Executor",
-    "Insight",
     "MIDSCENE_MODEL_NAME",
     "PointSchema",
     "RectSchema",
+    "Service",
+    "ServiceError",
     "SizeSchema",
     "TMultimodalPromptSchema",
     "TUserPromptSchema",
+    "TaskRunner",
     "createAgent",
     "default",
     "describeUserPage",
@@ -95,7 +99,7 @@ for(var __webpack_i__ in __webpack_exports__)if (-1 === [
     "getVersion",
     "plan",
     "z"
-].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+].indexOf(__rspack_i)) exports[__rspack_i] = __webpack_exports__[__rspack_i];
 Object.defineProperty(exports, '__esModule', {
     value: true
 });