@rpascene/core 0.30.8

package/dist/lib/ai-model/common.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model\\common.js","sources":["webpack://@rpascene/core/webpack/runtime/define_property_getters","webpack://@rpascene/core/webpack/runtime/has_own_property","webpack://@rpascene/core/webpack/runtime/make_namespace_object","webpack://@rpascene/core/./src/ai-model/common.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  RpasceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@rpascene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@rpascene/shared/constants';\nimport type { TVlModeTypes } from '@rpascene/shared/env';\nimport { treeToList } from '@rpascene/shared/extractor';\nimport { compositeElementInfoImg } from '@rpascene/shared/img';\nimport { getDebug } from '@rpascene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n  TEXT = 5,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(\n      locate.bbox,\n      width,\n      height,\n      rightLimit,\n      bottomLimit,\n      vlMode,\n    );\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  let result: [number, number, number, number] = [0, 0, 0, 0];\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    result = adaptDoubaoBbox(bbox, width, height);\n  } else if (vlMode === 'gemini') {\n    result = adaptGeminiBbox(bbox, width, height);\n  } else if (vlMode === 'qwen3-vl') {\n    result = normalized01000(bbox, width, height);\n  } else {\n    result = adaptQwenBbox(bbox);\n  }\n\n  result[2] = Math.min(result[2], rightLimit);\n  result[3] = Math.min(result[3], bottomLimit);\n\n  return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  return [\n    Math.round((bbox[0] * width) / 1000),\n    Math.round((bbox[1] * height) / 1000),\n    Math.round((bbox[2] * width) / 1000),\n    Math.round((bbox[3] * height) / 1000),\n  ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  rightLimit = width,\n  bottomLimit = height,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils(\n    'adaptBboxToRect',\n    bbox,\n    width,\n    height,\n    'offset',\n    offsetX,\n    offsetY,\n    'limit',\n    rightLimit,\n    bottomLimit,\n    'vlMode',\n    vlMode,\n  );\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    rightLimit,\n    bottomLimit,\n    vlMode,\n  );\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(size: Size, modelName: string) {\n  if (warned) return;\n  if (modelName.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  const minEdgeSize = vlMode === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): RpasceneYamlFlowItem[] {\n  const flow: RpasceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: RpasceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'rpascene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst RpasceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepThink: z.boolean().optional(),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst RpasceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type RpasceneLocationResultType = z.infer<typeof RpasceneLocationResult>;\n\n// Export the input type - this is the inferred type from getRpasceneLocationSchema()\nexport type RpasceneLocationInputType = z.infer<typeof RpasceneLocationInput>;\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getRpasceneLocationSchema = () => {\n  return RpasceneLocationInput;\n};\n\nexport const ifRpasceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nexport const dumpRpasceneLocatorField = (field: any): string => {\n  assert(\n    ifRpasceneLocatorField(field),\n    'field is not a rpascene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllRpasceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifRpasceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllRpasceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a RpasceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllRpasceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllRpasceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement)\n  for (const fieldName in locateFieldValues) {\n    validated[fieldName] = locateFieldValues[fieldName];\n  }\n\n  return validated;\n};\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","warned","warnGPT4oSizeLimit","size","modelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","RpasceneLocationInput","getRpasceneLocationSchema","ifRpasceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpRpasceneLocatorField","String","findAllRpasceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","_field__def","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ACkBO,IAAKI,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;;WAAZA;;AASZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;QACN,YAAnB,OAAOY,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;KAC1B;IACD,OAAOgB;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEda,IAAAA,sBAAAA,MAAAA,AAAAA,EACEd,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BO,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;YAC5CW,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAIH,IACEiB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQN,kBAAkB;QAEhEiB,KAAK,GAAG,CACNZ,OACAY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACNX,QACAW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIY,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;QACpCW,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGlB,QAAS;QACnCY,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGjB,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,IAAIO,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBS,gBAAgBN,MAAMP,OAAOC,UAC7BG,AAAW,aAAXA,SACAkB,gBAAgBf,MAAMP,OAAOC,UAC7BG,AAAW,eAAXA,SACAmB,gBAAgBhB,MAAMP,OAAOC,UAE7BK,cAAcC;IAGzBI,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASY,gBACdhB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASqB,gBACdf,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMuB,OAAOZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAMyB,MAAMb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAMyB,QAAQd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM2B,SAASf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACuB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdrB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd4B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX5B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjCR,kBACE,mBACAW,MACAP,OACAC,QACA,UACA4B,SACAC,SACA,SACA5B,YACAC,aACA,UACAC;IAEF,MAAM,CAACoB,MAAMC,KAAKC,OAAOC,OAAO,GAAGtB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM2B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYjC,OACzBiC,YAAYjC,QAAQ+B;IAItB,IAAIC,UAAUE,aAAajC,QACzBiC,aAAajC,SAAS+B;IAIxBC,YAAYrB,KAAK,GAAG,CAAC,GAAGqB;IACxBC,aAAatB,KAAK,GAAG,CAAC,GAAGsB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAtC,kBAAkB,4BAA4BuC;IAE9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBAAmBC,IAAU,EAAEC,SAAiB;IAC9D,IAAIH,QAAQ;IACZ,IAAIG,UAAU,WAAW,GAAG,QAAQ,CAAC,WAAW;QAC9C,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,6FAA6F,CAAC;QAErN,IACE1B,KAAK,GAAG,CAAC0B,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpC1B,KAAK,GAAG,CAAC0B,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAUhC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAASlC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAWnC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYpC,KAAK,GAAG,IAAI+B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChB9C,MAAgC;IAEhC,MAAM+C,cAAc/C,AAAW,oBAAXA,SAA6B,MAAM;IACvD,MAAMgD,iBAAiB;IAGvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTvC,KAAK,IAAI,CAAEuC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVvC,KAAK,IAAI,CAAEuC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IAGN,IAAIG,WAAW3C,KAAK,GAAG,CAACuC,aAAahB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA;IAClD,IAAIG,YAAY5C,KAAK,GAAG,CAACuC,aAAahB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUtB,KAAK,IAAI,GAAGkB;IAC1B,IAAIK,SAASvB,KAAK,GAAG,GAAGmB;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAU7C,KAAK,GAAG,CAAC,GAAG6C;IACtBC,SAAS9C,KAAK,GAAG,CAAC,GAAG8C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCvB,KAAK,IAAI,GAAGsB;IACZtB,KAAK,GAAG,GAAGuB;IACXvB,KAAK,KAAK,GAAGoB;IACbpB,KAAK,MAAM,GAAGqB;IAEd,OAAOrB;AACT;AAEO,eAAewB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCvB,IAAU;IAEV,MAAMwB,eAAeC,AAAAA,IAAAA,0BAAAA,UAAAA,AAAAA,EAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,0BAAAA,QAAAA,CAAAA,IAAa,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtB1B;IACF;IACA,OAAO6B;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXnC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEkC,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAME,UAAUD,OAAO,cAAc,IAAID;QACzC,MAAMG,YAAYF,OAAO,WAAW,GAChCG,gBAAgBL,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMI,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAL,KAAK,IAAI,CAACO;IACZ;IAEA,IAAIR,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMQ,cAAcC,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAClC,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;IACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;AACf;AAEO,MAAMC,aAAaD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACjC,OAAOA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAChB,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;IAC9C,QAAQA,6BAAAA,CAAAA,CAAAA,KACA,CACJA,6BAAAA,CAAAA,CAAAA,MAAQ,CAAC;QACP,MAAMA,6BAAAA,CAAAA,CAAAA,MAAQ;QACd,KAAKA,6BAAAA,CAAAA,CAAAA,MAAQ;IACf,IAED,QAAQ;IACX,yBAAyBA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;IACvCA,6BAAAA,CAAAA,CAAAA,MAAQ;IACRA,6BAAAA,CAAAA,CAAAA,MACS,CAAC;QACN,QAAQA,6BAAAA,CAAAA,CAAAA,MAAQ;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,6BAAAA,CAAAA,CAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,OAAS;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,6BAAAA,CAAAA,CAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,6BAAAA,CAAAA,CAAAA,OAAS,CAAC;IACjC,QAAQI;IAGR,WAAWJ,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,WAAWA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAC/B,OAAOA,6BAAAA,CAAAA,CAAAA,OAAS,GAAG,QAAQ;IAG3B,QAAQA,6BAAAA,CAAAA,CAAAA,KAAO,CAAC;QAACA,6BAAAA,CAAAA,CAAAA,MAAQ;QAAIA,6BAAAA,CAAAA,CAAAA,MAAQ;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIP,uBAAuBQ,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvC7E,IAAAA,sBAAAA,MAAAA,AAAAA,EACE4E,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOjH,OAAO,IAAI,CAACgH,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAAClH;YAClB,MAAMsG,QAAQW,UAAU,KAAK,CAACjH,IAAI;YAClC,IAAI,CAACqG,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTI;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMzB,kBAAkB,CAC7B0B,YACAC;IAEA,MAAMC,gBAAgBT,4BAA4BQ;IAClD,MAAM/F,SAAS;QAAE,GAAG8F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAalG,MAAM,CAACiG,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTlG,MAAM,CAACiG,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BlG,MAAM,CAACiG,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBlG,MAAM,CAACiG,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAOlG;AACT;AAEO,MAAMmG,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBT,4BAA4BQ;IAClD,MAAM/F,SAAS;QAAE,GAAG8F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAalG,MAAM,CAACiG,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvBlG,MAAM,CAACiG,UAAU,GAAG;YAClB,CAACrB,oBAAoB,EAAE;YACvB,QAAQsB;QACV;IAEJ;IAEA,OAAOlG;AACT;AAUO,MAAMoG,mBAAmB,CAC9BC,UACAN;IAGA,MAAMO,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAehB,4BAA4BQ;IAGjD,IAAIQ,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOR,UAAU,KAAK,CAACO;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAM/H,OAAO4H,MAChB,IAAIC,aAAa,QAAQ,CAAC7H,MAExB+H,mBAAmB,CAAC/H,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/C+H,mBAAmB,CAAC/H,IAAI,GAAG4H,KAAK,CAAC5H,IAAI;IAKzC,MAAMgI,YAAYX,UAAU,KAAK,CAACU;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT"}

package/dist/lib/ai-model/conversation-history.js

@@ -0,0 +1,92 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    ConversationHistory: ()=>ConversationHistory
+});
+function _define_property(obj, key, value) {
+    if (key in obj) Object.defineProperty(obj, key, {
+        value: value,
+        enumerable: true,
+        configurable: true,
+        writable: true
+    });
+    else obj[key] = value;
+    return obj;
+}
+var _computedKey;
+_computedKey = Symbol.iterator;
+let _computedKey1 = _computedKey;
+class ConversationHistory {
+    append(message) {
+        if ('user' === message.role) this.pruneOldestUserMessageIfNecessary();
+        this.messages.push(message);
+    }
+    seed(messages) {
+        this.reset();
+        messages.forEach((message)=>{
+            this.append(message);
+        });
+    }
+    reset() {
+        this.messages.length = 0;
+    }
+    snapshot() {
+        return [
+            ...this.messages
+        ];
+    }
+    get length() {
+        return this.messages.length;
+    }
+    [_computedKey1]() {
+        return this.messages[Symbol.iterator]();
+    }
+    toJSON() {
+        return this.snapshot();
+    }
+    pruneOldestUserMessageIfNecessary() {
+        const userMessages = this.messages.filter((item)=>'user' === item.role);
+        if (userMessages.length < this.maxUserImageMessages) return;
+        const firstUserMessageIndex = this.messages.findIndex((item)=>'user' === item.role);
+        if (firstUserMessageIndex >= 0) this.messages.splice(firstUserMessageIndex, 1);
+    }
+    constructor(options){
+        var _options_initialMessages;
+        _define_property(this, "maxUserImageMessages", void 0);
+        _define_property(this, "messages", []);
+        this.maxUserImageMessages = (null == options ? void 0 : options.maxUserImageMessages) ?? 4;
+        if (null == options ? void 0 : null == (_options_initialMessages = options.initialMessages) ? void 0 : _options_initialMessages.length) this.seed(options.initialMessages);
+    }
+}
+exports.ConversationHistory = __webpack_exports__.ConversationHistory;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "ConversationHistory"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=conversation-history.js.map

package/dist/lib/ai-model/conversation-history.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model\\conversation-history.js","sources":["webpack://@rpascene/core/webpack/runtime/define_property_getters","webpack://@rpascene/core/webpack/runtime/has_own_property","webpack://@rpascene/core/webpack/runtime/make_namespace_object","webpack://@rpascene/core/./src/ai-model/conversation-history.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nexport interface ConversationHistoryOptions {\n  maxUserImageMessages?: number;\n  initialMessages?: ChatCompletionMessageParam[];\n}\n\nexport class ConversationHistory {\n  private readonly maxUserImageMessages: number;\n  private readonly messages: ChatCompletionMessageParam[] = [];\n\n  constructor(options?: ConversationHistoryOptions) {\n    this.maxUserImageMessages = options?.maxUserImageMessages ?? 4;\n    if (options?.initialMessages?.length) {\n      this.seed(options.initialMessages);\n    }\n  }\n\n  append(message: ChatCompletionMessageParam) {\n    if (message.role === 'user') {\n      this.pruneOldestUserMessageIfNecessary();\n    }\n\n    this.messages.push(message);\n  }\n\n  seed(messages: ChatCompletionMessageParam[]) {\n    this.reset();\n    messages.forEach((message) => {\n      this.append(message);\n    });\n  }\n\n  reset() {\n    this.messages.length = 0;\n  }\n\n  snapshot(): ChatCompletionMessageParam[] {\n    return [...this.messages];\n  }\n\n  get length(): number {\n    return this.messages.length;\n  }\n\n  [Symbol.iterator](): IterableIterator<ChatCompletionMessageParam> {\n    return this.messages[Symbol.iterator]();\n  }\n\n  toJSON(): ChatCompletionMessageParam[] {\n    return this.snapshot();\n  }\n\n  private pruneOldestUserMessageIfNecessary() {\n    const userMessages = this.messages.filter((item) => item.role === 'user');\n    if (userMessages.length < this.maxUserImageMessages) {\n      return;\n    }\n\n    const firstUserMessageIndex = this.messages.findIndex(\n      (item) => item.role === 'user',\n    );\n\n    if (firstUserMessageIndex >= 0) {\n      this.messages.splice(firstUserMessageIndex, 1);\n    }\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","ConversationHistory","message","messages","userMessages","item","firstUserMessageIndex","options","_options_initialMessages"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;eCuCGG,OAAO,QAAQ;;AAtCX,MAAMC;IAWX,OAAOC,OAAmC,EAAE;QAC1C,IAAIA,AAAiB,WAAjBA,QAAQ,IAAI,EACd,IAAI,CAAC,iCAAiC;QAGxC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAACA;IACrB;IAEA,KAAKC,QAAsC,EAAE;QAC3C,IAAI,CAAC,KAAK;QACVA,SAAS,OAAO,CAAC,CAACD;YAChB,IAAI,CAAC,MAAM,CAACA;QACd;IACF;IAEA,QAAQ;QACN,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG;IACzB;IAEA,WAAyC;QACvC,OAAO;eAAI,IAAI,CAAC,QAAQ;SAAC;IAC3B;IAEA,IAAI,SAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM;IAC7B;IAEA,CAAC,cAAD,GAAkE;QAChE,OAAO,IAAI,CAAC,QAAQ,CAACF,OAAO,QAAQ,CAAC;IACvC;IAEA,SAAuC;QACrC,OAAO,IAAI,CAAC,QAAQ;IACtB;IAEQ,oCAAoC;QAC1C,MAAMI,eAAe,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAACC,OAASA,AAAc,WAAdA,KAAK,IAAI;QAC7D,IAAID,aAAa,MAAM,GAAG,IAAI,CAAC,oBAAoB,EACjD;QAGF,MAAME,wBAAwB,IAAI,CAAC,QAAQ,CAAC,SAAS,CACnD,CAACD,OAASA,AAAc,WAAdA,KAAK,IAAI;QAGrB,IAAIC,yBAAyB,GAC3B,IAAI,CAAC,QAAQ,CAAC,MAAM,CAACA,uBAAuB;IAEhD;IAvDA,YAAYC,OAAoC,CAAE;YAE5CC;QALN,uBAAiB,wBAAjB;QACA,uBAAiB,YAAyC,EAAE;QAG1D,IAAI,CAAC,oBAAoB,GAAGD,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,oBAAoB,AAAD,KAAK;QAC7D,IAAIC,QAAAA,UAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,QAAS,eAAe,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAClC,IAAI,CAAC,IAAI,CAACD,QAAQ,eAAe;IAErC;AAmDF"}

package/dist/lib/ai-model/index.js

@@ -0,0 +1,131 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    uiTarsPlanning: ()=>external_ui_tars_planning_js_namespaceObject.uiTarsPlanning,
+    plan: ()=>external_llm_planning_js_namespaceObject.plan,
+    SizeSchema: ()=>external_common_js_namespaceObject.SizeSchema,
+    TMultimodalPromptSchema: ()=>external_common_js_namespaceObject.TMultimodalPromptSchema,
+    TUserPromptSchema: ()=>external_common_js_namespaceObject.TUserPromptSchema,
+    resizeImageForUiTars: ()=>external_ui_tars_planning_js_namespaceObject.resizeImageForUiTars,
+    getRpasceneLocationSchema: ()=>external_common_js_namespaceObject.getRpasceneLocationSchema,
+    AiLocateElement: ()=>external_inspect_js_namespaceObject.AiLocateElement,
+    generatePlaywrightTestStream: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTestStream,
+    dumpActionParam: ()=>external_common_js_namespaceObject.dumpActionParam,
+    generateYamlTest: ()=>yaml_generator_js_namespaceObject.generateYamlTest,
+    systemPromptToLocateElement: ()=>llm_locator_js_namespaceObject.systemPromptToLocateElement,
+    elementByPositionWithElementInfo: ()=>util_js_namespaceObject.elementByPositionWithElementInfo,
+    AiLocateSection: ()=>external_inspect_js_namespaceObject.AiLocateSection,
+    AIActionType: ()=>external_common_js_namespaceObject.AIActionType,
+    callAI: ()=>index_js_namespaceObject.callAI,
+    ConversationHistory: ()=>external_conversation_history_js_namespaceObject.ConversationHistory,
+    generatePlaywrightTest: ()=>playwright_generator_js_namespaceObject.generatePlaywrightTest,
+    AiExtractElementInfo: ()=>external_inspect_js_namespaceObject.AiExtractElementInfo,
+    generateYamlTestStream: ()=>yaml_generator_js_namespaceObject.generateYamlTestStream,
+    loadActionParam: ()=>external_common_js_namespaceObject.loadActionParam,
+    RectSchema: ()=>external_common_js_namespaceObject.RectSchema,
+    PointSchema: ()=>external_common_js_namespaceObject.PointSchema,
+    adaptBboxToRect: ()=>external_common_js_namespaceObject.adaptBboxToRect,
+    callAIWithObjectResponse: ()=>index_js_namespaceObject.callAIWithObjectResponse,
+    parseActionParam: ()=>external_common_js_namespaceObject.parseActionParam,
+    callAIWithStringResponse: ()=>index_js_namespaceObject.callAIWithStringResponse,
+    describeUserPage: ()=>util_js_namespaceObject.describeUserPage,
+    findAllRpasceneLocatorField: ()=>external_common_js_namespaceObject.findAllRpasceneLocatorField
+});
+const index_js_namespaceObject = require("./service-caller/index.js");
+const llm_locator_js_namespaceObject = require("./prompt/llm-locator.js");
+const util_js_namespaceObject = require("./prompt/util.js");
+const playwright_generator_js_namespaceObject = require("./prompt/playwright-generator.js");
+const yaml_generator_js_namespaceObject = require("./prompt/yaml-generator.js");
+const external_inspect_js_namespaceObject = require("./inspect.js");
+const external_llm_planning_js_namespaceObject = require("./llm-planning.js");
+const external_common_js_namespaceObject = require("./common.js");
+const external_ui_tars_planning_js_namespaceObject = require("./ui-tars-planning.js");
+const external_conversation_history_js_namespaceObject = require("./conversation-history.js");
+exports.AIActionType = __webpack_exports__.AIActionType;
+exports.AiExtractElementInfo = __webpack_exports__.AiExtractElementInfo;
+exports.AiLocateElement = __webpack_exports__.AiLocateElement;
+exports.AiLocateSection = __webpack_exports__.AiLocateSection;
+exports.ConversationHistory = __webpack_exports__.ConversationHistory;
+exports.PointSchema = __webpack_exports__.PointSchema;
+exports.RectSchema = __webpack_exports__.RectSchema;
+exports.SizeSchema = __webpack_exports__.SizeSchema;
+exports.TMultimodalPromptSchema = __webpack_exports__.TMultimodalPromptSchema;
+exports.TUserPromptSchema = __webpack_exports__.TUserPromptSchema;
+exports.adaptBboxToRect = __webpack_exports__.adaptBboxToRect;
+exports.callAI = __webpack_exports__.callAI;
+exports.callAIWithObjectResponse = __webpack_exports__.callAIWithObjectResponse;
+exports.callAIWithStringResponse = __webpack_exports__.callAIWithStringResponse;
+exports.describeUserPage = __webpack_exports__.describeUserPage;
+exports.dumpActionParam = __webpack_exports__.dumpActionParam;
+exports.elementByPositionWithElementInfo = __webpack_exports__.elementByPositionWithElementInfo;
+exports.findAllRpasceneLocatorField = __webpack_exports__.findAllRpasceneLocatorField;
+exports.generatePlaywrightTest = __webpack_exports__.generatePlaywrightTest;
+exports.generatePlaywrightTestStream = __webpack_exports__.generatePlaywrightTestStream;
+exports.generateYamlTest = __webpack_exports__.generateYamlTest;
+exports.generateYamlTestStream = __webpack_exports__.generateYamlTestStream;
+exports.getRpasceneLocationSchema = __webpack_exports__.getRpasceneLocationSchema;
+exports.loadActionParam = __webpack_exports__.loadActionParam;
+exports.parseActionParam = __webpack_exports__.parseActionParam;
+exports.plan = __webpack_exports__.plan;
+exports.resizeImageForUiTars = __webpack_exports__.resizeImageForUiTars;
+exports.systemPromptToLocateElement = __webpack_exports__.systemPromptToLocateElement;
+exports.uiTarsPlanning = __webpack_exports__.uiTarsPlanning;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "AIActionType",
+    "AiExtractElementInfo",
+    "AiLocateElement",
+    "AiLocateSection",
+    "ConversationHistory",
+    "PointSchema",
+    "RectSchema",
+    "SizeSchema",
+    "TMultimodalPromptSchema",
+    "TUserPromptSchema",
+    "adaptBboxToRect",
+    "callAI",
+    "callAIWithObjectResponse",
+    "callAIWithStringResponse",
+    "describeUserPage",
+    "dumpActionParam",
+    "elementByPositionWithElementInfo",
+    "findAllRpasceneLocatorField",
+    "generatePlaywrightTest",
+    "generatePlaywrightTestStream",
+    "generateYamlTest",
+    "generateYamlTestStream",
+    "getRpasceneLocationSchema",
+    "loadActionParam",
+    "parseActionParam",
+    "plan",
+    "resizeImageForUiTars",
+    "systemPromptToLocateElement",
+    "uiTarsPlanning"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=index.js.map

package/dist/lib/ai-model/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model\\index.js","sources":["webpack://@rpascene/core/webpack/runtime/define_property_getters","webpack://@rpascene/core/webpack/runtime/has_own_property","webpack://@rpascene/core/webpack/runtime/make_namespace_object"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D"}

package/dist/lib/ai-model/inspect.js

@@ -0,0 +1,326 @@
+"use strict";
+var __webpack_require__ = {};
+(()=>{
+    __webpack_require__.d = (exports1, definition)=>{
+        for(var key in definition)if (__webpack_require__.o(definition, key) && !__webpack_require__.o(exports1, key)) Object.defineProperty(exports1, key, {
+            enumerable: true,
+            get: definition[key]
+        });
+    };
+})();
+(()=>{
+    __webpack_require__.o = (obj, prop)=>Object.prototype.hasOwnProperty.call(obj, prop);
+})();
+(()=>{
+    __webpack_require__.r = (exports1)=>{
+        if ('undefined' != typeof Symbol && Symbol.toStringTag) Object.defineProperty(exports1, Symbol.toStringTag, {
+            value: 'Module'
+        });
+        Object.defineProperty(exports1, '__esModule', {
+            value: true
+        });
+    };
+})();
+var __webpack_exports__ = {};
+__webpack_require__.r(__webpack_exports__);
+__webpack_require__.d(__webpack_exports__, {
+    AiLocateElement: ()=>AiLocateElement,
+    AiLocateSection: ()=>AiLocateSection,
+    AiExtractElementInfo: ()=>AiExtractElementInfo
+});
+const img_namespaceObject = require("@rpascene/shared/img");
+const logger_namespaceObject = require("@rpascene/shared/logger");
+const utils_namespaceObject = require("@rpascene/shared/utils");
+const external_common_js_namespaceObject = require("./common.js");
+const extraction_js_namespaceObject = require("./prompt/extraction.js");
+const llm_locator_js_namespaceObject = require("./prompt/llm-locator.js");
+const llm_section_locator_js_namespaceObject = require("./prompt/llm-section-locator.js");
+const util_js_namespaceObject = require("./prompt/util.js");
+const index_js_namespaceObject = require("./service-caller/index.js");
+const debugInspect = (0, logger_namespaceObject.getDebug)('ai:inspect');
+const debugSection = (0, logger_namespaceObject.getDebug)('ai:section');
+const extraTextFromUserPrompt = (prompt)=>{
+    if ('string' == typeof prompt) return prompt;
+    return prompt.prompt;
+};
+const promptsToChatParam = async (multimodalPrompt)=>{
+    var _multimodalPrompt_images;
+    const msgs = [];
+    if (null == multimodalPrompt ? void 0 : null == (_multimodalPrompt_images = multimodalPrompt.images) ? void 0 : _multimodalPrompt_images.length) {
+        msgs.push({
+            role: 'user',
+            content: [
+                {
+                    type: 'text',
+                    text: 'Next, I will provide all the reference images.'
+                }
+            ]
+        });
+        for (const item of multimodalPrompt.images){
+            const base64 = await (0, img_namespaceObject.preProcessImageUrl)(item.url, !!multimodalPrompt.convertHttpImage2Base64);
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'text',
+                        text: `reference image ${item.name}:`
+                    }
+                ]
+            });
+            msgs.push({
+                role: 'user',
+                content: [
+                    {
+                        type: 'image_url',
+                        image_url: {
+                            url: base64,
+                            detail: 'high'
+                        }
+                    }
+                ]
+            });
+        }
+    }
+    return msgs;
+};
+async function AiLocateElement(options) {
+    const { context, targetElementDescription, callAIFn, modelConfig } = options;
+    const { vlMode } = modelConfig;
+    const { screenshotBase64 } = context;
+    const { description, elementById, insertElementByPosition } = await (0, util_js_namespaceObject.describeUserPage)(context, {
+        vlMode
+    });
+    (0, utils_namespaceObject.assert)(targetElementDescription, "cannot find the target element description");
+    const userInstructionPrompt = await llm_locator_js_namespaceObject.findElementPrompt.format({
+        pageDescription: description,
+        targetElementDescription: extraTextFromUserPrompt(targetElementDescription)
+    });
+    const systemPrompt = (0, llm_locator_js_namespaceObject.systemPromptToLocateElement)(vlMode);
+    let imagePayload = screenshotBase64;
+    let imageWidth = context.size.width;
+    let imageHeight = context.size.height;
+    let originalImageWidth = imageWidth;
+    let originalImageHeight = imageHeight;
+    if (options.searchConfig) {
+        var _options_searchConfig_rect, _options_searchConfig_rect1;
+        (0, utils_namespaceObject.assert)(options.searchConfig.rect, 'searchArea is provided but its rect cannot be found. Failed to locate element');
+        (0, utils_namespaceObject.assert)(options.searchConfig.imageBase64, 'searchArea is provided but its imageBase64 cannot be found. Failed to locate element');
+        imagePayload = options.searchConfig.imageBase64;
+        imageWidth = null == (_options_searchConfig_rect = options.searchConfig.rect) ? void 0 : _options_searchConfig_rect.width;
+        imageHeight = null == (_options_searchConfig_rect1 = options.searchConfig.rect) ? void 0 : _options_searchConfig_rect1.height;
+        originalImageWidth = imageWidth;
+        originalImageHeight = imageHeight;
+    } else if ('qwen-vl' === vlMode) {
+        const paddedResult = await (0, img_namespaceObject.paddingToMatchBlockByBase64)(imagePayload);
+        imageWidth = paddedResult.width;
+        imageHeight = paddedResult.height;
+        imagePayload = paddedResult.imageBase64;
+    } else if ('qwen3-vl' === vlMode) ;
+    else if (!vlMode) imagePayload = await (0, external_common_js_namespaceObject.markupImageForLLM)(screenshotBase64, context.tree, context.size);
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: imagePayload,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: userInstructionPrompt
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof targetElementDescription) {
+        const addOns = await promptsToChatParam({
+            images: targetElementDescription.images,
+            convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const res = await callAIFn(msgs, external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig);
+    const rawResponse = JSON.stringify(res.content);
+    let resRect;
+    let matchedElements = 'elements' in res.content ? res.content.elements : [];
+    let errors = 'errors' in res.content ? res.content.errors : [];
+    try {
+        if ('bbox' in res.content && Array.isArray(res.content.bbox)) {
+            var _options_searchConfig_rect2, _options_searchConfig, _options_searchConfig_rect3, _options_searchConfig1;
+            resRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(res.content.bbox, imageWidth, imageHeight, null == (_options_searchConfig = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect2 = _options_searchConfig.rect) ? void 0 : _options_searchConfig_rect2.left, null == (_options_searchConfig1 = options.searchConfig) ? void 0 : null == (_options_searchConfig_rect3 = _options_searchConfig1.rect) ? void 0 : _options_searchConfig_rect3.top, originalImageWidth, originalImageHeight, vlMode);
+            debugInspect('resRect', resRect);
+            const rectCenter = {
+                x: resRect.left + resRect.width / 2,
+                y: resRect.top + resRect.height / 2
+            };
+            let element = (0, util_js_namespaceObject.elementByPositionWithElementInfo)(context.tree, rectCenter);
+            const distanceToCenter = element ? (0, util_js_namespaceObject.distance)({
+                x: element.center[0],
+                y: element.center[1]
+            }, rectCenter) : 0;
+            if (!element || distanceToCenter > util_js_namespaceObject.distanceThreshold) element = insertElementByPosition(rectCenter);
+            if (element) {
+                matchedElements = [
+                    element
+                ];
+                errors = [];
+            }
+        }
+    } catch (e) {
+        const msg = e instanceof Error ? `Failed to parse bbox: ${e.message}` : 'unknown error in locate';
+        if (errors && (null == errors ? void 0 : errors.length) !== 0) errors.push(`(${msg})`);
+        else errors = [
+            msg
+        ];
+    }
+    return {
+        rect: resRect,
+        parseResult: {
+            elements: matchedElements,
+            errors
+        },
+        rawResponse,
+        elementById,
+        usage: res.usage,
+        isOrderSensitive: 'object' == typeof res.content && null !== res.content && 'isOrderSensitive' in res.content ? res.content.isOrderSensitive : void 0
+    };
+}
+async function AiLocateSection(options) {
+    const { context, sectionDescription, modelConfig } = options;
+    const { vlMode } = modelConfig;
+    const { screenshotBase64 } = context;
+    const systemPrompt = (0, llm_section_locator_js_namespaceObject.systemPromptToLocateSection)(vlMode);
+    const sectionLocatorInstructionText = await llm_section_locator_js_namespaceObject.sectionLocatorInstruction.format({
+        sectionDescription: extraTextFromUserPrompt(sectionDescription)
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: [
+                {
+                    type: 'image_url',
+                    image_url: {
+                        url: screenshotBase64,
+                        detail: 'high'
+                    }
+                },
+                {
+                    type: 'text',
+                    text: sectionLocatorInstructionText
+                }
+            ]
+        }
+    ];
+    if ('string' != typeof sectionDescription) {
+        const addOns = await promptsToChatParam({
+            images: sectionDescription.images,
+            convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const result = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig);
+    let sectionRect;
+    const sectionBbox = result.content.bbox;
+    if (sectionBbox) {
+        const targetRect = (0, external_common_js_namespaceObject.adaptBboxToRect)(sectionBbox, context.size.width, context.size.height, 0, 0, context.size.width, context.size.height, vlMode);
+        debugSection('original targetRect %j', targetRect);
+        const referenceBboxList = result.content.references_bbox || [];
+        debugSection('referenceBboxList %j', referenceBboxList);
+        const referenceRects = referenceBboxList.filter((bbox)=>Array.isArray(bbox)).map((bbox)=>(0, external_common_js_namespaceObject.adaptBboxToRect)(bbox, context.size.width, context.size.height, 0, 0, context.size.width, context.size.height, vlMode));
+        debugSection('referenceRects %j', referenceRects);
+        const mergedRect = (0, external_common_js_namespaceObject.mergeRects)([
+            targetRect,
+            ...referenceRects
+        ]);
+        debugSection('mergedRect %j', mergedRect);
+        sectionRect = (0, external_common_js_namespaceObject.expandSearchArea)(mergedRect, context.size, vlMode);
+        debugSection('expanded sectionRect %j', sectionRect);
+    }
+    let imageBase64 = screenshotBase64;
+    if (sectionRect) {
+        const croppedResult = await (0, img_namespaceObject.cropByRect)(screenshotBase64, sectionRect, 'qwen-vl' === vlMode);
+        imageBase64 = croppedResult.imageBase64;
+        sectionRect.width = croppedResult.width;
+        sectionRect.height = croppedResult.height;
+    }
+    return {
+        rect: sectionRect,
+        imageBase64,
+        error: result.content.error,
+        rawResponse: JSON.stringify(result.content),
+        usage: result.usage
+    };
+}
+async function AiExtractElementInfo(options) {
+    const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } = options;
+    const { vlMode } = modelConfig;
+    const systemPrompt = (0, extraction_js_namespaceObject.systemPromptToExtract)();
+    const { screenshotBase64 } = context;
+    const { description, elementById } = await (0, util_js_namespaceObject.describeUserPage)(context, {
+        truncateTextLength: 200,
+        filterNonTextContent: false,
+        visibleOnly: false,
+        domIncluded: null == extractOption ? void 0 : extractOption.domIncluded,
+        vlMode
+    });
+    const extractDataPromptText = await (0, extraction_js_namespaceObject.extractDataQueryPrompt)(description, dataQuery);
+    const userContent = [];
+    if ((null == extractOption ? void 0 : extractOption.screenshotIncluded) !== false) userContent.push({
+        type: 'image_url',
+        image_url: {
+            url: screenshotBase64,
+            detail: 'high'
+        }
+    });
+    userContent.push({
+        type: 'text',
+        text: extractDataPromptText
+    });
+    const msgs = [
+        {
+            role: 'system',
+            content: systemPrompt
+        },
+        {
+            role: 'user',
+            content: userContent
+        }
+    ];
+    if (multimodalPrompt) {
+        const addOns = await promptsToChatParam({
+            images: multimodalPrompt.images,
+            convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64
+        });
+        msgs.push(...addOns);
+    }
+    const result = await (0, index_js_namespaceObject.callAIWithObjectResponse)(msgs, external_common_js_namespaceObject.AIActionType.EXTRACT_DATA, modelConfig);
+    return {
+        parseResult: result.content,
+        elementById,
+        usage: result.usage
+    };
+}
+exports.AiExtractElementInfo = __webpack_exports__.AiExtractElementInfo;
+exports.AiLocateElement = __webpack_exports__.AiLocateElement;
+exports.AiLocateSection = __webpack_exports__.AiLocateSection;
+for(var __webpack_i__ in __webpack_exports__)if (-1 === [
+    "AiExtractElementInfo",
+    "AiLocateElement",
+    "AiLocateSection"
+].indexOf(__webpack_i__)) exports[__webpack_i__] = __webpack_exports__[__webpack_i__];
+Object.defineProperty(exports, '__esModule', {
+    value: true
+});
+
+//# sourceMappingURL=inspect.js.map