@midscene/core 0.28.7-beta-20250915040112.0 → 0.28.7-beta-20250915094412.0

package/dist/es/agent/utils.mjs

@@ -139,7 +139,7 @@ function trimContextByViewport(execution) {
         }) : execution.tasks
     };
 }
-const getMidsceneVersion = ()=>"0.28.7-beta-20250915040112.0";
+const getMidsceneVersion = ()=>"0.28.7-beta-20250915094412.0";
 const parsePrompt = (prompt)=>{
     if ('string' == typeof prompt) return {
         textPrompt: prompt,

package/dist/es/ai-model/common.mjs

@@ -10,6 +10,7 @@ var common_AIActionType = /*#__PURE__*/ function(AIActionType) {
     AIActionType[AIActionType["EXTRACT_DATA"] = 2] = "EXTRACT_DATA";
     AIActionType[AIActionType["PLAN"] = 3] = "PLAN";
     AIActionType[AIActionType["DESCRIBE_ELEMENT"] = 4] = "DESCRIBE_ELEMENT";
+    AIActionType[AIActionType["TEXT"] = 5] = "TEXT";
     return AIActionType;
 }({});
 const defaultBboxSize = 20;

package/dist/es/ai-model/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height, vlMode);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlMode === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(bbox, width, height, vlMode);\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(size: Size, modelName: string) {\n  if (warned) return;\n  if (modelName.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  const minEdgeSize = vlMode === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationResult;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodUnion (the new MidsceneLocation structure)\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n    return locateFieldFlagName in shape;\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n"],"names":["AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","vlMode","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","warned","warnGPT4oSizeLimit","size","modelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationResult","getMidsceneLocationSchema","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","Object","key","_field__def","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam"],"mappings":";;;;;;AAwBO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;WAAZA;;AAQZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,MAAgC;IAGhC,IAAKH,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGI,UAAUJ,OAAO,IAAI,EAAEC,OAAOC,QAAQC;IAGtD,OAAOH;AACT;AAEO,SAASK,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGV;QACN,YAAnB,OAAOU,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGV;KAC1B;IACD,OAAOc;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCL,KAAa,EACbC,MAAc;IAEdW,OACEZ,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOI,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIb,QAAS;YAC3CU,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,SAAU;YAC5CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIb,QAAS;YAC3CU,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,SAAU;SAC7C;QAEH,MAAM,IAAIO,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;QACpCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;KACrC;IAIH,IACEe,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS,QAAQL,kBAAkB;QAE/De,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU,QAAQN,kBAAkB;QAEhEe,KAAK,GAAG,CACNV,OACAU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS,QAAQL,kBAAkB;QAE/De,KAAK,GAAG,CACNT,QACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIU,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;QACpCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;KACrC;IAGH,MAAMK,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdL,KAAa,EACbC,MAAc,EACdC,MAAgC;IAEhC,IAAIA,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAChC,OAAOS,gBAAgBN,MAAML,OAAOC;IAGtC,IAAIC,AAAW,aAAXA,QACF,OAAOkB,gBAAgBf,MAAML,OAAOC;IAGtC,OAAOG,cAAcC;AACvB;AAEO,SAASe,gBACdf,IAAc,EACdL,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOX,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGL,QAAS;IAC5C,MAAMsB,MAAMZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,SAAU;IAC5C,MAAMsB,QAAQb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGL,QAAS;IAC7C,MAAMwB,SAASd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdpB,IAAc,EACdL,KAAa,EACbC,MAAc,EACdyB,UAAU,CAAC,EACXC,UAAU,CAAC,EACXzB,MAAiC;IAEjCN,kBAAkB,mBAAmBS,MAAML,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGrB,UAAUE,MAAML,OAAOC,QAAQC;IAGlE,MAAM0B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAY9B,OACzB8B,YAAY9B,QAAQ4B;IAItB,IAAIC,UAAUE,aAAa9B,QACzB8B,aAAa9B,SAAS4B;IAIxBC,YAAYpB,KAAK,GAAG,CAAC,GAAGoB;IACxBC,aAAarB,KAAK,GAAG,CAAC,GAAGqB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAnC,kBAAkB,4BAA4BoC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBAAmBC,IAAU,EAAEC,SAAiB;IAC9D,IAAIH,QAAQ;IACZ,IAAIG,UAAU,WAAW,GAAG,QAAQ,CAAC,WAAW;QAC9C,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,6FAA6F,CAAC;QAErN,IACEzB,KAAK,GAAG,CAACyB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCzB,KAAK,GAAG,CAACyB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU/B,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAASjC,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAWlC,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYnC,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChB7C,MAAgC;IAEhC,MAAM8C,cAAc9C,AAAW,oBAAXA,SAA6B,MAAM;IACvD,MAAM+C,iBAAiB;IAGvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTtC,KAAK,IAAI,CAAEsC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVtC,KAAK,IAAI,CAAEsC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IAGN,IAAIG,WAAW1C,KAAK,GAAG,CAACsC,aAAahB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA;IAClD,IAAIG,YAAY3C,KAAK,GAAG,CAACsC,aAAahB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUtB,KAAK,IAAI,GAAGkB;IAC1B,IAAIK,SAASvB,KAAK,GAAG,GAAGmB;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAU5C,KAAK,GAAG,CAAC,GAAG4C;IACtBC,SAAS7C,KAAK,GAAG,CAAC,GAAG6C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCvB,KAAK,IAAI,GAAGsB;IACZtB,KAAK,GAAG,GAAGuB;IACXvB,KAAK,KAAK,GAAGoB;IACbpB,KAAK,MAAM,GAAGqB;IAEd,OAAOrB;AACT;AAEO,eAAewB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCvB,IAAU;IAEV,MAAMwB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtB1B;IACF;IACA,OAAO6B;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXnC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEkC,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAME,UAAUD,OAAO,cAAc,IAAID;QACzC,MAAMG,YAAYF,OAAO,WAAW,GAChCG,gBAAgBL,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMI,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAL,KAAK,IAAI,CAACO;IACZ;IAEA,IAAIR,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMQ,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAE5B,MAAMC,yBAAyBN,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAGP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QACpC,OAAOP,uBAAuBQ;IAChC;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvC5E,OACE2E,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMd,QAAQW,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTM;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAM3B,kBAAkB,CAC7B4B,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAMhG,SAAS;QAAE,GAAG+F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAanG,MAAM,CAACkG,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTnG,MAAM,CAACkG,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BnG,MAAM,CAACkG,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBnG,MAAM,CAACkG,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAOnG;AACT;AAEO,MAAMoG,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAMhG,SAAS;QAAE,GAAG+F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAanG,MAAM,CAACkG,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvBnG,MAAM,CAACkG,UAAU,GAAG;YAClB,CAACvB,oBAAoB,EAAE;YACvB,QAAQwB;QACV;IAEJ;IAEA,OAAOnG;AACT"}
+{"version":3,"file":"ai-model/common.mjs","sources":["webpack://@midscene/core/./src/ai-model/common.ts"],"sourcesContent":["import type {\n  AIUsageInfo,\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nexport enum AIActionType {\n  ASSERT = 0,\n  INSPECT_ELEMENT = 1,\n  EXTRACT_DATA = 2,\n  PLAN = 3,\n  DESCRIBE_ELEMENT = 4,\n  TEXT = 5,\n}\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(locate.bbox, width, height, vlMode);\n  }\n\n  return locate;\n}\n\nexport function adaptQwenBbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  if (Array.isArray(bbox) && Array.isArray(bbox[0])) {\n    bbox = bbox[0];\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nexport function adaptBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    return adaptDoubaoBbox(bbox, width, height);\n  }\n\n  if (vlMode === 'gemini') {\n    return adaptGeminiBbox(bbox, width, height);\n  }\n\n  return adaptQwenBbox(bbox);\n}\n\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils('adaptBboxToRect', bbox, width, height, offsetX, offsetY);\n  const [left, top, right, bottom] = adaptBbox(bbox, width, height, vlMode);\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n  return rect;\n}\n\nlet warned = false;\nexport function warnGPT4oSizeLimit(size: Size, modelName: string) {\n  if (warned) return;\n  if (modelName.toLowerCase().includes('gpt-4o')) {\n    const warningMsg = `GPT-4o has a maximum image input size of 2000x768 or 768x2000, but got ${size.width}x${size.height}. Please set your interface to a smaller resolution. Otherwise, the result may be inaccurate.`;\n\n    if (\n      Math.max(size.width, size.height) > 2000 ||\n      Math.min(size.width, size.height) > 768\n    ) {\n      console.warn(warningMsg);\n      warned = true;\n    }\n  } else if (size.width > 1800 || size.height > 1800) {\n    console.warn(\n      `The image size seems too large (${size.width}x${size.height}). It may lead to more token usage, slower response, and inaccurate result.`,\n    );\n    warned = true;\n  }\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  const minEdgeSize = vlMode === 'doubao-vision' ? 500 : 300;\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n  sleep?: number,\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  if (sleep) {\n    flow.push({\n      sleep,\n    });\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationResult;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodUnion (the new MidsceneLocation structure)\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n    return locateFieldFlagName in shape;\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n"],"names":["AIActionType","defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","vlMode","adaptBbox","adaptQwenBbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","Array","bboxList","item","x","y","adaptGeminiBbox","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","warned","warnGPT4oSizeLimit","size","modelName","warningMsg","console","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","sleep","flow","plan","verb","action","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationResult","getMidsceneLocationSchema","ifMidsceneLocatorField","field","_actualField__def","_actualField__def1","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","_zodObject__def","zodObject","keys","Object","key","_field__def","jsonObject","zodSchema","locatorFields","fieldName","fieldValue","loadActionParam"],"mappings":";;;;;;AAwBO,IAAKA,sBAAYA,WAAAA,GAAAA,SAAZA,YAAY;;;;;;;WAAZA;;AASZ,MAAMC,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAG5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,MAAgC;IAGhC,IAAKH,OAAe,OAAO,IAAI,CAACA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,AAAD,GAAG;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,IAAI,EACdA,OAAO,IAAI,GAAGI,UAAUJ,OAAO,IAAI,EAAEC,OAAOC,QAAQC;IAGtD,OAAOH;AACT;AAEO,SAASK,cACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGV;QACN,YAAnB,OAAOU,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGV;KAC1B;IACD,OAAOc;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCL,KAAa,EACbC,MAAc;IAEdW,OACEZ,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOI,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIb,QAAS;YAC3CU,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,SAAU;YAC5CS,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIb,QAAS;YAC3CU,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIZ,SAAU;SAC7C;QAEH,MAAM,IAAIO,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,MAAM,OAAO,CAACV,SAASU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GAC9CA,OAAOA,IAAI,CAAC,EAAE;IAGhB,IAAIW,WAAqB,EAAE;IAC3B,IAAID,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BD,SAAS,IAAI,CAACF,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEH,SAAS,IAAI,CAACF,OAAOG;IAEzB;SAEAD,WAAWX;IAGb,IAAIW,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLN,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;QACpCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;KACrC;IAIH,IACEe,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLN,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS,QAAQL,kBAAkB;QAE/De,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU,QAAQN,kBAAkB;QAEhEe,KAAK,GAAG,CACNV,OACAU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS,QAAQL,kBAAkB;QAE/De,KAAK,GAAG,CACNT,QACAS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIU,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;QACpCS,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCU,KAAK,KAAK,CAAEM,QAAQ,CAAC,EAAE,GAAGf,SAAU;KACrC;IAGH,MAAMK,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEO,SAASH,UACdE,IAAc,EACdL,KAAa,EACbC,MAAc,EACdC,MAAgC;IAEhC,IAAIA,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAChC,OAAOS,gBAAgBN,MAAML,OAAOC;IAGtC,IAAIC,AAAW,aAAXA,QACF,OAAOkB,gBAAgBf,MAAML,OAAOC;IAGtC,OAAOG,cAAcC;AACvB;AAEO,SAASe,gBACdf,IAAc,EACdL,KAAa,EACbC,MAAc;IAEd,MAAMoB,OAAOX,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGL,QAAS;IAC5C,MAAMsB,MAAMZ,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,SAAU;IAC5C,MAAMsB,QAAQb,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGL,QAAS;IAC7C,MAAMwB,SAASd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGJ,SAAU;IAC/C,OAAO;QAACoB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdpB,IAAc,EACdL,KAAa,EACbC,MAAc,EACdyB,UAAU,CAAC,EACXC,UAAU,CAAC,EACXzB,MAAiC;IAEjCN,kBAAkB,mBAAmBS,MAAML,OAAOC,QAAQyB,SAASC;IACnE,MAAM,CAACN,MAAMC,KAAKC,OAAOC,OAAO,GAAGrB,UAAUE,MAAML,OAAOC,QAAQC;IAGlE,MAAM0B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAY9B,OACzB8B,YAAY9B,QAAQ4B;IAItB,IAAIC,UAAUE,aAAa9B,QACzB8B,aAAa9B,SAAS4B;IAIxBC,YAAYpB,KAAK,GAAG,CAAC,GAAGoB;IACxBC,aAAarB,KAAK,GAAG,CAAC,GAAGqB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAnC,kBAAkB,4BAA4BoC;IAC9C,OAAOA;AACT;AAEA,IAAIC,SAAS;AACN,SAASC,mBAAmBC,IAAU,EAAEC,SAAiB;IAC9D,IAAIH,QAAQ;IACZ,IAAIG,UAAU,WAAW,GAAG,QAAQ,CAAC,WAAW;QAC9C,MAAMC,aAAa,CAAC,uEAAuE,EAAEF,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,6FAA6F,CAAC;QAErN,IACEzB,KAAK,GAAG,CAACyB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,QACpCzB,KAAK,GAAG,CAACyB,KAAK,KAAK,EAAEA,KAAK,MAAM,IAAI,KACpC;YACAG,QAAQ,IAAI,CAACD;YACbJ,SAAS;QACX;IACF,OAAO,IAAIE,KAAK,KAAK,GAAG,QAAQA,KAAK,MAAM,GAAG,MAAM;QAClDG,QAAQ,IAAI,CACV,CAAC,gCAAgC,EAAEH,KAAK,KAAK,CAAC,CAAC,EAAEA,KAAK,MAAM,CAAC,2EAA2E,CAAC;QAE3IF,SAAS;IACX;AACF;AAEO,SAASM,WAAWC,KAAa;IACtC,MAAMC,UAAU/B,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAASjC,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAWlC,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYnC,KAAK,GAAG,IAAI8B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdd,IAAU,EACVe,UAAgB,EAChB7C,MAAgC;IAEhC,MAAM8C,cAAc9C,AAAW,oBAAXA,SAA6B,MAAM;IACvD,MAAM+C,iBAAiB;IAGvB,MAAMC,wBACJlB,KAAK,KAAK,GAAGgB,cACTtC,KAAK,IAAI,CAAEsC,AAAAA,CAAAA,cAAchB,KAAK,KAAI,IAAK,KACvCiB;IACN,MAAME,sBACJnB,KAAK,MAAM,GAAGgB,cACVtC,KAAK,IAAI,CAAEsC,AAAAA,CAAAA,cAAchB,KAAK,MAAK,IAAK,KACxCiB;IAGN,IAAIG,WAAW1C,KAAK,GAAG,CAACsC,aAAahB,KAAK,KAAK,GAAGkB,AAAwB,IAAxBA;IAClD,IAAIG,YAAY3C,KAAK,GAAG,CAACsC,aAAahB,KAAK,MAAM,GAAGmB,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUtB,KAAK,IAAI,GAAGkB;IAC1B,IAAIK,SAASvB,KAAK,GAAG,GAAGmB;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAU5C,KAAK,GAAG,CAAC,GAAG4C;IACtBC,SAAS7C,KAAK,GAAG,CAAC,GAAG6C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCvB,KAAK,IAAI,GAAGsB;IACZtB,KAAK,GAAG,GAAGuB;IACXvB,KAAK,KAAK,GAAGoB;IACbpB,KAAK,MAAM,GAAGqB;IAEd,OAAOrB;AACT;AAEO,eAAewB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCvB,IAAU;IAEV,MAAMwB,eAAeC,WAAWF;IAChC,MAAMG,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBR;QAChB,sBAAsBI;QACtB1B;IACF;IACA,OAAO6B;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC,EAChCC,KAAc;IAEd,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQJ,MAAO;QACxB,MAAMK,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASL,YAAY,IAAI,CAAC,CAACK,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXnC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEkC,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAME,UAAUD,OAAO,cAAc,IAAID;QACzC,MAAMG,YAAYF,OAAO,WAAW,GAChCG,gBAAgBL,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMI,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAL,KAAK,IAAI,CAACO;IACZ;IAEA,IAAIR,OACFC,KAAK,IAAI,CAAC;QACRD;IACF;IAGF,OAAOC;AACT;AAGO,MAAMQ,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAE5B,MAAMC,yBAAyBN,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAGP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;QAGjCC,mBAKAC;IANJ,IAAIC,cAAcH;IAClB,IAAIC,AAAAA,SAAAA,CAAAA,oBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,kBAAkB,QAAQ,AAAD,MAAM,eACjCE,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAID,AAAAA,SAAAA,CAAAA,qBAAAA,YAAY,IAAI,AAAD,IAAfA,KAAAA,IAAAA,mBAAkB,QAAQ,AAAD,MAAM,aAAa;QAC9C,MAAME,QAAQD,YAAY,IAAI,CAAC,KAAK;QACpC,OAAOP,uBAAuBQ;IAChC;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACL;IACvC5E,OACE2E,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOM,OAAON;AAChB;AAEO,MAAMO,8BAA8B,CACzCC,SACAC;QAQIC;IANJ,IAAI,CAACF,SACH,OAAO,EAAE;IAIX,MAAMG,YAAYH;IAClB,IAAIE,AAAAA,SAAAA,CAAAA,kBAAAA,UAAU,IAAI,AAAD,IAAbA,KAAAA,IAAAA,gBAAgB,QAAQ,AAAD,MAAM,eAAeC,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMd,QAAQW,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACf,uBAAuBC,QAC1B,OAAO;YAIT,IAAIS,cAAc;oBACTM;gBAAP,OAAOA,AAAAA,SAAAA,CAAAA,cAAAA,MAAM,IAAI,AAAD,IAATA,KAAAA,IAAAA,YAAY,QAAQ,AAAD,MAAM;YAClC;YAEA,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAM3B,kBAAkB,CAC7B4B,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAMhG,SAAS;QAAE,GAAG+F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAanG,MAAM,CAACkG,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACTnG,MAAM,CAACkG,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1BnG,MAAM,CAACkG,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxBnG,MAAM,CAACkG,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAOnG;AACT;AAEO,MAAMoG,kBAAkB,CAC7BL,YACAC;IAEA,MAAMC,gBAAgBX,4BAA4BU;IAClD,MAAMhG,SAAS;QAAE,GAAG+F,UAAU;IAAC;IAE/B,KAAK,MAAMG,aAAaD,cAAe;QACrC,MAAME,aAAanG,MAAM,CAACkG,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvBnG,MAAM,CAACkG,UAAU,GAAG;YAClB,CAACvB,oBAAoB,EAAE;YACvB,QAAQwB;QACV;IAEJ;IAEA,OAAOnG;AACT"}

package/dist/es/ai-model/prompt/playwright-generator.mjs

@@ -31,7 +31,7 @@ Generated code should:
 6. Follow best practices for Playwright tests
 7. Be ready to execute without further modification
 
-Respond ONLY with the complete Playwright test code, no explanations.`;
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
     const messageContent = createMessageContent(promptText, screenshots, false !== options.includeScreenshots);
     const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
 Your task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.
@@ -47,7 +47,7 @@ ${PLAYWRIGHT_EXAMPLE_CODE}`;
             content: messageContent
         }
     ];
-    const response = await callAIWithStringResponse(prompt, AIActionType.EXTRACT_DATA, modelConfig);
+    const response = await callAIWithStringResponse(prompt, AIActionType.TEXT, modelConfig);
     if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return response.content;
     throw new Error('Failed to generate Playwright test code');
 };
@@ -82,7 +82,7 @@ Generated code should:
 7. Be ready to execute without further modification
 8. can't wrap this test code in markdown code block
 
-Respond ONLY with the complete Playwright test code, no explanations.`;
+Important: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \`\`\`typescript, \`\`\`javascript or \`\`\`). Start directly with the code content.`;
     const messageContent = createMessageContent(promptText, screenshots, false !== options.includeScreenshots);
     const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. 
 Your task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.
@@ -98,12 +98,12 @@ ${PLAYWRIGHT_EXAMPLE_CODE}`;
             content: messageContent
         }
     ];
-    if (options.stream && options.onChunk) return await callAI(prompt, AIActionType.EXTRACT_DATA, modelConfig, {
+    if (options.stream && options.onChunk) return await callAI(prompt, AIActionType.TEXT, modelConfig, {
         stream: true,
         onChunk: options.onChunk
     });
     {
-        const response = await callAIWithStringResponse(prompt, AIActionType.EXTRACT_DATA, modelConfig);
+        const response = await callAIWithStringResponse(prompt, AIActionType.TEXT, modelConfig);
         if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return {
             content: response.content,
             usage: response.usage,

package/dist/es/ai-model/prompt/playwright-generator.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/playwright-generator.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/playwright-generator.ts"],"sourcesContent":["import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType, callAI, callAIWithStringResponse } from '../index';\n// Import shared utilities and types from yaml-generator\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './yaml-generator';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nRespond ONLY with the complete Playwright test code, no explanations.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(\n    prompt,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nRespond ONLY with the complete Playwright test code, no explanations.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, AIActionType.EXTRACT_DATA, modelConfig, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(\n      prompt,\n      AIActionType.EXTRACT_DATA,\n      modelConfig,\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["generatePlaywrightTest","events","options","modelConfig","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","AIActionType","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;AA6DO,MAAMA,yBAAyB,OACpCC,QACAC,SACAC;IAGAC,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;qEAWwB,CAAC;IAGpE,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,yBACrBF,QACAG,aAAa,YAAY,EACzBhB;IAGF,IAAIc,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIG,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CpB,QACAC,SACAC;IAGAC,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;qEAYwB,CAAC;IAGpE,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMoB,OAAON,QAAQG,aAAa,YAAY,EAAEhB,aAAa;QAClE,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,yBACrBF,QACAG,aAAa,YAAY,EACzBhB;QAGF,IAAIc,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIG,MAAM;IAClB;AACF"}
+{"version":3,"file":"ai-model/prompt/playwright-generator.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/playwright-generator.ts"],"sourcesContent":["import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { PLAYWRIGHT_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType, callAI, callAIWithStringResponse } from '../index';\n// Import shared utilities and types from yaml-generator\nimport {\n  type ChromeRecordedEvent,\n  type EventCounts,\n  type EventSummary,\n  type InputDescription,\n  type ProcessedEvent,\n  createEventCounts,\n  createMessageContent,\n  extractInputDescriptions,\n  filterEventsByType,\n  getScreenshotsForLLM,\n  prepareEventSummary,\n  processEventsForLLM,\n  validateEvents,\n} from './yaml-generator';\n\n// Playwright-specific interfaces\nexport interface PlaywrightGenerationOptions {\n  testName?: string;\n  includeScreenshots?: boolean;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n  viewportSize?: { width: number; height: number };\n  waitForNetworkIdle?: boolean;\n  waitForNetworkIdleTimeout?: number;\n}\n\n// Re-export shared types for backward compatibility\nexport type {\n  ChromeRecordedEvent,\n  EventCounts,\n  InputDescription,\n  ProcessedEvent,\n  EventSummary,\n};\n\n// Re-export shared utilities for backward compatibility\nexport {\n  getScreenshotsForLLM,\n  filterEventsByType,\n  createEventCounts,\n  extractInputDescriptions,\n  processEventsForLLM,\n  prepareEventSummary,\n  createMessageContent,\n  validateEvents,\n};\n\n/**\n * Generates Playwright test code from recorded events\n */\nexport const generatePlaywrightTest = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  const response = await callAIWithStringResponse(\n    prompt,\n    AIActionType.TEXT,\n    modelConfig,\n  );\n\n  if (response?.content && typeof response.content === 'string') {\n    return response.content;\n  }\n\n  throw new Error('Failed to generate Playwright test code');\n};\n\n/**\n * Generates Playwright test code from recorded events with streaming support\n */\nexport const generatePlaywrightTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: PlaywrightGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  // Validate input\n  validateEvents(events);\n\n  // Prepare event summary using shared utilities\n  const summary = prepareEventSummary(events, {\n    testName: options.testName,\n    maxScreenshots: options.maxScreenshots || 3,\n  });\n\n  // Add Playwright-specific options to summary\n  const playwrightSummary = {\n    ...summary,\n    waitForNetworkIdle: options.waitForNetworkIdle !== false,\n    waitForNetworkIdleTimeout: options.waitForNetworkIdleTimeout || 2000,\n    viewportSize: options.viewportSize || { width: 1280, height: 800 },\n  };\n\n  // Get screenshots for visual context\n  const screenshots = getScreenshotsForLLM(events, options.maxScreenshots || 3);\n\n  // Create prompt text\n  const promptText = `Generate a Playwright test using @midscene/web/playwright that reproduces this recorded browser session. The test should be based on the following events and follow the structure of the example provided. Make the test descriptive with appropriate assertions and validations.\n\nEvent Summary:\n${JSON.stringify(playwrightSummary, null, 2)}\n\nGenerated code should:\n1. Import required dependencies\n2. Set up the test with proper configuration\n3. Include a beforeEach hook to navigate to the starting URL\n4. Implement a test that uses Midscene AI methods (aiTap, aiInput, aiAssert, etc.)\n5. Include appropriate assertions and validations\n6. Follow best practices for Playwright tests\n7. Be ready to execute without further modification\n8. can't wrap this test code in markdown code block\n\nImportant: Return ONLY the raw Playwright test code. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`typescript, \\`\\`\\`javascript or \\`\\`\\`). Start directly with the code content.`;\n\n  // Create message content with screenshots\n  const messageContent = createMessageContent(\n    promptText,\n    screenshots,\n    options.includeScreenshots !== false,\n  );\n\n  // Create system prompt\n  const systemPrompt = `You are an expert test automation engineer specializing in Playwright and Midscene. \nYour task is to generate a complete, executable Playwright test using @midscene/web/playwright that reproduces a recorded browser session.\n\n${PLAYWRIGHT_EXAMPLE_CODE}`;\n\n  // Use LLM to generate the Playwright test code with streaming\n  const prompt: ChatCompletionMessageParam[] = [\n    {\n      role: 'system',\n      content: systemPrompt,\n    },\n    {\n      role: 'user',\n      content: messageContent,\n    },\n  ];\n\n  if (options.stream && options.onChunk) {\n    // Use streaming\n    return await callAI(prompt, AIActionType.TEXT, modelConfig, {\n      stream: true,\n      onChunk: options.onChunk,\n    });\n  } else {\n    // Fallback to non-streaming\n    const response = await callAIWithStringResponse(\n      prompt,\n      AIActionType.TEXT,\n      modelConfig,\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return {\n        content: response.content,\n        usage: response.usage,\n        isStreamed: false,\n      };\n    }\n\n    throw new Error('Failed to generate Playwright test code');\n  }\n};\n"],"names":["generatePlaywrightTest","events","options","modelConfig","validateEvents","summary","prepareEventSummary","playwrightSummary","screenshots","getScreenshotsForLLM","promptText","JSON","messageContent","createMessageContent","systemPrompt","PLAYWRIGHT_EXAMPLE_CODE","prompt","response","callAIWithStringResponse","AIActionType","Error","generatePlaywrightTestStream","callAI"],"mappings":";;;AA6DO,MAAMA,yBAAyB,OACpCC,QACAC,SACAC;IAGAC,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;8LAWiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,MAAMK,WAAW,MAAMC,yBACrBF,QACAG,aAAa,IAAI,EACjBhB;IAGF,IAAIc,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;IAGzB,MAAM,IAAIG,MAAM;AAClB;AAKO,MAAMC,+BAA+B,OAC1CpB,QACAC,SACAC;IAGAC,eAAeH;IAGf,MAAMI,UAAUC,oBAAoBL,QAAQ;QAC1C,UAAUC,QAAQ,QAAQ;QAC1B,gBAAgBA,QAAQ,cAAc,IAAI;IAC5C;IAGA,MAAMK,oBAAoB;QACxB,GAAGF,OAAO;QACV,oBAAoBH,AAA+B,UAA/BA,QAAQ,kBAAkB;QAC9C,2BAA2BA,QAAQ,yBAAyB,IAAI;QAChE,cAAcA,QAAQ,YAAY,IAAI;YAAE,OAAO;YAAM,QAAQ;QAAI;IACnE;IAGA,MAAMM,cAAcC,qBAAqBR,QAAQC,QAAQ,cAAc,IAAI;IAG3E,MAAMQ,aAAa,CAAC;;;AAGtB,EAAEC,KAAK,SAAS,CAACJ,mBAAmB,MAAM,GAAG;;;;;;;;;;;;8LAYiJ,CAAC;IAG7L,MAAMK,iBAAiBC,qBACrBH,YACAF,aACAN,AAA+B,UAA/BA,QAAQ,kBAAkB;IAI5B,MAAMY,eAAe,CAAC;;;AAGxB,EAAEC,yBAAyB;IAGzB,MAAMC,SAAuC;QAC3C;YACE,MAAM;YACN,SAASF;QACX;QACA;YACE,MAAM;YACN,SAASF;QACX;KACD;IAED,IAAIV,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAMoB,OAAON,QAAQG,aAAa,IAAI,EAAEhB,aAAa;QAC1D,QAAQ;QACR,SAASD,QAAQ,OAAO;IAC1B;IACK;QAEL,MAAMe,WAAW,MAAMC,yBACrBF,QACAG,aAAa,IAAI,EACjBhB;QAGF,IAAIc,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;YACL,SAASA,SAAS,OAAO;YACzB,OAAOA,SAAS,KAAK;YACrB,YAAY;QACd;QAGF,MAAM,IAAIG,MAAM;IAClB;AACF"}

package/dist/es/ai-model/prompt/yaml-generator.mjs

@@ -123,7 +123,7 @@ Convert events:
 - scroll \u{2192} aiScroll with appropriate direction
 - Add aiAssert for important state changes
 
-Respond with YAML only, no explanations.`
+Important: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \`\`\`yaml or \`\`\`). Start directly with the YAML content.`
             }
         ];
         if (screenshots.length > 0) {
@@ -141,7 +141,7 @@ Respond with YAML only, no explanations.`
                     }))
             });
         }
-        const response = await callAIWithStringResponse(prompt, AIActionType.EXTRACT_DATA, modelConfig);
+        const response = await callAIWithStringResponse(prompt, AIActionType.TEXT, modelConfig);
         if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return response.content;
         throw new Error('Failed to generate YAML test configuration');
     } catch (error) {
@@ -179,7 +179,7 @@ Convert events:
 - scroll \u{2192} aiScroll with appropriate direction
 - Add aiAssert for important state changes
 
-Respond with YAML only, no explanations.`
+Important: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \`\`\`yaml or \`\`\`). Start directly with the YAML content.`
             }
         ];
         if (screenshots.length > 0) {
@@ -197,12 +197,12 @@ Respond with YAML only, no explanations.`
                     }))
             });
         }
-        if (options.stream && options.onChunk) return await callAI(prompt, AIActionType.EXTRACT_DATA, modelConfig, {
+        if (options.stream && options.onChunk) return await callAI(prompt, AIActionType.TEXT, modelConfig, {
             stream: true,
             onChunk: options.onChunk
         });
         {
-            const response = await callAIWithStringResponse(prompt, AIActionType.EXTRACT_DATA, modelConfig);
+            const response = await callAIWithStringResponse(prompt, AIActionType.TEXT, modelConfig);
             if ((null == response ? void 0 : response.content) && 'string' == typeof response.content) return {
                 content: response.content,
                 usage: response.usage,

package/dist/es/ai-model/prompt/yaml-generator.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/prompt/yaml-generator.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  AIActionType,\n  type ChatCompletionMessageParam,\n  callAI,\n  callAIWithStringResponse,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotBefore?: string;\n  screenshotAfter?: string;\n  screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  // Find events with screenshots, prioritizing navigation and click events\n  const eventsWithScreenshots = events.filter(\n    (event) =>\n      event.screenshotBefore ||\n      event.screenshotAfter ||\n      event.screenshotWithBox,\n  );\n\n  // Sort them by priority (navigation first, then clicks, then others)\n  const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n    if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n    if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n    if (a.type === 'click' && b.type !== 'click') return -1;\n    if (a.type !== 'click' && b.type === 'click') return 1;\n    return 0;\n  });\n\n  // Extract up to maxScreenshots screenshots\n  const screenshots: string[] = [];\n  for (const event of sortedEvents) {\n    // Prefer the most informative screenshot\n    const screenshot =\n      event.screenshotWithBox ||\n      event.screenshotAfter ||\n      event.screenshotBefore;\n    if (screenshot && !screenshots.includes(screenshot)) {\n      screenshots.push(screenshot);\n      if (screenshots.length >= maxScreenshots) break;\n    }\n  }\n\n  return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => ({\n      description: event.elementDescription || '',\n      value: event.value || '',\n    }))\n    .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n  return events.map((event) => ({\n    type: event.type,\n    timestamp: event.timestamp,\n    url: event.url,\n    title: event.title,\n    elementDescription: event.elementDescription,\n    value: event.value,\n    pageInfo: event.pageInfo,\n    elementRect: event.elementRect,\n  }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  // Extract useful information from events\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => event.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(events);\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  // Add screenshots if available and requested\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nRespond with YAML only, no explanations.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    const response = await callAIWithStringResponse(\n      prompt,\n      AIActionType.EXTRACT_DATA,\n      modelConfig,\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return response.content;\n    }\n\n    throw new Error('Failed to generate YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nRespond with YAML only, no explanations.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    if (options.stream && options.onChunk) {\n      // Use streaming\n      return await callAI(prompt, AIActionType.EXTRACT_DATA, modelConfig, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    } else {\n      // Fallback to non-streaming\n      const response = await callAIWithStringResponse(\n        prompt,\n        AIActionType.EXTRACT_DATA,\n        modelConfig,\n      );\n\n      if (response?.content && typeof response.content === 'string') {\n        return {\n          content: response.content,\n          usage: response.usage,\n          isStreamed: false,\n        };\n      }\n\n      throw new Error('Failed to generate YAML test configuration');\n    }\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n"],"names":["getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","validateEvents","Error","generateYamlTest","modelConfig","summary","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","response","callAIWithStringResponse","AIActionType","error","generateYamlTestStream","callAI"],"mappings":";;AAkFO,MAAMA,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAKO,MAAMC,iBAAiB,CAACjC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIkC,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9BnC,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,mBAAmB;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;wCASC,CAAC;YACnC;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,MAAMkC,WAAW,MAAMC,yBACrBJ,QACAK,aAAa,YAAY,EACzBR;QAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIR,MAAM;IAClB,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpC9C,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,mBAAmB;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;wCASC,CAAC;YACnC;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,IAAIU,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAM6B,OAAOR,QAAQK,aAAa,YAAY,EAAER,aAAa;YAClE,QAAQ;YACR,SAASlB,QAAQ,OAAO;QAC1B;QACK;YAEL,MAAMwB,WAAW,MAAMC,yBACrBJ,QACAK,aAAa,YAAY,EACzBR;YAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIR,MAAM;QAClB;IACF,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF"}
+{"version":3,"file":"ai-model/prompt/yaml-generator.mjs","sources":["webpack://@midscene/core/./src/ai-model/prompt/yaml-generator.ts"],"sourcesContent":["import type {\n  StreamingAIResponse,\n  StreamingCodeGenerationOptions,\n} from '@/types';\nimport { YAML_EXAMPLE_CODE } from '@midscene/shared/constants';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport {\n  AIActionType,\n  type ChatCompletionMessageParam,\n  callAI,\n  callAIWithStringResponse,\n} from '../index';\n\n// Common interfaces for test generation (shared between YAML and Playwright)\nexport interface EventCounts {\n  navigation: number;\n  click: number;\n  input: number;\n  scroll: number;\n  total: number;\n}\n\nexport interface InputDescription {\n  description: string;\n  value: string;\n}\n\nexport interface ProcessedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n}\n\nexport interface EventSummary {\n  testName: string;\n  startUrl: string;\n  eventCounts: EventCounts;\n  urls: string[];\n  clickDescriptions: string[];\n  inputDescriptions: InputDescription[];\n  events: ProcessedEvent[];\n}\n\n// Common ChromeRecordedEvent interface\nexport interface ChromeRecordedEvent {\n  type: string;\n  timestamp: number;\n  url?: string;\n  title?: string;\n  elementDescription?: string;\n  value?: string;\n  pageInfo?: any;\n  elementRect?: any;\n  screenshotBefore?: string;\n  screenshotAfter?: string;\n  screenshotWithBox?: string;\n}\n\nexport interface YamlGenerationOptions {\n  testName?: string;\n  includeTimestamps?: boolean;\n  maxScreenshots?: number;\n  description?: string;\n}\n\nexport interface FilteredEvents {\n  navigationEvents: ChromeRecordedEvent[];\n  clickEvents: ChromeRecordedEvent[];\n  inputEvents: ChromeRecordedEvent[];\n  scrollEvents: ChromeRecordedEvent[];\n}\n\n// Common utility functions (shared between YAML and Playwright generators)\n\n/**\n * Get screenshots from events for LLM context\n */\nexport const getScreenshotsForLLM = (\n  events: ChromeRecordedEvent[],\n  maxScreenshots = 1,\n): string[] => {\n  // Find events with screenshots, prioritizing navigation and click events\n  const eventsWithScreenshots = events.filter(\n    (event) =>\n      event.screenshotBefore ||\n      event.screenshotAfter ||\n      event.screenshotWithBox,\n  );\n\n  // Sort them by priority (navigation first, then clicks, then others)\n  const sortedEvents = [...eventsWithScreenshots].sort((a, b) => {\n    if (a.type === 'navigation' && b.type !== 'navigation') return -1;\n    if (a.type !== 'navigation' && b.type === 'navigation') return 1;\n    if (a.type === 'click' && b.type !== 'click') return -1;\n    if (a.type !== 'click' && b.type === 'click') return 1;\n    return 0;\n  });\n\n  // Extract up to maxScreenshots screenshots\n  const screenshots: string[] = [];\n  for (const event of sortedEvents) {\n    // Prefer the most informative screenshot\n    const screenshot =\n      event.screenshotWithBox ||\n      event.screenshotAfter ||\n      event.screenshotBefore;\n    if (screenshot && !screenshots.includes(screenshot)) {\n      screenshots.push(screenshot);\n      if (screenshots.length >= maxScreenshots) break;\n    }\n  }\n\n  return screenshots;\n};\n\n/**\n * Filter events by type for easier processing\n */\nexport const filterEventsByType = (\n  events: ChromeRecordedEvent[],\n): FilteredEvents => {\n  return {\n    navigationEvents: events.filter((event) => event.type === 'navigation'),\n    clickEvents: events.filter((event) => event.type === 'click'),\n    inputEvents: events.filter((event) => event.type === 'input'),\n    scrollEvents: events.filter((event) => event.type === 'scroll'),\n  };\n};\n\n/**\n * Create event counts summary\n */\nexport const createEventCounts = (\n  filteredEvents: FilteredEvents,\n  totalEvents: number,\n): EventCounts => {\n  return {\n    navigation: filteredEvents.navigationEvents.length,\n    click: filteredEvents.clickEvents.length,\n    input: filteredEvents.inputEvents.length,\n    scroll: filteredEvents.scrollEvents.length,\n    total: totalEvents,\n  };\n};\n\n/**\n * Extract input descriptions from input events\n */\nexport const extractInputDescriptions = (\n  inputEvents: ChromeRecordedEvent[],\n): InputDescription[] => {\n  return inputEvents\n    .map((event) => ({\n      description: event.elementDescription || '',\n      value: event.value || '',\n    }))\n    .filter((item) => item.description && item.value);\n};\n\n/**\n * Process events for LLM consumption\n */\nexport const processEventsForLLM = (\n  events: ChromeRecordedEvent[],\n): ProcessedEvent[] => {\n  return events.map((event) => ({\n    type: event.type,\n    timestamp: event.timestamp,\n    url: event.url,\n    title: event.title,\n    elementDescription: event.elementDescription,\n    value: event.value,\n    pageInfo: event.pageInfo,\n    elementRect: event.elementRect,\n  }));\n};\n\n/**\n * Prepare comprehensive event summary for LLM\n */\nexport const prepareEventSummary = (\n  events: ChromeRecordedEvent[],\n  options: { testName?: string; maxScreenshots?: number } = {},\n): EventSummary => {\n  const filteredEvents = filterEventsByType(events);\n  const eventCounts = createEventCounts(filteredEvents, events.length);\n\n  // Extract useful information from events\n  const startUrl =\n    filteredEvents.navigationEvents.length > 0\n      ? filteredEvents.navigationEvents[0].url || ''\n      : '';\n\n  const clickDescriptions = filteredEvents.clickEvents\n    .map((event) => event.elementDescription)\n    .filter((desc): desc is string => Boolean(desc))\n    .slice(0, 10);\n\n  const inputDescriptions = extractInputDescriptions(\n    filteredEvents.inputEvents,\n  ).slice(0, 10);\n\n  const urls = filteredEvents.navigationEvents\n    .map((e) => e.url)\n    .filter((url): url is string => Boolean(url))\n    .slice(0, 5);\n\n  const processedEvents = processEventsForLLM(events);\n\n  return {\n    testName: options.testName || 'Automated test from recorded events',\n    startUrl,\n    eventCounts,\n    urls,\n    clickDescriptions,\n    inputDescriptions,\n    events: processedEvents,\n  };\n};\n\n/**\n * Create message content for LLM with optional screenshots\n */\nexport const createMessageContent = (\n  promptText: string,\n  screenshots: string[] = [],\n  includeScreenshots = true,\n) => {\n  const messageContent: any[] = [\n    {\n      type: 'text',\n      text: promptText,\n    },\n  ];\n\n  // Add screenshots if available and requested\n  if (includeScreenshots && screenshots.length > 0) {\n    messageContent.unshift({\n      type: 'text',\n      text: 'Here are screenshots from the recording session to help you understand the context:',\n    });\n\n    screenshots.forEach((screenshot) => {\n      messageContent.push({\n        type: 'image_url',\n        image_url: {\n          url: screenshot,\n        },\n      });\n    });\n  }\n\n  return messageContent;\n};\n\n/**\n * Validate events before processing\n */\nexport const validateEvents = (events: ChromeRecordedEvent[]): void => {\n  if (!events.length) {\n    throw new Error('No events provided for test generation');\n  }\n};\n\n// YAML-specific generation functions\n\n/**\n * Generates YAML test configuration from recorded events using AI\n */\nexport const generateYamlTest = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<string> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    const response = await callAIWithStringResponse(\n      prompt,\n      AIActionType.TEXT,\n      modelConfig,\n    );\n\n    if (response?.content && typeof response.content === 'string') {\n      return response.content;\n    }\n\n    throw new Error('Failed to generate YAML test configuration');\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n\n/**\n * Generates YAML test configuration from recorded events using AI with streaming support\n */\nexport const generateYamlTestStream = async (\n  events: ChromeRecordedEvent[],\n  options: YamlGenerationOptions & StreamingCodeGenerationOptions,\n  modelConfig: IModelConfig,\n): Promise<StreamingAIResponse> => {\n  try {\n    // Validate input\n    validateEvents(events);\n\n    // Prepare event summary using shared utilities\n    const summary = prepareEventSummary(events, {\n      testName: options.testName,\n      maxScreenshots: options.maxScreenshots || 3,\n    });\n\n    // Add YAML-specific options to summary\n    const yamlSummary = {\n      ...summary,\n      includeTimestamps: options.includeTimestamps || false,\n    };\n\n    // Get screenshots for visual context\n    const screenshots = getScreenshotsForLLM(\n      events,\n      options.maxScreenshots || 3,\n    );\n\n    // Use LLM to generate the YAML test configuration\n    const prompt: ChatCompletionMessageParam[] = [\n      {\n        role: 'system',\n        content: `You are an expert in Midscene.js YAML test generation. Generate clean, accurate YAML following these rules: ${YAML_EXAMPLE_CODE}`,\n      },\n      {\n        role: 'user',\n        content: `Generate YAML test for Midscene.js automation from recorded browser events.\n\nEvent Summary:\n${JSON.stringify(yamlSummary, null, 2)}\n\nConvert events:\n- navigation → target.url\n- click → aiTap with element description\n- input → aiInput with value and locate\n- scroll → aiScroll with appropriate direction\n- Add aiAssert for important state changes\n\nImportant: Return ONLY the raw YAML content. Do NOT wrap the response in markdown code blocks (no \\`\\`\\`yaml or \\`\\`\\`). Start directly with the YAML content.`,\n      },\n    ];\n\n    // Add screenshots if available and requested\n    if (screenshots.length > 0) {\n      prompt.push({\n        role: 'user',\n        content:\n          'Here are screenshots from the recording session to help you understand the context:',\n      });\n\n      prompt.push({\n        role: 'user',\n        content: screenshots.map((screenshot) => ({\n          type: 'image_url',\n          image_url: {\n            url: screenshot,\n          },\n        })),\n      });\n    }\n\n    if (options.stream && options.onChunk) {\n      // Use streaming\n      return await callAI(prompt, AIActionType.TEXT, modelConfig, {\n        stream: true,\n        onChunk: options.onChunk,\n      });\n    } else {\n      // Fallback to non-streaming\n      const response = await callAIWithStringResponse(\n        prompt,\n        AIActionType.TEXT,\n        modelConfig,\n      );\n\n      if (response?.content && typeof response.content === 'string') {\n        return {\n          content: response.content,\n          usage: response.usage,\n          isStreamed: false,\n        };\n      }\n\n      throw new Error('Failed to generate YAML test configuration');\n    }\n  } catch (error) {\n    throw new Error(`Failed to generate YAML test: ${error}`);\n  }\n};\n"],"names":["getScreenshotsForLLM","events","maxScreenshots","eventsWithScreenshots","event","sortedEvents","a","b","screenshots","screenshot","filterEventsByType","createEventCounts","filteredEvents","totalEvents","extractInputDescriptions","inputEvents","item","processEventsForLLM","prepareEventSummary","options","eventCounts","startUrl","clickDescriptions","desc","Boolean","inputDescriptions","urls","e","url","processedEvents","createMessageContent","promptText","includeScreenshots","messageContent","validateEvents","Error","generateYamlTest","modelConfig","summary","yamlSummary","prompt","YAML_EXAMPLE_CODE","JSON","response","callAIWithStringResponse","AIActionType","error","generateYamlTestStream","callAI"],"mappings":";;AAkFO,MAAMA,uBAAuB,CAClCC,QACAC,iBAAiB,CAAC;IAGlB,MAAMC,wBAAwBF,OAAO,MAAM,CACzC,CAACG,QACCA,MAAM,gBAAgB,IACtBA,MAAM,eAAe,IACrBA,MAAM,iBAAiB;IAI3B,MAAMC,eAAe;WAAIF;KAAsB,CAAC,IAAI,CAAC,CAACG,GAAGC;QACvD,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,iBAAXA,EAAE,IAAI,IAAqBC,AAAW,iBAAXA,EAAE,IAAI,EAAmB,OAAO;QAC/D,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,IAAID,AAAW,YAAXA,EAAE,IAAI,IAAgBC,AAAW,YAAXA,EAAE,IAAI,EAAc,OAAO;QACrD,OAAO;IACT;IAGA,MAAMC,cAAwB,EAAE;IAChC,KAAK,MAAMJ,SAASC,aAAc;QAEhC,MAAMI,aACJL,MAAM,iBAAiB,IACvBA,MAAM,eAAe,IACrBA,MAAM,gBAAgB;QACxB,IAAIK,cAAc,CAACD,YAAY,QAAQ,CAACC,aAAa;YACnDD,YAAY,IAAI,CAACC;YACjB,IAAID,YAAY,MAAM,IAAIN,gBAAgB;QAC5C;IACF;IAEA,OAAOM;AACT;AAKO,MAAME,qBAAqB,CAChCT,SAEO;QACL,kBAAkBA,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,iBAAfA,MAAM,IAAI;QACrD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,aAAaH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,YAAfA,MAAM,IAAI;QAChD,cAAcH,OAAO,MAAM,CAAC,CAACG,QAAUA,AAAe,aAAfA,MAAM,IAAI;IACnD;AAMK,MAAMO,oBAAoB,CAC/BC,gBACAC,cAEO;QACL,YAAYD,eAAe,gBAAgB,CAAC,MAAM;QAClD,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,OAAOA,eAAe,WAAW,CAAC,MAAM;QACxC,QAAQA,eAAe,YAAY,CAAC,MAAM;QAC1C,OAAOC;IACT;AAMK,MAAMC,2BAA2B,CACtCC,cAEOA,YACJ,GAAG,CAAC,CAACX,QAAW;YACf,aAAaA,MAAM,kBAAkB,IAAI;YACzC,OAAOA,MAAM,KAAK,IAAI;QACxB,IACC,MAAM,CAAC,CAACY,OAASA,KAAK,WAAW,IAAIA,KAAK,KAAK;AAM7C,MAAMC,sBAAsB,CACjChB,SAEOA,OAAO,GAAG,CAAC,CAACG,QAAW;YAC5B,MAAMA,MAAM,IAAI;YAChB,WAAWA,MAAM,SAAS;YAC1B,KAAKA,MAAM,GAAG;YACd,OAAOA,MAAM,KAAK;YAClB,oBAAoBA,MAAM,kBAAkB;YAC5C,OAAOA,MAAM,KAAK;YAClB,UAAUA,MAAM,QAAQ;YACxB,aAAaA,MAAM,WAAW;QAChC;AAMK,MAAMc,sBAAsB,CACjCjB,QACAkB,UAA0D,CAAC,CAAC;IAE5D,MAAMP,iBAAiBF,mBAAmBT;IAC1C,MAAMmB,cAAcT,kBAAkBC,gBAAgBX,OAAO,MAAM;IAGnE,MAAMoB,WACJT,eAAe,gBAAgB,CAAC,MAAM,GAAG,IACrCA,eAAe,gBAAgB,CAAC,EAAE,CAAC,GAAG,IAAI,KAC1C;IAEN,MAAMU,oBAAoBV,eAAe,WAAW,CACjD,GAAG,CAAC,CAACR,QAAUA,MAAM,kBAAkB,EACvC,MAAM,CAAC,CAACmB,OAAyBC,QAAQD,OACzC,KAAK,CAAC,GAAG;IAEZ,MAAME,oBAAoBX,yBACxBF,eAAe,WAAW,EAC1B,KAAK,CAAC,GAAG;IAEX,MAAMc,OAAOd,eAAe,gBAAgB,CACzC,GAAG,CAAC,CAACe,IAAMA,EAAE,GAAG,EAChB,MAAM,CAAC,CAACC,MAAuBJ,QAAQI,MACvC,KAAK,CAAC,GAAG;IAEZ,MAAMC,kBAAkBZ,oBAAoBhB;IAE5C,OAAO;QACL,UAAUkB,QAAQ,QAAQ,IAAI;QAC9BE;QACAD;QACAM;QACAJ;QACAG;QACA,QAAQI;IACV;AACF;AAKO,MAAMC,uBAAuB,CAClCC,YACAvB,cAAwB,EAAE,EAC1BwB,qBAAqB,IAAI;IAEzB,MAAMC,iBAAwB;QAC5B;YACE,MAAM;YACN,MAAMF;QACR;KACD;IAGD,IAAIC,sBAAsBxB,YAAY,MAAM,GAAG,GAAG;QAChDyB,eAAe,OAAO,CAAC;YACrB,MAAM;YACN,MAAM;QACR;QAEAzB,YAAY,OAAO,CAAC,CAACC;YACnBwB,eAAe,IAAI,CAAC;gBAClB,MAAM;gBACN,WAAW;oBACT,KAAKxB;gBACP;YACF;QACF;IACF;IAEA,OAAOwB;AACT;AAKO,MAAMC,iBAAiB,CAACjC;IAC7B,IAAI,CAACA,OAAO,MAAM,EAChB,MAAM,IAAIkC,MAAM;AAEpB;AAOO,MAAMC,mBAAmB,OAC9BnC,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,mBAAmB;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;8JASuH,CAAC;YACzJ;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,MAAMkC,WAAW,MAAMC,yBACrBJ,QACAK,aAAa,IAAI,EACjBR;QAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAOA,SAAS,OAAO;QAGzB,MAAM,IAAIR,MAAM;IAClB,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF;AAKO,MAAMC,yBAAyB,OACpC9C,QACAkB,SACAkB;IAEA,IAAI;QAEFH,eAAejC;QAGf,MAAMqC,UAAUpB,oBAAoBjB,QAAQ;YAC1C,UAAUkB,QAAQ,QAAQ;YAC1B,gBAAgBA,QAAQ,cAAc,IAAI;QAC5C;QAGA,MAAMoB,cAAc;YAClB,GAAGD,OAAO;YACV,mBAAmBnB,QAAQ,iBAAiB,IAAI;QAClD;QAGA,MAAMX,cAAcR,qBAClBC,QACAkB,QAAQ,cAAc,IAAI;QAI5B,MAAMqB,SAAuC;YAC3C;gBACE,MAAM;gBACN,SAAS,CAAC,4GAA4G,EAAEC,mBAAmB;YAC7I;YACA;gBACE,MAAM;gBACN,SAAS,CAAC;;;AAGlB,EAAEC,KAAK,SAAS,CAACH,aAAa,MAAM,GAAG;;;;;;;;;8JASuH,CAAC;YACzJ;SACD;QAGD,IAAI/B,YAAY,MAAM,GAAG,GAAG;YAC1BgC,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SACE;YACJ;YAEAA,OAAO,IAAI,CAAC;gBACV,MAAM;gBACN,SAAShC,YAAY,GAAG,CAAC,CAACC,aAAgB;wBACxC,MAAM;wBACN,WAAW;4BACT,KAAKA;wBACP;oBACF;YACF;QACF;QAEA,IAAIU,QAAQ,MAAM,IAAIA,QAAQ,OAAO,EAEnC,OAAO,MAAM6B,OAAOR,QAAQK,aAAa,IAAI,EAAER,aAAa;YAC1D,QAAQ;YACR,SAASlB,QAAQ,OAAO;QAC1B;QACK;YAEL,MAAMwB,WAAW,MAAMC,yBACrBJ,QACAK,aAAa,IAAI,EACjBR;YAGF,IAAIM,AAAAA,CAAAA,QAAAA,WAAAA,KAAAA,IAAAA,SAAU,OAAO,AAAD,KAAK,AAA4B,YAA5B,OAAOA,SAAS,OAAO,EAC9C,OAAO;gBACL,SAASA,SAAS,OAAO;gBACzB,OAAOA,SAAS,KAAK;gBACrB,YAAY;YACd;YAGF,MAAM,IAAIR,MAAM;QAClB;IACF,EAAE,OAAOW,OAAO;QACd,MAAM,IAAIX,MAAM,CAAC,8BAA8B,EAAEW,OAAO;IAC1D;AACF"}

package/dist/es/ai-model/service-caller/index.mjs

@@ -324,8 +324,11 @@ const getResponseFormat = (modelName, AIActionTypeValue)=>{
                 type: AIResponseFormat.JSON
             };
             break;
+        case AIActionType.TEXT:
+            responseFormat = void 0;
+            break;
     }
-    if ('gpt-4o-2024-05-13' === modelName) responseFormat = {
+    if ('gpt-4o-2024-05-13' === modelName && AIActionTypeValue !== AIActionType.TEXT) responseFormat = {
         type: AIResponseFormat.JSON
     };
     return responseFormat;

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport { Anthropic } from '@anthropic-ai/sdk';\nimport {\n  DefaultAzureCredential,\n  getBearerTokenProvider,\n} from '@azure/identity';\nimport {\n  type IModelConfig,\n  MIDSCENE_API_TYPE,\n  MIDSCENE_LANGSMITH_DEBUG,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { parseBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI, { AzureOpenAI } from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { locatorSchema } from '../prompt/llm-locator';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  style: 'openai' | 'anthropic';\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    openaiUseAzureDeprecated,\n    useAzureOpenai,\n    azureOpenaiScope,\n    azureOpenaiKey,\n    azureOpenaiEndpoint,\n    azureOpenaiApiVersion,\n    azureOpenaiDeployment,\n    azureExtraConfig,\n    useAnthropicSdk,\n    anthropicApiKey,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n  } = modelConfig;\n\n  let openai: OpenAI | AzureOpenAI | undefined;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  if (openaiUseAzureDeprecated) {\n    // this is deprecated\n    openai = new AzureOpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      dangerouslyAllowBrowser: true,\n    }) as OpenAI;\n  } else if (useAzureOpenai) {\n    // https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=bash%2Cjavascript-key%2Ctypescript-keyless%2Cpython&pivots=programming-language-javascript#rest-api\n    // keyless authentication\n    let tokenProvider: any = undefined;\n    if (azureOpenaiScope) {\n      assert(\n        !ifInBrowser,\n        'Azure OpenAI is not supported in browser with Midscene.',\n      );\n      const credential = new DefaultAzureCredential();\n\n      tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);\n\n      openai = new AzureOpenAI({\n        azureADTokenProvider: tokenProvider,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    } else {\n      // endpoint, apiKey, apiVersion, deployment\n      openai = new AzureOpenAI({\n        apiKey: azureOpenaiKey,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        dangerouslyAllowBrowser: true,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    }\n  } else if (!useAnthropicSdk) {\n    openai = new OpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      defaultHeaders: {\n        ...(openaiExtraConfig?.defaultHeaders || {}),\n        [MIDSCENE_API_TYPE]: AIActionTypeValue.toString(),\n      },\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    const { wrapOpenAI } = await import('langsmith/wrappers');\n    openai = wrapOpenAI(openai);\n  }\n\n  if (typeof openai !== 'undefined') {\n    return {\n      completion: openai.chat.completions,\n      style: 'openai',\n      modelName,\n      modelDescription,\n      uiTarsVersion,\n      vlMode,\n    };\n  }\n\n  // Anthropic\n  if (useAnthropicSdk) {\n    openai = new Anthropic({\n      apiKey: anthropicApiKey,\n      httpAgent: proxyAgent,\n      dangerouslyAllowBrowser: true,\n    }) as any;\n  }\n\n  if (typeof openai !== 'undefined' && (openai as any).messages) {\n    return {\n      completion: (openai as any).messages,\n      style: 'anthropic',\n      modelName,\n      modelDescription,\n      uiTarsVersion,\n      vlMode,\n    };\n  }\n\n  throw new Error('Openai SDK or Anthropic SDK is not initialized');\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const {\n    completion,\n    style,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  } = await createChatClient({\n    AIActionTypeValue,\n    modelConfig,\n  });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlMode === 'qwen-vl' // qwen specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    if (style === 'openai') {\n      debugCall(\n        `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n      );\n\n      if (isStreaming) {\n        const stream = (await completion.create(\n          {\n            model: modelName,\n            messages,\n            response_format: responseFormat,\n            ...commonConfig,\n          },\n          {\n            stream: true,\n          },\n        )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n          _request_id?: string | null;\n        };\n\n        for await (const chunk of stream) {\n          const content = chunk.choices?.[0]?.delta?.content || '';\n          const reasoning_content =\n            (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n          // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n          if (chunk.usage) {\n            usage = chunk.usage;\n          }\n\n          if (content || reasoning_content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              reasoning_content,\n              accumulated,\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.choices?.[0]?.finish_reason) {\n            timeCost = Date.now() - startTime;\n\n            // If usage is not available from the stream, provide a basic usage info\n            if (!usage) {\n              // Estimate token counts based on content length (rough approximation)\n              const estimatedTokens = Math.max(\n                1,\n                Math.floor(accumulated.length / 4),\n              );\n              usage = {\n                prompt_tokens: estimatedTokens,\n                completion_tokens: estimatedTokens,\n                total_tokens: estimatedTokens * 2,\n              };\n            }\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: {\n                prompt_tokens: usage.prompt_tokens ?? 0,\n                completion_tokens: usage.completion_tokens ?? 0,\n                total_tokens: usage.total_tokens ?? 0,\n                time_cost: timeCost ?? 0,\n                model_name: modelName,\n                model_description: modelDescription,\n                intent: modelConfig.intent,\n              },\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n        debugProfileStats(\n          `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n        );\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n\n        debugProfileStats(\n          `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n        );\n\n        debugProfileDetail(\n          `model usage detail: ${JSON.stringify(result.usage)}`,\n        );\n\n        assert(\n          result.choices,\n          `invalid response from LLM service: ${JSON.stringify(result)}`,\n        );\n        content = result.choices[0].message.content!;\n        usage = result.usage;\n      }\n\n      debugCall(`response: ${content}`);\n      assert(content, 'empty content');\n    } else if (style === 'anthropic') {\n      const convertImageContent = (content: any) => {\n        if (content.type === 'image_url') {\n          const imgBase64 = content.image_url.url;\n          assert(imgBase64, 'image_url is required');\n          const { mimeType, body } = parseBase64(content.image_url.url);\n          return {\n            source: {\n              type: 'base64',\n              media_type: mimeType,\n              data: body,\n            },\n            type: 'image',\n          };\n        }\n        return content;\n      };\n\n      if (isStreaming) {\n        const stream = (await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any)) as any;\n\n        for await (const chunk of stream) {\n          const content = chunk.delta?.text || '';\n          if (content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              accumulated,\n              reasoning_content: '',\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.type === 'message_stop') {\n            timeCost = Date.now() - startTime;\n            const anthropicUsage = chunk.usage;\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: anthropicUsage\n                ? {\n                    prompt_tokens: anthropicUsage.input_tokens ?? 0,\n                    completion_tokens: anthropicUsage.output_tokens ?? 0,\n                    total_tokens:\n                      (anthropicUsage.input_tokens ?? 0) +\n                      (anthropicUsage.output_tokens ?? 0),\n                    time_cost: timeCost ?? 0,\n                    model_name: modelName,\n                    model_description: modelDescription,\n                    intent: modelConfig.intent,\n                  }\n                : undefined,\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n        content = (result as any).content[0].text as string;\n        usage = result.usage;\n      }\n\n      assert(content, 'empty content');\n    }\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.INSPECT_ELEMENT:\n        responseFormat = locatorSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  if (modelName === 'gpt-4o-2024-05-13') {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","openaiUseAzureDeprecated","useAzureOpenai","azureOpenaiScope","azureOpenaiKey","azureOpenaiEndpoint","azureOpenaiApiVersion","azureOpenaiDeployment","azureExtraConfig","useAnthropicSdk","anthropicApiKey","modelDescription","uiTarsVersion","vlMode","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","AzureOpenAI","tokenProvider","assert","ifInBrowser","credential","DefaultAzureCredential","getBearerTokenProvider","OpenAI","MIDSCENE_API_TYPE","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","Error","console","wrapOpenAI","Anthropic","callAI","messages","options","completion","style","responseFormat","getResponseFormat","maxTokens","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","convertImageContent","imgBase64","mimeType","body","parseBase64","m","Array","_chunk_delta","anthropicUsage","e","newError","AIActionType","assertSchema","locatorSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;;;;;AAgCA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAQC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,wBAAwB,EACxBC,cAAc,EACdC,gBAAgB,EAChBC,cAAc,EACdC,mBAAmB,EACnBC,qBAAqB,EACrBC,qBAAqB,EACrBC,gBAAgB,EAChBC,eAAe,EACfC,eAAe,EACfC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACP,GAAGnB;IAEJ,IAAIoB;IAEJ,IAAIC;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIrB,WAAW;QACboB,WAAW,oBAAoBpB;QAC/BmB,aAAa,IAAIG,gBAAgBtB;IACnC,OAAO,IAAID,YAAY;QACrBqB,WAAW,qBAAqBrB;QAChCoB,aAAa,IAAII,gBAAgBxB;IACnC;IAEA,IAAIM,0BAEFa,SAAS,IAAIM,YAAY;QACvB,SAAStB;QACT,QAAQC;QACR,WAAWgB;QACX,GAAGf,iBAAiB;QACpB,yBAAyB;IAC3B;SACK,IAAIE,gBAAgB;QAGzB,IAAImB;QACJ,IAAIlB,kBAAkB;YACpBmB,OACE,CAACC,aACD;YAEF,MAAMC,aAAa,IAAIC;YAEvBJ,gBAAgBK,uBAAuBF,YAAYrB;YAEnDW,SAAS,IAAIM,YAAY;gBACvB,sBAAsBC;gBACtB,UAAUhB;gBACV,YAAYC;gBACZ,YAAYC;gBACZ,GAAGP,iBAAiB;gBACpB,GAAGQ,gBAAgB;YACrB;QACF,OAEEM,SAAS,IAAIM,YAAY;YACvB,QAAQhB;YACR,UAAUC;YACV,YAAYC;YACZ,YAAYC;YACZ,yBAAyB;YACzB,GAAGP,iBAAiB;YACpB,GAAGQ,gBAAgB;QACrB;IAEJ,OAAO,IAAI,CAACC,iBACVK,SAAS,IAAIa,SAAO;QAClB,SAAS7B;QACT,QAAQC;QACR,WAAWgB;QACX,GAAGf,iBAAiB;QACpB,gBAAgB;YACd,GAAIA,AAAAA,CAAAA,QAAAA,oBAAAA,KAAAA,IAAAA,kBAAmB,cAAc,AAAD,KAAK,CAAC,CAAC;YAC3C,CAAC4B,kBAAkB,EAAEnC,kBAAkB,QAAQ;QACjD;QACA,yBAAyB;IAC3B;IAGF,IACEqB,UACAe,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIP,aACF,MAAM,IAAIQ,MAAM;QAElBC,QAAQ,GAAG,CAAC;QACZ,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC;QACpCnB,SAASmB,WAAWnB;IACtB;IAEA,IAAI,AAAkB,WAAXA,QACT,OAAO;QACL,YAAYA,OAAO,IAAI,CAAC,WAAW;QACnC,OAAO;QACPjB;QACAc;QACAC;QACAC;IACF;IAIF,IAAIJ,iBACFK,SAAS,IAAIoB,UAAU;QACrB,QAAQxB;QACR,WAAWK;QACX,yBAAyB;IAC3B;IAGF,IAAI,AAAkB,WAAXD,UAA2BA,OAAe,QAAQ,EAC3D,OAAO;QACL,YAAaA,OAAe,QAAQ;QACpC,OAAO;QACPjB;QACAc;QACAC;QACAC;IACF;IAGF,MAAM,IAAIkB,MAAM;AAClB;AAEO,eAAeI,OACpBC,QAAsC,EACtC3C,iBAA+B,EAC/BC,WAAyB,EACzB2C,OAGC;IAED,MAAM,EACJC,UAAU,EACVC,KAAK,EACL1C,SAAS,EACTc,gBAAgB,EAChBC,aAAa,EACbC,MAAM,EACP,GAAG,MAAMrB,iBAAiB;QACzBC;QACAC;IACF;IAEA,MAAM8C,iBAAiBC,kBAAkB5C,WAAWJ;IAEpD,MAAMiD,YAAYb,oBAAoB,iBAAiB,CAACc;IACxD,MAAMC,YAAY3B,SAAS;IAC3B,MAAM4B,oBAAoB5B,SAAS;IACnC,MAAM6B,qBAAqB7B,SAAS;IAEpC,MAAM8B,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAazC,AAAW,kBAAXA,SAA2B,MAAM;QAC9C,QAAQ,CAAC,CAACoC;QACV,YACE,AAAqB,YAArB,OAAOP,YACHA,YACAa,OAAO,QAAQ,CAACb,aAAa,QAAQ;QAC3C,GAAI7B,AAAW,cAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACF,IAAI0B,AAAU,aAAVA,OAAoB;YACtBK,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAEpD,WAAW;YAGrE,IAAIoD,aAAa;gBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;oBACE,OAAOzC;oBACPuC;oBACA,iBAAiBI;oBACjB,GAAGc,YAAY;gBACjB,GACA;oBACE,QAAQ;gBACV;gBAKF,WAAW,MAAMG,SAASD,OAAQ;wBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;oBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;oBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;oBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;oBAGrB,IAAIP,WAAWW,mBAAmB;wBAChCV,eAAeD;wBACf,MAAMY,YAAiC;4BACrCZ;4BACAW;4BACAV;4BACA,YAAY;4BACZ,OAAOY;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;wBACrCP,WAAWL,KAAK,GAAG,KAAKD;wBAGxB,IAAI,CAACK,OAAO;4BAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;4BAElCC,QAAQ;gCACN,eAAeY;gCACf,mBAAmBA;gCACnB,cAAcA,AAAkB,IAAlBA;4BAChB;wBACF;wBAGA,MAAME,aAAkC;4BACtC,SAAS;4BACTf;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO;gCACL,eAAeC,MAAM,aAAa,IAAI;gCACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gCAC9C,cAAcA,MAAM,YAAY,IAAI;gCACpC,WAAWC,YAAY;gCACvB,YAAYxD;gCACZ,mBAAmBc;gCACnB,QAAQjB,YAAY,MAAM;4BAC5B;wBACF;wBACA2C,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAhB,UAAUC;gBACVN,kBACE,CAAC,iBAAiB,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,WAAW,EAAEwC,UAAU;YAEvF,OAAO;oBAUqGc,eAAyDC,gBAAwDC;gBAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrC,OAAOzC;oBACPuC;oBACA,iBAAiBI;oBACjB,GAAGc,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBAExBF,kBACE,CAAC,OAAO,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAEuD,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;gBAG3TxB,mBACE,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;gBAGvDhD,OACEgD,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;gBAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;gBAC3ClB,QAAQkB,OAAO,KAAK;YACtB;YAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;YAChC5B,OAAO4B,SAAS;QAClB,OAAO,IAAIX,AAAU,gBAAVA,OAAuB;YAChC,MAAMiC,sBAAsB,CAACtB;gBAC3B,IAAIA,AAAiB,gBAAjBA,QAAQ,IAAI,EAAkB;oBAChC,MAAMuB,YAAYvB,QAAQ,SAAS,CAAC,GAAG;oBACvC5B,OAAOmD,WAAW;oBAClB,MAAM,EAAEC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,YAAY1B,QAAQ,SAAS,CAAC,GAAG;oBAC5D,OAAO;wBACL,QAAQ;4BACN,MAAM;4BACN,YAAYwB;4BACZ,MAAMC;wBACR;wBACA,MAAM;oBACR;gBACF;gBACA,OAAOzB;YACT;YAEA,IAAID,aAAa;gBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CAAC;oBACtC,OAAOzC;oBACP,QAAQ;oBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBrC;oBACjB,GAAGc,YAAY;gBACjB;gBAEA,WAAW,MAAMG,SAASD,OAAQ;wBAChBuB;oBAAhB,MAAM7B,UAAU6B,AAAAA,SAAAA,CAAAA,eAAAA,MAAM,KAAK,AAAD,IAAVA,KAAAA,IAAAA,aAAa,IAAI,AAAD,KAAK;oBACrC,IAAI7B,SAAS;wBACXC,eAAeD;wBACf,MAAMY,YAAiC;4BACrCZ;4BACAC;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAOY;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAIL,AAAe,mBAAfA,MAAM,IAAI,EAAqB;wBACjCJ,WAAWL,KAAK,GAAG,KAAKD;wBACxB,MAAMiC,iBAAiBvB,MAAM,KAAK;wBAGlC,MAAMS,aAAkC;4BACtC,SAAS;4BACTf;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO6B,iBACH;gCACE,eAAeA,eAAe,YAAY,IAAI;gCAC9C,mBAAmBA,eAAe,aAAa,IAAI;gCACnD,cACGA,AAAAA,CAAAA,eAAe,YAAY,IAAI,KAC/BA,CAAAA,eAAe,aAAa,IAAI;gCACnC,WAAW3B,YAAY;gCACvB,YAAYxD;gCACZ,mBAAmBc;gCACnB,QAAQjB,YAAY,MAAM;4BAC5B,IACAqE;wBACN;wBACA1B,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAhB,UAAUC;YACZ,OAAO;gBACL,MAAMmB,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrC,OAAOzC;oBACP,QAAQ;oBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBrC;oBACjB,GAAGc,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBACxBG,UAAWoB,OAAe,OAAO,CAAC,EAAE,CAAC,IAAI;gBACzClB,QAAQkB,OAAO,KAAK;YACtB;YAEAhD,OAAO4B,SAAS;QAClB;QAEA,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeY;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASd,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYxD;gBACZ,mBAAmBc;gBACnB,QAAQjB,YAAY,MAAM;YAC5B,IACAqE;YACJ,YAAY,CAAC,CAACd;QAChB;IACF,EAAE,OAAOgC,GAAQ;QACfjD,QAAQ,KAAK,CAAC,kBAAkBiD;QAChC,MAAMC,WAAW,IAAInD,MACnB,CAAC,eAAe,EAAEkB,cAAc,eAAe,GAAG,kBAAkB,EAAEgC,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,MAAMzC,oBAAoB,CAC/B5C,WACAJ;IAIA,IAAI+C;IAKJ,IAAI3C,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAK0F,aAAa,MAAM;YACtB3C,iBAAiB4C;YACjB;QACF,KAAKD,aAAa,eAAe;YAC/B3C,iBAAiB6C;YACjB;QACF,KAAKF,aAAa,IAAI;YACpB3C,iBAAiB8C;YACjB;QACF,KAAKH,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChC3C,iBAAiB;gBAAE,MAAM+C,iBAAiB,IAAI;YAAC;YAC/C;IACJ;IAIF,IAAI1F,AAAc,wBAAdA,WACF2C,iBAAiB;QAAE,MAAM+C,iBAAiB,IAAI;IAAC;IAGjD,OAAO/C;AACT;AAEO,eAAegD,yBACpBpD,QAAgB,EAChB3C,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM+F,WAAW,MAAMtD,OAAOC,UAAU3C,mBAAmBC;IAC3D4B,OAAOmE,UAAU;IACjB,MAAM5E,SAASnB,YAAY,MAAM;IACjC,MAAMgG,cAAcC,cAAcF,SAAS,OAAO,EAAE5E;IACpD,OAAO;QAAE,SAAS6E;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,yBACpBC,IAAY,EACZpG,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAEwD,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAO0D,MAAMpG,mBAAmBC;IACjE,OAAO;QAAEwD;QAASE;IAAM;AAC1B;AAEO,SAAS0C,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa,EAAEtF,MAAgC;IAC3E,MAAMuF,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAAC9C;IACT;IACA,IAAI;QACF,OAAOgB,KAAK,KAAK,CAAC6B;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAO7B,KAAK,KAAK,CAAC+B,WAAWF;IAC/B,EAAE,OAAOnB,GAAG,CAAC;IAEb,IAAIpE,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAM0F,aAAaL,yBAAyBE;QAC5C,OAAO7B,KAAK,KAAK,CAAC+B,WAAWC;IAC/B;IACA,MAAMxE,MAAM,CAAC,+BAA+B,EAAEoE,OAAO;AACvD"}
+{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport { Anthropic } from '@anthropic-ai/sdk';\nimport {\n  DefaultAzureCredential,\n  getBearerTokenProvider,\n} from '@azure/identity';\nimport {\n  type IModelConfig,\n  MIDSCENE_API_TYPE,\n  MIDSCENE_LANGSMITH_DEBUG,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { parseBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI, { AzureOpenAI } from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { locatorSchema } from '../prompt/llm-locator';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  style: 'openai' | 'anthropic';\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    openaiUseAzureDeprecated,\n    useAzureOpenai,\n    azureOpenaiScope,\n    azureOpenaiKey,\n    azureOpenaiEndpoint,\n    azureOpenaiApiVersion,\n    azureOpenaiDeployment,\n    azureExtraConfig,\n    useAnthropicSdk,\n    anthropicApiKey,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n  } = modelConfig;\n\n  let openai: OpenAI | AzureOpenAI | undefined;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  if (openaiUseAzureDeprecated) {\n    // this is deprecated\n    openai = new AzureOpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      dangerouslyAllowBrowser: true,\n    }) as OpenAI;\n  } else if (useAzureOpenai) {\n    // https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=bash%2Cjavascript-key%2Ctypescript-keyless%2Cpython&pivots=programming-language-javascript#rest-api\n    // keyless authentication\n    let tokenProvider: any = undefined;\n    if (azureOpenaiScope) {\n      assert(\n        !ifInBrowser,\n        'Azure OpenAI is not supported in browser with Midscene.',\n      );\n      const credential = new DefaultAzureCredential();\n\n      tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);\n\n      openai = new AzureOpenAI({\n        azureADTokenProvider: tokenProvider,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    } else {\n      // endpoint, apiKey, apiVersion, deployment\n      openai = new AzureOpenAI({\n        apiKey: azureOpenaiKey,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        dangerouslyAllowBrowser: true,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    }\n  } else if (!useAnthropicSdk) {\n    openai = new OpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      defaultHeaders: {\n        ...(openaiExtraConfig?.defaultHeaders || {}),\n        [MIDSCENE_API_TYPE]: AIActionTypeValue.toString(),\n      },\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    const { wrapOpenAI } = await import('langsmith/wrappers');\n    openai = wrapOpenAI(openai);\n  }\n\n  if (typeof openai !== 'undefined') {\n    return {\n      completion: openai.chat.completions,\n      style: 'openai',\n      modelName,\n      modelDescription,\n      uiTarsVersion,\n      vlMode,\n    };\n  }\n\n  // Anthropic\n  if (useAnthropicSdk) {\n    openai = new Anthropic({\n      apiKey: anthropicApiKey,\n      httpAgent: proxyAgent,\n      dangerouslyAllowBrowser: true,\n    }) as any;\n  }\n\n  if (typeof openai !== 'undefined' && (openai as any).messages) {\n    return {\n      completion: (openai as any).messages,\n      style: 'anthropic',\n      modelName,\n      modelDescription,\n      uiTarsVersion,\n      vlMode,\n    };\n  }\n\n  throw new Error('Openai SDK or Anthropic SDK is not initialized');\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const {\n    completion,\n    style,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  } = await createChatClient({\n    AIActionTypeValue,\n    modelConfig,\n  });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlMode === 'qwen-vl' // qwen specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    if (style === 'openai') {\n      debugCall(\n        `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n      );\n\n      if (isStreaming) {\n        const stream = (await completion.create(\n          {\n            model: modelName,\n            messages,\n            response_format: responseFormat,\n            ...commonConfig,\n          },\n          {\n            stream: true,\n          },\n        )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n          _request_id?: string | null;\n        };\n\n        for await (const chunk of stream) {\n          const content = chunk.choices?.[0]?.delta?.content || '';\n          const reasoning_content =\n            (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n          // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n          if (chunk.usage) {\n            usage = chunk.usage;\n          }\n\n          if (content || reasoning_content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              reasoning_content,\n              accumulated,\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.choices?.[0]?.finish_reason) {\n            timeCost = Date.now() - startTime;\n\n            // If usage is not available from the stream, provide a basic usage info\n            if (!usage) {\n              // Estimate token counts based on content length (rough approximation)\n              const estimatedTokens = Math.max(\n                1,\n                Math.floor(accumulated.length / 4),\n              );\n              usage = {\n                prompt_tokens: estimatedTokens,\n                completion_tokens: estimatedTokens,\n                total_tokens: estimatedTokens * 2,\n              };\n            }\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: {\n                prompt_tokens: usage.prompt_tokens ?? 0,\n                completion_tokens: usage.completion_tokens ?? 0,\n                total_tokens: usage.total_tokens ?? 0,\n                time_cost: timeCost ?? 0,\n                model_name: modelName,\n                model_description: modelDescription,\n                intent: modelConfig.intent,\n              },\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n        debugProfileStats(\n          `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n        );\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n\n        debugProfileStats(\n          `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n        );\n\n        debugProfileDetail(\n          `model usage detail: ${JSON.stringify(result.usage)}`,\n        );\n\n        assert(\n          result.choices,\n          `invalid response from LLM service: ${JSON.stringify(result)}`,\n        );\n        content = result.choices[0].message.content!;\n        usage = result.usage;\n      }\n\n      debugCall(`response: ${content}`);\n      assert(content, 'empty content');\n    } else if (style === 'anthropic') {\n      const convertImageContent = (content: any) => {\n        if (content.type === 'image_url') {\n          const imgBase64 = content.image_url.url;\n          assert(imgBase64, 'image_url is required');\n          const { mimeType, body } = parseBase64(content.image_url.url);\n          return {\n            source: {\n              type: 'base64',\n              media_type: mimeType,\n              data: body,\n            },\n            type: 'image',\n          };\n        }\n        return content;\n      };\n\n      if (isStreaming) {\n        const stream = (await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any)) as any;\n\n        for await (const chunk of stream) {\n          const content = chunk.delta?.text || '';\n          if (content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              accumulated,\n              reasoning_content: '',\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.type === 'message_stop') {\n            timeCost = Date.now() - startTime;\n            const anthropicUsage = chunk.usage;\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: anthropicUsage\n                ? {\n                    prompt_tokens: anthropicUsage.input_tokens ?? 0,\n                    completion_tokens: anthropicUsage.output_tokens ?? 0,\n                    total_tokens:\n                      (anthropicUsage.input_tokens ?? 0) +\n                      (anthropicUsage.output_tokens ?? 0),\n                    time_cost: timeCost ?? 0,\n                    model_name: modelName,\n                    model_description: modelDescription,\n                    intent: modelConfig.intent,\n                  }\n                : undefined,\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n        content = (result as any).content[0].text as string;\n        usage = result.usage;\n      }\n\n      assert(content, 'empty content');\n    }\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.INSPECT_ELEMENT:\n        responseFormat = locatorSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n      case AIActionType.TEXT:\n        // No response format for plain text - return as-is\n        responseFormat = undefined;\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  // Skip for plain text to allow string output\n  if (\n    modelName === 'gpt-4o-2024-05-13' &&\n    AIActionTypeValue !== AIActionType.TEXT\n  ) {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","openaiUseAzureDeprecated","useAzureOpenai","azureOpenaiScope","azureOpenaiKey","azureOpenaiEndpoint","azureOpenaiApiVersion","azureOpenaiDeployment","azureExtraConfig","useAnthropicSdk","anthropicApiKey","modelDescription","uiTarsVersion","vlMode","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","AzureOpenAI","tokenProvider","assert","ifInBrowser","credential","DefaultAzureCredential","getBearerTokenProvider","OpenAI","MIDSCENE_API_TYPE","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","Error","console","wrapOpenAI","Anthropic","callAI","messages","options","completion","style","responseFormat","getResponseFormat","maxTokens","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","convertImageContent","imgBase64","mimeType","body","parseBase64","m","Array","_chunk_delta","anthropicUsage","e","newError","AIActionType","assertSchema","locatorSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;;;;;AAgCA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAQC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,wBAAwB,EACxBC,cAAc,EACdC,gBAAgB,EAChBC,cAAc,EACdC,mBAAmB,EACnBC,qBAAqB,EACrBC,qBAAqB,EACrBC,gBAAgB,EAChBC,eAAe,EACfC,eAAe,EACfC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACP,GAAGnB;IAEJ,IAAIoB;IAEJ,IAAIC;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIrB,WAAW;QACboB,WAAW,oBAAoBpB;QAC/BmB,aAAa,IAAIG,gBAAgBtB;IACnC,OAAO,IAAID,YAAY;QACrBqB,WAAW,qBAAqBrB;QAChCoB,aAAa,IAAII,gBAAgBxB;IACnC;IAEA,IAAIM,0BAEFa,SAAS,IAAIM,YAAY;QACvB,SAAStB;QACT,QAAQC;QACR,WAAWgB;QACX,GAAGf,iBAAiB;QACpB,yBAAyB;IAC3B;SACK,IAAIE,gBAAgB;QAGzB,IAAImB;QACJ,IAAIlB,kBAAkB;YACpBmB,OACE,CAACC,aACD;YAEF,MAAMC,aAAa,IAAIC;YAEvBJ,gBAAgBK,uBAAuBF,YAAYrB;YAEnDW,SAAS,IAAIM,YAAY;gBACvB,sBAAsBC;gBACtB,UAAUhB;gBACV,YAAYC;gBACZ,YAAYC;gBACZ,GAAGP,iBAAiB;gBACpB,GAAGQ,gBAAgB;YACrB;QACF,OAEEM,SAAS,IAAIM,YAAY;YACvB,QAAQhB;YACR,UAAUC;YACV,YAAYC;YACZ,YAAYC;YACZ,yBAAyB;YACzB,GAAGP,iBAAiB;YACpB,GAAGQ,gBAAgB;QACrB;IAEJ,OAAO,IAAI,CAACC,iBACVK,SAAS,IAAIa,SAAO;QAClB,SAAS7B;QACT,QAAQC;QACR,WAAWgB;QACX,GAAGf,iBAAiB;QACpB,gBAAgB;YACd,GAAIA,AAAAA,CAAAA,QAAAA,oBAAAA,KAAAA,IAAAA,kBAAmB,cAAc,AAAD,KAAK,CAAC,CAAC;YAC3C,CAAC4B,kBAAkB,EAAEnC,kBAAkB,QAAQ;QACjD;QACA,yBAAyB;IAC3B;IAGF,IACEqB,UACAe,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIP,aACF,MAAM,IAAIQ,MAAM;QAElBC,QAAQ,GAAG,CAAC;QACZ,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC;QACpCnB,SAASmB,WAAWnB;IACtB;IAEA,IAAI,AAAkB,WAAXA,QACT,OAAO;QACL,YAAYA,OAAO,IAAI,CAAC,WAAW;QACnC,OAAO;QACPjB;QACAc;QACAC;QACAC;IACF;IAIF,IAAIJ,iBACFK,SAAS,IAAIoB,UAAU;QACrB,QAAQxB;QACR,WAAWK;QACX,yBAAyB;IAC3B;IAGF,IAAI,AAAkB,WAAXD,UAA2BA,OAAe,QAAQ,EAC3D,OAAO;QACL,YAAaA,OAAe,QAAQ;QACpC,OAAO;QACPjB;QACAc;QACAC;QACAC;IACF;IAGF,MAAM,IAAIkB,MAAM;AAClB;AAEO,eAAeI,OACpBC,QAAsC,EACtC3C,iBAA+B,EAC/BC,WAAyB,EACzB2C,OAGC;IAED,MAAM,EACJC,UAAU,EACVC,KAAK,EACL1C,SAAS,EACTc,gBAAgB,EAChBC,aAAa,EACbC,MAAM,EACP,GAAG,MAAMrB,iBAAiB;QACzBC;QACAC;IACF;IAEA,MAAM8C,iBAAiBC,kBAAkB5C,WAAWJ;IAEpD,MAAMiD,YAAYb,oBAAoB,iBAAiB,CAACc;IACxD,MAAMC,YAAY3B,SAAS;IAC3B,MAAM4B,oBAAoB5B,SAAS;IACnC,MAAM6B,qBAAqB7B,SAAS;IAEpC,MAAM8B,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAazC,AAAW,kBAAXA,SAA2B,MAAM;QAC9C,QAAQ,CAAC,CAACoC;QACV,YACE,AAAqB,YAArB,OAAOP,YACHA,YACAa,OAAO,QAAQ,CAACb,aAAa,QAAQ;QAC3C,GAAI7B,AAAW,cAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACF,IAAI0B,AAAU,aAAVA,OAAoB;YACtBK,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAEpD,WAAW;YAGrE,IAAIoD,aAAa;gBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;oBACE,OAAOzC;oBACPuC;oBACA,iBAAiBI;oBACjB,GAAGc,YAAY;gBACjB,GACA;oBACE,QAAQ;gBACV;gBAKF,WAAW,MAAMG,SAASD,OAAQ;wBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;oBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;oBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;oBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;oBAGrB,IAAIP,WAAWW,mBAAmB;wBAChCV,eAAeD;wBACf,MAAMY,YAAiC;4BACrCZ;4BACAW;4BACAV;4BACA,YAAY;4BACZ,OAAOY;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;wBACrCP,WAAWL,KAAK,GAAG,KAAKD;wBAGxB,IAAI,CAACK,OAAO;4BAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;4BAElCC,QAAQ;gCACN,eAAeY;gCACf,mBAAmBA;gCACnB,cAAcA,AAAkB,IAAlBA;4BAChB;wBACF;wBAGA,MAAME,aAAkC;4BACtC,SAAS;4BACTf;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO;gCACL,eAAeC,MAAM,aAAa,IAAI;gCACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gCAC9C,cAAcA,MAAM,YAAY,IAAI;gCACpC,WAAWC,YAAY;gCACvB,YAAYxD;gCACZ,mBAAmBc;gCACnB,QAAQjB,YAAY,MAAM;4BAC5B;wBACF;wBACA2C,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAhB,UAAUC;gBACVN,kBACE,CAAC,iBAAiB,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,WAAW,EAAEwC,UAAU;YAEvF,OAAO;oBAUqGc,eAAyDC,gBAAwDC;gBAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrC,OAAOzC;oBACPuC;oBACA,iBAAiBI;oBACjB,GAAGc,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBAExBF,kBACE,CAAC,OAAO,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAEuD,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;gBAG3TxB,mBACE,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;gBAGvDhD,OACEgD,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;gBAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;gBAC3ClB,QAAQkB,OAAO,KAAK;YACtB;YAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;YAChC5B,OAAO4B,SAAS;QAClB,OAAO,IAAIX,AAAU,gBAAVA,OAAuB;YAChC,MAAMiC,sBAAsB,CAACtB;gBAC3B,IAAIA,AAAiB,gBAAjBA,QAAQ,IAAI,EAAkB;oBAChC,MAAMuB,YAAYvB,QAAQ,SAAS,CAAC,GAAG;oBACvC5B,OAAOmD,WAAW;oBAClB,MAAM,EAAEC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,YAAY1B,QAAQ,SAAS,CAAC,GAAG;oBAC5D,OAAO;wBACL,QAAQ;4BACN,MAAM;4BACN,YAAYwB;4BACZ,MAAMC;wBACR;wBACA,MAAM;oBACR;gBACF;gBACA,OAAOzB;YACT;YAEA,IAAID,aAAa;gBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CAAC;oBACtC,OAAOzC;oBACP,QAAQ;oBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBrC;oBACjB,GAAGc,YAAY;gBACjB;gBAEA,WAAW,MAAMG,SAASD,OAAQ;wBAChBuB;oBAAhB,MAAM7B,UAAU6B,AAAAA,SAAAA,CAAAA,eAAAA,MAAM,KAAK,AAAD,IAAVA,KAAAA,IAAAA,aAAa,IAAI,AAAD,KAAK;oBACrC,IAAI7B,SAAS;wBACXC,eAAeD;wBACf,MAAMY,YAAiC;4BACrCZ;4BACAC;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAOY;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAIL,AAAe,mBAAfA,MAAM,IAAI,EAAqB;wBACjCJ,WAAWL,KAAK,GAAG,KAAKD;wBACxB,MAAMiC,iBAAiBvB,MAAM,KAAK;wBAGlC,MAAMS,aAAkC;4BACtC,SAAS;4BACTf;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO6B,iBACH;gCACE,eAAeA,eAAe,YAAY,IAAI;gCAC9C,mBAAmBA,eAAe,aAAa,IAAI;gCACnD,cACGA,AAAAA,CAAAA,eAAe,YAAY,IAAI,KAC/BA,CAAAA,eAAe,aAAa,IAAI;gCACnC,WAAW3B,YAAY;gCACvB,YAAYxD;gCACZ,mBAAmBc;gCACnB,QAAQjB,YAAY,MAAM;4BAC5B,IACAqE;wBACN;wBACA1B,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAhB,UAAUC;YACZ,OAAO;gBACL,MAAMmB,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrC,OAAOzC;oBACP,QAAQ;oBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBrC;oBACjB,GAAGc,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBACxBG,UAAWoB,OAAe,OAAO,CAAC,EAAE,CAAC,IAAI;gBACzClB,QAAQkB,OAAO,KAAK;YACtB;YAEAhD,OAAO4B,SAAS;QAClB;QAEA,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeY;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASd,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYxD;gBACZ,mBAAmBc;gBACnB,QAAQjB,YAAY,MAAM;YAC5B,IACAqE;YACJ,YAAY,CAAC,CAACd;QAChB;IACF,EAAE,OAAOgC,GAAQ;QACfjD,QAAQ,KAAK,CAAC,kBAAkBiD;QAChC,MAAMC,WAAW,IAAInD,MACnB,CAAC,eAAe,EAAEkB,cAAc,eAAe,GAAG,kBAAkB,EAAEgC,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,MAAMzC,oBAAoB,CAC/B5C,WACAJ;IAIA,IAAI+C;IAKJ,IAAI3C,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAK0F,aAAa,MAAM;YACtB3C,iBAAiB4C;YACjB;QACF,KAAKD,aAAa,eAAe;YAC/B3C,iBAAiB6C;YACjB;QACF,KAAKF,aAAa,IAAI;YACpB3C,iBAAiB8C;YACjB;QACF,KAAKH,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChC3C,iBAAiB;gBAAE,MAAM+C,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKJ,aAAa,IAAI;YAEpB3C,iBAAiBuB;YACjB;IACJ;IAKF,IACElE,AAAc,wBAAdA,aACAJ,sBAAsB0F,aAAa,IAAI,EAEvC3C,iBAAiB;QAAE,MAAM+C,iBAAiB,IAAI;IAAC;IAGjD,OAAO/C;AACT;AAEO,eAAegD,yBACpBpD,QAAgB,EAChB3C,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM+F,WAAW,MAAMtD,OAAOC,UAAU3C,mBAAmBC;IAC3D4B,OAAOmE,UAAU;IACjB,MAAM5E,SAASnB,YAAY,MAAM;IACjC,MAAMgG,cAAcC,cAAcF,SAAS,OAAO,EAAE5E;IACpD,OAAO;QAAE,SAAS6E;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,yBACpBC,IAAY,EACZpG,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAEwD,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAO0D,MAAMpG,mBAAmBC;IACjE,OAAO;QAAEwD;QAASE;IAAM;AAC1B;AAEO,SAAS0C,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa,EAAEtF,MAAgC;IAC3E,MAAMuF,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAAC9C;IACT;IACA,IAAI;QACF,OAAOgB,KAAK,KAAK,CAAC6B;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAO7B,KAAK,KAAK,CAAC+B,WAAWF;IAC/B,EAAE,OAAOnB,GAAG,CAAC;IAEb,IAAIpE,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAM0F,aAAaL,yBAAyBE;QAC5C,OAAO7B,KAAK,KAAK,CAAC+B,WAAWC;IAC/B;IACA,MAAMxE,MAAM,CAAC,+BAA+B,EAAEoE,OAAO;AACvD"}