@midscene/core 1.0.1-beta-20251126092253.0 → 1.0.1-beta-20251127021613.0

package/dist/es/ai-model/inspect.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from '../common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(\n    targetElementDescriptionText,\n  );\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(\n    msgs,\n    AIActionType.INSPECT_ELEMENT, // Reuse existing action type for now\n    modelConfig,\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","generateElementByPosition","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;AAsDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IASC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,+BAA+BlB,wBACnCY;IAEF,MAAMO,wBAAwBC,kBAC5BF;IAEF,MAAMG,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;YAWXkB,4BACCC;QAXdZ,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,iBAAXA,QAAyB;QAClC,MAAMe,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMoB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMC,MAAM,MAAMpB,SAASR,MAAM6B,aAAa,eAAe,EAAEpB;IAE/D,MAAMqB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;gBAKEQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAZ;YAGFlB,aAAa,WAAWwC;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMQ,UAA+BC,0BACnCF,YACA1B;YAEFqB,SAAS,EAAE;YAEX,IAAIM,SACFP,kBAAkB;gBAACO;aAAQ;QAE/B;IACF,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACR,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;aAFtBT,SAAS;YAACS;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMX;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;IAClB;AACF;AAEO,eAAeiB,gBAAgBxC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEwC,kBAAkB,EAAErC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAe+B,4BAA4BrC;IACjD,MAAMsC,gCAAgCC,0BACpCtD,wBAAwBmD;IAE1B,MAAM9C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMqC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMnB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQiD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA9C,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMuB,SAAS,MAAMC,yBACnBnD,MACA6B,aAAa,YAAY,EACzBpB;IAGF,IAAI2C;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAahB,gBACjBe,aACA/C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0B4D;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DxD,aAAa,wBAAwB6D;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAStB,MAAM,OAAO,CAACsB,OAC/B,GAAG,CAAC,CAACA,OACGnB,gBACLmB,MACAnD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqB8D;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7D9D,aAAa,iBAAiBgE;QAG9BN,cAAcQ,iBAAiBF,YAAYpD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B0D;IAC1C;IAEA,IAAIS,cAAclD;IAClB,IAAIyC,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BpD,kBACAyC,aACA1C,AAAW,iBAAXA;QAEFmD,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAanB,KAAK,SAAS,CAACmB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwB3D,OAO7C;IACC,MAAM,EAAE4D,SAAS,EAAE3D,OAAO,EAAE4D,aAAa,EAAEpE,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAemD;IACrB,MAAM,EAAExD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM8D,wBAAwBC,uBAC5BhE,QAAQ,eAAe,IAAI,IAC3B4D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAK3D;YACL,QAAQ;QACV;IACF;IAGF2D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMpE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASsD;QACX;KACD;IAED,IAAIxE,kBAAkB;QACpB,MAAM6B,SAAS,MAAM9B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMuB,SAAS,MAAMC,yBACnBnD,MACA6B,aAAa,YAAY,EACzBpB;IAEF,OAAO;QACL,aAAayC,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnBhE,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeyD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMxE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS0D;QACX;KACD;IAED,MAAMxB,SAAS,MAAM1C,SACnBR,MACA6B,aAAa,eAAe,EAC5BpB;IAGF,OAAO;QACL,kBAAkByC,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}
+{"version":3,"file":"ai-model/inspect.mjs","sources":["webpack://@midscene/core/./src/ai-model/inspect.ts"],"sourcesContent":["import type {\n  AIDataExtractionResponse,\n  AIElementResponse,\n  AISectionLocatorResponse,\n  AIUsageInfo,\n  Rect,\n  ReferenceImage,\n  ServiceExtractOption,\n  UIContext,\n} from '@/types';\nimport type { IModelConfig } from '@midscene/shared/env';\nimport { generateElementByPosition } from '@midscene/shared/extractor/dom-util';\nimport {\n  cropByRect,\n  paddingToMatchBlockByBase64,\n  preProcessImageUrl,\n} from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport type { LocateResultElement } from '@midscene/shared/types';\nimport { assert } from '@midscene/shared/utils';\nimport type {\n  ChatCompletionSystemMessageParam,\n  ChatCompletionUserMessageParam,\n} from 'openai/resources/index';\nimport type { TMultimodalPrompt, TUserPrompt } from '../common';\nimport {\n  AIActionType,\n  adaptBboxToRect,\n  expandSearchArea,\n  mergeRects,\n} from '../common';\nimport {\n  extractDataQueryPrompt,\n  systemPromptToExtract,\n} from './prompt/extraction';\nimport {\n  findElementPrompt,\n  systemPromptToLocateElement,\n} from './prompt/llm-locator';\nimport {\n  sectionLocatorInstruction,\n  systemPromptToLocateSection,\n} from './prompt/llm-section-locator';\nimport {\n  orderSensitiveJudgePrompt,\n  systemPromptToJudgeOrderSensitive,\n} from './prompt/order-sensitive-judge';\nimport { callAIWithObjectResponse } from './service-caller/index';\n\nexport type AIArgs = [\n  ChatCompletionSystemMessageParam,\n  ...ChatCompletionUserMessageParam[],\n];\n\nconst debugInspect = getDebug('ai:inspect');\nconst debugSection = getDebug('ai:section');\n\nconst extraTextFromUserPrompt = (prompt: TUserPrompt): string => {\n  if (typeof prompt === 'string') {\n    return prompt;\n  } else {\n    return prompt.prompt;\n  }\n};\n\nconst promptsToChatParam = async (\n  multimodalPrompt: TMultimodalPrompt,\n): Promise<ChatCompletionUserMessageParam[]> => {\n  const msgs: ChatCompletionUserMessageParam[] = [];\n  if (multimodalPrompt?.images?.length) {\n    msgs.push({\n      role: 'user',\n      content: [\n        {\n          type: 'text',\n          text: 'Next, I will provide all the reference images.',\n        },\n      ],\n    });\n\n    for (const item of multimodalPrompt.images) {\n      const base64 = await preProcessImageUrl(\n        item.url,\n        !!multimodalPrompt.convertHttpImage2Base64,\n      );\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'text',\n            text: `this is the reference image named '${item.name}':`,\n          },\n        ],\n      });\n\n      msgs.push({\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: base64,\n              detail: 'high',\n            },\n          },\n        ],\n      });\n    }\n  }\n  return msgs;\n};\n\nexport async function AiLocateElement(options: {\n  context: UIContext;\n  targetElementDescription: TUserPrompt;\n  referenceImage?: ReferenceImage;\n  callAIFn: typeof callAIWithObjectResponse<\n    AIElementResponse | [number, number]\n  >;\n  searchConfig?: Awaited<ReturnType<typeof AiLocateSection>>;\n  modelConfig: IModelConfig;\n}): Promise<{\n  parseResult: {\n    elements: LocateResultElement[];\n    errors?: string[];\n  };\n  rect?: Rect;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, targetElementDescription, callAIFn, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  assert(\n    targetElementDescription,\n    'cannot find the target element description',\n  );\n  const targetElementDescriptionText = extraTextFromUserPrompt(\n    targetElementDescription,\n  );\n  const userInstructionPrompt = findElementPrompt(targetElementDescriptionText);\n  const systemPrompt = systemPromptToLocateElement(vlMode);\n\n  let imagePayload = screenshotBase64;\n  let imageWidth = context.size.width;\n  let imageHeight = context.size.height;\n  let originalImageWidth = imageWidth;\n  let originalImageHeight = imageHeight;\n\n  if (options.searchConfig) {\n    assert(\n      options.searchConfig.rect,\n      'searchArea is provided but its rect cannot be found. Failed to locate element',\n    );\n    assert(\n      options.searchConfig.imageBase64,\n      'searchArea is provided but its imageBase64 cannot be found. Failed to locate element',\n    );\n\n    imagePayload = options.searchConfig.imageBase64;\n    imageWidth = options.searchConfig.rect?.width;\n    imageHeight = options.searchConfig.rect?.height;\n    originalImageWidth = imageWidth;\n    originalImageHeight = imageHeight;\n  } else if (vlMode === 'qwen2.5-vl') {\n    const paddedResult = await paddingToMatchBlockByBase64(imagePayload);\n    imageWidth = paddedResult.width;\n    imageHeight = paddedResult.height;\n    imagePayload = paddedResult.imageBase64;\n  }\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: imagePayload,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: userInstructionPrompt,\n        },\n      ],\n    },\n  ];\n\n  if (typeof targetElementDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: targetElementDescription.images,\n      convertHttpImage2Base64: targetElementDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const res = await callAIFn(msgs, AIActionType.INSPECT_ELEMENT, modelConfig);\n\n  const rawResponse = JSON.stringify(res.content);\n\n  let resRect: Rect | undefined;\n  let matchedElements = 'elements' in res.content ? res.content.elements : [];\n  let errors: string[] | undefined =\n    'errors' in res.content ? res.content.errors : [];\n  try {\n    if (\n      'bbox' in res.content &&\n      Array.isArray(res.content.bbox) &&\n      res.content.bbox.length >= 1\n    ) {\n      resRect = adaptBboxToRect(\n        res.content.bbox,\n        imageWidth,\n        imageHeight,\n        options.searchConfig?.rect?.left,\n        options.searchConfig?.rect?.top,\n        originalImageWidth,\n        originalImageHeight,\n        vlMode,\n      );\n\n      debugInspect('resRect', resRect);\n\n      const rectCenter = {\n        x: resRect.left + resRect.width / 2,\n        y: resRect.top + resRect.height / 2,\n      };\n\n      const element: LocateResultElement = generateElementByPosition(\n        rectCenter,\n        targetElementDescriptionText as string,\n      );\n      errors = [];\n\n      if (element) {\n        matchedElements = [element];\n      }\n    }\n  } catch (e) {\n    const msg =\n      e instanceof Error\n        ? `Failed to parse bbox: ${e.message}`\n        : 'unknown error in locate';\n    if (!errors || errors?.length === 0) {\n      errors = [msg];\n    } else {\n      errors.push(`(${msg})`);\n    }\n  }\n\n  return {\n    rect: resRect,\n    parseResult: {\n      elements: matchedElements as LocateResultElement[],\n      errors: errors as string[],\n    },\n    rawResponse,\n    usage: res.usage,\n  };\n}\n\nexport async function AiLocateSection(options: {\n  context: UIContext;\n  sectionDescription: TUserPrompt;\n  modelConfig: IModelConfig;\n}): Promise<{\n  rect?: Rect;\n  imageBase64?: string;\n  error?: string;\n  rawResponse: string;\n  usage?: AIUsageInfo;\n}> {\n  const { context, sectionDescription, modelConfig } = options;\n  const { vlMode } = modelConfig;\n  const { screenshotBase64 } = context;\n\n  const systemPrompt = systemPromptToLocateSection(vlMode);\n  const sectionLocatorInstructionText = sectionLocatorInstruction(\n    extraTextFromUserPrompt(sectionDescription),\n  );\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: [\n        {\n          type: 'image_url',\n          image_url: {\n            url: screenshotBase64,\n            detail: 'high',\n          },\n        },\n        {\n          type: 'text',\n          text: sectionLocatorInstructionText,\n        },\n      ],\n    },\n  ];\n\n  if (typeof sectionDescription !== 'string') {\n    const addOns = await promptsToChatParam({\n      images: sectionDescription.images,\n      convertHttpImage2Base64: sectionDescription.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AISectionLocatorResponse>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n\n  let sectionRect: Rect | undefined;\n  const sectionBbox = result.content.bbox;\n  if (sectionBbox) {\n    const targetRect = adaptBboxToRect(\n      sectionBbox,\n      context.size.width,\n      context.size.height,\n      0,\n      0,\n      context.size.width,\n      context.size.height,\n      vlMode,\n    );\n    debugSection('original targetRect %j', targetRect);\n\n    const referenceBboxList = result.content.references_bbox || [];\n    debugSection('referenceBboxList %j', referenceBboxList);\n\n    const referenceRects = referenceBboxList\n      .filter((bbox) => Array.isArray(bbox))\n      .map((bbox) => {\n        return adaptBboxToRect(\n          bbox,\n          context.size.width,\n          context.size.height,\n          0,\n          0,\n          context.size.width,\n          context.size.height,\n          vlMode,\n        );\n      });\n    debugSection('referenceRects %j', referenceRects);\n\n    // merge the sectionRect and referenceRects\n    const mergedRect = mergeRects([targetRect, ...referenceRects]);\n    debugSection('mergedRect %j', mergedRect);\n\n    // expand search area to at least 200 x 200\n    sectionRect = expandSearchArea(mergedRect, context.size, vlMode);\n    debugSection('expanded sectionRect %j', sectionRect);\n  }\n\n  let imageBase64 = screenshotBase64;\n  if (sectionRect) {\n    const croppedResult = await cropByRect(\n      screenshotBase64,\n      sectionRect,\n      vlMode === 'qwen2.5-vl',\n    );\n    imageBase64 = croppedResult.imageBase64;\n    sectionRect.width = croppedResult.width;\n    sectionRect.height = croppedResult.height;\n  }\n\n  return {\n    rect: sectionRect,\n    imageBase64,\n    error: result.content.error,\n    rawResponse: JSON.stringify(result.content),\n    usage: result.usage,\n  };\n}\n\nexport async function AiExtractElementInfo<T>(options: {\n  dataQuery: string | Record<string, string>;\n  multimodalPrompt?: TMultimodalPrompt;\n  context: UIContext;\n  pageDescription?: string;\n  extractOption?: ServiceExtractOption;\n  modelConfig: IModelConfig;\n}) {\n  const { dataQuery, context, extractOption, multimodalPrompt, modelConfig } =\n    options;\n  const systemPrompt = systemPromptToExtract();\n  const { screenshotBase64 } = context;\n\n  const extractDataPromptText = extractDataQueryPrompt(\n    options.pageDescription || '',\n    dataQuery,\n  );\n\n  const userContent: ChatCompletionUserMessageParam['content'] = [];\n\n  if (extractOption?.screenshotIncluded !== false) {\n    userContent.push({\n      type: 'image_url',\n      image_url: {\n        url: screenshotBase64,\n        detail: 'high',\n      },\n    });\n  }\n\n  userContent.push({\n    type: 'text',\n    text: extractDataPromptText,\n  });\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userContent,\n    },\n  ];\n\n  if (multimodalPrompt) {\n    const addOns = await promptsToChatParam({\n      images: multimodalPrompt.images,\n      convertHttpImage2Base64: multimodalPrompt.convertHttpImage2Base64,\n    });\n    msgs.push(...addOns);\n  }\n\n  const result = await callAIWithObjectResponse<AIDataExtractionResponse<T>>(\n    msgs,\n    AIActionType.EXTRACT_DATA,\n    modelConfig,\n  );\n  return {\n    parseResult: result.content,\n    usage: result.usage,\n  };\n}\n\nexport async function AiJudgeOrderSensitive(\n  description: string,\n  callAIFn: typeof callAIWithObjectResponse<{ isOrderSensitive: boolean }>,\n  modelConfig: IModelConfig,\n): Promise<{\n  isOrderSensitive: boolean;\n  usage?: AIUsageInfo;\n}> {\n  const systemPrompt = systemPromptToJudgeOrderSensitive();\n  const userPrompt = orderSensitiveJudgePrompt(description);\n\n  const msgs: AIArgs = [\n    { role: 'system', content: systemPrompt },\n    {\n      role: 'user',\n      content: userPrompt,\n    },\n  ];\n\n  const result = await callAIFn(\n    msgs,\n    AIActionType.INSPECT_ELEMENT, // Reuse existing action type for now\n    modelConfig,\n  );\n\n  return {\n    isOrderSensitive: result.content.isOrderSensitive ?? false,\n    usage: result.usage,\n  };\n}\n"],"names":["debugInspect","getDebug","debugSection","extraTextFromUserPrompt","prompt","promptsToChatParam","multimodalPrompt","_multimodalPrompt_images","msgs","item","base64","preProcessImageUrl","AiLocateElement","options","context","targetElementDescription","callAIFn","modelConfig","vlMode","screenshotBase64","assert","targetElementDescriptionText","userInstructionPrompt","findElementPrompt","systemPrompt","systemPromptToLocateElement","imagePayload","imageWidth","imageHeight","originalImageWidth","originalImageHeight","_options_searchConfig_rect","_options_searchConfig_rect1","paddedResult","paddingToMatchBlockByBase64","addOns","res","AIActionType","rawResponse","JSON","resRect","matchedElements","errors","Array","_options_searchConfig_rect2","_options_searchConfig_rect3","adaptBboxToRect","rectCenter","element","generateElementByPosition","e","msg","Error","AiLocateSection","sectionDescription","systemPromptToLocateSection","sectionLocatorInstructionText","sectionLocatorInstruction","result","callAIWithObjectResponse","sectionRect","sectionBbox","targetRect","referenceBboxList","referenceRects","bbox","mergedRect","mergeRects","expandSearchArea","imageBase64","croppedResult","cropByRect","AiExtractElementInfo","dataQuery","extractOption","systemPromptToExtract","extractDataPromptText","extractDataQueryPrompt","userContent","AiJudgeOrderSensitive","description","systemPromptToJudgeOrderSensitive","userPrompt","orderSensitiveJudgePrompt"],"mappings":";;;;;;;;;;AAsDA,MAAMA,eAAeC,SAAS;AAC9B,MAAMC,eAAeD,SAAS;AAE9B,MAAME,0BAA0B,CAACC;IAC/B,IAAI,AAAkB,YAAlB,OAAOA,QACT,OAAOA;IAEP,OAAOA,OAAO,MAAM;AAExB;AAEA,MAAMC,qBAAqB,OACzBC;QAGIC;IADJ,MAAMC,OAAyC,EAAE;IACjD,IAAID,QAAAA,mBAAAA,KAAAA,IAAAA,QAAAA,CAAAA,2BAAAA,iBAAkB,MAAM,AAAD,IAAvBA,KAAAA,IAAAA,yBAA0B,MAAM,EAAE;QACpCC,KAAK,IAAI,CAAC;YACR,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,MAAM;gBACR;aACD;QACH;QAEA,KAAK,MAAMC,QAAQH,iBAAiB,MAAM,CAAE;YAC1C,MAAMI,SAAS,MAAMC,mBACnBF,KAAK,GAAG,EACR,CAAC,CAACH,iBAAiB,uBAAuB;YAG5CE,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,MAAM,CAAC,mCAAmC,EAAEC,KAAK,IAAI,CAAC,EAAE,CAAC;oBAC3D;iBACD;YACH;YAEAD,KAAK,IAAI,CAAC;gBACR,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKE;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;QACF;IACF;IACA,OAAOF;AACT;AAEO,eAAeI,gBAAgBC,OASrC;IASC,MAAM,EAAEC,OAAO,EAAEC,wBAAwB,EAAEC,QAAQ,EAAEC,WAAW,EAAE,GAAGJ;IACrE,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7BM,OACEL,0BACA;IAEF,MAAMM,+BAA+BlB,wBACnCY;IAEF,MAAMO,wBAAwBC,kBAAkBF;IAChD,MAAMG,eAAeC,4BAA4BP;IAEjD,IAAIQ,eAAeP;IACnB,IAAIQ,aAAab,QAAQ,IAAI,CAAC,KAAK;IACnC,IAAIc,cAAcd,QAAQ,IAAI,CAAC,MAAM;IACrC,IAAIe,qBAAqBF;IACzB,IAAIG,sBAAsBF;IAE1B,IAAIf,QAAQ,YAAY,EAAE;YAWXkB,4BACCC;QAXdZ,OACEP,QAAQ,YAAY,CAAC,IAAI,EACzB;QAEFO,OACEP,QAAQ,YAAY,CAAC,WAAW,EAChC;QAGFa,eAAeb,QAAQ,YAAY,CAAC,WAAW;QAC/Cc,aAAa,QAAAI,CAAAA,6BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,2BAA2B,KAAK;QAC7CH,cAAc,QAAAI,CAAAA,8BAAAA,QAAQ,YAAY,CAAC,IAAI,AAAD,IAAxBA,KAAAA,IAAAA,4BAA2B,MAAM;QAC/CH,qBAAqBF;QACrBG,sBAAsBF;IACxB,OAAO,IAAIV,AAAW,iBAAXA,QAAyB;QAClC,MAAMe,eAAe,MAAMC,4BAA4BR;QACvDC,aAAaM,aAAa,KAAK;QAC/BL,cAAcK,aAAa,MAAM;QACjCP,eAAeO,aAAa,WAAW;IACzC;IAEA,MAAMzB,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKE;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMJ;gBACR;aACD;QACH;KACD;IAED,IAAI,AAAoC,YAApC,OAAOP,0BAAuC;QAChD,MAAMoB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQU,yBAAyB,MAAM;YACvC,yBAAyBA,yBAAyB,uBAAuB;QAC3E;QACAP,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMC,MAAM,MAAMpB,SAASR,MAAM6B,aAAa,eAAe,EAAEpB;IAE/D,MAAMqB,cAAcC,KAAK,SAAS,CAACH,IAAI,OAAO;IAE9C,IAAII;IACJ,IAAIC,kBAAkB,cAAcL,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,QAAQ,GAAG,EAAE;IAC3E,IAAIM,SACF,YAAYN,IAAI,OAAO,GAAGA,IAAI,OAAO,CAAC,MAAM,GAAG,EAAE;IACnD,IAAI;QACF,IACE,UAAUA,IAAI,OAAO,IACrBO,MAAM,OAAO,CAACP,IAAI,OAAO,CAAC,IAAI,KAC9BA,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,GAC3B;gBAKEQ,6BAAAA,uBACAC,6BAAAA;YALFL,UAAUM,gBACRV,IAAI,OAAO,CAAC,IAAI,EAChBT,YACAC,aAAAA,QACAgB,CAAAA,wBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,sBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,IAAI,UAChCC,CAAAA,yBAAAA,QAAQ,YAAY,AAAD,IAAnBA,KAAAA,IAAAA,QAAAA,CAAAA,8BAAAA,uBAAsB,IAAI,AAAD,IAAzBA,KAAAA,IAAAA,4BAA4B,GAAG,EAC/BhB,oBACAC,qBACAZ;YAGFlB,aAAa,WAAWwC;YAExB,MAAMO,aAAa;gBACjB,GAAGP,QAAQ,IAAI,GAAGA,QAAQ,KAAK,GAAG;gBAClC,GAAGA,QAAQ,GAAG,GAAGA,QAAQ,MAAM,GAAG;YACpC;YAEA,MAAMQ,UAA+BC,0BACnCF,YACA1B;YAEFqB,SAAS,EAAE;YAEX,IAAIM,SACFP,kBAAkB;gBAACO;aAAQ;QAE/B;IACF,EAAE,OAAOE,GAAG;QACV,MAAMC,MACJD,aAAaE,QACT,CAAC,sBAAsB,EAAEF,EAAE,OAAO,EAAE,GACpC;QACN,IAAI,AAACR,UAAUA,AAAAA,CAAAA,QAAAA,SAAAA,KAAAA,IAAAA,OAAQ,MAAM,AAAD,MAAM,GAGhCA,OAAO,IAAI,CAAC,CAAC,CAAC,EAAES,IAAI,CAAC,CAAC;aAFtBT,SAAS;YAACS;SAAI;IAIlB;IAEA,OAAO;QACL,MAAMX;QACN,aAAa;YACX,UAAUC;YACV,QAAQC;QACV;QACAJ;QACA,OAAOF,IAAI,KAAK;IAClB;AACF;AAEO,eAAeiB,gBAAgBxC,OAIrC;IAOC,MAAM,EAAEC,OAAO,EAAEwC,kBAAkB,EAAErC,WAAW,EAAE,GAAGJ;IACrD,MAAM,EAAEK,MAAM,EAAE,GAAGD;IACnB,MAAM,EAAEE,gBAAgB,EAAE,GAAGL;IAE7B,MAAMU,eAAe+B,4BAA4BrC;IACjD,MAAMsC,gCAAgCC,0BACpCtD,wBAAwBmD;IAE1B,MAAM9C,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS;gBACP;oBACE,MAAM;oBACN,WAAW;wBACT,KAAKL;wBACL,QAAQ;oBACV;gBACF;gBACA;oBACE,MAAM;oBACN,MAAMqC;gBACR;aACD;QACH;KACD;IAED,IAAI,AAA8B,YAA9B,OAAOF,oBAAiC;QAC1C,MAAMnB,SAAS,MAAM9B,mBAAmB;YACtC,QAAQiD,mBAAmB,MAAM;YACjC,yBAAyBA,mBAAmB,uBAAuB;QACrE;QACA9C,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMuB,SAAS,MAAMC,yBACnBnD,MACA6B,aAAa,YAAY,EACzBpB;IAGF,IAAI2C;IACJ,MAAMC,cAAcH,OAAO,OAAO,CAAC,IAAI;IACvC,IAAIG,aAAa;QACf,MAAMC,aAAahB,gBACjBe,aACA/C,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAEFhB,aAAa,0BAA0B4D;QAEvC,MAAMC,oBAAoBL,OAAO,OAAO,CAAC,eAAe,IAAI,EAAE;QAC9DxD,aAAa,wBAAwB6D;QAErC,MAAMC,iBAAiBD,kBACpB,MAAM,CAAC,CAACE,OAAStB,MAAM,OAAO,CAACsB,OAC/B,GAAG,CAAC,CAACA,OACGnB,gBACLmB,MACAnD,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnB,GACA,GACAA,QAAQ,IAAI,CAAC,KAAK,EAClBA,QAAQ,IAAI,CAAC,MAAM,EACnBI;QAGNhB,aAAa,qBAAqB8D;QAGlC,MAAME,aAAaC,WAAW;YAACL;eAAeE;SAAe;QAC7D9D,aAAa,iBAAiBgE;QAG9BN,cAAcQ,iBAAiBF,YAAYpD,QAAQ,IAAI,EAAEI;QACzDhB,aAAa,2BAA2B0D;IAC1C;IAEA,IAAIS,cAAclD;IAClB,IAAIyC,aAAa;QACf,MAAMU,gBAAgB,MAAMC,WAC1BpD,kBACAyC,aACA1C,AAAW,iBAAXA;QAEFmD,cAAcC,cAAc,WAAW;QACvCV,YAAY,KAAK,GAAGU,cAAc,KAAK;QACvCV,YAAY,MAAM,GAAGU,cAAc,MAAM;IAC3C;IAEA,OAAO;QACL,MAAMV;QACNS;QACA,OAAOX,OAAO,OAAO,CAAC,KAAK;QAC3B,aAAanB,KAAK,SAAS,CAACmB,OAAO,OAAO;QAC1C,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAec,qBAAwB3D,OAO7C;IACC,MAAM,EAAE4D,SAAS,EAAE3D,OAAO,EAAE4D,aAAa,EAAEpE,gBAAgB,EAAEW,WAAW,EAAE,GACxEJ;IACF,MAAMW,eAAemD;IACrB,MAAM,EAAExD,gBAAgB,EAAE,GAAGL;IAE7B,MAAM8D,wBAAwBC,uBAC5BhE,QAAQ,eAAe,IAAI,IAC3B4D;IAGF,MAAMK,cAAyD,EAAE;IAEjE,IAAIJ,AAAAA,CAAAA,QAAAA,gBAAAA,KAAAA,IAAAA,cAAe,kBAAkB,AAAD,MAAM,OACxCI,YAAY,IAAI,CAAC;QACf,MAAM;QACN,WAAW;YACT,KAAK3D;YACL,QAAQ;QACV;IACF;IAGF2D,YAAY,IAAI,CAAC;QACf,MAAM;QACN,MAAMF;IACR;IAEA,MAAMpE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAASsD;QACX;KACD;IAED,IAAIxE,kBAAkB;QACpB,MAAM6B,SAAS,MAAM9B,mBAAmB;YACtC,QAAQC,iBAAiB,MAAM;YAC/B,yBAAyBA,iBAAiB,uBAAuB;QACnE;QACAE,KAAK,IAAI,IAAI2B;IACf;IAEA,MAAMuB,SAAS,MAAMC,yBACnBnD,MACA6B,aAAa,YAAY,EACzBpB;IAEF,OAAO;QACL,aAAayC,OAAO,OAAO;QAC3B,OAAOA,OAAO,KAAK;IACrB;AACF;AAEO,eAAeqB,sBACpBC,WAAmB,EACnBhE,QAAwE,EACxEC,WAAyB;IAKzB,MAAMO,eAAeyD;IACrB,MAAMC,aAAaC,0BAA0BH;IAE7C,MAAMxE,OAAe;QACnB;YAAE,MAAM;YAAU,SAASgB;QAAa;QACxC;YACE,MAAM;YACN,SAAS0D;QACX;KACD;IAED,MAAMxB,SAAS,MAAM1C,SACnBR,MACA6B,aAAa,eAAe,EAC5BpB;IAGF,OAAO;QACL,kBAAkByC,OAAO,OAAO,CAAC,gBAAgB,IAAI;QACrD,OAAOA,OAAO,KAAK;IACrB;AACF"}

package/dist/es/ai-model/service-caller/index.mjs

@@ -73,6 +73,21 @@ async function callAI(messages, AIActionTypeValue, modelConfig, options) {
     let accumulated = '';
     let usage;
     let timeCost;
+    const buildUsageInfo = (usageData)=>{
+        var _usageData_prompt_tokens_details;
+        if (!usageData) return;
+        const cachedInputTokens = null == usageData ? void 0 : null == (_usageData_prompt_tokens_details = usageData.prompt_tokens_details) ? void 0 : _usageData_prompt_tokens_details.cached_tokens;
+        return {
+            prompt_tokens: usageData.prompt_tokens ?? 0,
+            completion_tokens: usageData.completion_tokens ?? 0,
+            total_tokens: usageData.total_tokens ?? 0,
+            cached_input: cachedInputTokens ?? 0,
+            time_cost: timeCost ?? 0,
+            model_name: modelName,
+            model_description: modelDescription,
+            intent: modelConfig.intent
+        };
+    };
     const commonConfig = {
         temperature: 'vlm-ui-tars' === vlMode ? 0.0 : void 0,
         stream: !!isStreaming,
@@ -123,15 +138,7 @@ async function callAI(messages, AIActionTypeValue, modelConfig, options) {
                         accumulated,
                         reasoning_content: '',
                         isComplete: true,
-                        usage: {
-                            prompt_tokens: usage.prompt_tokens ?? 0,
-                            completion_tokens: usage.completion_tokens ?? 0,
-                            total_tokens: usage.total_tokens ?? 0,
-                            time_cost: timeCost ?? 0,
-                            model_name: modelName,
-                            model_description: modelDescription,
-                            intent: modelConfig.intent
-                        }
+                        usage: buildUsageInfo(usage)
                     };
                     options.onChunk(finalChunk);
                     break;
@@ -166,15 +173,7 @@ async function callAI(messages, AIActionTypeValue, modelConfig, options) {
         }
         return {
             content: content || '',
-            usage: usage ? {
-                prompt_tokens: usage.prompt_tokens ?? 0,
-                completion_tokens: usage.completion_tokens ?? 0,
-                total_tokens: usage.total_tokens ?? 0,
-                time_cost: timeCost ?? 0,
-                model_name: modelName,
-                model_description: modelDescription,
-                intent: modelConfig.intent
-            } : void 0,
+            usage: buildUsageInfo(usage),
             isStreamed: !!isStreaming
         };
     } catch (e) {

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_LANGFUSE_DEBUG,\n  MIDSCENE_LANGSMITH_DEBUG,\n  MIDSCENE_MODEL_MAX_TOKENS,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n  } = modelConfig;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    ...(proxyAgent ? { httpAgent: proxyAgent as any } : {}),\n    ...openaiExtraConfig,\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  // LangSmith wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langsmithModule = 'langsmith/wrappers';\n    const { wrapOpenAI } = await import(langsmithModule);\n    openai = wrapOpenAI(openai);\n  }\n\n  // Langfuse wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGFUSE_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langfuse is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langfuse wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langfuseModule = 'langfuse';\n    const { observeOpenAI } = await import(langfuseModule);\n    openai = observeOpenAI(openai);\n  }\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      AIActionTypeValue,\n      modelConfig,\n    });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens =\n    globalConfigManager.getEnvConfigValue(MIDSCENE_MODEL_MAX_TOKENS) ??\n    globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : undefined,\n    stream: !!isStreaming,\n    max_tokens: typeof maxTokens === 'number' ? maxTokens : undefined,\n    ...(vlMode === 'qwen2.5-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: {\n              prompt_tokens: usage.prompt_tokens ?? 0,\n              completion_tokens: usage.completion_tokens ?? 0,\n              total_tokens: usage.total_tokens ?? 0,\n              time_cost: timeCost ?? 0,\n              model_name: modelName,\n              model_description: modelDescription,\n              intent: modelConfig.intent,\n            },\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        response_format: responseFormat,\n        ...commonConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      usage = result.usage;\n    }\n\n    debugCall(`response: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n      case AIActionType.TEXT:\n        // No response format for plain text - return as-is\n        responseFormat = undefined;\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  // Skip for plain text to allow string output\n  if (\n    modelName === 'gpt-4o-2024-05-13' &&\n    AIActionTypeValue !== AIActionType.TEXT\n  ) {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; contentString: string; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return {\n    content: jsonContent,\n    contentString: response.content,\n    usage: response.usage,\n  };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\n/**\n * Normalize a parsed JSON object by trimming whitespace from:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction normalizeJsonObject(obj: any): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n\n      // Recursively normalize the value\n      let normalizedValue = normalizeJsonObject(value);\n\n      // Trim all string values\n      if (typeof normalizedValue === 'string') {\n        normalizedValue = normalizedValue.trim();\n      }\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n\n  let parsed: any;\n  try {\n    parsed = JSON.parse(cleanJsonString);\n    return normalizeJsonObject(parsed);\n  } catch {}\n  try {\n    parsed = JSON.parse(jsonrepair(cleanJsonString));\n    return normalizeJsonObject(parsed);\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    parsed = JSON.parse(jsonrepair(jsonString));\n    return normalizeJsonObject(parsed);\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","openAIOptions","baseOpenAI","OpenAI","openai","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","ifInBrowser","Error","console","langsmithModule","wrapOpenAI","MIDSCENE_LANGFUSE_DEBUG","langfuseModule","observeOpenAI","wrappedClient","callAI","messages","options","completion","responseFormat","getResponseFormat","maxTokens","MIDSCENE_MODEL_MAX_TOKENS","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","undefined","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","assert","e","newError","AIActionType","assertSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","normalizeJsonObject","obj","Array","item","normalized","key","value","Object","trimmedKey","normalizedValue","cleanJsonString","_cleanJsonString_match","Number","parsed","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;AAyBA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EACnB,GAAGV;IAEJ,IAAIW;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIX,WAAW;QACbU,WAAW,oBAAoBV;QAC/BS,aAAa,IAAIG,gBAAgBZ;IACnC,OAAO,IAAID,YAAY;QACrBW,WAAW,qBAAqBX;QAChCU,aAAa,IAAII,gBAAgBd;IACnC;IAEA,MAAMe,gBAAgB;QACpB,SAASZ;QACT,QAAQC;QACR,GAAIM,aAAa;YAAE,WAAWA;QAAkB,IAAI,CAAC,CAAC;QACtD,GAAGL,iBAAiB;QACpB,yBAAyB;IAC3B;IAEA,MAAMW,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAGrB,IACEE,UACAC,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIC,aACF,MAAM,IAAIC,MAAM;QAElBC,QAAQ,GAAG,CAAC;QAEZ,MAAMC,kBAAkB;QACxB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;QACpCN,SAASO,WAAWP;IACtB;IAGA,IACEA,UACAC,oBAAoB,qBAAqB,CAACO,0BAC1C;QACA,IAAIL,aACF,MAAM,IAAIC,MAAM;QAElBC,QAAQ,GAAG,CAAC;QAEZ,MAAMI,iBAAiB;QACvB,MAAM,EAAEC,aAAa,EAAE,GAAG,MAAM,MAAM,CAACD;QACvCT,SAASU,cAAcV;IACzB;IAEA,IAAIT,oBAAoB;QACtB,MAAMoB,gBAAgB,MAAMpB,mBAAmBO,YAAYD;QAE3D,IAAIc,eACFX,SAASW;IAEb;IAEA,OAAO;QACL,YAAYX,OAAO,IAAI,CAAC,WAAW;QACnChB;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAesB,OACpBC,QAAsC,EACtCjC,iBAA+B,EAC/BC,WAAyB,EACzBiC,OAGC;IAED,MAAM,EAAEC,UAAU,EAAE/B,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMX,iBAAiB;QACrBC;QACAC;IACF;IAEF,MAAMmC,iBAAiBC,kBAAkBjC,WAAWJ;IAEpD,MAAMsC,YACJjB,oBAAoB,iBAAiB,CAACkB,8BACtClB,oBAAoB,iBAAiB,CAACmB;IACxC,MAAMC,YAAY3B,SAAS;IAC3B,MAAM4B,oBAAoB5B,SAAS;IACnC,MAAM6B,qBAAqB7B,SAAS;IAEpC,MAAM8B,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAazC,AAAW,kBAAXA,SAA2B,MAAM0C;QAC9C,QAAQ,CAAC,CAACN;QACV,YAAY,AAAqB,YAArB,OAAOR,YAAyBA,YAAYc;QACxD,GAAI1C,AAAW,iBAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACF+B,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAE1C,WAAW;QAGrE,IAAI0C,aAAa;YACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;gBACE,OAAO/B;gBACP6B;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMG,SAASD,OAAQ;oBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;gBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;gBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;gBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;gBAGrB,IAAIP,WAAWW,mBAAmB;oBAChCV,eAAeD;oBACf,MAAMY,YAAiC;wBACrCZ;wBACAW;wBACAV;wBACA,YAAY;wBACZ,OAAOI;oBACT;oBACAlB,QAAQ,OAAO,CAAEyB;gBACnB;gBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;oBACrCP,WAAWL,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACK,OAAO;wBAEV,MAAMW,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACb,YAAY,MAAM,GAAG;wBAElCC,QAAQ;4BACN,eAAeW;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTd;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAO;4BACL,eAAeC,MAAM,aAAa,IAAI;4BACtC,mBAAmBA,MAAM,iBAAiB,IAAI;4BAC9C,cAAcA,MAAM,YAAY,IAAI;4BACpC,WAAWC,YAAY;4BACvB,YAAY9C;4BACZ,mBAAmBI;4BACnB,QAAQP,YAAY,MAAM;wBAC5B;oBACF;oBACAiC,QAAQ,OAAO,CAAE4B;oBACjB;gBACF;YACF;YACAf,UAAUC;YACVN,kBACE,CAAC,iBAAiB,EAAEtC,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAEwC,UAAU;QAEvF,OAAO;gBAUqGa,eAAyDC,gBAAwDC;YAT3N,MAAMC,SAAS,MAAM/B,WAAW,MAAM,CAAC;gBACrC,OAAO/B;gBACP6B;gBACA,iBAAiBG;gBACjB,GAAGe,YAAY;YACjB;YACAD,WAAWL,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAEtC,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAEsD,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEf,SAAS,aAAa,EAAEgB,OAAO,WAAW,IAAI,IAAI;YAG3TvB,mBAAmB,CAAC,oBAAoB,EAAEwB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhEnB,UAAUmB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3CjB,QAAQiB,OAAO,KAAK;QACtB;QAEAzB,UAAU,CAAC,UAAU,EAAEM,SAAS;QAChCqB,OAAOrB,SAAS;QAGhB,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMW,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEd,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeW;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASb,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAY9C;gBACZ,mBAAmBI;gBACnB,QAAQP,YAAY,MAAM;YAC5B,IACAmD;YACJ,YAAY,CAAC,CAACN;QAChB;IACF,EAAE,OAAOuB,GAAQ;QACf5C,QAAQ,KAAK,CAAC,kBAAkB4C;QAChC,MAAMC,WAAW,IAAI9C,MACnB,CAAC,eAAe,EAAEsB,cAAc,eAAe,GAAG,kBAAkB,EAAEuB,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,MAAMjC,oBAAoB,CAC/BjC,WACAJ;IAIA,IAAIoC;IAKJ,IAAIhC,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAKuE,aAAa,MAAM;YACtBnC,iBAAiBoC;YACjB;QACF,KAAKD,aAAa,IAAI;YACpBnC,iBAAiBqC;YACjB;QACF,KAAKF,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChCnC,iBAAiB;gBAAE,MAAMsC,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKH,aAAa,IAAI;YAEpBnC,iBAAiBgB;YACjB;IACJ;IAKF,IACEhD,AAAc,wBAAdA,aACAJ,sBAAsBuE,aAAa,IAAI,EAEvCnC,iBAAiB;QAAE,MAAMsC,iBAAiB,IAAI;IAAC;IAGjD,OAAOtC;AACT;AAEO,eAAeuC,yBACpB1C,QAAsC,EACtCjC,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM2E,WAAW,MAAM5C,OAAOC,UAAUjC,mBAAmBC;IAC3DmE,OAAOQ,UAAU;IACjB,MAAMlE,SAAST,YAAY,MAAM;IACjC,MAAM4E,cAAcC,cAAcF,SAAS,OAAO,EAAElE;IACpD,OAAO;QACL,SAASmE;QACT,eAAeD,SAAS,OAAO;QAC/B,OAAOA,SAAS,KAAK;IACvB;AACF;AAEO,eAAeG,yBACpBC,IAAY,EACZhF,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAE8C,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAOgD,MAAMhF,mBAAmBC;IACjE,OAAO;QAAE8C;QAASE;IAAM;AAC1B;AAEO,SAASgC,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAQA,SAASC,oBAAoBC,GAAQ;IAEnC,IAAIA,QAAAA,KACF,OAAOA;IAIT,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACE,OAASH,oBAAoBG;IAI/C,IAAI,AAAe,YAAf,OAAOF,KAAkB;QAC3B,MAAMG,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACN,KAAM;YAE9C,MAAMO,aAAaH,IAAI,IAAI;YAG3B,IAAII,kBAAkBT,oBAAoBM;YAG1C,IAAI,AAA2B,YAA3B,OAAOG,iBACTA,kBAAkBA,gBAAgB,IAAI;YAGxCL,UAAU,CAACI,WAAW,GAAGC;QAC3B;QAEA,OAAOL;IACT;IAGA,IAAI,AAAe,YAAf,OAAOH,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEO,SAASV,cAAcQ,KAAa,EAAE5E,MAAgC;IAC3E,MAAMuF,kBAAkBhB,yBAAyBK;IAEjD,IAAIW,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAACC;IACT;IAEA,IAAIC;IACJ,IAAI;QACFA,SAASjC,KAAK,KAAK,CAAC8B;QACpB,OAAOV,oBAAoBa;IAC7B,EAAE,OAAM,CAAC;IACT,IAAI;QACFA,SAASjC,KAAK,KAAK,CAACkC,WAAWJ;QAC/B,OAAOV,oBAAoBa;IAC7B,EAAE,OAAO/B,GAAG,CAAC;IAEb,IAAI3D,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAM4F,aAAajB,yBAAyBY;QAC5CG,SAASjC,KAAK,KAAK,CAACkC,WAAWC;QAC/B,OAAOf,oBAAoBa;IAC7B;IACA,MAAM5E,MAAM,CAAC,+BAA+B,EAAE8D,OAAO;AACvD"}
+{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_LANGFUSE_DEBUG,\n  MIDSCENE_LANGSMITH_DEBUG,\n  MIDSCENE_MODEL_MAX_TOKENS,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    createOpenAIClient,\n  } = modelConfig;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    ...(proxyAgent ? { httpAgent: proxyAgent as any } : {}),\n    ...openaiExtraConfig,\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  // LangSmith wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langsmithModule = 'langsmith/wrappers';\n    const { wrapOpenAI } = await import(langsmithModule);\n    openai = wrapOpenAI(openai);\n  }\n\n  // Langfuse wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGFUSE_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langfuse is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langfuse wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langfuseModule = 'langfuse';\n    const { observeOpenAI } = await import(langfuseModule);\n    openai = observeOpenAI(openai);\n  }\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, modelName, modelDescription, uiTarsVersion, vlMode } =\n    await createChatClient({\n      AIActionTypeValue,\n      modelConfig,\n    });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens =\n    globalConfigManager.getEnvConfigValue(MIDSCENE_MODEL_MAX_TOKENS) ??\n    globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const buildUsageInfo = (usageData?: OpenAI.CompletionUsage) => {\n    if (!usageData) return undefined;\n\n    const cachedInputTokens = (\n      usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n    )?.prompt_tokens_details?.cached_tokens;\n\n    return {\n      prompt_tokens: usageData.prompt_tokens ?? 0,\n      completion_tokens: usageData.completion_tokens ?? 0,\n      total_tokens: usageData.total_tokens ?? 0,\n      cached_input: cachedInputTokens ?? 0,\n      time_cost: timeCost ?? 0,\n      model_name: modelName,\n      model_description: modelDescription,\n      intent: modelConfig.intent,\n    } satisfies AIUsageInfo;\n  };\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : undefined,\n    stream: !!isStreaming,\n    max_tokens: typeof maxTokens === 'number' ? maxTokens : undefined,\n    ...(vlMode === 'qwen2.5-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: buildUsageInfo(usage),\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n      );\n    } else {\n      const result = await completion.create({\n        model: modelName,\n        messages,\n        response_format: responseFormat,\n        ...commonConfig,\n      } as any);\n      timeCost = Date.now() - startTime;\n\n      debugProfileStats(\n        `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n      );\n\n      debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);\n\n      assert(\n        result.choices,\n        `invalid response from LLM service: ${JSON.stringify(result)}`,\n      );\n      content = result.choices[0].message.content!;\n      usage = result.usage;\n    }\n\n    debugCall(`response: ${content}`);\n    assert(content, 'empty content');\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      } as OpenAI.CompletionUsage;\n    }\n\n    return {\n      content: content || '',\n      usage: buildUsageInfo(usage),\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n      case AIActionType.TEXT:\n        // No response format for plain text - return as-is\n        responseFormat = undefined;\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  // Skip for plain text to allow string output\n  if (\n    modelName === 'gpt-4o-2024-05-13' &&\n    AIActionTypeValue !== AIActionType.TEXT\n  ) {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; contentString: string; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return {\n    content: jsonContent,\n    contentString: response.content,\n    usage: response.usage,\n  };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\n/**\n * Normalize a parsed JSON object by trimming whitespace from:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction normalizeJsonObject(obj: any): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n\n      // Recursively normalize the value\n      let normalizedValue = normalizeJsonObject(value);\n\n      // Trim all string values\n      if (typeof normalizedValue === 'string') {\n        normalizedValue = normalizedValue.trim();\n      }\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n\n  let parsed: any;\n  try {\n    parsed = JSON.parse(cleanJsonString);\n    return normalizeJsonObject(parsed);\n  } catch {}\n  try {\n    parsed = JSON.parse(jsonrepair(cleanJsonString));\n    return normalizeJsonObject(parsed);\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    parsed = JSON.parse(jsonrepair(jsonString));\n    return normalizeJsonObject(parsed);\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","createOpenAIClient","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","openAIOptions","baseOpenAI","OpenAI","openai","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","ifInBrowser","Error","console","langsmithModule","wrapOpenAI","MIDSCENE_LANGFUSE_DEBUG","langfuseModule","observeOpenAI","wrappedClient","callAI","messages","options","completion","responseFormat","getResponseFormat","maxTokens","MIDSCENE_MODEL_MAX_TOKENS","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","buildUsageInfo","usageData","_usageData_prompt_tokens_details","cachedInputTokens","commonConfig","undefined","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","assert","e","newError","AIActionType","assertSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","normalizeJsonObject","obj","Array","item","normalized","key","value","Object","trimmedKey","normalizedValue","cleanJsonString","_cleanJsonString_match","Number","parsed","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;AAyBA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAOC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,kBAAkB,EACnB,GAAGV;IAEJ,IAAIW;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIX,WAAW;QACbU,WAAW,oBAAoBV;QAC/BS,aAAa,IAAIG,gBAAgBZ;IACnC,OAAO,IAAID,YAAY;QACrBW,WAAW,qBAAqBX;QAChCU,aAAa,IAAII,gBAAgBd;IACnC;IAEA,MAAMe,gBAAgB;QACpB,SAASZ;QACT,QAAQC;QACR,GAAIM,aAAa;YAAE,WAAWA;QAAkB,IAAI,CAAC,CAAC;QACtD,GAAGL,iBAAiB;QACpB,yBAAyB;IAC3B;IAEA,MAAMW,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAGrB,IACEE,UACAC,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIC,aACF,MAAM,IAAIC,MAAM;QAElBC,QAAQ,GAAG,CAAC;QAEZ,MAAMC,kBAAkB;QACxB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;QACpCN,SAASO,WAAWP;IACtB;IAGA,IACEA,UACAC,oBAAoB,qBAAqB,CAACO,0BAC1C;QACA,IAAIL,aACF,MAAM,IAAIC,MAAM;QAElBC,QAAQ,GAAG,CAAC;QAEZ,MAAMI,iBAAiB;QACvB,MAAM,EAAEC,aAAa,EAAE,GAAG,MAAM,MAAM,CAACD;QACvCT,SAASU,cAAcV;IACzB;IAEA,IAAIT,oBAAoB;QACtB,MAAMoB,gBAAgB,MAAMpB,mBAAmBO,YAAYD;QAE3D,IAAIc,eACFX,SAASW;IAEb;IAEA,OAAO;QACL,YAAYX,OAAO,IAAI,CAAC,WAAW;QACnChB;QACAI;QACAC;QACAC;IACF;AACF;AAEO,eAAesB,OACpBC,QAAsC,EACtCjC,iBAA+B,EAC/BC,WAAyB,EACzBiC,OAGC;IAED,MAAM,EAAEC,UAAU,EAAE/B,SAAS,EAAEI,gBAAgB,EAAEC,aAAa,EAAEC,MAAM,EAAE,GACtE,MAAMX,iBAAiB;QACrBC;QACAC;IACF;IAEF,MAAMmC,iBAAiBC,kBAAkBjC,WAAWJ;IAEpD,MAAMsC,YACJjB,oBAAoB,iBAAiB,CAACkB,8BACtClB,oBAAoB,iBAAiB,CAACmB;IACxC,MAAMC,YAAY3B,SAAS;IAC3B,MAAM4B,oBAAoB5B,SAAS;IACnC,MAAM6B,qBAAqB7B,SAAS;IAEpC,MAAM8B,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,iBAAiB,CAACC;YAGIC;QAF1B,IAAI,CAACD,WAAW;QAEhB,MAAME,oBACJF,QAAAA,YAAAA,KAAAA,IAAAA,QADwBC,CAAAA,mCACxBD,UACC,qBAAqB,AAAD,IAFGC,KAAAA,IAAAA,iCAEA,aAAa;QAEvC,OAAO;YACL,eAAeD,UAAU,aAAa,IAAI;YAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;YAClD,cAAcA,UAAU,YAAY,IAAI;YACxC,cAAcE,qBAAqB;YACnC,WAAWJ,YAAY;YACvB,YAAY9C;YACZ,mBAAmBI;YACnB,QAAQP,YAAY,MAAM;QAC5B;IACF;IAEA,MAAMsD,eAAe;QACnB,aAAa7C,AAAW,kBAAXA,SAA2B,MAAM8C;QAC9C,QAAQ,CAAC,CAACV;QACV,YAAY,AAAqB,YAArB,OAAOR,YAAyBA,YAAYkB;QACxD,GAAI9C,AAAW,iBAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACF+B,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAE1C,WAAW;QAGrE,IAAI0C,aAAa;YACf,MAAMW,SAAU,MAAMtB,WAAW,MAAM,CACrC;gBACE,OAAO/B;gBACP6B;gBACA,iBAAiBG;gBACjB,GAAGmB,YAAY;YACjB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMG,SAASD,OAAQ;oBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;gBAtBJ,MAAMd,UAAUY,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;gBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;gBAG3D,IAAIF,MAAM,KAAK,EACbT,QAAQS,MAAM,KAAK;gBAGrB,IAAIX,WAAWe,mBAAmB;oBAChCd,eAAeD;oBACf,MAAMgB,YAAiC;wBACrChB;wBACAe;wBACAd;wBACA,YAAY;wBACZ,OAAOQ;oBACT;oBACAtB,QAAQ,OAAO,CAAE6B;gBACnB;gBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;oBACrCX,WAAWL,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACK,OAAO;wBAEV,MAAMe,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACjB,YAAY,MAAM,GAAG;wBAElCC,QAAQ;4BACN,eAAee;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTlB;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAOG,eAAeF;oBACxB;oBACAf,QAAQ,OAAO,CAAEgC;oBACjB;gBACF;YACF;YACAnB,UAAUC;YACVN,kBACE,CAAC,iBAAiB,EAAEtC,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAEwC,UAAU;QAEvF,OAAO;gBAUqGiB,eAAyDC,gBAAwDC;YAT3N,MAAMC,SAAS,MAAMnC,WAAW,MAAM,CAAC;gBACrC,OAAO/B;gBACP6B;gBACA,iBAAiBG;gBACjB,GAAGmB,YAAY;YACjB;YACAL,WAAWL,KAAK,GAAG,KAAKD;YAExBF,kBACE,CAAC,OAAO,EAAEtC,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAE0D,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEnB,SAAS,aAAa,EAAEoB,OAAO,WAAW,IAAI,IAAI;YAG3T3B,mBAAmB,CAAC,oBAAoB,EAAE4B,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;YAExEE,OACEF,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;YAEhEvB,UAAUuB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC3CrB,QAAQqB,OAAO,KAAK;QACtB;QAEA7B,UAAU,CAAC,UAAU,EAAEM,SAAS;QAChCyB,OAAOzB,SAAS;QAGhB,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMe,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAElB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAee;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASjB,WAAW;YACpB,OAAOI,eAAeF;YACtB,YAAY,CAAC,CAACH;QAChB;IACF,EAAE,OAAO2B,GAAQ;QACfhD,QAAQ,KAAK,CAAC,kBAAkBgD;QAChC,MAAMC,WAAW,IAAIlD,MACnB,CAAC,eAAe,EAAEsB,cAAc,eAAe,GAAG,kBAAkB,EAAE2B,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,MAAMrC,oBAAoB,CAC/BjC,WACAJ;IAIA,IAAIoC;IAKJ,IAAIhC,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAK2E,aAAa,MAAM;YACtBvC,iBAAiBwC;YACjB;QACF,KAAKD,aAAa,IAAI;YACpBvC,iBAAiByC;YACjB;QACF,KAAKF,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChCvC,iBAAiB;gBAAE,MAAM0C,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKH,aAAa,IAAI;YAEpBvC,iBAAiBoB;YACjB;IACJ;IAKF,IACEpD,AAAc,wBAAdA,aACAJ,sBAAsB2E,aAAa,IAAI,EAEvCvC,iBAAiB;QAAE,MAAM0C,iBAAiB,IAAI;IAAC;IAGjD,OAAO1C;AACT;AAEO,eAAe2C,yBACpB9C,QAAsC,EACtCjC,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM+E,WAAW,MAAMhD,OAAOC,UAAUjC,mBAAmBC;IAC3DuE,OAAOQ,UAAU;IACjB,MAAMtE,SAAST,YAAY,MAAM;IACjC,MAAMgF,cAAcC,cAAcF,SAAS,OAAO,EAAEtE;IACpD,OAAO;QACL,SAASuE;QACT,eAAeD,SAAS,OAAO;QAC/B,OAAOA,SAAS,KAAK;IACvB;AACF;AAEO,eAAeG,yBACpBC,IAAY,EACZpF,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAE8C,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAOoD,MAAMpF,mBAAmBC;IACjE,OAAO;QAAE8C;QAASE;IAAM;AAC1B;AAEO,SAASoC,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAQA,SAASC,oBAAoBC,GAAQ;IAEnC,IAAIA,QAAAA,KACF,OAAOA;IAIT,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACE,OAASH,oBAAoBG;IAI/C,IAAI,AAAe,YAAf,OAAOF,KAAkB;QAC3B,MAAMG,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACN,KAAM;YAE9C,MAAMO,aAAaH,IAAI,IAAI;YAG3B,IAAII,kBAAkBT,oBAAoBM;YAG1C,IAAI,AAA2B,YAA3B,OAAOG,iBACTA,kBAAkBA,gBAAgB,IAAI;YAGxCL,UAAU,CAACI,WAAW,GAAGC;QAC3B;QAEA,OAAOL;IACT;IAGA,IAAI,AAAe,YAAf,OAAOH,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEO,SAASV,cAAcQ,KAAa,EAAEhF,MAAgC;IAC3E,MAAM2F,kBAAkBhB,yBAAyBK;IAEjD,IAAIW,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAACC;IACT;IAEA,IAAIC;IACJ,IAAI;QACFA,SAASjC,KAAK,KAAK,CAAC8B;QACpB,OAAOV,oBAAoBa;IAC7B,EAAE,OAAM,CAAC;IACT,IAAI;QACFA,SAASjC,KAAK,KAAK,CAACkC,WAAWJ;QAC/B,OAAOV,oBAAoBa;IAC7B,EAAE,OAAO/B,GAAG,CAAC;IAEb,IAAI/D,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAMgG,aAAajB,yBAAyBY;QAC5CG,SAASjC,KAAK,KAAK,CAACkC,WAAWC;QAC/B,OAAOf,oBAAoBa;IAC7B;IACA,MAAMhF,MAAM,CAAC,+BAA+B,EAAEkE,OAAO;AACvD"}

package/dist/es/task-runner.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"task-runner.mjs","sources":["webpack://@midscene/core/./src/task-runner.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n  PlanningActionParamError,\n  UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\ntype TaskRunnerInitOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n};\n\ntype TaskRunnerOperationOptions = {\n  allowWhenError?: boolean;\n};\n\nexport class TaskRunner {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  private readonly onTaskUpdate?:\n    | ((runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void)\n    | undefined;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: TaskRunnerInitOptions,\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n    this.onTaskUpdate = options?.onTaskUpdate;\n  }\n\n  private async emitOnTaskUpdate(error?: TaskExecutionError): Promise<void> {\n    if (!this.onTaskUpdate) {\n      return;\n    }\n    await this.onTaskUpdate(this, error);\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<string | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshotBase64;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    contextOrScreenshot: UIContext | string | undefined,\n    phase: 'after-calling',\n  ): void {\n    const timing = phase;\n    const screenshot =\n      typeof contextOrScreenshot === 'string'\n        ? contextOrScreenshot\n        : contextOrScreenshot?.screenshotBase64;\n    if (!timing || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private normalizeStatusFromError(\n    options?: TaskRunnerOperationOptions,\n    errorMessage?: string,\n  ): void {\n    if (this.status !== 'error') {\n      return;\n    }\n    assert(\n      options?.allowWhenError,\n      errorMessage ||\n        `task runner is in error state, cannot proceed\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    // reset runner state so new tasks can run\n    this.status = this.tasks.length > 0 ? 'pending' : 'init';\n  }\n\n  private findPreviousNonSubTaskUIContext(\n    currentIndex: number,\n  ): UIContext | undefined {\n    for (let i = currentIndex - 1; i >= 0; i--) {\n      const candidate = this.tasks[i];\n      if (!candidate || candidate.subTask) {\n        continue;\n      }\n      if (candidate.uiContext) {\n        return candidate.uiContext;\n      }\n    }\n    return undefined;\n  }\n\n  async append(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<void> {\n    this.normalizeStatusFromError(\n      options,\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n    await this.emitOnTaskUpdate();\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task, options);\n    return this.flush(options);\n  }\n\n  async flush(\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    this.normalizeStatusFromError(\n      options,\n      'task runner is in error state',\n    );\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    await this.emitOnTaskUpdate();\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskPlanningLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        await this.emitOnTaskUpdate();\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action Space', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        let uiContext: UIContext | undefined;\n        if (task.subTask) {\n          uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);\n          assert(\n            uiContext,\n            'subTask requires uiContext from previous non-subTask task',\n          );\n        } else {\n          uiContext = await this.getUiContext();\n        }\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        } else if (task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskPlanningLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action Space') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n    if (taskIndex + 1 < this.tasks.length) {\n      await this.emitOnTaskUpdate();\n    }\n\n    let finalizeError: TaskExecutionError | undefined;\n    if (!successfullyCompleted) {\n      this.status = 'error';\n      const errorTask = this.latestErrorTask();\n      const messageBase =\n        errorTask?.errorMessage ||\n        (errorTask?.error ? String(errorTask.error) : 'Task execution failed');\n      const stack = errorTask?.errorStack;\n      const message = stack ? `${messageBase}\\n${stack}` : messageBase;\n      finalizeError = new TaskExecutionError(message, this, errorTask, {\n        cause: errorTask?.error,\n      });\n      await this.emitOnTaskUpdate(finalizeError);\n    } else {\n      this.status = 'completed';\n      await this.emitOnTaskUpdate();\n    }\n\n    if (finalizeError) {\n      throw finalizeError;\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action Space',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n\nexport class TaskExecutionError extends Error {\n  runner: TaskRunner;\n\n  errorTask: ExecutionTask | null;\n\n  constructor(\n    message: string,\n    runner: TaskRunner,\n    errorTask: ExecutionTask | null,\n    options?: { cause?: unknown },\n  ) {\n    super(message, options);\n    this.runner = runner;\n    this.errorTask = errorTask;\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","error","options","_this_lastUiContext","now","Date","shouldReuse","_this_lastUiContext1","uiContext","undefined","console","task","contextOrScreenshot","phase","timing","screenshot","recorderItem","errorMessage","_this_latestErrorTask","_this_latestErrorTask1","assert","currentIndex","i","candidate","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","executorContext","isLastTask","Object","finalizeError","errorTask","messageBase","String","stack","message","TaskExecutionError","outputIndex","Math","thought","output","errorTaskIndex","dumpData","errorMsg","Error","name","uiContextBuilder","runner"],"mappings":";;;;;;;;;;;;AAgBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAczB,MAAMC;IAgCX,MAAc,iBAAiBC,KAA0B,EAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAEA;IAChC;IAOA,MAAc,aAAaC,OAAoC,EAE7D;YAOmBC;QANnB,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACJ,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,YAAY,AAAD,KACrB,IAAI,CAAC,aAAa,IAClBE,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIL;QAEzC,IAAIO,eAAAA,SAAeH,CAAAA,sBAAAA,IAAI,CAAC,aAAa,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,OAAO,AAAD,GAAG;gBAIvCI;YAHPV,MACE,CAAC,gCAAgC,EAAEO,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,QAAAG,CAAAA,uBAAAA,IAAI,CAAC,aAAa,AAAD,IAAjBA,KAAAA,IAAAA,qBAAoB,OAAO;QACpC;QAEA,IAAI;YACF,MAAMC,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYH,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGI;YAEvB,OAAOD;QACT,EAAE,OAAOP,OAAO;YACd,IAAI,CAAC,aAAa,GAAGQ;YACrB,MAAMR;QACR;IACF;IAEA,MAAc,oBAAiD;QAC7D,IAAI;YACF,MAAMO,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,gBAAgB;QACpC,EAAE,OAAOP,OAAO;YACdS,QAAQ,KAAK,CAAC,oCAAoCT;QACpD;IAEF;IAEQ,mBACNU,IAAmB,EACnBC,mBAAmD,EACnDC,KAAsB,EAChB;QACN,MAAMC,SAASD;QACf,MAAME,aACJ,AAA+B,YAA/B,OAAOH,sBACHA,sBACAA,QAAAA,sBAAAA,KAAAA,IAAAA,oBAAqB,gBAAgB;QAC3C,IAAI,CAACE,UAAU,CAACC,YACd;QAGF,MAAMC,eAAsC;YAC1C,MAAM;YACN,IAAIX,KAAK,GAAG;YACZU;YACAD;QACF;QAEA,IAAI,CAACH,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACK;aAAa;YAC9B;QACF;QACAL,KAAK,QAAQ,CAAC,IAAI,CAACK;IACrB;IAEQ,kBAAkBL,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEQ,yBACNT,OAAoC,EACpCe,YAAqB,EACf;YAOsDC,uBAAkCC;QAN9F,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb;QAEFC,OACElB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,cAAc,EACvBe,gBACE,CAAC,qDAAqD,EAAE,QAAAC,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAGlI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;IACpD;IAEQ,gCACNE,YAAoB,EACG;QACvB,IAAK,IAAIC,IAAID,eAAe,GAAGC,KAAK,GAAGA,IAAK;YAC1C,MAAMC,YAAY,IAAI,CAAC,KAAK,CAACD,EAAE;YAC/B,IAAI,AAACC,cAAaA,UAAU,OAAO,EAGnC;gBAAA,IAAIA,UAAU,SAAS,EACrB,OAAOA,UAAU,SAAS;YAC5B;QACF;IAEF;IAEA,MAAM,OACJZ,IAA+C,EAC/CT,OAAoC,EACrB;YAG+CgB,uBAAkCC;QAFhG,IAAI,CAAC,wBAAwB,CAC3BjB,SACA,CAAC,yDAAyD,EAAE,QAAAgB,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAEpI,IAAIK,MAAM,OAAO,CAACb,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACc,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACd;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;QAEhB,MAAM,IAAI,CAAC,gBAAgB;IAC7B;IAEA,MAAM,eACJA,IAA+C,EAC/CT,OAAoC,EACoB;QACxD,MAAM,IAAI,CAAC,MAAM,CAACS,MAAMT;QACxB,OAAO,IAAI,CAAC,KAAK,CAACA;IACpB;IAEA,MAAM,MACJA,OAAoC,EACoB;QACxD,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDQ,QAAQ,IAAI,CACV;QAIJ,IAAI,CAAC,wBAAwB,CAC3BR,SACA;QAEFkB,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QAEpC,MAAMM,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACf,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIe,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,MAAM,IAAI,CAAC,gBAAgB;QAC3B,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMhB,OAAO,IAAI,CAAC,KAAK,CAACgB,UAAU;YAClCP,OACET,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAON,KAAK,GAAG;YACjB;YACA,IAAI;gBACFM,KAAK,MAAM,GAAG;gBACd,MAAM,IAAI,CAAC,gBAAgB;gBAC3B,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOmB,GAAG;oBACVpB,QAAQ,KAAK,CAAC,wBAAwBoB;gBACxC;gBACAV,OACE;oBAAC;oBAAW;oBAAgB;iBAAW,CAAC,OAAO,CAACT,KAAK,IAAI,KAAK,GAC9D,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEoB,QAAQ,EAAEC,KAAK,EAAE,GAAGrB;gBAC5BS,OAAOW,UAAU,CAAC,oCAAoC,EAAEpB,KAAK,IAAI,EAAE;gBAEnE,IAAIsB;gBACJ,IAAIzB;gBACJ,IAAIG,KAAK,OAAO,EAAE;oBAChBH,YAAY,IAAI,CAAC,+BAA+B,CAACmB;oBACjDP,OACEZ,WACA;gBAEJ,OACEA,YAAY,MAAM,IAAI,CAAC,YAAY;gBAErCG,KAAK,SAAS,GAAGH;gBACjB,MAAM0B,kBAAmC;oBACvCvB;oBACA,SAASkB,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,OAAO;oBACpCrB;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BS,OACET,AAAiB,YAAjBA,KAAK,OAAO,IACVA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;gBAC3C,OAAO,IAAIvB,AAAc,eAAdA,KAAK,IAAI,EAAiB;oBACnCsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;oBACzC,IAAIvB,AAAiB,aAAjBA,KAAK,OAAO,EACdkB,qBACEI,QAAAA,cAAAA,KAAAA,IAAAA,YACC,MAAM;gBAEb,OAAO,IAAItB,AAAc,mBAAdA,KAAK,IAAI,EAClBsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;qBACpC;oBACLxB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;gBAC3C;gBAEA,MAAMC,aAAaR,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIQ,YAAY;oBACd,MAAMpB,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACJ,MAAMI,YAAY;gBAC5C;gBAEAqB,OAAO,MAAM,CAACzB,MAAMsB;gBACpBtB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGN,KAAK,GAAG;gBAC1BM,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3BgB;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBjB,KAAK,KAAK,GAAGmB;gBACbnB,KAAK,YAAY,GACfmB,AAAAA,CAAAA,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO,AAAD,KAAM,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEnB,KAAK,UAAU,GAAGmB,EAAE,KAAK;gBAEzBnB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGN,KAAK,GAAG;gBAC1BM,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3B;YACF;QACF;QAGA,IAAK,IAAIW,IAAIK,YAAY,GAAGL,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAEzB,IAAIK,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EACnC,MAAM,IAAI,CAAC,gBAAgB;QAG7B,IAAIU;QACJ,IAAKT,uBAYE;YACL,IAAI,CAAC,MAAM,GAAG;YACd,MAAM,IAAI,CAAC,gBAAgB;QAC7B,OAf4B;YAC1B,IAAI,CAAC,MAAM,GAAG;YACd,MAAMU,YAAY,IAAI,CAAC,eAAe;YACtC,MAAMC,cACJD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,YAAY,AAAD,KACrBA,CAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK,AAAD,IAAIE,OAAOF,UAAU,KAAK,IAAI,uBAAsB;YACtE,MAAMG,QAAQH,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,UAAU;YACnC,MAAMI,UAAUD,QAAQ,GAAGF,YAAY,EAAE,EAAEE,OAAO,GAAGF;YACrDF,gBAAgB,IAAIM,mBAAmBD,SAAS,IAAI,EAAEJ,WAAW;gBAC/D,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK;YACzB;YACA,MAAM,IAAI,CAAC,gBAAgB,CAACD;QAC9B;QAKA,IAAIA,eACF,MAAMA;QAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMO,cAAcC,KAAK,GAAG,CAAClB,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEmB,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAACrC,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAIqC,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,MAAMC,WAA0B;YAC9B,SAAS5C,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;QACA,OAAO4C;IACT;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMZ,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASY;YACX;YACA,SAASA;YACT,UAAU;gBACR,MAAM,IAAIC,MAAMD,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACZ;QAE1B,OAAO;YACL,QAAQ7B;YACR,QAAQ,IAAI;QACd;IACF;IA/XA,YACE2C,IAAY,EACZC,gBAA0C,EAC1CnD,OAA+B,CAC/B;QAnBF;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAEA,uBAAiB,gBAAjB;QA2BA,uBAAQ,iBAAR;QAlBE,IAAI,CAAC,MAAM,GACTA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAKA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAGkD;QACZ,IAAI,CAAC,KAAK,GAAIlD,AAAAA,CAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAK,EAAC,EAAG,GAAG,CAAC,CAACuB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGvB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,WAAW;QACvC,IAAI,CAAC,gBAAgB,GAAGmD;QACxB,IAAI,CAAC,YAAY,GAAGnD,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,YAAY;IAC3C;AAkXF;AAEO,MAAMyC,2BAA2BQ;IAKtC,YACET,OAAe,EACfY,MAAkB,EAClBhB,SAA+B,EAC/BpC,OAA6B,CAC7B;QACA,KAAK,CAACwC,SAASxC,UAVjB,0CAEA;QASE,IAAI,CAAC,MAAM,GAAGoD;QACd,IAAI,CAAC,SAAS,GAAGhB;IACnB;AACF"}
+{"version":3,"file":"task-runner.mjs","sources":["webpack://@midscene/core/./src/task-runner.ts"],"sourcesContent":["import type {\n  ExecutionDump,\n  ExecutionRecorderItem,\n  ExecutionTask,\n  ExecutionTaskActionApply,\n  ExecutionTaskApply,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskProgressOptions,\n  ExecutionTaskReturn,\n  ExecutorContext,\n  PlanningActionParamError,\n  UIContext,\n} from '@/types';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\n\nconst debug = getDebug('task-runner');\nconst UI_CONTEXT_CACHE_TTL_MS = 300;\n\ntype TaskRunnerInitOptions = ExecutionTaskProgressOptions & {\n  tasks?: ExecutionTaskApply[];\n  onTaskUpdate?: (\n    runner: TaskRunner,\n    error?: TaskExecutionError,\n  ) => Promise<void> | void;\n};\n\ntype TaskRunnerOperationOptions = {\n  allowWhenError?: boolean;\n};\n\nexport class TaskRunner {\n  name: string;\n\n  tasks: ExecutionTask[];\n\n  // status of runner\n  status: 'init' | 'pending' | 'running' | 'completed' | 'error';\n\n  onTaskStart?: ExecutionTaskProgressOptions['onTaskStart'];\n\n  private readonly uiContextBuilder: () => Promise<UIContext>;\n\n  private readonly onTaskUpdate?:\n    | ((runner: TaskRunner, error?: TaskExecutionError) => Promise<void> | void)\n    | undefined;\n\n  constructor(\n    name: string,\n    uiContextBuilder: () => Promise<UIContext>,\n    options?: TaskRunnerInitOptions,\n  ) {\n    this.status =\n      options?.tasks && options.tasks.length > 0 ? 'pending' : 'init';\n    this.name = name;\n    this.tasks = (options?.tasks || []).map((item) =>\n      this.markTaskAsPending(item),\n    );\n    this.onTaskStart = options?.onTaskStart;\n    this.uiContextBuilder = uiContextBuilder;\n    this.onTaskUpdate = options?.onTaskUpdate;\n  }\n\n  private async emitOnTaskUpdate(error?: TaskExecutionError): Promise<void> {\n    if (!this.onTaskUpdate) {\n      return;\n    }\n    await this.onTaskUpdate(this, error);\n  }\n\n  private lastUiContext?: {\n    context: UIContext;\n    capturedAt: number;\n  };\n\n  private async getUiContext(options?: { forceRefresh?: boolean }): Promise<\n    UIContext | undefined\n  > {\n    const now = Date.now();\n    const shouldReuse =\n      !options?.forceRefresh &&\n      this.lastUiContext &&\n      now - this.lastUiContext.capturedAt <= UI_CONTEXT_CACHE_TTL_MS;\n\n    if (shouldReuse && this.lastUiContext?.context) {\n      debug(\n        `reuse cached uiContext captured ${now - this.lastUiContext.capturedAt}ms ago`,\n      );\n      return this.lastUiContext?.context;\n    }\n\n    try {\n      const uiContext = await this.uiContextBuilder();\n      if (uiContext) {\n        this.lastUiContext = {\n          context: uiContext,\n          capturedAt: Date.now(),\n        };\n      } else {\n        this.lastUiContext = undefined;\n      }\n      return uiContext;\n    } catch (error) {\n      this.lastUiContext = undefined;\n      throw error;\n    }\n  }\n\n  private async captureScreenshot(): Promise<string | undefined> {\n    try {\n      const uiContext = await this.getUiContext({ forceRefresh: true });\n      return uiContext?.screenshotBase64;\n    } catch (error) {\n      console.error('error while capturing screenshot', error);\n    }\n    return undefined;\n  }\n\n  private attachRecorderItem(\n    task: ExecutionTask,\n    contextOrScreenshot: UIContext | string | undefined,\n    phase: 'after-calling',\n  ): void {\n    const timing = phase;\n    const screenshot =\n      typeof contextOrScreenshot === 'string'\n        ? contextOrScreenshot\n        : contextOrScreenshot?.screenshotBase64;\n    if (!timing || !screenshot) {\n      return;\n    }\n\n    const recorderItem: ExecutionRecorderItem = {\n      type: 'screenshot',\n      ts: Date.now(),\n      screenshot,\n      timing,\n    };\n\n    if (!task.recorder) {\n      task.recorder = [recorderItem];\n      return;\n    }\n    task.recorder.push(recorderItem);\n  }\n\n  private markTaskAsPending(task: ExecutionTaskApply): ExecutionTask {\n    return {\n      status: 'pending',\n      ...task,\n    };\n  }\n\n  private normalizeStatusFromError(\n    options?: TaskRunnerOperationOptions,\n    errorMessage?: string,\n  ): void {\n    if (this.status !== 'error') {\n      return;\n    }\n    assert(\n      options?.allowWhenError,\n      errorMessage ||\n        `task runner is in error state, cannot proceed\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    // reset runner state so new tasks can run\n    this.status = this.tasks.length > 0 ? 'pending' : 'init';\n  }\n\n  private findPreviousNonSubTaskUIContext(\n    currentIndex: number,\n  ): UIContext | undefined {\n    for (let i = currentIndex - 1; i >= 0; i--) {\n      const candidate = this.tasks[i];\n      if (!candidate || candidate.subTask) {\n        continue;\n      }\n      if (candidate.uiContext) {\n        return candidate.uiContext;\n      }\n    }\n    return undefined;\n  }\n\n  async append(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<void> {\n    this.normalizeStatusFromError(\n      options,\n      `task runner is in error state, cannot append task\\nerror=${this.latestErrorTask()?.error}\\n${this.latestErrorTask()?.errorStack}`,\n    );\n    if (Array.isArray(task)) {\n      this.tasks.push(...task.map((item) => this.markTaskAsPending(item)));\n    } else {\n      this.tasks.push(this.markTaskAsPending(task));\n    }\n    if (this.status !== 'running') {\n      this.status = 'pending';\n    }\n    await this.emitOnTaskUpdate();\n  }\n\n  async appendAndFlush(\n    task: ExecutionTaskApply[] | ExecutionTaskApply,\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    await this.append(task, options);\n    return this.flush(options);\n  }\n\n  async flush(\n    options?: TaskRunnerOperationOptions,\n  ): Promise<{ output: any; thought?: string } | undefined> {\n    if (this.status === 'init' && this.tasks.length > 0) {\n      console.warn(\n        'illegal state for task runner, status is init but tasks are not empty',\n      );\n    }\n\n    this.normalizeStatusFromError(options, 'task runner is in error state');\n    assert(this.status !== 'running', 'task runner is already running');\n    assert(this.status !== 'completed', 'task runner is already completed');\n\n    const nextPendingIndex = this.tasks.findIndex(\n      (task) => task.status === 'pending',\n    );\n    if (nextPendingIndex < 0) {\n      // all tasks are completed\n      return;\n    }\n\n    this.status = 'running';\n    await this.emitOnTaskUpdate();\n    let taskIndex = nextPendingIndex;\n    let successfullyCompleted = true;\n\n    let previousFindOutput: ExecutionTaskPlanningLocateOutput | undefined;\n\n    while (taskIndex < this.tasks.length) {\n      const task = this.tasks[taskIndex];\n      assert(\n        task.status === 'pending',\n        `task status should be pending, but got: ${task.status}`,\n      );\n      task.timing = {\n        start: Date.now(),\n      };\n      try {\n        task.status = 'running';\n        await this.emitOnTaskUpdate();\n        try {\n          if (this.onTaskStart) {\n            await this.onTaskStart(task);\n          }\n        } catch (e) {\n          console.error('error in onTaskStart', e);\n        }\n        assert(\n          ['Insight', 'Action Space', 'Planning'].indexOf(task.type) >= 0,\n          `unsupported task type: ${task.type}`,\n        );\n\n        const { executor, param } = task;\n        assert(executor, `executor is required for task type: ${task.type}`);\n\n        let returnValue;\n        let uiContext: UIContext | undefined;\n        if (task.subTask) {\n          uiContext = this.findPreviousNonSubTaskUIContext(taskIndex);\n          assert(\n            uiContext,\n            'subTask requires uiContext from previous non-subTask task',\n          );\n        } else {\n          uiContext = await this.getUiContext();\n        }\n        task.uiContext = uiContext;\n        const executorContext: ExecutorContext = {\n          task,\n          element: previousFindOutput?.element,\n          uiContext,\n        };\n\n        if (task.type === 'Insight') {\n          assert(\n            task.subType === 'Query' ||\n              task.subType === 'Assert' ||\n              task.subType === 'WaitFor' ||\n              task.subType === 'Boolean' ||\n              task.subType === 'Number' ||\n              task.subType === 'String',\n            `unsupported service subType: ${task.subType}`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        } else if (task.type === 'Planning') {\n          returnValue = await task.executor(param, executorContext);\n          if (task.subType === 'Locate') {\n            previousFindOutput = (\n              returnValue as ExecutionTaskReturn<ExecutionTaskPlanningLocateOutput>\n            )?.output;\n          }\n        } else if (task.type === 'Action Space') {\n          returnValue = await task.executor(param, executorContext);\n        } else {\n          console.warn(\n            `unsupported task type: ${task.type}, will try to execute it directly`,\n          );\n          returnValue = await task.executor(param, executorContext);\n        }\n\n        const isLastTask = taskIndex === this.tasks.length - 1;\n\n        if (isLastTask) {\n          const screenshot = await this.captureScreenshot();\n          this.attachRecorderItem(task, screenshot, 'after-calling');\n        }\n\n        Object.assign(task, returnValue);\n        task.status = 'finished';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        taskIndex++;\n      } catch (e: any) {\n        successfullyCompleted = false;\n        task.error = e;\n        task.errorMessage =\n          e?.message || (typeof e === 'string' ? e : 'error-without-message');\n        task.errorStack = e.stack;\n\n        task.status = 'failed';\n        task.timing.end = Date.now();\n        task.timing.cost = task.timing.end - task.timing.start;\n        await this.emitOnTaskUpdate();\n        break;\n      }\n    }\n\n    // set all remaining tasks as cancelled\n    for (let i = taskIndex + 1; i < this.tasks.length; i++) {\n      this.tasks[i].status = 'cancelled';\n    }\n    if (taskIndex + 1 < this.tasks.length) {\n      await this.emitOnTaskUpdate();\n    }\n\n    let finalizeError: TaskExecutionError | undefined;\n    if (!successfullyCompleted) {\n      this.status = 'error';\n      const errorTask = this.latestErrorTask();\n      const messageBase =\n        errorTask?.errorMessage ||\n        (errorTask?.error ? String(errorTask.error) : 'Task execution failed');\n      const stack = errorTask?.errorStack;\n      const message = stack ? `${messageBase}\\n${stack}` : messageBase;\n      finalizeError = new TaskExecutionError(message, this, errorTask, {\n        cause: errorTask?.error,\n      });\n      await this.emitOnTaskUpdate(finalizeError);\n    } else {\n      this.status = 'completed';\n      await this.emitOnTaskUpdate();\n    }\n\n    if (finalizeError) {\n      throw finalizeError;\n    }\n\n    if (this.tasks.length) {\n      // return the last output\n      const outputIndex = Math.min(taskIndex, this.tasks.length - 1);\n      const { thought, output } = this.tasks[outputIndex];\n      return {\n        thought,\n        output,\n      };\n    }\n  }\n\n  isInErrorState(): boolean {\n    return this.status === 'error';\n  }\n\n  latestErrorTask(): ExecutionTask | null {\n    if (this.status !== 'error') {\n      return null;\n    }\n    const errorTaskIndex = this.tasks.findIndex(\n      (task) => task.status === 'failed',\n    );\n    if (errorTaskIndex >= 0) {\n      return this.tasks[errorTaskIndex];\n    }\n    return null;\n  }\n\n  dump(): ExecutionDump {\n    const dumpData: ExecutionDump = {\n      logTime: Date.now(),\n      name: this.name,\n      tasks: this.tasks,\n    };\n    return dumpData;\n  }\n\n  async appendErrorPlan(errorMsg: string): Promise<{\n    output: undefined;\n    runner: TaskRunner;\n  }> {\n    const errorTask: ExecutionTaskActionApply<PlanningActionParamError> = {\n      type: 'Action Space',\n      subType: 'Error',\n      param: {\n        thought: errorMsg,\n      },\n      thought: errorMsg,\n      executor: async () => {\n        throw new Error(errorMsg || 'error without thought');\n      },\n    };\n    await this.appendAndFlush(errorTask);\n\n    return {\n      output: undefined,\n      runner: this,\n    };\n  }\n}\n\nexport class TaskExecutionError extends Error {\n  runner: TaskRunner;\n\n  errorTask: ExecutionTask | null;\n\n  constructor(\n    message: string,\n    runner: TaskRunner,\n    errorTask: ExecutionTask | null,\n    options?: { cause?: unknown },\n  ) {\n    super(message, options);\n    this.runner = runner;\n    this.errorTask = errorTask;\n  }\n}\n"],"names":["debug","getDebug","UI_CONTEXT_CACHE_TTL_MS","TaskRunner","error","options","_this_lastUiContext","now","Date","shouldReuse","_this_lastUiContext1","uiContext","undefined","console","task","contextOrScreenshot","phase","timing","screenshot","recorderItem","errorMessage","_this_latestErrorTask","_this_latestErrorTask1","assert","currentIndex","i","candidate","Array","item","nextPendingIndex","taskIndex","successfullyCompleted","previousFindOutput","e","executor","param","returnValue","executorContext","isLastTask","Object","finalizeError","errorTask","messageBase","String","stack","message","TaskExecutionError","outputIndex","Math","thought","output","errorTaskIndex","dumpData","errorMsg","Error","name","uiContextBuilder","runner"],"mappings":";;;;;;;;;;;;AAgBA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,0BAA0B;AAczB,MAAMC;IAgCX,MAAc,iBAAiBC,KAA0B,EAAiB;QACxE,IAAI,CAAC,IAAI,CAAC,YAAY,EACpB;QAEF,MAAM,IAAI,CAAC,YAAY,CAAC,IAAI,EAAEA;IAChC;IAOA,MAAc,aAAaC,OAAoC,EAE7D;YAOmBC;QANnB,MAAMC,MAAMC,KAAK,GAAG;QACpB,MAAMC,cACJ,CAACJ,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,YAAY,AAAD,KACrB,IAAI,CAAC,aAAa,IAClBE,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,IAAIL;QAEzC,IAAIO,eAAAA,SAAeH,CAAAA,sBAAAA,IAAI,CAAC,aAAa,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,OAAO,AAAD,GAAG;gBAIvCI;YAHPV,MACE,CAAC,gCAAgC,EAAEO,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,MAAM,CAAC;YAEhF,OAAO,QAAAG,CAAAA,uBAAAA,IAAI,CAAC,aAAa,AAAD,IAAjBA,KAAAA,IAAAA,qBAAoB,OAAO;QACpC;QAEA,IAAI;YACF,MAAMC,YAAY,MAAM,IAAI,CAAC,gBAAgB;YAC7C,IAAIA,WACF,IAAI,CAAC,aAAa,GAAG;gBACnB,SAASA;gBACT,YAAYH,KAAK,GAAG;YACtB;iBAEA,IAAI,CAAC,aAAa,GAAGI;YAEvB,OAAOD;QACT,EAAE,OAAOP,OAAO;YACd,IAAI,CAAC,aAAa,GAAGQ;YACrB,MAAMR;QACR;IACF;IAEA,MAAc,oBAAiD;QAC7D,IAAI;YACF,MAAMO,YAAY,MAAM,IAAI,CAAC,YAAY,CAAC;gBAAE,cAAc;YAAK;YAC/D,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,gBAAgB;QACpC,EAAE,OAAOP,OAAO;YACdS,QAAQ,KAAK,CAAC,oCAAoCT;QACpD;IAEF;IAEQ,mBACNU,IAAmB,EACnBC,mBAAmD,EACnDC,KAAsB,EAChB;QACN,MAAMC,SAASD;QACf,MAAME,aACJ,AAA+B,YAA/B,OAAOH,sBACHA,sBACAA,QAAAA,sBAAAA,KAAAA,IAAAA,oBAAqB,gBAAgB;QAC3C,IAAI,CAACE,UAAU,CAACC,YACd;QAGF,MAAMC,eAAsC;YAC1C,MAAM;YACN,IAAIX,KAAK,GAAG;YACZU;YACAD;QACF;QAEA,IAAI,CAACH,KAAK,QAAQ,EAAE;YAClBA,KAAK,QAAQ,GAAG;gBAACK;aAAa;YAC9B;QACF;QACAL,KAAK,QAAQ,CAAC,IAAI,CAACK;IACrB;IAEQ,kBAAkBL,IAAwB,EAAiB;QACjE,OAAO;YACL,QAAQ;YACR,GAAGA,IAAI;QACT;IACF;IAEQ,yBACNT,OAAoC,EACpCe,YAAqB,EACf;YAOsDC,uBAAkCC;QAN9F,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb;QAEFC,OACElB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,cAAc,EACvBe,gBACE,CAAC,qDAAqD,EAAE,QAAAC,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAGlI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;IACpD;IAEQ,gCACNE,YAAoB,EACG;QACvB,IAAK,IAAIC,IAAID,eAAe,GAAGC,KAAK,GAAGA,IAAK;YAC1C,MAAMC,YAAY,IAAI,CAAC,KAAK,CAACD,EAAE;YAC/B,IAAI,AAACC,cAAaA,UAAU,OAAO,EAGnC;gBAAA,IAAIA,UAAU,SAAS,EACrB,OAAOA,UAAU,SAAS;YAC5B;QACF;IAEF;IAEA,MAAM,OACJZ,IAA+C,EAC/CT,OAAoC,EACrB;YAG+CgB,uBAAkCC;QAFhG,IAAI,CAAC,wBAAwB,CAC3BjB,SACA,CAAC,yDAAyD,EAAE,QAAAgB,CAAAA,wBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,sBAAwB,KAAK,CAAC,EAAE,EAAE,QAAAC,CAAAA,yBAAAA,IAAI,CAAC,eAAe,EAAC,IAArBA,KAAAA,IAAAA,uBAAwB,UAAU,EAAE;QAEpI,IAAIK,MAAM,OAAO,CAACb,OAChB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAIA,KAAK,GAAG,CAAC,CAACc,OAAS,IAAI,CAAC,iBAAiB,CAACA;aAE7D,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAACd;QAEzC,IAAI,AAAgB,cAAhB,IAAI,CAAC,MAAM,EACb,IAAI,CAAC,MAAM,GAAG;QAEhB,MAAM,IAAI,CAAC,gBAAgB;IAC7B;IAEA,MAAM,eACJA,IAA+C,EAC/CT,OAAoC,EACoB;QACxD,MAAM,IAAI,CAAC,MAAM,CAACS,MAAMT;QACxB,OAAO,IAAI,CAAC,KAAK,CAACA;IACpB;IAEA,MAAM,MACJA,OAAoC,EACoB;QACxD,IAAI,AAAgB,WAAhB,IAAI,CAAC,MAAM,IAAe,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAChDQ,QAAQ,IAAI,CACV;QAIJ,IAAI,CAAC,wBAAwB,CAACR,SAAS;QACvCkB,OAAO,AAAgB,cAAhB,IAAI,CAAC,MAAM,EAAgB;QAClCA,OAAO,AAAgB,gBAAhB,IAAI,CAAC,MAAM,EAAkB;QAEpC,MAAMM,mBAAmB,IAAI,CAAC,KAAK,CAAC,SAAS,CAC3C,CAACf,OAASA,AAAgB,cAAhBA,KAAK,MAAM;QAEvB,IAAIe,mBAAmB,GAErB;QAGF,IAAI,CAAC,MAAM,GAAG;QACd,MAAM,IAAI,CAAC,gBAAgB;QAC3B,IAAIC,YAAYD;QAChB,IAAIE,wBAAwB;QAE5B,IAAIC;QAEJ,MAAOF,YAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAE;YACpC,MAAMhB,OAAO,IAAI,CAAC,KAAK,CAACgB,UAAU;YAClCP,OACET,AAAgB,cAAhBA,KAAK,MAAM,EACX,CAAC,wCAAwC,EAAEA,KAAK,MAAM,EAAE;YAE1DA,KAAK,MAAM,GAAG;gBACZ,OAAON,KAAK,GAAG;YACjB;YACA,IAAI;gBACFM,KAAK,MAAM,GAAG;gBACd,MAAM,IAAI,CAAC,gBAAgB;gBAC3B,IAAI;oBACF,IAAI,IAAI,CAAC,WAAW,EAClB,MAAM,IAAI,CAAC,WAAW,CAACA;gBAE3B,EAAE,OAAOmB,GAAG;oBACVpB,QAAQ,KAAK,CAAC,wBAAwBoB;gBACxC;gBACAV,OACE;oBAAC;oBAAW;oBAAgB;iBAAW,CAAC,OAAO,CAACT,KAAK,IAAI,KAAK,GAC9D,CAAC,uBAAuB,EAAEA,KAAK,IAAI,EAAE;gBAGvC,MAAM,EAAEoB,QAAQ,EAAEC,KAAK,EAAE,GAAGrB;gBAC5BS,OAAOW,UAAU,CAAC,oCAAoC,EAAEpB,KAAK,IAAI,EAAE;gBAEnE,IAAIsB;gBACJ,IAAIzB;gBACJ,IAAIG,KAAK,OAAO,EAAE;oBAChBH,YAAY,IAAI,CAAC,+BAA+B,CAACmB;oBACjDP,OACEZ,WACA;gBAEJ,OACEA,YAAY,MAAM,IAAI,CAAC,YAAY;gBAErCG,KAAK,SAAS,GAAGH;gBACjB,MAAM0B,kBAAmC;oBACvCvB;oBACA,SAASkB,QAAAA,qBAAAA,KAAAA,IAAAA,mBAAoB,OAAO;oBACpCrB;gBACF;gBAEA,IAAIG,AAAc,cAAdA,KAAK,IAAI,EAAgB;oBAC3BS,OACET,AAAiB,YAAjBA,KAAK,OAAO,IACVA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,cAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,IACZA,AAAiB,aAAjBA,KAAK,OAAO,EACd,CAAC,6BAA6B,EAAEA,KAAK,OAAO,EAAE;oBAEhDsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;gBAC3C,OAAO,IAAIvB,AAAc,eAAdA,KAAK,IAAI,EAAiB;oBACnCsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;oBACzC,IAAIvB,AAAiB,aAAjBA,KAAK,OAAO,EACdkB,qBACEI,QAAAA,cAAAA,KAAAA,IAAAA,YACC,MAAM;gBAEb,OAAO,IAAItB,AAAc,mBAAdA,KAAK,IAAI,EAClBsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;qBACpC;oBACLxB,QAAQ,IAAI,CACV,CAAC,uBAAuB,EAAEC,KAAK,IAAI,CAAC,iCAAiC,CAAC;oBAExEsB,cAAc,MAAMtB,KAAK,QAAQ,CAACqB,OAAOE;gBAC3C;gBAEA,MAAMC,aAAaR,cAAc,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;gBAErD,IAAIQ,YAAY;oBACd,MAAMpB,aAAa,MAAM,IAAI,CAAC,iBAAiB;oBAC/C,IAAI,CAAC,kBAAkB,CAACJ,MAAMI,YAAY;gBAC5C;gBAEAqB,OAAO,MAAM,CAACzB,MAAMsB;gBACpBtB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGN,KAAK,GAAG;gBAC1BM,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3BgB;YACF,EAAE,OAAOG,GAAQ;gBACfF,wBAAwB;gBACxBjB,KAAK,KAAK,GAAGmB;gBACbnB,KAAK,YAAY,GACfmB,AAAAA,CAAAA,QAAAA,IAAAA,KAAAA,IAAAA,EAAG,OAAO,AAAD,KAAM,CAAa,YAAb,OAAOA,IAAiBA,IAAI,uBAAsB;gBACnEnB,KAAK,UAAU,GAAGmB,EAAE,KAAK;gBAEzBnB,KAAK,MAAM,GAAG;gBACdA,KAAK,MAAM,CAAC,GAAG,GAAGN,KAAK,GAAG;gBAC1BM,KAAK,MAAM,CAAC,IAAI,GAAGA,KAAK,MAAM,CAAC,GAAG,GAAGA,KAAK,MAAM,CAAC,KAAK;gBACtD,MAAM,IAAI,CAAC,gBAAgB;gBAC3B;YACF;QACF;QAGA,IAAK,IAAIW,IAAIK,YAAY,GAAGL,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAEA,IACjD,IAAI,CAAC,KAAK,CAACA,EAAE,CAAC,MAAM,GAAG;QAEzB,IAAIK,YAAY,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EACnC,MAAM,IAAI,CAAC,gBAAgB;QAG7B,IAAIU;QACJ,IAAKT,uBAYE;YACL,IAAI,CAAC,MAAM,GAAG;YACd,MAAM,IAAI,CAAC,gBAAgB;QAC7B,OAf4B;YAC1B,IAAI,CAAC,MAAM,GAAG;YACd,MAAMU,YAAY,IAAI,CAAC,eAAe;YACtC,MAAMC,cACJD,AAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,YAAY,AAAD,KACrBA,CAAAA,CAAAA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK,AAAD,IAAIE,OAAOF,UAAU,KAAK,IAAI,uBAAsB;YACtE,MAAMG,QAAQH,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,UAAU;YACnC,MAAMI,UAAUD,QAAQ,GAAGF,YAAY,EAAE,EAAEE,OAAO,GAAGF;YACrDF,gBAAgB,IAAIM,mBAAmBD,SAAS,IAAI,EAAEJ,WAAW;gBAC/D,OAAOA,QAAAA,YAAAA,KAAAA,IAAAA,UAAW,KAAK;YACzB;YACA,MAAM,IAAI,CAAC,gBAAgB,CAACD;QAC9B;QAKA,IAAIA,eACF,MAAMA;QAGR,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE;YAErB,MAAMO,cAAcC,KAAK,GAAG,CAAClB,WAAW,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG;YAC5D,MAAM,EAAEmB,OAAO,EAAEC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAACH,YAAY;YACnD,OAAO;gBACLE;gBACAC;YACF;QACF;IACF;IAEA,iBAA0B;QACxB,OAAO,AAAgB,YAAhB,IAAI,CAAC,MAAM;IACpB;IAEA,kBAAwC;QACtC,IAAI,AAAgB,YAAhB,IAAI,CAAC,MAAM,EACb,OAAO;QAET,MAAMC,iBAAiB,IAAI,CAAC,KAAK,CAAC,SAAS,CACzC,CAACrC,OAASA,AAAgB,aAAhBA,KAAK,MAAM;QAEvB,IAAIqC,kBAAkB,GACpB,OAAO,IAAI,CAAC,KAAK,CAACA,eAAe;QAEnC,OAAO;IACT;IAEA,OAAsB;QACpB,MAAMC,WAA0B;YAC9B,SAAS5C,KAAK,GAAG;YACjB,MAAM,IAAI,CAAC,IAAI;YACf,OAAO,IAAI,CAAC,KAAK;QACnB;QACA,OAAO4C;IACT;IAEA,MAAM,gBAAgBC,QAAgB,EAGnC;QACD,MAAMZ,YAAgE;YACpE,MAAM;YACN,SAAS;YACT,OAAO;gBACL,SAASY;YACX;YACA,SAASA;YACT,UAAU;gBACR,MAAM,IAAIC,MAAMD,YAAY;YAC9B;QACF;QACA,MAAM,IAAI,CAAC,cAAc,CAACZ;QAE1B,OAAO;YACL,QAAQ7B;YACR,QAAQ,IAAI;QACd;IACF;IA5XA,YACE2C,IAAY,EACZC,gBAA0C,EAC1CnD,OAA+B,CAC/B;QAnBF;QAEA;QAGA;QAEA;QAEA,uBAAiB,oBAAjB;QAEA,uBAAiB,gBAAjB;QA2BA,uBAAQ,iBAAR;QAlBE,IAAI,CAAC,MAAM,GACTA,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAKA,QAAQ,KAAK,CAAC,MAAM,GAAG,IAAI,YAAY;QAC3D,IAAI,CAAC,IAAI,GAAGkD;QACZ,IAAI,CAAC,KAAK,GAAIlD,AAAAA,CAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,KAAK,AAAD,KAAK,EAAC,EAAG,GAAG,CAAC,CAACuB,OACvC,IAAI,CAAC,iBAAiB,CAACA;QAEzB,IAAI,CAAC,WAAW,GAAGvB,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,WAAW;QACvC,IAAI,CAAC,gBAAgB,GAAGmD;QACxB,IAAI,CAAC,YAAY,GAAGnD,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,YAAY;IAC3C;AA+WF;AAEO,MAAMyC,2BAA2BQ;IAKtC,YACET,OAAe,EACfY,MAAkB,EAClBhB,SAA+B,EAC/BpC,OAA6B,CAC7B;QACA,KAAK,CAACwC,SAASxC,UAVjB,0CAEA;QASE,IAAI,CAAC,MAAM,GAAGoD;QACd,IAAI,CAAC,SAAS,GAAGhB;IACnB;AACF"}

package/dist/es/types.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshotBase64: string;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\nexport type ThinkingStrategy = 'off' | 'cot';\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActionContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","ServiceError","Error","message","dump"],"mappings":";AAAqD;;;;;;;;;;AAuC9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AA0EL,MAAeC;AAMtB;AAyEO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF"}
+{"version":3,"file":"types.mjs","sources":["webpack://@midscene/core/./src/types.ts"],"sourcesContent":["/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { CreateOpenAIClientFn, TModelConfig } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  LocateResultElement,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  cached_input: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\nexport type { LocateResultElement };\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  errors?: string[];\n}\n\nexport type AIElementResponse = AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext {\n  abstract screenshotBase64: string;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type ServiceAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type ServiceExtractParam = string | Record<string, string>;\n\nexport type ElementCacheFeature = Record<string, unknown>;\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport type ThinkingLevel = 'off' | 'medium' | 'high';\nexport type ThinkingStrategy = 'off' | 'cot';\n\nexport interface ServiceTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface ServiceDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: ServiceExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: LocateResultElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: ServiceTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialServiceDumpFromSDK = Omit<\n  ServiceDump,\n  'logTime' | 'logId' | 'model_name'\n>;\n\nexport interface ServiceResultBase {\n  dump: ServiceDump;\n}\n\nexport type LocateResultWithDump = LocateResult & ServiceResultBase;\n\nexport interface ServiceExtractResult<T> extends ServiceResultBase {\n  data: T;\n  thought?: string;\n  usage?: AIUsageInfo;\n}\n\nexport class ServiceError extends Error {\n  dump: ServiceDump;\n\n  constructor(message: string, dump: ServiceDump) {\n    super(message);\n    this.name = 'ServiceError';\n    this.dump = dump;\n  }\n}\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type ServiceAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n}\n\nexport interface RawResponsePlanningAIResponse {\n  action: PlanningAction;\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n}\n\nexport interface PlanningAIResponse\n  extends Omit<RawResponsePlanningAIResponse, 'action'> {\n  actions?: PlanningAction[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface LongPressParam {\n  duration?: number;\n}\n\nexport interface PullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType = 'Planning' | 'Insight' | 'Action Space' | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n  uiContext?: UIContext;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  subTask?: boolean;\n  param?: TaskParam;\n  thought?: string;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n    searchAreaUsage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n  aiActionContext?: string;\n}\n\n/*\ntask - service-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskInsightDump = ServiceDump;\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - service-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: ServiceExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  ServiceAssertionResponse,\n  ExecutionTaskInsightDump\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action Space',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    aiActionContext?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\ntask - planning-locate\n*/\nexport type ExecutionTaskPlanningLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskPlanningLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport type ExecutionTaskPlanningDump = ServiceDump;\n\nexport type ExecutionTaskPlanningLocateApply = ExecutionTaskApply<\n  'Planning',\n  ExecutionTaskPlanningLocateParam,\n  ExecutionTaskPlanningLocateOutput,\n  ExecutionTaskPlanningDump\n>;\n\nexport type ExecutionTaskPlanningLocate =\n  ExecutionTask<ExecutionTaskPlanningLocateApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  sdkVersion: string;\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<TParam = any, TReturn = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<TParam>;\n  call: (param: TParam, context: ExecutorContext) => Promise<TReturn> | TReturn;\n  delayAfterRunner?: number;\n}\n\n/**\n * Type utilities for extracting types from DeviceAction definitions\n */\n\n/**\n * Extract parameter type from a DeviceAction\n */\nexport type ActionParam<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<infer P, any> ? P : never;\n\n/**\n * Extract return type from a DeviceAction\n */\nexport type ActionReturn<Action extends DeviceAction<any, any>> =\n  Action extends DeviceAction<any, infer R> ? R : never;\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext;\n\n/**\n * Agent\n */\n\nexport type CacheConfig = {\n  strategy?: 'read-only' | 'read-write' | 'write-only';\n  id: string;\n};\n\nexport type Cache =\n  | false // No read, no write\n  | true // Will throw error at runtime - deprecated\n  | CacheConfig; // Object configuration (requires explicit id)\n\nexport interface AgentOpt {\n  testId?: string;\n  // @deprecated\n  cacheId?: string; // Keep backward compatibility, but marked as deprecated\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfig;\n  cache?: Cache;\n  /**\n   * Maximum number of replanning cycles for aiAct.\n   * Defaults to 20 (40 for `vlm-ui-tars`) when not provided.\n   * If omitted, the agent will also read `MIDSCENE_REPLANNING_CYCLE_LIMIT` for backward compatibility.\n   */\n  replanningCycleLimit?: number;\n\n  /**\n   * Custom OpenAI client factory function\n   *\n   * If provided, this function will be called to create OpenAI client instances\n   * for each AI call, allowing you to:\n   * - Wrap clients with observability tools (langsmith, langfuse)\n   * - Use custom OpenAI-compatible clients\n   * - Apply different configurations based on intent\n   *\n   * @param config - Resolved model configuration\n   * @returns OpenAI client instance (original or wrapped)\n   *\n   * @example\n   * ```typescript\n   * createOpenAIClient: async (openai, opts) => {\n   *   // Wrap with langsmith for planning tasks\n   *   if (opts.baseURL?.includes('planning')) {\n   *     return wrapOpenAI(openai, { metadata: { task: 'planning' } });\n   *   }\n   *\n   *   return openai;\n   * }\n   * ```\n   */\n  createOpenAIClient?: CreateOpenAIClientFn;\n}\n\nexport type TestStatus =\n  | 'passed'\n  | 'failed'\n  | 'timedOut'\n  | 'skipped'\n  | 'interrupted';\n\nexport interface ReportFileWithAttributes {\n  reportFilePath: string;\n  reportAttributes: {\n    testDuration: number;\n    testStatus: TestStatus;\n    testTitle: string;\n    testId: string;\n    testDescription: string;\n  };\n}\n"],"names":["AIResponseFormat","UIContext","ServiceError","Error","message","dump"],"mappings":";AAAqD;;;;;;;;;;AAwC9C,IAAKA,yBAAgBA,WAAAA,GAAAA,SAAhBA,gBAAgB;;;WAAhBA;;AA0EL,MAAeC;AAMtB;AAyEO,MAAMC,qBAAqBC;IAGhC,YAAYC,OAAe,EAAEC,IAAiB,CAAE;QAC9C,KAAK,CAACD,UAHR;QAIE,IAAI,CAAC,IAAI,GAAG;QACZ,IAAI,CAAC,IAAI,GAAGC;IACd;AACF"}