@midscene/core 0.28.2 → 0.28.3-beta-20250910120845.0

package/dist/lib/ai-model/service-caller/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport { Anthropic } from '@anthropic-ai/sdk';\nimport {\n  DefaultAzureCredential,\n  getBearerTokenProvider,\n} from '@azure/identity';\nimport {\n  type IModelPreferences,\n  MIDSCENE_API_TYPE,\n  MIDSCENE_LANGSMITH_DEBUG,\n  OPENAI_MAX_TOKENS,\n  globalConfigManager,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { parseBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI, { AzureOpenAI } from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { locatorSchema } from '../prompt/llm-locator';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelPreferences,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelPreferences: IModelPreferences;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  style: 'openai' | 'anthropic';\n  modelName: string;\n  modelDescription: string;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    openaiUseAzureDeprecated,\n    useAzureOpenai,\n    azureOpenaiScope,\n    azureOpenaiKey,\n    azureOpenaiEndpoint,\n    azureOpenaiApiVersion,\n    azureOpenaiDeployment,\n    azureExtraConfig,\n    useAnthropicSdk,\n    anthropicApiKey,\n    modelDescription,\n  } = globalConfigManager.getModelConfigByIntent(modelPreferences.intent);\n\n  let openai: OpenAI | AzureOpenAI | undefined;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  if (openaiUseAzureDeprecated) {\n    // this is deprecated\n    openai = new AzureOpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      dangerouslyAllowBrowser: true,\n    }) as OpenAI;\n  } else if (useAzureOpenai) {\n    // https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=bash%2Cjavascript-key%2Ctypescript-keyless%2Cpython&pivots=programming-language-javascript#rest-api\n    // keyless authentication\n    let tokenProvider: any = undefined;\n    if (azureOpenaiScope) {\n      assert(\n        !ifInBrowser,\n        'Azure OpenAI is not supported in browser with Midscene.',\n      );\n      const credential = new DefaultAzureCredential();\n\n      tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);\n\n      openai = new AzureOpenAI({\n        azureADTokenProvider: tokenProvider,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    } else {\n      // endpoint, apiKey, apiVersion, deployment\n      openai = new AzureOpenAI({\n        apiKey: azureOpenaiKey,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        dangerouslyAllowBrowser: true,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    }\n  } else if (!useAnthropicSdk) {\n    openai = new OpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      defaultHeaders: {\n        ...(openaiExtraConfig?.defaultHeaders || {}),\n        [MIDSCENE_API_TYPE]: AIActionTypeValue.toString(),\n      },\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    const { wrapOpenAI } = await import('langsmith/wrappers');\n    openai = wrapOpenAI(openai);\n  }\n\n  if (typeof openai !== 'undefined') {\n    return {\n      completion: openai.chat.completions,\n      style: 'openai',\n      modelName,\n      modelDescription,\n    };\n  }\n\n  // Anthropic\n  if (useAnthropicSdk) {\n    openai = new Anthropic({\n      apiKey: anthropicApiKey,\n      httpAgent: proxyAgent,\n      dangerouslyAllowBrowser: true,\n    }) as any;\n  }\n\n  if (typeof openai !== 'undefined' && (openai as any).messages) {\n    return {\n      completion: (openai as any).messages,\n      style: 'anthropic',\n      modelName,\n      modelDescription,\n    };\n  }\n\n  throw new Error('Openai SDK or Anthropic SDK is not initialized');\n}\n\nexport async function call(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const { completion, style, modelName, modelDescription } =\n    await createChatClient({\n      AIActionTypeValue,\n      modelPreferences,\n    });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlLocateMode(modelPreferences) === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlLocateMode(modelPreferences) === 'qwen-vl' // qwen specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    if (style === 'openai') {\n      debugCall(\n        `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n      );\n\n      if (isStreaming) {\n        const stream = (await completion.create(\n          {\n            model: modelName,\n            messages,\n            response_format: responseFormat,\n            ...commonConfig,\n          },\n          {\n            stream: true,\n          },\n        )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n          _request_id?: string | null;\n        };\n\n        for await (const chunk of stream) {\n          const content = chunk.choices?.[0]?.delta?.content || '';\n          const reasoning_content =\n            (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n          // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n          if (chunk.usage) {\n            usage = chunk.usage;\n          }\n\n          if (content || reasoning_content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              reasoning_content,\n              accumulated,\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.choices?.[0]?.finish_reason) {\n            timeCost = Date.now() - startTime;\n\n            // If usage is not available from the stream, provide a basic usage info\n            if (!usage) {\n              // Estimate token counts based on content length (rough approximation)\n              const estimatedTokens = Math.max(\n                1,\n                Math.floor(accumulated.length / 4),\n              );\n              usage = {\n                prompt_tokens: estimatedTokens,\n                completion_tokens: estimatedTokens,\n                total_tokens: estimatedTokens * 2,\n              };\n            }\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: {\n                prompt_tokens: usage.prompt_tokens ?? 0,\n                completion_tokens: usage.completion_tokens ?? 0,\n                total_tokens: usage.total_tokens ?? 0,\n                time_cost: timeCost ?? 0,\n                model_name: modelName,\n                model_description: modelDescription,\n                intent: modelPreferences.intent,\n              },\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n        debugProfileStats(\n          `streaming model, ${modelName}, mode, ${vlLocateMode(modelPreferences) || 'default'}, cost-ms, ${timeCost}`,\n        );\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n\n        debugProfileStats(\n          `model, ${modelName}, mode, ${vlLocateMode(modelPreferences) || 'default'}, ui-tars-version, ${uiTarsModelVersion(modelPreferences)}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n        );\n\n        debugProfileDetail(\n          `model usage detail: ${JSON.stringify(result.usage)}`,\n        );\n\n        assert(\n          result.choices,\n          `invalid response from LLM service: ${JSON.stringify(result)}`,\n        );\n        content = result.choices[0].message.content!;\n        usage = result.usage;\n      }\n\n      debugCall(`response: ${content}`);\n      assert(content, 'empty content');\n    } else if (style === 'anthropic') {\n      const convertImageContent = (content: any) => {\n        if (content.type === 'image_url') {\n          const imgBase64 = content.image_url.url;\n          assert(imgBase64, 'image_url is required');\n          const { mimeType, body } = parseBase64(content.image_url.url);\n          return {\n            source: {\n              type: 'base64',\n              media_type: mimeType,\n              data: body,\n            },\n            type: 'image',\n          };\n        }\n        return content;\n      };\n\n      if (isStreaming) {\n        const stream = (await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any)) as any;\n\n        for await (const chunk of stream) {\n          const content = chunk.delta?.text || '';\n          if (content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              accumulated,\n              reasoning_content: '',\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.type === 'message_stop') {\n            timeCost = Date.now() - startTime;\n            const anthropicUsage = chunk.usage;\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: anthropicUsage\n                ? {\n                    prompt_tokens: anthropicUsage.input_tokens ?? 0,\n                    completion_tokens: anthropicUsage.output_tokens ?? 0,\n                    total_tokens:\n                      (anthropicUsage.input_tokens ?? 0) +\n                      (anthropicUsage.output_tokens ?? 0),\n                    time_cost: timeCost ?? 0,\n                    model_name: modelName,\n                    model_description: modelDescription,\n                    intent: modelPreferences.intent,\n                  }\n                : undefined,\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n        content = (result as any).content[0].text as string;\n        usage = result.usage;\n      }\n\n      assert(content, 'empty content');\n    }\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelPreferences.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.INSPECT_ELEMENT:\n        responseFormat = locatorSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  if (modelName === 'gpt-4o-2024-05-13') {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callToGetJSONObject<T>(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await call(messages, AIActionTypeValue, modelPreferences);\n  assert(response, 'empty response');\n  const jsonContent = safeParseJson(response.content, modelPreferences);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAiFnWithStringResponse<T>(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelPreferences: IModelPreferences,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await call(\n    msgs,\n    AIActionTypeValue,\n    modelPreferences,\n  );\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(\n  input: string,\n  modelPreferences: IModelPreferences,\n) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (\n    vlLocateMode(modelPreferences) === 'doubao-vision' ||\n    vlLocateMode(modelPreferences) === 'vlm-ui-tars'\n  ) {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","createChatClient","AIActionTypeValue","modelPreferences","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","openaiUseAzureDeprecated","useAzureOpenai","azureOpenaiScope","azureOpenaiKey","azureOpenaiEndpoint","azureOpenaiApiVersion","azureOpenaiDeployment","azureExtraConfig","useAnthropicSdk","anthropicApiKey","modelDescription","globalConfigManager","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","AzureOpenAI","tokenProvider","assert","ifInBrowser","credential","DefaultAzureCredential","getBearerTokenProvider","OpenAI","MIDSCENE_API_TYPE","MIDSCENE_LANGSMITH_DEBUG","Error","console","wrapOpenAI","Anthropic","call","messages","options","completion","style","responseFormat","getResponseFormat","maxTokens","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","vlLocateMode","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","uiTarsModelVersion","JSON","convertImageContent","imgBase64","mimeType","body","parseBase64","m","Array","_chunk_delta","anthropicUsage","e","newError","AIActionType","assertSchema","locatorSchema","planSchema","AIResponseFormat","callToGetJSONObject","response","jsonContent","safeParseJson","callAiFnWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;ICyBA,eAAeI,iBAAiB,EAC9BC,iBAAiB,EACjBC,gBAAgB,EAIjB;QAMC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,wBAAwB,EACxBC,cAAc,EACdC,gBAAgB,EAChBC,cAAc,EACdC,mBAAmB,EACnBC,qBAAqB,EACrBC,qBAAqB,EACrBC,gBAAgB,EAChBC,eAAe,EACfC,eAAe,EACfC,gBAAgB,EACjB,GAAGC,oBAAAA,mBAAAA,CAAAA,sBAA0C,CAAClB,iBAAiB,MAAM;QAEtE,IAAImB;QAEJ,IAAIC;QACJ,MAAMC,aAAaC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QAC5B,IAAIpB,WAAW;YACbmB,WAAW,oBAAoBnB;YAC/BkB,aAAa,IAAIG,2CAAAA,eAAeA,CAACrB;QACnC,OAAO,IAAID,YAAY;YACrBoB,WAAW,qBAAqBpB;YAChCmB,aAAa,IAAII,2CAAAA,eAAeA,CAACvB;QACnC;QAEA,IAAIM,0BAEFY,SAAS,IAAIM,gCAAAA,WAAWA,CAAC;YACvB,SAASrB;YACT,QAAQC;YACR,WAAWe;YACX,GAAGd,iBAAiB;YACpB,yBAAyB;QAC3B;aACK,IAAIE,gBAAgB;YAGzB,IAAIkB;YACJ,IAAIjB,kBAAkB;gBACpBkB,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACC,sBAAAA,WAAWA,EACZ;gBAEF,MAAMC,aAAa,IAAIC,yBAAAA,sBAAsBA;gBAE7CJ,gBAAgBK,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAAuBF,YAAYpB;gBAEnDU,SAAS,IAAIM,gCAAAA,WAAWA,CAAC;oBACvB,sBAAsBC;oBACtB,UAAUf;oBACV,YAAYC;oBACZ,YAAYC;oBACZ,GAAGP,iBAAiB;oBACpB,GAAGQ,gBAAgB;gBACrB;YACF,OAEEK,SAAS,IAAIM,gCAAAA,WAAWA,CAAC;gBACvB,QAAQf;gBACR,UAAUC;gBACV,YAAYC;gBACZ,YAAYC;gBACZ,yBAAyB;gBACzB,GAAGP,iBAAiB;gBACpB,GAAGQ,gBAAgB;YACrB;QAEJ,OAAO,IAAI,CAACC,iBACVI,SAAS,IAAIa,CAAAA,yBAAAA,EAAO;YAClB,SAAS5B;YACT,QAAQC;YACR,WAAWe;YACX,GAAGd,iBAAiB;YACpB,gBAAgB;gBACd,GAAIA,AAAAA,CAAAA,QAAAA,oBAAAA,KAAAA,IAAAA,kBAAmB,cAAc,AAAD,KAAK,CAAC,CAAC;gBAC3C,CAAC2B,oBAAAA,iBAAiBA,CAAC,EAAElC,kBAAkB,QAAQ;YACjD;YACA,yBAAyB;QAC3B;QAGF,IACEoB,UACAD,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACgB,oBAAAA,wBAAwBA,GAClE;YACA,IAAIN,sBAAAA,WAAWA,EACb,MAAM,IAAIO,MAAM;YAElBC,QAAQ,GAAG,CAAC;YACZ,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM;YAC7BlB,SAASkB,WAAWlB;QACtB;QAEA,IAAI,AAAkB,WAAXA,QACT,OAAO;YACL,YAAYA,OAAO,IAAI,CAAC,WAAW;YACnC,OAAO;YACPhB;YACAc;QACF;QAIF,IAAIF,iBACFI,SAAS,IAAImB,oBAAAA,SAASA,CAAC;YACrB,QAAQtB;YACR,WAAWI;YACX,yBAAyB;QAC3B;QAGF,IAAI,AAAkB,WAAXD,UAA2BA,OAAe,QAAQ,EAC3D,OAAO;YACL,YAAaA,OAAe,QAAQ;YACpC,OAAO;YACPhB;YACAc;QACF;QAGF,MAAM,IAAIkB,MAAM;IAClB;IAEO,eAAeI,KACpBC,QAAsC,EACtCzC,iBAA+B,EAC/BC,gBAAmC,EACnCyC,OAGC;QAED,MAAM,EAAEC,UAAU,EAAEC,KAAK,EAAExC,SAAS,EAAEc,gBAAgB,EAAE,GACtD,MAAMnB,iBAAiB;YACrBC;YACAC;QACF;QAEF,MAAM4C,iBAAiBC,kBAAkB1C,WAAWJ;QAEpD,MAAM+C,YAAY5B,oBAAAA,mBAAAA,CAAAA,iBAAqC,CAAC6B,oBAAAA,iBAAiBA;QACzE,MAAMC,YAAY1B,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QAC3B,MAAM2B,oBAAoB3B,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QACnC,MAAM4B,qBAAqB5B,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QAEpC,MAAM6B,YAAYC,KAAK,GAAG;QAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;QACtD,IAAIa;QACJ,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,MAAMC,eAAe;YACnB,aAAaC,AAAmC,kBAAnCA,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAa3D,oBAAsC,MAAM;YACtE,QAAQ,CAAC,CAACqD;YACV,YACE,AAAqB,YAArB,OAAOP,YACHA,YACAc,OAAO,QAAQ,CAACd,aAAa,QAAQ;YAC3C,GAAIa,AAAmC,cAAnCA,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAa3D,oBACb;gBACE,2BAA2B;YAC7B,IACA,CAAC,CAAC;QACR;QAEA,IAAI;YACF,IAAI2C,AAAU,aAAVA,OAAoB;gBACtBK,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAElD,WAAW;gBAGrE,IAAIkD,aAAa;oBACf,MAAMQ,SAAU,MAAMnB,WAAW,MAAM,CACrC;wBACE,OAAOvC;wBACPqC;wBACA,iBAAiBI;wBACjB,GAAGc,YAAY;oBACjB,GACA;wBACE,QAAQ;oBACV;oBAKF,WAAW,MAAMI,SAASD,OAAQ;4BAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;wBAtBJ,MAAMX,UAAUS,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;wBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;wBAG3D,IAAIF,MAAM,KAAK,EACbN,QAAQM,MAAM,KAAK;wBAGrB,IAAIR,WAAWY,mBAAmB;4BAChCX,eAAeD;4BACf,MAAMa,YAAiC;gCACrCb;gCACAY;gCACAX;gCACA,YAAY;gCACZ,OAAOa;4BACT;4BACA3B,QAAQ,OAAO,CAAE0B;wBACnB;wBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;4BACrCR,WAAWL,KAAK,GAAG,KAAKD;4BAGxB,IAAI,CAACK,OAAO;gCAEV,MAAMa,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACf,YAAY,MAAM,GAAG;gCAElCC,QAAQ;oCACN,eAAea;oCACf,mBAAmBA;oCACnB,cAAcA,AAAkB,IAAlBA;gCAChB;4BACF;4BAGA,MAAME,aAAkC;gCACtC,SAAS;gCACThB;gCACA,mBAAmB;gCACnB,YAAY;gCACZ,OAAO;oCACL,eAAeC,MAAM,aAAa,IAAI;oCACtC,mBAAmBA,MAAM,iBAAiB,IAAI;oCAC9C,cAAcA,MAAM,YAAY,IAAI;oCACpC,WAAWC,YAAY;oCACvB,YAAYtD;oCACZ,mBAAmBc;oCACnB,QAAQjB,iBAAiB,MAAM;gCACjC;4BACF;4BACAyC,QAAQ,OAAO,CAAE8B;4BACjB;wBACF;oBACF;oBACAjB,UAAUC;oBACVN,kBACE,CAAC,iBAAiB,EAAE9C,UAAU,QAAQ,EAAEwD,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAa3D,qBAAqB,UAAU,WAAW,EAAEyD,UAAU;gBAE/G,OAAO;wBAUoJe,eAAyDC,gBAAwDC;oBAT1Q,MAAMC,SAAS,MAAMjC,WAAW,MAAM,CAAC;wBACrC,OAAOvC;wBACPqC;wBACA,iBAAiBI;wBACjB,GAAGc,YAAY;oBACjB;oBACAD,WAAWL,KAAK,GAAG,KAAKD;oBAExBF,kBACE,CAAC,OAAO,EAAE9C,UAAU,QAAQ,EAAEwD,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAa3D,qBAAqB,UAAU,mBAAmB,EAAE4E,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EAAmB5E,kBAAkB,iBAAiB,EAAEwE,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEjB,SAAS,aAAa,EAAEkB,OAAO,WAAW,IAAI,IAAI;oBAG1WzB,mBACE,CAAC,oBAAoB,EAAE2B,KAAK,SAAS,CAACF,OAAO,KAAK,GAAG;oBAGvDhD,IAAAA,sBAAAA,MAAAA,AAAAA,EACEgD,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEE,KAAK,SAAS,CAACF,SAAS;oBAEhErB,UAAUqB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;oBAC3CnB,QAAQmB,OAAO,KAAK;gBACtB;gBAEA3B,UAAU,CAAC,UAAU,EAAEM,SAAS;gBAChC3B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2B,SAAS;YAClB,OAAO,IAAIX,AAAU,gBAAVA,OAAuB;gBAChC,MAAMmC,sBAAsB,CAACxB;oBAC3B,IAAIA,AAAiB,gBAAjBA,QAAQ,IAAI,EAAkB;wBAChC,MAAMyB,YAAYzB,QAAQ,SAAS,CAAC,GAAG;wBACvC3B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOoD,WAAW;wBAClB,MAAM,EAAEC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAY5B,QAAQ,SAAS,CAAC,GAAG;wBAC5D,OAAO;4BACL,QAAQ;gCACN,MAAM;gCACN,YAAY0B;gCACZ,MAAMC;4BACR;4BACA,MAAM;wBACR;oBACF;oBACA,OAAO3B;gBACT;gBAEA,IAAID,aAAa;oBACf,MAAMQ,SAAU,MAAMnB,WAAW,MAAM,CAAC;wBACtC,OAAOvC;wBACP,QAAQ;wBACR,UAAUqC,SAAS,GAAG,CAAC,CAAC2C,IAAO;gCAC7B,MAAM;gCACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;4BACf;wBACA,iBAAiBvC;wBACjB,GAAGc,YAAY;oBACjB;oBAEA,WAAW,MAAMI,SAASD,OAAQ;4BAChBwB;wBAAhB,MAAM/B,UAAU+B,AAAAA,SAAAA,CAAAA,eAAAA,MAAM,KAAK,AAAD,IAAVA,KAAAA,IAAAA,aAAa,IAAI,AAAD,KAAK;wBACrC,IAAI/B,SAAS;4BACXC,eAAeD;4BACf,MAAMa,YAAiC;gCACrCb;gCACAC;gCACA,mBAAmB;gCACnB,YAAY;gCACZ,OAAOa;4BACT;4BACA3B,QAAQ,OAAO,CAAE0B;wBACnB;wBAGA,IAAIL,AAAe,mBAAfA,MAAM,IAAI,EAAqB;4BACjCL,WAAWL,KAAK,GAAG,KAAKD;4BACxB,MAAMmC,iBAAiBxB,MAAM,KAAK;4BAGlC,MAAMS,aAAkC;gCACtC,SAAS;gCACThB;gCACA,mBAAmB;gCACnB,YAAY;gCACZ,OAAO+B,iBACH;oCACE,eAAeA,eAAe,YAAY,IAAI;oCAC9C,mBAAmBA,eAAe,aAAa,IAAI;oCACnD,cACGA,AAAAA,CAAAA,eAAe,YAAY,IAAI,KAC/BA,CAAAA,eAAe,aAAa,IAAI;oCACnC,WAAW7B,YAAY;oCACvB,YAAYtD;oCACZ,mBAAmBc;oCACnB,QAAQjB,iBAAiB,MAAM;gCACjC,IACAoE;4BACN;4BACA3B,QAAQ,OAAO,CAAE8B;4BACjB;wBACF;oBACF;oBACAjB,UAAUC;gBACZ,OAAO;oBACL,MAAMoB,SAAS,MAAMjC,WAAW,MAAM,CAAC;wBACrC,OAAOvC;wBACP,QAAQ;wBACR,UAAUqC,SAAS,GAAG,CAAC,CAAC2C,IAAO;gCAC7B,MAAM;gCACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;4BACf;wBACA,iBAAiBvC;wBACjB,GAAGc,YAAY;oBACjB;oBACAD,WAAWL,KAAK,GAAG,KAAKD;oBACxBG,UAAWqB,OAAe,OAAO,CAAC,EAAE,CAAC,IAAI;oBACzCnB,QAAQmB,OAAO,KAAK;gBACtB;gBAEAhD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2B,SAAS;YAClB;YAEA,IAAID,eAAe,CAACG,OAAO;gBAEzB,MAAMa,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEhB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;gBAEtCE,QAAQ;oBACN,eAAea;oBACf,mBAAmBA;oBACnB,cAAcA,AAAkB,IAAlBA;gBAChB;YACF;YAEA,OAAO;gBACL,SAASf,WAAW;gBACpB,OAAOE,QACH;oBACE,eAAeA,MAAM,aAAa,IAAI;oBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;oBAC9C,cAAcA,MAAM,YAAY,IAAI;oBACpC,WAAWC,YAAY;oBACvB,YAAYtD;oBACZ,mBAAmBc;oBACnB,QAAQjB,iBAAiB,MAAM;gBACjC,IACAoE;gBACJ,YAAY,CAAC,CAACf;YAChB;QACF,EAAE,OAAOkC,GAAQ;YACfnD,QAAQ,KAAK,CAAC,kBAAkBmD;YAChC,MAAMC,WAAW,IAAIrD,MACnB,CAAC,eAAe,EAAEkB,cAAc,eAAe,GAAG,kBAAkB,EAAEkC,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;gBACE,OAAOA;YACT;YAEF,MAAMC;QACR;IACF;IAEO,MAAM3C,oBAAoB,CAC/B1C,WACAJ;QAIA,IAAI6C;QAKJ,IAAIzC,UAAU,QAAQ,CAAC,UACrB,OAAQJ;YACN,KAAK0F,mCAAAA,YAAAA,CAAAA,MAAmB;gBACtB7C,iBAAiB8C,6BAAAA,YAAYA;gBAC7B;YACF,KAAKD,mCAAAA,YAAAA,CAAAA,eAA4B;gBAC/B7C,iBAAiB+C,+BAAAA,aAAaA;gBAC9B;YACF,KAAKF,mCAAAA,YAAAA,CAAAA,IAAiB;gBACpB7C,iBAAiBgD,gCAAAA,UAAUA;gBAC3B;YACF,KAAKH,mCAAAA,YAAAA,CAAAA,YAAyB;YAC9B,KAAKA,mCAAAA,YAAAA,CAAAA,gBAA6B;gBAChC7C,iBAAiB;oBAAE,MAAMiD,kCAAAA,gBAAAA,CAAAA,IAAqB;gBAAC;gBAC/C;QACJ;QAIF,IAAI1F,AAAc,wBAAdA,WACFyC,iBAAiB;YAAE,MAAMiD,kCAAAA,gBAAAA,CAAAA,IAAqB;QAAC;QAGjD,OAAOjD;IACT;IAEO,eAAekD,oBACpBtD,QAAsC,EACtCzC,iBAA+B,EAC/BC,gBAAmC;QAEnC,MAAM+F,WAAW,MAAMxD,KAAKC,UAAUzC,mBAAmBC;QACzD2B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOoE,UAAU;QACjB,MAAMC,cAAcC,cAAcF,SAAS,OAAO,EAAE/F;QACpD,OAAO;YAAE,SAASgG;YAAa,OAAOD,SAAS,KAAK;QAAC;IACvD;IAEO,eAAeG,2BACpBC,IAAY,EACZpG,iBAA+B,EAC/BC,gBAAmC;QAEnC,MAAM,EAAEsD,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,KAC/B4D,MACApG,mBACAC;QAEF,OAAO;YAAEsD;YAASE;QAAM;IAC1B;IAEO,SAAS4C,yBAAyBL,QAAgB;QACvD,IAAI;YAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;YACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;YAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;YAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;YAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;YACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;QAE3B,EAAE,OAAM,CAAC;QAET,OAAOR;IACT;IAEO,SAASS,yBAAyBC,KAAa;QACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;QAG5C,OAAOA;IACT;IAEO,SAASR,cACdQ,KAAa,EACbzG,gBAAmC;QAEnC,MAAM0G,kBAAkBN,yBAAyBK;QAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;gBACtCC;YAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAAC/C;QACT;QACA,IAAI;YACF,OAAOiB,KAAK,KAAK,CAAC6B;QACpB,EAAE,OAAM,CAAC;QACT,IAAI;YACF,OAAO7B,KAAK,KAAK,CAAC+B,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWF;QAC/B,EAAE,OAAOnB,GAAG,CAAC;QAEb,IACE5B,AAAmC,oBAAnCA,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAa3D,qBACb2D,AAAmC,kBAAnCA,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAa3D,mBACb;YACA,MAAM6G,aAAaL,yBAAyBE;YAC5C,OAAO7B,KAAK,KAAK,CAAC+B,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWC;QAC/B;QACA,MAAM1E,MAAM,CAAC,+BAA+B,EAAEsE,OAAO;IACvD"}
+{"version":3,"file":"ai-model/service-caller/index.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { AIResponseFormat, type AIUsageInfo } from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport { Anthropic } from '@anthropic-ai/sdk';\nimport {\n  DefaultAzureCredential,\n  getBearerTokenProvider,\n} from '@azure/identity';\nimport {\n  type IModelConfig,\n  MIDSCENE_API_TYPE,\n  MIDSCENE_LANGSMITH_DEBUG,\n  OPENAI_MAX_TOKENS,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { parseBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport { ifInBrowser } from '@midscene/shared/utils';\nimport { HttpsProxyAgent } from 'https-proxy-agent';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI, { AzureOpenAI } from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport { SocksProxyAgent } from 'socks-proxy-agent';\nimport { AIActionType, type AIArgs } from '../common';\nimport { assertSchema } from '../prompt/assertion';\nimport { locatorSchema } from '../prompt/llm-locator';\nimport { planSchema } from '../prompt/llm-planning';\n\nasync function createChatClient({\n  AIActionTypeValue,\n  modelConfig,\n}: {\n  AIActionTypeValue: AIActionType;\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  style: 'openai' | 'anthropic';\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    openaiUseAzureDeprecated,\n    useAzureOpenai,\n    azureOpenaiScope,\n    azureOpenaiKey,\n    azureOpenaiEndpoint,\n    azureOpenaiApiVersion,\n    azureOpenaiDeployment,\n    azureExtraConfig,\n    useAnthropicSdk,\n    anthropicApiKey,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n  } = modelConfig;\n\n  let openai: OpenAI | AzureOpenAI | undefined;\n\n  let proxyAgent = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n  if (httpProxy) {\n    debugProxy('using http proxy', httpProxy);\n    proxyAgent = new HttpsProxyAgent(httpProxy);\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', socksProxy);\n    proxyAgent = new SocksProxyAgent(socksProxy);\n  }\n\n  if (openaiUseAzureDeprecated) {\n    // this is deprecated\n    openai = new AzureOpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      dangerouslyAllowBrowser: true,\n    }) as OpenAI;\n  } else if (useAzureOpenai) {\n    // https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=bash%2Cjavascript-key%2Ctypescript-keyless%2Cpython&pivots=programming-language-javascript#rest-api\n    // keyless authentication\n    let tokenProvider: any = undefined;\n    if (azureOpenaiScope) {\n      assert(\n        !ifInBrowser,\n        'Azure OpenAI is not supported in browser with Midscene.',\n      );\n      const credential = new DefaultAzureCredential();\n\n      tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);\n\n      openai = new AzureOpenAI({\n        azureADTokenProvider: tokenProvider,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    } else {\n      // endpoint, apiKey, apiVersion, deployment\n      openai = new AzureOpenAI({\n        apiKey: azureOpenaiKey,\n        endpoint: azureOpenaiEndpoint,\n        apiVersion: azureOpenaiApiVersion,\n        deployment: azureOpenaiDeployment,\n        dangerouslyAllowBrowser: true,\n        ...openaiExtraConfig,\n        ...azureExtraConfig,\n      });\n    }\n  } else if (!useAnthropicSdk) {\n    openai = new OpenAI({\n      baseURL: openaiBaseURL,\n      apiKey: openaiApiKey,\n      httpAgent: proxyAgent,\n      ...openaiExtraConfig,\n      defaultHeaders: {\n        ...(openaiExtraConfig?.defaultHeaders || {}),\n        [MIDSCENE_API_TYPE]: AIActionTypeValue.toString(),\n      },\n      dangerouslyAllowBrowser: true,\n    });\n  }\n\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    const { wrapOpenAI } = await import('langsmith/wrappers');\n    openai = wrapOpenAI(openai);\n  }\n\n  if (typeof openai !== 'undefined') {\n    return {\n      completion: openai.chat.completions,\n      style: 'openai',\n      modelName,\n      modelDescription,\n      uiTarsVersion,\n      vlMode,\n    };\n  }\n\n  // Anthropic\n  if (useAnthropicSdk) {\n    openai = new Anthropic({\n      apiKey: anthropicApiKey,\n      httpAgent: proxyAgent,\n      dangerouslyAllowBrowser: true,\n    }) as any;\n  }\n\n  if (typeof openai !== 'undefined' && (openai as any).messages) {\n    return {\n      completion: (openai as any).messages,\n      style: 'anthropic',\n      modelName,\n      modelDescription,\n      uiTarsVersion,\n      vlMode,\n    };\n  }\n\n  throw new Error('Openai SDK or Anthropic SDK is not initialized');\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n  },\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\n  const {\n    completion,\n    style,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n  } = await createChatClient({\n    AIActionTypeValue,\n    modelConfig,\n  });\n\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\n\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const commonConfig = {\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\n    stream: !!isStreaming,\n    max_tokens:\n      typeof maxTokens === 'number'\n        ? maxTokens\n        : Number.parseInt(maxTokens || '2048', 10),\n    ...(vlMode === 'qwen-vl' // qwen specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  try {\n    if (style === 'openai') {\n      debugCall(\n        `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n      );\n\n      if (isStreaming) {\n        const stream = (await completion.create(\n          {\n            model: modelName,\n            messages,\n            response_format: responseFormat,\n            ...commonConfig,\n          },\n          {\n            stream: true,\n          },\n        )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n          _request_id?: string | null;\n        };\n\n        for await (const chunk of stream) {\n          const content = chunk.choices?.[0]?.delta?.content || '';\n          const reasoning_content =\n            (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n          // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n          if (chunk.usage) {\n            usage = chunk.usage;\n          }\n\n          if (content || reasoning_content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              reasoning_content,\n              accumulated,\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.choices?.[0]?.finish_reason) {\n            timeCost = Date.now() - startTime;\n\n            // If usage is not available from the stream, provide a basic usage info\n            if (!usage) {\n              // Estimate token counts based on content length (rough approximation)\n              const estimatedTokens = Math.max(\n                1,\n                Math.floor(accumulated.length / 4),\n              );\n              usage = {\n                prompt_tokens: estimatedTokens,\n                completion_tokens: estimatedTokens,\n                total_tokens: estimatedTokens * 2,\n              };\n            }\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: {\n                prompt_tokens: usage.prompt_tokens ?? 0,\n                completion_tokens: usage.completion_tokens ?? 0,\n                total_tokens: usage.total_tokens ?? 0,\n                time_cost: timeCost ?? 0,\n                model_name: modelName,\n                model_description: modelDescription,\n                intent: modelConfig.intent,\n              },\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n        debugProfileStats(\n          `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\n        );\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          messages,\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n\n        debugProfileStats(\n          `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\n        );\n\n        debugProfileDetail(\n          `model usage detail: ${JSON.stringify(result.usage)}`,\n        );\n\n        assert(\n          result.choices,\n          `invalid response from LLM service: ${JSON.stringify(result)}`,\n        );\n        content = result.choices[0].message.content!;\n        usage = result.usage;\n      }\n\n      debugCall(`response: ${content}`);\n      assert(content, 'empty content');\n    } else if (style === 'anthropic') {\n      const convertImageContent = (content: any) => {\n        if (content.type === 'image_url') {\n          const imgBase64 = content.image_url.url;\n          assert(imgBase64, 'image_url is required');\n          const { mimeType, body } = parseBase64(content.image_url.url);\n          return {\n            source: {\n              type: 'base64',\n              media_type: mimeType,\n              data: body,\n            },\n            type: 'image',\n          };\n        }\n        return content;\n      };\n\n      if (isStreaming) {\n        const stream = (await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any)) as any;\n\n        for await (const chunk of stream) {\n          const content = chunk.delta?.text || '';\n          if (content) {\n            accumulated += content;\n            const chunkData: CodeGenerationChunk = {\n              content,\n              accumulated,\n              reasoning_content: '',\n              isComplete: false,\n              usage: undefined,\n            };\n            options.onChunk!(chunkData);\n          }\n\n          // Check if stream is complete\n          if (chunk.type === 'message_stop') {\n            timeCost = Date.now() - startTime;\n            const anthropicUsage = chunk.usage;\n\n            // Send final chunk\n            const finalChunk: CodeGenerationChunk = {\n              content: '',\n              accumulated,\n              reasoning_content: '',\n              isComplete: true,\n              usage: anthropicUsage\n                ? {\n                    prompt_tokens: anthropicUsage.input_tokens ?? 0,\n                    completion_tokens: anthropicUsage.output_tokens ?? 0,\n                    total_tokens:\n                      (anthropicUsage.input_tokens ?? 0) +\n                      (anthropicUsage.output_tokens ?? 0),\n                    time_cost: timeCost ?? 0,\n                    model_name: modelName,\n                    model_description: modelDescription,\n                    intent: modelConfig.intent,\n                  }\n                : undefined,\n            };\n            options.onChunk!(finalChunk);\n            break;\n          }\n        }\n        content = accumulated;\n      } else {\n        const result = await completion.create({\n          model: modelName,\n          system: 'You are a versatile professional in software UI automation',\n          messages: messages.map((m) => ({\n            role: 'user',\n            content: Array.isArray(m.content)\n              ? (m.content as any).map(convertImageContent)\n              : m.content,\n          })),\n          response_format: responseFormat,\n          ...commonConfig,\n        } as any);\n        timeCost = Date.now() - startTime;\n        content = (result as any).content[0].text as string;\n        usage = result.usage;\n      }\n\n      assert(content, 'empty content');\n    }\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      };\n    }\n\n    return {\n      content: content || '',\n      usage: usage\n        ? {\n            prompt_tokens: usage.prompt_tokens ?? 0,\n            completion_tokens: usage.completion_tokens ?? 0,\n            total_tokens: usage.total_tokens ?? 0,\n            time_cost: timeCost ?? 0,\n            model_name: modelName,\n            model_description: modelDescription,\n            intent: modelConfig.intent,\n          }\n        : undefined,\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport const getResponseFormat = (\n  modelName: string,\n  AIActionTypeValue: AIActionType,\n):\n  | OpenAI.ChatCompletionCreateParams['response_format']\n  | OpenAI.ResponseFormatJSONObject => {\n  let responseFormat:\n    | OpenAI.ChatCompletionCreateParams['response_format']\n    | OpenAI.ResponseFormatJSONObject\n    | undefined;\n\n  if (modelName.includes('gpt-4')) {\n    switch (AIActionTypeValue) {\n      case AIActionType.ASSERT:\n        responseFormat = assertSchema;\n        break;\n      case AIActionType.INSPECT_ELEMENT:\n        responseFormat = locatorSchema;\n        break;\n      case AIActionType.PLAN:\n        responseFormat = planSchema;\n        break;\n      case AIActionType.EXTRACT_DATA:\n      case AIActionType.DESCRIBE_ELEMENT:\n        responseFormat = { type: AIResponseFormat.JSON };\n        break;\n    }\n  }\n\n  // gpt-4o-2024-05-13 only supports json_object response format\n  if (modelName === 'gpt-4o-2024-05-13') {\n    responseFormat = { type: AIResponseFormat.JSON };\n  }\n\n  return responseFormat;\n};\n\nexport async function callAIWithObjectResponse<T>(\n  messages: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: T; usage?: AIUsageInfo }> {\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  return { content: jsonContent, usage: response.usage };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  AIActionTypeValue: AIActionType,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n  try {\n    return JSON.parse(cleanJsonString);\n  } catch {}\n  try {\n    return JSON.parse(jsonrepair(cleanJsonString));\n  } catch (e) {}\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    return JSON.parse(jsonrepair(jsonString));\n  }\n  throw Error(`failed to parse json response: ${input}`);\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","openaiUseAzureDeprecated","useAzureOpenai","azureOpenaiScope","azureOpenaiKey","azureOpenaiEndpoint","azureOpenaiApiVersion","azureOpenaiDeployment","azureExtraConfig","useAnthropicSdk","anthropicApiKey","modelDescription","uiTarsVersion","vlMode","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","AzureOpenAI","tokenProvider","assert","ifInBrowser","credential","DefaultAzureCredential","getBearerTokenProvider","OpenAI","MIDSCENE_API_TYPE","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","Error","console","wrapOpenAI","Anthropic","callAI","messages","options","completion","style","responseFormat","getResponseFormat","maxTokens","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","convertImageContent","imgBase64","mimeType","body","parseBase64","m","Array","_chunk_delta","anthropicUsage","e","newError","AIActionType","assertSchema","locatorSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IC0BA,eAAeI,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;QAQC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,wBAAwB,EACxBC,cAAc,EACdC,gBAAgB,EAChBC,cAAc,EACdC,mBAAmB,EACnBC,qBAAqB,EACrBC,qBAAqB,EACrBC,gBAAgB,EAChBC,eAAe,EACfC,eAAe,EACfC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACP,GAAGnB;QAEJ,IAAIoB;QAEJ,IAAIC;QACJ,MAAMC,aAAaC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QAC5B,IAAIrB,WAAW;YACboB,WAAW,oBAAoBpB;YAC/BmB,aAAa,IAAIG,2CAAAA,eAAeA,CAACtB;QACnC,OAAO,IAAID,YAAY;YACrBqB,WAAW,qBAAqBrB;YAChCoB,aAAa,IAAII,2CAAAA,eAAeA,CAACxB;QACnC;QAEA,IAAIM,0BAEFa,SAAS,IAAIM,gCAAAA,WAAWA,CAAC;YACvB,SAAStB;YACT,QAAQC;YACR,WAAWgB;YACX,GAAGf,iBAAiB;YACpB,yBAAyB;QAC3B;aACK,IAAIE,gBAAgB;YAGzB,IAAImB;YACJ,IAAIlB,kBAAkB;gBACpBmB,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,CAACC,sBAAAA,WAAWA,EACZ;gBAEF,MAAMC,aAAa,IAAIC,yBAAAA,sBAAsBA;gBAE7CJ,gBAAgBK,AAAAA,IAAAA,yBAAAA,sBAAAA,AAAAA,EAAuBF,YAAYrB;gBAEnDW,SAAS,IAAIM,gCAAAA,WAAWA,CAAC;oBACvB,sBAAsBC;oBACtB,UAAUhB;oBACV,YAAYC;oBACZ,YAAYC;oBACZ,GAAGP,iBAAiB;oBACpB,GAAGQ,gBAAgB;gBACrB;YACF,OAEEM,SAAS,IAAIM,gCAAAA,WAAWA,CAAC;gBACvB,QAAQhB;gBACR,UAAUC;gBACV,YAAYC;gBACZ,YAAYC;gBACZ,yBAAyB;gBACzB,GAAGP,iBAAiB;gBACpB,GAAGQ,gBAAgB;YACrB;QAEJ,OAAO,IAAI,CAACC,iBACVK,SAAS,IAAIa,CAAAA,yBAAAA,EAAO;YAClB,SAAS7B;YACT,QAAQC;YACR,WAAWgB;YACX,GAAGf,iBAAiB;YACpB,gBAAgB;gBACd,GAAIA,AAAAA,CAAAA,QAAAA,oBAAAA,KAAAA,IAAAA,kBAAmB,cAAc,AAAD,KAAK,CAAC,CAAC;gBAC3C,CAAC4B,oBAAAA,iBAAiBA,CAAC,EAAEnC,kBAAkB,QAAQ;YACjD;YACA,yBAAyB;QAC3B;QAGF,IACEqB,UACAe,oBAAAA,mBAAAA,CAAAA,qBAAyC,CAACC,oBAAAA,wBAAwBA,GAClE;YACA,IAAIP,sBAAAA,WAAWA,EACb,MAAM,IAAIQ,MAAM;YAElBC,QAAQ,GAAG,CAAC;YACZ,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM;YAC7BnB,SAASmB,WAAWnB;QACtB;QAEA,IAAI,AAAkB,WAAXA,QACT,OAAO;YACL,YAAYA,OAAO,IAAI,CAAC,WAAW;YACnC,OAAO;YACPjB;YACAc;YACAC;YACAC;QACF;QAIF,IAAIJ,iBACFK,SAAS,IAAIoB,oBAAAA,SAASA,CAAC;YACrB,QAAQxB;YACR,WAAWK;YACX,yBAAyB;QAC3B;QAGF,IAAI,AAAkB,WAAXD,UAA2BA,OAAe,QAAQ,EAC3D,OAAO;YACL,YAAaA,OAAe,QAAQ;YACpC,OAAO;YACPjB;YACAc;YACAC;YACAC;QACF;QAGF,MAAM,IAAIkB,MAAM;IAClB;IAEO,eAAeI,OACpBC,QAAsC,EACtC3C,iBAA+B,EAC/BC,WAAyB,EACzB2C,OAGC;QAED,MAAM,EACJC,UAAU,EACVC,KAAK,EACL1C,SAAS,EACTc,gBAAgB,EAChBC,aAAa,EACbC,MAAM,EACP,GAAG,MAAMrB,iBAAiB;YACzBC;YACAC;QACF;QAEA,MAAM8C,iBAAiBC,kBAAkB5C,WAAWJ;QAEpD,MAAMiD,YAAYb,oBAAAA,mBAAAA,CAAAA,iBAAqC,CAACc,oBAAAA,iBAAiBA;QACzE,MAAMC,YAAY3B,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QAC3B,MAAM4B,oBAAoB5B,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QACnC,MAAM6B,qBAAqB7B,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;QAEpC,MAAM8B,YAAYC,KAAK,GAAG;QAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;QACtD,IAAIa;QACJ,IAAIC,cAAc;QAClB,IAAIC;QACJ,IAAIC;QAEJ,MAAMC,eAAe;YACnB,aAAazC,AAAW,kBAAXA,SAA2B,MAAM;YAC9C,QAAQ,CAAC,CAACoC;YACV,YACE,AAAqB,YAArB,OAAOP,YACHA,YACAa,OAAO,QAAQ,CAACb,aAAa,QAAQ;YAC3C,GAAI7B,AAAW,cAAXA,SACA;gBACE,2BAA2B;YAC7B,IACA,CAAC,CAAC;QACR;QAEA,IAAI;YACF,IAAI0B,AAAU,aAAVA,OAAoB;gBACtBK,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAEpD,WAAW;gBAGrE,IAAIoD,aAAa;oBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;wBACE,OAAOzC;wBACPuC;wBACA,iBAAiBI;wBACjB,GAAGc,YAAY;oBACjB,GACA;wBACE,QAAQ;oBACV;oBAKF,WAAW,MAAMG,SAASD,OAAQ;4BAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;wBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;wBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;wBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;wBAGrB,IAAIP,WAAWW,mBAAmB;4BAChCV,eAAeD;4BACf,MAAMY,YAAiC;gCACrCZ;gCACAW;gCACAV;gCACA,YAAY;gCACZ,OAAOY;4BACT;4BACA1B,QAAQ,OAAO,CAAEyB;wBACnB;wBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;4BACrCP,WAAWL,KAAK,GAAG,KAAKD;4BAGxB,IAAI,CAACK,OAAO;gCAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;gCAElCC,QAAQ;oCACN,eAAeY;oCACf,mBAAmBA;oCACnB,cAAcA,AAAkB,IAAlBA;gCAChB;4BACF;4BAGA,MAAME,aAAkC;gCACtC,SAAS;gCACTf;gCACA,mBAAmB;gCACnB,YAAY;gCACZ,OAAO;oCACL,eAAeC,MAAM,aAAa,IAAI;oCACtC,mBAAmBA,MAAM,iBAAiB,IAAI;oCAC9C,cAAcA,MAAM,YAAY,IAAI;oCACpC,WAAWC,YAAY;oCACvB,YAAYxD;oCACZ,mBAAmBc;oCACnB,QAAQjB,YAAY,MAAM;gCAC5B;4BACF;4BACA2C,QAAQ,OAAO,CAAE6B;4BACjB;wBACF;oBACF;oBACAhB,UAAUC;oBACVN,kBACE,CAAC,iBAAiB,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,WAAW,EAAEwC,UAAU;gBAEvF,OAAO;wBAUqGc,eAAyDC,gBAAwDC;oBAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;wBACrC,OAAOzC;wBACPuC;wBACA,iBAAiBI;wBACjB,GAAGc,YAAY;oBACjB;oBACAD,WAAWL,KAAK,GAAG,KAAKD;oBAExBF,kBACE,CAAC,OAAO,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAEuD,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;oBAG3TxB,mBACE,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;oBAGvDhD,IAAAA,sBAAAA,MAAAA,AAAAA,EACEgD,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;oBAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;oBAC3ClB,QAAQkB,OAAO,KAAK;gBACtB;gBAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;gBAChC5B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4B,SAAS;YAClB,OAAO,IAAIX,AAAU,gBAAVA,OAAuB;gBAChC,MAAMiC,sBAAsB,CAACtB;oBAC3B,IAAIA,AAAiB,gBAAjBA,QAAQ,IAAI,EAAkB;wBAChC,MAAMuB,YAAYvB,QAAQ,SAAS,CAAC,GAAG;wBACvC5B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOmD,WAAW;wBAClB,MAAM,EAAEC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,AAAAA,IAAAA,oBAAAA,WAAAA,AAAAA,EAAY1B,QAAQ,SAAS,CAAC,GAAG;wBAC5D,OAAO;4BACL,QAAQ;gCACN,MAAM;gCACN,YAAYwB;gCACZ,MAAMC;4BACR;4BACA,MAAM;wBACR;oBACF;oBACA,OAAOzB;gBACT;gBAEA,IAAID,aAAa;oBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CAAC;wBACtC,OAAOzC;wBACP,QAAQ;wBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;gCAC7B,MAAM;gCACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;4BACf;wBACA,iBAAiBrC;wBACjB,GAAGc,YAAY;oBACjB;oBAEA,WAAW,MAAMG,SAASD,OAAQ;4BAChBuB;wBAAhB,MAAM7B,UAAU6B,AAAAA,SAAAA,CAAAA,eAAAA,MAAM,KAAK,AAAD,IAAVA,KAAAA,IAAAA,aAAa,IAAI,AAAD,KAAK;wBACrC,IAAI7B,SAAS;4BACXC,eAAeD;4BACf,MAAMY,YAAiC;gCACrCZ;gCACAC;gCACA,mBAAmB;gCACnB,YAAY;gCACZ,OAAOY;4BACT;4BACA1B,QAAQ,OAAO,CAAEyB;wBACnB;wBAGA,IAAIL,AAAe,mBAAfA,MAAM,IAAI,EAAqB;4BACjCJ,WAAWL,KAAK,GAAG,KAAKD;4BACxB,MAAMiC,iBAAiBvB,MAAM,KAAK;4BAGlC,MAAMS,aAAkC;gCACtC,SAAS;gCACTf;gCACA,mBAAmB;gCACnB,YAAY;gCACZ,OAAO6B,iBACH;oCACE,eAAeA,eAAe,YAAY,IAAI;oCAC9C,mBAAmBA,eAAe,aAAa,IAAI;oCACnD,cACGA,AAAAA,CAAAA,eAAe,YAAY,IAAI,KAC/BA,CAAAA,eAAe,aAAa,IAAI;oCACnC,WAAW3B,YAAY;oCACvB,YAAYxD;oCACZ,mBAAmBc;oCACnB,QAAQjB,YAAY,MAAM;gCAC5B,IACAqE;4BACN;4BACA1B,QAAQ,OAAO,CAAE6B;4BACjB;wBACF;oBACF;oBACAhB,UAAUC;gBACZ,OAAO;oBACL,MAAMmB,SAAS,MAAMhC,WAAW,MAAM,CAAC;wBACrC,OAAOzC;wBACP,QAAQ;wBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;gCAC7B,MAAM;gCACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;4BACf;wBACA,iBAAiBrC;wBACjB,GAAGc,YAAY;oBACjB;oBACAD,WAAWL,KAAK,GAAG,KAAKD;oBACxBG,UAAWoB,OAAe,OAAO,CAAC,EAAE,CAAC,IAAI;oBACzClB,QAAQkB,OAAO,KAAK;gBACtB;gBAEAhD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO4B,SAAS;YAClB;YAEA,IAAID,eAAe,CAACG,OAAO;gBAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;gBAEtCE,QAAQ;oBACN,eAAeY;oBACf,mBAAmBA;oBACnB,cAAcA,AAAkB,IAAlBA;gBAChB;YACF;YAEA,OAAO;gBACL,SAASd,WAAW;gBACpB,OAAOE,QACH;oBACE,eAAeA,MAAM,aAAa,IAAI;oBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;oBAC9C,cAAcA,MAAM,YAAY,IAAI;oBACpC,WAAWC,YAAY;oBACvB,YAAYxD;oBACZ,mBAAmBc;oBACnB,QAAQjB,YAAY,MAAM;gBAC5B,IACAqE;gBACJ,YAAY,CAAC,CAACd;YAChB;QACF,EAAE,OAAOgC,GAAQ;YACfjD,QAAQ,KAAK,CAAC,kBAAkBiD;YAChC,MAAMC,WAAW,IAAInD,MACnB,CAAC,eAAe,EAAEkB,cAAc,eAAe,GAAG,kBAAkB,EAAEgC,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;gBACE,OAAOA;YACT;YAEF,MAAMC;QACR;IACF;IAEO,MAAMzC,oBAAoB,CAC/B5C,WACAJ;QAIA,IAAI+C;QAKJ,IAAI3C,UAAU,QAAQ,CAAC,UACrB,OAAQJ;YACN,KAAK0F,mCAAAA,YAAAA,CAAAA,MAAmB;gBACtB3C,iBAAiB4C,6BAAAA,YAAYA;gBAC7B;YACF,KAAKD,mCAAAA,YAAAA,CAAAA,eAA4B;gBAC/B3C,iBAAiB6C,+BAAAA,aAAaA;gBAC9B;YACF,KAAKF,mCAAAA,YAAAA,CAAAA,IAAiB;gBACpB3C,iBAAiB8C,gCAAAA,UAAUA;gBAC3B;YACF,KAAKH,mCAAAA,YAAAA,CAAAA,YAAyB;YAC9B,KAAKA,mCAAAA,YAAAA,CAAAA,gBAA6B;gBAChC3C,iBAAiB;oBAAE,MAAM+C,kCAAAA,gBAAAA,CAAAA,IAAqB;gBAAC;gBAC/C;QACJ;QAIF,IAAI1F,AAAc,wBAAdA,WACF2C,iBAAiB;YAAE,MAAM+C,kCAAAA,gBAAAA,CAAAA,IAAqB;QAAC;QAGjD,OAAO/C;IACT;IAEO,eAAegD,yBACpBpD,QAAgB,EAChB3C,iBAA+B,EAC/BC,WAAyB;QAEzB,MAAM+F,WAAW,MAAMtD,OAAOC,UAAU3C,mBAAmBC;QAC3D4B,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOmE,UAAU;QACjB,MAAM5E,SAASnB,YAAY,MAAM;QACjC,MAAMgG,cAAcC,cAAcF,SAAS,OAAO,EAAE5E;QACpD,OAAO;YAAE,SAAS6E;YAAa,OAAOD,SAAS,KAAK;QAAC;IACvD;IAEO,eAAeG,yBACpBC,IAAY,EACZpG,iBAA+B,EAC/BC,WAAyB;QAEzB,MAAM,EAAEwD,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAO0D,MAAMpG,mBAAmBC;QACjE,OAAO;YAAEwD;YAASE;QAAM;IAC1B;IAEO,SAAS0C,yBAAyBL,QAAgB;QACvD,IAAI;YAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;YACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;YAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;YAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;YAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;YACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;QAE3B,EAAE,OAAM,CAAC;QAET,OAAOR;IACT;IAEO,SAASS,yBAAyBC,KAAa;QACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;QAG5C,OAAOA;IACT;IAEO,SAASR,cAAcQ,KAAa,EAAEtF,MAAgC;QAC3E,MAAMuF,kBAAkBN,yBAAyBK;QAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;gBACtCC;YAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAAC9C;QACT;QACA,IAAI;YACF,OAAOgB,KAAK,KAAK,CAAC6B;QACpB,EAAE,OAAM,CAAC;QACT,IAAI;YACF,OAAO7B,KAAK,KAAK,CAAC+B,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWF;QAC/B,EAAE,OAAOnB,GAAG,CAAC;QAEb,IAAIpE,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;YAC1D,MAAM0F,aAAaL,yBAAyBE;YAC5C,OAAO7B,KAAK,KAAK,CAAC+B,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWC;QAC/B;QACA,MAAMxE,MAAM,CAAC,+BAA+B,EAAEoE,OAAO;IACvD"}

package/dist/lib/ai-model/ui-tars-planning.js

@@ -45,17 +45,17 @@ const pointToBbox = (point, width, height)=>[
         Math.round(Math.min(point.y + bboxSize / 2, height))
     ];
 async function vlmPlanning(options) {
-    const { conversationHistory, userInstruction, size, modelPreferences } = options;
+    const { conversationHistory, userInstruction, size, modelConfig } = options;
+    const { uiTarsModelVersion } = modelConfig;
     const systemPrompt = (0, ui_tars_planning_js_namespaceObject.getUiTarsPlanningPrompt)() + userInstruction;
-    const res = await (0, index_js_namespaceObject.call)([
+    const res = await (0, index_js_namespaceObject.callAIWithStringResponse)([
         {
             role: 'user',
             content: systemPrompt
         },
         ...conversationHistory
-    ], external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelPreferences);
+    ], external_common_js_namespaceObject.AIActionType.INSPECT_ELEMENT, modelConfig);
     const convertedText = convertBboxToCoordinates(res.content);
-    const modelVer = (0, env_namespaceObject.uiTarsModelVersion)(modelPreferences);
     const { parsed } = (0, action_parser_namespaceObject.actionParser)({
         prediction: convertedText,
         factor: [
@@ -66,9 +66,9 @@ async function vlmPlanning(options) {
             width: size.width,
             height: size.height
         },
-        modelVer: modelVer || void 0
+        modelVer: uiTarsModelVersion
     });
-    debug('ui-tars modelVer', modelVer, ', parsed', JSON.stringify(parsed));
+    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
     const transformActions = [];
     parsed.forEach((action)=>{
         if ('click' === action.action_type) {
@@ -184,8 +184,8 @@ function getPoint(startBox, size) {
         y * size.height
     ];
 }
-async function resizeImageForUiTars(imageBase64, size, modelPreferences) {
-    if ('vlm-ui-tars' === (0, env_namespaceObject.vlLocateMode)(modelPreferences) && (0, env_namespaceObject.uiTarsModelVersion)(modelPreferences) === env_namespaceObject.UITarsModelVersion.V1_5) {
+async function resizeImageForUiTars(imageBase64, size, uiTarsVersion) {
+    if (uiTarsVersion === env_namespaceObject.UITarsModelVersion.V1_5) {
         debug('ui-tars-v1.5, will check image size', size);
         const currentPixels = size.width * size.height;
         const maxPixels = 12845056;

package/dist/lib/ai-model/ui-tars-planning.js.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/ui-tars-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIUsageInfo,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Size,\n} from '@/types';\nimport {\n  type IModelPreferences,\n  UITarsModelVersion,\n  uiTarsModelVersion,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType } from './common';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { call } from './service-caller/index';\ntype ActionType =\n  | 'click'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function vlmPlanning(options: {\n  userInstruction: string;\n  conversationHistory: ChatCompletionMessageParam[];\n  size: { width: number; height: number };\n  modelPreferences: IModelPreferences;\n}): Promise<{\n  actions: PlanningAction<any>[];\n  actionsFromModel: ReturnType<typeof actionParser>['parsed'];\n  action_summary: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const { conversationHistory, userInstruction, size, modelPreferences } =\n    options;\n  const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;\n\n  const res = await call(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory,\n    ],\n    AIActionType.INSPECT_ELEMENT,\n    modelPreferences,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const modelVer = uiTarsModelVersion(modelPreferences);\n\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: modelVer || undefined,\n  });\n\n  debug('ui-tars modelVer', modelVer, ', parsed', JSON.stringify(parsed));\n\n  const transformActions: PlanningAction[] = [];\n  parsed.forEach((action) => {\n    if (action.action_type === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n      });\n    } else if (action.action_type === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys,\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (action.action_type === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(`No actions found, response: ${res.content}`, {\n      cause: {\n        prediction: res.content,\n        parsed,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n\n  return {\n    actions: transformActions,\n    actionsFromModel: parsed,\n    action_summary: getSummary(res.content),\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  modelPreferences: IModelPreferences,\n) {\n  if (\n    vlLocateMode(modelPreferences) === 'vlm-ui-tars' &&\n    uiTarsModelVersion(modelPreferences) === UITarsModelVersion.V1_5\n  ) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","vlmPlanning","options","conversationHistory","userInstruction","size","modelPreferences","systemPrompt","getUiTarsPlanningPrompt","res","call","AIActionType","convertedText","convertBboxToCoordinates","modelVer","uiTarsModelVersion","parsed","actionParser","undefined","JSON","transformActions","action","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","Error","getSummary","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","resizeImageForUiTars","imageBase64","vlLocateMode","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACwBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,YAAYC,OAKjC;IAQC,MAAM,EAAEC,mBAAmB,EAAEC,eAAe,EAAEC,IAAI,EAAEC,gBAAgB,EAAE,GACpEJ;IACF,MAAMK,eAAeC,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,MAA4BJ;IAEjD,MAAMK,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,IAAAA,AAAAA,EAChB;QACE;YACE,MAAM;YACN,SAASH;QACX;WACGJ;KACJ,EACDQ,mCAAAA,YAAAA,CAAAA,eAA4B,EAC5BL;IAEF,MAAMM,gBAAgBC,yBAAyBJ,IAAI,OAAO;IAE1D,MAAMK,WAAWC,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EAAmBT;IAEpC,MAAM,EAAEU,MAAM,EAAE,GAAGC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC9B,YAAYL;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOP,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUS,YAAYI;IACxB;IAEAzB,MAAM,oBAAoBqB,UAAU,YAAYK,KAAK,SAAS,CAACH;IAE/D,MAAMI,mBAAqC,EAAE;IAC7CJ,OAAO,OAAO,CAAC,CAACK;QACd,IAAIA,AAAuB,YAAvBA,OAAO,WAAW,EAAc;YAClCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMxB,QAAQ0B,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEhB;YACvDe,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMzB,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BQ,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIgB,AAAuB,WAAvBA,OAAO,WAAW,EAAa;YACxCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMG,aAAaD,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEhB;YAC5D,MAAMoB,WAAWF,SAASF,OAAO,aAAa,CAAC,OAAO,EAAEhB;YACxDe,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMzB,YACJ;4BAAE,GAAG4B,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCnB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQgB,OAAO,OAAO,IAAI;wBAC1B,MAAMzB,YACJ;4BAAE,GAAG6B,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjCpB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASgB,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIA,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOC,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWC,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,eAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASC,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3B,IAAKA,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMK,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBN,OAAO,aAAa,CAAC,GAAG;YAE1DD,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASM;gBACX;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEO,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASC,OAAO,OAAO,IAAI;QAC7B;IAEJ;IAEA,IAAID,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIS,MAAM,CAAC,4BAA4B,EAAEpB,IAAI,OAAO,EAAE,EAAE;QAC5D,OAAO;YACL,YAAYA,IAAI,OAAO;YACvBO;QACF;IACF;IAGFvB,MAAM,oBAAoB0B,KAAK,SAAS,CAACC,kBAAkB,MAAM;IAEjE,OAAO;QACL,SAASA;QACT,kBAAkBJ;QAClB,gBAAgBc,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWrB,IAAI,OAAO;QACtC,OAAOA,IAAI,KAAK;QAChB,aAAaU,KAAK,SAAS,CAACV,IAAI,OAAO,EAAES,QAAW;IACtD;AACF;AAOA,SAASL,yBAAyBkB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAI5C,KAAK,KAAK,CAAEuC,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAI7C,KAAK,KAAK,CAAEyC,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASV,SAASwB,QAAgB,EAAE1C,IAAuC;IACzE,MAAM,CAACuC,GAAGC,EAAE,GAAG1B,KAAK,KAAK,CAAC4B;IAC1B,OAAO;QAACH,IAAIvC,KAAK,KAAK;QAAEwC,IAAIxC,KAAK,MAAM;KAAC;AAC1C;AAkEO,eAAe2C,qBACpBC,WAAmB,EACnB5C,IAAU,EACVC,gBAAmC;IAEnC,IACE4C,AAAmC,kBAAnCA,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAa5C,qBACbS,AAAAA,IAAAA,oBAAAA,kBAAAA,AAAAA,EAAmBT,sBAAsB6C,oBAAAA,kBAAAA,CAAAA,IAAuB,EAChE;QACA1D,MAAM,uCAAuCY;QAC7C,MAAM+C,gBAAgB/C,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAMgD,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAetD,KAAK,IAAI,CAACqD,YAAYD;YAC3C,MAAMG,WAAWvD,KAAK,KAAK,CAACK,KAAK,KAAK,GAAGiD;YACzC,MAAME,YAAYxD,KAAK,KAAK,CAACK,KAAK,MAAM,GAAGiD;YAC3C7D,MACE,2DACA8D,UACAC;YAEF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}
+{"version":3,"file":"ai-model/ui-tars-planning.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/ai-model/ui-tars-planning.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import type {\n  AIUsageInfo,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Size,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport { AIActionType } from './common';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { callAIWithStringResponse } from './service-caller/index';\ntype ActionType =\n  | 'click'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function vlmPlanning(options: {\n  userInstruction: string;\n  conversationHistory: ChatCompletionMessageParam[];\n  size: { width: number; height: number };\n  modelConfig: IModelConfig;\n}): Promise<{\n  actions: PlanningAction<any>[];\n  actionsFromModel: ReturnType<typeof actionParser>['parsed'];\n  action_summary: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n}> {\n  const { conversationHistory, userInstruction, size, modelConfig } = options;\n  const { uiTarsModelVersion } = modelConfig;\n  const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory,\n    ],\n    AIActionType.INSPECT_ELEMENT,\n    modelConfig,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  parsed.forEach((action) => {\n    if (action.action_type === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n      });\n    } else if (action.action_type === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'finished') {\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (action.action_type === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys,\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (action.action_type === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    }\n  });\n\n  if (transformActions.length === 0) {\n    throw new Error(`No actions found, response: ${res.content}`, {\n      cause: {\n        prediction: res.content,\n        parsed,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n\n  return {\n    actions: transformActions,\n    actionsFromModel: parsed,\n    action_summary: getSummary(res.content),\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  uiTarsVersion: UITarsModelVersion | undefined,\n) {\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","vlmPlanning","options","conversationHistory","userInstruction","size","modelConfig","uiTarsModelVersion","systemPrompt","getUiTarsPlanningPrompt","res","callAIWithStringResponse","AIActionType","convertedText","convertBboxToCoordinates","parsed","actionParser","JSON","transformActions","action","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","Error","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","resizeImageForUiTars","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;ACmBA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,YAAYC,OAKjC;IAQC,MAAM,EAAEC,mBAAmB,EAAEC,eAAe,EAAEC,IAAI,EAAEC,WAAW,EAAE,GAAGJ;IACpE,MAAM,EAAEK,kBAAkB,EAAE,GAAGD;IAC/B,MAAME,eAAeC,AAAAA,IAAAA,oCAAAA,uBAAAA,AAAAA,MAA4BL;IAEjD,MAAMM,MAAM,MAAMC,AAAAA,IAAAA,yBAAAA,wBAAAA,AAAAA,EAChB;QACE;YACE,MAAM;YACN,SAASH;QACX;WACGL;KACJ,EACDS,mCAAAA,YAAAA,CAAAA,eAA4B,EAC5BN;IAEF,MAAMO,gBAAgBC,yBAAyBJ,IAAI,OAAO;IAE1D,MAAM,EAAEK,MAAM,EAAE,GAAGC,AAAAA,IAAAA,8BAAAA,YAAAA,AAAAA,EAAa;QAC9B,YAAYH;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOR,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUE;IACZ;IAEAd,MACE,oBACAc,oBACA,YACAU,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7CH,OAAO,OAAO,CAAC,CAACI;QACd,IAAIA,AAAuB,YAAvBA,OAAO,WAAW,EAAc;YAClCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAMtB,QAAQwB,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEd;YACvDa,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BQ,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIc,AAAuB,WAAvBA,OAAO,WAAW,EAAa;YACxCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCC,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMG,aAAaD,SAASF,OAAO,aAAa,CAAC,SAAS,EAAEd;YAC5D,MAAMkB,WAAWF,SAASF,OAAO,aAAa,CAAC,OAAO,EAAEd;YACxDa,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQC,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAG0B,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCjB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQc,OAAO,OAAO,IAAI;wBAC1B,MAAMvB,YACJ;4BAAE,GAAG2B,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjClB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASc,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIA,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOC,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWC,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,eAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO,CAAC;YACR,SAASC,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIA,AAAuB,aAAvBA,OAAO,WAAW,EAC3B,IAAKA,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMK,OAAOC,AAAAA,IAAAA,mCAAAA,oBAAAA,AAAAA,EAAqBN,OAAO,aAAa,CAAC,GAAG;YAE1DD,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASM;gBACX;gBACA,SAASL,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEO,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAuB,WAAvBA,OAAO,WAAW,EAC3BD,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASC,OAAO,OAAO,IAAI;QAC7B;IAEJ;IAEA,IAAID,AAA4B,MAA5BA,iBAAiB,MAAM,EACzB,MAAM,IAAIS,MAAM,CAAC,4BAA4B,EAAEjB,IAAI,OAAO,EAAE,EAAE;QAC5D,OAAO;YACL,YAAYA,IAAI,OAAO;YACvBK;QACF;IACF;IAGFtB,MAAM,oBAAoBwB,KAAK,SAAS,CAACC,kBAAkB,MAAM;IAEjE,OAAO;QACL,SAASA;QACT,kBAAkBH;QAClB,gBAAgBa,AAAAA,IAAAA,oCAAAA,UAAAA,AAAAA,EAAWlB,IAAI,OAAO;QACtC,OAAOA,IAAI,KAAK;QAChB,aAAaO,KAAK,SAAS,CAACP,IAAI,OAAO,EAAEmB,QAAW;IACtD;AACF;AAOA,SAASf,yBAAyBgB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAI3C,KAAK,KAAK,CAAEsC,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAI5C,KAAK,KAAK,CAAEwC,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASX,SAASyB,QAAgB,EAAEzC,IAAuC;IACzE,MAAM,CAACsC,GAAGC,EAAE,GAAG3B,KAAK,KAAK,CAAC6B;IAC1B,OAAO;QAACH,IAAItC,KAAK,KAAK;QAAEuC,IAAIvC,KAAK,MAAM;KAAC;AAC1C;AAkEO,eAAe0C,qBACpBC,WAAmB,EACnB3C,IAAU,EACV4C,aAA6C;IAE7C,IAAIA,kBAAkBC,oBAAAA,kBAAAA,CAAAA,IAAuB,EAAE;QAC7CzD,MAAM,uCAAuCY;QAC7C,MAAM8C,gBAAgB9C,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAM+C,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAerD,KAAK,IAAI,CAACoD,YAAYD;YAC3C,MAAMG,WAAWtD,KAAK,KAAK,CAACK,KAAK,KAAK,GAAGgD;YACzC,MAAME,YAAYvD,KAAK,KAAK,CAACK,KAAK,MAAM,GAAGgD;YAC3C5D,MACE,2DACA6D,UACAC;YAEF,MAAMC,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,eAAAA,AAAAA,EAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}

package/dist/lib/insight/index.js

@@ -47,9 +47,8 @@ function _define_property(obj, key, value) {
 }
 const debug = (0, logger_namespaceObject.getDebug)('ai:insight');
 class Insight {
-    async locate(query, opt) {
+    async locate(query, opt, modelConfig) {
         var _parseResult_errors;
-        const { callAI } = opt || {};
         const queryPrompt = 'string' == typeof query ? query : query.prompt;
         (0, utils_namespaceObject.assert)(queryPrompt, 'query is required for locate');
         const dumpSubscriber = this.onceDumpUpdatedFn;
@@ -59,10 +58,8 @@ class Insight {
         if (globalDeepThinkSwitch) debug('globalDeepThinkSwitch', globalDeepThinkSwitch);
         let searchAreaPrompt;
         if (query.deepThink || globalDeepThinkSwitch) searchAreaPrompt = query.prompt;
-        const modelPreferences = {
-            intent: 'grounding'
-        };
-        if (searchAreaPrompt && !(0, env_namespaceObject.vlLocateMode)(modelPreferences)) {
+        const { vlMode } = modelConfig;
+        if (searchAreaPrompt && !vlMode) {
             console.warn('The "deepThink" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model');
             searchAreaPrompt = void 0;
         }
@@ -74,7 +71,8 @@ class Insight {
         if (searchAreaPrompt) {
             searchAreaResponse = await (0, inspect_js_namespaceObject.AiLocateSection)({
                 context,
-                sectionDescription: searchAreaPrompt
+                sectionDescription: searchAreaPrompt,
+                modelConfig
             });
             (0, utils_namespaceObject.assert)(searchAreaResponse.rect, `cannot find search area for "${searchAreaPrompt}"${searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''}`);
             searchAreaRawResponse = searchAreaResponse.rawResponse;
@@ -83,10 +81,11 @@ class Insight {
         }
         const startTime = Date.now();
         const { parseResult, rect, elementById, rawResponse, usage, isOrderSensitive } = await (0, index_js_namespaceObject.AiLocateElement)({
-            callAI: callAI || this.aiVendorFn,
+            callAIFn: this.aiVendorFn,
             context,
             targetElementDescription: queryPrompt,
-            searchConfig: searchAreaResponse
+            searchConfig: searchAreaResponse,
+            modelConfig
         });
         const timeCost = Date.now() - startTime;
         const taskInfo = {
@@ -144,14 +143,11 @@ class Insight {
             rect
         };
     }
-    async extract(dataDemand, opt, multimodalPrompt) {
+    async extract(dataDemand, modelConfig, opt, multimodalPrompt) {
         var _parseResult_errors;
         (0, utils_namespaceObject.assert)('object' == typeof dataDemand || 'string' == typeof dataDemand, `dataDemand should be object or string, but get ${typeof dataDemand}`);
         const dumpSubscriber = this.onceDumpUpdatedFn;
         this.onceDumpUpdatedFn = void 0;
-        const modelPreferences = {
-            intent: 'VQA'
-        };
         const context = await this.contextRetrieverFn('extract');
         const startTime = Date.now();
         const { parseResult, usage } = await (0, index_js_namespaceObject.AiExtractElementInfo)({
@@ -159,7 +155,7 @@ class Insight {
             dataQuery: dataDemand,
             multimodalPrompt,
             extractOption: opt,
-            modelPreferences
+            modelConfig
         });
         const timeCost = Date.now() - startTime;
         const taskInfo = {
@@ -191,14 +187,12 @@ class Insight {
             usage
         };
     }
-    async describe(target, opt) {
+    async describe(target, modelConfig, opt) {
         (0, utils_namespaceObject.assert)(target, 'target is required for insight.describe');
         const context = await this.contextRetrieverFn('describe');
         const { screenshotBase64, size } = context;
         (0, utils_namespaceObject.assert)(screenshotBase64, 'screenshot is required for insight.describe');
-        const modelPreferences = {
-            intent: 'grounding'
-        };
+        const { vlMode } = modelConfig;
         const systemPrompt = (0, describe_js_namespaceObject.elementDescriberInstruction)();
         const defaultRectSize = 30;
         const targetRect = Array.isArray(target) ? {
@@ -218,9 +212,9 @@ class Insight {
             borderThickness: 3
         });
         if (null == opt ? void 0 : opt.deepThink) {
-            const searchArea = (0, common_js_namespaceObject.expandSearchArea)(targetRect, context.size, modelPreferences);
+            const searchArea = (0, common_js_namespaceObject.expandSearchArea)(targetRect, context.size, vlMode);
             debug('describe: set searchArea', searchArea);
-            imagePayload = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, (0, env_namespaceObject.getIsUseQwenVl)(modelPreferences));
+            imagePayload = await (0, img_namespaceObject.cropByRect)(imagePayload, searchArea, 'qwen-vl' === vlMode);
         }
         const msgs = [
             {
@@ -240,8 +234,8 @@ class Insight {
                 ]
             }
         ];
-        const callAIFn = this.aiVendorFn || index_js_namespaceObject.callToGetJSONObject;
-        const res = await callAIFn(msgs, common_js_namespaceObject.AIActionType.DESCRIBE_ELEMENT, modelPreferences);
+        const callAIFn = this.aiVendorFn;
+        const res = await callAIFn(msgs, common_js_namespaceObject.AIActionType.DESCRIBE_ELEMENT, modelConfig);
         const { content } = res;
         (0, utils_namespaceObject.assert)(!content.error, `describe failed: ${content.error}`);
         (0, utils_namespaceObject.assert)(content.description, 'failed to describe the element');
@@ -249,7 +243,7 @@ class Insight {
     }
     constructor(context, opt){
         _define_property(this, "contextRetrieverFn", void 0);
-        _define_property(this, "aiVendorFn", common_js_namespaceObject.callAiFn);
+        _define_property(this, "aiVendorFn", index_js_namespaceObject.callAIWithObjectResponse);
         _define_property(this, "onceDumpUpdatedFn", void 0);
         _define_property(this, "taskInfo", void 0);
         (0, utils_namespaceObject.assert)(context, 'context is required for Insight');

package/dist/lib/insight/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"insight/index.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import {\n  AIActionType,\n  type AIArgs,\n  callAiFn,\n  expandSearchArea,\n} from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callToGetJSONObject,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightOptions,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport {\n  type IModelPreferences,\n  MIDSCENE_FORCE_DEEP_THINK,\n  getIsUseQwenVl,\n  globalConfigManager,\n  vlLocateMode,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n  callAI?: typeof callAiFn<AIElementResponse>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: (...args: Array<any>) => Promise<any> = callAiFn;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt?: LocateOpts,\n  ): Promise<LocateResult> {\n    const { callAI } = opt || {};\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n    const modelPreferences: IModelPreferences = {\n      intent: 'grounding',\n    };\n\n    if (searchAreaPrompt && !vlLocateMode(modelPreferences)) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAI: callAI || this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<{\n    data: T;\n    thought?: string;\n    usage?: AIUsageInfo;\n  }> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const modelPreferences: IModelPreferences = {\n      intent: 'VQA',\n    };\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelPreferences,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const modelPreferences: IModelPreferences = { intent: 'grounding' };\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(\n        targetRect,\n        context.size,\n        modelPreferences,\n      );\n      debug('describe: set searchArea', searchArea);\n      imagePayload = await cropByRect(\n        imagePayload,\n        searchArea,\n        getIsUseQwenVl(modelPreferences),\n      );\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn =\n      this.aiVendorFn || callToGetJSONObject<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelPreferences,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Insight","query","opt","_parseResult_errors","callAI","queryPrompt","assert","dumpSubscriber","undefined","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","modelPreferences","vlLocateMode","console","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","elementById","rawResponse","usage","isOrderSensitive","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","item","element","emitInsightDump","Error","dataDemand","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","cropByRect","getIsUseQwenVl","msgs","callAIFn","callToGetJSONObject","res","AIActionType","content","callAiFn","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;AC8CA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAmCnB,MAAM,OACJC,KAA0B,EAC1BC,GAAgB,EACO;YAkFnBC;QAjFJ,MAAM,EAAEC,MAAM,EAAE,GAAGF,OAAO,CAAC;QAC3B,MAAMG,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QACpB,MAAME,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzBF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMQ,wBAAwBC,oBAAAA,mBAAAA,CAAAA,qBAAyC,CACrEC,oBAAAA,yBAAyBA;QAE3B,IAAIF,uBACFX,MAAM,yBAAyBW;QAEjC,IAAIG;QACJ,IAAIX,MAAM,SAAS,IAAIQ,uBACrBG,mBAAmBX,MAAM,MAAM;QAEjC,MAAMY,mBAAsC;YAC1C,QAAQ;QACV;QAEA,IAAID,oBAAoB,CAACE,AAAAA,IAAAA,oBAAAA,YAAAA,AAAAA,EAAaD,mBAAmB;YACvDE,QAAQ,IAAI,CACV;YAEFH,mBAAmBJ;QACrB;QAEA,MAAMQ,UAAUd,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE/D,IAAIe;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIR,kBAAkB;YACpBQ,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCL;gBACA,oBAAoBJ;YACtB;YACAN,IAAAA,sBAAAA,MAAAA,AAAAA,EACEc,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAER,iBAAiB,CAAC,EAChDQ,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,WAAW,EACXC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgB;YACxB,QAAQ1B,UAAU,IAAI,CAAC,UAAU;YACjCY;YACA,0BAA0BX;YAC1B,cAAce;QAChB;QAEA,MAAMW,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAX;YACAC;YACAC;QACF;QAEA,IAAIe;QACJ,IAAI,QAAA/B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B+B,WAAW,CAAC,6BAA6B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS9B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAaoB;YACb,MAAM;YACNO;YACA,WAAW,CAAC,CAACf;YACb,OAAOiB;QACT;QAEA,MAAME,WAA0B,EAAE;QACjCZ,CAAAA,YAAY,QAAQ,IAAI,EAAC,EAAG,OAAO,CAAC,CAACa;YACpC,IAAI,QAAQA,MAAM;gBAChB,MAAMC,UAAUZ,YAAYW,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE;gBAEpC,IAAI,CAACC,SAAS,YACZvB,QAAQ,IAAI,CACV,CAAC,+BAA+B,EAAEsB,KAAK,EAAE,CAAC,0CAA0C,CAAC;gBAIzFD,SAAS,IAAI,CAACE;YAChB;QACF;QAEAC,IAAAA,kCAAAA,eAAAA,AAAAA,EACE;YACE,GAAGJ,QAAQ;YACX,gBAAgBC;QAClB,GACA7B;QAGF,IAAI2B,UACF,MAAM,IAAIM,MAAMN;QAGlB5B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE8B,SAAS,MAAM,IAAI,GACnB,CAAC,0CAA0C,EAAEA,SAAS,MAAM,EAAE;QAGhE,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,SAASA,QAAQ,CAAC,EAAE,CAAE,OAAO;gBAC7B,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM,IAAI,EAAE;gBACjC,YAAYA,QAAQ,CAAC,EAAE,CAAE,UAAU;gBACnCP;YACF;YACAJ;QACF;QAEF,OAAO;YACL,SAAS;YACTA;QACF;IACF;IAEA,MAAM,QACJgB,UAA+B,EAC/BvC,GAA0B,EAC1BwC,gBAAoC,EAKnC;YA+BGvC;QA9BJG,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOmC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMlC,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzB,MAAMK,mBAAsC;YAC1C,QAAQ;QACV;QAEA,MAAMG,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE9C,MAAMM,YAAYC,KAAK,GAAG;QAC1B,MAAM,EAAEC,WAAW,EAAEI,KAAK,EAAE,GAAG,MAAMe,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EAAwB;YAC3D3B;YACA,WAAWyB;YACXC;YACA,eAAexC;YACfW;QACF;QAEA,MAAMkB,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACT;QAC9B;QAEA,IAAIU;QACJ,IAAI,QAAA/B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B+B,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTM;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNT;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGrB,eAAe,CAAC;QAG1Ce,IAAAA,kCAAAA,eAAAA,AAAAA,EACE;YACE,GAAGJ,QAAQ;YACXS;QACF,GACArC;QAGF,IAAI2B,YAAY,CAACU,QAAQ,CAAC1C,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIsC,MAAMN;QAGlB,OAAO;YACLU;YACAC;YACAjB;QACF;IACF;IAEA,MAAM,SACJkB,MAA+B,EAC/B5C,GAEC,EACwD;QACzDI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOwC,QAAQ;QACf,MAAM9B,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAC9C,MAAM,EAAE+B,gBAAgB,EAAEC,IAAI,EAAE,GAAGhC;QACnCV,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOyC,kBAAkB;QAEzB,MAAMlC,mBAAsC;YAAE,QAAQ;QAAY;QAClE,MAAMoC,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIlD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMe,aAAawC,AAAAA,IAAAA,0BAAAA,gBAAAA,AAAAA,EACjBL,YACApC,QAAQ,IAAI,EACZH;YAEFf,MAAM,4BAA4BmB;YAClCsC,eAAe,MAAMG,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EACnBH,cACAtC,YACA0C,AAAAA,IAAAA,oBAAAA,cAAAA,AAAAA,EAAe9C;QAEnB;QAEA,MAAM+C,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASX;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMM,WACJ,IAAI,CAAC,UAAU,IAAIC,yBAAAA,mBAAmBA;QAExC,MAAMC,MAAM,MAAMF,SAChBD,MACAI,0BAAAA,YAAAA,CAAAA,gBAA6B,EAC7BnD;QAGF,MAAM,EAAEoD,OAAO,EAAE,GAAGF;QACpBzD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAAC2D,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1D3D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO2D,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IA1UA,YACEjD,OAEmE,EACnEd,GAAoB,CACpB;QAfF;QAIA,qCAAoDgE,0BAAAA,QAAQA;QAE5D;QAEA;QAQE5D,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOU,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMmD,QAAQ,OAAO,CAACnD;QAGlD,IAAI,AAA2B,WAApBd,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAwTF"}
+{"version":3,"file":"insight/index.js","sources":["webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/insight/index.ts"],"sourcesContent":["__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","import { AIActionType, type AIArgs, expandSearchArea } from '@/ai-model/common';\nimport {\n  AiExtractElementInfo,\n  AiLocateElement,\n  callAIWithObjectResponse,\n} from '@/ai-model/index';\nimport { AiLocateSection } from '@/ai-model/inspect';\nimport { elementDescriberInstruction } from '@/ai-model/prompt/describe';\nimport type {\n  AIDescribeElementResponse,\n  AIUsageInfo,\n  BaseElement,\n  DetailedLocateParam,\n  DumpSubscriber,\n  InsightAction,\n  InsightExtractOption,\n  InsightExtractParam,\n  InsightTaskInfo,\n  LocateResult,\n  PartialInsightDumpFromSDK,\n  Rect,\n  UIContext,\n} from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_FORCE_DEEP_THINK,\n  globalConfigManager,\n} from '@midscene/shared/env';\nimport { compositeElementInfoImg, cropByRect } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert } from '@midscene/shared/utils';\nimport type { TMultimodalPrompt } from '../ai-model/common';\nimport { emitInsightDump } from './utils';\n\nexport interface LocateOpts {\n  context?: UIContext<BaseElement>;\n}\n\nexport type AnyValue<T> = {\n  [K in keyof T]: unknown extends T[K] ? any : T[K];\n};\n\ninterface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: typeof callAIWithObjectResponse;\n}\n\nconst debug = getDebug('ai:insight');\nexport default class Insight<\n  ElementType extends BaseElement = BaseElement,\n  ContextType extends UIContext<ElementType> = UIContext<ElementType>,\n> {\n  contextRetrieverFn: (\n    action: InsightAction,\n  ) => Promise<ContextType> | ContextType;\n\n  aiVendorFn: Exclude<InsightOptions['aiVendorFn'], undefined> =\n    callAIWithObjectResponse;\n\n  onceDumpUpdatedFn?: DumpSubscriber;\n\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n\n  constructor(\n    context:\n      | ContextType\n      | ((action: InsightAction) => Promise<ContextType> | ContextType),\n    opt?: InsightOptions,\n  ) {\n    assert(context, 'context is required for Insight');\n    if (typeof context === 'function') {\n      this.contextRetrieverFn = context;\n    } else {\n      this.contextRetrieverFn = () => Promise.resolve(context);\n    }\n\n    // just for unit test, aiVendorFn is callAIWithObjectResponse by default\n    if (typeof opt?.aiVendorFn !== 'undefined') {\n      this.aiVendorFn = opt.aiVendorFn;\n    }\n    if (typeof opt?.taskInfo !== 'undefined') {\n      this.taskInfo = opt.taskInfo;\n    }\n  }\n\n  async locate(\n    query: DetailedLocateParam,\n    opt: LocateOpts,\n    modelConfig: IModelConfig,\n  ): Promise<LocateResult> {\n    const queryPrompt = typeof query === 'string' ? query : query.prompt;\n    assert(queryPrompt, 'query is required for locate');\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    assert(typeof query === 'object', 'query should be an object for locate');\n\n    const globalDeepThinkSwitch = globalConfigManager.getEnvConfigInBoolean(\n      MIDSCENE_FORCE_DEEP_THINK,\n    );\n    if (globalDeepThinkSwitch) {\n      debug('globalDeepThinkSwitch', globalDeepThinkSwitch);\n    }\n    let searchAreaPrompt;\n    if (query.deepThink || globalDeepThinkSwitch) {\n      searchAreaPrompt = query.prompt;\n    }\n\n    const { vlMode } = modelConfig;\n\n    if (searchAreaPrompt && !vlMode) {\n      console.warn(\n        'The \"deepThink\" feature is not supported with multimodal LLM. Please config VL model for Midscene. https://midscenejs.com/choose-a-model',\n      );\n      searchAreaPrompt = undefined;\n    }\n\n    const context = opt?.context || (await this.contextRetrieverFn('locate'));\n\n    let searchArea: Rect | undefined = undefined;\n    let searchAreaRawResponse: string | undefined = undefined;\n    let searchAreaUsage: AIUsageInfo | undefined = undefined;\n    let searchAreaResponse:\n      | Awaited<ReturnType<typeof AiLocateSection>>\n      | undefined = undefined;\n    if (searchAreaPrompt) {\n      searchAreaResponse = await AiLocateSection({\n        context,\n        sectionDescription: searchAreaPrompt,\n        modelConfig,\n      });\n      assert(\n        searchAreaResponse.rect,\n        `cannot find search area for \"${searchAreaPrompt}\"${\n          searchAreaResponse.error ? `: ${searchAreaResponse.error}` : ''\n        }`,\n      );\n      searchAreaRawResponse = searchAreaResponse.rawResponse;\n      searchAreaUsage = searchAreaResponse.usage;\n      searchArea = searchAreaResponse.rect;\n    }\n\n    const startTime = Date.now();\n    const {\n      parseResult,\n      rect,\n      elementById,\n      rawResponse,\n      usage,\n      isOrderSensitive,\n    } = await AiLocateElement({\n      callAIFn: this.aiVendorFn,\n      context,\n      targetElementDescription: queryPrompt,\n      searchConfig: searchAreaResponse,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(rawResponse),\n      formatResponse: JSON.stringify(parseResult),\n      usage,\n      searchArea,\n      searchAreaRawResponse,\n      searchAreaUsage,\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI model failed to locate: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'locate',\n      userQuery: {\n        element: queryPrompt,\n      },\n      matchedElement: [],\n      matchedRect: rect,\n      data: null,\n      taskInfo,\n      deepThink: !!searchArea,\n      error: errorLog,\n    };\n\n    const elements: BaseElement[] = [];\n    (parseResult.elements || []).forEach((item) => {\n      if ('id' in item) {\n        const element = elementById(item?.id);\n\n        if (!element) {\n          console.warn(\n            `locate: cannot find element id=${item.id}. Maybe an unstable response from AI model`,\n          );\n          return;\n        }\n        elements.push(element);\n      }\n    });\n\n    emitInsightDump(\n      {\n        ...dumpData,\n        matchedElement: elements,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog) {\n      throw new Error(errorLog);\n    }\n\n    assert(\n      elements.length <= 1,\n      `locate: multiple elements found, length = ${elements.length}`,\n    );\n\n    if (elements.length === 1) {\n      return {\n        element: {\n          id: elements[0]!.id,\n          indexId: elements[0]!.indexId,\n          center: elements[0]!.center,\n          rect: elements[0]!.rect,\n          xpaths: elements[0]!.xpaths || [],\n          attributes: elements[0]!.attributes,\n          isOrderSensitive,\n        },\n        rect,\n      };\n    }\n    return {\n      element: null,\n      rect,\n    };\n  }\n\n  async extract<T>(\n    dataDemand: InsightExtractParam,\n    modelConfig: IModelConfig,\n    opt?: InsightExtractOption,\n    multimodalPrompt?: TMultimodalPrompt,\n  ): Promise<{\n    data: T;\n    thought?: string;\n    usage?: AIUsageInfo;\n  }> {\n    assert(\n      typeof dataDemand === 'object' || typeof dataDemand === 'string',\n      `dataDemand should be object or string, but get ${typeof dataDemand}`,\n    );\n    const dumpSubscriber = this.onceDumpUpdatedFn;\n    this.onceDumpUpdatedFn = undefined;\n\n    const context = await this.contextRetrieverFn('extract');\n\n    const startTime = Date.now();\n\n    const { parseResult, usage } = await AiExtractElementInfo<T>({\n      context,\n      dataQuery: dataDemand,\n      multimodalPrompt,\n      extractOption: opt,\n      modelConfig,\n    });\n\n    const timeCost = Date.now() - startTime;\n    const taskInfo: InsightTaskInfo = {\n      ...(this.taskInfo ? this.taskInfo : {}),\n      durationMs: timeCost,\n      rawResponse: JSON.stringify(parseResult),\n    };\n\n    let errorLog: string | undefined;\n    if (parseResult.errors?.length) {\n      errorLog = `AI response error: \\n${parseResult.errors.join('\\n')}`;\n    }\n\n    const dumpData: PartialInsightDumpFromSDK = {\n      type: 'extract',\n      userQuery: {\n        dataDemand,\n      },\n      matchedElement: [],\n      data: null,\n      taskInfo,\n      error: errorLog,\n    };\n\n    const { data, thought } = parseResult || {};\n\n    // 4\n    emitInsightDump(\n      {\n        ...dumpData,\n        data,\n      },\n      dumpSubscriber,\n    );\n\n    if (errorLog && !data && !opt?.doNotThrowError) {\n      throw new Error(errorLog);\n    }\n\n    return {\n      data,\n      thought,\n      usage,\n    };\n  }\n\n  async describe(\n    target: Rect | [number, number],\n    modelConfig: IModelConfig,\n    opt?: {\n      deepThink?: boolean;\n    },\n  ): Promise<Pick<AIDescribeElementResponse, 'description'>> {\n    assert(target, 'target is required for insight.describe');\n    const context = await this.contextRetrieverFn('describe');\n    const { screenshotBase64, size } = context;\n    assert(screenshotBase64, 'screenshot is required for insight.describe');\n    // The result of the \"describe\" function will be used for positioning, so essentially it is a form of grounding.\n    const { vlMode } = modelConfig;\n    const systemPrompt = elementDescriberInstruction();\n\n    // Convert [x,y] center point to Rect if needed\n    const defaultRectSize = 30;\n    const targetRect: Rect = Array.isArray(target)\n      ? {\n          left: Math.floor(target[0] - defaultRectSize / 2),\n          top: Math.floor(target[1] - defaultRectSize / 2),\n          width: defaultRectSize,\n          height: defaultRectSize,\n        }\n      : target;\n\n    let imagePayload = await compositeElementInfoImg({\n      inputImgBase64: screenshotBase64,\n      size,\n      elementsPositionInfo: [\n        {\n          rect: targetRect,\n        },\n      ],\n      borderThickness: 3,\n    });\n\n    if (opt?.deepThink) {\n      const searchArea = expandSearchArea(targetRect, context.size, vlMode);\n      debug('describe: set searchArea', searchArea);\n      imagePayload = await cropByRect(\n        imagePayload,\n        searchArea,\n        vlMode === 'qwen-vl',\n      );\n    }\n\n    const msgs: AIArgs = [\n      { role: 'system', content: systemPrompt },\n      {\n        role: 'user',\n        content: [\n          {\n            type: 'image_url',\n            image_url: {\n              url: imagePayload,\n              detail: 'high',\n            },\n          },\n        ],\n      },\n    ];\n\n    const callAIFn = this\n      .aiVendorFn as typeof callAIWithObjectResponse<AIDescribeElementResponse>;\n\n    const res = await callAIFn(\n      msgs,\n      AIActionType.DESCRIBE_ELEMENT,\n      modelConfig,\n    );\n\n    const { content } = res;\n    assert(!content.error, `describe failed: ${content.error}`);\n    assert(content.description, 'failed to describe the element');\n    return content;\n  }\n}\n"],"names":["__webpack_require__","definition","key","Object","obj","prop","Symbol","debug","getDebug","Insight","query","opt","modelConfig","_parseResult_errors","queryPrompt","assert","dumpSubscriber","undefined","globalDeepThinkSwitch","globalConfigManager","MIDSCENE_FORCE_DEEP_THINK","searchAreaPrompt","vlMode","console","context","searchArea","searchAreaRawResponse","searchAreaUsage","searchAreaResponse","AiLocateSection","startTime","Date","parseResult","rect","elementById","rawResponse","usage","isOrderSensitive","AiLocateElement","timeCost","taskInfo","JSON","errorLog","dumpData","elements","item","element","emitInsightDump","Error","dataDemand","multimodalPrompt","AiExtractElementInfo","data","thought","target","screenshotBase64","size","systemPrompt","elementDescriberInstruction","defaultRectSize","targetRect","Array","Math","imagePayload","compositeElementInfoImg","expandSearchArea","cropByRect","msgs","callAIFn","res","AIActionType","content","callAIWithObjectResponse","Promise"],"mappings":";;;IAAAA,oBAAoB,CAAC,GAAG,CAAC,UAASC;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGD,oBAAoB,CAAC,CAACC,YAAYC,QAAQ,CAACF,oBAAoB,CAAC,CAAC,UAASE,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAF,oBAAoB,CAAC,GAAG,CAACI,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFL,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOM,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;;;;;;;;ACyCA,MAAMI,QAAQC,AAAAA,IAAAA,uBAAAA,QAAAA,AAAAA,EAAS;AACR,MAAMC;IAqCnB,MAAM,OACJC,KAA0B,EAC1BC,GAAe,EACfC,WAAyB,EACF;YAkFnBC;QAjFJ,MAAMC,cAAc,AAAiB,YAAjB,OAAOJ,QAAqBA,QAAQA,MAAM,MAAM;QACpEK,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOD,aAAa;QACpB,MAAME,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzBF,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,AAAiB,YAAjB,OAAOL,OAAoB;QAElC,MAAMQ,wBAAwBC,oBAAAA,mBAAAA,CAAAA,qBAAyC,CACrEC,oBAAAA,yBAAyBA;QAE3B,IAAIF,uBACFX,MAAM,yBAAyBW;QAEjC,IAAIG;QACJ,IAAIX,MAAM,SAAS,IAAIQ,uBACrBG,mBAAmBX,MAAM,MAAM;QAGjC,MAAM,EAAEY,MAAM,EAAE,GAAGV;QAEnB,IAAIS,oBAAoB,CAACC,QAAQ;YAC/BC,QAAQ,IAAI,CACV;YAEFF,mBAAmBJ;QACrB;QAEA,MAAMO,UAAUb,AAAAA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,OAAO,AAAD,KAAM,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE/D,IAAIc;QACJ,IAAIC;QACJ,IAAIC;QACJ,IAAIC;QAGJ,IAAIP,kBAAkB;YACpBO,qBAAqB,MAAMC,AAAAA,IAAAA,2BAAAA,eAAAA,AAAAA,EAAgB;gBACzCL;gBACA,oBAAoBH;gBACpBT;YACF;YACAG,IAAAA,sBAAAA,MAAAA,AAAAA,EACEa,mBAAmB,IAAI,EACvB,CAAC,6BAA6B,EAAEP,iBAAiB,CAAC,EAChDO,mBAAmB,KAAK,GAAG,CAAC,EAAE,EAAEA,mBAAmB,KAAK,EAAE,GAAG,IAC7D;YAEJF,wBAAwBE,mBAAmB,WAAW;YACtDD,kBAAkBC,mBAAmB,KAAK;YAC1CH,aAAaG,mBAAmB,IAAI;QACtC;QAEA,MAAME,YAAYC,KAAK,GAAG;QAC1B,MAAM,EACJC,WAAW,EACXC,IAAI,EACJC,WAAW,EACXC,WAAW,EACXC,KAAK,EACLC,gBAAgB,EACjB,GAAG,MAAMC,AAAAA,IAAAA,yBAAAA,eAAAA,AAAAA,EAAgB;YACxB,UAAU,IAAI,CAAC,UAAU;YACzBd;YACA,0BAA0BV;YAC1B,cAAcc;YACdhB;QACF;QAEA,MAAM2B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACN;YAC5B,gBAAgBM,KAAK,SAAS,CAACT;YAC/BI;YACAX;YACAC;YACAC;QACF;QAEA,IAAIe;QACJ,IAAI,QAAA7B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B6B,WAAW,CAAC,6BAA6B,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAG5E,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACT,SAAS7B;YACX;YACA,gBAAgB,EAAE;YAClB,aAAamB;YACb,MAAM;YACNO;YACA,WAAW,CAAC,CAACf;YACb,OAAOiB;QACT;QAEA,MAAME,WAA0B,EAAE;QACjCZ,CAAAA,YAAY,QAAQ,IAAI,EAAC,EAAG,OAAO,CAAC,CAACa;YACpC,IAAI,QAAQA,MAAM;gBAChB,MAAMC,UAAUZ,YAAYW,QAAAA,OAAAA,KAAAA,IAAAA,KAAM,EAAE;gBAEpC,IAAI,CAACC,SAAS,YACZvB,QAAQ,IAAI,CACV,CAAC,+BAA+B,EAAEsB,KAAK,EAAE,CAAC,0CAA0C,CAAC;gBAIzFD,SAAS,IAAI,CAACE;YAChB;QACF;QAEAC,IAAAA,kCAAAA,eAAAA,AAAAA,EACE;YACE,GAAGJ,QAAQ;YACX,gBAAgBC;QAClB,GACA5B;QAGF,IAAI0B,UACF,MAAM,IAAIM,MAAMN;QAGlB3B,IAAAA,sBAAAA,MAAAA,AAAAA,EACE6B,SAAS,MAAM,IAAI,GACnB,CAAC,0CAA0C,EAAEA,SAAS,MAAM,EAAE;QAGhE,IAAIA,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACL,SAAS;gBACP,IAAIA,QAAQ,CAAC,EAAE,CAAE,EAAE;gBACnB,SAASA,QAAQ,CAAC,EAAE,CAAE,OAAO;gBAC7B,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM;gBAC3B,MAAMA,QAAQ,CAAC,EAAE,CAAE,IAAI;gBACvB,QAAQA,QAAQ,CAAC,EAAE,CAAE,MAAM,IAAI,EAAE;gBACjC,YAAYA,QAAQ,CAAC,EAAE,CAAE,UAAU;gBACnCP;YACF;YACAJ;QACF;QAEF,OAAO;YACL,SAAS;YACTA;QACF;IACF;IAEA,MAAM,QACJgB,UAA+B,EAC/BrC,WAAyB,EACzBD,GAA0B,EAC1BuC,gBAAoC,EAKnC;YA4BGrC;QA3BJE,IAAAA,sBAAAA,MAAAA,AAAAA,EACE,AAAsB,YAAtB,OAAOkC,cAA2B,AAAsB,YAAtB,OAAOA,YACzC,CAAC,+CAA+C,EAAE,OAAOA,YAAY;QAEvE,MAAMjC,iBAAiB,IAAI,CAAC,iBAAiB;QAC7C,IAAI,CAAC,iBAAiB,GAAGC;QAEzB,MAAMO,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAE9C,MAAMM,YAAYC,KAAK,GAAG;QAE1B,MAAM,EAAEC,WAAW,EAAEI,KAAK,EAAE,GAAG,MAAMe,AAAAA,IAAAA,yBAAAA,oBAAAA,AAAAA,EAAwB;YAC3D3B;YACA,WAAWyB;YACXC;YACA,eAAevC;YACfC;QACF;QAEA,MAAM2B,WAAWR,KAAK,GAAG,KAAKD;QAC9B,MAAMU,WAA4B;YAChC,GAAI,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC;YACtC,YAAYD;YACZ,aAAaE,KAAK,SAAS,CAACT;QAC9B;QAEA,IAAIU;QACJ,IAAI,QAAA7B,CAAAA,sBAAAA,YAAY,MAAM,AAAD,IAAjBA,KAAAA,IAAAA,oBAAoB,MAAM,EAC5B6B,WAAW,CAAC,qBAAqB,EAAEV,YAAY,MAAM,CAAC,IAAI,CAAC,OAAO;QAGpE,MAAMW,WAAsC;YAC1C,MAAM;YACN,WAAW;gBACTM;YACF;YACA,gBAAgB,EAAE;YAClB,MAAM;YACNT;YACA,OAAOE;QACT;QAEA,MAAM,EAAEU,IAAI,EAAEC,OAAO,EAAE,GAAGrB,eAAe,CAAC;QAG1Ce,IAAAA,kCAAAA,eAAAA,AAAAA,EACE;YACE,GAAGJ,QAAQ;YACXS;QACF,GACApC;QAGF,IAAI0B,YAAY,CAACU,QAAQ,CAACzC,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,eAAe,AAAD,GAC3C,MAAM,IAAIqC,MAAMN;QAGlB,OAAO;YACLU;YACAC;YACAjB;QACF;IACF;IAEA,MAAM,SACJkB,MAA+B,EAC/B1C,WAAyB,EACzBD,GAEC,EACwD;QACzDI,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOuC,QAAQ;QACf,MAAM9B,UAAU,MAAM,IAAI,CAAC,kBAAkB,CAAC;QAC9C,MAAM,EAAE+B,gBAAgB,EAAEC,IAAI,EAAE,GAAGhC;QACnCT,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOwC,kBAAkB;QAEzB,MAAM,EAAEjC,MAAM,EAAE,GAAGV;QACnB,MAAM6C,eAAeC,AAAAA,IAAAA,4BAAAA,2BAAAA,AAAAA;QAGrB,MAAMC,kBAAkB;QACxB,MAAMC,aAAmBC,MAAM,OAAO,CAACP,UACnC;YACE,MAAMQ,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC/C,KAAKG,KAAK,KAAK,CAACR,MAAM,CAAC,EAAE,GAAGK,kBAAkB;YAC9C,OAAOA;YACP,QAAQA;QACV,IACAL;QAEJ,IAAIS,eAAe,MAAMC,AAAAA,IAAAA,oBAAAA,uBAAAA,AAAAA,EAAwB;YAC/C,gBAAgBT;YAChBC;YACA,sBAAsB;gBACpB;oBACE,MAAMI;gBACR;aACD;YACD,iBAAiB;QACnB;QAEA,IAAIjD,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,SAAS,EAAE;YAClB,MAAMc,aAAawC,AAAAA,IAAAA,0BAAAA,gBAAAA,AAAAA,EAAiBL,YAAYpC,QAAQ,IAAI,EAAEF;YAC9Df,MAAM,4BAA4BkB;YAClCsC,eAAe,MAAMG,AAAAA,IAAAA,oBAAAA,UAAAA,AAAAA,EACnBH,cACAtC,YACAH,AAAW,cAAXA;QAEJ;QAEA,MAAM6C,OAAe;YACnB;gBAAE,MAAM;gBAAU,SAASV;YAAa;YACxC;gBACE,MAAM;gBACN,SAAS;oBACP;wBACE,MAAM;wBACN,WAAW;4BACT,KAAKM;4BACL,QAAQ;wBACV;oBACF;iBACD;YACH;SACD;QAED,MAAMK,WAAW,IAAI,CAClB,UAAU;QAEb,MAAMC,MAAM,MAAMD,SAChBD,MACAG,0BAAAA,YAAAA,CAAAA,gBAA6B,EAC7B1D;QAGF,MAAM,EAAE2D,OAAO,EAAE,GAAGF;QACpBtD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAO,CAACwD,QAAQ,KAAK,EAAE,CAAC,iBAAiB,EAAEA,QAAQ,KAAK,EAAE;QAC1DxD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOwD,QAAQ,WAAW,EAAE;QAC5B,OAAOA;IACT;IAvUA,YACE/C,OAEmE,EACnEb,GAAoB,CACpB;QAhBF;QAIA,qCACE6D,yBAAAA,wBAAwBA;QAE1B;QAEA;QAQEzD,IAAAA,sBAAAA,MAAAA,AAAAA,EAAOS,SAAS;QAChB,IAAI,AAAmB,cAAnB,OAAOA,SACT,IAAI,CAAC,kBAAkB,GAAGA;aAE1B,IAAI,CAAC,kBAAkB,GAAG,IAAMiD,QAAQ,OAAO,CAACjD;QAIlD,IAAI,AAA2B,WAApBb,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,UAAU,AAAD,GACvB,IAAI,CAAC,UAAU,GAAGA,IAAI,UAAU;QAElC,IAAI,AAAyB,WAAlBA,CAAAA,QAAAA,MAAAA,KAAAA,IAAAA,IAAK,QAAQ,AAAD,GACrB,IAAI,CAAC,QAAQ,GAAGA,IAAI,QAAQ;IAEhC;AAoTF"}

package/dist/lib/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/types.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { TModelConfigFn } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\nexport interface AIElementLocatorResponse {\n  elements: {\n    id: string;\n    reason?: string;\n    text?: string;\n    xpaths?: string[];\n  }[];\n  bbox?: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse =\n  | AIElementLocatorResponse\n  | AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext<ElementType extends BaseElement = BaseElement> {\n  abstract screenshotBase64: string;\n\n  abstract tree: ElementTreeNode<ElementType>;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\n/**\n * insight\n */\n\nexport type CallAIFn = <T>(\n  messages: ChatCompletionMessageParam[],\n) => Promise<T>;\n\nexport interface InsightOptions {\n  taskInfo?: Omit<InsightTaskInfo, 'durationMs'>;\n  aiVendorFn?: CallAIFn;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type LocateResultElement = {\n  center: [number, number];\n  rect: Rect;\n  id: string;\n  indexId?: number;\n  xpaths: string[];\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n  isOrderSensitive?: boolean;\n};\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  sdkVersion: string;\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: BaseElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'sdkVersion' | 'logTime' | 'logId' | 'model_name'\n>;\n\nexport type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  id?: string;\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface PlanningAIResponse {\n  action?: PlanningAction; // this is the qwen mode\n  actions?: PlanningAction[];\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport type PlanningActionParamTap = null;\nexport type PlanningActionParamHover = null;\nexport type PlanningActionParamRightClick = null;\n\nexport interface PlanningActionParamInputOrKeyPress {\n  value: string;\n  autoDismissKeyboard?: boolean;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface AndroidLongPressParam {\n  duration?: number;\n}\n\nexport interface AndroidPullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskInsightDumpLog {\n  dump?: InsightDump;\n}\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    log?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext<WebElementInfo>;\n\n/**\n * Agent\n */\n\nexport interface AgentOpt {\n  testId?: string;\n  cacheId?: string;\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","AIResponseFormat","UIContext"],"mappings":";;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ICgCO,IAAKI,mBAAAA,WAAAA,GAAAA,SAAAA,gBAAgB;;;eAAhBA;;IAwFL,MAAeC;IAQtB"}
+{"version":3,"file":"types.js","sources":["webpack://@midscene/core/webpack/runtime/compat_get_default_export","webpack://@midscene/core/webpack/runtime/define_property_getters","webpack://@midscene/core/webpack/runtime/has_own_property","webpack://@midscene/core/webpack/runtime/make_namespace_object","webpack://@midscene/core/./src/types.ts"],"sourcesContent":["// getDefaultExport function for compatibility with non-ESM modules\n__webpack_require__.n = (module) => {\n\tvar getter = module && module.__esModule ?\n\t\t() => (module['default']) :\n\t\t() => (module);\n\t__webpack_require__.d(getter, { a: getter });\n\treturn getter;\n};\n","__webpack_require__.d = (exports, definition) => {\n\tfor(var key in definition) {\n        if(__webpack_require__.o(definition, key) && !__webpack_require__.o(exports, key)) {\n            Object.defineProperty(exports, key, { enumerable: true, get: definition[key] });\n        }\n    }\n};","__webpack_require__.o = (obj, prop) => (Object.prototype.hasOwnProperty.call(obj, prop))","// define __esModule on exports\n__webpack_require__.r = (exports) => {\n\tif(typeof Symbol !== 'undefined' && Symbol.toStringTag) {\n\t\tObject.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });\n\t}\n\tObject.defineProperty(exports, '__esModule', { value: true });\n};","/* eslint-disable @typescript-eslint/no-explicit-any */\n\nimport type { NodeType } from '@midscene/shared/constants';\nimport type { TModelConfigFn } from '@midscene/shared/env';\nimport type {\n  BaseElement,\n  ElementTreeNode,\n  Rect,\n  Size,\n} from '@midscene/shared/types';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { z } from 'zod';\nimport type { TUserPrompt } from './ai-model/common';\nimport type { DetailedLocateParam, MidsceneYamlFlowItem } from './yaml';\n\nexport type {\n  ElementTreeNode,\n  BaseElement,\n  Rect,\n  Size,\n  Point,\n} from '@midscene/shared/types';\nexport * from './yaml';\n\nexport type AIUsageInfo = Record<string, any> & {\n  prompt_tokens: number | undefined;\n  completion_tokens: number | undefined;\n  total_tokens: number | undefined;\n  time_cost: number | undefined;\n  model_name: string | undefined;\n  model_description: string | undefined;\n  intent: string | undefined;\n};\n\n/**\n * openai\n *\n */\nexport enum AIResponseFormat {\n  JSON = 'json_object',\n  TEXT = 'text',\n}\n\nexport type AISingleElementResponseById = {\n  id: string;\n  reason?: string;\n  text?: string;\n  xpaths?: string[];\n};\n\nexport type AISingleElementResponseByPosition = {\n  position?: {\n    x: number;\n    y: number;\n  };\n  bbox?: [number, number, number, number];\n  reason: string;\n  text: string;\n};\n\nexport type AISingleElementResponse = AISingleElementResponseById;\nexport interface AIElementLocatorResponse {\n  elements: {\n    id: string;\n    reason?: string;\n    text?: string;\n    xpaths?: string[];\n  }[];\n  bbox?: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport interface AIElementCoordinatesResponse {\n  bbox: [number, number, number, number];\n  isOrderSensitive?: boolean;\n  errors?: string[];\n}\n\nexport type AIElementResponse =\n  | AIElementLocatorResponse\n  | AIElementCoordinatesResponse;\n\nexport interface AIDataExtractionResponse<DataDemand> {\n  data: DataDemand;\n  errors?: string[];\n  thought?: string;\n}\n\nexport interface AISectionLocatorResponse {\n  bbox: [number, number, number, number];\n  references_bbox?: [number, number, number, number][];\n  error?: string;\n}\n\nexport interface AIAssertionResponse {\n  pass: boolean;\n  thought: string;\n}\n\nexport interface AIDescribeElementResponse {\n  description: string;\n  error?: string;\n}\n\nexport interface LocatorValidatorOption {\n  centerDistanceThreshold?: number;\n}\n\nexport interface LocateValidatorResult {\n  pass: boolean;\n  rect: Rect;\n  center: [number, number];\n  centerDistance?: number;\n}\n\nexport interface AgentDescribeElementAtPointResult {\n  prompt: string;\n  deepThink: boolean;\n  verifyResult?: LocateValidatorResult;\n}\n\n/**\n * context\n */\n\nexport abstract class UIContext<ElementType extends BaseElement = BaseElement> {\n  abstract screenshotBase64: string;\n\n  abstract tree: ElementTreeNode<ElementType>;\n\n  abstract size: Size;\n\n  abstract _isFrozen?: boolean;\n}\n\nexport type EnsureObject<T> = { [K in keyof T]: any };\n\nexport type InsightAction = 'locate' | 'extract' | 'assert' | 'describe';\n\nexport type InsightExtractParam = string | Record<string, string>;\n\nexport type LocateResultElement = {\n  center: [number, number];\n  rect: Rect;\n  id: string;\n  indexId?: number;\n  xpaths: string[];\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n  isOrderSensitive?: boolean;\n};\n\nexport interface LocateResult {\n  element: LocateResultElement | null;\n  rect?: Rect;\n}\n\nexport interface InsightTaskInfo {\n  durationMs: number;\n  formatResponse?: string;\n  rawResponse?: string;\n  usage?: AIUsageInfo;\n  searchArea?: Rect;\n  searchAreaRawResponse?: string;\n  searchAreaUsage?: AIUsageInfo;\n}\n\nexport interface DumpMeta {\n  sdkVersion: string;\n  logTime: number;\n}\n\nexport interface ReportDumpWithAttributes {\n  dumpString: string;\n  attributes?: Record<string, any>;\n}\n\nexport interface InsightDump extends DumpMeta {\n  type: 'locate' | 'extract' | 'assert';\n  logId: string;\n  userQuery: {\n    element?: TUserPrompt;\n    dataDemand?: InsightExtractParam;\n    assertion?: TUserPrompt;\n  };\n  matchedElement: BaseElement[];\n  matchedRect?: Rect;\n  deepThink?: boolean;\n  data: any;\n  assertionPass?: boolean;\n  assertionThought?: string;\n  taskInfo: InsightTaskInfo;\n  error?: string;\n  output?: any;\n}\n\nexport type PartialInsightDumpFromSDK = Omit<\n  InsightDump,\n  'sdkVersion' | 'logTime' | 'logId' | 'model_name'\n>;\n\nexport type DumpSubscriber = (dump: InsightDump) => Promise<void> | void;\n\n// intermediate variables to optimize the return value by AI\nexport interface LiteUISection {\n  name: string;\n  description: string;\n  sectionCharacteristics: string;\n  textIds: string[];\n}\n\nexport type ElementById = (id: string) => BaseElement | null;\n\nexport type InsightAssertionResponse = AIAssertionResponse & {\n  usage?: AIUsageInfo;\n};\n\n/**\n * agent\n */\n\nexport type OnTaskStartTip = (tip: string) => Promise<void> | void;\n\nexport interface AgentWaitForOpt {\n  checkIntervalMs?: number;\n  timeoutMs?: number;\n}\n\nexport interface AgentAssertOpt {\n  keepRawResponse?: boolean;\n}\n\n/**\n * planning\n *\n */\n\nexport interface PlanningLocateParam extends DetailedLocateParam {\n  id?: string;\n  bbox?: [number, number, number, number];\n}\n\nexport interface PlanningAction<ParamType = any> {\n  thought?: string;\n  type: string;\n  param: ParamType;\n  locate?: PlanningLocateParam | null;\n}\n\nexport interface PlanningAIResponse {\n  action?: PlanningAction; // this is the qwen mode\n  actions?: PlanningAction[];\n  more_actions_needed_by_instruction: boolean;\n  log: string;\n  sleep?: number;\n  error?: string;\n  usage?: AIUsageInfo;\n  rawResponse?: string;\n  yamlFlow?: MidsceneYamlFlowItem[];\n  yamlString?: string;\n}\n\nexport type PlanningActionParamTap = null;\nexport type PlanningActionParamHover = null;\nexport type PlanningActionParamRightClick = null;\n\nexport interface PlanningActionParamInputOrKeyPress {\n  value: string;\n  autoDismissKeyboard?: boolean;\n}\n\nexport interface PlanningActionParamSleep {\n  timeMs: number;\n}\n\nexport interface PlanningActionParamError {\n  thought: string;\n}\n\nexport type PlanningActionParamWaitFor = AgentWaitForOpt & {};\n\nexport interface AndroidLongPressParam {\n  duration?: number;\n}\n\nexport interface AndroidPullParam {\n  direction: 'up' | 'down';\n  distance?: number;\n  duration?: number;\n}\n/**\n * misc\n */\n\nexport interface Color {\n  name: string;\n  hex: string;\n}\n\nexport interface BaseAgentParserOpt {\n  selector?: string;\n}\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PuppeteerParserOpt extends BaseAgentParserOpt {}\n\n// eslint-disable-next-line @typescript-eslint/no-empty-interface\nexport interface PlaywrightParserOpt extends BaseAgentParserOpt {}\n\n/*\naction\n*/\nexport interface ExecutionTaskProgressOptions {\n  onTaskStart?: (task: ExecutionTask) => Promise<void> | void;\n}\n\nexport interface ExecutionRecorderItem {\n  type: 'screenshot';\n  ts: number;\n  screenshot?: string;\n  timing?: string;\n}\n\nexport type ExecutionTaskType =\n  | 'Planning'\n  | 'Insight'\n  | 'Action'\n  | 'Assertion'\n  | 'Log';\n\nexport interface ExecutorContext {\n  task: ExecutionTask;\n  element?: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskApply<\n  Type extends ExecutionTaskType = any,\n  TaskParam = any,\n  TaskOutput = any,\n  TaskLog = any,\n> {\n  type: Type;\n  subType?: string;\n  param?: TaskParam;\n  thought?: string;\n  locate?: PlanningLocateParam | null;\n  uiContext?: UIContext;\n  executor: (\n    param: TaskParam,\n    context: ExecutorContext,\n  ) => // biome-ignore lint/suspicious/noConfusingVoidType: <explanation>\n    | Promise<ExecutionTaskReturn<TaskOutput, TaskLog> | undefined | void>\n    | undefined\n    | void;\n}\n\nexport interface ExecutionTaskHitBy {\n  from: string;\n  context: Record<string, any>;\n}\n\nexport interface ExecutionTaskReturn<TaskOutput = unknown, TaskLog = unknown> {\n  output?: TaskOutput;\n  log?: TaskLog;\n  recorder?: ExecutionRecorderItem[];\n  hitBy?: ExecutionTaskHitBy;\n}\n\nexport type ExecutionTask<\n  E extends ExecutionTaskApply<any, any, any> = ExecutionTaskApply<\n    any,\n    any,\n    any\n  >,\n> = E &\n  ExecutionTaskReturn<\n    E extends ExecutionTaskApply<any, any, infer TaskOutput, any>\n      ? TaskOutput\n      : unknown,\n    E extends ExecutionTaskApply<any, any, any, infer TaskLog>\n      ? TaskLog\n      : unknown\n  > & {\n    status: 'pending' | 'running' | 'finished' | 'failed' | 'cancelled';\n    error?: Error;\n    errorMessage?: string;\n    errorStack?: string;\n    timing?: {\n      start: number;\n      end?: number;\n      cost?: number;\n    };\n    usage?: AIUsageInfo;\n  };\n\nexport interface ExecutionDump extends DumpMeta {\n  name: string;\n  description?: string;\n  tasks: ExecutionTask[];\n}\n\n/*\ntask - insight-locate\n*/\nexport type ExecutionTaskInsightLocateParam = PlanningLocateParam;\n\nexport interface ExecutionTaskInsightLocateOutput {\n  element: LocateResultElement | null;\n}\n\nexport interface ExecutionTaskInsightDumpLog {\n  dump?: InsightDump;\n}\n\nexport type ExecutionTaskInsightLocateApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightLocateParam,\n  ExecutionTaskInsightLocateOutput,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightLocate =\n  ExecutionTask<ExecutionTaskInsightLocateApply>;\n\n/*\ntask - insight-query\n*/\nexport interface ExecutionTaskInsightQueryParam {\n  dataDemand: InsightExtractParam;\n}\n\nexport interface ExecutionTaskInsightQueryOutput {\n  data: any;\n}\n\nexport type ExecutionTaskInsightQueryApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightQueryParam,\n  any,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightQuery =\n  ExecutionTask<ExecutionTaskInsightQueryApply>;\n\n/*\ntask - assertion\n*/\nexport interface ExecutionTaskInsightAssertionParam {\n  assertion: string;\n}\n\nexport type ExecutionTaskInsightAssertionApply = ExecutionTaskApply<\n  'Insight',\n  ExecutionTaskInsightAssertionParam,\n  InsightAssertionResponse,\n  ExecutionTaskInsightDumpLog\n>;\n\nexport type ExecutionTaskInsightAssertion =\n  ExecutionTask<ExecutionTaskInsightAssertionApply>;\n\n/*\ntask - action (i.e. interact) \n*/\nexport type ExecutionTaskActionApply<ActionParam = any> = ExecutionTaskApply<\n  'Action',\n  ActionParam,\n  void,\n  void\n>;\n\nexport type ExecutionTaskAction = ExecutionTask<ExecutionTaskActionApply>;\n\n/*\ntask - Log\n*/\n\nexport type ExecutionTaskLogApply<\n  LogParam = {\n    content: string;\n  },\n> = ExecutionTaskApply<'Log', LogParam, void, void>;\n\nexport type ExecutionTaskLog = ExecutionTask<ExecutionTaskLogApply>;\n\n/*\ntask - planning\n*/\n\nexport type ExecutionTaskPlanningApply = ExecutionTaskApply<\n  'Planning',\n  {\n    userInstruction: string;\n    log?: string;\n  },\n  PlanningAIResponse\n>;\n\nexport type ExecutionTaskPlanning = ExecutionTask<ExecutionTaskPlanningApply>;\n\n/*\nGrouped dump\n*/\nexport interface GroupedActionDump {\n  groupName: string;\n  groupDescription?: string;\n  modelBriefs: string[];\n  executions: ExecutionDump[];\n}\n\nexport type InterfaceType =\n  | 'puppeteer'\n  | 'playwright'\n  | 'static'\n  | 'chrome-extension-proxy'\n  | 'android'\n  | string;\n\nexport interface StreamingCodeGenerationOptions {\n  /** Whether to enable streaming output */\n  stream?: boolean;\n  /** Callback function to handle streaming chunks */\n  onChunk?: StreamingCallback;\n  /** Callback function to handle streaming completion */\n  onComplete?: (finalCode: string) => void;\n  /** Callback function to handle streaming errors */\n  onError?: (error: Error) => void;\n}\n\nexport type StreamingCallback = (chunk: CodeGenerationChunk) => void;\n\nexport interface CodeGenerationChunk {\n  /** The incremental content chunk */\n  content: string;\n  /** The reasoning content */\n  reasoning_content: string;\n  /** The accumulated content so far */\n  accumulated: string;\n  /** Whether this is the final chunk */\n  isComplete: boolean;\n  /** Token usage information if available */\n  usage?: AIUsageInfo;\n}\n\nexport interface StreamingAIResponse {\n  /** The final accumulated content */\n  content: string;\n  /** Token usage information */\n  usage?: AIUsageInfo;\n  /** Whether the response was streamed */\n  isStreamed: boolean;\n}\n\nexport interface DeviceAction<T = any> {\n  name: string;\n  description?: string;\n  interfaceAlias?: string;\n  paramSchema?: z.ZodType<T>;\n  call: (param: T, context: ExecutorContext) => Promise<void> | void;\n}\n\n/**\n * Web-specific types\n */\nexport interface WebElementInfo extends BaseElement {\n  id: string;\n  attributes: {\n    nodeType: NodeType;\n    [key: string]: string;\n  };\n}\n\nexport type WebUIContext = UIContext<WebElementInfo>;\n\n/**\n * Agent\n */\n\nexport interface AgentOpt {\n  testId?: string;\n  cacheId?: string;\n  groupName?: string;\n  groupDescription?: string;\n  /* if auto generate report, default true */\n  generateReport?: boolean;\n  /* if auto print report msg, default true */\n  autoPrintReportMsg?: boolean;\n  onTaskStartTip?: OnTaskStartTip;\n  aiActionContext?: string;\n  /* custom report file name */\n  reportFileName?: string;\n  modelConfig?: TModelConfigFn;\n}\n"],"names":["__webpack_require__","module","getter","definition","key","Object","obj","prop","Symbol","AIResponseFormat","UIContext"],"mappings":";;;;;;;;;;;;;;;;;IACAA,oBAAoB,CAAC,GAAG,CAACC;QACxB,IAAIC,SAASD,UAAUA,OAAO,UAAU,GACvC,IAAOA,MAAM,CAAC,UAAU,GACxB,IAAOA;QACRD,oBAAoB,CAAC,CAACE,QAAQ;YAAE,GAAGA;QAAO;QAC1C,OAAOA;IACR;;;ICPAF,oBAAoB,CAAC,GAAG,CAAC,UAASG;QACjC,IAAI,IAAIC,OAAOD,WACR,IAAGH,oBAAoB,CAAC,CAACG,YAAYC,QAAQ,CAACJ,oBAAoB,CAAC,CAAC,UAASI,MACzEC,OAAO,cAAc,CAAC,UAASD,KAAK;YAAE,YAAY;YAAM,KAAKD,UAAU,CAACC,IAAI;QAAC;IAGzF;;;ICNAJ,oBAAoB,CAAC,GAAG,CAACM,KAAKC,OAAUF,OAAO,SAAS,CAAC,cAAc,CAAC,IAAI,CAACC,KAAKC;;;ICClFP,oBAAoB,CAAC,GAAG,CAAC;QACxB,IAAG,AAAkB,eAAlB,OAAOQ,UAA0BA,OAAO,WAAW,EACrDH,OAAO,cAAc,CAAC,UAASG,OAAO,WAAW,EAAE;YAAE,OAAO;QAAS;QAEtEH,OAAO,cAAc,CAAC,UAAS,cAAc;YAAE,OAAO;QAAK;IAC5D;;;;;;;;;;;;;;;;;;;ICgCO,IAAKI,mBAAAA,WAAAA,GAAAA,SAAAA,gBAAgB;;;eAAhBA;;IAwFL,MAAeC;IAQtB"}