@midscene/core 1.2.2-beta-20260116064919.0 → 1.2.2-beta-20260116083700.0

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["../../../../src/ai-model/service-caller/index.ts"],"sourcesContent":["import {\n  AIResponseFormat,\n  type AIUsageInfo,\n  type DeepThinkOption,\n} from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_LANGFUSE_DEBUG,\n  MIDSCENE_LANGSMITH_DEBUG,\n  MIDSCENE_MODEL_MAX_TOKENS,\n  OPENAI_MAX_TOKENS,\n  type TModelFamily,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport type { AIArgs } from '../../common';\n\nasync function createChatClient({\n  modelConfig,\n}: {\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n  modelFamily: TModelFamily | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    modelFamily,\n    createOpenAIClient,\n    timeout,\n  } = modelConfig;\n\n  let proxyAgent: any = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n\n  // Helper function to sanitize proxy URL for logging (remove credentials)\n  // Uses URL API instead of regex to avoid ReDoS vulnerabilities\n  const sanitizeProxyUrl = (url: string): string => {\n    try {\n      const parsed = new URL(url);\n      if (parsed.username) {\n        // Keep username for debugging, hide password for security\n        parsed.password = '****';\n        return parsed.href;\n      }\n      return url;\n    } catch {\n      // If URL parsing fails, return original URL (will be caught later)\n      return url;\n    }\n  };\n\n  if (httpProxy) {\n    debugProxy('using http proxy', sanitizeProxyUrl(httpProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'HTTP proxy is configured but not supported in browser environment',\n      );\n    } else {\n      // Dynamic import with variable to avoid bundler static analysis\n      const moduleName = 'undici';\n      const { ProxyAgent } = await import(moduleName);\n      proxyAgent = new ProxyAgent({\n        uri: httpProxy,\n        // Note: authentication is handled via the URI (e.g., http://user:pass@proxy.com:8080)\n      });\n    }\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', sanitizeProxyUrl(socksProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'SOCKS proxy is configured but not supported in browser environment',\n      );\n    } else {\n      try {\n        // Dynamic import with variable to avoid bundler static analysis\n        const moduleName = 'fetch-socks';\n        const { socksDispatcher } = await import(moduleName);\n        // Parse SOCKS proxy URL (e.g., socks5://127.0.0.1:1080)\n        const proxyUrl = new URL(socksProxy);\n\n        // Validate hostname\n        if (!proxyUrl.hostname) {\n          throw new Error('SOCKS proxy URL must include a valid hostname');\n        }\n\n        // Validate and parse port\n        const port = Number.parseInt(proxyUrl.port, 10);\n        if (!proxyUrl.port || Number.isNaN(port)) {\n          throw new Error('SOCKS proxy URL must include a valid port');\n        }\n\n        // Parse SOCKS version from protocol\n        const protocol = proxyUrl.protocol.replace(':', '');\n        const socksType =\n          protocol === 'socks4' ? 4 : protocol === 'socks5' ? 5 : 5;\n\n        proxyAgent = socksDispatcher({\n          type: socksType,\n          host: proxyUrl.hostname,\n          port,\n          ...(proxyUrl.username\n            ? {\n                userId: decodeURIComponent(proxyUrl.username),\n                password: decodeURIComponent(proxyUrl.password || ''),\n              }\n            : {}),\n        });\n        debugProxy('socks proxy configured successfully', {\n          type: socksType,\n          host: proxyUrl.hostname,\n          port: port,\n        });\n      } catch (error) {\n        console.error('Failed to configure SOCKS proxy:', error);\n        throw new Error(\n          `Invalid SOCKS proxy URL: ${socksProxy}. Expected format: socks4://host:port, socks5://host:port, or with authentication: socks5://user:pass@host:port`,\n        );\n      }\n    }\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    // Use fetchOptions.dispatcher for fetch-based SDK instead of httpAgent\n    // Note: Type assertion needed due to undici version mismatch between dependencies\n    ...(proxyAgent ? { fetchOptions: { dispatcher: proxyAgent as any } } : {}),\n    ...openaiExtraConfig,\n    ...(typeof timeout === 'number' ? { timeout } : {}),\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  // LangSmith wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langsmithModule = 'langsmith/wrappers';\n    const { wrapOpenAI } = await import(langsmithModule);\n    openai = wrapOpenAI(openai);\n  }\n\n  // Langfuse wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGFUSE_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langfuse is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langfuse wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langfuseModule = 'langfuse';\n    const { observeOpenAI } = await import(langfuseModule);\n    openai = observeOpenAI(openai);\n  }\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n    modelFamily,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: string;\n  reasoning_content?: string;\n  usage?: AIUsageInfo;\n  isStreamed: boolean;\n}> {\n  const {\n    completion,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n    modelFamily,\n  } = await createChatClient({\n    modelConfig,\n  });\n\n  const maxTokens =\n    globalConfigManager.getEnvConfigValueAsNumber(MIDSCENE_MODEL_MAX_TOKENS) ??\n    globalConfigManager.getEnvConfigValueAsNumber(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n  const temperature = modelConfig.temperature ?? 0;\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let accumulatedReasoning = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const buildUsageInfo = (usageData?: OpenAI.CompletionUsage) => {\n    if (!usageData) return undefined;\n\n    const cachedInputTokens = (\n      usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n    )?.prompt_tokens_details?.cached_tokens;\n\n    return {\n      prompt_tokens: usageData.prompt_tokens ?? 0,\n      completion_tokens: usageData.completion_tokens ?? 0,\n      total_tokens: usageData.total_tokens ?? 0,\n      cached_input: cachedInputTokens ?? 0,\n      time_cost: timeCost ?? 0,\n      model_name: modelName,\n      model_description: modelDescription,\n      intent: modelConfig.intent,\n    } satisfies AIUsageInfo;\n  };\n\n  const commonConfig = {\n    temperature,\n    stream: !!isStreaming,\n    max_tokens: maxTokens,\n    ...(vlMode === 'qwen2.5-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n  const {\n    config: deepThinkConfig,\n    debugMessage,\n    warningMessage,\n  } = resolveDeepThinkConfig({\n    deepThink: options?.deepThink,\n    modelFamily,\n  });\n  if (debugMessage) {\n    debugCall(debugMessage);\n  }\n  if (warningMessage) {\n    debugCall(warningMessage);\n    console.warn(warningMessage);\n  }\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          ...commonConfig,\n          ...deepThinkConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          accumulatedReasoning += reasoning_content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: buildUsageInfo(usage),\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}, temperature, ${temperature ?? ''}`,\n      );\n    } else {\n      // Non-streaming with retry logic\n      const retryCount = modelConfig.retryCount ?? 1;\n      const retryInterval = modelConfig.retryInterval ?? 2000;\n      const maxAttempts = retryCount + 1; // retryCount=1 means 2 total attempts (1 initial + 1 retry)\n\n      let lastError: Error | undefined;\n\n      for (let attempt = 1; attempt <= maxAttempts; attempt++) {\n        try {\n          const result = await completion.create({\n            model: modelName,\n            messages,\n            ...commonConfig,\n            ...deepThinkConfig,\n          } as any);\n\n          timeCost = Date.now() - startTime;\n\n          debugProfileStats(\n            `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}, temperature, ${temperature ?? ''}`,\n          );\n\n          debugProfileDetail(\n            `model usage detail: ${JSON.stringify(result.usage)}`,\n          );\n\n          if (!result.choices) {\n            throw new Error(\n              `invalid response from LLM service: ${JSON.stringify(result)}`,\n            );\n          }\n\n          content = result.choices[0].message.content!;\n          if (!content) {\n            throw new Error('empty content from AI model');\n          }\n\n          accumulatedReasoning =\n            (result.choices[0].message as any)?.reasoning_content || '';\n          usage = result.usage;\n          break; // Success, exit retry loop\n        } catch (error) {\n          lastError = error as Error;\n          if (attempt < maxAttempts) {\n            console.warn(\n              `[Midscene] AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n            );\n            await new Promise((resolve) => setTimeout(resolve, retryInterval));\n          }\n        }\n      }\n\n      if (!content) {\n        throw lastError;\n      }\n    }\n\n    debugCall(`response reasoning content: ${accumulatedReasoning}`);\n    debugCall(`response content: ${content}`);\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      } as OpenAI.CompletionUsage;\n    }\n\n    return {\n      content: content || '',\n      reasoning_content: accumulatedReasoning || undefined,\n      usage: buildUsageInfo(usage),\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${e.message}\\nTrouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: T;\n  contentString: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const response = await callAI(messages, modelConfig, {\n    deepThink: options?.deepThink,\n  });\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  assert(\n    typeof jsonContent === 'object',\n    `failed to parse json response from model (${modelConfig.modelName}): ${response.content}`,\n  );\n  return {\n    content: jsonContent,\n    contentString: response.content,\n    usage: response.usage,\n    reasoning_content: response.reasoning_content,\n  };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function resolveDeepThinkConfig({\n  deepThink,\n  modelFamily,\n}: {\n  deepThink?: DeepThinkOption;\n  modelFamily?: TModelFamily;\n}): {\n  config: Record<string, unknown>;\n  debugMessage?: string;\n  warningMessage?: string;\n} {\n  const normalizedDeepThink = deepThink === 'unset' ? undefined : deepThink;\n\n  if (normalizedDeepThink === undefined) {\n    return { config: {}, debugMessage: undefined };\n  }\n\n  if (modelFamily === 'qwen3-vl') {\n    return {\n      config: { enable_thinking: normalizedDeepThink },\n      debugMessage: `deepThink mapped to enable_thinking=${normalizedDeepThink} for qwen3-vl`,\n    };\n  }\n\n  if (modelFamily === 'doubao-vision') {\n    return {\n      config: {\n        thinking: { type: normalizedDeepThink ? 'enabled' : 'disabled' },\n      },\n      debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for doubao-vision`,\n    };\n  }\n\n  if (modelFamily === 'glm-v') {\n    return {\n      config: {\n        thinking: { type: normalizedDeepThink ? 'enabled' : 'disabled' },\n      },\n      debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for glm-v`,\n    };\n  }\n\n  return {\n    config: {},\n    debugMessage: `deepThink ignored: unsupported model_family \"${modelFamily ?? 'default'}\"`,\n    warningMessage: `The \"deepThink\" option is not supported for model_family \"${modelFamily ?? 'default'}\".`,\n  };\n}\n\n/**\n * Normalize a parsed JSON object by trimming whitespace from:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction normalizeJsonObject(obj: any): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n\n      // Recursively normalize the value\n      let normalizedValue = normalizeJsonObject(value);\n\n      // Trim all string values\n      if (typeof normalizedValue === 'string') {\n        normalizedValue = normalizedValue.trim();\n      }\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n\n  let parsed: any;\n  let lastError: unknown;\n  try {\n    parsed = JSON.parse(cleanJsonString);\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n  try {\n    parsed = JSON.parse(jsonrepair(cleanJsonString));\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    try {\n      parsed = JSON.parse(jsonrepair(jsonString));\n      return normalizeJsonObject(parsed);\n    } catch (error) {\n      lastError = error;\n    }\n  }\n  throw Error(\n    `failed to parse LLM response into JSON. Error - ${String(\n      lastError ?? 'unknown error',\n    )}. Response - \\n ${input}`,\n  );\n}\n"],"names":["createChatClient","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","modelFamily","createOpenAIClient","timeout","proxyAgent","debugProxy","getDebug","sanitizeProxyUrl","url","parsed","URL","ifInBrowser","console","moduleName","ProxyAgent","socksDispatcher","proxyUrl","Error","port","Number","protocol","socksType","decodeURIComponent","error","openAIOptions","baseOpenAI","OpenAI","openai","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","langsmithModule","wrapOpenAI","MIDSCENE_LANGFUSE_DEBUG","langfuseModule","observeOpenAI","wrappedClient","callAI","messages","options","completion","maxTokens","MIDSCENE_MODEL_MAX_TOKENS","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","temperature","isStreaming","content","accumulated","accumulatedReasoning","usage","timeCost","buildUsageInfo","usageData","cachedInputTokens","commonConfig","deepThinkConfig","debugMessage","warningMessage","resolveDeepThinkConfig","stream","chunk","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","retryCount","retryInterval","maxAttempts","lastError","attempt","result","JSON","Promise","resolve","setTimeout","e","newError","callAIWithObjectResponse","response","assert","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","deepThink","normalizedDeepThink","normalizeJsonObject","obj","Array","item","normalized","key","value","Object","trimmedKey","normalizedValue","cleanJsonString","jsonrepair","jsonString","String"],"mappings":";;;;;AA0BA,eAAeA,iBAAiB,EAC9BC,WAAW,EAGZ;IAQC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,WAAW,EACXC,kBAAkB,EAClBC,OAAO,EACR,GAAGZ;IAEJ,IAAIa;IACJ,MAAMC,aAAaC,SAAS;IAI5B,MAAMC,mBAAmB,CAACC;QACxB,IAAI;YACF,MAAMC,SAAS,IAAIC,IAAIF;YACvB,IAAIC,OAAO,QAAQ,EAAE;gBAEnBA,OAAO,QAAQ,GAAG;gBAClB,OAAOA,OAAO,IAAI;YACpB;YACA,OAAOD;QACT,EAAE,OAAM;YAEN,OAAOA;QACT;IACF;IAEA,IAAIf,WAAW;QACbY,WAAW,oBAAoBE,iBAAiBd;QAChD,IAAIkB,aACFC,QAAQ,IAAI,CACV;aAEG;YAEL,MAAMC,aAAa;YACnB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;YACpCT,aAAa,IAAIU,WAAW;gBAC1B,KAAKrB;YAEP;QACF;IACF,OAAO,IAAID,YAAY;QACrBa,WAAW,qBAAqBE,iBAAiBf;QACjD,IAAImB,aACFC,QAAQ,IAAI,CACV;aAGF,IAAI;YAEF,MAAMC,aAAa;YACnB,MAAM,EAAEE,eAAe,EAAE,GAAG,MAAM,MAAM,CAACF;YAEzC,MAAMG,WAAW,IAAIN,IAAIlB;YAGzB,IAAI,CAACwB,SAAS,QAAQ,EACpB,MAAM,IAAIC,MAAM;YAIlB,MAAMC,OAAOC,OAAO,QAAQ,CAACH,SAAS,IAAI,EAAE;YAC5C,IAAI,CAACA,SAAS,IAAI,IAAIG,OAAO,KAAK,CAACD,OACjC,MAAM,IAAID,MAAM;YAIlB,MAAMG,WAAWJ,SAAS,QAAQ,CAAC,OAAO,CAAC,KAAK;YAChD,MAAMK,YACJD,AAAa,aAAbA,WAAwB,IAAIA,AAAa,aAAbA,WAAwB,IAAI;YAE1DhB,aAAaW,gBAAgB;gBAC3B,MAAMM;gBACN,MAAML,SAAS,QAAQ;gBACvBE;gBACA,GAAIF,SAAS,QAAQ,GACjB;oBACE,QAAQM,mBAAmBN,SAAS,QAAQ;oBAC5C,UAAUM,mBAAmBN,SAAS,QAAQ,IAAI;gBACpD,IACA,CAAC,CAAC;YACR;YACAX,WAAW,uCAAuC;gBAChD,MAAMgB;gBACN,MAAML,SAAS,QAAQ;gBACvB,MAAME;YACR;QACF,EAAE,OAAOK,OAAO;YACdX,QAAQ,KAAK,CAAC,oCAAoCW;YAClD,MAAM,IAAIN,MACR,CAAC,yBAAyB,EAAEzB,WAAW,+GAA+G,CAAC;QAE3J;IAEJ;IAEA,MAAMgC,gBAAgB;QACpB,SAAS7B;QACT,QAAQC;QAGR,GAAIQ,aAAa;YAAE,cAAc;gBAAE,YAAYA;YAAkB;QAAE,IAAI,CAAC,CAAC;QACzE,GAAGP,iBAAiB;QACpB,GAAI,AAAmB,YAAnB,OAAOM,UAAuB;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAClD,yBAAyB;IAC3B;IAEA,MAAMsB,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAGrB,IACEE,UACAC,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIlB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMkB,kBAAkB;QACxB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;QACpCH,SAASI,WAAWJ;IACtB;IAGA,IACEA,UACAC,oBAAoB,qBAAqB,CAACI,0BAC1C;QACA,IAAIrB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMqB,iBAAiB;QACvB,MAAM,EAAEC,aAAa,EAAE,GAAG,MAAM,MAAM,CAACD;QACvCN,SAASO,cAAcP;IACzB;IAEA,IAAIzB,oBAAoB;QACtB,MAAMiC,gBAAgB,MAAMjC,mBAAmBuB,YAAYD;QAE3D,IAAIW,eACFR,SAASQ;IAEb;IAEA,OAAO;QACL,YAAYR,OAAO,IAAI,CAAC,WAAW;QACnCjC;QACAI;QACAC;QACAC;QACAC;IACF;AACF;AAEO,eAAemC,OACpBC,QAAsC,EACtC9C,WAAyB,EACzB+C,OAIC;IAOD,MAAM,EACJC,UAAU,EACV7C,SAAS,EACTI,gBAAgB,EAChBC,aAAa,EACbC,MAAM,EACNC,WAAW,EACZ,GAAG,MAAMX,iBAAiB;QACzBC;IACF;IAEA,MAAMiD,YACJZ,oBAAoB,yBAAyB,CAACa,8BAC9Cb,oBAAoB,yBAAyB,CAACc;IAChD,MAAMC,YAAYrC,SAAS;IAC3B,MAAMsC,oBAAoBtC,SAAS;IACnC,MAAMuC,qBAAqBvC,SAAS;IAEpC,MAAMwC,YAAYC,KAAK,GAAG;IAC1B,MAAMC,cAAczD,YAAY,WAAW,IAAI;IAE/C,MAAM0D,cAAcX,SAAS,UAAUA,SAAS;IAChD,IAAIY;IACJ,IAAIC,cAAc;IAClB,IAAIC,uBAAuB;IAC3B,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,iBAAiB,CAACC;QACtB,IAAI,CAACA,WAAW;QAEhB,MAAMC,oBACJD,WACC,uBAAuB;QAE1B,OAAO;YACL,eAAeA,UAAU,aAAa,IAAI;YAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;YAClD,cAAcA,UAAU,YAAY,IAAI;YACxC,cAAcC,qBAAqB;YACnC,WAAWH,YAAY;YACvB,YAAY5D;YACZ,mBAAmBI;YACnB,QAAQP,YAAY,MAAM;QAC5B;IACF;IAEA,MAAMmE,eAAe;QACnBV;QACA,QAAQ,CAAC,CAACC;QACV,YAAYT;QACZ,GAAIxC,AAAW,iBAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IACA,MAAM,EACJ,QAAQ2D,eAAe,EACvBC,YAAY,EACZC,cAAc,EACf,GAAGC,uBAAuB;QACzB,WAAWxB,SAAS;QACpBrC;IACF;IACA,IAAI2D,cACFjB,UAAUiB;IAEZ,IAAIC,gBAAgB;QAClBlB,UAAUkB;QACVjD,QAAQ,IAAI,CAACiD;IACf;IAEA,IAAI;QACFlB,UACE,CAAC,QAAQ,EAAEM,cAAc,eAAe,GAAG,WAAW,EAAEvD,WAAW;QAGrE,IAAIuD,aAAa;YACf,MAAMc,SAAU,MAAMxB,WAAW,MAAM,CACrC;gBACE,OAAO7C;gBACP2C;gBACA,GAAGqB,YAAY;gBACf,GAAGC,eAAe;YACpB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMK,SAASD,OAAQ;gBAChC,MAAMb,UAAUc,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAO,WAAW;gBACtD,MAAMC,oBACHD,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAe,qBAAqB;gBAG3D,IAAIA,MAAM,KAAK,EACbX,QAAQW,MAAM,KAAK;gBAGrB,IAAId,WAAWe,mBAAmB;oBAChCd,eAAeD;oBACfE,wBAAwBa;oBACxB,MAAMC,YAAiC;wBACrChB;wBACAe;wBACAd;wBACA,YAAY;wBACZ,OAAOgB;oBACT;oBACA7B,QAAQ,OAAO,CAAE4B;gBACnB;gBAGA,IAAIF,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,eAAe;oBACrCV,WAAWP,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACO,OAAO;wBAEV,MAAMe,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAClB,YAAY,MAAM,GAAG;wBAElCE,QAAQ;4BACN,eAAee;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTnB;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAOI,eAAeF;oBACxB;oBACAf,QAAQ,OAAO,CAAEgC;oBACjB;gBACF;YACF;YACApB,UAAUC;YACVP,kBACE,CAAC,iBAAiB,EAAElD,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAEsD,SAAS,eAAe,EAAEN,eAAe,IAAI;QAE1H,OAAO;YAEL,MAAMuB,aAAahF,YAAY,UAAU,IAAI;YAC7C,MAAMiF,gBAAgBjF,YAAY,aAAa,IAAI;YACnD,MAAMkF,cAAcF,aAAa;YAEjC,IAAIG;YAEJ,IAAK,IAAIC,UAAU,GAAGA,WAAWF,aAAaE,UAC5C,IAAI;gBACF,MAAMC,SAAS,MAAMrC,WAAW,MAAM,CAAC;oBACrC,OAAO7C;oBACP2C;oBACA,GAAGqB,YAAY;oBACf,GAAGC,eAAe;gBACpB;gBAEAL,WAAWP,KAAK,GAAG,KAAKD;gBAExBF,kBACE,CAAC,OAAO,EAAElD,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAE6E,OAAO,KAAK,EAAE,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,KAAK,EAAE,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,KAAK,EAAE,gBAAgB,GAAG,WAAW,EAAEtB,SAAS,aAAa,EAAEsB,OAAO,WAAW,IAAI,GAAG,eAAe,EAAE5B,eAAe,IAAI;gBAG9VH,mBACE,CAAC,oBAAoB,EAAEgC,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;gBAGvD,IAAI,CAACA,OAAO,OAAO,EACjB,MAAM,IAAI3D,MACR,CAAC,mCAAmC,EAAE4D,KAAK,SAAS,CAACD,SAAS;gBAIlE1B,UAAU0B,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;gBAC3C,IAAI,CAAC1B,SACH,MAAM,IAAIjC,MAAM;gBAGlBmC,uBACGwB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,EAAU,qBAAqB;gBAC3DvB,QAAQuB,OAAO,KAAK;gBACpB;YACF,EAAE,OAAOrD,OAAO;gBACdmD,YAAYnD;gBACZ,IAAIoD,UAAUF,aAAa;oBACzB7D,QAAQ,IAAI,CACV,CAAC,mCAAmC,EAAE+D,QAAQ,CAAC,EAAEF,YAAY,eAAe,EAAED,cAAc,aAAa,EAAEE,UAAU,OAAO,EAAE;oBAEhI,MAAM,IAAII,QAAQ,CAACC,UAAYC,WAAWD,SAASP;gBACrD;YACF;YAGF,IAAI,CAACtB,SACH,MAAMwB;QAEV;QAEA/B,UAAU,CAAC,4BAA4B,EAAES,sBAAsB;QAC/DT,UAAU,CAAC,kBAAkB,EAAEO,SAAS;QAGxC,IAAID,eAAe,CAACI,OAAO;YAEzB,MAAMe,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEnB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCG,QAAQ;gBACN,eAAee;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASlB,WAAW;YACpB,mBAAmBE,wBAAwBe;YAC3C,OAAOZ,eAAeF;YACtB,YAAY,CAAC,CAACJ;QAChB;IACF,EAAE,OAAOgC,GAAQ;QACfrE,QAAQ,KAAK,CAAC,kBAAkBqE;QAChC,MAAMC,WAAW,IAAIjE,MACnB,CAAC,eAAe,EAAEgC,cAAc,eAAe,GAAG,kBAAkB,EAAEvD,UAAU,GAAG,EAAEuF,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC9J;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,eAAeC,yBACpB9C,QAAsC,EACtC9C,WAAyB,EACzB+C,OAEC;IAOD,MAAM8C,WAAW,MAAMhD,OAAOC,UAAU9C,aAAa;QACnD,WAAW+C,SAAS;IACtB;IACA+C,OAAOD,UAAU;IACjB,MAAMpF,SAAST,YAAY,MAAM;IACjC,MAAM+F,cAAcC,cAAcH,SAAS,OAAO,EAAEpF;IACpDqF,OACE,AAAuB,YAAvB,OAAOC,aACP,CAAC,0CAA0C,EAAE/F,YAAY,SAAS,CAAC,GAAG,EAAE6F,SAAS,OAAO,EAAE;IAE5F,OAAO;QACL,SAASE;QACT,eAAeF,SAAS,OAAO;QAC/B,OAAOA,SAAS,KAAK;QACrB,mBAAmBA,SAAS,iBAAiB;IAC/C;AACF;AAEO,eAAeI,yBACpBC,IAAY,EACZlG,WAAyB;IAEzB,MAAM,EAAE2D,OAAO,EAAEG,KAAK,EAAE,GAAG,MAAMjB,OAAOqD,MAAMlG;IAC9C,OAAO;QAAE2D;QAASG;IAAM;AAC1B;AAEO,SAASqC,yBAAyBN,QAAgB;IACvD,IAAI;QAEF,MAAMO,YAAYP,SAAS,KAAK,CAAC;QACjC,IAAIO,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBR,SAAS,KAAK,CACnC;QAEF,IAAIQ,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBT,SAAS,KAAK,CAAC;QACrC,IAAIS,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOT;AACT;AAEO,SAASU,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASjC,uBAAuB,EACrCkC,SAAS,EACT/F,WAAW,EAIZ;IAKC,MAAMgG,sBAAsBD,AAAc,YAAdA,YAAwB7B,SAAY6B;IAEhE,IAAIC,AAAwB9B,WAAxB8B,qBACF,OAAO;QAAE,QAAQ,CAAC;QAAG,cAAc9B;IAAU;IAG/C,IAAIlE,AAAgB,eAAhBA,aACF,OAAO;QACL,QAAQ;YAAE,iBAAiBgG;QAAoB;QAC/C,cAAc,CAAC,oCAAoC,EAAEA,oBAAoB,aAAa,CAAC;IACzF;IAGF,IAAIhG,AAAgB,oBAAhBA,aACF,OAAO;QACL,QAAQ;YACN,UAAU;gBAAE,MAAMgG,sBAAsB,YAAY;YAAW;QACjE;QACA,cAAc,CAAC,kCAAkC,EAAEA,sBAAsB,YAAY,WAAW,kBAAkB,CAAC;IACrH;IAGF,IAAIhG,AAAgB,YAAhBA,aACF,OAAO;QACL,QAAQ;YACN,UAAU;gBAAE,MAAMgG,sBAAsB,YAAY;YAAW;QACjE;QACA,cAAc,CAAC,kCAAkC,EAAEA,sBAAsB,YAAY,WAAW,UAAU,CAAC;IAC7G;IAGF,OAAO;QACL,QAAQ,CAAC;QACT,cAAc,CAAC,6CAA6C,EAAEhG,eAAe,UAAU,CAAC,CAAC;QACzF,gBAAgB,CAAC,0DAA0D,EAAEA,eAAe,UAAU,EAAE,CAAC;IAC3G;AACF;AAQA,SAASiG,oBAAoBC,GAAQ;IAEnC,IAAIA,QAAAA,KACF,OAAOA;IAIT,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACE,OAASH,oBAAoBG;IAI/C,IAAI,AAAe,YAAf,OAAOF,KAAkB;QAC3B,MAAMG,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACN,KAAM;YAE9C,MAAMO,aAAaH,IAAI,IAAI;YAG3B,IAAII,kBAAkBT,oBAAoBM;YAG1C,IAAI,AAA2B,YAA3B,OAAOG,iBACTA,kBAAkBA,gBAAgB,IAAI;YAGxCL,UAAU,CAACI,WAAW,GAAGC;QAC3B;QAEA,OAAOL;IACT;IAGA,IAAI,AAAe,YAAf,OAAOH,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEO,SAASZ,cAAcQ,KAAa,EAAE/F,MAAgC;IAC3E,MAAM4G,kBAAkBlB,yBAAyBK;IAEjD,IAAIa,iBAAiB,MAAM,oBACzB,OAAOA,gBACJ,KAAK,CAAC,oBACL,MAAM,GACP,IAAIzF;IAGT,IAAIV;IACJ,IAAIiE;IACJ,IAAI;QACFjE,SAASoE,KAAK,KAAK,CAAC+B;QACpB,OAAOV,oBAAoBzF;IAC7B,EAAE,OAAOc,OAAO;QACdmD,YAAYnD;IACd;IACA,IAAI;QACFd,SAASoE,KAAK,KAAK,CAACgC,WAAWD;QAC/B,OAAOV,oBAAoBzF;IAC7B,EAAE,OAAOc,OAAO;QACdmD,YAAYnD;IACd;IAEA,IAAIvB,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAM8G,aAAahB,yBAAyBc;QAC5C,IAAI;YACFnG,SAASoE,KAAK,KAAK,CAACgC,WAAWC;YAC/B,OAAOZ,oBAAoBzF;QAC7B,EAAE,OAAOc,OAAO;YACdmD,YAAYnD;QACd;IACF;IACA,MAAMN,MACJ,CAAC,gDAAgD,EAAE8F,OACjDrC,aAAa,iBACb,gBAAgB,EAAEqB,OAAO;AAE/B"}
+{"version":3,"file":"ai-model/service-caller/index.mjs","sources":["../../../../src/ai-model/service-caller/index.ts"],"sourcesContent":["import {\n  AIResponseFormat,\n  type AIUsageInfo,\n  type DeepThinkOption,\n} from '@/types';\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\nimport {\n  type IModelConfig,\n  MIDSCENE_LANGFUSE_DEBUG,\n  MIDSCENE_LANGSMITH_DEBUG,\n  MIDSCENE_MODEL_MAX_TOKENS,\n  OPENAI_MAX_TOKENS,\n  type TModelFamily,\n  type TVlModeTypes,\n  type UITarsModelVersion,\n  globalConfigManager,\n} from '@midscene/shared/env';\n\nimport { getDebug } from '@midscene/shared/logger';\nimport { assert, ifInBrowser } from '@midscene/shared/utils';\nimport { jsonrepair } from 'jsonrepair';\nimport OpenAI from 'openai';\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\nimport type { Stream } from 'openai/streaming';\nimport type { AIArgs } from '../../common';\nimport { isAutoGLM } from '../auto-glm/util';\n\nasync function createChatClient({\n  modelConfig,\n}: {\n  modelConfig: IModelConfig;\n}): Promise<{\n  completion: OpenAI.Chat.Completions;\n  modelName: string;\n  modelDescription: string;\n  uiTarsVersion?: UITarsModelVersion;\n  vlMode: TVlModeTypes | undefined;\n  modelFamily: TModelFamily | undefined;\n}> {\n  const {\n    socksProxy,\n    httpProxy,\n    modelName,\n    openaiBaseURL,\n    openaiApiKey,\n    openaiExtraConfig,\n    modelDescription,\n    uiTarsModelVersion: uiTarsVersion,\n    vlMode,\n    modelFamily,\n    createOpenAIClient,\n    timeout,\n  } = modelConfig;\n\n  let proxyAgent: any = undefined;\n  const debugProxy = getDebug('ai:call:proxy');\n\n  // Helper function to sanitize proxy URL for logging (remove credentials)\n  // Uses URL API instead of regex to avoid ReDoS vulnerabilities\n  const sanitizeProxyUrl = (url: string): string => {\n    try {\n      const parsed = new URL(url);\n      if (parsed.username) {\n        // Keep username for debugging, hide password for security\n        parsed.password = '****';\n        return parsed.href;\n      }\n      return url;\n    } catch {\n      // If URL parsing fails, return original URL (will be caught later)\n      return url;\n    }\n  };\n\n  if (httpProxy) {\n    debugProxy('using http proxy', sanitizeProxyUrl(httpProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'HTTP proxy is configured but not supported in browser environment',\n      );\n    } else {\n      // Dynamic import with variable to avoid bundler static analysis\n      const moduleName = 'undici';\n      const { ProxyAgent } = await import(moduleName);\n      proxyAgent = new ProxyAgent({\n        uri: httpProxy,\n        // Note: authentication is handled via the URI (e.g., http://user:pass@proxy.com:8080)\n      });\n    }\n  } else if (socksProxy) {\n    debugProxy('using socks proxy', sanitizeProxyUrl(socksProxy));\n    if (ifInBrowser) {\n      console.warn(\n        'SOCKS proxy is configured but not supported in browser environment',\n      );\n    } else {\n      try {\n        // Dynamic import with variable to avoid bundler static analysis\n        const moduleName = 'fetch-socks';\n        const { socksDispatcher } = await import(moduleName);\n        // Parse SOCKS proxy URL (e.g., socks5://127.0.0.1:1080)\n        const proxyUrl = new URL(socksProxy);\n\n        // Validate hostname\n        if (!proxyUrl.hostname) {\n          throw new Error('SOCKS proxy URL must include a valid hostname');\n        }\n\n        // Validate and parse port\n        const port = Number.parseInt(proxyUrl.port, 10);\n        if (!proxyUrl.port || Number.isNaN(port)) {\n          throw new Error('SOCKS proxy URL must include a valid port');\n        }\n\n        // Parse SOCKS version from protocol\n        const protocol = proxyUrl.protocol.replace(':', '');\n        const socksType =\n          protocol === 'socks4' ? 4 : protocol === 'socks5' ? 5 : 5;\n\n        proxyAgent = socksDispatcher({\n          type: socksType,\n          host: proxyUrl.hostname,\n          port,\n          ...(proxyUrl.username\n            ? {\n                userId: decodeURIComponent(proxyUrl.username),\n                password: decodeURIComponent(proxyUrl.password || ''),\n              }\n            : {}),\n        });\n        debugProxy('socks proxy configured successfully', {\n          type: socksType,\n          host: proxyUrl.hostname,\n          port: port,\n        });\n      } catch (error) {\n        console.error('Failed to configure SOCKS proxy:', error);\n        throw new Error(\n          `Invalid SOCKS proxy URL: ${socksProxy}. Expected format: socks4://host:port, socks5://host:port, or with authentication: socks5://user:pass@host:port`,\n        );\n      }\n    }\n  }\n\n  const openAIOptions = {\n    baseURL: openaiBaseURL,\n    apiKey: openaiApiKey,\n    // Use fetchOptions.dispatcher for fetch-based SDK instead of httpAgent\n    // Note: Type assertion needed due to undici version mismatch between dependencies\n    ...(proxyAgent ? { fetchOptions: { dispatcher: proxyAgent as any } } : {}),\n    ...openaiExtraConfig,\n    ...(typeof timeout === 'number' ? { timeout } : {}),\n    dangerouslyAllowBrowser: true,\n  };\n\n  const baseOpenAI = new OpenAI(openAIOptions);\n\n  let openai: OpenAI = baseOpenAI;\n\n  // LangSmith wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGSMITH_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langsmith is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langsmithModule = 'langsmith/wrappers';\n    const { wrapOpenAI } = await import(langsmithModule);\n    openai = wrapOpenAI(openai);\n  }\n\n  // Langfuse wrapper\n  if (\n    openai &&\n    globalConfigManager.getEnvConfigInBoolean(MIDSCENE_LANGFUSE_DEBUG)\n  ) {\n    if (ifInBrowser) {\n      throw new Error('langfuse is not supported in browser');\n    }\n    console.log('DEBUGGING MODE: langfuse wrapper enabled');\n    // Use variable to prevent static analysis by bundlers\n    const langfuseModule = 'langfuse';\n    const { observeOpenAI } = await import(langfuseModule);\n    openai = observeOpenAI(openai);\n  }\n\n  if (createOpenAIClient) {\n    const wrappedClient = await createOpenAIClient(baseOpenAI, openAIOptions);\n\n    if (wrappedClient) {\n      openai = wrappedClient as OpenAI;\n    }\n  }\n\n  return {\n    completion: openai.chat.completions,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n    modelFamily,\n  };\n}\n\nexport async function callAI(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    stream?: boolean;\n    onChunk?: StreamingCallback;\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: string;\n  reasoning_content?: string;\n  usage?: AIUsageInfo;\n  isStreamed: boolean;\n}> {\n  const {\n    completion,\n    modelName,\n    modelDescription,\n    uiTarsVersion,\n    vlMode,\n    modelFamily,\n  } = await createChatClient({\n    modelConfig,\n  });\n\n  const maxTokens =\n    globalConfigManager.getEnvConfigValueAsNumber(MIDSCENE_MODEL_MAX_TOKENS) ??\n    globalConfigManager.getEnvConfigValueAsNumber(OPENAI_MAX_TOKENS);\n  const debugCall = getDebug('ai:call');\n  const debugProfileStats = getDebug('ai:profile:stats');\n  const debugProfileDetail = getDebug('ai:profile:detail');\n\n  const startTime = Date.now();\n  const temperature = modelConfig.temperature ?? 0;\n\n  const isStreaming = options?.stream && options?.onChunk;\n  let content: string | undefined;\n  let accumulated = '';\n  let accumulatedReasoning = '';\n  let usage: OpenAI.CompletionUsage | undefined;\n  let timeCost: number | undefined;\n\n  const buildUsageInfo = (usageData?: OpenAI.CompletionUsage) => {\n    if (!usageData) return undefined;\n\n    const cachedInputTokens = (\n      usageData as { prompt_tokens_details?: { cached_tokens?: number } }\n    )?.prompt_tokens_details?.cached_tokens;\n\n    return {\n      prompt_tokens: usageData.prompt_tokens ?? 0,\n      completion_tokens: usageData.completion_tokens ?? 0,\n      total_tokens: usageData.total_tokens ?? 0,\n      cached_input: cachedInputTokens ?? 0,\n      time_cost: timeCost ?? 0,\n      model_name: modelName,\n      model_description: modelDescription,\n      intent: modelConfig.intent,\n    } satisfies AIUsageInfo;\n  };\n\n  const commonConfig = {\n    temperature,\n    stream: !!isStreaming,\n    max_tokens: maxTokens,\n    ...(vlMode === 'qwen2.5-vl' // qwen vl v2 specific config\n      ? {\n          vl_high_resolution_images: true,\n        }\n      : {}),\n  };\n\n  if (isAutoGLM(vlMode)) {\n    (commonConfig as unknown as Record<string, number>).top_p = 0.85;\n    (commonConfig as unknown as Record<string, number>).frequency_penalty = 0.2;\n  }\n\n  const {\n    config: deepThinkConfig,\n    debugMessage,\n    warningMessage,\n  } = resolveDeepThinkConfig({\n    deepThink: options?.deepThink,\n    modelFamily,\n  });\n  if (debugMessage) {\n    debugCall(debugMessage);\n  }\n  if (warningMessage) {\n    debugCall(warningMessage);\n    console.warn(warningMessage);\n  }\n\n  try {\n    debugCall(\n      `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\n    );\n\n    if (isStreaming) {\n      const stream = (await completion.create(\n        {\n          model: modelName,\n          messages,\n          ...commonConfig,\n          ...deepThinkConfig,\n        },\n        {\n          stream: true,\n        },\n      )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\n        _request_id?: string | null;\n      };\n\n      for await (const chunk of stream) {\n        const content = chunk.choices?.[0]?.delta?.content || '';\n        const reasoning_content =\n          (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\n\n        // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\n        if (chunk.usage) {\n          usage = chunk.usage;\n        }\n\n        if (content || reasoning_content) {\n          accumulated += content;\n          accumulatedReasoning += reasoning_content;\n          const chunkData: CodeGenerationChunk = {\n            content,\n            reasoning_content,\n            accumulated,\n            isComplete: false,\n            usage: undefined,\n          };\n          options.onChunk!(chunkData);\n        }\n\n        // Check if stream is complete\n        if (chunk.choices?.[0]?.finish_reason) {\n          timeCost = Date.now() - startTime;\n\n          // If usage is not available from the stream, provide a basic usage info\n          if (!usage) {\n            // Estimate token counts based on content length (rough approximation)\n            const estimatedTokens = Math.max(\n              1,\n              Math.floor(accumulated.length / 4),\n            );\n            usage = {\n              prompt_tokens: estimatedTokens,\n              completion_tokens: estimatedTokens,\n              total_tokens: estimatedTokens * 2,\n            };\n          }\n\n          // Send final chunk\n          const finalChunk: CodeGenerationChunk = {\n            content: '',\n            accumulated,\n            reasoning_content: '',\n            isComplete: true,\n            usage: buildUsageInfo(usage),\n          };\n          options.onChunk!(finalChunk);\n          break;\n        }\n      }\n      content = accumulated;\n      debugProfileStats(\n        `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}, temperature, ${temperature ?? ''}`,\n      );\n    } else {\n      // Non-streaming with retry logic\n      const retryCount = modelConfig.retryCount ?? 1;\n      const retryInterval = modelConfig.retryInterval ?? 2000;\n      const maxAttempts = retryCount + 1; // retryCount=1 means 2 total attempts (1 initial + 1 retry)\n\n      let lastError: Error | undefined;\n\n      for (let attempt = 1; attempt <= maxAttempts; attempt++) {\n        try {\n          const result = await completion.create({\n            model: modelName,\n            messages,\n            ...commonConfig,\n            ...deepThinkConfig,\n          } as any);\n\n          timeCost = Date.now() - startTime;\n\n          debugProfileStats(\n            `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}, temperature, ${temperature ?? ''}`,\n          );\n\n          debugProfileDetail(\n            `model usage detail: ${JSON.stringify(result.usage)}`,\n          );\n\n          if (!result.choices) {\n            throw new Error(\n              `invalid response from LLM service: ${JSON.stringify(result)}`,\n            );\n          }\n\n          content = result.choices[0].message.content!;\n          if (!content) {\n            throw new Error('empty content from AI model');\n          }\n\n          accumulatedReasoning =\n            (result.choices[0].message as any)?.reasoning_content || '';\n          usage = result.usage;\n          break; // Success, exit retry loop\n        } catch (error) {\n          lastError = error as Error;\n          if (attempt < maxAttempts) {\n            console.warn(\n              `[Midscene] AI call failed (attempt ${attempt}/${maxAttempts}), retrying in ${retryInterval}ms... Error: ${lastError.message}`,\n            );\n            await new Promise((resolve) => setTimeout(resolve, retryInterval));\n          }\n        }\n      }\n\n      if (!content) {\n        throw lastError;\n      }\n    }\n\n    debugCall(`response reasoning content: ${accumulatedReasoning}`);\n    debugCall(`response content: ${content}`);\n\n    // Ensure we always have usage info for streaming responses\n    if (isStreaming && !usage) {\n      // Estimate token counts based on content length (rough approximation)\n      const estimatedTokens = Math.max(\n        1,\n        Math.floor((content || '').length / 4),\n      );\n      usage = {\n        prompt_tokens: estimatedTokens,\n        completion_tokens: estimatedTokens,\n        total_tokens: estimatedTokens * 2,\n      } as OpenAI.CompletionUsage;\n    }\n\n    return {\n      content: content || '',\n      reasoning_content: accumulatedReasoning || undefined,\n      usage: buildUsageInfo(usage),\n      isStreamed: !!isStreaming,\n    };\n  } catch (e: any) {\n    console.error(' call AI error', e);\n    const newError = new Error(\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service (${modelName}): ${e.message}\\nTrouble shooting: https://midscenejs.com/model-provider.html`,\n      {\n        cause: e,\n      },\n    );\n    throw newError;\n  }\n}\n\nexport async function callAIWithObjectResponse<T>(\n  messages: ChatCompletionMessageParam[],\n  modelConfig: IModelConfig,\n  options?: {\n    deepThink?: DeepThinkOption;\n  },\n): Promise<{\n  content: T;\n  contentString: string;\n  usage?: AIUsageInfo;\n  reasoning_content?: string;\n}> {\n  const response = await callAI(messages, modelConfig, {\n    deepThink: options?.deepThink,\n  });\n  assert(response, 'empty response');\n  const vlMode = modelConfig.vlMode;\n  const jsonContent = safeParseJson(response.content, vlMode);\n  assert(\n    typeof jsonContent === 'object',\n    `failed to parse json response from model (${modelConfig.modelName}): ${response.content}`,\n  );\n  return {\n    content: jsonContent,\n    contentString: response.content,\n    usage: response.usage,\n    reasoning_content: response.reasoning_content,\n  };\n}\n\nexport async function callAIWithStringResponse(\n  msgs: AIArgs,\n  modelConfig: IModelConfig,\n): Promise<{ content: string; usage?: AIUsageInfo }> {\n  const { content, usage } = await callAI(msgs, modelConfig);\n  return { content, usage };\n}\n\nexport function extractJSONFromCodeBlock(response: string) {\n  try {\n    // First, try to match a JSON object directly in the response\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\n    if (jsonMatch) {\n      return jsonMatch[1];\n    }\n\n    // If no direct JSON object is found, try to extract JSON from a code block\n    const codeBlockMatch = response.match(\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\n    );\n    if (codeBlockMatch) {\n      return codeBlockMatch[1];\n    }\n\n    // If no code block is found, try to find a JSON-like structure in the text\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\n    if (jsonLikeMatch) {\n      return jsonLikeMatch[0];\n    }\n  } catch {}\n  // If no JSON-like structure is found, return the original response\n  return response;\n}\n\nexport function preprocessDoubaoBboxJson(input: string) {\n  if (input.includes('bbox')) {\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\n    while (/\\d+\\s+\\d+/.test(input)) {\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\n    }\n  }\n  return input;\n}\n\nexport function resolveDeepThinkConfig({\n  deepThink,\n  modelFamily,\n}: {\n  deepThink?: DeepThinkOption;\n  modelFamily?: TModelFamily;\n}): {\n  config: Record<string, unknown>;\n  debugMessage?: string;\n  warningMessage?: string;\n} {\n  const normalizedDeepThink = deepThink === 'unset' ? undefined : deepThink;\n\n  if (normalizedDeepThink === undefined) {\n    return { config: {}, debugMessage: undefined };\n  }\n\n  if (modelFamily === 'qwen3-vl') {\n    return {\n      config: { enable_thinking: normalizedDeepThink },\n      debugMessage: `deepThink mapped to enable_thinking=${normalizedDeepThink} for qwen3-vl`,\n    };\n  }\n\n  if (modelFamily === 'doubao-vision') {\n    return {\n      config: {\n        thinking: { type: normalizedDeepThink ? 'enabled' : 'disabled' },\n      },\n      debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for doubao-vision`,\n    };\n  }\n\n  if (modelFamily === 'glm-v') {\n    return {\n      config: {\n        thinking: { type: normalizedDeepThink ? 'enabled' : 'disabled' },\n      },\n      debugMessage: `deepThink mapped to thinking.type=${normalizedDeepThink ? 'enabled' : 'disabled'} for glm-v`,\n    };\n  }\n\n  return {\n    config: {},\n    debugMessage: `deepThink ignored: unsupported model_family \"${modelFamily ?? 'default'}\"`,\n    warningMessage: `The \"deepThink\" option is not supported for model_family \"${modelFamily ?? 'default'}\".`,\n  };\n}\n\n/**\n * Normalize a parsed JSON object by trimming whitespace from:\n * 1. All object keys (e.g., \" prompt \" -> \"prompt\")\n * 2. All string values (e.g., \" Tap \" -> \"Tap\")\n * This handles LLM output that may include leading/trailing spaces.\n */\nfunction normalizeJsonObject(obj: any): any {\n  // Handle null and undefined\n  if (obj === null || obj === undefined) {\n    return obj;\n  }\n\n  // Handle arrays - recursively normalize each element\n  if (Array.isArray(obj)) {\n    return obj.map((item) => normalizeJsonObject(item));\n  }\n\n  // Handle objects\n  if (typeof obj === 'object') {\n    const normalized: any = {};\n\n    for (const [key, value] of Object.entries(obj)) {\n      // Trim the key to remove leading/trailing spaces\n      const trimmedKey = key.trim();\n\n      // Recursively normalize the value\n      let normalizedValue = normalizeJsonObject(value);\n\n      // Trim all string values\n      if (typeof normalizedValue === 'string') {\n        normalizedValue = normalizedValue.trim();\n      }\n\n      normalized[trimmedKey] = normalizedValue;\n    }\n\n    return normalized;\n  }\n\n  // Handle primitive strings\n  if (typeof obj === 'string') {\n    return obj.trim();\n  }\n\n  // Return other primitives as-is\n  return obj;\n}\n\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\n  const cleanJsonString = extractJSONFromCodeBlock(input);\n  // match the point\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\n    return cleanJsonString\n      .match(/\\((\\d+),(\\d+)\\)/)\n      ?.slice(1)\n      .map(Number);\n  }\n\n  let parsed: any;\n  let lastError: unknown;\n  try {\n    parsed = JSON.parse(cleanJsonString);\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n  try {\n    parsed = JSON.parse(jsonrepair(cleanJsonString));\n    return normalizeJsonObject(parsed);\n  } catch (error) {\n    lastError = error;\n  }\n\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\n    try {\n      parsed = JSON.parse(jsonrepair(jsonString));\n      return normalizeJsonObject(parsed);\n    } catch (error) {\n      lastError = error;\n    }\n  }\n  throw Error(\n    `failed to parse LLM response into JSON. Error - ${String(\n      lastError ?? 'unknown error',\n    )}. Response - \\n ${input}`,\n  );\n}\n"],"names":["createChatClient","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","modelDescription","uiTarsVersion","vlMode","modelFamily","createOpenAIClient","timeout","proxyAgent","debugProxy","getDebug","sanitizeProxyUrl","url","parsed","URL","ifInBrowser","console","moduleName","ProxyAgent","socksDispatcher","proxyUrl","Error","port","Number","protocol","socksType","decodeURIComponent","error","openAIOptions","baseOpenAI","OpenAI","openai","globalConfigManager","MIDSCENE_LANGSMITH_DEBUG","langsmithModule","wrapOpenAI","MIDSCENE_LANGFUSE_DEBUG","langfuseModule","observeOpenAI","wrappedClient","callAI","messages","options","completion","maxTokens","MIDSCENE_MODEL_MAX_TOKENS","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","temperature","isStreaming","content","accumulated","accumulatedReasoning","usage","timeCost","buildUsageInfo","usageData","cachedInputTokens","commonConfig","isAutoGLM","deepThinkConfig","debugMessage","warningMessage","resolveDeepThinkConfig","stream","chunk","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","retryCount","retryInterval","maxAttempts","lastError","attempt","result","JSON","Promise","resolve","setTimeout","e","newError","callAIWithObjectResponse","response","assert","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","deepThink","normalizedDeepThink","normalizeJsonObject","obj","Array","item","normalized","key","value","Object","trimmedKey","normalizedValue","cleanJsonString","jsonrepair","jsonString","String"],"mappings":";;;;;;AA2BA,eAAeA,iBAAiB,EAC9BC,WAAW,EAGZ;IAQC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACNC,WAAW,EACXC,kBAAkB,EAClBC,OAAO,EACR,GAAGZ;IAEJ,IAAIa;IACJ,MAAMC,aAAaC,SAAS;IAI5B,MAAMC,mBAAmB,CAACC;QACxB,IAAI;YACF,MAAMC,SAAS,IAAIC,IAAIF;YACvB,IAAIC,OAAO,QAAQ,EAAE;gBAEnBA,OAAO,QAAQ,GAAG;gBAClB,OAAOA,OAAO,IAAI;YACpB;YACA,OAAOD;QACT,EAAE,OAAM;YAEN,OAAOA;QACT;IACF;IAEA,IAAIf,WAAW;QACbY,WAAW,oBAAoBE,iBAAiBd;QAChD,IAAIkB,aACFC,QAAQ,IAAI,CACV;aAEG;YAEL,MAAMC,aAAa;YACnB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;YACpCT,aAAa,IAAIU,WAAW;gBAC1B,KAAKrB;YAEP;QACF;IACF,OAAO,IAAID,YAAY;QACrBa,WAAW,qBAAqBE,iBAAiBf;QACjD,IAAImB,aACFC,QAAQ,IAAI,CACV;aAGF,IAAI;YAEF,MAAMC,aAAa;YACnB,MAAM,EAAEE,eAAe,EAAE,GAAG,MAAM,MAAM,CAACF;YAEzC,MAAMG,WAAW,IAAIN,IAAIlB;YAGzB,IAAI,CAACwB,SAAS,QAAQ,EACpB,MAAM,IAAIC,MAAM;YAIlB,MAAMC,OAAOC,OAAO,QAAQ,CAACH,SAAS,IAAI,EAAE;YAC5C,IAAI,CAACA,SAAS,IAAI,IAAIG,OAAO,KAAK,CAACD,OACjC,MAAM,IAAID,MAAM;YAIlB,MAAMG,WAAWJ,SAAS,QAAQ,CAAC,OAAO,CAAC,KAAK;YAChD,MAAMK,YACJD,AAAa,aAAbA,WAAwB,IAAIA,AAAa,aAAbA,WAAwB,IAAI;YAE1DhB,aAAaW,gBAAgB;gBAC3B,MAAMM;gBACN,MAAML,SAAS,QAAQ;gBACvBE;gBACA,GAAIF,SAAS,QAAQ,GACjB;oBACE,QAAQM,mBAAmBN,SAAS,QAAQ;oBAC5C,UAAUM,mBAAmBN,SAAS,QAAQ,IAAI;gBACpD,IACA,CAAC,CAAC;YACR;YACAX,WAAW,uCAAuC;gBAChD,MAAMgB;gBACN,MAAML,SAAS,QAAQ;gBACvB,MAAME;YACR;QACF,EAAE,OAAOK,OAAO;YACdX,QAAQ,KAAK,CAAC,oCAAoCW;YAClD,MAAM,IAAIN,MACR,CAAC,yBAAyB,EAAEzB,WAAW,+GAA+G,CAAC;QAE3J;IAEJ;IAEA,MAAMgC,gBAAgB;QACpB,SAAS7B;QACT,QAAQC;QAGR,GAAIQ,aAAa;YAAE,cAAc;gBAAE,YAAYA;YAAkB;QAAE,IAAI,CAAC,CAAC;QACzE,GAAGP,iBAAiB;QACpB,GAAI,AAAmB,YAAnB,OAAOM,UAAuB;YAAEA;QAAQ,IAAI,CAAC,CAAC;QAClD,yBAAyB;IAC3B;IAEA,MAAMsB,aAAa,IAAIC,SAAOF;IAE9B,IAAIG,SAAiBF;IAGrB,IACEE,UACAC,oBAAoB,qBAAqB,CAACC,2BAC1C;QACA,IAAIlB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMkB,kBAAkB;QACxB,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAACD;QACpCH,SAASI,WAAWJ;IACtB;IAGA,IACEA,UACAC,oBAAoB,qBAAqB,CAACI,0BAC1C;QACA,IAAIrB,aACF,MAAM,IAAIM,MAAM;QAElBL,QAAQ,GAAG,CAAC;QAEZ,MAAMqB,iBAAiB;QACvB,MAAM,EAAEC,aAAa,EAAE,GAAG,MAAM,MAAM,CAACD;QACvCN,SAASO,cAAcP;IACzB;IAEA,IAAIzB,oBAAoB;QACtB,MAAMiC,gBAAgB,MAAMjC,mBAAmBuB,YAAYD;QAE3D,IAAIW,eACFR,SAASQ;IAEb;IAEA,OAAO;QACL,YAAYR,OAAO,IAAI,CAAC,WAAW;QACnCjC;QACAI;QACAC;QACAC;QACAC;IACF;AACF;AAEO,eAAemC,OACpBC,QAAsC,EACtC9C,WAAyB,EACzB+C,OAIC;IAOD,MAAM,EACJC,UAAU,EACV7C,SAAS,EACTI,gBAAgB,EAChBC,aAAa,EACbC,MAAM,EACNC,WAAW,EACZ,GAAG,MAAMX,iBAAiB;QACzBC;IACF;IAEA,MAAMiD,YACJZ,oBAAoB,yBAAyB,CAACa,8BAC9Cb,oBAAoB,yBAAyB,CAACc;IAChD,MAAMC,YAAYrC,SAAS;IAC3B,MAAMsC,oBAAoBtC,SAAS;IACnC,MAAMuC,qBAAqBvC,SAAS;IAEpC,MAAMwC,YAAYC,KAAK,GAAG;IAC1B,MAAMC,cAAczD,YAAY,WAAW,IAAI;IAE/C,MAAM0D,cAAcX,SAAS,UAAUA,SAAS;IAChD,IAAIY;IACJ,IAAIC,cAAc;IAClB,IAAIC,uBAAuB;IAC3B,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,iBAAiB,CAACC;QACtB,IAAI,CAACA,WAAW;QAEhB,MAAMC,oBACJD,WACC,uBAAuB;QAE1B,OAAO;YACL,eAAeA,UAAU,aAAa,IAAI;YAC1C,mBAAmBA,UAAU,iBAAiB,IAAI;YAClD,cAAcA,UAAU,YAAY,IAAI;YACxC,cAAcC,qBAAqB;YACnC,WAAWH,YAAY;YACvB,YAAY5D;YACZ,mBAAmBI;YACnB,QAAQP,YAAY,MAAM;QAC5B;IACF;IAEA,MAAMmE,eAAe;QACnBV;QACA,QAAQ,CAAC,CAACC;QACV,YAAYT;QACZ,GAAIxC,AAAW,iBAAXA,SACA;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI2D,UAAU3D,SAAS;QACpB0D,aAAmD,KAAK,GAAG;QAC3DA,aAAmD,iBAAiB,GAAG;IAC1E;IAEA,MAAM,EACJ,QAAQE,eAAe,EACvBC,YAAY,EACZC,cAAc,EACf,GAAGC,uBAAuB;QACzB,WAAWzB,SAAS;QACpBrC;IACF;IACA,IAAI4D,cACFlB,UAAUkB;IAEZ,IAAIC,gBAAgB;QAClBnB,UAAUmB;QACVlD,QAAQ,IAAI,CAACkD;IACf;IAEA,IAAI;QACFnB,UACE,CAAC,QAAQ,EAAEM,cAAc,eAAe,GAAG,WAAW,EAAEvD,WAAW;QAGrE,IAAIuD,aAAa;YACf,MAAMe,SAAU,MAAMzB,WAAW,MAAM,CACrC;gBACE,OAAO7C;gBACP2C;gBACA,GAAGqB,YAAY;gBACf,GAAGE,eAAe;YACpB,GACA;gBACE,QAAQ;YACV;YAKF,WAAW,MAAMK,SAASD,OAAQ;gBAChC,MAAMd,UAAUe,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAO,WAAW;gBACtD,MAAMC,oBACHD,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,OAAe,qBAAqB;gBAG3D,IAAIA,MAAM,KAAK,EACbZ,QAAQY,MAAM,KAAK;gBAGrB,IAAIf,WAAWgB,mBAAmB;oBAChCf,eAAeD;oBACfE,wBAAwBc;oBACxB,MAAMC,YAAiC;wBACrCjB;wBACAgB;wBACAf;wBACA,YAAY;wBACZ,OAAOiB;oBACT;oBACA9B,QAAQ,OAAO,CAAE6B;gBACnB;gBAGA,IAAIF,MAAM,OAAO,EAAE,CAAC,EAAE,EAAE,eAAe;oBACrCX,WAAWP,KAAK,GAAG,KAAKD;oBAGxB,IAAI,CAACO,OAAO;wBAEV,MAAMgB,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACnB,YAAY,MAAM,GAAG;wBAElCE,QAAQ;4BACN,eAAegB;4BACf,mBAAmBA;4BACnB,cAAcA,AAAkB,IAAlBA;wBAChB;oBACF;oBAGA,MAAME,aAAkC;wBACtC,SAAS;wBACTpB;wBACA,mBAAmB;wBACnB,YAAY;wBACZ,OAAOI,eAAeF;oBACxB;oBACAf,QAAQ,OAAO,CAAEiC;oBACjB;gBACF;YACF;YACArB,UAAUC;YACVP,kBACE,CAAC,iBAAiB,EAAElD,UAAU,QAAQ,EAAEM,UAAU,UAAU,WAAW,EAAEsD,SAAS,eAAe,EAAEN,eAAe,IAAI;QAE1H,OAAO;YAEL,MAAMwB,aAAajF,YAAY,UAAU,IAAI;YAC7C,MAAMkF,gBAAgBlF,YAAY,aAAa,IAAI;YACnD,MAAMmF,cAAcF,aAAa;YAEjC,IAAIG;YAEJ,IAAK,IAAIC,UAAU,GAAGA,WAAWF,aAAaE,UAC5C,IAAI;gBACF,MAAMC,SAAS,MAAMtC,WAAW,MAAM,CAAC;oBACrC,OAAO7C;oBACP2C;oBACA,GAAGqB,YAAY;oBACf,GAAGE,eAAe;gBACpB;gBAEAN,WAAWP,KAAK,GAAG,KAAKD;gBAExBF,kBACE,CAAC,OAAO,EAAElD,UAAU,QAAQ,EAAEM,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAE8E,OAAO,KAAK,EAAE,iBAAiB,GAAG,qBAAqB,EAAEA,OAAO,KAAK,EAAE,qBAAqB,GAAG,gBAAgB,EAAEA,OAAO,KAAK,EAAE,gBAAgB,GAAG,WAAW,EAAEvB,SAAS,aAAa,EAAEuB,OAAO,WAAW,IAAI,GAAG,eAAe,EAAE7B,eAAe,IAAI;gBAG9VH,mBACE,CAAC,oBAAoB,EAAEiC,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;gBAGvD,IAAI,CAACA,OAAO,OAAO,EACjB,MAAM,IAAI5D,MACR,CAAC,mCAAmC,EAAE6D,KAAK,SAAS,CAACD,SAAS;gBAIlE3B,UAAU2B,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;gBAC3C,IAAI,CAAC3B,SACH,MAAM,IAAIjC,MAAM;gBAGlBmC,uBACGyB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,EAAU,qBAAqB;gBAC3DxB,QAAQwB,OAAO,KAAK;gBACpB;YACF,EAAE,OAAOtD,OAAO;gBACdoD,YAAYpD;gBACZ,IAAIqD,UAAUF,aAAa;oBACzB9D,QAAQ,IAAI,CACV,CAAC,mCAAmC,EAAEgE,QAAQ,CAAC,EAAEF,YAAY,eAAe,EAAED,cAAc,aAAa,EAAEE,UAAU,OAAO,EAAE;oBAEhI,MAAM,IAAII,QAAQ,CAACC,UAAYC,WAAWD,SAASP;gBACrD;YACF;YAGF,IAAI,CAACvB,SACH,MAAMyB;QAEV;QAEAhC,UAAU,CAAC,4BAA4B,EAAES,sBAAsB;QAC/DT,UAAU,CAAC,kBAAkB,EAAEO,SAAS;QAGxC,IAAID,eAAe,CAACI,OAAO;YAEzB,MAAMgB,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEpB,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCG,QAAQ;gBACN,eAAegB;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASnB,WAAW;YACpB,mBAAmBE,wBAAwBgB;YAC3C,OAAOb,eAAeF;YACtB,YAAY,CAAC,CAACJ;QAChB;IACF,EAAE,OAAOiC,GAAQ;QACftE,QAAQ,KAAK,CAAC,kBAAkBsE;QAChC,MAAMC,WAAW,IAAIlE,MACnB,CAAC,eAAe,EAAEgC,cAAc,eAAe,GAAG,kBAAkB,EAAEvD,UAAU,GAAG,EAAEwF,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC9J;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,eAAeC,yBACpB/C,QAAsC,EACtC9C,WAAyB,EACzB+C,OAEC;IAOD,MAAM+C,WAAW,MAAMjD,OAAOC,UAAU9C,aAAa;QACnD,WAAW+C,SAAS;IACtB;IACAgD,OAAOD,UAAU;IACjB,MAAMrF,SAAST,YAAY,MAAM;IACjC,MAAMgG,cAAcC,cAAcH,SAAS,OAAO,EAAErF;IACpDsF,OACE,AAAuB,YAAvB,OAAOC,aACP,CAAC,0CAA0C,EAAEhG,YAAY,SAAS,CAAC,GAAG,EAAE8F,SAAS,OAAO,EAAE;IAE5F,OAAO;QACL,SAASE;QACT,eAAeF,SAAS,OAAO;QAC/B,OAAOA,SAAS,KAAK;QACrB,mBAAmBA,SAAS,iBAAiB;IAC/C;AACF;AAEO,eAAeI,yBACpBC,IAAY,EACZnG,WAAyB;IAEzB,MAAM,EAAE2D,OAAO,EAAEG,KAAK,EAAE,GAAG,MAAMjB,OAAOsD,MAAMnG;IAC9C,OAAO;QAAE2D;QAASG;IAAM;AAC1B;AAEO,SAASsC,yBAAyBN,QAAgB;IACvD,IAAI;QAEF,MAAMO,YAAYP,SAAS,KAAK,CAAC;QACjC,IAAIO,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBR,SAAS,KAAK,CACnC;QAEF,IAAIQ,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBT,SAAS,KAAK,CAAC;QACrC,IAAIS,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOT;AACT;AAEO,SAASU,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASjC,uBAAuB,EACrCkC,SAAS,EACThG,WAAW,EAIZ;IAKC,MAAMiG,sBAAsBD,AAAc,YAAdA,YAAwB7B,SAAY6B;IAEhE,IAAIC,AAAwB9B,WAAxB8B,qBACF,OAAO;QAAE,QAAQ,CAAC;QAAG,cAAc9B;IAAU;IAG/C,IAAInE,AAAgB,eAAhBA,aACF,OAAO;QACL,QAAQ;YAAE,iBAAiBiG;QAAoB;QAC/C,cAAc,CAAC,oCAAoC,EAAEA,oBAAoB,aAAa,CAAC;IACzF;IAGF,IAAIjG,AAAgB,oBAAhBA,aACF,OAAO;QACL,QAAQ;YACN,UAAU;gBAAE,MAAMiG,sBAAsB,YAAY;YAAW;QACjE;QACA,cAAc,CAAC,kCAAkC,EAAEA,sBAAsB,YAAY,WAAW,kBAAkB,CAAC;IACrH;IAGF,IAAIjG,AAAgB,YAAhBA,aACF,OAAO;QACL,QAAQ;YACN,UAAU;gBAAE,MAAMiG,sBAAsB,YAAY;YAAW;QACjE;QACA,cAAc,CAAC,kCAAkC,EAAEA,sBAAsB,YAAY,WAAW,UAAU,CAAC;IAC7G;IAGF,OAAO;QACL,QAAQ,CAAC;QACT,cAAc,CAAC,6CAA6C,EAAEjG,eAAe,UAAU,CAAC,CAAC;QACzF,gBAAgB,CAAC,0DAA0D,EAAEA,eAAe,UAAU,EAAE,CAAC;IAC3G;AACF;AAQA,SAASkG,oBAAoBC,GAAQ;IAEnC,IAAIA,QAAAA,KACF,OAAOA;IAIT,IAAIC,MAAM,OAAO,CAACD,MAChB,OAAOA,IAAI,GAAG,CAAC,CAACE,OAASH,oBAAoBG;IAI/C,IAAI,AAAe,YAAf,OAAOF,KAAkB;QAC3B,MAAMG,aAAkB,CAAC;QAEzB,KAAK,MAAM,CAACC,KAAKC,MAAM,IAAIC,OAAO,OAAO,CAACN,KAAM;YAE9C,MAAMO,aAAaH,IAAI,IAAI;YAG3B,IAAII,kBAAkBT,oBAAoBM;YAG1C,IAAI,AAA2B,YAA3B,OAAOG,iBACTA,kBAAkBA,gBAAgB,IAAI;YAGxCL,UAAU,CAACI,WAAW,GAAGC;QAC3B;QAEA,OAAOL;IACT;IAGA,IAAI,AAAe,YAAf,OAAOH,KACT,OAAOA,IAAI,IAAI;IAIjB,OAAOA;AACT;AAEO,SAASZ,cAAcQ,KAAa,EAAEhG,MAAgC;IAC3E,MAAM6G,kBAAkBlB,yBAAyBK;IAEjD,IAAIa,iBAAiB,MAAM,oBACzB,OAAOA,gBACJ,KAAK,CAAC,oBACL,MAAM,GACP,IAAI1F;IAGT,IAAIV;IACJ,IAAIkE;IACJ,IAAI;QACFlE,SAASqE,KAAK,KAAK,CAAC+B;QACpB,OAAOV,oBAAoB1F;IAC7B,EAAE,OAAOc,OAAO;QACdoD,YAAYpD;IACd;IACA,IAAI;QACFd,SAASqE,KAAK,KAAK,CAACgC,WAAWD;QAC/B,OAAOV,oBAAoB1F;IAC7B,EAAE,OAAOc,OAAO;QACdoD,YAAYpD;IACd;IAEA,IAAIvB,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAM+G,aAAahB,yBAAyBc;QAC5C,IAAI;YACFpG,SAASqE,KAAK,KAAK,CAACgC,WAAWC;YAC/B,OAAOZ,oBAAoB1F;QAC7B,EAAE,OAAOc,OAAO;YACdoD,YAAYpD;QACd;IACF;IACA,MAAMN,MACJ,CAAC,gDAAgD,EAAE+F,OACjDrC,aAAa,iBACb,gBAAgB,EAAEqB,OAAO;AAE/B"}

package/dist/es/ai-model/ui-tars-planning.mjs

@@ -20,7 +20,7 @@ async function uiTarsPlanning(userInstruction, options) {
     let instruction = userInstruction;
     if (actionContext) instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\n<user_instruction>${userInstruction}</user_instruction>`;
     const systemPrompt = getUiTarsPlanningPrompt() + instruction;
-    const screenshotBase64 = context.screenshot.getData();
+    const screenshotBase64 = context.screenshot.base64;
     const imagePayload = await resizeImageForUiTars(screenshotBase64, context.size, uiTarsModelVersion);
     conversationHistory.append({
         role: 'user',
@@ -63,47 +63,50 @@ async function uiTarsPlanning(userInstruction, options) {
         if ('click' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
             const point = getPoint(action.action_inputs.start_box, size);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, size.width, size.height)
+            };
             transformActions.push({
                 type: 'Tap',
                 param: {
-                    locate: {
-                        prompt: action.thought || '',
-                        bbox: pointToBbox({
-                            x: point[0],
-                            y: point[1]
-                        }, size.width, size.height)
-                    }
+                    locate: locate
                 }
             });
         } else if ('left_double' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
             const point = getPoint(action.action_inputs.start_box, size);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, size.width, size.height)
+            };
             transformActions.push({
                 type: 'DoubleClick',
                 param: {
-                    locate: {
-                        prompt: action.thought || '',
-                        bbox: pointToBbox({
-                            x: point[0],
-                            y: point[1]
-                        }, size.width, size.height)
-                    }
+                    locate: locate
                 },
                 thought: action.thought || ''
             });
         } else if ('right_single' === actionType) {
             assert(action.action_inputs.start_box, 'start_box is required');
             const point = getPoint(action.action_inputs.start_box, size);
+            const locate = {
+                prompt: action.thought || '',
+                bbox: pointToBbox({
+                    x: point[0],
+                    y: point[1]
+                }, size.width, size.height)
+            };
             transformActions.push({
                 type: 'RightClick',
                 param: {
-                    locate: {
-                        prompt: action.thought || '',
-                        bbox: pointToBbox({
-                            x: point[0],
-                            y: point[1]
-                        }, size.width, size.height)
-                    }
+                    locate: locate
                 },
                 thought: action.thought || ''
             });

package/dist/es/ai-model/ui-tars-planning.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"ai-model/ui-tars-planning.mjs","sources":["../../../src/ai-model/ui-tars-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  Size,\n  UIContext,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport type { ConversationHistory } from './conversation-history';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { callAIWithStringResponse } from './service-caller/index';\ntype ActionType =\n  | 'click'\n  | 'left_double'\n  | 'right_single'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function uiTarsPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n  const { uiTarsModelVersion } = modelConfig;\n\n  let instruction = userInstruction;\n  if (actionContext) {\n    instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n<user_instruction>${userInstruction}</user_instruction>`;\n  }\n\n  const systemPrompt = getUiTarsPlanningPrompt() + instruction;\n\n  const screenshotBase64 = context.screenshot.getData();\n  const imagePayload = await resizeImageForUiTars(\n    screenshotBase64,\n    context.size,\n    uiTarsModelVersion,\n  );\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: {\n          url: imagePayload,\n        },\n      },\n    ],\n  });\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory.snapshot(),\n    ],\n    modelConfig,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const { size } = context;\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  const unhandledActions: Array<{ type: string; thought: string }> = [];\n  let shouldContinue = true;\n  parsed.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n      });\n    } else if (actionType === 'left_double') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'DoubleClick',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'right_single') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n      transformActions.push({\n        type: 'RightClick',\n        param: {\n          locate: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: point[0], y: point[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      shouldContinue = false;\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys.join('+'),\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType) {\n      // Track unhandled action types\n      unhandledActions.push({\n        type: actionType,\n        thought: action.thought || '',\n      });\n      debug('Unhandled action type:', actionType, 'thought:', action.thought);\n    }\n  });\n\n  if (transformActions.length === 0) {\n    const errorDetails: string[] = [];\n\n    // Check if parsing failed\n    if (parsed.length === 0) {\n      errorDetails.push('Action parser returned no actions');\n\n      // Check if response has Thought but no Action\n      if (\n        res.content.includes('Thought:') &&\n        !res.content.includes('Action:')\n      ) {\n        errorDetails.push(\n          'Response contains \"Thought:\" but missing \"Action:\" line',\n        );\n      } else {\n        errorDetails.push('Response may be malformed or empty');\n      }\n    }\n\n    // Check if we have unhandled action types\n    if (unhandledActions.length > 0) {\n      const types = unhandledActions.map((a) => a.type).join(', ');\n      errorDetails.push(`Unhandled action types: ${types}`);\n    }\n\n    const errorMessage = [\n      'No actions found in UI-TARS response.',\n      ...errorDetails,\n      `\\nRaw response: ${res.content}`,\n    ].join('\\n');\n\n    throw new Error(errorMessage, {\n      cause: {\n        prediction: res.content,\n        parsed,\n        unhandledActions,\n        convertedText,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  const log = getSummary(res.content);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: log,\n  });\n\n  return {\n    actions: transformActions,\n    log,\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n    shouldContinuePlanning: shouldContinue,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface LeftDoubleAction extends BaseAction {\n  action_type: 'left_double';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface RightSingleAction extends BaseAction {\n  action_type: 'right_single';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | LeftDoubleAction\n  | RightSingleAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  uiTarsVersion: UITarsModelVersion | undefined,\n) {\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","uiTarsPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","uiTarsModelVersion","instruction","systemPrompt","getUiTarsPlanningPrompt","screenshotBase64","imagePayload","resizeImageForUiTars","res","callAIWithStringResponse","convertedText","convertBboxToCoordinates","size","parsed","actionParser","JSON","transformActions","unhandledActions","shouldContinue","action","actionType","assert","getPoint","startPoint","endPoint","keys","transformHotkeyInput","console","errorDetails","types","a","errorMessage","Error","log","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;;;;;;AA0BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,eACpBC,eAAuB,EACvBC,OAKC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IACrE,MAAM,EAAEK,kBAAkB,EAAE,GAAGF;IAE/B,IAAIG,cAAcP;IAClB,IAAIK,eACFE,cAAc,CAAC,yBAAyB,EAAEF,cAAc,8CAA8C,EAAEL,gBAAgB,mBAAmB,CAAC;IAG9I,MAAMQ,eAAeC,4BAA4BF;IAEjD,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,OAAO;IACnD,MAAMQ,eAAe,MAAMC,qBACzBF,kBACAP,QAAQ,IAAI,EACZG;IAGFJ,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKS;gBACP;YACF;SACD;IACH;IAEA,MAAME,MAAM,MAAMC,yBAChB;QACE;YACE,MAAM;YACN,SAASN;QACX;WACGN,oBAAoB,QAAQ;KAChC,EACDE;IAEF,MAAMW,gBAAgBC,yBAAyBH,IAAI,OAAO;IAE1D,MAAM,EAAEI,IAAI,EAAE,GAAGd;IACjB,MAAM,EAAEe,MAAM,EAAE,GAAGC,aAAa;QAC9B,YAAYJ;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOE,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUX;IACZ;IAEAf,MACE,oBACAe,oBACA,YACAc,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7C,MAAMC,mBAA6D,EAAE;IACrE,IAAIC,iBAAiB;IACrBL,OAAO,OAAO,CAAC,CAACM;QACd,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM7B,QAAQgC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YACvDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQG,OAAO,OAAO,IAAI;wBAC1B,MAAM9B,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BsB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;YACF;QACF,OAAO,IAAIQ,AAAe,kBAAfA,YAA8B;YACvCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM7B,QAAQgC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YACvDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQG,OAAO,OAAO,IAAI;wBAC1B,MAAM9B,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BsB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASO,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,mBAAfA,YAA+B;YACxCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM7B,QAAQgC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YACvDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQ;wBACN,QAAQG,OAAO,OAAO,IAAI;wBAC1B,MAAM9B,YACJ;4BAAE,GAAGC,KAAK,CAAC,EAAE;4BAAE,GAAGA,KAAK,CAAC,EAAE;wBAAC,GAC3BsB,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASO,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YAAuB;YAChCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,OAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMI,aAAaD,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YAC5D,MAAMY,WAAWF,SAASH,OAAO,aAAa,CAAC,OAAO,EAAEP;YACxDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQG,OAAO,OAAO,IAAI;wBAC1B,MAAM9B,YACJ;4BAAE,GAAGkC,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCX,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQO,OAAO,OAAO,IAAI;wBAC1B,MAAM9B,YACJ;4BAAE,GAAGmC,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjCZ,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASO,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOG,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWG,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YAA2B;YACpCF,iBAAiB;YACjBF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO,CAAC;gBACR,SAASG,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMM,OAAOC,qBAAqBP,OAAO,aAAa,CAAC,GAAG;YAE1DH,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASS,KAAK,IAAI,CAAC;gBACrB;gBACA,SAASN,OAAO,OAAO,IAAI;YAC7B;QACF,OAbEQ,QAAQ,IAAI,CACV;aAaC,IAAIP,AAAe,WAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASG,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,YAAY;YAErBH,iBAAiB,IAAI,CAAC;gBACpB,MAAMG;gBACN,SAASD,OAAO,OAAO,IAAI;YAC7B;YACAjC,MAAM,0BAA0BkC,YAAY,YAAYD,OAAO,OAAO;QACxE;IACF;IAEA,IAAIH,AAA4B,MAA5BA,iBAAiB,MAAM,EAAQ;QACjC,MAAMY,eAAyB,EAAE;QAGjC,IAAIf,AAAkB,MAAlBA,OAAO,MAAM,EAAQ;YACvBe,aAAa,IAAI,CAAC;YAGlB,IACEpB,IAAI,OAAO,CAAC,QAAQ,CAAC,eACrB,CAACA,IAAI,OAAO,CAAC,QAAQ,CAAC,YAEtBoB,aAAa,IAAI,CACf;iBAGFA,aAAa,IAAI,CAAC;QAEtB;QAGA,IAAIX,iBAAiB,MAAM,GAAG,GAAG;YAC/B,MAAMY,QAAQZ,iBAAiB,GAAG,CAAC,CAACa,IAAMA,EAAE,IAAI,EAAE,IAAI,CAAC;YACvDF,aAAa,IAAI,CAAC,CAAC,wBAAwB,EAAEC,OAAO;QACtD;QAEA,MAAME,eAAe;YACnB;eACGH;YACH,CAAC,gBAAgB,EAAEpB,IAAI,OAAO,EAAE;SACjC,CAAC,IAAI,CAAC;QAEP,MAAM,IAAIwB,MAAMD,cAAc;YAC5B,OAAO;gBACL,YAAYvB,IAAI,OAAO;gBACvBK;gBACAI;gBACAP;YACF;QACF;IACF;IAEAxB,MAAM,oBAAoB6B,KAAK,SAAS,CAACC,kBAAkB,MAAM;IACjE,MAAMiB,MAAMC,WAAW1B,IAAI,OAAO;IAElCX,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAASoC;IACX;IAEA,OAAO;QACL,SAASjB;QACTiB;QACA,OAAOzB,IAAI,KAAK;QAChB,aAAaO,KAAK,SAAS,CAACP,IAAI,OAAO,EAAE2B,QAAW;QACpD,wBAAwBjB;IAC1B;AACF;AAOA,SAASP,yBAAyByB,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAIxD,KAAK,KAAK,CAAEmD,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAIzD,KAAK,KAAK,CAAEqD,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAAShB,SAAS8B,QAAgB,EAAExC,IAAuC;IACzE,MAAM,CAACqC,GAAGC,EAAE,GAAGnC,KAAK,KAAK,CAACqC;IAC1B,OAAO;QAACH,IAAIrC,KAAK,KAAK;QAAEsC,IAAItC,KAAK,MAAM;KAAC;AAC1C;AAkFO,eAAeL,qBACpB8C,WAAmB,EACnBzC,IAAU,EACV0C,aAA6C;IAE7C,IAAIA,kBAAkBC,mBAAmB,IAAI,EAAE;QAC7CrE,MAAM,uCAAuC0B;QAC7C,MAAM4C,gBAAgB5C,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAM6C,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAejE,KAAK,IAAI,CAACgE,YAAYD;YAC3C,MAAMG,WAAWlE,KAAK,KAAK,CAACmB,KAAK,KAAK,GAAG8C;YACzC,MAAME,YAAYnE,KAAK,KAAK,CAACmB,KAAK,MAAM,GAAG8C;YAC3CxE,MACE,2DACAyE,UACAC;YAEF,MAAMC,eAAe,MAAMC,gBAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}
+{"version":3,"file":"ai-model/ui-tars-planning.mjs","sources":["../../../src/ai-model/ui-tars-planning.ts"],"sourcesContent":["import type {\n  PlanningAIResponse,\n  PlanningAction,\n  Size,\n  UIContext,\n} from '@/types';\nimport { type IModelConfig, UITarsModelVersion } from '@midscene/shared/env';\nimport { resizeImgBase64 } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { transformHotkeyInput } from '@midscene/shared/us-keyboard-layout';\nimport { assert } from '@midscene/shared/utils';\nimport { actionParser } from '@ui-tars/action-parser';\nimport type { ConversationHistory } from './conversation-history';\nimport { getSummary, getUiTarsPlanningPrompt } from './prompt/ui-tars-planning';\nimport { callAIWithStringResponse } from './service-caller/index';\n\ntype ActionType =\n  | 'click'\n  | 'left_double'\n  | 'right_single'\n  | 'drag'\n  | 'type'\n  | 'hotkey'\n  | 'finished'\n  | 'scroll'\n  | 'wait';\n\nconst debug = getDebug('ui-tars-planning');\nconst bboxSize = 10;\nconst pointToBbox = (\n  point: { x: number; y: number },\n  width: number,\n  height: number,\n): [number, number, number, number] => {\n  return [\n    Math.round(Math.max(point.x - bboxSize / 2, 0)),\n    Math.round(Math.max(point.y - bboxSize / 2, 0)),\n    Math.round(Math.min(point.x + bboxSize / 2, width)),\n    Math.round(Math.min(point.y + bboxSize / 2, height)),\n  ];\n};\n\nexport async function uiTarsPlanning(\n  userInstruction: string,\n  options: {\n    conversationHistory: ConversationHistory;\n    context: UIContext;\n    modelConfig: IModelConfig;\n    actionContext?: string;\n  },\n): Promise<PlanningAIResponse> {\n  const { conversationHistory, context, modelConfig, actionContext } = options;\n  const { uiTarsModelVersion } = modelConfig;\n\n  let instruction = userInstruction;\n  if (actionContext) {\n    instruction = `<high_priority_knowledge>${actionContext}</high_priority_knowledge>\\n<user_instruction>${userInstruction}</user_instruction>`;\n  }\n\n  const systemPrompt = getUiTarsPlanningPrompt() + instruction;\n\n  const screenshotBase64 = context.screenshot.base64;\n  const imagePayload = await resizeImageForUiTars(\n    screenshotBase64,\n    context.size,\n    uiTarsModelVersion,\n  );\n\n  conversationHistory.append({\n    role: 'user',\n    content: [\n      {\n        type: 'image_url',\n        image_url: {\n          url: imagePayload,\n        },\n      },\n    ],\n  });\n\n  const res = await callAIWithStringResponse(\n    [\n      {\n        role: 'user',\n        content: systemPrompt,\n      },\n      ...conversationHistory.snapshot(),\n    ],\n    modelConfig,\n  );\n  const convertedText = convertBboxToCoordinates(res.content);\n\n  const { size } = context;\n  const { parsed } = actionParser({\n    prediction: convertedText,\n    factor: [1000, 1000],\n    screenContext: {\n      width: size.width,\n      height: size.height,\n    },\n    modelVer: uiTarsModelVersion,\n  });\n\n  debug(\n    'ui-tars modelVer',\n    uiTarsModelVersion,\n    ', parsed',\n    JSON.stringify(parsed),\n  );\n\n  const transformActions: PlanningAction[] = [];\n  const unhandledActions: Array<{ type: string; thought: string }> = [];\n  let shouldContinue = true;\n  parsed.forEach((action) => {\n    const actionType = (action.action_type || '').toLowerCase();\n    if (actionType === 'click') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n\n      const locate = {\n        prompt: action.thought || '',\n        bbox: pointToBbox(\n          { x: point[0], y: point[1] },\n          size.width,\n          size.height,\n        ),\n      };\n\n      transformActions.push({\n        type: 'Tap',\n        param: {\n          locate: locate,\n        },\n      });\n    } else if (actionType === 'left_double') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n\n      const locate = {\n        prompt: action.thought || '',\n        bbox: pointToBbox(\n          { x: point[0], y: point[1] },\n          size.width,\n          size.height,\n        ),\n      };\n\n      transformActions.push({\n        type: 'DoubleClick',\n        param: {\n          locate: locate,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'right_single') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      const point = getPoint(action.action_inputs.start_box, size);\n\n      const locate = {\n        prompt: action.thought || '',\n        bbox: pointToBbox(\n          { x: point[0], y: point[1] },\n          size.width,\n          size.height,\n        ),\n      };\n\n      transformActions.push({\n        type: 'RightClick',\n        param: {\n          locate: locate,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'drag') {\n      assert(action.action_inputs.start_box, 'start_box is required');\n      assert(action.action_inputs.end_box, 'end_box is required');\n      const startPoint = getPoint(action.action_inputs.start_box, size);\n      const endPoint = getPoint(action.action_inputs.end_box, size);\n      transformActions.push({\n        type: 'DragAndDrop',\n        param: {\n          from: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: startPoint[0], y: startPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n          to: {\n            prompt: action.thought || '',\n            bbox: pointToBbox(\n              { x: endPoint[0], y: endPoint[1] },\n              size.width,\n              size.height,\n            ),\n          },\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'type') {\n      transformActions.push({\n        type: 'Input',\n        param: {\n          value: action.action_inputs.content,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'scroll') {\n      transformActions.push({\n        type: 'Scroll',\n        param: {\n          direction: action.action_inputs.direction,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'finished') {\n      shouldContinue = false;\n      transformActions.push({\n        type: 'Finished',\n        param: {},\n        thought: action.thought || '',\n      });\n    } else if (actionType === 'hotkey') {\n      if (!action.action_inputs.key) {\n        console.warn(\n          'No key found in action: hotkey. Will not perform action.',\n        );\n      } else {\n        const keys = transformHotkeyInput(action.action_inputs.key);\n\n        transformActions.push({\n          type: 'KeyboardPress',\n          param: {\n            keyName: keys.join('+'),\n          },\n          thought: action.thought || '',\n        });\n      }\n    } else if (actionType === 'wait') {\n      transformActions.push({\n        type: 'Sleep',\n        param: {\n          timeMs: 1000,\n        },\n        thought: action.thought || '',\n      });\n    } else if (actionType) {\n      // Track unhandled action types\n      unhandledActions.push({\n        type: actionType,\n        thought: action.thought || '',\n      });\n      debug('Unhandled action type:', actionType, 'thought:', action.thought);\n    }\n  });\n\n  if (transformActions.length === 0) {\n    const errorDetails: string[] = [];\n\n    // Check if parsing failed\n    if (parsed.length === 0) {\n      errorDetails.push('Action parser returned no actions');\n\n      // Check if response has Thought but no Action\n      if (\n        res.content.includes('Thought:') &&\n        !res.content.includes('Action:')\n      ) {\n        errorDetails.push(\n          'Response contains \"Thought:\" but missing \"Action:\" line',\n        );\n      } else {\n        errorDetails.push('Response may be malformed or empty');\n      }\n    }\n\n    // Check if we have unhandled action types\n    if (unhandledActions.length > 0) {\n      const types = unhandledActions.map((a) => a.type).join(', ');\n      errorDetails.push(`Unhandled action types: ${types}`);\n    }\n\n    const errorMessage = [\n      'No actions found in UI-TARS response.',\n      ...errorDetails,\n      `\\nRaw response: ${res.content}`,\n    ].join('\\n');\n\n    throw new Error(errorMessage, {\n      cause: {\n        prediction: res.content,\n        parsed,\n        unhandledActions,\n        convertedText,\n      },\n    });\n  }\n\n  debug('transformActions', JSON.stringify(transformActions, null, 2));\n  const log = getSummary(res.content);\n\n  conversationHistory.append({\n    role: 'assistant',\n    content: log,\n  });\n\n  return {\n    actions: transformActions,\n    log,\n    usage: res.usage,\n    rawResponse: JSON.stringify(res.content, undefined, 2),\n    shouldContinuePlanning: shouldContinue,\n  };\n}\n\n/**\n * Converts bounding box notation to coordinate points\n * @param text - The text containing bbox tags to be converted\n * @returns The text with bbox tags replaced by coordinate points\n */\nfunction convertBboxToCoordinates(text: string): string {\n  // Match the four numbers after <bbox>\n  const pattern = /<bbox>(\\d+)\\s+(\\d+)\\s+(\\d+)\\s+(\\d+)<\\/bbox>/g;\n\n  function replaceMatch(\n    match: string,\n    x1: string,\n    y1: string,\n    x2: string,\n    y2: string,\n  ): string {\n    // Convert strings to numbers and calculate center point\n    const x1Num = Number.parseInt(x1, 10);\n    const y1Num = Number.parseInt(y1, 10);\n    const x2Num = Number.parseInt(x2, 10);\n    const y2Num = Number.parseInt(y2, 10);\n\n    // Use Math.floor to truncate and calculate center point\n    const x = Math.floor((x1Num + x2Num) / 2);\n    const y = Math.floor((y1Num + y2Num) / 2);\n\n    // Return formatted coordinate string\n    return `(${x},${y})`;\n  }\n\n  // Remove [EOS] and replace <bbox> coordinates\n  const cleanedText = text.replace(/\\[EOS\\]/g, '');\n  return cleanedText.replace(pattern, replaceMatch).trim();\n}\n\nfunction getPoint(startBox: string, size: { width: number; height: number }) {\n  const [x, y] = JSON.parse(startBox);\n  return [x * size.width, y * size.height];\n}\n\ninterface BaseAction {\n  action_type: ActionType;\n  action_inputs: Record<string, any>;\n  reflection: string | null;\n  thought: string | null;\n}\n\ninterface ClickAction extends BaseAction {\n  action_type: 'click';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface DragAction extends BaseAction {\n  action_type: 'drag';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n    end_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface WaitAction extends BaseAction {\n  action_type: 'wait';\n  action_inputs: {\n    time: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface LeftDoubleAction extends BaseAction {\n  action_type: 'left_double';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface RightSingleAction extends BaseAction {\n  action_type: 'right_single';\n  action_inputs: {\n    start_box: string; // JSON string of [x, y] coordinates\n  };\n}\n\ninterface TypeAction extends BaseAction {\n  action_type: 'type';\n  action_inputs: {\n    content: string;\n  };\n}\n\ninterface HotkeyAction extends BaseAction {\n  action_type: 'hotkey';\n  action_inputs: {\n    key: string;\n  };\n}\n\ninterface ScrollAction extends BaseAction {\n  action_type: 'scroll';\n  action_inputs: {\n    direction: 'up' | 'down';\n  };\n}\n\ninterface FinishedAction extends BaseAction {\n  action_type: 'finished';\n  action_inputs: Record<string, never>;\n}\n\nexport type Action =\n  | ClickAction\n  | LeftDoubleAction\n  | RightSingleAction\n  | DragAction\n  | TypeAction\n  | HotkeyAction\n  | ScrollAction\n  | FinishedAction\n  | WaitAction;\n\nexport async function resizeImageForUiTars(\n  imageBase64: string,\n  size: Size,\n  uiTarsVersion: UITarsModelVersion | undefined,\n) {\n  if (uiTarsVersion === UITarsModelVersion.V1_5) {\n    debug('ui-tars-v1.5, will check image size', size);\n    const currentPixels = size.width * size.height;\n    const maxPixels = 16384 * 28 * 28; //\n    if (currentPixels > maxPixels) {\n      const resizeFactor = Math.sqrt(maxPixels / currentPixels);\n      const newWidth = Math.floor(size.width * resizeFactor);\n      const newHeight = Math.floor(size.height * resizeFactor);\n      debug(\n        'resize image for ui-tars, new width: %s, new height: %s',\n        newWidth,\n        newHeight,\n      );\n      const resizedImage = await resizeImgBase64(imageBase64, {\n        width: newWidth,\n        height: newHeight,\n      });\n      return resizedImage;\n    }\n  }\n  return imageBase64;\n}\n"],"names":["debug","getDebug","bboxSize","pointToBbox","point","width","height","Math","uiTarsPlanning","userInstruction","options","conversationHistory","context","modelConfig","actionContext","uiTarsModelVersion","instruction","systemPrompt","getUiTarsPlanningPrompt","screenshotBase64","imagePayload","resizeImageForUiTars","res","callAIWithStringResponse","convertedText","convertBboxToCoordinates","size","parsed","actionParser","JSON","transformActions","unhandledActions","shouldContinue","action","actionType","assert","getPoint","locate","startPoint","endPoint","keys","transformHotkeyInput","console","errorDetails","types","a","errorMessage","Error","log","getSummary","undefined","text","pattern","replaceMatch","match","x1","y1","x2","y2","x1Num","Number","y1Num","x2Num","y2Num","x","y","cleanedText","startBox","imageBase64","uiTarsVersion","UITarsModelVersion","currentPixels","maxPixels","resizeFactor","newWidth","newHeight","resizedImage","resizeImgBase64"],"mappings":";;;;;;;;AA2BA,MAAMA,QAAQC,SAAS;AACvB,MAAMC,WAAW;AACjB,MAAMC,cAAc,CAClBC,OACAC,OACAC,SAEO;QACLC,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAG;QAC5CK,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGG;QAC5CE,KAAK,KAAK,CAACA,KAAK,GAAG,CAACH,MAAM,CAAC,GAAGF,WAAW,GAAGI;KAC7C;AAGI,eAAeE,eACpBC,eAAuB,EACvBC,OAKC;IAED,MAAM,EAAEC,mBAAmB,EAAEC,OAAO,EAAEC,WAAW,EAAEC,aAAa,EAAE,GAAGJ;IACrE,MAAM,EAAEK,kBAAkB,EAAE,GAAGF;IAE/B,IAAIG,cAAcP;IAClB,IAAIK,eACFE,cAAc,CAAC,yBAAyB,EAAEF,cAAc,8CAA8C,EAAEL,gBAAgB,mBAAmB,CAAC;IAG9I,MAAMQ,eAAeC,4BAA4BF;IAEjD,MAAMG,mBAAmBP,QAAQ,UAAU,CAAC,MAAM;IAClD,MAAMQ,eAAe,MAAMC,qBACzBF,kBACAP,QAAQ,IAAI,EACZG;IAGFJ,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAAS;YACP;gBACE,MAAM;gBACN,WAAW;oBACT,KAAKS;gBACP;YACF;SACD;IACH;IAEA,MAAME,MAAM,MAAMC,yBAChB;QACE;YACE,MAAM;YACN,SAASN;QACX;WACGN,oBAAoB,QAAQ;KAChC,EACDE;IAEF,MAAMW,gBAAgBC,yBAAyBH,IAAI,OAAO;IAE1D,MAAM,EAAEI,IAAI,EAAE,GAAGd;IACjB,MAAM,EAAEe,MAAM,EAAE,GAAGC,aAAa;QAC9B,YAAYJ;QACZ,QAAQ;YAAC;YAAM;SAAK;QACpB,eAAe;YACb,OAAOE,KAAK,KAAK;YACjB,QAAQA,KAAK,MAAM;QACrB;QACA,UAAUX;IACZ;IAEAf,MACE,oBACAe,oBACA,YACAc,KAAK,SAAS,CAACF;IAGjB,MAAMG,mBAAqC,EAAE;IAC7C,MAAMC,mBAA6D,EAAE;IACrE,IAAIC,iBAAiB;IACrBL,OAAO,OAAO,CAAC,CAACM;QACd,MAAMC,aAAcD,AAAAA,CAAAA,OAAO,WAAW,IAAI,EAAC,EAAG,WAAW;QACzD,IAAIC,AAAe,YAAfA,YAAwB;YAC1BC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM7B,QAAQgC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YAEvD,MAAMW,SAAS;gBACb,QAAQJ,OAAO,OAAO,IAAI;gBAC1B,MAAM9B,YACJ;oBAAE,GAAGC,KAAK,CAAC,EAAE;oBAAE,GAAGA,KAAK,CAAC,EAAE;gBAAC,GAC3BsB,KAAK,KAAK,EACVA,KAAK,MAAM;YAEf;YAEAI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQO;gBACV;YACF;QACF,OAAO,IAAIH,AAAe,kBAAfA,YAA8B;YACvCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM7B,QAAQgC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YAEvD,MAAMW,SAAS;gBACb,QAAQJ,OAAO,OAAO,IAAI;gBAC1B,MAAM9B,YACJ;oBAAE,GAAGC,KAAK,CAAC,EAAE;oBAAE,GAAGA,KAAK,CAAC,EAAE;gBAAC,GAC3BsB,KAAK,KAAK,EACVA,KAAK,MAAM;YAEf;YAEAI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQO;gBACV;gBACA,SAASJ,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,mBAAfA,YAA+B;YACxCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvC,MAAM7B,QAAQgC,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YAEvD,MAAMW,SAAS;gBACb,QAAQJ,OAAO,OAAO,IAAI;gBAC1B,MAAM9B,YACJ;oBAAE,GAAGC,KAAK,CAAC,EAAE;oBAAE,GAAGA,KAAK,CAAC,EAAE;gBAAC,GAC3BsB,KAAK,KAAK,EACVA,KAAK,MAAM;YAEf;YAEAI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,QAAQO;gBACV;gBACA,SAASJ,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YAAuB;YAChCC,OAAOF,OAAO,aAAa,CAAC,SAAS,EAAE;YACvCE,OAAOF,OAAO,aAAa,CAAC,OAAO,EAAE;YACrC,MAAMK,aAAaF,SAASH,OAAO,aAAa,CAAC,SAAS,EAAEP;YAC5D,MAAMa,WAAWH,SAASH,OAAO,aAAa,CAAC,OAAO,EAAEP;YACxDI,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,MAAM;wBACJ,QAAQG,OAAO,OAAO,IAAI;wBAC1B,MAAM9B,YACJ;4BAAE,GAAGmC,UAAU,CAAC,EAAE;4BAAE,GAAGA,UAAU,CAAC,EAAE;wBAAC,GACrCZ,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;oBACA,IAAI;wBACF,QAAQO,OAAO,OAAO,IAAI;wBAC1B,MAAM9B,YACJ;4BAAE,GAAGoC,QAAQ,CAAC,EAAE;4BAAE,GAAGA,QAAQ,CAAC,EAAE;wBAAC,GACjCb,KAAK,KAAK,EACVA,KAAK,MAAM;oBAEf;gBACF;gBACA,SAASO,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,WAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,OAAOG,OAAO,aAAa,CAAC,OAAO;YACrC;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,aAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,WAAWG,OAAO,aAAa,CAAC,SAAS;YAC3C;YACA,SAASA,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,AAAe,eAAfA,YAA2B;YACpCF,iBAAiB;YACjBF,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO,CAAC;gBACR,SAASG,OAAO,OAAO,IAAI;YAC7B;QACF,OAAO,IAAIC,AAAe,aAAfA,YACT,IAAKD,OAAO,aAAa,CAAC,GAAG,EAItB;YACL,MAAMO,OAAOC,qBAAqBR,OAAO,aAAa,CAAC,GAAG;YAE1DH,iBAAiB,IAAI,CAAC;gBACpB,MAAM;gBACN,OAAO;oBACL,SAASU,KAAK,IAAI,CAAC;gBACrB;gBACA,SAASP,OAAO,OAAO,IAAI;YAC7B;QACF,OAbES,QAAQ,IAAI,CACV;aAaC,IAAIR,AAAe,WAAfA,YACTJ,iBAAiB,IAAI,CAAC;YACpB,MAAM;YACN,OAAO;gBACL,QAAQ;YACV;YACA,SAASG,OAAO,OAAO,IAAI;QAC7B;aACK,IAAIC,YAAY;YAErBH,iBAAiB,IAAI,CAAC;gBACpB,MAAMG;gBACN,SAASD,OAAO,OAAO,IAAI;YAC7B;YACAjC,MAAM,0BAA0BkC,YAAY,YAAYD,OAAO,OAAO;QACxE;IACF;IAEA,IAAIH,AAA4B,MAA5BA,iBAAiB,MAAM,EAAQ;QACjC,MAAMa,eAAyB,EAAE;QAGjC,IAAIhB,AAAkB,MAAlBA,OAAO,MAAM,EAAQ;YACvBgB,aAAa,IAAI,CAAC;YAGlB,IACErB,IAAI,OAAO,CAAC,QAAQ,CAAC,eACrB,CAACA,IAAI,OAAO,CAAC,QAAQ,CAAC,YAEtBqB,aAAa,IAAI,CACf;iBAGFA,aAAa,IAAI,CAAC;QAEtB;QAGA,IAAIZ,iBAAiB,MAAM,GAAG,GAAG;YAC/B,MAAMa,QAAQb,iBAAiB,GAAG,CAAC,CAACc,IAAMA,EAAE,IAAI,EAAE,IAAI,CAAC;YACvDF,aAAa,IAAI,CAAC,CAAC,wBAAwB,EAAEC,OAAO;QACtD;QAEA,MAAME,eAAe;YACnB;eACGH;YACH,CAAC,gBAAgB,EAAErB,IAAI,OAAO,EAAE;SACjC,CAAC,IAAI,CAAC;QAEP,MAAM,IAAIyB,MAAMD,cAAc;YAC5B,OAAO;gBACL,YAAYxB,IAAI,OAAO;gBACvBK;gBACAI;gBACAP;YACF;QACF;IACF;IAEAxB,MAAM,oBAAoB6B,KAAK,SAAS,CAACC,kBAAkB,MAAM;IACjE,MAAMkB,MAAMC,WAAW3B,IAAI,OAAO;IAElCX,oBAAoB,MAAM,CAAC;QACzB,MAAM;QACN,SAASqC;IACX;IAEA,OAAO;QACL,SAASlB;QACTkB;QACA,OAAO1B,IAAI,KAAK;QAChB,aAAaO,KAAK,SAAS,CAACP,IAAI,OAAO,EAAE4B,QAAW;QACpD,wBAAwBlB;IAC1B;AACF;AAOA,SAASP,yBAAyB0B,IAAY;IAE5C,MAAMC,UAAU;IAEhB,SAASC,aACPC,KAAa,EACbC,EAAU,EACVC,EAAU,EACVC,EAAU,EACVC,EAAU;QAGV,MAAMC,QAAQC,OAAO,QAAQ,CAACL,IAAI;QAClC,MAAMM,QAAQD,OAAO,QAAQ,CAACJ,IAAI;QAClC,MAAMM,QAAQF,OAAO,QAAQ,CAACH,IAAI;QAClC,MAAMM,QAAQH,OAAO,QAAQ,CAACF,IAAI;QAGlC,MAAMM,IAAIzD,KAAK,KAAK,CAAEoD,AAAAA,CAAAA,QAAQG,KAAI,IAAK;QACvC,MAAMG,IAAI1D,KAAK,KAAK,CAAEsD,AAAAA,CAAAA,QAAQE,KAAI,IAAK;QAGvC,OAAO,CAAC,CAAC,EAAEC,EAAE,CAAC,EAAEC,EAAE,CAAC,CAAC;IACtB;IAGA,MAAMC,cAAcf,KAAK,OAAO,CAAC,YAAY;IAC7C,OAAOe,YAAY,OAAO,CAACd,SAASC,cAAc,IAAI;AACxD;AAEA,SAASjB,SAAS+B,QAAgB,EAAEzC,IAAuC;IACzE,MAAM,CAACsC,GAAGC,EAAE,GAAGpC,KAAK,KAAK,CAACsC;IAC1B,OAAO;QAACH,IAAItC,KAAK,KAAK;QAAEuC,IAAIvC,KAAK,MAAM;KAAC;AAC1C;AAkFO,eAAeL,qBACpB+C,WAAmB,EACnB1C,IAAU,EACV2C,aAA6C;IAE7C,IAAIA,kBAAkBC,mBAAmB,IAAI,EAAE;QAC7CtE,MAAM,uCAAuC0B;QAC7C,MAAM6C,gBAAgB7C,KAAK,KAAK,GAAGA,KAAK,MAAM;QAC9C,MAAM8C,YAAY;QAClB,IAAID,gBAAgBC,WAAW;YAC7B,MAAMC,eAAelE,KAAK,IAAI,CAACiE,YAAYD;YAC3C,MAAMG,WAAWnE,KAAK,KAAK,CAACmB,KAAK,KAAK,GAAG+C;YACzC,MAAME,YAAYpE,KAAK,KAAK,CAACmB,KAAK,MAAM,GAAG+C;YAC3CzE,MACE,2DACA0E,UACAC;YAEF,MAAMC,eAAe,MAAMC,gBAAgBT,aAAa;gBACtD,OAAOM;gBACP,QAAQC;YACV;YACA,OAAOC;QACT;IACF;IACA,OAAOR;AACT"}

package/dist/es/common.mjs

@@ -6,6 +6,19 @@ import { getDebug } from "@midscene/shared/logger";
 import { z } from "zod";
 const defaultBboxSize = 20;
 const debugInspectUtils = getDebug('ai:common');
+function pointToBbox(x, y, bboxSize = defaultBboxSize) {
+    const halfSize = bboxSize / 2;
+    const x1 = Math.max(x - halfSize, 0);
+    const y1 = Math.max(y - halfSize, 0);
+    const x2 = Math.min(x + halfSize, 1000);
+    const y2 = Math.min(y + halfSize, 1000);
+    return [
+        x1,
+        y1,
+        x2,
+        y2
+    ];
+}
 function fillBboxParam(locate, width, height, rightLimit, bottomLimit, vlMode) {
     if (locate.bbox_2d && !locate?.bbox) {
         locate.bbox = locate.bbox_2d;
@@ -86,7 +99,7 @@ function adaptBbox(bbox, width, height, rightLimit, bottomLimit, vlMode) {
         0,
         0
     ];
-    result = 'doubao-vision' === vlMode || 'vlm-ui-tars' === vlMode ? adaptDoubaoBbox(normalizedBbox, width, height) : 'gemini' === vlMode ? adaptGeminiBbox(normalizedBbox, width, height) : 'qwen3-vl' === vlMode || 'glm-v' === vlMode ? normalized01000(normalizedBbox, width, height) : adaptQwen2_5Bbox(normalizedBbox);
+    result = 'doubao-vision' === vlMode || 'vlm-ui-tars' === vlMode ? adaptDoubaoBbox(normalizedBbox, width, height) : 'gemini' === vlMode ? adaptGeminiBbox(normalizedBbox, width, height) : 'qwen2.5-vl' === vlMode ? adaptQwen2_5Bbox(normalizedBbox) : normalized01000(normalizedBbox, width, height);
     result[2] = Math.min(result[2], rightLimit);
     result[3] = Math.min(result[3], bottomLimit);
     return result;
@@ -340,6 +353,6 @@ const getReadableTimeString = (format = 'YYYY-MM-DD HH:mm:ss')=>{
     const timeString = format.replace('YYYY', String(year)).replace('MM', month).replace('DD', day).replace('HH', hours).replace('mm', minutes).replace('ss', seconds);
     return `${timeString} (${format})`;
 };
-export { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwen2_5Bbox, buildYamlFlowFromPlans, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, fillBboxParam, finalizeActionName, findAllMidsceneLocatorField, getMidsceneLocationSchema, getReadableTimeString, ifMidsceneLocatorField, loadActionParam, markupImageForLLM, mergeRects, normalized01000, parseActionParam };
+export { PointSchema, RectSchema, SizeSchema, TMultimodalPromptSchema, TUserPromptSchema, adaptBbox, adaptBboxToRect, adaptDoubaoBbox, adaptGeminiBbox, adaptQwen2_5Bbox, buildYamlFlowFromPlans, dumpActionParam, dumpMidsceneLocatorField, expandSearchArea, fillBboxParam, finalizeActionName, findAllMidsceneLocatorField, getMidsceneLocationSchema, getReadableTimeString, ifMidsceneLocatorField, loadActionParam, markupImageForLLM, mergeRects, normalized01000, parseActionParam, pointToBbox };
 
 //# sourceMappingURL=common.mjs.map

package/dist/es/common.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert, isPlainObject } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\ntype AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(\n      locate.bbox,\n      width,\n      height,\n      rightLimit,\n      bottomLimit,\n      vlMode,\n    );\n  }\n\n  return locate;\n}\n\nexport function adaptQwen2_5Bbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction normalizeBboxInput(\n  bbox: AdaptBboxInput,\n): number[] | string[] | string {\n  if (Array.isArray(bbox)) {\n    if (Array.isArray(bbox[0])) {\n      return bbox[0] as number[] | string[];\n    }\n    return bbox as number[] | string[];\n  }\n  return bbox as string;\n}\n\nexport function adaptBbox(\n  bbox: AdaptBboxInput,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  const normalizedBbox = normalizeBboxInput(bbox);\n\n  let result: [number, number, number, number] = [0, 0, 0, 0];\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    result = adaptDoubaoBbox(normalizedBbox, width, height);\n  } else if (vlMode === 'gemini') {\n    result = adaptGeminiBbox(normalizedBbox as number[], width, height);\n  } else if (vlMode === 'qwen3-vl' || vlMode === 'glm-v') {\n    result = normalized01000(normalizedBbox as number[], width, height);\n  } else {\n    result = adaptQwen2_5Bbox(normalizedBbox as number[]);\n  }\n\n  result[2] = Math.min(result[2], rightLimit);\n  result[3] = Math.min(result[3], bottomLimit);\n\n  return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  return [\n    Math.round((bbox[0] * width) / 1000),\n    Math.round((bbox[1] * height) / 1000),\n    Math.round((bbox[2] * width) / 1000),\n    Math.round((bbox[3] * height) / 1000),\n  ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  rightLimit = width,\n  bottomLimit = height,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils(\n    'adaptBboxToRect',\n    bbox,\n    width,\n    height,\n    'offset',\n    offsetX,\n    offsetY,\n    'limit',\n    rightLimit,\n    bottomLimit,\n    'vlMode',\n    vlMode,\n  );\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    rightLimit,\n    bottomLimit,\n    vlMode,\n  );\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n\n  return rect;\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  let minEdgeSize = 500;\n  if (vlMode === 'qwen3-vl') {\n    minEdgeSize = 1200;\n  }\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepThink: z.boolean().optional(),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\n\n// Export the input type - this is the inferred type from getMidsceneLocationSchema()\nexport type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement)\n  for (const fieldName in locateFieldValues) {\n    validated[fieldName] = locateFieldValues[fieldName];\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n): string => {\n  const now = new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["defaultBboxSize","debugInspectUtils","getDebug","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwen2_5Bbox","bbox","msg","JSON","Error","result","Math","adaptDoubaoBbox","assert","splitted","Number","bboxList","Array","item","x","y","normalizeBboxInput","normalizedBbox","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","Object","key","jsonObject","zodSchema","isPlainObject","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated","finalizeActionName","getReadableTimeString","format","now","Date","year","month","day","hours","minutes","seconds","timeString"],"mappings":";;;;;;AAuBA,MAAMA,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAI5B,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,QAAQ,MAAM;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAQ,MACVA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,iBACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CC,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;QACN,YAAnB,OAAOY,IAAI,CAAC,EAAE,GACVK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,IAClBK,KAAK,KAAK,CAACL,IAAI,CAAC,EAAE,GAAGZ;KAC1B;IACD,OAAOgB;AACT;AAEO,SAASE,gBACdN,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEda,OACEd,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BO,OACE,+BAA+B,IAAI,CAACP,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMQ,WAAWR,KAAK,KAAK,CAAC;QAC5B,IAAIQ,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLH,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;YAC5CW,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAIf,QAAS;YAC3CY,KAAK,KAAK,CAAEI,OAAOD,QAAQ,CAAC,EAAE,IAAId,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIU,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACX,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACY;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAOF,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAACC,GAAGC,EAAE,GAAGF,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAOI,EAAE,IAAI,KAAKJ,OAAOK,EAAE,IAAI;QAC/C,OACEJ,SAAS,IAAI,CAACD,OAAOG;IAEzB;SAEAF,WAAWV;IAGb,IAAIU,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLL,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAIH,IACEgB,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLL,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;QAEhEiB,KAAK,GAAG,CACNZ,OACAY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS,QAAQL,kBAAkB;QAE/DiB,KAAK,GAAG,CACNX,QACAW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU,QAAQN,kBAAkB;KAEjE;IAGH,IAAIY,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLK,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;QACpCW,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGjB,QAAS;QACnCY,KAAK,KAAK,CAAEK,QAAQ,CAAC,EAAE,GAAGhB,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEA,SAASc,mBACPf,IAAoB;IAEpB,IAAIW,MAAM,OAAO,CAACX,OAChB;QAAA,IAAIW,MAAM,OAAO,CAACX,IAAI,CAAC,EAAE,GACvB,OAAOA,IAAI,CAAC,EAAE;IAChB;IAGF,OAAOA;AACT;AAEO,SAASF,UACdE,IAAoB,EACpBP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,MAAMmB,iBAAiBD,mBAAmBf;IAE1C,IAAII,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBS,gBAAgBU,gBAAgBvB,OAAOC,UACvCG,AAAW,aAAXA,SACAoB,gBAAgBD,gBAA4BvB,OAAOC,UACnDG,AAAW,eAAXA,UAAyBA,AAAW,YAAXA,SACzBqB,gBAAgBF,gBAA4BvB,OAAOC,UAEnDK,iBAAiBiB;IAG5BZ,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGC,KAAK,GAAG,CAACD,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASc,gBACdlB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCW,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BY,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASuB,gBACdjB,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMyB,OAAOd,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAM2B,MAAMf,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAM2B,QAAQhB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM6B,SAASjB,KAAK,KAAK,CAAEL,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACyB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdvB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd8B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX9B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjCR,kBACE,mBACAW,MACAP,OACAC,QACA,UACA8B,SACAC,SACA,SACA9B,YACAC,aACA,UACAC;IAEF,MAAM,CAACsB,MAAMC,KAAKC,OAAOC,OAAO,GAAGxB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM6B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYnC,OACzBmC,YAAYnC,QAAQiC;IAItB,IAAIC,UAAUE,aAAanC,QACzBmC,aAAanC,SAASiC;IAIxBC,YAAYvB,KAAK,GAAG,CAAC,GAAGuB;IACxBC,aAAaxB,KAAK,GAAG,CAAC,GAAGwB;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACAxC,kBAAkB,4BAA4ByC;IAE9C,OAAOA;AACT;AAEO,SAASC,WAAWC,KAAa;IACtC,MAAMC,UAAU5B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS9B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW/B,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAYhC,KAAK,GAAG,IAAI2B,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdR,IAAU,EACVS,UAAgB,EAChB1C,MAAgC;IAEhC,IAAI2C,cAAc;IAClB,IAAI3C,AAAW,eAAXA,QACF2C,cAAc;IAEhB,MAAMC,iBAAiB;IAGvB,MAAMC,wBACJZ,KAAK,KAAK,GAAGU,cACTnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,KAAI,IAAK,KACvCW;IACN,MAAME,sBACJb,KAAK,MAAM,GAAGU,cACVnC,KAAK,IAAI,CAAEmC,AAAAA,CAAAA,cAAcV,KAAK,MAAK,IAAK,KACxCW;IAGN,IAAIG,WAAWvC,KAAK,GAAG,CAACmC,aAAaV,KAAK,KAAK,GAAGY,AAAwB,IAAxBA;IAClD,IAAIG,YAAYxC,KAAK,GAAG,CAACmC,aAAaV,KAAK,MAAM,GAAGa,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUhB,KAAK,IAAI,GAAGY;IAC1B,IAAIK,SAASjB,KAAK,GAAG,GAAGa;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAUzC,KAAK,GAAG,CAAC,GAAGyC;IACtBC,SAAS1C,KAAK,GAAG,CAAC,GAAG0C;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCjB,KAAK,IAAI,GAAGgB;IACZhB,KAAK,GAAG,GAAGiB;IACXjB,KAAK,KAAK,GAAGc;IACbd,KAAK,MAAM,GAAGe;IAEd,OAAOf;AACT;AAEO,eAAekB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,WAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC;IAEhC,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQH,MAAO;QACxB,MAAMI,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASJ,YAAY,IAAI,CAAC,CAACI,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMK,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAN,KAAK,IAAI,CAACQ;IACZ;IAEA,OAAOR;AACT;AAGO,MAAMS,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACH;IACvC1E,OACEyE,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOI,OAAOJ;AAChB;AAEO,MAAMK,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMX,QAAQQ,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACZ,uBAAuBC,QAC1B,OAAO;YAIT,IAAIO,cACF,OAAOP,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMZ,kBAAkB,CAC7BwB,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBV,4BAA4BQ;IAClD,MAAM1F,SAAS;QAAE,GAAGyF,UAAU;IAAC;IAE/B,KAAK,MAAMI,aAAaD,cAAe;QACrC,MAAME,aAAa9F,MAAM,CAAC6F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT9F,MAAM,CAAC6F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B9F,MAAM,CAAC6F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB9F,MAAM,CAAC6F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO9F;AACT;AAEO,MAAM+F,kBAAkB,CAC7BN,YACAC;IAEA,MAAME,gBAAgBV,4BAA4BQ;IAClD,MAAM1F,SAAS;QAAE,GAAGyF,UAAU;IAAC;IAE/B,KAAK,MAAMI,aAAaD,cAAe;QACrC,MAAME,aAAa9F,MAAM,CAAC6F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB9F,MAAM,CAAC6F,UAAU,GAAG;YAClB,CAACpB,oBAAoB,EAAE;YACvB,QAAQqB;QACV;IAEJ;IAEA,OAAO9F;AACT;AAUO,MAAMgG,mBAAmB,CAC9BC,UACAP;IAGA,IAAI,CAACA,WACH;IAIF,MAAMQ,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAejB,4BAA4BQ;IAGjD,IAAIS,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOT,UAAU,KAAK,CAACQ;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMb,OAAOU,MAChB,IAAIC,aAAa,QAAQ,CAACX,MAExBa,mBAAmB,CAACb,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Ca,mBAAmB,CAACb,IAAI,GAAGU,KAAK,CAACV,IAAI;IAKzC,MAAMc,YAAYZ,UAAU,KAAK,CAACW;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT;AAEO,MAAMC,qBAAqB;AAO3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB;IAE9B,MAAMC,MAAM,IAAIC;IAChB,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQ5B,OAAOyB,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMI,MAAM7B,OAAOyB,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMK,QAAQ9B,OAAOyB,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,UAAU/B,OAAOyB,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMO,UAAUhC,OAAOyB,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMQ,aAAaT,OAChB,OAAO,CAAC,QAAQxB,OAAO2B,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAET,OAAO,CAAC,CAAC;AACpC"}
+{"version":3,"file":"common.mjs","sources":["../../src/common.ts"],"sourcesContent":["import type {\n  BaseElement,\n  DeviceAction,\n  ElementTreeNode,\n  MidsceneYamlFlowItem,\n  PlanningAction,\n  Rect,\n  Size,\n} from '@/types';\nimport { assert, isPlainObject } from '@midscene/shared/utils';\n\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\n\nimport type { PlanningLocateParam } from '@/types';\nimport { NodeType } from '@midscene/shared/constants';\nimport type { TVlModeTypes } from '@midscene/shared/env';\nimport { treeToList } from '@midscene/shared/extractor';\nimport { compositeElementInfoImg } from '@midscene/shared/img';\nimport { getDebug } from '@midscene/shared/logger';\nimport { z } from 'zod';\n\nexport type AIArgs = ChatCompletionMessageParam[];\n\nconst defaultBboxSize = 20; // must be even number\nconst debugInspectUtils = getDebug('ai:common');\ntype AdaptBboxInput = number[] | string[] | string | (number[] | string[])[];\n\n/**\n * Convert a point coordinate [0, 1000] to a small bbox [0, 1000]\n * Creates a small bbox around the center point in the same coordinate space\n *\n * @param x - X coordinate in [0, 1000] range\n * @param y - Y coordinate in [0, 1000] range\n * @param bboxSize - Size of the bbox to create (default: 20)\n * @returns [x1, y1, x2, y2] bbox in [0, 1000] coordinate space\n */\nexport function pointToBbox(\n  x: number,\n  y: number,\n  bboxSize = defaultBboxSize,\n): [number, number, number, number] {\n  const halfSize = bboxSize / 2;\n  const x1 = Math.max(x - halfSize, 0);\n  const y1 = Math.max(y - halfSize, 0);\n  const x2 = Math.min(x + halfSize, 1000);\n  const y2 = Math.min(y + halfSize, 1000);\n\n  return [x1, y1, x2, y2];\n}\n\n// transform the param of locate from qwen mode\nexport function fillBboxParam(\n  locate: PlanningLocateParam,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n) {\n  // The Qwen model might have hallucinations of naming bbox as bbox_2d.\n  if ((locate as any).bbox_2d && !locate?.bbox) {\n    locate.bbox = (locate as any).bbox_2d;\n    // biome-ignore lint/performance/noDelete: <explanation>\n    delete (locate as any).bbox_2d;\n  }\n\n  if (locate?.bbox) {\n    locate.bbox = adaptBbox(\n      locate.bbox,\n      width,\n      height,\n      rightLimit,\n      bottomLimit,\n      vlMode,\n    );\n  }\n\n  return locate;\n}\n\nexport function adaptQwen2_5Bbox(\n  bbox: number[],\n): [number, number, number, number] {\n  if (bbox.length < 2) {\n    const msg = `invalid bbox data for qwen-vl mode: ${JSON.stringify(bbox)} `;\n    throw new Error(msg);\n  }\n\n  const result: [number, number, number, number] = [\n    Math.round(bbox[0]),\n    Math.round(bbox[1]),\n    typeof bbox[2] === 'number'\n      ? Math.round(bbox[2])\n      : Math.round(bbox[0] + defaultBboxSize),\n    typeof bbox[3] === 'number'\n      ? Math.round(bbox[3])\n      : Math.round(bbox[1] + defaultBboxSize),\n  ];\n  return result;\n}\n\nexport function adaptDoubaoBbox(\n  bbox: string[] | number[] | string,\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  assert(\n    width > 0 && height > 0,\n    'width and height must be greater than 0 in doubao mode',\n  );\n\n  if (typeof bbox === 'string') {\n    assert(\n      /^(\\d+)\\s(\\d+)\\s(\\d+)\\s(\\d+)$/.test(bbox.trim()),\n      `invalid bbox data string for doubao-vision mode: ${bbox}`,\n    );\n    const splitted = bbox.split(' ');\n    if (splitted.length === 4) {\n      return [\n        Math.round((Number(splitted[0]) * width) / 1000),\n        Math.round((Number(splitted[1]) * height) / 1000),\n        Math.round((Number(splitted[2]) * width) / 1000),\n        Math.round((Number(splitted[3]) * height) / 1000),\n      ];\n    }\n    throw new Error(`invalid bbox data string for doubao-vision mode: ${bbox}`);\n  }\n\n  let bboxList: number[] = [];\n  if (Array.isArray(bbox) && typeof bbox[0] === 'string') {\n    bbox.forEach((item) => {\n      if (typeof item === 'string' && item.includes(',')) {\n        const [x, y] = item.split(',');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else if (typeof item === 'string' && item.includes(' ')) {\n        const [x, y] = item.split(' ');\n        bboxList.push(Number(x.trim()), Number(y.trim()));\n      } else {\n        bboxList.push(Number(item));\n      }\n    });\n  } else {\n    bboxList = bbox as any;\n  }\n\n  if (bboxList.length === 4 || bboxList.length === 5) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[2] * width) / 1000),\n      Math.round((bboxList[3] * height) / 1000),\n    ];\n  }\n\n  // treat the bbox as a center point\n  if (\n    bboxList.length === 6 ||\n    bboxList.length === 2 ||\n    bboxList.length === 3 ||\n    bboxList.length === 7\n  ) {\n    return [\n      Math.max(\n        0,\n        Math.round((bboxList[0] * width) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.max(\n        0,\n        Math.round((bboxList[1] * height) / 1000) - defaultBboxSize / 2,\n      ),\n      Math.min(\n        width,\n        Math.round((bboxList[0] * width) / 1000) + defaultBboxSize / 2,\n      ),\n      Math.min(\n        height,\n        Math.round((bboxList[1] * height) / 1000) + defaultBboxSize / 2,\n      ),\n    ];\n  }\n\n  if (bbox.length === 8) {\n    return [\n      Math.round((bboxList[0] * width) / 1000),\n      Math.round((bboxList[1] * height) / 1000),\n      Math.round((bboxList[4] * width) / 1000),\n      Math.round((bboxList[5] * height) / 1000),\n    ];\n  }\n\n  const msg = `invalid bbox data for doubao-vision mode: ${JSON.stringify(bbox)} `;\n  throw new Error(msg);\n}\n\nfunction normalizeBboxInput(\n  bbox: AdaptBboxInput,\n): number[] | string[] | string {\n  if (Array.isArray(bbox)) {\n    if (Array.isArray(bbox[0])) {\n      return bbox[0] as number[] | string[];\n    }\n    return bbox as number[] | string[];\n  }\n  return bbox as string;\n}\n\nexport function adaptBbox(\n  bbox: AdaptBboxInput,\n  width: number,\n  height: number,\n  rightLimit: number,\n  bottomLimit: number,\n  vlMode: TVlModeTypes | undefined,\n): [number, number, number, number] {\n  const normalizedBbox = normalizeBboxInput(bbox);\n\n  let result: [number, number, number, number] = [0, 0, 0, 0];\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\n    result = adaptDoubaoBbox(normalizedBbox, width, height);\n  } else if (vlMode === 'gemini') {\n    result = adaptGeminiBbox(normalizedBbox as number[], width, height);\n  } else if (vlMode === 'qwen2.5-vl') {\n    result = adaptQwen2_5Bbox(normalizedBbox as number[]);\n  } else {\n    // Default: normalized 0-1000 coordinate system\n    // Includes: qwen3-vl, glm-v, auto-glm, auto-glm-multilingual, and future models\n    result = normalized01000(normalizedBbox as number[], width, height);\n  }\n\n  result[2] = Math.min(result[2], rightLimit);\n  result[3] = Math.min(result[3], bottomLimit);\n\n  return result;\n}\n\n// x1, y1, x2, y2 -> 0-1000\nexport function normalized01000(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  return [\n    Math.round((bbox[0] * width) / 1000),\n    Math.round((bbox[1] * height) / 1000),\n    Math.round((bbox[2] * width) / 1000),\n    Math.round((bbox[3] * height) / 1000),\n  ];\n}\n\n// y1, x1, y2, x2 -> 0-1000\nexport function adaptGeminiBbox(\n  bbox: number[],\n  width: number,\n  height: number,\n): [number, number, number, number] {\n  const left = Math.round((bbox[1] * width) / 1000);\n  const top = Math.round((bbox[0] * height) / 1000);\n  const right = Math.round((bbox[3] * width) / 1000);\n  const bottom = Math.round((bbox[2] * height) / 1000);\n  return [left, top, right, bottom];\n}\n\nexport function adaptBboxToRect(\n  bbox: number[],\n  width: number,\n  height: number,\n  offsetX = 0,\n  offsetY = 0,\n  rightLimit = width,\n  bottomLimit = height,\n  vlMode?: TVlModeTypes | undefined,\n): Rect {\n  debugInspectUtils(\n    'adaptBboxToRect',\n    bbox,\n    width,\n    height,\n    'offset',\n    offsetX,\n    offsetY,\n    'limit',\n    rightLimit,\n    bottomLimit,\n    'vlMode',\n    vlMode,\n  );\n  const [left, top, right, bottom] = adaptBbox(\n    bbox,\n    width,\n    height,\n    rightLimit,\n    bottomLimit,\n    vlMode,\n  );\n\n  // Calculate initial rect dimensions\n  const rectLeft = left;\n  const rectTop = top;\n  let rectWidth = right - left;\n  let rectHeight = bottom - top;\n\n  // Ensure the rect doesn't exceed image boundaries\n  // If right edge exceeds width, adjust the width\n  if (rectLeft + rectWidth > width) {\n    rectWidth = width - rectLeft;\n  }\n\n  // If bottom edge exceeds height, adjust the height\n  if (rectTop + rectHeight > height) {\n    rectHeight = height - rectTop;\n  }\n\n  // Ensure minimum dimensions (width and height should be at least 1)\n  rectWidth = Math.max(1, rectWidth);\n  rectHeight = Math.max(1, rectHeight);\n\n  const rect = {\n    left: rectLeft + offsetX,\n    top: rectTop + offsetY,\n    width: rectWidth,\n    height: rectHeight,\n  };\n  debugInspectUtils('adaptBboxToRect, result=', rect);\n\n  return rect;\n}\n\nexport function mergeRects(rects: Rect[]) {\n  const minLeft = Math.min(...rects.map((r) => r.left));\n  const minTop = Math.min(...rects.map((r) => r.top));\n  const maxRight = Math.max(...rects.map((r) => r.left + r.width));\n  const maxBottom = Math.max(...rects.map((r) => r.top + r.height));\n  return {\n    left: minLeft,\n    top: minTop,\n    width: maxRight - minLeft,\n    height: maxBottom - minTop,\n  };\n}\n\n// expand the search area to at least 300 x 300, or add a default padding\nexport function expandSearchArea(\n  rect: Rect,\n  screenSize: Size,\n  vlMode: TVlModeTypes | undefined,\n) {\n  let minEdgeSize = 500;\n  if (vlMode === 'qwen3-vl') {\n    minEdgeSize = 1200;\n  }\n  const defaultPadding = 160;\n\n  // Calculate padding needed to reach minimum edge size\n  const paddingSizeHorizontal =\n    rect.width < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.width) / 2)\n      : defaultPadding;\n  const paddingSizeVertical =\n    rect.height < minEdgeSize\n      ? Math.ceil((minEdgeSize - rect.height) / 2)\n      : defaultPadding;\n\n  // Calculate new dimensions (ensure minimum edge size)\n  let newWidth = Math.max(minEdgeSize, rect.width + paddingSizeHorizontal * 2);\n  let newHeight = Math.max(minEdgeSize, rect.height + paddingSizeVertical * 2);\n\n  // Calculate initial position with padding\n  let newLeft = rect.left - paddingSizeHorizontal;\n  let newTop = rect.top - paddingSizeVertical;\n\n  // Ensure the rect doesn't exceed screen boundaries by adjusting position\n  // If the rect goes beyond the right edge, shift it left\n  if (newLeft + newWidth > screenSize.width) {\n    newLeft = screenSize.width - newWidth;\n  }\n\n  // If the rect goes beyond the bottom edge, shift it up\n  if (newTop + newHeight > screenSize.height) {\n    newTop = screenSize.height - newHeight;\n  }\n\n  // Ensure the rect doesn't go beyond the left/top edges\n  newLeft = Math.max(0, newLeft);\n  newTop = Math.max(0, newTop);\n\n  // If after position adjustment, the rect still exceeds screen boundaries,\n  // clamp the dimensions to fit within screen\n  if (newLeft + newWidth > screenSize.width) {\n    newWidth = screenSize.width - newLeft;\n  }\n  if (newTop + newHeight > screenSize.height) {\n    newHeight = screenSize.height - newTop;\n  }\n\n  rect.left = newLeft;\n  rect.top = newTop;\n  rect.width = newWidth;\n  rect.height = newHeight;\n\n  return rect;\n}\n\nexport async function markupImageForLLM(\n  screenshotBase64: string,\n  tree: ElementTreeNode<BaseElement>,\n  size: Size,\n) {\n  const elementsInfo = treeToList(tree);\n  const elementsPositionInfoWithoutText = elementsInfo!.filter(\n    (elementInfo) => {\n      if (elementInfo.attributes.nodeType === NodeType.TEXT) {\n        return false;\n      }\n      return true;\n    },\n  );\n\n  const imagePayload = await compositeElementInfoImg({\n    inputImgBase64: screenshotBase64,\n    elementsPositionInfo: elementsPositionInfoWithoutText,\n    size,\n  });\n  return imagePayload;\n}\n\nexport function buildYamlFlowFromPlans(\n  plans: PlanningAction[],\n  actionSpace: DeviceAction<any>[],\n): MidsceneYamlFlowItem[] {\n  const flow: MidsceneYamlFlowItem[] = [];\n\n  for (const plan of plans) {\n    const verb = plan.type;\n\n    const action = actionSpace.find((action) => action.name === verb);\n    if (!action) {\n      console.warn(\n        `Cannot convert action ${verb} to yaml flow. Will ignore it.`,\n      );\n      continue;\n    }\n\n    const flowKey = action.interfaceAlias || verb;\n    const flowParam = action.paramSchema\n      ? dumpActionParam(plan.param || {}, action.paramSchema)\n      : {};\n\n    const flowItem: MidsceneYamlFlowItem = {\n      [flowKey]: '',\n      ...flowParam,\n    };\n\n    flow.push(flowItem);\n  }\n\n  return flow;\n}\n\n// Zod schemas for shared types\nexport const PointSchema = z.object({\n  left: z.number(),\n  top: z.number(),\n});\n\nexport const SizeSchema = z.object({\n  width: z.number(),\n  height: z.number(),\n  dpr: z.number().optional(),\n});\n\nexport const RectSchema = PointSchema.and(SizeSchema).and(\n  z.object({\n    zoom: z.number().optional(),\n  }),\n);\n\n// Zod schema for TMultimodalPrompt\nexport const TMultimodalPromptSchema = z.object({\n  images: z\n    .array(\n      z.object({\n        name: z.string(),\n        url: z.string(),\n      }),\n    )\n    .optional(),\n  convertHttpImage2Base64: z.boolean().optional(),\n});\n\n// Zod schema for TUserPrompt\nexport const TUserPromptSchema = z.union([\n  z.string(),\n  z\n    .object({\n      prompt: z.string(),\n    })\n    .and(TMultimodalPromptSchema.partial()),\n]);\n\n// Generate TypeScript types from Zod schemas\nexport type TMultimodalPrompt = z.infer<typeof TMultimodalPromptSchema>;\nexport type TUserPrompt = z.infer<typeof TUserPromptSchema>;\n\nconst locateFieldFlagName = 'midscene_location_field_flag';\n\n// Schema for locator field input (when users provide locate parameters)\nconst MidsceneLocationInput = z\n  .object({\n    prompt: TUserPromptSchema,\n    deepThink: z.boolean().optional(),\n    cacheable: z.boolean().optional(),\n    xpath: z.union([z.string(), z.boolean()]).optional(),\n  })\n  .passthrough();\n\n// Schema for locator field result (when AI returns locate results)\nconst MidsceneLocationResult = z\n  .object({\n    [locateFieldFlagName]: z.literal(true),\n    prompt: TUserPromptSchema,\n\n    // optional fields\n    deepThink: z.boolean().optional(), // only available in vl model\n    cacheable: z.boolean().optional(),\n    xpath: z.boolean().optional(), // preset result for xpath\n\n    // these two fields will only appear in the result\n    center: z.tuple([z.number(), z.number()]),\n    rect: RectSchema,\n  })\n  .passthrough();\n\n// Export the result type - this is used for runtime results that include center and rect\nexport type MidsceneLocationResultType = z.infer<typeof MidsceneLocationResult>;\n\n// Export the input type - this is the inferred type from getMidsceneLocationSchema()\nexport type MidsceneLocationInputType = z.infer<typeof MidsceneLocationInput>;\n\n/**\n * Returns the schema for locator fields.\n * This now returns the input schema which is more permissive and suitable for validation.\n */\nexport const getMidsceneLocationSchema = () => {\n  return MidsceneLocationInput;\n};\n\nexport const ifMidsceneLocatorField = (field: any): boolean => {\n  // Handle optional fields by getting the inner type\n  let actualField = field;\n  if (actualField._def?.typeName === 'ZodOptional') {\n    actualField = actualField._def.innerType;\n  }\n\n  // Check if this is a ZodObject\n  if (actualField._def?.typeName === 'ZodObject') {\n    const shape = actualField._def.shape();\n\n    // Method 1: Check for the location field flag (for result schema)\n    if (locateFieldFlagName in shape) {\n      return true;\n    }\n\n    // Method 2: Check if it's the input schema by checking for 'prompt' field\n    // Input schema has 'prompt' as a required field\n    if ('prompt' in shape && shape.prompt) {\n      return true;\n    }\n  }\n\n  return false;\n};\n\nexport const dumpMidsceneLocatorField = (field: any): string => {\n  assert(\n    ifMidsceneLocatorField(field),\n    'field is not a midscene locator field',\n  );\n\n  // If field is a string, return it directly\n  if (typeof field === 'string') {\n    return field;\n  }\n\n  // If field is an object with prompt property\n  if (field && typeof field === 'object' && field.prompt) {\n    // If prompt is a string, return it directly\n    if (typeof field.prompt === 'string') {\n      return field.prompt;\n    }\n    // If prompt is a TUserPrompt object, extract the prompt string\n    if (typeof field.prompt === 'object' && field.prompt.prompt) {\n      return field.prompt.prompt; // TODO: dump images if necessary\n    }\n  }\n\n  // Fallback: try to convert to string\n  return String(field);\n};\n\nexport const findAllMidsceneLocatorField = (\n  zodType?: z.ZodType<any>,\n  requiredOnly?: boolean,\n): string[] => {\n  if (!zodType) {\n    return [];\n  }\n\n  // Check if this is a ZodObject by checking if it has a shape property\n  const zodObject = zodType as any;\n  if (zodObject._def?.typeName === 'ZodObject' && zodObject.shape) {\n    const keys = Object.keys(zodObject.shape);\n    return keys.filter((key) => {\n      const field = zodObject.shape[key];\n      if (!ifMidsceneLocatorField(field)) {\n        return false;\n      }\n\n      // If requiredOnly is true, filter out optional fields\n      if (requiredOnly) {\n        return field._def?.typeName !== 'ZodOptional';\n      }\n\n      return true;\n    });\n  }\n\n  // For other ZodType instances, we can't extract field names\n  return [];\n};\n\nexport const dumpActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  // Prevent spreading strings into {0: 'c', 1: 'o', ...}\n  if (!isPlainObject(jsonObject)) {\n    return {};\n  }\n\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue) {\n      // If it's already a string, keep it as is\n      if (typeof fieldValue === 'string') {\n        result[fieldName] = fieldValue;\n      } else if (typeof fieldValue === 'object') {\n        // Check if this field is actually a MidsceneLocationType object\n        if (fieldValue.prompt) {\n          // If prompt is a string, use it directly\n          if (typeof fieldValue.prompt === 'string') {\n            result[fieldName] = fieldValue.prompt;\n          } else if (\n            typeof fieldValue.prompt === 'object' &&\n            fieldValue.prompt.prompt\n          ) {\n            // If prompt is a TUserPrompt object, extract the prompt string\n            result[fieldName] = fieldValue.prompt.prompt;\n          }\n        }\n      }\n    }\n  }\n\n  return result;\n};\n\nexport const loadActionParam = (\n  jsonObject: Record<string, any>,\n  zodSchema: z.ZodType<any>,\n): Record<string, any> => {\n  const locatorFields = findAllMidsceneLocatorField(zodSchema);\n  const result = { ...jsonObject };\n\n  for (const fieldName of locatorFields) {\n    const fieldValue = result[fieldName];\n    if (fieldValue && typeof fieldValue === 'string') {\n      result[fieldName] = {\n        [locateFieldFlagName]: true,\n        prompt: fieldValue,\n      };\n    }\n  }\n\n  return result;\n};\n\n/**\n * Parse and validate action parameters using Zod schema.\n * All fields are validated through Zod, EXCEPT locator fields which are skipped.\n * Default values defined in the schema are automatically applied.\n *\n * Locator fields are special business logic fields with complex validation requirements,\n * so they are intentionally excluded from Zod parsing and use existing validation logic.\n */\nexport const parseActionParam = (\n  rawParam: Record<string, any> | undefined,\n  zodSchema?: z.ZodType<any>,\n): Record<string, any> | undefined => {\n  // If no schema is provided, return undefined (action takes no parameters)\n  if (!zodSchema) {\n    return undefined;\n  }\n\n  // Handle undefined or null rawParam by providing an empty object\n  const param = rawParam ?? {};\n\n  // Find all locate fields in the schema\n  const locateFields = findAllMidsceneLocatorField(zodSchema);\n\n  // If there are no locate fields, just do normal validation\n  if (locateFields.length === 0) {\n    return zodSchema.parse(param);\n  }\n\n  // Extract locate field values to restore later\n  const locateFieldValues: Record<string, any> = {};\n  for (const fieldName of locateFields) {\n    if (fieldName in param) {\n      locateFieldValues[fieldName] = param[fieldName];\n    }\n  }\n\n  // Build params for validation - skip locate fields and use dummy values\n  const paramsForValidation: Record<string, any> = {};\n  for (const key in param) {\n    if (locateFields.includes(key)) {\n      // Use dummy value to satisfy schema validation\n      paramsForValidation[key] = { prompt: '_dummy_' };\n    } else {\n      paramsForValidation[key] = param[key];\n    }\n  }\n\n  // Validate with dummy locate values\n  const validated = zodSchema.parse(paramsForValidation);\n\n  // Restore the actual locate field values (unvalidated, as per business requirement)\n  for (const fieldName in locateFieldValues) {\n    validated[fieldName] = locateFieldValues[fieldName];\n  }\n\n  return validated;\n};\n\nexport const finalizeActionName = 'Finalize';\n\n/**\n * Get a readable time string for the current time\n * @param format - Optional format string. Supports: YYYY, MM, DD, HH, mm, ss. Default: 'YYYY-MM-DD HH:mm:ss'\n * @returns A formatted time string with format label\n */\nexport const getReadableTimeString = (\n  format = 'YYYY-MM-DD HH:mm:ss',\n): string => {\n  const now = new Date();\n  const year = now.getFullYear();\n  const month = String(now.getMonth() + 1).padStart(2, '0');\n  const day = String(now.getDate()).padStart(2, '0');\n  const hours = String(now.getHours()).padStart(2, '0');\n  const minutes = String(now.getMinutes()).padStart(2, '0');\n  const seconds = String(now.getSeconds()).padStart(2, '0');\n\n  const timeString = format\n    .replace('YYYY', String(year))\n    .replace('MM', month)\n    .replace('DD', day)\n    .replace('HH', hours)\n    .replace('mm', minutes)\n    .replace('ss', seconds);\n\n  return `${timeString} (${format})`;\n};\n"],"names":["defaultBboxSize","debugInspectUtils","getDebug","pointToBbox","x","y","bboxSize","halfSize","x1","Math","y1","x2","y2","fillBboxParam","locate","width","height","rightLimit","bottomLimit","vlMode","adaptBbox","adaptQwen2_5Bbox","bbox","msg","JSON","Error","result","adaptDoubaoBbox","assert","splitted","Number","bboxList","Array","item","normalizeBboxInput","normalizedBbox","adaptGeminiBbox","normalized01000","left","top","right","bottom","adaptBboxToRect","offsetX","offsetY","rectLeft","rectTop","rectWidth","rectHeight","rect","mergeRects","rects","minLeft","r","minTop","maxRight","maxBottom","expandSearchArea","screenSize","minEdgeSize","defaultPadding","paddingSizeHorizontal","paddingSizeVertical","newWidth","newHeight","newLeft","newTop","markupImageForLLM","screenshotBase64","tree","size","elementsInfo","treeToList","elementsPositionInfoWithoutText","elementInfo","NodeType","imagePayload","compositeElementInfoImg","buildYamlFlowFromPlans","plans","actionSpace","flow","plan","verb","action","console","flowKey","flowParam","dumpActionParam","flowItem","PointSchema","z","SizeSchema","RectSchema","TMultimodalPromptSchema","TUserPromptSchema","locateFieldFlagName","MidsceneLocationInput","getMidsceneLocationSchema","ifMidsceneLocatorField","field","actualField","shape","dumpMidsceneLocatorField","String","findAllMidsceneLocatorField","zodType","requiredOnly","zodObject","keys","Object","key","jsonObject","zodSchema","isPlainObject","locatorFields","fieldName","fieldValue","loadActionParam","parseActionParam","rawParam","param","locateFields","locateFieldValues","paramsForValidation","validated","finalizeActionName","getReadableTimeString","format","now","Date","year","month","day","hours","minutes","seconds","timeString"],"mappings":";;;;;;AAuBA,MAAMA,kBAAkB;AACxB,MAAMC,oBAAoBC,SAAS;AAY5B,SAASC,YACdC,CAAS,EACTC,CAAS,EACTC,WAAWN,eAAe;IAE1B,MAAMO,WAAWD,WAAW;IAC5B,MAAME,KAAKC,KAAK,GAAG,CAACL,IAAIG,UAAU;IAClC,MAAMG,KAAKD,KAAK,GAAG,CAACJ,IAAIE,UAAU;IAClC,MAAMI,KAAKF,KAAK,GAAG,CAACL,IAAIG,UAAU;IAClC,MAAMK,KAAKH,KAAK,GAAG,CAACJ,IAAIE,UAAU;IAElC,OAAO;QAACC;QAAIE;QAAIC;QAAIC;KAAG;AACzB;AAGO,SAASC,cACdC,MAA2B,EAC3BC,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAGhC,IAAKL,OAAe,OAAO,IAAI,CAACA,QAAQ,MAAM;QAC5CA,OAAO,IAAI,GAAIA,OAAe,OAAO;QAErC,OAAQA,OAAe,OAAO;IAChC;IAEA,IAAIA,QAAQ,MACVA,OAAO,IAAI,GAAGM,UACZN,OAAO,IAAI,EACXC,OACAC,QACAC,YACAC,aACAC;IAIJ,OAAOL;AACT;AAEO,SAASO,iBACdC,IAAc;IAEd,IAAIA,KAAK,MAAM,GAAG,GAAG;QACnB,MAAMC,MAAM,CAAC,oCAAoC,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;QAC1E,MAAM,IAAIG,MAAMF;IAClB;IAEA,MAAMG,SAA2C;QAC/CjB,KAAK,KAAK,CAACa,IAAI,CAAC,EAAE;QAClBb,KAAK,KAAK,CAACa,IAAI,CAAC,EAAE;QACC,YAAnB,OAAOA,IAAI,CAAC,EAAE,GACVb,KAAK,KAAK,CAACa,IAAI,CAAC,EAAE,IAClBb,KAAK,KAAK,CAACa,IAAI,CAAC,EAAE,GAAGtB;QACN,YAAnB,OAAOsB,IAAI,CAAC,EAAE,GACVb,KAAK,KAAK,CAACa,IAAI,CAAC,EAAE,IAClBb,KAAK,KAAK,CAACa,IAAI,CAAC,EAAE,GAAGtB;KAC1B;IACD,OAAO0B;AACT;AAEO,SAASC,gBACdL,IAAkC,EAClCP,KAAa,EACbC,MAAc;IAEdY,OACEb,QAAQ,KAAKC,SAAS,GACtB;IAGF,IAAI,AAAgB,YAAhB,OAAOM,MAAmB;QAC5BM,OACE,+BAA+B,IAAI,CAACN,KAAK,IAAI,KAC7C,CAAC,iDAAiD,EAAEA,MAAM;QAE5D,MAAMO,WAAWP,KAAK,KAAK,CAAC;QAC5B,IAAIO,AAAoB,MAApBA,SAAS,MAAM,EACjB,OAAO;YACLpB,KAAK,KAAK,CAAEqB,OAAOD,QAAQ,CAAC,EAAE,IAAId,QAAS;YAC3CN,KAAK,KAAK,CAAEqB,OAAOD,QAAQ,CAAC,EAAE,IAAIb,SAAU;YAC5CP,KAAK,KAAK,CAAEqB,OAAOD,QAAQ,CAAC,EAAE,IAAId,QAAS;YAC3CN,KAAK,KAAK,CAAEqB,OAAOD,QAAQ,CAAC,EAAE,IAAIb,SAAU;SAC7C;QAEH,MAAM,IAAIS,MAAM,CAAC,iDAAiD,EAAEH,MAAM;IAC5E;IAEA,IAAIS,WAAqB,EAAE;IAC3B,IAAIC,MAAM,OAAO,CAACV,SAAS,AAAmB,YAAnB,OAAOA,IAAI,CAAC,EAAE,EACvCA,KAAK,OAAO,CAAC,CAACW;QACZ,IAAI,AAAgB,YAAhB,OAAOA,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YAClD,MAAM,CAAC7B,GAAGC,EAAE,GAAG4B,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAO1B,EAAE,IAAI,KAAK0B,OAAOzB,EAAE,IAAI;QAC/C,OAAO,IAAI,AAAgB,YAAhB,OAAO4B,QAAqBA,KAAK,QAAQ,CAAC,MAAM;YACzD,MAAM,CAAC7B,GAAGC,EAAE,GAAG4B,KAAK,KAAK,CAAC;YAC1BF,SAAS,IAAI,CAACD,OAAO1B,EAAE,IAAI,KAAK0B,OAAOzB,EAAE,IAAI;QAC/C,OACE0B,SAAS,IAAI,CAACD,OAAOG;IAEzB;SAEAF,WAAWT;IAGb,IAAIS,AAAoB,MAApBA,SAAS,MAAM,IAAUA,AAAoB,MAApBA,SAAS,MAAM,EAC1C,OAAO;QACLtB,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCN,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGf,SAAU;QACpCP,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCN,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGf,SAAU;KACrC;IAIH,IACEe,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,IACfA,AAAoB,MAApBA,SAAS,MAAM,EAEf,OAAO;QACLtB,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGhB,QAAS,QAAQf,kBAAkB;QAE/DS,KAAK,GAAG,CACN,GACAA,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGf,SAAU,QAAQhB,kBAAkB;QAEhES,KAAK,GAAG,CACNM,OACAN,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGhB,QAAS,QAAQf,kBAAkB;QAE/DS,KAAK,GAAG,CACNO,QACAP,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGf,SAAU,QAAQhB,kBAAkB;KAEjE;IAGH,IAAIsB,AAAgB,MAAhBA,KAAK,MAAM,EACb,OAAO;QACLb,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCN,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGf,SAAU;QACpCP,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGhB,QAAS;QACnCN,KAAK,KAAK,CAAEsB,QAAQ,CAAC,EAAE,GAAGf,SAAU;KACrC;IAGH,MAAMO,MAAM,CAAC,0CAA0C,EAAEC,KAAK,SAAS,CAACF,MAAM,CAAC,CAAC;IAChF,MAAM,IAAIG,MAAMF;AAClB;AAEA,SAASW,mBACPZ,IAAoB;IAEpB,IAAIU,MAAM,OAAO,CAACV,OAChB;QAAA,IAAIU,MAAM,OAAO,CAACV,IAAI,CAAC,EAAE,GACvB,OAAOA,IAAI,CAAC,EAAE;IAChB;IAGF,OAAOA;AACT;AAEO,SAASF,UACdE,IAAoB,EACpBP,KAAa,EACbC,MAAc,EACdC,UAAkB,EAClBC,WAAmB,EACnBC,MAAgC;IAEhC,MAAMgB,iBAAiBD,mBAAmBZ;IAE1C,IAAII,SAA2C;QAAC;QAAG;QAAG;QAAG;KAAE;IAEzDA,SADEP,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,SACvBQ,gBAAgBQ,gBAAgBpB,OAAOC,UACvCG,AAAW,aAAXA,SACAiB,gBAAgBD,gBAA4BpB,OAAOC,UACnDG,AAAW,iBAAXA,SACAE,iBAAiBc,kBAIjBE,gBAAgBF,gBAA4BpB,OAAOC;IAG9DU,MAAM,CAAC,EAAE,GAAGjB,KAAK,GAAG,CAACiB,MAAM,CAAC,EAAE,EAAET;IAChCS,MAAM,CAAC,EAAE,GAAGjB,KAAK,GAAG,CAACiB,MAAM,CAAC,EAAE,EAAER;IAEhC,OAAOQ;AACT;AAGO,SAASW,gBACdf,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,OAAO;QACLP,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BN,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGN,SAAU;QAChCP,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGP,QAAS;QAC/BN,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGN,SAAU;KACjC;AACH;AAGO,SAASoB,gBACdd,IAAc,EACdP,KAAa,EACbC,MAAc;IAEd,MAAMsB,OAAO7B,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC5C,MAAMwB,MAAM9B,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC5C,MAAMwB,QAAQ/B,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGP,QAAS;IAC7C,MAAM0B,SAAShC,KAAK,KAAK,CAAEa,IAAI,CAAC,EAAE,GAAGN,SAAU;IAC/C,OAAO;QAACsB;QAAMC;QAAKC;QAAOC;KAAO;AACnC;AAEO,SAASC,gBACdpB,IAAc,EACdP,KAAa,EACbC,MAAc,EACd2B,UAAU,CAAC,EACXC,UAAU,CAAC,EACX3B,aAAaF,KAAK,EAClBG,cAAcF,MAAM,EACpBG,MAAiC;IAEjClB,kBACE,mBACAqB,MACAP,OACAC,QACA,UACA2B,SACAC,SACA,SACA3B,YACAC,aACA,UACAC;IAEF,MAAM,CAACmB,MAAMC,KAAKC,OAAOC,OAAO,GAAGrB,UACjCE,MACAP,OACAC,QACAC,YACAC,aACAC;IAIF,MAAM0B,WAAWP;IACjB,MAAMQ,UAAUP;IAChB,IAAIQ,YAAYP,QAAQF;IACxB,IAAIU,aAAaP,SAASF;IAI1B,IAAIM,WAAWE,YAAYhC,OACzBgC,YAAYhC,QAAQ8B;IAItB,IAAIC,UAAUE,aAAahC,QACzBgC,aAAahC,SAAS8B;IAIxBC,YAAYtC,KAAK,GAAG,CAAC,GAAGsC;IACxBC,aAAavC,KAAK,GAAG,CAAC,GAAGuC;IAEzB,MAAMC,OAAO;QACX,MAAMJ,WAAWF;QACjB,KAAKG,UAAUF;QACf,OAAOG;QACP,QAAQC;IACV;IACA/C,kBAAkB,4BAA4BgD;IAE9C,OAAOA;AACT;AAEO,SAASC,WAAWC,KAAa;IACtC,MAAMC,UAAU3C,KAAK,GAAG,IAAI0C,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI;IACnD,MAAMC,SAAS7C,KAAK,GAAG,IAAI0C,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG;IACjD,MAAME,WAAW9C,KAAK,GAAG,IAAI0C,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,IAAI,GAAGA,EAAE,KAAK;IAC9D,MAAMG,YAAY/C,KAAK,GAAG,IAAI0C,MAAM,GAAG,CAAC,CAACE,IAAMA,EAAE,GAAG,GAAGA,EAAE,MAAM;IAC/D,OAAO;QACL,MAAMD;QACN,KAAKE;QACL,OAAOC,WAAWH;QAClB,QAAQI,YAAYF;IACtB;AACF;AAGO,SAASG,iBACdR,IAAU,EACVS,UAAgB,EAChBvC,MAAgC;IAEhC,IAAIwC,cAAc;IAClB,IAAIxC,AAAW,eAAXA,QACFwC,cAAc;IAEhB,MAAMC,iBAAiB;IAGvB,MAAMC,wBACJZ,KAAK,KAAK,GAAGU,cACTlD,KAAK,IAAI,CAAEkD,AAAAA,CAAAA,cAAcV,KAAK,KAAI,IAAK,KACvCW;IACN,MAAME,sBACJb,KAAK,MAAM,GAAGU,cACVlD,KAAK,IAAI,CAAEkD,AAAAA,CAAAA,cAAcV,KAAK,MAAK,IAAK,KACxCW;IAGN,IAAIG,WAAWtD,KAAK,GAAG,CAACkD,aAAaV,KAAK,KAAK,GAAGY,AAAwB,IAAxBA;IAClD,IAAIG,YAAYvD,KAAK,GAAG,CAACkD,aAAaV,KAAK,MAAM,GAAGa,AAAsB,IAAtBA;IAGpD,IAAIG,UAAUhB,KAAK,IAAI,GAAGY;IAC1B,IAAIK,SAASjB,KAAK,GAAG,GAAGa;IAIxB,IAAIG,UAAUF,WAAWL,WAAW,KAAK,EACvCO,UAAUP,WAAW,KAAK,GAAGK;IAI/B,IAAIG,SAASF,YAAYN,WAAW,MAAM,EACxCQ,SAASR,WAAW,MAAM,GAAGM;IAI/BC,UAAUxD,KAAK,GAAG,CAAC,GAAGwD;IACtBC,SAASzD,KAAK,GAAG,CAAC,GAAGyD;IAIrB,IAAID,UAAUF,WAAWL,WAAW,KAAK,EACvCK,WAAWL,WAAW,KAAK,GAAGO;IAEhC,IAAIC,SAASF,YAAYN,WAAW,MAAM,EACxCM,YAAYN,WAAW,MAAM,GAAGQ;IAGlCjB,KAAK,IAAI,GAAGgB;IACZhB,KAAK,GAAG,GAAGiB;IACXjB,KAAK,KAAK,GAAGc;IACbd,KAAK,MAAM,GAAGe;IAEd,OAAOf;AACT;AAEO,eAAekB,kBACpBC,gBAAwB,EACxBC,IAAkC,EAClCC,IAAU;IAEV,MAAMC,eAAeC,WAAWH;IAChC,MAAMI,kCAAkCF,aAAc,MAAM,CAC1D,CAACG;QACC,IAAIA,YAAY,UAAU,CAAC,QAAQ,KAAKC,SAAS,IAAI,EACnD,OAAO;QAET,OAAO;IACT;IAGF,MAAMC,eAAe,MAAMC,wBAAwB;QACjD,gBAAgBT;QAChB,sBAAsBK;QACtBH;IACF;IACA,OAAOM;AACT;AAEO,SAASE,uBACdC,KAAuB,EACvBC,WAAgC;IAEhC,MAAMC,OAA+B,EAAE;IAEvC,KAAK,MAAMC,QAAQH,MAAO;QACxB,MAAMI,OAAOD,KAAK,IAAI;QAEtB,MAAME,SAASJ,YAAY,IAAI,CAAC,CAACI,SAAWA,OAAO,IAAI,KAAKD;QAC5D,IAAI,CAACC,QAAQ;YACXC,QAAQ,IAAI,CACV,CAAC,sBAAsB,EAAEF,KAAK,8BAA8B,CAAC;YAE/D;QACF;QAEA,MAAMG,UAAUF,OAAO,cAAc,IAAID;QACzC,MAAMI,YAAYH,OAAO,WAAW,GAChCI,gBAAgBN,KAAK,KAAK,IAAI,CAAC,GAAGE,OAAO,WAAW,IACpD,CAAC;QAEL,MAAMK,WAAiC;YACrC,CAACH,QAAQ,EAAE;YACX,GAAGC,SAAS;QACd;QAEAN,KAAK,IAAI,CAACQ;IACZ;IAEA,OAAOR;AACT;AAGO,MAAMS,cAAcC,EAAE,MAAM,CAAC;IAClC,MAAMA,EAAE,MAAM;IACd,KAAKA,EAAE,MAAM;AACf;AAEO,MAAMC,aAAaD,EAAE,MAAM,CAAC;IACjC,OAAOA,EAAE,MAAM;IACf,QAAQA,EAAE,MAAM;IAChB,KAAKA,EAAE,MAAM,GAAG,QAAQ;AAC1B;AAEO,MAAME,aAAaH,YAAY,GAAG,CAACE,YAAY,GAAG,CACvDD,EAAE,MAAM,CAAC;IACP,MAAMA,EAAE,MAAM,GAAG,QAAQ;AAC3B;AAIK,MAAMG,0BAA0BH,EAAE,MAAM,CAAC;IAC9C,QAAQA,EAAAA,KACA,CACJA,EAAE,MAAM,CAAC;QACP,MAAMA,EAAE,MAAM;QACd,KAAKA,EAAE,MAAM;IACf,IAED,QAAQ;IACX,yBAAyBA,EAAE,OAAO,GAAG,QAAQ;AAC/C;AAGO,MAAMI,oBAAoBJ,EAAE,KAAK,CAAC;IACvCA,EAAE,MAAM;IACRA,EAAAA,MACS,CAAC;QACN,QAAQA,EAAE,MAAM;IAClB,GACC,GAAG,CAACG,wBAAwB,OAAO;CACvC;AAMD,MAAME,sBAAsB;AAG5B,MAAMC,wBAAwBN,EAAAA,MACrB,CAAC;IACN,QAAQI;IACR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,OAAO;KAAG,EAAE,QAAQ;AACpD,GACC,WAAW;AAGiBA,EAAAA,MACtB,CAAC;IACN,CAACK,oBAAoB,EAAEL,EAAE,OAAO,CAAC;IACjC,QAAQI;IAGR,WAAWJ,EAAE,OAAO,GAAG,QAAQ;IAC/B,WAAWA,EAAE,OAAO,GAAG,QAAQ;IAC/B,OAAOA,EAAE,OAAO,GAAG,QAAQ;IAG3B,QAAQA,EAAE,KAAK,CAAC;QAACA,EAAE,MAAM;QAAIA,EAAE,MAAM;KAAG;IACxC,MAAME;AACR,GACC,WAAW;AAYP,MAAMK,4BAA4B,IAChCD;AAGF,MAAME,yBAAyB,CAACC;IAErC,IAAIC,cAAcD;IAClB,IAAIC,YAAY,IAAI,EAAE,aAAa,eACjCA,cAAcA,YAAY,IAAI,CAAC,SAAS;IAI1C,IAAIA,YAAY,IAAI,EAAE,aAAa,aAAa;QAC9C,MAAMC,QAAQD,YAAY,IAAI,CAAC,KAAK;QAGpC,IAAIL,uBAAuBM,OACzB,OAAO;QAKT,IAAI,YAAYA,SAASA,MAAM,MAAM,EACnC,OAAO;IAEX;IAEA,OAAO;AACT;AAEO,MAAMC,2BAA2B,CAACH;IACvCxE,OACEuE,uBAAuBC,QACvB;IAIF,IAAI,AAAiB,YAAjB,OAAOA,OACT,OAAOA;IAIT,IAAIA,SAAS,AAAiB,YAAjB,OAAOA,SAAsBA,MAAM,MAAM,EAAE;QAEtD,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,EACrB,OAAOA,MAAM,MAAM;QAGrB,IAAI,AAAwB,YAAxB,OAAOA,MAAM,MAAM,IAAiBA,MAAM,MAAM,CAAC,MAAM,EACzD,OAAOA,MAAM,MAAM,CAAC,MAAM;IAE9B;IAGA,OAAOI,OAAOJ;AAChB;AAEO,MAAMK,8BAA8B,CACzCC,SACAC;IAEA,IAAI,CAACD,SACH,OAAO,EAAE;IAIX,MAAME,YAAYF;IAClB,IAAIE,UAAU,IAAI,EAAE,aAAa,eAAeA,UAAU,KAAK,EAAE;QAC/D,MAAMC,OAAOC,OAAO,IAAI,CAACF,UAAU,KAAK;QACxC,OAAOC,KAAK,MAAM,CAAC,CAACE;YAClB,MAAMX,QAAQQ,UAAU,KAAK,CAACG,IAAI;YAClC,IAAI,CAACZ,uBAAuBC,QAC1B,OAAO;YAIT,IAAIO,cACF,OAAOP,MAAM,IAAI,EAAE,aAAa;YAGlC,OAAO;QACT;IACF;IAGA,OAAO,EAAE;AACX;AAEO,MAAMZ,kBAAkB,CAC7BwB,YACAC;IAGA,IAAI,CAACC,cAAcF,aACjB,OAAO,CAAC;IAGV,MAAMG,gBAAgBV,4BAA4BQ;IAClD,MAAMvF,SAAS;QAAE,GAAGsF,UAAU;IAAC;IAE/B,KAAK,MAAMI,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,YAEF;YAAA,IAAI,AAAsB,YAAtB,OAAOA,YACT3F,MAAM,CAAC0F,UAAU,GAAGC;iBACf,IAAI,AAAsB,YAAtB,OAAOA,YAEhB;gBAAA,IAAIA,WAAW,MAAM,EAEnB;oBAAA,IAAI,AAA6B,YAA7B,OAAOA,WAAW,MAAM,EAC1B3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM;yBAChC,IACL,AAA6B,YAA7B,OAAOA,WAAW,MAAM,IACxBA,WAAW,MAAM,CAAC,MAAM,EAGxB3F,MAAM,CAAC0F,UAAU,GAAGC,WAAW,MAAM,CAAC,MAAM;gBAC9C;YACF;QACF;IAEJ;IAEA,OAAO3F;AACT;AAEO,MAAM4F,kBAAkB,CAC7BN,YACAC;IAEA,MAAME,gBAAgBV,4BAA4BQ;IAClD,MAAMvF,SAAS;QAAE,GAAGsF,UAAU;IAAC;IAE/B,KAAK,MAAMI,aAAaD,cAAe;QACrC,MAAME,aAAa3F,MAAM,CAAC0F,UAAU;QACpC,IAAIC,cAAc,AAAsB,YAAtB,OAAOA,YACvB3F,MAAM,CAAC0F,UAAU,GAAG;YAClB,CAACpB,oBAAoB,EAAE;YACvB,QAAQqB;QACV;IAEJ;IAEA,OAAO3F;AACT;AAUO,MAAM6F,mBAAmB,CAC9BC,UACAP;IAGA,IAAI,CAACA,WACH;IAIF,MAAMQ,QAAQD,YAAY,CAAC;IAG3B,MAAME,eAAejB,4BAA4BQ;IAGjD,IAAIS,AAAwB,MAAxBA,aAAa,MAAM,EACrB,OAAOT,UAAU,KAAK,CAACQ;IAIzB,MAAME,oBAAyC,CAAC;IAChD,KAAK,MAAMP,aAAaM,aACtB,IAAIN,aAAaK,OACfE,iBAAiB,CAACP,UAAU,GAAGK,KAAK,CAACL,UAAU;IAKnD,MAAMQ,sBAA2C,CAAC;IAClD,IAAK,MAAMb,OAAOU,MAChB,IAAIC,aAAa,QAAQ,CAACX,MAExBa,mBAAmB,CAACb,IAAI,GAAG;QAAE,QAAQ;IAAU;SAE/Ca,mBAAmB,CAACb,IAAI,GAAGU,KAAK,CAACV,IAAI;IAKzC,MAAMc,YAAYZ,UAAU,KAAK,CAACW;IAGlC,IAAK,MAAMR,aAAaO,kBACtBE,SAAS,CAACT,UAAU,GAAGO,iBAAiB,CAACP,UAAU;IAGrD,OAAOS;AACT;AAEO,MAAMC,qBAAqB;AAO3B,MAAMC,wBAAwB,CACnCC,SAAS,qBAAqB;IAE9B,MAAMC,MAAM,IAAIC;IAChB,MAAMC,OAAOF,IAAI,WAAW;IAC5B,MAAMG,QAAQ5B,OAAOyB,IAAI,QAAQ,KAAK,GAAG,QAAQ,CAAC,GAAG;IACrD,MAAMI,MAAM7B,OAAOyB,IAAI,OAAO,IAAI,QAAQ,CAAC,GAAG;IAC9C,MAAMK,QAAQ9B,OAAOyB,IAAI,QAAQ,IAAI,QAAQ,CAAC,GAAG;IACjD,MAAMM,UAAU/B,OAAOyB,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IACrD,MAAMO,UAAUhC,OAAOyB,IAAI,UAAU,IAAI,QAAQ,CAAC,GAAG;IAErD,MAAMQ,aAAaT,OAChB,OAAO,CAAC,QAAQxB,OAAO2B,OACvB,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,KACd,OAAO,CAAC,MAAMC,OACd,OAAO,CAAC,MAAMC,SACd,OAAO,CAAC,MAAMC;IAEjB,OAAO,GAAGC,WAAW,EAAE,EAAET,OAAO,CAAC,CAAC;AACpC"}

package/dist/es/index.mjs.map

@@ -1 +1 @@
-{"version":3,"file":"index.mjs","sources":["../../src/index.ts"],"sourcesContent":["import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  getMidsceneLocationSchema,\n  type MidsceneLocationResultType,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport {\n  ServiceError,\n  ExecutionDump,\n  GroupedActionDump,\n  type IExecutionDump,\n  type IGroupedActionDump,\n} from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, type AiActOptions, createAgent } from './agent';\n\n// Dump utilities\nexport {\n  restoreImageReferences,\n  escapeContent,\n  unescapeContent,\n  parseImageScripts,\n  parseDumpScript,\n  parseDumpScriptAttributes,\n  generateImageScriptTag,\n  generateDumpScriptTag,\n} from './dump';\n\n// ScreenshotItem\nexport { ScreenshotItem } from './screenshot-item';\n"],"names":["Service"],"mappings":";;;;;;;;;;AAoCA,YAAeA"}
+{"version":3,"file":"index.mjs","sources":["../../src/index.ts"],"sourcesContent":["import { z } from 'zod';\nimport Service from './service/index';\nimport { TaskRunner } from './task-runner';\nimport { getVersion } from './utils';\n\nexport {\n  plan,\n  describeUserPage,\n  AiLocateElement,\n  getMidsceneLocationSchema,\n  type MidsceneLocationResultType,\n  PointSchema,\n  SizeSchema,\n  RectSchema,\n  TMultimodalPromptSchema,\n  TUserPromptSchema,\n  type TMultimodalPrompt,\n  type TUserPrompt,\n} from './ai-model/index';\n\nexport {\n  MIDSCENE_MODEL_NAME,\n  type CreateOpenAIClientFn,\n} from '@midscene/shared/env';\n\nexport type * from './types';\nexport {\n  ServiceError,\n  ExecutionDump,\n  GroupedActionDump,\n  type IExecutionDump,\n  type IGroupedActionDump,\n} from './types';\n\nexport { z };\n\nexport default Service;\nexport { TaskRunner, Service, getVersion };\n\nexport type {\n  MidsceneYamlScript,\n  MidsceneYamlTask,\n  MidsceneYamlFlowItem,\n  MidsceneYamlConfigResult,\n  MidsceneYamlConfig,\n  MidsceneYamlScriptWebEnv,\n  MidsceneYamlScriptAndroidEnv,\n  MidsceneYamlScriptIOSEnv,\n  MidsceneYamlScriptEnv,\n  LocateOption,\n  DetailedLocateParam,\n} from './yaml';\n\nexport { Agent, type AgentOpt, type AiActOptions, createAgent } from './agent';\n\n// Dump utilities\nexport {\n  restoreImageReferences,\n  escapeContent,\n  unescapeContent,\n  parseImageScripts,\n  parseDumpScript,\n  parseDumpScriptAttributes,\n  generateImageScriptTag,\n  generateDumpScriptTag,\n} from './dump';\n\n// ScreenshotItem\nexport { ScreenshotItem } from './screenshot-item';\nexport type { SerializedScreenshotItem } from './screenshot-item';\n"],"names":["Service"],"mappings":";;;;;;;;;;AAoCA,YAAeA"}

package/dist/es/screenshot-item.mjs

@@ -12,17 +12,22 @@ class ScreenshotItem {
     static create(base64) {
         return new ScreenshotItem(base64);
     }
-    getData() {
+    get base64() {
         return this._data;
     }
     toSerializable() {
-        return this._data;
+        return {
+            base64: this._data
+        };
+    }
+    toJSON() {
+        return this.toSerializable();
     }
     static isSerializedData(value) {
-        return 'string' == typeof value && value.length > 0;
+        return 'object' == typeof value && null !== value && 'base64' in value && 'string' == typeof value.base64;
     }
     static fromSerializedData(data) {
-        return new ScreenshotItem(data);
+        return new ScreenshotItem(data.base64);
     }
     constructor(data){
         _define_property(this, "_data", void 0);