@sqaitech/core 0.30.10

package/dist/es/ai-model/service-caller/index.mjs

@@ -0,0 +1,388 @@
+import { AIResponseFormat } from "../../types.mjs";
+import { Anthropic } from "@anthropic-ai/sdk";
+import { DefaultAzureCredential, getBearerTokenProvider } from "@azure/identity";
+import { OPENAI_MAX_TOKENS, SQAI_API_TYPE, SQAI_LANGSMITH_DEBUG, globalConfigManager } from "@sqaitech/shared/env";
+import { parseBase64 } from "@sqaitech/shared/img";
+import { getDebug } from "@sqaitech/shared/logger";
+import { assert, ifInBrowser } from "@sqaitech/shared/utils";
+import { HttpsProxyAgent } from "https-proxy-agent";
+import { jsonrepair } from "jsonrepair";
+import openai_0, { AzureOpenAI } from "openai";
+import { SocksProxyAgent } from "socks-proxy-agent";
+import { AIActionType } from "../common.mjs";
+import { assertSchema } from "../prompt/assertion.mjs";
+import { locatorSchema } from "../prompt/llm-locator.mjs";
+import { planSchema } from "../prompt/llm-planning.mjs";
+async function createChatClient({ AIActionTypeValue, modelConfig }) {
+    const { socksProxy, httpProxy, modelName, openaiBaseURL, openaiApiKey, openaiExtraConfig, openaiUseAzureDeprecated, useAzureOpenai, azureOpenaiScope, azureOpenaiKey, azureOpenaiEndpoint, azureOpenaiApiVersion, azureOpenaiDeployment, azureExtraConfig, useAnthropicSdk, anthropicApiKey, modelDescription, uiTarsModelVersion: uiTarsVersion, vlMode } = modelConfig;
+    let openai;
+    let proxyAgent;
+    const debugProxy = getDebug('ai:call:proxy');
+    if (httpProxy) {
+        debugProxy('using http proxy', httpProxy);
+        proxyAgent = new HttpsProxyAgent(httpProxy);
+    } else if (socksProxy) {
+        debugProxy('using socks proxy', socksProxy);
+        proxyAgent = new SocksProxyAgent(socksProxy);
+    }
+    if (openaiUseAzureDeprecated) openai = new AzureOpenAI({
+        baseURL: openaiBaseURL,
+        apiKey: openaiApiKey,
+        httpAgent: proxyAgent,
+        ...openaiExtraConfig,
+        dangerouslyAllowBrowser: true
+    });
+    else if (useAzureOpenai) {
+        let tokenProvider;
+        if (azureOpenaiScope) {
+            assert(!ifInBrowser, 'Azure OpenAI is not supported in browser with Midscene.');
+            const credential = new DefaultAzureCredential();
+            tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);
+            openai = new AzureOpenAI({
+                azureADTokenProvider: tokenProvider,
+                endpoint: azureOpenaiEndpoint,
+                apiVersion: azureOpenaiApiVersion,
+                deployment: azureOpenaiDeployment,
+                ...openaiExtraConfig,
+                ...azureExtraConfig
+            });
+        } else openai = new AzureOpenAI({
+            apiKey: azureOpenaiKey,
+            endpoint: azureOpenaiEndpoint,
+            apiVersion: azureOpenaiApiVersion,
+            deployment: azureOpenaiDeployment,
+            dangerouslyAllowBrowser: true,
+            ...openaiExtraConfig,
+            ...azureExtraConfig
+        });
+    } else if (!useAnthropicSdk) openai = new openai_0({
+        baseURL: openaiBaseURL,
+        apiKey: openaiApiKey,
+        httpAgent: proxyAgent,
+        ...openaiExtraConfig,
+        defaultHeaders: {
+            ...(null == openaiExtraConfig ? void 0 : openaiExtraConfig.defaultHeaders) || {},
+            [SQAI_API_TYPE]: AIActionTypeValue.toString()
+        },
+        dangerouslyAllowBrowser: true
+    });
+    if (openai && globalConfigManager.getEnvConfigInBoolean(SQAI_LANGSMITH_DEBUG)) {
+        if (ifInBrowser) throw new Error('langsmith is not supported in browser');
+        console.log('DEBUGGING MODE: langsmith wrapper enabled');
+        const { wrapOpenAI } = await import("langsmith/wrappers");
+        openai = wrapOpenAI(openai);
+    }
+    if (void 0 !== openai) return {
+        completion: openai.chat.completions,
+        style: 'openai',
+        modelName,
+        modelDescription,
+        uiTarsVersion,
+        vlMode
+    };
+    if (useAnthropicSdk) openai = new Anthropic({
+        apiKey: anthropicApiKey,
+        httpAgent: proxyAgent,
+        dangerouslyAllowBrowser: true
+    });
+    if (void 0 !== openai && openai.messages) return {
+        completion: openai.messages,
+        style: 'anthropic',
+        modelName,
+        modelDescription,
+        uiTarsVersion,
+        vlMode
+    };
+    throw new Error('Openai SDK or Anthropic SDK is not initialized');
+}
+async function callAI(messages, AIActionTypeValue, modelConfig, options) {
+    const { completion, style, modelName, modelDescription, uiTarsVersion, vlMode } = await createChatClient({
+        AIActionTypeValue,
+        modelConfig
+    });
+    const responseFormat = getResponseFormat(modelName, AIActionTypeValue);
+    const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);
+    const debugCall = getDebug('ai:call');
+    const debugProfileStats = getDebug('ai:profile:stats');
+    const debugProfileDetail = getDebug('ai:profile:detail');
+    const startTime = Date.now();
+    const isStreaming = (null == options ? void 0 : options.stream) && (null == options ? void 0 : options.onChunk);
+    let content;
+    let accumulated = '';
+    let usage;
+    let timeCost;
+    const commonConfig = {
+        temperature: 'vlm-ui-tars' === vlMode ? 0.0 : 0.1,
+        stream: !!isStreaming,
+        max_tokens: 'number' == typeof maxTokens ? maxTokens : Number.parseInt(maxTokens || '2048', 10),
+        ...'qwen-vl' === vlMode || 'qwen3-vl' === vlMode ? {
+            vl_high_resolution_images: true
+        } : {}
+    };
+    try {
+        if ('openai' === style) {
+            debugCall(`sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`);
+            if (isStreaming) {
+                const stream = await completion.create({
+                    model: modelName,
+                    messages,
+                    response_format: responseFormat,
+                    ...commonConfig
+                }, {
+                    stream: true
+                });
+                for await (const chunk of stream){
+                    var _chunk_choices__delta, _chunk_choices_, _chunk_choices, _chunk_choices__delta1, _chunk_choices_1, _chunk_choices1, _chunk_choices_2, _chunk_choices2;
+                    const content = (null == (_chunk_choices = chunk.choices) ? void 0 : null == (_chunk_choices_ = _chunk_choices[0]) ? void 0 : null == (_chunk_choices__delta = _chunk_choices_.delta) ? void 0 : _chunk_choices__delta.content) || '';
+                    const reasoning_content = (null == (_chunk_choices1 = chunk.choices) ? void 0 : null == (_chunk_choices_1 = _chunk_choices1[0]) ? void 0 : null == (_chunk_choices__delta1 = _chunk_choices_1.delta) ? void 0 : _chunk_choices__delta1.reasoning_content) || '';
+                    if (chunk.usage) usage = chunk.usage;
+                    if (content || reasoning_content) {
+                        accumulated += content;
+                        const chunkData = {
+                            content,
+                            reasoning_content,
+                            accumulated,
+                            isComplete: false,
+                            usage: void 0
+                        };
+                        options.onChunk(chunkData);
+                    }
+                    if (null == (_chunk_choices2 = chunk.choices) ? void 0 : null == (_chunk_choices_2 = _chunk_choices2[0]) ? void 0 : _chunk_choices_2.finish_reason) {
+                        timeCost = Date.now() - startTime;
+                        if (!usage) {
+                            const estimatedTokens = Math.max(1, Math.floor(accumulated.length / 4));
+                            usage = {
+                                prompt_tokens: estimatedTokens,
+                                completion_tokens: estimatedTokens,
+                                total_tokens: 2 * estimatedTokens
+                            };
+                        }
+                        const finalChunk = {
+                            content: '',
+                            accumulated,
+                            reasoning_content: '',
+                            isComplete: true,
+                            usage: {
+                                prompt_tokens: usage.prompt_tokens ?? 0,
+                                completion_tokens: usage.completion_tokens ?? 0,
+                                total_tokens: usage.total_tokens ?? 0,
+                                time_cost: timeCost ?? 0,
+                                model_name: modelName,
+                                model_description: modelDescription,
+                                intent: modelConfig.intent
+                            }
+                        };
+                        options.onChunk(finalChunk);
+                        break;
+                    }
+                }
+                content = accumulated;
+                debugProfileStats(`streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`);
+            } else {
+                var _result_usage, _result_usage1, _result_usage2;
+                const result = await completion.create({
+                    model: modelName,
+                    messages,
+                    response_format: responseFormat,
+                    ...commonConfig
+                });
+                timeCost = Date.now() - startTime;
+                debugProfileStats(`model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${(null == (_result_usage = result.usage) ? void 0 : _result_usage.prompt_tokens) || ''}, completion-tokens, ${(null == (_result_usage1 = result.usage) ? void 0 : _result_usage1.completion_tokens) || ''}, total-tokens, ${(null == (_result_usage2 = result.usage) ? void 0 : _result_usage2.total_tokens) || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`);
+                debugProfileDetail(`model usage detail: ${JSON.stringify(result.usage)}`);
+                assert(result.choices, `invalid response from LLM service: ${JSON.stringify(result)}`);
+                content = result.choices[0].message.content;
+                usage = result.usage;
+            }
+            debugCall(`response: ${content}`);
+            assert(content, 'empty content');
+        } else if ('anthropic' === style) {
+            const convertImageContent = (content)=>{
+                if ('image_url' === content.type) {
+                    const imgBase64 = content.image_url.url;
+                    assert(imgBase64, 'image_url is required');
+                    const { mimeType, body } = parseBase64(content.image_url.url);
+                    return {
+                        source: {
+                            type: 'base64',
+                            media_type: mimeType,
+                            data: body
+                        },
+                        type: 'image'
+                    };
+                }
+                return content;
+            };
+            if (isStreaming) {
+                const stream = await completion.create({
+                    model: modelName,
+                    system: 'You are a versatile professional in software UI automation',
+                    messages: messages.map((m)=>({
+                            role: 'user',
+                            content: Array.isArray(m.content) ? m.content.map(convertImageContent) : m.content
+                        })),
+                    response_format: responseFormat,
+                    ...commonConfig
+                });
+                for await (const chunk of stream){
+                    var _chunk_delta;
+                    const content = (null == (_chunk_delta = chunk.delta) ? void 0 : _chunk_delta.text) || '';
+                    if (content) {
+                        accumulated += content;
+                        const chunkData = {
+                            content,
+                            accumulated,
+                            reasoning_content: '',
+                            isComplete: false,
+                            usage: void 0
+                        };
+                        options.onChunk(chunkData);
+                    }
+                    if ('message_stop' === chunk.type) {
+                        timeCost = Date.now() - startTime;
+                        const anthropicUsage = chunk.usage;
+                        const finalChunk = {
+                            content: '',
+                            accumulated,
+                            reasoning_content: '',
+                            isComplete: true,
+                            usage: anthropicUsage ? {
+                                prompt_tokens: anthropicUsage.input_tokens ?? 0,
+                                completion_tokens: anthropicUsage.output_tokens ?? 0,
+                                total_tokens: (anthropicUsage.input_tokens ?? 0) + (anthropicUsage.output_tokens ?? 0),
+                                time_cost: timeCost ?? 0,
+                                model_name: modelName,
+                                model_description: modelDescription,
+                                intent: modelConfig.intent
+                            } : void 0
+                        };
+                        options.onChunk(finalChunk);
+                        break;
+                    }
+                }
+                content = accumulated;
+            } else {
+                const result = await completion.create({
+                    model: modelName,
+                    system: 'You are a versatile professional in software UI automation',
+                    messages: messages.map((m)=>({
+                            role: 'user',
+                            content: Array.isArray(m.content) ? m.content.map(convertImageContent) : m.content
+                        })),
+                    response_format: responseFormat,
+                    ...commonConfig
+                });
+                timeCost = Date.now() - startTime;
+                content = result.content[0].text;
+                usage = result.usage;
+            }
+            assert(content, 'empty content');
+        }
+        if (isStreaming && !usage) {
+            const estimatedTokens = Math.max(1, Math.floor((content || '').length / 4));
+            usage = {
+                prompt_tokens: estimatedTokens,
+                completion_tokens: estimatedTokens,
+                total_tokens: 2 * estimatedTokens
+            };
+        }
+        return {
+            content: content || '',
+            usage: usage ? {
+                prompt_tokens: usage.prompt_tokens ?? 0,
+                completion_tokens: usage.completion_tokens ?? 0,
+                total_tokens: usage.total_tokens ?? 0,
+                time_cost: timeCost ?? 0,
+                model_name: modelName,
+                model_description: modelDescription,
+                intent: modelConfig.intent
+            } : void 0,
+            isStreamed: !!isStreaming
+        };
+    } catch (e) {
+        console.error(' call AI error', e);
+        const newError = new Error(`failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`, {
+            cause: e
+        });
+        throw newError;
+    }
+}
+const getResponseFormat = (modelName, AIActionTypeValue)=>{
+    let responseFormat;
+    if (modelName.includes('gpt-4')) switch(AIActionTypeValue){
+        case AIActionType.ASSERT:
+            responseFormat = assertSchema;
+            break;
+        case AIActionType.INSPECT_ELEMENT:
+            responseFormat = locatorSchema;
+            break;
+        case AIActionType.PLAN:
+            responseFormat = planSchema;
+            break;
+        case AIActionType.EXTRACT_DATA:
+        case AIActionType.DESCRIBE_ELEMENT:
+            responseFormat = {
+                type: AIResponseFormat.JSON
+            };
+            break;
+        case AIActionType.TEXT:
+            responseFormat = void 0;
+            break;
+    }
+    if ('gpt-4o-2024-05-13' === modelName && AIActionTypeValue !== AIActionType.TEXT) responseFormat = {
+        type: AIResponseFormat.JSON
+    };
+    return responseFormat;
+};
+async function callAIWithObjectResponse(messages, AIActionTypeValue, modelConfig) {
+    const response = await callAI(messages, AIActionTypeValue, modelConfig);
+    assert(response, 'empty response');
+    const vlMode = modelConfig.vlMode;
+    const jsonContent = safeParseJson(response.content, vlMode);
+    return {
+        content: jsonContent,
+        usage: response.usage
+    };
+}
+async function callAIWithStringResponse(msgs, AIActionTypeValue, modelConfig) {
+    const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);
+    return {
+        content,
+        usage
+    };
+}
+function extractJSONFromCodeBlock(response) {
+    try {
+        const jsonMatch = response.match(/^\s*(\{[\s\S]*\})\s*$/);
+        if (jsonMatch) return jsonMatch[1];
+        const codeBlockMatch = response.match(/```(?:json)?\s*(\{[\s\S]*?\})\s*```/);
+        if (codeBlockMatch) return codeBlockMatch[1];
+        const jsonLikeMatch = response.match(/\{[\s\S]*\}/);
+        if (jsonLikeMatch) return jsonLikeMatch[0];
+    } catch  {}
+    return response;
+}
+function preprocessDoubaoBboxJson(input) {
+    if (input.includes('bbox')) while(/\d+\s+\d+/.test(input))input = input.replace(/(\d+)\s+(\d+)/g, '$1,$2');
+    return input;
+}
+function safeParseJson(input, vlMode) {
+    const cleanJsonString = extractJSONFromCodeBlock(input);
+    if (null == cleanJsonString ? void 0 : cleanJsonString.match(/\((\d+),(\d+)\)/)) {
+        var _cleanJsonString_match;
+        return null == (_cleanJsonString_match = cleanJsonString.match(/\((\d+),(\d+)\)/)) ? void 0 : _cleanJsonString_match.slice(1).map(Number);
+    }
+    try {
+        return JSON.parse(cleanJsonString);
+    } catch  {}
+    try {
+        return JSON.parse(jsonrepair(cleanJsonString));
+    } catch (e) {}
+    if ('doubao-vision' === vlMode || 'vlm-ui-tars' === vlMode) {
+        const jsonString = preprocessDoubaoBboxJson(cleanJsonString);
+        return JSON.parse(jsonrepair(jsonString));
+    }
+    throw Error(`failed to parse json response: ${input}`);
+}
+export { callAI, callAIWithObjectResponse, callAIWithStringResponse, extractJSONFromCodeBlock, getResponseFormat, preprocessDoubaoBboxJson, safeParseJson };
+
+//# sourceMappingURL=index.mjs.map

package/dist/es/ai-model/service-caller/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"ai-model\\service-caller\\index.mjs","sources":["webpack://@sqaitech/core/./src/ai-model/service-caller/index.ts"],"sourcesContent":["import { AIResponseFormat, type AIUsageInfo } from '@/types';\r\nimport type { CodeGenerationChunk, StreamingCallback } from '@/types';\r\nimport { Anthropic } from '@anthropic-ai/sdk';\r\nimport {\r\n  DefaultAzureCredential,\r\n  getBearerTokenProvider,\r\n} from '@azure/identity';\r\nimport {\r\n  type IModelConfig,\r\n  SQAI_API_TYPE,\r\n  SQAI_LANGSMITH_DEBUG,\r\n  OPENAI_MAX_TOKENS,\r\n  type TVlModeTypes,\r\n  type UITarsModelVersion,\r\n  globalConfigManager,\r\n} from '@sqaitech/shared/env';\r\n\r\nimport { parseBase64 } from '@sqaitech/shared/img';\r\nimport { getDebug } from '@sqaitech/shared/logger';\r\nimport { assert } from '@sqaitech/shared/utils';\r\nimport { ifInBrowser } from '@sqaitech/shared/utils';\r\nimport { HttpsProxyAgent } from 'https-proxy-agent';\r\nimport { jsonrepair } from 'jsonrepair';\r\nimport OpenAI, { AzureOpenAI } from 'openai';\r\nimport type { ChatCompletionMessageParam } from 'openai/resources/index';\r\nimport type { Stream } from 'openai/streaming';\r\nimport { SocksProxyAgent } from 'socks-proxy-agent';\r\nimport { AIActionType, type AIArgs } from '../common';\r\nimport { assertSchema } from '../prompt/assertion';\r\nimport { locatorSchema } from '../prompt/llm-locator';\r\nimport { planSchema } from '../prompt/llm-planning';\r\n\r\nasync function createChatClient({\r\n  AIActionTypeValue,\r\n  modelConfig,\r\n}: {\r\n  AIActionTypeValue: AIActionType;\r\n  modelConfig: IModelConfig;\r\n}): Promise<{\r\n  completion: OpenAI.Chat.Completions;\r\n  style: 'openai' | 'anthropic';\r\n  modelName: string;\r\n  modelDescription: string;\r\n  uiTarsVersion?: UITarsModelVersion;\r\n  vlMode: TVlModeTypes | undefined;\r\n}> {\r\n  const {\r\n    socksProxy,\r\n    httpProxy,\r\n    modelName,\r\n    openaiBaseURL,\r\n    openaiApiKey,\r\n    openaiExtraConfig,\r\n    openaiUseAzureDeprecated,\r\n    useAzureOpenai,\r\n    azureOpenaiScope,\r\n    azureOpenaiKey,\r\n    azureOpenaiEndpoint,\r\n    azureOpenaiApiVersion,\r\n    azureOpenaiDeployment,\r\n    azureExtraConfig,\r\n    useAnthropicSdk,\r\n    anthropicApiKey,\r\n    modelDescription,\r\n    uiTarsModelVersion: uiTarsVersion,\r\n    vlMode,\r\n  } = modelConfig;\r\n\r\n  let openai: OpenAI | AzureOpenAI | undefined;\r\n\r\n  let proxyAgent = undefined;\r\n  const debugProxy = getDebug('ai:call:proxy');\r\n  if (httpProxy) {\r\n    debugProxy('using http proxy', httpProxy);\r\n    proxyAgent = new HttpsProxyAgent(httpProxy);\r\n  } else if (socksProxy) {\r\n    debugProxy('using socks proxy', socksProxy);\r\n    proxyAgent = new SocksProxyAgent(socksProxy);\r\n  }\r\n\r\n  if (openaiUseAzureDeprecated) {\r\n    // this is deprecated\r\n    openai = new AzureOpenAI({\r\n      baseURL: openaiBaseURL,\r\n      apiKey: openaiApiKey,\r\n      httpAgent: proxyAgent,\r\n      ...openaiExtraConfig,\r\n      dangerouslyAllowBrowser: true,\r\n    }) as OpenAI;\r\n  } else if (useAzureOpenai) {\r\n    // https://learn.microsoft.com/en-us/azure/ai-services/openai/chatgpt-quickstart?tabs=bash%2Cjavascript-key%2Ctypescript-keyless%2Cpython&pivots=programming-language-javascript#rest-api\r\n    // keyless authentication\r\n    let tokenProvider: any = undefined;\r\n    if (azureOpenaiScope) {\r\n      assert(\r\n        !ifInBrowser,\r\n        'Azure OpenAI is not supported in browser with Midscene.',\r\n      );\r\n      const credential = new DefaultAzureCredential();\r\n\r\n      tokenProvider = getBearerTokenProvider(credential, azureOpenaiScope);\r\n\r\n      openai = new AzureOpenAI({\r\n        azureADTokenProvider: tokenProvider,\r\n        endpoint: azureOpenaiEndpoint,\r\n        apiVersion: azureOpenaiApiVersion,\r\n        deployment: azureOpenaiDeployment,\r\n        ...openaiExtraConfig,\r\n        ...azureExtraConfig,\r\n      });\r\n    } else {\r\n      // endpoint, apiKey, apiVersion, deployment\r\n      openai = new AzureOpenAI({\r\n        apiKey: azureOpenaiKey,\r\n        endpoint: azureOpenaiEndpoint,\r\n        apiVersion: azureOpenaiApiVersion,\r\n        deployment: azureOpenaiDeployment,\r\n        dangerouslyAllowBrowser: true,\r\n        ...openaiExtraConfig,\r\n        ...azureExtraConfig,\r\n      });\r\n    }\r\n  } else if (!useAnthropicSdk) {\r\n    openai = new OpenAI({\r\n      baseURL: openaiBaseURL,\r\n      apiKey: openaiApiKey,\r\n      httpAgent: proxyAgent,\r\n      ...openaiExtraConfig,\r\n      defaultHeaders: {\r\n        ...(openaiExtraConfig?.defaultHeaders || {}),\r\n        [SQAI_API_TYPE]: AIActionTypeValue.toString(),\r\n      },\r\n      dangerouslyAllowBrowser: true,\r\n    });\r\n  }\r\n\r\n  if (\r\n    openai &&\r\n    globalConfigManager.getEnvConfigInBoolean(SQAI_LANGSMITH_DEBUG)\r\n  ) {\r\n    if (ifInBrowser) {\r\n      throw new Error('langsmith is not supported in browser');\r\n    }\r\n    console.log('DEBUGGING MODE: langsmith wrapper enabled');\r\n    const { wrapOpenAI } = await import('langsmith/wrappers');\r\n    openai = wrapOpenAI(openai);\r\n  }\r\n\r\n  if (typeof openai !== 'undefined') {\r\n    return {\r\n      completion: openai.chat.completions,\r\n      style: 'openai',\r\n      modelName,\r\n      modelDescription,\r\n      uiTarsVersion,\r\n      vlMode,\r\n    };\r\n  }\r\n\r\n  // Anthropic\r\n  if (useAnthropicSdk) {\r\n    openai = new Anthropic({\r\n      apiKey: anthropicApiKey,\r\n      httpAgent: proxyAgent,\r\n      dangerouslyAllowBrowser: true,\r\n    }) as any;\r\n  }\r\n\r\n  if (typeof openai !== 'undefined' && (openai as any).messages) {\r\n    return {\r\n      completion: (openai as any).messages,\r\n      style: 'anthropic',\r\n      modelName,\r\n      modelDescription,\r\n      uiTarsVersion,\r\n      vlMode,\r\n    };\r\n  }\r\n\r\n  throw new Error('Openai SDK or Anthropic SDK is not initialized');\r\n}\r\n\r\nexport async function callAI(\r\n  messages: ChatCompletionMessageParam[],\r\n  AIActionTypeValue: AIActionType,\r\n  modelConfig: IModelConfig,\r\n  options?: {\r\n    stream?: boolean;\r\n    onChunk?: StreamingCallback;\r\n  },\r\n): Promise<{ content: string; usage?: AIUsageInfo; isStreamed: boolean }> {\r\n  const {\r\n    completion,\r\n    style,\r\n    modelName,\r\n    modelDescription,\r\n    uiTarsVersion,\r\n    vlMode,\r\n  } = await createChatClient({\r\n    AIActionTypeValue,\r\n    modelConfig,\r\n  });\r\n\r\n  const responseFormat = getResponseFormat(modelName, AIActionTypeValue);\r\n\r\n  const maxTokens = globalConfigManager.getEnvConfigValue(OPENAI_MAX_TOKENS);\r\n  const debugCall = getDebug('ai:call');\r\n  const debugProfileStats = getDebug('ai:profile:stats');\r\n  const debugProfileDetail = getDebug('ai:profile:detail');\r\n\r\n  const startTime = Date.now();\r\n\r\n  const isStreaming = options?.stream && options?.onChunk;\r\n  let content: string | undefined;\r\n  let accumulated = '';\r\n  let usage: OpenAI.CompletionUsage | undefined;\r\n  let timeCost: number | undefined;\r\n\r\n  const commonConfig = {\r\n    temperature: vlMode === 'vlm-ui-tars' ? 0.0 : 0.1,\r\n    stream: !!isStreaming,\r\n    max_tokens:\r\n      typeof maxTokens === 'number'\r\n        ? maxTokens\r\n        : Number.parseInt(maxTokens || '2048', 10),\r\n    ...(vlMode === 'qwen-vl' || vlMode === 'qwen3-vl' // qwen specific config\r\n      ? {\r\n          vl_high_resolution_images: true,\r\n        }\r\n      : {}),\r\n  };\r\n\r\n  try {\r\n    if (style === 'openai') {\r\n      debugCall(\r\n        `sending ${isStreaming ? 'streaming ' : ''}request to ${modelName}`,\r\n      );\r\n\r\n      if (isStreaming) {\r\n        const stream = (await completion.create(\r\n          {\r\n            model: modelName,\r\n            messages,\r\n            response_format: responseFormat,\r\n            ...commonConfig,\r\n          },\r\n          {\r\n            stream: true,\r\n          },\r\n        )) as Stream<OpenAI.Chat.Completions.ChatCompletionChunk> & {\r\n          _request_id?: string | null;\r\n        };\r\n\r\n        for await (const chunk of stream) {\r\n          const content = chunk.choices?.[0]?.delta?.content || '';\r\n          const reasoning_content =\r\n            (chunk.choices?.[0]?.delta as any)?.reasoning_content || '';\r\n\r\n          // Check for usage info in any chunk (OpenAI provides usage in separate chunks)\r\n          if (chunk.usage) {\r\n            usage = chunk.usage;\r\n          }\r\n\r\n          if (content || reasoning_content) {\r\n            accumulated += content;\r\n            const chunkData: CodeGenerationChunk = {\r\n              content,\r\n              reasoning_content,\r\n              accumulated,\r\n              isComplete: false,\r\n              usage: undefined,\r\n            };\r\n            options.onChunk!(chunkData);\r\n          }\r\n\r\n          // Check if stream is complete\r\n          if (chunk.choices?.[0]?.finish_reason) {\r\n            timeCost = Date.now() - startTime;\r\n\r\n            // If usage is not available from the stream, provide a basic usage info\r\n            if (!usage) {\r\n              // Estimate token counts based on content length (rough approximation)\r\n              const estimatedTokens = Math.max(\r\n                1,\r\n                Math.floor(accumulated.length / 4),\r\n              );\r\n              usage = {\r\n                prompt_tokens: estimatedTokens,\r\n                completion_tokens: estimatedTokens,\r\n                total_tokens: estimatedTokens * 2,\r\n              };\r\n            }\r\n\r\n            // Send final chunk\r\n            const finalChunk: CodeGenerationChunk = {\r\n              content: '',\r\n              accumulated,\r\n              reasoning_content: '',\r\n              isComplete: true,\r\n              usage: {\r\n                prompt_tokens: usage.prompt_tokens ?? 0,\r\n                completion_tokens: usage.completion_tokens ?? 0,\r\n                total_tokens: usage.total_tokens ?? 0,\r\n                time_cost: timeCost ?? 0,\r\n                model_name: modelName,\r\n                model_description: modelDescription,\r\n                intent: modelConfig.intent,\r\n              },\r\n            };\r\n            options.onChunk!(finalChunk);\r\n            break;\r\n          }\r\n        }\r\n        content = accumulated;\r\n        debugProfileStats(\r\n          `streaming model, ${modelName}, mode, ${vlMode || 'default'}, cost-ms, ${timeCost}`,\r\n        );\r\n      } else {\r\n        const result = await completion.create({\r\n          model: modelName,\r\n          messages,\r\n          response_format: responseFormat,\r\n          ...commonConfig,\r\n        } as any);\r\n        timeCost = Date.now() - startTime;\r\n\r\n        debugProfileStats(\r\n          `model, ${modelName}, mode, ${vlMode || 'default'}, ui-tars-version, ${uiTarsVersion}, prompt-tokens, ${result.usage?.prompt_tokens || ''}, completion-tokens, ${result.usage?.completion_tokens || ''}, total-tokens, ${result.usage?.total_tokens || ''}, cost-ms, ${timeCost}, requestId, ${result._request_id || ''}`,\r\n        );\r\n\r\n        debugProfileDetail(\r\n          `model usage detail: ${JSON.stringify(result.usage)}`,\r\n        );\r\n\r\n        assert(\r\n          result.choices,\r\n          `invalid response from LLM service: ${JSON.stringify(result)}`,\r\n        );\r\n        content = result.choices[0].message.content!;\r\n        usage = result.usage;\r\n      }\r\n\r\n      debugCall(`response: ${content}`);\r\n      assert(content, 'empty content');\r\n    } else if (style === 'anthropic') {\r\n      const convertImageContent = (content: any) => {\r\n        if (content.type === 'image_url') {\r\n          const imgBase64 = content.image_url.url;\r\n          assert(imgBase64, 'image_url is required');\r\n          const { mimeType, body } = parseBase64(content.image_url.url);\r\n          return {\r\n            source: {\r\n              type: 'base64',\r\n              media_type: mimeType,\r\n              data: body,\r\n            },\r\n            type: 'image',\r\n          };\r\n        }\r\n        return content;\r\n      };\r\n\r\n      if (isStreaming) {\r\n        const stream = (await completion.create({\r\n          model: modelName,\r\n          system: 'You are a versatile professional in software UI automation',\r\n          messages: messages.map((m) => ({\r\n            role: 'user',\r\n            content: Array.isArray(m.content)\r\n              ? (m.content as any).map(convertImageContent)\r\n              : m.content,\r\n          })),\r\n          response_format: responseFormat,\r\n          ...commonConfig,\r\n        } as any)) as any;\r\n\r\n        for await (const chunk of stream) {\r\n          const content = chunk.delta?.text || '';\r\n          if (content) {\r\n            accumulated += content;\r\n            const chunkData: CodeGenerationChunk = {\r\n              content,\r\n              accumulated,\r\n              reasoning_content: '',\r\n              isComplete: false,\r\n              usage: undefined,\r\n            };\r\n            options.onChunk!(chunkData);\r\n          }\r\n\r\n          // Check if stream is complete\r\n          if (chunk.type === 'message_stop') {\r\n            timeCost = Date.now() - startTime;\r\n            const anthropicUsage = chunk.usage;\r\n\r\n            // Send final chunk\r\n            const finalChunk: CodeGenerationChunk = {\r\n              content: '',\r\n              accumulated,\r\n              reasoning_content: '',\r\n              isComplete: true,\r\n              usage: anthropicUsage\r\n                ? {\r\n                    prompt_tokens: anthropicUsage.input_tokens ?? 0,\r\n                    completion_tokens: anthropicUsage.output_tokens ?? 0,\r\n                    total_tokens:\r\n                      (anthropicUsage.input_tokens ?? 0) +\r\n                      (anthropicUsage.output_tokens ?? 0),\r\n                    time_cost: timeCost ?? 0,\r\n                    model_name: modelName,\r\n                    model_description: modelDescription,\r\n                    intent: modelConfig.intent,\r\n                  }\r\n                : undefined,\r\n            };\r\n            options.onChunk!(finalChunk);\r\n            break;\r\n          }\r\n        }\r\n        content = accumulated;\r\n      } else {\r\n        const result = await completion.create({\r\n          model: modelName,\r\n          system: 'You are a versatile professional in software UI automation',\r\n          messages: messages.map((m) => ({\r\n            role: 'user',\r\n            content: Array.isArray(m.content)\r\n              ? (m.content as any).map(convertImageContent)\r\n              : m.content,\r\n          })),\r\n          response_format: responseFormat,\r\n          ...commonConfig,\r\n        } as any);\r\n        timeCost = Date.now() - startTime;\r\n        content = (result as any).content[0].text as string;\r\n        usage = result.usage;\r\n      }\r\n\r\n      assert(content, 'empty content');\r\n    }\r\n    // Ensure we always have usage info for streaming responses\r\n    if (isStreaming && !usage) {\r\n      // Estimate token counts based on content length (rough approximation)\r\n      const estimatedTokens = Math.max(\r\n        1,\r\n        Math.floor((content || '').length / 4),\r\n      );\r\n      usage = {\r\n        prompt_tokens: estimatedTokens,\r\n        completion_tokens: estimatedTokens,\r\n        total_tokens: estimatedTokens * 2,\r\n      };\r\n    }\r\n\r\n    return {\r\n      content: content || '',\r\n      usage: usage\r\n        ? {\r\n            prompt_tokens: usage.prompt_tokens ?? 0,\r\n            completion_tokens: usage.completion_tokens ?? 0,\r\n            total_tokens: usage.total_tokens ?? 0,\r\n            time_cost: timeCost ?? 0,\r\n            model_name: modelName,\r\n            model_description: modelDescription,\r\n            intent: modelConfig.intent,\r\n          }\r\n        : undefined,\r\n      isStreamed: !!isStreaming,\r\n    };\r\n  } catch (e: any) {\r\n    console.error(' call AI error', e);\r\n    const newError = new Error(\r\n      `failed to call ${isStreaming ? 'streaming ' : ''}AI model service: ${e.message}. Trouble shooting: https://midscenejs.com/model-provider.html`,\r\n      {\r\n        cause: e,\r\n      },\r\n    );\r\n    throw newError;\r\n  }\r\n}\r\n\r\nexport const getResponseFormat = (\r\n  modelName: string,\r\n  AIActionTypeValue: AIActionType,\r\n):\r\n  | OpenAI.ChatCompletionCreateParams['response_format']\r\n  | OpenAI.ResponseFormatJSONObject => {\r\n  let responseFormat:\r\n    | OpenAI.ChatCompletionCreateParams['response_format']\r\n    | OpenAI.ResponseFormatJSONObject\r\n    | undefined;\r\n\r\n  if (modelName.includes('gpt-4')) {\r\n    switch (AIActionTypeValue) {\r\n      case AIActionType.ASSERT:\r\n        responseFormat = assertSchema;\r\n        break;\r\n      case AIActionType.INSPECT_ELEMENT:\r\n        responseFormat = locatorSchema;\r\n        break;\r\n      case AIActionType.PLAN:\r\n        responseFormat = planSchema;\r\n        break;\r\n      case AIActionType.EXTRACT_DATA:\r\n      case AIActionType.DESCRIBE_ELEMENT:\r\n        responseFormat = { type: AIResponseFormat.JSON };\r\n        break;\r\n      case AIActionType.TEXT:\r\n        // No response format for plain text - return as-is\r\n        responseFormat = undefined;\r\n        break;\r\n    }\r\n  }\r\n\r\n  // gpt-4o-2024-05-13 only supports json_object response format\r\n  // Skip for plain text to allow string output\r\n  if (\r\n    modelName === 'gpt-4o-2024-05-13' &&\r\n    AIActionTypeValue !== AIActionType.TEXT\r\n  ) {\r\n    responseFormat = { type: AIResponseFormat.JSON };\r\n  }\r\n\r\n  return responseFormat;\r\n};\r\n\r\nexport async function callAIWithObjectResponse<T>(\r\n  messages: ChatCompletionMessageParam[],\r\n  AIActionTypeValue: AIActionType,\r\n  modelConfig: IModelConfig,\r\n): Promise<{ content: T; usage?: AIUsageInfo }> {\r\n  const response = await callAI(messages, AIActionTypeValue, modelConfig);\r\n  assert(response, 'empty response');\r\n  const vlMode = modelConfig.vlMode;\r\n  const jsonContent = safeParseJson(response.content, vlMode);\r\n  return { content: jsonContent, usage: response.usage };\r\n}\r\n\r\nexport async function callAIWithStringResponse(\r\n  msgs: AIArgs,\r\n  AIActionTypeValue: AIActionType,\r\n  modelConfig: IModelConfig,\r\n): Promise<{ content: string; usage?: AIUsageInfo }> {\r\n  const { content, usage } = await callAI(msgs, AIActionTypeValue, modelConfig);\r\n  return { content, usage };\r\n}\r\n\r\nexport function extractJSONFromCodeBlock(response: string) {\r\n  try {\r\n    // First, try to match a JSON object directly in the response\r\n    const jsonMatch = response.match(/^\\s*(\\{[\\s\\S]*\\})\\s*$/);\r\n    if (jsonMatch) {\r\n      return jsonMatch[1];\r\n    }\r\n\r\n    // If no direct JSON object is found, try to extract JSON from a code block\r\n    const codeBlockMatch = response.match(\r\n      /```(?:json)?\\s*(\\{[\\s\\S]*?\\})\\s*```/,\r\n    );\r\n    if (codeBlockMatch) {\r\n      return codeBlockMatch[1];\r\n    }\r\n\r\n    // If no code block is found, try to find a JSON-like structure in the text\r\n    const jsonLikeMatch = response.match(/\\{[\\s\\S]*\\}/);\r\n    if (jsonLikeMatch) {\r\n      return jsonLikeMatch[0];\r\n    }\r\n  } catch {}\r\n  // If no JSON-like structure is found, return the original response\r\n  return response;\r\n}\r\n\r\nexport function preprocessDoubaoBboxJson(input: string) {\r\n  if (input.includes('bbox')) {\r\n    // when its values like 940 445 969 490, replace all /\\d+\\s+\\d+/g with /$1,$2/g\r\n    while (/\\d+\\s+\\d+/.test(input)) {\r\n      input = input.replace(/(\\d+)\\s+(\\d+)/g, '$1,$2');\r\n    }\r\n  }\r\n  return input;\r\n}\r\n\r\nexport function safeParseJson(input: string, vlMode: TVlModeTypes | undefined) {\r\n  const cleanJsonString = extractJSONFromCodeBlock(input);\r\n  // match the point\r\n  if (cleanJsonString?.match(/\\((\\d+),(\\d+)\\)/)) {\r\n    return cleanJsonString\r\n      .match(/\\((\\d+),(\\d+)\\)/)\r\n      ?.slice(1)\r\n      .map(Number);\r\n  }\r\n  try {\r\n    return JSON.parse(cleanJsonString);\r\n  } catch {}\r\n  try {\r\n    return JSON.parse(jsonrepair(cleanJsonString));\r\n  } catch (e) {}\r\n\r\n  if (vlMode === 'doubao-vision' || vlMode === 'vlm-ui-tars') {\r\n    const jsonString = preprocessDoubaoBboxJson(cleanJsonString);\r\n    return JSON.parse(jsonrepair(jsonString));\r\n  }\r\n  throw Error(`failed to parse json response: ${input}`);\r\n}\r\n"],"names":["createChatClient","AIActionTypeValue","modelConfig","socksProxy","httpProxy","modelName","openaiBaseURL","openaiApiKey","openaiExtraConfig","openaiUseAzureDeprecated","useAzureOpenai","azureOpenaiScope","azureOpenaiKey","azureOpenaiEndpoint","azureOpenaiApiVersion","azureOpenaiDeployment","azureExtraConfig","useAnthropicSdk","anthropicApiKey","modelDescription","uiTarsVersion","vlMode","openai","proxyAgent","debugProxy","getDebug","HttpsProxyAgent","SocksProxyAgent","AzureOpenAI","tokenProvider","assert","ifInBrowser","credential","DefaultAzureCredential","getBearerTokenProvider","OpenAI","SQAI_API_TYPE","globalConfigManager","SQAI_LANGSMITH_DEBUG","Error","console","wrapOpenAI","Anthropic","callAI","messages","options","completion","style","responseFormat","getResponseFormat","maxTokens","OPENAI_MAX_TOKENS","debugCall","debugProfileStats","debugProfileDetail","startTime","Date","isStreaming","content","accumulated","usage","timeCost","commonConfig","Number","stream","chunk","_chunk_choices__delta","_chunk_choices__delta1","_chunk_choices_2","reasoning_content","chunkData","undefined","estimatedTokens","Math","finalChunk","_result_usage","_result_usage1","_result_usage2","result","JSON","convertImageContent","imgBase64","mimeType","body","parseBase64","m","Array","_chunk_delta","anthropicUsage","e","newError","AIActionType","assertSchema","locatorSchema","planSchema","AIResponseFormat","callAIWithObjectResponse","response","jsonContent","safeParseJson","callAIWithStringResponse","msgs","extractJSONFromCodeBlock","jsonMatch","codeBlockMatch","jsonLikeMatch","preprocessDoubaoBboxJson","input","cleanJsonString","_cleanJsonString_match","jsonrepair","jsonString"],"mappings":";;;;;;;;;;;;;;;AAgCA,eAAeA,iBAAiB,EAC9BC,iBAAiB,EACjBC,WAAW,EAIZ;IAQC,MAAM,EACJC,UAAU,EACVC,SAAS,EACTC,SAAS,EACTC,aAAa,EACbC,YAAY,EACZC,iBAAiB,EACjBC,wBAAwB,EACxBC,cAAc,EACdC,gBAAgB,EAChBC,cAAc,EACdC,mBAAmB,EACnBC,qBAAqB,EACrBC,qBAAqB,EACrBC,gBAAgB,EAChBC,eAAe,EACfC,eAAe,EACfC,gBAAgB,EAChB,oBAAoBC,aAAa,EACjCC,MAAM,EACP,GAAGnB;IAEJ,IAAIoB;IAEJ,IAAIC;IACJ,MAAMC,aAAaC,SAAS;IAC5B,IAAIrB,WAAW;QACboB,WAAW,oBAAoBpB;QAC/BmB,aAAa,IAAIG,gBAAgBtB;IACnC,OAAO,IAAID,YAAY;QACrBqB,WAAW,qBAAqBrB;QAChCoB,aAAa,IAAII,gBAAgBxB;IACnC;IAEA,IAAIM,0BAEFa,SAAS,IAAIM,YAAY;QACvB,SAAStB;QACT,QAAQC;QACR,WAAWgB;QACX,GAAGf,iBAAiB;QACpB,yBAAyB;IAC3B;SACK,IAAIE,gBAAgB;QAGzB,IAAImB;QACJ,IAAIlB,kBAAkB;YACpBmB,OACE,CAACC,aACD;YAEF,MAAMC,aAAa,IAAIC;YAEvBJ,gBAAgBK,uBAAuBF,YAAYrB;YAEnDW,SAAS,IAAIM,YAAY;gBACvB,sBAAsBC;gBACtB,UAAUhB;gBACV,YAAYC;gBACZ,YAAYC;gBACZ,GAAGP,iBAAiB;gBACpB,GAAGQ,gBAAgB;YACrB;QACF,OAEEM,SAAS,IAAIM,YAAY;YACvB,QAAQhB;YACR,UAAUC;YACV,YAAYC;YACZ,YAAYC;YACZ,yBAAyB;YACzB,GAAGP,iBAAiB;YACpB,GAAGQ,gBAAgB;QACrB;IAEJ,OAAO,IAAI,CAACC,iBACVK,SAAS,IAAIa,SAAO;QAClB,SAAS7B;QACT,QAAQC;QACR,WAAWgB;QACX,GAAGf,iBAAiB;QACpB,gBAAgB;YACd,GAAIA,AAAAA,CAAAA,QAAAA,oBAAAA,KAAAA,IAAAA,kBAAmB,cAAc,AAAD,KAAK,CAAC,CAAC;YAC3C,CAAC4B,cAAc,EAAEnC,kBAAkB,QAAQ;QAC7C;QACA,yBAAyB;IAC3B;IAGF,IACEqB,UACAe,oBAAoB,qBAAqB,CAACC,uBAC1C;QACA,IAAIP,aACF,MAAM,IAAIQ,MAAM;QAElBC,QAAQ,GAAG,CAAC;QACZ,MAAM,EAAEC,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC;QACpCnB,SAASmB,WAAWnB;IACtB;IAEA,IAAI,AAAkB,WAAXA,QACT,OAAO;QACL,YAAYA,OAAO,IAAI,CAAC,WAAW;QACnC,OAAO;QACPjB;QACAc;QACAC;QACAC;IACF;IAIF,IAAIJ,iBACFK,SAAS,IAAIoB,UAAU;QACrB,QAAQxB;QACR,WAAWK;QACX,yBAAyB;IAC3B;IAGF,IAAI,AAAkB,WAAXD,UAA2BA,OAAe,QAAQ,EAC3D,OAAO;QACL,YAAaA,OAAe,QAAQ;QACpC,OAAO;QACPjB;QACAc;QACAC;QACAC;IACF;IAGF,MAAM,IAAIkB,MAAM;AAClB;AAEO,eAAeI,OACpBC,QAAsC,EACtC3C,iBAA+B,EAC/BC,WAAyB,EACzB2C,OAGC;IAED,MAAM,EACJC,UAAU,EACVC,KAAK,EACL1C,SAAS,EACTc,gBAAgB,EAChBC,aAAa,EACbC,MAAM,EACP,GAAG,MAAMrB,iBAAiB;QACzBC;QACAC;IACF;IAEA,MAAM8C,iBAAiBC,kBAAkB5C,WAAWJ;IAEpD,MAAMiD,YAAYb,oBAAoB,iBAAiB,CAACc;IACxD,MAAMC,YAAY3B,SAAS;IAC3B,MAAM4B,oBAAoB5B,SAAS;IACnC,MAAM6B,qBAAqB7B,SAAS;IAEpC,MAAM8B,YAAYC,KAAK,GAAG;IAE1B,MAAMC,cAAcZ,AAAAA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,MAAM,AAAD,KAAKA,CAAAA,QAAAA,UAAAA,KAAAA,IAAAA,QAAS,OAAO,AAAD;IACtD,IAAIa;IACJ,IAAIC,cAAc;IAClB,IAAIC;IACJ,IAAIC;IAEJ,MAAMC,eAAe;QACnB,aAAazC,AAAW,kBAAXA,SAA2B,MAAM;QAC9C,QAAQ,CAAC,CAACoC;QACV,YACE,AAAqB,YAArB,OAAOP,YACHA,YACAa,OAAO,QAAQ,CAACb,aAAa,QAAQ;QAC3C,GAAI7B,AAAW,cAAXA,UAAwBA,AAAW,eAAXA,SACxB;YACE,2BAA2B;QAC7B,IACA,CAAC,CAAC;IACR;IAEA,IAAI;QACF,IAAI0B,AAAU,aAAVA,OAAoB;YACtBK,UACE,CAAC,QAAQ,EAAEK,cAAc,eAAe,GAAG,WAAW,EAAEpD,WAAW;YAGrE,IAAIoD,aAAa;gBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CACrC;oBACE,OAAOzC;oBACPuC;oBACA,iBAAiBI;oBACjB,GAAGc,YAAY;gBACjB,GACA;oBACE,QAAQ;gBACV;gBAKF,WAAW,MAAMG,SAASD,OAAQ;wBAChBE,uBAAAA,iBAAAA,gBAEbC,wBAAAA,kBAAAA,iBAoBCC,kBAAAA;oBAtBJ,MAAMV,UAAUQ,AAAAA,SAAAA,CAAAA,iBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,kBAAAA,cAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,wBAAAA,gBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,sBAA2B,OAAO,AAAD,KAAK;oBACtD,MAAMG,oBACJ,AAAC,SAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,QAAAA,CAAAA,yBAAAA,iBAAoB,KAAK,AAAD,IAAxBA,KAAAA,IAAAA,uBAAmC,iBAAiB,AAAD,KAAK;oBAG3D,IAAIF,MAAM,KAAK,EACbL,QAAQK,MAAM,KAAK;oBAGrB,IAAIP,WAAWW,mBAAmB;wBAChCV,eAAeD;wBACf,MAAMY,YAAiC;4BACrCZ;4BACAW;4BACAV;4BACA,YAAY;4BACZ,OAAOY;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAI,QAAAF,CAAAA,kBAAAA,MAAM,OAAO,AAAD,IAAZA,KAAAA,IAAAA,QAAAA,CAAAA,mBAAAA,eAAe,CAAC,EAAE,AAAD,IAAjBA,KAAAA,IAAAA,iBAAoB,aAAa,EAAE;wBACrCP,WAAWL,KAAK,GAAG,KAAKD;wBAGxB,IAAI,CAACK,OAAO;4BAEV,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAACd,YAAY,MAAM,GAAG;4BAElCC,QAAQ;gCACN,eAAeY;gCACf,mBAAmBA;gCACnB,cAAcA,AAAkB,IAAlBA;4BAChB;wBACF;wBAGA,MAAME,aAAkC;4BACtC,SAAS;4BACTf;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO;gCACL,eAAeC,MAAM,aAAa,IAAI;gCACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gCAC9C,cAAcA,MAAM,YAAY,IAAI;gCACpC,WAAWC,YAAY;gCACvB,YAAYxD;gCACZ,mBAAmBc;gCACnB,QAAQjB,YAAY,MAAM;4BAC5B;wBACF;wBACA2C,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAhB,UAAUC;gBACVN,kBACE,CAAC,iBAAiB,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,WAAW,EAAEwC,UAAU;YAEvF,OAAO;oBAUqGc,eAAyDC,gBAAwDC;gBAT3N,MAAMC,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrC,OAAOzC;oBACPuC;oBACA,iBAAiBI;oBACjB,GAAGc,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBAExBF,kBACE,CAAC,OAAO,EAAEhD,UAAU,QAAQ,EAAEgB,UAAU,UAAU,mBAAmB,EAAED,cAAc,iBAAiB,EAAEuD,AAAAA,SAAAA,CAAAA,gBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,cAAc,aAAa,AAAD,KAAK,GAAG,qBAAqB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,iBAAiB,AAAD,KAAK,GAAG,gBAAgB,EAAEC,AAAAA,SAAAA,CAAAA,iBAAAA,OAAO,KAAK,AAAD,IAAXA,KAAAA,IAAAA,eAAc,YAAY,AAAD,KAAK,GAAG,WAAW,EAAEhB,SAAS,aAAa,EAAEiB,OAAO,WAAW,IAAI,IAAI;gBAG3TxB,mBACE,CAAC,oBAAoB,EAAEyB,KAAK,SAAS,CAACD,OAAO,KAAK,GAAG;gBAGvDhD,OACEgD,OAAO,OAAO,EACd,CAAC,mCAAmC,EAAEC,KAAK,SAAS,CAACD,SAAS;gBAEhEpB,UAAUoB,OAAO,OAAO,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;gBAC3ClB,QAAQkB,OAAO,KAAK;YACtB;YAEA1B,UAAU,CAAC,UAAU,EAAEM,SAAS;YAChC5B,OAAO4B,SAAS;QAClB,OAAO,IAAIX,AAAU,gBAAVA,OAAuB;YAChC,MAAMiC,sBAAsB,CAACtB;gBAC3B,IAAIA,AAAiB,gBAAjBA,QAAQ,IAAI,EAAkB;oBAChC,MAAMuB,YAAYvB,QAAQ,SAAS,CAAC,GAAG;oBACvC5B,OAAOmD,WAAW;oBAClB,MAAM,EAAEC,QAAQ,EAAEC,IAAI,EAAE,GAAGC,YAAY1B,QAAQ,SAAS,CAAC,GAAG;oBAC5D,OAAO;wBACL,QAAQ;4BACN,MAAM;4BACN,YAAYwB;4BACZ,MAAMC;wBACR;wBACA,MAAM;oBACR;gBACF;gBACA,OAAOzB;YACT;YAEA,IAAID,aAAa;gBACf,MAAMO,SAAU,MAAMlB,WAAW,MAAM,CAAC;oBACtC,OAAOzC;oBACP,QAAQ;oBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBrC;oBACjB,GAAGc,YAAY;gBACjB;gBAEA,WAAW,MAAMG,SAASD,OAAQ;wBAChBuB;oBAAhB,MAAM7B,UAAU6B,AAAAA,SAAAA,CAAAA,eAAAA,MAAM,KAAK,AAAD,IAAVA,KAAAA,IAAAA,aAAa,IAAI,AAAD,KAAK;oBACrC,IAAI7B,SAAS;wBACXC,eAAeD;wBACf,MAAMY,YAAiC;4BACrCZ;4BACAC;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAOY;wBACT;wBACA1B,QAAQ,OAAO,CAAEyB;oBACnB;oBAGA,IAAIL,AAAe,mBAAfA,MAAM,IAAI,EAAqB;wBACjCJ,WAAWL,KAAK,GAAG,KAAKD;wBACxB,MAAMiC,iBAAiBvB,MAAM,KAAK;wBAGlC,MAAMS,aAAkC;4BACtC,SAAS;4BACTf;4BACA,mBAAmB;4BACnB,YAAY;4BACZ,OAAO6B,iBACH;gCACE,eAAeA,eAAe,YAAY,IAAI;gCAC9C,mBAAmBA,eAAe,aAAa,IAAI;gCACnD,cACGA,AAAAA,CAAAA,eAAe,YAAY,IAAI,KAC/BA,CAAAA,eAAe,aAAa,IAAI;gCACnC,WAAW3B,YAAY;gCACvB,YAAYxD;gCACZ,mBAAmBc;gCACnB,QAAQjB,YAAY,MAAM;4BAC5B,IACAqE;wBACN;wBACA1B,QAAQ,OAAO,CAAE6B;wBACjB;oBACF;gBACF;gBACAhB,UAAUC;YACZ,OAAO;gBACL,MAAMmB,SAAS,MAAMhC,WAAW,MAAM,CAAC;oBACrC,OAAOzC;oBACP,QAAQ;oBACR,UAAUuC,SAAS,GAAG,CAAC,CAACyC,IAAO;4BAC7B,MAAM;4BACN,SAASC,MAAM,OAAO,CAACD,EAAE,OAAO,IAC3BA,EAAE,OAAO,CAAS,GAAG,CAACL,uBACvBK,EAAE,OAAO;wBACf;oBACA,iBAAiBrC;oBACjB,GAAGc,YAAY;gBACjB;gBACAD,WAAWL,KAAK,GAAG,KAAKD;gBACxBG,UAAWoB,OAAe,OAAO,CAAC,EAAE,CAAC,IAAI;gBACzClB,QAAQkB,OAAO,KAAK;YACtB;YAEAhD,OAAO4B,SAAS;QAClB;QAEA,IAAID,eAAe,CAACG,OAAO;YAEzB,MAAMY,kBAAkBC,KAAK,GAAG,CAC9B,GACAA,KAAK,KAAK,CAAEf,AAAAA,CAAAA,WAAW,EAAC,EAAG,MAAM,GAAG;YAEtCE,QAAQ;gBACN,eAAeY;gBACf,mBAAmBA;gBACnB,cAAcA,AAAkB,IAAlBA;YAChB;QACF;QAEA,OAAO;YACL,SAASd,WAAW;YACpB,OAAOE,QACH;gBACE,eAAeA,MAAM,aAAa,IAAI;gBACtC,mBAAmBA,MAAM,iBAAiB,IAAI;gBAC9C,cAAcA,MAAM,YAAY,IAAI;gBACpC,WAAWC,YAAY;gBACvB,YAAYxD;gBACZ,mBAAmBc;gBACnB,QAAQjB,YAAY,MAAM;YAC5B,IACAqE;YACJ,YAAY,CAAC,CAACd;QAChB;IACF,EAAE,OAAOgC,GAAQ;QACfjD,QAAQ,KAAK,CAAC,kBAAkBiD;QAChC,MAAMC,WAAW,IAAInD,MACnB,CAAC,eAAe,EAAEkB,cAAc,eAAe,GAAG,kBAAkB,EAAEgC,EAAE,OAAO,CAAC,8DAA8D,CAAC,EAC/I;YACE,OAAOA;QACT;QAEF,MAAMC;IACR;AACF;AAEO,MAAMzC,oBAAoB,CAC/B5C,WACAJ;IAIA,IAAI+C;IAKJ,IAAI3C,UAAU,QAAQ,CAAC,UACrB,OAAQJ;QACN,KAAK0F,aAAa,MAAM;YACtB3C,iBAAiB4C;YACjB;QACF,KAAKD,aAAa,eAAe;YAC/B3C,iBAAiB6C;YACjB;QACF,KAAKF,aAAa,IAAI;YACpB3C,iBAAiB8C;YACjB;QACF,KAAKH,aAAa,YAAY;QAC9B,KAAKA,aAAa,gBAAgB;YAChC3C,iBAAiB;gBAAE,MAAM+C,iBAAiB,IAAI;YAAC;YAC/C;QACF,KAAKJ,aAAa,IAAI;YAEpB3C,iBAAiBuB;YACjB;IACJ;IAKF,IACElE,AAAc,wBAAdA,aACAJ,sBAAsB0F,aAAa,IAAI,EAEvC3C,iBAAiB;QAAE,MAAM+C,iBAAiB,IAAI;IAAC;IAGjD,OAAO/C;AACT;AAEO,eAAegD,yBACpBpD,QAAsC,EACtC3C,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM+F,WAAW,MAAMtD,OAAOC,UAAU3C,mBAAmBC;IAC3D4B,OAAOmE,UAAU;IACjB,MAAM5E,SAASnB,YAAY,MAAM;IACjC,MAAMgG,cAAcC,cAAcF,SAAS,OAAO,EAAE5E;IACpD,OAAO;QAAE,SAAS6E;QAAa,OAAOD,SAAS,KAAK;IAAC;AACvD;AAEO,eAAeG,yBACpBC,IAAY,EACZpG,iBAA+B,EAC/BC,WAAyB;IAEzB,MAAM,EAAEwD,OAAO,EAAEE,KAAK,EAAE,GAAG,MAAMjB,OAAO0D,MAAMpG,mBAAmBC;IACjE,OAAO;QAAEwD;QAASE;IAAM;AAC1B;AAEO,SAAS0C,yBAAyBL,QAAgB;IACvD,IAAI;QAEF,MAAMM,YAAYN,SAAS,KAAK,CAAC;QACjC,IAAIM,WACF,OAAOA,SAAS,CAAC,EAAE;QAIrB,MAAMC,iBAAiBP,SAAS,KAAK,CACnC;QAEF,IAAIO,gBACF,OAAOA,cAAc,CAAC,EAAE;QAI1B,MAAMC,gBAAgBR,SAAS,KAAK,CAAC;QACrC,IAAIQ,eACF,OAAOA,aAAa,CAAC,EAAE;IAE3B,EAAE,OAAM,CAAC;IAET,OAAOR;AACT;AAEO,SAASS,yBAAyBC,KAAa;IACpD,IAAIA,MAAM,QAAQ,CAAC,SAEjB,MAAO,YAAY,IAAI,CAACA,OACtBA,QAAQA,MAAM,OAAO,CAAC,kBAAkB;IAG5C,OAAOA;AACT;AAEO,SAASR,cAAcQ,KAAa,EAAEtF,MAAgC;IAC3E,MAAMuF,kBAAkBN,yBAAyBK;IAEjD,IAAIC,QAAAA,kBAAAA,KAAAA,IAAAA,gBAAiB,KAAK,CAAC,oBAAoB;YACtCC;QAAP,OAAO,QAAAA,CAAAA,yBAAAA,gBACJ,KAAK,CAAC,kBAAiB,IADnBA,KAAAA,IAAAA,uBAEH,KAAK,CAAC,GACP,GAAG,CAAC9C;IACT;IACA,IAAI;QACF,OAAOgB,KAAK,KAAK,CAAC6B;IACpB,EAAE,OAAM,CAAC;IACT,IAAI;QACF,OAAO7B,KAAK,KAAK,CAAC+B,WAAWF;IAC/B,EAAE,OAAOnB,GAAG,CAAC;IAEb,IAAIpE,AAAW,oBAAXA,UAA8BA,AAAW,kBAAXA,QAA0B;QAC1D,MAAM0F,aAAaL,yBAAyBE;QAC5C,OAAO7B,KAAK,KAAK,CAAC+B,WAAWC;IAC/B;IACA,MAAMxE,MAAM,CAAC,+BAA+B,EAAEoE,OAAO;AACvD"}

package/dist/es/ai-model/ui-tars-planning.mjs

@@ -0,0 +1,201 @@
+import { UITarsModelVersion } from "@sqaitech/shared/env";
+import { resizeImgBase64 } from "@sqaitech/shared/img";
+import { getDebug } from "@sqaitech/shared/logger";
+import { transformHotkeyInput } from "@sqaitech/shared/us-keyboard-layout";
+import { assert } from "@sqaitech/shared/utils";
+import { actionParser } from "@ui-tars/action-parser";
+import { AIActionType } from "./common.mjs";
+import { getSummary, getUiTarsPlanningPrompt } from "./prompt/ui-tars-planning.mjs";
+import { callAIWithStringResponse } from "./service-caller/index.mjs";
+const debug = getDebug('ui-tars-planning');
+const bboxSize = 10;
+const pointToBbox = (point, width, height)=>[
+        Math.round(Math.max(point.x - bboxSize / 2, 0)),
+        Math.round(Math.max(point.y - bboxSize / 2, 0)),
+        Math.round(Math.min(point.x + bboxSize / 2, width)),
+        Math.round(Math.min(point.y + bboxSize / 2, height))
+    ];
+async function uiTarsPlanning(userInstruction, options) {
+    const { conversationHistory, context, modelConfig } = options;
+    const { uiTarsModelVersion } = modelConfig;
+    const systemPrompt = getUiTarsPlanningPrompt() + userInstruction;
+    const imagePayload = await resizeImageForUiTars(context.screenshotBase64, context.size, uiTarsModelVersion);
+    conversationHistory.append({
+        role: 'user',
+        content: [
+            {
+                type: 'image_url',
+                image_url: {
+                    url: imagePayload
+                }
+            }
+        ]
+    });
+    const res = await callAIWithStringResponse([
+        {
+            role: 'user',
+            content: systemPrompt
+        },
+        ...conversationHistory.snapshot()
+    ], AIActionType.INSPECT_ELEMENT, modelConfig);
+    const convertedText = convertBboxToCoordinates(res.content);
+    const { size } = context;
+    const { parsed } = actionParser({
+        prediction: convertedText,
+        factor: [
+            1000,
+            1000
+        ],
+        screenContext: {
+            width: size.width,
+            height: size.height
+        },
+        modelVer: uiTarsModelVersion
+    });
+    debug('ui-tars modelVer', uiTarsModelVersion, ', parsed', JSON.stringify(parsed));
+    const transformActions = [];
+    let shouldContinue = true;
+    parsed.forEach((action)=>{
+        const actionType = (action.action_type || '').toLowerCase();
+        if ('click' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            const point = getPoint(action.action_inputs.start_box, size);
+            transformActions.push({
+                type: 'Tap',
+                param: {
+                    locate: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: point[0],
+                            y: point[1]
+                        }, size.width, size.height)
+                    }
+                }
+            });
+        } else if ('drag' === actionType) {
+            assert(action.action_inputs.start_box, 'start_box is required');
+            assert(action.action_inputs.end_box, 'end_box is required');
+            const startPoint = getPoint(action.action_inputs.start_box, size);
+            const endPoint = getPoint(action.action_inputs.end_box, size);
+            transformActions.push({
+                type: 'DragAndDrop',
+                param: {
+                    from: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: startPoint[0],
+                            y: startPoint[1]
+                        }, size.width, size.height)
+                    },
+                    to: {
+                        prompt: action.thought || '',
+                        bbox: pointToBbox({
+                            x: endPoint[0],
+                            y: endPoint[1]
+                        }, size.width, size.height)
+                    }
+                },
+                thought: action.thought || ''
+            });
+        } else if ('type' === actionType) transformActions.push({
+            type: 'Input',
+            param: {
+                value: action.action_inputs.content
+            },
+            thought: action.thought || ''
+        });
+        else if ('scroll' === actionType) transformActions.push({
+            type: 'Scroll',
+            param: {
+                direction: action.action_inputs.direction
+            },
+            thought: action.thought || ''
+        });
+        else if ('finished' === actionType) {
+            shouldContinue = false;
+            transformActions.push({
+                type: 'Finished',
+                param: {},
+                thought: action.thought || ''
+            });
+        } else if ('hotkey' === actionType) if (action.action_inputs.key) {
+            const keys = transformHotkeyInput(action.action_inputs.key);
+            transformActions.push({
+                type: 'KeyboardPress',
+                param: {
+                    keyName: keys
+                },
+                thought: action.thought || ''
+            });
+        } else console.warn('No key found in action: hotkey. Will not perform action.');
+        else if ('wait' === actionType) transformActions.push({
+            type: 'Sleep',
+            param: {
+                timeMs: 1000
+            },
+            thought: action.thought || ''
+        });
+    });
+    if (0 === transformActions.length) throw new Error(`No actions found, response: ${res.content}`, {
+        cause: {
+            prediction: res.content,
+            parsed
+        }
+    });
+    debug('transformActions', JSON.stringify(transformActions, null, 2));
+    const log = getSummary(res.content);
+    conversationHistory.append({
+        role: 'assistant',
+        content: log
+    });
+    return {
+        actions: transformActions,
+        log,
+        usage: res.usage,
+        rawResponse: JSON.stringify(res.content, void 0, 2),
+        more_actions_needed_by_instruction: shouldContinue
+    };
+}
+function convertBboxToCoordinates(text) {
+    const pattern = /<bbox>(\d+)\s+(\d+)\s+(\d+)\s+(\d+)<\/bbox>/g;
+    function replaceMatch(match, x1, y1, x2, y2) {
+        const x1Num = Number.parseInt(x1, 10);
+        const y1Num = Number.parseInt(y1, 10);
+        const x2Num = Number.parseInt(x2, 10);
+        const y2Num = Number.parseInt(y2, 10);
+        const x = Math.floor((x1Num + x2Num) / 2);
+        const y = Math.floor((y1Num + y2Num) / 2);
+        return `(${x},${y})`;
+    }
+    const cleanedText = text.replace(/\[EOS\]/g, '');
+    return cleanedText.replace(pattern, replaceMatch).trim();
+}
+function getPoint(startBox, size) {
+    const [x, y] = JSON.parse(startBox);
+    return [
+        x * size.width,
+        y * size.height
+    ];
+}
+async function resizeImageForUiTars(imageBase64, size, uiTarsVersion) {
+    if (uiTarsVersion === UITarsModelVersion.V1_5) {
+        debug('ui-tars-v1.5, will check image size', size);
+        const currentPixels = size.width * size.height;
+        const maxPixels = 12845056;
+        if (currentPixels > maxPixels) {
+            const resizeFactor = Math.sqrt(maxPixels / currentPixels);
+            const newWidth = Math.floor(size.width * resizeFactor);
+            const newHeight = Math.floor(size.height * resizeFactor);
+            debug('resize image for ui-tars, new width: %s, new height: %s', newWidth, newHeight);
+            const resizedImage = await resizeImgBase64(imageBase64, {
+                width: newWidth,
+                height: newHeight
+            });
+            return resizedImage;
+        }
+    }
+    return imageBase64;
+}
+export { resizeImageForUiTars, uiTarsPlanning };
+
+//# sourceMappingURL=ui-tars-planning.mjs.map