@rpascene/shared 0.30.11 → 0.30.13

package/dist/es/constants/index.mjs

@@ -21,54 +21,100 @@ const DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT = 5000;
 const DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT = 2000;
 const DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME = 300;
 const DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY = 2;
+const rpasceneCommand = {
+    Tap: "Tap",
+    RightClick: "RightClick",
+    DoubleClick: "DoubleClick",
+    Hover: "Hover",
+    Input: "Input",
+    KeyboardPress: "KeyboardPress",
+    Scroll: "Scroll",
+    DragAndDrop: "DragAndDrop",
+    ClearInput: "ClearInput",
+    Sleep: "Sleep",
+    LongPress: "LongPress",
+    Swipe: "Swipe",
+    Reload: "Reload",
+    GoBack: "GoBack",
+    Navigate: 'Navigate'
+};
+const rpastudioCommand = {
+    SeleniumClickCommand: "SeleniumClickCommand",
+    SeleniumRightClickCommand: "SeleniumRightClickCommand",
+    SeleniumDoubleClickCommand: "SeleniumDoubleClickCommand",
+    SeleniumMouseHoverCommand: "SeleniumMouseHoverCommand",
+    SeleniumSetValueCommand: "SeleniumSetValueCommand",
+    SeleniumElementSendKeysCommand: "SeleniumElementSendKeysCommand",
+    SeleniumSendKeysCommand: "SeleniumSendKeysCommand",
+    SeleniumElementScrollCommand: 'SeleniumElementScrollCommand',
+    SeleniumWindowScrollCommand: 'SeleniumWindowScrollCommand',
+    SeleniumDragAndDropCommand: 'SeleniumDragAndDropCommand',
+    SeleniumClearValueCommand: "SeleniumClearValueCommand",
+    PauseCommand: "PauseCommand",
+    SeleniumBrowserRefreshCommand: "SeleniumBrowserRefreshCommand",
+    SeleniumBrowserNavigateBackCommand: 'SeleniumBrowserNavigateBackCommand',
+    SeleniumBrowserNavigateCommand: 'SeleniumBrowserNavigateCommand'
+};
 const commandMap = {
-    Tap: {
-        CommandName: "SeleniumClickCommand",
+    [rpasceneCommand.Tap]: {
+        CommandName: rpastudioCommand.SeleniumClickCommand,
         DisplayName: "\u5143\u7D20\u70B9\u51FB"
     },
-    RightClick: {
-        CommandName: "SeleniumRightClickCommand",
+    [rpasceneCommand.RightClick]: {
+        CommandName: rpastudioCommand.SeleniumRightClickCommand,
         DisplayName: "\u5143\u7D20\u53F3\u952E\u70B9\u51FB"
     },
-    DoubleClick: {
-        CommandName: "SeleniumDoubleClickCommand",
+    [rpasceneCommand.DoubleClick]: {
+        CommandName: rpastudioCommand.SeleniumDoubleClickCommand,
         DisplayName: "\u5143\u7D20\u53CC\u51FB"
     },
-    Hover: {
-        CommandName: "SeleniumMouseHoverCommand",
+    [rpasceneCommand.Hover]: {
+        CommandName: rpastudioCommand.SeleniumMouseHoverCommand,
         DisplayName: "\u9F20\u6807\u60AC\u505C"
     },
-    Input: {
-        CommandName: "SeleniumSetValueCommand",
+    [rpasceneCommand.Input]: {
+        CommandName: rpastudioCommand.SeleniumSetValueCommand,
         DisplayName: "\u8BBE\u7F6E\u6587\u672C"
     },
-    KeyboardPress: {
-        CommandName: "SeleniumSendKeysCommand",
+    [rpasceneCommand.KeyboardPress]: {
+        CommandName: rpastudioCommand.SeleniumElementSendKeysCommand,
         DisplayName: "\u6A21\u62DF\u6309\u952E"
     },
-    Scroll: {
-        CommandName: "SeleniumElementScrollCommand",
+    [rpasceneCommand.Scroll]: {
+        CommandName: rpastudioCommand.SeleniumElementScrollCommand,
         DisplayName: "\u5143\u7D20\u6EDA\u52A8"
     },
-    DragAndDrop: {
-        CommandName: "SeleniumDragDropCommand",
+    [rpasceneCommand.DragAndDrop]: {
+        CommandName: rpastudioCommand.SeleniumDragAndDropCommand,
         DisplayName: "\u5143\u7D20\u62D6\u62FD\uFF08\u81F3\u6307\u5B9A\u5143\u7D20\uFF09"
     },
-    ClearInput: {
-        CommandName: "SeleniumClearTextCommand",
+    [rpasceneCommand.ClearInput]: {
+        CommandName: rpastudioCommand.SeleniumClearValueCommand,
         DisplayName: "\u6E05\u7A7A\u6587\u672C"
     },
-    Sleep: {
-        CommandName: "SeleniumPauseCommand",
+    [rpasceneCommand.Sleep]: {
+        CommandName: rpastudioCommand.PauseCommand,
         DisplayName: "\u6682\u505C\u811A\u672C"
     },
-    LongPress: {
-        CommandName: "SeleniumSendKeysCommand",
-        DisplayName: "\u6A21\u62DF\u6309\u952E"
+    [rpasceneCommand.LongPress]: {
+        CommandName: "",
+        DisplayName: ""
     },
-    Swipe: {
-        CommandName: "SeleniumClickCommand",
-        DisplayName: "\u5143\u7D20\u70B9\u51FB"
+    [rpasceneCommand.Swipe]: {
+        CommandName: "",
+        DisplayName: ""
+    },
+    [rpasceneCommand.Reload]: {
+        CommandName: rpastudioCommand.SeleniumBrowserRefreshCommand,
+        DisplayName: "\u5237\u65B0"
+    },
+    [rpasceneCommand.GoBack]: {
+        CommandName: rpastudioCommand.SeleniumBrowserNavigateBackCommand,
+        DisplayName: "\u540E\u9000"
+    },
+    [rpasceneCommand.Navigate]: {
+        CommandName: rpastudioCommand.SeleniumBrowserNavigateCommand,
+        DisplayName: "\u6253\u5F00\u9875\u9762"
     }
 };
-export { CONTAINER_MINI_HEIGHT, CONTAINER_MINI_WIDTH, DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT, DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY, DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME, DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT, DEFAULT_WDA_PORT, constants_NodeType as NodeType, PLAYGROUND_SERVER_PORT, PLAYWRIGHT_EXAMPLE_CODE, SCRCPY_SERVER_PORT, TEXT_MAX_SIZE, TEXT_SIZE_THRESHOLD, WEBDRIVER_ELEMENT_ID_KEY, YAML_EXAMPLE_CODE, commandMap };
+export { CONTAINER_MINI_HEIGHT, CONTAINER_MINI_WIDTH, DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT, DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY, DEFAULT_WAIT_FOR_NETWORK_IDLE_TIME, DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT, DEFAULT_WDA_PORT, constants_NodeType as NodeType, PLAYGROUND_SERVER_PORT, PLAYWRIGHT_EXAMPLE_CODE, SCRCPY_SERVER_PORT, TEXT_MAX_SIZE, TEXT_SIZE_THRESHOLD, WEBDRIVER_ELEMENT_ID_KEY, YAML_EXAMPLE_CODE, commandMap, rpasceneCommand, rpastudioCommand };

package/dist/es/env/constants.mjs

@@ -16,7 +16,16 @@ const VQA_MODEL_CONFIG_KEYS = {
     azureExtraConfig: MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON,
     useAnthropicSdk: MIDSCENE_VQA_USE_ANTHROPIC_SDK,
     anthropicApiKey: MIDSCENE_VQA_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_VQA_VL_MODE
+    vlMode: MIDSCENE_VQA_VL_MODE,
+    domain: 'domain',
+    yht_access_token: 'yht_access_token',
+    model: 'model',
+    baseURL: 'baseURL',
+    chatType: 'chatType',
+    modelCategory: 'modelCategory',
+    stream: 'stream',
+    temperature: 'temperature',
+    top_p: 'top_p'
 };
 const GROUNDING_MODEL_CONFIG_KEYS = {
     modelName: MIDSCENE_GROUNDING_MODEL_NAME,
@@ -35,7 +44,16 @@ const GROUNDING_MODEL_CONFIG_KEYS = {
     azureExtraConfig: MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON,
     useAnthropicSdk: MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK,
     anthropicApiKey: MIDSCENE_GROUNDING_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_GROUNDING_VL_MODE
+    vlMode: MIDSCENE_GROUNDING_VL_MODE,
+    domain: 'domain',
+    yht_access_token: 'yht_access_token',
+    model: 'model',
+    baseURL: 'baseURL',
+    chatType: 'chatType',
+    modelCategory: 'modelCategory',
+    stream: 'stream',
+    temperature: 'temperature',
+    top_p: 'top_p'
 };
 const PLANNING_MODEL_CONFIG_KEYS = {
     modelName: MIDSCENE_PLANNING_MODEL_NAME,
@@ -54,7 +72,16 @@ const PLANNING_MODEL_CONFIG_KEYS = {
     azureExtraConfig: MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON,
     useAnthropicSdk: MIDSCENE_PLANNING_USE_ANTHROPIC_SDK,
     anthropicApiKey: MIDSCENE_PLANNING_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_PLANNING_VL_MODE
+    vlMode: MIDSCENE_PLANNING_VL_MODE,
+    domain: 'domain',
+    yht_access_token: 'yht_access_token',
+    model: 'model',
+    baseURL: 'baseURL',
+    chatType: 'chatType',
+    modelCategory: 'modelCategory',
+    stream: 'stream',
+    temperature: 'temperature',
+    top_p: 'top_p'
 };
 const DEFAULT_MODEL_CONFIG_KEYS = {
     modelName: MIDSCENE_MODEL_NAME,
@@ -73,7 +100,16 @@ const DEFAULT_MODEL_CONFIG_KEYS = {
     azureExtraConfig: MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
     useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
     anthropicApiKey: MIDSCENE_ANTHROPIC_API_KEY,
-    vlMode: MIDSCENE_VL_MODE
+    vlMode: MIDSCENE_VL_MODE,
+    domain: 'domain',
+    yht_access_token: 'yht_access_token',
+    model: 'model',
+    baseURL: 'baseURL',
+    chatType: 'chatType',
+    modelCategory: 'modelCategory',
+    stream: 'stream',
+    temperature: 'temperature',
+    top_p: 'top_p'
 };
 const DEFAULT_MODEL_CONFIG_KEYS_LEGACY = {
     modelName: MIDSCENE_MODEL_NAME,
@@ -92,6 +128,15 @@ const DEFAULT_MODEL_CONFIG_KEYS_LEGACY = {
     azureExtraConfig: MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
     useAnthropicSdk: MIDSCENE_USE_ANTHROPIC_SDK,
     anthropicApiKey: ANTHROPIC_API_KEY,
-    vlMode: 'DEFAULT_MODEL_CONFIG_KEYS has no vlMode key'
+    vlMode: 'DEFAULT_MODEL_CONFIG_KEYS has no vlMode key',
+    domain: 'domain',
+    yht_access_token: 'yht_access_token',
+    model: 'model',
+    baseURL: 'baseURL',
+    chatType: 'chatType',
+    modelCategory: 'modelCategory',
+    stream: 'stream',
+    temperature: 'temperature',
+    top_p: 'top_p'
 };
 export { DEFAULT_MODEL_CONFIG_KEYS, DEFAULT_MODEL_CONFIG_KEYS_LEGACY, GROUNDING_MODEL_CONFIG_KEYS, PLANNING_MODEL_CONFIG_KEYS, VQA_MODEL_CONFIG_KEYS };

package/dist/es/env/global-config-manager.mjs

@@ -1,5 +1,5 @@
 import { initDebugConfig } from "./init-debug.mjs";
-import { ALL_ENV_KEYS, BOOLEAN_ENV_KEYS, MATCH_BY_POSITION, NUMBER_ENV_KEYS, STRING_ENV_KEYS } from "./types.mjs";
+import { ALL_ENV_KEYS, BOOLEAN_ENV_KEYS, GLOBAL_ENV_KEYS, MATCH_BY_POSITION, MODEL_ENV_KEYS, NUMBER_ENV_KEYS, STRING_ENV_KEYS } from "./types.mjs";
 function _define_property(obj, key, value) {
     if (key in obj) Object.defineProperty(obj, key, {
         value: value,
@@ -59,6 +59,15 @@ class GlobalConfigManager {
     }
     overrideAIConfig(newConfig, extendMode = false) {
         var _this_override;
+        for(const key in newConfig){
+            if (![
+                ...GLOBAL_ENV_KEYS,
+                ...MODEL_ENV_KEYS
+            ].includes(key)) throw new Error(`Failed to override AI config, invalid key: ${key}`);
+            const value = newConfig[key];
+            if ('string' != typeof value) throw new Error(`Failed to override AI config, value for key ${key} must be a string, but got with type ${typeof value}`);
+            if (this.keysHaveBeenRead[key]) console.warn(`Warning: try to override AI config with key ${key} ,but it has been read.`);
+        }
         const savedNewConfig = extendMode ? {
             ...null == (_this_override = this.override) ? void 0 : _this_override.newConfig,
             ...newConfig

package/dist/es/env/types.mjs

@@ -144,6 +144,19 @@ const GLOBAL_ENV_KEYS = [
     ...NUMBER_ENV_KEYS,
     ...STRING_ENV_KEYS
 ];
+const YHT_STRING_ENV_KEYS = [
+    'domain',
+    'yht_access_token',
+    'model',
+    'baseURL'
+];
+const YHT_NUMBER_ENV_KEYS = [
+    'chatType',
+    'modelCategory',
+    'stream',
+    'temperature',
+    'top_p'
+];
 const MODEL_ENV_KEYS = [
     MIDSCENE_MODEL_NAME,
     MIDSCENE_OPENAI_INIT_CONFIG_JSON,
@@ -226,25 +239,15 @@ const MODEL_ENV_KEYS = [
     MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON,
     MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK,
     MIDSCENE_GROUNDING_ANTHROPIC_API_KEY,
-    MIDSCENE_GROUNDING_VL_MODE
-];
-const Yht_ENV_KEYS = [
-    'domain',
-    'yht_access_token',
-    'chatType',
-    'model',
-    'modelCategory',
-    'stream',
-    'temperature',
-    'top_p',
-    'baseURL'
+    MIDSCENE_GROUNDING_VL_MODE,
+    ...YHT_STRING_ENV_KEYS,
+    ...YHT_NUMBER_ENV_KEYS
 ];
 const ALL_ENV_KEYS = [
     ...UNUSED_ENV_KEYS,
     ...BASIC_ENV_KEYS,
     ...GLOBAL_ENV_KEYS,
-    ...MODEL_ENV_KEYS,
-    ...Yht_ENV_KEYS
+    ...MODEL_ENV_KEYS
 ];
 var types_UITarsModelVersion = /*#__PURE__*/ function(UITarsModelVersion) {
     UITarsModelVersion["V1_0"] = "1.0";
@@ -262,4 +265,4 @@ const VL_MODE_RAW_VALID_VALUES = [
     'vlm-ui-tars-doubao',
     'vlm-ui-tars-doubao-1.5'
 ];
-export { ALL_ENV_KEYS, ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, BASIC_ENV_KEYS, BOOLEAN_ENV_KEYS, DOCKER_CONTAINER, GLOBAL_ENV_KEYS, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, MIDSCENE_ANTHROPIC_API_KEY, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_API_VERSION, MIDSCENE_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_AZURE_OPENAI_ENDPOINT, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_KEY, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_CACHE_MAX_FILENAME_LENGTH, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_GROUNDING_ANTHROPIC_API_KEY, MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION, MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT, MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_GROUNDING_AZURE_OPENAI_KEY, MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE, MIDSCENE_GROUNDING_MODEL_NAME, MIDSCENE_GROUNDING_OPENAI_API_KEY, MIDSCENE_GROUNDING_OPENAI_BASE_URL, MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY, MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON, MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY, MIDSCENE_GROUNDING_OPENAI_USE_AZURE, MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK, MIDSCENE_GROUNDING_USE_AZURE_OPENAI, MIDSCENE_GROUNDING_VL_MODE, MIDSCENE_IOS_DEVICE_UDID, MIDSCENE_IOS_SIMULATOR_UDID, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_ANDROID_MODE, MIDSCENE_MCP_CHROME_PATH, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_API_KEY, MIDSCENE_OPENAI_BASE_URL, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_OPENAI_USE_AZURE, MIDSCENE_PLANNING_ANTHROPIC_API_KEY, MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION, MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT, MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_PLANNING_AZURE_OPENAI_KEY, MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE, MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_OPENAI_API_KEY, MIDSCENE_PLANNING_OPENAI_BASE_URL, MIDSCENE_PLANNING_OPENAI_HTTP_PROXY, MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON, MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY, MIDSCENE_PLANNING_OPENAI_USE_AZURE, MIDSCENE_PLANNING_USE_ANTHROPIC_SDK, MIDSCENE_PLANNING_USE_AZURE_OPENAI, MIDSCENE_PLANNING_VL_MODE, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPLANNING_CYCLE_LIMIT, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN3_VL, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, MIDSCENE_VL_MODE, MIDSCENE_VQA_ANTHROPIC_API_KEY, MIDSCENE_VQA_AZURE_OPENAI_API_VERSION, MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT, MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_VQA_AZURE_OPENAI_KEY, MIDSCENE_VQA_AZURE_OPENAI_SCOPE, MIDSCENE_VQA_MODEL_NAME, MIDSCENE_VQA_OPENAI_API_KEY, MIDSCENE_VQA_OPENAI_BASE_URL, MIDSCENE_VQA_OPENAI_HTTP_PROXY, MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON, MIDSCENE_VQA_OPENAI_SOCKS_PROXY, MIDSCENE_VQA_OPENAI_USE_AZURE, MIDSCENE_VQA_USE_ANTHROPIC_SDK, MIDSCENE_VQA_USE_AZURE_OPENAI, MIDSCENE_VQA_VL_MODE, MODEL_ENV_KEYS, NUMBER_ENV_KEYS, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, STRING_ENV_KEYS, types_UITarsModelVersion as UITarsModelVersion, UNUSED_ENV_KEYS, VL_MODE_RAW_VALID_VALUES, Yht_ENV_KEYS };
+export { ALL_ENV_KEYS, ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, BASIC_ENV_KEYS, BOOLEAN_ENV_KEYS, DOCKER_CONTAINER, GLOBAL_ENV_KEYS, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_ADB_REMOTE_HOST, MIDSCENE_ADB_REMOTE_PORT, MIDSCENE_ANDROID_IME_STRATEGY, MIDSCENE_ANTHROPIC_API_KEY, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_API_VERSION, MIDSCENE_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_AZURE_OPENAI_ENDPOINT, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_KEY, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_CACHE_MAX_FILENAME_LENGTH, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_GROUNDING_ANTHROPIC_API_KEY, MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION, MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT, MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_GROUNDING_AZURE_OPENAI_KEY, MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE, MIDSCENE_GROUNDING_MODEL_NAME, MIDSCENE_GROUNDING_OPENAI_API_KEY, MIDSCENE_GROUNDING_OPENAI_BASE_URL, MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY, MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON, MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY, MIDSCENE_GROUNDING_OPENAI_USE_AZURE, MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK, MIDSCENE_GROUNDING_USE_AZURE_OPENAI, MIDSCENE_GROUNDING_VL_MODE, MIDSCENE_IOS_DEVICE_UDID, MIDSCENE_IOS_SIMULATOR_UDID, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_ANDROID_MODE, MIDSCENE_MCP_CHROME_PATH, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_API_KEY, MIDSCENE_OPENAI_BASE_URL, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_OPENAI_USE_AZURE, MIDSCENE_PLANNING_ANTHROPIC_API_KEY, MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION, MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT, MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_PLANNING_AZURE_OPENAI_KEY, MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE, MIDSCENE_PLANNING_MODEL_NAME, MIDSCENE_PLANNING_OPENAI_API_KEY, MIDSCENE_PLANNING_OPENAI_BASE_URL, MIDSCENE_PLANNING_OPENAI_HTTP_PROXY, MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON, MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY, MIDSCENE_PLANNING_OPENAI_USE_AZURE, MIDSCENE_PLANNING_USE_ANTHROPIC_SDK, MIDSCENE_PLANNING_USE_AZURE_OPENAI, MIDSCENE_PLANNING_VL_MODE, MIDSCENE_PREFERRED_LANGUAGE, MIDSCENE_REPLANNING_CYCLE_LIMIT, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN3_VL, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, MIDSCENE_VL_MODE, MIDSCENE_VQA_ANTHROPIC_API_KEY, MIDSCENE_VQA_AZURE_OPENAI_API_VERSION, MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT, MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT, MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_VQA_AZURE_OPENAI_KEY, MIDSCENE_VQA_AZURE_OPENAI_SCOPE, MIDSCENE_VQA_MODEL_NAME, MIDSCENE_VQA_OPENAI_API_KEY, MIDSCENE_VQA_OPENAI_BASE_URL, MIDSCENE_VQA_OPENAI_HTTP_PROXY, MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON, MIDSCENE_VQA_OPENAI_SOCKS_PROXY, MIDSCENE_VQA_OPENAI_USE_AZURE, MIDSCENE_VQA_USE_ANTHROPIC_SDK, MIDSCENE_VQA_USE_AZURE_OPENAI, MIDSCENE_VQA_VL_MODE, MODEL_ENV_KEYS, NUMBER_ENV_KEYS, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, STRING_ENV_KEYS, types_UITarsModelVersion as UITarsModelVersion, UNUSED_ENV_KEYS, VL_MODE_RAW_VALID_VALUES, YHT_NUMBER_ENV_KEYS, YHT_STRING_ENV_KEYS };

package/dist/es/env/utils.mjs

@@ -13,6 +13,10 @@ const getPreferredLanguage = ()=>{
     return isChina ? 'Chinese' : 'English';
 };
 const overrideAIConfig = (newConfig, extendMode = false)=>{
-    globalConfigManager.overrideAIConfig(newConfig, extendMode);
+    const config = Object.entries(newConfig).reduce((pre, [key, value])=>({
+            ...pre,
+            [key]: String(value)
+        }), {});
+    globalConfigManager.overrideAIConfig(config, extendMode);
 };
 export { getPreferredLanguage, globalConfigManager, globalModelConfigManager, overrideAIConfig };

package/dist/es/extractor/index.mjs

@@ -1,7 +1,7 @@
 import { descriptionOfTree, traverseTree, treeToList, trimAttributes, truncateText } from "./tree.mjs";
-import { extractTextWithPosition, extractTreeNode, extractTreeNodeAsString } from "./web-extractor.mjs";
+import { extractTextWithPosition, extractTreeNode, extractTreeNodeAsString, getElementFromPoint } from "./web-extractor.mjs";
 import { getNodeFromCacheList, setNodeHashCacheListOnWindow } from "./util.mjs";
 import { getElementInfoByXpath, getElementXpath, getNodeInfoByXpath, getXpathsById, getXpathsByPoint } from "./locator.mjs";
 import { generateElementByPosition, isNotContainerElement } from "./dom-util.mjs";
 export * from "./customLocator.mjs";
-export { descriptionOfTree, generateElementByPosition, getElementInfoByXpath, getElementXpath, getNodeFromCacheList, getNodeInfoByXpath, getXpathsById, getXpathsByPoint, isNotContainerElement, setNodeHashCacheListOnWindow, traverseTree, treeToList, trimAttributes, truncateText, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition as webExtractTextWithPosition };
+export { descriptionOfTree, generateElementByPosition, getElementFromPoint, getElementInfoByXpath, getElementXpath, getNodeFromCacheList, getNodeInfoByXpath, getXpathsById, getXpathsByPoint, isNotContainerElement, setNodeHashCacheListOnWindow, traverseTree, treeToList, trimAttributes, truncateText, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition as webExtractTextWithPosition };

package/dist/es/extractor/web-extractor.mjs

@@ -357,5 +357,65 @@ if ('undefined' != typeof window) onWindowMessage(window, async ({ action, data
             error: 'error message'
         };
     }
+    if ('getElementFromPoint' === action) try {
+        return await getElementFromPoint({
+            x: data.x,
+            y: data.y,
+            containerPaths: data.containerPaths
+        });
+    } catch (error) {
+        return {
+            error: 'error message'
+        };
+    }
 });
-export { collectElementInfo, extractTextWithPosition, extractTreeNode, extractTreeNodeAsString, mergeElementAndChildrenRects };
+const getElementOffset = (element)=>{
+    const pLeft = parseInt(window.getComputedStyle(element, null).getPropertyValue('padding-left'), 10) || 0;
+    const pTop = parseInt(window.getComputedStyle(element, null).getPropertyValue('padding-top'), 10) || 0;
+    const bLeft = parseInt(window.getComputedStyle(element, null).getPropertyValue('border-left-width'), 10) || 0;
+    const bTop = parseInt(window.getComputedStyle(element, null).getPropertyValue('border-top-width'), 10) || 0;
+    return {
+        x: pLeft + bLeft,
+        y: pTop + bTop
+    };
+};
+const web_extractor_getCssScale = (element)=>{
+    try {
+        const { width, height } = element.getBoundingClientRect();
+        return {
+            cssScaleX: Number((width / element.offsetWidth).toFixed(1)),
+            cssScaleY: Number((height / element.offsetHeight).toFixed(1))
+        };
+    } catch (error) {
+        return {
+            cssScaleX: 1,
+            cssScaleY: 1
+        };
+    }
+};
+const getElementFromPoint = async (args)=>{
+    if (!args.containerPaths) args.containerPaths = [];
+    const dom = document.elementFromPoint(args.x, args.y);
+    if ((null == dom ? void 0 : dom.nodeName) === 'IFRAME') {
+        const containerPath = getContainerPath(dom);
+        const bounding = dom.getBoundingClientRect();
+        const offset = getElementOffset(dom);
+        const { cssScaleX, cssScaleY } = web_extractor_getCssScale(dom);
+        return await postWindowMessage(dom.contentWindow, window, {
+            action: 'getElementFromPoint',
+            data: {
+                x: (args.x - bounding.x - offset.x) / cssScaleX,
+                y: (args.y - bounding.y - offset.y) / cssScaleY,
+                containerPaths: args.containerPaths.concat(containerPath)
+            }
+        });
+    }
+    {
+        const elementInfo = {
+            allPaths: getLocators(dom),
+            containerPaths: args.containerPaths
+        };
+        return elementInfo;
+    }
+};
+export { collectElementInfo, extractTextWithPosition, extractTreeNode, extractTreeNodeAsString, getElementFromPoint, mergeElementAndChildrenRects };