@rpascene/shared 0.30.11 → 0.30.13

package/dist/types/env/constants.d.ts

@@ -31,6 +31,15 @@ interface IModelConfigKeys {
      * Extra
      */
     vlMode: string;
+    domain: string;
+    yht_access_token: string;
+    model: string;
+    baseURL: string;
+    chatType: string;
+    modelCategory: string;
+    stream: string;
+    temperature: string;
+    top_p: string;
 }
 export declare const VQA_MODEL_CONFIG_KEYS: IModelConfigKeys;
 export declare const GROUNDING_MODEL_CONFIG_KEYS: IModelConfigKeys;

package/dist/types/env/types.d.ts

@@ -123,14 +123,15 @@ export declare const STRING_ENV_KEYS: readonly ["OPENAI_MAX_TOKENS", "MIDSCENE_A
  * Can be access at any time
  */
 export declare const GLOBAL_ENV_KEYS: readonly ["MIDSCENE_CACHE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_FORCE_DEEP_THINK", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER"];
+export declare const YHT_STRING_ENV_KEYS: readonly ["domain", "yht_access_token", "model", "baseURL"];
+export declare const YHT_NUMBER_ENV_KEYS: readonly ["chatType", "modelCategory", "stream", "temperature", "top_p"];
 /**
  * Model related eve keys, used for declare which model to use.
  * Can be override by both agent.modelConfig and overrideAIConfig
  * Can only be access after agent.constructor
  */
-export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_API_KEY", "MIDSCENE_OPENAI_BASE_URL", "MIDSCENE_OPENAI_USE_AZURE", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_USE_AZURE_OPENAI", "MIDSCENE_AZURE_OPENAI_SCOPE", "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_USE_ANTHROPIC_SDK", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "ANTHROPIC_API_KEY", "MIDSCENE_AZURE_OPENAI_ENDPOINT", "MIDSCENE_AZURE_OPENAI_KEY", "MIDSCENE_AZURE_OPENAI_API_VERSION", "MIDSCENE_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VL_MODE", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_USE_AZURE", "ANTHROPIC_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_KEY", "AZURE_OPENAI_API_VERSION", "AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VQA_MODEL_NAME", "MIDSCENE_VQA_OPENAI_SOCKS_PROXY", "MIDSCENE_VQA_OPENAI_HTTP_PROXY", "MIDSCENE_VQA_OPENAI_BASE_URL", "MIDSCENE_VQA_OPENAI_API_KEY", "MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_VQA_OPENAI_USE_AZURE", "MIDSCENE_VQA_USE_AZURE_OPENAI", "MIDSCENE_VQA_AZURE_OPENAI_SCOPE", "MIDSCENE_VQA_AZURE_OPENAI_KEY", "MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT", "MIDSCENE_VQA_AZURE_OPENAI_API_VERSION", "MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_VQA_USE_ANTHROPIC_SDK", "MIDSCENE_VQA_ANTHROPIC_API_KEY", "MIDSCENE_VQA_VL_MODE", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY", "MIDSCENE_PLANNING_OPENAI_HTTP_PROXY", "MIDSCENE_PLANNING_OPENAI_BASE_URL", "MIDSCENE_PLANNING_OPENAI_API_KEY", "MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_OPENAI_USE_AZURE", "MIDSCENE_PLANNING_USE_AZURE_OPENAI", "MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE", "MIDSCENE_PLANNING_AZURE_OPENAI_KEY", "MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT", "MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION", "MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_USE_ANTHROPIC_SDK", "MIDSCENE_PLANNING_ANTHROPIC_API_KEY", "MIDSCENE_PLANNING_VL_MODE", "MIDSCENE_GROUNDING_MODEL_NAME", "MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY", "MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY", "MIDSCENE_GROUNDING_OPENAI_BASE_URL", "MIDSCENE_GROUNDING_OPENAI_API_KEY", "MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_GROUNDING_OPENAI_USE_AZURE", "MIDSCENE_GROUNDING_USE_AZURE_OPENAI", "MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE", "MIDSCENE_GROUNDING_AZURE_OPENAI_KEY", "MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT", "MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION", "MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK", "MIDSCENE_GROUNDING_ANTHROPIC_API_KEY", "MIDSCENE_GROUNDING_VL_MODE"];
-export declare const Yht_ENV_KEYS: string[];
-export declare const ALL_ENV_KEYS: readonly string[];
+export declare const MODEL_ENV_KEYS: readonly ["MIDSCENE_MODEL_NAME", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_API_KEY", "MIDSCENE_OPENAI_BASE_URL", "MIDSCENE_OPENAI_USE_AZURE", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_USE_AZURE_OPENAI", "MIDSCENE_AZURE_OPENAI_SCOPE", "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_USE_ANTHROPIC_SDK", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "ANTHROPIC_API_KEY", "MIDSCENE_AZURE_OPENAI_ENDPOINT", "MIDSCENE_AZURE_OPENAI_KEY", "MIDSCENE_AZURE_OPENAI_API_VERSION", "MIDSCENE_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VL_MODE", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_USE_AZURE", "ANTHROPIC_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_KEY", "AZURE_OPENAI_API_VERSION", "AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VQA_MODEL_NAME", "MIDSCENE_VQA_OPENAI_SOCKS_PROXY", "MIDSCENE_VQA_OPENAI_HTTP_PROXY", "MIDSCENE_VQA_OPENAI_BASE_URL", "MIDSCENE_VQA_OPENAI_API_KEY", "MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_VQA_OPENAI_USE_AZURE", "MIDSCENE_VQA_USE_AZURE_OPENAI", "MIDSCENE_VQA_AZURE_OPENAI_SCOPE", "MIDSCENE_VQA_AZURE_OPENAI_KEY", "MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT", "MIDSCENE_VQA_AZURE_OPENAI_API_VERSION", "MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_VQA_USE_ANTHROPIC_SDK", "MIDSCENE_VQA_ANTHROPIC_API_KEY", "MIDSCENE_VQA_VL_MODE", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY", "MIDSCENE_PLANNING_OPENAI_HTTP_PROXY", "MIDSCENE_PLANNING_OPENAI_BASE_URL", "MIDSCENE_PLANNING_OPENAI_API_KEY", "MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_OPENAI_USE_AZURE", "MIDSCENE_PLANNING_USE_AZURE_OPENAI", "MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE", "MIDSCENE_PLANNING_AZURE_OPENAI_KEY", "MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT", "MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION", "MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_USE_ANTHROPIC_SDK", "MIDSCENE_PLANNING_ANTHROPIC_API_KEY", "MIDSCENE_PLANNING_VL_MODE", "MIDSCENE_GROUNDING_MODEL_NAME", "MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY", "MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY", "MIDSCENE_GROUNDING_OPENAI_BASE_URL", "MIDSCENE_GROUNDING_OPENAI_API_KEY", "MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_GROUNDING_OPENAI_USE_AZURE", "MIDSCENE_GROUNDING_USE_AZURE_OPENAI", "MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE", "MIDSCENE_GROUNDING_AZURE_OPENAI_KEY", "MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT", "MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION", "MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK", "MIDSCENE_GROUNDING_ANTHROPIC_API_KEY", "MIDSCENE_GROUNDING_VL_MODE", "domain", "yht_access_token", "model", "baseURL", "chatType", "modelCategory", "stream", "temperature", "top_p"];
+export declare const ALL_ENV_KEYS: readonly [...string[], "MIDSCENE_DEBUG_MODE", "MIDSCENE_DEBUG_AI_PROFILE", "MIDSCENE_DEBUG_AI_RESPONSE", "MIDSCENE_RUN_DIR", "MIDSCENE_CACHE", "MIDSCENE_LANGSMITH_DEBUG", "MIDSCENE_FORCE_DEEP_THINK", "MIDSCENE_MCP_USE_PUPPETEER_MODE", "MIDSCENE_MCP_ANDROID_MODE", "MIDSCENE_CACHE_MAX_FILENAME_LENGTH", "MIDSCENE_REPLANNING_CYCLE_LIMIT", "OPENAI_MAX_TOKENS", "MIDSCENE_ADB_PATH", "MIDSCENE_ADB_REMOTE_HOST", "MIDSCENE_ADB_REMOTE_PORT", "MIDSCENE_ANDROID_IME_STRATEGY", "MIDSCENE_IOS_DEVICE_UDID", "MIDSCENE_IOS_SIMULATOR_UDID", "MIDSCENE_REPORT_TAG_NAME", "MIDSCENE_PREFERRED_LANGUAGE", "MATCH_BY_POSITION", "MIDSCENE_MCP_CHROME_PATH", "DOCKER_CONTAINER", "MIDSCENE_MODEL_NAME", "MIDSCENE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_OPENAI_API_KEY", "MIDSCENE_OPENAI_BASE_URL", "MIDSCENE_OPENAI_USE_AZURE", "MIDSCENE_OPENAI_SOCKS_PROXY", "MIDSCENE_OPENAI_HTTP_PROXY", "MIDSCENE_USE_AZURE_OPENAI", "MIDSCENE_AZURE_OPENAI_SCOPE", "MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_USE_ANTHROPIC_SDK", "MIDSCENE_USE_VLM_UI_TARS", "MIDSCENE_USE_QWEN_VL", "MIDSCENE_USE_QWEN3_VL", "MIDSCENE_USE_DOUBAO_VISION", "MIDSCENE_USE_GEMINI", "MIDSCENE_USE_VL_MODEL", "ANTHROPIC_API_KEY", "MIDSCENE_AZURE_OPENAI_ENDPOINT", "MIDSCENE_AZURE_OPENAI_KEY", "MIDSCENE_AZURE_OPENAI_API_VERSION", "MIDSCENE_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VL_MODE", "OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_USE_AZURE", "ANTHROPIC_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_KEY", "AZURE_OPENAI_API_VERSION", "AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VQA_MODEL_NAME", "MIDSCENE_VQA_OPENAI_SOCKS_PROXY", "MIDSCENE_VQA_OPENAI_HTTP_PROXY", "MIDSCENE_VQA_OPENAI_BASE_URL", "MIDSCENE_VQA_OPENAI_API_KEY", "MIDSCENE_VQA_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_VQA_OPENAI_USE_AZURE", "MIDSCENE_VQA_USE_AZURE_OPENAI", "MIDSCENE_VQA_AZURE_OPENAI_SCOPE", "MIDSCENE_VQA_AZURE_OPENAI_KEY", "MIDSCENE_VQA_AZURE_OPENAI_ENDPOINT", "MIDSCENE_VQA_AZURE_OPENAI_API_VERSION", "MIDSCENE_VQA_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_VQA_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_VQA_USE_ANTHROPIC_SDK", "MIDSCENE_VQA_ANTHROPIC_API_KEY", "MIDSCENE_VQA_VL_MODE", "MIDSCENE_PLANNING_MODEL_NAME", "MIDSCENE_PLANNING_OPENAI_SOCKS_PROXY", "MIDSCENE_PLANNING_OPENAI_HTTP_PROXY", "MIDSCENE_PLANNING_OPENAI_BASE_URL", "MIDSCENE_PLANNING_OPENAI_API_KEY", "MIDSCENE_PLANNING_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_OPENAI_USE_AZURE", "MIDSCENE_PLANNING_USE_AZURE_OPENAI", "MIDSCENE_PLANNING_AZURE_OPENAI_SCOPE", "MIDSCENE_PLANNING_AZURE_OPENAI_KEY", "MIDSCENE_PLANNING_AZURE_OPENAI_ENDPOINT", "MIDSCENE_PLANNING_AZURE_OPENAI_API_VERSION", "MIDSCENE_PLANNING_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_PLANNING_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_PLANNING_USE_ANTHROPIC_SDK", "MIDSCENE_PLANNING_ANTHROPIC_API_KEY", "MIDSCENE_PLANNING_VL_MODE", "MIDSCENE_GROUNDING_MODEL_NAME", "MIDSCENE_GROUNDING_OPENAI_SOCKS_PROXY", "MIDSCENE_GROUNDING_OPENAI_HTTP_PROXY", "MIDSCENE_GROUNDING_OPENAI_BASE_URL", "MIDSCENE_GROUNDING_OPENAI_API_KEY", "MIDSCENE_GROUNDING_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_GROUNDING_OPENAI_USE_AZURE", "MIDSCENE_GROUNDING_USE_AZURE_OPENAI", "MIDSCENE_GROUNDING_AZURE_OPENAI_SCOPE", "MIDSCENE_GROUNDING_AZURE_OPENAI_KEY", "MIDSCENE_GROUNDING_AZURE_OPENAI_ENDPOINT", "MIDSCENE_GROUNDING_AZURE_OPENAI_API_VERSION", "MIDSCENE_GROUNDING_AZURE_OPENAI_DEPLOYMENT", "MIDSCENE_GROUNDING_AZURE_OPENAI_INIT_CONFIG_JSON", "MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK", "MIDSCENE_GROUNDING_ANTHROPIC_API_KEY", "MIDSCENE_GROUNDING_VL_MODE", "domain", "yht_access_token", "model", "baseURL", "chatType", "modelCategory", "stream", "temperature", "top_p"];
 export type TEnvKeys = (typeof ALL_ENV_KEYS)[number];
 export type TGlobalConfig = Record<TEnvKeys, string | undefined>;
 export type TVlModeValues = 'qwen-vl' | 'qwen3-vl' | 'doubao-vision' | 'gemini' | 'vlm-ui-tars' | 'vlm-ui-tars-doubao' | 'vlm-ui-tars-doubao-1.5';

package/dist/types/env/utils.d.ts

@@ -4,4 +4,4 @@ import { type GLOBAL_ENV_KEYS, type MODEL_ENV_KEYS } from './types';
 export declare const globalModelConfigManager: ModelConfigManager;
 export declare const globalConfigManager: GlobalConfigManager;
 export declare const getPreferredLanguage: () => string;
-export declare const overrideAIConfig: (newConfig: Partial<Record<(typeof GLOBAL_ENV_KEYS)[number] | (typeof MODEL_ENV_KEYS)[number], string>>, extendMode?: boolean) => void;
+export declare const overrideAIConfig: (newConfig: Partial<Record<(typeof GLOBAL_ENV_KEYS)[number] | (typeof MODEL_ENV_KEYS)[number], string | number>>, extendMode?: boolean) => void;

package/dist/types/extractor/index.d.ts

@@ -27,7 +27,7 @@ export interface ElementNode {
 }
 export { descriptionOfTree, traverseTree, treeToList, truncateText, trimAttributes, } from './tree';
 export { extractTextWithPosition as webExtractTextWithPosition } from './web-extractor';
-export { extractTreeNode as webExtractNodeTree } from './web-extractor';
+export { extractTreeNode as webExtractNodeTree, getElementFromPoint } from './web-extractor';
 export { extractTreeNodeAsString as webExtractNodeTreeAsString } from './web-extractor';
 export { setNodeHashCacheListOnWindow, getNodeFromCacheList } from './util';
 export { getXpathsById, getXpathsByPoint, getNodeInfoByXpath, getElementInfoByXpath, getElementXpath, } from './locator';

package/dist/types/extractor/web-extractor.d.ts

@@ -16,4 +16,9 @@ export declare function mergeElementAndChildrenRects(node: Node, currentWindow:
     zoom: number;
     isVisible: boolean;
 } | null;
+export declare const getElementFromPoint: (args: {
+    x: number;
+    y: number;
+    containerPaths: string[];
+}) => Promise<any>;
 export {};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@rpascene/shared",
   "description": "RPA shared",
-  "version": "0.30.11",
+  "version": "0.30.13",
   "repository": "",
   "homepage": "",
   "types": "./dist/types/index.d.ts",

package/src/constants/index.ts

@@ -29,53 +29,104 @@ export const DEFAULT_WAIT_FOR_NETWORK_IDLE_CONCURRENCY = 2;
 
 export { PLAYWRIGHT_EXAMPLE_CODE, YAML_EXAMPLE_CODE } from './example-code';
 
+
+export const rpasceneCommand = {
+  Tap: "Tap",
+  RightClick: "RightClick",
+  DoubleClick: "DoubleClick",
+  Hover: "Hover",
+  Input: "Input",
+  KeyboardPress: "KeyboardPress",
+  Scroll: "Scroll",
+  DragAndDrop: "DragAndDrop",
+  ClearInput: "ClearInput",
+  Sleep: "Sleep",
+  LongPress: "LongPress",
+  Swipe: "Swipe",
+  Reload: "Reload",
+  GoBack: "GoBack",
+  Navigate: 'Navigate'
+}
+
+export const rpastudioCommand = {
+  SeleniumClickCommand: "SeleniumClickCommand",
+  SeleniumRightClickCommand: "SeleniumRightClickCommand",
+  SeleniumDoubleClickCommand: "SeleniumDoubleClickCommand",
+  SeleniumMouseHoverCommand: "SeleniumMouseHoverCommand",
+  SeleniumSetValueCommand: "SeleniumSetValueCommand",
+  SeleniumElementSendKeysCommand: "SeleniumElementSendKeysCommand",
+  SeleniumSendKeysCommand: "SeleniumSendKeysCommand",
+  SeleniumElementScrollCommand: 'SeleniumElementScrollCommand',
+  SeleniumWindowScrollCommand: 'SeleniumWindowScrollCommand',
+  SeleniumDragAndDropCommand: 'SeleniumDragAndDropCommand',
+  SeleniumClearValueCommand: "SeleniumClearValueCommand",
+  PauseCommand: "PauseCommand",
+  SeleniumBrowserRefreshCommand: "SeleniumBrowserRefreshCommand",
+  SeleniumBrowserNavigateBackCommand: 'SeleniumBrowserNavigateBackCommand',
+  SeleniumBrowserNavigateCommand: 'SeleniumBrowserNavigateCommand'
+}
+
+
+
 export const commandMap: any = {
-  Tap: {
-    CommandName: "SeleniumClickCommand",
+  [rpasceneCommand.Tap]: {
+    CommandName: rpastudioCommand.SeleniumClickCommand,
     DisplayName: '元素点击'
   },
-  RightClick: {
-    CommandName: "SeleniumRightClickCommand",
+  [rpasceneCommand.RightClick]: {
+    CommandName: rpastudioCommand.SeleniumRightClickCommand,
     DisplayName: "元素右键点击"
   },
-  DoubleClick: {
-    CommandName: "SeleniumDoubleClickCommand",
+  [rpasceneCommand.DoubleClick]: {
+    CommandName: rpastudioCommand.SeleniumDoubleClickCommand,
     DisplayName: "元素双击"
   },
-  Hover: {
-    CommandName: "SeleniumMouseHoverCommand",
+  [rpasceneCommand.Hover]: {
+    CommandName: rpastudioCommand.SeleniumMouseHoverCommand,
     DisplayName: "鼠标悬停"
   },
-  Input: {
-    CommandName: "SeleniumSetValueCommand",
+  [rpasceneCommand.Input]: {
+    CommandName: rpastudioCommand.SeleniumSetValueCommand,
     DisplayName: "设置文本"
   },
-  KeyboardPress: {
-    CommandName: "SeleniumSendKeysCommand",
+  [rpasceneCommand.KeyboardPress]: {
+    CommandName: rpastudioCommand.SeleniumElementSendKeysCommand,
     DisplayName: "模拟按键"
   },
-  Scroll: {
-    CommandName: "SeleniumElementScrollCommand",
+  [rpasceneCommand.Scroll]: {
+    CommandName: rpastudioCommand.SeleniumElementScrollCommand,
     DisplayName: "元素滚动"
   },
-  DragAndDrop: {
-    CommandName: "SeleniumDragDropCommand",
+  [rpasceneCommand.DragAndDrop]: {
+    CommandName: rpastudioCommand.SeleniumDragAndDropCommand,
     DisplayName: "元素拖拽（至指定元素）"
   },
-  ClearInput: {
-    CommandName: "SeleniumClearTextCommand",
+  [rpasceneCommand.ClearInput]: {
+    CommandName: rpastudioCommand.SeleniumClearValueCommand,
     DisplayName: "清空文本"
   },
-  Sleep: {
-    CommandName: "SeleniumPauseCommand",
+  [rpasceneCommand.Sleep]: {
+    CommandName: rpastudioCommand.PauseCommand,
     DisplayName: "暂停脚本"
   },
-  LongPress: {
-    CommandName: "SeleniumSendKeysCommand",
-    DisplayName: "模拟按键"
+  [rpasceneCommand.LongPress]: {
+    CommandName: "",
+    DisplayName: ""
+  },
+  [rpasceneCommand.Swipe]: {
+    CommandName: "",
+    DisplayName: ""
+  },
+  [rpasceneCommand.Reload]: {
+    CommandName: rpastudioCommand.SeleniumBrowserRefreshCommand,
+    DisplayName: "刷新"
   },
-  Swipe: {
-    CommandName: "SeleniumClickCommand",
-    DisplayName: "元素点击"
+  [rpasceneCommand.GoBack]: {
+    CommandName: rpastudioCommand.SeleniumBrowserNavigateBackCommand,
+    DisplayName: "后退"
   },
+  [rpasceneCommand.Navigate]: {
+    CommandName: rpastudioCommand.SeleniumBrowserNavigateCommand,
+    DisplayName: "打开页面"
+  }
 }

package/src/env/constants.ts

@@ -111,6 +111,20 @@ interface IModelConfigKeys {
    * Extra
    */
   vlMode: string;
+
+  domain: string,
+  yht_access_token: string,
+  model: string,
+  baseURL: string,
+
+  chatType: string,
+  modelCategory: string,
+  stream: string,
+  temperature: string,
+  top_p: string,
+
+
+
 }
 
 export const VQA_MODEL_CONFIG_KEYS: IModelConfigKeys = {
@@ -146,6 +160,18 @@ export const VQA_MODEL_CONFIG_KEYS: IModelConfigKeys = {
    * Extra
    */
   vlMode: MIDSCENE_VQA_VL_MODE,
+
+  // 友互通
+  domain: 'domain',
+  yht_access_token: 'yht_access_token',
+  model: 'model',
+  baseURL: 'baseURL',
+  chatType: 'chatType',
+  modelCategory: 'modelCategory',
+  stream: 'stream',
+  temperature: 'temperature',
+  top_p: 'top_p',
+
 } as const;
 
 export const GROUNDING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
@@ -181,6 +207,17 @@ export const GROUNDING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
    * Extra
    */
   vlMode: MIDSCENE_GROUNDING_VL_MODE,
+
+  // 友互通
+  domain: 'domain',
+  yht_access_token: 'yht_access_token',
+  model: 'model',
+  baseURL: 'baseURL',
+  chatType: 'chatType',
+  modelCategory: 'modelCategory',
+  stream: 'stream',
+  temperature: 'temperature',
+  top_p: 'top_p',
 } as const;
 
 export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
@@ -216,6 +253,17 @@ export const PLANNING_MODEL_CONFIG_KEYS: IModelConfigKeys = {
    * Extra
    */
   vlMode: MIDSCENE_PLANNING_VL_MODE,
+
+  // 友互通
+  domain: 'domain',
+  yht_access_token: 'yht_access_token',
+  model: 'model',
+  baseURL: 'baseURL',
+  chatType: 'chatType',
+  modelCategory: 'modelCategory',
+  stream: 'stream',
+  temperature: 'temperature',
+  top_p: 'top_p',
 } as const;
 
 // modelConfig return default
@@ -252,6 +300,17 @@ export const DEFAULT_MODEL_CONFIG_KEYS: IModelConfigKeys = {
    * Extra
    */
   vlMode: MIDSCENE_VL_MODE,
+
+  // 友互通
+  domain: 'domain',
+  yht_access_token: 'yht_access_token',
+  model: 'model',
+  baseURL: 'baseURL',
+  chatType: 'chatType',
+  modelCategory: 'modelCategory',
+  stream: 'stream',
+  temperature: 'temperature',
+  top_p: 'top_p',
 } as const;
 
 // read from process.env
@@ -288,4 +347,16 @@ export const DEFAULT_MODEL_CONFIG_KEYS_LEGACY: IModelConfigKeys = {
    * Extra
    */
   vlMode: 'DEFAULT_MODEL_CONFIG_KEYS has no vlMode key',
-} as const;
+
+  // 友互通
+  domain: 'domain',
+  yht_access_token: 'yht_access_token',
+  model: 'model',
+  baseURL: 'baseURL',
+  chatType: 'chatType',
+  modelCategory: 'modelCategory',
+  stream: 'stream',
+  temperature: 'temperature',
+  top_p: 'top_p',
+
+} as const;

package/src/env/decide-model-config.ts

@@ -30,10 +30,10 @@ type TModelConfigKeys =
   | typeof DEFAULT_MODEL_CONFIG_KEYS_LEGACY;
 
 const KEYS_MAP: Record<TIntent, TModelConfigKeys> = {
-  VQA: VQA_MODEL_CONFIG_KEYS,
-  grounding: GROUNDING_MODEL_CONFIG_KEYS,
-  planning: PLANNING_MODEL_CONFIG_KEYS,
-  default: DEFAULT_MODEL_CONFIG_KEYS,
+  VQA: VQA_MODEL_CONFIG_KEYS, // 验证、断言、提取数据
+  grounding: GROUNDING_MODEL_CONFIG_KEYS, // 元素定位
+  planning: PLANNING_MODEL_CONFIG_KEYS, // 规划
+  default: DEFAULT_MODEL_CONFIG_KEYS,  // 默认
 } as const;
 
 /**
@@ -248,6 +248,7 @@ export const decideModelConfigFromEnv = (
   const keysForEnv =
     intent === 'default' ? DEFAULT_MODEL_CONFIG_KEYS_LEGACY : KEYS_MAP[intent];
 
+  // 按照modelName判断是否配置了对应的模型策略
   if (intent !== 'default' && allEnvConfig[keysForEnv.modelName]) {
     const modelName = allEnvConfig[keysForEnv.modelName]!;
 
@@ -283,7 +284,7 @@ export const decideModelConfigFromEnv = (
   }
 
   debugLog(`decideModelConfig as legacy logic with intent ${intent}.`);
-
+  // 走默认环境变量策略
   // TODO: when fallback to legacy logic, prefer to read MIDSCENE_OPENAI_API_KEY rather than OPENAI_API_KEY
   const result = decideOpenaiSdkConfig({
     keys: DEFAULT_MODEL_CONFIG_KEYS_LEGACY,

package/src/env/global-config-manager.ts

@@ -39,6 +39,7 @@ export class GlobalConfigManager {
    * recalculate allEnvConfig every time because process.env can be updated any time
    */
   public getAllEnvConfig() {
+    // .env中的配置
     const envConfig = ALL_ENV_KEYS.reduce(
       (p, name) => {
         p[name] = process.env[name];
@@ -50,8 +51,10 @@ export class GlobalConfigManager {
     if (this.override) {
       const { newConfig, extendMode } = this.override;
       if (extendMode) {
+        // 扩展模式
         return { ...envConfig, ...newConfig };
       } else {
+        // 覆盖模式
         return { ...newConfig };
       }
     } else {
@@ -134,22 +137,23 @@ export class GlobalConfigManager {
     >,
     extendMode = false, // true: merge with global config, false: override global config
   ) {
-    // for (const key in newConfig) {
-    //   if (![...GLOBAL_ENV_KEYS, ...MODEL_ENV_KEYS].includes(key as never)) {
-    //     throw new Error(`Failed to override AI config, invalid key: ${key}`);
-    //   }
-    //   const value = newConfig[key as keyof typeof newConfig];
-    //   if (typeof value !== 'string') {
-    //     throw new Error(
-    //       `Failed to override AI config, value for key ${key} must be a string, but got with type ${typeof value}`,
-    //     );
-    //   }
-    //   if (this.keysHaveBeenRead[key]) {
-    //     console.warn(
-    //       `Warning: try to override AI config with key ${key} ,but it has been read.`,
-    //     );
-    //   }
-    // }
+    for (const key in newConfig) {
+      if (![...GLOBAL_ENV_KEYS, ...MODEL_ENV_KEYS].includes(key as never)) {
+        throw new Error(`Failed to override AI config, invalid key: ${key}`);
+      }
+      const value = newConfig[key as keyof typeof newConfig];
+      if (typeof value !== 'string') {
+        throw new Error(
+          `Failed to override AI config, value for key ${key} must be a string, but got with type ${typeof value}`,
+        );
+      }
+      // 应该在使用前配置
+      if (this.keysHaveBeenRead[key]) {
+        console.warn(
+          `Warning: try to override AI config with key ${key} ,but it has been read.`,
+        );
+      }
+    }
     const savedNewConfig = extendMode
       ? {
         ...this.override?.newConfig,

package/src/env/types.ts

@@ -228,6 +228,22 @@ export const GLOBAL_ENV_KEYS = [
   ...STRING_ENV_KEYS,
 ] as const;
 
+export const YHT_STRING_ENV_KEYS = [
+  'domain',
+  'yht_access_token',
+  'model',
+  'baseURL'
+] as const;
+
+export const YHT_NUMBER_ENV_KEYS = [
+  'chatType',
+  'modelCategory',
+  'stream',
+  'temperature',
+  'top_p',
+] as const;
+
+
 /**
  * Model related eve keys, used for declare which model to use.
  * Can be override by both agent.modelConfig and overrideAIConfig
@@ -321,26 +337,15 @@ export const MODEL_ENV_KEYS = [
   MIDSCENE_GROUNDING_USE_ANTHROPIC_SDK,
   MIDSCENE_GROUNDING_ANTHROPIC_API_KEY,
   MIDSCENE_GROUNDING_VL_MODE,
+  ...YHT_STRING_ENV_KEYS,
+  ...YHT_NUMBER_ENV_KEYS
 ] as const;
 
-export const Yht_ENV_KEYS = [
-  'domain',
-  'yht_access_token',
-  'chatType',
-  'model',
-  'modelCategory',
-  'stream',
-  'temperature',
-  'top_p',
-  'baseURL'
-]
-
 export const ALL_ENV_KEYS = [
   ...UNUSED_ENV_KEYS,
   ...BASIC_ENV_KEYS,
   ...GLOBAL_ENV_KEYS,
-  ...MODEL_ENV_KEYS,
-  ...Yht_ENV_KEYS
+  ...MODEL_ENV_KEYS
 ] as const;
 
 export type TEnvKeys = (typeof ALL_ENV_KEYS)[number];

package/src/env/utils.ts

@@ -30,10 +30,19 @@ export const overrideAIConfig = (
   newConfig: Partial<
     Record<
       (typeof GLOBAL_ENV_KEYS)[number] | (typeof MODEL_ENV_KEYS)[number],
-      string
+      string | number
     >
   >,
   extendMode = false, // true: merge with global config, false: override global config
 ) => {
-  globalConfigManager.overrideAIConfig(newConfig, extendMode);
+
+  // 全部转string
+  const config = Object.entries(newConfig).reduce((pre, [key, value]) => {
+    return {
+      ...pre,
+      [key]: String(value)
+    }
+  }, {})
+
+  globalConfigManager.overrideAIConfig(config, extendMode);
 };

package/src/extractor/index.ts

@@ -33,7 +33,7 @@ export {
 
 export { extractTextWithPosition as webExtractTextWithPosition } from './web-extractor';
 
-export { extractTreeNode as webExtractNodeTree } from './web-extractor';
+export { extractTreeNode as webExtractNodeTree, getElementFromPoint } from './web-extractor';
 
 export { extractTreeNodeAsString as webExtractNodeTreeAsString } from './web-extractor';
 

package/src/extractor/web-extractor.ts

@@ -554,6 +554,72 @@ if (typeof window !== 'undefined') {
           error: 'error message'
         }
       }
+    } else if (action === 'getElementFromPoint') {
+      // @ts-ignore
+      try {
+        return await getElementFromPoint({
+          x: data.x,
+          y: data.y,
+          containerPaths: data.containerPaths
+        })
+      } catch (error) {
+        return {
+          error: 'error message'
+        }
+      }
     }
   })
 }
+
+const getElementOffset = (element: any) => {
+  const pLeft = parseInt(window.getComputedStyle(element, null).getPropertyValue('padding-left'), 10) || 0;
+  const pTop = parseInt(window.getComputedStyle(element, null).getPropertyValue('padding-top'), 10) || 0;
+  const bLeft = parseInt(window.getComputedStyle(element, null).getPropertyValue('border-left-width'), 10) || 0;
+  const bTop = parseInt(window.getComputedStyle(element, null).getPropertyValue('border-top-width'), 10) || 0;
+  return {
+    x: pLeft + bLeft,
+    y: pTop + bTop,
+  };
+};
+
+const getCssScale = (element: any) => {
+  try {
+    const { width, height } = element.getBoundingClientRect();
+    return {
+      cssScaleX: Number((width / element.offsetWidth).toFixed(1)),
+      cssScaleY: Number((height / element.offsetHeight).toFixed(1)),
+    };
+  } catch (error) {
+    return {
+      cssScaleX: 1,
+      cssScaleY: 1,
+    };
+  }
+}
+
+export const getElementFromPoint = async (args: { x: number, y: number, containerPaths: string[] }) => {
+  if (!args.containerPaths) {
+    args.containerPaths = []
+  }
+  const dom: any = document.elementFromPoint(args.x, args.y);
+  if (dom?.nodeName === 'IFRAME') {
+    const containerPath: any = getContainerPath(dom)
+    const bounding = dom.getBoundingClientRect();
+    const offset = getElementOffset(dom);
+    const { cssScaleX, cssScaleY } = getCssScale(dom);
+    return await postWindowMessage(dom.contentWindow, window, {
+      action: 'getElementFromPoint',
+      data: {
+        x: (args.x - bounding.x - offset.x) / cssScaleX,
+        y: (args.y - bounding.y - offset.y) / cssScaleY,
+        containerPaths: args.containerPaths.concat(containerPath)
+      }
+    })
+  } else {
+    const elementInfo = {
+      allPaths: getLocators(dom),
+      containerPaths: args.containerPaths
+    }
+    return elementInfo
+  }
+}