@midscene/shared 0.16.9-beta-20250508125518.0 → 0.16.9

package/dist/lib/env.d.ts

@@ -8,6 +8,7 @@ declare const MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
 declare const MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
 declare const MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
 declare const MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
+declare const MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
 declare const OPENAI_API_KEY = "OPENAI_API_KEY";
 declare const OPENAI_BASE_URL = "OPENAI_BASE_URL";
 declare const OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -50,6 +51,7 @@ declare const allConfigFromEnv: () => {
     MATCH_BY_POSITION: string | undefined;
     MIDSCENE_REPORT_TAG_NAME: string | undefined;
     MIDSCENE_OPENAI_SOCKS_PROXY: string | undefined;
+    MIDSCENE_OPENAI_HTTP_PROXY: string | undefined;
     MIDSCENE_USE_AZURE_OPENAI: string | undefined;
     MIDSCENE_AZURE_OPENAI_SCOPE: string | undefined;
     MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON: string | undefined;
@@ -80,4 +82,4 @@ declare const getAIConfigInBoolean: (configKey: keyof ReturnType<typeof allConfi
 declare const getAIConfigInJson: (configKey: keyof ReturnType<typeof allConfigFromEnv>) => any;
 declare const overrideAIConfig: (newConfig: Partial<ReturnType<typeof allConfigFromEnv>>, extendMode?: boolean) => void;
 
-export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };
+export { ANTHROPIC_API_KEY, AZURE_OPENAI_API_VERSION, AZURE_OPENAI_DEPLOYMENT, AZURE_OPENAI_ENDPOINT, AZURE_OPENAI_KEY, MATCH_BY_POSITION, MIDSCENE_ADB_PATH, MIDSCENE_API_TYPE, MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_AZURE_OPENAI_SCOPE, MIDSCENE_CACHE, MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG, MIDSCENE_DEBUG_AI_PROFILE, MIDSCENE_DEBUG_AI_RESPONSE, MIDSCENE_DEBUG_MODE, MIDSCENE_FORCE_DEEP_THINK, MIDSCENE_LANGSMITH_DEBUG, MIDSCENE_MCP_USE_PUPPETEER_MODE, MIDSCENE_MODEL_NAME, MIDSCENE_OPENAI_HTTP_PROXY, MIDSCENE_OPENAI_INIT_CONFIG_JSON, MIDSCENE_OPENAI_SOCKS_PROXY, MIDSCENE_REPORT_TAG_NAME, MIDSCENE_RUN_DIR, MIDSCENE_USE_ANTHROPIC_SDK, MIDSCENE_USE_AZURE_OPENAI, MIDSCENE_USE_DOUBAO_VISION, MIDSCENE_USE_GEMINI, MIDSCENE_USE_QWEN_VL, MIDSCENE_USE_VLM_UI_TARS, MIDSCENE_USE_VL_MODEL, OPENAI_API_KEY, OPENAI_BASE_URL, OPENAI_MAX_TOKENS, OPENAI_USE_AZURE, UITarsModelVersion, allConfigFromEnv, getAIConfig, getAIConfigInBoolean, getAIConfigInJson, overrideAIConfig, uiTarsModelVersion, vlLocateMode };

package/dist/lib/env.js

@@ -39,6 +39,7 @@ __export(env_exports, {
   MIDSCENE_LANGSMITH_DEBUG: () => MIDSCENE_LANGSMITH_DEBUG,
   MIDSCENE_MCP_USE_PUPPETEER_MODE: () => MIDSCENE_MCP_USE_PUPPETEER_MODE,
   MIDSCENE_MODEL_NAME: () => MIDSCENE_MODEL_NAME,
+  MIDSCENE_OPENAI_HTTP_PROXY: () => MIDSCENE_OPENAI_HTTP_PROXY,
   MIDSCENE_OPENAI_INIT_CONFIG_JSON: () => MIDSCENE_OPENAI_INIT_CONFIG_JSON,
   MIDSCENE_OPENAI_SOCKS_PROXY: () => MIDSCENE_OPENAI_SOCKS_PROXY,
   MIDSCENE_REPORT_TAG_NAME: () => MIDSCENE_REPORT_TAG_NAME,
@@ -74,6 +75,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
 var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
 var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
 var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
+var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
 var OPENAI_API_KEY = "OPENAI_API_KEY";
 var OPENAI_BASE_URL = "OPENAI_BASE_URL";
 var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -117,6 +119,7 @@ var allConfigFromEnv = () => {
     [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
     [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
     [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
+    [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
     [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
     [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
     [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,
@@ -257,6 +260,7 @@ var overrideAIConfig = (newConfig, extendMode = false) => {
   MIDSCENE_LANGSMITH_DEBUG,
   MIDSCENE_MCP_USE_PUPPETEER_MODE,
   MIDSCENE_MODEL_NAME,
+  MIDSCENE_OPENAI_HTTP_PROXY,
   MIDSCENE_OPENAI_INIT_CONFIG_JSON,
   MIDSCENE_OPENAI_SOCKS_PROXY,
   MIDSCENE_REPORT_TAG_NAME,

package/dist/lib/extractor-debug.js

@@ -364,24 +364,8 @@ function getNodeAttributes(node, currentWindow) {
 }
 function midsceneGenerateHash(node, content, rect) {
   const slicedHash = generateHashId(rect, content);
-  if (node) {
-    if (!window.midsceneNodeHashCacheList) {
-      setNodeHashCacheListOnWindow();
-    }
-    setNodeToCacheList(node, slicedHash);
-  }
   return slicedHash;
 }
-function setNodeHashCacheListOnWindow() {
-  if (typeof window !== "undefined") {
-    window.midsceneNodeHashCacheList = [];
-  }
-}
-function setNodeToCacheList(node, id) {
-  if (typeof window !== "undefined") {
-    window.midsceneNodeHashCacheList.push({ node, id });
-  }
-}
 function setMidsceneVisibleRectOnWindow() {
   if (typeof window !== "undefined") {
     window.midsceneVisibleRect = visibleRect;
@@ -452,7 +436,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -479,7 +465,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -507,7 +495,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -538,9 +528,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
+      // attributes,
       content: text,
       rect,
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -565,7 +558,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }

package/dist/lib/extractor.d.ts

@@ -1,2 +1,47 @@
-import './constants.js';
-export { E as ElementInfo, a as ElementNode, d as descriptionOfTree, h as getElementInfoByNode, g as getXpathsById, s as setNodeHashCacheListOnWindow, t as traverseTree, b as treeToList, c as webExtractNodeTree, f as webExtractNodeTreeAsString, e as webExtractTextWithPosition } from './index-df4e781b.js';
+import { NodeType } from './constants.js';
+import { B as BaseElement, E as ElementTreeNode } from './index-305e7a7e.js';
+
+declare function descriptionOfTree<ElementType extends BaseElement = BaseElement>(tree: ElementTreeNode<ElementType>, truncateTextLength?: number, filterNonTextContent?: boolean): string;
+declare function treeToList<T extends BaseElement>(tree: ElementTreeNode<T>): T[];
+declare function traverseTree<T extends BaseElement, ReturnNodeType extends BaseElement>(tree: ElementTreeNode<T>, onNode: (node: T) => ReturnNodeType): ElementTreeNode<ReturnNodeType>;
+
+interface WebElementInfo extends ElementInfo {
+    zoom: number;
+    screenWidth?: number;
+    screenHeight?: number;
+}
+interface WebElementNode {
+    node: WebElementInfo | null;
+    children: WebElementNode[];
+}
+declare function extractTextWithPosition$1(initNode: globalThis.Node, debugMode?: boolean): WebElementInfo[];
+declare function extractTreeNodeAsString(initNode: globalThis.Node, debugMode?: boolean): string;
+declare function extractTreeNode(initNode: globalThis.Node, debugMode?: boolean): WebElementNode;
+
+declare function extractTextWithPosition(initNode: globalThis.Document): ElementInfo[];
+
+interface ElementInfo {
+    id: string;
+    indexId: number;
+    nodeHashId: string;
+    locator: string;
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    nodeType: NodeType;
+    content: string;
+    rect: {
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+    };
+    center: [number, number];
+}
+interface ElementNode {
+    node: ElementInfo | null;
+    children: ElementNode[];
+}
+
+export { type ElementInfo, type ElementNode, extractTextWithPosition as clientExtractTextWithPosition, descriptionOfTree, traverseTree, treeToList, extractTreeNode as webExtractNodeTree, extractTreeNodeAsString as webExtractNodeTreeAsString, extractTextWithPosition$1 as webExtractTextWithPosition };

package/dist/lib/extractor.js

@@ -20,10 +20,8 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/extractor/index.ts
 var extractor_exports = {};
 __export(extractor_exports, {
+  clientExtractTextWithPosition: () => extractTextWithPosition2,
   descriptionOfTree: () => descriptionOfTree,
-  getElementInfoByNode: () => getElementInfoByNode,
-  getXpathsById: () => getXpathsById,
-  setNodeHashCacheListOnWindow: () => setNodeHashCacheListOnWindow,
   traverseTree: () => traverseTree,
   treeToList: () => treeToList,
   webExtractNodeTree: () => extractTreeNode,
@@ -517,32 +515,8 @@ function getNodeAttributes(node, currentWindow) {
 }
 function midsceneGenerateHash(node, content, rect) {
   const slicedHash = generateHashId(rect, content);
-  if (node) {
-    if (!window.midsceneNodeHashCacheList) {
-      setNodeHashCacheListOnWindow();
-    }
-    setNodeToCacheList(node, slicedHash);
-  }
   return slicedHash;
 }
-function setNodeHashCacheListOnWindow() {
-  if (typeof window !== "undefined") {
-    window.midsceneNodeHashCacheList = [];
-  }
-}
-function setNodeToCacheList(node, id) {
-  if (typeof window !== "undefined") {
-    window.midsceneNodeHashCacheList.push({ node, id });
-  }
-}
-function getNodeFromCacheList(id) {
-  if (typeof window !== "undefined") {
-    return window.midsceneNodeHashCacheList.find(
-      (item) => item.id === id
-    )?.node;
-  }
-  return null;
-}
 function getTopDocument() {
   const container = document.body || document;
   return container;
@@ -603,7 +577,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -630,7 +606,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -658,7 +636,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -689,9 +669,12 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
+      // attributes,
       content: text,
       rect,
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -716,7 +699,9 @@ function collectElementInfo(node, currentWindow, currentDocument, baseZoom = 1,
         Math.round(rect.left + rect.width / 2),
         Math.round(rect.top + rect.height / 2)
       ],
-      zoom: rect.zoom
+      zoom: rect.zoom,
+      screenWidth: currentWindow.innerWidth,
+      screenHeight: currentWindow.innerHeight
     };
     return elementInfo;
   }
@@ -825,90 +810,167 @@ function extractTreeNode(initNode, debugMode2 = false) {
   };
 }
 
-// src/extractor/locator.ts
-var getElementIndex = (element) => {
-  let index = 1;
-  let prev = element.previousElementSibling;
-  while (prev) {
-    if (prev.nodeName.toLowerCase() === element.nodeName.toLowerCase()) {
-      index++;
-    }
-    prev = prev.previousElementSibling;
+// src/extractor/client-extractor.ts
+function getNodeAttributes2(node) {
+  const attrs = {};
+  if (node && node.nodeType === 1) {
+    const element = node;
+    for (let i = 0; i < element.attributes.length; i++) {
+      const attr = element.attributes[i];
+      attrs[attr.nodeName] = attr.nodeValue ?? "";
+    }
+  }
+  return attrs;
+}
+function getRect2(attributes) {
+  const x = Math.round(Number.parseFloat(attributes.x ?? "0"));
+  const y = Math.round(Number.parseFloat(attributes.y ?? "0"));
+  const width = Math.round(Number.parseFloat(attributes.width ?? "0"));
+  const height = Math.round(Number.parseFloat(attributes.height ?? "0"));
+  return {
+    left: Math.max(0, Math.floor(x)),
+    top: Math.max(0, Math.floor(y)),
+    width: Math.max(0, width),
+    height: Math.max(0, height)
+  };
+}
+function validTextNodeContent(node) {
+  if (node.nodeType === 3) {
+    return node.nodeValue?.trim() || "";
   }
-  return index;
-};
-var findFirstAncestorWithId = (element) => {
-  let current = element;
-  while (current?.parentElement) {
-    if (current.id) {
-      return current;
-    }
-    current = current.parentElement;
+  return "";
+}
+function getXPathForElement(element) {
+  if (element.nodeType !== 1) {
+    return "";
   }
-  return null;
-};
-var getElementXPath = (element) => {
-  if (element.nodeType !== Node.ELEMENT_NODE)
+  const getIndex = (sib, name) => {
+    let count = 1;
+    for (let cur = sib.previousSibling; cur; cur = cur.previousSibling) {
+      if (cur.nodeType === 1 && cur.nodeName === name) {
+        count++;
+      }
+    }
+    return count;
+  };
+  const buildAttributePart = (elem) => {
+    const attributes = ["id", "resource-id", "content-desc", "class"];
+    for (const attr of attributes) {
+      if (elem.hasAttribute(attr)) {
+        const value = elem.getAttribute(attr);
+        if (value && value.trim() !== "") {
+          return `[@${attr}="${value}"]`;
+        }
+      }
+    }
     return "";
-  const el = element;
-  if (el === document.documentElement) {
-    return "/html";
-  }
-  if (el === document.body) {
-    return "/html/body";
-  }
-  if (el.id) {
-    return `//*[@id="${el.id}"]`;
-  }
-  const ancestorWithId = findFirstAncestorWithId(el);
-  if (ancestorWithId) {
-    const ancestorPath = `//*[@id="${ancestorWithId.id}"]`;
-    let current = el;
-    const pathParts = [];
-    while (current && current !== ancestorWithId) {
-      const index2 = getElementIndex(current);
-      const tagName2 = current.nodeName.toLowerCase();
-      pathParts.unshift(`${tagName2}[${index2}]`);
-      current = current.parentElement;
-    }
-    return pathParts.length > 0 ? `${ancestorPath}/${pathParts.join("/")}` : ancestorPath;
-  }
-  if (!el.parentNode) {
-    return `/${el.nodeName.toLowerCase()}`;
-  }
-  const index = getElementIndex(el);
-  const tagName = el.nodeName.toLowerCase();
-  if (el.parentNode) {
-    const parentXPath = getElementXPath(el.parentNode);
-    return `${parentXPath}/${tagName}[${index}]`;
-  }
-  return `/${tagName}[${index}]`;
-};
-function generateXPaths(node) {
-  if (!node)
-    return [];
-  const xPath = getElementXPath(node);
-  return [xPath];
-}
-function getXpathsById(id) {
-  let node = getNodeFromCacheList(id);
-  if (!node) {
-    return null;
+  };
+  const getPath = (node, path = "") => {
+    if (node.parentNode) {
+      path = getPath(node.parentNode, path);
+    }
+    if (node.nodeType === 1) {
+      const elem = node;
+      const tagName = elem.nodeName.toLowerCase();
+      let part = `/${tagName}`;
+      const attributePart = buildAttributePart(elem);
+      if (attributePart) {
+        part += attributePart;
+      } else {
+        const index = getIndex(node, node.nodeName);
+        if (index > 1) {
+          part += `[${index}]`;
+        }
+      }
+      path += part;
+    }
+    return path;
+  };
+  return getPath(element);
+}
+function extractTextWithPosition2(initNode) {
+  const elementInfoArray = [];
+  let nodeIndex = 1;
+  function dfs(node, parentNode = null) {
+    if (!node) {
+      return;
+    }
+    const currentNodeDes = { node, children: [] };
+    if (parentNode) {
+      parentNode.children.push(currentNodeDes);
+    }
+    collectElementInfo2(node);
+    if (node.childNodes && node.childNodes.length > 0) {
+      for (let i = 0; i < node.childNodes.length; i++) {
+        dfs(node.childNodes[i], currentNodeDes);
+      }
+    }
   }
-  if (isTextElement(node)) {
-    node = node.parentElement;
+  function collectElementInfo2(node) {
+    const attributes = getNodeAttributes2(node);
+    const rect = getRect2(attributes);
+    const nodeHashId = midsceneGenerateHash(null, attributes.placeholder, rect);
+    const text = validTextNodeContent(node);
+    let nodeType;
+    switch (node.nodeName.toUpperCase()) {
+      case "TEXT":
+        nodeType = "TEXT Node" /* TEXT */;
+        break;
+      case "IMAGE":
+        nodeType = "IMG Node" /* IMG */;
+        break;
+      case "BUTTON":
+        nodeType = "BUTTON Node" /* BUTTON */;
+        break;
+      case "SEARCHINPUT":
+      case "TEXTINPUT":
+      case "INPUT":
+        nodeType = "FORM_ITEM Node" /* FORM_ITEM */;
+        break;
+      case "NAV":
+      case "LIST":
+      case "CELL":
+        nodeType = "CONTAINER Node" /* CONTAINER */;
+        break;
+      default:
+        if (attributes.id === "android:id/input" || attributes.id === "android:id/inputArea") {
+          nodeType = "FORM_ITEM Node" /* FORM_ITEM */;
+        } else {
+          nodeType = "CONTAINER Node" /* CONTAINER */;
+        }
+        break;
+    }
+    const xpath = getXPathForElement(node);
+    const elementInfo = {
+      id: nodeHashId,
+      indexId: nodeIndex++,
+      nodeHashId,
+      locator: xpath,
+      attributes: {
+        nodeType,
+        ...attributes
+      },
+      content: text,
+      rect,
+      center: [
+        Math.round(rect.left + rect.width / 2),
+        Math.round(rect.top + rect.height / 2)
+      ],
+      nodeType
+    };
+    if (elementInfo.nodeType !== "CONTAINER Node" /* CONTAINER */) {
+      elementInfoArray.push(elementInfo);
+    }
   }
-  return generateXPaths(node);
-}
-function getElementInfoByNode(node) {
-  return collectElementInfo(node, window, document);
+  const rootNode = initNode;
+  const rootDescriptor = { node: rootNode, children: [] };
+  dfs(rootNode, rootDescriptor);
+  return elementInfoArray;
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
+  clientExtractTextWithPosition,
   descriptionOfTree,
-  getElementInfoByNode,
-  getXpathsById,
-  setNodeHashCacheListOnWindow,
   traverseTree,
   treeToList,
   webExtractNodeTree,

package/dist/lib/fs.d.ts

@@ -10,7 +10,6 @@ declare function getRunningPkgInfo(dir?: string): PkgInfo | null;
  * @returns {string|null} - The most recent package.json file path or null
  */
 declare function findNearestPackageJson(dir: string): string | null;
-declare function getElementInfosScriptContent(): string;
 declare function getExtraReturnLogic(tree?: boolean): Promise<string | null>;
 
-export { findNearestPackageJson, getElementInfosScriptContent, getExtraReturnLogic, getRunningPkgInfo };
+export { findNearestPackageJson, getExtraReturnLogic, getRunningPkgInfo };

package/dist/lib/fs.js

@@ -31,7 +31,6 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var fs_exports = {};
 __export(fs_exports, {
   findNearestPackageJson: () => findNearestPackageJson,
-  getElementInfosScriptContent: () => getElementInfosScriptContent,
   getExtraReturnLogic: () => getExtraReturnLogic,
   getRunningPkgInfo: () => getRunningPkgInfo
 });
@@ -87,19 +86,15 @@ function findNearestPackageJson(dir) {
   }
   return findNearestPackageJson(parentDir);
 }
-function getElementInfosScriptContent() {
+async function getExtraReturnLogic(tree = false) {
+  if (ifInBrowser) {
+    return null;
+  }
   const currentFilePath = __filename;
   const pathDir = findNearestPackageJson((0, import_node_path.dirname)(currentFilePath));
   assert(pathDir, `can't find pathDir, with ${import_node_path.dirname}`);
   const scriptPath = import_node_path2.default.join(pathDir, "./dist/script/htmlElement.js");
   const elementInfosScriptContent = (0, import_node_fs.readFileSync)(scriptPath, "utf-8");
-  return elementInfosScriptContent;
-}
-async function getExtraReturnLogic(tree = false) {
-  if (ifInBrowser) {
-    return null;
-  }
-  const elementInfosScriptContent = `${getElementInfosScriptContent()}midscene_element_inspector.setNodeHashCacheListOnWindow();`;
   if (tree) {
     return `${elementInfosScriptContent}midscene_element_inspector.webExtractNodeTree()`;
   }
@@ -108,7 +103,6 @@ async function getExtraReturnLogic(tree = false) {
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   findNearestPackageJson,
-  getElementInfosScriptContent,
   getExtraReturnLogic,
   getRunningPkgInfo
 });

package/dist/lib/img.d.ts

@@ -1,6 +1,6 @@
 import { Buffer } from 'node:buffer';
 import Jimp from 'jimp';
-import { R as Rect, B as BaseElement } from './index-df4e781b.js';
+import { R as Rect, B as BaseElement } from './index-305e7a7e.js';
 import './constants.js';
 
 interface Size {

package/dist/lib/index-305e7a7e.d.ts

@@ -0,0 +1,32 @@
+import { NodeType } from './constants.js';
+
+interface Point {
+    left: number;
+    top: number;
+}
+interface Size {
+    width: number;
+    height: number;
+    dpr?: number;
+}
+type Rect = Point & Size & {
+    zoom?: number;
+};
+declare abstract class BaseElement {
+    abstract id: string;
+    abstract indexId?: number;
+    abstract attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    abstract content: string;
+    abstract rect: Rect;
+    abstract center: [number, number];
+    abstract locator?: string;
+}
+interface ElementTreeNode<ElementType extends BaseElement = BaseElement> {
+    node: ElementType | null;
+    children: ElementTreeNode<ElementType>[];
+}
+
+export { BaseElement as B, type ElementTreeNode as E, type Rect as R };

package/dist/lib/logger.js

@@ -55,6 +55,7 @@ var MIDSCENE_DEBUG_MODE = "MIDSCENE_DEBUG_MODE";
 var MIDSCENE_MCP_USE_PUPPETEER_MODE = "MIDSCENE_MCP_USE_PUPPETEER_MODE";
 var MIDSCENE_FORCE_DEEP_THINK = "MIDSCENE_FORCE_DEEP_THINK";
 var MIDSCENE_OPENAI_SOCKS_PROXY = "MIDSCENE_OPENAI_SOCKS_PROXY";
+var MIDSCENE_OPENAI_HTTP_PROXY = "MIDSCENE_OPENAI_HTTP_PROXY";
 var OPENAI_API_KEY = "OPENAI_API_KEY";
 var OPENAI_BASE_URL = "OPENAI_BASE_URL";
 var OPENAI_MAX_TOKENS = "OPENAI_MAX_TOKENS";
@@ -97,6 +98,7 @@ var allConfigFromEnv = () => {
     [MATCH_BY_POSITION]: process.env[MATCH_BY_POSITION] || void 0,
     [MIDSCENE_REPORT_TAG_NAME]: process.env[MIDSCENE_REPORT_TAG_NAME] || void 0,
     [MIDSCENE_OPENAI_SOCKS_PROXY]: process.env[MIDSCENE_OPENAI_SOCKS_PROXY] || void 0,
+    [MIDSCENE_OPENAI_HTTP_PROXY]: process.env[MIDSCENE_OPENAI_HTTP_PROXY] || void 0,
     [MIDSCENE_USE_AZURE_OPENAI]: process.env[MIDSCENE_USE_AZURE_OPENAI] || void 0,
     [MIDSCENE_AZURE_OPENAI_SCOPE]: process.env[MIDSCENE_AZURE_OPENAI_SCOPE] || void 0,
     [MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON]: process.env[MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON] || void 0,