@midscene/web 0.5.1 → 0.5.2-beta-20240929094445.0

package/dist/script/htmlElementDebug.js

@@ -583,17 +583,60 @@ var midscene_element_inspector = (() => {
 
   // src/extractor/dom-util.ts
   function isFormElement(node) {
-    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "label" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
+    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
   }
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
   }
   function isImgElement(node) {
+    if (!includeBaseElement(node) && node instanceof Element) {
+      const computedStyle = window.getComputedStyle(node);
+      const backgroundImage = computedStyle.getPropertyValue("background-image");
+      if (backgroundImage !== "none") {
+        return true;
+      }
+    }
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "img" || node instanceof SVGElement && node.tagName.toLowerCase() === "svg";
   }
   function isTextElement(node) {
     return node.nodeName.toLowerCase() === "#text";
   }
+  function isContainerElement(node) {
+    if (!(node instanceof HTMLElement))
+      return false;
+    if (includeBaseElement(node)) {
+      return false;
+    }
+    const computedStyle = window.getComputedStyle(node);
+    const backgroundColor = computedStyle.getPropertyValue("background-color");
+    if (backgroundColor) {
+      return true;
+    }
+    return false;
+  }
+  function includeBaseElement(node) {
+    if (!(node instanceof HTMLElement))
+      return false;
+    if (node.innerText) {
+      return true;
+    }
+    const includeList = [
+      "svg",
+      "button",
+      "input",
+      "textarea",
+      "select",
+      "option",
+      "img"
+    ];
+    for (const tagName of includeList) {
+      const element = node.querySelectorAll(tagName);
+      if (element.length > 0) {
+        return true;
+      }
+    }
+    return false;
+  }
 
   // src/extractor/util.ts
   var import_js_sha256 = __toESM(require_sha256());
@@ -621,9 +664,9 @@ var midscene_element_inspector = (() => {
   function selectorForValue(val) {
     return `[${taskIdKey}='${val}']`;
   }
-  function setDataForNode(node, nodeHash) {
+  function setDataForNode(node, nodeHash, setToParentNode = false) {
     const taskId = taskIdKey;
-    if (!(node instanceof HTMLElement)) {
+    if (!(node instanceof Element)) {
       return "";
     }
     if (!taskId) {
@@ -632,7 +675,13 @@ var midscene_element_inspector = (() => {
     }
     const selector = selectorForValue(nodeHash);
     if (getDebugMode()) {
-      node.setAttribute(taskIdKey, nodeHash.toString());
+      if (setToParentNode) {
+        if (node.parentNode instanceof HTMLElement) {
+          node.parentNode.setAttribute(taskIdKey, nodeHash.toString());
+        }
+      } else {
+        node.setAttribute(taskIdKey, nodeHash.toString());
+      }
     }
     return selector;
   }
@@ -690,6 +739,28 @@ var midscene_element_inspector = (() => {
       zoom
     };
   }
+  var isElementCovered = (el, rect) => {
+    const x = rect.left + rect.width / 2;
+    const y = rect.top + rect.height / 2;
+    const topElement = document.elementFromPoint(x, y);
+    if (topElement === el) {
+      return false;
+    }
+    if (el == null ? void 0 : el.contains(topElement)) {
+      return false;
+    }
+    if (topElement == null ? void 0 : topElement.contains(el)) {
+      return false;
+    }
+    logger(el, "Element is covered by another element", {
+      topElement,
+      el,
+      rect,
+      x,
+      y
+    });
+    return true;
+  };
   function visibleRect(el, baseZoom = 1) {
     if (!el) {
       logger(el, "Element is not in the DOM hierarchy");
@@ -711,6 +782,9 @@ var midscene_element_inspector = (() => {
       logger(el, "Element has no size");
       return false;
     }
+    if (baseZoom === 1 && isElementCovered(el, rect)) {
+      return false;
+    }
     const scrollLeft = window.pageXOffset || document.documentElement.scrollLeft;
     const scrollTop = window.pageYOffset || document.documentElement.scrollTop;
     const viewportWidth = window.innerWidth || document.documentElement.clientWidth;
@@ -805,12 +879,11 @@ var midscene_element_inspector = (() => {
       logger("Element is not visible", node);
       return null;
     }
-    const debugMode2 = getDebugMode();
     if (isFormElement(node)) {
-      const attributes2 = getNodeAttributes(node);
-      const nodeHashId2 = midsceneGenerateHash(attributes2.placeholder, rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      let valueContent = attributes2.value || attributes2.placeholder || node.textContent || "";
+      const attributes = getNodeAttributes(node);
+      const nodeHashId = midsceneGenerateHash(attributes.placeholder, rect);
+      const selector = setDataForNode(node, nodeHashId);
+      let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
       const tagName = node.tagName.toLowerCase();
       if (node.tagName.toLowerCase() === "select") {
         const selectedOption = node.options[node.selectedIndex];
@@ -819,13 +892,13 @@ var midscene_element_inspector = (() => {
       if ((node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea") && node.value) {
         valueContent = node.value;
       }
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
-        locator: selector2,
+        nodeHashId,
+        locator: selector,
         nodeType: NodeType.FORM_ITEM,
-        attributes: __spreadProps(__spreadValues({}, attributes2), {
+        attributes: __spreadProps(__spreadValues({}, attributes), {
           htmlTagName: `<${tagName}>`,
           nodeType: NodeType.FORM_ITEM
         }),
@@ -835,24 +908,25 @@ var midscene_element_inspector = (() => {
           Math.round(rect.left + rect.width / 2),
           Math.round(rect.top + rect.height / 2)
         ],
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
     if (isButtonElement(node)) {
-      const attributes2 = getNodeAttributes(node);
+      const attributes = getNodeAttributes(node);
       const pseudo = getPseudoElementContent(node);
       const content = node.innerText || pseudo.before || pseudo.after || "";
-      const nodeHashId2 = midsceneGenerateHash(content, rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const nodeHashId = midsceneGenerateHash(content, rect);
+      const selector = setDataForNode(node, nodeHashId);
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
+        nodeHashId,
         nodeType: NodeType.BUTTON,
-        locator: selector2,
-        attributes: __spreadProps(__spreadValues({}, attributes2), {
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
           nodeType: NodeType.BUTTON
         }),
         content,
@@ -861,21 +935,22 @@ var midscene_element_inspector = (() => {
           Math.round(rect.left + rect.width / 2),
           Math.round(rect.top + rect.height / 2)
         ],
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
     if (isImgElement(node)) {
-      const attributes2 = getNodeAttributes(node);
-      const nodeHashId2 = midsceneGenerateHash("", rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const attributes = getNodeAttributes(node);
+      const nodeHashId = midsceneGenerateHash("", rect);
+      const selector = setDataForNode(node, nodeHashId);
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
-        locator: selector2,
-        attributes: __spreadProps(__spreadValues(__spreadValues({}, attributes2), node.nodeName.toLowerCase() === "svg" ? {
+        nodeHashId,
+        locator: selector,
+        attributes: __spreadProps(__spreadValues(__spreadValues({}, attributes), node.nodeName.toLowerCase() === "svg" ? {
           svgContent: "true"
         } : {}), {
           nodeType: NodeType.IMG
@@ -887,30 +962,31 @@ var midscene_element_inspector = (() => {
           Math.round(rect.left + rect.width / 2),
           Math.round(rect.top + rect.height / 2)
         ],
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
     if (isTextElement(node)) {
       const text = (_a = node.textContent) == null ? void 0 : _a.trim().replace(/\n+/g, " ");
       if (!text) {
         return null;
       }
-      const attributes2 = getNodeAttributes(node);
-      const attributeKeys = Object.keys(attributes2);
+      const attributes = getNodeAttributes(node);
+      const attributeKeys = Object.keys(attributes);
       if (!text.trim() && attributeKeys.length === 0) {
         return null;
       }
-      const nodeHashId2 = midsceneGenerateHash(text, rect);
-      const selector2 = setDataForNode(node, nodeHashId2);
-      const elementInfo2 = {
-        id: nodeHashId2,
+      const nodeHashId = midsceneGenerateHash(text, rect);
+      const selector = setDataForNode(node, nodeHashId, true);
+      const elementInfo = {
+        id: nodeHashId,
         nodePath,
-        nodeHashId: nodeHashId2,
+        nodeHashId,
         nodeType: NodeType.TEXT,
-        locator: selector2,
-        attributes: __spreadProps(__spreadValues({}, attributes2), {
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
           nodeType: NodeType.TEXT
         }),
         center: [
@@ -920,33 +996,38 @@ var midscene_element_inspector = (() => {
         // attributes,
         content: text,
         rect,
-        htmlNode: debugMode2 ? node : null,
-        zoom: rect.zoom
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
       };
-      return elementInfo2;
+      return elementInfo;
     }
-    const attributes = getNodeAttributes(node);
-    const nodeHashId = midsceneGenerateHash("", rect);
-    const selector = setDataForNode(node, nodeHashId);
-    const elementInfo = {
-      id: nodeHashId,
-      nodePath,
-      nodeHashId,
-      nodeType: NodeType.CONTAINER,
-      locator: selector,
-      attributes: __spreadProps(__spreadValues({}, attributes), {
-        nodeType: NodeType.CONTAINER
-      }),
-      content: "",
-      rect,
-      center: [
-        Math.round(rect.left + rect.width / 2),
-        Math.round(rect.top + rect.height / 2)
-      ],
-      htmlNode: debugMode2 ? node : null,
-      zoom: rect.zoom
-    };
-    return elementInfo;
+    if (isContainerElement(node)) {
+      const attributes = getNodeAttributes(node);
+      const nodeHashId = midsceneGenerateHash("", rect);
+      const selector = setDataForNode(node, nodeHashId);
+      const elementInfo = {
+        id: nodeHashId,
+        nodePath,
+        nodeHashId,
+        nodeType: NodeType.CONTAINER,
+        locator: selector,
+        attributes: __spreadProps(__spreadValues({}, attributes), {
+          nodeType: NodeType.CONTAINER
+        }),
+        content: "",
+        rect,
+        center: [
+          Math.round(rect.left + rect.width / 2),
+          Math.round(rect.top + rect.height / 2)
+        ],
+        zoom: rect.zoom,
+        screenWidth: window.innerWidth,
+        screenHeight: window.innerHeight
+      };
+      return elementInfo;
+    }
+    return null;
   }
   function extractTextWithPosition(initNode, debugMode2 = false, currentFrame = { id: 0, left: 0, top: 0 }) {
     setDebugMode(debugMode2);
@@ -957,34 +1038,14 @@ var midscene_element_inspector = (() => {
         return null;
       }
       const elementInfo = collectElementInfo(node, nodePath, baseZoom);
-      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG) {
+      if ((elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.BUTTON || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.IMG || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.TEXT || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.FORM_ITEM || (elementInfo == null ? void 0 : elementInfo.nodeType) === NodeType.CONTAINER) {
+        elementInfoArray.push(elementInfo);
         return elementInfo;
       }
-      let hasNonContainerChildren = false;
-      const childrenPureContainers = [];
+      const rect = getRect(node, baseZoom);
       for (let i = 0; i < node.childNodes.length; i++) {
         logger("will dfs", node.childNodes[i]);
-        const resultLengthBeforeDfs = elementInfoArray.length;
-        const result = dfs(
-          node.childNodes[i],
-          `${nodePath}-${i}`,
-          elementInfo == null ? void 0 : elementInfo.zoom
-        );
-        if (!result)
-          continue;
-        if ((result == null ? void 0 : result.nodeType) === NodeType.CONTAINER && elementInfoArray.length > resultLengthBeforeDfs) {
-          hasNonContainerChildren = true;
-          continue;
-        }
-        if ((result == null ? void 0 : result.nodeType) === NodeType.CONTAINER) {
-          childrenPureContainers.push(result);
-        } else {
-          hasNonContainerChildren = true;
-          elementInfoArray.push(result);
-        }
-      }
-      if (hasNonContainerChildren) {
-        elementInfoArray.push(...childrenPureContainers);
+        dfs(node.childNodes[i], `${nodePath}-${i}`, rect.zoom);
       }
       if (!elementInfo) {
         logger("should NOT continue for node", node);
@@ -992,10 +1053,7 @@ var midscene_element_inspector = (() => {
       }
       return elementInfo;
     }
-    const outerMostElementInfo = dfs(initNode || getDocument(), "0");
-    if (outerMostElementInfo && !elementInfoArray.length) {
-      elementInfoArray.push(outerMostElementInfo);
-    }
+    dfs(initNode || getDocument(), "0");
     for (let i = 0; i < elementInfoArray.length; i++) {
       elementInfoArray[i].indexId = (i + 1).toString();
     }
@@ -1013,7 +1071,7 @@ var midscene_element_inspector = (() => {
 
   // src/extractor/debug.ts
   console.log(extractTextWithPosition(document.body, true));
-  console.log(JSON.stringify(extractTextWithPosition(document.body, false)));
+  console.log(JSON.stringify(extractTextWithPosition(document.body, true)));
   setExtractTextWithPositionOnWindow();
 })();
 /*! Bundled license information:

package/dist/types/appium.d.ts

@@ -1,5 +1,5 @@
-export { P as AppiumAgent } from './tasks-a905efe6.js';
-export { P as AppiumPage } from './page-2f02d5a8.js';
+export { P as AppiumAgent } from './tasks-876d60ec.js';
+export { P as AppiumPage } from './page-dd5d0f7b.js';
 import '@midscene/core';
 import '@midscene/shared/constants';
 import '@midscene/shared/fs';

package/dist/types/debug.d.ts

@@ -1,5 +1,5 @@
 import { writeFileSync } from 'node:fs';
-import { W as WebPage, E as ElementInfo } from './page-2f02d5a8.js';
+import { W as WebPage, E as ElementInfo } from './page-dd5d0f7b.js';
 import { NodeType } from '@midscene/shared/constants';
 import 'playwright';
 import 'puppeteer';

package/dist/types/index.d.ts

@@ -1,6 +1,6 @@
 export { PlayWrightAiFixtureType, PlaywrightAiFixture } from './playwright.js';
-export { P as AppiumAgent, P as PlaywrightAgent } from './tasks-a905efe6.js';
-export { P as AppiumPage } from './page-2f02d5a8.js';
+export { P as AppiumAgent, P as PlaywrightAgent } from './tasks-876d60ec.js';
+export { P as AppiumPage } from './page-dd5d0f7b.js';
 export { PuppeteerAgent } from './puppeteer.js';
 export { generateExtractData } from './debug.js';
 import '@midscene/core/.';

package/dist/types/{page-2f02d5a8.d.ts → page-dd5d0f7b.d.ts}

@@ -14,7 +14,6 @@ interface ElementInfo {
         [key: string]: string;
     };
     nodeType: NodeType;
-    htmlNode: Node | null;
     content: string;
     rect: {
         left: number;

package/dist/types/playwright.d.ts

@@ -1,9 +1,9 @@
 import { AgentWaitForOpt } from '@midscene/core/.';
 import { TestInfo } from '@playwright/test';
 import { Page } from 'playwright';
-import { b as PageTaskExecutor } from './tasks-a905efe6.js';
-export { P as PlaywrightAgent } from './tasks-a905efe6.js';
-export { b as PlaywrightWebPage } from './page-2f02d5a8.js';
+import { b as PageTaskExecutor } from './tasks-876d60ec.js';
+export { P as PlaywrightAgent } from './tasks-876d60ec.js';
+export { b as PlaywrightWebPage } from './page-dd5d0f7b.js';
 import '@midscene/core';
 import '@midscene/shared/constants';
 import '@midscene/shared/fs';

package/dist/types/puppeteer.d.ts

@@ -1,6 +1,6 @@
-import { P as PageAgent, a as PageAgentOpt } from './tasks-a905efe6.js';
+import { P as PageAgent, a as PageAgentOpt } from './tasks-876d60ec.js';
 import { Page } from 'puppeteer';
-export { a as PuppeteerWebPage } from './page-2f02d5a8.js';
+export { a as PuppeteerWebPage } from './page-dd5d0f7b.js';
 import '@midscene/core';
 import '@midscene/shared/constants';
 import '@midscene/shared/fs';

package/dist/types/{tasks-a905efe6.d.ts → tasks-876d60ec.d.ts}

@@ -1,4 +1,4 @@
-import { W as WebPage } from './page-2f02d5a8.js';
+import { W as WebPage } from './page-dd5d0f7b.js';
 import Insight, { BaseElement, Rect, UIContext, PlanningAction, AIElementParseResponse, GroupedActionDump, ExecutionDump, AgentWaitForOpt, InsightExtractParam, InsightAssertionResponse, PlanningActionParamWaitFor, Executor } from '@midscene/core';
 import { NodeType } from '@midscene/shared/constants';
 import { getMidscenePkgInfo } from '@midscene/shared/fs';

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/web",
   "description": "Web integration for Midscene.js",
-  "version": "0.5.1",
+  "version": "0.5.2-beta-20240929094445.0",
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
@@ -81,8 +81,8 @@
     "openai": "4.57.1",
     "inquirer": "10.1.5",
     "@xmldom/xmldom": "0.8.10",
-    "@midscene/core": "0.5.1",
-    "@midscene/shared": "0.5.1"
+    "@midscene/shared": "0.5.2-beta-20240929094445.0",
+    "@midscene/core": "0.5.2-beta-20240929094445.0"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.58.2",