@midscene/web 0.3.4 → 0.4.0

package/dist/script/htmlElement.js

@@ -580,9 +580,18 @@ var midscene_element_inspector = (() => {
     extractTextWithPosition: () => extractTextWithPosition
   });
 
+  // ../shared/dist/es/constants.js
+  var NodeType = /* @__PURE__ */ ((NodeType2) => {
+    NodeType2["FORM_ITEM"] = "FORM_ITEM Node";
+    NodeType2["BUTTON"] = "BUTTON Node";
+    NodeType2["IMG"] = "IMG Node";
+    NodeType2["TEXT"] = "TEXT Node";
+    return NodeType2;
+  })(NodeType || {});
+
   // src/extractor/dom-util.ts
-  function isInputElement(node) {
-    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea");
+  function isFormElement(node) {
+    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "label" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
   }
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
@@ -744,6 +753,7 @@ var midscene_element_inspector = (() => {
       }
       const shouldContinue = collectElementInfo(node);
       if (!shouldContinue) {
+        logger("should NOT continue for node", node);
         return;
       }
       for (let i = 0; i < node.childNodes.length; i++) {
@@ -759,20 +769,27 @@ var midscene_element_inspector = (() => {
         logger("Element is not visible", node);
         return true;
       }
-      if (isInputElement(node)) {
+      if (isFormElement(node)) {
         const attributes = getNodeAttributes(node);
         const nodeHashId = generateHash(attributes.placeholder, rect);
         const selector = setDataForNode(node, nodeHashId);
+        let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
+        const tagName = node.tagName.toLowerCase();
+        if (node.tagName.toLowerCase() === "select") {
+          const selectedOption = node.options[node.selectedIndex];
+          valueContent = selectedOption.textContent || "";
+        }
         elementInfoArray.push({
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
           locator: selector,
-          nodeType: "INPUT Node" /* INPUT */,
+          nodeType: NodeType.FORM_ITEM,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "INPUT Node" /* INPUT */
+            htmlTagName: `<${tagName}>`,
+            nodeType: NodeType.FORM_ITEM
           }),
-          content: attributes.placeholder || "",
+          content: valueContent.trim(),
           rect,
           center: [
             Math.round(rect.left + rect.width / 2),
@@ -780,6 +797,8 @@ var midscene_element_inspector = (() => {
           ],
           htmlNode: debugMode2 ? node : null
         });
+        if (tagName === "label")
+          return true;
         return;
       }
       if (isButtonElement(node)) {
@@ -792,10 +811,10 @@ var midscene_element_inspector = (() => {
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
-          nodeType: "BUTTON Node" /* BUTTON */,
+          nodeType: NodeType.BUTTON,
           locator: selector,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "BUTTON Node" /* BUTTON */
+            nodeType: NodeType.BUTTON
           }),
           content,
           rect,
@@ -817,9 +836,9 @@ var midscene_element_inspector = (() => {
           nodeHashId,
           locator: selector,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "IMG Node" /* IMG */
+            nodeType: NodeType.IMG
           }),
-          nodeType: "IMG Node" /* IMG */,
+          nodeType: NodeType.IMG,
           content: "",
           rect,
           center: [
@@ -836,16 +855,20 @@ var midscene_element_inspector = (() => {
           return;
         }
         const attributes = getNodeAttributes(node);
+        const attributeKeys = Object.keys(attributes);
+        if (!text.trim() && attributeKeys.length === 0) {
+          return;
+        }
         const nodeHashId = generateHash(text, rect);
         const selector = setDataForNode(node, nodeHashId);
         elementInfoArray.push({
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
-          nodeType: "TEXT Node" /* TEXT */,
+          nodeType: NodeType.TEXT,
           locator: selector,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "TEXT Node" /* TEXT */
+            nodeType: NodeType.TEXT
           }),
           center: [
             Math.round(rect.left + rect.width / 2),

package/dist/script/htmlElementDebug.js

@@ -569,9 +569,18 @@ var midscene_element_inspector = (() => {
     }
   });
 
+  // ../shared/dist/es/constants.js
+  var NodeType = /* @__PURE__ */ ((NodeType2) => {
+    NodeType2["FORM_ITEM"] = "FORM_ITEM Node";
+    NodeType2["BUTTON"] = "BUTTON Node";
+    NodeType2["IMG"] = "IMG Node";
+    NodeType2["TEXT"] = "TEXT Node";
+    return NodeType2;
+  })(NodeType || {});
+
   // src/extractor/dom-util.ts
-  function isInputElement(node) {
-    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea");
+  function isFormElement(node) {
+    return node instanceof HTMLElement && (node.tagName.toLowerCase() === "input" || node.tagName.toLowerCase() === "textarea" || node.tagName.toLowerCase() === "label" || node.tagName.toLowerCase() === "select" || node.tagName.toLowerCase() === "option");
   }
   function isButtonElement(node) {
     return node instanceof HTMLElement && node.tagName.toLowerCase() === "button";
@@ -733,6 +742,7 @@ var midscene_element_inspector = (() => {
       }
       const shouldContinue = collectElementInfo(node);
       if (!shouldContinue) {
+        logger("should NOT continue for node", node);
         return;
       }
       for (let i = 0; i < node.childNodes.length; i++) {
@@ -748,20 +758,27 @@ var midscene_element_inspector = (() => {
         logger("Element is not visible", node);
         return true;
       }
-      if (isInputElement(node)) {
+      if (isFormElement(node)) {
         const attributes = getNodeAttributes(node);
         const nodeHashId = generateHash(attributes.placeholder, rect);
         const selector = setDataForNode(node, nodeHashId);
+        let valueContent = attributes.value || attributes.placeholder || node.textContent || "";
+        const tagName = node.tagName.toLowerCase();
+        if (node.tagName.toLowerCase() === "select") {
+          const selectedOption = node.options[node.selectedIndex];
+          valueContent = selectedOption.textContent || "";
+        }
         elementInfoArray.push({
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
           locator: selector,
-          nodeType: "INPUT Node" /* INPUT */,
+          nodeType: NodeType.FORM_ITEM,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "INPUT Node" /* INPUT */
+            htmlTagName: `<${tagName}>`,
+            nodeType: NodeType.FORM_ITEM
           }),
-          content: attributes.placeholder || "",
+          content: valueContent.trim(),
           rect,
           center: [
             Math.round(rect.left + rect.width / 2),
@@ -769,6 +786,8 @@ var midscene_element_inspector = (() => {
           ],
           htmlNode: debugMode2 ? node : null
         });
+        if (tagName === "label")
+          return true;
         return;
       }
       if (isButtonElement(node)) {
@@ -781,10 +800,10 @@ var midscene_element_inspector = (() => {
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
-          nodeType: "BUTTON Node" /* BUTTON */,
+          nodeType: NodeType.BUTTON,
           locator: selector,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "BUTTON Node" /* BUTTON */
+            nodeType: NodeType.BUTTON
           }),
           content,
           rect,
@@ -806,9 +825,9 @@ var midscene_element_inspector = (() => {
           nodeHashId,
           locator: selector,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "IMG Node" /* IMG */
+            nodeType: NodeType.IMG
           }),
-          nodeType: "IMG Node" /* IMG */,
+          nodeType: NodeType.IMG,
           content: "",
           rect,
           center: [
@@ -825,16 +844,20 @@ var midscene_element_inspector = (() => {
           return;
         }
         const attributes = getNodeAttributes(node);
+        const attributeKeys = Object.keys(attributes);
+        if (!text.trim() && attributeKeys.length === 0) {
+          return;
+        }
         const nodeHashId = generateHash(text, rect);
         const selector = setDataForNode(node, nodeHashId);
         elementInfoArray.push({
           id: nodeHashId,
           indexId: generateId(nodeIndex++),
           nodeHashId,
-          nodeType: "TEXT Node" /* TEXT */,
+          nodeType: NodeType.TEXT,
           locator: selector,
           attributes: __spreadProps(__spreadValues({}, attributes), {
-            nodeType: "TEXT Node" /* TEXT */
+            nodeType: NodeType.TEXT
           }),
           center: [
             Math.round(rect.left + rect.width / 2),

package/dist/types/debug.d.ts

@@ -1,8 +1,30 @@
 import { writeFileSync } from 'node:fs';
 import { W as WebPage } from './page.d-70ed000f.js';
+import { NodeType } from '@midscene/shared/constants';
 import 'playwright';
 import 'puppeteer';
 
+interface ElementInfo {
+    id: string;
+    indexId: string;
+    nodeHashId: string;
+    locator: string;
+    attributes: {
+        nodeType: NodeType;
+        [key: string]: string;
+    };
+    nodeType: NodeType;
+    htmlNode: Node | null;
+    content: string;
+    rect: {
+        left: number;
+        top: number;
+        width: number;
+        height: number;
+    };
+    center: [number, number];
+}
+
 declare function generateExtractData(page: WebPage, targetDir: string, saveImgType?: {
     disableInputImage: boolean;
     disableOutputImage: boolean;
@@ -13,5 +35,30 @@ declare function generateExtractData(page: WebPage, targetDir: string, saveImgTy
 declare function generateTestDataPath(testDataName: string): string;
 type WriteFileSyncParams = Parameters<typeof writeFileSync>;
 declare function writeFileSyncWithDir(filePath: string, content: WriteFileSyncParams[1], options?: WriteFileSyncParams[2]): void;
+declare function getElementInfos(page: any): Promise<{
+    elementsPositionInfo: {
+        label: string;
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+        attributes: {
+            [key: string]: string;
+            nodeType: NodeType;
+        };
+    }[];
+    captureElementSnapshot: ElementInfo[];
+    elementsPositionInfoWithoutText: {
+        label: string;
+        x: number;
+        y: number;
+        width: number;
+        height: number;
+        attributes: {
+            [key: string]: string;
+            nodeType: NodeType;
+        };
+    }[];
+}>;
 
-export { generateExtractData, generateTestDataPath, writeFileSyncWithDir };
+export { generateExtractData, generateTestDataPath, getElementInfos, writeFileSyncWithDir };

package/dist/types/index.d.ts

@@ -4,8 +4,9 @@ export { generateExtractData } from './debug.js';
 import '@midscene/core/.';
 import '@playwright/test';
 import 'playwright';
-import './tasks-75eae15e.js';
+import './tasks-ce3a7499.js';
 import './page.d-70ed000f.js';
 import 'puppeteer';
 import '@midscene/core';
+import '@midscene/shared/constants';
 import 'node:fs';

package/dist/types/playwright.d.ts

@@ -1,10 +1,11 @@
 import { AgentWaitForOpt } from '@midscene/core/.';
 import { TestInfo } from '@playwright/test';
 import { Page } from 'playwright';
-import { P as PageTaskExecutor } from './tasks-75eae15e.js';
+import { P as PageTaskExecutor } from './tasks-ce3a7499.js';
 import './page.d-70ed000f.js';
 import 'puppeteer';
 import '@midscene/core';
+import '@midscene/shared/constants';
 
 declare const PlaywrightAiFixture: () => {
     ai: ({ page }: {

package/dist/types/puppeteer.d.ts

@@ -1,8 +1,9 @@
 import { W as WebPage } from './page.d-70ed000f.js';
 import { GroupedActionDump, ExecutionDump, AgentWaitForOpt } from '@midscene/core';
-import { P as PageTaskExecutor, A as AiTaskCache } from './tasks-75eae15e.js';
+import { P as PageTaskExecutor, A as AiTaskCache } from './tasks-ce3a7499.js';
 import 'playwright';
 import 'puppeteer';
+import '@midscene/shared/constants';
 
 interface PageAgentOpt {
     testId?: string;

package/dist/types/{tasks-75eae15e.d.ts → tasks-ce3a7499.d.ts}

@@ -1,12 +1,6 @@
 import { W as WebPage } from './page.d-70ed000f.js';
 import Insight, { BaseElement, Rect, UIContext, PlanningAction, AIElementParseResponse, InsightExtractParam, InsightAssertionResponse, PlanningActionParamWaitFor, Executor } from '@midscene/core';
-
-declare enum NodeType {
-    INPUT = "INPUT Node",
-    BUTTON = "BUTTON Node",
-    IMG = "IMG Node",
-    TEXT = "TEXT Node"
-}
+import { NodeType } from '@midscene/shared/constants';
 
 declare class WebElementInfo implements BaseElement {
     content: string;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/web",
   "description": "Web integration for Midscene.js",
-  "version": "0.3.4",
+  "version": "0.4.0",
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "module": "./dist/es/index.js",
@@ -74,12 +74,12 @@
   ],
   "dependencies": {
     "openai": "4.47.1",
-    "sharp": "0.33.3",
     "inquirer": "10.1.5",
-    "@midscene/core": "0.3.4"
+    "@midscene/core": "0.4.0",
+    "@midscene/shared": "0.4.0"
   },
   "devDependencies": {
-    "@modern-js/module-tools": "^2.56.1",
+    "@modern-js/module-tools": "2.58.2",
     "js-sha256": "0.11.0",
     "@types/node": "^18.0.0",
     "typescript": "~5.0.4",

package/dist/script/types/htmlElement.d.ts

@@ -1,30 +0,0 @@
-declare enum NodeType {
-    INPUT = "INPUT Node",
-    BUTTON = "BUTTON Node",
-    IMG = "IMG Node",
-    TEXT = "TEXT Node"
-}
-
-interface ElementInfo {
-    id: string;
-    indexId: string;
-    nodeHashId: string;
-    locator: string;
-    attributes: {
-        nodeType: NodeType;
-        [key: string]: string;
-    };
-    nodeType: NodeType;
-    htmlNode: Node | null;
-    content: string;
-    rect: {
-        left: number;
-        top: number;
-        width: number;
-        height: number;
-    };
-    center: [number, number];
-}
-declare function extractTextWithPosition(initNode?: Node, debugMode?: boolean): ElementInfo[];
-
-export { extractTextWithPosition };

package/dist/script/types/htmlElementDebug.d.ts

@@ -1,2 +0,0 @@
-
-export {  }