@midscene/web 0.5.2-beta-20241010035503.0 → 0.5.2

package/dist/lib/playwright.js

@@ -361,7 +361,7 @@ var import_utils5 = require("@midscene/core/utils");
 var import_node_assert2 = __toESM(require("assert"));
 var import_core = __toESM(require("@midscene/core"));
 var import_utils3 = require("@midscene/core/utils");
-var import_img2 = require("@midscene/shared/img");
+var import_img3 = require("@midscene/shared/img");
 
 // src/common/task-cache.ts
 var import_node_fs2 = require("fs");
@@ -374,8 +374,10 @@ var import_node_assert = __toESM(require("assert"));
 var import_node_crypto = require("crypto");
 var import_node_fs = require("fs");
 var import_node_path = __toESM(require("path"));
+var import_constants = require("@midscene/shared/constants");
 var import_fs = require("@midscene/shared/fs");
 var import_img = require("@midscene/shared/img");
+var import_img2 = require("@midscene/shared/img");
 var import_dayjs = __toESM(require_dayjs_min());
 
 // src/web-element.ts
@@ -386,7 +388,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -398,6 +401,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -406,19 +410,24 @@ async function parseContextFromWebPage(page, _opt) {
   (0, import_node_assert.default)(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
   const screenshotBase64 = (0, import_img.base64Encoded)(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
+  const screenshotBase64WithElementInfos = await (0, import_img2.compositeElementInfoImg)({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
@@ -430,13 +439,13 @@ async function getExtraReturnLogic() {
   return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -444,7 +453,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -629,7 +639,7 @@ var PageTaskExecutor = class {
     const item = {
       type: "screenshot",
       ts: Date.now(),
-      screenshot: (0, import_img2.base64Encoded)(file),
+      screenshot: (0, import_img3.base64Encoded)(file),
       timing
     };
     return item;
@@ -998,13 +1008,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1275,7 +1285,7 @@ function waitForNetworkIdle(page) {
 // src/puppeteer/base-page.ts
 var import_node_fs3 = require("fs");
 var import_utils7 = require("@midscene/core/utils");
-var import_img3 = require("@midscene/shared/img");
+var import_img4 = require("@midscene/shared/img");
 var Page = class {
   evaluate(pageFunction, arg) {
     if (this.pageType === "puppeteer") {
@@ -1300,14 +1310,14 @@ var Page = class {
         deviceScaleFactor: window.devicePixelRatio
       };
     });
-    const path2 = (0, import_utils7.getTmpFile)("jpeg");
+    const path2 = (0, import_utils7.getTmpFile)("png");
     await this.page.screenshot({
       path: path2,
       type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {
-      buf = await (0, import_img3.resizeImg)((0, import_node_fs3.readFileSync)(path2), {
+      buf = await (0, import_img4.resizeImg)((0, import_node_fs3.readFileSync)(path2), {
         width: viewportSize.width,
         height: viewportSize.height
       });

package/dist/lib/puppeteer.js

@@ -357,7 +357,7 @@ var import_utils5 = require("@midscene/core/utils");
 var import_node_assert2 = __toESM(require("assert"));
 var import_core = __toESM(require("@midscene/core"));
 var import_utils3 = require("@midscene/core/utils");
-var import_img2 = require("@midscene/shared/img");
+var import_img3 = require("@midscene/shared/img");
 
 // src/common/task-cache.ts
 var import_node_fs2 = require("fs");
@@ -370,8 +370,10 @@ var import_node_assert = __toESM(require("assert"));
 var import_node_crypto = require("crypto");
 var import_node_fs = require("fs");
 var import_node_path = __toESM(require("path"));
+var import_constants = require("@midscene/shared/constants");
 var import_fs = require("@midscene/shared/fs");
 var import_img = require("@midscene/shared/img");
+var import_img2 = require("@midscene/shared/img");
 var import_dayjs = __toESM(require_dayjs_min());
 
 // src/web-element.ts
@@ -382,7 +384,8 @@ var WebElementInfo = class {
     page,
     locator,
     id,
-    attributes
+    attributes,
+    indexId
   }) {
     this.content = content;
     this.rect = rect;
@@ -394,6 +397,7 @@ var WebElementInfo = class {
     this.locator = locator;
     this.id = id;
     this.attributes = attributes;
+    this.indexId = indexId;
   }
 };
 
@@ -402,19 +406,24 @@ async function parseContextFromWebPage(page, _opt) {
   (0, import_node_assert.default)(page, "page is required");
   const url = page.url();
   const file = await page.screenshot();
-  const screenshotBuffer = (0, import_node_fs.readFileSync)(file);
   const screenshotBase64 = (0, import_img.base64Encoded)(file);
   const captureElementSnapshot = await page.getElementInfos();
-  const elementsInfo = await alignElements(
-    screenshotBuffer,
-    captureElementSnapshot,
-    page
-  );
+  const elementsInfo = await alignElements(captureElementSnapshot, page);
+  const elementsPositionInfoWithoutText = elementsInfo.filter((elementInfo) => {
+    if (elementInfo.attributes.nodeType === import_constants.NodeType.TEXT) {
+      return false;
+    }
+    return true;
+  });
   const size = await (0, import_img.imageInfoOfBase64)(screenshotBase64);
+  const screenshotBase64WithElementInfos = await (0, import_img2.compositeElementInfoImg)({
+    inputImgBase64: screenshotBase64.split(";base64,").pop(),
+    elementsPositionInfo: elementsPositionInfoWithoutText
+  });
   return {
     content: elementsInfo,
     size,
-    screenshotBase64,
+    screenshotBase64: `data:image/png;base64,${screenshotBase64WithElementInfos}`,
     url
   };
 }
@@ -426,13 +435,13 @@ async function getExtraReturnLogic() {
   return `${elementInfosScriptContent}midscene_element_inspector.webExtractTextWithPosition()`;
 }
 var sizeThreshold = 3;
-async function alignElements(screenshotBuffer, elements, page) {
+async function alignElements(elements, page) {
   const validElements = elements.filter((item) => {
     return item.rect.height >= sizeThreshold && item.rect.width >= sizeThreshold;
   });
   const textsAligned = [];
   for (const item of validElements) {
-    const { rect, id, content, attributes, locator } = item;
+    const { rect, id, content, attributes, locator, indexId } = item;
     textsAligned.push(
       new WebElementInfo({
         rect,
@@ -440,7 +449,8 @@ async function alignElements(screenshotBuffer, elements, page) {
         id,
         content,
         attributes,
-        page
+        page,
+        indexId
       })
     );
   }
@@ -625,7 +635,7 @@ var PageTaskExecutor = class {
     const item = {
       type: "screenshot",
       ts: Date.now(),
-      screenshot: (0, import_img2.base64Encoded)(file),
+      screenshot: (0, import_img3.base64Encoded)(file),
       timing
     };
     return item;
@@ -994,13 +1004,13 @@ var PageTaskExecutor = class {
       const assertTask = await this.convertPlanToExecutable([assertPlan]);
       await taskExecutor.append(this.wrapExecutorWithScreenshot(assertTask[0]));
       const output = await taskExecutor.flush();
-      if (output.pass) {
+      if (output == null ? void 0 : output.pass) {
         return {
           output: void 0,
           executor: taskExecutor
         };
       }
-      errorThought = output.thought;
+      errorThought = (output == null ? void 0 : output.thought) || "unknown error";
       const now = Date.now();
       if (now - startTime < checkIntervalMs) {
         const timeRemaining = checkIntervalMs - (now - startTime);
@@ -1143,7 +1153,7 @@ ${errorTask == null ? void 0 : errorTask.errorStack}`);
 // src/puppeteer/base-page.ts
 var import_node_fs3 = require("fs");
 var import_utils7 = require("@midscene/core/utils");
-var import_img3 = require("@midscene/shared/img");
+var import_img4 = require("@midscene/shared/img");
 var Page = class {
   evaluate(pageFunction, arg) {
     if (this.pageType === "puppeteer") {
@@ -1168,14 +1178,14 @@ var Page = class {
         deviceScaleFactor: window.devicePixelRatio
       };
     });
-    const path2 = (0, import_utils7.getTmpFile)("jpeg");
+    const path2 = (0, import_utils7.getTmpFile)("png");
     await this.page.screenshot({
       path: path2,
       type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {
-      buf = await (0, import_img3.resizeImg)((0, import_node_fs3.readFileSync)(path2), {
+      buf = await (0, import_img4.resizeImg)((0, import_node_fs3.readFileSync)(path2), {
         width: viewportSize.width,
         height: viewportSize.height
       });

package/dist/script/htmlElement.js

@@ -872,15 +872,13 @@ var midscene_element_inspector = (() => {
     const hashHex = (0, import_js_sha256.default)(combined);
     return hashHex.slice(0, 10);
   }
-  function generateId(numberId) {
-    return `${numberId}`;
-  }
   function getDocument() {
     const container = document.body || document;
     return container;
   }
 
   // src/extractor/web-extractor.ts
+  var indexId = 0;
   function collectElementInfo(node, nodePath, baseZoom = 1) {
     var _a;
     const rect = visibleRect(node, baseZoom);
@@ -908,6 +906,7 @@ var midscene_element_inspector = (() => {
         nodeHashId,
         locator: selector,
         nodeType: NodeType.FORM_ITEM,
+        indexId: indexId++,
         attributes: __spreadProps(__spreadValues({}, attributes), {
           htmlTagName: `<${tagName}>`,
           nodeType: NodeType.FORM_ITEM
@@ -932,6 +931,7 @@ var midscene_element_inspector = (() => {
       const selector = setDataForNode(node, nodeHashId);
       const elementInfo = {
         id: nodeHashId,
+        indexId: indexId++,
         nodePath,
         nodeHashId,
         nodeType: NodeType.BUTTON,
@@ -957,6 +957,7 @@ var midscene_element_inspector = (() => {
       const selector = setDataForNode(node, nodeHashId);
       const elementInfo = {
         id: nodeHashId,
+        indexId: indexId++,
         nodePath,
         nodeHashId,
         locator: selector,
@@ -992,6 +993,7 @@ var midscene_element_inspector = (() => {
       const selector = setDataForNode(node, nodeHashId, true);
       const elementInfo = {
         id: nodeHashId,
+        indexId: indexId++,
         nodePath,
         nodeHashId,
         nodeType: NodeType.TEXT,
@@ -1020,6 +1022,7 @@ var midscene_element_inspector = (() => {
         id: nodeHashId,
         nodePath,
         nodeHashId,
+        indexId: indexId++,
         nodeType: NodeType.CONTAINER,
         locator: selector,
         attributes: __spreadProps(__spreadValues({}, attributes), {
@@ -1064,9 +1067,6 @@ var midscene_element_inspector = (() => {
       return elementInfo;
     }
     dfs(initNode || getDocument(), "0");
-    for (let i = 0; i < elementInfoArray.length; i++) {
-      elementInfoArray[i].indexId = (i + 1).toString();
-    }
     if (currentFrame.left !== 0 || currentFrame.top !== 0) {
       for (let i = 0; i < elementInfoArray.length; i++) {
         elementInfoArray[i].rect.left += currentFrame.left;
@@ -1214,7 +1214,7 @@ var midscene_element_inspector = (() => {
       const xpath = getXPathForElement(node);
       const elementInfo = {
         id: nodeHashId,
-        indexId: generateId(nodeIndex++),
+        indexId: nodeIndex++,
         nodeHashId,
         locator: xpath,
         attributes: __spreadValues({

package/dist/script/htmlElementDebug.js

@@ -871,6 +871,7 @@ var midscene_element_inspector = (() => {
   }
 
   // src/extractor/web-extractor.ts
+  var indexId = 0;
   function collectElementInfo(node, nodePath, baseZoom = 1) {
     var _a;
     const rect = visibleRect(node, baseZoom);
@@ -898,6 +899,7 @@ var midscene_element_inspector = (() => {
         nodeHashId,
         locator: selector,
         nodeType: NodeType.FORM_ITEM,
+        indexId: indexId++,
         attributes: __spreadProps(__spreadValues({}, attributes), {
           htmlTagName: `<${tagName}>`,
           nodeType: NodeType.FORM_ITEM
@@ -922,6 +924,7 @@ var midscene_element_inspector = (() => {
       const selector = setDataForNode(node, nodeHashId);
       const elementInfo = {
         id: nodeHashId,
+        indexId: indexId++,
         nodePath,
         nodeHashId,
         nodeType: NodeType.BUTTON,
@@ -947,6 +950,7 @@ var midscene_element_inspector = (() => {
       const selector = setDataForNode(node, nodeHashId);
       const elementInfo = {
         id: nodeHashId,
+        indexId: indexId++,
         nodePath,
         nodeHashId,
         locator: selector,
@@ -982,6 +986,7 @@ var midscene_element_inspector = (() => {
       const selector = setDataForNode(node, nodeHashId, true);
       const elementInfo = {
         id: nodeHashId,
+        indexId: indexId++,
         nodePath,
         nodeHashId,
         nodeType: NodeType.TEXT,
@@ -1010,6 +1015,7 @@ var midscene_element_inspector = (() => {
         id: nodeHashId,
         nodePath,
         nodeHashId,
+        indexId: indexId++,
         nodeType: NodeType.CONTAINER,
         locator: selector,
         attributes: __spreadProps(__spreadValues({}, attributes), {
@@ -1054,9 +1060,6 @@ var midscene_element_inspector = (() => {
       return elementInfo;
     }
     dfs(initNode || getDocument(), "0");
-    for (let i = 0; i < elementInfoArray.length; i++) {
-      elementInfoArray[i].indexId = (i + 1).toString();
-    }
     if (currentFrame.left !== 0 || currentFrame.top !== 0) {
       for (let i = 0; i < elementInfoArray.length; i++) {
         elementInfoArray[i].rect.left += currentFrame.left;

package/dist/types/appium.d.ts

@@ -1,5 +1,5 @@
-export { P as AppiumAgent } from './tasks-876d60ec.js';
-export { P as AppiumPage } from './page-dd5d0f7b.js';
+export { P as AppiumAgent } from './tasks-9af2ee2d.js';
+export { P as AppiumPage } from './page-ccbba1d9.js';
 import '@midscene/core';
 import '@midscene/shared/constants';
 import '@midscene/shared/fs';

package/dist/types/debug.d.ts

@@ -1,9 +1,9 @@
 import { writeFileSync } from 'node:fs';
-import { W as WebPage, E as ElementInfo } from './page-dd5d0f7b.js';
-import { NodeType } from '@midscene/shared/constants';
+import { W as WebPage, E as ElementInfo } from './page-ccbba1d9.js';
 import 'playwright';
 import 'puppeteer';
 import 'webdriverio';
+import '@midscene/shared/constants';
 
 declare function generateExtractData(page: WebPage, targetDir: string, saveImgType?: {
     disableInputImage: boolean;
@@ -16,29 +16,9 @@ declare function generateTestDataPath(testDataName: string): string;
 type WriteFileSyncParams = Parameters<typeof writeFileSync>;
 declare function writeFileSyncWithDir(filePath: string, content: WriteFileSyncParams[1], options?: WriteFileSyncParams[2]): void;
 declare function getElementInfos(page: WebPage): Promise<{
-    elementsPositionInfo: {
-        label: string;
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-        attributes: {
-            [key: string]: string;
-            nodeType: NodeType;
-        };
-    }[];
+    elementsPositionInfo: ElementInfo[];
     captureElementSnapshot: ElementInfo[];
-    elementsPositionInfoWithoutText: {
-        label: string;
-        x: number;
-        y: number;
-        width: number;
-        height: number;
-        attributes: {
-            [key: string]: string;
-            nodeType: NodeType;
-        };
-    }[];
+    elementsPositionInfoWithoutText: ElementInfo[];
 }>;
 
 export { generateExtractData, generateTestDataPath, getElementInfos, writeFileSyncWithDir };

package/dist/types/index.d.ts

@@ -1,6 +1,6 @@
 export { PlayWrightAiFixtureType, PlaywrightAiFixture } from './playwright.js';
-export { P as AppiumAgent, P as PlaywrightAgent } from './tasks-876d60ec.js';
-export { P as AppiumPage } from './page-dd5d0f7b.js';
+export { P as AppiumAgent, P as PlaywrightAgent } from './tasks-9af2ee2d.js';
+export { P as AppiumPage } from './page-ccbba1d9.js';
 export { PuppeteerAgent } from './puppeteer.js';
 export { generateExtractData } from './debug.js';
 import '@midscene/core/.';

package/dist/types/{page-dd5d0f7b.d.ts → page-ccbba1d9.d.ts}

@@ -5,7 +5,7 @@ import { NodeType } from '@midscene/shared/constants';
 
 interface ElementInfo {
     id: string;
-    indexId?: string;
+    indexId: number;
     nodePath: string;
     nodeHashId: string;
     locator: string;

package/dist/types/playwright.d.ts

@@ -1,9 +1,9 @@
 import { AgentWaitForOpt } from '@midscene/core/.';
 import { TestInfo } from '@playwright/test';
 import { Page } from 'playwright';
-import { b as PageTaskExecutor } from './tasks-876d60ec.js';
-export { P as PlaywrightAgent } from './tasks-876d60ec.js';
-export { b as PlaywrightWebPage } from './page-dd5d0f7b.js';
+import { b as PageTaskExecutor } from './tasks-9af2ee2d.js';
+export { P as PlaywrightAgent } from './tasks-9af2ee2d.js';
+export { b as PlaywrightWebPage } from './page-ccbba1d9.js';
 import '@midscene/core';
 import '@midscene/shared/constants';
 import '@midscene/shared/fs';

package/dist/types/puppeteer.d.ts

@@ -1,6 +1,6 @@
-import { P as PageAgent, a as PageAgentOpt } from './tasks-876d60ec.js';
+import { P as PageAgent, a as PageAgentOpt } from './tasks-9af2ee2d.js';
 import { Page } from 'puppeteer';
-export { a as PuppeteerWebPage } from './page-dd5d0f7b.js';
+export { a as PuppeteerWebPage } from './page-ccbba1d9.js';
 import '@midscene/core';
 import '@midscene/shared/constants';
 import '@midscene/shared/fs';

package/dist/types/{tasks-876d60ec.d.ts → tasks-9af2ee2d.d.ts}

@@ -1,4 +1,4 @@
-import { W as WebPage } from './page-dd5d0f7b.js';
+import { W as WebPage } from './page-ccbba1d9.js';
 import Insight, { BaseElement, Rect, UIContext, PlanningAction, AIElementParseResponse, GroupedActionDump, ExecutionDump, AgentWaitForOpt, InsightExtractParam, InsightAssertionResponse, PlanningActionParamWaitFor, Executor } from '@midscene/core';
 import { NodeType } from '@midscene/shared/constants';
 import { getMidscenePkgInfo } from '@midscene/shared/fs';
@@ -10,11 +10,12 @@ declare class WebElementInfo implements BaseElement {
     center: [number, number];
     page: WebPage;
     id: string;
+    indexId: number;
     attributes: {
         nodeType: NodeType;
         [key: string]: string;
     };
-    constructor({ content, rect, page, locator, id, attributes, }: {
+    constructor({ content, rect, page, locator, id, attributes, indexId, }: {
         content: string;
         rect: Rect;
         page: WebPage;
@@ -24,6 +25,7 @@ declare class WebElementInfo implements BaseElement {
             nodeType: NodeType;
             [key: string]: string;
         };
+        indexId: number;
     });
 }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@midscene/web",
   "description": "An AI-powered automation SDK can control the page, perform assertions, and extract data in JSON format using natural language. See https://midscenejs.com/ for details.",
-  "version": "0.5.2-beta-20241010035503.0",
+  "version": "0.5.2",
   "jsnext:source": "./src/index.ts",
   "main": "./dist/lib/index.js",
   "types": "./dist/types/index.d.ts",
@@ -47,8 +47,8 @@
     "openai": "4.57.1",
     "inquirer": "10.1.5",
     "@xmldom/xmldom": "0.8.10",
-    "@midscene/core": "0.5.2-beta-20241010035503.0",
-    "@midscene/shared": "0.5.2-beta-20241010035503.0"
+    "@midscene/core": "0.5.2",
+    "@midscene/shared": "0.5.2"
   },
   "devDependencies": {
     "@modern-js/module-tools": "2.58.2",