@midscene/web 0.5.1 → 0.5.2-beta-20240929094445.0

package/dist/es/appium.js

@@ -1188,6 +1188,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1707,7 +1708,7 @@ function getNodeAttributes2(node) {
   }
   return attrs;
 }
-function getRect(attributes) {
+function getRect2(attributes) {
   var _a, _b, _c, _d;
   const x = Math.round(Number.parseFloat((_a = attributes.x) != null ? _a : "0"));
   const y = Math.round(Number.parseFloat((_b = attributes.y) != null ? _b : "0"));
@@ -1795,7 +1796,7 @@ function extractTextWithPosition2(initNode) {
   }
   function collectElementInfo(node) {
     const attributes = getNodeAttributes2(node);
-    const rect = getRect(attributes);
+    const rect = getRect2(attributes);
     const nodeHashId = midsceneGenerateHash(attributes.placeholder, rect);
     const text = validTextNodeContent(node);
     let nodeType;
@@ -1842,7 +1843,6 @@ function extractTextWithPosition2(initNode) {
         Math.round(rect.top + rect.height / 2)
       ],
       nodeType,
-      htmlNode: null,
       nodePath: ""
     };
     if (elementInfo.nodeType !== NodeType2.CONTAINER) {

package/dist/es/debug.js

@@ -49,6 +49,11 @@ async function generateExtractData(page, targetDir, saveImgType) {
     inputImgBase64
   });
   const resizeImgBase64 = await resizeImg(inputImgBase64);
+  const existingSnapshot = existsSync(snapshotJsonPath) ? JSON.parse(readFileSync(snapshotJsonPath, "utf-8")) : null;
+  if (existingSnapshot && JSON.stringify(existingSnapshot) === JSON.stringify(captureElementSnapshot)) {
+    console.log("skip save snapshot for ", targetDir);
+    return;
+  }
   if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
     writeFileSyncWithDir(
       snapshotJsonPath,
@@ -84,7 +89,7 @@ function generateTestDataPath(testDataName) {
   const modulePath = __require.resolve("@midscene/core").replace("dist/lib/index.js", "");
   const midsceneTestDataPath = path.join(
     modulePath,
-    `tests/ai/inspector/test-data/${testDataName}`
+    `tests/ai/evaluate/test-data/${testDataName}`
   );
   return midsceneTestDataPath;
 }

package/dist/es/index.js

@@ -1198,6 +1198,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1725,7 +1726,7 @@ var PlaywrightAiFixture = () => {
         async (taskPrompt, opts) => {
           return new Promise((resolve, reject) => {
             test.step(`ai - ${taskPrompt}`, async () => {
-              await page.waitForLoadState("networkidle");
+              await waitForNetworkIdle(page);
               const actionType = (opts == null ? void 0 : opts.type) || "action";
               const result = await agent.ai(taskPrompt, actionType);
               resolve(result);
@@ -1739,7 +1740,7 @@ var PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt) => {
         test.step(`aiAction - ${taskPrompt}`, async () => {
-          await page.waitForLoadState("networkidle");
+          await waitForNetworkIdle(page);
           await agent.aiAction(taskPrompt);
         });
       });
@@ -1750,7 +1751,7 @@ var PlaywrightAiFixture = () => {
       await use(async (demand) => {
         return new Promise((resolve, reject) => {
           test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             const result = await agent.aiQuery(demand);
             resolve(result);
           });
@@ -1763,7 +1764,7 @@ var PlaywrightAiFixture = () => {
       await use(async (assertion, errorMsg) => {
         return new Promise((resolve, reject) => {
           test.step(`aiAssert - ${assertion}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             await agent.aiAssert(assertion, errorMsg);
             resolve(null);
           });
@@ -1785,6 +1786,13 @@ var PlaywrightAiFixture = () => {
     }
   };
 };
+function waitForNetworkIdle(page) {
+  const timeout = 20 * 1e3;
+  return Promise.race([
+    page.waitForLoadState("networkidle"),
+    new Promise((resolve) => setTimeout(resolve, timeout))
+  ]);
+}
 
 // src/puppeteer/base-page.ts
 import { readFileSync as readFileSync3, writeFileSync } from "fs";
@@ -1817,8 +1825,7 @@ var Page = class {
     const path3 = getTmpFile("jpeg");
     await this.page.screenshot({
       path: path3,
-      type: "jpeg",
-      quality: 75
+      type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {
@@ -1963,7 +1970,7 @@ function getNodeAttributes2(node) {
   }
   return attrs;
 }
-function getRect(attributes) {
+function getRect2(attributes) {
   var _a, _b, _c, _d;
   const x = Math.round(Number.parseFloat((_a = attributes.x) != null ? _a : "0"));
   const y = Math.round(Number.parseFloat((_b = attributes.y) != null ? _b : "0"));
@@ -2051,7 +2058,7 @@ function extractTextWithPosition2(initNode) {
   }
   function collectElementInfo(node) {
     const attributes = getNodeAttributes2(node);
-    const rect = getRect(attributes);
+    const rect = getRect2(attributes);
     const nodeHashId = midsceneGenerateHash(attributes.placeholder, rect);
     const text = validTextNodeContent(node);
     let nodeType;
@@ -2098,7 +2105,6 @@ function extractTextWithPosition2(initNode) {
         Math.round(rect.top + rect.height / 2)
       ],
       nodeType,
-      htmlNode: null,
       nodePath: ""
     };
     if (elementInfo.nodeType !== NodeType2.CONTAINER) {
@@ -2323,6 +2329,11 @@ async function generateExtractData(page, targetDir, saveImgType) {
     inputImgBase64
   });
   const resizeImgBase64 = await resizeImg3(inputImgBase64);
+  const existingSnapshot = existsSync2(snapshotJsonPath) ? JSON.parse(readFileSync4(snapshotJsonPath, "utf-8")) : null;
+  if (existingSnapshot && JSON.stringify(existingSnapshot) === JSON.stringify(captureElementSnapshot)) {
+    console.log("skip save snapshot for ", targetDir);
+    return;
+  }
   if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
     writeFileSyncWithDir(
       snapshotJsonPath,

package/dist/es/playwright.js

@@ -672,6 +672,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1199,7 +1200,7 @@ var PlaywrightAiFixture = () => {
         async (taskPrompt, opts) => {
           return new Promise((resolve, reject) => {
             test.step(`ai - ${taskPrompt}`, async () => {
-              await page.waitForLoadState("networkidle");
+              await waitForNetworkIdle(page);
               const actionType = (opts == null ? void 0 : opts.type) || "action";
               const result = await agent.ai(taskPrompt, actionType);
               resolve(result);
@@ -1213,7 +1214,7 @@ var PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt) => {
         test.step(`aiAction - ${taskPrompt}`, async () => {
-          await page.waitForLoadState("networkidle");
+          await waitForNetworkIdle(page);
           await agent.aiAction(taskPrompt);
         });
       });
@@ -1224,7 +1225,7 @@ var PlaywrightAiFixture = () => {
       await use(async (demand) => {
         return new Promise((resolve, reject) => {
           test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             const result = await agent.aiQuery(demand);
             resolve(result);
           });
@@ -1237,7 +1238,7 @@ var PlaywrightAiFixture = () => {
       await use(async (assertion, errorMsg) => {
         return new Promise((resolve, reject) => {
           test.step(`aiAssert - ${assertion}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             await agent.aiAssert(assertion, errorMsg);
             resolve(null);
           });
@@ -1259,6 +1260,13 @@ var PlaywrightAiFixture = () => {
     }
   };
 };
+function waitForNetworkIdle(page) {
+  const timeout = 20 * 1e3;
+  return Promise.race([
+    page.waitForLoadState("networkidle"),
+    new Promise((resolve) => setTimeout(resolve, timeout))
+  ]);
+}
 
 // src/puppeteer/base-page.ts
 import { readFileSync as readFileSync3, writeFileSync } from "fs";
@@ -1291,8 +1299,7 @@ var Page = class {
     const path2 = getTmpFile("jpeg");
     await this.page.screenshot({
       path: path2,
-      type: "jpeg",
-      quality: 75
+      type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {

package/dist/es/puppeteer.js

@@ -669,6 +669,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1167,8 +1168,7 @@ var Page = class {
     const path2 = getTmpFile("jpeg");
     await this.page.screenshot({
       path: path2,
-      type: "jpeg",
-      quality: 75
+      type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {

package/dist/lib/appium.js

@@ -1184,6 +1184,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1695,7 +1696,7 @@ function getNodeAttributes2(node) {
   }
   return attrs;
 }
-function getRect(attributes) {
+function getRect2(attributes) {
   var _a, _b, _c, _d;
   const x = Math.round(Number.parseFloat((_a = attributes.x) != null ? _a : "0"));
   const y = Math.round(Number.parseFloat((_b = attributes.y) != null ? _b : "0"));
@@ -1783,7 +1784,7 @@ function extractTextWithPosition2(initNode) {
   }
   function collectElementInfo(node) {
     const attributes = getNodeAttributes2(node);
-    const rect = getRect(attributes);
+    const rect = getRect2(attributes);
     const nodeHashId = midsceneGenerateHash(attributes.placeholder, rect);
     const text = validTextNodeContent(node);
     let nodeType;
@@ -1830,7 +1831,6 @@ function extractTextWithPosition2(initNode) {
         Math.round(rect.top + rect.height / 2)
       ],
       nodeType,
-      htmlNode: null,
       nodePath: ""
     };
     if (elementInfo.nodeType !== import_constants2.NodeType.CONTAINER) {

package/dist/lib/debug.js

@@ -70,6 +70,11 @@ async function generateExtractData(page, targetDir, saveImgType) {
     inputImgBase64
   });
   const resizeImgBase64 = await (0, import_img.resizeImg)(inputImgBase64);
+  const existingSnapshot = (0, import_node_fs.existsSync)(snapshotJsonPath) ? JSON.parse((0, import_node_fs.readFileSync)(snapshotJsonPath, "utf-8")) : null;
+  if (existingSnapshot && JSON.stringify(existingSnapshot) === JSON.stringify(captureElementSnapshot)) {
+    console.log("skip save snapshot for ", targetDir);
+    return;
+  }
   if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
     writeFileSyncWithDir(
       snapshotJsonPath,
@@ -105,7 +110,7 @@ function generateTestDataPath(testDataName) {
   const modulePath = require.resolve("@midscene/core").replace("dist/lib/index.js", "");
   const midsceneTestDataPath = import_node_path.default.join(
     modulePath,
-    `tests/ai/inspector/test-data/${testDataName}`
+    `tests/ai/evaluate/test-data/${testDataName}`
   );
   return midsceneTestDataPath;
 }

package/dist/lib/index.js

@@ -1198,6 +1198,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1725,7 +1726,7 @@ var PlaywrightAiFixture = () => {
         async (taskPrompt, opts) => {
           return new Promise((resolve, reject) => {
             import_test.test.step(`ai - ${taskPrompt}`, async () => {
-              await page.waitForLoadState("networkidle");
+              await waitForNetworkIdle(page);
               const actionType = (opts == null ? void 0 : opts.type) || "action";
               const result = await agent.ai(taskPrompt, actionType);
               resolve(result);
@@ -1739,7 +1740,7 @@ var PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt) => {
         import_test.test.step(`aiAction - ${taskPrompt}`, async () => {
-          await page.waitForLoadState("networkidle");
+          await waitForNetworkIdle(page);
           await agent.aiAction(taskPrompt);
         });
       });
@@ -1750,7 +1751,7 @@ var PlaywrightAiFixture = () => {
       await use(async (demand) => {
         return new Promise((resolve, reject) => {
           import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             const result = await agent.aiQuery(demand);
             resolve(result);
           });
@@ -1763,7 +1764,7 @@ var PlaywrightAiFixture = () => {
       await use(async (assertion, errorMsg) => {
         return new Promise((resolve, reject) => {
           import_test.test.step(`aiAssert - ${assertion}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             await agent.aiAssert(assertion, errorMsg);
             resolve(null);
           });
@@ -1785,6 +1786,13 @@ var PlaywrightAiFixture = () => {
     }
   };
 };
+function waitForNetworkIdle(page) {
+  const timeout = 20 * 1e3;
+  return Promise.race([
+    page.waitForLoadState("networkidle"),
+    new Promise((resolve) => setTimeout(resolve, timeout))
+  ]);
+}
 
 // src/puppeteer/base-page.ts
 var import_node_fs3 = require("fs");
@@ -1817,8 +1825,7 @@ var Page = class {
     const path3 = (0, import_utils7.getTmpFile)("jpeg");
     await this.page.screenshot({
       path: path3,
-      type: "jpeg",
-      quality: 75
+      type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {
@@ -1955,7 +1962,7 @@ function getNodeAttributes2(node) {
   }
   return attrs;
 }
-function getRect(attributes) {
+function getRect2(attributes) {
   var _a, _b, _c, _d;
   const x = Math.round(Number.parseFloat((_a = attributes.x) != null ? _a : "0"));
   const y = Math.round(Number.parseFloat((_b = attributes.y) != null ? _b : "0"));
@@ -2043,7 +2050,7 @@ function extractTextWithPosition2(initNode) {
   }
   function collectElementInfo(node) {
     const attributes = getNodeAttributes2(node);
-    const rect = getRect(attributes);
+    const rect = getRect2(attributes);
     const nodeHashId = midsceneGenerateHash(attributes.placeholder, rect);
     const text = validTextNodeContent(node);
     let nodeType;
@@ -2090,7 +2097,6 @@ function extractTextWithPosition2(initNode) {
         Math.round(rect.top + rect.height / 2)
       ],
       nodeType,
-      htmlNode: null,
       nodePath: ""
     };
     if (elementInfo.nodeType !== import_constants2.NodeType.CONTAINER) {
@@ -2311,6 +2317,11 @@ async function generateExtractData(page, targetDir, saveImgType) {
     inputImgBase64
   });
   const resizeImgBase64 = await (0, import_img5.resizeImg)(inputImgBase64);
+  const existingSnapshot = (0, import_node_fs5.existsSync)(snapshotJsonPath) ? JSON.parse((0, import_node_fs5.readFileSync)(snapshotJsonPath, "utf-8")) : null;
+  if (existingSnapshot && JSON.stringify(existingSnapshot) === JSON.stringify(captureElementSnapshot)) {
+    console.log("skip save snapshot for ", targetDir);
+    return;
+  }
   if (!(saveImgType == null ? void 0 : saveImgType.disableSnapshot)) {
     writeFileSyncWithDir(
       snapshotJsonPath,

package/dist/lib/playwright.js

@@ -676,6 +676,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1203,7 +1204,7 @@ var PlaywrightAiFixture = () => {
         async (taskPrompt, opts) => {
           return new Promise((resolve, reject) => {
             import_test.test.step(`ai - ${taskPrompt}`, async () => {
-              await page.waitForLoadState("networkidle");
+              await waitForNetworkIdle(page);
               const actionType = (opts == null ? void 0 : opts.type) || "action";
               const result = await agent.ai(taskPrompt, actionType);
               resolve(result);
@@ -1217,7 +1218,7 @@ var PlaywrightAiFixture = () => {
       const agent = agentForPage(page, testInfo);
       await use(async (taskPrompt) => {
         import_test.test.step(`aiAction - ${taskPrompt}`, async () => {
-          await page.waitForLoadState("networkidle");
+          await waitForNetworkIdle(page);
           await agent.aiAction(taskPrompt);
         });
       });
@@ -1228,7 +1229,7 @@ var PlaywrightAiFixture = () => {
       await use(async (demand) => {
         return new Promise((resolve, reject) => {
           import_test.test.step(`aiQuery - ${JSON.stringify(demand)}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             const result = await agent.aiQuery(demand);
             resolve(result);
           });
@@ -1241,7 +1242,7 @@ var PlaywrightAiFixture = () => {
       await use(async (assertion, errorMsg) => {
         return new Promise((resolve, reject) => {
           import_test.test.step(`aiAssert - ${assertion}`, async () => {
-            await page.waitForLoadState("networkidle");
+            await waitForNetworkIdle(page);
             await agent.aiAssert(assertion, errorMsg);
             resolve(null);
           });
@@ -1263,6 +1264,13 @@ var PlaywrightAiFixture = () => {
     }
   };
 };
+function waitForNetworkIdle(page) {
+  const timeout = 20 * 1e3;
+  return Promise.race([
+    page.waitForLoadState("networkidle"),
+    new Promise((resolve) => setTimeout(resolve, timeout))
+  ]);
+}
 
 // src/puppeteer/base-page.ts
 var import_node_fs3 = require("fs");
@@ -1295,8 +1303,7 @@ var Page = class {
     const path2 = (0, import_utils7.getTmpFile)("jpeg");
     await this.page.screenshot({
       path: path2,
-      type: "jpeg",
-      quality: 75
+      type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {

package/dist/lib/puppeteer.js

@@ -672,6 +672,7 @@ var PageTaskExecutor = class {
             let locateResult;
             const callAI = this.insight.aiVendorFn;
             const element = await this.insight.locate(param.prompt, {
+              quickAnswer: plan2.quickAnswer,
               callAI: async (...message) => {
                 if (locateCache) {
                   locateResult = locateCache;
@@ -1170,8 +1171,7 @@ var Page = class {
     const path2 = (0, import_utils7.getTmpFile)("jpeg");
     await this.page.screenshot({
       path: path2,
-      type: "jpeg",
-      quality: 75
+      type: "png"
     });
     let buf;
     if (viewportSize.deviceScaleFactor > 1) {