misoai-web 1.5.6 → 1.5.7

package/dist/lib/chrome-extension.js

@@ -78,7 +78,8 @@ var WebElementInfo = class {
     id,
     attributes,
     indexId,
-    xpaths
+    xpaths,
+    isVisible
   }) {
     this.content = content;
     this.rect = rect;
@@ -91,6 +92,7 @@ var WebElementInfo = class {
     this.attributes = attributes;
     this.indexId = indexId;
     this.xpaths = xpaths;
+    this.isVisible = isVisible;
   }
 };
 
@@ -113,14 +115,15 @@ async function parseContextFromWebPage(page, _opt) {
     })
   ]);
   const webTree = (0, import_extractor.traverseTree)(tree, (elementInfo) => {
-    const { rect, id, content, attributes, locator, indexId } = elementInfo;
+    const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
     return new WebElementInfo({
       rect,
       locator,
       id,
       content,
       attributes,
-      indexId
+      indexId,
+      isVisible
     });
   });
   (0, import_utils2.assert)(screenshotBase64, "screenshotBase64 is required");
@@ -151,7 +154,7 @@ function printReportMsg(filepath) {
 }
 var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
 function replaceIllegalPathCharsAndSpace(str) {
-  return str.replace(/[/\\:*?"<>| ]/g, "-");
+  return str.replace(/[:*?"<>| ]/g, "-");
 }
 function matchElementFromPlan(planLocateParam, tree) {
   if (!planLocateParam) {
@@ -281,6 +284,10 @@ var ScriptPlayer = class {
       } else if ("aiQuery" in flowItem) {
         const queryTask = flowItem;
         const prompt = queryTask.aiQuery;
+        const options = {
+          domIncluded: queryTask.domIncluded,
+          screenshotIncluded: queryTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for aiQuery");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -291,6 +298,10 @@ var ScriptPlayer = class {
       } else if ("aiNumber" in flowItem) {
         const numberTask = flowItem;
         const prompt = numberTask.aiNumber;
+        const options = {
+          domIncluded: numberTask.domIncluded,
+          screenshotIncluded: numberTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for number");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -301,6 +312,10 @@ var ScriptPlayer = class {
       } else if ("aiString" in flowItem) {
         const stringTask = flowItem;
         const prompt = stringTask.aiString;
+        const options = {
+          domIncluded: stringTask.domIncluded,
+          screenshotIncluded: stringTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for string");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -311,6 +326,10 @@ var ScriptPlayer = class {
       } else if ("aiBoolean" in flowItem) {
         const booleanTask = flowItem;
         const prompt = booleanTask.aiBoolean;
+        const options = {
+          domIncluded: booleanTask.domIncluded,
+          screenshotIncluded: booleanTask.screenshotIncluded
+        };
         (0, import_utils3.assert)(prompt, "missing prompt for boolean");
         (0, import_utils3.assert)(
           typeof prompt === "string",
@@ -353,6 +372,9 @@ var ScriptPlayer = class {
       } else if ("aiTap" in flowItem) {
         const tapTask = flowItem;
         await agent.aiTap(tapTask.aiTap, tapTask);
+      } else if ("aiRightClick" in flowItem) {
+        const rightClickTask = flowItem;
+        await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
       } else if ("aiHover" in flowItem) {
         const hoverTask = flowItem;
         await agent.aiHover(hoverTask.aiHover, hoverTask);
@@ -375,6 +397,11 @@ var ScriptPlayer = class {
           evaluateJavaScriptTask.javascript
         );
         this.setResult(evaluateJavaScriptTask.name, result);
+      } else if ("logScreenshot" in flowItem) {
+        const logScreenshotTask = flowItem;
+        await agent.logScreenshot(logScreenshotTask.logScreenshot, {
+          content: logScreenshotTask.content || ""
+        });
       } else {
         throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
       }
@@ -890,10 +917,10 @@ var PageTaskExecutor = class {
               if (!taskParam || !taskParam.value) {
                 return;
               }
-              await this.page.keyboard.type(taskParam.value);
-            } else {
-              await this.page.keyboard.type(taskParam.value);
             }
+            await this.page.keyboard.type(taskParam.value, {
+              autoDismissKeyboard: taskParam.autoDismissKeyboard
+            });
           }
         };
         tasks.push(taskActionInput);
@@ -922,6 +949,22 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionTap);
+      } else if (plan2.type === "RightClick") {
+        const taskActionRightClick = {
+          type: "Action",
+          subType: "RightClick",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            (0, import_utils6.assert)(element, "Element not found, cannot right click");
+            await this.page.mouse.click(
+              element.center[0],
+              element.center[1],
+              { button: "right" }
+            );
+          }
+        };
+        tasks.push(taskActionRightClick);
       } else if (plan2.type === "Drag") {
         const taskActionDrag = {
           type: "Action",
@@ -1450,7 +1493,7 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async createTypeQueryTask(type, demand) {
+  async createTypeQueryTask(type, demand, opt) {
     const taskExecutor = new import_misoai_core.Executor(
       taskTitleStr(
         type,
@@ -1481,7 +1524,10 @@ var PageTaskExecutor = class {
             result: `${type}, ${demand}`
           };
         }
-        const { data, usage } = await this.insight.extract(demandInput);
+        const { data, usage } = await this.insight.extract(
+          demandInput,
+          opt
+        );
         let outputResult = data;
         if (ifTypeRestricted) {
           (0, import_utils6.assert)(data?.result !== void 0, "No result in query data");
@@ -1501,17 +1547,17 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand) {
-    return this.createTypeQueryTask("Query", demand);
+  async query(demand, opt) {
+    return this.createTypeQueryTask("Query", demand, opt);
   }
-  async boolean(prompt) {
-    return this.createTypeQueryTask("Boolean", prompt);
+  async boolean(prompt, opt) {
+    return this.createTypeQueryTask("Boolean", prompt, opt);
   }
-  async number(prompt) {
-    return this.createTypeQueryTask("Number", prompt);
+  async number(prompt, opt) {
+    return this.createTypeQueryTask("Number", prompt, opt);
   }
-  async string(prompt) {
-    return this.createTypeQueryTask("String", prompt);
+  async string(prompt, opt) {
+    return this.createTypeQueryTask("String", prompt, opt);
   }
   async assert(assertion) {
     const description = `assert: ${assertion}`;
@@ -1647,7 +1693,7 @@ function buildPlans(type, locateParam, param) {
     param: locateParam,
     thought: ""
   } : null;
-  if (type === "Tap" || type === "Hover") {
+  if (type === "Tap" || type === "Hover" || type === "RightClick") {
     (0, import_utils8.assert)(locateParam, `missing locate info for action "${type}"`);
     (0, import_utils8.assert)(locatePlan, `missing locate info for action "${type}"`);
     const tapPlan = {
@@ -1727,7 +1773,7 @@ var import_js_yaml3 = __toESM(require("js-yaml"));
 var import_semver = __toESM(require("semver"));
 
 // package.json
-var version = "1.5.6";
+var version = "1.0.3";
 
 // src/common/task-cache.ts
 var debug3 = (0, import_logger3.getDebug)("cache");
@@ -1858,8 +1904,14 @@ cache file: ${cacheFile}`
       return;
     }
     try {
+      const dir = (0, import_node_path2.dirname)(this.cacheFilePath);
+      if (!(0, import_node_fs2.existsSync)(dir)) {
+        (0, import_node_fs2.mkdirSync)(dir, { recursive: true });
+        debug3("created cache directory: %s", dir);
+      }
       const yamlData = import_js_yaml3.default.dump(this.cache);
       (0, import_node_fs2.writeFileSync)(this.cacheFilePath, yamlData);
+      debug3("cache flushed to file: %s", this.cacheFilePath);
     } catch (err) {
       debug3(
         "write cache to file failed, path: %s, error: %s",
@@ -2117,6 +2169,23 @@ var PageAgent = class {
       metadata
     };
   }
+  async aiRightClick(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("RightClick", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
   async aiInput(value, locatePrompt, opt) {
     (0, import_utils12.assert)(
       typeof value === "string",
@@ -2548,6 +2617,40 @@ ${errors}`);
     }
     throw new Error("evaluateJavaScript is not supported in current agent");
   }
+  async logScreenshot(title, options) {
+    const screenshotTitle = title || "untitled";
+    const content = options?.content || "";
+    const screenshot = await this.page.screenshotBase64?.();
+    if (screenshot) {
+      const executionDump = {
+        name: screenshotTitle,
+        description: content,
+        tasks: [{
+          type: "Screenshot",
+          subType: "log",
+          status: "finished",
+          executor: null,
+          param: {
+            title: screenshotTitle,
+            content
+          },
+          output: {
+            screenshot
+          },
+          thought: `Logged screenshot: ${screenshotTitle}`,
+          timing: {
+            start: Date.now(),
+            end: Date.now(),
+            cost: 0
+          }
+        }],
+        sdkVersion: "1.0.0",
+        logTime: Date.now(),
+        model_name: "screenshot"
+      };
+      this.appendExecutionDump(executionDump);
+    }
+  }
   async destroy() {
     await this.page.destroy();
   }
@@ -2768,7 +2871,7 @@ function sleep2(ms) {
 var ChromeExtensionProxyPage = class {
   constructor(forceSameTabNavigation) {
     this.pageType = "chrome-extension-proxy";
-    this.version = "1.5.6";
+    this.version = "1.0.3";
     this.activeTabId = null;
     this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;
@@ -2777,7 +2880,8 @@ var ChromeExtensionProxyPage = class {
     this.latestMouseX = 100;
     this.latestMouseY = 100;
     this.mouse = {
-      click: async (x, y) => {
+      click: async (x, y, options) => {
+        const { button = "left", count = 1 } = options || {};
         await this.mouse.move(x, y);
         if (this.isMobileEmulation === null) {
           const result = await this.sendCommandToDebugger("Runtime.evaluate", {
@@ -2788,7 +2892,7 @@ var ChromeExtensionProxyPage = class {
           });
           this.isMobileEmulation = result?.result?.value;
         }
-        if (this.isMobileEmulation) {
+        if (this.isMobileEmulation && button === "left") {
           const touchPoints = [{ x: Math.round(x), y: Math.round(y) }];
           await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
             type: "touchStart",
@@ -2805,15 +2909,15 @@ var ChromeExtensionProxyPage = class {
             type: "mousePressed",
             x,
             y,
-            button: "left",
-            clickCount: 1
+            button,
+            clickCount: count
           });
           await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
             type: "mouseReleased",
             x,
             y,
-            button: "left",
-            clickCount: 1
+            button,
+            clickCount: count
           });
         }
       },