misoai-web 1.5.6 → 1.5.7

package/dist/es/chrome-extension.js

@@ -45,7 +45,8 @@ var WebElementInfo = class {
     id,
     attributes,
     indexId,
-    xpaths
+    xpaths,
+    isVisible
   }) {
     this.content = content;
     this.rect = rect;
@@ -58,6 +59,7 @@ var WebElementInfo = class {
     this.attributes = attributes;
     this.indexId = indexId;
     this.xpaths = xpaths;
+    this.isVisible = isVisible;
   }
 };
 
@@ -80,14 +82,15 @@ async function parseContextFromWebPage(page, _opt) {
     })
   ]);
   const webTree = traverseTree(tree, (elementInfo) => {
-    const { rect, id, content, attributes, locator, indexId } = elementInfo;
+    const { rect, id, content, attributes, locator, indexId, isVisible } = elementInfo;
     return new WebElementInfo({
       rect,
       locator,
       id,
       content,
       attributes,
-      indexId
+      indexId,
+      isVisible
     });
   });
   assert(screenshotBase64, "screenshotBase64 is required");
@@ -118,7 +121,7 @@ function printReportMsg(filepath) {
 }
 var ERROR_CODE_NOT_IMPLEMENTED_AS_DESIGNED = "NOT_IMPLEMENTED_AS_DESIGNED";
 function replaceIllegalPathCharsAndSpace(str) {
-  return str.replace(/[/\\:*?"<>| ]/g, "-");
+  return str.replace(/[:*?"<>| ]/g, "-");
 }
 function matchElementFromPlan(planLocateParam, tree) {
   if (!planLocateParam) {
@@ -248,6 +251,10 @@ var ScriptPlayer = class {
       } else if ("aiQuery" in flowItem) {
         const queryTask = flowItem;
         const prompt = queryTask.aiQuery;
+        const options = {
+          domIncluded: queryTask.domIncluded,
+          screenshotIncluded: queryTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for aiQuery");
         assert2(
           typeof prompt === "string",
@@ -258,6 +265,10 @@ var ScriptPlayer = class {
       } else if ("aiNumber" in flowItem) {
         const numberTask = flowItem;
         const prompt = numberTask.aiNumber;
+        const options = {
+          domIncluded: numberTask.domIncluded,
+          screenshotIncluded: numberTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for number");
         assert2(
           typeof prompt === "string",
@@ -268,6 +279,10 @@ var ScriptPlayer = class {
       } else if ("aiString" in flowItem) {
         const stringTask = flowItem;
         const prompt = stringTask.aiString;
+        const options = {
+          domIncluded: stringTask.domIncluded,
+          screenshotIncluded: stringTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for string");
         assert2(
           typeof prompt === "string",
@@ -278,6 +293,10 @@ var ScriptPlayer = class {
       } else if ("aiBoolean" in flowItem) {
         const booleanTask = flowItem;
         const prompt = booleanTask.aiBoolean;
+        const options = {
+          domIncluded: booleanTask.domIncluded,
+          screenshotIncluded: booleanTask.screenshotIncluded
+        };
         assert2(prompt, "missing prompt for boolean");
         assert2(
           typeof prompt === "string",
@@ -320,6 +339,9 @@ var ScriptPlayer = class {
       } else if ("aiTap" in flowItem) {
         const tapTask = flowItem;
         await agent.aiTap(tapTask.aiTap, tapTask);
+      } else if ("aiRightClick" in flowItem) {
+        const rightClickTask = flowItem;
+        await agent.aiRightClick(rightClickTask.aiRightClick, rightClickTask);
       } else if ("aiHover" in flowItem) {
         const hoverTask = flowItem;
         await agent.aiHover(hoverTask.aiHover, hoverTask);
@@ -342,6 +364,11 @@ var ScriptPlayer = class {
           evaluateJavaScriptTask.javascript
         );
         this.setResult(evaluateJavaScriptTask.name, result);
+      } else if ("logScreenshot" in flowItem) {
+        const logScreenshotTask = flowItem;
+        await agent.logScreenshot(logScreenshotTask.logScreenshot, {
+          content: logScreenshotTask.content || ""
+        });
       } else {
         throw new Error(`unknown flowItem: ${JSON.stringify(flowItem)}`);
       }
@@ -872,10 +899,10 @@ var PageTaskExecutor = class {
               if (!taskParam || !taskParam.value) {
                 return;
               }
-              await this.page.keyboard.type(taskParam.value);
-            } else {
-              await this.page.keyboard.type(taskParam.value);
             }
+            await this.page.keyboard.type(taskParam.value, {
+              autoDismissKeyboard: taskParam.autoDismissKeyboard
+            });
           }
         };
         tasks.push(taskActionInput);
@@ -904,6 +931,22 @@ var PageTaskExecutor = class {
           }
         };
         tasks.push(taskActionTap);
+      } else if (plan2.type === "RightClick") {
+        const taskActionRightClick = {
+          type: "Action",
+          subType: "RightClick",
+          thought: plan2.thought,
+          locate: plan2.locate,
+          executor: async (param, { element }) => {
+            assert4(element, "Element not found, cannot right click");
+            await this.page.mouse.click(
+              element.center[0],
+              element.center[1],
+              { button: "right" }
+            );
+          }
+        };
+        tasks.push(taskActionRightClick);
       } else if (plan2.type === "Drag") {
         const taskActionDrag = {
           type: "Action",
@@ -1432,7 +1475,7 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async createTypeQueryTask(type, demand) {
+  async createTypeQueryTask(type, demand, opt) {
     const taskExecutor = new Executor(
       taskTitleStr(
         type,
@@ -1463,7 +1506,10 @@ var PageTaskExecutor = class {
             result: `${type}, ${demand}`
           };
         }
-        const { data, usage } = await this.insight.extract(demandInput);
+        const { data, usage } = await this.insight.extract(
+          demandInput,
+          opt
+        );
         let outputResult = data;
         if (ifTypeRestricted) {
           assert4(data?.result !== void 0, "No result in query data");
@@ -1483,17 +1529,17 @@ var PageTaskExecutor = class {
       executor: taskExecutor
     };
   }
-  async query(demand) {
-    return this.createTypeQueryTask("Query", demand);
+  async query(demand, opt) {
+    return this.createTypeQueryTask("Query", demand, opt);
   }
-  async boolean(prompt) {
-    return this.createTypeQueryTask("Boolean", prompt);
+  async boolean(prompt, opt) {
+    return this.createTypeQueryTask("Boolean", prompt, opt);
   }
-  async number(prompt) {
-    return this.createTypeQueryTask("Number", prompt);
+  async number(prompt, opt) {
+    return this.createTypeQueryTask("Number", prompt, opt);
   }
-  async string(prompt) {
-    return this.createTypeQueryTask("String", prompt);
+  async string(prompt, opt) {
+    return this.createTypeQueryTask("String", prompt, opt);
   }
   async assert(assertion) {
     const description = `assert: ${assertion}`;
@@ -1629,7 +1675,7 @@ function buildPlans(type, locateParam, param) {
     param: locateParam,
     thought: ""
   } : null;
-  if (type === "Tap" || type === "Hover") {
+  if (type === "Tap" || type === "Hover" || type === "RightClick") {
     assert5(locateParam, `missing locate info for action "${type}"`);
     assert5(locatePlan, `missing locate info for action "${type}"`);
     const tapPlan = {
@@ -1700,8 +1746,8 @@ function buildPlans(type, locateParam, param) {
 
 // src/common/task-cache.ts
 import assert6 from "assert";
-import { existsSync as existsSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
-import { join as join2 } from "path";
+import { existsSync as existsSync2, mkdirSync as mkdirSync2, readFileSync, writeFileSync as writeFileSync2 } from "fs";
+import { dirname as dirname2, join as join2 } from "path";
 import { getMidsceneRunSubDir as getMidsceneRunSubDir2 } from "misoai-shared/common";
 import { getDebug as getDebug3 } from "misoai-shared/logger";
 import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
@@ -1709,7 +1755,7 @@ import yaml3 from "js-yaml";
 import semver from "semver";
 
 // package.json
-var version = "1.5.6";
+var version = "1.0.3";
 
 // src/common/task-cache.ts
 var debug3 = getDebug3("cache");
@@ -1840,8 +1886,14 @@ cache file: ${cacheFile}`
       return;
     }
     try {
+      const dir = dirname2(this.cacheFilePath);
+      if (!existsSync2(dir)) {
+        mkdirSync2(dir, { recursive: true });
+        debug3("created cache directory: %s", dir);
+      }
       const yamlData = yaml3.dump(this.cache);
       writeFileSync2(this.cacheFilePath, yamlData);
+      debug3("cache flushed to file: %s", this.cacheFilePath);
     } catch (err) {
       debug3(
         "write cache to file failed, path: %s, error: %s",
@@ -2099,6 +2151,23 @@ var PageAgent = class {
       metadata
     };
   }
+  async aiRightClick(locatePrompt, opt) {
+    const detailedLocateParam = this.buildDetailedLocateParam(
+      locatePrompt,
+      opt
+    );
+    const plans = buildPlans("RightClick", detailedLocateParam);
+    const { executor, output } = await this.taskExecutor.runPlans(
+      taskTitleStr("RightClick", locateParamStr(detailedLocateParam)),
+      plans,
+      { cacheable: opt?.cacheable }
+    );
+    const metadata = this.afterTaskRunning(executor);
+    return {
+      result: output,
+      metadata
+    };
+  }
   async aiInput(value, locatePrompt, opt) {
     assert7(
       typeof value === "string",
@@ -2530,6 +2599,40 @@ ${errors}`);
     }
     throw new Error("evaluateJavaScript is not supported in current agent");
   }
+  async logScreenshot(title, options) {
+    const screenshotTitle = title || "untitled";
+    const content = options?.content || "";
+    const screenshot = await this.page.screenshotBase64?.();
+    if (screenshot) {
+      const executionDump = {
+        name: screenshotTitle,
+        description: content,
+        tasks: [{
+          type: "Screenshot",
+          subType: "log",
+          status: "finished",
+          executor: null,
+          param: {
+            title: screenshotTitle,
+            content
+          },
+          output: {
+            screenshot
+          },
+          thought: `Logged screenshot: ${screenshotTitle}`,
+          timing: {
+            start: Date.now(),
+            end: Date.now(),
+            cost: 0
+          }
+        }],
+        sdkVersion: "1.0.0",
+        logTime: Date.now(),
+        model_name: "screenshot"
+      };
+      this.appendExecutionDump(executionDump);
+    }
+  }
   async destroy() {
     await this.page.destroy();
   }
@@ -2752,7 +2855,7 @@ function sleep2(ms) {
 var ChromeExtensionProxyPage = class {
   constructor(forceSameTabNavigation) {
     this.pageType = "chrome-extension-proxy";
-    this.version = "1.5.6";
+    this.version = "1.0.3";
     this.activeTabId = null;
     this.tabIdOfDebuggerAttached = null;
     this.attachingDebugger = null;
@@ -2761,7 +2864,8 @@ var ChromeExtensionProxyPage = class {
     this.latestMouseX = 100;
     this.latestMouseY = 100;
     this.mouse = {
-      click: async (x, y) => {
+      click: async (x, y, options) => {
+        const { button = "left", count = 1 } = options || {};
         await this.mouse.move(x, y);
         if (this.isMobileEmulation === null) {
           const result = await this.sendCommandToDebugger("Runtime.evaluate", {
@@ -2772,7 +2876,7 @@ var ChromeExtensionProxyPage = class {
           });
           this.isMobileEmulation = result?.result?.value;
         }
-        if (this.isMobileEmulation) {
+        if (this.isMobileEmulation && button === "left") {
           const touchPoints = [{ x: Math.round(x), y: Math.round(y) }];
           await this.sendCommandToDebugger("Input.dispatchTouchEvent", {
             type: "touchStart",
@@ -2789,15 +2893,15 @@ var ChromeExtensionProxyPage = class {
             type: "mousePressed",
             x,
             y,
-            button: "left",
-            clickCount: 1
+            button,
+            clickCount: count
           });
           await this.sendCommandToDebugger("Input.dispatchMouseEvent", {
             type: "mouseReleased",
             x,
             y,
-            button: "left",
-            clickCount: 1
+            button,
+            clickCount: count
           });
         }
       },