@midscene/web 0.16.0 → 0.16.1

package/dist/es/puppeteer-agent-launcher.js

@@ -1,6 +1,6 @@
 // src/puppeteer/agent-launcher.ts
 import { readFileSync as readFileSync2 } from "fs";
-import { getDebug as getDebug6 } from "@midscene/shared/logger";
+import { getDebug as getDebug7 } from "@midscene/shared/logger";
 import { assert as assert8 } from "@midscene/shared/utils";
 
 // src/common/agent.ts
@@ -285,8 +285,8 @@ import {
   DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT,
   DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT
 } from "@midscene/shared/constants";
-import { vlLocateMode } from "@midscene/shared/env";
-import { getDebug as getDebug3 } from "@midscene/shared/logger";
+import { vlLocateMode as vlLocateMode2 } from "@midscene/shared/env";
+import { getDebug as getDebug4 } from "@midscene/shared/logger";
 import { assert as assert6 } from "@midscene/shared/utils";
 
 // src/common/tasks.ts
@@ -298,6 +298,14 @@ import {
   vlmPlanning
 } from "@midscene/core/ai-model";
 import { sleep } from "@midscene/core/utils";
+import { UITarsModelVersion } from "@midscene/shared/env";
+import { uiTarsModelVersion } from "@midscene/shared/env";
+import { vlLocateMode } from "@midscene/shared/env";
+import {
+  imageInfo,
+  resizeImgBase64 as resizeImgBase642
+} from "@midscene/shared/img";
+import { getDebug as getDebug2 } from "@midscene/shared/logger";
 import { assert as assert4 } from "@midscene/shared/utils";
 
 // src/common/task-cache.ts
@@ -400,7 +408,7 @@ function printReportMsg(filepath) {
 function replaceIllegalPathCharsAndSpace(str) {
   return str.replace(/[/\\:*?"<>| ]/g, "-");
 }
-function forceClosePopup(page, debug5) {
+function forceClosePopup(page, debug6) {
   page.on("popup", async (popup) => {
     if (!popup) {
       console.warn("got a popup event, but the popup is not ready yet, skip");
@@ -412,19 +420,19 @@ function forceClosePopup(page, debug5) {
       try {
         await popup.close();
       } catch (error) {
-        debug5(`failed to close popup ${url}, error: ${error}`);
+        debug6(`failed to close popup ${url}, error: ${error}`);
       }
     } else {
-      debug5(`popup is already closed, skip close ${url}`);
+      debug6(`popup is already closed, skip close ${url}`);
     }
     if (!page.isClosed()) {
       try {
         await page.goto(url);
       } catch (error) {
-        debug5(`failed to goto ${url}, error: ${error}`);
+        debug6(`failed to goto ${url}, error: ${error}`);
       }
     } else {
-      debug5(`page is already closed, skip goto ${url}`);
+      debug6(`page is already closed, skip goto ${url}`);
     }
   });
 }
@@ -706,6 +714,7 @@ function paramStr(task) {
 }
 
 // src/common/tasks.ts
+var debug2 = getDebug2("page-task-executor");
 var replanningCountLimit = 10;
 var isAndroidPage = (page) => {
   return page.pageType === "android";
@@ -1294,13 +1303,37 @@ var PageTaskExecutor = class {
         };
         executorContext.task.recorder = [recordItem];
         executorContext.task.pageContext = pageContext;
+        let imagePayload = pageContext.screenshotBase64;
+        if (vlLocateMode() === "vlm-ui-tars" && uiTarsModelVersion() === UITarsModelVersion.V1_5) {
+          const size = pageContext.size;
+          debug2("ui-tars-v1.5, will check image size", size);
+          const currentPixels = size.width * size.height;
+          const maxPixels = 16384 * 28 * 28;
+          if (currentPixels > maxPixels) {
+            const resizeFactor = Math.sqrt(maxPixels / currentPixels);
+            const newWidth = Math.floor(size.width * resizeFactor);
+            const newHeight = Math.floor(size.height * resizeFactor);
+            debug2(
+              "resize image",
+              imageInfo,
+              "new width",
+              newWidth,
+              "new height",
+              newHeight
+            );
+            imagePayload = await resizeImgBase642(imagePayload, {
+              width: newWidth,
+              height: newHeight
+            });
+          }
+        }
         this.appendConversationHistory({
           role: "user",
           content: [
             {
               type: "image_url",
               image_url: {
-                url: pageContext.screenshotBase64
+                url: imagePayload
               }
             }
           ]
@@ -1643,9 +1676,9 @@ var PageTaskExecutor = class {
 };
 
 // src/common/plan-builder.ts
-import { getDebug as getDebug2 } from "@midscene/shared/logger";
+import { getDebug as getDebug3 } from "@midscene/shared/logger";
 import { assert as assert5 } from "@midscene/shared/utils";
-var debug2 = getDebug2("plan-builder");
+var debug3 = getDebug3("plan-builder");
 function buildPlans(type, locateParam, param) {
   let returnPlans = [];
   const locatePlan = locateParam ? {
@@ -1707,14 +1740,14 @@ function buildPlans(type, locateParam, param) {
     returnPlans = [sleepPlan];
   }
   if (returnPlans) {
-    debug2("buildPlans", returnPlans);
+    debug3("buildPlans", returnPlans);
     return returnPlans;
   }
   throw new Error(`Not supported type: ${type}`);
 }
 
 // src/common/agent.ts
-var debug3 = getDebug3("web-integration");
+var debug4 = getDebug4("web-integration");
 var PageAgent = class {
   constructor(page, opts) {
     /**
@@ -1757,7 +1790,7 @@ var PageAgent = class {
       });
     }
     return await parseContextFromWebPage(this.page, {
-      ignoreMarker: !!vlLocateMode()
+      ignoreMarker: !!vlLocateMode2()
     });
   }
   async setAIActionContext(prompt) {
@@ -1792,7 +1825,7 @@ var PageAgent = class {
       type: "dump",
       generateReport
     });
-    debug3("writeOutActionDumps", this.reportFile);
+    debug4("writeOutActionDumps", this.reportFile);
     if (generateReport && autoPrintReportMsg && this.reportFile) {
       printReportMsg(this.reportFile);
     }
@@ -1896,7 +1929,7 @@ ${errorTask?.errorStack}`);
     return output;
   }
   async aiAction(taskPrompt) {
-    const { output, executor } = await (vlLocateMode() === "vlm-ui-tars" ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext));
+    const { output, executor } = await (vlLocateMode2() === "vlm-ui-tars" ? this.taskExecutor.actionToGoal(taskPrompt) : this.taskExecutor.action(taskPrompt, this.opts.aiActionContext));
     this.afterTaskRunning(executor);
     return output;
   }
@@ -1979,7 +2012,7 @@ ${errors}`);
 };
 
 // src/puppeteer/index.ts
-import { getDebug as getDebug5 } from "@midscene/shared/logger";
+import { getDebug as getDebug6 } from "@midscene/shared/logger";
 
 // src/puppeteer/page.ts
 import {
@@ -1994,9 +2027,9 @@ import { sleep as sleep2 } from "@midscene/core/utils";
 import { DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT as DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT2 } from "@midscene/shared/constants";
 import { treeToList as treeToList2 } from "@midscene/shared/extractor";
 import { getExtraReturnLogic } from "@midscene/shared/fs";
-import { getDebug as getDebug4 } from "@midscene/shared/logger";
+import { getDebug as getDebug5 } from "@midscene/shared/logger";
 import { assert as assert7 } from "@midscene/shared/utils";
-var debugPage = getDebug4("web:page");
+var debugPage = getDebug5("web:page");
 var Page = class {
   constructor(underlyingPage, pageType, opts) {
     this.everMoved = false;
@@ -2279,14 +2312,14 @@ var WebPage = class extends Page {
 
 // src/puppeteer/index.ts
 import { overrideAIConfig } from "@midscene/shared/env";
-var debug4 = getDebug5("puppeteer:agent");
+var debug5 = getDebug6("puppeteer:agent");
 var PuppeteerAgent = class extends PageAgent {
   constructor(page, opts) {
     const webPage = new WebPage(page);
     super(webPage, opts);
     const { forceSameTabNavigation = true } = opts ?? {};
     if (forceSameTabNavigation) {
-      forceClosePopup(page, debug4);
+      forceClosePopup(page, debug5);
     }
   }
 };
@@ -2299,7 +2332,7 @@ var defaultViewportWidth = 1440;
 var defaultViewportHeight = 768;
 var defaultViewportScale = process.platform === "darwin" ? 2 : 1;
 var defaultWaitForNetworkIdleTimeout = DEFAULT_WAIT_FOR_NETWORK_IDLE_TIMEOUT3;
-var launcherDebug = getDebug6("puppeteer:launcher");
+var launcherDebug = getDebug7("puppeteer:launcher");
 async function launchPuppeteerPage(target, preference) {
   assert8(target.url, "url is required");
   const freeFn = [];
@@ -2361,10 +2394,13 @@ async function launchPuppeteerPage(target, preference) {
     // add 200px for the address bar
   ];
   launcherDebug(
-    "launching browser with viewport, headed: %s, viewport: %j, args: %j, preference: %j",
+    "launching browser with viewport, headed",
     headed,
+    "viewport",
     viewportConfig,
+    "args",
     args,
+    "preference",
     preference
   );
   const browser = await puppeteer.launch({