misoai-web 1.6.1 → 1.6.3

package/dist/es/puppeteer-agent-launcher.js

@@ -1285,6 +1285,7 @@ var PageTaskExecutor = class {
             actions: [],
             more_actions_needed_by_instruction: false,
             log: "",
+            summary: "Loaded YAML workflow configuration",
             yamlString
           },
           cache: {
@@ -1386,6 +1387,7 @@ var PageTaskExecutor = class {
             actions: finalActions,
             more_actions_needed_by_instruction,
             log: log2,
+            summary: planResult.summary || "Generated action plan from user instruction",
             yamlFlow: planResult.yamlFlow
           },
           cache: {
@@ -1441,6 +1443,7 @@ var PageTaskExecutor = class {
             actionType: actions[0].type,
             more_actions_needed_by_instruction: true,
             log: "",
+            summary: action_summary || "Generated VLM action plan",
             yamlFlow: planResult.yamlFlow
           },
           cache: {
@@ -2035,7 +2038,7 @@ import { ifInBrowser as ifInBrowser2 } from "misoai-shared/utils";
 import semver from "semver";
 
 // package.json
-var version = "1.6.1";
+var version = "1.6.3";
 
 // src/common/task-cache.ts
 var debug3 = getDebug3("cache");
@@ -3096,7 +3099,7 @@ ${errors}`);
   }
   countDataExtractions(memory) {
     return memory.filter(
-      (item) => item.context?.dataExtracted || item.taskType === "Insight" && item.summary.includes("Extracted")
+      (item) => item.context?.dataExtracted || item.taskType === "Query" && item.summary.includes("Extracted")
     ).length;
   }
   extractWorkflowSteps(memory) {
@@ -3134,13 +3137,236 @@ import {
 } from "misoai-shared/fs";
 import { getDebug as getDebug5 } from "misoai-shared/logger";
 import { assert as assert8 } from "misoai-shared/utils";
+
+// src/common/mouse-utils.ts
+function generateHumanMousePath(from, to, options = {}) {
+  const {
+    steps = 20,
+    easing = "easeInOut"
+  } = options;
+  if (steps <= 1) {
+    return [to];
+  }
+  const path = [];
+  const controlPoint1 = {
+    x: from.x + (to.x - from.x) * 0.25 + (Math.random() - 0.5) * 50,
+    y: from.y + (to.y - from.y) * 0.25 + (Math.random() - 0.5) * 50
+  };
+  const controlPoint2 = {
+    x: from.x + (to.x - from.x) * 0.75 + (Math.random() - 0.5) * 50,
+    y: from.y + (to.y - from.y) * 0.75 + (Math.random() - 0.5) * 50
+  };
+  for (let i = 0; i <= steps; i++) {
+    const t = i / steps;
+    const easedT = applyEasing(t, easing);
+    const point = calculateBezierPoint(from, controlPoint1, controlPoint2, to, easedT);
+    path.push(point);
+  }
+  return path;
+}
+function calculateBezierPoint(p0, p1, p2, p3, t) {
+  const oneMinusT = 1 - t;
+  const oneMinusTSquared = oneMinusT * oneMinusT;
+  const oneMinusTCubed = oneMinusTSquared * oneMinusT;
+  const tSquared = t * t;
+  const tCubed = tSquared * t;
+  return {
+    x: oneMinusTCubed * p0.x + 3 * oneMinusTSquared * t * p1.x + 3 * oneMinusT * tSquared * p2.x + tCubed * p3.x,
+    y: oneMinusTCubed * p0.y + 3 * oneMinusTSquared * t * p1.y + 3 * oneMinusT * tSquared * p2.y + tCubed * p3.y
+  };
+}
+function applyEasing(t, easing) {
+  switch (easing) {
+    case "linear":
+      return t;
+    case "easeIn":
+      return t * t;
+    case "easeOut":
+      return 1 - (1 - t) * (1 - t);
+    case "easeInOut":
+      return t < 0.5 ? 2 * t * t : 1 - Math.pow(-2 * t + 2, 2) / 2;
+    default:
+      return t;
+  }
+}
+function calculateStepDelay(totalDuration, currentStep, totalSteps) {
+  const baseDelay = totalDuration / totalSteps;
+  const variation = 0.3;
+  const randomFactor = 1 + (Math.random() - 0.5) * variation;
+  return Math.max(1, baseDelay * randomFactor);
+}
+
+// src/common/visual-feedback.ts
+function getMousePointerScript(x, y, options = {}) {
+  const {
+    pointerSize = 20,
+    pointerColor = "#ff4444",
+    trailLength = 5,
+    showTrail = true,
+    animationDuration = 200
+  } = options;
+  return `
+    (() => {
+      // Create or update mouse pointer
+      if (!window.misoaiMousePointer) {
+        const pointer = document.createElement('div');
+        pointer.id = 'misoai-mouse-pointer';
+        pointer.style.cssText = \`
+          position: fixed;
+          width: ${pointerSize}px;
+          height: ${pointerSize}px;
+          background: ${pointerColor};
+          border: 2px solid white;
+          border-radius: 50%;
+          pointer-events: none;
+          z-index: 999999;
+          box-shadow: 0 0 10px rgba(0,0,0,0.3);
+          transition: all ${animationDuration}ms ease-out;
+          transform: translate(-50%, -50%);
+        \`;
+        document.body.appendChild(pointer);
+        window.misoaiMousePointer = pointer;
+        
+        // Initialize trail array
+        window.misoaiMouseTrail = [];
+      }
+      
+      const pointer = window.misoaiMousePointer;
+      pointer.style.left = ${x}px;
+      pointer.style.top = ${y}px;
+      
+      ${showTrail ? getTrailScript(x, y, trailLength, pointerColor) : ""}
+      
+      // Auto-hide after 3 seconds of inactivity
+      clearTimeout(window.misoaiMouseTimeout);
+      pointer.style.opacity = '1';
+      window.misoaiMouseTimeout = setTimeout(() => {
+        if (pointer) pointer.style.opacity = '0.3';
+      }, 3000);
+    })();
+  `;
+}
+function getTrailScript(x, y, trailLength, color) {
+  return `
+    // Add current position to trail
+    window.misoaiMouseTrail.push({ x: ${x}, y: ${y}, timestamp: Date.now() });
+    
+    // Keep only recent trail points
+    if (window.misoaiMouseTrail.length > ${trailLength}) {
+      window.misoaiMouseTrail.shift();
+    }
+    
+    // Remove old trail elements
+    document.querySelectorAll('.misoai-mouse-trail').forEach(el => el.remove());
+    
+    // Create new trail elements
+    window.misoaiMouseTrail.forEach((point, index) => {
+      if (index === window.misoaiMouseTrail.length - 1) return; // Skip current position
+      
+      const trailElement = document.createElement('div');
+      trailElement.className = 'misoai-mouse-trail';
+      const opacity = (index + 1) / window.misoaiMouseTrail.length * 0.6;
+      const size = 8 + (index + 1) / window.misoaiMouseTrail.length * 8;
+      
+      trailElement.style.cssText = \`
+        position: fixed;
+        width: \${size}px;
+        height: \${size}px;
+        background: ${color};
+        border-radius: 50%;
+        pointer-events: none;
+        z-index: 999998;
+        opacity: \${opacity};
+        transform: translate(-50%, -50%);
+        left: \${point.x}px;
+        top: \${point.y}px;
+      \`;
+      
+      document.body.appendChild(trailElement);
+      
+      // Auto-remove trail element after animation
+      setTimeout(() => {
+        if (trailElement.parentNode) {
+          trailElement.parentNode.removeChild(trailElement);
+        }
+      }, 1000);
+    });
+  `;
+}
+function getClickAnimationScript(x, y) {
+  return `
+    (() => {
+      const clickRipple = document.createElement('div');
+      clickRipple.style.cssText = \`
+        position: fixed;
+        left: ${x}px;
+        top: ${y}px;
+        width: 0;
+        height: 0;
+        border: 2px solid #ff4444;
+        border-radius: 50%;
+        pointer-events: none;
+        z-index: 999999;
+        transform: translate(-50%, -50%);
+        animation: misoai-click-ripple 0.6s ease-out forwards;
+      \`;
+      
+      // Add CSS animation if not exists
+      if (!document.getElementById('misoai-click-styles')) {
+        const style = document.createElement('style');
+        style.id = 'misoai-click-styles';
+        style.textContent = \`
+          @keyframes misoai-click-ripple {
+            0% {
+              width: 0;
+              height: 0;
+              opacity: 1;
+            }
+            100% {
+              width: 40px;
+              height: 40px;
+              opacity: 0;
+            }
+          }
+        \`;
+        document.head.appendChild(style);
+      }
+      
+      document.body.appendChild(clickRipple);
+      
+      // Remove after animation
+      setTimeout(() => {
+        if (clickRipple.parentNode) {
+          clickRipple.parentNode.removeChild(clickRipple);
+        }
+      }, 600);
+    })();
+  `;
+}
+
+// src/puppeteer/base-page.ts
 var debugPage = getDebug5("web:page");
 var Page = class {
   constructor(underlyingPage, pageType, opts) {
+    this.currentMousePosition = { x: 0, y: 0 };
+    this.mouseMovementOptions = {
+      steps: 15,
+      duration: 300,
+      easing: "easeInOut",
+      showVisualFeedback: true
+    };
+    this.visualFeedbackOptions = {
+      pointerSize: 16,
+      pointerColor: "#ff4444",
+      trailLength: 4,
+      showTrail: true,
+      animationDuration: 150
+    };
     this.everMoved = false;
     this.underlyingPage = underlyingPage;
     this.pageType = pageType;
     this.waitForNavigationTimeout = opts?.waitForNavigationTimeout ?? DEFAULT_WAIT_FOR_NAVIGATION_TIMEOUT2;
+    this.initializeMousePosition();
   }
   async evaluate(pageFunction, arg) {
     let result;
@@ -3159,9 +3385,76 @@ var Page = class {
     debugPage("evaluate function end");
     return result;
   }
+  /**
+   * Initialize mouse position to center of viewport
+   */
+  async initializeMousePosition() {
+    try {
+      const size = await this.size();
+      this.currentMousePosition = {
+        x: Math.floor(size.width / 2),
+        y: Math.floor(size.height / 2)
+      };
+    } catch (error) {
+      this.currentMousePosition = { x: 400, y: 300 };
+    }
+  }
   async evaluateJavaScript(script) {
     return this.evaluate(script);
   }
+  /**
+   * Show visual feedback for mouse pointer
+   */
+  async showMousePointer(x, y) {
+    if (!this.mouseMovementOptions.showVisualFeedback)
+      return;
+    const script = getMousePointerScript(x, y, this.visualFeedbackOptions);
+    try {
+      await this.evaluate(script);
+    } catch (error) {
+      debugPage("Visual feedback error:", error);
+    }
+  }
+  /**
+   * Show click animation
+   */
+  async showClickAnimation(x, y) {
+    if (!this.mouseMovementOptions.showVisualFeedback)
+      return;
+    const script = getClickAnimationScript(x, y);
+    try {
+      await this.evaluate(script);
+    } catch (error) {
+      debugPage("Click animation error:", error);
+    }
+  }
+  /**
+   * Move mouse with human-like movement and visual feedback
+   */
+  async moveMouseHumanLike(toX, toY, options) {
+    const moveOptions = { ...this.mouseMovementOptions, ...options };
+    const path = generateHumanMousePath(
+      this.currentMousePosition,
+      { x: toX, y: toY },
+      moveOptions
+    );
+    for (let i = 0; i < path.length; i++) {
+      const point = path[i];
+      if (moveOptions.showVisualFeedback) {
+        await this.showMousePointer(point.x, point.y);
+      }
+      await this.underlyingPage.mouse.move(point.x, point.y);
+      this.currentMousePosition = point;
+      if (i < path.length - 1) {
+        const delay = calculateStepDelay(
+          moveOptions.duration || 300,
+          i,
+          path.length
+        );
+        await sleep2(delay);
+      }
+    }
+  }
   async waitForNavigation() {
     if (this.pageType === "puppeteer" || this.pageType === "playwright") {
       debugPage("waitForNavigation begin");
@@ -3251,7 +3544,8 @@ var Page = class {
     return {
       click: async (x, y, options) => {
         await this.mouse.move(x, y);
-        this.underlyingPage.mouse.click(x, y, {
+        await this.showClickAnimation(x, y);
+        await this.underlyingPage.mouse.click(x, y, {
           button: options?.button || "left",
           count: options?.count || 1
         });
@@ -3271,27 +3565,23 @@ var Page = class {
       },
       move: async (x, y) => {
         this.everMoved = true;
-        return this.underlyingPage.mouse.move(x, y);
+        await this.moveMouseHumanLike(x, y);
+        return Promise.resolve();
       },
       drag: async (from, to) => {
         if (this.pageType === "puppeteer") {
-          await this.underlyingPage.mouse.drag(
-            {
-              x: from.x,
-              y: from.y
-            },
-            {
-              x: to.x,
-              y: to.y
-            }
-          );
+          await this.mouse.move(from.x, from.y);
+          await this.underlyingPage.mouse.down();
+          await this.moveMouseHumanLike(to.x, to.y, {
+            duration: 500,
+            // Slower for drag operations
+            steps: 25
+          });
+          await this.underlyingPage.mouse.up();
         } else if (this.pageType === "playwright") {
-          await this.underlyingPage.mouse.move(
-            from.x,
-            from.y
-          );
+          await this.mouse.move(from.x, from.y);
           await this.underlyingPage.mouse.down();
-          await this.underlyingPage.mouse.move(to.x, to.y);
+          await this.mouse.move(to.x, to.y);
           await this.underlyingPage.mouse.up();
         }
       }
@@ -3354,6 +3644,18 @@ var Page = class {
       await this.mouse.move(targetX, targetY);
     }
   }
+  /**
+   * Configure mouse movement behavior
+   */
+  configureMouseMovement(options) {
+    this.mouseMovementOptions = { ...this.mouseMovementOptions, ...options };
+  }
+  /**
+   * Configure visual feedback
+   */
+  configureVisualFeedback(options) {
+    this.visualFeedbackOptions = { ...this.visualFeedbackOptions, ...options };
+  }
   async scrollUntilTop(startingPoint) {
     await this.moveToPointBeforeScroll(startingPoint);
     return this.mouse.wheel(0, -9999999);