@lov3kaizen/agentsea-surf 0.5.1

package/dist/index.mjs

@@ -0,0 +1,582 @@
+import {
+  BaseBackend,
+  CoordinateScaler,
+  DEFAULT_SURF_CONFIG,
+  DockerBackend,
+  LinuxBackend,
+  MacOSBackend,
+  PuppeteerBackend,
+  SecurityValidator,
+  SurfAgent,
+  VisionAnalyzer,
+  WindowsBackend,
+  clickInputSchema,
+  clickOutputSchema,
+  createBackend,
+  createNativeBackend,
+  cursorMoveInputSchema,
+  cursorMoveOutputSchema,
+  dragInputSchema,
+  dragOutputSchema,
+  keyPressInputSchema,
+  keyPressOutputSchema,
+  screenshotInputSchema,
+  screenshotOutputSchema,
+  scrollInputSchema,
+  scrollOutputSchema,
+  typeTextInputSchema,
+  typeTextOutputSchema,
+  waitInputSchema,
+  waitOutputSchema
+} from "./chunk-QCNRPRJB.mjs";
+
+// src/tools/screenshot.tool.ts
+import { serverTool } from "@lov3kaizen/agentsea-core";
+function createScreenshotTool(backend) {
+  return serverTool({
+    name: "computer_screenshot",
+    description: "Take a screenshot of the current screen or a specific region. Returns the image as base64 for vision analysis. Use this to see what is currently displayed on the screen.",
+    inputSchema: screenshotInputSchema,
+    outputSchema: screenshotOutputSchema,
+    execute: async (input) => {
+      const _startTime = Date.now();
+      try {
+        const result = await backend.screenshot({
+          region: input.region,
+          format: input.format,
+          quality: input.quality
+        });
+        return {
+          success: true,
+          base64: result.base64,
+          mimeType: result.mimeType,
+          width: result.dimensions.width,
+          height: result.dimensions.height,
+          scaleFactor: result.dimensions.scaleFactor
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        throw new Error(`Screenshot failed: ${errorMessage}`);
+      }
+    }
+  });
+}
+var screenshotToolDefinition = {
+  name: "computer_screenshot",
+  description: "Take a screenshot of the current screen or a specific region. Returns the image as base64 for vision analysis.",
+  inputSchema: screenshotInputSchema,
+  outputSchema: screenshotOutputSchema
+};
+
+// src/tools/click.tool.ts
+import { serverTool as serverTool2 } from "@lov3kaizen/agentsea-core";
+function createClickTool(backend) {
+  return serverTool2({
+    name: "computer_click",
+    description: "Click at specified screen coordinates. Supports single/double click, different mouse buttons (left, right, middle), and modifier keys (ctrl, alt, shift, meta). Use this to interact with buttons, links, and other clickable elements.",
+    inputSchema: clickInputSchema,
+    outputSchema: clickOutputSchema,
+    execute: async (input) => {
+      const startTime = Date.now();
+      const point = { x: input.x, y: input.y };
+      try {
+        const options = {
+          button: input.button,
+          holdMs: input.holdMs,
+          modifiers: input.modifiers
+        };
+        let result;
+        if (input.clickType === "double") {
+          result = await backend.doubleClick(point, options);
+        } else {
+          result = await backend.click(point, options);
+        }
+        return {
+          success: result.success,
+          x: input.x,
+          y: input.y,
+          action: input.clickType === "double" ? "doubleClick" : "click",
+          duration: Date.now() - startTime,
+          error: result.error
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        return {
+          success: false,
+          x: input.x,
+          y: input.y,
+          action: input.clickType === "double" ? "doubleClick" : "click",
+          duration: Date.now() - startTime,
+          error: `Click failed: ${errorMessage}`
+        };
+      }
+    }
+  });
+}
+var clickToolDefinition = {
+  name: "computer_click",
+  description: "Click at specified screen coordinates. Supports single/double click, different mouse buttons, and modifier keys.",
+  inputSchema: clickInputSchema,
+  outputSchema: clickOutputSchema
+};
+
+// src/tools/type-text.tool.ts
+import { serverTool as serverTool3 } from "@lov3kaizen/agentsea-core";
+function createTypeTextTool(backend) {
+  return serverTool3({
+    name: "computer_type",
+    description: "Type text at the current cursor position or at specified coordinates. Can optionally clear existing text first (useful for text fields). Use this to enter text into input fields, search boxes, editors, etc.",
+    inputSchema: typeTextInputSchema,
+    outputSchema: typeTextOutputSchema,
+    execute: async (input) => {
+      const startTime = Date.now();
+      try {
+        const options = {
+          point: input.x !== void 0 && input.y !== void 0 ? { x: input.x, y: input.y } : void 0,
+          delayMs: input.delayMs,
+          clearFirst: input.clearFirst
+        };
+        const result = await backend.typeText(input.text, options);
+        return {
+          success: result.success,
+          textLength: input.text.length,
+          duration: Date.now() - startTime,
+          error: result.error
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        return {
+          success: false,
+          textLength: input.text.length,
+          duration: Date.now() - startTime,
+          error: `Type text failed: ${errorMessage}`
+        };
+      }
+    }
+  });
+}
+var typeTextToolDefinition = {
+  name: "computer_type",
+  description: "Type text at the current cursor position or at specified coordinates. Can optionally clear existing text first.",
+  inputSchema: typeTextInputSchema,
+  outputSchema: typeTextOutputSchema
+};
+
+// src/tools/scroll.tool.ts
+import { serverTool as serverTool4 } from "@lov3kaizen/agentsea-core";
+function createScrollTool(backend) {
+  return serverTool4({
+    name: "computer_scroll",
+    description: "Scroll the screen in a specified direction (up, down, left, right) at given coordinates. Use this to navigate through long pages, lists, or documents.",
+    inputSchema: scrollInputSchema,
+    outputSchema: scrollOutputSchema,
+    execute: async (input) => {
+      const startTime = Date.now();
+      const point = { x: input.x, y: input.y };
+      try {
+        const result = await backend.scroll(
+          input.direction,
+          point,
+          {
+            amount: input.amount,
+            smooth: input.smooth
+          }
+        );
+        return {
+          success: result.success,
+          direction: input.direction,
+          amount: input.amount,
+          duration: Date.now() - startTime,
+          error: result.error
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        return {
+          success: false,
+          direction: input.direction,
+          amount: input.amount,
+          duration: Date.now() - startTime,
+          error: `Scroll failed: ${errorMessage}`
+        };
+      }
+    }
+  });
+}
+var scrollToolDefinition = {
+  name: "computer_scroll",
+  description: "Scroll the screen in a specified direction at given coordinates.",
+  inputSchema: scrollInputSchema,
+  outputSchema: scrollOutputSchema
+};
+
+// src/tools/drag.tool.ts
+import { serverTool as serverTool5 } from "@lov3kaizen/agentsea-core";
+function createDragTool(backend) {
+  return serverTool5({
+    name: "computer_drag",
+    description: "Drag from one point to another. Useful for drag-and-drop operations, selecting text, moving windows, resizing elements, or drawing.",
+    inputSchema: dragInputSchema,
+    outputSchema: dragOutputSchema,
+    execute: async (input) => {
+      const startTime = Date.now();
+      const from = { x: input.fromX, y: input.fromY };
+      const to = { x: input.toX, y: input.toY };
+      try {
+        const result = await backend.drag(from, to, {
+          button: input.button,
+          durationMs: input.durationMs
+        });
+        return {
+          success: result.success,
+          fromX: input.fromX,
+          fromY: input.fromY,
+          toX: input.toX,
+          toY: input.toY,
+          duration: Date.now() - startTime,
+          error: result.error
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        return {
+          success: false,
+          fromX: input.fromX,
+          fromY: input.fromY,
+          toX: input.toX,
+          toY: input.toY,
+          duration: Date.now() - startTime,
+          error: `Drag failed: ${errorMessage}`
+        };
+      }
+    }
+  });
+}
+var dragToolDefinition = {
+  name: "computer_drag",
+  description: "Drag from one point to another. Useful for drag-and-drop operations, selecting text, or moving windows.",
+  inputSchema: dragInputSchema,
+  outputSchema: dragOutputSchema
+};
+
+// src/tools/key-press.tool.ts
+import { serverTool as serverTool6 } from "@lov3kaizen/agentsea-core";
+function createKeyPressTool(backend) {
+  return serverTool6({
+    name: "computer_key",
+    description: "Press a keyboard key or key combination. Supports all standard keys (enter, escape, tab, backspace, delete, arrows, function keys, etc.) and modifier combinations (ctrl, alt, shift, meta/command). Use this for keyboard shortcuts, navigation, and special key inputs.",
+    inputSchema: keyPressInputSchema,
+    outputSchema: keyPressOutputSchema,
+    execute: async (input) => {
+      const startTime = Date.now();
+      try {
+        for (let i = 0; i < input.repeat; i++) {
+          const result = await backend.keyPress(
+            input.key,
+            input.modifiers
+          );
+          if (!result.success) {
+            return {
+              success: false,
+              key: input.key,
+              modifiers: input.modifiers,
+              repeat: i + 1,
+              duration: Date.now() - startTime,
+              error: result.error
+            };
+          }
+          if (input.holdMs && input.holdMs > 0) {
+            await new Promise((resolve) => setTimeout(resolve, input.holdMs));
+          }
+        }
+        return {
+          success: true,
+          key: input.key,
+          modifiers: input.modifiers,
+          repeat: input.repeat,
+          duration: Date.now() - startTime
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        return {
+          success: false,
+          key: input.key,
+          modifiers: input.modifiers,
+          repeat: input.repeat,
+          duration: Date.now() - startTime,
+          error: `Key press failed: ${errorMessage}`
+        };
+      }
+    }
+  });
+}
+var keyPressToolDefinition = {
+  name: "computer_key",
+  description: "Press a keyboard key or key combination. Supports all standard keys and modifier combinations.",
+  inputSchema: keyPressInputSchema,
+  outputSchema: keyPressOutputSchema
+};
+
+// src/tools/cursor-move.tool.ts
+import { serverTool as serverTool7 } from "@lov3kaizen/agentsea-core";
+function createCursorMoveTool(backend) {
+  return serverTool7({
+    name: "computer_cursor_move",
+    description: "Move the cursor to specified coordinates without clicking. Useful for hover effects, tooltip activation, or positioning before other actions.",
+    inputSchema: cursorMoveInputSchema,
+    outputSchema: cursorMoveOutputSchema,
+    execute: async (input) => {
+      const startTime = Date.now();
+      const point = { x: input.x, y: input.y };
+      try {
+        const result = await backend.moveCursor(point);
+        if (input.smooth && input.durationMs > 0) {
+          await new Promise((resolve) => setTimeout(resolve, input.durationMs));
+        }
+        return {
+          success: result.success,
+          x: input.x,
+          y: input.y,
+          duration: Date.now() - startTime,
+          error: result.error
+        };
+      } catch (error) {
+        const errorMessage = error instanceof Error ? error.message : "Unknown error";
+        return {
+          success: false,
+          x: input.x,
+          y: input.y,
+          duration: Date.now() - startTime,
+          error: `Cursor move failed: ${errorMessage}`
+        };
+      }
+    }
+  });
+}
+var cursorMoveToolDefinition = {
+  name: "computer_cursor_move",
+  description: "Move the cursor to specified coordinates without clicking. Useful for hover effects or positioning.",
+  inputSchema: cursorMoveInputSchema,
+  outputSchema: cursorMoveOutputSchema
+};
+
+// src/tools/wait.tool.ts
+import { serverTool as serverTool8 } from "@lov3kaizen/agentsea-core";
+function createWaitTool(backend) {
+  return serverTool8({
+    name: "computer_wait",
+    description: "Wait for a specified duration in milliseconds. Use this to wait for page loads, animations, network requests, or UI updates to complete before taking the next action.",
+    inputSchema: waitInputSchema,
+    outputSchema: waitOutputSchema,
+    execute: async (input) => {
+      const startTime = Date.now();
+      try {
+        await backend.wait(input.ms);
+        return {
+          success: true,
+          waitedMs: Date.now() - startTime,
+          reason: input.reason
+        };
+      } catch (error) {
+        const _errorMessage = error instanceof Error ? error.message : "Unknown error";
+        return {
+          success: false,
+          waitedMs: Date.now() - startTime,
+          reason: input.reason
+        };
+      }
+    }
+  });
+}
+var waitToolDefinition = {
+  name: "computer_wait",
+  description: "Wait for a specified duration. Useful for waiting for page loads, animations, or UI updates.",
+  inputSchema: waitInputSchema,
+  outputSchema: waitOutputSchema
+};
+
+// src/tools/index.ts
+function createSurfTools(backend) {
+  return {
+    screenshot: createScreenshotTool(backend),
+    click: createClickTool(backend),
+    typeText: createTypeTextTool(backend),
+    scroll: createScrollTool(backend),
+    drag: createDragTool(backend),
+    keyPress: createKeyPressTool(backend),
+    cursorMove: createCursorMoveTool(backend),
+    wait: createWaitTool(backend)
+  };
+}
+function createSurfToolsArray(backend) {
+  return [
+    createScreenshotTool(backend),
+    createClickTool(backend),
+    createTypeTextTool(backend),
+    createScrollTool(backend),
+    createDragTool(backend),
+    createKeyPressTool(backend),
+    createCursorMoveTool(backend),
+    createWaitTool(backend)
+  ];
+}
+var SURF_TOOL_NAMES = [
+  "computer_screenshot",
+  "computer_click",
+  "computer_type",
+  "computer_scroll",
+  "computer_drag",
+  "computer_key",
+  "computer_cursor_move",
+  "computer_wait"
+];
+
+// src/utils/image-utils.ts
+async function resizeImage(imageBuffer, targetWidth, targetHeight) {
+  try {
+    const sharp = await import("sharp");
+    return sharp.default(imageBuffer).resize(targetWidth, targetHeight, {
+      fit: "contain",
+      background: { r: 0, g: 0, b: 0, alpha: 0 }
+    }).toBuffer();
+  } catch {
+    return imageBuffer;
+  }
+}
+function imageToBase64(imageBuffer) {
+  return imageBuffer.toString("base64");
+}
+function base64ToImage(base64) {
+  return Buffer.from(base64, "base64");
+}
+async function getImageDimensions(imageBuffer) {
+  try {
+    const sharp = await import("sharp");
+    const metadata = await sharp.default(imageBuffer).metadata();
+    return {
+      width: metadata.width || 0,
+      height: metadata.height || 0
+    };
+  } catch {
+    return null;
+  }
+}
+async function cropImage(imageBuffer, region) {
+  try {
+    const sharp = await import("sharp");
+    return sharp.default(imageBuffer).extract({
+      left: region.x,
+      top: region.y,
+      width: region.width,
+      height: region.height
+    }).toBuffer();
+  } catch {
+    return imageBuffer;
+  }
+}
+async function convertImageFormat(imageBuffer, format, quality) {
+  try {
+    const sharp = await import("sharp");
+    let sharpInstance = sharp.default(imageBuffer);
+    switch (format) {
+      case "png":
+        sharpInstance = sharpInstance.png();
+        break;
+      case "jpeg":
+        sharpInstance = sharpInstance.jpeg({ quality: quality || 90 });
+        break;
+      case "webp":
+        sharpInstance = sharpInstance.webp({ quality: quality || 90 });
+        break;
+    }
+    return sharpInstance.toBuffer();
+  } catch {
+    return imageBuffer;
+  }
+}
+async function calculateImageHash(imageBuffer) {
+  try {
+    const sharp = await import("sharp");
+    const { data } = await sharp.default(imageBuffer).resize(8, 8, { fit: "fill" }).grayscale().raw().toBuffer({ resolveWithObject: true });
+    let sum = 0;
+    for (let i = 0; i < data.length; i++) {
+      sum += data[i];
+    }
+    const avg = sum / data.length;
+    let hash = "";
+    for (let i = 0; i < data.length; i++) {
+      hash += data[i] >= avg ? "1" : "0";
+    }
+    return hash;
+  } catch {
+    return "";
+  }
+}
+function compareImageHashes(hash1, hash2) {
+  if (hash1.length !== hash2.length || hash1.length === 0) {
+    return 0;
+  }
+  let matching = 0;
+  for (let i = 0; i < hash1.length; i++) {
+    if (hash1[i] === hash2[i]) {
+      matching++;
+    }
+  }
+  return matching / hash1.length * 100;
+}
+export {
+  BaseBackend,
+  CoordinateScaler,
+  DEFAULT_SURF_CONFIG,
+  DockerBackend,
+  LinuxBackend,
+  MacOSBackend,
+  PuppeteerBackend,
+  SURF_TOOL_NAMES,
+  SecurityValidator,
+  SurfAgent,
+  VisionAnalyzer,
+  WindowsBackend,
+  base64ToImage,
+  calculateImageHash,
+  clickInputSchema,
+  clickOutputSchema,
+  clickToolDefinition,
+  compareImageHashes,
+  convertImageFormat,
+  createBackend,
+  createClickTool,
+  createCursorMoveTool,
+  createDragTool,
+  createKeyPressTool,
+  createNativeBackend,
+  createScreenshotTool,
+  createScrollTool,
+  createSurfTools,
+  createSurfToolsArray,
+  createTypeTextTool,
+  createWaitTool,
+  cropImage,
+  cursorMoveInputSchema,
+  cursorMoveOutputSchema,
+  cursorMoveToolDefinition,
+  dragInputSchema,
+  dragOutputSchema,
+  dragToolDefinition,
+  getImageDimensions,
+  imageToBase64,
+  keyPressInputSchema,
+  keyPressOutputSchema,
+  keyPressToolDefinition,
+  resizeImage,
+  screenshotInputSchema,
+  screenshotOutputSchema,
+  screenshotToolDefinition,
+  scrollInputSchema,
+  scrollOutputSchema,
+  scrollToolDefinition,
+  typeTextInputSchema,
+  typeTextOutputSchema,
+  typeTextToolDefinition,
+  waitInputSchema,
+  waitOutputSchema,
+  waitToolDefinition
+};