@intuned/browser-use-dev 0.0.0-stage → 0.1.2-dev.0

package/dist/browser-use/playwrightComputer.js

@@ -0,0 +1,442 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.PlaywrightComputer = void 0;
+var _networkSettled = require("../common/networkSettled");
+var _Logger = require("../common/Logger");
+var _eventCollector = require("./eventCollector");
+const NETWORK_SETTLE_TIMEOUT_MS = 15_000;
+const PRE_CLICK_DELAY_MS = 500;
+const TYPE_DELAY_MS = 12;
+const SCROLL_STEP_DELAY_MS = 200;
+const CONSECUTIVE_SCROLL_WARNING_THRESHOLD = 5;
+const HOME_END_SCROLL_AMOUNT = 100;
+const HOME_END_SCROLL_PIXELS_PER_UNIT = 100;
+const HOME_END_SCROLL_STEP_DELAY_MS = 50;
+const MAX_WAIT_MS = 100_000;
+const MAX_ECHOED_VALUE = 120;
+const KEY_MAP = {
+  "/": "/",
+  "\\": "\\",
+  alt: "Alt",
+  arrowdown: "ArrowDown",
+  arrowleft: "ArrowLeft",
+  arrowright: "ArrowRight",
+  arrowup: "ArrowUp",
+  backspace: "Backspace",
+  capslock: "CapsLock",
+  cmd: "ControlOrMeta",
+  command: "ControlOrMeta",
+  control: "Control",
+  ctrl: "Control",
+  delete: "Delete",
+  down: "ArrowDown",
+  end: "End",
+  enter: "Enter",
+  esc: "Escape",
+  escape: "Escape",
+  home: "Home",
+  insert: "Insert",
+  left: "ArrowLeft",
+  meta: "Meta",
+  option: "Alt",
+  page_down: "PageDown",
+  page_up: "PageUp",
+  pagedown: "PageDown",
+  pageup: "PageUp",
+  return: "Enter",
+  right: "ArrowRight",
+  shift: "Shift",
+  space: " ",
+  super: "Meta",
+  tab: "Tab",
+  up: "ArrowUp",
+  win: "Meta"
+};
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
+const truncate = text => text.length > MAX_ECHOED_VALUE ? `${text.slice(0, MAX_ECHOED_VALUE)}…` : text;
+class PlaywrightComputer {
+  _mouseX = 0;
+  _mouseY = 0;
+  _consecutiveScrollCount = 0;
+  _lastScrollDirection = null;
+  _viewportWidth = 1280;
+  _viewportHeight = 720;
+  constructor(page) {
+    this._page = page;
+  }
+  syncViewport = async () => {
+    try {
+      const {
+        width,
+        height
+      } = await this._page.evaluate(() => ({
+        width: window.innerWidth,
+        height: window.innerHeight
+      }));
+      if (width > 0 && height > 0) {
+        await this._page.setViewportSize({
+          width,
+          height
+        });
+        this._viewportWidth = width;
+        this._viewportHeight = height;
+      }
+    } catch (e) {
+      _Logger.logger.warn(`[browser-use] failed to sync viewport: ${e}`);
+    }
+  };
+  runAction = async (fn, {
+    collectEvents = true
+  } = {}) => {
+    await this.syncViewport();
+    if (!collectEvents) {
+      const result = await fn();
+      let url = "unknown";
+      try {
+        url = this._page.url();
+      } catch {}
+      return `CURRENT URL: ${url}\nResult: ${result}`;
+    }
+    const collector = new _eventCollector.EventCollector(this._page);
+    collector.start();
+    try {
+      const result = await fn();
+      await collector.stop();
+      for (const event of collector.getEvents()) {
+        _Logger.logger.info(`[browser-use] event: ${event}`);
+      }
+      return collector.formatResponse(result);
+    } catch (e) {
+      await collector.stop().catch(() => undefined);
+      throw e;
+    }
+  };
+  settled = fn => (0, _networkSettled.withNetworkSettled)(fn, {
+    page: this._page,
+    timeoutInMs: NETWORK_SETTLE_TIMEOUT_MS
+  });
+  resetScrollStreak = () => {
+    this._consecutiveScrollCount = 0;
+    this._lastScrollDirection = null;
+  };
+  validateCoordinates = (x, y) => {
+    if (!Number.isFinite(x) || !Number.isFinite(y) || x < 0 || y < 0) {
+      throw new Error(`Coordinates (${x}, ${y}) must be non-negative numbers`);
+    }
+    if (x > this._viewportWidth || y > this._viewportHeight) {
+      throw new Error(`Coordinates (${x}, ${y}) are out of bounds — the viewport is ` + `${this._viewportWidth}x${this._viewportHeight} pixels`);
+    }
+  };
+  url = () => {
+    try {
+      return this._page.url();
+    } catch {
+      return "unknown";
+    }
+  };
+  screenshot = async () => {
+    await this.syncViewport();
+    await this._page.bringToFront();
+    const buf = await this._page.screenshot({
+      fullPage: false,
+      scale: "css",
+      timeout: 30_000
+    });
+    return buf.toString("base64");
+  };
+  click = ({
+    x,
+    y,
+    button = "left",
+    click_count = 1
+  }) => this.runAction(async () => {
+    this.resetScrollStreak();
+    switch (button) {
+      case "back":
+        return this.goBack();
+      case "forward":
+        return this.goForward();
+      case "wheel":
+        _Logger.logger.info(`[browser-use] scroll wheel by (${x}, ${y})`);
+        await this._page.mouse.wheel(x, y);
+        return `Scrolled wheel by (${x}, ${y})`;
+      default:
+        {
+          if (button !== "left" && click_count !== 1) {
+            throw new Error(`click_count is only supported for left clicks, not ${button}`);
+          }
+          if (![1, 2, 3].includes(click_count)) {
+            throw new Error(`click_count must be 1, 2, or 3, got ${click_count}`);
+          }
+          this.validateCoordinates(x, y);
+          this._mouseX = x;
+          this._mouseY = y;
+          const clickDesc = click_count === 1 ? `${button} click` : `${click_count}x ${button} click`;
+          _Logger.logger.info(`[browser-use] ${clickDesc} at (${x}, ${y})`);
+          await sleep(PRE_CLICK_DELAY_MS);
+          await this.settled(async p => {
+            if (button === "left" && click_count === 2) {
+              await p.mouse.dblclick(x, y);
+            } else {
+              await p.mouse.click(x, y, {
+                button,
+                clickCount: click_count
+              });
+            }
+          });
+          return `Clicked ${clickDesc} at (${x}, ${y})`;
+        }
+    }
+  });
+  double_click = ({
+    x,
+    y
+  }) => this.click({
+    x,
+    y,
+    button: "left",
+    click_count: 2
+  });
+  scrollSteps = async (amount, direction, pixelsPerUnit, delayBetweenStepsMs = SCROLL_STEP_DELAY_MS) => {
+    let deltaX = 0;
+    let deltaY = 0;
+    if (direction === "up") deltaY = -pixelsPerUnit;else if (direction === "down") deltaY = pixelsPerUnit;else if (direction === "left") deltaX = -pixelsPerUnit;else deltaX = pixelsPerUnit;
+    for (let i = 0; i < amount; i++) {
+      await sleep(delayBetweenStepsMs);
+      await this._page.mouse.wheel(deltaX, deltaY);
+    }
+    return `Scrolled ${direction} by ${amount}`;
+  };
+  scroll = ({
+    x,
+    y,
+    scroll_direction,
+    scroll_amount
+  }) => this.runAction(async () => {
+    if (!Number.isFinite(scroll_amount) || scroll_amount <= 0) {
+      throw new Error(`scroll_amount must be a positive number`);
+    }
+    if (x !== undefined && y !== undefined) {
+      this.validateCoordinates(x, y);
+      await this._page.mouse.move(x, y);
+      this._mouseX = x;
+      this._mouseY = y;
+    }
+    _Logger.logger.info(`[browser-use] scroll ${scroll_direction} by ${scroll_amount} at ` + `(${this._mouseX}, ${this._mouseY})`);
+    const pixelsPerUnit = Math.max(1, Math.floor(this._viewportHeight / 8));
+    let result = await this.scrollSteps(scroll_amount, scroll_direction, pixelsPerUnit);
+    if (scroll_direction !== this._lastScrollDirection) {
+      this._consecutiveScrollCount = 0;
+    }
+    this._lastScrollDirection = scroll_direction;
+    this._consecutiveScrollCount += 1;
+    if (this._consecutiveScrollCount >= CONSECUTIVE_SCROLL_WARNING_THRESHOLD) {
+      const warning = `⚠️ WARNING: You have performed ${this._consecutiveScrollCount} consecutive scroll operations.\n` + `- If skipping through the content is not an issue, consider using a bigger scroll_amount\n` + `- If content is essential to the task, keep scrolling\n`;
+      _Logger.logger.warn(`[browser-use] ${warning}`);
+      result = `${warning}\n${result}`;
+    }
+    return result;
+  }, {
+    collectEvents: false
+  });
+  type = ({
+    text
+  }) => this.runAction(async () => {
+    this.resetScrollStreak();
+    _Logger.logger.info(`[browser-use] type "${text}"`);
+    await this.settled(p => p.keyboard.type(text, {
+      delay: TYPE_DELAY_MS
+    }));
+    return `Typed ${text}`;
+  });
+  focusedValue = () => this._page.evaluate(() => {
+    const el = document.activeElement;
+    if (!el) return null;
+    if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) return el.value;
+    if (el.isContentEditable) return el.textContent ?? "";
+    return null;
+  });
+  clearFocused = async () => {
+    const p = this._page;
+    await p.keyboard.press("ControlOrMeta+a");
+    await p.keyboard.press("Delete");
+    let value = await this.focusedValue();
+    if (value) {
+      await p.evaluate(() => {
+        const el = document.activeElement;
+        try {
+          if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) el.select();else if (el !== null && el !== void 0 && el.isContentEditable) document.execCommand("selectAll");
+        } catch {}
+      });
+      await p.keyboard.press("Delete");
+      value = await this.focusedValue();
+    }
+    return value;
+  };
+  clear_field = ({
+    x,
+    y
+  }) => this.runAction(async () => {
+    this.resetScrollStreak();
+    this.validateCoordinates(x, y);
+    _Logger.logger.info(`[browser-use] clear the field at (${x}, ${y})`);
+    const left = await this.settled(async p => {
+      await p.mouse.click(x, y);
+      return this.clearFocused();
+    });
+    if (left === null) return `Clicked (${x}, ${y}) and sent select-all + delete, but no text field has focus there — check the screenshot`;
+    if (left) return `Tried to clear the field at (${x}, ${y}) but it still holds "${truncate(left)}"`;
+    return `Cleared the field at (${x}, ${y})`;
+  });
+  fill_field = ({
+    x,
+    y,
+    text
+  }) => this.runAction(async () => {
+    this.resetScrollStreak();
+    this.validateCoordinates(x, y);
+    _Logger.logger.info(`[browser-use] fill the field at (${x}, ${y}) with "${text}"`);
+    const value = await this.settled(async p => {
+      await p.mouse.click(x, y);
+      await this.clearFocused();
+      await p.keyboard.type(text, {
+        delay: TYPE_DELAY_MS
+      });
+      return this.focusedValue();
+    });
+    if (value === null) return `Clicked (${x}, ${y}) and typed "${text}", but no text field has focus there — check the screenshot`;
+    if (value === text) return `Set the field at (${x}, ${y}) to "${text}"`;
+    return `Typed "${text}" into the field at (${x}, ${y}); it now holds "${truncate(value)}"`;
+  });
+  keypress = ({
+    keys
+  }) => this.runAction(async () => {
+    this.resetScrollStreak();
+    if (!Array.isArray(keys) || keys.length === 0) {
+      throw new Error("keys must be a non-empty array");
+    }
+    if (keys.length === 1) {
+      const single = keys[0].toLowerCase();
+      if (single === "home" || single === "end") {
+        _Logger.logger.info(`[browser-use] press ${keys[0]} (slow scroll)`);
+        const direction = single === "home" ? "up" : "down";
+        await this.scrollSteps(HOME_END_SCROLL_AMOUNT, direction, HOME_END_SCROLL_PIXELS_PER_UNIT, HOME_END_SCROLL_STEP_DELAY_MS);
+        return `Pressed key ${keys[0]} (scrolled to ${direction === "up" ? "top" : "bottom"} with scroll amount of ${HOME_END_SCROLL_AMOUNT})`;
+      }
+    }
+    _Logger.logger.info(`[browser-use] press ${keys.join("+")}`);
+    const mapped = keys.map(k => KEY_MAP[k.toLowerCase()] ?? k);
+    await this.settled(async p => {
+      const modifiers = mapped.slice(0, -1);
+      const mainKey = mapped[mapped.length - 1];
+      for (const mod of modifiers) await p.keyboard.down(mod);
+      try {
+        await p.keyboard.press(mainKey);
+      } finally {
+        for (const mod of [...modifiers].reverse()) await p.keyboard.up(mod);
+      }
+    });
+    return `Pressed key ${keys.join("+")}`;
+  });
+  wait = ({
+    ms = 1000
+  } = {}) => this.runAction(async () => {
+    this.resetScrollStreak();
+    if (!Number.isFinite(ms) || ms < 0) {
+      throw new Error(`wait ms must be a non-negative number, got ${ms}`);
+    }
+    const capped = Math.min(ms, MAX_WAIT_MS);
+    _Logger.logger.info(`[browser-use] wait ${capped}ms`);
+    await sleep(capped);
+    return `Waited for ${capped}ms`;
+  }, {
+    collectEvents: false
+  });
+  move = ({
+    x,
+    y
+  }) => this.runAction(async () => {
+    this.resetScrollStreak();
+    this.validateCoordinates(x, y);
+    _Logger.logger.info(`[browser-use] move mouse to (${x}, ${y})`);
+    await this._page.mouse.move(x, y);
+    this._mouseX = x;
+    this._mouseY = y;
+    return `Moved mouse to (${x}, ${y})`;
+  }, {
+    collectEvents: false
+  });
+  drag = ({
+    path
+  }) => this.runAction(async () => {
+    this.resetScrollStreak();
+    const first = path[0];
+    if (!first) return "Nothing to drag — empty path";
+    const last = path[path.length - 1];
+    _Logger.logger.info(`[browser-use] drag from (${first.x}, ${first.y}) to (${last.x}, ${last.y})`);
+    await this.settled(async p => {
+      await p.mouse.move(first.x, first.y);
+      await p.mouse.down();
+      for (const pt of path.slice(1)) await p.mouse.move(pt.x, pt.y);
+      await p.mouse.up();
+    });
+    this._mouseX = last.x;
+    this._mouseY = last.y;
+    return `Dragged from (${first.x}, ${first.y}) to (${last.x}, ${last.y})`;
+  });
+  list_tabs = async () => {
+    const pages = this._page.context().pages();
+    const tabs = await Promise.all(pages.map(async (p, index) => ({
+      index,
+      url: p.url(),
+      title: await p.title().catch(() => ""),
+      is_current: p === this._page
+    })));
+    const current = tabs.find(t => t.is_current);
+    _Logger.logger.info(`[browser-use] list tabs: ${tabs.length} open, current: ${(current === null || current === void 0 ? void 0 : current.url) ?? "unknown"}`);
+    return {
+      tabs
+    };
+  };
+  executeJavascript = async code => {
+    this.resetScrollStreak();
+    _Logger.logger.info(`[browser-use] execute javascript: ${code.slice(0, 200)}${code.length > 200 ? "…" : ""}`);
+    const collector = new _eventCollector.EventCollector(this._page);
+    collector.start();
+    try {
+      const value = await this._page.evaluate(src => new Function(`return (async () => { ${src} })()`)(), code);
+      await collector.stop();
+      return {
+        value,
+        events: collector.getEvents()
+      };
+    } catch (e) {
+      await collector.stop().catch(() => undefined);
+      throw e;
+    }
+  };
+  goBack = async () => {
+    _Logger.logger.info("[browser-use] go back");
+    try {
+      await this._page.goBack();
+      return "Navigated back";
+    } catch {
+      _Logger.logger.info("[browser-use] cannot go back — no previous page in history");
+      return "Cannot go back — no previous page in history";
+    }
+  };
+  goForward = async () => {
+    _Logger.logger.info("[browser-use] go forward");
+    try {
+      await this._page.goForward();
+      return "Navigated forward";
+    } catch {
+      _Logger.logger.info("[browser-use] cannot go forward — no next page in history");
+      return "Cannot go forward — no next page in history";
+    }
+  };
+}
+exports.PlaywrightComputer = PlaywrightComputer;

package/dist/browser-use/prompts.js

@@ -0,0 +1,11 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.composeDefaultSystemPrompt = composeDefaultSystemPrompt;
+function composeDefaultSystemPrompt() {
+  const capability = ["<SYSTEM_CAPABILITY>", "- You are controlling a web browser already open on the current page, through screenshots and mouse/keyboard actions.", "- Every action waits for the page to settle (network requests to finish) before it returns, so you never need a wait after a click, a navigation or a form submit. Use wait only for animations and timers.", "- Actions return text, not a screenshot. Call the screenshot tool to see the page. scroll and wait already return a screenshot of the settled page, so don't add one after them.", "- Tool results report the current URL and any browser events the action triggered (file downloads, popup tabs opening or closing).", "</SYSTEM_CAPABILITY>"].join("\n");
+  const instructions = ["<IMPORTANT_INSTRUCTIONS>", "- You are a fully autonomous agent. Never ask the user for permission or confirmation.", "- Complete the task immediately and fully, then report what you did.", "- Act only on what you can actually see on screen — never invent values.", "- Always use parallel tool calls (several tool calls in one response) for actions that don't depend on each other's result, e.g. fill_field for every field of a form, click Submit, then screenshot. They run one after another in the order you list them and stop at the first failure.", "- When an action navigates or changes the page, call it together with screenshot in the same response: the screenshot will show the new state.", "- Use separate responses only when you need to see the screen between steps.", "</IMPORTANT_INSTRUCTIONS>"].join("\n");
+  return `${capability}\n\n${instructions}`;
+}