@intuned/browser-use-dev 0.0.0-stage → 0.1.2-dev.0

package/dist/browser-use/index.d.ts

@@ -0,0 +1,228 @@
+import { Page } from "playwright";
+import { ModelMessage } from "ai";
+
+/**
+ * A caller-supplied function tool the browser-use model can invoke during a
+ * run, alongside the built-in `computer`, `list_tabs`, `clear_field` and `fill_field`
+ * tools. The handler's return value is serialized and fed back to the model
+ * as the tool output ("success" when it returns nothing); a thrown error is
+ * fed back as an `error: ...` output the model can react to.
+ *
+ * @interface BrowserUseFunctionTool
+ */
+export interface BrowserUseFunctionTool {
+  /** Tool name the model calls it by. Must not collide with built-in tools. */
+  name: string;
+  /** What the tool does and when the model should use it. */
+  description: string;
+  /** JSON schema object describing the tool's arguments. */
+  parameters: Record<string, unknown>;
+  /** Executes the tool call with the model-provided arguments. */
+  handler: (args: Record<string, unknown>) => unknown | Promise<unknown>;
+}
+
+/**
+ * A debug frame captured immediately BEFORE a mouse action executes, with the
+ * action's target point(s) so a recording shows exactly where each action
+ * landed on the state the model acted on.
+ *
+ * @interface BrowserUseFrame
+ */
+export interface BrowserUseFrame {
+  /** PNG screenshot, one pixel per CSS pixel. */
+  image: Buffer;
+  /** Screenshot width in CSS pixels. */
+  width: number;
+  /** Screenshot height in CSS pixels. */
+  height: number;
+  /** The action about to run (e.g. "click", "drag", "clear_field"). */
+  action: string;
+  /** The action's target coordinate(s) in CSS pixels. */
+  markers: { x: number; y: number }[];
+  /** Extra action context (e.g. mouse button, scroll deltas). */
+  label: string;
+}
+
+/**
+ * Options for {@link executeTask}.
+ *
+ * @interface ExecuteTaskOptions
+ */
+export interface ExecuteTaskOptions {
+  /** The Playwright page the agent acts on. The agent stays pinned to this
+   * exact page for screenshots and actions (it never switches tabs). */
+  page: Page;
+  /** Model id to drive the run. Any vision-capable model id supported by the
+   * SDK's AI gateway (Anthropic, OpenAI, Google) — the provider is detected
+   * from the id. Defaults to "gpt-5.6-luna". */
+  model?: string;
+  /** Explicit API key for the detected provider. When omitted, the
+   * provider's env var (`ANTHROPIC_API_KEY` / `OPENAI_API_KEY` /
+   * `GOOGLE_API_KEY`) is used; when that is also absent, the Intuned AI
+   * gateway (via `@intuned/runtime`) provides credentials. */
+  apiKey?: string;
+  /** Extra function tools the model may call during the run. */
+  tools?: BrowserUseFunctionTool[];
+  /** Log each model message, tool call, and request cost. Defaults to true. */
+  logSteps?: boolean;
+  /** Expose the built-in `execute_javascript` tool, letting the model run
+   * arbitrary JavaScript in the page and get its result back — an escape
+   * hatch for cases the standard actions can't reach. Disabled by default. */
+  enableJavaScriptTool?: boolean;
+  /** The task for the agent, in plain English. Scope it tightly — say exactly
+   * what to act on and what outcome is expected. */
+  task: string;
+  /** Appended (after a blank line) to the built-in system prompt — the place
+   * for caller-specific rules the agent must follow at system priority. The
+   * built-in prompt (browser capability + viewport size + full-autonomy
+   * framing) is always present and never replaced, so callers only supply
+   * what's specific to them. */
+  systemPrompt?: string;
+  /** JSON schema for a structured result of the run. When provided, the
+   * model gets a reserved `submit_result` tool with these parameters and is
+   * instructed to call it once when the task is done; the submission is
+   * validated against the schema (an invalid one is sent back for the model
+   * to correct), and the validated object is returned as the result's
+   * `output`. If the model finishes without submitting, it is nudged once;
+   * `output` is null when it still doesn't submit. Reserves the tool name
+   * "submit_result". */
+  outputSchema?: Record<string, unknown>;
+  /** Model-call cap; the run throws when exceeded (runaway guard).
+   * Defaults to 100. */
+  maxTurns?: number;
+  /** Overall wall-clock cap for the run. On expiry the loop is aborted (no
+   * further page actions) and the returned promise rejects. */
+  timeoutInMs?: number;
+  /** External abort signal; aborting stops the loop between model calls and
+   * before each page action. */
+  signal?: AbortSignal;
+  /** Keep raw screenshots in the returned `messages` instead of replacing
+   * them with "[omitted]". Defaults to false. */
+  showImages?: boolean;
+}
+
+/**
+ * Result of a completed {@link executeTask} run.
+ *
+ * @interface BrowserUseTurnResult
+ */
+export interface BrowserUseTurnResult<TOutput = unknown> {
+  /** The final text of the model's last assistant message ("" when none). */
+  finalText: string;
+  /** The full run transcript (screenshots omitted unless `showImages`). */
+  messages: ModelMessage[];
+  /** The schema-validated structured result the model submitted via the
+   * `submit_result` tool (see `outputSchema`). Null when no `outputSchema`
+   * was given or the model never submitted a valid result. */
+  output: TOutput | null;
+  /** Summed per-request cost of the run in cents, read from the Intuned AI
+   * gateway's `x-ai-cost-in-cents` response header (0 when unavailable,
+   * e.g. when calling a provider directly). */
+  totalCostInCents: number;
+  /** Number of model calls the run made. */
+  turns: number;
+  /** Token usage summed over the run's model calls (input, output, and total tokens). */
+  usage: { inputTokens: number; outputTokens: number; totalTokens: number };
+}
+
+/**
+ * Run a browser-use agent (BETA) that drives a Playwright page: screenshots
+ * in, click/type/scroll actions out, executed on the exact page you pass in.
+ * Built on the AI SDK, so any vision-capable model the SDK's AI gateway
+ * supports (Anthropic, OpenAI, Google) can drive the browser — not just
+ * models with a provider-specific "computer use" tool.
+ *
+ * ## Features and limitations
+ * - Composes the system and user messages for you — pass a plain-English
+ *   `task` (and optionally extra system-priority rules via `systemPrompt`).
+ * - Structured results: pass an `outputSchema` (JSON schema) and the run
+ *   returns a schema-validated `output` object the model submits via a
+ *   reserved `submit_result` tool when the task is done.
+ * - Built-in function tools beyond the standard actions (click, double_click,
+ *   scroll, type, keypress, wait, move, drag): `screenshot` (see the page),
+ *   `list_tabs` (check whether a popup tab opened), `clear_field`
+ *   (deterministically empty an input) and `fill_field` (click, clear and
+ *   type into a field in one action). Add your own via `tools`.
+ * - Every action reports the browser events it triggered — file downloads,
+ *   popup tabs opening or closing — plus the current URL back to the model,
+ *   so out-of-viewport side effects aren't invisible.
+ * - Actions return text and run one at a time in the order the model issued
+ *   them; the agent calls `screenshot` to see the page (`scroll` and `wait`
+ *   return one). Screenshots are one pixel per CSS pixel, viewport-synced, so
+ *   model coordinates and mouse coordinates share one space — safe on HiDPI
+ *   headful browsers.
+ * - Actions wait for the network they trigger to settle, and scrolling is
+ *   slow and stepped so lazy-loaded content renders instead of being skipped.
+ * - Credentials resolve like the rest of the SDK: explicit `apiKey`, else
+ *   the provider's env var, else the Intuned AI gateway. Per-request gateway
+ *   cost is summed into the result's `totalCostInCents`.
+ * - BETA: the API surface may change between minor versions.
+ *
+ * @param {ExecuteTaskOptions} options - Task and agent configuration.
+ * @returns {Promise<BrowserUseTurnResult>} The completed run's result.
+ * @example
+ * ```typescript Run a task on the current page
+ * import { executeTask } from "@intuned/browser-use";
+ * import { BrowserContext, Page } from "playwright";
+ *
+ * interface Params {}
+ *
+ * export default async function handler(params: Params, page: Page, context: BrowserContext) {
+ *   await page.goto("https://sandbox.intuned.dev/infinite-scroll");
+ *   const result = await executeTask({
+ *     page,
+ *     task: "Scroll down until the year 2021 is visible and report the first 2021 entry's title.",
+ *     maxTurns: 20,
+ *     timeoutInMs: 120_000,
+ *   });
+ *   return { answer: result.finalText, costInCents: result.totalCostInCents };
+ * }
+ * ```
+ * @example
+ * ```typescript Custom tool and system prompt
+ * import { executeTask } from "@intuned/browser-use";
+ * import { BrowserContext, Page } from "playwright";
+ *
+ * interface Params {}
+ *
+ * export default async function handler(params: Params, page: Page, context: BrowserContext) {
+ *   const result = await executeTask({
+ *     page,
+ *     task: "Use the get_reference_code tool, type the code into the Lookup field, and click Search.",
+ *     systemPrompt:
+ *       "- Act ONLY on the current screen of the active tab. Never navigate to another URL.",
+ *     tools: [
+ *       {
+ *         name: "get_reference_code",
+ *         description: "Get the reference code to type into the lookup field.",
+ *         parameters: { type: "object", properties: {}, additionalProperties: false, required: [] },
+ *         handler: () => ({ code: "REF-1234" }),
+ *       },
+ *     ],
+ *   });
+ *   return { answer: result.finalText };
+ * }
+ * ```
+ */
+export declare function executeTask<TOutput = unknown>(
+  options: ExecuteTaskOptions
+): Promise<BrowserUseTurnResult<TOutput>>;
+
+/**
+ * Thrown when a run doesn't finish cleanly (timeout or maxTurns exceeded).
+ * Carries spend/progress up to the point of failure — the ONLY way to read
+ * it in these cases, since there is no persistent agent object to poll and
+ * no successful {@link BrowserUseTurnResult} to return.
+ *
+ * @class BrowserUseError
+ */
+export declare class BrowserUseError extends Error {
+  /** Accumulated cost (in cents) up to the point of failure. */
+  readonly totalCostInCents: number;
+  /** Model calls completed before the failure. */
+  readonly turns: number;
+  constructor(
+    message: string,
+    info: { totalCostInCents: number; turns: number }
+  );
+}

package/dist/browser-use/index.js

@@ -0,0 +1,19 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+Object.defineProperty(exports, "BrowserUseError", {
+  enumerable: true,
+  get: function () {
+    return _errors.BrowserUseError;
+  }
+});
+Object.defineProperty(exports, "executeTask", {
+  enumerable: true,
+  get: function () {
+    return _executeTask.executeTask;
+  }
+});
+var _executeTask = require("./executeTask");
+var _errors = require("./errors");

package/dist/browser-use/messages.js

@@ -0,0 +1,71 @@
+"use strict";
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.SCREENSHOT_OMITTED_TEXT = void 0;
+exports.pruneImages = pruneImages;
+const SCREENSHOT_OMITTED_TEXT = exports.SCREENSHOT_OMITTED_TEXT = "[screenshot omitted to save context]";
+const isImageMedia = item => item.type === "media" && item.mediaType.startsWith("image/");
+function pruneImages(messages, keep) {
+  if (keep <= 0) return messages;
+  let total = 0;
+  for (const message of messages) {
+    if (message.role === "user" && Array.isArray(message.content)) {
+      total += message.content.filter(part => part.type === "image").length;
+    } else if (message.role === "tool") {
+      for (const part of message.content) {
+        if (part.output.type === "content") total += part.output.value.filter(isImageMedia).length;
+      }
+    }
+  }
+  let toRemove = total - keep;
+  toRemove -= toRemove % keep;
+  if (toRemove <= 0) return messages;
+  return messages.map(message => {
+    if (toRemove <= 0) return message;
+    if (message.role === "user" && Array.isArray(message.content)) {
+      const content = message.content.map(part => {
+        if (part.type === "image" && toRemove > 0) {
+          toRemove -= 1;
+          return {
+            type: "text",
+            text: SCREENSHOT_OMITTED_TEXT
+          };
+        }
+        return part;
+      });
+      return {
+        ...message,
+        content
+      };
+    }
+    if (message.role === "tool") {
+      const content = message.content.map(part => {
+        if (part.output.type !== "content") return part;
+        const value = part.output.value.map(item => {
+          if (isImageMedia(item) && toRemove > 0) {
+            toRemove -= 1;
+            return {
+              type: "text",
+              text: SCREENSHOT_OMITTED_TEXT
+            };
+          }
+          return item;
+        });
+        return {
+          ...part,
+          output: {
+            type: "content",
+            value
+          }
+        };
+      });
+      return {
+        ...message,
+        content
+      };
+    }
+    return message;
+  });
+}