@agentium/browser 2.0.7 → 2.1.0

package/dist/index.d.cts

@@ -1,5 +1,5 @@
 import * as _agentium_core from '@agentium/core';
-import { ModelProvider, UnifiedMemoryConfig, CostTracker, LogLevel, EventBus, MemoryManager, ToolDef } from '@agentium/core';
+import { ModelProvider, UnifiedMemoryConfig, ToolDef, CostTracker, LogLevel, EventBus, MemoryManager } from '@agentium/core';
 
 /**
  * Secure credential store for BrowserAgent.
@@ -38,20 +38,45 @@ declare class CredentialVault {
     mask(text: string): string;
 }
 
+/**
+ * The complete set of actions a `BrowserAgent` can take during a run.
+ *
+ * Click / type / scroll accept EITHER an `index` (preferred — resolved
+ * against the DOM tree we showed the model) OR `x`/`y` coordinates as a
+ * fallback. Indexed actions are far more reliable on dynamic pages because
+ * they survive layout shifts, devicePixelRatio mismatches, and "the
+ * sibling 4 pixels away" failure modes.
+ */
 type BrowserAction = {
     action: "click";
-    x: number;
-    y: number;
-    description: string;
+    /** Element index from the DOM snapshot. Preferred. */
+    index?: number;
+    /** Fallback x coordinate (CSS pixels). */
+    x?: number;
+    /** Fallback y coordinate (CSS pixels). */
+    y?: number;
+    /** Free-form description; if it contains a quoted label, used as a text-locator fallback. */
+    description?: string;
 } | {
     action: "type";
+    /** Element index from the DOM snapshot. Preferred. */
+    index?: number;
     text: string;
+    /** Fallback x coordinate (CSS pixels). */
     x?: number;
+    /** Fallback y coordinate (CSS pixels). */
     y?: number;
+    /** If true (default), clear the field first. */
+    clear?: boolean;
+    /** If true (default), press Enter after typing when text ends with `\n`. */
+    submit?: boolean;
 } | {
     action: "scroll";
     direction: "up" | "down";
+    /** Pixels to scroll. Default: 400. */
     amount?: number;
+    /** If provided, scroll the element with this index into view instead. */
+    index?: number;
 } | {
     action: "navigate";
     url: string;
@@ -62,6 +87,37 @@ type BrowserAction = {
     ms: number;
 } | {
     action: "screenshot";
+} | {
+    action: "send_keys";
+    keys: string;
+} | {
+    action: "find_text";
+    text: string;
+} | {
+    action: "evaluate";
+    code: string;
+} | {
+    action: "dropdown_options";
+    index: number;
+} | {
+    action: "select_dropdown";
+    index: number;
+    text: string;
+} | {
+    action: "upload_file";
+    index: number;
+    path: string;
+} | {
+    action: "extract";
+    /** Natural-language description of what to extract. */
+    query: string;
+    /** Include link hrefs in the extracted content. Default: false. */
+    extractLinks?: boolean;
+} | {
+    action: "tool";
+    /** Name of a custom tool registered on the BrowserAgent. */
+    name: string;
+    args?: Record<string, unknown>;
 } | {
     action: "done";
     result: string;
@@ -69,14 +125,87 @@ type BrowserAction = {
     action: "fail";
     reason: string;
 };
+/**
+ * One entry in the DOM snapshot the model sees. Returned by
+ * `BrowserProvider.extractDOM()` alongside the human-readable string form.
+ */
+interface DomElement {
+    /** Stable 1-based index for this step. Used by indexed actions. */
+    index: number;
+    /** Center coordinate in CSS pixels (fallback for the model if needed). */
+    cx: number;
+    cy: number;
+    /** ARIA role or tag name. */
+    role: string;
+    /** Input type, if applicable. */
+    type?: string;
+    /** Visible label (text / aria-label / placeholder / href / …). */
+    label: string;
+    /** Tag name. */
+    tag: string;
+    /** Whether it's an `<input>` / `<textarea>` / `[contenteditable]`. */
+    isInput: boolean;
+    /** Whether it's a `<select>`. */
+    isSelect: boolean;
+    /** Whether it's an `<input type="file">`. */
+    isFile: boolean;
+}
 interface BrowserAgentConfig {
     name: string;
     /** Vision-capable model (GPT-4o, Gemini, etc.) */
     model: ModelProvider;
-    /** Extra instructions appended to the system prompt */
+    /**
+     * Optional secondary (usually cheaper) model used for the `extract`
+     * action and other text-only sub-tasks. Falls back to `model`.
+     */
+    pageExtractionLLM?: ModelProvider;
+    /** Extra instructions appended to the default system prompt. */
     instructions?: string;
+    /**
+     * Append additional instructions to the default system prompt.
+     * Alias for `instructions` (browser-use parity). Both are concatenated.
+     */
+    extendSystemMessage?: string;
+    /**
+     * Completely replace the default system prompt with this string.
+     * The credentials / DOM / coordinate sections are still appended at the
+     * end automatically. Most users should use `instructions` instead.
+     */
+    overrideSystemMessage?: string;
     /** Max vision loop iterations. Default: 30 */
     maxSteps?: number;
+    /**
+     * Maximum number of consecutive step failures (action threw, invalid
+     * JSON from model, locator timeout) before the agent gives up. Default: 3.
+     */
+    maxFailures?: number;
+    /**
+     * Maximum number of actions the model can return in a single step.
+     * If the model returns an array, we execute them in order until the page
+     * navigates or the DOM changes substantially. Default: 3.
+     * Set to 1 to force one-action-per-step (the v2.0.x behaviour).
+     */
+    maxActionsPerStep?: number;
+    /**
+     * Actions to run before the LLM loop starts. Useful for boilerplate
+     * (cookie-banner click, login flow, scrolling) you already know the
+     * answer to — saves vision tokens.
+     */
+    initialActions?: BrowserAction[];
+    /**
+     * Vision mode. Default: `"auto"`.
+     * - `true`: always send a screenshot with every step (v2.0.x behaviour).
+     * - `false`: never send screenshots; DOM-only operation.
+     * - `"auto"`: send a screenshot on the first step and whenever the model
+     *   used the `screenshot` action in the previous step. Saves vision
+     *   tokens dramatically on DOM-only-suitable workloads.
+     */
+    useVision?: boolean | "auto";
+    /**
+     * If true (default), detect a URL in the task string and navigate to it
+     * before the first LLM call — saves one round trip on simple tasks.
+     */
+    directlyOpenUrl?: boolean;
     /** Run browser without visible window. Default: true */
     headless?: boolean;
     /** Browser viewport size. Default: 1280x720 */
@@ -97,11 +226,36 @@ interface BrowserAgentConfig {
      * Default: true
      */
     useDOM?: boolean;
+    /**
+     * Allow the `evaluate` action to run arbitrary JavaScript inside the
+     * page. Default: false (security). Only enable if you trust the source
+     * of task strings — a malicious task could exfiltrate page contents.
+     */
+    allowEvaluate?: boolean;
+    /**
+     * Restrict navigation to specific domains. Wildcard patterns supported:
+     * `"example.com"`, `"*.example.com"`, `"http*://example.com"`.
+     * When set, any `navigate` action to a non-matching URL throws.
+     */
+    allowedDomains?: string[];
+    /**
+     * Block navigation to specific domains. Same pattern format as
+     * `allowedDomains`. Evaluated AFTER `allowedDomains`, so if both are
+     * set a URL must be in `allowedDomains` AND not in `prohibitedDomains`.
+     */
+    prohibitedDomains?: string[];
     /**
      * Path to a Playwright storageState JSON file.
      * Restores cookies, localStorage, and sessionStorage from a previous session.
      */
     storageState?: string;
+    /**
+     * Connect to an existing browser via Chrome DevTools Protocol instead
+     * of launching one. Format: `"http://localhost:9222"`. When set,
+     * `headless`, `stealth.args`, `recordVideo`, etc. are ignored — the
+     * existing browser's configuration is used.
+     */
+    cdpUrl?: string;
     /**
      * Enable video recording of the browser session.
      * Pass `true` for default dir (`./browser-videos`) or `{ dir: "/path" }`.
@@ -134,6 +288,14 @@ interface BrowserAgentConfig {
     memory?: UnifiedMemoryConfig;
     /** Skills — pre-packaged or learned tool bundles. */
     skills?: Array<_agentium_core.Skill | string>;
+    /**
+     * Custom tools that the BrowserAgent itself can invoke during a run.
+     * The agent emits `{ "action": "tool", "name": "<tool>", "args": {...} }`
+     * and we dispatch to the tool's `execute(args)`. Use this for 2FA codes,
+     * API calls, file I/O, calling out to other agents — anything the
+     * browser can't do alone.
+     */
+    tools?: ToolDef[];
     /** Cost tracker — track vision model token usage and enforce budgets across browser runs. */
     costTracker?: CostTracker;
     logLevel?: LogLevel;
@@ -150,6 +312,8 @@ interface BrowserRunOpts {
     userId?: string;
     /** Path to save storageState (cookies/auth) after the run completes */
     saveStorageState?: string;
+    /** Per-run override of `maxSteps`. */
+    maxSteps?: number;
 }
 interface BrowserRunOutput {
     /** Final text result produced by the agent */
@@ -166,17 +330,23 @@ interface BrowserRunOutput {
     durationMs: number;
     /** Video file path (if recordVideo was enabled) */
     videoPath?: string;
+    /** Extracted content from every `extract` action, in chronological order. */
+    extractedContent?: string[];
 }
 interface BrowserStep {
     index: number;
     action: BrowserAction;
-    /** Screenshot taken before this action was executed */
+    /** Screenshot taken before this action was executed. May be empty if useVision=false. */
     screenshot: Buffer;
     pageUrl: string;
     pageTitle: string;
     timestamp: Date;
     /** Simplified DOM snapshot (if useDOM is enabled) */
     dom?: string;
+    /** Free-form result string produced by the action (e.g. extract output). */
+    output?: string;
+    /** Whether this step succeeded (vs threw / failed locator). Default: true. */
+    ok?: boolean;
 }
 interface StealthConfig {
     /**
@@ -198,6 +368,13 @@ interface StealthConfig {
     };
     /** Ignore HTTPS certificate errors. Default: false (secure). Only enable for local testing. */
     ignoreHTTPSErrors?: boolean;
+    /**
+     * `window.devicePixelRatio` to emulate. Default: `1`. Set to `2` to mimic
+     * a Retina display (sharper screenshots at 2× cost). Setting to 2 on a
+     * non-Retina host display can cause the headed window to look zoomed-out
+     * or stretched because the OS compositor downsamples a 2× surface.
+     */
+    deviceScaleFactor?: number;
     /** HTTP/SOCKS proxy. Format: "http://user:pass@host:port" */
     proxy?: {
         server: string;
@@ -228,19 +405,32 @@ declare class BrowserAgent {
     readonly name: string;
     readonly eventBus: EventBus;
     private model;
+    private pageExtractionLLM;
     private instructions?;
+    private extendSystemMessage?;
+    private overrideSystemMessage?;
     private maxSteps;
+    private maxFailures;
+    private maxActionsPerStep;
+    private initialActions;
+    private useVision;
+    private directlyOpenUrl;
     private headless;
     private viewport;
     private defaultStartUrl?;
     private waitAfterAction;
     private maxRepeats;
     private useDOM;
+    private allowEvaluate;
+    private allowedDomains?;
+    private prohibitedDomains?;
     private storageState?;
+    private cdpUrl?;
     private recordVideo?;
     private credentials?;
     private stealth?;
     private humanize?;
+    private tools;
     private costTracker;
     private memoryManager;
     private logger;
@@ -256,13 +446,32 @@ declare class BrowserAgent {
         name?: string;
         description?: string;
     }): ToolDef;
+    private shouldCaptureVision;
+    private detectUrlInTask;
+    private assertDomainAllowed;
     private finalize;
+    /**
+     * Execute a single action. Returns `{ output?, didNavigate? }` for the
+     * caller's state tracking. May throw — the loop handles failure-budget
+     * accounting in that case.
+     */
     private executeAction;
     private sleep;
+    /**
+     * Parse a quoted target keyword from a click action's `description`.
+     * Returns `undefined` for generic / ambiguous labels (login buttons,
+     * close, OK, etc.) where a substring text match could fire on the
+     * wrong element.
+     */
+    private extractClickKeyword;
 }
 
 /**
- * Playwright wrapper with stealth anti-detection and human-like behavior.
+ * Playwright wrapper with stealth anti-detection, human-like behavior,
+ * indexed DOM element resolution, and a rich action vocabulary.
+ *
+ * The `BrowserProvider` is intentionally LLM-agnostic — it exposes the
+ * primitives that `BrowserAgent` orchestrates via vision+DOM reasoning.
  */
 declare class BrowserProvider {
     private browser;
@@ -274,6 +483,13 @@ declare class BrowserProvider {
     private _viewport;
     private _videoDir?;
     private _humanize?;
+    /**
+     * Most recent DOM snapshot (one per `extractDOM` call). Indexed actions
+     * (`clickByIndex`, `inputByIndex`, …) resolve their `index` against this.
+     */
+    private _lastDom;
+    /** True if we connected over CDP (don't tear down the browser on close). */
+    private _attached;
     constructor();
     launch(opts?: {
         headless?: boolean;
@@ -287,6 +503,7 @@ declare class BrowserProvider {
         };
         stealth?: boolean | StealthConfig;
         humanize?: boolean | HumanizeConfig;
+        cdpUrl?: string;
     }): Promise<void>;
     saveStorageState(path: string): Promise<void>;
     navigate(url: string): Promise<void>;
@@ -297,14 +514,114 @@ declare class BrowserProvider {
         width: number;
         height: number;
     };
+    /** Most recent DOM snapshot. Each entry has a stable `index`. */
+    get lastDom(): DomElement[];
     click(x: number, y: number): Promise<void>;
     type(text: string): Promise<void>;
     clickAndType(x: number, y: number, text: string): Promise<void>;
     pressKey(key: string): Promise<void>;
+    /**
+     * Send arbitrary keyboard keys / shortcuts. Accepts a single
+     * Playwright key spec (`"Enter"`, `"Control+l"`, `"Shift+ArrowDown"`)
+     * or a space-separated sequence (`"Tab Tab Enter"`).
+     */
+    sendKeys(keys: string): Promise<void>;
     scroll(direction: "up" | "down", amount?: number): Promise<void>;
+    /**
+     * Build a Playwright locator for a DOM-snapshot index. Each `extractDOM`
+     * call tags surviving elements with `data-bua-idx="<n>"`; we resolve by
+     * that attribute. Returns null if the index is unknown.
+     */
+    private locatorForIndex;
+    /**
+     * Click an element by its DOM-snapshot index. The most reliable click
+     * path on dynamic pages — survives layout shifts and DPR oddities.
+     */
+    clickByIndex(index: number, opts?: {
+        timeout?: number;
+    }): Promise<boolean>;
+    /**
+     * Focus an indexed input, optionally clear it, and type. Returns false
+     * if the index couldn't be resolved or the input couldn't be focused.
+     */
+    inputByIndex(index: number, text: string, opts?: {
+        clear?: boolean;
+        submit?: boolean;
+        timeout?: number;
+    }): Promise<boolean>;
+    uploadFileByIndex(index: number, path: string): Promise<boolean>;
+    /** Scroll the indexed element into view (no click). */
+    scrollIntoViewByIndex(index: number): Promise<boolean>;
+    /**
+     * Deterministic, DOM-based click using Playwright's text locator.
+     *
+     * Returns `true` if a matching, visible, clickable element was found and
+     * clicked within `timeout` ms; `false` otherwise (so the caller can fall
+     * back to coordinate clicking). Substring-matches by default — e.g.
+     * `clickByText("Cheapest")` matches "Cheapest · 23-28 days · $2,550".
+     */
+    clickByText(keyword: string, opts?: {
+        timeout?: number;
+    }): Promise<boolean>;
+    /**
+     * Scroll the first occurrence of `text` into view. Returns false if no
+     * match was found within the timeout.
+     */
+    findText(text: string, opts?: {
+        timeout?: number;
+    }): Promise<boolean>;
+    /**
+     * Read the options of a native `<select>` at the given DOM-snapshot
+     * index. Returns `[]` if the element is not a `<select>`.
+     */
+    dropdownOptions(index: number): Promise<{
+        value: string;
+        label: string;
+        selected: boolean;
+    }[]>;
+    /**
+     * Select an option in a native `<select>` by its visible text or value.
+     * Returns false if the element isn't a `<select>` or no option matched.
+     */
+    selectDropdown(index: number, text: string): Promise<boolean>;
+    /**
+     * Run arbitrary JS in the page context. The caller is responsible for
+     * gating this behind a config flag — the BrowserAgent only routes the
+     * `evaluate` action here when `allowEvaluate: true`.
+     *
+     * The code is wrapped in `(async () => { ... })()` and the return value
+     * is coerced to a string for the model.
+     */
+    evaluate(code: string): Promise<string>;
+    /**
+     * Returns a clean text representation of the visible page body, with
+     * optional link extraction. Used by the BrowserAgent's `extract` action
+     * — the text is passed to a (usually cheap) LLM with the user's query.
+     */
+    pageText(opts?: {
+        extractLinks?: boolean;
+        maxChars?: number;
+    }): Promise<string>;
+    /**
+     * Snapshot the interactive elements visible in the viewport, tag each
+     * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
+     * return BOTH a human-readable string (for the model) and the
+     * structured list (for the runtime).
+     *
+     * Three properties matter for accuracy:
+     *  - Hit-tested: each listed coordinate / index actually reaches the
+     *    labeled element (overlays / occlusion skip the entry).
+     *  - Visibility-filtered: invisible, zero-size, `pointer-events: none`,
+     *    and near-zero-opacity elements are excluded.
+     *  - `cursor: pointer` fallback pass: a second scan catches custom React
+     *    widgets that have no semantic role/href/onclick but are clickable.
+     */
     extractDOM(opts?: {
         maxElements?: number;
-    }): Promise<string>;
+    }): Promise<{
+        text: string;
+        elements: DomElement[];
+    }>;
     getPageInfo(): Promise<PageInfo>;
     waitForStable(minWait?: number): Promise<void>;
     newTab(url?: string): Promise<string>;
@@ -330,8 +647,7 @@ declare class BrowserProvider {
      */
     private clampToViewport;
     /**
-     * Simulate human mouse movement using Bézier-like interpolation.
-     * Moves from the current mouse position to the target in small steps.
+     * Simulate human mouse movement using smoothstep interpolation.
      */
     private humanMouseMove;
     /** Small random pause after an interaction. */
@@ -342,4 +658,4 @@ declare class BrowserProvider {
     private sleep;
 }
 
-export { type BrowserAction, BrowserAgent, type BrowserAgentConfig, BrowserProvider, type BrowserRunOpts, type BrowserRunOutput, type BrowserStep, CredentialVault, type HumanizeConfig, type PageInfo, type StealthConfig };
+export { type BrowserAction, BrowserAgent, type BrowserAgentConfig, BrowserProvider, type BrowserRunOpts, type BrowserRunOutput, type BrowserStep, CredentialVault, type DomElement, type HumanizeConfig, type PageInfo, type StealthConfig };