@agentium/browser 2.1.0 → 2.2.1

package/dist/index.d.cts

@@ -149,6 +149,32 @@ interface DomElement {
     isSelect: boolean;
     /** Whether it's an `<input type="file">`. */
     isFile: boolean;
+    /** Origin frame (`"main"` or the iframe `src`/path). */
+    frame?: string;
+}
+/**
+ * Scroll-context metadata returned alongside `DomElement[]`. Gives the
+ * model spatial awareness so it can decide when to scroll vs when more
+ * content is below / above the fold.
+ */
+interface DomScrollContext {
+    /** Approximate viewports of scrollable content above the current view. */
+    pagesAbove: number;
+    /** Approximate viewports of scrollable content below the current view. */
+    pagesBelow: number;
+    /** Total interactive elements found (visible + hidden combined). */
+    totalInteractive: number;
+    /** Count of interactive elements that exist on the page but aren't in the viewport. */
+    hiddenInteractive: number;
+}
+/** Combined return value of `BrowserProvider.extractDOM()`. */
+interface DomSnapshot {
+    /** Human-readable string fed to the model. */
+    text: string;
+    /** Structured list (stable indices) for runtime resolution. */
+    elements: DomElement[];
+    /** Spatial/scroll context. */
+    scroll: DomScrollContext;
 }
 interface BrowserAgentConfig {
     name: string;
@@ -159,6 +185,27 @@ interface BrowserAgentConfig {
      * action and other text-only sub-tasks. Falls back to `model`.
      */
     pageExtractionLLM?: ModelProvider;
+    /**
+     * Fallback model used automatically when the primary `model` returns a
+     * rate-limit / auth / 5xx error or fails to produce valid JSON several
+     * times in a row. Failure-budget aware. Leave unset to disable.
+     */
+    fallbackModel?: ModelProvider;
+    /**
+     * Ask the model to emit a structured thinking/evaluation/memory/next_goal
+     * envelope around its action(s). Significantly improves accuracy and
+     * self-correction on multi-step tasks. Default: `true`. Set `false` for
+     * a `flash_mode` that just returns the raw action(s) — useful for very
+     * fast / cheap models that are bad at long outputs.
+     */
+    useThinking?: boolean;
+    /**
+     * Maximum number of recent step turns kept verbatim in the conversation
+     * history sent to the model. Older turns are compacted into a single
+     * summary line. Default: 6. Set 0 to disable conversation history
+     * entirely (each step rebuilt from scratch — v2.0 behaviour).
+     */
+    historyWindow?: number;
     /** Extra instructions appended to the default system prompt. */
     instructions?: string;
     /**
@@ -347,6 +394,26 @@ interface BrowserStep {
     output?: string;
     /** Whether this step succeeded (vs threw / failed locator). Default: true. */
     ok?: boolean;
+    /** Model's chain-of-thought reasoning (if `useThinking` was on). */
+    thinking?: string;
+    /** Model's evaluation of whether the previous action met its goal. */
+    evaluationPreviousGoal?: string;
+    /** Model's running memory of important state. */
+    memory?: string;
+    /** Model's stated next goal for this step. */
+    nextGoal?: string;
+}
+/**
+ * Structured envelope the model returns when `useThinking: true`. Inspired
+ * by browser-use's `AgentOutput`. Every field is optional from a runtime
+ * standpoint — only `action` is required for execution.
+ */
+interface AgentOutput {
+    thinking?: string;
+    evaluationPreviousGoal?: string;
+    memory?: string;
+    nextGoal?: string;
+    action: BrowserAction | BrowserAction[];
 }
 interface StealthConfig {
     /**
@@ -406,6 +473,9 @@ declare class BrowserAgent {
     readonly eventBus: EventBus;
     private model;
     private pageExtractionLLM;
+    private fallbackModel;
+    private useThinking;
+    private historyWindow;
     private instructions?;
     private extendSystemMessage?;
     private overrideSystemMessage?;
@@ -449,6 +519,52 @@ declare class BrowserAgent {
     private shouldCaptureVision;
     private detectUrlInTask;
     private assertDomainAllowed;
+    /**
+     * Build the message array sent to the model: system prompt + a compact
+     * summary of older turns (if any) + the most recent `historyWindow`
+     * turns verbatim + the current step's user message.
+     *
+     * Inspired by browser-use's history compaction. Keeps tokens bounded
+     * while giving the model meaningful context about what it already
+     * tried.
+     */
+    private buildMessages;
+    /**
+     * Call the primary model, retrying once with `fallbackModel` on
+     * transient errors (5xx, 429, network). Returns the response or
+     * `null` if both models failed.
+     */
+    private callModelWithFallback;
+    private isTransientError;
+    /**
+     * Parse the model's raw response into an `AgentOutput`. Tolerant to
+     * three shapes:
+     *  - Full envelope: { thinking, evaluation_previous_goal, action, ... }
+     *  - Raw action object (legacy / `useThinking: false`)
+     *  - Raw action array
+     *
+     * Also strips ```json fences the model occasionally adds.
+     */
+    private parseEnvelope;
+    /**
+     * Quick post-navigation health check. If the page came back blank
+     * (no body text and no interactive elements), reload once and wait
+     * for stable. This catches the "FreightOS half-loaded font test" class
+     * of failure before the LLM ever sees it.
+     */
+    private navigationHealthCheck;
+    /**
+     * Force-finalize the run with whatever partial data the agent has.
+     * Called when:
+     *   - `maxSteps` is exhausted without an explicit `done`,
+     *   - `maxFailures` is exceeded,
+     *   - the model can't produce parseable JSON enough times to make
+     *     forward progress.
+     * The result is composed from `extractedContent` + the last few
+     * action summaries so the caller gets something useful instead of
+     * just a one-line error.
+     */
+    private forceDone;
     private finalize;
     /**
      * Execute a single action. Returns `{ output?, didNavigate? }` for the
@@ -605,23 +721,47 @@ declare class BrowserProvider {
     /**
      * Snapshot the interactive elements visible in the viewport, tag each
      * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
-     * return BOTH a human-readable string (for the model) and the
-     * structured list (for the runtime).
+     * return:
+     *   - `text`: a human-readable string fed to the model
+     *   - `elements`: the structured list with stable indices
+     *   - `scroll`: spatial context (pages above/below, hidden interactive count)
      *
-     * Three properties matter for accuracy:
-     *  - Hit-tested: each listed coordinate / index actually reaches the
+     * Five properties matter for accuracy:
+     *  - **Hit-tested**: each listed coordinate / index actually reaches the
      *    labeled element (overlays / occlusion skip the entry).
-     *  - Visibility-filtered: invisible, zero-size, `pointer-events: none`,
-     *    and near-zero-opacity elements are excluded.
-     *  - `cursor: pointer` fallback pass: a second scan catches custom React
-     *    widgets that have no semantic role/href/onclick but are clickable.
+     *  - **Visibility filtered (with parent chain)**: an element is dropped
+     *    if itself OR any ancestor is `display:none`, `visibility:hidden`,
+     *    `pointer-events:none`, or near-zero opacity.
+     *  - **Shadow DOM piercing**: traverses open shadow roots so custom
+     *    elements / web components are visible to the agent.
+     *  - **Same-origin iframes**: walks into each accessible iframe and
+     *    includes its interactive elements (offset by the iframe's screen
+     *    position so the coordinates the model sees are still viewport-
+     *    relative).
+     *  - **`cursor: pointer` fallback pass**: catches custom React widgets
+     *    that have no semantic role/href/onclick but are clickable.
      */
     extractDOM(opts?: {
         maxElements?: number;
-    }): Promise<{
-        text: string;
-        elements: DomElement[];
-    }>;
+    }): Promise<DomSnapshot>;
+    /**
+     * Install the `__buaExtract` global on the main page. Idempotent —
+     * subsequent calls are no-ops.
+     */
+    private installExtractorScript;
+    /**
+     * The extractor source. Lives in its own method so we can also inject
+     * it into iframes that haven't yet had it loaded.
+     *
+     * This function intentionally runs entirely in the page context. It:
+     *   - traverses the regular DOM + open shadow roots (deep)
+     *   - applies a parent-chain visibility filter
+     *   - applies a `cursor:pointer` second pass for custom widgets
+     *   - hit-tests each candidate at its center to avoid overlay collisions
+     *   - returns scroll context (pages above/below, hidden counts)
+     *   - tags survivors with `data-bua-idx` for indexed actions
+     */
+    private extractorScriptSource;
     getPageInfo(): Promise<PageInfo>;
     waitForStable(minWait?: number): Promise<void>;
     newTab(url?: string): Promise<string>;
@@ -658,4 +798,4 @@ declare class BrowserProvider {
     private sleep;
 }
 
-export { type BrowserAction, BrowserAgent, type BrowserAgentConfig, BrowserProvider, type BrowserRunOpts, type BrowserRunOutput, type BrowserStep, CredentialVault, type DomElement, type HumanizeConfig, type PageInfo, type StealthConfig };
+export { type AgentOutput, type BrowserAction, BrowserAgent, type BrowserAgentConfig, BrowserProvider, type BrowserRunOpts, type BrowserRunOutput, type BrowserStep, CredentialVault, type DomElement, type DomScrollContext, type DomSnapshot, type HumanizeConfig, type PageInfo, type StealthConfig };

package/dist/index.d.ts

@@ -149,6 +149,32 @@ interface DomElement {
     isSelect: boolean;
     /** Whether it's an `<input type="file">`. */
     isFile: boolean;
+    /** Origin frame (`"main"` or the iframe `src`/path). */
+    frame?: string;
+}
+/**
+ * Scroll-context metadata returned alongside `DomElement[]`. Gives the
+ * model spatial awareness so it can decide when to scroll vs when more
+ * content is below / above the fold.
+ */
+interface DomScrollContext {
+    /** Approximate viewports of scrollable content above the current view. */
+    pagesAbove: number;
+    /** Approximate viewports of scrollable content below the current view. */
+    pagesBelow: number;
+    /** Total interactive elements found (visible + hidden combined). */
+    totalInteractive: number;
+    /** Count of interactive elements that exist on the page but aren't in the viewport. */
+    hiddenInteractive: number;
+}
+/** Combined return value of `BrowserProvider.extractDOM()`. */
+interface DomSnapshot {
+    /** Human-readable string fed to the model. */
+    text: string;
+    /** Structured list (stable indices) for runtime resolution. */
+    elements: DomElement[];
+    /** Spatial/scroll context. */
+    scroll: DomScrollContext;
 }
 interface BrowserAgentConfig {
     name: string;
@@ -159,6 +185,27 @@ interface BrowserAgentConfig {
      * action and other text-only sub-tasks. Falls back to `model`.
      */
     pageExtractionLLM?: ModelProvider;
+    /**
+     * Fallback model used automatically when the primary `model` returns a
+     * rate-limit / auth / 5xx error or fails to produce valid JSON several
+     * times in a row. Failure-budget aware. Leave unset to disable.
+     */
+    fallbackModel?: ModelProvider;
+    /**
+     * Ask the model to emit a structured thinking/evaluation/memory/next_goal
+     * envelope around its action(s). Significantly improves accuracy and
+     * self-correction on multi-step tasks. Default: `true`. Set `false` for
+     * a `flash_mode` that just returns the raw action(s) — useful for very
+     * fast / cheap models that are bad at long outputs.
+     */
+    useThinking?: boolean;
+    /**
+     * Maximum number of recent step turns kept verbatim in the conversation
+     * history sent to the model. Older turns are compacted into a single
+     * summary line. Default: 6. Set 0 to disable conversation history
+     * entirely (each step rebuilt from scratch — v2.0 behaviour).
+     */
+    historyWindow?: number;
     /** Extra instructions appended to the default system prompt. */
     instructions?: string;
     /**
@@ -347,6 +394,26 @@ interface BrowserStep {
     output?: string;
     /** Whether this step succeeded (vs threw / failed locator). Default: true. */
     ok?: boolean;
+    /** Model's chain-of-thought reasoning (if `useThinking` was on). */
+    thinking?: string;
+    /** Model's evaluation of whether the previous action met its goal. */
+    evaluationPreviousGoal?: string;
+    /** Model's running memory of important state. */
+    memory?: string;
+    /** Model's stated next goal for this step. */
+    nextGoal?: string;
+}
+/**
+ * Structured envelope the model returns when `useThinking: true`. Inspired
+ * by browser-use's `AgentOutput`. Every field is optional from a runtime
+ * standpoint — only `action` is required for execution.
+ */
+interface AgentOutput {
+    thinking?: string;
+    evaluationPreviousGoal?: string;
+    memory?: string;
+    nextGoal?: string;
+    action: BrowserAction | BrowserAction[];
 }
 interface StealthConfig {
     /**
@@ -406,6 +473,9 @@ declare class BrowserAgent {
     readonly eventBus: EventBus;
     private model;
     private pageExtractionLLM;
+    private fallbackModel;
+    private useThinking;
+    private historyWindow;
     private instructions?;
     private extendSystemMessage?;
     private overrideSystemMessage?;
@@ -449,6 +519,52 @@ declare class BrowserAgent {
     private shouldCaptureVision;
     private detectUrlInTask;
     private assertDomainAllowed;
+    /**
+     * Build the message array sent to the model: system prompt + a compact
+     * summary of older turns (if any) + the most recent `historyWindow`
+     * turns verbatim + the current step's user message.
+     *
+     * Inspired by browser-use's history compaction. Keeps tokens bounded
+     * while giving the model meaningful context about what it already
+     * tried.
+     */
+    private buildMessages;
+    /**
+     * Call the primary model, retrying once with `fallbackModel` on
+     * transient errors (5xx, 429, network). Returns the response or
+     * `null` if both models failed.
+     */
+    private callModelWithFallback;
+    private isTransientError;
+    /**
+     * Parse the model's raw response into an `AgentOutput`. Tolerant to
+     * three shapes:
+     *  - Full envelope: { thinking, evaluation_previous_goal, action, ... }
+     *  - Raw action object (legacy / `useThinking: false`)
+     *  - Raw action array
+     *
+     * Also strips ```json fences the model occasionally adds.
+     */
+    private parseEnvelope;
+    /**
+     * Quick post-navigation health check. If the page came back blank
+     * (no body text and no interactive elements), reload once and wait
+     * for stable. This catches the "FreightOS half-loaded font test" class
+     * of failure before the LLM ever sees it.
+     */
+    private navigationHealthCheck;
+    /**
+     * Force-finalize the run with whatever partial data the agent has.
+     * Called when:
+     *   - `maxSteps` is exhausted without an explicit `done`,
+     *   - `maxFailures` is exceeded,
+     *   - the model can't produce parseable JSON enough times to make
+     *     forward progress.
+     * The result is composed from `extractedContent` + the last few
+     * action summaries so the caller gets something useful instead of
+     * just a one-line error.
+     */
+    private forceDone;
     private finalize;
     /**
      * Execute a single action. Returns `{ output?, didNavigate? }` for the
@@ -605,23 +721,47 @@ declare class BrowserProvider {
     /**
      * Snapshot the interactive elements visible in the viewport, tag each
      * with a `data-bua-idx="<n>"` attribute (used by indexed actions), and
-     * return BOTH a human-readable string (for the model) and the
-     * structured list (for the runtime).
+     * return:
+     *   - `text`: a human-readable string fed to the model
+     *   - `elements`: the structured list with stable indices
+     *   - `scroll`: spatial context (pages above/below, hidden interactive count)
      *
-     * Three properties matter for accuracy:
-     *  - Hit-tested: each listed coordinate / index actually reaches the
+     * Five properties matter for accuracy:
+     *  - **Hit-tested**: each listed coordinate / index actually reaches the
      *    labeled element (overlays / occlusion skip the entry).
-     *  - Visibility-filtered: invisible, zero-size, `pointer-events: none`,
-     *    and near-zero-opacity elements are excluded.
-     *  - `cursor: pointer` fallback pass: a second scan catches custom React
-     *    widgets that have no semantic role/href/onclick but are clickable.
+     *  - **Visibility filtered (with parent chain)**: an element is dropped
+     *    if itself OR any ancestor is `display:none`, `visibility:hidden`,
+     *    `pointer-events:none`, or near-zero opacity.
+     *  - **Shadow DOM piercing**: traverses open shadow roots so custom
+     *    elements / web components are visible to the agent.
+     *  - **Same-origin iframes**: walks into each accessible iframe and
+     *    includes its interactive elements (offset by the iframe's screen
+     *    position so the coordinates the model sees are still viewport-
+     *    relative).
+     *  - **`cursor: pointer` fallback pass**: catches custom React widgets
+     *    that have no semantic role/href/onclick but are clickable.
      */
     extractDOM(opts?: {
         maxElements?: number;
-    }): Promise<{
-        text: string;
-        elements: DomElement[];
-    }>;
+    }): Promise<DomSnapshot>;
+    /**
+     * Install the `__buaExtract` global on the main page. Idempotent —
+     * subsequent calls are no-ops.
+     */
+    private installExtractorScript;
+    /**
+     * The extractor source. Lives in its own method so we can also inject
+     * it into iframes that haven't yet had it loaded.
+     *
+     * This function intentionally runs entirely in the page context. It:
+     *   - traverses the regular DOM + open shadow roots (deep)
+     *   - applies a parent-chain visibility filter
+     *   - applies a `cursor:pointer` second pass for custom widgets
+     *   - hit-tests each candidate at its center to avoid overlay collisions
+     *   - returns scroll context (pages above/below, hidden counts)
+     *   - tags survivors with `data-bua-idx` for indexed actions
+     */
+    private extractorScriptSource;
     getPageInfo(): Promise<PageInfo>;
     waitForStable(minWait?: number): Promise<void>;
     newTab(url?: string): Promise<string>;
@@ -658,4 +798,4 @@ declare class BrowserProvider {
     private sleep;
 }
 
-export { type BrowserAction, BrowserAgent, type BrowserAgentConfig, BrowserProvider, type BrowserRunOpts, type BrowserRunOutput, type BrowserStep, CredentialVault, type DomElement, type HumanizeConfig, type PageInfo, type StealthConfig };
+export { type AgentOutput, type BrowserAction, BrowserAgent, type BrowserAgentConfig, BrowserProvider, type BrowserRunOpts, type BrowserRunOutput, type BrowserStep, CredentialVault, type DomElement, type DomScrollContext, type DomSnapshot, type HumanizeConfig, type PageInfo, type StealthConfig };