@roll-agent/browser-use-agent 0.11.0 → 0.12.1

package/dist/tools/browser-dom-action-candidates.d.ts

@@ -0,0 +1,8 @@
+import type { BrowserDomActionHint, NativeCdpController } from "@roll-agent/browser";
+type DomActionCandidateController = Pick<NativeCdpController, "createIsolatedWorld" | "describeNode" | "evaluateJson" | "getDocument" | "querySelectorAllByNodeId">;
+type DomActionCandidateOptions = {
+    readonly frameId?: string;
+    readonly maxCandidates?: number;
+};
+export declare function collectDomActionHints(controller: DomActionCandidateController, optionsOrMaxCandidates?: DomActionCandidateOptions | number): Promise<readonly BrowserDomActionHint[]>;
+export {};

package/dist/tools/browser-native-page.d.ts

@@ -0,0 +1,6 @@
+import type { BrowserContextManager, BrowserInspectablePage, BrowserRuntime } from "@roll-agent/browser";
+export declare function resolveNativePageForBrowserTool(input: {
+    readonly ctxManager: BrowserContextManager;
+    readonly runtime: BrowserRuntime;
+    readonly pageId?: string;
+}): Promise<BrowserInspectablePage>;

package/dist/tools/browser-ref-schemas.d.ts

@@ -0,0 +1,99 @@
+import { z } from "zod";
+export declare const BrowserElementRefResolveStrategySchema: z.ZodEnum<["backend_node_id", "role_name_nth"]>;
+export declare const BrowserElementRefTargetSchema: z.ZodObject<{
+    ref: z.ZodString;
+    role: z.ZodString;
+    name: z.ZodString;
+    x: z.ZodNumber;
+    y: z.ZodNumber;
+    resolvedBy: z.ZodEnum<["backend_node_id", "role_name_nth"]>;
+    backendNodeId: z.ZodOptional<z.ZodNumber>;
+    frameId: z.ZodOptional<z.ZodString>;
+    disabled: z.ZodBoolean;
+}, "strip", z.ZodTypeAny, {
+    ref: string;
+    role: string;
+    name: string;
+    disabled: boolean;
+    x: number;
+    y: number;
+    resolvedBy: "backend_node_id" | "role_name_nth";
+    backendNodeId?: number | undefined;
+    frameId?: string | undefined;
+}, {
+    ref: string;
+    role: string;
+    name: string;
+    disabled: boolean;
+    x: number;
+    y: number;
+    resolvedBy: "backend_node_id" | "role_name_nth";
+    backendNodeId?: number | undefined;
+    frameId?: string | undefined;
+}>;
+export declare const BrowserElementRefActionOutputSchema: z.ZodObject<{
+    success: z.ZodLiteral<true>;
+    ref: z.ZodString;
+    resolvedBy: z.ZodEnum<["backend_node_id", "role_name_nth"]>;
+    target: z.ZodObject<{
+        ref: z.ZodString;
+        role: z.ZodString;
+        name: z.ZodString;
+        x: z.ZodNumber;
+        y: z.ZodNumber;
+        resolvedBy: z.ZodEnum<["backend_node_id", "role_name_nth"]>;
+        backendNodeId: z.ZodOptional<z.ZodNumber>;
+        frameId: z.ZodOptional<z.ZodString>;
+        disabled: z.ZodBoolean;
+    }, "strip", z.ZodTypeAny, {
+        ref: string;
+        role: string;
+        name: string;
+        disabled: boolean;
+        x: number;
+        y: number;
+        resolvedBy: "backend_node_id" | "role_name_nth";
+        backendNodeId?: number | undefined;
+        frameId?: string | undefined;
+    }, {
+        ref: string;
+        role: string;
+        name: string;
+        disabled: boolean;
+        x: number;
+        y: number;
+        resolvedBy: "backend_node_id" | "role_name_nth";
+        backendNodeId?: number | undefined;
+        frameId?: string | undefined;
+    }>;
+}, "strip", z.ZodTypeAny, {
+    ref: string;
+    target: {
+        ref: string;
+        role: string;
+        name: string;
+        disabled: boolean;
+        x: number;
+        y: number;
+        resolvedBy: "backend_node_id" | "role_name_nth";
+        backendNodeId?: number | undefined;
+        frameId?: string | undefined;
+    };
+    success: true;
+    resolvedBy: "backend_node_id" | "role_name_nth";
+}, {
+    ref: string;
+    target: {
+        ref: string;
+        role: string;
+        name: string;
+        disabled: boolean;
+        x: number;
+        y: number;
+        resolvedBy: "backend_node_id" | "role_name_nth";
+        backendNodeId?: number | undefined;
+        frameId?: string | undefined;
+    };
+    success: true;
+    resolvedBy: "backend_node_id" | "role_name_nth";
+}>;

package/dist/tools/browser-ref-visual.d.ts

@@ -0,0 +1,5 @@
+import type { BrowserElementRefTarget, NativeCdpController } from "@roll-agent/browser";
+import { NativeVisualActivitySession } from "../native-visual-activity-session.ts";
+export type BrowserRefVisualSession = Pick<NativeVisualActivitySession, "begin" | "succeed" | "fail" | "previewMouseMotion" | "previewMouseClick">;
+export declare function createBrowserRefVisualSession(controller: Pick<NativeCdpController, "evaluateJson">): BrowserRefVisualSession;
+export declare function clickBrowserRefVisualTarget(controller: Pick<NativeCdpController, "dispatchMouseEvent">, session: BrowserRefVisualSession, target: BrowserElementRefTarget): Promise<void>;

package/dist/tools/browser-snapshot.d.ts

@@ -0,0 +1,33 @@
+import { type BrowserAxNode } from "@roll-agent/browser";
+export declare const browserSnapshot: import("@roll-agent/sdk").ToolDefinition<{
+    pageId?: string | undefined;
+    maxNodes?: number | undefined;
+    interactiveOnly?: boolean | undefined;
+    maxDepth?: number | undefined;
+}, {
+    page: {
+        pageId: string;
+        url: string;
+        title: string;
+        boundPlatform: "zhipin" | "yupao" | null;
+        detectedPlatform: "zhipin" | "yupao" | null;
+        isSelectedForPlatform: boolean;
+    };
+    snapshot: {
+        truncated: boolean;
+        nodes: BrowserAxNode[];
+        refs: {
+            ref: string;
+            role: string;
+            name: string;
+            nth: number;
+            disabled: boolean;
+            backendNodeId?: number | undefined;
+            frameId?: string | undefined;
+        }[];
+        nodeCount: number;
+        maxNodes: number;
+        interactiveOnly: boolean;
+        maxDepth?: number | undefined;
+    };
+}>;

package/dist/tools/click-ref.d.ts

@@ -0,0 +1,22 @@
+export declare const clickRef: import("@roll-agent/sdk").ToolDefinition<{
+    ref: string;
+    pageId?: string | undefined;
+    browserActionApproval?: {
+        id: string;
+    } | undefined;
+}, {
+    ref: string;
+    target: {
+        ref: string;
+        role: string;
+        name: string;
+        disabled: boolean;
+        x: number;
+        y: number;
+        resolvedBy: "backend_node_id" | "role_name_nth";
+        backendNodeId?: number | undefined;
+        frameId?: string | undefined;
+    };
+    success: true;
+    resolvedBy: "backend_node_id" | "role_name_nth";
+}>;

package/dist/tools/type-ref.d.ts

@@ -0,0 +1,24 @@
+export declare const typeRef: import("@roll-agent/sdk").ToolDefinition<{
+    ref: string;
+    text: string;
+    pageId?: string | undefined;
+    clear?: boolean | undefined;
+    browserActionApproval?: {
+        id: string;
+    } | undefined;
+}, {
+    ref: string;
+    target: {
+        ref: string;
+        role: string;
+        name: string;
+        disabled: boolean;
+        x: number;
+        y: number;
+        resolvedBy: "backend_node_id" | "role_name_nth";
+        backendNodeId?: number | undefined;
+        frameId?: string | undefined;
+    };
+    success: true;
+    resolvedBy: "backend_node_id" | "role_name_nth";
+}>;

package/dist/tools/zhipin-get-candidate-info.d.ts

@@ -35,8 +35,8 @@ export declare const zhipinGetCandidateInfo: import("@roll-agent/sdk").ToolDefin
     };
     chatMessages: {
         index: number;
-        time: string;
         content: string;
+        time: string;
         sender: "candidate" | "recruiter" | "system";
         messageType: "text" | "resume" | "system" | "wechat-exchange";
     }[];

package/dist/tools/zhipin-get-candidate-list.d.ts

@@ -14,8 +14,8 @@ export declare const zhipinGetCandidateList: import("@roll-agent/sdk").ToolDefin
 }, {
     success: boolean;
     candidates: {
-        index: number;
         name: string;
+        index: number;
         candidateId: string;
         age: string;
         experience: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@roll-agent/browser-use-agent",
-  "version": "0.11.0",
+  "version": "0.12.1",
   "type": "module",
   "repository": {
     "type": "git",
@@ -47,7 +47,7 @@
     "zod": "^3.25.76",
     "@roll-agent/reply-authority-client": "0.1.2",
     "@roll-agent/sdk": "0.2.0",
-    "@roll-agent/browser": "0.4.0"
+    "@roll-agent/browser": "0.5.0"
   },
   "devDependencies": {
     "@types/node": "^22.0.0"

package/references/generic-browser-refs.md

@@ -0,0 +1,151 @@
+# Generic Browser Refs
+
+Use this reference when an orchestrator needs to operate a browser page element that is not covered
+by a platform-specific tool.
+
+## Purpose
+
+`browser_snapshot`, `click_ref`, and `type_ref` provide a generic accessibility-driven observe/action
+loop:
+
+```text
+AX snapshot -> select emitted @eN -> click/type -> re-observe or read back
+```
+
+This is a fallback layer for unmodeled page operations. For BOSS workflows, prefer `zhipin_*` tools
+when they already express the business action.
+
+## Tool Summary
+
+| Tool | Input | Output | Use it for |
+| --- | --- | --- | --- |
+| `browser_snapshot` | `pageId?`, `maxDepth?`, `maxNodes?`, `interactiveOnly?` | `page`, `snapshot` | Observe the current page's AX tree and get `@eN` refs. It also merges limited DOM-action hints for non-semantic controls such as clickable tabs rendered as `span`, including same-target iframes when Chrome exposes child `frameId` values. |
+| `click_ref` | `ref`, `pageId?`, `browserActionApproval?` | `success`, `ref`, `resolvedBy`, `target` | Click a ref returned by `browser_snapshot`. |
+| `type_ref` | `ref`, `text`, `clear?`, `pageId?`, `browserActionApproval?` | `success`, `ref`, `resolvedBy`, `target` | Focus a ref, optionally clear it, then insert text. |
+
+`browser_snapshot.snapshot` contains:
+
+| Field | Meaning |
+| --- | --- |
+| `nodes` | AX nodes returned to the orchestrator. With `interactiveOnly:true`, this is a flat list of interactive nodes. |
+| `refs` | Current-snapshot handles, shaped as `@e1`, `@e2`, ... |
+| `nodeCount` | Number of returned nodes. |
+| `truncated` | `true` when `maxNodes` stopped traversal. |
+| `maxNodes` | Effective node cap after applying `BROWSER_SECURITY_JSON.maxSnapshotNodes`. |
+| `interactiveOnly` | Whether non-interactive context nodes were omitted. |
+| `maxDepth` | Optional AX tree depth cap used for this snapshot. |
+
+Each `refs[]` item contains `ref`, optional `backendNodeId`, optional `frameId`, `role`, `name`,
+`nth`, and `disabled`. AX-native refs use their AX role. DOM-action refs use `role:"clickable"`,
+`role:"focusable"`, or `role:"editable"`, preserve a real `backendNodeId` when Chrome exposes one,
+and appear in `nodes[]` with `properties.domActionable:true`, `properties.domActionKind`, and
+`properties.domActionHints`.
+
+Iframe handling:
+
+```text
+main AX tree -> iframe node backendNodeId -> DOM.describeNode -> child frameId
+  -> Page.createIsolatedWorld({ frameId }) + Runtime.evaluate in that frame
+  -> DOM.getDocument({ pierce:true }) maps marker attributes back to backendNodeId
+  -> Accessibility.getFullAXTree({ frameId })
+  -> repeat for nested same-target iframes until maxNodes or frame de-duplication
+  -> child refs carry frameId
+```
+
+When an iframe child ref is clicked or typed, `click_ref` / `type_ref` keep the `frameId` in the
+resolved `target`. If the original `backendNodeId` is stale, the fallback re-queries that same frame's
+AX tree before dispatching the viewport action.
+
+## Selection Logic
+
+Choose a target in this order:
+
+1. Match the user's intent against `role` and `name`, for example `role:"button"` and `name:"交换电话"`.
+2. Reject refs with `disabled:true`.
+3. For non-semantic clickable text, match `role:"clickable"`, the visible label, and
+   `properties.domActionable:true`; examples include tab/filter labels such as `未读`.
+4. If there are multiple matching nodes, prefer visible task context from nearby `nodes` data or take
+   another snapshot with `interactiveOnly:false` and a small `maxDepth`.
+5. If a matching ref includes `frameId`, pass the ref normally; do not pass `frameId` manually.
+6. Keep the selected `page.pageId` and pass it back to `click_ref` / `type_ref` when multiple pages are open.
+
+Do not construct refs manually. Only pass refs emitted by the most recent snapshot for that page.
+
+## Action Flow
+
+```bash
+roll run browser-use-agent browser_snapshot --input-json '{"interactiveOnly":true}' --json
+roll run browser-use-agent click_ref --input-json '{"ref":"@e3"}' --json
+roll run browser-use-agent browser_snapshot --input-json '{"interactiveOnly":true}' --json
+```
+
+For text input:
+
+```bash
+roll run browser-use-agent type_ref --input-json '{"ref":"@e5","text":"hello","clear":true}' --json
+```
+
+When `BROWSER_SECURITY_JSON.actionPolicy` is `confirm`, the first side-effecting action can return
+`needs_confirmation`. In that case, take `details.approvalRequest.retryInput` from the structured
+error and merge it into the retry input unchanged.
+
+## Visual Feedback
+
+The generic ref tools use the native CDP visual feedback path:
+
+| Tool | Activity capsule | Visual cursor |
+| --- | --- | --- |
+| `browser_snapshot` | Shows reading and completion/failure state. | Not applicable because it is read-only. |
+| `click_ref` | Shows click progress and completion/failure state. | Shows pointer placement and click pulse for the resolved target point. |
+| `type_ref` | Shows input progress and completion/failure state. | Shows pointer placement and click pulse before text insertion. |
+
+`BROWSER_VISUAL_ACTIVITY=false` disables the capsule. `BROWSER_VISUAL_CURSOR=false` disables the
+pointer and click pulse.
+
+## Staleness Rules
+
+Refresh the snapshot before reusing `@eN` after any of these events:
+
+- navigation, reload, redirect, or platform switch
+- modal open/close
+- list filtering, search, sort, or virtual-scroll loading
+- a prior `click_ref` that may re-render the target area
+- `click_ref` / `type_ref` returns a stale-ref or not-found error
+
+`click_ref` and `type_ref` first resolve by `backendNodeId` when the ref has one. If that fails, they
+fall back to `role/name/nth`. The fallback is useful for small re-renders, but it is not a business
+identity.
+
+## Relationship To BOSS Refs
+
+| Ref family | Produced by | Consumed by | Meaning |
+| --- | --- | --- | --- |
+| `@eN` | `browser_snapshot` | `click_ref`, `type_ref` | Generic AX element handle for the current page snapshot. |
+| `@cN` | `zhipin_get_candidate_list` | `zhipin_say_hello`, `zhipin_open_resume` | BOSS recommendation candidate handle. |
+| `@jN` | `zhipin_list_recommend_jobs` | `zhipin_select_recommend_job` | BOSS recommendation job-filter handle. |
+
+Do not pass one ref family into another tool family.
+
+## Boundary Conditions
+
+- This is an Accessibility Tree snapshot, not a full HTML dump, screenshot, network log, or page state database.
+- DOM-action refs are intentionally narrow: short visible non-semantic elements with click hints such as
+  `cursor:pointer`, `onclick`, `tabIndex`, or nearby class names like `filter`, `tab`, `menu`, `dropdown`,
+  `button`, or `toggle`. Plain article text is not exposed as clickable.
+- DOM-action augmentation is collected from the active document and same-target iframe execution contexts.
+  Non-semantic iframe controls can be promoted when they are visible short-label `span`/`div`/`li`-style
+  elements with action hints and Chrome can map their marker attributes back to `backendNodeId`.
+- Composite dropdown option rows such as `li.company-item` are promoted by using their visible descendant
+  text only inside dropdown/menu/select/option contexts, so large page containers are still filtered out.
+- Canvas, image-map hotspots, non-accessible custom widgets, and deeply nested iframe/Shadow DOM flows may not expose
+  enough AX semantics for reliable operation.
+- Same-target iframe refs are recursively inlined while Chrome's normal page-scoped CDP session can
+  resolve child `frameId` values. The recursion stops at `maxNodes`, skipped frame errors, or repeated
+  frame IDs. Cross-target/OOPIF iframe traversal is not implemented here because it requires flattened
+  CDP `sessionId` routing, which this native controller intentionally does not expose yet.
+- Only click and text input are covered. Drag, hover, keyboard shortcuts, file upload, and complex gestures still need
+  dedicated tools.
+- The implementation does not call `Runtime.enable()` for the native CDP path. Fallback matching may use
+  `Runtime.evaluate`, but it does not enable the Runtime domain. This avoids that specific detection point, not every
+  possible anti-automation signal.
+- `domainAllowlist`, `maxSnapshotNodes`, and `actionPolicy` from `BROWSER_SECURITY_JSON` still apply.