@usecrow/client 0.1.36 → 0.1.37

package/dist/index.d.ts

@@ -1,9 +1,28 @@
+declare interface ActionResult {
+    success: boolean;
+    message: string;
+}
+
 export declare interface ActiveWorkflow {
     name: string;
     todos: WorkflowTodo[];
     isComplete?: boolean;
 }
 
+/**
+ * Structured browser state for LLM consumption
+ */
+declare interface BrowserState {
+    url: string;
+    title: string;
+    /** Page info + scroll position hint (e.g. "Page info: 1920x1080px...\n[Start of page]") */
+    header: string;
+    /** Simplified HTML of interactive elements */
+    content: string;
+    /** Page footer hint (e.g. "... 300 pixels below ..." or "[End of page]") */
+    footer: string;
+}
+
 declare interface BrowserUseConfig {
     productId: string;
     apiUrl: string;
@@ -20,6 +39,8 @@ export declare interface Citation {
     page?: number;
 }
 
+declare function cleanUpHighlights(): void;
+
 export declare interface ContextData {
     /** Current page URL or path */
     page?: string;
@@ -317,6 +338,87 @@ export declare const DEFAULT_TOOLS: ToolHandlers;
 
 export declare type DefaultToolName = keyof typeof DEFAULT_TOOLS;
 
+declare namespace dom {
+    export {
+        getFlatTree,
+        flatTreeToString,
+        getSelectorMap,
+        getElementTextMap,
+        cleanUpHighlights,
+        DomConfig,
+        getAllTextTillNextClickableElement
+    }
+}
+
+declare interface DomConfig {
+    interactiveBlacklist?: (Element | (() => Element))[];
+    interactiveWhitelist?: (Element | (() => Element))[];
+    include_attributes?: string[];
+    highlightOpacity?: number;
+    highlightLabelOpacity?: number;
+}
+
+declare type DomNode = TextDomNode | ElementDomNode | InteractiveElementDomNode;
+
+declare interface ElementDomNode {
+    tagName: string;
+    attributes?: Record<string, string>;
+    xpath?: string;
+    children?: string[];
+    isVisible?: boolean;
+    isTopElement?: boolean;
+    isInViewport?: boolean;
+    isNew?: boolean;
+    isInteractive?: false;
+    highlightIndex?: number;
+    extra?: Record<string, any>;
+    [key: string]: unknown;
+}
+
+/**
+ * Derived from @page-agent/page-controller
+ * Original: https://github.com/alibaba/page-agent
+ * Copyright (c) 2025 Alibaba Group Holding Limited
+ * Licensed under MIT License
+ */
+declare interface FlatDomTree {
+    rootId: string;
+    map: Record<string, DomNode>;
+}
+
+/**
+ * 对应 python 中的 views::clickable_elements_to_string,
+ * 将 dom 信息处理成适合 llm 阅读的文本格式
+ * @形如
+ * ``` text
+ * [0]<a aria-label=page-agent.js 首页 />
+ * [1]<div >P />
+ * [2]<div >page-agent.js
+ * UI Agent in your webpage />
+ * [3]<a >文档 />
+ * [4]<a aria-label=查看源码（在新窗口打开）>源码 />
+ * UI Agent in your webpage
+ * 用户输入需求，AI 理解页面并自动操作。
+ * [5]<a role=button>快速开始 />
+ * [6]<a role=button>查看文档 />
+ * 无需后端
+ * ```
+ * 其中可交互元素用序号标出，提示llm可以用序号操作。
+ * 缩进代表父子关系。
+ * 普通文本则直接列出来。
+ *
+ * @todo 数据脱敏过滤器
+ */
+declare function flatTreeToString(flatTree: FlatDomTree, include_attributes?: string[]): string;
+
+declare const getAllTextTillNextClickableElement: (node: TreeNode, maxDepth?: number) => string;
+
+declare function getElementTextMap(simplifiedHTML: string): Map<number, string>;
+
+declare function getFlatTree(config: DomConfig): FlatDomTree;
+
+declare function getSelectorMap(flatTree: FlatDomTree): Map<number, InteractiveElementDomNode>;
+
 export declare interface IdentifyOptions {
     /** JWT token from your backend for user verification */
     token: string;
@@ -372,6 +474,30 @@ export declare interface IdentityState {
     isVerified: boolean;
 }
 
+export declare interface InputSchemaFieldDef {
+    name: string;
+    type: 'string' | 'number' | 'bool';
+    required: boolean;
+    default_value?: string;
+}
+
+declare interface InteractiveElementDomNode {
+    tagName: string;
+    attributes?: Record<string, string>;
+    xpath?: string;
+    children?: string[];
+    isVisible?: boolean;
+    isTopElement?: boolean;
+    isInViewport?: boolean;
+    isInteractive: true;
+    highlightIndex: number;
+    /**
+     * 可交互元素的 dom 引用
+     */
+    ref: HTMLElement;
+    [key: string]: unknown;
+}
+
 export declare interface Message {
     id: string;
     content: string;
@@ -392,6 +518,193 @@ export declare interface NavigationRoute {
     params?: Record<string, string>;
 }
 
+/**
+ * PageController manages DOM state and element interactions.
+ * It provides async methods for all DOM operations, keeping state isolated.
+ *
+ * @lifecycle
+ * - beforeUpdate: Emitted before the DOM tree is updated.
+ * - afterUpdate: Emitted after the DOM tree is updated.
+ */
+declare class PageController extends EventTarget {
+    private config;
+    /** Corresponds to eval_page in browser-use */
+    private flatTree;
+    /**
+     * All highlighted index-mapped interactive elements
+     * Corresponds to DOMState.selector_map in browser-use
+     */
+    private selectorMap;
+    /** Index -> element text description mapping */
+    private elementTextMap;
+    /**
+     * Simplified HTML for LLM consumption.
+     * Corresponds to clickable_elements_to_string in browser-use
+     */
+    private simplifiedHTML;
+    /** last time the tree was updated */
+    private lastTimeUpdate;
+    /** Whether the tree has been indexed at least once */
+    private isIndexed;
+    /** Visual mask overlay for blocking user interaction during automation */
+    private mask;
+    private maskReady;
+    constructor(config?: PageControllerConfig);
+    /**
+     * Initialize mask asynchronously (dynamic import to avoid CSS loading in Node)
+     */
+    initMask(): void;
+    /**
+     * Get current page URL
+     */
+    getCurrentUrl(): Promise<string>;
+    /**
+     * Get last tree update timestamp
+     */
+    getLastUpdateTime(): Promise<number>;
+    /**
+     * Get structured browser state for LLM consumption.
+     * Automatically calls updateTree() to refresh the DOM state.
+     */
+    getBrowserState(): Promise<BrowserState>;
+    /**
+     * Update DOM tree, returns simplified HTML for LLM.
+     * This is the main method to refresh the page state.
+     * Automatically bypasses mask during DOM extraction if enabled.
+     */
+    updateTree(): Promise<string>;
+    /**
+     * Clean up all element highlights
+     */
+    cleanUpHighlights(): Promise<void>;
+    /**
+     * Ensure the tree has been indexed before any index-based operation.
+     * Throws if updateTree() hasn't been called yet.
+     */
+    private assertIndexed;
+    /**
+     * Click element by index
+     */
+    clickElement(index: number): Promise<ActionResult>;
+    /**
+     * Input text into element by index
+     */
+    inputText(index: number, text: string): Promise<ActionResult>;
+    /**
+     * Select dropdown option by index and option text
+     */
+    selectOption(index: number, optionText: string): Promise<ActionResult>;
+    /**
+     * Scroll vertically
+     */
+    scroll(options: {
+        down: boolean;
+        numPages: number;
+        pixels?: number;
+        index?: number;
+    }): Promise<ActionResult>;
+    /**
+     * Scroll horizontally
+     */
+    scrollHorizontally(options: {
+        right: boolean;
+        pixels: number;
+        index?: number;
+    }): Promise<ActionResult>;
+    /**
+     * Execute arbitrary JavaScript on the page
+     */
+    executeJavascript(script: string): Promise<ActionResult>;
+    /**
+     * Find an interactive element index by visible text (exact match).
+     * Scans the elementTextMap for a match.
+     */
+    findElementByText(text: string, exact?: boolean): number | null;
+    /**
+     * Find an interactive element index by CSS selector.
+     * Queries the DOM, then reverse-looks up in selectorMap.
+     */
+    findElementBySelector(cssSelector: string): number | null;
+    /**
+     * Find an interactive element index by XPath.
+     * Evaluates the XPath, then reverse-looks up in selectorMap.
+     */
+    findElementByXPath(xpath: string): number | null;
+    /**
+     * Find an interactive element index by aria-label attribute.
+     */
+    findElementByAriaLabel(label: string): number | null;
+    /**
+     * Find an interactive element index by placeholder text.
+     */
+    findElementByPlaceholder(placeholder: string): number | null;
+    /**
+     * Multi-strategy element finder. Tries strategies in priority order.
+     * Returns the index of the first match, or null if nothing found.
+     */
+    findElementByStrategies(strategies: Array<{
+        type: 'text_exact' | 'text_contains' | 'aria_label' | 'placeholder' | 'css' | 'xpath';
+        value: string;
+        priority: number;
+    }>): number | null;
+    /**
+     * Press a key on an element (or the active element if no index provided).
+     */
+    pressKey(key: string, index?: number): Promise<ActionResult>;
+    /**
+     * Navigate to a URL. Restricts to same-origin http(s) URLs for security.
+     */
+    navigateToUrl(url: string): Promise<ActionResult>;
+    /**
+     * Go back in browser history.
+     */
+    goBack(): Promise<ActionResult>;
+    /**
+     * Go forward in browser history.
+     */
+    goForward(): Promise<ActionResult>;
+    /**
+     * Wait until an element matching the given strategies appears in the DOM,
+     * or until the timeout is reached. Refreshes the DOM tree on each poll.
+     */
+    waitForElement(strategies: Array<{
+        type: 'text_exact' | 'text_contains' | 'aria_label' | 'placeholder' | 'css' | 'xpath';
+        value: string;
+        priority: number;
+    }>, timeoutMs?: number, pollIntervalMs?: number): Promise<number | null>;
+    /**
+     * Get the current selectorMap (for external use like WorkflowExecutor).
+     */
+    getSelectorMap(): Map<number, InteractiveElementDomNode>;
+    /**
+     * Get the current elementTextMap (for external use like WorkflowExecutor).
+     */
+    getElementTextMap(): Map<number, string>;
+    /**
+     * Show the visual mask overlay.
+     * Only works after mask is setup.
+     */
+    showMask(): Promise<void>;
+    /**
+     * Hide the visual mask overlay.
+     * Only works after mask is setup.
+     */
+    hideMask(): Promise<void>;
+    /**
+     * Dispose and clean up resources
+     */
+    dispose(): void;
+}
+
+/**
+ * Configuration for PageController
+ */
+declare interface PageControllerConfig extends dom.DomConfig {
+    viewportExpansion?: number;
+    /** Enable visual mask overlay during operations (default: false) */
+    enableMask?: boolean;
+}
+
 /**
  * Parse SSE chunk into lines and extract data
  */
@@ -402,6 +715,26 @@ export declare function parseSSEChunk(chunk: string): Generator<string>;
  */
 export declare function parseSSEData(data: string): StreamEvent | null;
 
+export declare interface RecordedStepDef {
+    type: 'click' | 'input' | 'select' | 'keypress' | 'navigation';
+    description: string;
+    target_text: string;
+    selector_strategies: SelectorStrategy[];
+    value?: string;
+    url?: string;
+    element_tag: string;
+    container_hint?: string;
+}
+
+export declare interface RecordedWorkflowDef {
+    id: string;
+    name: string;
+    description?: string;
+    steps: RecordedStepDef[];
+    input_schema: InputSchemaFieldDef[];
+    enabled: boolean;
+}
+
 /**
  * Resolve a route template by interpolating :param placeholders with actual values.
  *
@@ -411,6 +744,21 @@ export declare function parseSSEData(data: string): StreamEvent | null;
  */
 export declare function resolveRoute(routes: NavigationRoute[], pageName: string, params?: Record<string, unknown>): string | null;
 
+export declare interface SelectorStrategy {
+    type: 'text_exact' | 'text_contains' | 'aria_label' | 'placeholder' | 'css' | 'xpath';
+    value: string;
+    priority: number;
+}
+
+export declare interface StepResult {
+    step_index: number;
+    step_type: string;
+    description: string;
+    success: boolean;
+    message: string;
+    element_index?: number;
+}
+
 export declare type StreamEvent = {
     type: 'content';
     text: string;
@@ -481,6 +829,13 @@ export declare type StreamEvent = {
  */
 export declare function streamResponse(response: Response, signal?: AbortSignal): AsyncGenerator<StreamEvent>;
 
+declare interface TextDomNode {
+    type: 'TEXT_NODE';
+    text: string;
+    isVisible: boolean;
+    [key: string]: unknown;
+}
+
 export declare type ToolHandler = (args: Record<string, unknown>) => Promise<ToolResult> | ToolResult;
 
 export declare type ToolHandlers = Record<string, ToolHandler>;
@@ -515,6 +870,66 @@ export declare interface ToolResult {
     error?: string;
 }
 
+/**
+ * elementsToString 内部使用的类型
+ */
+declare interface TreeNode {
+    type: 'text' | 'element';
+    parent: TreeNode | null;
+    children: TreeNode[];
+    isVisible: boolean;
+    text?: string;
+    tagName?: string;
+    attributes?: Record<string, string>;
+    isInteractive?: boolean;
+    isTopElement?: boolean;
+    isNew?: boolean;
+    highlightIndex?: number;
+    extra?: Record<string, any>;
+}
+
+export declare class WorkflowExecutor {
+    private controller;
+    private config;
+    constructor(controller: PageController, config?: WorkflowExecutorConfig);
+    /**
+     * Execute a recorded workflow with variable substitution.
+     */
+    execute(workflow: RecordedWorkflowDef, inputs?: Record<string, string | number | boolean>): Promise<WorkflowResult>;
+    /**
+     * Execute a single step.
+     */
+    private executeStep;
+    /**
+     * Replace {variable_name} placeholders in step fields with actual values.
+     */
+    private resolveVariables;
+    private delay;
+}
+
+export declare interface WorkflowExecutorConfig {
+    /** Time to wait for elements to appear after actions (ms). Default: 3000 */
+    waitTimeout?: number;
+    /** Time between DOM polls when waiting for elements (ms). Default: 300 */
+    pollInterval?: number;
+    /** Default delay between steps (ms). Default: 500 */
+    stepDelay?: number;
+    /** Stop execution on first failure. Default: false */
+    stopOnFailure?: boolean;
+    /** Callback for step progress updates */
+    onStepProgress?: (result: StepResult) => void;
+}
+
+export declare interface WorkflowResult {
+    success: boolean;
+    workflow_name: string;
+    total_steps: number;
+    completed_steps: number;
+    failed_steps: number;
+    step_results: StepResult[];
+    error?: string;
+}
+
 export declare interface WorkflowTodo {
     id: string;
     text: string;