openclaw-node-harness 2.0.0

package/skills/fast-browser-use/src/dom/mod.rs

@@ -0,0 +1,14 @@
+//! DOM extraction and manipulation module
+//!
+//! This module provides functionality for extracting and working with the DOM structure
+//! of web pages. It includes:
+//! - ElementNode: Representation of DOM elements
+//! - DomTree: Complete DOM tree with indexing for interactive elements
+
+pub mod element;
+pub mod tree;
+pub mod yaml;
+
+pub use element::{AriaChild, AriaNode, BoundingBox, ElementNode};
+pub use tree::DomTree;
+pub use yaml::{yaml_escape_key_if_needed, yaml_escape_value_if_needed};

package/skills/fast-browser-use/src/dom/tree.rs

@@ -0,0 +1,296 @@
+use crate::{dom::element::{AriaChild, AriaNode},
+            error::{BrowserError, Result}};
+use headless_chrome::Tab;
+use std::sync::Arc;
+
+/// Represents the ARIA snapshot of a web page
+/// Based on Playwright's AriaSnapshot structure
+#[derive(Debug, Clone)]
+pub struct DomTree {
+    /// Root AriaNode (usually a fragment)
+    pub root: AriaNode,
+
+    /// Array of CSS selectors indexed by element index
+    pub selectors: Vec<String>,
+
+    /// List of iframe indices (for multi-frame snapshots)
+    pub iframe_indices: Vec<usize>,
+}
+
+/// Snapshot extraction response from JavaScript
+#[derive(Debug, serde::Deserialize)]
+struct SnapshotResponse {
+    root: AriaNode,
+    selectors: Vec<String>,
+    #[serde(rename = "iframeIndices")]
+    iframe_indices: Vec<usize>,
+}
+
+impl DomTree {
+    /// Create a new DomTree from an AriaNode
+    pub fn new(root: AriaNode) -> Self {
+        let mut tree = Self { root, selectors: Vec::new(), iframe_indices: Vec::new() };
+        tree.rebuild_maps();
+        tree
+    }
+
+    /// Build DOM tree from a browser tab
+    pub fn from_tab(tab: &Arc<Tab>) -> Result<Self> {
+        Self::from_tab_with_prefix(tab, "")
+    }
+
+    /// Build DOM tree from a browser tab with a ref prefix (for iframe handling)
+    pub fn from_tab_with_prefix(tab: &Arc<Tab>, _ref_prefix: &str) -> Result<Self> {
+        // Note: ref_prefix is deprecated but kept for API compatibility
+        // JavaScript code to extract ARIA snapshot
+        let js_code = include_str!("extract_dom.js");
+
+        // Execute JavaScript to extract DOM
+        let result = tab
+            .evaluate(js_code, false)
+            .map_err(|e| BrowserError::DomParseFailed(format!("Failed to execute DOM extraction script: {}", e)))?;
+
+        // Get the JSON string value
+        let json_value = result
+            .value
+            .ok_or_else(|| BrowserError::DomParseFailed("No value returned from DOM extraction".to_string()))?;
+
+        // The JavaScript returns a JSON string, so we need to parse it as a string first
+        let json_str: String = serde_json::from_value(json_value)
+            .map_err(|e| BrowserError::DomParseFailed(format!("Failed to get JSON string: {}", e)))?;
+
+        // Then parse the JSON string into SnapshotResponse
+        let response: SnapshotResponse = serde_json::from_str(&json_str)
+            .map_err(|e| BrowserError::DomParseFailed(format!("Failed to parse snapshot JSON: {}", e)))?;
+
+        Ok(Self { root: response.root, selectors: response.selectors, iframe_indices: response.iframe_indices })
+    }
+
+    /// Rebuild the selectors array by traversing the tree
+    /// Note: This only resizes the array based on indices found.
+    /// Actual selectors are populated from JavaScript extraction.
+    fn rebuild_maps(&mut self) {
+        self.iframe_indices.clear();
+
+        // Find the maximum index in the tree
+        let max_index = self.find_max_index(&self.root.clone());
+
+        // Resize selectors array if needed
+        if let Some(max_idx) = max_index {
+            if self.selectors.len() <= max_idx {
+                self.selectors.resize(max_idx + 1, String::new());
+            }
+        }
+
+        // Collect iframe indices
+        let root = self.root.clone();
+        self.collect_iframe_indices(&root);
+    }
+
+    fn find_max_index(&self, node: &AriaNode) -> Option<usize> {
+        let mut max = node.index;
+
+        for child in &node.children {
+            if let AriaChild::Node(child_node) = child {
+                if let Some(child_max) = self.find_max_index(child_node) {
+                    max = match max {
+                        Some(current) => Some(current.max(child_max)),
+                        None => Some(child_max),
+                    };
+                }
+            }
+        }
+
+        max
+    }
+
+    fn collect_iframe_indices(&mut self, node: &AriaNode) {
+        if let Some(index) = node.index {
+            if node.role == "iframe" {
+                self.iframe_indices.push(index);
+            }
+        }
+
+        for child in &node.children {
+            if let AriaChild::Node(child_node) = child {
+                self.collect_iframe_indices(child_node);
+            }
+        }
+    }
+
+    /// Get CSS selector for a given index
+    pub fn get_selector(&self, index: usize) -> Option<&String> {
+        self.selectors.get(index).filter(|s| !s.is_empty())
+    }
+
+    /// Get all interactive element indices
+    pub fn interactive_indices(&self) -> Vec<usize> {
+        let mut indices = Vec::new();
+        self.collect_indices(&self.root, &mut indices);
+        indices.sort();
+        indices
+    }
+
+    fn collect_indices(&self, node: &AriaNode, indices: &mut Vec<usize>) {
+        if let Some(index) = node.index {
+            indices.push(index);
+        }
+        for child in &node.children {
+            if let AriaChild::Node(child_node) = child {
+                self.collect_indices(child_node, indices);
+            }
+        }
+    }
+
+    /// Count total nodes in the tree
+    pub fn count_nodes(&self) -> usize {
+        self.root.count_nodes()
+    }
+
+    /// Count interactive elements (elements with indices)
+    pub fn count_interactive(&self) -> usize {
+        self.root.count_interactive()
+    }
+
+    /// Find node by index
+    pub fn find_node_by_index(&self, index: usize) -> Option<&AriaNode> {
+        self.root.find_by_index(index)
+    }
+
+    /// Find node by index (mutable)
+    pub fn find_node_by_index_mut(&mut self, index: usize) -> Option<&mut AriaNode> {
+        self.root.find_by_index_mut(index)
+    }
+
+    /// Get all iframe indices for multi-frame snapshot handling
+    pub fn get_iframe_indices(&self) -> &[usize] {
+        &self.iframe_indices
+    }
+
+    /// Convert the DOM tree to JSON
+    pub fn to_json(&self) -> Result<String> {
+        serde_json::to_string_pretty(&self.root)
+            .map_err(|e| BrowserError::DomParseFailed(format!("Failed to serialize DOM to JSON: {}", e)))
+    }
+
+    /// Replace an iframe node's children with content from another snapshot
+    /// Used for multi-frame snapshot assembly
+    pub fn inject_iframe_content(&mut self, iframe_index: usize, iframe_snapshot: DomTree) {
+        if let Some(iframe_node) = self.find_node_by_index_mut(iframe_index) {
+            // Replace iframe's children with the snapshot's root children
+            iframe_node.children = iframe_snapshot.root.children;
+
+            // Merge selectors (offset by current length)
+            let offset = self.selectors.len();
+            for selector in iframe_snapshot.selectors {
+                if !selector.is_empty() {
+                    self.selectors.push(selector);
+                }
+            }
+
+            // Update iframe indices with offset
+            for idx in iframe_snapshot.iframe_indices {
+                self.iframe_indices.push(idx + offset);
+            }
+        }
+    }
+
+    /// Create a snapshot with multiple frames assembled
+    /// Takes a function that can retrieve snapshots for iframe elements
+    pub fn assemble_with_iframes<F>(mut self, mut get_iframe_snapshot: F) -> Self
+    where
+        F: FnMut(usize) -> Option<DomTree>,
+    {
+        let iframe_indices = self.iframe_indices.clone();
+
+        for iframe_index in iframe_indices {
+            if let Some(iframe_snapshot) = get_iframe_snapshot(iframe_index) {
+                self.inject_iframe_content(iframe_index, iframe_snapshot);
+            }
+        }
+
+        self
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn create_test_tree() -> AriaNode {
+        let mut root = AriaNode::fragment();
+
+        root.children.push(AriaChild::Node(Box::new(
+            AriaNode::new("button", "Click me").with_index(0).with_box(true, Some("pointer".to_string())),
+        )));
+
+        root.children
+            .push(AriaChild::Node(Box::new(AriaNode::new("link", "Go to page").with_index(1).with_box(true, None))));
+
+        root.children.push(AriaChild::Node(Box::new(
+            AriaNode::new("paragraph", "").with_child(AriaChild::Text("Some text".to_string())),
+        )));
+
+        root
+    }
+
+    #[test]
+    fn test_find_node_by_index() {
+        let root = create_test_tree();
+        let tree = DomTree::new(root);
+
+        let button = tree.find_node_by_index(0);
+        assert!(button.is_some());
+        assert_eq!(button.unwrap().role, "button");
+        assert_eq!(button.unwrap().name, "Click me");
+
+        let not_found = tree.find_node_by_index(999);
+        assert!(not_found.is_none());
+    }
+
+    #[test]
+    fn test_count_nodes() {
+        let root = create_test_tree();
+        let tree = DomTree::new(root);
+
+        // fragment + button + link + paragraph = 4
+        assert_eq!(tree.count_nodes(), 4);
+    }
+
+    #[test]
+    fn test_interactive_indices() {
+        let root = create_test_tree();
+        let tree = DomTree::new(root);
+
+        let indices = tree.interactive_indices();
+        assert_eq!(indices.len(), 2);
+        assert!(indices.contains(&0));
+        assert!(indices.contains(&1));
+    }
+
+    #[test]
+    fn test_inject_iframe_content() {
+        let mut main_tree = AriaNode::fragment();
+        main_tree.children.push(AriaChild::Node(Box::new(AriaNode::new("iframe", "").with_index(0))));
+
+        let mut iframe_tree = AriaNode::fragment();
+        iframe_tree.children.push(AriaChild::Node(Box::new(AriaNode::new("button", "Inside iframe").with_index(0))));
+
+        let mut main = DomTree::new(main_tree);
+        let iframe = DomTree::new(iframe_tree);
+
+        main.inject_iframe_content(0, iframe);
+
+        // Check that iframe now has the button as a child
+        let iframe_node = main.find_node_by_index(0).unwrap();
+        assert_eq!(iframe_node.children.len(), 1);
+
+        match &iframe_node.children[0] {
+            AriaChild::Node(n) => {
+                assert_eq!(n.role, "button");
+                assert_eq!(n.name, "Inside iframe");
+            }
+            _ => panic!("Expected node child"),
+        }
+    }
+}

package/skills/fast-browser-use/src/dom/yaml.rs

@@ -0,0 +1,149 @@
+// YAML rendering utilities for ARIA snapshots
+// Based on Playwright's yaml.ts and renderAriaTree logic
+
+/// Check if a YAML string needs quotes
+pub fn yaml_string_needs_quotes(s: &str) -> bool {
+    if s.is_empty() {
+        return true;
+    }
+
+    // Leading or trailing whitespace
+    if s.starts_with(char::is_whitespace) || s.ends_with(char::is_whitespace) {
+        return true;
+    }
+
+    // Control characters
+    if s.chars().any(|c| c.is_control()) {
+        return true;
+    }
+
+    // Starts with '-'
+    if s.starts_with('-') {
+        return true;
+    }
+
+    // Contains ':' or '\n' followed by space or at end
+    if s.contains(":\n") || s.contains(": ") || s.ends_with(':') {
+        return true;
+    }
+
+    // Contains '#' preceded by space (comment indicator)
+    if s.contains(" #") {
+        return true;
+    }
+
+    // Line breaks
+    if s.contains('\n') || s.contains('\r') {
+        return true;
+    }
+
+    // Indicator characters
+    if s.starts_with(['&', '*', ',', '?', '!', '>', '|', '@', '"', '\'', '#', '%']) {
+        return true;
+    }
+
+    // Contains quotes or backslashes (need escaping)
+    if s.contains('"') || s.contains('\\') || s.contains('\'') {
+        return true;
+    }
+
+    // Special characters
+    if s.chars().any(|c| matches!(c, '{' | '}' | '`')) {
+        return true;
+    }
+
+    // Starts with '['
+    if s.starts_with('[') {
+        return true;
+    }
+
+    // YAML boolean/null values
+    let lower = s.to_lowercase();
+    if matches!(lower.as_str(), "y" | "n" | "yes" | "no" | "true" | "false" | "on" | "off" | "null") {
+        return true;
+    }
+
+    // Numeric values
+    if s.parse::<f64>().is_ok() {
+        return true;
+    }
+
+    false
+}
+
+/// Escape a YAML key if needed (uses single quotes)
+pub fn yaml_escape_key_if_needed(s: &str) -> String {
+    if !yaml_string_needs_quotes(s) {
+        return s.to_string();
+    }
+
+    // Use single quotes and escape single quotes by doubling them
+    format!("'{}'", s.replace('\'', "''"))
+}
+
+/// Escape a YAML value if needed (uses double quotes)
+pub fn yaml_escape_value_if_needed(s: &str) -> String {
+    if !yaml_string_needs_quotes(s) {
+        return s.to_string();
+    }
+
+    // Use double quotes and escape special characters
+    let mut result = String::from('"');
+
+    for ch in s.chars() {
+        match ch {
+            '\\' => result.push_str("\\\\"),
+            '"' => result.push_str("\\\""),
+            '\x08' => result.push_str("\\b"),
+            '\x0C' => result.push_str("\\f"),
+            '\n' => result.push_str("\\n"),
+            '\r' => result.push_str("\\r"),
+            '\t' => result.push_str("\\t"),
+            c if c.is_control() => {
+                // Escape control characters as \xNN
+                result.push_str(&format!("\\x{:02x}", c as u32));
+            }
+            c => result.push(c),
+        }
+    }
+
+    result.push('"');
+    result
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_yaml_string_needs_quotes() {
+        assert!(yaml_string_needs_quotes("")); // empty
+        assert!(yaml_string_needs_quotes(" text")); // leading space
+        assert!(yaml_string_needs_quotes("text ")); // trailing space
+        assert!(yaml_string_needs_quotes("-text")); // starts with dash
+        assert!(yaml_string_needs_quotes("foo: bar")); // contains colon-space
+        assert!(yaml_string_needs_quotes("foo #bar")); // contains space-hash
+        assert!(yaml_string_needs_quotes("true")); // boolean
+        assert!(yaml_string_needs_quotes("123")); // number
+        assert!(yaml_string_needs_quotes("[array]")); // starts with bracket
+
+        assert!(!yaml_string_needs_quotes("simple"));
+        assert!(!yaml_string_needs_quotes("hello-world"));
+        assert!(!yaml_string_needs_quotes("foo_bar"));
+    }
+
+    #[test]
+    fn test_yaml_escape_key_if_needed() {
+        assert_eq!(yaml_escape_key_if_needed("simple"), "simple");
+        assert_eq!(yaml_escape_key_if_needed("it's"), "'it''s'");
+        assert_eq!(yaml_escape_key_if_needed("-start"), "'-start'");
+    }
+
+    #[test]
+    fn test_yaml_escape_value_if_needed() {
+        assert_eq!(yaml_escape_value_if_needed("simple"), "simple");
+        assert_eq!(yaml_escape_value_if_needed("hello\nworld"), "\"hello\\nworld\"");
+        assert_eq!(yaml_escape_value_if_needed("quote\"here"), "\"quote\\\"here\"");
+        assert_eq!(yaml_escape_value_if_needed("back\\slash"), "\"back\\\\slash\"");
+    }
+}

package/skills/fast-browser-use/src/error.rs

@@ -0,0 +1,115 @@
+use thiserror::Error;
+
+/// Core error type for browser-use operations
+#[derive(Error, Debug)]
+pub enum BrowserError {
+    /// Browser launch failed
+    #[error("Failed to launch browser: {0}")]
+    LaunchFailed(String),
+
+    /// Browser connection failed
+    #[error("Failed to connect to browser: {0}")]
+    ConnectionFailed(String),
+
+    /// Operation timed out
+    #[error("Operation timed out: {0}")]
+    Timeout(String),
+
+    /// Invalid CSS selector
+    #[error("Invalid selector: {0}")]
+    SelectorInvalid(String),
+
+    /// Element not found in DOM
+    #[error("Element not found: {0}")]
+    ElementNotFound(String),
+
+    /// DOM parsing failed
+    #[error("Failed to parse DOM: {0}")]
+    DomParseFailed(String),
+
+    /// Tool execution failed
+    #[error("Tool '{tool}' execution failed: {reason}")]
+    ToolExecutionFailed { tool: String, reason: String },
+
+    /// Invalid argument provided to a function
+    #[error("Invalid argument: {0}")]
+    InvalidArgument(String),
+
+    /// Navigation failed
+    #[error("Navigation failed: {0}")]
+    NavigationFailed(String),
+
+    /// JavaScript evaluation failed
+    #[error("JavaScript evaluation failed: {0}")]
+    EvaluationFailed(String),
+
+    /// Screenshot capture failed
+    #[error("Screenshot failed: {0}")]
+    ScreenshotFailed(String),
+
+    /// Download operation failed
+    #[error("Download failed: {0}")]
+    DownloadFailed(String),
+
+    /// Tab operation failed
+    #[error("Tab operation failed: {0}")]
+    TabOperationFailed(String),
+
+    /// Chrome/CDP error from headless_chrome crate
+    #[error("Chrome error: {0}")]
+    ChromeError(String),
+
+    /// JSON serialization/deserialization error
+    #[error("JSON error: {0}")]
+    JsonError(#[from] serde_json::Error),
+
+    /// IO error
+    #[error("IO error: {0}")]
+    IoError(#[from] std::io::Error),
+}
+
+/// Result type alias for browser-use operations
+pub type Result<T> = std::result::Result<T, BrowserError>;
+
+/// Convert anyhow::Error from headless_chrome to BrowserError
+impl From<anyhow::Error> for BrowserError {
+    fn from(err: anyhow::Error) -> Self {
+        BrowserError::ChromeError(err.to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_error_display() {
+        let err = BrowserError::LaunchFailed("Chrome not found".to_string());
+        assert_eq!(err.to_string(), "Failed to launch browser: Chrome not found");
+    }
+
+    #[test]
+    fn test_tool_execution_error() {
+        let err = BrowserError::ToolExecutionFailed { tool: "navigate".to_string(), reason: "Invalid URL".to_string() };
+        assert_eq!(err.to_string(), "Tool 'navigate' execution failed: Invalid URL");
+    }
+
+    #[test]
+    fn test_json_error_conversion() {
+        let json_err = serde_json::from_str::<serde_json::Value>("invalid json");
+        assert!(json_err.is_err());
+
+        let browser_err: BrowserError = json_err.unwrap_err().into();
+        assert!(matches!(browser_err, BrowserError::JsonError(_)));
+    }
+
+    #[test]
+    fn test_result_type_alias() {
+        fn example_function() -> Result<String> {
+            Err(BrowserError::InvalidArgument("test".to_string()))
+        }
+
+        let result = example_function();
+        assert!(result.is_err());
+    }
+}

package/skills/fast-browser-use/src/lib.rs

@@ -0,0 +1,17 @@
+pub mod browser;
+pub mod dom;
+pub mod error;
+pub mod tools;
+
+#[cfg(feature = "mcp-handler")]
+pub mod mcp;
+
+pub use browser::{BrowserSession, ConnectionOptions, LaunchOptions};
+pub use dom::{BoundingBox, DomTree, ElementNode};
+pub use error::{BrowserError, Result};
+pub use tools::{Tool, ToolContext, ToolRegistry, ToolResult};
+
+#[cfg(feature = "mcp-handler")]
+pub use mcp::BrowserServer;
+#[cfg(feature = "mcp-handler")]
+pub use rmcp::ServiceExt;

package/skills/fast-browser-use/src/mcp/handler.rs

@@ -0,0 +1,63 @@
+//! ServerHandler implementation for BrowserSession
+
+use crate::browser::BrowserSession;
+use log::debug;
+use rmcp::{ServerHandler,
+           handler::server::tool::ToolRouter,
+           model::{ServerCapabilities, ServerInfo},
+           tool_handler};
+use std::sync::{Arc, Mutex};
+
+/// MCP Server wrapper for BrowserSession
+///
+/// This struct holds a browser session and provides thread-safe access
+/// for MCP tool execution.
+#[derive(Clone)]
+pub struct BrowserServer {
+    session: Arc<Mutex<BrowserSession>>,
+    tool_router: ToolRouter<Self>,
+}
+
+impl BrowserServer {
+    /// Create a new browser server with default launch options
+    pub fn new() -> Result<Self, String> {
+        let session = BrowserSession::new().map_err(|e| format!("Failed to launch browser: {}", e))?;
+
+        Ok(Self { session: Arc::new(Mutex::new(session)), tool_router: Self::tool_router() })
+    }
+
+    /// Create a new browser server with custom launch options
+    pub fn with_options(options: crate::browser::LaunchOptions) -> Result<Self, String> {
+        let session = BrowserSession::launch(options).map_err(|e| format!("Failed to launch browser: {}", e))?;
+
+        Ok(Self { session: Arc::new(Mutex::new(session)), tool_router: Self::tool_router() })
+    }
+
+    /// Get a reference to the browser session (blocking lock)
+    pub(crate) fn session(&self) -> std::sync::MutexGuard<'_, BrowserSession> {
+        self.session.lock().expect("Failed to lock browser session")
+    }
+}
+
+impl Default for BrowserServer {
+    fn default() -> Self {
+        Self::new().expect("Failed to create default browser server")
+    }
+}
+
+impl Drop for BrowserServer {
+    fn drop(&mut self) {
+        debug!("BrowserServer dropped");
+    }
+}
+
+#[tool_handler]
+impl ServerHandler for BrowserServer {
+    fn get_info(&self) -> ServerInfo {
+        ServerInfo {
+            instructions: Some("Browser-use MCP Server".into()),
+            capabilities: ServerCapabilities::builder().enable_tools().build(),
+            ..Default::default()
+        }
+    }
+}