openclaw-node-harness 2.0.0

package/skills/fast-browser-use/src/tools/annotate.rs

@@ -0,0 +1,165 @@
+use crate::{
+    error::{BrowserError, Result},
+    tools::{Tool, ToolContext, ToolResult},
+    dom::element::AriaChild,
+};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use image::Rgba;
+use imageproc::drawing::{draw_filled_rect_mut, draw_text_mut};
+use imageproc::rect::Rect as ImageRect;
+use rusttype::{Font, Scale};
+use std::io::Cursor;
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
+
+/// Parameters for the annotate tool
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct AnnotateParams {
+    /// Whether to return the base64 image (default: false, saves to file)
+    #[serde(default)]
+    pub return_base64: bool,
+
+    /// Path to save the annotated screenshot (if not returning base64)
+    pub path: Option<String>,
+}
+
+/// Tool for capturing a screenshot with annotated interactive elements
+#[derive(Default)]
+pub struct AnnotateTool;
+
+impl Tool for AnnotateTool {
+    type Params = AnnotateParams;
+
+    fn name(&self) -> &str {
+        "annotate"
+    }
+
+    fn execute_typed(&self, params: AnnotateParams, context: &mut ToolContext) -> Result<ToolResult> {
+        // 1. Capture screenshot
+        let screenshot_data = context
+            .session
+            .tab()?
+            .capture_screenshot(
+                headless_chrome::protocol::cdp::Page::CaptureScreenshotFormatOption::Png,
+                None,
+                None,
+                false, // Viewport only for annotation usually makes more sense to match coordinates
+            )
+            .map_err(|e| BrowserError::ScreenshotFailed(e.to_string()))?;
+
+        // 2. Extract DOM with bounding boxes
+        let dom = context.get_dom()?;
+
+        // 3. Load image
+        let mut img = image::load_from_memory(&screenshot_data)
+            .map_err(|e| BrowserError::ScreenshotFailed(format!("Failed to load screenshot image: {}", e)))?
+            .to_rgba8();
+
+        let (width, height) = img.dimensions();
+
+        // 4. Load font (using a built-in font or loading from bytes if possible, otherwise we might fail)
+        // Since we can't easily rely on system fonts in a portable way, we'll try to use a bundled font or fallback.
+        // For this environment, let's assume we can't bundle a font easily without adding it to the repo.
+        // Actually, we can use `ab_glyph` with a font file.
+        // A better approach for a self-contained binary is to include a font as bytes.
+        // Let's use a very simple fallback or a known system font path if we can't embed.
+        // Wait, `imageproc` examples often use `DejaVuSans`.
+        // I'll try to look for a system font, or if that fails, just draw boxes without text? 
+        // No, numbers are crucial.
+        // I will embed a font. `DejaVuSans.ttf` is open.
+        // Since I can't download files easily right now, I'll assume a system path or try to find one.
+        // MacOS: /System/Library/Fonts/Helvetica.ttc
+        // Linux: /usr/share/fonts/truetype/dejavu/DejaVuSans.ttf
+        
+        let font_path = if cfg!(target_os = "macos") {
+            "/System/Library/Fonts/Helvetica.ttc"
+        } else {
+             "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf"
+        };
+        
+        // Very basic font loading fallback
+        let font_bytes = std::fs::read(font_path).unwrap_or_default();
+        let font = Font::try_from_bytes(&font_bytes);
+
+        // 5. Draw annotations
+        let mut map = HashMap::new();
+        let mut valid_indices = Vec::new();
+
+        // Collect all interactive elements with valid rects
+        // We traverse the DOM tree
+        let mut queue = vec![&dom.root];
+        while let Some(node) = queue.pop() {
+            if let Some(index) = node.index {
+                if let Some(rect) = &node.box_info.rect {
+                    if rect.width > 0.0 && rect.height > 0.0 && rect.x >= 0.0 && rect.y >= 0.0 {
+                        // Check if rect is within viewport roughly
+                         if rect.x < width as f64 && rect.y < height as f64 {
+                             valid_indices.push((index, rect.clone()));
+                             
+                             // Add to selector map
+                             if let Some(selector) = dom.get_selector(index) {
+                                 map.insert(index.to_string(), selector.clone());
+                             }
+                         }
+                    }
+                }
+            }
+            
+            for child in &node.children {
+                if let AriaChild::Node(child_node) = child {
+                    queue.push(child_node);
+                }
+            }
+        }
+        
+        // Sort indices for consistent visualization if needed, but they are already indexed
+        
+        for (index, rect) in valid_indices {
+             // Draw yellow box
+             let x = rect.x as i32;
+             let y = rect.y as i32;
+             let w = rect.width as u32;
+             let h = rect.height as u32;
+             
+             // Define color: Yellow with alpha
+             let color = Rgba([255, 255, 0, 128]); // Semi-transparent yellow
+             let border_color = Rgba([255, 0, 0, 255]); // Red border
+             
+             // Draw filled rect (marker)
+             // We'll draw a small badge at the top-left corner of the element
+             let badge_size = 20;
+             let badge_rect = ImageRect::at(x, y).of_size(badge_size, badge_size);
+             
+             draw_filled_rect_mut(&mut img, badge_rect, border_color);
+             
+             // Draw text number
+             if let Some(font) = &font {
+                 let scale = Scale::uniform(16.0);
+                 let text = index.to_string();
+                 draw_text_mut(&mut img, Rgba([255, 255, 255, 255]), x + 2, y + 2, scale, font, &text);
+             }
+        }
+
+        // 6. Save or return
+        let mut bytes: Vec<u8> = Vec::new();
+        img.write_to(&mut Cursor::new(&mut bytes), image::ImageOutputFormat::Png)
+             .map_err(|e| BrowserError::ScreenshotFailed(format!("Failed to encode annotated image: {}", e)))?;
+
+        let mut result_data = serde_json::Map::new();
+        result_data.insert("map".to_string(), serde_json::to_value(&map).unwrap());
+        
+        if params.return_base64 {
+            let base64_string = BASE64.encode(&bytes);
+            result_data.insert("image_base64".to_string(), serde_json::Value::String(base64_string));
+        }
+        
+        if let Some(path) = params.path {
+            std::fs::write(&path, &bytes)
+                .map_err(|e| BrowserError::ScreenshotFailed(format!("Failed to save annotated screenshot: {}", e)))?;
+            result_data.insert("path".to_string(), serde_json::Value::String(path));
+        }
+
+        Ok(ToolResult::success(Some(serde_json::Value::Object(result_data))))
+    }
+}

package/skills/fast-browser-use/src/tools/click.rs

@@ -0,0 +1,84 @@
+use crate::{error::{BrowserError, Result},
+            tools::{Tool, ToolContext, ToolResult}};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+/// Parameters for the click tool
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct ClickParams {
+    /// CSS selector (use either this or index, not both)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub selector: Option<String>,
+
+    /// Element index from DOM tree (use either this or selector, not both)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub index: Option<usize>,
+}
+
+/// Tool for clicking elements
+#[derive(Default)]
+pub struct ClickTool;
+
+impl Tool for ClickTool {
+    type Params = ClickParams;
+
+    fn name(&self) -> &str {
+        "click"
+    }
+
+    fn execute_typed(&self, params: ClickParams, context: &mut ToolContext) -> Result<ToolResult> {
+        // Validate that exactly one selector method is provided
+        match (&params.selector, &params.index) {
+            (Some(_), Some(_)) => {
+                return Err(BrowserError::ToolExecutionFailed {
+                    tool: "click".to_string(),
+                    reason: "Cannot specify both 'selector' and 'index'. Use one or the other.".to_string(),
+                });
+            }
+            (None, None) => {
+                return Err(BrowserError::ToolExecutionFailed {
+                    tool: "click".to_string(),
+                    reason: "Must specify either 'selector' or 'index'.".to_string(),
+                });
+            }
+            _ => {}
+        }
+
+        if let Some(selector) = params.selector {
+            // CSS selector path
+            let tab = context.session.tab()?;
+            let element = context.session.find_element(&tab, &selector)?;
+            element
+                .click()
+                .map_err(|e| BrowserError::ToolExecutionFailed { tool: "click".to_string(), reason: e.to_string() })?;
+
+            Ok(ToolResult::success_with(serde_json::json!({
+                "selector": selector,
+                "method": "css"
+            })))
+        } else if let Some(index) = params.index {
+            // Index path - convert index to CSS selector
+            let css_selector = {
+                let dom = context.get_dom()?;
+                let selector = dom
+                    .get_selector(index)
+                    .ok_or_else(|| BrowserError::ElementNotFound(format!("No element with index {}", index)))?;
+                selector.clone()
+            };
+
+            let tab = context.session.tab()?;
+            let element = context.session.find_element(&tab, &css_selector)?;
+            element
+                .click()
+                .map_err(|e| BrowserError::ToolExecutionFailed { tool: "click".to_string(), reason: e.to_string() })?;
+
+            Ok(ToolResult::success_with(serde_json::json!({
+                "index": index,
+                "selector": css_selector,
+                "method": "index"
+            })))
+        } else {
+            unreachable!("Validation above ensures one field is Some")
+        }
+    }
+}

package/skills/fast-browser-use/src/tools/close.rs

@@ -0,0 +1,35 @@
+use crate::{error::{BrowserError, Result},
+            tools::{Tool, ToolContext, ToolResult}};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+/// Parameters for the close tool (no parameters needed)
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct CloseParams {}
+
+/// Tool for closing the browser
+#[derive(Default)]
+pub struct CloseTool;
+
+impl Tool for CloseTool {
+    type Params = CloseParams;
+
+    fn name(&self) -> &str {
+        "close"
+    }
+
+    fn execute_typed(&self, _params: CloseParams, context: &mut ToolContext) -> Result<ToolResult> {
+        // Note: Closing the browser via BrowserSession is tricky because we hold a reference
+        // In a real implementation, this would need to signal the session owner to close
+        // For now, we'll close all tabs as a proxy for closing the browser
+
+        context
+            .session
+            .close()
+            .map_err(|e| BrowserError::ToolExecutionFailed { tool: "close".to_string(), reason: e.to_string() })?;
+
+        Ok(ToolResult::success_with(serde_json::json!({
+            "message": "Browser closed successfully"
+        })))
+    }
+}

package/skills/fast-browser-use/src/tools/close_tab.rs

@@ -0,0 +1,45 @@
+use crate::{error::Result,
+            tools::{Tool, ToolContext, ToolResult}};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+/// Parameters for the close_tab tool (no parameters needed)
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct CloseTabParams {}
+
+/// Tool for closing the current active tab
+#[derive(Default)]
+pub struct CloseTabTool;
+
+impl Tool for CloseTabTool {
+    type Params = CloseTabParams;
+
+    fn name(&self) -> &str {
+        "close_tab"
+    }
+
+    fn execute_typed(&self, _params: CloseTabParams, context: &mut ToolContext) -> Result<ToolResult> {
+        // Get the current tab info before closing
+        let active_tab = context.session.tab()?;
+        let tab_title = active_tab.get_title().unwrap_or_default();
+        let tab_url = active_tab.get_url();
+
+        // Get the current tab index
+        let tabs = context.session.get_tabs()?;
+        let current_index = tabs.iter().position(|tab| std::sync::Arc::ptr_eq(tab, &active_tab)).unwrap_or(0);
+
+        // Close the active tab
+        active_tab
+            .close(true)
+            .map_err(|e| crate::error::BrowserError::TabOperationFailed(format!("Failed to close tab: {}", e)))?;
+
+        let message = format!("Closed tab [{}]: {} ({})", current_index, tab_title, tab_url);
+
+        Ok(ToolResult::success_with(serde_json::json!({
+            "index": current_index,
+            "title": tab_title,
+            "url": tab_url,
+            "message": message
+        })))
+    }
+}

package/skills/fast-browser-use/src/tools/convert_to_markdown.js

@@ -0,0 +1,117 @@
+// Enhanced JavaScript code to convert HTML to markdown using Mozilla Readability
+// Based on UI-TARS implementation
+// Returns JSON string with title, HTML content, and metadata
+
+(function () {
+  try {
+    // Load Readability constructor from the injected script
+    // The script should be injected as READABILITY_SCRIPT constant
+    if (typeof READABILITY_SCRIPT === "undefined") {
+      throw new Error(
+        "READABILITY_SCRIPT not defined. Readability.js must be injected first.",
+      );
+    }
+
+    // Always re-evaluate Readability to ensure fresh instance
+    // This prevents issues with state pollution between multiple calls
+    // Use Function constructor instead of eval to create a clean scope
+    var ReadabilityConstructor;
+    var __readabilityModule = { exports: {} };
+
+    // Create an isolated function scope for Readability
+    var loadReadability = new Function(
+      "module",
+      "exports",
+      READABILITY_SCRIPT + "; return module.exports;",
+    );
+    ReadabilityConstructor = loadReadability(
+      __readabilityModule,
+      __readabilityModule.exports,
+    );
+
+    if (!ReadabilityConstructor) {
+      throw new Error("Failed to load Readability constructor");
+    }
+
+    // Clone the document to avoid DOM flickering (visual artifacts)
+    // This prevents the page from changing appearance during extraction
+    // Use deep clone with true parameter to ensure all children are cloned
+    var documentClone = document.cloneNode(true);
+
+    // Clean up unwanted elements from the clone
+    // These elements don't contribute to the main content and can interfere with extraction
+    var elementsToRemove = [
+      "script", // JavaScript code
+      "noscript", // Fallback content
+      "style", // CSS styles
+      "link", // External resources
+      "svg", // Vector graphics
+      "img", // Images (handled separately by Readability)
+      "video", // Videos
+      "iframe", // Embedded frames
+      "canvas", // Canvas elements
+      ".reflist", // Reference lists (Wikipedia-style)
+    ];
+
+    documentClone
+      .querySelectorAll(elementsToRemove.join(","))
+      .forEach(function (el) {
+        el.remove();
+      });
+
+    // Use Mozilla Readability algorithm to extract main content
+    // This filters out navigation, ads, sidebars, etc.
+    var reader = new ReadabilityConstructor(documentClone);
+    var article = reader.parse();
+
+    if (!article) {
+      // Readability failed to extract content, fall back to basic extraction
+      // This can happen on pages with insufficient content or unusual structure
+      var fallbackContent = document.body ? document.body.innerHTML : "";
+      var fallbackText = document.body ? document.body.textContent : "";
+
+      return JSON.stringify({
+        title: document.title || "",
+        content: fallbackContent,
+        textContent: fallbackText,
+        url: window.location.href,
+        excerpt: "",
+        byline: "",
+        siteName: "",
+        length: fallbackText.length,
+        lang: document.documentElement.lang || "",
+        dir: document.documentElement.dir || "",
+        publishedTime: "",
+        readabilityFailed: true,
+        error: "Readability.parse() returned null - using fallback extraction",
+      });
+    }
+
+    // Return structured data as JSON string
+    // The HTML content will be converted to Markdown on the Rust side
+    return JSON.stringify({
+      title: article.title || document.title || "",
+      content: article.content || "", // Main HTML content
+      textContent: article.textContent || "", // Plain text version
+      url: window.location.href,
+      excerpt: article.excerpt || "",
+      byline: article.byline || "",
+      siteName: article.siteName || "",
+      length: article.length || 0,
+      lang: article.lang || document.documentElement.lang || "",
+      dir: article.dir || document.documentElement.dir || "",
+      publishedTime: article.publishedTime || "",
+      readabilityFailed: false,
+    });
+  } catch (error) {
+    // If anything goes wrong, return error information
+    return JSON.stringify({
+      title: document.title || "",
+      content: "",
+      textContent: "",
+      url: window.location.href,
+      error: error.message + " (stack: " + (error.stack || "no stack") + ")",
+      readabilityFailed: true,
+    });
+  }
+})();

package/skills/fast-browser-use/src/tools/cookies.rs

@@ -0,0 +1,58 @@
+use crate::{error::Result, tools::{Tool, ToolContext, ToolResult}};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
+pub struct GetCookiesParams {
+    /// Optional list of URLs to filter cookies by
+    pub urls: Option<Vec<String>>,
+}
+
+#[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
+pub struct CookieParam {
+    pub name: String,
+    pub value: String,
+    pub domain: Option<String>,
+    pub path: Option<String>,
+    pub secure: Option<bool>,
+    pub http_only: Option<bool>,
+    pub same_site: Option<String>,
+    pub expires: Option<f64>,
+    pub url: Option<String>,
+}
+
+#[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
+pub struct SetCookiesParams {
+    pub cookies: Vec<CookieParam>,
+}
+
+#[derive(Default)]
+pub struct GetCookiesTool;
+
+impl Tool for GetCookiesTool {
+    type Params = GetCookiesParams;
+
+    fn name(&self) -> &str {
+        "get_cookies"
+    }
+
+    fn execute_typed(&self, _params: Self::Params, context: &mut ToolContext) -> Result<ToolResult> {
+        let cookies = context.session.get_cookies()?;
+        Ok(ToolResult::success_with(cookies))
+    }
+}
+
+#[derive(Default)]
+pub struct SetCookiesTool;
+
+impl Tool for SetCookiesTool {
+    type Params = SetCookiesParams;
+
+    fn name(&self) -> &str {
+        "set_cookies"
+    }
+
+    fn execute_typed(&self, params: Self::Params, context: &mut ToolContext) -> Result<ToolResult> {
+        context.session.set_cookies(params.cookies)?;
+        Ok(ToolResult::success(None))
+    }
+}

package/skills/fast-browser-use/src/tools/debug.rs

@@ -0,0 +1,44 @@
+use crate::{browser::debug::{ConsoleLog, NetworkError}, error::Result, tools::{Tool, ToolContext, ToolResult}};
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
+pub struct GetConsoleLogsParams {
+    // No params needed, gets all logs since session start
+}
+
+#[derive(Debug, Serialize, Deserialize, schemars::JsonSchema)]
+pub struct GetNetworkErrorsParams {
+    // No params needed
+}
+
+#[derive(Default)]
+pub struct GetConsoleLogsTool;
+
+impl Tool for GetConsoleLogsTool {
+    type Params = GetConsoleLogsParams;
+
+    fn name(&self) -> &str {
+        "get_console_logs"
+    }
+
+    fn execute_typed(&self, _params: Self::Params, context: &mut ToolContext) -> Result<ToolResult> {
+        let logs = context.session.get_console_logs()?;
+        Ok(ToolResult::success_with(logs))
+    }
+}
+
+#[derive(Default)]
+pub struct GetNetworkErrorsTool;
+
+impl Tool for GetNetworkErrorsTool {
+    type Params = GetNetworkErrorsParams;
+
+    fn name(&self) -> &str {
+        "get_network_errors"
+    }
+
+    fn execute_typed(&self, _params: Self::Params, context: &mut ToolContext) -> Result<ToolResult> {
+        let errors = context.session.get_network_errors()?;
+        Ok(ToolResult::success_with(errors))
+    }
+}

package/skills/fast-browser-use/src/tools/evaluate.rs

@@ -0,0 +1,40 @@
+use crate::{error::{BrowserError, Result},
+            tools::{Tool, ToolContext, ToolResult}};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct EvaluateParams {
+    /// JavaScript code to execute
+    pub code: String,
+
+    /// Wait for promise resolution (default: false)
+    #[serde(default)]
+    pub await_promise: bool,
+}
+
+#[derive(Default)]
+pub struct EvaluateTool;
+
+impl Tool for EvaluateTool {
+    type Params = EvaluateParams;
+
+    fn name(&self) -> &str {
+        "evaluate"
+    }
+
+    fn execute_typed(&self, params: EvaluateParams, context: &mut ToolContext) -> Result<ToolResult> {
+        let result = context
+            .session
+            .tab()?
+            .evaluate(&params.code, params.await_promise)
+            .map_err(|e| BrowserError::EvaluationFailed(e.to_string()))?;
+
+        let result_value = result.value.unwrap_or(Value::Null);
+
+        Ok(ToolResult::success_with(serde_json::json!({
+            "result": result_value
+        })))
+    }
+}

package/skills/fast-browser-use/src/tools/extract.rs

@@ -0,0 +1,66 @@
+use crate::{error::{BrowserError, Result},
+            tools::{Tool, ToolContext, ToolResult}};
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)]
+pub struct ExtractParams {
+    /// CSS selector (optional, defaults to body)
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub selector: Option<String>,
+
+    /// Format: "text" or "html"
+    #[serde(default = "default_format")]
+    pub format: String,
+}
+
+fn default_format() -> String {
+    "text".to_string()
+}
+
+#[derive(Default)]
+pub struct ExtractContentTool;
+
+impl Tool for ExtractContentTool {
+    type Params = ExtractParams;
+
+    fn name(&self) -> &str {
+        "extract"
+    }
+
+    fn execute_typed(&self, params: ExtractParams, context: &mut ToolContext) -> Result<ToolResult> {
+        let content = if let Some(selector) = &params.selector {
+            let tab = context.session.tab()?;
+            let element = context.session.find_element(&tab, selector)?;
+
+            if params.format == "html" {
+                element.get_content().map_err(|e| BrowserError::ToolExecutionFailed {
+                    tool: "extract".to_string(),
+                    reason: e.to_string(),
+                })?
+            } else {
+                element.get_inner_text().map_err(|e| BrowserError::ToolExecutionFailed {
+                    tool: "extract".to_string(),
+                    reason: e.to_string(),
+                })?
+            }
+        } else {
+            // Extract from body
+            let js_code = if params.format == "html" { "document.body.innerHTML" } else { "document.body.innerText" };
+
+            let result = context
+                .session
+                .tab()?
+                .evaluate(js_code, false)
+                .map_err(|e| BrowserError::EvaluationFailed(e.to_string()))?;
+
+            result.value.and_then(|v| v.as_str().map(String::from)).unwrap_or_default()
+        };
+
+        Ok(ToolResult::success_with(serde_json::json!({
+            "content": content,
+            "format": params.format,
+            "length": content.len()
+        })))
+    }
+}