@elizaos/computeruse 0.24.20

package/src/exceptions.rs

@@ -0,0 +1,65 @@
+use napi::{self, Status};
+use computeruse::errors::AutomationError;
+
+/// Map ComputerUse errors to NAPI errors
+pub fn map_error(err: AutomationError) -> napi::Error {
+    match err {
+        AutomationError::ElementNotFound(msg) => {
+            napi::Error::new(Status::InvalidArg, format!("ELEMENT_NOT_FOUND: {msg}"))
+        }
+        AutomationError::Timeout(msg) => napi::Error::new(
+            Status::GenericFailure,
+            format!("OPERATION_TIMED_OUT: {msg}"),
+        ),
+        AutomationError::PermissionDenied(msg) => {
+            napi::Error::new(Status::GenericFailure, format!("PERMISSION_DENIED: {msg}"))
+        }
+        AutomationError::PlatformError(e) => {
+            napi::Error::new(Status::GenericFailure, format!("PLATFORM_ERROR: {e}"))
+        }
+        AutomationError::UnsupportedOperation(msg) => {
+            napi::Error::new(Status::InvalidArg, format!("UNSUPPORTED_OPERATION: {msg}"))
+        }
+        AutomationError::UnsupportedPlatform(msg) => {
+            napi::Error::new(Status::InvalidArg, format!("UNSUPPORTED_PLATFORM: {msg}"))
+        }
+        AutomationError::InvalidArgument(e) => {
+            napi::Error::new(Status::InvalidArg, format!("INVALID_ARGUMENT: {e}"))
+        }
+        AutomationError::Internal(e) => {
+            napi::Error::new(Status::GenericFailure, format!("INTERNAL_ERROR: {e}"))
+        }
+        AutomationError::InvalidSelector(e) => {
+            napi::Error::new(Status::InvalidArg, format!("INVALID_SELECTOR: {e}"))
+        }
+        AutomationError::UIAutomationAPIError { message, .. } => napi::Error::new(
+            Status::GenericFailure,
+            format!("UI_AUTOMATION_API_ERROR: {message}"),
+        ),
+        AutomationError::ElementDetached(msg) => {
+            napi::Error::new(Status::InvalidArg, format!("ELEMENT_DETACHED: {msg}"))
+        }
+        AutomationError::ElementNotVisible(msg) => {
+            napi::Error::new(Status::InvalidArg, format!("ELEMENT_NOT_VISIBLE: {msg}"))
+        }
+        AutomationError::ElementNotEnabled(msg) => {
+            napi::Error::new(Status::InvalidArg, format!("ELEMENT_NOT_ENABLED: {msg}"))
+        }
+        AutomationError::ElementNotStable(msg) => {
+            napi::Error::new(Status::GenericFailure, format!("ELEMENT_NOT_STABLE: {msg}"))
+        }
+        AutomationError::ElementObscured(msg) => {
+            napi::Error::new(Status::InvalidArg, format!("ELEMENT_OBSCURED: {msg}"))
+        }
+        AutomationError::ScrollFailed(msg) => {
+            napi::Error::new(Status::GenericFailure, format!("SCROLL_FAILED: {msg}"))
+        }
+        AutomationError::OperationCancelled(msg) => {
+            napi::Error::new(Status::Cancelled, format!("OPERATION_CANCELLED: {msg}"))
+        }
+        AutomationError::VerificationFailed(msg) => napi::Error::new(
+            Status::GenericFailure,
+            format!("VERIFICATION_FAILED: {msg}"),
+        ),
+    }
+}

package/src/lib.rs

@@ -0,0 +1,26 @@
+mod desktop;
+mod element;
+mod exceptions;
+mod locator;
+mod selector;
+mod types;
+mod window_manager;
+
+// Main types first
+pub use desktop::Desktop;
+pub use element::{Element, TypeTextOptions};
+pub use locator::Locator;
+pub use selector::Selector;
+pub use types::{
+    ActionResult, Bounds, BoundsEntry, BrowserDomElement, BrowserDomResult, ClickResult, ClickType,
+    ClusteredBoundsEntry, ClusteredFormattingResult, CommandOutput, Coordinates, DomBoundsEntry,
+    ElementSource, FontStyle, GeminiVisionResult, HighlightHandle, InspectElement, Monitor,
+    MonitorScreenshotPair, OcrBoundsEntry, OcrElement, OcrResult, OmniparserBoundsEntry,
+    OmniparserItem, OmniparserResult, OverlayDisplayMode, PropertyLoadingMode, ScreenshotResult,
+    TextPosition, TreeBuildConfig, TreeOutputFormat, UIElementAttributes, UINode,
+    VisionBoundsEntry, VisionElement, VisionType, WindowTreeResult,
+};
+pub use window_manager::{WindowInfo, WindowManager};
+
+// Error handling - see exceptions.rs for detailed architecture
+pub use exceptions::map_error;

package/src/locator.rs

@@ -0,0 +1,172 @@
+use napi_derive::napi;
+use computeruse::locator::WaitCondition as ComputerUseWaitCondition;
+use computeruse::Locator as ComputerUseLocator;
+
+use crate::map_error;
+use crate::Element;
+use crate::Selector;
+use napi::bindgen_prelude::Either;
+
+/// Locator for finding UI elements by selector.
+#[napi(js_name = "Locator")]
+pub struct Locator {
+    inner: ComputerUseLocator,
+}
+
+impl std::fmt::Display for Locator {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "Locator({})", self.inner.selector_string())
+    }
+}
+
+impl From<ComputerUseLocator> for Locator {
+    fn from(l: ComputerUseLocator) -> Self {
+        Locator { inner: l }
+    }
+}
+
+#[napi]
+impl Locator {
+    /// (async) Get the first matching element.
+    ///
+    /// @param {number} [timeoutMs] - Timeout in milliseconds (default: 10000).
+    /// @returns {Promise<Element>} The first matching element.
+    #[napi]
+    pub async fn first(&self, timeout_ms: Option<f64>) -> napi::Result<Element> {
+        use std::time::Duration;
+        let timeout = Duration::from_millis(timeout_ms.unwrap_or(10000.0) as u64);
+        self.inner
+            .first(Some(timeout))
+            .await
+            .map(Element::from)
+            .map_err(map_error)
+    }
+
+    /// (async) Get all matching elements.
+    ///
+    /// @param {number} timeoutMs - Timeout in milliseconds (required).
+    /// @param {number} [depth] - Maximum depth to search.
+    /// @returns {Promise<Array<Element>>} List of matching elements.
+    #[napi]
+    pub async fn all(&self, timeout_ms: f64, depth: Option<u32>) -> napi::Result<Vec<Element>> {
+        use std::time::Duration;
+        let timeout = Duration::from_millis(timeout_ms as u64);
+        let depth = depth.map(|d| d as usize);
+        self.inner
+            .all(Some(timeout), depth)
+            .await
+            .map(|els| els.into_iter().map(Element::from).collect())
+            .map_err(map_error)
+    }
+
+    /// Set a default timeout for this locator.
+    ///
+    /// @param {number} timeoutMs - Timeout in milliseconds.
+    /// @returns {Locator} A new locator with the specified timeout.
+    #[napi]
+    pub fn timeout(&self, timeout_ms: f64) -> Locator {
+        let loc = self
+            .inner
+            .clone()
+            .set_default_timeout(std::time::Duration::from_millis(timeout_ms as u64));
+        Locator::from(loc)
+    }
+
+    /// Set the root element for this locator.
+    ///
+    /// @param {Element} element - The root element.
+    /// @returns {Locator} A new locator with the specified root element.
+    #[napi]
+    pub fn within(&self, element: &Element) -> Locator {
+        let loc = self.inner.clone().within(element.inner.clone());
+        Locator::from(loc)
+    }
+
+    /// Chain another selector.
+    /// Accepts either a selector string or a Selector object.
+    ///
+    /// @param {string | Selector} selector - The selector.
+    /// @returns {Locator} A new locator with the chained selector.
+    #[napi]
+    pub fn locator(
+        &self,
+        #[napi(ts_arg_type = "string | Selector")] selector: Either<String, &Selector>,
+    ) -> napi::Result<Locator> {
+        use napi::bindgen_prelude::Either::*;
+        let sel_rust: computeruse::selector::Selector = match selector {
+            A(sel_str) => sel_str.as_str().into(),
+            B(sel_obj) => sel_obj.inner.clone(),
+        };
+        let loc = self.inner.clone().locator(sel_rust);
+        Ok(Locator::from(loc))
+    }
+
+    /// (async) Validate element existence without throwing an error.
+    ///
+    /// @param {number} timeoutMs - Timeout in milliseconds (required).
+    /// @returns {Promise<ValidationResult>} Validation result with exists flag and optional element.
+    #[napi]
+    pub async fn validate(&self, timeout_ms: f64) -> napi::Result<ValidationResult> {
+        use std::time::Duration;
+        let timeout = Duration::from_millis(timeout_ms as u64);
+        match self.inner.validate(Some(timeout)).await {
+            Ok(Some(element)) => Ok(ValidationResult {
+                exists: true,
+                element: Some(Element::from(element)),
+                error: None,
+            }),
+            Ok(None) => Ok(ValidationResult {
+                exists: false,
+                element: None,
+                error: None,
+            }),
+            Err(e) => Ok(ValidationResult {
+                exists: false,
+                element: None,
+                error: Some(e.to_string()),
+            }),
+        }
+    }
+
+    /// (async) Wait for an element to meet a specific condition.
+    ///
+    /// @param {string} condition - Condition to wait for: 'exists', 'visible', 'enabled', 'focused'
+    /// @param {number} timeoutMs - Timeout in milliseconds (required).
+    /// @returns {Promise<Element>} The element when condition is met.
+    #[napi]
+    pub async fn wait_for(&self, condition: String, timeout_ms: f64) -> napi::Result<Element> {
+        use std::time::Duration;
+        let wait_condition = parse_condition(&condition)?;
+        let timeout = Duration::from_millis(timeout_ms as u64);
+
+        self.inner
+            .wait_for(wait_condition, Some(timeout))
+            .await
+            .map(Element::from)
+            .map_err(map_error)
+    }
+}
+
+/// Result of element validation
+#[napi(object)]
+pub struct ValidationResult {
+    /// Whether the element exists
+    pub exists: bool,
+    /// The element if found
+    pub element: Option<Element>,
+    /// Error message if validation failed (not element not found, but actual error)
+    pub error: Option<String>,
+}
+/// Convert string condition to WaitCondition enum
+fn parse_condition(condition: &str) -> napi::Result<ComputerUseWaitCondition> {
+    match condition.to_lowercase().as_str() {
+        "exists" => Ok(ComputerUseWaitCondition::Exists),
+        "visible" => Ok(ComputerUseWaitCondition::Visible),
+        "enabled" => Ok(ComputerUseWaitCondition::Enabled),
+        "focused" => Ok(ComputerUseWaitCondition::Focused),
+        _ => Err(napi::Error::new(
+            napi::Status::InvalidArg,
+            format!("Invalid condition '{condition}'. Valid: exists, visible, enabled, focused"),
+        )),
+    }
+}

package/src/selector.rs

@@ -0,0 +1,158 @@
+use napi::bindgen_prelude::FromNapiValue;
+use napi_derive::napi;
+use std::collections::BTreeMap;
+use computeruse::selector::Selector as ComputerUseSelector;
+
+/// Selector for locating UI elements. Provides a typed alternative to the string based selector API.
+#[napi(js_name = "Selector")]
+pub struct Selector {
+    pub(crate) inner: ComputerUseSelector,
+}
+
+impl std::fmt::Display for Selector {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{:?}", self.inner)
+    }
+}
+
+impl From<ComputerUseSelector> for Selector {
+    fn from(sel: ComputerUseSelector) -> Self {
+        Selector { inner: sel }
+    }
+}
+
+impl From<&Selector> for ComputerUseSelector {
+    fn from(sel: &Selector) -> Self {
+        sel.inner.clone()
+    }
+}
+
+impl FromNapiValue for Selector {
+    unsafe fn from_napi_value(
+        env: napi::sys::napi_env,
+        napi_val: napi::sys::napi_value,
+    ) -> napi::Result<Self> {
+        let mut result = std::ptr::null_mut();
+        let status = napi::sys::napi_get_value_external(env, napi_val, &mut result);
+        if status != napi::sys::Status::napi_ok {
+            return Err(napi::Error::new(
+                napi::Status::InvalidArg,
+                "Failed to get external value for Selector".to_string(),
+            ));
+        }
+        Ok(std::ptr::read(result as *const Selector))
+    }
+}
+
+#[napi]
+impl Selector {
+    /// Create a selector that matches elements by their accessibility `name`.
+    #[napi(factory)]
+    pub fn name(name: String) -> Self {
+        Selector::from(ComputerUseSelector::Name(name))
+    }
+
+    /// Create a selector that matches elements by role (and optionally name).
+    #[napi(factory)]
+    pub fn role(role: String, name: Option<String>) -> Self {
+        Selector::from(ComputerUseSelector::Role { role, name })
+    }
+
+    /// Create a selector that matches elements by accessibility `id`.
+    #[napi(factory, js_name = "id")]
+    pub fn id_factory(id: String) -> Self {
+        Selector::from(ComputerUseSelector::Id(id))
+    }
+
+    /// Create a selector that matches elements by the text they display.
+    #[napi(factory)]
+    pub fn text(text: String) -> Self {
+        Selector::from(ComputerUseSelector::Text(text))
+    }
+
+    /// Create a selector from an XPath-like path string.
+    #[napi(factory)]
+    pub fn path(path: String) -> Self {
+        Selector::from(ComputerUseSelector::Path(path))
+    }
+
+    /// Create a selector that matches elements by a native automation id (e.g., AutomationID on Windows).
+    #[napi(factory, js_name = "nativeId")]
+    pub fn native_id(id: String) -> Self {
+        Selector::from(ComputerUseSelector::NativeId(id))
+    }
+
+    /// Create a selector that matches elements by their class name.
+    #[napi(factory, js_name = "className")]
+    pub fn class_name(name: String) -> Self {
+        Selector::from(ComputerUseSelector::ClassName(name))
+    }
+
+    /// Create a selector from an arbitrary attribute map.
+    #[napi(factory)]
+    pub fn attributes(attributes: std::collections::HashMap<String, String>) -> Self {
+        let map: BTreeMap<String, String> = attributes.into_iter().collect();
+        Selector::from(ComputerUseSelector::Attributes(map))
+    }
+
+    /// Chain another selector onto this selector.
+    #[napi]
+    pub fn chain(&self, other: &Selector) -> Selector {
+        Selector::from(ComputerUseSelector::Chain(vec![
+            self.inner.clone(),
+            other.inner.clone(),
+        ]))
+    }
+
+    /// Filter by visibility.
+    #[napi]
+    pub fn visible(&self, is_visible: bool) -> Selector {
+        Selector::from(ComputerUseSelector::Chain(vec![
+            self.inner.clone(),
+            ComputerUseSelector::Visible(is_visible),
+        ]))
+    }
+
+    /// Create a selector that selects the nth element from matches.
+    /// Positive values are 0-based from the start (0 = first, 1 = second).
+    /// Negative values are from the end (-1 = last, -2 = second-to-last).
+    #[napi(factory)]
+    pub fn nth(index: i32) -> Self {
+        Selector::from(ComputerUseSelector::Nth(index))
+    }
+
+    /// Create a selector that matches elements having at least one descendant matching the inner selector.
+    /// This is similar to Playwright's :has() pseudo-class.
+    #[napi(factory)]
+    pub fn has(inner_selector: &Selector) -> Self {
+        Selector::from(ComputerUseSelector::Has(Box::new(
+            inner_selector.inner.clone(),
+        )))
+    }
+
+    /// Create a selector that navigates to the parent element.
+    /// This is similar to Playwright's .. syntax.
+    #[napi(factory)]
+    pub fn parent() -> Self {
+        Selector::from(ComputerUseSelector::Parent)
+    }
+
+    /// Create a selector that scopes the search to a specific process.
+    /// This is typically used as the first part of a chained selector.
+    /// Example: `Selector.process("chrome").chain(Selector.role("Button", "Submit"))`
+    #[napi(factory)]
+    pub fn process(process_name: String) -> Self {
+        Selector::from(ComputerUseSelector::Process(process_name))
+    }
+
+    /// Create a selector that scopes the search to a specific window within a process.
+    /// Typically chained after a process selector.
+    /// Example: `Selector.process("notepad").chain(Selector.window("Untitled"))`
+    #[napi(factory)]
+    pub fn window(title: String) -> Self {
+        Selector::from(ComputerUseSelector::Role {
+            role: "Window".to_string(),
+            name: Some(title),
+        })
+    }
+}