@agent-native/core 0.114.6 → 0.114.8

package/corpus/templates/clips/desktop/src-tauri/Cargo.toml

@@ -77,6 +77,11 @@ libc = "0.2"
 core-foundation = "0.10"
 core-graphics = "0.25"
 objc2 = { version = "0.6", features = ["exception"] }
+objc2-app-kit = { version = "0.3", default-features = false, features = [
+  "std",
+  "NSEvent",
+  "block2",
+] }
 # Grapheme-cluster-aware chunking for voice-dictation Unicode typing in
 # terminal apps, so a chunk boundary never splits a multi-scalar emoji/
 # combining-mark sequence across two synthetic keyboard events. Already

package/corpus/templates/clips/desktop/src-tauri/build.rs

@@ -1,12 +1,78 @@
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 
 fn main() {
     emit_sentry_env_reruns();
+    compile_screen_memory_ocr_helper();
     add_swift_runtime_rpaths();
     tauri_build::build()
 }
 
+/// Build the tiny, macOS-only AVFoundation/Vision bridge used by local Screen
+/// Memory OCR. Keeping this outside the Rust dependency graph avoids adding a
+/// large Objective-C binding surface for a single, OS-provided capability.
+fn compile_screen_memory_ocr_helper() {
+    if std::env::var("CARGO_CFG_TARGET_OS").as_deref() != Ok("macos") {
+        return;
+    }
+
+    let source = Path::new("native/screen_memory_ocr.swift");
+    println!("cargo:rerun-if-changed={}", source.display());
+
+    let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("Cargo sets OUT_DIR"));
+    let object = out_dir.join("screen_memory_ocr_helper.o");
+    let archive = out_dir.join("libscreen_memory_ocr_helper.a");
+
+    // `tauri build --target universal-apple-darwin` invokes Cargo once per
+    // architecture. swiftc otherwise emits an object for the runner's host
+    // architecture, which makes the x86_64 link fail on arm64 CI runners.
+    let swift_arch = match std::env::var("CARGO_CFG_TARGET_ARCH").as_deref() {
+        Ok("aarch64") => "arm64",
+        Ok("x86_64") => "x86_64",
+        Ok(arch) => panic!("unsupported macOS OCR helper architecture: {arch}"),
+        Err(error) => panic!("Cargo target architecture is required: {error}"),
+    };
+    let deployment_target =
+        std::env::var("MACOSX_DEPLOYMENT_TARGET").unwrap_or_else(|_| "13.0".to_string());
+    let swift_target = format!("{swift_arch}-apple-macosx{deployment_target}");
+
+    let swift_status = Command::new("xcrun")
+        .args([
+            "swiftc",
+            "-parse-as-library",
+            "-emit-object",
+            "-target",
+            &swift_target,
+            source.to_str().expect("UTF-8 source path"),
+            "-o",
+            object.to_str().expect("UTF-8 output path"),
+        ])
+        .status()
+        .expect("Xcode's swiftc is required to build the macOS OCR helper");
+    assert!(swift_status.success(), "failed to compile macOS OCR helper");
+
+    let archive_status = Command::new("ar")
+        .args([
+            "crus",
+            archive.to_str().expect("UTF-8 archive path"),
+            object.to_str().expect("UTF-8 object path"),
+        ])
+        .status()
+        .expect("ar is required to archive the macOS OCR helper");
+    assert!(
+        archive_status.success(),
+        "failed to archive macOS OCR helper"
+    );
+
+    println!("cargo:rustc-link-search=native={}", out_dir.display());
+    println!("cargo:rustc-link-lib=static=screen_memory_ocr_helper");
+    // The helper is written in Swift, while the rest of the native desktop
+    // stack already carries Swift runtime rpaths through ScreenCaptureKit.
+    for library in ["swiftCore", "swiftFoundation", "swift_Concurrency"] {
+        println!("cargo:rustc-link-lib=dylib={library}");
+    }
+}
+
 fn emit_sentry_env_reruns() {
     for name in [
         "CLIPS_DESKTOP_SENTRY_DSN",
@@ -47,6 +113,8 @@ fn add_swift_runtime_rpaths() {
     // CMTime symbols we touch via raw `msg_send!` / `extern "C"`.
     println!("cargo:rustc-link-lib=framework=AVFoundation");
     println!("cargo:rustc-link-lib=framework=CoreMedia");
+    println!("cargo:rustc-link-lib=framework=Vision");
+    println!("cargo:rustc-link-lib=framework=CoreGraphics");
     println!("cargo:rustc-link-lib=framework=IOKit");
 
     // The screencapturekit crate builds a Swift bridge. Its build script adds

package/corpus/templates/clips/desktop/src-tauri/src/accessibility.rs

@@ -54,7 +54,120 @@
 //!   - AXValue can also be a CFNumber (sliders) or CFURL — we only return
 //!     a String when it's a CFString; otherwise "".
 
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
+
+/// A deliberately small, retention-bound summary of the current accessibility
+/// surface. This is evidence for local Rewind scenes, not a serialised AX tree.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub struct AccessibilityFingerprint {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub document: Option<AccessibilitySemanticNode>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub focused: Option<AccessibilitySemanticNode>,
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    pub visible_labels: Vec<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub struct AccessibilitySemanticNode {
+    pub role: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub title: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub document: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub url: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub selected_text: Option<String>,
+}
+
+const MAX_AX_NODES: usize = 6;
+const MAX_AX_DEPTH: usize = 1;
+const MAX_AX_STRING_BYTES: usize = 160;
+const MAX_AX_LABELS: usize = 8;
+const MAX_AX_TOTAL_BYTES: usize = 1_200;
+
+/// Keep text useful for scene continuity without retaining arbitrary field
+/// contents. Password-like controls and credential-shaped values are omitted.
+fn safe_ax_text(value: Option<String>, remaining: &mut usize) -> Option<String> {
+    let value = value?;
+    let value = value.split_whitespace().collect::<Vec<_>>().join(" ");
+    if value.is_empty() || looks_like_credential(&value) {
+        return None;
+    }
+    let limit = MAX_AX_STRING_BYTES.min(*remaining);
+    if limit == 0 {
+        return None;
+    }
+    let mut end = value.len().min(limit);
+    while end > 0 && !value.is_char_boundary(end) {
+        end -= 1;
+    }
+    let value = value[..end].to_string();
+    if value.is_empty() {
+        return None;
+    }
+    *remaining = remaining.saturating_sub(value.len());
+    Some(value)
+}
+
+fn looks_like_credential(value: &str) -> bool {
+    let lower = value.to_ascii_lowercase();
+    if [
+        "password",
+        "passcode",
+        "secret=",
+        "client secret",
+        "api key",
+        "api_key",
+        "access token",
+        "auth token",
+        "bearer ",
+    ]
+    .iter()
+    .any(|marker| lower.contains(marker))
+    {
+        return true;
+    }
+    let compact = value
+        .chars()
+        .filter(|c| !c.is_whitespace())
+        .collect::<String>();
+    let tokenish = compact.len() >= 32
+        && compact
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-' | '.'));
+    let jwtish = compact.matches('.').count() == 2 && compact.len() >= 32;
+    let private_key = value.contains("-----BEGIN") && value.contains("PRIVATE KEY-----");
+    tokenish || jwtish || private_key
+}
+
+fn role_is_secure(role: &str) -> bool {
+    let role = role.to_ascii_lowercase();
+    role.contains("secure") || role.contains("password")
+}
+
+fn safe_ax_url(url: String, remaining: &mut usize) -> Option<String> {
+    let url = url.split(['?', '#']).next()?.to_string();
+    // Never retain a URL that embeds credentials in its authority component.
+    let authority = url
+        .split_once("://")?
+        .1
+        .split('/')
+        .next()
+        .unwrap_or_default();
+    if authority.contains('@') {
+        return None;
+    }
+    if url.split('/').skip(3).any(looks_like_credential) {
+        return None;
+    }
+    safe_ax_text(Some(url), remaining)
+}
 
 #[derive(Debug, Clone, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -122,9 +235,53 @@ pub async fn accessibility_request_permission() -> Result<bool, String> {
     }
 }
 
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ax_text_is_bounded_and_budgeted() {
+        let mut remaining = 12;
+        let value = safe_ax_text(Some("a very long accessible title".into()), &mut remaining)
+            .expect("safe title");
+        assert_eq!(value.len(), 12);
+        assert_eq!(remaining, 0);
+        assert!(safe_ax_text(Some("another title".into()), &mut remaining).is_none());
+    }
+
+    #[test]
+    fn credential_shaped_text_is_not_retained() {
+        let mut remaining = MAX_AX_TOTAL_BYTES;
+        assert!(safe_ax_text(Some("password: hunter2".into()), &mut remaining).is_none());
+        assert!(
+            safe_ax_text(Some("API key: short-but-sensitive".into()), &mut remaining).is_none()
+        );
+        assert!(safe_ax_text(
+            Some("eyJhbGciOiJIUzI1NiJ9.abcdefghijklmnopqrstuvwxyz.0123456789abcdef".into()),
+            &mut remaining
+        )
+        .is_none());
+        assert!(role_is_secure("AXSecureTextField"));
+        assert!(safe_ax_url(
+            "https://alice:secret@example.test/document".into(),
+            &mut remaining
+        )
+        .is_none());
+        assert!(safe_ax_url(
+            "https://example.test/session/abcdefghijklmnopqrstuvwxyz123456".into(),
+            &mut remaining
+        )
+        .is_none());
+    }
+}
+
 #[cfg(target_os = "macos")]
 pub(crate) mod macos {
-    use super::ActiveWindowContext;
+    use super::{
+        role_is_secure, safe_ax_text, safe_ax_url, AccessibilityFingerprint,
+        AccessibilitySemanticNode, ActiveWindowContext, MAX_AX_DEPTH, MAX_AX_LABELS, MAX_AX_NODES,
+        MAX_AX_TOTAL_BYTES,
+    };
     use core_foundation::array::CFArray;
     use core_foundation::base::{CFType, TCFType};
     use core_foundation::dictionary::CFDictionary;
@@ -140,6 +297,7 @@ pub(crate) mod macos {
     type CFTypeRef = *const c_void;
     type CFStringRef = *const c_void;
     type CFDictionaryRef = *const c_void;
+    type CFArrayRef = *const c_void;
     type CFBooleanRef = *const c_void;
     type CFAllocatorRef = *const c_void;
     type CFIndex = isize;
@@ -166,6 +324,7 @@ pub(crate) mod macos {
     extern "C" {
         fn AXIsProcessTrustedWithOptions(options: CFDictionaryRef) -> Boolean;
         fn AXUIElementCreateSystemWide() -> AXUIElementRef;
+        fn AXUIElementCreateApplication(pid: i32) -> AXUIElementRef;
         fn AXUIElementCopyAttributeValue(
             element: AXUIElementRef,
             attribute: CFStringRef,
@@ -194,6 +353,10 @@ pub(crate) mod macos {
         fn CFRelease(cf: CFTypeRef);
         fn CFGetTypeID(cf: CFTypeRef) -> CFTypeID;
         fn CFStringGetTypeID() -> CFTypeID;
+        fn CFURLGetTypeID() -> CFTypeID;
+        fn CFArrayGetCount(array: CFArrayRef) -> CFIndex;
+        fn CFArrayGetValueAtIndex(array: CFArrayRef, index: CFIndex) -> *const c_void;
+        fn CFURLGetString(url: CFTypeRef) -> CFStringRef;
         fn CFDictionaryCreate(
             allocator: CFAllocatorRef,
             keys: *const *const c_void,
@@ -341,25 +504,214 @@ pub(crate) mod macos {
         }
     }
 
+    /// Collect the semantic surface from the same application selected by the
+    /// Core Graphics foreground-window sample. This avoids combining an
+    /// ignored Clips utility window's AX tree with the underlying work app.
+    pub fn semantic_fingerprint_for_pid_impl(pid: i32) -> Option<AccessibilityFingerprint> {
+        unsafe {
+            if !is_trusted(false) {
+                return None;
+            }
+            let application = AXUIElementCreateApplication(pid);
+            if application.is_null() {
+                return None;
+            }
+            let window = copy_ax_element_attribute(application, "AXFocusedWindow");
+            let focused = copy_ax_element_attribute(application, "AXFocusedUIElement");
+            CFRelease(application as CFTypeRef);
+
+            let document_element = window.or(focused)?;
+            let mut remaining = MAX_AX_TOTAL_BYTES;
+            let document = semantic_node(document_element, false, &mut remaining);
+            let focused_node =
+                focused.and_then(|element| semantic_node(element, true, &mut remaining));
+            let mut labels = Vec::new();
+            collect_visible_labels(document_element, 0, &mut labels, &mut remaining);
+
+            if let Some(window) = window {
+                CFRelease(window);
+            }
+            if let Some(focused) = focused {
+                CFRelease(focused);
+            }
+
+            if document.is_none() && focused_node.is_none() && labels.is_empty() {
+                None
+            } else {
+                Some(AccessibilityFingerprint {
+                    document,
+                    focused: focused_node,
+                    visible_labels: labels,
+                })
+            }
+        }
+    }
+
+    unsafe fn copy_ax_element_attribute(
+        element: AXUIElementRef,
+        attribute: &str,
+    ) -> Option<AXUIElementRef> {
+        let attribute = cfstr(attribute);
+        if attribute.is_null() {
+            return None;
+        }
+        let mut value = ptr::null();
+        let error = AXUIElementCopyAttributeValue(element, attribute, &mut value);
+        CFRelease(attribute as CFTypeRef);
+        (error == AX_ERROR_SUCCESS && !value.is_null()).then_some(value as AXUIElementRef)
+    }
+
+    unsafe fn ax_string_attribute(element: AXUIElementRef, attribute: &str) -> Option<String> {
+        let attribute = cfstr(attribute);
+        if attribute.is_null() {
+            return None;
+        }
+        let mut raw_value = ptr::null();
+        let error = AXUIElementCopyAttributeValue(element, attribute, &mut raw_value);
+        CFRelease(attribute as CFTypeRef);
+        if error != AX_ERROR_SUCCESS || raw_value.is_null() {
+            return None;
+        }
+        let value = if CFGetTypeID(raw_value) == CFStringGetTypeID() {
+            cfstring_to_string(raw_value as CFStringRef)
+        } else {
+            None
+        };
+        CFRelease(raw_value);
+        value
+    }
+
+    unsafe fn ax_url_attribute(element: AXUIElementRef) -> Option<String> {
+        let attribute = cfstr("AXURL");
+        if attribute.is_null() {
+            return None;
+        }
+        let mut value = ptr::null();
+        let error = AXUIElementCopyAttributeValue(element, attribute, &mut value);
+        CFRelease(attribute as CFTypeRef);
+        if error != AX_ERROR_SUCCESS || value.is_null() {
+            return None;
+        }
+        // URLs may contain query-string credentials. Keep only the stable
+        // origin/path portion, and omit anything that cannot be read safely.
+        let string = if CFGetTypeID(value) == CFStringGetTypeID() {
+            value as CFStringRef
+        } else if CFGetTypeID(value) == CFURLGetTypeID() {
+            CFURLGetString(value)
+        } else {
+            ptr::null()
+        };
+        let result = cfstring_to_string(string)
+            .and_then(|url| url.split(['?', '#']).next().map(str::to_string));
+        CFRelease(value);
+        result
+    }
+
+    unsafe fn semantic_node(
+        element: AXUIElementRef,
+        focused: bool,
+        remaining: &mut usize,
+    ) -> Option<AccessibilitySemanticNode> {
+        let role = ax_string_attribute(element, "AXRole")?;
+        if role_is_secure(&role) {
+            return None;
+        }
+        let role = safe_ax_text(Some(role), remaining)?;
+        let title = safe_ax_text(ax_string_attribute(element, "AXTitle"), remaining);
+        let description = safe_ax_text(ax_string_attribute(element, "AXDescription"), remaining);
+        let document = safe_ax_text(ax_string_attribute(element, "AXDocument"), remaining);
+        let url = ax_url_attribute(element).and_then(|url| safe_ax_url(url, remaining));
+        let selected_text = focused
+            .then(|| ax_string_attribute(element, "AXSelectedText"))
+            .flatten()
+            .and_then(|text| safe_ax_text(Some(text), remaining));
+        Some(AccessibilitySemanticNode {
+            role,
+            title,
+            description,
+            document,
+            url,
+            selected_text,
+        })
+    }
+
+    unsafe fn collect_visible_labels(
+        element: AXUIElementRef,
+        depth: usize,
+        labels: &mut Vec<String>,
+        remaining: &mut usize,
+    ) {
+        if depth >= MAX_AX_DEPTH || labels.len() >= MAX_AX_LABELS || *remaining == 0 {
+            return;
+        }
+        let attribute = cfstr("AXChildren");
+        if attribute.is_null() {
+            return;
+        }
+        let mut children = ptr::null();
+        let error = AXUIElementCopyAttributeValue(element, attribute, &mut children);
+        CFRelease(attribute as CFTypeRef);
+        if error != AX_ERROR_SUCCESS || children.is_null() {
+            return;
+        }
+        let count = (CFArrayGetCount(children) as usize).min(MAX_AX_NODES);
+        for index in 0..count {
+            if labels.len() >= MAX_AX_LABELS || *remaining == 0 {
+                break;
+            }
+            let child = CFArrayGetValueAtIndex(children, index as CFIndex) as AXUIElementRef;
+            if child.is_null() {
+                continue;
+            }
+            let role = ax_string_attribute(child, "AXRole");
+            if role.as_deref().is_some_and(role_is_secure) {
+                continue;
+            }
+            let label = ax_string_attribute(child, "AXTitle")
+                .or_else(|| ax_string_attribute(child, "AXDescription"))
+                .and_then(|text| safe_ax_text(Some(text), remaining));
+            if let Some(label) = label {
+                labels.push(label);
+            }
+        }
+        CFRelease(children);
+    }
+
     pub fn active_window_context_impl() -> ActiveWindowContext {
+        active_window_context_with_pid_impl().0
+    }
+
+    pub(crate) fn active_window_context_with_pid_impl() -> (ActiveWindowContext, Option<i32>) {
         let window = active_window_from_core_graphics();
         let frontmost = frontmost_application();
-
-        ActiveWindowContext {
-            app_name: window
-                .as_ref()
-                .and_then(|w| w.app_name.clone())
-                .or_else(|| frontmost.as_ref().and_then(|app| app.app_name.clone())),
-            window_title: window.and_then(|w| w.window_title),
-            bundle_id: frontmost.and_then(|app| app.bundle_id),
-            source: "core-graphics".to_string(),
-        }
+        let owner = window
+            .as_ref()
+            .and_then(|window| window.owner_pid)
+            .and_then(application_for_pid);
+        let owner_pid = window.as_ref().and_then(|window| window.owner_pid);
+
+        (
+            ActiveWindowContext {
+                app_name: window
+                    .as_ref()
+                    .and_then(|w| w.app_name.clone())
+                    .or_else(|| owner.as_ref().and_then(|app| app.app_name.clone()))
+                    .or_else(|| frontmost.as_ref().and_then(|app| app.app_name.clone())),
+                window_title: window.and_then(|w| w.window_title),
+                bundle_id: owner
+                    .and_then(|app| app.bundle_id)
+                    .or_else(|| frontmost.and_then(|app| app.bundle_id)),
+                source: "core-graphics".to_string(),
+            },
+            owner_pid,
+        )
     }
 
     #[derive(Debug, Clone)]
     struct WindowInfo {
         app_name: Option<String>,
         window_title: Option<String>,
+        owner_pid: Option<i32>,
     }
 
     #[derive(Debug, Clone)]
@@ -400,6 +752,8 @@ pub(crate) mod macos {
                 return Some(WindowInfo {
                     app_name: owner,
                     window_title: title,
+                    owner_pid: dict_number_i64(&window, "kCGWindowOwnerPID")
+                        .and_then(|pid| i32::try_from(pid).ok()),
                 });
             }
         }
@@ -430,6 +784,26 @@ pub(crate) mod macos {
         }
     }
 
+    fn application_for_pid(pid: i32) -> Option<AppInfo> {
+        use objc2::msg_send;
+        use objc2::runtime::{AnyClass, AnyObject};
+
+        unsafe {
+            let class_name = std::ffi::CString::new("NSRunningApplication").ok()?;
+            let cls: &AnyClass = AnyClass::get(&class_name)?;
+            let app: *mut AnyObject = msg_send![cls, runningApplicationWithProcessIdentifier: pid];
+            if app.is_null() {
+                return None;
+            }
+            let app_name: *mut AnyObject = msg_send![app, localizedName];
+            let bundle_id: *mut AnyObject = msg_send![app, bundleIdentifier];
+            Some(AppInfo {
+                app_name: ns_string_to_owned(app_name),
+                bundle_id: ns_string_to_owned(bundle_id),
+            })
+        }
+    }
+
     fn dict_string(dict: &CFDictionary<CFString, CFType>, key: &'static str) -> Option<String> {
         let key = CFString::from_static_string(key);
         let value = dict.find(&key)?;