@desklink/host 0.5.9 → 0.5.11

package/engine/src/keymap_mac.rs

@@ -1,4 +1,6 @@
 use anyhow::Result;
+#[cfg(desklink_vpx)]
+use crate::protocol::ObservedInput;
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub struct Keystroke {
@@ -182,3 +184,258 @@ fn us_ansi_key(character: char) -> Option<(i16, bool)> {
         _ => return None,
     })
 }
+
+/// Event types of a CGEventTap that carry a key.
+#[cfg(desklink_vpx)]
+pub mod key_event {
+    pub const DOWN: u32 = 10;
+    pub const UP: u32 = 11;
+    pub const FLAGS_CHANGED: u32 = 12;
+}
+
+/// Generic modifier bits of a CGEvent's flags.
+#[cfg(desklink_vpx)]
+const SHIFT: u64 = 0x2_0000;
+#[cfg(desklink_vpx)]
+const CONTROL: u64 = 0x4_0000;
+#[cfg(desklink_vpx)]
+const ALT: u64 = 0x8_0000;
+#[cfg(desklink_vpx)]
+const META: u64 = 0x10_0000;
+
+/// A modifier key: its name, and the bit of the event flags that is set while
+/// this key (not just its twin on the other side) is held. Caps Lock and Fn have
+/// no per-side bit, so theirs is the generic one.
+#[cfg(desklink_vpx)]
+const MODIFIER_KEYS: [(i16, &str, u64); 10] = [
+    (59, "Control", 0x1), (62, "ControlRight", 0x2000), (56, "Shift", 0x2),
+    (60, "ShiftRight", 0x4), (58, "Alt", 0x20), (61, "AltRight", 0x40),
+    (55, "Meta", 0x8), (54, "MetaRight", 0x10), (57, "CapsLock", 0x1_0000),
+    (63, "Fn", 0x80_0000),
+];
+
+/// Keys that have a name rather than a character, following `KeyboardEvent.key`.
+/// Every name here that the control channel accepts maps to the same virtual key
+/// code through `named_key`.
+#[cfg(desklink_vpx)]
+const NAMED_KEYS: [(i16, &str); 36] = [
+    (36, "Enter"), (76, "Enter"), (48, "Tab"), (53, "Escape"),
+    (51, "Backspace"), (117, "Delete"), (126, "ArrowUp"), (125, "ArrowDown"),
+    (123, "ArrowLeft"), (124, "ArrowRight"), (115, "Home"), (119, "End"),
+    (116, "PageUp"), (121, "PageDown"), (122, "F1"), (120, "F2"),
+    (99, "F3"), (118, "F4"), (96, "F5"), (97, "F6"),
+    (98, "F7"), (100, "F8"), (101, "F9"), (109, "F10"),
+    (103, "F11"), (111, "F12"), (105, "F13"), (107, "F14"),
+    (113, "F15"), (106, "F16"), (64, "F17"), (79, "F18"),
+    (80, "F19"), (90, "F20"), (114, "Help"), (110, "ContextMenu"),
+];
+
+/// Names of the modifiers held in `flags`, in the order the control channel's
+/// own examples use.
+#[cfg(desklink_vpx)]
+pub fn modifier_names(flags: u64) -> Vec<String> {
+    [
+        (CONTROL, "Control"),
+        (ALT, "Alt"),
+        (SHIFT, "Shift"),
+        (META, "Meta"),
+    ]
+    .into_iter()
+    .filter(|(bit, _)| flags & bit != 0)
+    .map(|(_, name)| name.to_owned())
+    .collect()
+}
+
+/// The character a key makes on the US ANSI map this engine injects with. Only
+/// a fallback: a control chord, or an event the system gave no text, has none.
+#[cfg(desklink_vpx)]
+fn base_character(code: i16, shift: bool) -> Option<char> {
+    (' '..='~').find(|character| us_ansi_key(*character) == Some((code, shift)))
+}
+
+/// The single printable character a key produced on the layout in use.
+#[cfg(desklink_vpx)]
+fn printable(text: &str) -> Option<char> {
+    let mut characters = text.chars();
+    let character = characters.next()?;
+    // U+F700..U+F8FF is where AppKit puts the function and arrow keys.
+    (characters.next().is_none() && !character.is_control() && !('\u{f700}'..='\u{f8ff}').contains(&character))
+        .then_some(character)
+}
+
+/// What a CGEventTap key event means, in the control channel's `key` vocabulary.
+///
+/// `text` is what the event types on the active layout. Auto-repeat is not an
+/// event: a held key is one down and one up. A modifier arrives as a flags
+/// change, whose direction is its own bit being set.
+#[cfg(desklink_vpx)]
+pub fn observed_key(
+    event_type: u32,
+    code: i16,
+    flags: u64,
+    autorepeat: bool,
+    text: &str,
+) -> Option<ObservedInput> {
+    let modifiers = modifier_names(flags);
+    if event_type == key_event::FLAGS_CHANGED {
+        let (_, name, bit) = MODIFIER_KEYS.iter().find(|(key, _, _)| *key == code)?;
+        return Some(ObservedInput::Key {
+            name: Some((*name).to_owned()),
+            character: None,
+            down: flags & bit != 0,
+            modifiers,
+        });
+    }
+    let down = match event_type {
+        key_event::DOWN if autorepeat => return None,
+        key_event::DOWN => true,
+        key_event::UP => false,
+        _ => return None,
+    };
+    let named = NAMED_KEYS.iter().find(|(key, _)| *key == code);
+    let character = if named.is_some() {
+        None
+    } else {
+        printable(text).or_else(|| base_character(code, flags & SHIFT != 0))
+    };
+    let name = match (named, character) {
+        (Some((_, name)), _) => Some((*name).to_owned()),
+        (None, Some(_)) => None,
+        (None, None) => Some(String::from("Unidentified")),
+    };
+    Some(ObservedInput::Key {
+        name,
+        character: character.map(String::from),
+        down,
+        modifiers,
+    })
+}
+
+#[cfg(all(test, desklink_vpx))]
+mod tests {
+    use super::*;
+
+    fn key(
+        event_type: u32,
+        code: i16,
+        flags: u64,
+        autorepeat: bool,
+        text: &str,
+    ) -> Option<ObservedInput> {
+        observed_key(event_type, code, flags, autorepeat, text)
+    }
+
+    #[test]
+    fn a_letter_is_a_character_with_the_modifier_state_after_the_event() {
+        assert_eq!(
+            key(key_event::DOWN, 0, SHIFT, false, "A"),
+            Some(ObservedInput::Key {
+                name: None,
+                character: Some(String::from("A")),
+                down: true,
+                modifiers: vec![String::from("Shift")],
+            })
+        );
+    }
+
+    #[test]
+    fn a_named_key_carries_no_character() {
+        assert_eq!(
+            key(key_event::UP, 36, 0, false, "\r"),
+            Some(ObservedInput::Key {
+                name: Some(String::from("Enter")),
+                character: None,
+                down: false,
+                modifiers: Vec::new(),
+            })
+        );
+    }
+
+    #[test]
+    fn a_chord_with_no_text_falls_back_to_the_key_it_is() {
+        // Control-C types U+0003, which is not a character a keycap can show.
+        let Some(ObservedInput::Key {
+            character,
+            modifiers,
+            ..
+        }) = key(key_event::DOWN, 8, CONTROL, false, "\u{3}")
+        else {
+            panic!("a chord is a key");
+        };
+        assert_eq!(character.as_deref(), Some("c"));
+        assert_eq!(modifiers, vec![String::from("Control")]);
+    }
+
+    #[test]
+    fn an_event_with_no_text_is_still_the_shifted_character() {
+        let Some(ObservedInput::Key { character, .. }) =
+            key(key_event::DOWN, 4, SHIFT, false, "")
+        else {
+            panic!("a letter is a key");
+        };
+        assert_eq!(character.as_deref(), Some("H"));
+    }
+
+    #[test]
+    fn a_held_key_is_one_down_and_one_up() {
+        assert_eq!(key(key_event::DOWN, 0, 0, true, "a"), None);
+        assert!(key(key_event::UP, 0, 0, false, "a").is_some());
+    }
+
+    #[test]
+    fn releasing_one_shift_while_the_other_is_held_is_an_up() {
+        // Left shift went up; right shift (bit 0x4) and the generic bit remain.
+        let Some(ObservedInput::Key {
+            name,
+            down,
+            modifiers,
+            ..
+        }) = key(key_event::FLAGS_CHANGED, 56, SHIFT | 0x4, false, "")
+        else {
+            panic!("a modifier change is a key");
+        };
+        assert_eq!(name.as_deref(), Some("Shift"));
+        assert!(!down);
+        assert_eq!(modifiers, vec![String::from("Shift")]);
+        let Some(ObservedInput::Key { down, .. }) =
+            key(key_event::FLAGS_CHANGED, 56, SHIFT | 0x2 | 0x4, false, "")
+        else {
+            panic!("a modifier change is a key");
+        };
+        assert!(down);
+    }
+
+    #[test]
+    fn a_key_with_no_name_and_no_text_is_unidentified_not_empty() {
+        let Some(ObservedInput::Key {
+            name, character, ..
+        }) = key(key_event::DOWN, 127, 0, false, "")
+        else {
+            panic!("an unknown key is still a key");
+        };
+        assert_eq!(name.as_deref(), Some("Unidentified"));
+        assert_eq!(character, None);
+    }
+
+    #[test]
+    fn the_private_use_characters_appkit_gives_arrows_are_not_text() {
+        assert_eq!(printable("\u{f702}"), None);
+        assert_eq!(printable("é"), Some('é'));
+        assert_eq!(printable("ab"), None);
+    }
+
+    #[test]
+    fn what_is_observed_is_what_the_control_channel_would_inject() {
+        for (code, name) in NAMED_KEYS {
+            if let Some(stroke) = named_key(name) {
+                // Keypad Enter (76) shares its name with Return (36).
+                assert!(stroke.code == code || name == "Enter", "{name}");
+            }
+        }
+        for (code, name, _) in MODIFIER_KEYS {
+            if let Some(injected) = modifier_key(name) {
+                assert_eq!(injected, code, "{name}");
+            }
+        }
+    }
+}

package/engine/src/mac.rs

@@ -39,6 +39,8 @@ unsafe extern "C" {
     pub(crate) fn CGPreflightScreenCaptureAccess() -> bool;
     pub(crate) fn CGPreflightPostEventAccess() -> bool;
     pub(crate) fn CGRequestPostEventAccess() -> bool;
+    pub(crate) fn CGPreflightListenEventAccess() -> bool;
+    pub(crate) fn CGRequestListenEventAccess() -> bool;
     fn CGGetActiveDisplayList(max: u32, displays: *mut DisplayId, count: *mut u32) -> i32;
     fn CGMainDisplayID() -> DisplayId;
     fn CGDisplayPixelsWide(display: DisplayId) -> usize;
@@ -713,6 +715,12 @@ pub fn run() -> i32 {
                 let _ = crate::input::request_access();
                 println!("If the Accessibility alert did not open, allow {} in System Settings › Privacy & Security › Accessibility.", tcc_responsible_app_name());
             }
+            if crate::input::observe_probe().is_ok() {
+                println!("Input Monitoring permission is already granted to {}.", tcc_responsible_app_name());
+            } else {
+                let _ = crate::input::request_observe_access();
+                println!("If the Input Monitoring alert did not open, allow {} in System Settings › Privacy & Security › Input Monitoring.", tcc_responsible_app_name());
+            }
         }
         _ => println!("desklink-host {}\nUSAGE: desklink-host [serve|capabilities|capture-probe|encode-probe|setup-input|version]", env!("CARGO_PKG_VERSION")),
     }

package/engine/src/main.rs

@@ -26,7 +26,7 @@ mod clipboard;
 mod clipboard;
 #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
 mod convert;
-#[cfg(target_os = "linux")]
+#[cfg(any(target_os = "linux", all(target_os = "macos", desklink_vpx)))]
 mod cursor;
 #[cfg(target_os = "linux")]
 mod encoder;
@@ -72,6 +72,8 @@ mod keymap;
 mod keymap;
 #[cfg(target_os = "macos")]
 mod mac;
+#[cfg(any(target_os = "linux", all(any(target_os = "macos", target_os = "windows"), desklink_vpx)))]
+mod observe;
 #[cfg(any(
     target_os = "linux",
     all(any(target_os = "macos", target_os = "windows"), desklink_vpx)
@@ -666,6 +668,13 @@ fn render_event(notice: session::Notice) -> Option<String> {
             "session.input",
             serde_json::json!({ "sessionId": session_id, "input": input }),
         ),
+        session::SessionEvent::Observed {
+            timestamp_us,
+            input,
+        } => (
+            "session.input.observed",
+            serde_json::json!({ "sessionId": session_id, "timestamp_us": timestamp_us, "input": input }),
+        ),
         session::SessionEvent::KeyframeRequest { generation } => (
             "session.keyframeRequest",
             serde_json::json!({ "sessionId": session_id, "generation": generation }),
@@ -752,6 +761,7 @@ async fn dispatch(
                 agent_indicator: params.agent_indicator,
                 local_frames: params.local_frames,
                 audio: params.audio,
+                observe_input: params.observe_input,
                 ice_servers: usable.0,
             };
             let session = session::Session::open(open, events.clone())
@@ -770,6 +780,7 @@ async fn dispatch(
                 "cursor": session.cursor_info(),
                 "agentIndicator": session.agent_indicator(),
                 "audio": session.audio_info(),
+                "inputObservation": session.observation_info(),
             });
             // URLs the WebRTC stack cannot dial were left out of the peer's
             // configuration; each skipped one is named once instead of

package/engine/src/observe.rs

@@ -0,0 +1,94 @@
+//! Passive observation of the desktop's own pointer buttons, cursor and keys.
+//!
+//! The engine only listens; events are the ones `docs/PROTOCOL.md` describes. A
+//! platform that cannot observe says why with a typed reason and delivers nothing.
+
+use crate::protocol::{CursorPosition, ObservedInput};
+use std::sync::Arc;
+
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+pub enum Observation {
+    Input {
+        timestamp_us: u64,
+        input: ObservedInput,
+    },
+    Cursor(CursorPosition),
+}
+
+pub type Sink = Arc<dyn Fn(Observation) + Send + Sync + 'static>;
+
+/// Keeps an observer running; dropping it stops it.
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+pub struct Tap {
+    pub mechanism: &'static str,
+    pub guard: Box<dyn Send>,
+}
+
+/// Why the desktop's input cannot be observed, and what the owner can do.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Unavailable {
+    pub code: &'static str,
+    pub reason: String,
+    pub remedy: String,
+}
+
+#[cfg(target_os = "macos")]
+pub fn start(display_id: u32, pixels: (usize, usize), sink: Sink) -> Result<Tap, Unavailable> {
+    crate::input::observe(display_id, pixels, sink)
+}
+
+#[cfg(not(target_os = "macos"))]
+pub fn start(_: u32, _: (usize, usize), _: Sink) -> Result<Tap, Unavailable> {
+    Err(Unavailable {
+        code: "unsupported-platform",
+        reason: String::from("observing the desktop's own input is only implemented on macOS"),
+        remedy: String::from("none: this platform delivers no input events"),
+    })
+}
+
+/// The `inputObservation` member of the `session.open` reply.
+pub fn status(result: Option<&Result<Tap, Unavailable>>) -> serde_json::Value {
+    match result {
+        None => serde_json::json!({"observing": false}),
+        Some(Ok(tap)) => serde_json::json!({"observing": true, "mechanism": tap.mechanism}),
+        Some(Err(error)) => serde_json::json!({
+            "observing": false,
+            "code": error.code,
+            "reason": error.reason,
+            "remedy": error.remedy,
+        }),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[cfg(not(target_os = "macos"))]
+    fn a_platform_that_cannot_observe_says_so_and_delivers_nothing() {
+        let delivered = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let counted = delivered.clone();
+        let sink: Sink = Arc::new(move |_| {
+            counted.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        });
+        let refused = start(0, (1920, 1080), sink);
+        let status = status(Some(&refused));
+        assert_eq!(status["observing"], false);
+        assert_eq!(status["code"], "unsupported-platform");
+        assert_eq!(delivered.load(std::sync::atomic::Ordering::SeqCst), 0);
+    }
+
+    #[test]
+    fn a_refusal_carries_its_code_reason_and_remedy() {
+        let refused: Result<Tap, Unavailable> = Err(Unavailable {
+            code: "missing-input-monitoring",
+            reason: String::from("reason"),
+            remedy: String::from("remedy"),
+        });
+        assert_eq!(
+            status(Some(&refused)),
+            serde_json::json!({"observing": false, "code": "missing-input-monitoring", "reason": "reason", "remedy": "remedy"})
+        );
+    }
+}

package/engine/src/portal.rs

@@ -47,13 +47,20 @@ pub async fn open_display(_: Option<u32>) -> Result<PortalSession> {
     anyhow::bail!("display selection is only supported on macOS")
 }
 
+/// The portal service answered without a ScreenCast interface. That happens
+/// when xdg-desktop-portal started before the compositor's portal and cached an
+/// interface list without it; only a restart makes it look again.
+const NO_SCREENCAST: &str = "compositor has no ScreenCast portal: xdg-desktop-portal does not export \
+org.freedesktop.portal.ScreenCast. If it started before the compositor's portal, run \
+`systemctl --user restart xdg-desktop-portal` once the compositor portal is running";
+
 pub async fn open(
     restore_token: Option<&str>,
     cursor: crate::protocol::CursorMode,
 ) -> Result<PortalSession> {
     let proxy = Screencast::new()
         .await
-        .context("compositor has no ScreenCast portal")?;
+        .context(NO_SCREENCAST)?;
 
     if proxy
         .available_source_types()
@@ -142,3 +149,78 @@ fn choose_cursor_mode(available: BitFlags<CursorMode>) -> Result<CursorMode> {
         .find(|mode| available.contains(*mode))
         .context("cursor-unavailable: portal cannot leave the cursor out of capture")
 }
+
+#[cfg(all(test, target_os = "linux"))]
+mod tests {
+    use super::*;
+    use std::process::{Child, Command, Stdio};
+    use zbus::{connection::Builder, message::Type, MessageStream};
+
+    struct Bus(Child);
+
+    impl Drop for Bus {
+        fn drop(&mut self) {
+            let _ = self.0.kill();
+            let _ = self.0.wait();
+        }
+    }
+
+    /// A private session bus whose `org.freedesktop.portal.Desktop` owner
+    /// answers every property read the way a stale xdg-desktop-portal answers
+    /// for ScreenCast: no such interface.
+    #[tokio::test]
+    async fn open_names_the_fix_when_the_portal_exports_no_screencast() {
+        use zbus::export::futures_core::Stream;
+
+        let mut daemon = Command::new("dbus-daemon")
+            .args(["--session", "--nofork", "--print-address=1"])
+            .stdout(Stdio::piped())
+            .spawn()
+            .expect("dbus-daemon");
+        let mut address = String::new();
+        std::io::BufRead::read_line(
+            &mut std::io::BufReader::new(daemon.stdout.take().unwrap()),
+            &mut address,
+        )
+        .unwrap();
+        let _bus = Bus(daemon);
+        let address = address.trim().to_owned();
+
+        let fake = Builder::address(address.as_str())
+            .unwrap()
+            .name("org.freedesktop.portal.Desktop")
+            .unwrap()
+            .build()
+            .await
+            .unwrap();
+        let service = fake.clone();
+        let answering = tokio::spawn(async move {
+            let mut messages = MessageStream::from(&service);
+            while let Some(Ok(message)) =
+                std::future::poll_fn(|cx| std::pin::Pin::new(&mut messages).poll_next(cx)).await
+            {
+                if message.message_type() != Type::MethodCall {
+                    continue;
+                }
+                let _ = service
+                    .reply_dbus_error(
+                        &message.header(),
+                        zbus::fdo::Error::InvalidArgs(
+                            "No such interface \u{201c}org.freedesktop.portal.ScreenCast\u{201d}".into(),
+                        ),
+                    )
+                    .await;
+            }
+        });
+        // Screencast::new reads the session bus from the environment.
+        std::env::set_var("DBUS_SESSION_BUS_ADDRESS", &address);
+
+        let error = match open(None, crate::protocol::CursorMode::Embedded).await {
+            Ok(_) => panic!("a portal without ScreenCast must not open a session"),
+            Err(error) => format!("{error:#}"),
+        };
+        answering.abort();
+        assert!(error.starts_with("compositor has no ScreenCast portal"), "{error}");
+        assert!(error.contains("systemctl --user restart xdg-desktop-portal"), "{error}");
+    }
+}

package/engine/src/protocol.rs

@@ -149,6 +149,11 @@ pub struct OpenParams {
     /// cannot be answered is refused.
     #[serde(default)]
     pub audio: bool,
+    /// Observe this computer's own pointer buttons, cursor and keys and raise
+    /// them as `session.input.observed` and `session.cursor`. Off unless asked:
+    /// it reports every key typed on the desktop, in every application.
+    #[serde(default)]
+    pub observe_input: bool,
 }
 
 fn default_local_frames() -> bool {
@@ -379,7 +384,7 @@ impl ControlMessage {
     }
 }
 
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum PointerPhase {
     Move,
@@ -388,6 +393,30 @@ pub enum PointerPhase {
     Cancel,
 }
 
+/// A pointer button or key the person pressed on the observed desktop. The
+/// vocabulary is the control channel's own (`pointer` and `key`), so a consumer
+/// reads what it sent and what it saw the same way. `x`/`y` are in the source's
+/// own pixels (`geometry.source`); `modifiers` is the modifier state after the
+/// event.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum ObservedInput {
+    Pointer {
+        phase: PointerPhase,
+        x: i64,
+        y: i64,
+        button: i64,
+    },
+    Key {
+        #[serde(skip_serializing_if = "Option::is_none")]
+        name: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        character: Option<String>,
+        down: bool,
+        modifiers: Vec<String>,
+    },
+}
+
 /// Messages the engine sends back on the control channel.
 #[derive(Debug, Serialize)]
 #[serde(tag = "kind", rename_all = "snake_case")]