@desklink/host 0.5.9 → 0.5.10

package/engine/src/mac.rs

@@ -39,6 +39,8 @@ unsafe extern "C" {
     pub(crate) fn CGPreflightScreenCaptureAccess() -> bool;
     pub(crate) fn CGPreflightPostEventAccess() -> bool;
     pub(crate) fn CGRequestPostEventAccess() -> bool;
+    pub(crate) fn CGPreflightListenEventAccess() -> bool;
+    pub(crate) fn CGRequestListenEventAccess() -> bool;
     fn CGGetActiveDisplayList(max: u32, displays: *mut DisplayId, count: *mut u32) -> i32;
     fn CGMainDisplayID() -> DisplayId;
     fn CGDisplayPixelsWide(display: DisplayId) -> usize;
@@ -713,6 +715,12 @@ pub fn run() -> i32 {
                 let _ = crate::input::request_access();
                 println!("If the Accessibility alert did not open, allow {} in System Settings › Privacy & Security › Accessibility.", tcc_responsible_app_name());
             }
+            if crate::input::observe_probe().is_ok() {
+                println!("Input Monitoring permission is already granted to {}.", tcc_responsible_app_name());
+            } else {
+                let _ = crate::input::request_observe_access();
+                println!("If the Input Monitoring alert did not open, allow {} in System Settings › Privacy & Security › Input Monitoring.", tcc_responsible_app_name());
+            }
         }
         _ => println!("desklink-host {}\nUSAGE: desklink-host [serve|capabilities|capture-probe|encode-probe|setup-input|version]", env!("CARGO_PKG_VERSION")),
     }

package/engine/src/main.rs

@@ -26,7 +26,7 @@ mod clipboard;
 mod clipboard;
 #[cfg(any(target_os = "linux", target_os = "macos", target_os = "windows"))]
 mod convert;
-#[cfg(target_os = "linux")]
+#[cfg(any(target_os = "linux", all(target_os = "macos", desklink_vpx)))]
 mod cursor;
 #[cfg(target_os = "linux")]
 mod encoder;
@@ -72,6 +72,8 @@ mod keymap;
 mod keymap;
 #[cfg(target_os = "macos")]
 mod mac;
+#[cfg(any(target_os = "linux", all(any(target_os = "macos", target_os = "windows"), desklink_vpx)))]
+mod observe;
 #[cfg(any(
     target_os = "linux",
     all(any(target_os = "macos", target_os = "windows"), desklink_vpx)
@@ -666,6 +668,13 @@ fn render_event(notice: session::Notice) -> Option<String> {
             "session.input",
             serde_json::json!({ "sessionId": session_id, "input": input }),
         ),
+        session::SessionEvent::Observed {
+            timestamp_us,
+            input,
+        } => (
+            "session.input.observed",
+            serde_json::json!({ "sessionId": session_id, "timestamp_us": timestamp_us, "input": input }),
+        ),
         session::SessionEvent::KeyframeRequest { generation } => (
             "session.keyframeRequest",
             serde_json::json!({ "sessionId": session_id, "generation": generation }),
@@ -752,6 +761,7 @@ async fn dispatch(
                 agent_indicator: params.agent_indicator,
                 local_frames: params.local_frames,
                 audio: params.audio,
+                observe_input: params.observe_input,
                 ice_servers: usable.0,
             };
             let session = session::Session::open(open, events.clone())
@@ -770,6 +780,7 @@ async fn dispatch(
                 "cursor": session.cursor_info(),
                 "agentIndicator": session.agent_indicator(),
                 "audio": session.audio_info(),
+                "inputObservation": session.observation_info(),
             });
             // URLs the WebRTC stack cannot dial were left out of the peer's
             // configuration; each skipped one is named once instead of

package/engine/src/observe.rs

@@ -0,0 +1,94 @@
+//! Passive observation of the desktop's own pointer buttons, cursor and keys.
+//!
+//! The engine only listens; events are the ones `docs/PROTOCOL.md` describes. A
+//! platform that cannot observe says why with a typed reason and delivers nothing.
+
+use crate::protocol::{CursorPosition, ObservedInput};
+use std::sync::Arc;
+
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+pub enum Observation {
+    Input {
+        timestamp_us: u64,
+        input: ObservedInput,
+    },
+    Cursor(CursorPosition),
+}
+
+pub type Sink = Arc<dyn Fn(Observation) + Send + Sync + 'static>;
+
+/// Keeps an observer running; dropping it stops it.
+#[cfg_attr(not(target_os = "macos"), allow(dead_code))]
+pub struct Tap {
+    pub mechanism: &'static str,
+    pub guard: Box<dyn Send>,
+}
+
+/// Why the desktop's input cannot be observed, and what the owner can do.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Unavailable {
+    pub code: &'static str,
+    pub reason: String,
+    pub remedy: String,
+}
+
+#[cfg(target_os = "macos")]
+pub fn start(display_id: u32, pixels: (usize, usize), sink: Sink) -> Result<Tap, Unavailable> {
+    crate::input::observe(display_id, pixels, sink)
+}
+
+#[cfg(not(target_os = "macos"))]
+pub fn start(_: u32, _: (usize, usize), _: Sink) -> Result<Tap, Unavailable> {
+    Err(Unavailable {
+        code: "unsupported-platform",
+        reason: String::from("observing the desktop's own input is only implemented on macOS"),
+        remedy: String::from("none: this platform delivers no input events"),
+    })
+}
+
+/// The `inputObservation` member of the `session.open` reply.
+pub fn status(result: Option<&Result<Tap, Unavailable>>) -> serde_json::Value {
+    match result {
+        None => serde_json::json!({"observing": false}),
+        Some(Ok(tap)) => serde_json::json!({"observing": true, "mechanism": tap.mechanism}),
+        Some(Err(error)) => serde_json::json!({
+            "observing": false,
+            "code": error.code,
+            "reason": error.reason,
+            "remedy": error.remedy,
+        }),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    #[cfg(not(target_os = "macos"))]
+    fn a_platform_that_cannot_observe_says_so_and_delivers_nothing() {
+        let delivered = Arc::new(std::sync::atomic::AtomicUsize::new(0));
+        let counted = delivered.clone();
+        let sink: Sink = Arc::new(move |_| {
+            counted.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+        });
+        let refused = start(0, (1920, 1080), sink);
+        let status = status(Some(&refused));
+        assert_eq!(status["observing"], false);
+        assert_eq!(status["code"], "unsupported-platform");
+        assert_eq!(delivered.load(std::sync::atomic::Ordering::SeqCst), 0);
+    }
+
+    #[test]
+    fn a_refusal_carries_its_code_reason_and_remedy() {
+        let refused: Result<Tap, Unavailable> = Err(Unavailable {
+            code: "missing-input-monitoring",
+            reason: String::from("reason"),
+            remedy: String::from("remedy"),
+        });
+        assert_eq!(
+            status(Some(&refused)),
+            serde_json::json!({"observing": false, "code": "missing-input-monitoring", "reason": "reason", "remedy": "remedy"})
+        );
+    }
+}

package/engine/src/portal.rs

@@ -47,13 +47,20 @@ pub async fn open_display(_: Option<u32>) -> Result<PortalSession> {
     anyhow::bail!("display selection is only supported on macOS")
 }
 
+/// The portal service answered without a ScreenCast interface. That happens
+/// when xdg-desktop-portal started before the compositor's portal and cached an
+/// interface list without it; only a restart makes it look again.
+const NO_SCREENCAST: &str = "compositor has no ScreenCast portal: xdg-desktop-portal does not export \
+org.freedesktop.portal.ScreenCast. If it started before the compositor's portal, run \
+`systemctl --user restart xdg-desktop-portal` once the compositor portal is running";
+
 pub async fn open(
     restore_token: Option<&str>,
     cursor: crate::protocol::CursorMode,
 ) -> Result<PortalSession> {
     let proxy = Screencast::new()
         .await
-        .context("compositor has no ScreenCast portal")?;
+        .context(NO_SCREENCAST)?;
 
     if proxy
         .available_source_types()
@@ -142,3 +149,78 @@ fn choose_cursor_mode(available: BitFlags<CursorMode>) -> Result<CursorMode> {
         .find(|mode| available.contains(*mode))
         .context("cursor-unavailable: portal cannot leave the cursor out of capture")
 }
+
+#[cfg(all(test, target_os = "linux"))]
+mod tests {
+    use super::*;
+    use std::process::{Child, Command, Stdio};
+    use zbus::{connection::Builder, message::Type, MessageStream};
+
+    struct Bus(Child);
+
+    impl Drop for Bus {
+        fn drop(&mut self) {
+            let _ = self.0.kill();
+            let _ = self.0.wait();
+        }
+    }
+
+    /// A private session bus whose `org.freedesktop.portal.Desktop` owner
+    /// answers every property read the way a stale xdg-desktop-portal answers
+    /// for ScreenCast: no such interface.
+    #[tokio::test]
+    async fn open_names_the_fix_when_the_portal_exports_no_screencast() {
+        use zbus::export::futures_core::Stream;
+
+        let mut daemon = Command::new("dbus-daemon")
+            .args(["--session", "--nofork", "--print-address=1"])
+            .stdout(Stdio::piped())
+            .spawn()
+            .expect("dbus-daemon");
+        let mut address = String::new();
+        std::io::BufRead::read_line(
+            &mut std::io::BufReader::new(daemon.stdout.take().unwrap()),
+            &mut address,
+        )
+        .unwrap();
+        let _bus = Bus(daemon);
+        let address = address.trim().to_owned();
+
+        let fake = Builder::address(address.as_str())
+            .unwrap()
+            .name("org.freedesktop.portal.Desktop")
+            .unwrap()
+            .build()
+            .await
+            .unwrap();
+        let service = fake.clone();
+        let answering = tokio::spawn(async move {
+            let mut messages = MessageStream::from(&service);
+            while let Some(Ok(message)) =
+                std::future::poll_fn(|cx| std::pin::Pin::new(&mut messages).poll_next(cx)).await
+            {
+                if message.message_type() != Type::MethodCall {
+                    continue;
+                }
+                let _ = service
+                    .reply_dbus_error(
+                        &message.header(),
+                        zbus::fdo::Error::InvalidArgs(
+                            "No such interface \u{201c}org.freedesktop.portal.ScreenCast\u{201d}".into(),
+                        ),
+                    )
+                    .await;
+            }
+        });
+        // Screencast::new reads the session bus from the environment.
+        std::env::set_var("DBUS_SESSION_BUS_ADDRESS", &address);
+
+        let error = match open(None, crate::protocol::CursorMode::Embedded).await {
+            Ok(_) => panic!("a portal without ScreenCast must not open a session"),
+            Err(error) => format!("{error:#}"),
+        };
+        answering.abort();
+        assert!(error.starts_with("compositor has no ScreenCast portal"), "{error}");
+        assert!(error.contains("systemctl --user restart xdg-desktop-portal"), "{error}");
+    }
+}

package/engine/src/protocol.rs

@@ -149,6 +149,11 @@ pub struct OpenParams {
     /// cannot be answered is refused.
     #[serde(default)]
     pub audio: bool,
+    /// Observe this computer's own pointer buttons, cursor and keys and raise
+    /// them as `session.input.observed` and `session.cursor`. Off unless asked:
+    /// it reports every key typed on the desktop, in every application.
+    #[serde(default)]
+    pub observe_input: bool,
 }
 
 fn default_local_frames() -> bool {
@@ -379,7 +384,7 @@ impl ControlMessage {
     }
 }
 
-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum PointerPhase {
     Move,
@@ -388,6 +393,30 @@ pub enum PointerPhase {
     Cancel,
 }
 
+/// A pointer button or key the person pressed on the observed desktop. The
+/// vocabulary is the control channel's own (`pointer` and `key`), so a consumer
+/// reads what it sent and what it saw the same way. `x`/`y` are in the source's
+/// own pixels (`geometry.source`); `modifiers` is the modifier state after the
+/// event.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+#[serde(tag = "kind", rename_all = "snake_case")]
+pub enum ObservedInput {
+    Pointer {
+        phase: PointerPhase,
+        x: i64,
+        y: i64,
+        button: i64,
+    },
+    Key {
+        #[serde(skip_serializing_if = "Option::is_none")]
+        name: Option<String>,
+        #[serde(skip_serializing_if = "Option::is_none")]
+        character: Option<String>,
+        down: bool,
+        modifiers: Vec<String>,
+    },
+}
+
 /// Messages the engine sends back on the control channel.
 #[derive(Debug, Serialize)]
 #[serde(tag = "kind", rename_all = "snake_case")]

package/engine/src/session.rs

@@ -714,6 +714,8 @@ pub struct OpenRequest {
     pub local_frames: bool,
     /// Capture the desktop's own system audio for `session.audio`.
     pub audio: bool,
+    /// Listen to the desktop's own pointer buttons, cursor and keys.
+    pub observe_input: bool,
 }
 
 /// Events a session raises for its consumer.
@@ -755,6 +757,11 @@ pub enum SessionEvent {
     Input {
         input: serde_json::Value,
     },
+    /// A pointer button or key the person pressed, observed rather than applied.
+    Observed {
+        timestamp_us: u64,
+        input: crate::protocol::ObservedInput,
+    },
     /// The far end needs a fresh reference frame and only the consumer can make
     /// one: the encoder for this track is on its side. Raised by an RTCP PLI or
     /// FIR, and once when the transport connects, because everything sent before
@@ -854,6 +861,9 @@ struct Inner {
     #[cfg(target_os = "linux")]
     audio: Mutex<Option<crate::audio::AudioCapture>>,
     cursor_info: serde_json::Value,
+    /// `session.open`'s `inputObservation` member, fixed when the session opened.
+    observation: serde_json::Value,
+    observer: Mutex<Option<crate::observe::Tap>>,
 }
 
 /// A source whose video arrives already encoded.
@@ -978,6 +988,7 @@ pub fn capabilities() -> serde_json::Value {
             }),
         ),
     };
+    let observe = crate::input::observe_probe();
     serde_json::json!({
         "protocol": crate::protocol::PROTOCOL_VERSION,
         "engine": format!("desklink-host/{}", env!("CARGO_PKG_VERSION")),
@@ -1009,6 +1020,11 @@ pub fn capabilities() -> serde_json::Value {
             "layout": keymap::LayoutNames::from_environment().identity(),
             "unavailable_reason": unavailable,
             "grant": input_grant,
+            "observe": {
+                "mechanism": "cgeventtap",
+                "grant": if observe.is_ok() { "granted" } else { "missing-input-monitoring" },
+                "unavailable_reason": observe.as_ref().err().map(|e| serde_json::json!({"reason": e.reason, "remedy": e.remedy})),
+            },
         },
         "clipboard": { "read": true, "write": true, "mime": ["text/plain;charset=utf-8"],
             "maxBytes": clipboard::MAX_CLIPBOARD_BYTES },
@@ -1480,6 +1496,29 @@ impl Session {
             "origin": { "x": source.origin_x, "y": source.origin_y },
         });
 
+        // Observation is an addition to the stream, never a condition of it: a
+        // desktop that cannot be observed still streams, and the reply says why.
+        let observed = request.observe_input.then(|| {
+            let sink: crate::observe::Sink = {
+                let events = events.clone();
+                let id = id.clone();
+                Arc::new(move |observation| {
+                    let event = match observation {
+                        crate::observe::Observation::Input { timestamp_us, input } => {
+                            SessionEvent::Observed { timestamp_us, input }
+                        }
+                        crate::observe::Observation::Cursor(position) => SessionEvent::Cursor { position },
+                    };
+                    let _ = events.send(Notice { session_id: id.clone(), event });
+                })
+            };
+            crate::observe::start(source.node_id, (source_w, source_h), sink)
+        });
+        if let Some(Err(error)) = &observed {
+            log::warn!("input observation unavailable: {} ({})", error.reason, error.code);
+        }
+        let observation = crate::observe::status(observed.as_ref());
+
         let pipeline = Arc::new(AtomicBool::new(true));
         let inner = Arc::new(Inner {
             id,
@@ -1526,6 +1565,8 @@ impl Session {
             encoded: Mutex::new(None),
             local_frames: request.local_frames,
             cursor_info,
+            observation,
+            observer: Mutex::new(observed.and_then(Result::ok)),
         });
 
         inner.notify(SessionEvent::Description {
@@ -1641,6 +1682,13 @@ impl Session {
             })),
             local_frames: false,
             cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
+            observation: serde_json::json!({
+                "observing": false,
+                "code": "unsupported-source",
+                "reason": "consumer-encoded video has no desktop of its own to observe",
+                "remedy": "observe the device where its pixels come from",
+            }),
+            observer: Mutex::new(None),
         });
         inner.notify(SessionEvent::State {
             capture: "consented",
@@ -1706,6 +1754,10 @@ impl Session {
         &self.inner.cursor_info
     }
 
+    pub fn observation_info(&self) -> &serde_json::Value {
+        &self.inner.observation
+    }
+
     pub fn geometry(&self) -> &serde_json::Value {
         &self.inner.geometry
     }
@@ -2176,6 +2228,8 @@ impl Inner {
         // handlers, and `shutdown` exits right after this. The encode thread
         // finds it gone and stops.
         drop(lock(&self.encoder).take());
+        // Stop listening to the person's keys the moment the session ends.
+        drop(lock(&self.observer).take());
         if let Ok(mut indicator) = self.indicator.lock() {
             indicator.take();
         }
@@ -3829,6 +3883,8 @@ mod tests {
             })),
             local_frames: false,
             cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
+            observation: serde_json::json!({"observing": false}),
+            observer: Mutex::new(None),
         });
         spawn_peer_events(&inner, peer_events_rx);
         spawn_encoded(&inner, feed_rx);
@@ -4084,6 +4140,8 @@ mod tests {
             encoded: Mutex::new(None),
             local_frames: true,
             cursor_info: serde_json::json!({"mode":"embedded", "positions":false}),
+            observation: serde_json::json!({"observing": false}),
+            observer: Mutex::new(None),
         });
         (inner, recorded)
     }
@@ -4484,6 +4542,7 @@ mod tests {
             agent_indicator: false,
             local_frames: true,
             audio: false,
+            observe_input: false,
         }
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@desklink/host",
-    "version": "0.5.9",
+    "version": "0.5.10",
     "license": "Apache-2.0",
     "repository": {
         "type": "git",
@@ -38,7 +38,7 @@
         "ws": "8.21.3"
     },
     "optionalDependencies": {
-        "@desklink/host-linux-x64-gnu": "0.5.9",
-        "@desklink/host-darwin-arm64": "0.5.9"
+        "@desklink/host-linux-x64-gnu": "0.5.10",
+        "@desklink/host-darwin-arm64": "0.5.10"
     }
 }