@desklink/host 0.5.9 → 0.5.11

package/engine/src/session.rs

@@ -714,6 +714,8 @@ pub struct OpenRequest {
     pub local_frames: bool,
     /// Capture the desktop's own system audio for `session.audio`.
     pub audio: bool,
+    /// Listen to the desktop's own pointer buttons, cursor and keys.
+    pub observe_input: bool,
 }
 
 /// Events a session raises for its consumer.
@@ -755,6 +757,11 @@ pub enum SessionEvent {
     Input {
         input: serde_json::Value,
     },
+    /// A pointer button or key the person pressed, observed rather than applied.
+    Observed {
+        timestamp_us: u64,
+        input: crate::protocol::ObservedInput,
+    },
     /// The far end needs a fresh reference frame and only the consumer can make
     /// one: the encoder for this track is on its side. Raised by an RTCP PLI or
     /// FIR, and once when the transport connects, because everything sent before
@@ -854,6 +861,9 @@ struct Inner {
     #[cfg(target_os = "linux")]
     audio: Mutex<Option<crate::audio::AudioCapture>>,
     cursor_info: serde_json::Value,
+    /// `session.open`'s `inputObservation` member, fixed when the session opened.
+    observation: serde_json::Value,
+    observer: Mutex<Option<crate::observe::Tap>>,
 }
 
 /// A source whose video arrives already encoded.
@@ -978,6 +988,7 @@ pub fn capabilities() -> serde_json::Value {
             }),
         ),
     };
+    let observe = crate::input::observe_probe();
     serde_json::json!({
         "protocol": crate::protocol::PROTOCOL_VERSION,
         "engine": format!("desklink-host/{}", env!("CARGO_PKG_VERSION")),
@@ -1009,6 +1020,11 @@ pub fn capabilities() -> serde_json::Value {
             "layout": keymap::LayoutNames::from_environment().identity(),
             "unavailable_reason": unavailable,
             "grant": input_grant,
+            "observe": {
+                "mechanism": "cgeventtap",
+                "grant": if observe.is_ok() { "granted" } else { "missing-input-monitoring" },
+                "unavailable_reason": observe.as_ref().err().map(|e| serde_json::json!({"reason": e.reason, "remedy": e.remedy})),
+            },
         },
         "clipboard": { "read": true, "write": true, "mime": ["text/plain;charset=utf-8"],
             "maxBytes": clipboard::MAX_CLIPBOARD_BYTES },
@@ -1480,6 +1496,29 @@ impl Session {
             "origin": { "x": source.origin_x, "y": source.origin_y },
         });
 
+        // Observation is an addition to the stream, never a condition of it: a
+        // desktop that cannot be observed still streams, and the reply says why.
+        let observed = request.observe_input.then(|| {
+            let sink: crate::observe::Sink = {
+                let events = events.clone();
+                let id = id.clone();
+                Arc::new(move |observation| {
+                    let event = match observation {
+                        crate::observe::Observation::Input { timestamp_us, input } => {
+                            SessionEvent::Observed { timestamp_us, input }
+                        }
+                        crate::observe::Observation::Cursor(position) => SessionEvent::Cursor { position },
+                    };
+                    let _ = events.send(Notice { session_id: id.clone(), event });
+                })
+            };
+            crate::observe::start(source.node_id, (source_w, source_h), sink)
+        });
+        if let Some(Err(error)) = &observed {
+            log::warn!("input observation unavailable: {} ({})", error.reason, error.code);
+        }
+        let observation = crate::observe::status(observed.as_ref());
+
         let pipeline = Arc::new(AtomicBool::new(true));
         let inner = Arc::new(Inner {
             id,
@@ -1526,6 +1565,8 @@ impl Session {
             encoded: Mutex::new(None),
             local_frames: request.local_frames,
             cursor_info,
+            observation,
+            observer: Mutex::new(observed.and_then(Result::ok)),
         });
 
         inner.notify(SessionEvent::Description {
@@ -1641,6 +1682,13 @@ impl Session {
             })),
             local_frames: false,
             cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
+            observation: serde_json::json!({
+                "observing": false,
+                "code": "unsupported-source",
+                "reason": "consumer-encoded video has no desktop of its own to observe",
+                "remedy": "observe the device where its pixels come from",
+            }),
+            observer: Mutex::new(None),
         });
         inner.notify(SessionEvent::State {
             capture: "consented",
@@ -1706,6 +1754,10 @@ impl Session {
         &self.inner.cursor_info
     }
 
+    pub fn observation_info(&self) -> &serde_json::Value {
+        &self.inner.observation
+    }
+
     pub fn geometry(&self) -> &serde_json::Value {
         &self.inner.geometry
     }
@@ -2176,6 +2228,8 @@ impl Inner {
         // handlers, and `shutdown` exits right after this. The encode thread
         // finds it gone and stops.
         drop(lock(&self.encoder).take());
+        // Stop listening to the person's keys the moment the session ends.
+        drop(lock(&self.observer).take());
         if let Ok(mut indicator) = self.indicator.lock() {
             indicator.take();
         }
@@ -3829,6 +3883,8 @@ mod tests {
             })),
             local_frames: false,
             cursor_info: serde_json::json!({"mode":"unavailable", "positions":false}),
+            observation: serde_json::json!({"observing": false}),
+            observer: Mutex::new(None),
         });
         spawn_peer_events(&inner, peer_events_rx);
         spawn_encoded(&inner, feed_rx);
@@ -4084,6 +4140,8 @@ mod tests {
             encoded: Mutex::new(None),
             local_frames: true,
             cursor_info: serde_json::json!({"mode":"embedded", "positions":false}),
+            observation: serde_json::json!({"observing": false}),
+            observer: Mutex::new(None),
         });
         (inner, recorded)
     }
@@ -4484,6 +4542,7 @@ mod tests {
             agent_indicator: false,
             local_frames: true,
             audio: false,
+            observe_input: false,
         }
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@desklink/host",
-    "version": "0.5.9",
+    "version": "0.5.11",
     "license": "Apache-2.0",
     "repository": {
         "type": "git",
@@ -38,7 +38,7 @@
         "ws": "8.21.3"
     },
     "optionalDependencies": {
-        "@desklink/host-linux-x64-gnu": "0.5.9",
-        "@desklink/host-darwin-arm64": "0.5.9"
+        "@desklink/host-linux-x64-gnu": "0.5.11",
+        "@desklink/host-darwin-arm64": "0.5.11"
     }
 }