@desklink/host 0.5.3 → 0.5.6

package/engine/src/session.rs

@@ -93,6 +93,10 @@ pub struct Metrics {
     /// Control messages forwarded back to the consumer that owns the device,
     /// instead of being applied to a desktop this engine holds.
     pub input_forwarded: u64,
+    pub audio_frames: u64,
+    pub audio_bytes: u64,
+    pub audio_dropped_bytes: u64,
+    pub audio_gap_micros: u64,
 }
 
 fn tile_hashes(raw: &[u8], width: usize, height: usize) -> Vec<u64> {
@@ -708,6 +712,8 @@ pub struct OpenRequest {
     /// Keep the latest frame and its tile hashes for `session.frame` and
     /// `session.frame.changed`; off, a frame is converted once, for the encoder.
     pub local_frames: bool,
+    /// Capture the desktop's own system audio for `session.audio`.
+    pub audio: bool,
 }
 
 /// Events a session raises for its consumer.
@@ -736,6 +742,14 @@ pub enum SessionEvent {
         seq: u64,
         damage: Vec<[usize; 4]>,
     },
+    /// One chunk was captured, even if the full queue dropped it, on the same
+    /// clock `session.frame.changed` uses.
+    Audio {
+        seq: u64,
+        timestamp_us: u64,
+        bytes: u64,
+        gap_micros: u64,
+    },
     /// One control-channel message from the client, for a source whose device
     /// the consumer drives. The engine applied nothing locally.
     Input {
@@ -817,6 +831,9 @@ struct Inner {
     encoded: Mutex<Option<Encoded>>,
     /// `OpenRequest::local_frames`: whether `latest` is kept at all.
     local_frames: bool,
+    /// Dropped on close, stopping the PipeWire stream.
+    #[cfg(target_os = "linux")]
+    audio: Mutex<Option<crate::audio::AudioCapture>>,
     cursor_info: serde_json::Value,
 }
 
@@ -869,6 +886,9 @@ pub fn capabilities() -> serde_json::Value {
         .map(|desktop| serde_json::json!([desktop.screen_size().0, desktop.screen_size().1]))
         .unwrap_or(serde_json::Value::Null);
     let clipboard = wayland_clipboard_available();
+    // Asked, not taken: the probe connects to this session's PipeWire, reads the
+    // registry and leaves. No portal, no prompt, no capture.
+    let audio = crate::audio::probe();
     let (input, grant_state, unavailable) = match crate::input::probe() {
         Ok(()) => (serde_json::json!(true), "granted", serde_json::Value::Null),
         Err(unavailable) => (
@@ -896,7 +916,9 @@ pub fn capabilities() -> serde_json::Value {
             "cursor": "embedded",
             "cursor_modes": ["embedded", "hidden"],
             "cursor_positions": {"x11": true, "wayland": "ext-image-copy-capture-v1"},
-            "audio": false,
+            "audio": audio.is_some(),
+            "audio_formats": crate::audio::FORMATS,
+            "audio_source": audio.map(|source| source.name),
         },
         "encode": encode_capabilities(),
         "input": {
@@ -949,6 +971,7 @@ pub fn capabilities() -> serde_json::Value {
             "formats": ["bgra"],
             "cursor": "embedded",
             "cursor_modes": ["embedded"],
+            // ScreenCaptureKit audio is not wired up; macOS is out of scope.
             "audio": false,
             "displays": crate::mac::displays(),
             "grant": if capture_granted { "granted" } else { "missing-screen-recording" },
@@ -1007,6 +1030,13 @@ impl Session {
         request: OpenRequest,
         events: tokio_mpsc::UnboundedSender<Notice>,
     ) -> std::result::Result<Self, SessionError> {
+        #[cfg(not(target_os = "linux"))]
+        if request.audio {
+            return Err(SessionError::new(
+                "audio-unavailable",
+                "system audio capture requires Linux PipeWire",
+            ));
+        }
         if request.cursor == crate::protocol::CursorMode::Hidden {
             #[cfg(not(target_os = "linux"))]
             return Err(SessionError::new(
@@ -1370,6 +1400,14 @@ impl Session {
             SessionError::new("input", format!("no keyboard layout: {error:#}"))
         })?;
 
+        // Refuse before allocating the peer; never silently downgrade.
+        #[cfg(target_os = "linux")]
+        let audio = if request.audio {
+            Some(start_audio(&id, &events)?)
+        } else {
+            None
+        };
+
         // H.264 is offered beside VP9 only when an encoder for it starts on
         // this machine; the answer then decides which one is coded.
         let codec = if !crate::h264::fits(width, height) {
@@ -1433,6 +1471,8 @@ impl Session {
             },
             capture: Mutex::new(Some(capture)),
             layout: Mutex::new(Some(layout)),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(audio),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(false),
             closed: AtomicBool::new(false),
@@ -1474,6 +1514,12 @@ impl Session {
         height: usize,
         events: tokio_mpsc::UnboundedSender<Notice>,
     ) -> std::result::Result<Self, SessionError> {
+        if request.audio {
+            return Err(SessionError::new(
+                "operation",
+                "an encoded source carries video the consumer already encoded; there is no local desktop audio to capture",
+            ));
+        }
         if width == 0 || height == 0 || width > 7680 || height > 4320 {
             return Err(SessionError::new(
                 "source",
@@ -1523,6 +1569,8 @@ impl Session {
             point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
             capture: Mutex::new(None),
             layout: Mutex::new(None),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(None),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(false),
             closed: AtomicBool::new(false),
@@ -1850,11 +1898,92 @@ impl Session {
     }
 
     pub fn metrics(&self) -> Metrics {
-        self.inner
+        let mut metrics = self
+            .inner
             .metrics
             .lock()
             .map(|m| m.clone())
-            .unwrap_or_default()
+            .unwrap_or_default();
+        #[cfg(target_os = "linux")]
+        if let Some(audio) = lock(&self.inner.audio).as_ref() {
+            let (frames, bytes, dropped_bytes, gap_micros) = audio.totals();
+            metrics.audio_frames = frames;
+            metrics.audio_bytes = bytes;
+            metrics.audio_dropped_bytes = dropped_bytes;
+            metrics.audio_gap_micros = gap_micros;
+        }
+        metrics
+    }
+
+    pub fn audio_info(&self) -> Option<serde_json::Value> {
+        #[cfg(target_os = "linux")]
+        {
+            lock(&self.inner.audio).as_ref()?; // start() accepted the fixed format.
+            return Some(serde_json::json!({
+                "source": "pipewire-monitor", "format": crate::audio::FORMAT,
+                "sample_rate": crate::audio::SAMPLE_RATE, "channels": crate::audio::CHANNELS,
+            }));
+        }
+        #[cfg(not(target_os = "linux"))]
+        None
+    }
+
+    /// Consume one queued PCM chunk; samples never travel through JSON.
+    #[cfg(target_os = "linux")]
+    pub fn audio(
+        &self,
+        path: &str,
+        max_bytes: usize,
+    ) -> std::result::Result<serde_json::Value, SessionError> {
+        let no_audio = if lock(&self.inner.encoded).is_some() {
+            "this session forwards an encoded stream and has no local audio"
+        } else {
+            "this session was opened without audio"
+        };
+        let held = lock(&self.inner.audio);
+        let capture = held
+            .as_ref()
+            .ok_or_else(|| SessionError::new("operation", no_audio))?;
+        let frame_bytes = crate::audio::BYTES_PER_FRAME as usize;
+        if max_bytes < frame_bytes {
+            return Err(SessionError::new(
+                "malformed",
+                "max_bytes must hold at least one sample frame",
+            ));
+        }
+        let chunk = capture
+            .take()
+            .ok_or_else(|| SessionError::new("audio", "no audio chunk has arrived yet"))?;
+        let wanted = (max_bytes / frame_bytes * frame_bytes).min(chunk.bytes.len());
+        let written = if path.is_empty() { 0 } else { wanted };
+        let result = serde_json::json!({
+            "seq": chunk.seq,
+            "timestamp_us": crate::protocol::timestamp_us(chunk.at),
+            "frames": written as u64 / crate::audio::BYTES_PER_FRAME,
+            "bytes": written,
+            "format": crate::audio::FORMAT,
+            "sample_rate": crate::audio::SAMPLE_RATE,
+            "channels": crate::audio::CHANNELS,
+            "gap_ms": chunk.gap_micros / 1000,
+            "written": !path.is_empty(),
+        });
+        if written > 0 {
+            std::fs::write(path, &chunk.bytes[..written])
+                .map_err(|error| SessionError::new("audio", error.to_string()))?;
+        }
+        Ok(result)
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    pub fn audio(
+        &self,
+        _path: &str,
+        _max_bytes: usize,
+    ) -> std::result::Result<serde_json::Value, SessionError> {
+        Err(SessionError::new(
+            "operation",
+            "this platform has no system audio capture",
+        ))
     }
 
     pub async fn accept_answer(&self, sdp: String) -> Result<()> {
@@ -1987,6 +2116,8 @@ impl Inner {
         // out of scope, so the consumer can truthfully say the desktop stopped.
         let capture = self.capture.lock().ok().and_then(|mut held| held.take());
         drop(capture);
+        #[cfg(target_os = "linux")]
+        drop(self.audio.lock().ok().and_then(|mut held| held.take()));
         self.notify(SessionEvent::State {
             capture: "ended",
             transport: String::from("closed"),
@@ -2112,6 +2243,15 @@ impl Inner {
             self.forward(message);
             return;
         }
+        // A shared window's place on the desktop is unknown (the portal reports
+        // no origin for it), while the pointer device spans the whole desktop:
+        // a tap would land somewhere the person never pointed.
+        if self.source.source_type.as_deref() == Some("Window")
+            && matches!(&message, ControlMessage::Pointer { phase, .. } if !matches!(phase, PointerPhase::Cancel))
+        {
+            self.reject(seq, "input-unavailable", WINDOW_POINTER_REFUSAL);
+            return;
+        }
         if !matches!(
             &message,
             ControlMessage::ClipboardRead { .. } | ControlMessage::ClipboardWrite { .. }
@@ -2445,6 +2585,27 @@ fn frame_interval(max_fps: u32) -> Duration {
     Duration::from_secs_f64(1.0 / max_fps.max(1) as f64)
 }
 
+#[cfg(target_os = "linux")]
+fn start_audio(
+    id: &str,
+    events: &tokio_mpsc::UnboundedSender<Notice>,
+) -> std::result::Result<crate::audio::AudioCapture, SessionError> {
+    let events = events.clone();
+    let id = id.to_owned();
+    crate::audio::AudioCapture::start(Box::new(move |notice| {
+        let _ = events.send(Notice {
+            session_id: id.clone(),
+            event: SessionEvent::Audio {
+                seq: notice.seq,
+                timestamp_us: crate::protocol::timestamp_us(notice.at),
+                bytes: notice.bytes,
+                gap_micros: notice.gap_micros,
+            },
+        });
+    }))
+    .map_err(|reason| SessionError::new("audio-unavailable", reason))
+}
+
 /// The newest captured frame and when it was captured, handed from the capture
 /// thread to the encoder. A newer frame replaces one the encoder has not taken,
 /// so however far behind it falls, what it codes next is the desktop as it is
@@ -3243,6 +3404,11 @@ fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiv
                         first_frame: inner.metrics.lock().map(|m| m.encoded_frames).unwrap_or(0)
                             > 0,
                     });
+                    // desklink scout trace: the session's transport emit, one
+                    // line after the peer event that caused it.
+                    if crate::peer::dl_trace_on() {
+                        crate::peer::dl_eng!("session_transport_emit {}", format!("{state}").to_lowercase());
+                    }
                 }
                 PeerEvent::ControlOpen => {
                     inner.control_open.store(true, Ordering::SeqCst);
@@ -3301,6 +3467,8 @@ fn available_parallelism() -> usize {
         .unwrap_or(4)
 }
 
+const WINDOW_POINTER_REFUSAL: &str = "a shared window cannot take taps: its place on the desktop is unknown; share the whole screen to point and click";
+
 /// Encoded-surface pixels to the source's own pixels. The client aims at what it
 /// sees; the applier is sized and clamped to the source, so the layout origin
 /// must not be added here.
@@ -3417,6 +3585,8 @@ mod tests {
             point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
             capture: Mutex::new(None),
             layout: Mutex::new(None),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(None),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(false),
             closed: AtomicBool::new(false),
@@ -3673,6 +3843,8 @@ mod tests {
             #[cfg(target_os = "linux")]
             point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
             layout: Mutex::new(Some(Layout::from_environment().expect("a keymap"))),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(None),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(true),
             closed: AtomicBool::new(false),
@@ -4080,6 +4252,7 @@ mod tests {
             loopback_tcp: false,
             agent_indicator: false,
             local_frames: true,
+            audio: false,
         }
     }
 

package/engine/src/win.rs

@@ -15,6 +15,7 @@ pub fn capabilities() -> Value {
             "formats": [],
             "cursor": "unavailable",
             "cursor_modes": [],
+            // No Windows capture path exists to carry audio from; out of scope.
             "audio": false,
             "displays": crate::portal::displays(),
             "grant": "unavailable",

package/engine/src/x11.rs

@@ -465,16 +465,3 @@ impl X11Desktop {
         )
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn fitting_never_upscales_and_keeps_even_dimensions() {
-        assert_eq!(fit(800, 600, 1280, 800), (800, 600));
-        assert_eq!(fit(1920, 1080, 1280, 720), (1280, 720));
-        let (w, h) = fit(1000, 1000, 600, 600);
-        assert_eq!((w, h), (600, 600));
-    }
-}

package/engine/vendor/rtc-ice/Cargo.toml

@@ -0,0 +1,137 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2024"
+name = "rtc-ice"
+version = "0.21.0"
+authors = ["Rain Liu <yliu@webrtc.rs>"]
+build = false
+autolib = false
+autobins = false
+autoexamples = false
+autotests = false
+autobenches = false
+description = "RTC ICE in Rust"
+homepage = "https://webrtc.rs"
+documentation = "https://docs.rs/rtc-ice"
+readme = "README.md"
+keywords = [
+    "sansio",
+    "networking",
+    "protocols",
+]
+categories = ["network-programming"]
+license = "MIT/Apache-2.0"
+repository = "https://github.com/webrtc-rs/rtc"
+resolver = "2"
+
+[features]
+crypto-aws-lc-rs = [
+    "crypto/crypto-aws-lc-rs",
+    "stun/crypto-aws-lc-rs",
+]
+crypto-ring = [
+    "crypto/crypto-ring",
+    "stun/crypto-ring",
+]
+default = ["crypto-ring"]
+
+[lib]
+name = "rtc_ice"
+path = "src/lib.rs"
+
+[[example]]
+name = "ping_pong"
+path = "examples/ping_pong.rs"
+bench = false
+
+[dependencies.bytes]
+version = "1.12.0"
+
+[dependencies.crc]
+version = "3.0.1"
+
+[dependencies.crypto]
+version = "0.21.0"
+default-features = false
+package = "rtc-crypto"
+
+[dependencies.log]
+version = "0.4.33"
+
+[dependencies.mdns]
+version = "0.21.0"
+package = "rtc-mdns"
+
+[dependencies.sansio]
+version = "1"
+
+[dependencies.serde]
+version = "1.0.228"
+features = ["derive"]
+
+[dependencies.shared]
+version = "0.21.0"
+features = []
+default-features = false
+package = "rtc-shared"
+
+[dependencies.stun]
+version = "0.21.0"
+default-features = false
+package = "rtc-stun"
+
+[dependencies.url]
+version = "2.5.0"
+
+[dependencies.uuid]
+version = "1"
+features = ["v4"]
+
+[dev-dependencies.chrono]
+version = "0.4.45"
+
+[dev-dependencies.clap]
+version = "4.6.1"
+features = ["derive"]
+
+[dev-dependencies.ctrlc]
+version = "3.5.2"
+
+[dev-dependencies.env_logger]
+version = "0.11.11"
+
+[dev-dependencies.futures]
+version = "0.3.30"
+
+[dev-dependencies.hyper]
+version = "0.14.28"
+features = ["full"]
+
+[dev-dependencies.ipnet]
+version = "2.9.0"
+
+[dev-dependencies.lazy_static]
+version = "1.4.0"
+
+[dev-dependencies.regex]
+version = "1.10.3"
+
+[dev-dependencies.serde_json]
+version = "1.0.114"
+
+[dev-dependencies.tokio]
+version = "1.52.3"
+features = ["full"]
+
+[dev-dependencies.waitgroup]
+version = "0.1.2"

package/engine/vendor/rtc-ice/DESKLINK-PATCH.md

@@ -0,0 +1,111 @@
+# desklink patch to vendored `rtc-ice` 0.21.0
+
+Two shapes, both for the roam-persistence goal: keep media flowing across an
+ICE restart, and nominate the restarted generation without waiting for a check
+tick.
+
+## What
+
+### 1. Media keeps flowing on the previous pair across a restart
+
+`Agent::apply_restart` (src/agent/mod.rs) no longer interrupts media that is
+already flowing on the current selected ICE pair. When a restart is applied, the
+pair media was using is remembered (`restart_media_pair`), and
+`get_selected_candidate_pair` keeps returning it until the restarted generation
+nominates a new pair. Everything else about the restart is unchanged: the new
+local credentials are installed, remote credentials are dropped, binding
+requests and candidate pairs are cleared, the agent re-checks, and the
+connection state goes to `Checking`.
+
+### 2. The nomination does not wait for the next check tick
+
+`ControllingSelector::handle_success_response` (src/agent/agent_selector.rs)
+requests an immediate check pass when the response makes a pair `Succeeded` and
+that pair is the one the next pass would nominate. Nomination is decided in
+`contact_candidates`, which the driver only reaches on a `check_interval` tick
+(200 ms) or when an inbound Binding request happens to arrive while a valid pair
+already exists; a pair that becomes valid just after a tick therefore waits a
+full tick for its `USE-CANDIDATE`. The request uses
+`Agent::request_connectivity_check()`, the deferred-check mechanism added for
+issue #88, so `contact()` is not called from the read path and `poll_timeout`
+reports an already-due deadline instead.
+
+Every changed spot is marked with a `// desklink patch:` comment.
+
+## Why
+
+An ICE restart on the engine (a `session.restart_ice` from the client) used to
+drop the selected pair before the client had even seen the offer. The engine's
+RTP/DTLS send path stamps outgoing packets from
+`Agent::get_selected_candidate_pair`, and drops them when there is none, so
+media stopped until the new generation re-nominated a pair — 1.3–2.5 s on real
+networks, which is inside the roam-persistence 2 s bar.
+
+RFC 8445 §9 and libwebrtc keep media on the **previous selected pair** while a
+restart's connectivity checks run, and move to the new pair once it is
+nominated. rtc-ice 0.21.0 did not; this patch makes it do so. It is needed for
+the desklink roam goal: after a network path change, the live picture must be
+advancing again within 2 s of path-back on the same session, including the
+client's 300 ms reattach grace being removed later.
+
+The behavior is pinned by the engine integration test
+`a_restart_keeps_media_flowing_on_the_selected_pair_until_the_new_one_nominates`
+in `packages/desktop-host/engine/src/peer.rs`.
+
+Shape 2 exists because the tick is the engine's largest single bounded cost
+between a restart offer and `transport=connected`: in the routed roam leg the
+engine's first pair became valid 332 ms after the answer and its `USE-CANDIDATE`
+left 197 ms later — one 200 ms tick. A peer that runs no connectivity checks of
+its own leaves an inbound request unavailable, and the nomination then waits for
+the tick. The engine integration test asserts the reconnect bound; the loopback
+harness cannot grade sub-tick timing on its own (its client answers checks and
+sends triggered checks in the same burst, so an inbound request is available
+almost immediately), which is why the field measurement is the reference.
+
+## Scope of the change
+
+### Shape 1
+
+- `Agent` gains `restart_media_pair: Option<(Candidate, Candidate)>`, initialised
+  to `None`.
+- `apply_restart` clones the current `(local, remote)` into it before the
+  teardown.
+- `get_selected_candidate_pair` returns that clone when there is no live
+  `selected_pair`.
+- `set_selected_pair(.., Some(..))` clears it once a new pair is nominated.
+- `update_connection_state` clears it on `Failed` / `Closed`, so a restart that
+  never nominates does not keep media pointed at an unreachable pair.
+
+### Shape 2
+
+- `ControllingSelector::handle_success_response` calls
+  `request_connectivity_check()` after a pair becomes `Succeeded`, when no pair is
+  selected or nominated yet, this pair is the best valid pair, and both of its
+  candidates have passed their acceptance wait (`is_nominatable`).
+- Nothing else changes: the same `contact_candidates` pass decides the
+  nomination, the acceptance waits still gate it, and a nomination already in
+  flight is not re-requested, so the extra pass cannot repeat per response.
+
+No public API changed; no signalling or wire change.
+
+## How to drop the patch
+
+1. Remove the `[patch.crates-io] rtc-ice = { path = "vendor/rtc-ice" }` entry from
+   `packages/desktop-host/engine/Cargo.toml`.
+2. Delete `packages/desktop-host/engine/vendor/rtc-ice`.
+3. Update `packages/desktop-host/engine/Cargo.lock` (`cargo update -p rtc-ice`).
+
+Do this once upstream rtc-ice keeps the previous selected pair across a restart
+on its own and nominates a freshly validated pair without waiting for the next
+check tick (an upstream PR is a follow-up; it needs the captain's word first,
+since it would carry his name).
+
+## Dev-only measurement instrumentation (not part of the patch)
+
+`src/dl_trace.rs` and its call sites — marked `// desklink scout trace:` — are
+a private-lab instrument, not a shipped behaviour change. They are off unless
+`DESKLINK_ICE_TRACE=1` is set, and when on they only write `[ice <us>]` lines to
+stderr, which the bridge forwards untouched. To drop them: delete
+`src/dl_trace.rs`, remove `pub(crate) mod dl_trace;` from `src/lib.rs`, and
+remove the marked blocks from `src/agent/mod.rs` and
+`src/agent/agent_selector.rs` (grep `desklink scout trace`).