@desklink/host 0.5.4 → 0.5.6

package/engine/src/session.rs

@@ -93,6 +93,10 @@ pub struct Metrics {
     /// Control messages forwarded back to the consumer that owns the device,
     /// instead of being applied to a desktop this engine holds.
     pub input_forwarded: u64,
+    pub audio_frames: u64,
+    pub audio_bytes: u64,
+    pub audio_dropped_bytes: u64,
+    pub audio_gap_micros: u64,
 }
 
 fn tile_hashes(raw: &[u8], width: usize, height: usize) -> Vec<u64> {
@@ -708,6 +712,8 @@ pub struct OpenRequest {
     /// Keep the latest frame and its tile hashes for `session.frame` and
     /// `session.frame.changed`; off, a frame is converted once, for the encoder.
     pub local_frames: bool,
+    /// Capture the desktop's own system audio for `session.audio`.
+    pub audio: bool,
 }
 
 /// Events a session raises for its consumer.
@@ -736,6 +742,14 @@ pub enum SessionEvent {
         seq: u64,
         damage: Vec<[usize; 4]>,
     },
+    /// One chunk was captured, even if the full queue dropped it, on the same
+    /// clock `session.frame.changed` uses.
+    Audio {
+        seq: u64,
+        timestamp_us: u64,
+        bytes: u64,
+        gap_micros: u64,
+    },
     /// One control-channel message from the client, for a source whose device
     /// the consumer drives. The engine applied nothing locally.
     Input {
@@ -817,6 +831,9 @@ struct Inner {
     encoded: Mutex<Option<Encoded>>,
     /// `OpenRequest::local_frames`: whether `latest` is kept at all.
     local_frames: bool,
+    /// Dropped on close, stopping the PipeWire stream.
+    #[cfg(target_os = "linux")]
+    audio: Mutex<Option<crate::audio::AudioCapture>>,
     cursor_info: serde_json::Value,
 }
 
@@ -869,6 +886,9 @@ pub fn capabilities() -> serde_json::Value {
         .map(|desktop| serde_json::json!([desktop.screen_size().0, desktop.screen_size().1]))
         .unwrap_or(serde_json::Value::Null);
     let clipboard = wayland_clipboard_available();
+    // Asked, not taken: the probe connects to this session's PipeWire, reads the
+    // registry and leaves. No portal, no prompt, no capture.
+    let audio = crate::audio::probe();
     let (input, grant_state, unavailable) = match crate::input::probe() {
         Ok(()) => (serde_json::json!(true), "granted", serde_json::Value::Null),
         Err(unavailable) => (
@@ -896,7 +916,9 @@ pub fn capabilities() -> serde_json::Value {
             "cursor": "embedded",
             "cursor_modes": ["embedded", "hidden"],
             "cursor_positions": {"x11": true, "wayland": "ext-image-copy-capture-v1"},
-            "audio": false,
+            "audio": audio.is_some(),
+            "audio_formats": crate::audio::FORMATS,
+            "audio_source": audio.map(|source| source.name),
         },
         "encode": encode_capabilities(),
         "input": {
@@ -949,6 +971,7 @@ pub fn capabilities() -> serde_json::Value {
             "formats": ["bgra"],
             "cursor": "embedded",
             "cursor_modes": ["embedded"],
+            // ScreenCaptureKit audio is not wired up; macOS is out of scope.
             "audio": false,
             "displays": crate::mac::displays(),
             "grant": if capture_granted { "granted" } else { "missing-screen-recording" },
@@ -1007,6 +1030,13 @@ impl Session {
         request: OpenRequest,
         events: tokio_mpsc::UnboundedSender<Notice>,
     ) -> std::result::Result<Self, SessionError> {
+        #[cfg(not(target_os = "linux"))]
+        if request.audio {
+            return Err(SessionError::new(
+                "audio-unavailable",
+                "system audio capture requires Linux PipeWire",
+            ));
+        }
         if request.cursor == crate::protocol::CursorMode::Hidden {
             #[cfg(not(target_os = "linux"))]
             return Err(SessionError::new(
@@ -1370,6 +1400,14 @@ impl Session {
             SessionError::new("input", format!("no keyboard layout: {error:#}"))
         })?;
 
+        // Refuse before allocating the peer; never silently downgrade.
+        #[cfg(target_os = "linux")]
+        let audio = if request.audio {
+            Some(start_audio(&id, &events)?)
+        } else {
+            None
+        };
+
         // H.264 is offered beside VP9 only when an encoder for it starts on
         // this machine; the answer then decides which one is coded.
         let codec = if !crate::h264::fits(width, height) {
@@ -1433,6 +1471,8 @@ impl Session {
             },
             capture: Mutex::new(Some(capture)),
             layout: Mutex::new(Some(layout)),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(audio),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(false),
             closed: AtomicBool::new(false),
@@ -1474,6 +1514,12 @@ impl Session {
         height: usize,
         events: tokio_mpsc::UnboundedSender<Notice>,
     ) -> std::result::Result<Self, SessionError> {
+        if request.audio {
+            return Err(SessionError::new(
+                "operation",
+                "an encoded source carries video the consumer already encoded; there is no local desktop audio to capture",
+            ));
+        }
         if width == 0 || height == 0 || width > 7680 || height > 4320 {
             return Err(SessionError::new(
                 "source",
@@ -1523,6 +1569,8 @@ impl Session {
             point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
             capture: Mutex::new(None),
             layout: Mutex::new(None),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(None),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(false),
             closed: AtomicBool::new(false),
@@ -1850,11 +1898,92 @@ impl Session {
     }
 
     pub fn metrics(&self) -> Metrics {
-        self.inner
+        let mut metrics = self
+            .inner
             .metrics
             .lock()
             .map(|m| m.clone())
-            .unwrap_or_default()
+            .unwrap_or_default();
+        #[cfg(target_os = "linux")]
+        if let Some(audio) = lock(&self.inner.audio).as_ref() {
+            let (frames, bytes, dropped_bytes, gap_micros) = audio.totals();
+            metrics.audio_frames = frames;
+            metrics.audio_bytes = bytes;
+            metrics.audio_dropped_bytes = dropped_bytes;
+            metrics.audio_gap_micros = gap_micros;
+        }
+        metrics
+    }
+
+    pub fn audio_info(&self) -> Option<serde_json::Value> {
+        #[cfg(target_os = "linux")]
+        {
+            lock(&self.inner.audio).as_ref()?; // start() accepted the fixed format.
+            return Some(serde_json::json!({
+                "source": "pipewire-monitor", "format": crate::audio::FORMAT,
+                "sample_rate": crate::audio::SAMPLE_RATE, "channels": crate::audio::CHANNELS,
+            }));
+        }
+        #[cfg(not(target_os = "linux"))]
+        None
+    }
+
+    /// Consume one queued PCM chunk; samples never travel through JSON.
+    #[cfg(target_os = "linux")]
+    pub fn audio(
+        &self,
+        path: &str,
+        max_bytes: usize,
+    ) -> std::result::Result<serde_json::Value, SessionError> {
+        let no_audio = if lock(&self.inner.encoded).is_some() {
+            "this session forwards an encoded stream and has no local audio"
+        } else {
+            "this session was opened without audio"
+        };
+        let held = lock(&self.inner.audio);
+        let capture = held
+            .as_ref()
+            .ok_or_else(|| SessionError::new("operation", no_audio))?;
+        let frame_bytes = crate::audio::BYTES_PER_FRAME as usize;
+        if max_bytes < frame_bytes {
+            return Err(SessionError::new(
+                "malformed",
+                "max_bytes must hold at least one sample frame",
+            ));
+        }
+        let chunk = capture
+            .take()
+            .ok_or_else(|| SessionError::new("audio", "no audio chunk has arrived yet"))?;
+        let wanted = (max_bytes / frame_bytes * frame_bytes).min(chunk.bytes.len());
+        let written = if path.is_empty() { 0 } else { wanted };
+        let result = serde_json::json!({
+            "seq": chunk.seq,
+            "timestamp_us": crate::protocol::timestamp_us(chunk.at),
+            "frames": written as u64 / crate::audio::BYTES_PER_FRAME,
+            "bytes": written,
+            "format": crate::audio::FORMAT,
+            "sample_rate": crate::audio::SAMPLE_RATE,
+            "channels": crate::audio::CHANNELS,
+            "gap_ms": chunk.gap_micros / 1000,
+            "written": !path.is_empty(),
+        });
+        if written > 0 {
+            std::fs::write(path, &chunk.bytes[..written])
+                .map_err(|error| SessionError::new("audio", error.to_string()))?;
+        }
+        Ok(result)
+    }
+
+    #[cfg(not(target_os = "linux"))]
+    pub fn audio(
+        &self,
+        _path: &str,
+        _max_bytes: usize,
+    ) -> std::result::Result<serde_json::Value, SessionError> {
+        Err(SessionError::new(
+            "operation",
+            "this platform has no system audio capture",
+        ))
     }
 
     pub async fn accept_answer(&self, sdp: String) -> Result<()> {
@@ -1987,6 +2116,8 @@ impl Inner {
         // out of scope, so the consumer can truthfully say the desktop stopped.
         let capture = self.capture.lock().ok().and_then(|mut held| held.take());
         drop(capture);
+        #[cfg(target_os = "linux")]
+        drop(self.audio.lock().ok().and_then(|mut held| held.take()));
         self.notify(SessionEvent::State {
             capture: "ended",
             transport: String::from("closed"),
@@ -2112,6 +2243,15 @@ impl Inner {
             self.forward(message);
             return;
         }
+        // A shared window's place on the desktop is unknown (the portal reports
+        // no origin for it), while the pointer device spans the whole desktop:
+        // a tap would land somewhere the person never pointed.
+        if self.source.source_type.as_deref() == Some("Window")
+            && matches!(&message, ControlMessage::Pointer { phase, .. } if !matches!(phase, PointerPhase::Cancel))
+        {
+            self.reject(seq, "input-unavailable", WINDOW_POINTER_REFUSAL);
+            return;
+        }
         if !matches!(
             &message,
             ControlMessage::ClipboardRead { .. } | ControlMessage::ClipboardWrite { .. }
@@ -2445,6 +2585,27 @@ fn frame_interval(max_fps: u32) -> Duration {
     Duration::from_secs_f64(1.0 / max_fps.max(1) as f64)
 }
 
+#[cfg(target_os = "linux")]
+fn start_audio(
+    id: &str,
+    events: &tokio_mpsc::UnboundedSender<Notice>,
+) -> std::result::Result<crate::audio::AudioCapture, SessionError> {
+    let events = events.clone();
+    let id = id.to_owned();
+    crate::audio::AudioCapture::start(Box::new(move |notice| {
+        let _ = events.send(Notice {
+            session_id: id.clone(),
+            event: SessionEvent::Audio {
+                seq: notice.seq,
+                timestamp_us: crate::protocol::timestamp_us(notice.at),
+                bytes: notice.bytes,
+                gap_micros: notice.gap_micros,
+            },
+        });
+    }))
+    .map_err(|reason| SessionError::new("audio-unavailable", reason))
+}
+
 /// The newest captured frame and when it was captured, handed from the capture
 /// thread to the encoder. A newer frame replaces one the encoder has not taken,
 /// so however far behind it falls, what it codes next is the desktop as it is
@@ -3243,6 +3404,11 @@ fn spawn_peer_events(inner: &Arc<Inner>, mut events: tokio_mpsc::UnboundedReceiv
                         first_frame: inner.metrics.lock().map(|m| m.encoded_frames).unwrap_or(0)
                             > 0,
                     });
+                    // desklink scout trace: the session's transport emit, one
+                    // line after the peer event that caused it.
+                    if crate::peer::dl_trace_on() {
+                        crate::peer::dl_eng!("session_transport_emit {}", format!("{state}").to_lowercase());
+                    }
                 }
                 PeerEvent::ControlOpen => {
                     inner.control_open.store(true, Ordering::SeqCst);
@@ -3301,6 +3467,8 @@ fn available_parallelism() -> usize {
         .unwrap_or(4)
 }
 
+const WINDOW_POINTER_REFUSAL: &str = "a shared window cannot take taps: its place on the desktop is unknown; share the whole screen to point and click";
+
 /// Encoded-surface pixels to the source's own pixels. The client aims at what it
 /// sees; the applier is sized and clamped to the source, so the layout origin
 /// must not be added here.
@@ -3417,6 +3585,8 @@ mod tests {
             point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
             capture: Mutex::new(None),
             layout: Mutex::new(None),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(None),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(false),
             closed: AtomicBool::new(false),
@@ -3673,6 +3843,8 @@ mod tests {
             #[cfg(target_os = "linux")]
             point_desktop: PointDesktop::Unavailable("layer_shell_unavailable"),
             layout: Mutex::new(Some(Layout::from_environment().expect("a keymap"))),
+            #[cfg(target_os = "linux")]
+            audio: Mutex::new(None),
             last_seq: Mutex::new(0),
             control_open: AtomicBool::new(true),
             closed: AtomicBool::new(false),
@@ -4080,6 +4252,7 @@ mod tests {
             loopback_tcp: false,
             agent_indicator: false,
             local_frames: true,
+            audio: false,
         }
     }
 

package/engine/src/win.rs

@@ -15,6 +15,7 @@ pub fn capabilities() -> Value {
             "formats": [],
             "cursor": "unavailable",
             "cursor_modes": [],
+            // No Windows capture path exists to carry audio from; out of scope.
             "audio": false,
             "displays": crate::portal::displays(),
             "grant": "unavailable",

package/engine/vendor/rtc-ice/DESKLINK-PATCH.md

@@ -1,7 +1,13 @@
 # desklink patch to vendored `rtc-ice` 0.21.0
 
+Two shapes, both for the roam-persistence goal: keep media flowing across an
+ICE restart, and nominate the restarted generation without waiting for a check
+tick.
+
 ## What
 
+### 1. Media keeps flowing on the previous pair across a restart
+
 `Agent::apply_restart` (src/agent/mod.rs) no longer interrupts media that is
 already flowing on the current selected ICE pair. When a restart is applied, the
 pair media was using is remembered (`restart_media_pair`), and
@@ -11,6 +17,19 @@ local credentials are installed, remote credentials are dropped, binding
 requests and candidate pairs are cleared, the agent re-checks, and the
 connection state goes to `Checking`.
 
+### 2. The nomination does not wait for the next check tick
+
+`ControllingSelector::handle_success_response` (src/agent/agent_selector.rs)
+requests an immediate check pass when the response makes a pair `Succeeded` and
+that pair is the one the next pass would nominate. Nomination is decided in
+`contact_candidates`, which the driver only reaches on a `check_interval` tick
+(200 ms) or when an inbound Binding request happens to arrive while a valid pair
+already exists; a pair that becomes valid just after a tick therefore waits a
+full tick for its `USE-CANDIDATE`. The request uses
+`Agent::request_connectivity_check()`, the deferred-check mechanism added for
+issue #88, so `contact()` is not called from the read path and `poll_timeout`
+reports an already-due deadline instead.
+
 Every changed spot is marked with a `// desklink patch:` comment.
 
 ## Why
@@ -33,8 +52,20 @@ The behavior is pinned by the engine integration test
 `a_restart_keeps_media_flowing_on_the_selected_pair_until_the_new_one_nominates`
 in `packages/desktop-host/engine/src/peer.rs`.
 
+Shape 2 exists because the tick is the engine's largest single bounded cost
+between a restart offer and `transport=connected`: in the routed roam leg the
+engine's first pair became valid 332 ms after the answer and its `USE-CANDIDATE`
+left 197 ms later — one 200 ms tick. A peer that runs no connectivity checks of
+its own leaves an inbound request unavailable, and the nomination then waits for
+the tick. The engine integration test asserts the reconnect bound; the loopback
+harness cannot grade sub-tick timing on its own (its client answers checks and
+sends triggered checks in the same burst, so an inbound request is available
+almost immediately), which is why the field measurement is the reference.
+
 ## Scope of the change
 
+### Shape 1
+
 - `Agent` gains `restart_media_pair: Option<(Candidate, Candidate)>`, initialised
   to `None`.
 - `apply_restart` clones the current `(local, remote)` into it before the
@@ -45,6 +76,16 @@ in `packages/desktop-host/engine/src/peer.rs`.
 - `update_connection_state` clears it on `Failed` / `Closed`, so a restart that
   never nominates does not keep media pointed at an unreachable pair.
 
+### Shape 2
+
+- `ControllingSelector::handle_success_response` calls
+  `request_connectivity_check()` after a pair becomes `Succeeded`, when no pair is
+  selected or nominated yet, this pair is the best valid pair, and both of its
+  candidates have passed their acceptance wait (`is_nominatable`).
+- Nothing else changes: the same `contact_candidates` pass decides the
+  nomination, the acceptance waits still gate it, and a nomination already in
+  flight is not re-requested, so the extra pass cannot repeat per response.
+
 No public API changed; no signalling or wire change.
 
 ## How to drop the patch
@@ -55,5 +96,16 @@ No public API changed; no signalling or wire change.
 3. Update `packages/desktop-host/engine/Cargo.lock` (`cargo update -p rtc-ice`).
 
 Do this once upstream rtc-ice keeps the previous selected pair across a restart
-on its own (an upstream PR is a follow-up; it needs the captain's word first,
+on its own and nominates a freshly validated pair without waiting for the next
+check tick (an upstream PR is a follow-up; it needs the captain's word first,
 since it would carry his name).
+
+## Dev-only measurement instrumentation (not part of the patch)
+
+`src/dl_trace.rs` and its call sites — marked `// desklink scout trace:` — are
+a private-lab instrument, not a shipped behaviour change. They are off unless
+`DESKLINK_ICE_TRACE=1` is set, and when on they only write `[ice <us>]` lines to
+stderr, which the bridge forwards untouched. To drop them: delete
+`src/dl_trace.rs`, remove `pub(crate) mod dl_trace;` from `src/lib.rs`, and
+remove the marked blocks from `src/agent/mod.rs` and
+`src/agent/agent_selector.rs` (grep `desklink scout trace`).

package/engine/vendor/rtc-ice/src/agent/agent_selector.rs

@@ -10,6 +10,9 @@ use stun::textattrs::*;
 
 use crate::attributes::{control::*, priority::*, use_candidate::*};
 use crate::candidate::{candidate_pair::*, *};
+// desklink scout trace (dl-roam-postnom-attr):
+use crate::dl_trace;
+use crate::dl_trace::dl_ice;
 
 trait ControllingSelector {
     fn start(&mut self, now: Instant);
@@ -133,6 +136,20 @@ impl Agent {
         };
 
         if let Some((msg, local, remote)) = result {
+            // desklink scout trace: a USE-CANDIDATE (nomination) request going
+            // out — the first transmission, or a tick-gated retransmission
+            // while the previous one is still unacknowledged.
+            if dl_trace::enabled() {
+                dl_ice!(
+                    "{} NOMINATE txn={:?} local={} remote={} nominated_pair={:?} outstanding={}",
+                    self.get_name(),
+                    msg.transaction_id,
+                    self.local_candidates[local].addr(),
+                    self.remote_candidates[remote].addr(),
+                    self.nominated_pair,
+                    self.pending_binding_requests.len(),
+                );
+            }
             self.send_binding_request(now, &msg, local, remote);
         }
     }
@@ -222,6 +239,27 @@ impl ControllingSelector for Agent {
 
         let nominated_pair_is_some = self.nominated_pair.is_some();
 
+        // desklink scout trace: which nomination branch this contact() pass took.
+        if dl_trace::enabled() {
+            let best = self
+                .get_best_valid_candidate_pair()
+                .map(|i| {
+                    let p = &self.candidate_pairs[i];
+                    format!(
+                        "{}<->{}",
+                        self.local_candidates[p.local_index].addr(),
+                        self.remote_candidates[p.remote_index].addr()
+                    )
+                })
+                .unwrap_or_else(|| "-none-valid-".to_string());
+            dl_ice!(
+                "{} contact_branch selected={:?} nominated={nominated_pair_is_some} best_valid={best} pairs={}",
+                self.get_name(),
+                self.selected_pair,
+                self.candidate_pairs.len(),
+            );
+        }
+
         if self.get_selected_pair().is_some() {
             if self.validate_selected_pair(now) {
                 self.check_keepalive(now);
@@ -336,7 +374,39 @@ impl ControllingSelector for Agent {
                     rtt.as_millis()
                 );
                 if pending_request.is_use_candidate && selected_pair_is_none {
+                    // desklink scout trace: the browser's answer to a nomination
+                    // request — this is what converts nominated into selected.
+                    if dl_trace::enabled() {
+                        dl_ice!(
+                            "{} NOMINATION_ACK txn={:?} rtt_us={} local={} remote={}",
+                            self.get_name(),
+                            m.transaction_id,
+                            rtt.as_micros(),
+                            self.local_candidates[local_index].addr(),
+                            self.remote_candidates[remote_index].addr(),
+                        );
+                    }
                     self.set_selected_pair(Some(now), Some(pair_index));
+                } else {
+                    // desklink patch: nomination is decided in `contact_candidates`,
+                    // and that only runs on a `check_interval` tick, so a pair that
+                    // becomes valid just after a tick waits a full tick for its
+                    // USE-CANDIDATE. Ask for the check pass now, when this is the pair
+                    // that tick would nominate. `request_connectivity_check` is the
+                    // deferred-check request (issue #88): it makes `poll_timeout`
+                    // report an already-due deadline, so the driver runs `contact()`
+                    // at once instead of at the next tick. See DESKLINK-PATCH.md.
+                    if self.get_selected_pair().is_none()
+                        && self.nominated_pair.is_none()
+                        && self.get_best_valid_candidate_pair() == Some(pair_index)
+                    {
+                        let p = &self.candidate_pairs[pair_index];
+                        if self.is_nominatable(now, p.local_index, true)
+                            && self.is_nominatable(now, p.remote_index, false)
+                        {
+                            self.request_connectivity_check();
+                        }
+                    }
                 }
             } else {
                 // This shouldn't happen
@@ -367,6 +437,17 @@ impl ControllingSelector for Agent {
             let p = &self.candidate_pairs[pair_index];
             let nominated_pair_is_none = self.nominated_pair.is_none();
 
+            // desklink scout trace: an inbound binding request on the
+            // controlling side — the browser's own connectivity check.
+            if dl_trace::enabled() {
+                dl_ice!(
+                    "{} RECV_REQUEST controlling use_candidate={} pair_state={:?} nominated={nominated_pair_is_none}",
+                    self.get_name(),
+                    m.contains(ATTR_USE_CANDIDATE),
+                    p.state,
+                );
+            }
+
             trace!(
                 "controllingSelector: after findPair {}, p.state: {}, {}",
                 p,