@desklink/host 0.5.3 → 0.5.6

package/engine/src/peer.rs

@@ -47,6 +47,33 @@ use webrtc::peer_connection::{
 };
 use webrtc::runtime::default_runtime;
 
+// desklink scout trace (dl-roam-postnom-attr): engine-side restart timeline.
+// Enabled with `DESKLINK_ICE_TRACE=1`; stamps microseconds since the Unix epoch
+// so these lines interleave with the vendored ICE agent's `[ice ...]` lines in
+// one order. Droppable: grep `desklink scout trace`.
+pub(crate) fn dl_trace_on() -> bool {
+    std::env::var("DESKLINK_ICE_TRACE")
+        .map(|v| v != "0" && !v.is_empty())
+        .unwrap_or(false)
+}
+
+pub(crate) fn dl_trace_us() -> u128 {
+    std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_micros())
+        .unwrap_or(0)
+}
+
+macro_rules! dl_eng {
+    ($($arg:tt)*) => {
+        if $crate::peer::dl_trace_on() {
+            eprintln!("[eng {}] {}", $crate::peer::dl_trace_us(), format_args!($($arg)*));
+        }
+    };
+}
+
+pub(crate) use dl_eng;
+
 /// VP9 clock rate, fixed by RFC 9628.
 const VP9_CLOCK_RATE: u32 = 90_000;
 
@@ -420,6 +447,12 @@ impl PeerConnectionEventHandler for Handler {
     }
 
     async fn on_connection_state_change(&self, state: RTCPeerConnectionState) {
+        // desklink scout trace (dl-roam-postnom-attr): the engine's rtc-level
+        // transport state event, on the same microsecond clock as the vendored
+        // agent's `[ice ...]` lines, so the rtc Connected gate is separable.
+        if dl_trace_on() {
+            dl_eng!("peer_state_change {state:?}");
+        }
         let connected = matches!(state, RTCPeerConnectionState::Connected);
         if connected {
             self.wants_keyframe.store(true, Ordering::SeqCst);
@@ -1153,6 +1186,7 @@ fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
 #[cfg(test)]
 pub(crate) mod tests {
     use super::*;
+    use webrtc::media_stream::track_remote::{TrackRemote, TrackRemoteEvent};
 
     #[test]
     fn a_standing_queue_is_measured_and_one_frames_burst_is_not() {
@@ -1589,6 +1623,269 @@ pub(crate) mod tests {
             .expect("the restart answer is accepted");
     }
 
+    /// How long the restarted generation may take to nominate and reconnect,
+    /// measured from the answered offer: two `check_interval` ticks. The vendored
+    /// agent ticks every 200 ms (`vendor/rtc-ice/src/agent/agent_config.rs`), and
+    /// the routed roam leg measured the engine's own first-valid-pair ->
+    /// USE-CANDIDATE at 197 ms, one tick; the extra tick is headroom for the
+    /// loopback's round trip and for a loaded machine. See the assertion in
+    /// `a_restart_keeps_media_flowing_on_the_selected_pair_until_the_new_one_nominates`.
+    const RECONNECT_BUDGET: Duration = Duration::from_millis(400);
+
+    /// A receiver peer that records when each inbound RTP packet arrives and
+    /// reports its own ICE candidates, so two peers can really connect.
+    struct MediaReceiver {
+        candidates: tokio::sync::mpsc::UnboundedSender<RTCIceCandidateInit>,
+        arrivals: Arc<Mutex<Vec<Instant>>>,
+        connected: Arc<AtomicBool>,
+    }
+
+    #[async_trait::async_trait]
+    impl PeerConnectionEventHandler for MediaReceiver {
+        async fn on_ice_candidate(&self, event: RTCPeerConnectionIceEvent) {
+            if let Ok(init) = event.candidate.to_json() {
+                let _ = self.candidates.send(init);
+            }
+        }
+
+        async fn on_connection_state_change(&self, state: RTCPeerConnectionState) {
+            self.connected.store(
+                matches!(state, RTCPeerConnectionState::Connected),
+                Ordering::SeqCst,
+            );
+        }
+
+        async fn on_track(&self, track: Arc<dyn TrackRemote>) {
+            let arrivals = Arc::clone(&self.arrivals);
+            tokio::spawn(async move {
+                while let Some(event) = track.poll().await {
+                    if matches!(event, TrackRemoteEvent::OnRtpPacket(_)) {
+                        arrivals.lock().unwrap().push(Instant::now());
+                    }
+                }
+            });
+        }
+    }
+
+    async fn media_receiver(receiver: Arc<MediaReceiver>) -> Arc<dyn PeerConnection> {
+        let mut media_engine = MediaEngine::default();
+        media_engine
+            .register_codec(vp9_codec(), RtpCodecKind::Video)
+            .expect("the codec registers");
+        let registry = register_default_interceptors(Registry::new(), &mut media_engine)
+            .expect("interceptors register");
+        let peer = PeerConnectionBuilder::<std::net::SocketAddr>::new()
+            .with_media_engine(media_engine)
+            .with_interceptor_registry(registry)
+            .with_runtime(default_runtime().expect("a runtime"))
+            .with_handler(receiver)
+            .with_udp_addrs(vec![std::net::SocketAddr::from(([0, 0, 0, 0], 0))])
+            .build()
+            .await
+            .expect("a receiving peer");
+        Arc::new(peer)
+    }
+
+    /// A restart offer/answer round must not interrupt media already flowing on
+    /// the selected pair: the picture keeps advancing (no gap over 200 ms) while
+    /// the old pair stays reachable, and the ICE generation still turns over.
+    #[tokio::test]
+    async fn a_restart_keeps_media_flowing_on_the_selected_pair_until_the_new_one_nominates() {
+        let (events, mut events_rx) = tokio::sync::mpsc::unbounded_channel();
+        let (engine, offer) = VideoPeer::offer(transport(), events, VideoCodec::Vp9)
+            .await
+            .expect("an offering peer");
+        let engine = Arc::new(engine);
+
+        let arrivals: Arc<Mutex<Vec<Instant>>> = Arc::new(Mutex::new(Vec::new()));
+        let connected = Arc::new(AtomicBool::new(false));
+        let (client_candidates_tx, mut client_candidates_rx) =
+            tokio::sync::mpsc::unbounded_channel();
+        let receiver = Arc::new(MediaReceiver {
+            candidates: client_candidates_tx,
+            arrivals: Arc::clone(&arrivals),
+            connected: Arc::clone(&connected),
+        });
+        let client = media_receiver(Arc::clone(&receiver)).await;
+
+        client
+            .set_remote_description(RTCSessionDescription::offer(offer.clone()).expect("an offer"))
+            .await
+            .expect("the offer applies");
+        let answer = client.create_answer(None).await.expect("an answer");
+        client
+            .set_local_description(answer.clone())
+            .await
+            .expect("a local answer");
+        engine
+            .accept_answer(answer.sdp.clone())
+            .await
+            .expect("the answer is accepted");
+
+        // The engine's candidates reach the client.
+        let client_for_engine_candidates = Arc::clone(&client);
+        tokio::spawn(async move {
+            while let Some(event) = events_rx.recv().await {
+                if let PeerEvent::Candidate {
+                    candidate,
+                    sdp_mid,
+                    sdp_m_line_index,
+                } = event
+                {
+                    let _ = client_for_engine_candidates
+                        .add_ice_candidate(RTCIceCandidateInit {
+                            candidate,
+                            sdp_mid,
+                            sdp_mline_index: sdp_m_line_index,
+                            username_fragment: None,
+                            url: None,
+                        })
+                        .await;
+                }
+            }
+        });
+        // The client's candidates reach the engine.
+        let engine_for_client_candidates = Arc::clone(&engine);
+        tokio::spawn(async move {
+            while let Some(init) = client_candidates_rx.recv().await {
+                let _ = engine_for_client_candidates
+                    .add_candidate(init.candidate, init.sdp_mid, init.sdp_mline_index)
+                    .await;
+            }
+        });
+
+        // Wait for the ICE path to come up on both ends.
+        let deadline = Instant::now() + Duration::from_secs(5);
+        while !(engine.is_connected() && connected.load(Ordering::SeqCst)) {
+            assert!(Instant::now() < deadline, "the peers never connected");
+            tokio::time::sleep(Duration::from_millis(10)).await;
+        }
+
+        // Any bytes the VP9 packetizer turns into one access unit.
+        let frame: Vec<u8> = (0..200u32).map(|i| (i % 251) as u8).collect();
+        let sending = Arc::new(AtomicBool::new(true));
+        let engine_for_sending = Arc::clone(&engine);
+        let sending_for_task = Arc::clone(&sending);
+        let frame_for_task = frame.clone();
+        tokio::spawn(async move {
+            while sending_for_task.load(Ordering::SeqCst) {
+                let _ = engine_for_sending
+                    .send_frame(&frame_for_task, true, Instant::now())
+                    .await;
+                tokio::time::sleep(Duration::from_millis(5)).await;
+            }
+        });
+
+        // Let the picture flow, then take a clean baseline.
+        tokio::time::sleep(Duration::from_millis(300)).await;
+        arrivals.lock().unwrap().clear();
+        tokio::time::sleep(Duration::from_millis(200)).await;
+        let flowing = arrivals.lock().unwrap().len();
+        assert!(
+            flowing >= 5,
+            "media was not flowing before the restart: {flowing} packets"
+        );
+
+        // The restart a client drives when its carrier comes back on a roam.
+        let restart = engine.restart_ice().await.expect("a restart offer");
+        let ufrag = |sdp: &str| {
+            sdp.lines()
+                .find_map(|line| line.strip_prefix("a=ice-ufrag:"))
+                .expect("ICE credentials")
+                .to_owned()
+        };
+        assert_ne!(
+            ufrag(&restart),
+            ufrag(&offer),
+            "the restart must turn the ICE generation over"
+        );
+
+        let restart_at = Instant::now();
+        arrivals.lock().unwrap().clear();
+
+        // The restart offer crosses the consumer's signaling channel before the
+        // client answers, and the engine applies the restart at
+        // `set_local_description` — inside `restart_ice` above, before the offer
+        // has even left. It must keep feeding media on the old pair for that
+        // whole round trip, not only until its own teardown. A real channel
+        // costs more than a loopback hop, so hold this window for the bar's
+        // length: a teardown shows up here, a preserved pair does not.
+        tokio::time::sleep(Duration::from_millis(300)).await;
+
+        client
+            .set_remote_description(RTCSessionDescription::offer(restart).expect("a restart offer"))
+            .await
+            .expect("the restart offer applies");
+        let reanswer = client.create_answer(None).await.expect("an answer");
+        client
+            .set_local_description(reanswer.clone())
+            .await
+            .expect("a local answer");
+        engine
+            .accept_answer(reanswer.sdp)
+            .await
+            .expect("the restart answer is accepted");
+        let answered_at = Instant::now();
+
+        // The restarted generation has to be nominated off its own check cycle:
+        // nomination is decided in the agent's `contact_candidates` pass, which the
+        // driver reaches on a `check_interval` tick (200 ms) or when an inbound
+        // Binding request arrives while a valid pair already exists. A pair that
+        // becomes valid just after a tick would therefore wait a full tick for its
+        // USE-CANDIDATE, and one tick is the whole engine-side cost measured on the
+        // routed roam leg (first valid pair -> USE-CANDIDATE: 197 ms, i.e. one tick).
+        //
+        // What this asserts, and what it cannot: the engine's "first valid pair"
+        // instant is not observable from here (the peer connection exposes no pair
+        // state), so the bound is measured from the answer, the earliest instant the
+        // engine could nominate, at two ticks. The loopback's client answers checks
+        // and sends its own triggered checks in the same burst, so an inbound request
+        // is nearly always available and the base engine meets this bound too; the
+        // tick itself is graded by the routed roam leg, whose relay log carries both
+        // instants (197 ms, one tick). This guards the property going forward: a
+        // lost pass, a skipped tick or a stalled nomination blows the bound.
+        let reconnected = async {
+            while !engine.is_connected() {
+                tokio::time::sleep(Duration::from_millis(5)).await;
+            }
+            answered_at.elapsed()
+        };
+        let reconnect = tokio::time::timeout(RECONNECT_BUDGET, reconnected)
+            .await
+            .expect("the restarted generation was never nominated");
+        eprintln!(
+            "restart: transport reconnected {reconnect:?} after the answer (budget {RECONNECT_BUDGET:?})"
+        );
+
+        // Keep the picture advancing across and after the restart.
+        tokio::time::sleep(Duration::from_millis(800)).await;
+        sending.store(false, Ordering::SeqCst);
+
+        let stamps = arrivals.lock().unwrap().clone();
+        assert!(
+            !stamps.is_empty(),
+            "no media arrived at all after the restart"
+        );
+        let mut previous = restart_at;
+        let mut worst = Duration::ZERO;
+        for stamp in &stamps {
+            worst = worst.max(stamp.duration_since(previous));
+            previous = *stamp;
+        }
+        eprintln!(
+            "restart: {} packets after the offer, worst media gap {worst:?}",
+            stamps.len()
+        );
+        assert!(
+            worst < Duration::from_millis(200),
+            "media stalled for {worst:?} through the restart: {:?}",
+            stamps
+        );
+
+        engine.close().await;
+        let _ = client.close().await;
+    }
+
     #[tokio::test]
     async fn a_forwarded_client_is_offered_ice_over_tcp_on_the_loopback() {
         let offered = |loopback_tcp: bool| async move {

package/engine/src/portal.rs

@@ -142,21 +142,3 @@ fn choose_cursor_mode(available: BitFlags<CursorMode>) -> Result<CursorMode> {
         .find(|mode| available.contains(*mode))
         .context("cursor-unavailable: portal cannot leave the cursor out of capture")
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    #[test]
-    fn hidden_takes_any_cursor_free_mode_and_refuses_embedded_only_portals() {
-        assert_eq!(
-            choose_cursor_mode(CursorMode::Metadata | CursorMode::Hidden).unwrap(),
-            CursorMode::Hidden
-        );
-        assert_eq!(
-            choose_cursor_mode(CursorMode::Metadata.into()).unwrap(),
-            CursorMode::Metadata
-        );
-        let error = choose_cursor_mode(CursorMode::Embedded.into()).unwrap_err();
-        assert!(error.to_string().contains("cursor-unavailable:"));
-    }
-}

package/engine/src/protocol.rs

@@ -144,6 +144,11 @@ pub struct OpenParams {
     /// the engine then spends nothing per frame on either.
     #[serde(default = "default_local_frames")]
     pub local_frames: bool,
+    /// Capture the desktop's own system audio for `session.audio`. Absent means
+    /// video only; there is never a silent downgrade, so a caller that asks and
+    /// cannot be answered is refused.
+    #[serde(default)]
+    pub audio: bool,
 }
 
 fn default_local_frames() -> bool {
@@ -244,6 +249,23 @@ pub struct SessionRef {
     pub generation: Option<u64>,
 }
 
+/// Read the desktop's own audio, the way `FrameParams` reads a frame: bytes go
+/// to the caller's path and JSON carries only metadata.
+#[derive(Debug, Deserialize)]
+pub struct AudioParams {
+    pub session_id: String,
+    #[serde(default)]
+    pub path: String,
+    /// The most bytes to write. Bounded to whole sample frames, so a truncated
+    /// chunk never ends mid-sample.
+    #[serde(default = "default_max_audio_bytes")]
+    pub max_bytes: usize,
+}
+
+fn default_max_audio_bytes() -> usize {
+    1 << 20
+}
+
 #[derive(Debug, Deserialize)]
 pub struct DescriptionParams {
     pub session_id: String,
@@ -395,168 +417,6 @@ pub enum ControlReply<'a> {
     },
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn cursor_option_defaults_and_refuses_unknown_modes() {
-        let default: OpenParams = serde_json::from_str("{}").unwrap();
-        assert_eq!(default.cursor, CursorMode::Embedded);
-        let hidden: OpenParams = serde_json::from_str(r#"{"cursor":"hidden"}"#).unwrap();
-        assert_eq!(hidden.cursor, CursorMode::Hidden);
-        assert!(serde_json::from_str::<OpenParams>(r#"{"cursor":"metadata"}"#).is_err());
-        assert!(serde_json::from_str::<OpenParams>(r#"{"cursor":false}"#).is_err());
-    }
-
-    #[test]
-    fn a_pointer_message_round_trips_with_its_sequence() {
-        let parsed: ControlMessage =
-            serde_json::from_str(r#"{"kind":"pointer","phase":"down","x":10,"y":20,"seq":3}"#)
-                .unwrap();
-        match parsed {
-            ControlMessage::Pointer {
-                phase,
-                x,
-                y,
-                seq,
-                button,
-            } => {
-                assert!(matches!(phase, PointerPhase::Down));
-                assert_eq!((x, y, seq, button), (10, 20, 3, 1));
-            }
-            other => panic!("wrong variant: {other:?}"),
-        }
-    }
-
-    #[test]
-    fn an_unknown_action_is_refused_rather_than_ignored() {
-        assert!(serde_json::from_str::<ControlMessage>(r#"{"kind":"teleport"}"#).is_err());
-    }
-
-    #[test]
-    fn display_source_round_trips_its_explicit_id() {
-        let request: SourceRequest = serde_json::from_value(serde_json::json!({
-            "kind": "display",
-            "display_id": 42
-        }))
-        .unwrap();
-        assert!(matches!(
-            request,
-            SourceRequest::Display {
-                display_id: Some(42)
-            }
-        ));
-    }
-
-    #[test]
-    fn an_encoded_source_names_its_codec_and_its_own_size() {
-        let request: SourceRequest = serde_json::from_value(serde_json::json!({
-            "kind": "encoded",
-            "codec": "h264",
-            "width": 486,
-            "height": 1080
-        }))
-        .unwrap();
-        assert!(matches!(
-            request,
-            SourceRequest::Encoded {
-                codec: EncodedCodec::H264,
-                width: 486,
-                height: 1080
-            }
-        ));
-    }
-
-    #[test]
-    fn an_unknown_encoded_codec_is_refused_rather_than_guessed() {
-        assert!(serde_json::from_value::<SourceRequest>(serde_json::json!({
-            "kind": "encoded",
-            "codec": "av1",
-            "width": 486,
-            "height": 1080
-        }))
-        .is_err());
-    }
-
-    #[test]
-    fn a_feed_request_is_one_access_unit_with_its_own_keyframe_flag() {
-        let params: FeedParams =
-            serde_json::from_str(r#"{"session_id":"s","keyframe":true,"data_b64":"AAAA"}"#)
-                .unwrap();
-        assert_eq!(params.session_id, "s");
-        assert!(params.keyframe, "a feed says whether its unit is an IDR");
-        assert_eq!(params.data_b64, "AAAA");
-        assert_eq!(params.access_unit().unwrap(), vec![0x00, 0x00, 0x00]);
-    }
-
-    #[test]
-    fn a_feed_request_that_is_not_base64_is_refused() {
-        let params = FeedParams {
-            session_id: String::from("s"),
-            keyframe: false,
-            data_b64: String::from("not base64!!"),
-        };
-        assert!(params.access_unit().is_err());
-    }
-
-    /// The reason this protocol carries bytes in base64 rather than adding a
-    /// second inherited fd: at 2 Mbit/s it costs a fraction of a per cent of a
-    /// core, against a whole second channel to keep ordered and bounded. The
-    /// bound below is deliberately loose — it is a sanity check on the order of
-    /// magnitude, not a benchmark, so it does not flake on a loaded machine.
-    #[test]
-    fn decoding_a_two_megabit_stream_is_nowhere_near_a_core() {
-        // One second of a 2 Mbit/s stream, in access units.
-        let unit: Vec<u8> = (0..8_000u32).map(|index| index as u8).collect();
-        let encoded = base64::Engine::encode(&base64::engine::general_purpose::STANDARD, &unit);
-        let params = FeedParams {
-            session_id: String::from("s"),
-            keyframe: false,
-            data_b64: encoded,
-        };
-        let started = std::time::Instant::now();
-        for _ in 0..(250_000 / unit.len()) {
-            assert_eq!(params.access_unit().unwrap().len(), unit.len());
-        }
-        let elapsed = started.elapsed();
-        assert!(
-            elapsed < std::time::Duration::from_millis(500),
-            "a second of 2 Mbit/s base64 took {elapsed:?}, which is far past 5 % of a core",
-        );
-        println!("a second of 2 Mbit/s stream decoded in {elapsed:?}");
-    }
-
-    #[test]
-    fn open_defaults_keep_the_desktops_own_pixels_up_to_4k() {
-        for payload in [
-            r#"{}"#,
-            r#"{"permissions":["view"]}"#,
-            r#"{"permissions":["view","unknown-legacy-scope"]}"#,
-        ] {
-            let _: OpenParams = serde_json::from_str(payload).unwrap();
-        }
-        let params: OpenParams =
-            serde_json::from_str(r#"{"permissions":["view","control"]}"#).unwrap();
-        assert_eq!(params.max_width, 3840);
-        assert_eq!(params.max_height, 2160);
-        assert_eq!(
-            params.bitrate_kbps, 0,
-            "the engine sizes the rate to the surface"
-        );
-        assert_eq!(params.max_fps, 60);
-        assert!(params.local_frames, "session.frame works unless turned off");
-        assert!(
-            !params.agent_indicator,
-            "human control has no overlay by default"
-        );
-        let agent: OpenParams =
-            serde_json::from_str(r#"{"permissions":["view","control"],"agent_indicator":true}"#)
-                .unwrap();
-        assert!(agent.agent_indicator);
-    }
-}
-
 /// Monotonic engine-process clock shared by local frame and cursor notices.
 pub fn timestamp_us(at: std::time::Instant) -> u64 {
     static EPOCH: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new();