@desklink/host 0.4.0 → 0.5.1

package/engine/src/session.rs

@@ -895,7 +895,7 @@ pub fn capabilities() -> serde_json::Value {
             "formats": ["bgrx", "bgra", "rgbx", "rgba"],
             "cursor": "embedded",
             "cursor_modes": ["embedded", "hidden"],
-            "cursor_positions": {"x11": true, "portal": "negotiated"},
+            "cursor_positions": {"x11": true, "wayland": "ext-image-copy-capture-v1"},
             "audio": false,
         },
         "encode": encode_capabilities(),
@@ -1205,8 +1205,8 @@ impl Session {
                 }
                 .map_err(|error| {
                     let message = format!("{error:#}");
-                    let code = if message.contains("cursor_positions_unavailable:") {
-                        "cursor_positions_unavailable"
+                    let code = if message.contains("cursor-unavailable:") {
+                        "cursor-unavailable"
                     } else {
                         "source"
                     };
@@ -1264,12 +1264,11 @@ impl Session {
                 )
                 .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
                 #[cfg(target_os = "linux")]
-                let capture = capture::start(
+                let mut capture = capture::start(
                     portal,
                     width,
                     height,
                     max_fps,
-                    cursor_sink,
                     sink,
                     indicator
                         .as_ref()
@@ -1281,16 +1280,31 @@ impl Session {
                 )
                 .map_err(|error| SessionError::new("source", format!("{error:#}")))?;
                 #[cfg(target_os = "linux")]
-                {
-                    if request.cursor == crate::protocol::CursorMode::Hidden
-                        && !capture.cursor_positions()
-                    {
-                        return Err(SessionError::new(
-                            "cursor_positions_unavailable",
-                            "portal stream did not provide SPA cursor metadata",
-                        ));
+                if let Some(cursor_sink) = cursor_sink {
+                    // Positions are an addition to hidden capture, never a
+                    // condition of it: without them the stream still runs and
+                    // the reply says why the receiver gets no cursor.
+                    cursor_info["source"] = serde_json::json!(crate::wl_cursor::PROTOCOL);
+                    let limitation = match crate::wl_cursor::start(
+                        (source.origin_x, source.origin_y),
+                        (source_w, source_h),
+                        cursor_sink,
+                        Duration::from_secs(1),
+                    ) {
+                        Ok((tracker, observed)) => {
+                            capture.cursor = Some(tracker);
+                            cursor_info["positions"] = serde_json::json!(observed);
+                            (!observed).then_some("the compositor has sent no cursor position yet: the cursor is off the captured output, the seat has no pointer, or the compositor withholds positions from this engine")
+                        }
+                        Err(error) => {
+                            log::warn!("hidden cursor capture has no cursor positions: {error:#}");
+                            Some("this compositor does not offer ext-image-copy-capture-v1 cursor sessions, so the cursor is hidden and its position is unknown")
+                        }
+                    };
+                    if let Some(limitation) = limitation {
+                        log::warn!("{limitation}");
+                        cursor_info["limitation"] = serde_json::json!(limitation);
                     }
-                    cursor_info["positions"] = serde_json::json!(capture.cursor_positions());
                 }
                 #[cfg(target_os = "linux")]
                 if let Some(indicator) = indicator.as_mut() {
@@ -1319,7 +1333,7 @@ impl Session {
         let encoder = Encoder::new(
             width,
             height,
-            bitrate_kbps,
+            RateControl::starting(bitrate_kbps).target,
             max_fps,
             available_parallelism().min(8) as u32,
         )
@@ -2563,6 +2577,15 @@ struct RateControl {
 }
 
 impl RateControl {
+    /// Begin below an unknown weak link's capacity, then use the existing
+    /// loss/delay feedback growth to approach the requested ceiling.
+    fn starting(ceiling: u32) -> Self {
+        Self {
+            target: ceiling.min(3000),
+            ..Self::new(ceiling)
+        }
+    }
+
     fn new(ceiling: u32) -> Self {
         Self {
             ceiling,
@@ -2659,9 +2682,9 @@ impl SendBudget {
     }
 }
 
-/// Whether motion is coded at half size. A full-size frame is slow when it
-/// took longer to code than a frame interval and the source already had a
-/// newer frame waiting: the encoder, not the source, set the pace. Content that
+/// Whether motion is coded at half size. A full-size frame is slow when its
+/// encoding or constrained-link drain time exceeds a frame interval and a
+/// newer frame or send-budget debt is waiting. Content that
 /// arrives slower than the frame cap (30 fps into a 60 fps session) is never
 /// slow however long a frame takes, as long as it is done before the next one.
 /// Several slow frames that each change much of the picture (a packet over a
@@ -2677,15 +2700,31 @@ impl SendBudget {
 struct MotionSize {
     half: bool,
     run: u32,
+    link_run: u32,
 }
 
 impl MotionSize {
     const SLOW_RUN: u32 = 5;
     const SMALL_RUN: u32 = 30;
 
-    /// Account for a motion frame coded at the current size in `took`; `large`
+    /// Persistent send debt needs a quicker response than a transient slow
+    /// encode: every extra full-size motion frame extends the remote queue.
+    fn queued(&mut self, drain: Duration, debt: bool, interval: Duration) {
+        self.link_run = if !self.half && debt && drain > interval {
+            self.link_run + 1
+        } else {
+            0
+        };
+        if self.link_run >= 2 {
+            self.half = true;
+            self.run = 0;
+            self.link_run = 0;
+        }
+    }
+
+    /// Account for a motion frame serviced at the current size in `took`; `large`
     /// when its packet was over a quarter of the per-frame rate budget, `behind`
-    /// when a newer frame was already waiting once it was coded.
+    /// when a newer frame or send-budget debt was waiting once it was coded.
     fn coded(&mut self, took: Duration, large: bool, behind: bool, interval: Duration) {
         let slow = took > interval && behind;
         let (counts, needed) = if self.half {
@@ -2762,9 +2801,9 @@ fn spawn_pipeline(
         .name("desklink-encode".into())
         .spawn(move || {
             let interval = frame_interval(max_fps);
-            let mut rate = RateControl::new(bitrate_kbps);
+            let mut rate = RateControl::starting(bitrate_kbps);
             if let Ok(mut m) = inner.metrics.lock() {
-                m.target_kbps = bitrate_kbps;
+                m.target_kbps = rate.target;
                 (m.codec, m.encoder) = ("vp9", "libvpx");
             }
             // Whether motion may be coded at half size: VP9's alone.
@@ -2944,16 +2983,26 @@ fn spawn_pipeline(
                         Pass::Motion { half: false, .. } | Pass::Keepalive => {}
                     }
                 }
+                budget.sent(now, packet.data.len(), rate.target);
                 match pass {
                     Pass::Motion { .. } => {
                         pending = false;
                         keyframe = false;
                         refined = false;
                         still_since = now;
-                        let budget = rate.target as u64 * 1000 / 8 / max_fps.max(1) as u64;
-                        let large = packet.data.len() as u64 * 4 > budget;
+                        let frame_budget = rate.target as u64 * 1000 / 8 / max_fps.max(1) as u64;
+                        let large = packet.data.len() as u64 * 4 > frame_budget;
                         if halves {
+                            // A fast encoder can still outrun a constrained link.
+                            // Include the packet's drain time once the send
+                            // budget is holding frames, so sustained motion
+                            // reduces its size before a backlog builds.
+                            let drain = Duration::from_secs_f64(
+                                packet.data.len() as f64 * 8.0 / SendBudget::rate(rate.target),
+                            );
+                            let link_behind = !budget.wait(Instant::now(), rate.target).is_zero();
                             size.coded(took, large, frame_rx.waiting(), interval);
+                            size.queued(drain, link_behind, interval);
                         }
                     }
                     Pass::Refine => {
@@ -2964,7 +3013,6 @@ fn spawn_pipeline(
                 }
                 last_sent = Some(now);
                 cadence.sent(now);
-                budget.sent(now, packet.data.len(), rate.target);
                 // A new picture carries the moment it was captured, so the
                 // receiver paces playout by the desktop's clock rather than by
                 // encode time; a re-coded one (refinement, keepalive) is new
@@ -3403,66 +3451,6 @@ mod tests {
         );
     }
 
-    #[test]
-    fn tile_damage_at_origin_does_not_underflow_or_hide_other_tiles() {
-        assert_eq!(
-            dirty_regions(&[1, 0, 0, 1], &[0; 4], 64, 64),
-            vec![[0, 0, 32, 32], [32, 32, 32, 32]]
-        );
-    }
-
-    #[test]
-    fn frames_that_jitter_around_the_cap_go_at_once_and_a_flood_stays_capped() {
-        let interval = Duration::from_millis(20);
-        let start = Instant::now();
-        let mut cadence = Cadence::new(interval, start);
-        // Captured at the cap, each a few ms early or late: none waits, so
-        // none is delayed or superseded by the next.
-        for (n, jitter) in [0i64, -6, 5, -8, 7, -3, 0, -9, 9, -5].iter().enumerate() {
-            let at = start + interval * n as u32 + Duration::from_millis(9);
-            let at = if *jitter < 0 {
-                at - Duration::from_millis(jitter.unsigned_abs())
-            } else {
-                at + Duration::from_millis(*jitter as u64)
-            };
-            assert_eq!(cadence.wait(at), Duration::ZERO, "frame {n} waited");
-            cadence.sent(at);
-        }
-
-        // A source far above the cap: one second of 1 ms frames sends at most
-        // one frame over the cap.
-        let start = start + Duration::from_secs(10);
-        let mut cadence = Cadence::new(interval, start);
-        let sent = (0..1000)
-            .map(|ms| start + Duration::from_millis(ms))
-            .filter(|&at| {
-                let go = cadence.wait(at).is_zero();
-                if go {
-                    cadence.sent(at);
-                }
-                go
-            })
-            .count();
-        assert!((50..=51).contains(&sent), "sent {sent} at a 50 fps cap");
-    }
-
-    #[test]
-    fn a_small_source_is_never_upscaled() {
-        assert_eq!(fit(1000, 600, 1280, 800), (1000, 600));
-    }
-
-    #[test]
-    fn a_large_source_is_fitted_into_the_box_with_even_dimensions() {
-        let (w, h) = fit(2560, 1440, 1280, 800);
-        assert_eq!((w, h), (1280, 720));
-        assert_eq!((w % 2, h % 2), (0, 0));
-    }
-
-    #[test]
-    fn fit_keeps_the_aspect_ratio_when_the_box_is_the_constraint() {
-        assert_eq!(fit(2560, 1440, 1200, 600), (1066, 600));
-    }
-
     #[test]
     fn an_encoded_surface_pixel_maps_to_the_sources_own_pixel() {
         let source = SelectedSource {
@@ -4324,149 +4312,6 @@ mod tests {
         assert!(inner.input.lock().unwrap().is_none());
     }
 
-    #[test]
-    fn a_standing_queue_backs_the_rate_off_to_what_the_path_delivers() {
-        let start = Instant::now();
-        let queued = |ms: u64, delivered_kbps: u32| {
-            Some(PathReport {
-                queue_delay: Duration::from_millis(ms),
-                delivered_kbps,
-            })
-        };
-        let mut rate = RateControl::new(8000);
-        // A burst's worth of queue and no loss moves nothing.
-        assert_eq!(rate.update(&[], queued(15, 3000), start), None);
-        // A standing queue on a 4 Mbps link: 85% of what got through, at once,
-        // with no loss reported at all.
-        assert_eq!(rate.update(&[], queued(80, 4000), start), Some(3400));
-        // Not again until the back-off has had time to show in the feedback.
-        assert_eq!(
-            rate.update(&[], queued(80, 3400), start + Duration::from_millis(100)),
-            None
-        );
-        assert_eq!(
-            rate.update(&[], queued(60, 3400), start + Duration::from_millis(300)),
-            Some(2890)
-        );
-        // A stream that sent little is not taken for a slow link: at most half.
-        assert_eq!(
-            rate.update(&[], queued(60, 100), start + Duration::from_millis(600)),
-            Some(1228)
-        );
-        // Clean loss reports do not raise it while the queue has not drained...
-        assert_eq!(
-            rate.update(&[0], queued(15, 1200), start + Duration::from_secs(2)),
-            None
-        );
-        // ...and do once it has.
-        assert_eq!(
-            rate.update(&[0], queued(2, 1200), start + Duration::from_secs(3)),
-            Some(1390)
-        );
-        // Nothing drops below the floor.
-        let mut rate = RateControl::new(8000);
-        for step in 0..20u64 {
-            rate.update(
-                &[],
-                queued(200, 1),
-                start + DELAY_BACKOFF_EVERY * step as u32,
-            );
-        }
-        assert_eq!(rate.target, 1000);
-    }
-
-    #[test]
-    fn without_transport_feedback_the_rate_follows_loss_alone() {
-        let start = Instant::now();
-        let mut rate = RateControl::new(8000);
-        assert_eq!(rate.update(&[], None, start), None);
-        assert_eq!(rate.update(&[64], None, start), Some(5600));
-        assert_eq!(
-            rate.update(&[64], None, start + Duration::from_millis(500)),
-            None
-        );
-        assert_eq!(
-            rate.update(&[10], None, start + Duration::from_secs(1)),
-            None
-        );
-        assert_eq!(
-            rate.update(&[0], None, start + Duration::from_secs(2)),
-            Some(6112)
-        );
-    }
-
-    #[test]
-    fn a_large_frame_holds_the_next_until_the_link_has_carried_it() {
-        let start = Instant::now();
-        let mut budget = SendBudget::default();
-        // Frames at the target, one per 20 ms at 4000 kbps, never wait, and
-        // neither does the ordinary overshoot of a frame at twice the budget.
-        for n in 0..50u32 {
-            let at = start + Duration::from_millis(20) * n;
-            assert_eq!(budget.wait(at, 4000), Duration::ZERO, "frame {n} waited");
-            budget.sent(at, if n % 5 == 0 { 20_000 } else { 10_000 }, 4000);
-        }
-        // A 100 kB key frame drains in 160 ms (4000 kbps and a quarter): it
-        // goes at once, and the next waits out all of it past the 60 ms burst
-        // allowance.
-        let at = start + Duration::from_secs(1);
-        assert_eq!(budget.wait(at, 4000), Duration::ZERO);
-        budget.sent(at, 100_000, 4000);
-        let wait = budget.wait(at, 4000);
-        assert!(
-            (Duration::from_millis(99)..=Duration::from_millis(101)).contains(&wait),
-            "waited {wait:?}"
-        );
-        assert_eq!(budget.wait(at + wait, 4000), Duration::ZERO);
-    }
-
-    #[test]
-    fn motion_halves_only_when_the_encoder_sets_the_pace_and_does_not_flap() {
-        let interval = Duration::from_millis(16);
-        let (slow, fast) = (Duration::from_millis(30), Duration::from_millis(2));
-        let mut size = MotionSize::default();
-        for _ in 0..MotionSize::SLOW_RUN * 4 {
-            size.coded(slow, false, true, interval);
-        }
-        assert!(
-            !size.half,
-            "slow small frames (typing on a busy host) keep full size"
-        );
-        for _ in 0..MotionSize::SLOW_RUN * 4 {
-            size.coded(slow, true, false, interval);
-        }
-        assert!(
-            !size.half,
-            "a source slower than the cap (30 fps, 30 ms encode) never halves"
-        );
-        for _ in 0..MotionSize::SLOW_RUN - 1 {
-            size.coded(slow, true, true, interval);
-        }
-        size.coded(fast, true, true, interval);
-        assert!(!size.half, "a broken run of slow frames keeps full size");
-        for _ in 0..MotionSize::SLOW_RUN {
-            size.coded(slow, true, true, interval);
-        }
-        assert!(size.half, "a run of slow large frames halves");
-        // The marginal band: half-size frames that are small but still slow,
-        // between large ones. Neither may send it back to full size.
-        for i in 0..MotionSize::SMALL_RUN * 10 {
-            size.coded(slow, i % 7 == 0, true, interval);
-            assert!(size.half, "marginal half-size motion must not flap back");
-        }
-        for _ in 0..MotionSize::SMALL_RUN * 2 {
-            size.coded(fast, true, true, interval);
-        }
-        assert!(size.half, "large half-size frames stay half, however fast");
-        for _ in 0..MotionSize::SMALL_RUN {
-            size.coded(fast, false, true, interval);
-        }
-        assert!(
-            !size.half,
-            "a run of small, fast half-size frames returns to full size"
-        );
-    }
-
     #[tokio::test]
     async fn the_pipeline_caps_the_rate_codes_the_last_frame_and_refines_a_still_desktop_once() {
         let (events, _events_rx) = tokio_mpsc::unbounded_channel();